Merge lp:~openstack-charmers/charm-helpers/to_upstream into lp:charm-helpers

[James Page]

Bulk update for all openstack python redux charm-helper changes.

[lp:~openstack-charmers/charm-helpers/to_upstream]

109. By James Page on 2013-10-11

Update for swift havana release 1.10.0

110. By Adam Gandelman on 2013-10-11

neutron: Ensure correct headers for running kernel are installed.

111. By Adam Gandelman on 2013-10-12

Reorder list for neutron package.

112. By Adam Gandelman on 2013-10-14

Adds new SubordinateConfigContexts, updates tests.

=== modified file 'charmhelpers/contrib/hahelpers/cluster.py'
--- charmhelpers/contrib/hahelpers/cluster.py	2013-07-23 11:50:28 +0000
+++ charmhelpers/contrib/hahelpers/cluster.py	2013-10-15 01:15:14 +0000
@@ -97,12 +97,14 @@
         return True
     for r_id in relation_ids('identity-service'):
         for unit in relation_list(r_id):
-            if None not in [
+            rel_state = [
                 relation_get('https_keystone', rid=r_id, unit=unit),
                 relation_get('ssl_cert', rid=r_id, unit=unit),
                 relation_get('ssl_key', rid=r_id, unit=unit),
                 relation_get('ca_cert', rid=r_id, unit=unit),
-            ]:
+            ]
+            # NOTE: works around (LP: #1203241)
+            if (None not in rel_state) and ('' not in rel_state):
                 return True
     return False
 

=== modified file 'charmhelpers/contrib/openstack/context.py'
--- charmhelpers/contrib/openstack/context.py	2013-07-19 23:29:59 +0000
+++ charmhelpers/contrib/openstack/context.py	2013-10-15 01:15:14 +0000
@@ -1,3 +1,4 @@
+import json
 import os
 
 from base64 import b64decode
@@ -6,6 +7,12 @@
     check_call
 )
 
+
+from charmhelpers.fetch import (
+    apt_install,
+    filter_installed_packages,
+)
+
 from charmhelpers.core.hookenv import (
     config,
     local_unit,
@@ -14,6 +21,9 @@
     relation_ids,
     related_units,
     unit_get,
+    unit_private_ip,
+    ERROR,
+    WARNING,
 )
 
 from charmhelpers.contrib.hahelpers.cluster import (
@@ -29,6 +39,10 @@
     get_ca_cert,
 )
 
+from charmhelpers.contrib.openstack.neutron import (
+    neutron_plugin_attribute,
+)
+
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
 
 
@@ -36,6 +50,13 @@
     pass
 
 
+def ensure_packages(packages):
+    '''Install but do not upgrade required plugin packages'''
+    required = filter_installed_packages(packages)
+    if required:
+        apt_install(required, fatal=True)
+
+
 def context_complete(ctxt):
     _missing = []
     for k, v in ctxt.iteritems():
@@ -57,30 +78,43 @@
 class SharedDBContext(OSContextGenerator):
     interfaces = ['shared-db']
 
+    def __init__(self, database=None, user=None, relation_prefix=None):
+        '''
+        Allows inspecting relation for settings prefixed with relation_prefix.
+        This is useful for parsing access for multiple databases returned via
+        the shared-db interface (eg, nova_password, quantum_password)
+        '''
+        self.relation_prefix = relation_prefix
+        self.database = database
+        self.user = user
+
     def __call__(self):
-        log('Generating template context for shared-db')
-        conf = config()
-        try:
-            database = conf['database']
-            username = conf['database-user']
-        except KeyError as e:
+        self.database = self.database or config('database')
+        self.user = self.user or config('database-user')
+        if None in [self.database, self.user]:
             log('Could not generate shared_db context. '
-                'Missing required charm config options: %s.' % e)
+                'Missing required charm config options. '
+                '(database name and user)')
             raise OSContextError
         ctxt = {}
+
+        password_setting = 'password'
+        if self.relation_prefix:
+            password_setting = self.relation_prefix + '_password'
+
         for rid in relation_ids('shared-db'):
             for unit in related_units(rid):
+                passwd = relation_get(password_setting, rid=rid, unit=unit)
                 ctxt = {
                     'database_host': relation_get('db_host', rid=rid,
                                                   unit=unit),
-                    'database': database,
-                    'database_user': username,
-                    'database_password': relation_get('password', rid=rid,
-                                                      unit=unit)
+                    'database': self.database,
+                    'database_user': self.user,
+                    'database_password': passwd,
                 }
-        if not context_complete(ctxt):
-            return {}
-        return ctxt
+                if context_complete(ctxt):
+                    return ctxt
+        return {}
 
 
 class IdentityServiceContext(OSContextGenerator):
@@ -109,9 +143,9 @@
                     'service_protocol': 'http',
                     'auth_protocol': 'http',
                 }
-        if not context_complete(ctxt):
-            return {}
-        return ctxt
+                if context_complete(ctxt):
+                    return ctxt
+        return {}
 
 
 class AMQPContext(OSContextGenerator):
@@ -132,20 +166,30 @@
         for rid in relation_ids('amqp'):
             for unit in related_units(rid):
                 if relation_get('clustered', rid=rid, unit=unit):
-                    rabbitmq_host = relation_get('vip', rid=rid, unit=unit)
+                    ctxt['clustered'] = True
+                    ctxt['rabbitmq_host'] = relation_get('vip', rid=rid,
+                                                         unit=unit)
                 else:
-                    rabbitmq_host = relation_get('private-address',
-                                                 rid=rid, unit=unit)
-                ctxt = {
-                    'rabbitmq_host': rabbitmq_host,
+                    ctxt['rabbitmq_host'] = relation_get('private-address',
+                                                         rid=rid, unit=unit)
+                ctxt.update({
                     'rabbitmq_user': username,
                     'rabbitmq_password': relation_get('password', rid=rid,
                                                       unit=unit),
                     'rabbitmq_virtual_host': vhost,
-                }
+                })
+                if context_complete(ctxt):
+                    # Sufficient information found = break out!
+                    break
+            # Used for active/active rabbitmq >= grizzly
+            ctxt['rabbitmq_hosts'] = []
+            for unit in related_units(rid):
+                ctxt['rabbitmq_hosts'].append(relation_get('private-address',
+                                                           rid=rid, unit=unit))
         if not context_complete(ctxt):
             return {}
-        return ctxt
+        else:
+            return ctxt
 
 
 class CephContext(OSContextGenerator):
@@ -153,21 +197,33 @@
 
     def __call__(self):
         '''This generates context for /etc/ceph/ceph.conf templates'''
-        log('Generating tmeplate context for ceph')
+        if not relation_ids('ceph'):
+            return {}
+        log('Generating template context for ceph')
         mon_hosts = []
         auth = None
+        key = None
         for rid in relation_ids('ceph'):
             for unit in related_units(rid):
                 mon_hosts.append(relation_get('private-address', rid=rid,
                                               unit=unit))
                 auth = relation_get('auth', rid=rid, unit=unit)
+                key = relation_get('key', rid=rid, unit=unit)
 
         ctxt = {
             'mon_hosts': ' '.join(mon_hosts),
             'auth': auth,
+            'key': key,
         }
+
+        if not os.path.isdir('/etc/ceph'):
+            os.mkdir('/etc/ceph')
+
         if not context_complete(ctxt):
             return {}
+
+        ensure_packages(['ceph-common'])
+
         return ctxt
 
 
@@ -207,7 +263,7 @@
 
 
 class ImageServiceContext(OSContextGenerator):
-    interfaces = ['image-servce']
+    interfaces = ['image-service']
 
     def __call__(self):
         '''
@@ -269,6 +325,7 @@
         if ca_cert:
             with open(CA_CERT_PATH, 'w') as ca_out:
                 ca_out.write(b64decode(ca_cert))
+            check_call(['update-ca-certificates'])
 
     def __call__(self):
         if isinstance(self.external_ports, basestring):
@@ -292,3 +349,174 @@
             portmap = (int(ext_port), int(int_port))
             ctxt['endpoints'].append(portmap)
         return ctxt
+
+
+class NeutronContext(object):
+    interfaces = []
+
+    @property
+    def plugin(self):
+        return None
+
+    @property
+    def network_manager(self):
+        return None
+
+    @property
+    def packages(self):
+        return neutron_plugin_attribute(
+            self.plugin, 'packages', self.network_manager)
+
+    @property
+    def neutron_security_groups(self):
+        return None
+
+    def _ensure_packages(self):
+        [ensure_packages(pkgs) for pkgs in self.packages]
+
+    def _save_flag_file(self):
+        if self.network_manager == 'quantum':
+            _file = '/etc/nova/quantum_plugin.conf'
+        else:
+            _file = '/etc/nova/neutron_plugin.conf'
+        with open(_file, 'wb') as out:
+            out.write(self.plugin + '\n')
+
+    def ovs_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+
+        ovs_ctxt = {
+            'core_plugin': driver,
+            'neutron_plugin': 'ovs',
+            'neutron_security_groups': self.neutron_security_groups,
+            'local_ip': unit_private_ip(),
+        }
+
+        return ovs_ctxt
+
+    def __call__(self):
+        self._ensure_packages()
+
+        if self.network_manager not in ['quantum', 'neutron']:
+            return {}
+
+        if not self.plugin:
+            return {}
+
+        ctxt = {'network_manager': self.network_manager}
+
+        if self.plugin == 'ovs':
+            ctxt.update(self.ovs_ctxt())
+
+        self._save_flag_file()
+        return ctxt
+
+
+class OSConfigFlagContext(OSContextGenerator):
+        '''
+        Responsible adding user-defined config-flags in charm config to a
+        to a template context.
+        '''
+        def __call__(self):
+            config_flags = config('config-flags')
+            if not config_flags or config_flags in ['None', '']:
+                return {}
+            config_flags = config_flags.split(',')
+            flags = {}
+            for flag in config_flags:
+                if '=' not in flag:
+                    log('Improperly formatted config-flag, expected k=v '
+                        'got %s' % flag, level=WARNING)
+                    continue
+                k, v = flag.split('=')
+                flags[k.strip()] = v
+            ctxt = {'user_config_flags': flags}
+            return ctxt
+
+
+class SubordinateConfigContext(OSContextGenerator):
+    """
+    Responsible for inspecting relations to subordinates that
+    may be exporting required config via a json blob.
+
+    The subordinate interface allows subordinates to export their
+    configuration requirements to the principle for multiple config
+    files and multiple serivces.  Ie, a subordinate that has interfaces
+    to both glance and nova may export to following yaml blob as json:
+
+        glance:
+            /etc/glance/glance-api.conf:
+                sections:
+                    DEFAULT:
+                        - [key1, value1]
+            /etc/glance/glance-registry.conf:
+                    MYSECTION:
+                        - [key2, value2]
+        nova:
+            /etc/nova/nova.conf:
+                sections:
+                    DEFAULT:
+                        - [key3, value3]
+
+
+    It is then up to the principle charms to subscribe this context to
+    the service+config file it is interestd in.  Configuration data will
+    be available in the template context, in glance's case, as:
+        ctxt = {
+            ... other context ...
+            'subordinate_config': {
+                'DEFAULT': {
+                    'key1': 'value1',
+                },
+                'MYSECTION': {
+                    'key2': 'value2',
+                },
+            }
+        }
+
+    """
+    def __init__(self, service, config_file, interface):
+        """
+        :param service     : Service name key to query in any subordinate
+                             data found
+        :param config_file : Service's config file to query sections
+        :param interface   : Subordinate interface to inspect
+        """
+        self.service = service
+        self.config_file = config_file
+        self.interface = interface
+
+    def __call__(self):
+        ctxt = {}
+        for rid in relation_ids(self.interface):
+            for unit in related_units(rid):
+                sub_config = relation_get('subordinate_configuration',
+                                          rid=rid, unit=unit)
+                if sub_config and sub_config != '':
+                    try:
+                        sub_config = json.loads(sub_config)
+                    except:
+                        log('Could not parse JSON from subordinate_config '
+                            'setting from %s' % rid, level=ERROR)
+                        continue
+
+                    if self.service not in sub_config:
+                        log('Found subordinate_config on %s but it contained'
+                            'nothing for %s service' % (rid, self.service))
+                        continue
+
+                    sub_config = sub_config[self.service]
+                    if self.config_file not in sub_config:
+                        log('Found subordinate_config on %s but it contained'
+                            'nothing for %s' % (rid, self.config_file))
+                        continue
+
+                    sub_config = sub_config[self.config_file]
+                    for k, v in sub_config.iteritems():
+                        ctxt[k] = v
+
+        if not ctxt:
+            ctxt['sections'] = {}
+
+        return ctxt

=== added file 'charmhelpers/contrib/openstack/neutron.py'
--- charmhelpers/contrib/openstack/neutron.py	1970-01-01 00:00:00 +0000
+++ charmhelpers/contrib/openstack/neutron.py	2013-10-15 01:15:14 +0000
@@ -0,0 +1,117 @@
+# Various utilies for dealing with Neutron and the renaming from Quantum.
+
+from subprocess import check_output
+
+from charmhelpers.core.hookenv import (
+    config,
+    log,
+    ERROR,
+)
+
+from charmhelpers.contrib.openstack.utils import os_release
+
+
+def headers_package():
+    """Ensures correct linux-headers for running kernel are installed,
+    for building DKMS package"""
+    kver = check_output(['uname', '-r']).strip()
+    return 'linux-headers-%s' % kver
+
+
+# legacy
+def quantum_plugins():
+    from charmhelpers.contrib.openstack import context
+    return {
+        'ovs': {
+            'config': '/etc/quantum/plugins/openvswitch/'
+                      'ovs_quantum_plugin.ini',
+            'driver': 'quantum.plugins.openvswitch.ovs_quantum_plugin.'
+                      'OVSQuantumPluginV2',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron')],
+            'services': ['quantum-plugin-openvswitch-agent'],
+            'packages': [[headers_package(), 'openvswitch-datapath-dkms'],
+                         ['quantum-plugin-openvswitch-agent']],
+        },
+        'nvp': {
+            'config': '/etc/quantum/plugins/nicira/nvp.ini',
+            'driver': 'quantum.plugins.nicira.nicira_nvp_plugin.'
+                      'QuantumPlugin.NvpPluginV2',
+            'services': [],
+            'packages': [],
+        }
+    }
+
+
+def neutron_plugins():
+    from charmhelpers.contrib.openstack import context
+    return {
+        'ovs': {
+            'config': '/etc/neutron/plugins/openvswitch/'
+                      'ovs_neutron_plugin.ini',
+            'driver': 'neutron.plugins.openvswitch.ovs_neutron_plugin.'
+                      'OVSNeutronPluginV2',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron')],
+            'services': ['neutron-plugin-openvswitch-agent'],
+            'packages': [[headers_package(), 'openvswitch-datapath-dkms'],
+                         ['quantum-plugin-openvswitch-agent']],
+        },
+        'nvp': {
+            'config': '/etc/neutron/plugins/nicira/nvp.ini',
+            'driver': 'neutron.plugins.nicira.nicira_nvp_plugin.'
+                      'NeutronPlugin.NvpPluginV2',
+            'services': [],
+            'packages': [],
+        }
+    }
+
+
+def neutron_plugin_attribute(plugin, attr, net_manager=None):
+    manager = net_manager or network_manager()
+    if manager == 'quantum':
+        plugins = quantum_plugins()
+    elif manager == 'neutron':
+        plugins = neutron_plugins()
+    else:
+        log('Error: Network manager does not support plugins.')
+        raise Exception
+
+    try:
+        _plugin = plugins[plugin]
+    except KeyError:
+        log('Unrecognised plugin for %s: %s' % (manager, plugin), level=ERROR)
+        raise Exception
+
+    try:
+        return _plugin[attr]
+    except KeyError:
+        return None
+
+
+def network_manager():
+    '''
+    Deals with the renaming of Quantum to Neutron in H and any situations
+    that require compatability (eg, deploying H with network-manager=quantum,
+    upgrading from G).
+    '''
+    release = os_release('nova-common')
+    manager = config('network-manager').lower()
+
+    if manager not in ['quantum', 'neutron']:
+        return manager
+
+    if release in ['essex']:
+        # E does not support neutron
+        log('Neutron networking not supported in Essex.', level=ERROR)
+        raise Exception
+    elif release in ['folsom', 'grizzly']:
+        # neutron is named quantum in F and G
+        return 'quantum'
+    else:
+        # ensure accurate naming for all releases post-H
+        return 'neutron'

=== added symlink 'charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf'
=== target is u'openstack_https_frontend'
=== modified file 'charmhelpers/contrib/openstack/templating.py'
--- charmhelpers/contrib/openstack/templating.py	2013-08-07 23:14:14 +0000
+++ charmhelpers/contrib/openstack/templating.py	2013-10-15 01:15:14 +0000
@@ -11,10 +11,10 @@
 from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES
 
 try:
-    from jinja2 import FileSystemLoader, ChoiceLoader, Environment
+    from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
 except ImportError:
     # python-jinja2 may not be installed yet, or we're running unittests.
-    FileSystemLoader = ChoiceLoader = Environment = None
+    FileSystemLoader = ChoiceLoader = Environment = exceptions = None
 
 
 class OSConfigException(Exception):
@@ -220,9 +220,24 @@
             log('Config not registered: %s' % config_file, level=ERROR)
             raise OSConfigException
         ctxt = self.templates[config_file].context()
+
         _tmpl = os.path.basename(config_file)
+        try:
+            template = self._get_template(_tmpl)
+        except exceptions.TemplateNotFound:
+            # if no template is found with basename, try looking for it
+            # using a munged full path, eg:
+            #   /etc/apache2/apache2.conf -> etc_apache2_apache2.conf
+            _tmpl = '_'.join(config_file.split('/')[1:])
+            try:
+                template = self._get_template(_tmpl)
+            except exceptions.TemplateNotFound as e:
+                log('Could not load template from %s by %s or %s.' %
+                    (self.templates_dir, os.path.basename(config_file), _tmpl),
+                    level=ERROR)
+                raise e
+
         log('Rendering from template: %s' % _tmpl, level=INFO)
-        template = self._get_template(_tmpl)
         return template.render(ctxt)
 
     def write(self, config_file):
@@ -232,8 +247,12 @@
         if config_file not in self.templates:
             log('Config not registered: %s' % config_file, level=ERROR)
             raise OSConfigException
+
+        _out = self.render(config_file)
+
         with open(config_file, 'wb') as out:
-            out.write(self.render(config_file))
+            out.write(_out)
+
         log('Wrote template %s.' % config_file, level=INFO)
 
     def write_all(self):

=== modified file 'charmhelpers/contrib/openstack/utils.py'
--- charmhelpers/contrib/openstack/utils.py	2013-08-23 14:12:19 +0000
+++ charmhelpers/contrib/openstack/utils.py	2013-10-15 01:15:14 +0000
@@ -1,12 +1,12 @@
 #!/usr/bin/python
 
 # Common python helper functions used for OpenStack charms.
-
 from collections import OrderedDict
 
 import apt_pkg as apt
 import subprocess
 import os
+import socket
 import sys
 
 from charmhelpers.core.hookenv import (
@@ -45,16 +45,17 @@
 ])
 
 # The ugly duckling
-SWIFT_CODENAMES = {
-    '1.4.3': 'diablo',
-    '1.4.8': 'essex',
-    '1.7.4': 'folsom',
-    '1.7.6': 'grizzly',
-    '1.7.7': 'grizzly',
-    '1.8.0': 'grizzly',
-    '1.9.0': 'havana',
-    '1.9.1': 'havana',
-}
+SWIFT_CODENAMES = OrderedDict([
+    ('1.4.3', 'diablo'),
+    ('1.4.8', 'essex'),
+    ('1.7.4', 'folsom'),
+    ('1.8.0', 'grizzly'),
+    ('1.7.7', 'grizzly'),
+    ('1.7.6', 'grizzly'),
+    ('1.10.0', 'havana'),
+    ('1.9.1', 'havana'),
+    ('1.9.0', 'havana'),
+])
 
 
 def error_out(msg):
@@ -137,8 +138,11 @@
 
     try:
         if 'swift' in pkg.name:
-            vers = vers[:5]
-            return SWIFT_CODENAMES[vers]
+            swift_vers = vers[:5]
+            if swift_vers not in SWIFT_CODENAMES:
+                # Deal with 1.10.0 upward
+                swift_vers = vers[:6]
+            return SWIFT_CODENAMES[swift_vers]
         else:
             vers = vers[:6]
             return OPENSTACK_CODENAMES[vers]
@@ -166,6 +170,25 @@
     #error_out(e)
 
 
+os_rel = None
+
+
+def os_release(package, base='essex'):
+    '''
+    Returns OpenStack release codename from a cached global.
+    If the codename can not be determined from either an installed package or
+    the installation source, the earliest release supported by the charm should
+    be returned.
+    '''
+    global os_rel
+    if os_rel:
+        return os_rel
+    os_rel = (get_os_codename_package(package, fatal=False) or
+              get_os_codename_install_source(config('openstack-origin')) or
+              base)
+    return os_rel
+
+
 def import_key(keyid):
     cmd = "apt-key adv --keyserver keyserver.ubuntu.com " \
           "--recv-keys %s" % keyid
@@ -274,3 +297,69 @@
     available_vers = get_os_version_install_source(src)
     apt.init()
     return apt.version_compare(available_vers, cur_vers) == 1
+
+
+def is_ip(address):
+    """
+    Returns True if address is a valid IP address.
+    """
+    try:
+        # Test to see if already an IPv4 address
+        socket.inet_aton(address)
+        return True
+    except socket.error:
+        return False
+
+
+def ns_query(address):
+    try:
+        import dns.resolver
+    except ImportError:
+        apt_install('python-dnspython')
+        import dns.resolver
+
+    if isinstance(address, dns.name.Name):
+        rtype = 'PTR'
+    elif isinstance(address, basestring):
+        rtype = 'A'
+
+    answers = dns.resolver.query(address, rtype)
+    if answers:
+        return str(answers[0])
+    return None
+
+
+def get_host_ip(hostname):
+    """
+    Resolves the IP for a given hostname, or returns
+    the input if it is already an IP.
+    """
+    if is_ip(hostname):
+        return hostname
+
+    return ns_query(hostname)
+
+
+def get_hostname(address):
+    """
+    Resolves hostname for given IP, or returns the input
+    if it is already a hostname.
+    """
+    if not is_ip(address):
+        return address
+
+    try:
+        import dns.reversename
+    except ImportError:
+        apt_install('python-dnspython')
+        import dns.reversename
+
+    rev = dns.reversename.from_address(address)
+    result = ns_query(rev)
+    if not result:
+        return None
+
+    # strip trailing .
+    if result.endswith('.'):
+        return result[:-1]
+    return result

=== modified file 'charmhelpers/contrib/storage/linux/ceph.py'
--- charmhelpers/contrib/storage/linux/ceph.py	2013-09-24 14:46:53 +0000
+++ charmhelpers/contrib/storage/linux/ceph.py	2013-10-15 01:15:14 +0000
@@ -335,3 +335,25 @@
             log('ceph: Starting service {} after migrating data.'
                 .format(svc))
             service_start(svc)
+
+
+def ensure_ceph_keyring(service, user=None, group=None):
+    '''
+    Ensures a ceph keyring is created for a named service
+    and optionally ensures user and group ownership.
+
+    Returns False if no ceph key is available in relation state.
+    '''
+    key = None
+    for rid in relation_ids('ceph'):
+        for unit in related_units(rid):
+            key = relation_get('key', rid=rid, unit=unit)
+            if key:
+                break
+    if not key:
+        return False
+    create_keyring(service=service, key=key)
+    keyring = _keyring_path(service)
+    if user and group:
+        check_call(['chown', '%s.%s' % (user, group), keyring])
+    return True

=== modified file 'tests/contrib/openstack/test_openstack_utils.py'
--- tests/contrib/openstack/test_openstack_utils.py	2013-07-18 11:13:34 +0000
+++ tests/contrib/openstack/test_openstack_utils.py	2013-10-15 01:15:14 +0000
@@ -43,6 +43,11 @@
         'os_release': 'havana',
         'os_version': '1.9.0'
     },
+    'swift-proxy': {
+        'pkg_vers': '1.10.0~rc1-0ubuntu1',
+        'os_release': 'havana',
+        'os_version': '1.10.0'
+    },
     # a package thats available in the cache but is not installed
     'cinder-common': {
         'os_release': 'havana',
@@ -71,6 +76,36 @@
 ]
 
 
+# Mock python-dnspython resolver used by get_host_ip()
+class FakeAnswer(object):
+    def __init__(self, ip):
+        self.ip = ip
+
+    def __str__(self):
+        return self.ip
+
+
+class FakeResolver(object):
+    def __init__(self, ip):
+        self.ip = ip
+
+    def query(self, hostname, query_type):
+        return [FakeAnswer(self.ip)]
+
+
+class FakeReverse(object):
+    def from_address(self, address):
+        return '156.94.189.91.in-addr.arpa'
+
+
+class FakeDNS(object):
+    def __init__(self, ip):
+        self.resolver = FakeResolver(ip)
+        self.reversename = FakeReverse()
+        self.name = MagicMock()
+        self.name.Name = basestring
+
+
 class OpenStackHelpersTestCase(TestCase):
     def _apt_cache(self):
         # mocks out the apt cache
@@ -266,6 +301,16 @@
                 openstack.get_os_version_package('foo', fatal=False)
             )
 
+    @patch.object(openstack, 'get_os_codename_package')
+    def test_os_release_uncached(self, get_cn):
+        openstack.os_rel = None
+        get_cn.return_value = 'folsom'
+        self.assertEquals('folsom', openstack.os_release('nova-common'))
+
+    def test_os_release_cached(self):
+        openstack.os_rel = 'foo'
+        self.assertEquals('foo', openstack.os_release('nova-common'))
+
     @patch.object(openstack, 'juju_log')
     @patch('sys.exit')
     def test_error_out(self, mocked_exit, juju_log):
@@ -371,10 +416,11 @@
                    '--recv-keys', 'foo']
         mocked_error.assert_called_with('Error importing repo key foo')
 
+    @patch('os.mkdir')
     @patch('os.path.exists')
     @patch('charmhelpers.contrib.openstack.utils.charm_dir')
     @patch('__builtin__.open')
-    def test_save_scriptrc(self, _open, _charm_dir, _exists):
+    def test_save_scriptrc(self, _open, _charm_dir, _exists, _mkdir):
         '''Test generation of scriptrc from environment'''
         scriptrc = ['#!/bin/bash\n',
                     'export setting1=foo\n',
@@ -382,11 +428,12 @@
         _file = MagicMock(spec=file)
         _open.return_value = _file
         _charm_dir.return_value = '/var/lib/juju/units/testing-foo-0/charm'
-        _exists.return_value = True
+        _exists.return_value = False
         os.environ['JUJU_UNIT_NAME'] = 'testing-foo/0'
         openstack.save_script_rc(setting1='foo', setting2='bar')
         expected_f = '/var/lib/juju/units/testing-foo-0/charm/scripts/scriptrc'
         _open.assert_called_with(expected_f, 'wb')
+        _mkdir.assert_called_with(os.path.dirname(expected_f))
         for line in scriptrc:
             _file.__enter__().write.assert_has_calls(call(line))
 
@@ -416,6 +463,37 @@
         vers_pkg.return_value = '2013.1~b1'
         self.assertFalse(openstack.openstack_upgrade_available('nova-common'))
 
+    def test_is_ip(self):
+        self.assertTrue(openstack.is_ip('10.0.0.1'))
+        self.assertFalse(openstack.is_ip('www.ubuntu.com'))
+
+    @patch.object(openstack, 'apt_install')
+    def test_get_host_ip_with_hostname(self, apt_install):
+        fake_dns = FakeDNS('10.0.0.1')
+        with patch('__builtin__.__import__', side_effect=[fake_dns]):
+            ip = openstack.get_host_ip('www.ubuntu.com')
+        self.assertEquals(ip, '10.0.0.1')
+
+    @patch.object(openstack, 'apt_install')
+    def test_get_host_ip_with_ip(self, apt_install):
+        fake_dns = FakeDNS('5.5.5.5')
+        with patch('__builtin__.__import__', side_effect=[fake_dns]):
+            ip = openstack.get_host_ip('4.2.2.1')
+        self.assertEquals(ip, '4.2.2.1')
+
+    @patch.object(openstack, 'apt_install')
+    def test_get_hostname_with_ip(self, apt_install):
+        fake_dns = FakeDNS('www.ubuntu.com')
+        with patch('__builtin__.__import__', side_effect=[fake_dns, fake_dns]):
+            hn = openstack.get_hostname('4.2.2.1')
+        self.assertEquals(hn, 'www.ubuntu.com')
+
+    @patch.object(openstack, 'apt_install')
+    def test_get_hostname_with_hostname(self, apt_install):
+        fake_dns = FakeDNS('5.5.5.5')
+        with patch('__builtin__.__import__', side_effect=[fake_dns]):
+            hn = openstack.get_hostname('www.ubuntu.com')
+        self.assertEquals(hn, 'www.ubuntu.com')
 
 if __name__ == '__main__':
     unittest.main()

=== modified file 'tests/contrib/openstack/test_os_contexts.py'
--- tests/contrib/openstack/test_os_contexts.py	2013-07-19 23:29:59 +0000
+++ tests/contrib/openstack/test_os_contexts.py	2013-10-15 01:15:14 +0000
@@ -1,3 +1,5 @@
+import yaml
+import json
 import unittest
 
 from mock import patch, MagicMock, call
@@ -109,6 +111,18 @@
     'vip': '10.0.0.1',
 }
 
+AMQP_AA_RELATION = {
+    'amqp:0': {
+        'rabbitmq/0': {
+            'private-address': 'rabbithost1',
+            'password': 'foobar',
+        },
+        'rabbitmq/1': {
+            'private-address': 'rabbithost2',
+        }
+    }
+}
+
 AMQP_CONFIG = {
     'rabbit-user': 'adam',
     'rabbit-vhost': 'foo',
@@ -119,14 +133,51 @@
         'ceph/0': {
             'private-address': 'ceph_node1',
             'auth': 'foo',
+            'key': 'bar',
         },
         'ceph/1': {
             'private-address': 'ceph_node2',
             'auth': 'foo',
-        },
-    }
-}
-
+            'key': 'bar',
+        },
+    }
+}
+
+SUB_CONFIG = """
+nova:
+    /etc/nova/nova.conf:
+        sections:
+            DEFAULT:
+                - [nova-key1, value1]
+                - [nova-key2, value2]
+glance:
+    /etc/glance/glance.conf:
+        sections:
+            DEFAULT:
+                - [glance-key1, value1]
+                - [glance-key2, value2]
+"""
+
+SUB_CONFIG_RELATION = {
+    'nova-subordinate:0': {
+        'nova-subordinate/0': {
+            'private-address': 'nova_node1',
+            'subordinate_configuration': json.dumps(yaml.load(SUB_CONFIG)),
+        },
+    },
+    'glance-subordinate:0': {
+        'glance-subordinate/0': {
+            'private-address': 'glance_node1',
+            'subordinate_configuration': json.dumps(yaml.load(SUB_CONFIG)),
+        },
+    },
+    'foo-subordinate:0': {
+        'foo-subordinate/0': {
+            'private-address': 'foo_node1',
+            'subordinate_configuration': 'ea8e09324jkadsfh',
+        },
+    }
+}
 
 # Imported in contexts.py and needs patching in setUp()
 TO_PATCH = [
@@ -148,6 +199,16 @@
 ]
 
 
+class fake_config(object):
+    def __init__(self, data):
+        self.data = data
+
+    def __call__(self, attr):
+        if attr in self.data:
+            return self.data[attr]
+        return None
+
+
 class ContextTests(unittest.TestCase):
     def setUp(self):
         for m in TO_PATCH:
@@ -170,7 +231,7 @@
         '''Test shared-db context with all required data'''
         relation = FakeRelation(relation_data=SHARED_DB_RELATION)
         self.relation_get.side_effect = relation.get
-        self.config.return_value = SHARED_DB_CONFIG
+        self.config.side_effect = fake_config(SHARED_DB_CONFIG)
         shared_db = context.SharedDBContext()
         result = shared_db()
         expected = {
@@ -196,12 +257,23 @@
         '''Test shared-db context missing relation data'''
         incomplete_config = copy(SHARED_DB_CONFIG)
         del incomplete_config['database-user']
+        self.config.side_effect = fake_config(incomplete_config)
         relation = FakeRelation(relation_data=SHARED_DB_RELATION)
         self.relation_get.side_effect = relation.get
         self.config.return_value = incomplete_config
         shared_db = context.SharedDBContext()
         self.assertRaises(context.OSContextError, shared_db)
 
+    def test_shared_db_context_with_params(self):
+        '''Test shared-db context with object parameters'''
+        shared_db = context.SharedDBContext(
+            database='quantum', user='quantum', relation_prefix='quantum')
+        result = shared_db()
+        self.assertIn(call('quantum_password', rid='foo:0', unit='foo/0'),
+                      self.relation_get.call_args_list)
+        self.assertEquals(result['database'], 'quantum')
+        self.assertEquals(result['database_user'], 'quantum')
+
     def test_identity_service_context_with_data(self):
         '''Test shared-db context with all required data'''
         relation = FakeRelation(relation_data=IDENTITY_SERVICE_RELATION)
@@ -238,12 +310,12 @@
         self.config.return_value = AMQP_CONFIG
         amqp = context.AMQPContext()
         result = amqp()
-
         expected = {
             'rabbitmq_host': 'rabbithost',
             'rabbitmq_password': 'foobar',
             'rabbitmq_user': 'adam',
-            'rabbitmq_virtual_host': 'foo'
+            'rabbitmq_virtual_host': 'foo',
+            'rabbitmq_hosts': ['rabbithost'],
         }
         self.assertEquals(result, expected)
 
@@ -256,12 +328,32 @@
         self.config.return_value = AMQP_CONFIG
         amqp = context.AMQPContext()
         result = amqp()
-
         expected = {
+            'clustered': True,
             'rabbitmq_host': relation_data['vip'],
             'rabbitmq_password': 'foobar',
             'rabbitmq_user': 'adam',
-            'rabbitmq_virtual_host': 'foo'
+            'rabbitmq_virtual_host': 'foo',
+            'rabbitmq_hosts': ['rabbithost'],
+        }
+        self.assertEquals(result, expected)
+
+    def test_amqp_context_with_data_active_active(self):
+        '''Test amqp context with required data with active/active rabbit'''
+        relation_data = copy(AMQP_AA_RELATION)
+        relation = FakeRelation(relation_data=relation_data)
+        self.relation_get.side_effect = relation.get
+        self.relation_ids.side_effect = relation.relation_ids
+        self.related_units.side_effect = relation.relation_units
+        self.config.return_value = AMQP_CONFIG
+        amqp = context.AMQPContext()
+        result = amqp()
+        expected = {
+            'rabbitmq_host': 'rabbithost1',
+            'rabbitmq_password': 'foobar',
+            'rabbitmq_user': 'adam',
+            'rabbitmq_virtual_host': 'foo',
+            'rabbitmq_hosts': ['rabbithost2', 'rabbithost1'],
         }
         self.assertEquals(result, expected)
 
@@ -286,8 +378,12 @@
         amqp = context.AMQPContext()
         self.assertRaises(context.OSContextError, amqp)
 
-    def test_ceph_context_with_data(self):
+    @patch('os.path.isdir')
+    @patch('os.mkdir')
+    @patch.object(context, 'ensure_packages')
+    def test_ceph_context_with_data(self, ensure_packages, mkdir, isdir):
         '''Test ceph context with all relation data'''
+        isdir.return_value = False
         relation = FakeRelation(relation_data=CEPH_RELATION)
         self.relation_get.side_effect = relation.get
         self.relation_ids.side_effect = relation.relation_ids
@@ -296,11 +392,16 @@
         result = ceph()
         expected = {
             'mon_hosts': 'ceph_node2 ceph_node1',
-            'auth': 'foo'
+            'auth': 'foo',
+            'key': 'bar',
         }
         self.assertEquals(result, expected)
+        ensure_packages.assert_called_with(['ceph-common'])
+        mkdir.assert_called_with('/etc/ceph')
 
-    def test_ceph_context_with_missing_data(self):
+    @patch('os.mkdir')
+    @patch.object(context, 'ensure_packages')
+    def test_ceph_context_with_missing_data(self, ensure_packages, mkdir):
         '''Test ceph context with missing relation data'''
         relation = copy(CEPH_RELATION)
         for k, v in relation.iteritems():
@@ -313,6 +414,7 @@
         ceph = context.CephContext()
         result = ceph()
         self.assertEquals(result, {})
+        self.assertFalse(ensure_packages.called)
 
     @patch('charmhelpers.contrib.openstack.context.unit_get')
     @patch('charmhelpers.contrib.openstack.context.local_unit')
@@ -471,3 +573,146 @@
         self.relation_get.return_value = 'http://glancehost:9292'
         self.assertEquals({'glance_api_servers': 'http://glancehost:9292'},
                           image_service())
+
+    @patch.object(context, 'neutron_plugin_attribute')
+    def test_neutron_context_base_properties(self, attr):
+        '''Test neutron context base properties'''
+        neutron = context.NeutronContext()
+        attr.return_value = 'quantum-plugin-package'
+        self.assertEquals(None, neutron.plugin)
+        self.assertEquals(None, neutron.network_manager)
+        self.assertEquals(None, neutron.neutron_security_groups)
+        self.assertEquals('quantum-plugin-package', neutron.packages)
+
+    @patch.object(context, 'neutron_plugin_attribute')
+    @patch.object(context, 'apt_install')
+    @patch.object(context, 'filter_installed_packages')
+    def test_neutron_ensure_package(self, _filter, _install, _packages):
+        '''Test neutron context installed required packages'''
+        _filter.return_value = ['quantum-plugin-package']
+        _packages.return_value = [['quantum-plugin-package']]
+        neutron = context.NeutronContext()
+        neutron._ensure_packages()
+        _install.assert_called_with(['quantum-plugin-package'], fatal=True)
+
+    @patch.object(context.NeutronContext, 'network_manager')
+    @patch.object(context.NeutronContext, 'plugin')
+    def test_neutron_save_flag_file(self, plugin, nm):
+        neutron = context.NeutronContext()
+        plugin.__get__ = MagicMock(return_value='ovs')
+        nm.__get__ = MagicMock(return_value='quantum')
+        with patch_open() as (_o, _f):
+            neutron._save_flag_file()
+            _o.assert_called_with('/etc/nova/quantum_plugin.conf', 'wb')
+            _f.write.assert_called_with('ovs\n')
+
+        nm.__get__ = MagicMock(return_value='neutron')
+        with patch_open() as (_o, _f):
+            neutron._save_flag_file()
+            _o.assert_called_with('/etc/nova/neutron_plugin.conf', 'wb')
+            _f.write.assert_called_with('ovs\n')
+
+    @patch.object(context.NeutronContext, 'neutron_security_groups')
+    @patch.object(context, 'unit_private_ip')
+    @patch.object(context, 'neutron_plugin_attribute')
+    def test_neutron_ovs_plugin_context(self, attr, ip, sec_groups):
+        ip.return_value = '10.0.0.1'
+        sec_groups.__get__ = MagicMock(return_value=True)
+        attr.return_value = 'some.quantum.driver.class'
+        neutron = context.NeutronContext()
+        self.assertEquals({
+            'core_plugin': 'some.quantum.driver.class',
+            'neutron_plugin': 'ovs',
+            'neutron_security_groups': True,
+            'local_ip': '10.0.0.1'}, neutron.ovs_ctxt())
+
+    @patch.object(context.NeutronContext, '_save_flag_file')
+    @patch.object(context.NeutronContext, 'ovs_ctxt')
+    @patch.object(context.NeutronContext, 'plugin')
+    @patch.object(context.NeutronContext, '_ensure_packages')
+    @patch.object(context.NeutronContext, 'network_manager')
+    def test_neutron_main_context_generation(self, nm, pkgs, plugin, ovs, ff):
+        neutron = context.NeutronContext()
+        nm.__get__ = MagicMock(return_value='flatdhcpmanager')
+        self.assertEquals({}, neutron())
+
+        nm.__get__ = MagicMock(return_value='neutron')
+        plugin.__get__ = MagicMock(return_value=None)
+        self.assertEquals({}, neutron())
+
+        nm.__get__ = MagicMock(return_value='neutron')
+        ovs.return_value = {'ovs': 'ovs_context'}
+        plugin.__get__ = MagicMock(return_value='ovs')
+        self.assertEquals(
+            {'network_manager': 'neutron', 'ovs': 'ovs_context'},
+            neutron()
+        )
+
+    @patch.object(context, 'config')
+    def test_os_configflag_context(self, config):
+        flags = context.OSConfigFlagContext()
+
+        config.return_value = 'floating_ip=True,use_virtio=False,max=5'
+        self.assertEquals({
+            'user_config_flags': {
+                'floating_ip': 'True',
+                'use_virtio': 'False',
+                'max': '5',
+            }
+        }, flags())
+
+        for empty in [None, '']:
+            config.return_value = empty
+            self.assertEquals({}, flags())
+
+        config.return_value = 'good_flag=woot,badflag,great_flag=w00t'
+        self.assertEquals({
+            'user_config_flags': {
+                'good_flag': 'woot',
+                'great_flag': 'w00t',
+            }
+        }, flags())
+
+    def test_os_subordinate_config_context(self):
+        relation = FakeRelation(relation_data=SUB_CONFIG_RELATION)
+        self.relation_get.side_effect = relation.get
+        self.relation_ids.side_effect = relation.relation_ids
+        self.related_units.side_effect = relation.relation_units
+        nova_sub_ctxt = context.SubordinateConfigContext(
+            service='nova',
+            config_file='/etc/nova/nova.conf',
+            interface='nova-subordinate',
+        )
+        glance_sub_ctxt = context.SubordinateConfigContext(
+            service='glance',
+            config_file='/etc/glance/glance.conf',
+            interface='glance-subordinate',
+        )
+        foo_sub_ctxt = context.SubordinateConfigContext(
+            service='foo',
+            config_file='/etc/foo/foo.conf',
+            interface='foo-subordinate',
+        )
+        self.assertEquals(
+            nova_sub_ctxt(),
+            {'sections': {
+                'DEFAULT': [
+                    ['nova-key1', 'value1'],
+                    ['nova-key2', 'value2']]
+                }}
+        )
+        self.assertEquals(
+            glance_sub_ctxt(),
+            {'sections': {
+                'DEFAULT': [
+                    ['glance-key1', 'value1'],
+                    ['glance-key2', 'value2']]
+                }}
+        )
+
+        # subrodinate supplies nothing for given config
+        glance_sub_ctxt.config_file = '/etc/glance/glance-api-paste.ini'
+        self.assertEquals(glance_sub_ctxt(), {'sections': {}})
+
+        # subordinate supplies bad input
+        self.assertEquals(foo_sub_ctxt(), {'sections': {}})

=== modified file 'tests/contrib/openstack/test_os_templating.py'
--- tests/contrib/openstack/test_os_templating.py	2013-07-11 19:51:05 +0000
+++ tests/contrib/openstack/test_os_templating.py	2013-10-15 01:15:14 +0000
@@ -6,6 +6,8 @@
 
 import charmhelpers.contrib.openstack.templating as templating
 
+from jinja2.exceptions import TemplateNotFound
+
 
 class FakeContextGenerator(object):
     interfaces = None
@@ -114,6 +116,50 @@
             fake_tmpl.render.assert_called_with({})
             self.assertNotIn('fooservice', self.renderer.complete_contexts())
 
+    def test_render_template_registered_but_not_found(self):
+        '''It loads a template by basename of config file first'''
+        path = os.path.dirname(__file__)
+        renderer = templating.OSConfigRenderer(templates_dir=path,
+                                               openstack_release='folsom')
+        e = TemplateNotFound('')
+        renderer._get_template = MagicMock()
+        renderer._get_template.side_effect = e
+        renderer.register('/etc/nova/nova.conf', contexts=[])
+        self.assertRaises(
+            TemplateNotFound, renderer.render, '/etc/nova/nova.conf')
+
+    def test_render_template_by_basename_first(self):
+        '''It loads a template by basename of config file first'''
+        path = os.path.dirname(__file__)
+        renderer = templating.OSConfigRenderer(templates_dir=path,
+                                               openstack_release='folsom')
+        renderer._get_template = MagicMock()
+        renderer.register('/etc/nova/nova.conf', contexts=[])
+        renderer.render('/etc/nova/nova.conf')
+        self.assertEquals(1, len(renderer._get_template.call_args_list))
+        self.assertEquals(
+            [call('nova.conf')], renderer._get_template.call_args_list)
+
+    def test_render_template_by_munged_full_path_last(self):
+        '''It loads a template by full path of config file second'''
+        path = os.path.dirname(__file__)
+        renderer = templating.OSConfigRenderer(templates_dir=path,
+                                               openstack_release='folsom')
+        tmp = MagicMock()
+        tmp.render = MagicMock()
+        e = TemplateNotFound('')
+        renderer._get_template = MagicMock()
+        renderer._get_template.side_effect = [e, tmp]
+        renderer.register('/etc/nova/nova.conf', contexts=[])
+        renderer.render('/etc/nova/nova.conf')
+        self.assertEquals(2, len(renderer._get_template.call_args_list))
+        self.assertEquals(
+            [call('nova.conf'), call('etc_nova_nova.conf')],
+            renderer._get_template.call_args_list)
+
+    def test_render_template_by_basename(self):
+        '''It renders template if it finds it by config file basename'''
+
     @patch('__builtin__.open')
     @patch.object(templating, 'get_loader')
     def test_write_out_config(self, loader, _open):

=== modified file 'tests/contrib/storage/test_linux_ceph.py'
--- tests/contrib/storage/test_linux_ceph.py	2013-09-24 14:46:53 +0000
+++ tests/contrib/storage/test_linux_ceph.py	2013-10-15 01:15:14 +0000
@@ -460,3 +460,43 @@
             'ceph: Formatting block device %s as '
             'filesystem %s.' % (device, fstype), level='INFO'
         )
+
+    @patch.object(ceph_utils, 'relation_ids')
+    @patch.object(ceph_utils, 'related_units')
+    @patch.object(ceph_utils, 'relation_get')
+    def test_ensure_ceph_keyring_no_relation_no_data(self, rget, runits, rids):
+        rids.return_value = []
+        self.assertEquals(False, ceph_utils.ensure_ceph_keyring(service='foo'))
+        rids.return_value = ['ceph:0']
+        runits.return_value = ['ceph/0']
+        rget.return_value = ''
+        self.assertEquals(False, ceph_utils.ensure_ceph_keyring(service='foo'))
+
+    @patch.object(ceph_utils, '_keyring_path')
+    @patch.object(ceph_utils, 'create_keyring')
+    @patch.object(ceph_utils, 'relation_ids')
+    @patch.object(ceph_utils, 'related_units')
+    @patch.object(ceph_utils, 'relation_get')
+    def test_ensure_ceph_keyring_with_data(self, rget, runits,
+                                           rids, create, _path):
+        rids.return_value = ['ceph:0']
+        runits.return_value = ['ceph/0']
+        rget.return_value = 'fookey'
+        self.assertEquals(True,
+                          ceph_utils.ensure_ceph_keyring(service='foo'))
+        create.assert_called_with(service='foo', key='fookey')
+        _path.assert_called_with('foo')
+        self.assertFalse(self.check_call.called)
+
+        _path.return_value = '/etc/ceph/client.foo.keyring'
+        self.assertEquals(
+            True,
+            ceph_utils.ensure_ceph_keyring(
+                service='foo', user='adam', group='users'))
+        create.assert_called_with(service='foo', key='fookey')
+        _path.assert_called_with('foo')
+        self.check_call.assert_called_with([
+            'chown',
+            'adam.users',
+            '/etc/ceph/client.foo.keyring'
+        ])