Merge lp:~james-page/charms/trusty/glance/network-splits into lp:~openstack-charmers-archive/charms/trusty/glance/next

[James Page]

Add support for multiple network configuration.

[James Troup (elmo)]

This has merge conflicts artifacts in the diff.

review: Needs Fixing

[lp:~james-page/charms/trusty/glance/network-splits]

72. By James Page on 2014-07-25

Rebase

[James Page (james-page)]

review: Needs Resubmitting

=== added file '.bzrignore'
--- .bzrignore	1970-01-01 00:00:00 +0000
+++ .bzrignore	2014-07-25 09:37:37 +0000
@@ -0,0 +1,2 @@
+.coverage
+bin

=== modified file 'Makefile'
--- Makefile	2014-07-18 09:45:47 +0000
+++ Makefile	2014-07-25 09:37:37 +0000
@@ -1,4 +1,5 @@
 #!/usr/bin/make
+PYTHON := /usr/bin/env python
 
 lint:
 	@echo "Running flake8 tests: "
@@ -8,12 +9,17 @@
 	@charm proof
 	@echo "OK"
 
-sync:
-	@charm-helper-sync -c charm-helpers-hooks.yaml
-	@charm-helper-sync -c charm-helpers-tests.yaml
-
 unit_test:
-	@$(PYTHON) /usr/bin/nosetests --nologcapture --with-coverage  unit_tests
+	@$(PYTHON) /usr/bin/nosetests --nologcapture --with-coverage unit_tests
+
+bin/charm_helpers_sync.py:
+	@mkdir -p bin
+	@bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \
+        > bin/charm_helpers_sync.py
+
+sync: bin/charm_helpers_sync.py
+	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-hooks.yaml
+	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-tests.yaml
 
 test:
 	@echo Starting Amulet tests...

=== modified file 'charm-helpers-hooks.yaml'
--- charm-helpers-hooks.yaml	2014-06-24 19:54:45 +0000
+++ charm-helpers-hooks.yaml	2014-07-25 09:37:37 +0000
@@ -3,7 +3,8 @@
 include:
     - core
     - fetch
-    - contrib.openstack
+    - contrib.openstack|inc=*
     - contrib.hahelpers
     - contrib.storage.linux.ceph
     - payload.execd
+    - contrib.network.ip

=== modified file 'config.yaml'
--- config.yaml	2014-04-12 16:55:29 +0000
+++ config.yaml	2014-07-25 09:37:37 +0000
@@ -52,15 +52,11 @@
   # HA configuration settings
   vip:
     type: string
-    description: "Virtual IP to use to front Glance API in ha configuration"
-  vip_iface:
-    type: string
-    default: eth0
-    description: "Network Interface where to place the Virtual IP"
-  vip_cidr:
-    type: int
-    default: 24
-    description: "Netmask that will be used for the Virtual IP"
+    description: |
+      Virtual IP(s) to use to front API services in HA configuration.
+      .
+      If multiple networks are being used, a VIP should be provided for each
+      network, separated by spaces.
   ha-bindiface:
     type: string
     default: eth0
@@ -96,4 +92,27 @@
     default: openstack
     type: string
     description: RabbitMQ virtual host to request access on rabbitmq-server.
+  # Network configuration options
+  # by default all access is over 'private-address'
+  os-admin-network:
+    type: string
+    description: |
+      The IP address and netmask of the OpenStack Admin network (e.g.,
+      192.168.0.0/24)
+      .
+      This network will be used for admin endpoints.
+  os-internal-network:
+    type: string
+    description: |
+      The IP address and netmask of the OpenStack Internal network (e.g.,
+      192.168.0.0/24)
+      .
+      This network will be used for internal endpoints.
+  os-public-network:
+    type: string
+    description: |
+      The IP address and netmask of the OpenStack Public network (e.g.,
+      192.168.0.0/24)
+      .
+      This network will be used for public endpoints.
 

=== modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
--- hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-07-10 21:43:51 +0000
+++ hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-07-25 09:37:37 +0000
@@ -146,12 +146,12 @@
     Obtains all relevant configuration from charm configuration required
     for initiating a relation to hacluster:
 
-        ha-bindiface, ha-mcastport, vip, vip_iface, vip_cidr
+        ha-bindiface, ha-mcastport, vip
 
     returns: dict: A dict containing settings keyed by setting name.
     raises: HAIncompleteConfig if settings are missing.
     '''
-    settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'vip_iface', 'vip_cidr']
+    settings = ['ha-bindiface', 'ha-mcastport', 'vip']
     conf = {}
     for setting in settings:
         conf[setting] = config_get(setting)

=== added directory 'hooks/charmhelpers/contrib/network'
=== added file 'hooks/charmhelpers/contrib/network/__init__.py'
=== added file 'hooks/charmhelpers/contrib/network/ip.py'
--- hooks/charmhelpers/contrib/network/ip.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/network/ip.py	2014-07-25 09:37:37 +0000
@@ -0,0 +1,156 @@
+import sys
+
+from functools import partial
+
+from charmhelpers.fetch import apt_install
+from charmhelpers.core.hookenv import (
+    ERROR, log,
+)
+
+try:
+    import netifaces
+except ImportError:
+    apt_install('python-netifaces')
+    import netifaces
+
+try:
+    import netaddr
+except ImportError:
+    apt_install('python-netaddr')
+    import netaddr
+
+
+def _validate_cidr(network):
+    try:
+        netaddr.IPNetwork(network)
+    except (netaddr.core.AddrFormatError, ValueError):
+        raise ValueError("Network (%s) is not in CIDR presentation format" %
+                         network)
+
+
+def get_address_in_network(network, fallback=None, fatal=False):
+    """
+    Get an IPv4 or IPv6 address within the network from the host.
+
+    :param network (str): CIDR presentation format. For example,
+        '192.168.1.0/24'.
+    :param fallback (str): If no address is found, return fallback.
+    :param fatal (boolean): If no address is found, fallback is not
+        set and fatal is True then exit(1).
+
+    """
+
+    def not_found_error_out():
+        log("No IP address found in network: %s" % network,
+            level=ERROR)
+        sys.exit(1)
+
+    if network is None:
+        if fallback is not None:
+            return fallback
+        else:
+            if fatal:
+                not_found_error_out()
+
+    _validate_cidr(network)
+    network = netaddr.IPNetwork(network)
+    for iface in netifaces.interfaces():
+        addresses = netifaces.ifaddresses(iface)
+        if network.version == 4 and netifaces.AF_INET in addresses:
+            addr = addresses[netifaces.AF_INET][0]['addr']
+            netmask = addresses[netifaces.AF_INET][0]['netmask']
+            cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
+            if cidr in network:
+                return str(cidr.ip)
+        if network.version == 6 and netifaces.AF_INET6 in addresses:
+            for addr in addresses[netifaces.AF_INET6]:
+                if not addr['addr'].startswith('fe80'):
+                    cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
+                                                        addr['netmask']))
+                    if cidr in network:
+                        return str(cidr.ip)
+
+    if fallback is not None:
+        return fallback
+
+    if fatal:
+        not_found_error_out()
+
+    return None
+
+
+def is_ipv6(address):
+    '''Determine whether provided address is IPv6 or not'''
+    try:
+        address = netaddr.IPAddress(address)
+    except netaddr.AddrFormatError:
+        # probably a hostname - so not an address at all!
+        return False
+    else:
+        return address.version == 6
+
+
+def is_address_in_network(network, address):
+    """
+    Determine whether the provided address is within a network range.
+
+    :param network (str): CIDR presentation format. For example,
+        '192.168.1.0/24'.
+    :param address: An individual IPv4 or IPv6 address without a net
+        mask or subnet prefix. For example, '192.168.1.1'.
+    :returns boolean: Flag indicating whether address is in network.
+    """
+    try:
+        network = netaddr.IPNetwork(network)
+    except (netaddr.core.AddrFormatError, ValueError):
+        raise ValueError("Network (%s) is not in CIDR presentation format" %
+                         network)
+    try:
+        address = netaddr.IPAddress(address)
+    except (netaddr.core.AddrFormatError, ValueError):
+        raise ValueError("Address (%s) is not in correct presentation format" %
+                         address)
+    if address in network:
+        return True
+    else:
+        return False
+
+
+def _get_for_address(address, key):
+    """Retrieve an attribute of or the physical interface that
+    the IP address provided could be bound to.
+
+    :param address (str): An individual IPv4 or IPv6 address without a net
+        mask or subnet prefix. For example, '192.168.1.1'.
+    :param key: 'iface' for the physical interface name or an attribute
+        of the configured interface, for example 'netmask'.
+    :returns str: Requested attribute or None if address is not bindable.
+    """
+    address = netaddr.IPAddress(address)
+    for iface in netifaces.interfaces():
+        addresses = netifaces.ifaddresses(iface)
+        if address.version == 4 and netifaces.AF_INET in addresses:
+            addr = addresses[netifaces.AF_INET][0]['addr']
+            netmask = addresses[netifaces.AF_INET][0]['netmask']
+            cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
+            if address in cidr:
+                if key == 'iface':
+                    return iface
+                else:
+                    return addresses[netifaces.AF_INET][0][key]
+        if address.version == 6 and netifaces.AF_INET6 in addresses:
+            for addr in addresses[netifaces.AF_INET6]:
+                if not addr['addr'].startswith('fe80'):
+                    cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
+                                                        addr['netmask']))
+                    if address in cidr:
+                        if key == 'iface':
+                            return iface
+                        else:
+                            return addr[key]
+    return None
+
+
+get_iface_for_address = partial(_get_for_address, key='iface')
+
+get_netmask_for_address = partial(_get_for_address, key='netmask')

=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2014-07-10 21:43:51 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2014-07-25 09:37:37 +0000
@@ -21,6 +21,7 @@
     relation_get,
     relation_ids,
     related_units,
+    relation_set,
     unit_get,
     unit_private_ip,
     ERROR,
@@ -43,6 +44,8 @@
     neutron_plugin_attribute,
 )
 
+from charmhelpers.contrib.network.ip import get_address_in_network
+
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
 
 
@@ -135,8 +138,26 @@
                 'Missing required charm config options. '
                 '(database name and user)')
             raise OSContextError
+
         ctxt = {}
 
+        # NOTE(jamespage) if mysql charm provides a network upon which
+        # access to the database should be made, reconfigure relation
+        # with the service units local address and defer execution
+        access_network = relation_get('access-network')
+        if access_network is not None:
+            if self.relation_prefix is not None:
+                hostname_key = "{}_hostname".format(self.relation_prefix)
+            else:
+                hostname_key = "hostname"
+            access_hostname = get_address_in_network(access_network,
+                                                     unit_get('private-address'))
+            set_hostname = relation_get(attribute=hostname_key,
+                                        unit=local_unit())
+            if set_hostname != access_hostname:
+                relation_set(relation_settings={hostname_key: access_hostname})
+                return ctxt  # Defer any further hook execution for now....
+
         password_setting = 'password'
         if self.relation_prefix:
             password_setting = self.relation_prefix + '_password'
@@ -341,10 +362,12 @@
         use_syslog = str(config('use-syslog')).lower()
         for rid in relation_ids('ceph'):
             for unit in related_units(rid):
-                mon_hosts.append(relation_get('private-address', rid=rid,
-                                              unit=unit))
                 auth = relation_get('auth', rid=rid, unit=unit)
                 key = relation_get('key', rid=rid, unit=unit)
+                ceph_addr = \
+                    relation_get('ceph-public-address', rid=rid, unit=unit) or \
+                    relation_get('private-address', rid=rid, unit=unit)
+                mon_hosts.append(ceph_addr)
 
         ctxt = {
             'mon_hosts': ' '.join(mon_hosts),
@@ -378,7 +401,9 @@
 
         cluster_hosts = {}
         l_unit = local_unit().replace('/', '-')
-        cluster_hosts[l_unit] = unit_get('private-address')
+        cluster_hosts[l_unit] = \
+            get_address_in_network(config('os-internal-network'),
+                                   unit_get('private-address'))
 
         for rid in relation_ids('cluster'):
             for unit in related_units(rid):

=== added file 'hooks/charmhelpers/contrib/openstack/ip.py'
--- hooks/charmhelpers/contrib/openstack/ip.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/ip.py	2014-07-25 09:37:37 +0000
@@ -0,0 +1,75 @@
+from charmhelpers.core.hookenv import (
+    config,
+    unit_get,
+)
+
+from charmhelpers.contrib.network.ip import (
+    get_address_in_network,
+    is_address_in_network,
+    is_ipv6,
+)
+
+from charmhelpers.contrib.hahelpers.cluster import is_clustered
+
+PUBLIC = 'public'
+INTERNAL = 'int'
+ADMIN = 'admin'
+
+_address_map = {
+    PUBLIC: {
+        'config': 'os-public-network',
+        'fallback': 'public-address'
+    },
+    INTERNAL: {
+        'config': 'os-internal-network',
+        'fallback': 'private-address'
+    },
+    ADMIN: {
+        'config': 'os-admin-network',
+        'fallback': 'private-address'
+    }
+}
+
+
+def canonical_url(configs, endpoint_type=PUBLIC):
+    '''
+    Returns the correct HTTP URL to this host given the state of HTTPS
+    configuration, hacluster and charm configuration.
+
+    :configs OSTemplateRenderer: A config tempating object to inspect for
+        a complete https context.
+    :endpoint_type str: The endpoint type to resolve.
+
+    :returns str: Base URL for services on the current service unit.
+    '''
+    scheme = 'http'
+    if 'https' in configs.complete_contexts():
+        scheme = 'https'
+    address = resolve_address(endpoint_type)
+    if is_ipv6(address):
+        address = "[{}]".format(address)
+    return '%s://%s' % (scheme, address)
+
+
+def resolve_address(endpoint_type=PUBLIC):
+    resolved_address = None
+    if is_clustered():
+        if config(_address_map[endpoint_type]['config']) is None:
+            # Assume vip is simple and pass back directly
+            resolved_address = config('vip')
+        else:
+            for vip in config('vip').split():
+                if is_address_in_network(
+                        config(_address_map[endpoint_type]['config']),
+                        vip):
+                    resolved_address = vip
+    else:
+        resolved_address = get_address_in_network(
+            config(_address_map[endpoint_type]['config']),
+            unit_get(_address_map[endpoint_type]['fallback'])
+        )
+    if resolved_address is None:
+        raise ValueError('Unable to resolve a suitable IP address'
+                         ' based on charm state and configuration')
+    else:
+        return resolved_address

=== added file 'hooks/charmhelpers/contrib/openstack/templates/ceph.conf'
--- hooks/charmhelpers/contrib/openstack/templates/ceph.conf	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/ceph.conf	2014-07-25 09:37:37 +0000
@@ -0,0 +1,15 @@
+###############################################################################
+# [ WARNING ]
+# cinder configuration file maintained by Juju
+# local changes may be overwritten.
+###############################################################################
+[global]
+{% if auth -%}
+ auth_supported = {{ auth }}
+ keyring = /etc/ceph/$cluster.$name.keyring
+ mon host = {{ mon_hosts }}
+{% endif -%}
+ log to syslog = {{ use_syslog }}
+ err to syslog = {{ use_syslog }}
+ clog to syslog = {{ use_syslog }}
+

=== added file 'hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg'
--- hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg	2014-07-25 09:37:37 +0000
@@ -0,0 +1,41 @@
+global
+    log 127.0.0.1 local0
+    log 127.0.0.1 local1 notice
+    maxconn 20000
+    user haproxy
+    group haproxy
+    spread-checks 0
+
+defaults
+    log global
+    mode tcp
+    option tcplog
+    option dontlognull
+    retries 3
+    timeout queue 1000
+    timeout connect 1000
+    timeout client 30000
+    timeout server 30000
+
+listen stats :8888
+    mode http
+    stats enable
+    stats hide-version
+    stats realm Haproxy\ Statistics
+    stats uri /
+    stats auth admin:password
+
+{% if units -%}
+{% for service, ports in service_ports.iteritems() -%}
+listen {{ service }}_ipv4 0.0.0.0:{{ ports[0] }}
+    balance roundrobin
+    {% for unit, address in units.iteritems() -%}
+    server {{ unit }} {{ address }}:{{ ports[1] }} check
+    {% endfor %}
+listen {{ service }}_ipv6 :::{{ ports[0] }}
+    balance roundrobin
+    {% for unit, address in units.iteritems() -%}
+    server {{ unit }} {{ address }}:{{ ports[1] }} check
+    {% endfor %}
+{% endfor -%}
+{% endif -%}

=== added file 'hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend'
--- hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend	2014-07-25 09:37:37 +0000
@@ -0,0 +1,23 @@
+{% if endpoints -%}
+{% for ext, int in endpoints -%}
+Listen {{ ext }}
+NameVirtualHost *:{{ ext }}
+<VirtualHost *:{{ ext }}>
+    ServerName {{ private_address }}
+    SSLEngine on
+    SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert
+    SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key
+    ProxyPass / http://localhost:{{ int }}/
+    ProxyPassReverse / http://localhost:{{ int }}/
+    ProxyPreserveHost on
+</VirtualHost>
+<Proxy *>
+    Order deny,allow
+    Allow from all
+</Proxy>
+<Location />
+    Order allow,deny
+    Allow from all
+</Location>
+{% endfor -%}
+{% endif -%}

=== added file 'hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf'
--- hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf	2014-07-25 09:37:37 +0000
@@ -0,0 +1,23 @@
+{% if endpoints -%}
+{% for ext, int in endpoints -%}
+Listen {{ ext }}
+NameVirtualHost *:{{ ext }}
+<VirtualHost *:{{ ext }}>
+    ServerName {{ private_address }}
+    SSLEngine on
+    SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert
+    SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key
+    ProxyPass / http://localhost:{{ int }}/
+    ProxyPassReverse / http://localhost:{{ int }}/
+    ProxyPreserveHost on
+</VirtualHost>
+<Proxy *>
+    Order deny,allow
+    Allow from all
+</Proxy>
+<Location />
+    Order allow,deny
+    Allow from all
+</Location>
+{% endfor -%}
+{% endif -%}

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2014-07-10 21:43:51 +0000
+++ hooks/charmhelpers/core/host.py	2014-07-25 09:37:37 +0000
@@ -322,6 +322,10 @@
     import apt_pkg
     if not pkgcache:
         apt_pkg.init()
+        # Force Apt to build its cache in memory. That way we avoid race
+        # conditions with other applications building the cache in the same
+        # place.
+        apt_pkg.config.set("Dir::Cache::pkgcache", "")
         pkgcache = apt_pkg.Cache()
     pkg = pkgcache[package]
     return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)

=== added symlink 'hooks/cluster-relation-joined'
=== target is u'glance_relations.py'
=== modified file 'hooks/glance_contexts.py'
--- hooks/glance_contexts.py	2014-04-16 08:18:06 +0000
+++ hooks/glance_contexts.py	2014-07-25 09:37:37 +0000
@@ -78,5 +78,6 @@
 
 
 class LoggingConfigContext(OSContextGenerator):
+
     def __call__(self):
         return {'debug': config('debug'), 'verbose': config('verbose')}

=== modified file 'hooks/glance_relations.py'
--- hooks/glance_relations.py	2014-04-10 15:48:30 +0000
+++ hooks/glance_relations.py	2014-07-25 09:37:37 +0000
@@ -44,7 +44,9 @@
 )
 
 from charmhelpers.contrib.hahelpers.cluster import (
-    canonical_url, eligible_leader)
+    eligible_leader,
+    get_hacluster_config
+)
 
 from charmhelpers.contrib.openstack.utils import (
     configure_installation_source,
@@ -54,6 +56,15 @@
 
 from charmhelpers.contrib.storage.linux.ceph import ensure_ceph_keyring
 from charmhelpers.payload.execd import execd_preinstall
+from charmhelpers.contrib.network.ip import (
+    get_address_in_network,
+    get_netmask_for_address,
+    get_iface_for_address,
+)
+from charmhelpers.contrib.openstack.ip import (
+    canonical_url,
+    PUBLIC, INTERNAL, ADMIN
+)
 
 from subprocess import (
     check_call,
@@ -70,7 +81,7 @@
     execd_preinstall()
     src = config('openstack-origin')
     if (lsb_release()['DISTRIB_CODENAME'] == 'precise' and
-       src == 'distro'):
+            src == 'distro'):
         src = 'cloud:precise-folsom'
 
     configure_installation_source(src)
@@ -163,7 +174,8 @@
         return
 
     relation_data = {
-        'glance-api-server': canonical_url(CONFIGS) + ":9292"
+        'glance-api-server':
+        "{}:9292".format(canonical_url(CONFIGS, INTERNAL))
     }
 
     juju_log("%s: image-service_joined: To peer glance-api-server=%s" %
@@ -222,13 +234,15 @@
         juju_log('Deferring keystone_joined() to service leader.')
         return
 
-    url = canonical_url(CONFIGS) + ":9292"
+    public_url = '{}:9292'.format(canonical_url(CONFIGS, PUBLIC))
+    internal_url = '{}:9292'.format(canonical_url(CONFIGS, INTERNAL))
+    admin_url = '{}:9292'.format(canonical_url(CONFIGS, ADMIN))
     relation_data = {
         'service': 'glance',
         'region': config('region'),
-        'public_url': url,
-        'admin_url': url,
-        'internal_url': url, }
+        'public_url': public_url,
+        'admin_url': admin_url,
+        'internal_url': internal_url, }
 
     relation_set(relation_id=relation_id, **relation_data)
 
@@ -265,10 +279,19 @@
     open_port(9292)
     configure_https()
 
-    # env_vars = {'OPENSTACK_PORT_MCASTPORT': config("ha-mcastport"),
-    #            'OPENSTACK_SERVICE_API': "glance-api",
-    #            'OPENSTACK_SERVICE_REGISTRY': "glance-registry"}
-    # save_script_rc(**env_vars)
+    # Pickup and changes due to network reference architecture
+    # configuration
+    [keystone_joined(rid) for rid in relation_ids('identity-service')]
+    [image_service_joined(rid) for rid in relation_ids('image-service')]
+    [cluster_joined(rid) for rid in relation_ids('cluster')]
+
+
[e-mail]('cluster-relation-joined')
+def cluster_joined(relation_id=None):
+    address = get_address_in_network(config('os-internal-network'),
+                                     unit_get('private-address'))
+    relation_set(relation_id=relation_id,
+                 relation_settings={'private-address': address})
 
 
 @hooks.hook('cluster-relation-changed')
@@ -289,33 +312,44 @@
 
 @hooks.hook('ha-relation-joined')
 def ha_relation_joined():
-    corosync_bindiface = config("ha-bindiface")
-    corosync_mcastport = config("ha-mcastport")
-    vip = config("vip")
-    vip_iface = config("vip_iface")
-    vip_cidr = config("vip_cidr")
-
-    # if vip and vip_iface and vip_cidr and \
-    #    corosync_bindiface and corosync_mcastport:
+    config = get_hacluster_config()
 
     resources = {
-        'res_glance_vip': 'ocf:heartbeat:IPaddr2',
-        'res_glance_haproxy': 'lsb:haproxy', }
+        'res_glance_haproxy': 'lsb:haproxy'
+    }
 
     resource_params = {
-        'res_glance_vip': 'params ip="%s" cidr_netmask="%s" nic="%s"' %
-                          (vip, vip_cidr, vip_iface),
-        'res_glance_haproxy': 'op monitor interval="5s"', }
+        'res_glance_haproxy': 'op monitor interval="5s"'
+    }
+
+    vip_group = []
+    for vip in config['vip'].split():
+        iface = get_iface_for_address(vip)
+        if iface is not None:
+            vip_key = 'res_glance_{}_vip'.format(iface)
+            resources[vip_key] = 'ocf:heartbeat:IPaddr2'
+            resource_params[vip_key] = (
+                'params ip="{vip}" cidr_netmask="{netmask}"'
+                ' nic="{iface}"'.format(vip=vip,
+                                        iface=iface,
+                                        netmask=get_netmask_for_address(vip))
+            )
+            vip_group.append(vip_key)
+
+    if len(vip_group) > 1:
+        relation_set(groups={'grp_glance_vips': ' '.join(vip_group)})
 
     init_services = {
-        'res_glance_haproxy': 'haproxy', }
+        'res_glance_haproxy': 'haproxy',
+    }
 
     clones = {
-        'cl_glance_haproxy': 'res_glance_haproxy', }
+        'cl_glance_haproxy': 'res_glance_haproxy',
+    }
 
     relation_set(init_services=init_services,
-                 corosync_bindiface=corosync_bindiface,
-                 corosync_mcastport=corosync_mcastport,
+                 corosync_bindiface=config['ha-bindiface'],
+                 corosync_mcastport=config['ha-mcastport'],
                  resources=resources,
                  resource_params=resource_params,
                  clones=clones)

=== modified file 'hooks/glance_utils.py'
--- hooks/glance_utils.py	2014-04-12 17:10:59 +0000
+++ hooks/glance_utils.py	2014-07-25 09:37:37 +0000
@@ -42,7 +42,7 @@
     get_os_codename_package,
     configure_installation_source)
 
-CLUSTER_RES = "res_glance_vip"
+CLUSTER_RES = "grp_glance_vips"
 
 PACKAGES = [
     "apache2", "glance", "python-mysqldb", "python-swift",

=== removed file 'templates/haproxy.cfg'
--- templates/haproxy.cfg	2014-02-16 20:40:22 +0000
+++ templates/haproxy.cfg	1970-01-01 00:00:00 +0000
@@ -1,36 +0,0 @@
-global
-    log 127.0.0.1 local0
-    log 127.0.0.1 local1 notice
-    maxconn 20000
-    user haproxy
-    group haproxy
-    spread-checks 0
-
-defaults
-    log global
-    mode tcp
-    option tcplog
-    option dontlognull
-    retries 3
-    timeout queue 1000
-    timeout connect 1000
-    timeout client 30000
-    timeout server 30000
-
-listen stats :8888
-    mode http
-    stats enable
-    stats hide-version
-    stats realm Haproxy\ Statistics
-    stats uri /
-    stats auth admin:password
-
-{% if units %}
-{% for service, ports in service_ports.iteritems() -%}
-listen {{ service }} 0.0.0.0:{{ ports[0] }}
-    balance roundrobin
-    {% for unit, address in units.iteritems() -%}
-    server {{ unit }} {{ address }}:{{ ports[1] }} check
-    {% endfor %}
-{% endfor %}
-{% endif %}

=== modified file 'tests/charmhelpers/contrib/amulet/deployment.py'
--- tests/charmhelpers/contrib/amulet/deployment.py	2014-07-10 21:43:51 +0000
+++ tests/charmhelpers/contrib/amulet/deployment.py	2014-07-25 09:37:37 +0000
@@ -1,40 +1,35 @@
 import amulet
-import re
 
 
 class AmuletDeployment(object):
     """This class provides generic Amulet deployment and test runner
        methods."""
 
-    def __init__(self, series):
+    def __init__(self, series=None):
         """Initialize the deployment environment."""
-        self.series = series
-        self.d = amulet.Deployment(series=self.series)
+        self.series = None
 
-    def _get_charm_name(self, service_name):
-        """Gets the charm name from the service name. Unique service names can
-           be specified with a '-service#' suffix (e.g. mysql-service1)."""
-        if re.match(r"^.*-service\d{1,3}$", service_name):
-            charm_name = re.sub('\-service\d{1,3}$', '', service_name)
+        if series:
+            self.series = series
+            self.d = amulet.Deployment(series=self.series)
         else:
-            charm_name = service_name
-        return charm_name
+            self.d = amulet.Deployment()
 
     def _add_services(self, this_service, other_services):
         """Add services to the deployment where this_service is the local charm
            that we're focused on testing and other_services are the other
            charms that come from the charm store."""
         name, units = range(2)
-
-        charm_name = self._get_charm_name(this_service[name])
-        self.d.add(this_service[name],
-                   units=this_service[units])
+        self.this_service = this_service[name]
+        self.d.add(this_service[name], units=this_service[units])
 
         for svc in other_services:
-            charm_name = self._get_charm_name(svc[name])
-            self.d.add(svc[name],
-                       charm='cs:{}/{}'.format(self.series, charm_name),
-                       units=svc[units])
+            if self.series:
+                self.d.add(svc[name],
+                           charm='cs:{}/{}'.format(self.series, svc[name]),
+                           units=svc[units])
+            else:
+                self.d.add(svc[name], units=svc[units])
 
     def _add_relations(self, relations):
         """Add all of the relations for the services."""

=== modified file 'tests/charmhelpers/contrib/amulet/utils.py'
--- tests/charmhelpers/contrib/amulet/utils.py	2014-07-10 21:43:51 +0000
+++ tests/charmhelpers/contrib/amulet/utils.py	2014-07-25 09:37:37 +0000
@@ -139,11 +139,11 @@
         return self._get_dir_mtime(sentry_unit, proc_dir)
 
     def service_restarted(self, sentry_unit, service, filename,
-                          pgrep_full=False, sleep_time=20):
+                          pgrep_full=False):
         """Compare a service's start time vs a file's last modification time
            (such as a config file for that service) to determine if the service
            has been restarted."""
-        sleep(sleep_time)
+        sleep(10)
         if self._get_proc_start_time(sentry_unit, service, pgrep_full) >= \
            self._get_file_mtime(sentry_unit, filename):
             return True

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
--- tests/charmhelpers/contrib/openstack/amulet/deployment.py	2014-07-10 21:43:51 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py	2014-07-25 09:37:37 +0000
@@ -7,7 +7,7 @@
     """This class inherits from AmuletDeployment and has additional support
        that is specifically for use by OpenStack charms."""
 
-    def __init__(self, series, openstack=None, source=None):
+    def __init__(self, series=None, openstack=None, source=None):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
         self.openstack = openstack
@@ -24,15 +24,13 @@
 
         if self.openstack:
             for svc in services:
-                charm_name = self._get_charm_name(svc[name])
-                if charm_name not in use_source:
+                if svc[name] not in use_source:
                     config = {'openstack-origin': self.openstack}
                     self.d.configure(svc[name], config)
 
         if self.source:
             for svc in services:
-                charm_name = self._get_charm_name(svc[name])
-                if charm_name in use_source:
+                if svc[name] in use_source:
                     config = {'source': self.source}
                     self.d.configure(svc[name], config)
 

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
--- tests/charmhelpers/contrib/openstack/amulet/utils.py	2014-07-10 21:43:51 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/utils.py	2014-07-25 09:37:37 +0000
@@ -177,40 +177,12 @@
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)
-        count = 1
-        status = image.status
-        while status != 'active' and count < 10:
-            time.sleep(3)
-            image = glance.images.get(image.id)
-            status = image.status
-            self.log.debug('image status: {}'.format(status))
-            count += 1
-
-        if status != 'active':
-            self.log.error('image creation timed out')
-            return None
-
         return image
 
     def delete_image(self, glance, image):
         """Delete the specified image."""
-        num_before = len(list(glance.images.list()))
         glance.images.delete(image)
 
-        count = 1
-        num_after = len(list(glance.images.list()))
-        while num_after != (num_before - 1) and count < 10:
-            time.sleep(3)
-            num_after = len(list(glance.images.list()))
-            self.log.debug('number of images: {}'.format(num_after))
-            count += 1
-
-        if num_after != (num_before - 1):
-            self.log.error('image deletion timed out')
-            return False
-
-        return True
-
     def create_instance(self, nova, image_name, instance_name, flavor):
         """Create the specified instance."""
         image = nova.images.find(name=image_name)
@@ -227,27 +199,11 @@
             self.log.debug('instance status: {}'.format(status))
             count += 1
 
-        if status != 'ACTIVE':
-            self.log.error('instance creation timed out')
+        if status == 'BUILD':
             return None
 
         return instance
 
     def delete_instance(self, nova, instance):
         """Delete the specified instance."""
-        num_before = len(list(nova.servers.list()))
         nova.servers.delete(instance)
-
-        count = 1
-        num_after = len(list(nova.servers.list()))
-        while num_after != (num_before - 1) and count < 10:
-            time.sleep(3)
-            num_after = len(list(nova.servers.list()))
-            self.log.debug('number of instances: {}'.format(num_after))
-            count += 1
-
-        if num_after != (num_before - 1):
-            self.log.error('instance deletion timed out')
-            return False
-
-        return True

=== modified file 'unit_tests/test_glance_relations.py'
--- unit_tests/test_glance_relations.py	2014-03-31 11:38:11 +0000
+++ unit_tests/test_glance_relations.py	2014-07-25 09:37:37 +0000
@@ -54,7 +54,10 @@
     'check_call',
     'execd_preinstall',
     'lsb_release',
-    'filter_installed_packages'
+    'filter_installed_packages',
+    'get_hacluster_config',
+    'get_netmask_for_address',
+    'get_iface_for_address'
 ]
 
 
@@ -435,21 +438,23 @@
         self.assertTrue(configs.write_all.called)
 
     def test_ha_relation_joined(self):
-        self.test_config.set('ha-bindiface', 'em0')
-        self.test_config.set('ha-mcastport', '8080')
-        self.test_config.set('vip', '10.10.10.10')
-        self.test_config.set('vip_iface', 'em1')
-        self.test_config.set('vip_cidr', '24')
+        self.get_hacluster_config.return_value = {
+            'ha-bindiface': 'em0',
+            'ha-mcastport': '8080',
+            'vip': '10.10.10.10',
+        }
+        self.get_iface_for_address.return_value = 'eth1'
+        self.get_netmask_for_address.return_value = '255.255.0.0'
         relations.ha_relation_joined()
         args = {
             'corosync_bindiface': 'em0',
             'corosync_mcastport': '8080',
             'init_services': {'res_glance_haproxy': 'haproxy'},
-            'resources': {'res_glance_vip': 'ocf:heartbeat:IPaddr2',
+            'resources': {'res_glance_eth1_vip': 'ocf:heartbeat:IPaddr2',
                           'res_glance_haproxy': 'lsb:haproxy'},
             'resource_params': {
-                'res_glance_vip': 'params ip="10.10.10.10"'
-                                  ' cidr_netmask="24" nic="em1"',
+                'res_glance_eth1_vip': 'params ip="10.10.10.10"'
+                ' cidr_netmask="255.255.0.0" nic="eth1"',
                 'res_glance_haproxy': 'op monitor interval="5s"'},
             'clones': {'cl_glance_haproxy': 'res_glance_haproxy'}
         }

=== modified file 'unit_tests/test_utils.py'
--- unit_tests/test_utils.py	2014-01-15 13:04:21 +0000
+++ unit_tests/test_utils.py	2014-07-25 09:37:37 +0000
@@ -80,9 +80,9 @@
         return self.config
 
     def set(self, attr, value):
-            if attr not in self.config:
-                raise KeyError
-            self.config[attr] = value
+        if attr not in self.config:
+            raise KeyError
+        self.config[attr] = value
 
 
 class TestRelation(object):