Merge lp:~james-page/charms/trusty/percona-cluster/network-splits into lp:~openstack-charmers-archive/charms/trusty/percona-cluster/trunk

[James Page]

Add support for separate 'access-network' configuration.

[Liam Young (gnuoy)]

See inline comment

review: Needs Fixing

[James Page (james-page)]

review: Needs Resubmitting

[Liam Young (gnuoy)]

LGTM

review: Approve

=== added file '.bzrignore'
--- .bzrignore	1970-01-01 00:00:00 +0000
+++ .bzrignore	2014-07-24 15:25:17 +0000
@@ -0,0 +1,2 @@
+bin
+.coverage

=== modified file 'Makefile'
--- Makefile	2013-09-20 13:04:42 +0000
+++ Makefile	2014-07-24 15:25:17 +0000
@@ -3,12 +3,15 @@
 
 lint:
 	@flake8 --exclude hooks/charmhelpers hooks
-	#@flake8 --exclude hooks/charmhelpers unit_tests
 	@charm proof
 
-test:
-	@echo Starting tests...
-	@$(PYTHON) /usr/bin/nosetests --nologcapture unit_tests
-
-sync:
-	@charm-helper-sync -c charm-helpers-sync.yaml
+bin/charm_helpers_sync.py:
+	@mkdir -p bin
+	@bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \
+        > bin/charm_helpers_sync.py
+
+sync: bin/charm_helpers_sync.py
+	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers.yaml
+
+publish: lint
+	bzr push lp:charms/trusty/percona-cluster

=== renamed file 'charm-helpers-sync.yaml' => 'charm-helpers.yaml'
--- charm-helpers-sync.yaml	2014-06-23 09:47:35 +0000
+++ charm-helpers.yaml	2014-07-24 15:25:17 +0000
@@ -6,3 +6,4 @@
     - contrib.hahelpers.cluster
     - contrib.peerstorage
     - payload.execd
+    - contrib.network.ip

=== modified file 'config.yaml'
--- config.yaml	2014-01-13 16:07:19 +0000
+++ config.yaml	2014-07-24 15:25:17 +0000
@@ -1,37 +1,46 @@
 options:
-    source:
-        type: string
-        description: Package install location for Percona XtraDB Cluster (defaults to distro for >= 14.04)
-    dataset-size:
-        default: '80%'
-        type: string
-        description: How much data do you want to keep in memory in the DB. This will be used to tune settings in the database server appropriately. Suffix this value with 'K','M','G', or 'T' to get the relevant kilo/mega/etc. bytes. If suffixed with %, one will get that percentage of RAM devoted to dataset.
-    max-connections:
-        default: -1
-        type: int
-        description: Maximum connections to allow. -1 means use the server's compiled in default.
-    root-password:
-        type: string
-        description: Root password for MySQL access; must be configured pre-deployment for Active-Active clusters.
-    sst-password:
-        type: string
-        description: Re-sync account password for new cluster nodes; must be configured pre-deployment for Active-Active clusters.
-    vip:
-        type: string
-        description: Virtual IP to use to front Percona XtraDB Cluster in active/active HA configuration
-    vip_iface:
-        type: string
-        default: eth0
-        description: Network interface on which to place the Virtual IP
-    vip_cidr:
-        type: int
-        default: 24
-        description: Netmask that will be used for the Virtual IP
-    ha-bindiface:
-        type: string
-        default: eth0
-        description: Default network interface on which HA cluster will bind to communication with the other members of the HA Cluster.
-    ha-mcastport:
-        type: int
-        default: 5490
-        description: Default multicast port number that will be used to communicate between HA Cluster nodes.
+  source:
+    type: string
+    description: Package install location for Percona XtraDB Cluster (defaults to distro for >= 14.04)
+  dataset-size:
+    default: '80%'
+    type: string
+    description: How much data do you want to keep in memory in the DB. This will be used to tune settings in the database server appropriately. Suffix this value with 'K','M','G', or 'T' to get the relevant kilo/mega/etc. bytes. If suffixed with %, one will get that percentage of RAM devoted to dataset.
+  max-connections:
+    default: -1
+    type: int
+    description: Maximum connections to allow. -1 means use the server's compiled in default.
+  root-password:
+    type: string
+    description: Root password for MySQL access; must be configured pre-deployment for Active-Active clusters.
+  sst-password:
+    type: string
+    description: Re-sync account password for new cluster nodes; must be configured pre-deployment for Active-Active clusters.
+  vip:
+    type: string
+    description: Virtual IP to use to front Percona XtraDB Cluster in active/active HA configuration
+  vip_iface:
+    type: string
+    default: eth0
+    description: Network interface on which to place the Virtual IP
+  vip_cidr:
+    type: int
+    default: 24
+    description: Netmask that will be used for the Virtual IP
+  ha-bindiface:
+    type: string
+    default: eth0
+    description: Default network interface on which HA cluster will bind to communication with the other members of the HA Cluster.
+  ha-mcastport:
+    type: int
+    default: 5490
+    description: Default multicast port number that will be used to communicate between HA Cluster nodes.
+  # Network configuration options
+  # by default all access is over 'private-address'
+  access-network:
+    type: string
+    description: |
+      The IP address and netmask of the 'access' network (e.g., 192.168.0.0/24)
+      .
+      This network will be used for access to database services.
+

=== modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
--- hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-03-07 10:20:46 +0000
+++ hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-07-24 15:25:17 +0000
@@ -146,12 +146,12 @@
     Obtains all relevant configuration from charm configuration required
     for initiating a relation to hacluster:
 
-        ha-bindiface, ha-mcastport, vip, vip_iface, vip_cidr
+        ha-bindiface, ha-mcastport, vip
 
     returns: dict: A dict containing settings keyed by setting name.
     raises: HAIncompleteConfig if settings are missing.
     '''
-    settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'vip_iface', 'vip_cidr']
+    settings = ['ha-bindiface', 'ha-mcastport', 'vip']
     conf = {}
     for setting in settings:
         conf[setting] = config_get(setting)
@@ -170,6 +170,7 @@
 
     :configs    : OSTemplateRenderer: A config tempating object to inspect for
                                       a complete https context.
+
     :vip_setting:                str: Setting in charm config that specifies
                                       VIP address.
     '''

=== added directory 'hooks/charmhelpers/contrib/network'
=== added file 'hooks/charmhelpers/contrib/network/__init__.py'
=== added file 'hooks/charmhelpers/contrib/network/ip.py'
--- hooks/charmhelpers/contrib/network/ip.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/network/ip.py	2014-07-24 15:25:17 +0000
@@ -0,0 +1,156 @@
+import sys
+
+from functools import partial
+
+from charmhelpers.fetch import apt_install
+from charmhelpers.core.hookenv import (
+    ERROR, log,
+)
+
+try:
+    import netifaces
+except ImportError:
+    apt_install('python-netifaces')
+    import netifaces
+
+try:
+    import netaddr
+except ImportError:
+    apt_install('python-netaddr')
+    import netaddr
+
+
+def _validate_cidr(network):
+    try:
+        netaddr.IPNetwork(network)
+    except (netaddr.core.AddrFormatError, ValueError):
+        raise ValueError("Network (%s) is not in CIDR presentation format" %
+                         network)
+
+
+def get_address_in_network(network, fallback=None, fatal=False):
+    """
+    Get an IPv4 or IPv6 address within the network from the host.
+
+    :param network (str): CIDR presentation format. For example,
+        '192.168.1.0/24'.
+    :param fallback (str): If no address is found, return fallback.
+    :param fatal (boolean): If no address is found, fallback is not
+        set and fatal is True then exit(1).
+
+    """
+
+    def not_found_error_out():
+        log("No IP address found in network: %s" % network,
+            level=ERROR)
+        sys.exit(1)
+
+    if network is None:
+        if fallback is not None:
+            return fallback
+        else:
+            if fatal:
+                not_found_error_out()
+
+    _validate_cidr(network)
+    network = netaddr.IPNetwork(network)
+    for iface in netifaces.interfaces():
+        addresses = netifaces.ifaddresses(iface)
+        if network.version == 4 and netifaces.AF_INET in addresses:
+            addr = addresses[netifaces.AF_INET][0]['addr']
+            netmask = addresses[netifaces.AF_INET][0]['netmask']
+            cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
+            if cidr in network:
+                return str(cidr.ip)
+        if network.version == 6 and netifaces.AF_INET6 in addresses:
+            for addr in addresses[netifaces.AF_INET6]:
+                if not addr['addr'].startswith('fe80'):
+                    cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
+                                                        addr['netmask']))
+                    if cidr in network:
+                        return str(cidr.ip)
+
+    if fallback is not None:
+        return fallback
+
+    if fatal:
+        not_found_error_out()
+
+    return None
+
+
+def is_ipv6(address):
+    '''Determine whether provided address is IPv6 or not'''
+    try:
+        address = netaddr.IPAddress(address)
+    except netaddr.AddrFormatError:
+        # probably a hostname - so not an address at all!
+        return False
+    else:
+        return address.version == 6
+
+
+def is_address_in_network(network, address):
+    """
+    Determine whether the provided address is within a network range.
+
+    :param network (str): CIDR presentation format. For example,
+        '192.168.1.0/24'.
+    :param address: An individual IPv4 or IPv6 address without a net
+        mask or subnet prefix. For example, '192.168.1.1'.
+    :returns boolean: Flag indicating whether address is in network.
+    """
+    try:
+        network = netaddr.IPNetwork(network)
+    except (netaddr.core.AddrFormatError, ValueError):
+        raise ValueError("Network (%s) is not in CIDR presentation format" %
+                         network)
+    try:
+        address = netaddr.IPAddress(address)
+    except (netaddr.core.AddrFormatError, ValueError):
+        raise ValueError("Address (%s) is not in correct presentation format" %
+                         address)
+    if address in network:
+        return True
+    else:
+        return False
+
+
+def _get_for_address(address, key):
+    """Retrieve an attribute of or the physical interface that
+    the IP address provided could be bound to.
+
+    :param address (str): An individual IPv4 or IPv6 address without a net
+        mask or subnet prefix. For example, '192.168.1.1'.
+    :param key: 'iface' for the physical interface name or an attribute
+        of the configured interface, for example 'netmask'.
+    :returns str: Requested attribute or None if address is not bindable.
+    """
+    address = netaddr.IPAddress(address)
+    for iface in netifaces.interfaces():
+        addresses = netifaces.ifaddresses(iface)
+        if address.version == 4 and netifaces.AF_INET in addresses:
+            addr = addresses[netifaces.AF_INET][0]['addr']
+            netmask = addresses[netifaces.AF_INET][0]['netmask']
+            cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
+            if address in cidr:
+                if key == 'iface':
+                    return iface
+                else:
+                    return addresses[netifaces.AF_INET][0][key]
+        if address.version == 6 and netifaces.AF_INET6 in addresses:
+            for addr in addresses[netifaces.AF_INET6]:
+                if not addr['addr'].startswith('fe80'):
+                    cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
+                                                        addr['netmask']))
+                    if address in cidr:
+                        if key == 'iface':
+                            return iface
+                        else:
+                            return addr[key]
+    return None
+
+
+get_iface_for_address = partial(_get_for_address, key='iface')
+
+get_netmask_for_address = partial(_get_for_address, key='netmask')

=== modified file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py	2014-06-23 09:47:35 +0000
+++ hooks/charmhelpers/core/hookenv.py	2014-07-24 15:25:17 +0000
@@ -25,7 +25,7 @@
 def cached(func):
     """Cache return values for multiple executions of func + args
 
-    For example:
+    For example::
 
         @cached
         def unit_get(attribute):
@@ -445,18 +445,19 @@
 class Hooks(object):
     """A convenient handler for hook functions.
 
-    Example:
+    Example::
+
         hooks = Hooks()
 
         # register a hook, taking its name from the function name
         @hooks.hook()
         def install():
-            ...
+            pass  # your code here
 
         # register a hook, providing a custom hook name
         @hooks.hook("config-changed")
         def config_changed():
-            ...
+            pass  # your code here
 
         if __name__ == "__main__":
             # execute a hook based on the name the program is called by

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2014-06-23 09:47:35 +0000
+++ hooks/charmhelpers/core/host.py	2014-07-24 15:25:17 +0000
@@ -211,13 +211,13 @@
 def restart_on_change(restart_map, stopstart=False):
     """Restart services based on configuration files changing
 
-    This function is used a decorator, for example
+    This function is used a decorator, for example::
 
         @restart_on_change({
             '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
             })
         def ceph_client_changed():
-            ...
+            pass  # your code here
 
     In this example, the cinder-api and cinder-volume services
     would be restarted if /etc/ceph/ceph.conf is changed by the
@@ -313,13 +313,19 @@
 
 def cmp_pkgrevno(package, revno, pkgcache=None):
     '''Compare supplied revno with the revno of the installed package
-       1 => Installed revno is greater than supplied arg
-       0 => Installed revno is the same as supplied arg
-      -1 => Installed revno is less than supplied arg
+
+    *  1 => Installed revno is greater than supplied arg
+    *  0 => Installed revno is the same as supplied arg
+    * -1 => Installed revno is less than supplied arg
+
     '''
     import apt_pkg
     if not pkgcache:
         apt_pkg.init()
+        # Force Apt to build its cache in memory. That way we avoid race
+        # conditions with other applications building the cache in the same
+        # place.
+        apt_pkg.config.set("Dir::Cache::pkgcache", "")
         pkgcache = apt_pkg.Cache()
     pkg = pkgcache[package]
     return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)

=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2014-06-23 09:47:35 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2014-07-24 15:25:17 +0000
@@ -235,31 +235,39 @@
                       sources_var='install_sources',
                       keys_var='install_keys'):
     """
-    Configure multiple sources from charm configuration
+    Configure multiple sources from charm configuration.
+
+    The lists are encoded as yaml fragments in the configuration.
+    The frament needs to be included as a string.
 
     Example config:
-        install_sources:
+        install_sources: |
           - "ppa:foo"
           - "http://example.com/repo precise main"
-        install_keys:
+        install_keys: |
           - null
           - "a1b2c3d4"
 
     Note that 'null' (a.k.a. None) should not be quoted.
     """
-    sources = safe_load(config(sources_var))
-    keys = config(keys_var)
-    if keys is not None:
-        keys = safe_load(keys)
-    if isinstance(sources, basestring) and (
-            keys is None or isinstance(keys, basestring)):
-        add_source(sources, keys)
+    sources = safe_load((config(sources_var) or '').strip()) or []
+    keys = safe_load((config(keys_var) or '').strip()) or None
+
+    if isinstance(sources, basestring):
+        sources = [sources]
+
+    if keys is None:
+        for source in sources:
+            add_source(source, None)
     else:
-        if not len(sources) == len(keys):
-            msg = 'Install sources and keys lists are different lengths'
-            raise SourceConfigError(msg)
-        for src_num in range(len(sources)):
-            add_source(sources[src_num], keys[src_num])
+        if isinstance(keys, basestring):
+            keys = [keys]
+
+        if len(sources) != len(keys):
+            raise SourceConfigError(
+                'Install sources and keys lists are different lengths')
+        for source, key in zip(sources, keys):
+            add_source(source, key)
     if update:
         apt_update(fatal=True)
 

=== modified file 'hooks/percona_hooks.py'
--- hooks/percona_hooks.py	2014-06-23 09:46:08 +0000
+++ hooks/percona_hooks.py	2014-07-24 15:25:17 +0000
@@ -54,6 +54,10 @@
 )
 from mysql import configure_db
 from charmhelpers.payload.execd import execd_preinstall
+from charmhelpers.contrib.network.ip import (
+    is_address_in_network,
+    get_address_in_network,
+)
 
 hooks = Hooks()
 
@@ -106,6 +110,10 @@
         elif not clustered:
             # Restart with new configuration
             service_restart('mysql')
+    # Notify any changes to the access network
+    for r_id in relation_ids('shared-db'):
+        for unit in related_units(r_id):
+            shared_db_changed(r_id, unit)            
 
 
 @hooks.hook('cluster-relation-changed')
@@ -142,11 +150,15 @@
                             database_name,
                             username,
                             admin=admin)
+
     relation_set(relation_id=relation_id,
-                 database=database_name,
-                 user=username,
-                 password=password,
-                 host=db_host)
+                 relation_settings={
+                     'user': username,
+                     'password': password,
+                     'host': db_host,
+                     'database': database_name,
+                 }
+                 )
 
 
 # TODO: This could be a hook common between mysql and percona-cluster
@@ -164,6 +176,9 @@
         db_host = config('vip')
     else:
         db_host = unit_get('private-address')
+
+    access_network = config('access-network')
+
     singleset = set([
         'database',
         'username',
@@ -175,6 +190,10 @@
         password = configure_db(settings['hostname'],
                                 settings['database'],
                                 settings['username'])
+        if (access_network is not None and
+                is_address_in_network(access_network,
+                                      get_host_ip(settings['hostname']))):
+            db_host = get_address_in_network(access_network)
         relation_set(relation_id=relation_id,
                      db_host=db_host,
                      password=password)
@@ -211,11 +230,18 @@
                     configure_db(databases[db]['hostname'],
                                  databases[db]['database'],
                                  databases[db]['username'])
+                if (access_network is not None and
+                        is_address_in_network(
+                            access_network,
+                            get_host_ip(databases[db]['hostname']))):
+                    db_host = get_address_in_network(access_network)
         if len(return_data) > 0:
             relation_set(relation_id=relation_id,
                          **return_data)
             relation_set(relation_id=relation_id,
                          db_host=db_host)
+    relation_set(relation_id=relation_id,
+                 relation_settings={'access-network': access_network})
 
 
 @hooks.hook('ha-relation-joined')

=== modified file 'hooks/percona_utils.py'
--- hooks/percona_utils.py	2014-03-07 11:04:44 +0000
+++ hooks/percona_utils.py	2014-07-24 15:25:17 +0000
@@ -93,7 +93,7 @@
     for relid in relation_ids('cluster'):
         for unit in related_units(relid):
             hosts.append(get_host_ip(relation_get('private-address',
-                                     unit, relid)))
+                                                  unit, relid)))
     return hosts
 
 SQL_SST_USER_SETUP = "GRANT RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.*" \