Merge lp:~corey.bryant/charms/trusty/swift-storage/amulet-updates into lp:~openstack-charmers-archive/charms/trusty/swift-storage/next

[Corey Bryant]

=== modified file 'Makefile'
--- Makefile	2014-08-13 13:13:37 +0000
+++ Makefile	2014-09-30 13:42:56 +0000
@@ -15,7 +15,8 @@
 	# coreycb note: The -v should only be temporary until Amulet sends
 	# raise_status() messages to stderr:
 	#   https://bugs.launchpad.net/amulet/+bug/1320357
-	@juju test -v -p AMULET_HTTP_PROXY
+	@juju test -v -p AMULET_HTTP_PROXY --timeout 900 \
+        00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse
 
 bin/charm_helpers_sync.py:
 	@mkdir -p bin

=== modified file 'hooks/charmhelpers/contrib/hahelpers/apache.py'
--- hooks/charmhelpers/contrib/hahelpers/apache.py	2014-03-27 10:20:58 +0000
+++ hooks/charmhelpers/contrib/hahelpers/apache.py	2014-09-30 13:42:56 +0000
@@ -20,20 +20,27 @@
 )
 
 
-def get_cert():
+def get_cert(cn=None):
+    # TODO: deal with multiple https endpoints via charm config
     cert = config_get('ssl_cert')
     key = config_get('ssl_key')
     if not (cert and key):
         log("Inspecting identity-service relations for SSL certificate.",
             level=INFO)
         cert = key = None
+        if cn:
+            ssl_cert_attr = 'ssl_cert_{}'.format(cn)
+            ssl_key_attr = 'ssl_key_{}'.format(cn)
+        else:
+            ssl_cert_attr = 'ssl_cert'
+            ssl_key_attr = 'ssl_key'
         for r_id in relation_ids('identity-service'):
             for unit in relation_list(r_id):
                 if not cert:
-                    cert = relation_get('ssl_cert',
+                    cert = relation_get(ssl_cert_attr,
                                         rid=r_id, unit=unit)
                 if not key:
-                    key = relation_get('ssl_key',
+                    key = relation_get(ssl_key_attr,
                                        rid=r_id, unit=unit)
     return (cert, key)
 

=== modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
--- hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-08-13 13:13:37 +0000
+++ hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-09-30 13:42:56 +0000
@@ -139,10 +139,9 @@
         return True
     for r_id in relation_ids('identity-service'):
         for unit in relation_list(r_id):
+            # TODO - needs fixing for new helper as ssl_cert/key suffixes with CN
             rel_state = [
                 relation_get('https_keystone', rid=r_id, unit=unit),
-                relation_get('ssl_cert', rid=r_id, unit=unit),
-                relation_get('ssl_key', rid=r_id, unit=unit),
                 relation_get('ca_cert', rid=r_id, unit=unit),
             ]
             # NOTE: works around (LP: #1203241)

=== modified file 'hooks/charmhelpers/contrib/network/ip.py'
--- hooks/charmhelpers/contrib/network/ip.py	2014-08-13 13:13:37 +0000
+++ hooks/charmhelpers/contrib/network/ip.py	2014-09-30 13:42:56 +0000
@@ -1,10 +1,11 @@
+import glob
 import sys
 
 from functools import partial
 
 from charmhelpers.fetch import apt_install
 from charmhelpers.core.hookenv import (
-    ERROR, log, config,
+    ERROR, log,
 )
 
 try:
@@ -156,19 +157,102 @@
 get_netmask_for_address = partial(_get_for_address, key='netmask')
 
 
-def get_ipv6_addr(iface="eth0"):
+def format_ipv6_addr(address):
+    """
+    IPv6 needs to be wrapped with [] in url link to parse correctly.
+    """
+    if is_ipv6(address):
+        address = "[%s]" % address
+    else:
+        log("Not an valid ipv6 address: %s" % address,
+            level=ERROR)
+        address = None
+    return address
+
+
+def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, fatal=True, exc_list=None):
+    """
+    Return the assigned IP address for a given interface, if any, or [].
+    """
+    # Extract nic if passed /dev/ethX
+    if '/' in iface:
+        iface = iface.split('/')[-1]
+    if not exc_list:
+        exc_list = []
     try:
-        iface_addrs = netifaces.ifaddresses(iface)
-        if netifaces.AF_INET6 not in iface_addrs:
-            raise Exception("Interface '%s' doesn't have an ipv6 address." % iface)
-
-        addresses = netifaces.ifaddresses(iface)[netifaces.AF_INET6]
-        ipv6_addr = [a['addr'] for a in addresses if not a['addr'].startswith('fe80')
-                     and config('vip') != a['addr']]
-        if not ipv6_addr:
-            raise Exception("Interface '%s' doesn't have global ipv6 address." % iface)
-
-        return ipv6_addr[0]
-
-    except ValueError:
-        raise ValueError("Invalid interface '%s'" % iface)
+        inet_num = getattr(netifaces, inet_type)
+    except AttributeError:
+        raise Exception('Unknown inet type ' + str(inet_type))
+
+    interfaces = netifaces.interfaces()
+    if inc_aliases:
+        ifaces = []
+        for _iface in interfaces:
+            if iface == _iface or _iface.split(':')[0] == iface:
+                ifaces.append(_iface)
+        if fatal and not ifaces:
+            raise Exception("Invalid interface '%s'" % iface)
+        ifaces.sort()
+    else:
+        if iface not in interfaces:
+            if fatal:
+                raise Exception("%s not found " % (iface))
+            else:
+                return []
+        else:
+            ifaces = [iface]
+
+    addresses = []
+    for netiface in ifaces:
+        net_info = netifaces.ifaddresses(netiface)
+        if inet_num in net_info:
+            for entry in net_info[inet_num]:
+                if 'addr' in entry and entry['addr'] not in exc_list:
+                    addresses.append(entry['addr'])
+    if fatal and not addresses:
+        raise Exception("Interface '%s' doesn't have any %s addresses." % (iface, inet_type))
+    return addresses
+
+get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET')
+
+
+def get_ipv6_addr(iface='eth0', inc_aliases=False, fatal=True, exc_list=None):
+    """
+    Return the assigned IPv6 address for a given interface, if any, or [].
+    """
+    addresses = get_iface_addr(iface=iface, inet_type='AF_INET6',
+                               inc_aliases=inc_aliases, fatal=fatal,
+                               exc_list=exc_list)
+    remotly_addressable = []
+    for address in addresses:
+        if not address.startswith('fe80'):
+            remotly_addressable.append(address)
+    if fatal and not remotly_addressable:
+        raise Exception("Interface '%s' doesn't have global ipv6 address." % iface)
+    return remotly_addressable
+
+
+def get_bridges(vnic_dir='/sys/devices/virtual/net'):
+    """
+    Return a list of bridges on the system or []
+    """
+    b_rgex = vnic_dir + '/*/bridge'
+    return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_rgex)]
+
+
+def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'):
+    """
+    Return a list of nics comprising a given bridge on the system or []
+    """
+    brif_rgex = "%s/%s/brif/*" % (vnic_dir, bridge)
+    return [x.split('/')[-1] for x in glob.glob(brif_rgex)]
+
+
+def is_bridge_member(nic):
+    """
+    Check if a given nic is a member of a bridge
+    """
+    for bridge in get_bridges():
+        if nic in get_bridge_nics(bridge):
+            return True
+    return False

=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2014-08-13 13:13:37 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2014-09-30 13:42:56 +0000
@@ -10,32 +10,62 @@
        that is specifically for use by OpenStack charms.
        """
 
-    def __init__(self, series=None, openstack=None, source=None):
+    def __init__(self, series=None, openstack=None, source=None, stable=True):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
         self.openstack = openstack
         self.source = source
+        self.stable = stable
+        # Note(coreycb): this needs to be changed when new next branches come
+        # out.
+        self.current_next = "trusty"
+
+    def _determine_branch_locations(self, other_services):
+        """Determine the branch locations for the other services.
+
+           Determine if the local branch being tested is derived from its
+           stable or next (dev) branch, and based on this, use the corresonding
+           stable or next branches for the other_services."""
+        base_charms = ['mysql', 'mongodb', 'rabbitmq-server']
+
+        if self.stable:
+            for svc in other_services:
+                temp = 'lp:charms/{}'
+                svc['location'] = temp.format(svc['name'])
+        else:
+            for svc in other_services:
+                if svc['name'] in base_charms:
+                    temp = 'lp:charms/{}'
+                    svc['location'] = temp.format(svc['name'])
+                else:
+                    temp = 'lp:~openstack-charmers/charms/{}/{}/next'
+                    svc['location'] = temp.format(self.current_next,
+                                                  svc['name'])
+        return other_services
 
     def _add_services(self, this_service, other_services):
-        """Add services to the deployment and set openstack-origin."""
+        """Add services to the deployment and set openstack-origin/source."""
+        other_services = self._determine_branch_locations(other_services)
+
         super(OpenStackAmuletDeployment, self)._add_services(this_service,
                                                              other_services)
-        name = 0
+
         services = other_services
         services.append(this_service)
-        use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph']
+        use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
+                      'ceph-osd', 'ceph-radosgw']
 
         if self.openstack:
             for svc in services:
-                if svc[name] not in use_source:
+                if svc['name'] not in use_source:
                     config = {'openstack-origin': self.openstack}
-                    self.d.configure(svc[name], config)
+                    self.d.configure(svc['name'], config)
 
         if self.source:
             for svc in services:
-                if svc[name] in use_source:
+                if svc['name'] in use_source:
                     config = {'source': self.source}
-                    self.d.configure(svc[name], config)
+                    self.d.configure(svc['name'], config)
 
     def _configure_services(self, configs):
         """Configure all of the services."""

=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
--- hooks/charmhelpers/contrib/openstack/amulet/utils.py	2014-08-13 13:13:37 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py	2014-09-30 13:42:56 +0000
@@ -187,15 +187,16 @@
 
         f = opener.open("http://download.cirros-cloud.net/version/released")
         version = f.read().strip()
-        cirros_img = "tests/cirros-{}-x86_64-disk.img".format(version)
+        cirros_img = "cirros-{}-x86_64-disk.img".format(version)
+        local_path = os.path.join('tests', cirros_img)
 
-        if not os.path.exists(cirros_img):
+        if not os.path.exists(local_path):
             cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net",
                                                   version, cirros_img)
-            opener.retrieve(cirros_url, cirros_img)
+            opener.retrieve(cirros_url, local_path)
         f.close()
 
-        with open(cirros_img) as f:
+        with open(local_path) as f:
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)

=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2014-08-13 13:13:37 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2014-09-30 13:42:56 +0000
@@ -8,7 +8,6 @@
     check_call
 )
 
-
 from charmhelpers.fetch import (
     apt_install,
     filter_installed_packages,
@@ -28,6 +27,11 @@
     INFO
 )
 
+from charmhelpers.core.host import (
+    mkdir,
+    write_file
+)
+
 from charmhelpers.contrib.hahelpers.cluster import (
     determine_apache_port,
     determine_api_port,
@@ -38,6 +42,7 @@
 from charmhelpers.contrib.hahelpers.apache import (
     get_cert,
     get_ca_cert,
+    install_ca_cert,
 )
 
 from charmhelpers.contrib.openstack.neutron import (
@@ -47,6 +52,7 @@
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
     get_ipv6_addr,
+    is_address_in_network
 )
 
 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
@@ -421,6 +427,11 @@
             'units': cluster_hosts,
         }
 
+        if config('haproxy-server-timeout'):
+            ctxt['haproxy_server_timeout'] = config('haproxy-server-timeout')
+        if config('haproxy-client-timeout'):
+            ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout')
+
         if config('prefer-ipv6'):
             ctxt['local_host'] = 'ip6-localhost'
             ctxt['haproxy_host'] = '::'
@@ -490,22 +501,36 @@
         cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http']
         check_call(cmd)
 
-    def configure_cert(self):
-        if not os.path.isdir('/etc/apache2/ssl'):
-            os.mkdir('/etc/apache2/ssl')
+    def configure_cert(self, cn=None):
         ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace)
-        if not os.path.isdir(ssl_dir):
-            os.mkdir(ssl_dir)
-        cert, key = get_cert()
-        with open(os.path.join(ssl_dir, 'cert'), 'w') as cert_out:
-            cert_out.write(b64decode(cert))
-        with open(os.path.join(ssl_dir, 'key'), 'w') as key_out:
-            key_out.write(b64decode(key))
+        mkdir(path=ssl_dir)
+        cert, key = get_cert(cn)
+        if cn:
+            cert_filename = 'cert_{}'.format(cn)
+            key_filename = 'key_{}'.format(cn)
+        else:
+            cert_filename = 'cert'
+            key_filename = 'key'
+        write_file(path=os.path.join(ssl_dir, cert_filename),
+                   content=b64decode(cert))
+        write_file(path=os.path.join(ssl_dir, key_filename),
+                   content=b64decode(key))
+
+    def configure_ca(self):
         ca_cert = get_ca_cert()
         if ca_cert:
-            with open(CA_CERT_PATH, 'w') as ca_out:
-                ca_out.write(b64decode(ca_cert))
-            check_call(['update-ca-certificates'])
+            install_ca_cert(b64decode(ca_cert))
+
+    def canonical_names(self):
+        '''Figure out which canonical names clients will access this service'''
+        cns = []
+        for r_id in relation_ids('identity-service'):
+            for unit in related_units(r_id):
+                rdata = relation_get(rid=r_id, unit=unit)
+                for k in rdata:
+                    if k.startswith('ssl_key_'):
+                        cns.append(k.lstrip('ssl_key_'))
+        return list(set(cns))
 
     def __call__(self):
         if isinstance(self.external_ports, basestring):
@@ -513,21 +538,47 @@
         if (not self.external_ports or not https()):
             return {}
 
-        self.configure_cert()
+        self.configure_ca()
         self.enable_modules()
 
         ctxt = {
             'namespace': self.service_namespace,
-            'private_address': unit_get('private-address'),
-            'endpoints': []
+            'endpoints': [],
+            'ext_ports': []
         }
-        if is_clustered():
-            ctxt['private_address'] = config('vip')
-        for api_port in self.external_ports:
-            ext_port = determine_apache_port(api_port)
-            int_port = determine_api_port(api_port)
-            portmap = (int(ext_port), int(int_port))
-            ctxt['endpoints'].append(portmap)
+
+        for cn in self.canonical_names():
+            self.configure_cert(cn)
+
+        addresses = []
+        vips = []
+        if config('vip'):
+            vips = config('vip').split()
+
+        for network_type in ['os-internal-network',
+                             'os-admin-network',
+                             'os-public-network']:
+            address = get_address_in_network(config(network_type),
+                                             unit_get('private-address'))
+            if len(vips) > 0 and is_clustered():
+                for vip in vips:
+                    if is_address_in_network(config(network_type),
+                                             vip):
+                        addresses.append((address, vip))
+                        break
+            elif is_clustered():
+                addresses.append((address, config('vip')))
+            else:
+                addresses.append((address, address))
+
+        for address, endpoint in set(addresses):
+            for api_port in self.external_ports:
+                ext_port = determine_apache_port(api_port)
+                int_port = determine_api_port(api_port)
+                portmap = (address, endpoint, int(ext_port), int(int_port))
+                ctxt['endpoints'].append(portmap)
+                ctxt['ext_ports'].append(int(ext_port))
+        ctxt['ext_ports'] = list(set(ctxt['ext_ports']))
         return ctxt
 
 

=== modified file 'hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg'
--- hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg	2014-08-13 13:13:37 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg	2014-09-30 13:42:56 +0000
@@ -14,8 +14,17 @@
     retries 3
     timeout queue 1000
     timeout connect 1000
+{% if haproxy_client_timeout -%}
+    timeout client {{ haproxy_client_timeout }}
+{% else -%}
     timeout client 30000
+{% endif -%}
+
+{% if haproxy_server_timeout -%}
+    timeout server {{ haproxy_server_timeout }}
+{% else -%}
     timeout server 30000
+{% endif -%}
 
 listen stats {{ stat_port }}
     mode http

=== modified file 'hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend'
--- hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend	2013-08-16 20:34:18 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend	2014-09-30 13:42:56 +0000
@@ -1,16 +1,18 @@
 {% if endpoints -%}
-{% for ext, int in endpoints -%}
-Listen {{ ext }}
-NameVirtualHost *:{{ ext }}
-<VirtualHost *:{{ ext }}>
-    ServerName {{ private_address }}
+{% for ext_port in ext_ports -%}
+Listen {{ ext_port }}
+{% endfor -%}
+{% for address, endpoint, ext, int in endpoints -%}
+<VirtualHost {{ address }}:{{ ext }}>
+    ServerName {{ endpoint }}
     SSLEngine on
-    SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert
-    SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key
+    SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
+    SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }}
     ProxyPass / http://localhost:{{ int }}/
     ProxyPassReverse / http://localhost:{{ int }}/
     ProxyPreserveHost on
 </VirtualHost>
+{% endfor -%}
 <Proxy *>
     Order deny,allow
     Allow from all
@@ -19,5 +21,4 @@
     Order allow,deny
     Allow from all
 </Location>
-{% endfor -%}
 {% endif -%}

=== modified file 'hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf'
--- hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf	2013-09-27 16:33:06 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf	2014-09-30 13:42:56 +0000
@@ -1,16 +1,18 @@
 {% if endpoints -%}
-{% for ext, int in endpoints -%}
-Listen {{ ext }}
-NameVirtualHost *:{{ ext }}
-<VirtualHost *:{{ ext }}>
-    ServerName {{ private_address }}
+{% for ext_port in ext_ports -%}
+Listen {{ ext_port }}
+{% endfor -%}
+{% for address, endpoint, ext, int in endpoints -%}
+<VirtualHost {{ address }}:{{ ext }}>
+    ServerName {{ endpoint }}
     SSLEngine on
-    SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert
-    SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key
+    SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
+    SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }}
     ProxyPass / http://localhost:{{ int }}/
     ProxyPassReverse / http://localhost:{{ int }}/
     ProxyPreserveHost on
 </VirtualHost>
+{% endfor -%}
 <Proxy *>
     Order deny,allow
     Allow from all
@@ -19,5 +21,4 @@
     Order allow,deny
     Allow from all
 </Location>
-{% endfor -%}
 {% endif -%}

=== modified file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py	2014-09-02 13:21:15 +0000
+++ hooks/charmhelpers/core/hookenv.py	2014-09-30 13:42:56 +0000
@@ -203,6 +203,17 @@
         if os.path.exists(self.path):
             self.load_previous()
 
+    def __getitem__(self, key):
+        """For regular dict lookups, check the current juju config first,
+        then the previous (saved) copy. This ensures that user-saved values
+        will be returned by a dict lookup.
+
+        """
+        try:
+            return dict.__getitem__(self, key)
+        except KeyError:
+            return (self._prev_dict or {})[key]
+
     def load_previous(self, path=None):
         """Load previous copy of config from disk.
 
@@ -475,9 +486,10 @@
             hooks.execute(sys.argv)
     """
 
-    def __init__(self):
+    def __init__(self, config_save=True):
         super(Hooks, self).__init__()
         self._hooks = {}
+        self._config_save = config_save
 
     def register(self, name, function):
         """Register a hook"""
@@ -488,9 +500,10 @@
         hook_name = os.path.basename(args[0])
         if hook_name in self._hooks:
             self._hooks[hook_name]()
-            cfg = config()
-            if cfg.implicit_save:
-                cfg.save()
+            if self._config_save:
+                cfg = config()
+                if cfg.implicit_save:
+                    cfg.save()
         else:
             raise UnregisteredHookError(hook_name)
 

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2014-08-26 13:19:40 +0000
+++ hooks/charmhelpers/core/host.py	2014-09-30 13:42:56 +0000
@@ -68,8 +68,8 @@
     """Determine whether a system service is available"""
     try:
         subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError:
-        return False
+    except subprocess.CalledProcessError as e:
+        return 'unrecognized service' not in e.output
     else:
         return True
 
@@ -209,10 +209,15 @@
     return system_mounts
 
 
-def file_hash(path):
-    """Generate a md5 hash of the contents of 'path' or None if not found """
+def file_hash(path, hash_type='md5'):
+    """
+    Generate a hash checksum of the contents of 'path' or None if not found.
+
+    :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`,
+                          such as md5, sha1, sha256, sha512, etc.
+    """
     if os.path.exists(path):
-        h = hashlib.md5()
+        h = getattr(hashlib, hash_type)()
         with open(path, 'r') as source:
             h.update(source.read())  # IGNORE:E1101 - it does have update
         return h.hexdigest()
@@ -220,6 +225,26 @@
         return None
 
 
+def check_hash(path, checksum, hash_type='md5'):
+    """
+    Validate a file using a cryptographic checksum.
+
+    :param str checksum: Value of the checksum used to validate the file.
+    :param str hash_type: Hash algorithm used to generate `checksum`.
+        Can be any hash alrgorithm supported by :mod:`hashlib`,
+        such as md5, sha1, sha256, sha512, etc.
+    :raises ChecksumError: If the file fails the checksum
+
+    """
+    actual_checksum = file_hash(path, hash_type)
+    if checksum != actual_checksum:
+        raise ChecksumError("'%s' != '%s'" % (checksum, actual_checksum))
+
+
+class ChecksumError(ValueError):
+    pass
+
+
 def restart_on_change(restart_map, stopstart=False):
     """Restart services based on configuration files changing
 

=== modified file 'hooks/charmhelpers/core/services/helpers.py'
--- hooks/charmhelpers/core/services/helpers.py	2014-08-13 13:13:37 +0000
+++ hooks/charmhelpers/core/services/helpers.py	2014-09-30 13:42:56 +0000
@@ -1,3 +1,5 @@
+import os
+import yaml
 from charmhelpers.core import hookenv
 from charmhelpers.core import templating
 
@@ -19,15 +21,21 @@
     the `name` attribute that are complete will used to populate the dictionary
     values (see `get_data`, below).
 
-    The generated context will be namespaced under the interface type, to prevent
-    potential naming conflicts.
+    The generated context will be namespaced under the relation :attr:`name`,
+    to prevent potential naming conflicts.
+
+    :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
+    :param list additional_required_keys: Extend the list of :attr:`required_keys`
     """
     name = None
     interface = None
     required_keys = []
 
-    def __init__(self, *args, **kwargs):
-        super(RelationContext, self).__init__(*args, **kwargs)
+    def __init__(self, name=None, additional_required_keys=None):
+        if name is not None:
+            self.name = name
+        if additional_required_keys is not None:
+            self.required_keys.extend(additional_required_keys)
         self.get_data()
 
     def __bool__(self):
@@ -101,9 +109,115 @@
         return {}
 
 
+class MysqlRelation(RelationContext):
+    """
+    Relation context for the `mysql` interface.
+
+    :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
+    :param list additional_required_keys: Extend the list of :attr:`required_keys`
+    """
+    name = 'db'
+    interface = 'mysql'
+    required_keys = ['host', 'user', 'password', 'database']
+
+
+class HttpRelation(RelationContext):
+    """
+    Relation context for the `http` interface.
+
+    :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
+    :param list additional_required_keys: Extend the list of :attr:`required_keys`
+    """
+    name = 'website'
+    interface = 'http'
+    required_keys = ['host', 'port']
+
+    def provide_data(self):
+        return {
+            'host': hookenv.unit_get('private-address'),
+            'port': 80,
+        }
+
+
+class RequiredConfig(dict):
+    """
+    Data context that loads config options with one or more mandatory options.
+
+    Once the required options have been changed from their default values, all
+    config options will be available, namespaced under `config` to prevent
+    potential naming conflicts (for example, between a config option and a
+    relation property).
+
+    :param list *args: List of options that must be changed from their default values.
+    """
+
+    def __init__(self, *args):
+        self.required_options = args
+        self['config'] = hookenv.config()
+        with open(os.path.join(hookenv.charm_dir(), 'config.yaml')) as fp:
+            self.config = yaml.load(fp).get('options', {})
+
+    def __bool__(self):
+        for option in self.required_options:
+            if option not in self['config']:
+                return False
+            current_value = self['config'][option]
+            default_value = self.config[option].get('default')
+            if current_value == default_value:
+                return False
+            if current_value in (None, '') and default_value in (None, ''):
+                return False
+        return True
+
+    def __nonzero__(self):
+        return self.__bool__()
+
+
+class StoredContext(dict):
+    """
+    A data context that always returns the data that it was first created with.
+
+    This is useful to do a one-time generation of things like passwords, that
+    will thereafter use the same value that was originally generated, instead
+    of generating a new value each time it is run.
+    """
+    def __init__(self, file_name, config_data):
+        """
+        If the file exists, populate `self` with the data from the file.
+        Otherwise, populate with the given data and persist it to the file.
+        """
+        if os.path.exists(file_name):
+            self.update(self.read_context(file_name))
+        else:
+            self.store_context(file_name, config_data)
+            self.update(config_data)
+
+    def store_context(self, file_name, config_data):
+        if not os.path.isabs(file_name):
+            file_name = os.path.join(hookenv.charm_dir(), file_name)
+        with open(file_name, 'w') as file_stream:
+            os.fchmod(file_stream.fileno(), 0600)
+            yaml.dump(config_data, file_stream)
+
+    def read_context(self, file_name):
+        if not os.path.isabs(file_name):
+            file_name = os.path.join(hookenv.charm_dir(), file_name)
+        with open(file_name, 'r') as file_stream:
+            data = yaml.load(file_stream)
+            if not data:
+                raise OSError("%s is empty" % file_name)
+            return data
+
+
 class TemplateCallback(ManagerCallback):
     """
-    Callback class that will render a template, for use as a ready action.
+    Callback class that will render a Jinja2 template, for use as a ready action.
+
+    :param str source: The template source file, relative to `$CHARM_DIR/templates`
+    :param str target: The target to write the rendered template to
+    :param str owner: The owner of the rendered file
+    :param str group: The group of the rendered file
+    :param int perms: The permissions of the rendered file
     """
     def __init__(self, source, target, owner='root', group='root', perms=0444):
         self.source = source

=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2014-08-26 13:19:40 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2014-09-30 13:42:56 +0000
@@ -208,7 +208,8 @@
     """Add a package source to this system.
 
     @param source: a URL or sources.list entry, as supported by
-    add-apt-repository(1). Examples:
+    add-apt-repository(1). Examples::
+
         ppa:charmers/example
         deb https://stub:[e-mail]/ubuntu trusty main
 
@@ -311,22 +312,35 @@
         apt_update(fatal=True)
 
 
-def install_remote(source):
+def install_remote(source, *args, **kwargs):
     """
     Install a file tree from a remote source
 
     The specified source should be a url of the form:
         scheme://[host]/path[#[option=value][&...]]
 
-    Schemes supported are based on this modules submodules
-    Options supported are submodule-specific"""
+    Schemes supported are based on this modules submodules.
+    Options supported are submodule-specific.
+    Additional arguments are passed through to the submodule.
+
+    For example::
+
+        dest = install_remote('http://example.com/archive.tgz',
+                              checksum='deadbeef',
+                              hash_type='sha1')
+
+    This will download `archive.tgz`, validate it using SHA1 and, if
+    the file is ok, extract it and return the directory in which it
+    was extracted.  If the checksum fails, it will raise
+    :class:`charmhelpers.core.host.ChecksumError`.
+    """
     # We ONLY check for True here because can_handle may return a string
     # explaining why it can't handle a given source.
     handlers = [h for h in plugins() if h.can_handle(source) is True]
     installed_to = None
     for handler in handlers:
         try:
-            installed_to = handler.install(source)
+            installed_to = handler.install(source, *args, **kwargs)
         except UnhandledSource:
             pass
     if not installed_to:

=== modified file 'hooks/charmhelpers/fetch/archiveurl.py'
--- hooks/charmhelpers/fetch/archiveurl.py	2014-03-20 13:47:54 +0000
+++ hooks/charmhelpers/fetch/archiveurl.py	2014-09-30 13:42:56 +0000
@@ -1,6 +1,8 @@
 import os
 import urllib2
+from urllib import urlretrieve
 import urlparse
+import hashlib
 
 from charmhelpers.fetch import (
     BaseFetchHandler,
@@ -10,11 +12,19 @@
     get_archive_handler,
     extract,
 )
-from charmhelpers.core.host import mkdir
+from charmhelpers.core.host import mkdir, check_hash
 
 
 class ArchiveUrlFetchHandler(BaseFetchHandler):
-    """Handler for archives via generic URLs"""
+    """
+    Handler to download archive files from arbitrary URLs.
+
+    Can fetch from http, https, ftp, and file URLs.
+
+    Can install either tarballs (.tar, .tgz, .tbz2, etc) or zip files.
+
+    Installs the contents of the archive in $CHARM_DIR/fetched/.
+    """
     def can_handle(self, source):
         url_parts = self.parse_url(source)
         if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
@@ -24,6 +34,12 @@
         return False
 
     def download(self, source, dest):
+        """
+        Download an archive file.
+
+        :param str source: URL pointing to an archive file.
+        :param str dest: Local path location to download archive file to.
+        """
         # propogate all exceptions
         # URLError, OSError, etc
         proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
@@ -48,7 +64,30 @@
                 os.unlink(dest)
             raise e
 
-    def install(self, source):
+    # Mandatory file validation via Sha1 or MD5 hashing.
+    def download_and_validate(self, url, hashsum, validate="sha1"):
+        tempfile, headers = urlretrieve(url)
+        check_hash(tempfile, hashsum, validate)
+        return tempfile
+
+    def install(self, source, dest=None, checksum=None, hash_type='sha1'):
+        """
+        Download and install an archive file, with optional checksum validation.
+
+        The checksum can also be given on the `source` URL's fragment.
+        For example::
+
+            handler.install('http://example.com/file.tgz#sha1=deadbeef')
+
+        :param str source: URL pointing to an archive file.
+        :param str dest: Local destination path to install to. If not given,
+            installs to `$CHARM_DIR/archives/archive_file_name`.
+        :param str checksum: If given, validate the archive file after download.
+        :param str hash_type: Algorithm used to generate `checksum`.
+            Can be any hash alrgorithm supported by :mod:`hashlib`,
+            such as md5, sha1, sha256, sha512, etc.
+
+        """
         url_parts = self.parse_url(source)
         dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
         if not os.path.exists(dest_dir):
@@ -60,4 +99,10 @@
             raise UnhandledSource(e.reason)
         except OSError as e:
             raise UnhandledSource(e.strerror)
-        return extract(dld_file)
+        options = urlparse.parse_qs(url_parts.fragment)
+        for key, value in options.items():
+            if key in hashlib.algorithms:
+                check_hash(dld_file, value, key)
+        if checksum:
+            check_hash(dld_file, checksum, hash_type)
+        return extract(dld_file, dest)

=== modified file 'tests/00-setup'
--- tests/00-setup	2014-07-11 16:41:12 +0000
+++ tests/00-setup	2014-09-30 13:42:56 +0000
@@ -4,8 +4,8 @@
 
 sudo add-apt-repository --yes ppa:juju/stable
 sudo apt-get update --yes
-sudo apt-get install --yes python-amulet
-sudo apt-get install --yes python-swiftclient
-sudo apt-get install --yes python-glanceclient
-sudo apt-get install --yes python-keystoneclient
-sudo apt-get install --yes python-novaclient
+sudo apt-get install --yes python-amulet \
+                           python-swiftclient \
+                           python-glanceclient \
+                           python-keystoneclient \
+                           python-novaclient

=== modified file 'tests/README'
--- tests/README	2014-07-11 16:41:12 +0000
+++ tests/README	2014-09-30 13:42:56 +0000
@@ -1,6 +1,12 @@
 This directory provides Amulet tests that focus on verification of swift-storage
 deployments.
 
+In order to run tests, you'll need charm-tools installed (in addition to
+juju, of course):
+    sudo add-apt-repository ppa:juju/stable
+    sudo apt-get update
+    sudo apt-get install charm-tools
+
 If you use a web proxy server to access the web, you'll need to set the
 AMULET_HTTP_PROXY environment variable to the http URL of the proxy server.
 

=== modified file 'tests/basic_deployment.py'
--- tests/basic_deployment.py	2014-07-11 16:41:12 +0000
+++ tests/basic_deployment.py	2014-09-30 13:42:56 +0000
@@ -20,10 +20,10 @@
 class SwiftStorageBasicDeployment(OpenStackAmuletDeployment):
     """Amulet tests on a basic swift-storage deployment."""
 
-    def __init__(self, series, openstack=None, source=None):
+    def __init__(self, series, openstack=None, source=None, stable=False):
         """Deploy the entire test environment."""
         super(SwiftStorageBasicDeployment, self).__init__(series, openstack,
-                                                          source)
+                                                          source, stable)
         self._add_services()
         self._add_relations()
         self._configure_services()
@@ -31,12 +31,15 @@
         self._initialize_tests()
 
     def _add_services(self):
-        """Add the service that we're testing, including the number of units,
-           where swift-storage is local, and the other charms are from
-           the charm store."""
-        this_service = ('swift-storage', 1)
-        other_services = [('mysql', 1),
-                          ('keystone', 1), ('glance', 1), ('swift-proxy', 1)]
+        """Add services
+
+           Add the services that we're testing, where swift-storage is local,
+           and the rest of the service are from lp branches that are
+           compatible with the local charm (e.g. stable or next).
+           """
+        this_service = {'name': 'swift-storage'}
+        other_services = [{'name': 'mysql'}, {'name': 'keystone'},
+                          {'name': 'glance'}, {'name': 'swift-proxy'}]
         super(SwiftStorageBasicDeployment, self)._add_services(this_service,
                                                                other_services)
 
@@ -249,9 +252,14 @@
             message = u.relation_error('swift-proxy swift-storage', ret)
             amulet.raise_status(amulet.FAIL, msg=message)
 
-    def test_restart_on_config_change(self):
+    def test_z_restart_on_config_change(self):
         """Verify that the specified services are restarted when the config
-           is changed."""
+           is changed.
+
+           Note(coreycb): The method name with the _z_ is a little odd
+           but it forces the test to run last.  It just makes things
+           easier because restarting services requires re-authorization.
+           """
         # NOTE(coreycb): Skipping failing test on until resolved. This test
         #                fails because the config file's last mod time is
         #                slightly after the process' last mod time.
@@ -282,6 +290,8 @@
             config = '/etc/swift/{}'.format(conf)
             if not u.service_restarted(self.swift_storage_sentry, s, config,
                                        pgrep_full=True, sleep_time=time):
+                self.d.configure('swift-storage',
+                                 {'object-server-threads-per-disk': '4'})
                 msg = "service {} didn't restart after config change".format(s)
                 amulet.raise_status(amulet.FAIL, msg=msg)
             time = 0

=== modified file 'tests/charmhelpers/contrib/amulet/deployment.py'
--- tests/charmhelpers/contrib/amulet/deployment.py	2014-08-13 13:13:37 +0000
+++ tests/charmhelpers/contrib/amulet/deployment.py	2014-09-30 13:42:56 +0000
@@ -24,25 +24,31 @@
         """Add services.
 
            Add services to the deployment where this_service is the local charm
-           that we're focused on testing and other_services are the other
-           charms that come from the charm store.
+           that we're testing and other_services are the other services that
+           are being used in the local amulet tests.
            """
-        name, units = range(2)
-
-        if this_service[name] != os.path.basename(os.getcwd()):
-            s = this_service[name]
+        if this_service['name'] != os.path.basename(os.getcwd()):
+            s = this_service['name']
             msg = "The charm's root directory name needs to be {}".format(s)
             amulet.raise_status(amulet.FAIL, msg=msg)
 
-        self.d.add(this_service[name], units=this_service[units])
+        if 'units' not in this_service:
+            this_service['units'] = 1
+
+        self.d.add(this_service['name'], units=this_service['units'])
 
         for svc in other_services:
-            if self.series:
-                self.d.add(svc[name],
-                           charm='cs:{}/{}'.format(self.series, svc[name]),
-                           units=svc[units])
+            if 'location' in svc:
+                branch_location = svc['location']
+            elif self.series:
+                branch_location = 'cs:{}/{}'.format(self.series, svc['name']),
             else:
-                self.d.add(svc[name], units=svc[units])
+                branch_location = None
+
+            if 'units' not in svc:
+                svc['units'] = 1
+
+            self.d.add(svc['name'], charm=branch_location, units=svc['units'])
 
     def _add_relations(self, relations):
         """Add all of the relations for the services."""
@@ -57,7 +63,7 @@
     def _deploy(self):
         """Deploy environment and wait for all hooks to finish executing."""
         try:
-            self.d.setup()
+            self.d.setup(timeout=900)
             self.d.sentry.wait(timeout=900)
         except amulet.helpers.TimeoutError:
             amulet.raise_status(amulet.FAIL, msg="Deployment timed out")

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
--- tests/charmhelpers/contrib/openstack/amulet/deployment.py	2014-08-13 13:13:37 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py	2014-09-30 13:42:56 +0000
@@ -10,32 +10,62 @@
        that is specifically for use by OpenStack charms.
        """
 
-    def __init__(self, series=None, openstack=None, source=None):
+    def __init__(self, series=None, openstack=None, source=None, stable=True):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
         self.openstack = openstack
         self.source = source
+        self.stable = stable
+        # Note(coreycb): this needs to be changed when new next branches come
+        # out.
+        self.current_next = "trusty"
+
+    def _determine_branch_locations(self, other_services):
+        """Determine the branch locations for the other services.
+
+           Determine if the local branch being tested is derived from its
+           stable or next (dev) branch, and based on this, use the corresonding
+           stable or next branches for the other_services."""
+        base_charms = ['mysql', 'mongodb', 'rabbitmq-server']
+
+        if self.stable:
+            for svc in other_services:
+                temp = 'lp:charms/{}'
+                svc['location'] = temp.format(svc['name'])
+        else:
+            for svc in other_services:
+                if svc['name'] in base_charms:
+                    temp = 'lp:charms/{}'
+                    svc['location'] = temp.format(svc['name'])
+                else:
+                    temp = 'lp:~openstack-charmers/charms/{}/{}/next'
+                    svc['location'] = temp.format(self.current_next,
+                                                  svc['name'])
+        return other_services
 
     def _add_services(self, this_service, other_services):
-        """Add services to the deployment and set openstack-origin."""
+        """Add services to the deployment and set openstack-origin/source."""
+        other_services = self._determine_branch_locations(other_services)
+
         super(OpenStackAmuletDeployment, self)._add_services(this_service,
                                                              other_services)
-        name = 0
+
         services = other_services
         services.append(this_service)
-        use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph']
+        use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
+                      'ceph-osd', 'ceph-radosgw']
 
         if self.openstack:
             for svc in services:
-                if svc[name] not in use_source:
+                if svc['name'] not in use_source:
                     config = {'openstack-origin': self.openstack}
-                    self.d.configure(svc[name], config)
+                    self.d.configure(svc['name'], config)
 
         if self.source:
             for svc in services:
-                if svc[name] in use_source:
+                if svc['name'] in use_source:
                     config = {'source': self.source}
-                    self.d.configure(svc[name], config)
+                    self.d.configure(svc['name'], config)
 
     def _configure_services(self, configs):
         """Configure all of the services."""

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
--- tests/charmhelpers/contrib/openstack/amulet/utils.py	2014-08-13 13:13:37 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/utils.py	2014-09-30 13:42:56 +0000
@@ -187,15 +187,16 @@
 
         f = opener.open("http://download.cirros-cloud.net/version/released")
         version = f.read().strip()
-        cirros_img = "tests/cirros-{}-x86_64-disk.img".format(version)
+        cirros_img = "cirros-{}-x86_64-disk.img".format(version)
+        local_path = os.path.join('tests', cirros_img)
 
-        if not os.path.exists(cirros_img):
+        if not os.path.exists(local_path):
             cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net",
                                                   version, cirros_img)
-            opener.retrieve(cirros_url, cirros_img)
+            opener.retrieve(cirros_url, local_path)
         f.close()
 
-        with open(cirros_img) as f:
+        with open(local_path) as f:
             image = glance.images.create(name=image_name, is_public=True,
                                          disk_format='qcow2',
                                          container_format='bare', data=f)