Merge lp:~openstack-charmers/charms/precise/swift-proxy/ha-support into lp:~charmers/charms/precise/swift-proxy/trunk

[Adam Gandelman]

* Python rewrite

* Support for manually assigned storage zones

* Support for high availability via hacluster subordinate.

* Adds Grizzly compatibility.

=== added file '.project'
--- .project	1970-01-01 00:00:00 +0000
+++ .project	2013-05-29 18:03:27 +0000
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>swift-proxy</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>

=== added file '.pydevproject'
--- .pydevproject	1970-01-01 00:00:00 +0000
+++ .pydevproject	2013-05-29 18:03:27 +0000
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?><pydev_project>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
+<path>/swift-proxy/hooks</path>
+</pydev_pathproperty>
+</pydev_project>

=== modified file 'config.yaml'
--- config.yaml	2012-12-14 23:07:01 +0000
+++ config.yaml	2013-05-29 18:03:27 +0000
@@ -46,7 +46,25 @@
       zones before the storage ring will be initially balance.  Deployment
       requirements differ based on the zone-assignment policy configured, see
       this charm's README for details.
-  # CA Cert info
+  # User provided SSL cert and key
+  ssl_cert:
+    type: string
+    description: |
+      Base64 encoded SSL certificate to install and use for API ports.
+      .
+         juju set swift-proxy ssl_cert="$(cat cert | base64)" \
+                              ssl_key="$(cat key | base64)"
+      .
+      Setting this value (and ssl_key) will enable reverse proxying, point
+      Swifts's entry in the Keystone catalog to use https, and override
+      any certficiate and key issued by Keystone (if it is configured to
+      do so).
+  ssl_key:
+    type: string
+    description: |
+      Base64 encoded SSL key to use with certificate specified as ssl_cert.
+  # Locally generated CA Cert info (only use without keystone)
+  # These options are deprecated and will be removed sometime
   use-https:
     default: "yes"
     type: string
@@ -67,6 +85,7 @@
     default: CN
     type: string
     description: Common Name
+  # General Swift Proxy configuration
   bind-port:
     default: 8080
     type: int
@@ -83,6 +102,10 @@
     default: tempauth
     type: string
     description: Auth method to use, tempauth or keystone
+  delay-auth-decision:
+    default: true
+    type: boolean
+    description: Delay authentication to downstream WSGI services.
   # Manual Keystone configuration.
   keystone-auth-host:
     type: string
@@ -105,3 +128,30 @@
   keystone-admin-password:
     type: string
     description: Keystone admin password
+  # HA configuration settings
+  swift-hash:
+    type: string
+    description: Hash to use across all swift-proxy servers - don't loose
+  vip:
+    type: string
+    description: "Virtual IP to use to front swift-proxy in ha configuration"
+  vip_iface:
+    type: string
+    default: eth0
+    description: "Network Interface where to place the Virtual IP"
+  vip_cidr:
+    type: int
+    default: 24
+    description: "Netmask that will be used for the Virtual IP"
+  ha-bindiface:
+    type: string
+    default: eth0
+    description: |
+      Default network interface on which HA cluster will bind to communication
+      with the other members of the HA Cluster.
+  ha-mcastport:
+    type: int
+    default: 5414
+    description: |
+      Default multicast port number that will be used to communicate between
+      HA Cluster nodes.

=== added symlink 'hooks/cluster-relation-changed'
=== target is u'swift_hooks.py'
=== added symlink 'hooks/cluster-relation-joined'
=== target is u'swift_hooks.py'
=== modified symlink 'hooks/config-changed'
=== target changed u'swift-hooks.py' => u'swift_hooks.py'
=== added symlink 'hooks/ha-relation-changed'
=== target is u'swift_hooks.py'
=== added symlink 'hooks/ha-relation-joined'
=== target is u'swift_hooks.py'
=== modified symlink 'hooks/identity-service-relation-changed'
=== target changed u'swift-hooks.py' => u'swift_hooks.py'
=== modified symlink 'hooks/identity-service-relation-joined'
=== target changed u'swift-hooks.py' => u'swift_hooks.py'
=== modified symlink 'hooks/install'
=== target changed u'swift-hooks.py' => u'swift_hooks.py'
=== added file 'hooks/lib/apache_utils.py'
--- hooks/lib/apache_utils.py	1970-01-01 00:00:00 +0000
+++ hooks/lib/apache_utils.py	2013-05-29 18:03:27 +0000
@@ -0,0 +1,193 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# Authors:
+#  James Page <[e-mail]>
+#
+
+from lib.utils import (
+    relation_ids,
+    relation_list,
+    relation_get,
+    render_template,
+    juju_log,
+    config_get,
+    install,
+    get_host_ip,
+    restart
+    )
+from lib.cluster_utils import https
+
+import os
+import subprocess
+from base64 import b64decode
+
+APACHE_SITE_DIR = "/etc/apache2/sites-available"
+SITE_TEMPLATE = "apache2_site.tmpl"
+RELOAD_CHECK = "To activate the new configuration"
+
+
+def get_cert():
+    cert = config_get('ssl_cert')
+    key = config_get('ssl_key')
+    if not (cert and key):
+        juju_log('INFO',
+                 "Inspecting identity-service relations for SSL certificate.")
+        cert = key = None
+        for r_id in relation_ids('identity-service'):
+            for unit in relation_list(r_id):
+                if not cert:
+                    cert = relation_get('ssl_cert',
+                                        rid=r_id, unit=unit)
+                if not key:
+                    key = relation_get('ssl_key',
+                                       rid=r_id, unit=unit)
+    return (cert, key)
+
+
+def get_ca_cert():
+    ca_cert = None
+    juju_log('INFO',
+             "Inspecting identity-service relations for CA SSL certificate.")
+    for r_id in relation_ids('identity-service'):
+        for unit in relation_list(r_id):
+            if not ca_cert:
+                ca_cert = relation_get('ca_cert',
+                                       rid=r_id, unit=unit)
+    return ca_cert
+
+
+def install_ca_cert(ca_cert):
+    if ca_cert:
+        with open('/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt',
+                  'w') as crt:
+            crt.write(ca_cert)
+        subprocess.check_call(['update-ca-certificates', '--fresh'])
+
+
+def enable_https(port_maps, namespace, cert, key, ca_cert=None):
+    '''
+    For a given number of port mappings, configures apache2
+    HTTPs local reverse proxying using certficates and keys provided in
+    either configuration data (preferred) or relation data.  Assumes ports
+    are not in use (calling charm should ensure that).
+
+    port_maps: dict: external to internal port mappings
+    namespace: str: name of charm
+    '''
+    def _write_if_changed(path, new_content):
+        content = None
+        if os.path.exists(path):
+            with open(path, 'r') as f:
+                content = f.read().strip()
+        if content != new_content:
+            with open(path, 'w') as f:
+                f.write(new_content)
+            return True
+        else:
+            return False
+
+    juju_log('INFO', "Enabling HTTPS for port mappings: {}".format(port_maps))
+    http_restart = False
+
+    if cert:
+        cert = b64decode(cert)
+    if key:
+        key = b64decode(key)
+    if ca_cert:
+        ca_cert = b64decode(ca_cert)
+
+    if not cert and not key:
+        juju_log('ERROR',
+                 "Expected but could not find SSL certificate data, not "
+                 "configuring HTTPS!")
+        return False
+
+    install('apache2')
+    if RELOAD_CHECK in subprocess.check_output(['a2enmod', 'ssl',
+                                                'proxy', 'proxy_http']):
+        http_restart = True
+
+    ssl_dir = os.path.join('/etc/apache2/ssl', namespace)
+    if not os.path.exists(ssl_dir):
+        os.makedirs(ssl_dir)
+
+    if (_write_if_changed(os.path.join(ssl_dir, 'cert'), cert)):
+        http_restart = True
+    if (_write_if_changed(os.path.join(ssl_dir, 'key'), key)):
+        http_restart = True
+    os.chmod(os.path.join(ssl_dir, 'key'), 0600)
+
+    install_ca_cert(ca_cert)
+
+    sites_dir = '/etc/apache2/sites-available'
+    for ext_port, int_port in port_maps.items():
+        juju_log('INFO',
+                 'Creating apache2 reverse proxy vhost'
+                 ' for {}:{}'.format(ext_port,
+                                     int_port))
+        site = "{}_{}".format(namespace, ext_port)
+        site_path = os.path.join(sites_dir, site)
+        with open(site_path, 'w') as fsite:
+            context = {
+                "ext": ext_port,
+                "int": int_port,
+                "namespace": namespace,
+                "private_address": get_host_ip()
+                }
+            fsite.write(render_template(SITE_TEMPLATE,
+                                        context))
+
+        if RELOAD_CHECK in subprocess.check_output(['a2ensite', site]):
+            http_restart = True
+
+    if http_restart:
+        restart('apache2')
+
+    return True
+
+
+def disable_https(port_maps, namespace):
+    '''
+    Ensure HTTPS reverse proxying is disables for given port mappings
+
+    port_maps: dict: of ext -> int port mappings
+    namespace: str: name of chamr
+    '''
+    juju_log('INFO', 'Ensuring HTTPS disabled for {}'.format(port_maps))
+
+    if (not os.path.exists('/etc/apache2') or
+        not os.path.exists(os.path.join('/etc/apache2/ssl', namespace))):
+        return
+
+    http_restart = False
+    for ext_port in port_maps.keys():
+        if os.path.exists(os.path.join(APACHE_SITE_DIR,
+                                       "{}_{}".format(namespace,
+                                                      ext_port))):
+            juju_log('INFO',
+                     "Disabling HTTPS reverse proxy"
+                     " for {} {}.".format(namespace,
+                                          ext_port))
+            if (RELOAD_CHECK in
+                subprocess.check_output(['a2dissite',
+                                         '{}_{}'.format(namespace,
+                                                        ext_port)])):
+                http_restart = True
+
+    if http_restart:
+        restart(['apache2'])
+
+
+def setup_https(port_maps, namespace, cert, key, ca_cert=None):
+    '''
+    Ensures HTTPS is either enabled or disabled for given port
+    mapping.
+
+    port_maps: dict: of ext -> int port mappings
+    namespace: str: name of charm
+    '''
+    if not https:
+        disable_https(port_maps, namespace)
+    else:
+        enable_https(port_maps, namespace, cert, key, ca_cert)

=== added file 'hooks/lib/cluster_utils.py'
--- hooks/lib/cluster_utils.py	1970-01-01 00:00:00 +0000
+++ hooks/lib/cluster_utils.py	2013-05-29 18:03:27 +0000
@@ -0,0 +1,130 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# This file is sourced from lp:openstack-charm-helpers
+#
+# Authors:
+#  James Page <[e-mail]>
+#  Adam Gandelman <[e-mail]>
+#
+
+from lib.utils import (
+    juju_log,
+    relation_ids,
+    relation_list,
+    relation_get,
+    get_unit_hostname,
+    config_get
+    )
+import subprocess
+import os
+
+
+def is_clustered():
+    for r_id in (relation_ids('ha') or []):
+        for unit in (relation_list(r_id) or []):
+            clustered = relation_get('clustered',
+                                     rid=r_id,
+                                     unit=unit)
+            if clustered:
+                return True
+    return False
+
+
+def is_leader(resource):
+    cmd = [
+        "crm", "resource",
+        "show", resource
+        ]
+    try:
+        status = subprocess.check_output(cmd)
+    except subprocess.CalledProcessError:
+        return False
+    else:
+        if get_unit_hostname() in status:
+            return True
+        else:
+            return False
+
+
+def peer_units():
+    peers = []
+    for r_id in (relation_ids('cluster') or []):
+        for unit in (relation_list(r_id) or []):
+            peers.append(unit)
+    return peers
+
+
+def oldest_peer(peers):
+    local_unit_no = int(os.getenv('JUJU_UNIT_NAME').split('/')[1])
+    for peer in peers:
+        remote_unit_no = int(peer.split('/')[1])
+        if remote_unit_no < local_unit_no:
+            return False
+    return True
+
+
+def eligible_leader(resource):
+    if is_clustered():
+        if not is_leader(resource):
+            juju_log('INFO', 'Deferring action to CRM leader.')
+            return False
+    else:
+        peers = peer_units()
+        if peers and not oldest_peer(peers):
+            juju_log('INFO', 'Deferring action to oldest service unit.')
+            return False
+    return True
+
+
+def https():
+    '''
+    Determines whether enough data has been provided in configuration
+    or relation data to configure HTTPS
+    .
+    returns: boolean
+    '''
+    if config_get('use-https') == "yes":
+        return True
+    if config_get('ssl_cert') and config_get('ssl_key'):
+        return True
+    for r_id in relation_ids('identity-service'):
+        for unit in relation_list(r_id):
+            if (relation_get('https_keystone', rid=r_id, unit=unit) and
+                relation_get('ssl_cert', rid=r_id, unit=unit) and
+                relation_get('ssl_key', rid=r_id, unit=unit) and
+                relation_get('ca_cert', rid=r_id, unit=unit)):
+                return True
+    return False
+
+
+def determine_api_port(public_port):
+    '''
+    Determine correct API server listening port based on
+    existence of HTTPS reverse proxy and/or haproxy.
+
+    public_port: int: standard public port for given service
+
+    returns: int: the correct listening port for the API service
+    '''
+    i = 0
+    if len(peer_units()) > 0 or is_clustered():
+        i += 1
+    if https():
+        i += 1
+    return public_port - (i * 10)
+
+
+def determine_haproxy_port(public_port):
+    '''
+    Description: Determine correct proxy listening port based on public IP +
+    existence of HTTPS reverse proxy.
+
+    public_port: int: standard public port for given service
+
+    returns: int: the correct listening port for the HAProxy service
+    '''
+    i = 0
+    if https():
+        i += 1
+    return public_port - (i * 10)

=== added file 'hooks/lib/haproxy_utils.py'
--- hooks/lib/haproxy_utils.py	1970-01-01 00:00:00 +0000
+++ hooks/lib/haproxy_utils.py	2013-05-29 18:03:27 +0000
@@ -0,0 +1,52 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# Authors:
+#  James Page <[e-mail]>
+#
+
+from lib.utils import (
+    relation_ids,
+    relation_list,
+    relation_get,
+    unit_get,
+    reload,
+    render_template
+    )
+import os
+
+HAPROXY_CONF = '/etc/haproxy/haproxy.cfg'
+HAPROXY_DEFAULT = '/etc/default/haproxy'
+
+
+def configure_haproxy(service_ports):
+    '''
+    Configure HAProxy based on the current peers in the service
+    cluster using the provided port map:
+
+        "swift": [ 8080, 8070 ]
+
+    HAproxy will also be reloaded/started if required
+
+    service_ports: dict: dict of lists of [ frontend, backend ]
+    '''
+    cluster_hosts = {}
+    cluster_hosts[os.getenv('JUJU_UNIT_NAME').replace('/', '-')] = \
+        unit_get('private-address')
+    for r_id in relation_ids('cluster'):
+        for unit in relation_list(r_id):
+            cluster_hosts[unit.replace('/', '-')] = \
+                relation_get(attribute='private-address',
+                             rid=r_id,
+                             unit=unit)
+    context = {
+        'units': cluster_hosts,
+        'service_ports': service_ports
+        }
+    with open(HAPROXY_CONF, 'w') as f:
+        f.write(render_template(os.path.basename(HAPROXY_CONF),
+                                context))
+    with open(HAPROXY_DEFAULT, 'w') as f:
+        f.write('ENABLED=1')
+
+    reload('haproxy')

=== modified file 'hooks/lib/openstack_common.py'
--- hooks/lib/openstack_common.py	2013-04-26 10:34:27 +0000
+++ hooks/lib/openstack_common.py	2013-05-29 18:03:27 +0000
@@ -2,7 +2,9 @@
 
 # Common python helper functions used for OpenStack charms.
 
+import apt_pkg as apt
 import subprocess
+import os
 
 CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
 CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
@@ -11,7 +13,7 @@
     'oneiric': 'diablo',
     'precise': 'essex',
     'quantal': 'folsom',
-    'raring' : 'grizzly'
+    'raring': 'grizzly',
 }
 
 
@@ -19,7 +21,8 @@
     '2011.2': 'diablo',
     '2012.1': 'essex',
     '2012.2': 'folsom',
-    '2013.1': 'grizzly'
+    '2013.1': 'grizzly',
+    '2013.2': 'havana',
 }
 
 # The ugly duckling
@@ -27,9 +30,12 @@
     '1.4.3': 'diablo',
     '1.4.8': 'essex',
     '1.7.4': 'folsom',
-    '1.8.0': 'grizzly'
+    '1.7.6': 'grizzly',
+    '1.7.7': 'grizzly',
+    '1.8.0': 'grizzly',
 }
 
+
 def juju_log(msg):
     subprocess.check_call(['juju-log', msg])
 
@@ -68,12 +74,13 @@
         ca_rel = ca_rel.split('%s-' % ubuntu_rel)[1].split('/')[0]
         return ca_rel
 
-    # Best guess match based on deb or ppa provided strings
+    # Best guess match based on deb string provided
     if src.startswith('deb') or src.startswith('ppa'):
         for k, v in openstack_codenames.iteritems():
             if v in src:
                 return v
 
+
 def get_os_codename_version(vers):
     '''Determine OpenStack codename from version number.'''
     try:
@@ -95,37 +102,18 @@
 
 def get_os_codename_package(pkg):
     '''Derive OpenStack release codename from an installed package.'''
-    cmd = ['dpkg', '-l', pkg]
-
+    apt.init()
+    cache = apt.Cache()
     try:
-        output = subprocess.check_output(cmd)
-    except subprocess.CalledProcessError:
-        e = 'Could not derive OpenStack version from package that is not '\
-            'installed; %s' % pkg
-        error_out(e)
-
-    def _clean(line):
-        line = line.split(' ')
-        clean = []
-        for c in line:
-            if c != '':
-                clean.append(c)
-        return clean
-
-    vers = None
-    for l in output.split('\n'):
-        if l.startswith('ii'):
-            l = _clean(l)
-            if l[1] == pkg:
-                vers = l[2]
-
-    if not vers:
+        pkg = cache[pkg]
+    except:
         e = 'Could not determine version of installed package: %s' % pkg
         error_out(e)
 
-    vers = vers[:6]
+    vers = apt.UpstreamVersion(pkg.current_ver.ver_str)
+
     try:
-        if 'swift' in pkg:
+        if 'swift' in pkg.name:
             vers = vers[:5]
             return swift_codenames[vers]
         else:
@@ -151,16 +139,17 @@
     e = "Could not determine OpenStack version for package: %s" % pkg
     error_out(e)
 
+
 def configure_installation_source(rel):
     '''Configure apt installation source.'''
 
-    def _import_key(id):
+    def _import_key(keyid):
         cmd = "apt-key adv --keyserver keyserver.ubuntu.com " \
-              "--recv-keys %s" % id
+              "--recv-keys %s" % keyid
         try:
             subprocess.check_call(cmd.split(' '))
-        except:
-            error_out("Error importing repo key %s" % id)
+        except subprocess.CalledProcessError:
+            error_out("Error importing repo key %s" % keyid)
 
     if rel == 'distro':
         return
@@ -169,7 +158,7 @@
         subprocess.check_call(["add-apt-repository", "-y", src])
     elif rel[:3] == "deb":
         l = len(rel.split('|'))
-        if l ==  2:
+        if l == 2:
             src, key = rel.split('|')
             juju_log("Importing PPA key from keyserver for %s" % src)
             _import_key(key)
@@ -222,3 +211,20 @@
             f.write(src)
     else:
         error_out("Invalid openstack-release specified: %s" % rel)
+
+
+def save_script_rc(script_path="scripts/scriptrc", **env_vars):
+    """
+    Write an rc file in the charm-delivered directory containing
+    exported environment variables provided by env_vars. Any charm scripts run
+    outside the juju hook environment can source this scriptrc to obtain
+    updated config information necessary to perform health checks or
+    service changes.
+    """
+    charm_dir = os.getenv('CHARM_DIR')
+    juju_rc_path = "%s/%s" % (charm_dir, script_path)
+    with open(juju_rc_path, 'wb') as rc_script:
+        rc_script.write(
+            "#!/bin/bash\n")
+        [rc_script.write('export %s=%s\n' % (u, p))
+         for u, p in env_vars.iteritems() if u != "script_path"]

=== added file 'hooks/lib/utils.py'
--- hooks/lib/utils.py	1970-01-01 00:00:00 +0000
+++ hooks/lib/utils.py	2013-05-29 18:03:27 +0000
@@ -0,0 +1,332 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# This file is sourced from lp:openstack-charm-helpers
+#
+# Authors:
+#  James Page <[e-mail]>
+#  Paul Collins <[e-mail]>
+#  Adam Gandelman <[e-mail]>
+#
+
+import json
+import os
+import subprocess
+import socket
+import sys
+
+
+def do_hooks(hooks):
+    hook = os.path.basename(sys.argv[0])
+
+    try:
+        hook_func = hooks[hook]
+    except KeyError:
+        juju_log('INFO',
+                 "This charm doesn't know how to handle '{}'.".format(hook))
+    else:
+        hook_func()
+
+
+def install(*pkgs):
+    cmd = [
+        'apt-get',
+        '-y',
+        'install'
+          ]
+    for pkg in pkgs:
+        cmd.append(pkg)
+    subprocess.check_call(cmd)
+
+TEMPLATES_DIR = 'templates'
+
+try:
+    import jinja2
+except ImportError:
+    install('python-jinja2')
+    import jinja2
+
+try:
+    import dns.resolver
+except ImportError:
+    install('python-dnspython')
+    import dns.resolver
+
+
+def render_template(template_name, context, template_dir=TEMPLATES_DIR):
+    templates = jinja2.Environment(
+                    loader=jinja2.FileSystemLoader(template_dir)
+                    )
+    template = templates.get_template(template_name)
+    return template.render(context)
+
+CLOUD_ARCHIVE = \
+""" # Ubuntu Cloud Archive
+deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
+"""
+
+CLOUD_ARCHIVE_POCKETS = {
+    'folsom': 'precise-updates/folsom',
+    'folsom/updates': 'precise-updates/folsom',
+    'folsom/proposed': 'precise-proposed/folsom',
+    'grizzly': 'precise-updates/grizzly',
+    'grizzly/updates': 'precise-updates/grizzly',
+    'grizzly/proposed': 'precise-proposed/grizzly'
+    }
+
+
+def configure_source():
+    source = str(config_get('openstack-origin'))
+    if not source:
+        return
+    if source.startswith('ppa:'):
+        cmd = [
+            'add-apt-repository',
+            source
+            ]
+        subprocess.check_call(cmd)
+    if source.startswith('cloud:'):
+        # CA values should be formatted as cloud:ubuntu-openstack/pocket, eg:
+        #   cloud:precise-folsom/updates or cloud:precise-folsom/proposed
+        install('ubuntu-cloud-keyring')
+        pocket = source.split(':')[1]
+        pocket = pocket.split('-')[1]
+        with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
+            apt.write(CLOUD_ARCHIVE.format(CLOUD_ARCHIVE_POCKETS[pocket]))
+    if source.startswith('deb'):
+        l = len(source.split('|'))
+        if l == 2:
+            (apt_line, key) = source.split('|')
+            cmd = [
+                'apt-key',
+                'adv', '--keyserver keyserver.ubuntu.com',
+                '--recv-keys', key
+                ]
+            subprocess.check_call(cmd)
+        elif l == 1:
+            apt_line = source
+
+        with open('/etc/apt/sources.list.d/quantum.list', 'w') as apt:
+            apt.write(apt_line + "\n")
+    cmd = [
+        'apt-get',
+        'update'
+        ]
+    subprocess.check_call(cmd)
+
+# Protocols
+TCP = 'TCP'
+UDP = 'UDP'
+
+
+def expose(port, protocol='TCP'):
+    cmd = [
+        'open-port',
+        '{}/{}'.format(port, protocol)
+        ]
+    subprocess.check_call(cmd)
+
+
+def juju_log(severity, message):
+    cmd = [
+        'juju-log',
+        '--log-level', severity,
+        message
+        ]
+    subprocess.check_call(cmd)
+
+
+cache = {}
+
+
+def cached(func):
+    def wrapper(*args, **kwargs):
+        global cache
+        key = str((func, args, kwargs))
+        try:
+            return cache[key]
+        except KeyError:
+            res = func(*args, **kwargs)
+            cache[key] = res
+            return res
+    return wrapper
+
+
+@cached
+def relation_ids(relation):
+    cmd = [
+        'relation-ids',
+        relation
+        ]
+    result = str(subprocess.check_output(cmd)).split()
+    if result == "":
+        return None
+    else:
+        return result
+
+
+@cached
+def relation_list(rid):
+    cmd = [
+        'relation-list',
+        '-r', rid,
+        ]
+    result = str(subprocess.check_output(cmd)).split()
+    if result == "":
+        return None
+    else:
+        return result
+
+
+@cached
+def relation_get(attribute, unit=None, rid=None):
+    cmd = [
+        'relation-get',
+        ]
+    if rid:
+        cmd.append('-r')
+        cmd.append(rid)
+    cmd.append(attribute)
+    if unit:
+        cmd.append(unit)
+    value = subprocess.check_output(cmd).strip()  # IGNORE:E1103
+    if value == "":
+        return None
+    else:
+        return value
+
+
+@cached
+def relation_get_dict(relation_id=None, remote_unit=None):
+    """Obtain all relation data as dict by way of JSON"""
+    cmd = [
+        'relation-get', '--format=json'
+        ]
+    if relation_id:
+        cmd.append('-r')
+        cmd.append(relation_id)
+    if remote_unit:
+        remote_unit_orig = os.getenv('JUJU_REMOTE_UNIT', None)
+        os.environ['JUJU_REMOTE_UNIT'] = remote_unit
+    j = subprocess.check_output(cmd)
+    if remote_unit and remote_unit_orig:
+        os.environ['JUJU_REMOTE_UNIT'] = remote_unit_orig
+    d = json.loads(j)
+    settings = {}
+    # convert unicode to strings
+    for k, v in d.iteritems():
+        settings[str(k)] = str(v)
+    return settings
+
+
+def relation_set(**kwargs):
+    cmd = [
+        'relation-set'
+        ]
+    args = []
+    for k, v in kwargs.items():
+        if k == 'rid':
+            if v:
+                cmd.append('-r')
+                cmd.append(v)
+        else:
+            args.append('{}={}'.format(k, v))
+    cmd += args
+    subprocess.check_call(cmd)
+
+
+@cached
+def unit_get(attribute):
+    cmd = [
+        'unit-get',
+        attribute
+        ]
+    value = subprocess.check_output(cmd).strip()  # IGNORE:E1103
+    if value == "":
+        return None
+    else:
+        return value
+
+
+@cached
+def config_get(attribute):
+    cmd = [
+        'config-get',
+        '--format',
+        'json',
+        ]
+    out = subprocess.check_output(cmd).strip()  # IGNORE:E1103
+    cfg = json.loads(out)
+
+    try:
+        return cfg[attribute]
+    except KeyError:
+        return None
+
+
+@cached
+def get_unit_hostname():
+    return socket.gethostname()
+
+
+@cached
+def get_host_ip(hostname=unit_get('private-address')):
+    try:
+        # Test to see if already an IPv4 address
+        socket.inet_aton(hostname)
+        return hostname
+    except socket.error:
+        answers = dns.resolver.query(hostname, 'A')
+        if answers:
+            return answers[0].address
+    return None
+
+
+def _svc_control(service, action):
+    subprocess.check_call(['service', service, action])
+
+
+def restart(*services):
+    for service in services:
+        _svc_control(service, 'restart')
+
+
+def stop(*services):
+    for service in services:
+        _svc_control(service, 'stop')
+
+
+def start(*services):
+    for service in services:
+        _svc_control(service, 'start')
+
+
+def reload(*services):
+    for service in services:
+        try:
+            _svc_control(service, 'reload')
+        except subprocess.CalledProcessError:
+            # Reload failed - either service does not support reload
+            # or it was not running - restart will fixup most things
+            _svc_control(service, 'restart')
+
+
+def running(service):
+    try:
+        output = subprocess.check_output(['service', service, 'status'])
+    except subprocess.CalledProcessError:
+        return False
+    else:
+        if ("start/running" in output or
+            "is running" in output):
+            return True
+        else:
+            return False
+
+
+def is_relation_made(relation, key='private-address'):
+    for r_id in (relation_ids(relation) or []):
+        for unit in (relation_list(r_id) or []):
+            if relation_get(key, rid=r_id, unit=unit):
+                return True
+    return False

=== modified symlink 'hooks/object-store-relation-joined'
=== target changed u'swift-hooks.py' => u'swift_hooks.py'
=== removed file 'hooks/swift-hooks.py'
--- hooks/swift-hooks.py	2013-04-10 20:33:37 +0000
+++ hooks/swift-hooks.py	1970-01-01 00:00:00 +0000
@@ -1,164 +0,0 @@
-#!/usr/bin/python
-
-import os
-import utils
-import sys
-import shutil
-import uuid
-from subprocess import check_call
-
-import lib.openstack_common as openstack
-import swift_utils as swift
-
-def install():
-    src = utils.config_get('openstack-origin')
-    if src != 'distro':
-        openstack.configure_installation_source(src)
-    check_call(['apt-get', 'update'])
-    rel = openstack.get_os_codename_install_source(src)
-
-    pkgs = swift.determine_packages(rel)
-    utils.install(*pkgs)
-
-    swift.ensure_swift_dir()
-
-    # initialize swift configs.
-    # swift.conf hash
-    ctxt = {
-        'swift_hash': swift.get_swift_hash()
-    }
-    with open(swift.SWIFT_CONF, 'w') as conf:
-        conf.write(swift.render_config(swift.SWIFT_CONF, ctxt))
-
-    # swift-proxy.conf
-    swift.write_proxy_config()
-
-    # memcached.conf
-    ctxt = { 'proxy_ip': utils.get_host_ip() }
-    with open(swift.MEMCACHED_CONF, 'w') as conf:
-        conf.write(swift.render_config(swift.MEMCACHED_CONF, ctxt))
-    check_call(['service', 'memcached', 'restart'])
-
-    # generate or setup SSL certificate
-    swift.configure_ssl()
-
-    # initialize new storage rings.
-    for ring in swift.SWIFT_RINGS.iteritems():
-        swift.initialize_ring(ring[1],
-                              utils.config_get('partition-power'),
-                              utils.config_get('replicas'),
-                              utils.config_get('min-hours'))
-
-    # configure a directory on webserver for distributing rings.
-    if not os.path.isdir(swift.WWW_DIR):
-        os.mkdir(swift.WWW_DIR, 0755)
-    uid, gid = swift.swift_user()
-    os.chown(swift.WWW_DIR, uid, gid)
-    swift.write_apache_config()
-
-
-def keystone_joined(relid=None):
-    hostname = utils.unit_get('private-address')
-    port = utils.config_get('bind-port')
-    ssl = utils.config_get('use-https')
-    if ssl == 'yes':
-        proto = 'https'
-    else:
-        proto = 'http'
-    admin_url = '%s://%s:%s' % (proto, hostname, port)
-    internal_url = public_url = '%s/v1/AUTH_$(tenant_id)s' % admin_url
-    utils.relation_set(service='swift',
-                       region=utils.config_get('region'),
-                       public_url=public_url, internal_url=internal_url,
-                       admin_url=admin_url,
-                       requested_roles=utils.config_get('operator-roles'),
-                       rid=relid)
-
-
-def keystone_changed():
-    swift.write_proxy_config()
-
-
-def balance_rings():
-    '''handle doing ring balancing and distribution.'''
-    new_ring = False
-    for ring in swift.SWIFT_RINGS.itervalues():
-        if swift.balance_ring(ring):
-            utils.juju_log('INFO', 'Balanced ring %s' % ring)
-            new_ring = True
-    if not new_ring:
-        return
-
-    for ring in swift.SWIFT_RINGS.keys():
-        f = '%s.ring.gz' % ring
-        shutil.copyfile(os.path.join(swift.SWIFT_CONF_DIR, f),
-                        os.path.join(swift.WWW_DIR, f))
-
-    msg = 'Broadcasting notification to all storage nodes that new '\
-          'ring is ready for consumption.'
-    utils.juju_log('INFO', msg)
-
-    www_dir = swift.WWW_DIR.split('/var/www/')[1]
-    trigger = uuid.uuid4()
-    swift_hash = swift.get_swift_hash()
-    # notify storage nodes that there is a new ring to fetch.
-    for relid in utils.relation_ids('swift-storage'):
-        utils.relation_set(rid=relid, swift_hash=swift_hash,
-                           www_dir=www_dir, trigger=trigger)
-    swift.proxy_control('restart')
-
-def storage_changed():
-    account_port = utils.config_get('account-ring-port')
-    object_port = utils.config_get('object-ring-port')
-    container_port = utils.config_get('container-ring-port')
-    zone = swift.get_zone(utils.config_get('zone-assignment'))
-    node_settings = {
-        'ip': utils.get_host_ip(utils.relation_get('private-address')),
-        'zone': zone,
-        'account_port': utils.relation_get('account_port'),
-        'object_port': utils.relation_get('object_port'),
-        'container_port': utils.relation_get('container_port'),
-    }
-    if None in node_settings.itervalues():
-        utils.juju_log('INFO', 'storage_changed: Relation not ready.')
-        return None
-
-    for k in ['zone', 'account_port', 'object_port', 'container_port']:
-        node_settings[k] = int(node_settings[k])
-
-    # Grant new node access to rings via apache.
-    swift.write_apache_config()
-
-    # allow for multiple devs per unit, passed along as a : separated list
-    devs = utils.relation_get('device').split(':')
-    for dev in devs:
-        node_settings['device'] = dev
-        for ring in swift.SWIFT_RINGS.itervalues():
-            if not swift.exists_in_ring(ring, node_settings):
-                swift.add_to_ring(ring, node_settings)
-
-    if swift.should_balance([r for r in swift.SWIFT_RINGS.itervalues()]):
-        balance_rings()
-
-def storage_broken():
-    swift.write_apache_config()
-
-def config_changed():
-    relids = utils.relation_ids('identity-service')
-    if relids:
-        for relid in relids:
-            keystone_joined(relid)
-    swift.write_proxy_config()
-
-hooks = {
-    'install': install,
-    'config-changed': config_changed,
-    'identity-service-relation-joined': keystone_joined,
-    'identity-service-relation-changed': keystone_changed,
-    'swift-storage-relation-changed': storage_changed,
-    'swift-storage-relation-broken': storage_broken,
-}
-
-utils.do_hooks(hooks)
-
-sys.exit(0)

=== modified symlink 'hooks/swift-storage-relation-broken'
=== target changed u'swift-hooks.py' => u'swift_hooks.py'
=== modified symlink 'hooks/swift-storage-relation-changed'
=== target changed u'swift-hooks.py' => u'swift_hooks.py'
=== added file 'hooks/swift_hooks.py'
--- hooks/swift_hooks.py	1970-01-01 00:00:00 +0000
+++ hooks/swift_hooks.py	2013-05-29 18:03:27 +0000
@@ -0,0 +1,257 @@
+#!/usr/bin/python
+
+import os
+import sys
+import shutil
+import uuid
+from subprocess import check_call
+
+import lib.openstack_common as openstack
+import lib.utils as utils
+import lib.cluster_utils as cluster
+import swift_utils as swift
+
+extra_pkgs = [
+    "haproxy",
+    "python-jinja2"
+    ]
+
+
+def install():
+    src = utils.config_get('openstack-origin')
+    if src != 'distro':
+        openstack.configure_installation_source(src)
+    check_call(['apt-get', 'update'])
+    rel = openstack.get_os_codename_install_source(src)
+
+    pkgs = swift.determine_packages(rel)
+    utils.install(*pkgs)
+    utils.install(*extra_pkgs)
+
+    swift.ensure_swift_dir()
+
+    # initialize swift configs.
+    # swift.conf hash
+    ctxt = {
+        'swift_hash': swift.get_swift_hash()
+    }
+    with open(swift.SWIFT_CONF, 'w') as conf:
+        conf.write(swift.render_config(swift.SWIFT_CONF, ctxt))
+
+    # swift-proxy.conf
+    swift.write_proxy_config()
+
+    # memcached.conf
+    ctxt = {'proxy_ip': utils.get_host_ip()}
+    with open(swift.MEMCACHED_CONF, 'w') as conf:
+        conf.write(swift.render_config(swift.MEMCACHED_CONF, ctxt))
+    check_call(['service', 'memcached', 'restart'])
+
+    # initialize new storage rings.
+    for ring in swift.SWIFT_RINGS.iteritems():
+        swift.initialize_ring(ring[1],
+                              utils.config_get('partition-power'),
+                              utils.config_get('replicas'),
+                              utils.config_get('min-hours'))
+
+    # configure a directory on webserver for distributing rings.
+    if not os.path.isdir(swift.WWW_DIR):
+        os.mkdir(swift.WWW_DIR, 0755)
+    uid, gid = swift.swift_user()
+    os.chown(swift.WWW_DIR, uid, gid)
+    swift.write_apache_config()
+    swift.configure_https()
+
+
+def keystone_joined(relid=None):
+    if not cluster.eligible_leader(swift.SWIFT_HA_RES):
+        return
+    if cluster.is_clustered():
+        hostname = utils.config_get('vip')
+    else:
+        hostname = utils.unit_get('private-address')
+    port = utils.config_get('bind-port')
+    if cluster.https():
+        proto = 'https'
+    else:
+        proto = 'http'
+    admin_url = '%s://%s:%s' % (proto, hostname, port)
+    internal_url = public_url = '%s/v1/AUTH_$(tenant_id)s' % admin_url
+    utils.relation_set(service='swift',
+                       region=utils.config_get('region'),
+                       public_url=public_url, internal_url=internal_url,
+                       admin_url=admin_url,
+                       requested_roles=utils.config_get('operator-roles'),
+                       rid=relid)
+
+
+def keystone_changed():
+    swift.write_proxy_config()
+    swift.configure_https()
+    # Re-fire keystone hooks to ripple back the HTTPS service entry
+    for relid in utils.relation_ids('identity-service'):
+        keystone_joined(relid=relid)
+
+
+def balance_rings():
+    '''handle doing ring balancing and distribution.'''
+    new_ring = False
+    for ring in swift.SWIFT_RINGS.itervalues():
+        if swift.balance_ring(ring):
+            utils.juju_log('INFO', 'Balanced ring %s' % ring)
+            new_ring = True
+    if not new_ring:
+        return
+
+    for ring in swift.SWIFT_RINGS.keys():
+        f = '%s.ring.gz' % ring
+        shutil.copyfile(os.path.join(swift.SWIFT_CONF_DIR, f),
+                        os.path.join(swift.WWW_DIR, f))
+
+    if cluster.eligible_leader(swift.SWIFT_HA_RES):
+        msg = 'Broadcasting notification to all storage nodes that new '\
+              'ring is ready for consumption.'
+        utils.juju_log('INFO', msg)
+        path = swift.WWW_DIR.split('/var/www/')[1]
+        trigger = uuid.uuid4()
+        swift_hash = swift.get_swift_hash()
+
+        if cluster.is_clustered():
+            hostname = utils.config_get('vip')
+        else:
+            hostname = utils.unit_get('private-address')
+
+        rings_url = 'http://%s/%s' % (hostname, path)
+        # notify storage nodes that there is a new ring to fetch.
+        for relid in utils.relation_ids('swift-storage'):
+            utils.relation_set(rid=relid, swift_hash=swift_hash,
+                               rings_url=rings_url, trigger=trigger)
+
+    swift.proxy_control('restart')
+
+
+def storage_changed():
+    zone = swift.get_zone(utils.config_get('zone-assignment'))
+    node_settings = {
+        'ip': utils.get_host_ip(utils.relation_get('private-address')),
+        'zone': zone,
+        'account_port': utils.relation_get('account_port'),
+        'object_port': utils.relation_get('object_port'),
+        'container_port': utils.relation_get('container_port'),
+    }
+    if None in node_settings.itervalues():
+        utils.juju_log('INFO', 'storage_changed: Relation not ready.')
+        return None
+
+    for k in ['zone', 'account_port', 'object_port', 'container_port']:
+        node_settings[k] = int(node_settings[k])
+
+    # Grant new node access to rings via apache.
+    swift.write_apache_config()
+
+    # allow for multiple devs per unit, passed along as a : separated list
+    devs = utils.relation_get('device').split(':')
+    for dev in devs:
+        node_settings['device'] = dev
+        for ring in swift.SWIFT_RINGS.itervalues():
+            if not swift.exists_in_ring(ring, node_settings):
+                swift.add_to_ring(ring, node_settings)
+
+    if swift.should_balance([r for r in swift.SWIFT_RINGS.itervalues()]):
+        balance_rings()
+
+
+def storage_broken():
+    swift.write_apache_config()
+
+
+def config_changed():
+    # Determine whether or not we should do an upgrade, based on the
+    # the version offered in keyston-release.
+    src = utils.config_get('openstack-origin')
+    available = openstack.get_os_codename_install_source(src)
+    installed = openstack.get_os_codename_package('python-swift')
+    if (available and
+        openstack.get_os_version_codename(available) > \
+        openstack.get_os_version_codename(installed)):
+        pkgs = swift.determine_packages(available)
+        swift.do_openstack_upgrade(src, pkgs)
+
+    relids = utils.relation_ids('identity-service')
+    if relids:
+        for relid in relids:
+            keystone_joined(relid)
+    swift.write_proxy_config()
+    swift.configure_https()
+
+
+def cluster_changed():
+    swift.configure_haproxy()
+
+
+def ha_relation_changed():
+    clustered = utils.relation_get('clustered')
+    if clustered and cluster.is_leader(swift.SWIFT_HA_RES):
+        utils.juju_log('INFO',
+                       'Cluster configured, notifying other services and'
+                       'updating keystone endpoint configuration')
+        # Tell all related services to start using
+        # the VIP instead
+        for r_id in utils.relation_ids('identity-service'):
+            keystone_joined(relid=r_id)
+
+
+def ha_relation_joined():
+    # Obtain the config values necessary for the cluster config. These
+    # include multicast port and interface to bind to.
+    corosync_bindiface = utils.config_get('ha-bindiface')
+    corosync_mcastport = utils.config_get('ha-mcastport')
+    vip = utils.config_get('vip')
+    vip_cidr = utils.config_get('vip_cidr')
+    vip_iface = utils.config_get('vip_iface')
+    if not vip:
+        utils.juju_log('ERROR',
+                       'Unable to configure hacluster as vip not provided')
+        sys.exit(1)
+
+    # Obtain resources
+    resources = {
+            'res_swift_vip': 'ocf:heartbeat:IPaddr2',
+            'res_swift_haproxy': 'lsb:haproxy'
+        }
+    resource_params = {
+            'res_swift_vip': 'params ip="%s" cidr_netmask="%s" nic="%s"' % \
+                              (vip, vip_cidr, vip_iface),
+            'res_swift_haproxy': 'op monitor interval="5s"'
+        }
+    init_services = {
+            'res_swift_haproxy': 'haproxy'
+        }
+    clones = {
+            'cl_swift_haproxy': 'res_swift_haproxy'
+        }
+
+    utils.relation_set(init_services=init_services,
+                       corosync_bindiface=corosync_bindiface,
+                       corosync_mcastport=corosync_mcastport,
+                       resources=resources,
+                       resource_params=resource_params,
+                       clones=clones)
+
+
+hooks = {
+    'install': install,
+    'config-changed': config_changed,
+    'identity-service-relation-joined': keystone_joined,
+    'identity-service-relation-changed': keystone_changed,
+    'swift-storage-relation-changed': storage_changed,
+    'swift-storage-relation-broken': storage_broken,
+    "cluster-relation-joined": cluster_changed,
+    "cluster-relation-changed": cluster_changed,
+    "ha-relation-joined": ha_relation_joined,
+    "ha-relation-changed": ha_relation_changed
+}
+
+utils.do_hooks(hooks)
+
+sys.exit(0)

=== modified file 'hooks/swift_utils.py'
--- hooks/swift_utils.py	2013-01-17 18:35:04 +0000
+++ hooks/swift_utils.py	2013-05-29 18:03:27 +0000
@@ -2,10 +2,16 @@
 import pwd
 import subprocess
 import lib.openstack_common as openstack
-import utils
+import lib.utils as utils
+import lib.haproxy_utils as haproxy
+import lib.apache_utils as apache
+import lib.cluster_utils as cluster
+import sys
+from base64 import b64encode
+
 
 # Various config files that are managed via templating.
-SWIFT_HASH_FILE='/var/lib/juju/swift-hash-path.conf'
+SWIFT_HASH_FILE = '/var/lib/juju/swift-hash-path.conf'
 SWIFT_CONF = '/etc/swift/swift.conf'
 SWIFT_PROXY_CONF = '/etc/swift/proxy-server.conf'
 SWIFT_CONF_DIR = os.path.dirname(SWIFT_CONF)
@@ -32,9 +38,12 @@
     'python-keystone',
 ]
 
+SWIFT_HA_RES = 'res_swift_vip'
+
 # Folsom-specific packages
 FOLSOM_PACKAGES = BASE_PACKAGES + ['swift-plugin-s3']
 
+
 def proxy_control(action):
     '''utility to work around swift-init's bad RCs.'''
     def _cmd(action):
@@ -49,8 +58,9 @@
         elif status == 0:
             return subprocess.check_call(_cmd('stop'))
 
-    # the proxy will not start unless there are balanced rings, gzip'd in /etc/swift
-    missing=False
+    # the proxy will not start unless there are balanced rings
+    # gzip'd in /etc/swift
+    missing = False
     for k in SWIFT_RINGS.keys():
         if not os.path.exists(os.path.join(SWIFT_CONF_DIR, '%s.ring.gz' % k)):
             missing = True
@@ -69,8 +79,9 @@
         elif status == 1:
             return subprocess.check_call(_cmd('start'))
 
+
 def swift_user(username='swift'):
-    user = pwd.getpwnam('swift')
+    user = pwd.getpwnam(username)
     return (user.pw_uid, user.pw_gid)
 
 
@@ -105,6 +116,10 @@
     if os.path.isfile(SWIFT_HASH_FILE):
         with open(SWIFT_HASH_FILE, 'r') as hashfile:
             swift_hash = hashfile.read().strip()
+    elif utils.config_get('swift-hash'):
+        swift_hash = utils.config_get('swift-hash')
+        with open(SWIFT_HASH_FILE, 'w') as hashfile:
+            hashfile.write(swift_hash)
     else:
         cmd = ['od', '-t', 'x8', '-N', '8', '-A', 'n']
         rand = open('/dev/random', 'r')
@@ -148,11 +163,16 @@
                 'keystone_host': utils.relation_get('auth_host',
                                                     unit, relid),
                 'auth_port': utils.relation_get('auth_port', unit, relid),
-                'service_user': utils.relation_get('service_username', unit, relid),
-                'service_password': utils.relation_get('service_password', unit, relid),
-                'service_tenant': utils.relation_get('service_tenant', unit, relid),
-                'service_port': utils.relation_get('service_port', unit, relid),
-                'admin_token': utils.relation_get('admin_token', unit, relid),
+                'service_user': utils.relation_get('service_username',
+                                                   unit, relid),
+                'service_password': utils.relation_get('service_password',
+                                                       unit, relid),
+                'service_tenant': utils.relation_get('service_tenant',
+                                                     unit, relid),
+                'service_port': utils.relation_get('service_port',
+                                                   unit, relid),
+                'admin_token': utils.relation_get('admin_token',
+                                                  unit, relid),
             }
             if None not in ks_auth.itervalues():
                 return ks_auth
@@ -167,19 +187,20 @@
         import multiprocessing
         workers = multiprocessing.cpu_count()
 
+    env_vars = {'OPENSTACK_SERVICE_SWIFT': 'proxy-server',
+                'OPENSTACK_PORT_API': bind_port,
+                'OPENSTACK_PORT_MEMCACHED': 11211}
+    openstack.save_script_rc(**env_vars)
+
     ctxt = {
         'proxy_ip': utils.get_host_ip(),
-        'bind_port': bind_port,
+        'bind_port': cluster.determine_api_port(bind_port),
         'workers': workers,
-        'operator_roles': utils.config_get('operator-roles')
+        'operator_roles': utils.config_get('operator-roles'),
+        'delay_auth_decision': utils.config_get('delay-auth-decision')
     }
 
-    if utils.config_get('use-https') == 'no':
-        ctxt['ssl'] = False
-    else:
-        ctxt['ssl'] = True
-        ctxt['ssl_cert'] = SSL_CERT
-        ctxt['ssl_key'] = SSL_KEY
+    ctxt['ssl'] = False
 
     ks_auth = get_keystone_auth()
     if ks_auth:
@@ -193,23 +214,10 @@
     proxy_control('restart')
     subprocess.check_call(['open-port', str(bind_port)])
 
-def configure_ssl():
-    # this should be expanded to cover setting up user-specified certificates
-    if (utils.config_get('use-https') == 'yes' and
-        not os.path.isfile(SSL_CERT) and
-        not os.path.isfile(SSL_KEY)):
-        subj = '/C=%s/ST=%s/L=%s/CN=%s' %\
-               (utils.config_get('country'), utils.config_get('state'),
-                utils.config_get('locale'), utils.config_get('common-name'))
-        cmd = ['openssl', 'req', '-new', '-x509', '-nodes',
-               '-out', SSL_CERT, '-keyout', SSL_KEY,
-               '-subj', subj]
-        subprocess.check_call(cmd)
-
 
 def _load_builder(path):
     # lifted straight from /usr/bin/swift-ring-builder
-    from swift.common.ring import RingBuilder, Ring
+    from swift.common.ring import RingBuilder
     import cPickle as pickle
     try:
         builder = pickle.load(open(path, 'rb'))
@@ -218,10 +226,8 @@
             builder = RingBuilder(1, 1, 1)
             builder.copy_from(builder_dict)
     except ImportError:  # Happens with really old builder pickles
-        modules['swift.ring_builder'] = \
-            modules['swift.common.ring.builder']
         builder = RingBuilder(1, 1, 1)
-        builder.copy_from(pickle.load(open(argv[1], 'rb')))
+        builder.copy_from(pickle.load(open(path, 'rb')))
     for dev in builder.devs:
         if dev and 'meta' not in dev:
             dev['meta'] = ''
@@ -233,8 +239,6 @@
     pickle.dump(ring.to_dict(), open(ring_path, 'wb'), protocol=2)
 
 
-
-
 def ring_port(ring_path, node):
     '''determine correct port from relation settings for a given ring file.'''
     for name in ['account', 'object', 'container']:
@@ -248,8 +252,8 @@
     ring = RingBuilder(part_power, replicas, min_hours)
     _write_ring(ring, path)
 
+
 def exists_in_ring(ring_path, node):
-    from swift.common.ring import RingBuilder, Ring
     ring = _load_builder(ring_path).to_dict()
     node['port'] = ring_port(ring_path, node)
 
@@ -266,7 +270,6 @@
 
 
 def add_to_ring(ring_path, node):
-    from swift.common.ring import RingBuilder, Ring
     ring = _load_builder(ring_path)
     port = ring_port(ring_path, node)
 
@@ -286,8 +289,9 @@
     }
     ring.add_dev(new_dev)
     _write_ring(ring, ring_path)
-    msg = 'Added new device to ring %s: %s' % (ring_path,
-                                               [k for k in new_dev.iteritems()])
+    msg = 'Added new device to ring %s: %s' %\
+         (ring_path,
+          [k for k in new_dev.iteritems()])
     utils.juju_log('INFO', msg)
 
 
@@ -332,8 +336,8 @@
             potential_zones.append(_get_zone(builder))
         return set(potential_zones).pop()
     else:
-        utils.juju_log('Invalid zone assignment policy: %s' %\
-                       assignemnt_policy)
+        utils.juju_log('ERROR', 'Invalid zone assignment policy: %s' %\
+                       assignment_policy)
         sys.exit(1)
 
 
@@ -351,9 +355,10 @@
         # swift-ring-builder returns 1 on WARNING (ring didn't require balance)
         return False
     else:
-        utils.juju_log('balance_ring: %s returned %s' % (cmd, rc))
+        utils.juju_log('ERROR', 'balance_ring: %s returned %s' % (cmd, rc))
         sys.exit(1)
 
+
 def should_balance(rings):
     '''Based on zones vs min. replicas, determine whether or not the rings
        should be balanaced during initial configuration.'''
@@ -379,8 +384,77 @@
             host = utils.relation_get('private-address', unit, relid)
             allowed_hosts.append(utils.get_host_ip(host))
 
-    ctxt = { 'www_dir': WWW_DIR, 'allowed_hosts': allowed_hosts }
+    ctxt = {
+        'www_dir': WWW_DIR,
+        'allowed_hosts': allowed_hosts
+        }
     with open(APACHE_CONF, 'w') as conf:
         conf.write(render_config(APACHE_CONF, ctxt))
-    subprocess.check_call(['service', 'apache2', 'reload'])
-
+    utils.reload('apache2')
+
+
+def generate_cert():
+    '''
+    Generates a self signed certificate and key using the
+    provided charm configuration data.
+
+    returns: tuple of (cert, key)
+    '''
+    CERT = '/etc/swift/ssl.cert'
+    KEY = '/etc/swift/ssl.key'
+    if (not os.path.exists(CERT) and
+        not os.path.exists(KEY)):
+        subj = '/C=%s/ST=%s/L=%s/CN=%s' %\
+            (utils.config_get('country'), utils.config_get('state'),
+             utils.config_get('locale'), utils.config_get('common-name'))
+        cmd = ['openssl', 'req', '-new', '-x509', '-nodes',
+               '-out', CERT, '-keyout', KEY,
+               '-subj', subj]
+        subprocess.check_call(cmd)
+        os.chmod(KEY, 0600)
+    # Slurp as base64 encoded - makes handling easier up the stack
+    with open(CERT, 'r') as cfile:
+        ssl_cert = b64encode(cfile.read())
+    with open(KEY, 'r') as kfile:
+        ssl_key = b64encode(kfile.read())
+    return (ssl_cert, ssl_key)
+
+
+def configure_haproxy():
+    api_port = utils.config_get('bind-port')
+    service_ports = {
+        "swift": [
+            cluster.determine_haproxy_port(api_port),
+            cluster.determine_api_port(api_port)
+            ]
+        }
+    write_proxy_config()
+    haproxy.configure_haproxy(service_ports)
+
+
+def configure_https():
+    if cluster.https():
+        api_port = utils.config_get('bind-port')
+        if (len(cluster.peer_units()) > 0 or
+            cluster.is_clustered()):
+            target_port = cluster.determine_haproxy_port(api_port)
+            configure_haproxy()
+        else:
+            target_port = cluster.determine_api_port(api_port)
+            write_proxy_config()
+        cert, key = apache.get_cert()
+        if None in (cert, key):
+            cert, key = generate_cert()
+        ca_cert = apache.get_ca_cert()
+        apache.setup_https(namespace="swift",
+                           port_maps={api_port: target_port},
+                           cert=cert, key=key, ca_cert=ca_cert)
+
+
+def do_openstack_upgrade(source, packages):
+    openstack.configure_installation_source(source)
+    os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
+    subprocess.check_call(['apt-get', 'update'])
+    cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confnew', '-y',
+           'install'] + packages
+    subprocess.check_call(cmd)

=== removed file 'hooks/utils.py'
--- hooks/utils.py	2012-12-14 23:07:01 +0000
+++ hooks/utils.py	1970-01-01 00:00:00 +0000
@@ -1,237 +0,0 @@
-
-#
-# Copyright 2012 Canonical Ltd.
-#
-# Authors:
-#  James Page <[e-mail]>
-#  Paul Collins <[e-mail]>
-#
-
-import json
-import os
-import subprocess
-import socket
-import sys
-
-
-def do_hooks(hooks):
-    hook = os.path.basename(sys.argv[0])
-
-    try:
-        hooks[hook]()
-    except KeyError:
-        juju_log('INFO',
-                 "This charm doesn't know how to handle '{}'.".format(hook))
-
-
-def install(*pkgs):
-    cmd = [
-        'apt-get',
-        '-y',
-        'install'
-          ]
-    for pkg in pkgs:
-        cmd.append(pkg)
-    subprocess.check_call(cmd)
-
-TEMPLATES_DIR = 'hooks/templates'
-
-try:
-    import jinja2
-except ImportError:
-    install('python-jinja2')
-    import jinja2
-
-try:
-    import dns.resolver
-    import dns.ipv4
-except ImportError:
-    install('python-dnspython')
-    import dns.resolver
-    import dns.ipv4
-
-
-def render_template(template_name, context, template_dir=TEMPLATES_DIR):
-    templates = jinja2.Environment(
-                    loader=jinja2.FileSystemLoader(template_dir)
-                    )
-    template = templates.get_template(template_name)
-    return template.render(context)
-
-CLOUD_ARCHIVE = \
-""" # Ubuntu Cloud Archive
-deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
-"""
-
-CLOUD_ARCHIVE_POCKETS = {
-    'folsom': 'precise-updates/folsom',
-    'folsom/updates': 'precise-updates/folsom',
-    'folsom/proposed': 'precise-proposed/folsom'
-    }
-
-
-def configure_source():
-    source = str(config_get('openstack-origin'))
-    if not source:
-        return
-    if source.startswith('ppa:'):
-        cmd = [
-            'add-apt-repository',
-            source
-            ]
-        subprocess.check_call(cmd)
-    if source.startswith('cloud:'):
-        install('ubuntu-cloud-keyring')
-        pocket = source.split(':')[1]
-        with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
-            apt.write(CLOUD_ARCHIVE.format(CLOUD_ARCHIVE_POCKETS[pocket]))
-    if source.startswith('deb'):
-        l = len(source.split('|'))
-        if l == 2:
-            (apt_line, key) = source.split('|')
-            cmd = [
-                'apt-key',
-                'adv', '--keyserver keyserver.ubuntu.com',
-                '--recv-keys', key
-                ]
-            subprocess.check_call(cmd)
-        elif l == 1:
-            apt_line = source
-
-        with open('/etc/apt/sources.list.d/quantum.list', 'w') as apt:
-            apt.write(apt_line + "\n")
-    cmd = [
-        'apt-get',
-        'update'
-        ]
-    subprocess.check_call(cmd)
-
-# Protocols
-TCP = 'TCP'
-UDP = 'UDP'
-
-
-def expose(port, protocol='TCP'):
-    cmd = [
-        'open-port',
-        '{}/{}'.format(port, protocol)
-        ]
-    subprocess.check_call(cmd)
-
-
-def juju_log(severity, message):
-    cmd = [
-        'juju-log',
-        '--log-level', severity,
-        message
-        ]
-    subprocess.check_call(cmd)
-
-
-def relation_ids(relation):
-    cmd = [
-        'relation-ids',
-        relation
-        ]
-    return subprocess.check_output(cmd).split()  # IGNORE:E1103
-
-
-def relation_list(rid):
-    cmd = [
-        'relation-list',
-        '-r', rid,
-        ]
-    return subprocess.check_output(cmd).split()  # IGNORE:E1103
-
-
-def relation_get(attribute, unit=None, rid=None):
-    cmd = [
-        'relation-get',
-        ]
-    if rid:
-        cmd.append('-r')
-        cmd.append(rid)
-    cmd.append(attribute)
-    if unit:
-        cmd.append(unit)
-    value = subprocess.check_output(cmd).strip()  # IGNORE:E1103
-    if value == "":
-        return None
-    else:
-        return value
-
-
-def relation_set(**kwargs):
-    cmd = [
-        'relation-set'
-        ]
-    args = []
-    for k, v in kwargs.items():
-        if k == 'rid':
-            if v:
-                cmd.append('-r')
-                cmd.append(v)
-        else:
-            args.append('{}={}'.format(k, v))
-    cmd += args
-    subprocess.check_call(cmd)
-
-
-def unit_get(attribute):
-    cmd = [
-        'unit-get',
-        attribute
-        ]
-    value = subprocess.check_output(cmd).strip()  # IGNORE:E1103
-    if value == "":
-        return None
-    else:
-        return value
-
-
-def config_get(attribute):
-    cmd = [
-        'config-get',
-        '--format',
-        'json',
-        ]
-    out = subprocess.check_output(cmd).strip()  # IGNORE:E1103
-    cfg = json.loads(out)
-
-    try:
-        return cfg[attribute]
-    except KeyError:
-        return None
-
-def get_unit_hostname():
-    return socket.gethostname()
-
-
-def get_host_ip(hostname=unit_get('private-address')):
-    try:
-      # Test to see if already an IPv4 address
-      socket.inet_aton(hostname)
-      return hostname
-    except socket.error:
-      try:
-        answers = dns.resolver.query(hostname, 'A')
-        if answers:
-          return answers[0].address
-      except dns.resolver.NXDOMAIN:
-        pass
-      return None
-
-
-def restart(*services):
-    for service in services:
-        subprocess.check_call(['service', service, 'restart'])
-
-
-def stop(*services):
-    for service in services:
-        subprocess.check_call(['service', service, 'stop'])
-
-
-def start(*services):
-    for service in services:
-        subprocess.check_call(['service', service, 'start'])

=== modified file 'metadata.yaml'
--- metadata.yaml	2012-12-18 19:59:19 +0000
+++ metadata.yaml	2013-05-29 18:03:27 +0000
@@ -12,3 +12,9 @@
     interface: swift
   identity-service:
     interface: keystone
+  ha:
+    interface: hacluster
+    scope: container
+peers:
+  cluster:
+    interface: swift-ha

=== modified file 'revision'
--- revision	2013-04-26 10:34:27 +0000
+++ revision	2013-05-29 18:03:27 +0000
@@ -1,1 +1,1 @@
-112
+132

=== added directory 'scripts'
=== added file 'scripts/add_to_cluster'
--- scripts/add_to_cluster	1970-01-01 00:00:00 +0000
+++ scripts/add_to_cluster	2013-05-29 18:03:27 +0000
@@ -0,0 +1,13 @@
+#!/bin/bash
+service corosync start || /bin/true
+sleep 2
+while ! service pacemaker start; do
+    echo "Attempting to start pacemaker"
+    sleep 1;
+done;
+crm node online
+sleep 2
+while crm status | egrep -q 'Stopped$'; do
+    echo "Waiting for nodes to come online"
+    sleep 1
+done

=== added file 'scripts/remove_from_cluster'
--- scripts/remove_from_cluster	1970-01-01 00:00:00 +0000
+++ scripts/remove_from_cluster	2013-05-29 18:03:27 +0000
@@ -0,0 +1,4 @@
+#!/bin/bash
+crm node standby
+service pacemaker stop
+service corosync stop

=== renamed directory 'hooks/templates' => 'templates'
=== added file 'templates/apache2_site.tmpl'
--- templates/apache2_site.tmpl	1970-01-01 00:00:00 +0000
+++ templates/apache2_site.tmpl	2013-05-29 18:03:27 +0000
@@ -0,0 +1,19 @@
+Listen {{ ext }}
+NameVirtualHost *:{{ ext }}
+<VirtualHost *:{{ ext }}>
+    ServerName {{ private_address }}
+    SSLEngine on
+    SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert
+    SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key
+    ProxyPass / http://localhost:{{ int }}/
+    ProxyPassReverse / http://localhost:{{ int }}/
+    ProxyPreserveHost on
+</VirtualHost>
+<Proxy *>
+    Order deny,allow
+    Allow from all
+</Proxy>
+<Location />
+    Order allow,deny
+    Allow from all
+</Location>

=== modified file 'templates/essex/proxy-server.conf'
--- hooks/templates/essex/proxy-server.conf	2012-12-14 23:07:01 +0000
+++ templates/essex/proxy-server.conf	2013-05-29 18:03:27 +0000
@@ -45,6 +45,15 @@
 admin_tenant_name = {{ service_tenant }}
 admin_user = {{ service_user }}
 admin_password = {{ service_password }}
+{% if os_release == 'essex' %}
+{% if delay_auth_decision|lower == 'true' %}
+delay_auth_decision = 1
+{% else %}
+delay_auth_decision = 0
+{% endif %}
+{% else %}
+delay_auth_decision = {{ delay_auth_decision|lower }}
+{% endif %}
 {% if os_release != 'essex' %}signing_dir = /etc/swift{% endif %}
 
 

=== modified file 'templates/grizzly/proxy-server.conf'
--- hooks/templates/grizzly/proxy-server.conf	2013-01-29 00:41:52 +0000
+++ templates/grizzly/proxy-server.conf	2013-05-29 18:03:27 +0000
@@ -45,6 +45,7 @@
 admin_tenant_name = {{ service_tenant }}
 admin_user = {{ service_user }}
 admin_password = {{ service_password }}
+delay_auth_decision = {{ delay_auth_decision|lower }}
 {% if os_release != 'essex' %}signing_dir = /etc/swift{% endif %}
 
 

=== added file 'templates/haproxy.cfg'
--- templates/haproxy.cfg	1970-01-01 00:00:00 +0000
+++ templates/haproxy.cfg	2013-05-29 18:03:27 +0000
@@ -0,0 +1,35 @@
+global
+    log 127.0.0.1 local0
+    log 127.0.0.1 local1 notice
+    maxconn 20000
+    user haproxy
+    group haproxy
+    spread-checks 0
+
+defaults
+    log global
+    mode http
+    option httplog
+    option dontlognull
+    retries 3
+    timeout queue 1000
+    timeout connect 1000
+    timeout client 30000
+    timeout server 30000
+
+listen stats :8888
+    mode http
+    stats enable
+    stats hide-version
+    stats realm Haproxy\ Statistics
+    stats uri /
+    stats auth admin:password
+
+{% for service, ports in service_ports.iteritems() -%}
+listen {{ service }} 0.0.0.0:{{ ports[0] }}
+    balance roundrobin
+    option tcplog
+    {% for unit, address in units.iteritems() -%}
+    server {{ unit }} {{ address }}:{{ ports[1] }} check
+    {% endfor %}
+{% endfor %}