Merge lp:~chad.smith/charms/precise/mysql/mysql-ha-with-health into lp:charms/mysql

[Chad Smith]

Add mysql health checks and add_to_cluster/remove_from_cluster scripts to mysql/ha-support charm. Landscape-client uses these scripts during rolling OpenStack upgrades to make sure mysql services are alive and well and that the HA vips are migrated off the current node in the event of a reboot.

Comparable changes have gone into most of the other openstack services.

[Chad Smith (chad.smith)]

We only have a very simplistic health_check in this branch because mysql service is actually completing inoperable on the passive unit due to the ceph shared rados filesystem being shifted to the active leader of the HA service. Without the filesystem, we can't validate that mysql is in good health until this unit cycles back to active leader.

Unmerged revisions

=== modified file 'config.yaml'
--- config.yaml	2012-09-13 21:44:01 +0000
+++ config.yaml	2013-03-26 14:36:21 +0000
@@ -31,3 +31,39 @@
         default: 'MIXED'
         type: string
         description: If binlogging is enabled, this is the format that will be used. Ignored when tuning-level == fast.
+    vip:
+        type: string
+        description: "Virtual IP to use to front mysql in ha configuration"
+    vip_iface:
+        type: string
+        default: eth0
+        description: "Network Interface where to place the Virtual IP"
+    vip_cidr:
+        type: int
+        default: 24
+        description: "Netmask that will be used for the Virtual IP"
+    ha-bindiface:
+        type: string
+        default: eth0
+        description: |
+          Default network interface on which HA cluster will bind to communication
+          with the other members of the HA Cluster.
+    ha-mcastport:
+        type: int
+        default: 5411
+        description: |
+          Default multicast port number that will be used to communicate between
+          HA Cluster nodes.
+    block-size:
+        type: int
+        default: 5
+        description: |
+          Default block storage size to create when setting up MySQL block storage.
+          This value should be specified in GB (e.g. 100 not 100GB).
+    rbd-name:
+        type: string
+        default: mysql1
+        description: |
+          The name that will be used to create the Ceph's RBD image with. If the
+          image name exists in Ceph, it will be re-used and the data will be
+          overwritten.

=== added symlink 'hooks/ceph-relation-changed'
=== target is u'ha_relations.py'
=== added symlink 'hooks/ceph-relation-joined'
=== target is u'ha_relations.py'
=== added symlink 'hooks/cluster-relation-changed'
=== target is u'ha_relations.py'
=== modified file 'hooks/common.py'
--- hooks/common.py	2012-11-29 16:31:14 +0000
+++ hooks/common.py	2013-03-26 14:36:21 +0000
@@ -7,7 +7,7 @@
 import uuid
 
 def get_service_user_file(service):
-    return '/var/lib/juju/%s.service_user2' % service
+    return '/var/lib/mysql/%s.service_user2' % service
 
 
 def get_service_user(service):
@@ -58,7 +58,7 @@
 
 def get_db_cursor():
     # Connect to mysql
-    passwd = open("/var/lib/juju/mysql.passwd").read().strip()
+    passwd = open("/var/lib/mysql/mysql.passwd").read().strip()
     connection = MySQLdb.connect(user="root", host="localhost", passwd=passwd)
     return connection.cursor()
 

=== modified file 'hooks/config-changed'
--- hooks/config-changed	2012-11-21 17:14:24 +0000
+++ hooks/config-changed	2013-03-26 14:36:21 +0000
@@ -89,7 +89,7 @@
     check_call(['add-apt-repository','-y','deb http://%s %s main' % (source, series)])
     check_call(['apt-get','update'])
 
-with open('/var/lib/juju/mysql.passwd','r') as rpw:
+with open('/var/lib/mysql/mysql.passwd','r') as rpw:
     root_pass = rpw.read()
 
 dconf = Popen(['debconf-set-selections'], stdin=PIPE)

=== added symlink 'hooks/ha-relation-changed'
=== target is u'ha_relations.py'
=== added symlink 'hooks/ha-relation-joined'
=== target is u'ha_relations.py'
=== added file 'hooks/ha_relations.py'
--- hooks/ha_relations.py	1970-01-01 00:00:00 +0000
+++ hooks/ha_relations.py	2013-03-26 14:36:21 +0000
@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+
+import sys
+import os
+
+import lib.utils as utils
+import lib.ceph_utils as ceph
+import lib.cluster_utils as cluster
+
+# CEPH
+DATA_SRC_DST = '/var/lib/mysql'
+SERVICE_NAME = os.getenv('JUJU_UNIT_NAME').split('/')[0]
+POOL_NAME = SERVICE_NAME
+LEADER_RES = 'res_mysql_vip'
+
+
+def ha_relation_joined():
+    vip = utils.config_get('vip')
+    vip_iface = utils.config_get('vip_iface')
+    vip_cidr = utils.config_get('vip_cidr')
+    corosync_bindiface = utils.config_get('ha-bindiface')
+    corosync_mcastport = utils.config_get('ha-mcastport')
+
+    if None in [vip, vip_cidr, vip_iface]:
+        utils.juju_log('WARNING',
+                       'Insufficient VIP information to configure cluster')
+        sys.exit(1)
+
+    # Starting configuring resources.
+    init_services = {
+            'res_mysqld': 'mysql',
+        }
+
+    # If the 'ha' relation has been made *before* the 'ceph' relation,
+    # it doesn't make sense to make it until after the 'ceph' relation is made
+    if not utils.is_relation_made('ceph', 'auth'):
+        utils.juju_log('INFO',
+                       '*ceph* relation does not exist. '
+                       'Not sending *ha* relation data yet')
+        return
+    else:
+        utils.juju_log('INFO',
+                       '*ceph* relation exists. Sending *ha* relation data')
+
+        block_storage = 'ceph'
+
+        resources = {
+            'res_mysql_rbd': 'ocf:ceph:rbd',
+            'res_mysql_fs': 'ocf:heartbeat:Filesystem',
+            'res_mysql_vip': 'ocf:heartbeat:IPaddr2',
+            'res_mysqld': 'upstart:mysql',
+            }
+
+        rbd_name = utils.config_get('rbd-name')
+        resource_params = {
+            'res_mysql_rbd': 'params name="%s" pool="%s" user="%s" '
+                             'secret="%s"' % \
+                             (rbd_name, POOL_NAME,
+                              SERVICE_NAME, ceph.keyfile_path(SERVICE_NAME)),
+            'res_mysql_fs': 'params device="/dev/rbd/%s/%s" directory="%s" '
+                            'fstype="ext4" op start start-delay="10s"' % \
+                            (POOL_NAME, rbd_name, DATA_SRC_DST),
+            'res_mysql_vip': 'params ip="%s" cidr_netmask="%s" nic="%s"' % \
+                             (vip, vip_cidr, vip_iface),
+            'res_mysqld': 'op start start-delay="5s" op monitor interval="5s"',
+            }
+
+        groups = {
+            'grp_mysql': 'res_mysql_rbd res_mysql_fs res_mysql_vip res_mysqld',
+            }
+
+        for rel_id in utils.relation_ids('ha'):
+            utils.relation_set(rid=rel_id,
+                               block_storage=block_storage,
+                               corosync_bindiface=corosync_bindiface,
+                               corosync_mcastport=corosync_mcastport,
+                               resources=resources,
+                               resource_params=resource_params,
+                               init_services=init_services,
+                               groups=groups)
+
+
+def ha_relation_changed():
+    clustered = utils.relation_get('clustered')
+    if (clustered and cluster.is_leader(LEADER_RES)):
+        utils.juju_log('INFO', 'Cluster configured, notifying other services')
+        # Tell all related services to start using the VIP
+        for r_id in utils.relation_ids('shared-db'):
+            utils.relation_set(rid=r_id,
+                               db_host=utils.config_get('vip'))
+
+
+def ceph_joined():
+    utils.juju_log('INFO', 'Start Ceph Relation Joined')
+    ceph.install()
+    utils.juju_log('INFO', 'Finish Ceph Relation Joined')
+
+
+def ceph_changed():
+    utils.juju_log('INFO', 'Start Ceph Relation Changed')
+    auth = utils.relation_get('auth')
+    key = utils.relation_get('key')
+    if None in [auth, key]:
+        utils.juju_log('INFO', 'Missing key or auth in relation')
+        return
+
+    ceph.configure(service=SERVICE_NAME, key=key, auth=auth)
+
+    if cluster.eligible_leader(LEADER_RES):
+        sizemb = int(utils.config_get('block-size')) * 1024
+        rbd_img = utils.config_get('rbd-name')
+        blk_device = '/dev/rbd/%s/%s' % (POOL_NAME, rbd_img)
+        ceph.ensure_ceph_storage(service=SERVICE_NAME, pool=POOL_NAME,
+                                 rbd_img=rbd_img, sizemb=sizemb,
+                                 fstype='ext4', mount_point=DATA_SRC_DST,
+                                 blk_device=blk_device,
+                                 system_services=['mysql'])
+    else:
+        utils.juju_log('INFO',
+                       'This is not the peer leader. Not configuring RBD.')
+        # Stopping MySQL
+        if utils.running('mysql'):
+            utils.juju_log('INFO', 'Stopping MySQL...')
+            utils.stop('mysql')
+
+    # If 'ha' relation has been made before the 'ceph' relation
+    # it is important to make sure the ha-relation data is being
+    # sent.
+    if utils.is_relation_made('ha'):
+        utils.juju_log('INFO',
+                       '*ha* relation exists. Making sure the ha'
+                       ' relation data is sent.')
+        ha_relation_joined()
+        return
+
+    utils.juju_log('INFO', 'Finish Ceph Relation Changed')
+
+
+hooks = {
+    "ha-relation-joined": ha_relation_joined,
+    "ha-relation-changed": ha_relation_changed,
+    "ceph-relation-joined": ceph_joined,
+    "ceph-relation-changed": ceph_changed,
+}
+
+utils.do_hooks(hooks)

=== modified file 'hooks/install'
--- hooks/install	2012-11-01 21:49:21 +0000
+++ hooks/install	2013-03-26 14:36:21 +0000
@@ -3,8 +3,9 @@
 apt-get update
 apt-get install -y debconf-utils python-mysqldb uuid pwgen dnsutils charm-helper-sh || exit 1
 
-PASSFILE=/var/lib/juju/mysql.passwd
+PASSFILE=/var/lib/mysql/mysql.passwd
 if ! [ -f $PASSFILE ] ; then
+  mkdir -p /var/lib/mysql
   touch $PASSFILE
 fi
 chmod 0600 $PASSFILE

=== added directory 'hooks/lib'
=== added file 'hooks/lib/__init__.py'
=== added file 'hooks/lib/ceph_utils.py'
--- hooks/lib/ceph_utils.py	1970-01-01 00:00:00 +0000
+++ hooks/lib/ceph_utils.py	2013-03-26 14:36:21 +0000
@@ -0,0 +1,256 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# This file is sourced from lp:openstack-charm-helpers
+#
+# Authors:
+#  James Page <[e-mail]>
+#  Adam Gandelman <[e-mail]>
+#
+
+import commands
+import subprocess
+import os
+import shutil
+import lib.utils as utils
+
+KEYRING = '/etc/ceph/ceph.client.%s.keyring'
+KEYFILE = '/etc/ceph/ceph.client.%s.key'
+
+CEPH_CONF = """[global]
+ auth supported = %(auth)s
+ keyring = %(keyring)s
+ mon host = %(mon_hosts)s
+"""
+
+
+def execute(cmd):
+    subprocess.check_call(cmd)
+
+
+def execute_shell(cmd):
+    subprocess.check_call(cmd, shell=True)
+
+
+def install():
+    ceph_dir = "/etc/ceph"
+    if not os.path.isdir(ceph_dir):
+        os.mkdir(ceph_dir)
+    utils.install('ceph-common')
+
+
+def rbd_exists(service, pool, rbd_img):
+    (rc, out) = commands.getstatusoutput('rbd list --id %s --pool %s' %\
+                                         (service, pool))
+    return rbd_img in out
+
+
+def create_rbd_image(service, pool, image, sizemb):
+    cmd = [
+        'rbd',
+        'create',
+        image,
+        '--size',
+        str(sizemb),
+        '--id',
+        service,
+        '--pool',
+        pool
+        ]
+    execute(cmd)
+
+
+def pool_exists(service, name):
+    (rc, out) = commands.getstatusoutput("rados --id %s lspools" % service)
+    return name in out
+
+
+def create_pool(service, name):
+    cmd = [
+        'rados',
+        '--id',
+        service,
+        'mkpool',
+        name
+        ]
+    execute(cmd)
+
+
+def keyfile_path(service):
+    return KEYFILE % service
+
+
+def keyring_path(service):
+    return KEYRING % service
+
+
+def create_keyring(service, key):
+    keyring = keyring_path(service)
+    if os.path.exists(keyring):
+        utils.juju_log('INFO', 'ceph: Keyring exists at %s.' % keyring)
+    cmd = [
+        'ceph-authtool',
+        keyring,
+        '--create-keyring',
+        '--name=client.%s' % service,
+        '--add-key=%s' % key
+        ]
+    execute(cmd)
+    utils.juju_log('INFO', 'ceph: Created new ring at %s.' % keyring)
+
+
+def create_key_file(service, key):
+    # create a file containing the key
+    keyfile = keyfile_path(service)
+    if os.path.exists(keyfile):
+        utils.juju_log('INFO', 'ceph: Keyfile exists at %s.' % keyfile)
+    fd = open(keyfile, 'w')
+    fd.write(key)
+    fd.close()
+    utils.juju_log('INFO', 'ceph: Created new keyfile at %s.' % keyfile)
+
+
+def get_ceph_nodes():
+    hosts = []
+    for r_id in utils.relation_ids('ceph'):
+        for unit in utils.relation_list(r_id):
+            hosts.append(utils.relation_get('private-address',
+                                            unit=unit, rid=r_id))
+    return hosts
+
+
+def configure(service, key, auth):
+    create_keyring(service, key)
+    create_key_file(service, key)
+    hosts = get_ceph_nodes()
+    mon_hosts = ",".join(map(str, hosts))
+    keyring = keyring_path(service)
+    with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
+        ceph_conf.write(CEPH_CONF % locals())
+    modprobe_kernel_module('rbd')
+
+
+def image_mapped(image_name):
+    (rc, out) = commands.getstatusoutput('rbd showmapped')
+    return image_name in out
+
+
+def map_block_storage(service, pool, image):
+    cmd = [
+        'rbd',
+        'map',
+        '%s/%s' % (pool, image),
+        '--user',
+        service,
+        '--secret',
+        keyfile_path(service),
+        ]
+    execute(cmd)
+
+
+def filesystem_mounted(fs):
+    return subprocess.call(['grep', '-wqs', fs, '/proc/mounts']) == 0
+
+
+def make_filesystem(blk_device, fstype='ext4'):
+    utils.juju_log('INFO',
+                   'ceph: Formatting block device %s as filesystem %s.' %\
+                   (blk_device, fstype))
+    cmd = ['mkfs', '-t', fstype, blk_device]
+    execute(cmd)
+
+
+def place_data_on_ceph(service, blk_device, data_src_dst, fstype='ext4'):
+    # mount block device into /mnt
+    cmd = ['mount', '-t', fstype, blk_device, '/mnt']
+    execute(cmd)
+
+    # copy data to /mnt
+    try:
+        copy_files(data_src_dst, '/mnt')
+    except:
+        pass
+
+    # umount block device
+    cmd = ['umount', '/mnt']
+    execute(cmd)
+
+    _dir = os.stat(data_src_dst)
+    uid = _dir.st_uid
+    gid = _dir.st_gid
+
+    # re-mount where the data should originally be
+    cmd = ['mount', '-t', fstype, blk_device, data_src_dst]
+    execute(cmd)
+
+    # ensure original ownership of new mount.
+    cmd = ['chown', '-R', '%s:%s' % (uid, gid), data_src_dst]
+    execute(cmd)
+
+
+# TODO: re-use
+def modprobe_kernel_module(module):
+    utils.juju_log('INFO', 'Loading kernel module')
+    cmd = ['modprobe', module]
+    execute(cmd)
+    cmd = 'echo %s >> /etc/modules' % module
+    execute_shell(cmd)
+
+
+def copy_files(src, dst, symlinks=False, ignore=None):
+    for item in os.listdir(src):
+        s = os.path.join(src, item)
+        d = os.path.join(dst, item)
+        if os.path.isdir(s):
+            shutil.copytree(s, d, symlinks, ignore)
+        else:
+            shutil.copy2(s, d)
+
+
+def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
+                        blk_device, fstype, system_services=[]):
+    """
+    To be called from the current cluster leader.
+    Ensures given pool and RBD image exists, is mapped to a block device,
+    and the device is formatted and mounted at the given mount_point.
+
+    If formatting a device for the first time, data existing at mount_point
+    will be migrated to the RBD device before being remounted.
+
+    All services listed in system_services will be stopped prior to data
+    migration and restarted when complete.
+    """
+    # Ensure pool, RBD image, RBD mappings are in place.
+    if not pool_exists(service, pool):
+        utils.juju_log('INFO', 'ceph: Creating new pool %s.' % pool)
+        create_pool(service, pool)
+
+    if not rbd_exists(service, pool, rbd_img):
+        utils.juju_log('INFO', 'ceph: Creating RBD image (%s).' % rbd_img)
+        create_rbd_image(service, pool, rbd_img, sizemb)
+
+    if not image_mapped(rbd_img):
+        utils.juju_log('INFO', 'ceph: Mapping RBD Image as a Block Device.')
+        map_block_storage(service, pool, rbd_img)
+
+    # make file system
+    # TODO: What happens if for whatever reason this is run again and
+    # the data is already in the rbd device and/or is mounted??
+    # When it is mounted already, it will fail to make the fs
+    # XXX: This is really sketchy!  Need to at least add an fstab entry
+    #      otherwise this hook will blow away existing data if its executed
+    #      after a reboot.
+    if not filesystem_mounted(mount_point):
+        make_filesystem(blk_device, fstype)
+
+        for svc in system_services:
+            if utils.running(svc):
+                utils.juju_log('INFO',
+                               'Stopping services %s prior to migrating '\
+                               'data' % svc)
+                utils.stop(svc)
+
+        place_data_on_ceph(service, blk_device, mount_point, fstype)
+
+        for svc in system_services:
+            utils.start(svc)

=== added file 'hooks/lib/cluster_utils.py'
--- hooks/lib/cluster_utils.py	1970-01-01 00:00:00 +0000
+++ hooks/lib/cluster_utils.py	2013-03-26 14:36:21 +0000
@@ -0,0 +1,130 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# This file is sourced from lp:openstack-charm-helpers
+#
+# Authors:
+#  James Page <[e-mail]>
+#  Adam Gandelman <[e-mail]>
+#
+
+from lib.utils import (
+    juju_log,
+    relation_ids,
+    relation_list,
+    relation_get,
+    get_unit_hostname,
+    config_get
+    )
+import subprocess
+import os
+
+
+def is_clustered():
+    for r_id in (relation_ids('ha') or []):
+        for unit in (relation_list(r_id) or []):
+            clustered = relation_get('clustered',
+                                     rid=r_id,
+                                     unit=unit)
+            if clustered:
+                return True
+    return False
+
+
+def is_leader(resource):
+    cmd = [
+        "crm", "resource",
+        "show", resource
+        ]
+    try:
+        status = subprocess.check_output(cmd)
+    except subprocess.CalledProcessError:
+        return False
+    else:
+        if get_unit_hostname() in status:
+            return True
+        else:
+            return False
+
+
+def peer_units():
+    peers = []
+    for r_id in (relation_ids('cluster') or []):
+        for unit in (relation_list(r_id) or []):
+            peers.append(unit)
+    return peers
+
+
+def oldest_peer(peers):
+    local_unit_no = os.getenv('JUJU_UNIT_NAME').split('/')[1]
+    for peer in peers:
+        remote_unit_no = peer.split('/')[1]
+        if remote_unit_no < local_unit_no:
+            return False
+    return True
+
+
+def eligible_leader(resource):
+    if is_clustered():
+        if not is_leader(resource):
+            juju_log('INFO', 'Deferring action to CRM leader.')
+            return False
+    else:
+        peers = peer_units()
+        if peers and not oldest_peer(peers):
+            juju_log('INFO', 'Deferring action to oldest service unit.')
+            return False
+    return True
+
+
+def https():
+    '''
+    Determines whether enough data has been provided in configuration
+    or relation data to configure HTTPS
+    .
+    returns: boolean
+    '''
+    if config_get('use-https') == "yes":
+        return True
+    if config_get('ssl_cert') and config_get('ssl_key'):
+        return True
+    for r_id in relation_ids('identity-service'):
+        for unit in relation_list(r_id):
+            if (relation_get('https_keystone', rid=r_id, unit=unit) and
+                relation_get('ssl_cert', rid=r_id, unit=unit) and
+                relation_get('ssl_key', rid=r_id, unit=unit) and
+                relation_get('ca_cert', rid=r_id, unit=unit)):
+                return True
+    return False
+
+
+def determine_api_port(public_port):
+    '''
+    Determine correct API server listening port based on
+    existence of HTTPS reverse proxy and/or haproxy.
+
+    public_port: int: standard public port for given service
+
+    returns: int: the correct listening port for the API service
+    '''
+    i = 0
+    if len(peer_units()) > 0 or is_clustered():
+        i += 1
+    if https():
+        i += 1
+    return public_port - (i * 10)
+
+
+def determine_haproxy_port(public_port):
+    '''
+    Description: Determine correct proxy listening port based on public IP +
+    existence of HTTPS reverse proxy.
+
+    public_port: int: standard public port for given service
+
+    returns: int: the correct listening port for the HAProxy service
+    '''
+    i = 0
+    if https():
+        i += 1
+    return public_port - (i * 10)

=== added file 'hooks/lib/utils.py'
--- hooks/lib/utils.py	1970-01-01 00:00:00 +0000
+++ hooks/lib/utils.py	2013-03-26 14:36:21 +0000
@@ -0,0 +1,283 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# This file is sourced from lp:openstack-charm-helpers
+#
+# Authors:
+#  James Page <[e-mail]>
+#  Paul Collins <[e-mail]>
+#  Adam Gandelman <[e-mail]>
+#
+
+import json
+import os
+import subprocess
+import socket
+import sys
+
+
+def do_hooks(hooks):
+    hook = os.path.basename(sys.argv[0])
+
+    try:
+        hook_func = hooks[hook]
+    except KeyError:
+        juju_log('INFO',
+                 "This charm doesn't know how to handle '{}'.".format(hook))
+    else:
+        hook_func()
+
+
+def install(*pkgs):
+    cmd = [
+        'apt-get',
+        '-y',
+        'install'
+          ]
+    for pkg in pkgs:
+        cmd.append(pkg)
+    subprocess.check_call(cmd)
+
+TEMPLATES_DIR = 'templates'
+
+try:
+    import jinja2
+except ImportError:
+    install('python-jinja2')
+    import jinja2
+
+try:
+    import dns.resolver
+except ImportError:
+    install('python-dnspython')
+    import dns.resolver
+
+
+def render_template(template_name, context, template_dir=TEMPLATES_DIR):
+    templates = jinja2.Environment(
+                    loader=jinja2.FileSystemLoader(template_dir)
+                    )
+    template = templates.get_template(template_name)
+    return template.render(context)
+
+CLOUD_ARCHIVE = \
+""" # Ubuntu Cloud Archive
+deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
+"""
+
+CLOUD_ARCHIVE_POCKETS = {
+    'folsom': 'precise-updates/folsom',
+    'folsom/updates': 'precise-updates/folsom',
+    'folsom/proposed': 'precise-proposed/folsom',
+    'grizzly': 'precise-updates/grizzly',
+    'grizzly/updates': 'precise-updates/grizzly',
+    'grizzly/proposed': 'precise-proposed/grizzly'
+    }
+
+
+def configure_source():
+    source = str(config_get('openstack-origin'))
+    if not source:
+        return
+    if source.startswith('ppa:'):
+        cmd = [
+            'add-apt-repository',
+            source
+            ]
+        subprocess.check_call(cmd)
+    if source.startswith('cloud:'):
+        install('ubuntu-cloud-keyring')
+        pocket = source.split(':')[1]
+        with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
+            apt.write(CLOUD_ARCHIVE.format(CLOUD_ARCHIVE_POCKETS[pocket]))
+    if source.startswith('deb'):
+        l = len(source.split('|'))
+        if l == 2:
+            (apt_line, key) = source.split('|')
+            cmd = [
+                'apt-key',
+                'adv', '--keyserver keyserver.ubuntu.com',
+                '--recv-keys', key
+                ]
+            subprocess.check_call(cmd)
+        elif l == 1:
+            apt_line = source
+
+        with open('/etc/apt/sources.list.d/quantum.list', 'w') as apt:
+            apt.write(apt_line + "\n")
+    cmd = [
+        'apt-get',
+        'update'
+        ]
+    subprocess.check_call(cmd)
+
+# Protocols
+TCP = 'TCP'
+UDP = 'UDP'
+
+
+def expose(port, protocol='TCP'):
+    cmd = [
+        'open-port',
+        '{}/{}'.format(port, protocol)
+        ]
+    subprocess.check_call(cmd)
+
+
+def juju_log(severity, message):
+    cmd = [
+        'juju-log',
+        '--log-level', severity,
+        message
+        ]
+    subprocess.check_call(cmd)
+
+
+def relation_ids(relation):
+    cmd = [
+        'relation-ids',
+        relation
+        ]
+    result = str(subprocess.check_output(cmd)).split()
+    if result == "":
+        return None
+    else:
+        return result
+
+
+def relation_list(rid):
+    cmd = [
+        'relation-list',
+        '-r', rid,
+        ]
+    result = str(subprocess.check_output(cmd)).split()
+    if result == "":
+        return None
+    else:
+        return result
+
+
+def relation_get(attribute, unit=None, rid=None):
+    cmd = [
+        'relation-get',
+        ]
+    if rid:
+        cmd.append('-r')
+        cmd.append(rid)
+    cmd.append(attribute)
+    if unit:
+        cmd.append(unit)
+    value = subprocess.check_output(cmd).strip()  # IGNORE:E1103
+    if value == "":
+        return None
+    else:
+        return value
+
+
+def relation_set(**kwargs):
+    cmd = [
+        'relation-set'
+        ]
+    args = []
+    for k, v in kwargs.items():
+        if k == 'rid':
+            if v:
+                cmd.append('-r')
+                cmd.append(v)
+        else:
+            args.append('{}={}'.format(k, v))
+    cmd += args
+    subprocess.check_call(cmd)
+
+
+def unit_get(attribute):
+    cmd = [
+        'unit-get',
+        attribute
+        ]
+    value = subprocess.check_output(cmd).strip()  # IGNORE:E1103
+    if value == "":
+        return None
+    else:
+        return value
+
+
+def config_get(attribute):
+    cmd = [
+        'config-get',
+        '--format',
+        'json',
+        ]
+    out = subprocess.check_output(cmd).strip()  # IGNORE:E1103
+    cfg = json.loads(out)
+
+    try:
+        return cfg[attribute]
+    except KeyError:
+        return None
+
+
+def get_unit_hostname():
+    return socket.gethostname()
+
+
+def get_host_ip(hostname=unit_get('private-address')):
+    try:
+        # Test to see if already an IPv4 address
+        socket.inet_aton(hostname)
+        return hostname
+    except socket.error:
+        answers = dns.resolver.query(hostname, 'A')
+        if answers:
+            return answers[0].address
+    return None
+
+
+def _svc_control(service, action):
+    subprocess.check_call(['service', service, action])
+
+
+def restart(*services):
+    for service in services:
+        _svc_control(service, 'restart')
+
+
+def stop(*services):
+    for service in services:
+        _svc_control(service, 'stop')
+
+
+def start(*services):
+    for service in services:
+        _svc_control(service, 'start')
+
+
+def reload(*services):
+    for service in services:
+        try:
+            _svc_control(service, 'reload')
+        except subprocess.CalledProcessError:
+            # Reload failed - either service does not support reload
+            # or it was not running - restart will fixup most things
+            _svc_control(service, 'restart')
+
+
+def running(service):
+    try:
+        output = subprocess.check_output(['service', service, 'status'])
+    except subprocess.CalledProcessError:
+        return False
+    else:
+        if ("start/running" in output or
+            "is running" in output):
+            return True
+        else:
+            return False
+
+
+def is_relation_made(relation, key='private-address'):
+    for r_id in (relation_ids(relation) or []):
+        for unit in (relation_list(r_id) or []):
+            if relation_get(key, rid=r_id, unit=unit):
+                return True
+    return False

=== modified file 'hooks/master-relation-changed'
--- hooks/master-relation-changed	2012-11-02 06:41:12 +0000
+++ hooks/master-relation-changed	2013-03-26 14:36:21 +0000
@@ -6,7 +6,7 @@
 
 . /usr/share/charm-helper/sh/net.sh
 
-ROOTARGS="-uroot -p`cat /var/lib/juju/mysql.passwd`"
+ROOTARGS="-uroot -p`cat /var/lib/mysql/mysql.passwd`"
 snapdir=/var/www/snaps
 mkdir -p $snapdir
 apt-get -y install apache2

=== modified file 'hooks/monitors.common.bash'
--- hooks/monitors.common.bash	2012-07-12 21:58:36 +0000
+++ hooks/monitors.common.bash	2013-03-26 14:36:21 +0000
@@ -1,4 +1,4 @@
-MYSQL="mysql -uroot -p`cat /var/lib/juju/mysql.passwd`"
+MYSQL="mysql -uroot -p`cat /var/lib/mysql/mysql.passwd`"
 monitor_user=monitors
 . /usr/share/charm-helper/sh/net.sh
 if [ -n "$JUJU_REMOTE_UNIT" ] ; then

=== modified symlink 'hooks/shared-db-relation-changed'
=== target changed u'shared-db-relations' => u'shared_db_relations.py'
=== modified symlink 'hooks/shared-db-relation-joined'
=== target changed u'shared-db-relations' => u'shared_db_relations.py'
=== removed file 'hooks/shared-db-relations'
--- hooks/shared-db-relations	2012-12-03 20:21:07 +0000
+++ hooks/shared-db-relations	1970-01-01 00:00:00 +0000
@@ -1,137 +0,0 @@
-#!/usr/bin/python
-#
-# Create relations between a shared database to many peers.
-# Join does nothing.   Peer requests access to $DATABASE from $REMOTE_HOST.
-# It's up to the hooks to ensure database exists, peer has access and
-# clean up grants after a broken/departed peer (TODO)
-#
-# Author: Adam Gandelman <[e-mail]>
-
-
-from common import *
-import sys
-import subprocess
-import json
-import socket
-import os
-
-
-def pwgen():
-    return subprocess.check_output(['pwgen', '-s', '16']).strip()
-
-
-def relation_get():
-    return json.loads(subprocess.check_output(
-                        ['relation-get',
-                         '--format',
-                         'json']
-                        )
-                      )
-
-
-def relation_set(**kwargs):
-    cmd = [ 'relation-set' ]
-    args = []
-    for k, v in kwargs.items():
-        if k == 'rid':
-            cmd.append('-r')
-            cmd.append(v)
-        else:
-            args.append('{}={}'.format(k, v))
-    cmd += args
-    subprocess.check_call(cmd)
-
-
-def shared_db_changed():
-
-    def configure_db(hostname,
-                     database,
-                     username):
-        passwd_file = "/var/lib/juju/mysql-{}.passwd"\
-                        .format(username)
-        if hostname != local_hostname:
-            remote_ip = socket.gethostbyname(hostname)
-        else:
-            remote_ip = '127.0.0.1'
-
-        if not os.path.exists(passwd_file):
-            password = pwgen()
-            with open(passwd_file, 'w') as pfile:
-                pfile.write(password)
-        else:
-            with open(passwd_file) as pfile:
-                password = pfile.read().strip()
-
-        if not database_exists(database):
-            create_database(database)
-        if not grant_exists(database,
-                            username,
-                            remote_ip):
-            create_grant(database,
-                         username,
-                         remote_ip, password)
-        return password
-
-    settings = relation_get()
-    local_hostname = socket.getfqdn()
-    singleset = set([ 
-        'database',
-        'username',
-        'hostname'
-        ])
-
-    if singleset.issubset(settings):
-        # Process a single database configuration
-        password = configure_db(settings['hostname'],
-                                settings['database'],
-                                settings['username'])
-        relation_set(db_host=local_hostname,
-                     password=password)
-    else:
-        # Process multiple database setup requests.
-        # from incoming relation data:
-        #  nova_database=xxx nova_username=xxx nova_hostname=xxx
-        #  quantum_database=xxx quantum_username=xxx quantum_hostname=xxx
-        # create
-        #{
-        #   "nova": {
-        #        "username": xxx,
-        #        "database": xxx,
-        #        "hostname": xxx
-        #    },
-        #    "quantum": {
-        #        "username": xxx,
-        #        "database": xxx,
-        #        "hostname": xxx
-        #    }
-        #}
-        #
-        databases = {}
-        for k, v in settings.iteritems():
-            db = k.split('_')[0]
-            x = '_'.join(k.split('_')[1:])
-            if db not in databases:
-                databases[db] = {}
-            databases[db][x] = v
-        return_data = {}
-        for db in databases:
-            if singleset.issubset(databases[db]):
-                return_data['_'.join([ db, 'password' ])] = \
-                    configure_db(databases[db]['hostname'],
-                                 databases[db]['database'],
-                                 databases[db]['username'])
-        relation_set(**return_data)
-        relation_set(db_host=local_hostname)
-
-hook = os.path.basename(sys.argv[0])
-hooks = {
-    "shared-db-relation-changed": shared_db_changed
-    }
-try:
-    hook_func = hooks[hook]
-except KeyError:
-    pass
-else:
-    hook_func()
-
-sys.exit(0)

=== added file 'hooks/shared_db_relations.py'
--- hooks/shared_db_relations.py	1970-01-01 00:00:00 +0000
+++ hooks/shared_db_relations.py	2013-03-26 14:36:21 +0000
@@ -0,0 +1,139 @@
+#!/usr/bin/python
+#
+# Create relations between a shared database to many peers.
+# Join does nothing.   Peer requests access to $DATABASE from $REMOTE_HOST.
+# It's up to the hooks to ensure database exists, peer has access and
+# clean up grants after a broken/departed peer (TODO)
+#
+# Author: Adam Gandelman <[e-mail]>
+
+
+from common import (
+    database_exists,
+    create_database,
+    grant_exists,
+    create_grant
+    )
+import subprocess
+import json
+import socket
+import os
+import lib.utils as utils
+import lib.cluster_utils as cluster
+
+LEADER_RES = 'res_mysql_vip'
+
+
+def pwgen():
+    return str(subprocess.check_output(['pwgen', '-s', '16'])).strip()
+
+
+def relation_get():
+    return json.loads(subprocess.check_output(
+                        ['relation-get',
+                         '--format',
+                         'json']
+                        )
+                      )
+
+
+def shared_db_changed():
+
+    def configure_db(hostname,
+                     database,
+                     username):
+        passwd_file = "/var/lib/mysql/mysql-{}.passwd"\
+                        .format(username)
+        if hostname != local_hostname:
+            remote_ip = socket.gethostbyname(hostname)
+        else:
+            remote_ip = '127.0.0.1'
+
+        if not os.path.exists(passwd_file):
+            password = pwgen()
+            with open(passwd_file, 'w') as pfile:
+                pfile.write(password)
+        else:
+            with open(passwd_file) as pfile:
+                password = pfile.read().strip()
+
+        if not database_exists(database):
+            create_database(database)
+        if not grant_exists(database,
+                            username,
+                            remote_ip):
+            create_grant(database,
+                         username,
+                         remote_ip, password)
+        return password
+
+    if not cluster.eligible_leader(LEADER_RES):
+        utils.juju_log('INFO',
+                       'MySQL service is peered, bailing shared-db relation'
+                       ' as this service unit is not the leader')
+        return
+
+    settings = relation_get()
+    local_hostname = socket.getfqdn()
+    singleset = set([
+        'database',
+        'username',
+        'hostname'
+        ])
+
+    if singleset.issubset(settings):
+        # Process a single database configuration
+        password = configure_db(settings['hostname'],
+                                settings['database'],
+                                settings['username'])
+        if not cluster.is_clustered():
+            utils.relation_set(db_host=local_hostname,
+                               password=password)
+        else:
+            utils.relation_set(db_host=utils.config_get("vip"),
+                               password=password)
+
+    else:
+        # Process multiple database setup requests.
+        # from incoming relation data:
+        #  nova_database=xxx nova_username=xxx nova_hostname=xxx
+        #  quantum_database=xxx quantum_username=xxx quantum_hostname=xxx
+        # create
+        #{
+        #   "nova": {
+        #        "username": xxx,
+        #        "database": xxx,
+        #        "hostname": xxx
+        #    },
+        #    "quantum": {
+        #        "username": xxx,
+        #        "database": xxx,
+        #        "hostname": xxx
+        #    }
+        #}
+        #
+        databases = {}
+        for k, v in settings.iteritems():
+            db = k.split('_')[0]
+            x = '_'.join(k.split('_')[1:])
+            if db not in databases:
+                databases[db] = {}
+            databases[db][x] = v
+        return_data = {}
+        for db in databases:
+            if singleset.issubset(databases[db]):
+                return_data['_'.join([db, 'password'])] = \
+                    configure_db(databases[db]['hostname'],
+                                 databases[db]['database'],
+                                 databases[db]['username'])
+        utils.relation_set(**return_data)
+        if not cluster.is_clustered():
+            utils.relation_set(db_host=local_hostname)
+        else:
+            utils.relation_set(db_host=utils.config_get("vip"))
+
+hooks = {
+    "shared-db-relation-changed": shared_db_changed
+    }
+
+utils.do_hooks(hooks)

=== modified file 'hooks/slave-relation-broken'
--- hooks/slave-relation-broken	2011-12-06 21:23:39 +0000
+++ hooks/slave-relation-broken	2013-03-26 14:36:21 +0000
@@ -1,8 +1,8 @@
 #!/bin/sh
 
 # Kill the replication
-mysql -uroot -p`cat /var/lib/juju/mysql.passwd` -e 'STOP SLAVE;'
-mysql -uroot -p`cat /var/lib/juju/mysql.passwd` -e 'RESET SLAVE;'
+mysql -uroot -p`cat /var/lib/mysql/mysql.passwd` -e 'STOP SLAVE;'
+mysql -uroot -p`cat /var/lib/mysql/mysql.passwd` -e 'RESET SLAVE;'
 # No longer a slave
 # XXX this may change the server-id .. not sure if thats what we
 # want!

=== modified file 'hooks/slave-relation-changed'
--- hooks/slave-relation-changed	2011-12-06 21:17:17 +0000
+++ hooks/slave-relation-changed	2013-03-26 14:36:21 +0000
@@ -2,7 +2,7 @@
 
 set -e
 
-ROOTARGS="-uroot -p`cat /var/lib/juju/mysql.passwd`"
+ROOTARGS="-uroot -p`cat /var/lib/mysql/mysql.passwd`"
 
 # Others can join that service but only the lowest will be the master
 # Note that we could be more automatic but for now we will wait for
@@ -39,7 +39,7 @@
 curl --silent --show-error $dumpurl |zcat| mysql $ROOTARGS
 # Root pw gets overwritten by import
 echo Re-setting Root Pasword -- can use ours because it hasnt been flushed
-myrootpw=`cat /var/lib/juju/mysql.passwd`
+myrootpw=`cat /var/lib/mysql/mysql.passwd`
 mysqladmin -uroot -p$myrootpw password $myrootpw
 # Debian packages expect debian-sys-maint@localhost to be root privileged and
 # configured in /etc/mysql/debian.cnf. we just broke that.. fix it

=== modified file 'hooks/upgrade-charm'
--- hooks/upgrade-charm	2012-11-02 06:41:12 +0000
+++ hooks/upgrade-charm	2013-03-26 14:36:21 +0000
@@ -2,10 +2,26 @@
 home=`dirname $0`
 # Remove any existing .service_user files, which will cause
 # new users/pw's to be generated, which is a good thing
-old_service_user_files=$(ls /var/lib/juju/$.service_user)
+old_service_user_files=$(ls /var/lib/juju/*.service_user)
 if [ -n "$old_service_user_files" ] ; then
     juju-log -l WARNING "Stale users left around, should be revoked: $(cat $old_service_user_files)"
     rm -f $old_service_user_files
 fi
+
+# Move service_user2 files to /var/lib/mysql as they are
+# now stored there to support HA clustering with ceph.
+new_service_user_files=$(ls /var/lib/juju/*.service_user2)
+if [ -n "$new_service_user_files" ]; then
+    juju-log -l INFO "Moving service_user files [$new_service_user_files] to [/var/lib/mysql]"
+    mv $new_service_user_files /var/lib/mysql/
+fi
+# Move passwd files to /var/lib/mysql as they are
+# now stored there to support HA clustering with ceph.
+password_files=$(ls /var/lib/juju/*.passwd)
+if [ -n "$password_files" ]; then
+    juju-log -l INFO "Moving passwd files [$password_files] to [/var/lib/mysql]"
+    mv $password_files /var/lib/mysql/
+fi
+
 $home/install
 exec $home/config-changed

=== modified file 'metadata.yaml'
--- metadata.yaml	2012-08-21 00:07:19 +0000
+++ metadata.yaml	2013-03-26 14:36:21 +0000
@@ -22,6 +22,14 @@
   local-monitors:
     interface: local-monitors
     scope: container
+peers:
+  cluster:
+    interface: mysql-ha
 requires:
   slave:
     interface: mysql-oneway-replication
+  ceph:
+    interface: ceph-client
+  ha:
+    interface: hacluster
+    scope: container

=== modified file 'revision'
--- revision	2012-11-29 16:31:14 +0000
+++ revision	2013-03-26 14:36:21 +0000
@@ -1,1 +1,1 @@
-165
+305

=== added directory 'scripts'
=== added file 'scripts/add_to_cluster'
--- scripts/add_to_cluster	1970-01-01 00:00:00 +0000
+++ scripts/add_to_cluster	2013-03-26 14:36:21 +0000
@@ -0,0 +1,2 @@
+#!/bin/bash
+crm node online

=== added directory 'scripts/health_checks.d'
=== added file 'scripts/health_checks.d/service_crm_access'
--- scripts/health_checks.d/service_crm_access	1970-01-01 00:00:00 +0000
+++ scripts/health_checks.d/service_crm_access	2013-03-26 14:36:21 +0000
@@ -0,0 +1,7 @@
+#!/bin/bash -e
+# ensure this host is present in a CRM config and has access to crm tools.
+#
+# Since mysql service isn't running on the secondary/passive node, we have no
+# other easy means for determining the health of the mysql service until this
+# node becomes the active leader for the HA service.
+crm status 2>/dev/null | grep -q `hostname`

=== added file 'scripts/remove_from_cluster'
--- scripts/remove_from_cluster	1970-01-01 00:00:00 +0000
+++ scripts/remove_from_cluster	2013-03-26 14:36:21 +0000
@@ -0,0 +1,2 @@
+#!/bin/bash
+crm node standby