Merge lp:~gandelman-a/charms/precise/mysql/ceph_refactor into lp:~openstack-charmers/charms/precise/mysql/ha-support

[Adam Gandelman]

Refactored ceph work to fit nicely into a ceph.py which can be easily reused across charms that need a ceph backed block device.

Charms can now specify the pool to be created for RBDs, mysql creates a pool that matches the service name.

Various other fixes here and there.

=== modified file 'hooks/ceph.py'
--- hooks/ceph.py	2013-01-28 17:59:02 +0000
+++ hooks/ceph.py	2013-03-02 03:51:18 +0000
@@ -6,6 +6,14 @@
 import os
 import shutil
 
+KEYRING = '/etc/ceph/ceph.client.%s.keyring'
+KEYFILE = '/etc/ceph/ceph.client.%s.key'
+
+CEPH_CONF = """[global]
+ auth supported = %(auth)s
+ keyring = %(keyring)s
+ mon host = %(mon_hosts)s
+"""
 
 def execute(cmd):
     subprocess.check_call(cmd)
@@ -15,33 +23,59 @@
     subprocess.check_call(cmd, shell=True)
 
 
-def create_image(service, image, sizemb):
+def install():
+    ceph_dir = "/etc/ceph"
+    if not os.path.isdir(ceph_dir):
+        os.mkdir(ceph_dir)
+    utils.install('ceph-common')
+
+
+def rbd_exists(service, pool, rbd_img):
+    (rc, out) = commands.getstatusoutput('rbd list --id %s --pool %s' %\
+                                         (service, pool))
+    return rbd_img in out
+
+
+def create_rbd_image(service, pool, image, sizemb):
     cmd = [
         'rbd',
         'create',
         image,
         '--size',
-        sizemb,
+        str(sizemb),
         '--id',
         service,
         '--pool',
-        'images'
+        pool
         ]
     execute(cmd)
 
 
-def create_image_pool(service):
+def pool_exists(service, name):
+    (rc, out) = commands.getstatusoutput("rados --id %s lspools" % service)
+    return name in out
+
+def create_pool(service, name):
     cmd = [
         'rados',
         '--id',
         service,
         'mkpool',
-        'images'
+        name
         ]
     execute(cmd)
 
 
-def create_keyring(service, keyring, key):
+def keyfile_path(service):
+    return KEYFILE % service
+
+def keyring_path(service):
+    return KEYRING % service
+
+def create_keyring(service, key):
+    keyring = keyring_path(service)
+    if os.path.exists(keyring):
+        utils.juju_log('INFO', 'ceph: Keyring exists at %s.' % keyring)
     cmd = [
         'ceph-authtool',
         keyring,
@@ -50,22 +84,64 @@
         '--add-key=%s' % key
         ]
     execute(cmd)
-
-
-def map_block_storage(service, image, keyfile):
+    utils.juju_log('INFO', 'ceph: Created new ring at %s.' % keyring)
+
+
+def create_key_file(service, key):
+    # create a file containing the key
+    keyfile = keyfile_path(service)
+    if os.path.exists(keyfile):
+        utils.juju_log('INFO', 'ceph: Keyfile exists at %s.'  % keyfile)
+    fd = open(keyfile, 'w')
+    fd.write(key)
+    fd.close()
+    utils.juju_log('INFO', 'ceph: Created new keyfile at %s.'  % keyfile)
+
+
+def get_ceph_nodes():
+    hosts = []
+    for r_id in utils.relation_ids('ceph'):
+        for unit in utils.relation_list(r_id):
+            hosts.append(utils.relation_get('private-address',
+                                            unit=unit, rid=r_id))
+    return hosts
+
+
+def configure(service, key, auth):
+    create_keyring(service, key)
+    create_key_file(service, key)
+    hosts = get_ceph_nodes()
+    mon_hosts = ",".join(map(str, hosts))
+    keyring = keyring_path(service)
+    with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
+        ceph_conf.write(CEPH_CONF % locals())
+    modprobe_kernel_module('rbd')
+
+
+def image_mapped(image_name):
+    (rc, out) = commands.getstatusoutput('rbd showmapped')
+    return image_name in out
+
+def map_block_storage(service, pool, image):
     cmd = [
         'rbd',
         'map',
-        'images/%s' % image,
+        '%s/%s' % (pool, image),
         '--user',
         service,
         '--secret',
-        keyfile,
+        keyfile_path(service),
         ]
     execute(cmd)
 
 
-def make_filesystem(service, blk_device, fstype='ext4'):
+def filesystem_mounted(fs):
+    return subprocess.call(['grep', '-wqs', fs, '/proc/mounts']) == 0
+
+def make_filesystem(blk_device, fstype='ext4'):
+    utils.juju_log('INFO',
+                   'ceph: Formatting block device %s as filesystem %s.' %\
+                   (blk_device, fstype))
     cmd = ['mkfs', '-t', fstype, blk_device]
     execute(cmd)
 
@@ -85,13 +161,21 @@
     cmd = ['umount', '/mnt']
     execute(cmd)
 
+    _dir = os.stat(data_src_dst)
+    uid = _dir.st_uid
+    gid = _dir.st_gid
+
     # re-mount where the data should originally be
     cmd = ['mount', '-t', fstype, blk_device, data_src_dst]
     execute(cmd)
 
+    # ensure original ownership of new mount.
+    cmd = ['chown', '-R', '%s:%s' % (uid, gid), data_src_dst]
+    execute(cmd)
 
 # TODO: re-use
 def modprobe_kernel_module(module):
+    utils.juju_log('INFO','Loading kernel module')
     cmd = ['modprobe', module]
     execute(cmd)
     cmd = 'echo %s >> /etc/modules' % module
@@ -106,3 +190,51 @@
             shutil.copytree(s, d, symlinks, ignore)
         else:
             shutil.copy2(s, d)
+
+def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
+                        blk_device, fstype, system_services=[]):
+    """
+    To be called from the current cluster leader.
+    Ensures given pool and RBD image exists, is mapped to a block device,
+    and the device is formatted and mounted at the given mount_point.
+
+    If formatting a device for the first time, data existing at mount_point
+    will be migrated to the RBD device before being remounted.
+
+    All services listed in system_services will be stopped prior to data
+    migration and restarted when complete.
+    """
+    # Ensure pool, RBD image, RBD mappings are in place.
+    if not pool_exists(service, pool):
+        utils.juju_log('INFO', 'ceph: Creating new pool %s.' % pool)
+        create_pool(service, pool)
+
+    if not rbd_exists(service, pool, rbd_img):
+        utils.juju_log('INFO', 'ceph: Creating RBD image (%s).' % rbd_img)
+        create_rbd_image(service, pool, rbd_img, sizemb)
+
+    if not image_mapped(rbd_img):
+        utils.juju_log('INFO', 'ceph: Mapping RBD Image as a Block Device.')
+        map_block_storage(service, pool, rbd_img)
+
+    # make file system
+    # TODO: What happens if for whatever reason this is run again and
+    # the data is already in the rbd device and/or is mounted??
+    # When it is mounted already, it will fail to make the fs
+    # XXX: This is really sketchy!  Need to at least add an fstab entry
+    #      otherwise this hook will blow away existing data if its executed
+    #      after a reboot.
+    if not filesystem_mounted(mount_point):
+        make_filesystem(blk_device, fstype)
+
+        for svc in system_services:
+            if utils.running(svc):
+                utils.juju_log('INFO',
+                               'Stopping services %s prior to migrating '\
+                               'data' % svc)
+                utils.stop(svc)
+
+        place_data_on_ceph(service, blk_device, mount_point, fstype)
+
+        for svc in system_services:
+            utils.start(svc)

=== modified file 'hooks/ha-relations'
--- hooks/ha-relations	2013-02-19 23:11:24 +0000
+++ hooks/ha-relations	2013-03-02 03:51:18 +0000
@@ -14,10 +14,9 @@
 
 # CEPH
 DATA_SRC_DST = '/var/lib/mysql'
-SERVICE_NAME = utils.get_unit_name().replace('-','/').split('/')[0]
-KEYRING = "/etc/ceph/ceph.client.%s.keyring" % SERVICE_NAME
-KEYFILE = "/etc/ceph/ceph.client.%s.key" % SERVICE_NAME
 
+SERVICE_NAME = os.getenv('JUJU_UNIT_NAME').split('/')[0]
+POOL_NAME = SERVICE_NAME
 
 config=json.loads(subprocess.check_output(['config-get','--format=json']))
 
@@ -69,10 +68,10 @@
             }
 
         resource_params = {
-                'res_mysql_rbd':'params name="%s" pool="images" user="%s" secret="%s"' % (
-                                config['rbd-name'], SERVICE_NAME, KEYFILE),
-                'res_mysql_fs':'params device="/dev/rbd/images/%s" directory="%s" fstype="ext4" op start start-delay="10s"' % (
-                                config['rbd-name'], DATA_SRC_DST),
+                'res_mysql_rbd':'params name="%s" pool="%s" user="%s" secret="%s"' % (
+                                config['rbd-name'], POOL_NAME, SERVICE_NAME, ceph.keyfile_path(SERVICE_NAME)),
+                'res_mysql_fs':'params device="/dev/rbd/%s/%s" directory="%s" fstype="ext4" op start start-delay="10s"' % (
+                                POOL_NAME, config['rbd-name'], DATA_SRC_DST),
                 'res_mysql_vip':'params ip="%s" cidr_netmask="%s" nic="%s"' % (config['vip'],
                                 config['vip_cidr'], config['vip_iface']),
                 'res_mysqld':'op start start-delay="5s" op monitor interval="5s"',
@@ -107,12 +106,7 @@
 
 def ceph_joined():
     utils.juju_log('INFO', 'Start Ceph Relation Joined')
-
-    ceph_dir = "/etc/ceph"
-    if not os.path.isdir(ceph_dir):
-        os.mkdir(ceph_dir)
-    utils.install('ceph-common')
-
+    ceph.install()
     utils.juju_log('INFO', 'Finish Ceph Relation Joined')
 
 
@@ -121,42 +115,32 @@
 
     # TODO: ask james: What happens if the relation data has changed?
     # do we reconfigure ceph? What do we do with the data?
+    auth = utils.relation_get('auth')
     key = utils.relation_get('key')
-
-    if key:
-        # create KEYRING file
-        if not os.path.exists(KEYRING):
-            ceph.create_keyring(SERVICE_NAME, KEYRING, key)
-        # create a file containing the key
-        if not os.path.exists(KEYFILE):
-            fd = open(KEYFILE, 'w')
-            fd.write(key)
-            fd.close()
-    else:
+    if None in [auth, key]:
+        utils.juju_log('INFO', 'Missing key or auth in relation')
         sys.exit(0)
 
-    # emit ceph config
-    hosts = get_ceph_nodes()
-    mon_hosts = ",".join(map(str, hosts))
-    conf_context = {
-        'auth': utils.relation_get('auth'),
-        'keyring': KEYRING,
-        'mon_hosts': mon_hosts,
-        }
-    with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
-        ceph_conf.write(utils.render_template('ceph.conf',
-                                              conf_context))
-
-    # Create the images pool if it does not already exist
+
+    ceph.configure(service=SERVICE_NAME, key=key, auth=auth)
+
     if utils.eligible_leader():
-        (status, output) = commands.getstatusoutput("rados --id %s lspools" % SERVICE_NAME)
-        pools = "images" in output
-        if not pools:
-            utils.juju_log('INFO','Creating image pool')
-            ceph.create_image_pool(SERVICE_NAME)
+        sizemb = int(config['block-size'].split('G')[0]) * 1024
+        rbd_img = config['rbd-name']
+        blk_device = '/dev/rbd/%s/%s' % (POOL_NAME, rbd_img)
+        ceph.ensure_ceph_storage(service=SERVICE_NAME, pool=POOL_NAME,
+                                 rbd_img=rbd_img, sizemb=sizemb,
+                                 fstype='ext4', mount_point=DATA_SRC_DST,
+                                 blk_device=blk_device,
+                                 system_services=['mysql'])
+    else:
+        utils.juju_log('INFO',
+                       'This is not the peer leader. Not configuring RBD.')
+        # Stopping MySQL
+        if utils.running('mysql'):
+            utils.juju_log('INFO','Stopping MySQL...')
+            utils.stop('mysql')
 
-    # Configure ceph()
-    configure_ceph()
 
     # If 'ha' relation has been made before the 'ceph' relation
     # it is important to make sure the ha-relation data is being
@@ -173,99 +157,6 @@
     utils.juju_log('INFO', 'Finish Ceph Relation Changed')
 
 
-def configure_ceph():
-    utils.juju_log('INFO', 'Start Ceph Configuration')
-
-    block_sizemb = int(config['block-size'].split('G')[0]) * 1024
-    image_name = config['rbd-name']
-    fstype = 'ext4'
-    data_src = DATA_SRC_DST
-    blk_device = '/dev/rbd/images/%s' % image_name
-
-    # modprobe the kernel module
-    utils.juju_log('INFO','Loading kernel module')
-    ceph.modprobe_kernel_module('rbd')
-
-
-    # configure mysql for ceph storage options
-    if not utils.eligible_leader():
-        utils.juju_log('INFO','This is not the peer leader. Not configuring RBD.')
-        # Stopping MySQL
-        if utils.running('mysql'):
-            utils.juju_log('INFO','Stopping MySQL...')
-            utils.stop('mysql')
-        return
-
-    elif utils.eligible_leader():
-        # create an image/block device
-        (status, output) = commands.getstatusoutput('rbd list --id %s --pool images' % SERVICE_NAME)
-        rbd = image_name in output
-        if not rbd:
-            utils.juju_log('INFO', 'Creating RBD Image...')
-            ceph.create_image(SERVICE_NAME, image_name, str(block_sizemb))
-        else:
-            utils.juju_log('INFO',
-                       'Looks like RBD already exists. Not creating a new one.')
-
-        # map the image to a block device if not already mapped.
-        (status, output) = commands.getstatusoutput('rbd showmapped')
-        mapped = image_name in output
-        if not mapped:
-            # map block storage
-            utils.juju_log('INFO', 'Mapping RBD Image as a Block Device')
-            ceph.map_block_storage(SERVICE_NAME, image_name, KEYFILE)
-        else:
-            utils.juju_log('INFO',
-                       'Looks like RBD is already mapped. Not re-mapping.')
-
-        # make file system
-        # TODO: What happens if for whatever reason this is run again and
-        # the data is already in the rbd device and/or is mounted??
-        # When it is mounted already, it will fail to make the fs
-        utils.juju_log('INFO', 'Trying to move data over to RBD.')
-        if not filesystem_mounted(data_src):
-            utils.juju_log('INFO', 'Formating RBD.')
-            ceph.make_filesystem(SERVICE_NAME, blk_device, fstype)
-
-            # Stopping MySQL
-            if utils.running('mysql'):
-                utils.juju_log('INFO','Stopping MySQL before moving data to RBD.')
-                utils.stop('mysql')
-
-            # mount block device to temporary location and copy the data
-            utils.juju_log('INFO', 'Copying MySQL data to RBD.')
-            ceph.place_data_on_ceph(SERVICE_NAME, blk_device, data_src, fstype)
-
-            # Make files be owned by mysql user/pass
-            cmd = ['chown', '-R', 'mysql:mysql', data_src]
-            subprocess.check_call(cmd)
-        else:
-            utils.juju_log('INFO',
-                       'Looks like data is already on the RBD, skipping...')
-
-        if not utils.running('mysql'):
-            utils.start('mysql')
-
-    else:
-        return
-
-    utils.juju_log('INFO', 'Finish Ceph Configuration')
-
-
-def filesystem_mounted(fs):
-    return subprocess.call(['grep', '-wqs', fs, '/proc/mounts']) == 0
-
-
-def get_ceph_nodes():
-    hosts = []
-    for r_id in utils.relation_ids('ceph'):
-        for unit in utils.relation_list(r_id):
-            #hosts.append(utils.relation_get_dict(relation_id=r_id,
-            #                                     remote_unit=unit)['private-address'])
-            hosts.append(utils.relation_get('private-address', unit=unit, rid=r_id))
-    return hosts
-
-
 def cluster_changed():
     utils.juju_log('INFO', 'Begin cluster changed hook.')
 

=== modified file 'revision'
--- revision	2013-02-20 15:55:13 +0000
+++ revision	2013-03-02 03:51:18 +0000
@@ -1,1 +1,1 @@
-293
+305