Merge lp:~james-page/charm-helpers/ceph-redux into lp:charm-helpers

[James Page]

Redux of ceph helper

Include unit testing and re-location to a more sensible location.

[lp:~james-page/charm-helpers/ceph-redux]

56. By James Page on 2013-08-23

Rebase on trunk

57. By James Page on 2013-08-23

Tidy lint in test_linux_ceph

58. By James Page on 2013-09-23

Rebase on trunk, fixup test errors

[Adam Gandelman (gandelman-a)]

LGTM. At this point we should probably have all tests that use patch_open()/mock_open() using the same copy.

review: Approve

[James Page (james-page)]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256

On 23/09/13 18:18, Adam Gandelman wrote:
> LGTM. At this point we should probably have all tests that use
> patch_open()/mock_open() using the same copy.

Agreed - I'll find some time todo a global refactor of that bit.

- --
James Page
Ubuntu and Debian Developer
<email address hidden>
<email address hidden>
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.14 (GNU/Linux)
Comment: Using GnuPG with Thunderbird - http://www.enigmail.net/

iQIcBAEBCAAGBQJSQKb7AAoJEL/srsug59jD6hkQAIajDJ8OoaP2dUv9IPeIoBSP
Rffbl+Wc0yZp6YdEP+Krw2u63LQZlplOXoYvwDnKonSaamOcYS1RpS51Mcv4TQ6i
YowB4X9/q7mPYczm/Sj3MDxOwWg+GWkhGX1DukzZSyrLiUs0Lfpd0s6EFISo70vT
0Z7FEqzLrULZ/5LEfkhIuZ/6Lw+xZbckOkvhoZU5mRgYe/p9jrLJsWastmk8m6No
jhsLEq5WoKro2Ytco4cBywSjyIcON6n36DGw1qR9lwFgep+N6d/WqNDdgvmS+DAJ
KP/d5+a6limRehw7zqPJiCMqG23zr3AoG7JRxRQkKjzRi3CJc/3QPg0+k8sp9Ll2
NUOIFCAOM8ST6hKoyi/rCTI0neVVzqTzc+sAnUYZU6cBA80LDTeVRxePPk6lY/fQ
Gz2YAUhBWpt1GVE+VOYDiECxsBFiPMFuNF0jwVnQxfxW7OOeqdBY8Su/u/Muge1M
djPoo6YpOHi9AVfMCNc8E8rmrnNJzW7vCOQVc0xtG05r9WJ+PoznSEwDzVjOqjhN
Nk/lyVRgdzpLdq3TAX/RKLMKuYJFNdt8I+xAu3KEc6+WE2GVM5gsvuBA1m/W0Mwq
zOLBTV7IHnMBfLIsK+iPTqOmNj9huDeHEWnrl/3/qmojN6keIIs9Mx1sB0shT9CW
2W+tTSOeYBPUJh3tvoUV
=dwG/
-----END PGP SIGNATURE-----

[Adam Gandelman (gandelman-a)]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

On 09/23/2013 01:42 PM, James Page wrote:
> On 23/09/13 18:18, Adam Gandelman wrote:
> > LGTM. At this point we should probably have all tests that use
> > patch_open()/mock_open() using the same copy.
>
> Agreed - I'll find some time todo a global refactor of that bit.
>

Filed this earlier for at least patch_open/mock_open:

https://code.launchpad.net/~gandelman-a/charm-helpers/test_helpers/+merge/187074
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.12 (GNU/Linux)
Comment: Using GnuPG with Thunderbird - http://www.enigmail.net/

iQEcBAEBAgAGBQJSQKpnAAoJEGAsEdGd4tyVCTEH/2yefCxcivHLVz6kMtAOvL8C
wPP+Q9s6e7bKlrAunF2IOQmOn2v46eOa6AOdrJ87kS5XKF/sulpwpx3asscfaVIM
9vQZalw6HCwGmmFqh5BepbdDDj3ULLNf1898JvhjPyvuuHJGqSQMY0qbCxe6L3Ec
T/1RLtJ8+JJKFhP4EWZbzl4Q8qYilr3ToWACGZ/yQgnYsVKC4n7vc0+BBisUtOpD
Bj6vLaIzT1SO1ZsEo9NnZmj5zdnnrXnwgSlHvoFHtL0UEr+ngjWY/CE3dRMerBoS
LGGONAXmPYTGnb7HYPF5Rj82CCkc4JX5vQ9f46vc+fWcebE1Z6N3A98YsGus+ZE=
=h4Sb
-----END PGP SIGNATURE-----

=== removed file 'charmhelpers/contrib/hahelpers/ceph.py'
--- charmhelpers/contrib/hahelpers/ceph.py	2013-08-21 09:10:58 +0000
+++ charmhelpers/contrib/hahelpers/ceph.py	1970-01-01 00:00:00 +0000
@@ -1,294 +0,0 @@
-#
-# Copyright 2012 Canonical Ltd.
-#
-# This file is sourced from lp:openstack-charm-helpers
-#
-# Authors:
-#  James Page <[e-mail]>
-#  Adam Gandelman <[e-mail]>
-#
-
-import commands
-import os
-import shutil
-import time
-
-from subprocess import (
-    check_call,
-    check_output,
-    CalledProcessError
-)
-
-from charmhelpers.core.hookenv import (
-    relation_get,
-    relation_ids,
-    related_units,
-    log,
-    INFO,
-    ERROR
-)
-
-from charmhelpers.fetch import (
-    apt_install,
-)
-
-from charmhelpers.core.host import (
-    mount,
-    mounts,
-    service_start,
-    service_stop,
-    umount,
-)
-
-KEYRING = '/etc/ceph/ceph.client.%s.keyring'
-KEYFILE = '/etc/ceph/ceph.client.%s.key'
-
-CEPH_CONF = """[global]
- auth supported = %(auth)s
- keyring = %(keyring)s
- mon host = %(mon_hosts)s
-"""
-
-
-def running(service):
-    # this local util can be dropped as soon the following branch lands
-    # in lp:charm-helpers
-    # https://code.launchpad.net/~gandelman-a/charm-helpers/service_running/
-    try:
-        output = check_output(['service', service, 'status'])
-    except CalledProcessError:
-        return False
-    else:
-        if ("start/running" in output or "is running" in output):
-            return True
-        else:
-            return False
-
-
-def install():
-    ceph_dir = "/etc/ceph"
-    if not os.path.isdir(ceph_dir):
-        os.mkdir(ceph_dir)
-    apt_install('ceph-common', fatal=True)
-
-
-def rbd_exists(service, pool, rbd_img):
-    (rc, out) = commands.getstatusoutput('rbd list --id %s --pool %s' %
-                                         (service, pool))
-    return rbd_img in out
-
-
-def create_rbd_image(service, pool, image, sizemb):
-    cmd = [
-        'rbd',
-        'create',
-        image,
-        '--size',
-        str(sizemb),
-        '--id',
-        service,
-        '--pool',
-        pool
-    ]
-    check_call(cmd)
-
-
-def pool_exists(service, name):
-    (rc, out) = commands.getstatusoutput("rados --id %s lspools" % service)
-    return name in out
-
-
-def create_pool(service, name):
-    cmd = [
-        'rados',
-        '--id',
-        service,
-        'mkpool',
-        name
-    ]
-    check_call(cmd)
-
-
-def keyfile_path(service):
-    return KEYFILE % service
-
-
-def keyring_path(service):
-    return KEYRING % service
-
-
-def create_keyring(service, key):
-    keyring = keyring_path(service)
-    if os.path.exists(keyring):
-        log('ceph: Keyring exists at %s.' % keyring, level=INFO)
-    cmd = [
-        'ceph-authtool',
-        keyring,
-        '--create-keyring',
-        '--name=client.%s' % service,
-        '--add-key=%s' % key
-    ]
-    check_call(cmd)
-    log('ceph: Created new ring at %s.' % keyring, level=INFO)
-
-
-def create_key_file(service, key):
-    # create a file containing the key
-    keyfile = keyfile_path(service)
-    if os.path.exists(keyfile):
-        log('ceph: Keyfile exists at %s.' % keyfile, level=INFO)
-    fd = open(keyfile, 'w')
-    fd.write(key)
-    fd.close()
-    log('ceph: Created new keyfile at %s.' % keyfile, level=INFO)
-
-
-def get_ceph_nodes():
-    hosts = []
-    for r_id in relation_ids('ceph'):
-        for unit in related_units(r_id):
-            hosts.append(relation_get('private-address', unit=unit, rid=r_id))
-    return hosts
-
-
-def configure(service, key, auth):
-    create_keyring(service, key)
-    create_key_file(service, key)
-    hosts = get_ceph_nodes()
-    mon_hosts = ",".join(map(str, hosts))
-    keyring = keyring_path(service)
-    with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
-        ceph_conf.write(CEPH_CONF % locals())
-    modprobe_kernel_module('rbd')
-
-
-def image_mapped(image_name):
-    (rc, out) = commands.getstatusoutput('rbd showmapped')
-    return image_name in out
-
-
-def map_block_storage(service, pool, image):
-    cmd = [
-        'rbd',
-        'map',
-        '%s/%s' % (pool, image),
-        '--user',
-        service,
-        '--secret',
-        keyfile_path(service),
-    ]
-    check_call(cmd)
-
-
-def filesystem_mounted(fs):
-    return fs in [f for m, f in mounts()]
-
-
-def make_filesystem(blk_device, fstype='ext4', timeout=10):
-    count = 0
-    e_noent = os.errno.ENOENT
-    while not os.path.exists(blk_device):
-        if count >= timeout:
-            log('ceph: gave up waiting on block device %s' % blk_device,
-                level=ERROR)
-            raise IOError(e_noent, os.strerror(e_noent), blk_device)
-        log('ceph: waiting for block device %s to appear' % blk_device,
-            level=INFO)
-        count += 1
-        time.sleep(1)
-    else:
-        log('ceph: Formatting block device %s as filesystem %s.' %
-            (blk_device, fstype), level=INFO)
-        check_call(['mkfs', '-t', fstype, blk_device])
-
-
-def place_data_on_ceph(service, blk_device, data_src_dst, fstype='ext4'):
-    # mount block device into /mnt
-    mount(blk_device, '/mnt')
-
-    # copy data to /mnt
-    try:
-        copy_files(data_src_dst, '/mnt')
-    except:
-        pass
-
-    # umount block device
-    umount('/mnt')
-
-    _dir = os.stat(data_src_dst)
-    uid = _dir.st_uid
-    gid = _dir.st_gid
-
-    # re-mount where the data should originally be
-    mount(blk_device, data_src_dst, persist=True)
-
-    # ensure original ownership of new mount.
-    cmd = ['chown', '-R', '%s:%s' % (uid, gid), data_src_dst]
-    check_call(cmd)
-
-
-# TODO: re-use
-def modprobe_kernel_module(module):
-    log('ceph: Loading kernel module', level=INFO)
-    cmd = ['modprobe', module]
-    check_call(cmd)
-    cmd = 'echo %s >> /etc/modules' % module
-    check_call(cmd, shell=True)
-
-
-def copy_files(src, dst, symlinks=False, ignore=None):
-    for item in os.listdir(src):
-        s = os.path.join(src, item)
-        d = os.path.join(dst, item)
-        if os.path.isdir(s):
-            shutil.copytree(s, d, symlinks, ignore)
-        else:
-            shutil.copy2(s, d)
-
-
-def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
-                        blk_device, fstype, system_services=[]):
-    """
-    To be called from the current cluster leader.
-    Ensures given pool and RBD image exists, is mapped to a block device,
-    and the device is formatted and mounted at the given mount_point.
-
-    If formatting a device for the first time, data existing at mount_point
-    will be migrated to the RBD device before being remounted.
-
-    All services listed in system_services will be stopped prior to data
-    migration and restarted when complete.
-    """
-    # Ensure pool, RBD image, RBD mappings are in place.
-    if not pool_exists(service, pool):
-        log('ceph: Creating new pool %s.' % pool, level=INFO)
-        create_pool(service, pool)
-
-    if not rbd_exists(service, pool, rbd_img):
-        log('ceph: Creating RBD image (%s).' % rbd_img, level=INFO)
-        create_rbd_image(service, pool, rbd_img, sizemb)
-
-    if not image_mapped(rbd_img):
-        log('ceph: Mapping RBD Image as a Block Device.', level=INFO)
-        map_block_storage(service, pool, rbd_img)
-
-    # make file system
-    # TODO: What happens if for whatever reason this is run again and
-    # the data is already in the rbd device and/or is mounted??
-    # When it is mounted already, it will fail to make the fs
-    # XXX: This is really sketchy!  Need to at least add an fstab entry
-    #      otherwise this hook will blow away existing data if its executed
-    #      after a reboot.
-    if not filesystem_mounted(mount_point):
-        make_filesystem(blk_device, fstype)
-
-        for svc in system_services:
-            if running(svc):
-                log('Stopping services %s prior to migrating data.' % svc,
-                    level=INFO)
-                service_stop(svc)
-
-        place_data_on_ceph(service, blk_device, mount_point, fstype)
-
-        for svc in system_services:
-            service_start(svc)

=== added file 'charmhelpers/contrib/storage/linux/ceph.py'
--- charmhelpers/contrib/storage/linux/ceph.py	1970-01-01 00:00:00 +0000
+++ charmhelpers/contrib/storage/linux/ceph.py	2013-09-23 10:51:06 +0000
@@ -0,0 +1,336 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# This file is sourced from lp:openstack-charm-helpers
+#
+# Authors:
+#  James Page <[e-mail]>
+#  Adam Gandelman <[e-mail]>
+#
+
+import os
+import shutil
+import json
+import time
+
+from subprocess import (
+    check_call,
+    check_output,
+    CalledProcessError
+)
+
+from charmhelpers.core.hookenv import (
+    relation_get,
+    relation_ids,
+    related_units,
+    log,
+    INFO,
+    WARNING,
+    ERROR
+)
+
+from charmhelpers.core.host import (
+    mount,
+    mounts,
+    service_start,
+    service_stop,
+    service_running,
+    umount,
+)
+
+from charmhelpers.fetch import (
+    apt_install,
+)
+
+KEYRING = '/etc/ceph/ceph.client.{}.keyring'
+KEYFILE = '/etc/ceph/ceph.client.{}.key'
+
+CEPH_CONF = """[global]
+ auth supported = {auth}
+ keyring = {keyring}
+ mon host = {mon_hosts}
+"""
+
+
+def install():
+    ''' Basic Ceph client installation '''
+    ceph_dir = "/etc/ceph"
+    if not os.path.exists(ceph_dir):
+        os.mkdir(ceph_dir)
+    apt_install('ceph-common', fatal=True)
+
+
+def rbd_exists(service, pool, rbd_img):
+    ''' Check to see if a RADOS block device exists '''
+    try:
+        out = check_output(['rbd', 'list', '--id', service,
+                            '--pool', pool])
+    except CalledProcessError:
+        return False
+    else:
+        return rbd_img in out
+
+
+def create_rbd_image(service, pool, image, sizemb):
+    ''' Create a new RADOS block device '''
+    cmd = [
+        'rbd',
+        'create',
+        image,
+        '--size',
+        str(sizemb),
+        '--id',
+        service,
+        '--pool',
+        pool
+    ]
+    check_call(cmd)
+
+
+def pool_exists(service, name):
+    ''' Check to see if a RADOS pool already exists '''
+    try:
+        out = check_output(['rados', '--id', service, 'lspools'])
+    except CalledProcessError:
+        return False
+    else:
+        return name in out
+
+
+def get_osds():
+    '''
+    Return a list of all Ceph Object Storage Daemons
+    currently in the cluster
+    '''
+    return json.loads(check_output(['ceph', 'osd', 'ls', '--format=json']))
+
+
+def create_pool(service, name, replicas=2):
+    ''' Create a new RADOS pool '''
+    if pool_exists(service, name):
+        log("Ceph pool {} already exists, skipping creation".format(name),
+            level=WARNING)
+        return
+    # Calculate the number of placement groups based
+    # on upstream recommended best practices.
+    pgnum = (len(get_osds()) * 100 / replicas)
+    cmd = [
+        'ceph', '--id', service,
+        'osd', 'pool', 'create',
+        name, pgnum
+    ]
+    check_call(cmd)
+    cmd = [
+        'ceph', '--id', service,
+        'osd', 'set', name,
+        'size', replicas
+    ]
+    check_call(cmd)
+
+
+def delete_pool(service, name):
+    ''' Delete a RADOS pool from ceph '''
+    cmd = [
+        'ceph', '--id', service,
+        'osd', 'pool', 'delete',
+        name, '--yes-i-really-really-mean-it'
+    ]
+    check_call(cmd)
+
+
+def _keyfile_path(service):
+    return KEYFILE.format(service)
+
+
+def _keyring_path(service):
+    return KEYRING.format(service)
+
+
+def create_keyring(service, key):
+    ''' Create a new Ceph keyring containing key'''
+    keyring = _keyring_path(service)
+    if os.path.exists(keyring):
+        log('ceph: Keyring exists at %s.' % keyring, level=WARNING)
+        return
+    cmd = [
+        'ceph-authtool',
+        keyring,
+        '--create-keyring',
+        '--name=client.{}'.format(service),
+        '--add-key={}'.format(key)
+    ]
+    check_call(cmd)
+    log('ceph: Created new ring at %s.' % keyring, level=INFO)
+
+
+def create_key_file(service, key):
+    ''' Create a file containing key '''
+    keyfile = _keyfile_path(service)
+    if os.path.exists(keyfile):
+        log('ceph: Keyfile exists at %s.' % keyfile, level=WARNING)
+        return
+    with open(keyfile, 'w') as fd:
+        fd.write(key)
+    log('ceph: Created new keyfile at %s.' % keyfile, level=INFO)
+
+
+def get_ceph_nodes():
+    ''' Query named relation 'ceph' to detemine current nodes '''
+    hosts = []
+    for r_id in relation_ids('ceph'):
+        for unit in related_units(r_id):
+            hosts.append(relation_get('private-address', unit=unit, rid=r_id))
+    return hosts
+
+
+def configure(service, key, auth):
+    ''' Perform basic configuration of Ceph '''
+    create_keyring(service, key)
+    create_key_file(service, key)
+    hosts = get_ceph_nodes()
+    with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
+        ceph_conf.write(CEPH_CONF.format(auth=auth,
+                                         keyring=_keyring_path(service),
+                                         mon_hosts=",".join(map(str, hosts))))
+    modprobe('rbd')
+
+
+def image_mapped(name):
+    ''' Determine whether a RADOS block device is mapped locally '''
+    try:
+        out = check_output(['rbd', 'showmapped'])
+    except CalledProcessError:
+        return False
+    else:
+        return name in out
+
+
+def map_block_storage(service, pool, image):
+    ''' Map a RADOS block device for local use '''
+    cmd = [
+        'rbd',
+        'map',
+        '{}/{}'.format(pool, image),
+        '--user',
+        service,
+        '--secret',
+        _keyfile_path(service),
+    ]
+    check_call(cmd)
+
+
+def filesystem_mounted(fs):
+    ''' Determine whether a filesytems is already mounted '''
+    return fs in [f for f, m in mounts()]
+
+
+def make_filesystem(blk_device, fstype='ext4', timeout=10):
+    ''' Make a new filesystem on the specified block device '''
+    count = 0
+    e_noent = os.errno.ENOENT
+    while not os.path.exists(blk_device):
+        if count >= timeout:
+            log('ceph: gave up waiting on block device %s' % blk_device,
+                level=ERROR)
+            raise IOError(e_noent, os.strerror(e_noent), blk_device)
+        log('ceph: waiting for block device %s to appear' % blk_device,
+            level=INFO)
+        count += 1
+        time.sleep(1)
+    else:
+        log('ceph: Formatting block device %s as filesystem %s.' %
+            (blk_device, fstype), level=INFO)
+        check_call(['mkfs', '-t', fstype, blk_device])
+
+
+def place_data_on_block_device(blk_device, data_src_dst):
+    ''' Migrate data in data_src_dst to blk_device and then remount '''
+    # mount block device into /mnt
+    mount(blk_device, '/mnt')
+    # copy data to /mnt
+    copy_files(data_src_dst, '/mnt')
+    # umount block device
+    umount('/mnt')
+    # Grab user/group ID's from original source
+    _dir = os.stat(data_src_dst)
+    uid = _dir.st_uid
+    gid = _dir.st_gid
+    # re-mount where the data should originally be
+    # TODO: persist is currently a NO-OP in core.host
+    mount(blk_device, data_src_dst, persist=True)
+    # ensure original ownership of new mount.
+    os.chown(data_src_dst, uid, gid)
+
+
+# TODO: re-use
+def modprobe(module):
+    ''' Load a kernel module and configure for auto-load on reboot '''
+    log('ceph: Loading kernel module', level=INFO)
+    cmd = ['modprobe', module]
+    check_call(cmd)
+    with open('/etc/modules', 'r+') as modules:
+        if module not in modules.read():
+            modules.write(module)
+
+
+def copy_files(src, dst, symlinks=False, ignore=None):
+    ''' Copy files from src to dst '''
+    for item in os.listdir(src):
+        s = os.path.join(src, item)
+        d = os.path.join(dst, item)
+        if os.path.isdir(s):
+            shutil.copytree(s, d, symlinks, ignore)
+        else:
+            shutil.copy2(s, d)
+
+
+def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
+                        blk_device, fstype, system_services=[]):
+    """
+    NOTE: This function must only be called from a single service unit for
+          the same rbd_img otherwise data loss will occur.
+
+    Ensures given pool and RBD image exists, is mapped to a block device,
+    and the device is formatted and mounted at the given mount_point.
+
+    If formatting a device for the first time, data existing at mount_point
+    will be migrated to the RBD device before being re-mounted.
+
+    All services listed in system_services will be stopped prior to data
+    migration and restarted when complete.
+    """
+    # Ensure pool, RBD image, RBD mappings are in place.
+    if not pool_exists(service, pool):
+        log('ceph: Creating new pool {}.'.format(pool))
+        create_pool(service, pool)
+
+    if not rbd_exists(service, pool, rbd_img):
+        log('ceph: Creating RBD image ({}).'.format(rbd_img))
+        create_rbd_image(service, pool, rbd_img, sizemb)
+
+    if not image_mapped(rbd_img):
+        log('ceph: Mapping RBD Image {} as a Block Device.'.format(rbd_img))
+        map_block_storage(service, pool, rbd_img)
+
+    # make file system
+    # TODO: What happens if for whatever reason this is run again and
+    # the data is already in the rbd device and/or is mounted??
+    # When it is mounted already, it will fail to make the fs
+    # XXX: This is really sketchy!  Need to at least add an fstab entry
+    #      otherwise this hook will blow away existing data if its executed
+    #      after a reboot.
+    if not filesystem_mounted(mount_point):
+        make_filesystem(blk_device, fstype)
+
+        for svc in system_services:
+            if service_running(svc):
+                log('ceph: Stopping services {} prior to migrating data.'
+                    .format(svc))
+                service_stop(svc)
+
+        place_data_on_block_device(blk_device, mount_point)
+
+        for svc in system_services:
+            log('ceph: Starting service {} after migrating data.'
+                .format(svc))
+            service_start(svc)

=== renamed file 'tests/contrib/hahelpers/test_ceph_utils.py' => 'tests/contrib/storage/test_linux_ceph.py'
--- tests/contrib/hahelpers/test_ceph_utils.py	2013-08-13 01:12:03 +0000
+++ tests/contrib/storage/test_linux_ceph.py	2013-09-23 10:51:06 +0000
@@ -1,12 +1,15 @@
-from mock import patch
+from mock import patch, call
 
 from shutil import rmtree
 from tempfile import mkdtemp
 from threading import Timer
 from testtools import TestCase
+import json
 
+import charmhelpers.contrib.storage.linux.ceph as ceph_utils
+from subprocess import CalledProcessError
+from tests.helpers import patch_open
 import nose.plugins.attrib
-import charmhelpers.contrib.hahelpers.ceph as ceph_utils
 import os
 import time
 
@@ -17,12 +20,24 @@
 rbd
 """
 
+LS_RBDS = """
+rbd1
+rbd2
+rbd3
+"""
+
+IMG_MAP = """
+bar
+baz
+"""
+
 
 class CephUtilsTests(TestCase):
     def setUp(self):
         super(CephUtilsTests, self).setUp()
         [self._patch(m) for m in [
             'check_call',
+            'check_output',
             'log',
         ]]
 
@@ -32,37 +47,352 @@
         self.addCleanup(_m.stop)
         setattr(self, method, mock)
 
-    def test_create_keyring(self):
+    @patch('os.path.exists')
+    def test_create_keyring(self, _exists):
         '''It creates a new ceph keyring'''
+        _exists.return_value = False
         ceph_utils.create_keyring('cinder', 'cephkey')
         _cmd = ['ceph-authtool', '/etc/ceph/ceph.client.cinder.keyring',
                 '--create-keyring', '--name=client.cinder',
                 '--add-key=cephkey']
         self.check_call.assert_called_with(_cmd)
 
-    def test_create_pool(self):
-        '''It creates rados pool correctly'''
-        ceph_utils.create_pool(service='cinder', name='foo')
-        self.check_call.assert_called_with(
-            ['rados', '--id', 'cinder', 'mkpool', 'foo']
-        )
+    @patch('os.path.exists')
+    def test_create_keyring_already_exists(self, _exists):
+        '''It creates a new ceph keyring'''
+        _exists.return_value = True
+        ceph_utils.create_keyring('cinder', 'cephkey')
+        self.log.assert_called()
+        self.check_call.assert_not_called()
+
+    @patch('os.path.exists')
+    def test_create_keyfile(self, _exists):
+        '''It creates a new ceph keyfile'''
+        _exists.return_value = False
+        with patch_open() as (_open, _file):
+            ceph_utils.create_key_file('cinder', 'cephkey')
+            _file.write.assert_called_with('cephkey')
+        self.log.assert_called()
+
+    @patch('os.path.exists')
+    def test_create_key_file_already_exists(self, _exists):
+        '''It creates a new ceph keyring'''
+        _exists.return_value = True
+        ceph_utils.create_key_file('cinder', 'cephkey')
+        self.log.assert_called()
+
+    @patch('os.mkdir')
+    @patch.object(ceph_utils, 'apt_install')
+    @patch('os.path.exists')
+    def test_install(self, _exists, _install, _mkdir):
+        _exists.return_value = False
+        ceph_utils.install()
+        _mkdir.assert_called_with('/etc/ceph')
+        _install.assert_called_with('ceph-common', fatal=True)
+
+    def test_get_osds(self):
+        self.check_output.return_value = json.dumps([1, 2, 3])
+        self.assertEquals(ceph_utils.get_osds(), [1, 2, 3])
+
+    def test_get_osds_none(self):
+        self.check_output.return_value = json.dumps(None)
+        self.assertEquals(ceph_utils.get_osds(), None)
+
+    @patch.object(ceph_utils, 'get_osds')
+    @patch.object(ceph_utils, 'pool_exists')
+    def test_create_pool(self, _exists, _get_osds):
+        '''It creates rados pool correctly with default replicas '''
+        _exists.return_value = False
+        _get_osds.return_value = [1, 2, 3]
+        ceph_utils.create_pool(service='cinder', name='foo')
+        self.check_call.assert_has_calls([
+            call(['ceph', '--id', 'cinder', 'osd', 'pool',
+                  'create', 'foo', 150]),
+            call(['ceph', '--id', 'cinder', 'osd', 'set',
+                  'foo', 'size', 2])
+        ])
+
+    @patch.object(ceph_utils, 'get_osds')
+    @patch.object(ceph_utils, 'pool_exists')
+    def test_create_pool_3_replicas(self, _exists, _get_osds):
+        '''It creates rados pool correctly with 3 replicas'''
+        _exists.return_value = False
+        _get_osds.return_value = [1, 2, 3]
+        ceph_utils.create_pool(service='cinder', name='foo', replicas=3)
+        self.check_call.assert_has_calls([
+            call(['ceph', '--id', 'cinder', 'osd', 'pool',
+                  'create', 'foo', 100]),
+            call(['ceph', '--id', 'cinder', 'osd', 'set',
+                  'foo', 'size', 3])
+        ])
+
+    def test_create_pool_already_exists(self):
+        self._patch('pool_exists')
+        self.pool_exists.return_value = True
+        ceph_utils.create_pool(service='cinder', name='foo')
+        self.log.assert_called()
+        self.check_call.assert_not_called()
 
     def test_keyring_path(self):
         '''It correctly dervies keyring path from service name'''
-        result = ceph_utils.keyring_path('cinder')
+        result = ceph_utils._keyring_path('cinder')
         self.assertEquals('/etc/ceph/ceph.client.cinder.keyring', result)
 
-    @patch('commands.getstatusoutput')
-    def test_pool_exists(self, get_output):
+    def test_keyfile_path(self):
+        '''It correctly dervies keyring path from service name'''
+        result = ceph_utils._keyfile_path('cinder')
+        self.assertEquals('/etc/ceph/ceph.client.cinder.key', result)
+
+    def test_pool_exists(self):
         '''It detects an rbd pool exists'''
-        get_output.return_value = (0, LS_POOLS)
+        self.check_output.return_value = LS_POOLS
         self.assertTrue(ceph_utils.pool_exists('cinder', 'volumes'))
 
-    @patch('commands.getstatusoutput')
-    def test_pool_does_not_exist(self, get_output):
+    def test_pool_does_not_exist(self):
         '''It detects an rbd pool exists'''
-        get_output.return_value = (0, LS_POOLS)
-        self.assertFalse(ceph_utils.pool_exists('cinder', 'foo'))
+        self.check_output.return_value = LS_POOLS
+        self.assertFalse(ceph_utils.pool_exists('cinder', 'foo'))
+
+    def test_pool_exists_error(self):
+        ''' Ensure subprocess errors and sandboxed with False '''
+        self.check_output.side_effect = CalledProcessError(1, 'rados')
+        self.assertFalse(ceph_utils.pool_exists('cinder', 'foo'))
+
+    def test_rbd_exists(self):
+        self.check_output.return_value = LS_RBDS
+        self.assertTrue(ceph_utils.rbd_exists('service', 'pool', 'rbd1'))
+        self.check_output.assert_call_with(
+            ['rbd', 'list', '--id', 'service', '--pool', 'pool']
+        )
+
+    def test_rbd_does_not_exist(self):
+        self.check_output.return_value = LS_RBDS
+        self.assertFalse(ceph_utils.rbd_exists('service', 'pool', 'rbd4'))
+        self.check_output.assert_call_with(
+            ['rbd', 'list', '--id', 'service', '--pool', 'pool']
+        )
+
+    def test_rbd_exists_error(self):
+        ''' Ensure subprocess errors and sandboxed with False '''
+        self.check_output.side_effect = CalledProcessError(1, 'rbd')
+        self.assertFalse(ceph_utils.rbd_exists('cinder', 'foo', 'rbd'))
+
+    def test_create_rbd_image(self):
+        ceph_utils.create_rbd_image('service', 'pool', 'image', 128)
+        _cmd = ['rbd', 'create', 'image',
+                '--size', '128',
+                '--id', 'service',
+                '--pool', 'pool']
+        self.check_call.assert_called_with(_cmd)
+
+    def test_delete_pool(self):
+        ceph_utils.delete_pool('cinder', 'pool')
+        _cmd = [
+            'ceph', '--id', 'cinder',
+            'osd', 'pool', 'delete',
+            'pool', '--yes-i-really-really-mean-it'
+        ]
+        self.check_call.assert_called_with(_cmd)
+
+    def test_get_ceph_nodes(self):
+        self._patch('relation_ids')
+        self._patch('related_units')
+        self._patch('relation_get')
+        units = ['ceph/1', 'ceph2', 'ceph/3']
+        self.relation_ids.return_value = ['ceph:0']
+        self.related_units.return_value = units
+        self.relation_get.return_value = '192.168.1.1'
+        self.assertEquals(len(ceph_utils.get_ceph_nodes()), 3)
+
+    def test_get_ceph_nodes_not_related(self):
+        self._patch('relation_ids')
+        self.relation_ids.return_value = []
+        self.assertEquals(ceph_utils.get_ceph_nodes(), [])
+
+    def test_configure(self):
+        self._patch('create_keyring')
+        self._patch('create_key_file')
+        self._patch('get_ceph_nodes')
+        self._patch('modprobe')
+        _hosts = ['192.168.1.1', '192.168.1.2']
+        self.get_ceph_nodes.return_value = _hosts
+        _conf = ceph_utils.CEPH_CONF.format(
+            auth='cephx',
+            keyring=ceph_utils._keyring_path('cinder'),
+            mon_hosts=",".join(map(str, _hosts))
+        )
+        with patch_open() as (_open, _file):
+            ceph_utils.configure('cinder', 'key', 'cephx')
+            _file.write.assert_called_with(_conf)
+            _open.assert_called_with('/etc/ceph/ceph.conf', 'w')
+        self.modprobe.assert_called_with('rbd')
+        self.create_keyring.assert_called_with('cinder', 'key')
+        self.create_key_file.assert_called_with('cinder', 'key')
+
+    def test_image_mapped(self):
+        self.check_output.return_value = IMG_MAP
+        self.assertTrue(ceph_utils.image_mapped('bar'))
+
+    def test_image_not_mapped(self):
+        self.check_output.return_value = IMG_MAP
+        self.assertFalse(ceph_utils.image_mapped('foo'))
+
+    def test_image_not_mapped_error(self):
+        self.check_output.side_effect = CalledProcessError(1, 'rbd')
+        self.assertFalse(ceph_utils.image_mapped('bar'))
+
+    def test_map_block_storage(self):
+        _service = 'cinder'
+        _pool = 'bar'
+        _img = 'foo'
+        _cmd = [
+            'rbd',
+            'map',
+            '{}/{}'.format(_pool, _img),
+            '--user',
+            _service,
+            '--secret',
+            ceph_utils._keyfile_path(_service),
+        ]
+        ceph_utils.map_block_storage(_service, _pool, _img)
+        self.check_call.assert_called_with(_cmd)
+
+    def test_modprobe(self):
+        with patch_open() as (_open, _file):
+            _file.read.return_value = 'anothermod\n'
+            ceph_utils.modprobe('mymod')
+            _open.assert_called_with('/etc/modules', 'r+')
+            _file.read.assert_called()
+            _file.write.assert_called_with('mymod')
+        self.check_call.assert_called_with(['modprobe', 'mymod'])
+
+    def test_filesystem_mounted(self):
+        self._patch('mounts')
+        self.mounts.return_value = [['/afs', '/dev/sdb'], ['/bfs', '/dev/sdd']]
+        self.assertTrue(ceph_utils.filesystem_mounted('/afs'))
+        self.assertFalse(ceph_utils.filesystem_mounted('/zfs'))
+
+    @patch('os.path.exists')
+    def test_make_filesystem(self, _exists):
+        _exists.return_value = True
+        ceph_utils.make_filesystem('/dev/sdd')
+        self.log.assert_called()
+        self.check_call.assert_called_with(['mkfs', '-t', 'ext4', '/dev/sdd'])
+
+    @patch('os.path.exists')
+    def test_make_filesystem_xfs(self, _exists):
+        _exists.return_value = True
+        ceph_utils.make_filesystem('/dev/sdd', 'xfs')
+        self.log.assert_called()
+        self.check_call.assert_called_with(['mkfs', '-t', 'xfs', '/dev/sdd'])
+
+    @patch('os.chown')
+    @patch('os.stat')
+    def test_place_data_on_block_device(self, _stat, _chown):
+        self._patch('mount')
+        self._patch('copy_files')
+        self._patch('umount')
+        _stat.return_value.st_uid = 100
+        _stat.return_value.st_gid = 100
+        ceph_utils.place_data_on_block_device('/dev/sdd', '/var/lib/mysql')
+        self.mount.assert_has_calls([
+            call('/dev/sdd', '/mnt'),
+            call('/dev/sdd', '/var/lib/mysql', persist=True)
+        ])
+        self.copy_files.assert_called_with('/var/lib/mysql', '/mnt')
+        self.umount.assert_called_with('/mnt')
+        _chown.assert_called_with('/var/lib/mysql', 100, 100)
+
+    @patch('shutil.copytree')
+    @patch('os.listdir')
+    @patch('os.path.isdir')
+    def test_copy_files_is_dir(self, _isdir, _listdir, _copytree):
+        _isdir.return_value = True
+        subdirs = ['a', 'b', 'c']
+        _listdir.return_value = subdirs
+        ceph_utils.copy_files('/source', '/dest')
+        for d in subdirs:
+            _copytree.assert_has_calls([
+                call('/source/{}'.format(d), '/dest/{}'.format(d),
+                     False, None)
+            ])
+
+    @patch('shutil.copytree')
+    @patch('os.listdir')
+    @patch('os.path.isdir')
+    def test_copy_files_include_symlinks(self, _isdir, _listdir, _copytree):
+        _isdir.return_value = True
+        subdirs = ['a', 'b', 'c']
+        _listdir.return_value = subdirs
+        ceph_utils.copy_files('/source', '/dest', True)
+        for d in subdirs:
+            _copytree.assert_has_calls([
+                call('/source/{}'.format(d), '/dest/{}'.format(d),
+                     True, None)
+            ])
+
+    @patch('shutil.copytree')
+    @patch('os.listdir')
+    @patch('os.path.isdir')
+    def test_copy_files_ignore(self, _isdir, _listdir, _copytree):
+        _isdir.return_value = True
+        subdirs = ['a', 'b', 'c']
+        _listdir.return_value = subdirs
+        ceph_utils.copy_files('/source', '/dest', True, False)
+        for d in subdirs:
+            _copytree.assert_has_calls([
+                call('/source/{}'.format(d), '/dest/{}'.format(d),
+                     True, False)
+            ])
+
+    @patch('shutil.copy2')
+    @patch('os.listdir')
+    @patch('os.path.isdir')
+    def test_copy_files_files(self, _isdir, _listdir, _copy2):
+        _isdir.return_value = False
+        files = ['a', 'b', 'c']
+        _listdir.return_value = files
+        ceph_utils.copy_files('/source', '/dest')
+        for f in files:
+            _copy2.assert_has_calls([
+                call('/source/{}'.format(f), '/dest/{}'.format(f))
+            ])
+
+    def test_ensure_ceph_storage(self):
+        self._patch('pool_exists')
+        self.pool_exists.return_value = False
+        self._patch('create_pool')
+        self._patch('rbd_exists')
+        self.rbd_exists.return_value = False
+        self._patch('create_rbd_image')
+        self._patch('image_mapped')
+        self.image_mapped.return_value = False
+        self._patch('map_block_storage')
+        self._patch('filesystem_mounted')
+        self.filesystem_mounted.return_value = False
+        self._patch('make_filesystem')
+        self._patch('service_stop')
+        self._patch('service_start')
+        self._patch('service_running')
+        self.service_running.return_value = True
+        self._patch('place_data_on_block_device')
+        _service = 'mysql'
+        _pool = 'bar'
+        _rbd_img = 'foo'
+        _mount = '/var/lib/mysql'
+        _services = ['mysql']
+        _blk_dev = '/dev/rbd1'
+        ceph_utils.ensure_ceph_storage(_service, _pool,
+                                       _rbd_img, 1024, _mount,
+                                       _blk_dev, 'ext4', _services)
+        self.create_pool.assert_called_with(_service, _pool)
+        self.create_rbd_image.assert_called_with(_service, _pool,
+                                                 _rbd_img, 1024)
+        self.map_block_storage.assert_called_with(_service, _pool, _rbd_img)
+        self.make_filesystem.assert_called_with(_blk_dev, 'ext4')
+        self.service_stop.assert_called_with(_services[0])
+        self.place_data_on_block_device.assert_called_with(_blk_dev, _mount)
+        self.service_start.assert_called_with(_services[0])
 
     def test_make_filesystem_default_filesystem(self):
         '''make_filesystem() uses ext4 as the default filesystem.'''
@@ -126,6 +456,7 @@
         fstype = 'xfs'
         ceph_utils.make_filesystem(device, fstype)
         self.check_call.assert_called_with(['mkfs', '-t', fstype, device])
-        self.log.assert_called_with('ceph: Formatting block device %s as '
-            'filesystem %s.' % (device, fstype), level='INFO')
-
+        self.log.assert_called_with(
+            'ceph: Formatting block device %s as '
+            'filesystem %s.' % (device, fstype), level='INFO'
+        )

=== added file 'tests/helpers.py'
--- tests/helpers.py	1970-01-01 00:00:00 +0000
+++ tests/helpers.py	2013-09-23 10:51:06 +0000
@@ -0,0 +1,34 @@
+''' General helper functions for tests '''
+from contextlib import contextmanager
+from mock import patch, MagicMock
+import io
+
+
+@contextmanager
+def patch_open():
+    '''Patch open() to allow mocking both open() itself and the file that is
+    yielded.
+
+    Yields the mock for "open" and "file", respectively.'''
+    mock_open = MagicMock(spec=open)
+    mock_file = MagicMock(spec=file)
+
+    @contextmanager
+    def stub_open(*args, **kwargs):
+        mock_open(*args, **kwargs)
+        yield mock_file
+
+    with patch('__builtin__.open', stub_open):
+        yield mock_open, mock_file
+
+
+@contextmanager
+def mock_open(filename, contents=None):
+    ''' Slightly simpler mock of open to return contents for filename '''
+    def mock_file(*args):
+        if args[0] == filename:
+            return io.StringIO(contents)
+        else:
+            return open(*args)
+    with patch('__builtin__.open', mock_file):
+        yield