Merge lp:~james-page/charms/precise/ceph/dir-support into lp:~charmers/charms/precise/ceph/trunk

[James Page]

Add support for using directories to host OSD filesytems instead of
requiring block devices for every OSD.

This allows use with the Juju local provider.

[Mark Mims (mark-mims)]

lgtm

review: Approve

=== modified file 'README.md'
--- README.md	2013-07-17 18:11:40 +0000
+++ README.md	2013-08-28 11:43:37 +0000
@@ -27,15 +27,20 @@
 to do this will cause a reconfiguration error and new service units will not join
 the existing ceph cluster.
 
-The charm also supports the specification of storage devices to be used in the 
+The charm also supports the specification of storage devices to be used in the
 ceph cluster.
 
     osd-devices:
         A list of devices that the charm will attempt to detect, initialise and
         activate as ceph storage.
 
-        This this can be a superset of the actual storage devices presented to
-        each service unit and can be changed post ceph bootstrap using `juju set`.
+        This can be a superset of the actual storage devices presented to each
+        service unit and can be changed post ceph bootstrap using `juju set`.
+
+        The full path of each device must be provided, e.g. /dev/vdb.
+
+        For Ceph >= 0.56.6 (Raring or the Grizzly Cloud Archive) use of
+        directories instead of devices is also supported.
 
 At a minimum you must provide a juju config file during initial deployment
 with the fsid and monitor-secret options (contents of cepy.yaml below):
@@ -66,10 +71,6 @@
 Technical Bootnotes
 ===================
 
-This charm is currently deliberately inflexible and potentially destructive.
-It is designed to deploy on exactly three machines. Each machine will run mon
-and osd.
-
 This charm uses the new-style Ceph deployment as reverse-engineered from the
 Chef cookbook at https://github.com/ceph/ceph-cookbooks, although we selected
 a different strategy to form the monitor cluster. Since we don't know the
 
=== modified file 'config.yaml'
--- config.yaml	2013-06-10 13:48:52 +0000
+++ config.yaml	2013-08-28 11:43:37 +0000
@@ -39,6 +39,9 @@
       .
       These devices are the range of devices that will be checked for and
       used across all service units.
+      .
+      For ceph >= 0.56.6 these can also be directories instead of devices - the
+      charm assumes anything not starting with /dev is a directory instead.
   osd-journal:
     type: string
     description: |
 
=== modified file 'hooks/ceph.py'
--- hooks/ceph.py	2013-07-08 08:32:38 +0000
+++ hooks/ceph.py	2013-08-28 11:43:37 +0000
@@ -15,14 +15,17 @@
 from charmhelpers.core.host import (
     mkdir,
     service_restart,
-    log
+)
+from charmhelpers.core.hookenv import (
+    log,
+    ERROR,
 )
 from charmhelpers.contrib.storage.linux.utils import (
     zap_disk,
-    is_block_device
+    is_block_device,
 )
 from utils import (
-    get_unit_hostname
+    get_unit_hostname,
 )
 
 LEADER = 'leader'
@@ -119,6 +122,16 @@
     return False
 
 
+def start_osds(devices):
+    # Scan for ceph block devices
+    rescan_osd_devices()
+    if get_ceph_version() >= "0.56.6":
+        # Use ceph-disk-activate for directory based OSD's
+        for dev_or_path in devices:
+            if os.path.exists(dev_or_path) and os.path.isdir(dev_or_path):
+                subprocess.check_call(['ceph-disk-activate', dev_or_path])
+
+
 def rescan_osd_devices():
     cmd = [
         'udevadm', 'trigger',
@@ -161,9 +174,38 @@
     ]
 }
 
+_osd_bootstrap_caps_profile = {
+    'mon': [
+        'allow profile bootstrap-osd'
+    ]
+}
+
+
+def parse_key(raw_key):
+    # get-or-create appears to have different output depending
+    # on whether its 'get' or 'create'
+    # 'create' just returns the key, 'get' is more verbose and
+    # needs parsing
+    key = None
+    if len(raw_key.splitlines()) == 1:
+        key = raw_key
+    else:
+        for element in raw_key.splitlines():
+            if 'key' in element:
+                key = element.split(' = ')[1].strip()  # IGNORE:E1103
+    return key
+
 
 def get_osd_bootstrap_key():
-    return get_named_key('bootstrap-osd', _osd_bootstrap_caps)
+    try:
+        # Attempt to get/create a key using the OSD bootstrap profile first
+        key = get_named_key('bootstrap-osd',
+                            _osd_bootstrap_caps_profile)
+    except:
+        # If that fails try with the older style permissions
+        key = get_named_key('bootstrap-osd',
+                            _osd_bootstrap_caps)
+    return key
 
 
 _radosgw_keyring = "/etc/ceph/keyring.rados.gateway"
@@ -214,19 +256,7 @@
             subsystem,
             '; '.join(subcaps),
         ])
-    output = subprocess.check_output(cmd).strip()  # IGNORE:E1103
-    # get-or-create appears to have different output depending
-    # on whether its 'get' or 'create'
-    # 'create' just returns the key, 'get' is more verbose and
-    # needs parsing
-    key = None
-    if len(output.splitlines()) == 1:
-        key = output
-    else:
-        for element in output.splitlines():
-            if 'key' in element:
-                key = element.split(' = ')[1].strip()  # IGNORE:E1103
-    return key
+    return parse_key(subprocess.check_output(cmd).strip())  # IGNORE:E1103
 
 
 def bootstrap_monitor_cluster(secret):
@@ -291,6 +321,13 @@
 
 
 def osdize(dev, osd_format, osd_journal, reformat_osd=False):
+    if dev.startswith('/dev'):
+        osdize_dev(dev, osd_format, osd_journal, reformat_osd)
+    else:
+        osdize_dir(dev)
+
+
+def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False):
     if not os.path.exists(dev):
         log('Path {} does not exist - bailing'.format(dev))
         return
@@ -327,6 +364,25 @@
     subprocess.check_call(cmd)
 
 
+def osdize_dir(path):
+    if os.path.exists(os.path.join(path, 'upstart')):
+        log('Path {} is already configured as an OSD - bailing'.format(path))
+        return
+
+    if get_ceph_version() < "0.56.6":
+        log('Unable to use directories for OSDs with ceph < 0.56.6',
+            level=ERROR)
+        raise
+
+    mkdir(path)
+    cmd = [
+        'ceph-disk-prepare',
+        '--data-dir',
+        path
+    ]
+    subprocess.check_call(cmd)
+
+
 def device_mounted(dev):
     return subprocess.call(['grep', '-wqs', dev + '1', '/proc/mounts']) == 0
 
 
=== modified file 'hooks/hooks.py'
--- hooks/hooks.py	2013-06-25 11:03:02 +0000
+++ hooks/hooks.py	2013-08-28 11:43:37 +0000
@@ -102,17 +102,16 @@
         with open(JOURNAL_ZAPPED, 'w') as zapped:
             zapped.write('DONE')
 
-    for dev in config('osd-devices').split(' '):
-        ceph.osdize(dev, config('osd-format'), config('osd-journal'),
-                    reformat_osd())
-
     # Support use of single node ceph
     if (not ceph.is_bootstrapped() and int(config('monitor-count')) == 1):
         ceph.bootstrap_monitor_cluster(config('monitor-secret'))
         ceph.wait_for_bootstrap()
 
     if ceph.is_bootstrapped():
-        ceph.rescan_osd_devices()
+        for dev in get_devices():
+            ceph.osdize(dev, config('osd-format'), config('osd-journal'),
+                        reformat_osd())
+        ceph.start_osds(get_devices())
 
     log('End config-changed hook.')
 
@@ -139,6 +138,13 @@
         return False
 
 
+def get_devices():
+    if config('osd-devices'):
+        return config('osd-devices').split(' ')
+    else:
+        return []
+
+
 @hooks.hook('mon-relation-departed',
             'mon-relation-joined')
 def mon_relation():
@@ -149,7 +155,10 @@
     if len(get_mon_hosts()) >= moncount:
         ceph.bootstrap_monitor_cluster(config('monitor-secret'))
         ceph.wait_for_bootstrap()
-        ceph.rescan_osd_devices()
+        for dev in get_devices():
+            ceph.osdize(dev, config('osd-format'), config('osd-journal'),
+                        reformat_osd())
+        ceph.start_osds(get_devices())
         notify_osds()
         notify_radosgws()
         notify_client()
@@ -258,7 +267,8 @@
     # In case we're being redeployed to the same machines, try
     # to make sure everything is running as soon as possible.
     service_restart('ceph-mon-all')
-    ceph.rescan_osd_devices()
+    if ceph.is_bootstrapped():
+        ceph.start_osds(get_devices())
 
 
 if __name__ == '__main__':
 
=== modified file 'revision'
--- revision	2013-06-20 23:58:01 +0000
+++ revision	2013-08-28 11:43:37 +0000
@@ -1,1 +1,1 @@
-92
+100