Merge lp:~gandelman-a/charm-helpers/openstack_storage_utils into lp:charm-helpers

[Adam Gandelman]

contrib.openstack.utils: Add ensure_block_device() + clean_storage().

Adds a couple of storage helpers to be shared among cinder + swift, and possibly others. They rely on the more generic helpers in contrib.openstack.storage, but help Openstack charms setup storage in an way expected way.

[James Page (james-page)]

Please can you rebase on trunk; generates a few conflicts now.

review: Needs Fixing

[lp:~gandelman-a/charm-helpers/openstack_storage_utils]

61. By Adam Gandelman on 2013-10-20

Rebase on trunk

[Adam Gandelman (gandelman-a)]

Rebased.

review: Needs Resubmitting

=== modified file 'charmhelpers/contrib/openstack/utils.py'
--- charmhelpers/contrib/openstack/utils.py	2013-10-11 13:38:29 +0000
+++ charmhelpers/contrib/openstack/utils.py	2013-10-20 20:20:48 +0000
@@ -13,15 +13,20 @@
     config,
     log as juju_log,
     charm_dir,
-)
-
-from charmhelpers.core.host import (
-    lsb_release,
-)
-
-from charmhelpers.fetch import (
-    apt_install,
-)
+    ERROR,
+    INFO
+)
+
+from charmhelpers.contrib.storage.linux.lvm import (
+    deactivate_lvm_volume_group,
+    is_lvm_physical_volume,
+    remove_lvm_physical_volume,
+)
+
+from charmhelpers.core.host import lsb_release, mounts, umount
+from charmhelpers.fetch import apt_install
+from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
+from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
 
 CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
 CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
@@ -57,6 +62,8 @@
     ('1.9.0', 'havana'),
 ])
 
+DEFAULT_LOOPBACK_SIZE = '5G'
+
 
 def error_out(msg):
     juju_log("FATAL ERROR: %s" % msg, level='ERROR')
@@ -299,6 +306,62 @@
     return apt.version_compare(available_vers, cur_vers) == 1
 
 
+def ensure_block_device(block_device):
+    '''
+    Confirm block_device, create as loopback if necessary.
+
+    :param block_device: str: Full path of block device to ensure.
+
+    :returns: str: Full path of ensured block device.
+    '''
+    _none = ['None', 'none', None]
+    if (block_device in _none):
+        error_out('prepare_storage(): Missing required input: '
+                  'block_device=%s.' % block_device, level=ERROR)
+
+    if block_device.startswith('/dev/'):
+        bdev = block_device
+    elif block_device.startswith('/'):
+        _bd = block_device.split('|')
+        if len(_bd) == 2:
+            bdev, size = _bd
+        else:
+            bdev = block_device
+            size = DEFAULT_LOOPBACK_SIZE
+        bdev = ensure_loopback_device(bdev, size)
+    else:
+        bdev = '/dev/%s' % block_device
+
+    if not is_block_device(bdev):
+        error_out('Failed to locate valid block device at %s' % bdev,
+                  level=ERROR)
+
+    return bdev
+
+
+def clean_storage(block_device):
+    '''
+    Ensures a block device is clean.  That is:
+        - unmounted
+        - any lvm volume groups are deactivated
+        - any lvm physical device signatures removed
+        - partition table wiped
+
+    :param block_device: str: Full path to block device to clean.
+    '''
+    for mp, d in mounts():
+        if d == block_device:
+            juju_log('clean_storage(): %s is mounted @ %s, unmounting.' %
+                     (d, mp), level=INFO)
+            umount(mp, persist=True)
+
+    if is_lvm_physical_volume(block_device):
+        deactivate_lvm_volume_group(block_device)
+        remove_lvm_physical_volume(block_device)
+    else:
+        zap_disk(block_device)
+
+
 def is_ip(address):
     """
     Returns True if address is a valid IP address.

=== modified file 'tests/contrib/openstack/test_openstack_utils.py'
--- tests/contrib/openstack/test_openstack_utils.py	2013-10-11 13:38:29 +0000
+++ tests/contrib/openstack/test_openstack_utils.py	2013-10-20 20:20:48 +0000
@@ -61,6 +61,9 @@
     }
 }
 
+MOUNTS = [
+    ['/mnt', '/dev/vdb']
+]
 
 url = 'deb ' + openstack.CLOUD_ARCHIVE_URL
 UCA_SOURCES = [
@@ -431,6 +434,8 @@
         _exists.return_value = False
         os.environ['JUJU_UNIT_NAME'] = 'testing-foo/0'
         openstack.save_script_rc(setting1='foo', setting2='bar')
+        rcdir = '/var/lib/juju/units/testing-foo-0/charm/scripts'
+        _mkdir.assert_called_with(rcdir)
         expected_f = '/var/lib/juju/units/testing-foo-0/charm/scripts/scriptrc'
         _open.assert_called_with(expected_f, 'wb')
         _mkdir.assert_called_with(os.path.dirname(expected_f))
@@ -463,6 +468,81 @@
         vers_pkg.return_value = '2013.1~b1'
         self.assertFalse(openstack.openstack_upgrade_available('nova-common'))
 
+    @patch.object(openstack, 'is_block_device')
+    @patch.object(openstack, 'error_out')
+    def test_ensure_block_device_bad_config(self, err, is_bd):
+        '''Test it doesn't prepare storage with bad config'''
+        openstack.ensure_block_device(block_device='none')
+        self.assertTrue(err.called)
+
+    @patch.object(openstack, 'is_block_device')
+    @patch.object(openstack, 'ensure_loopback_device')
+    def test_ensure_block_device_loopback(self, ensure_loopback, is_bd):
+        '''Test it ensures loopback device when checking block device'''
+        defsize = openstack.DEFAULT_LOOPBACK_SIZE
+        is_bd.return_value = True
+
+        ensure_loopback.return_value = '/tmp/cinder.img'
+        result = openstack.ensure_block_device('/tmp/cinder.img')
+        ensure_loopback.assert_called_with('/tmp/cinder.img', defsize)
+        self.assertEquals(result, '/tmp/cinder.img')
+
+        ensure_loopback.return_value = '/tmp/cinder-2.img'
+        result = openstack.ensure_block_device('/tmp/cinder-2.img|15G')
+        ensure_loopback.assert_called_with('/tmp/cinder-2.img', '15G')
+        self.assertEquals(result, '/tmp/cinder-2.img')
+
+    @patch.object(openstack, 'is_block_device')
+    def test_ensure_standard_block_device(self, is_bd):
+        '''Test it looks for storage at both relative and full device path'''
+        for dev in ['vdb', '/dev/vdb']:
+            openstack.ensure_block_device(dev)
+            is_bd.assert_called_with('/dev/vdb')
+
+    @patch.object(openstack, 'is_block_device')
+    @patch.object(openstack, 'error_out')
+    def test_ensure_nonexistent_block_device(self, error_out, is_bd):
+        '''Test it will not ensure a non-existant block device'''
+        is_bd.return_value = False
+        openstack.ensure_block_device(block_device='foo')
+        self.assertTrue(error_out.called)
+
+    @patch.object(openstack, 'juju_log')
+    @patch.object(openstack, 'umount')
+    @patch.object(openstack, 'mounts')
+    @patch.object(openstack, 'zap_disk')
+    @patch.object(openstack, 'is_lvm_physical_volume')
+    def test_clean_storage_unmount(self, is_pv, zap_disk, mounts, umount, log):
+        '''Test it unmounts block device when cleaning storage'''
+        is_pv.return_value = False
+        zap_disk.return_value = True
+        mounts.return_value = MOUNTS
+        openstack.clean_storage('/dev/vdb')
+        umount.called_with('/dev/vdb', True)
+
+    @patch.object(openstack, 'juju_log')
+    @patch.object(openstack, 'remove_lvm_physical_volume')
+    @patch.object(openstack, 'deactivate_lvm_volume_group')
+    @patch.object(openstack, 'mounts')
+    @patch.object(openstack, 'is_lvm_physical_volume')
+    def test_clean_storage_lvm_wipe(self, is_pv, mounts, rm_lv, rm_vg, log):
+        '''Test it removes traces of LVM when cleaning storage'''
+        mounts.return_value = []
+        is_pv.return_value = True
+        openstack.clean_storage('/dev/vdb')
+        rm_lv.assert_called_with('/dev/vdb')
+        rm_vg .assert_called_with('/dev/vdb')
+
+    @patch.object(openstack, 'zap_disk')
+    @patch.object(openstack, 'is_lvm_physical_volume')
+    @patch.object(openstack, 'mounts')
+    def test_clean_storage_zap_disk(self, mounts, is_pv, zap_disk):
+        '''It removes traces of LVM when cleaning storage'''
+        mounts.return_value = []
+        is_pv.return_value = False
+        openstack.clean_storage('/dev/vdb')
+        zap_disk.assert_called_with('/dev/vdb')
+
     def test_is_ip(self):
         self.assertTrue(openstack.is_ip('10.0.0.1'))
         self.assertFalse(openstack.is_ip('www.ubuntu.com'))
@@ -495,5 +575,6 @@
             hn = openstack.get_hostname('www.ubuntu.com')
         self.assertEquals(hn, 'www.ubuntu.com')
 
+
 if __name__ == '__main__':
     unittest.main()