Merge lp:~openstack-charmers/charms/precise/cinder/multi-backend into lp:~openstack-charmers-archive/charms/precise/cinder/trunk

[James Page]

Add support for multiple storage backends using subordinate charms.

[Marco Ceppi (marcoceppi)]

There appears to be a few conflicts

review: Abstain

[lp:~openstack-charmers/charms/precise/cinder/multi-backend]

36. By James Page on 2014-02-19

Resync helpers

37. By James Page on 2014-03-18

Rebase on trunk, re-sync helpers

=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2014-03-13 15:50:49 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2014-03-18 13:08:41 +0000
@@ -29,6 +29,7 @@
     determine_apache_port,
     determine_api_port,
     https,
+    is_clustered
 )
 
 from charmhelpers.contrib.hahelpers.apache import (
@@ -240,10 +241,13 @@
         '''This generates context for /etc/ceph/ceph.conf templates'''
         if not relation_ids('ceph'):
             return {}
+
         log('Generating template context for ceph')
+
         mon_hosts = []
         auth = None
         key = None
+        use_syslog = str(config('use-syslog')).lower()
         for rid in relation_ids('ceph'):
             for unit in related_units(rid):
                 mon_hosts.append(relation_get('private-address', rid=rid,
@@ -255,6 +259,7 @@
             'mon_hosts': ' '.join(mon_hosts),
             'auth': auth,
             'key': key,
+            'use_syslog': use_syslog
         }
 
         if not os.path.isdir('/etc/ceph'):
@@ -391,7 +396,7 @@
         return ctxt
 
 
-class NeutronContext(object):
+class NeutronContext(OSContextGenerator):
     interfaces = []
 
     @property
@@ -452,6 +457,22 @@
 
         return nvp_ctxt
 
+    def neutron_ctxt(self):
+        if https():
+            proto = 'https'
+        else:
+            proto = 'http'
+        if is_clustered():
+            host = config('vip')
+        else:
+            host = unit_get('private-address')
+        url = '%s://%s:%s' % (proto, host, '9696')
+        ctxt = {
+            'network_manager': self.network_manager,
+            'neutron_url': url,
+        }
+        return ctxt
+
     def __call__(self):
         self._ensure_packages()
 
@@ -461,7 +482,7 @@
         if not self.plugin:
             return {}
 
-        ctxt = {'network_manager': self.network_manager}
+        ctxt = self.neutron_ctxt()
 
         if self.plugin == 'ovs':
             ctxt.update(self.ovs_ctxt())
@@ -493,41 +514,7 @@
             if not config_flags:
                 return {}
 
-            if config_flags.find('==') >= 0:
-                log("config_flags is not in expected format (key=value)",
-                    level=ERROR)
-                raise OSContextError
-
-            # strip the following from each value.
-            post_strippers = ' ,'
-            # we strip any leading/trailing '=' or ' ' from the string then
-            # split on '='.
-            split = config_flags.strip(' =').split('=')
-            limit = len(split)
-            flags = {}
-            for i in xrange(0, limit - 1):
-                current = split[i]
-                next = split[i + 1]
-                vindex = next.rfind(',')
-                if (i == limit - 2) or (vindex < 0):
-                    value = next
-                else:
-                    value = next[:vindex]
-
-                if i == 0:
-                    key = current
-                else:
-                    # if this not the first entry, expect an embedded key.
-                    index = current.rfind(',')
-                    if index < 0:
-                        log("invalid config value(s) at index %s" % (i),
-                            level=ERROR)
-                        raise OSContextError
-                    key = current[index + 1:]
-
-                # Add to collection.
-                flags[key.strip(post_strippers)] = value.rstrip(post_strippers)
-
+            flags = config_flags_parser(config_flags)
             return {'user_config_flags': flags}
 
 

=== modified file 'hooks/charmhelpers/contrib/openstack/templates/ceph.conf'
--- hooks/charmhelpers/contrib/openstack/templates/ceph.conf	2013-10-17 21:48:08 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/ceph.conf	2014-03-18 13:08:41 +0000
@@ -9,3 +9,6 @@
  keyring = /etc/ceph/$cluster.$name.keyring
  mon host = {{ mon_hosts }}
 {% endif -%}
+log to syslog = {{ use_syslog }}
+err to syslog = {{ use_syslog }}
+clog to syslog = {{ use_syslog }}

=== modified file 'hooks/charmhelpers/contrib/storage/linux/ceph.py'
--- hooks/charmhelpers/contrib/storage/linux/ceph.py	2013-11-06 03:53:17 +0000
+++ hooks/charmhelpers/contrib/storage/linux/ceph.py	2014-03-18 13:08:41 +0000
@@ -49,6 +49,9 @@
  auth supported = {auth}
  keyring = {keyring}
  mon host = {mon_hosts}
+ log to syslog = {use_syslog}
+ err to syslog = {use_syslog}
+ clog to syslog = {use_syslog}
 """
 
 
@@ -194,7 +197,7 @@
     return hosts
 
 
-def configure(service, key, auth):
+def configure(service, key, auth, use_syslog):
     ''' Perform basic configuration of Ceph '''
     create_keyring(service, key)
     create_key_file(service, key)
@@ -202,7 +205,8 @@
     with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
         ceph_conf.write(CEPH_CONF.format(auth=auth,
                                          keyring=_keyring_path(service),
-                                         mon_hosts=",".join(map(str, hosts))))
+                                         mon_hosts=",".join(map(str, hosts)),
+                                         use_syslog=use_syslog))
     modprobe('rbd')
 
 

=== modified file 'hooks/charmhelpers/contrib/storage/linux/utils.py'
--- hooks/charmhelpers/contrib/storage/linux/utils.py	2014-02-18 12:25:01 +0000
+++ hooks/charmhelpers/contrib/storage/linux/utils.py	2014-03-18 13:08:41 +0000
@@ -22,4 +22,5 @@
 
     :param block_device: str: Full path of block device to clean.
     '''
-    check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device])
+    check_call(['sgdisk', '--zap-all', '--clear',
+                '--mbrtogpt', block_device])

=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2014-02-18 12:25:01 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2014-03-18 13:08:41 +0000
@@ -135,6 +135,10 @@
 
 
 def add_source(source, key=None):
+    if source is None:
+        log('Source is not present. Skipping')
+        return
+
     if (source.startswith('ppa:') or
         source.startswith('http') or
         source.startswith('deb ') or

=== modified file 'hooks/cinder_contexts.py'
--- hooks/cinder_contexts.py	2014-03-13 15:50:49 +0000
+++ hooks/cinder_contexts.py	2014-03-18 13:08:41 +0000
@@ -2,6 +2,8 @@
     config,
     relation_ids,
     service_name,
+    related_units,
+    relation_get,
 )
 
 from charmhelpers.contrib.openstack.context import (
@@ -74,3 +76,20 @@
         if not service_enabled('cinder-api'):
             return {}
         return super(ApacheSSLContext, self).__call__()
+
+
+class StorageBackendContext(OSContextGenerator):
+    interfaces = ['storage-backend']
+
+    def __call__(self):
+        backends = []
+        for rid in relation_ids('storage-backend'):
+            for unit in related_units(rid):
+                backend_name = relation_get('backend_name',
+                                            unit, rid)
+                if backend_name:
+                    backends.append(backend_name)
+        if len(backends) > 0:
+            return {'backends': ",".join(backends)}
+        else:
+            return {}

=== modified file 'hooks/cinder_hooks.py'
--- hooks/cinder_hooks.py	2014-03-13 15:50:49 +0000
+++ hooks/cinder_hooks.py	2014-03-18 13:08:41 +0000
@@ -271,6 +271,13 @@
         amqp_joined(relation_id=rel_id)
 
 
[e-mail]('storage-backend-relation-changed')
[e-mail]('storage-backend-relation-broken')
+@restart_on_change(restart_map())
+def storage_backend():
+    CONFIGS.write(CINDER_CONF)
+
+
 if __name__ == '__main__':
     try:
         hooks.execute(sys.argv)

=== modified file 'hooks/cinder_utils.py'
--- hooks/cinder_utils.py	2014-02-18 12:25:01 +0000
+++ hooks/cinder_utils.py	2014-03-18 13:08:41 +0000
@@ -98,6 +98,7 @@
 
 TEMPLATES = 'templates/'
 
+
 def ceph_config_file():
     return CHARM_CEPH_CONF.format(service_name())
 
@@ -112,7 +113,12 @@
                           context.SyslogContext(),
                           cinder_contexts.CephContext(),
                           cinder_contexts.HAProxyContext(),
-                          cinder_contexts.ImageServiceContext()],
+                          cinder_contexts.ImageServiceContext(),
+                          context.SubordinateConfigContext(
+                              interface='storage-backend',
+                              service='cinder',
+                              config_file=CINDER_CONF),
+                          cinder_contexts.StorageBackendContext()],
         'services': ['cinder-api', 'cinder-volume',
                      'cinder-scheduler', 'haproxy']
     }),

=== added symlink 'hooks/storage-backend-relation-broken'
=== target is u'cinder_hooks.py'
=== added symlink 'hooks/storage-backend-relation-changed'
=== target is u'cinder_hooks.py'
=== added symlink 'hooks/storage-backend-relation-departed'
=== target is u'cinder_hooks.py'
=== added symlink 'hooks/storage-backend-relation-joined'
=== target is u'cinder_hooks.py'
=== modified file 'metadata.yaml'
--- metadata.yaml	2013-10-17 21:48:08 +0000
+++ metadata.yaml	2014-03-18 13:08:41 +0000
@@ -22,6 +22,9 @@
   ha:
     interface: hacluster
     scope: container
+  storage-backend:
+    interface: cinder-backend
+    scope: container
 peers:
   cluster:
     interface: cinder-ha

=== modified file 'templates/cinder.conf'
--- templates/cinder.conf	2014-02-03 10:44:24 +0000
+++ templates/cinder.conf	2014-03-18 13:08:41 +0000
@@ -50,3 +50,21 @@
 {% endfor -%}
 {% endif -%}
 
+{% if sections and 'DEFAULT' in sections -%}
+{% for key, value in sections['DEFAULT'] -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{% endif -%}
+
+{% if backends -%}
+enabled_backends = {{ backends }}
+{% endif -%}
+
+{% for section in sections -%}
+{% if section != 'DEFAULT' -%}
+[{{ section }}]
+{% for key, value in sections[section] -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{% endif -%}
+{% endfor -%}

=== modified file 'unit_tests/test_cinder_contexts.py'
--- unit_tests/test_cinder_contexts.py	2014-03-13 15:50:49 +0000
+++ unit_tests/test_cinder_contexts.py	2014-03-18 13:08:41 +0000
@@ -16,10 +16,13 @@
     'service_name',
     'determine_apache_port',
     'determine_api_port',
+    'related_units',
+    'relation_get'
 ]
 
 
 class TestCinderContext(CharmTestCase):
+
     def setUp(self):
         super(TestCinderContext, self).setUp(contexts, TO_PATCH)
 
@@ -59,6 +62,25 @@
         service_enabled.return_value = False
         self.assertEquals(contexts.ApacheSSLContext()(), {})
 
+    def test_storage_backend_no_backends(self):
+        self.relation_ids.return_value = []
+        self.assertEquals(contexts.StorageBackendContext()(), {})
+
+    def test_storage_backend_single_backend(self):
+        self.relation_ids.return_value = ['cinder-ceph:0']
+        self.related_units.return_value = ['cinder-ceph/0']
+        self.relation_get.return_value = 'cinder-ceph'
+        self.assertEquals(contexts.StorageBackendContext()(),
+                          {'backends': 'cinder-ceph'})
+
+    def test_storage_backend_multi_backend(self):
+        self.relation_ids.return_value = ['cinder-ceph:0', 'cinder-vmware:0']
+        self.related_units.side_effect = [['cinder-ceph/0'],
+                                          ['cinder-vmware/0']]
+        self.relation_get.side_effect = ['cinder-ceph', 'cinder-vmware']
+        self.assertEquals(contexts.StorageBackendContext()(),
+                          {'backends': 'cinder-ceph,cinder-vmware'})
+
     @patch('charmhelpers.contrib.openstack.context.determine_apache_port')
     @patch('charmhelpers.contrib.openstack.context.determine_api_port')
     @patch('charmhelpers.contrib.openstack.context.unit_get')

=== modified file 'unit_tests/test_cinder_hooks.py'
--- unit_tests/test_cinder_hooks.py	2014-02-18 12:25:57 +0000
+++ unit_tests/test_cinder_hooks.py	2014-03-18 13:08:41 +0000
@@ -64,6 +64,7 @@
 
 
 class TestInstallHook(CharmTestCase):
+
     def setUp(self):
         super(TestInstallHook, self).setUp(hooks, TO_PATCH)
         self.config.side_effect = self.test_config.get_all
@@ -121,6 +122,7 @@
 
 
 class TestChangedHooks(CharmTestCase):
+
     def setUp(self):
         super(TestChangedHooks, self).setUp(hooks, TO_PATCH)
         self.config.side_effect = self.test_config.get_all
@@ -228,8 +230,17 @@
         hooks.hooks.execute(['hooks/image-service-relation-broken'])
         self.assertTrue(self.CONFIGS.write_all.called)
 
+    def test_storage_backend_changed(self):
+        hooks.hooks.execute(['hooks/storage-backend-relation-changed'])
+        self.CONFIGS.write.assert_called_with(utils.CINDER_CONF)
+
+    def test_storage_backend_broken(self):
+        hooks.hooks.execute(['hooks/storage-backend-relation-broken'])
+        self.CONFIGS.write.assert_called_with(utils.CINDER_CONF)
+
 
 class TestJoinedHooks(CharmTestCase):
+
     def setUp(self):
         super(TestJoinedHooks, self).setUp(hooks, TO_PATCH)
         self.config.side_effect = self.test_config.get_all

=== modified file 'unit_tests/test_cinder_utils.py'
--- unit_tests/test_cinder_utils.py	2014-02-12 10:55:14 +0000
+++ unit_tests/test_cinder_utils.py	2014-03-18 13:08:41 +0000
@@ -51,6 +51,7 @@
 
 
 class TestCinderUtils(CharmTestCase):
+
     def setUp(self):
         super(TestCinderUtils, self).setUp(cinder_utils, TO_PATCH)
         self.config.side_effect = self.test_config.get_all

=== modified file 'unit_tests/test_cluster_hooks.py'
--- unit_tests/test_cluster_hooks.py	2014-03-13 15:50:49 +0000
+++ unit_tests/test_cluster_hooks.py	2014-03-18 13:08:41 +0000
@@ -58,6 +58,7 @@
 
 
 class TestClusterHooks(CharmTestCase):
+
     def setUp(self):
         super(TestClusterHooks, self).setUp(hooks, TO_PATCH)
         self.config.side_effect = self.test_config.get_all

=== modified file 'unit_tests/test_utils.py'
--- unit_tests/test_utils.py	2014-01-15 15:30:21 +0000
+++ unit_tests/test_utils.py	2014-03-18 13:08:41 +0000
@@ -55,6 +55,7 @@
 
 
 class CharmTestCase(unittest.TestCase):
+
     def setUp(self, obj, patches):
         super(CharmTestCase, self).setUp()
         self.patches = patches
@@ -75,6 +76,7 @@
 
 
 class TestConfig(object):
+
     def __init__(self):
         self.config = get_default_config()
 
@@ -94,6 +96,7 @@
 
 
 class TestRelation(object):
+
     def __init__(self, relation_data={}):
         self.relation_data = relation_data