Merge lp:~openstack-charmers/charm-helpers/icehouse into lp:charm-helpers

[James Page]

WIP for icehouse release of openstack

[James Page (james-page)]

Note that this proposal also includes the active-active branch separately proposed.

[Marco Ceppi (marcoceppi)]

LGTM, following conflict was resolved:

+<<<<<<< TREE
             'packages': [[headers_package(), 'openvswitch-datapath-dkms'],
+=======
+ 'packages': [[headers_package()] + determine_dkms_package(),
+>>>>>>> MERGE-SOURCE

With MERGE-SOURCE being used

review: Approve

=== modified file 'charmhelpers/contrib/openstack/context.py'
--- charmhelpers/contrib/openstack/context.py	2014-02-27 15:50:39 +0000
+++ charmhelpers/contrib/openstack/context.py	2014-03-25 10:15:30 +0000
@@ -199,6 +199,7 @@
 
         ctxt = {}
         for rid in relation_ids('amqp'):
+            ha_vip_only = False
             for unit in related_units(rid):
                 if relation_get('clustered', rid=rid, unit=unit):
                     ctxt['clustered'] = True
@@ -213,16 +214,18 @@
                                                       unit=unit),
                     'rabbitmq_virtual_host': vhost,
                 })
+                if relation_get('ha_queues', rid=rid, unit=unit) is not None:
+                    ctxt['rabbitmq_ha_queues'] = True
+
+                ha_vip_only = relation_get('ha-vip-only',
+                                           rid=rid, unit=unit) is not None
+
                 if context_complete(ctxt):
                     # Sufficient information found = break out!
                     break
             # Used for active/active rabbitmq >= grizzly
-            if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \
-               len(related_units(rid)) > 1:
-                if relation_get('ha_queues'):
-                    ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues')
-                else:
-                    ctxt['rabbitmq_ha_queues'] = False
+            if ('clustered' not in ctxt or ha_vip_only) \
+                    and len(related_units(rid)) > 1:
                 rabbitmq_hosts = []
                 for unit in related_units(rid):
                     rabbitmq_hosts.append(relation_get('private-address',

=== modified file 'charmhelpers/contrib/openstack/neutron.py'
--- charmhelpers/contrib/openstack/neutron.py	2013-10-16 10:45:22 +0000
+++ charmhelpers/contrib/openstack/neutron.py	2014-03-25 10:15:30 +0000
@@ -18,6 +18,22 @@
     return 'linux-headers-%s' % kver
 
 
+def kernel_version():
+    """ Retrieve the current major kernel version as a tuple e.g. (3, 13) """
+    kver = check_output(['uname', '-r']).strip()
+    kver = kver.split('.')
+    return (int(kver[0]), int(kver[1]))
+
+
+def determine_dkms_package():
+    """ Determine which DKMS package should be used based on kernel version """
+    # NOTE: 3.13 kernels have support for GRE and VXLAN native
+    if kernel_version() >= (3, 13):
+        return []
+    else:
+        return ['openvswitch-datapath-dkms']
+
+
 # legacy
 def quantum_plugins():
     from charmhelpers.contrib.openstack import context
@@ -32,7 +48,7 @@
                                         database=config('neutron-database'),
                                         relation_prefix='neutron')],
             'services': ['quantum-plugin-openvswitch-agent'],
-            'packages': [[headers_package(), 'openvswitch-datapath-dkms'],
+            'packages': [[headers_package()] + determine_dkms_package(),
                          ['quantum-plugin-openvswitch-agent']],
             'server_packages': ['quantum-server',
                                 'quantum-plugin-openvswitch'],
@@ -57,7 +73,8 @@
 
 def neutron_plugins():
     from charmhelpers.contrib.openstack import context
-    return {
+    release = os_release('nova-common')
+    plugins = {
         'ovs': {
             'config': '/etc/neutron/plugins/openvswitch/'
                       'ovs_neutron_plugin.ini',
@@ -68,8 +85,8 @@
                                         database=config('neutron-database'),
                                         relation_prefix='neutron')],
             'services': ['neutron-plugin-openvswitch-agent'],
-            'packages': [[headers_package(), 'openvswitch-datapath-dkms'],
-                         ['quantum-plugin-openvswitch-agent']],
+            'packages': [[headers_package()] + determine_dkms_package(),
+                         ['neutron-plugin-openvswitch-agent']],
             'server_packages': ['neutron-server',
                                 'neutron-plugin-openvswitch'],
             'server_services': ['neutron-server']
@@ -89,6 +106,13 @@
             'server_services': ['neutron-server']
         }
     }
+    # NOTE: patch in ml2 plugin for icehouse onwards
+    if release >= 'icehouse':
+        plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini'
+        plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin'
+        plugins['ovs']['server_packages'] = ['neutron-server',
+                                             'neutron-plugin-ml2']
+    return plugins
 
 
 def neutron_plugin_attribute(plugin, attr, net_manager=None):

=== modified file 'charmhelpers/contrib/openstack/utils.py'
--- charmhelpers/contrib/openstack/utils.py	2014-01-14 12:14:15 +0000
+++ charmhelpers/contrib/openstack/utils.py	2014-03-25 10:15:30 +0000
@@ -65,6 +65,9 @@
     ('1.10.0', 'havana'),
     ('1.9.1', 'havana'),
     ('1.9.0', 'havana'),
+    ('1.13.0', 'icehouse'),
+    ('1.12.0', 'icehouse'),
+    ('1.11.0', 'icehouse'),
 ])
 
 DEFAULT_LOOPBACK_SIZE = '5G'
@@ -420,19 +423,19 @@
     Resolves hostname for given IP, or returns the input
     if it is already a hostname.
     """
-    if not is_ip(address):
-        return address
-
-    try:
-        import dns.reversename
-    except ImportError:
-        apt_install('python-dnspython')
-        import dns.reversename
-
-    rev = dns.reversename.from_address(address)
-    result = ns_query(rev)
-    if not result:
-        return None
+    if is_ip(address):
+        try:
+            import dns.reversename
+        except ImportError:
+            apt_install('python-dnspython')
+            import dns.reversename
+
+        rev = dns.reversename.from_address(address)
+        result = ns_query(rev)
+        if not result:
+            return None
+    else:
+        result = address
 
     if fqdn:
         # strip trailing .

=== added directory 'charmhelpers/contrib/peerstorage'
=== added file 'charmhelpers/contrib/peerstorage/__init__.py'
--- charmhelpers/contrib/peerstorage/__init__.py	1970-01-01 00:00:00 +0000
+++ charmhelpers/contrib/peerstorage/__init__.py	2014-03-25 10:15:30 +0000
@@ -0,0 +1,83 @@
+from charmhelpers.core.hookenv import (
+    relation_ids,
+    relation_get,
+    local_unit,
+    relation_set,
+)
+
+"""
+This helper provides functions to support use of a peer relation
+for basic key/value storage, with the added benefit that all storage
+can be replicated across peer units, so this is really useful for
+services that issue usernames/passwords to remote services.
+
+def shared_db_changed()
+    # Only the lead unit should create passwords
+    if not is_leader():
+        return
+    username = relation_get('username')
+    key = '{}.password'.format(username)
+    # Attempt to retrieve any existing password for this user
+    password = peer_retrieve(key)
+    if password is None:
+        # New user, create password and store
+        password = pwgen(length=64)
+        peer_store(key, password)
+    create_access(username, password)
+    relation_set(password=password)
+
+
+def cluster_changed()
+    # Echo any relation data other that *-address
+    # back onto the peer relation so all units have
+    # all *.password keys stored on their local relation
+    # for later retrieval.
+    peer_echo()
+
+"""
+
+
+def peer_retrieve(key, relation_name='cluster'):
+    """ Retrieve a named key from peer relation relation_name """
+    cluster_rels = relation_ids(relation_name)
+    if len(cluster_rels) > 0:
+        cluster_rid = cluster_rels[0]
+        return relation_get(attribute=key, rid=cluster_rid,
+                            unit=local_unit())
+    else:
+        raise ValueError('Unable to detect'
+                         'peer relation {}'.format(relation_name))
+
+
+def peer_store(key, value, relation_name='cluster'):
+    """ Store the key/value pair on the named peer relation relation_name """
+    cluster_rels = relation_ids(relation_name)
+    if len(cluster_rels) > 0:
+        cluster_rid = cluster_rels[0]
+        relation_set(relation_id=cluster_rid,
+                     relation_settings={key: value})
+    else:
+        raise ValueError('Unable to detect '
+                         'peer relation {}'.format(relation_name))
+
+
+def peer_echo(includes=None):
+    """Echo filtered attributes back onto the same relation for storage
+
+    Note that this helper must only be called within a peer relation
+    changed hook
+    """
+    rdata = relation_get()
+    echo_data = {}
+    if includes is None:
+        echo_data = rdata.copy()
+        for ex in ['private-address', 'public-address']:
+            if ex in echo_data:
+                echo_data.pop(ex)
+    else:
+        for attribute, value in rdata.iteritems():
+            for include in includes:
+                if include in attribute:
+                    echo_data[attribute] = value
+    if len(echo_data) > 0:
+        relation_set(relation_settings=echo_data)

=== modified file 'charmhelpers/fetch/__init__.py'
--- charmhelpers/fetch/__init__.py	2014-03-03 12:30:23 +0000
+++ charmhelpers/fetch/__init__.py	2014-03-25 10:15:30 +0000
@@ -97,6 +97,29 @@
         subprocess.call(cmd, env=env)
 
 
+def apt_upgrade(options=None, fatal=False, dist=False):
+    """Upgrade all packages"""
+    if options is None:
+        options = ['--option=Dpkg::Options::=--force-confold']
+
+    cmd = ['apt-get', '--assume-yes']
+    cmd.extend(options)
+    if dist:
+        cmd.append('dist-upgrade')
+    else:
+        cmd.append('upgrade')
+    log("Upgrading with options: {}".format(options))
+
+    env = os.environ.copy()
+    if 'DEBIAN_FRONTEND' not in env:
+        env['DEBIAN_FRONTEND'] = 'noninteractive'
+
+    if fatal:
+        subprocess.check_call(cmd, env=env)
+    else:
+        subprocess.call(cmd, env=env)
+
+
 def apt_update(fatal=False):
     """Update local apt cache"""
     cmd = ['apt-get', 'update']

=== modified file 'tests/contrib/openstack/test_openstack_utils.py'
--- tests/contrib/openstack/test_openstack_utils.py	2014-01-13 10:31:33 +0000
+++ tests/contrib/openstack/test_openstack_utils.py	2014-03-25 10:15:30 +0000
@@ -599,11 +599,14 @@
 
     @patch.object(openstack, 'apt_install')
     def test_get_hostname_with_hostname(self, apt_install):
-        fake_dns = FakeDNS('5.5.5.5')
-        with patch('__builtin__.__import__', side_effect=[fake_dns]):
-            hn = openstack.get_hostname('www.ubuntu.com')
+        hn = openstack.get_hostname('www.ubuntu.com')
         self.assertEquals(hn, 'www.ubuntu.com')
 
+    @patch.object(openstack, 'apt_install')
+    def test_get_hostname_with_hostname_not_fqdn(self, apt_install):
+        hn = openstack.get_hostname('packages.ubuntu.com', fqdn=False)
+        self.assertEquals(hn, 'packages')
+
 
 if __name__ == '__main__':
     unittest.main()

=== modified file 'tests/contrib/openstack/test_os_contexts.py'
--- tests/contrib/openstack/test_os_contexts.py	2014-02-27 16:20:51 +0000
+++ tests/contrib/openstack/test_os_contexts.py	2014-03-25 10:15:30 +0000
@@ -337,7 +337,6 @@
             'rabbitmq_user': 'adam',
             'rabbitmq_virtual_host': 'foo',
             'rabbitmq_hosts': 'rabbithost2,rabbithost1',
-            'rabbitmq_ha_queues': False
         }
         self.assertEquals(result, expected)
 

=== added directory 'tests/contrib/peerstorage'
=== added file 'tests/contrib/peerstorage/__init__.py'
=== added file 'tests/contrib/peerstorage/test_peerstorage.py'
--- tests/contrib/peerstorage/test_peerstorage.py	1970-01-01 00:00:00 +0000
+++ tests/contrib/peerstorage/test_peerstorage.py	2014-03-25 10:15:30 +0000
@@ -0,0 +1,69 @@
+from tests.helpers import FakeRelation
+from testtools import TestCase
+from mock import patch
+from charmhelpers.contrib import peerstorage
+
+
+TO_PATCH = ['relation_ids', 'relation_set', 'relation_get', 'local_unit']
+FAKE_RELATION_NAME = 'cluster'
+FAKE_RELATION = {
+    'cluster:0': {
+        'cluster/0': {
+        },
+        'cluster/1': {
+        },
+        'cluster/2': {
+        },
+    },
+
+}
+FAKE_RELATION_IDS = ['cluster:0']
+FAKE_LOCAL_UNIT = 'test_host'
+
+
+class TestPeerStorage(TestCase):
+    def setUp(self):
+        super(TestPeerStorage, self).setUp()
+        for m in TO_PATCH:
+            setattr(self, m, self._patch(m))
+        self.fake_relation_name = FAKE_RELATION_NAME
+        self.fake_relation = FakeRelation(FAKE_RELATION)
+        self.local_unit.return_value = FAKE_LOCAL_UNIT
+        self.relation_get.return_value = {'key1': 'value1',
+                                          'key2': 'value2',
+                                          'private-address': '127.0.0.1',
+                                          'public-address': '91.189.90.159'}
+
+    def _patch(self, method):
+        _m = patch('charmhelpers.contrib.peerstorage.' + method)
+        mock = _m.start()
+        self.addCleanup(_m.stop)
+        return mock
+
+    def test_peer_retrieve_no_relation(self):
+        self.relation_ids.return_value = []
+        self.assertRaises(ValueError, peerstorage.peer_retrieve, 'key', relation_name=self.fake_relation_name)
+
+    def test_peer_retrieve_with_relation(self):
+        self.relation_ids.return_value = FAKE_RELATION_IDS
+        peerstorage.peer_retrieve('key', self.fake_relation_name)
+        self.relation_get.assert_called_with(attribute='key', rid=FAKE_RELATION_IDS[0], unit=FAKE_LOCAL_UNIT)
+
+    def test_peer_store_no_relation(self):
+        self.relation_ids.return_value = []
+        self.assertRaises(ValueError, peerstorage.peer_store, 'key', 'value', relation_name=self.fake_relation_name)
+
+    def test_peer_store_with_relation(self):
+        self.relation_ids.return_value = FAKE_RELATION_IDS
+        peerstorage.peer_store('key', 'value', self.fake_relation_name)
+        self.relation_set.assert_called_with(relation_id=FAKE_RELATION_IDS[0],
+                                             relation_settings={'key': 'value'})
+
+    def test_peer_echo_no_includes(self):
+        peerstorage.peer_echo()
+        self.relation_set.assert_called_with(relation_settings={'key1': 'value1',
+                                                                'key2': 'value2'})
+
+    def test_peer_echo_includes(self):
+        peerstorage.peer_echo(['key1'])
+        self.relation_set.assert_called_with(relation_settings={'key1': 'value1'})

=== modified file 'tests/fetch/test_fetch.py'
--- tests/fetch/test_fetch.py	2014-03-05 10:37:16 +0000
+++ tests/fetch/test_fetch.py	2014-03-25 10:15:30 +0000
@@ -391,6 +391,36 @@
 
     @patch('subprocess.call')
     @patch.object(fetch, 'log')
+    def test_apt_upgrade_non_fatal(self, log, mock_call):
+        options = ['--foo', '--bar']
+        fetch.apt_upgrade(options)
+
+        mock_call.assert_called_with(['apt-get', '--assume-yes',
+                                      '--foo', '--bar', 'upgrade'],
+                                     env=getenv({'DEBIAN_FRONTEND': 'noninteractive'}))
+
+    @patch('subprocess.check_call')
+    @patch.object(fetch, 'log')
+    def test_apt_upgrade_fatal(self, log, mock_call):
+        options = ['--foo', '--bar']
+        fetch.apt_upgrade(options, fatal=True)
+
+        mock_call.assert_called_with(['apt-get', '--assume-yes',
+                                      '--foo', '--bar', 'upgrade'],
+                                     env=getenv({'DEBIAN_FRONTEND': 'noninteractive'}))
+
+    @patch('subprocess.check_call')
+    @patch.object(fetch, 'log')
+    def test_apt_dist_upgrade_fatal(self, log, mock_call):
+        options = ['--foo', '--bar']
+        fetch.apt_upgrade(options, fatal=True, dist=True)
+
+        mock_call.assert_called_with(['apt-get', '--assume-yes',
+                                      '--foo', '--bar', 'dist-upgrade'],
+                                     env=getenv({'DEBIAN_FRONTEND': 'noninteractive'}))
+
+    @patch('subprocess.call')
+    @patch.object(fetch, 'log')
     def test_installs_apt_packages(self, log, mock_call):
         packages = ['foo', 'bar']
         options = ['--foo', '--bar']