Merge lp:~hopem/charms/precise/quantum-gateway/lp1273067 into lp:charms/quantum-gateway

[Edward Hope-Morley]

The is part of a resubmit for all the charm use-syslog patches since the previous attempts were getting messy

[lp:~hopem/charms/precise/quantum-gateway/lp1273067]

48. By Edward Hope-Morley on 2014-03-26

[hopem] removed extraneous use_syslog from .ini files since neutron.conf & quantum.conf should suffice

[Edward Hope-Morley (hopem)]

review: Approve

=== modified file 'config.yaml'
--- config.yaml	2014-03-14 13:16:57 +0000
+++ config.yaml	2014-03-26 11:07:00 +0000
@@ -50,6 +50,11 @@
     type: string
     description: RabbitMQ Virtual Host
     default: openstack
+  use-syslog:
+    type: boolean
+    default: False
+    description: |
+      If set to True, supporting services will log to syslog.
   instance-mtu:
     type: int
     description: |
 
=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2014-03-14 13:24:39 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2014-03-26 11:07:00 +0000
@@ -199,6 +199,7 @@
 
         ctxt = {}
         for rid in relation_ids('amqp'):
+            ha_vip_only = False
             for unit in related_units(rid):
                 if relation_get('clustered', rid=rid, unit=unit):
                     ctxt['clustered'] = True
@@ -213,16 +214,18 @@
                                                       unit=unit),
                     'rabbitmq_virtual_host': vhost,
                 })
+                if relation_get('ha_queues', rid=rid, unit=unit) is not None:
+                    ctxt['rabbitmq_ha_queues'] = True
+
+                ha_vip_only = relation_get('ha-vip-only',
+                                           rid=rid, unit=unit) is not None
+
                 if context_complete(ctxt):
                     # Sufficient information found = break out!
                     break
             # Used for active/active rabbitmq >= grizzly
-            if ('clustered' not in ctxt or relation_get('ha-vip-only') == 'True') and \
-               len(related_units(rid)) > 1:
-                if relation_get('ha_queues'):
-                    ctxt['rabbitmq_ha_queues'] = relation_get('ha_queues')
-                else:
-                    ctxt['rabbitmq_ha_queues'] = False
+            if ('clustered' not in ctxt or ha_vip_only) \
+                    and len(related_units(rid)) > 1:
                 rabbitmq_hosts = []
                 for unit in related_units(rid):
                     rabbitmq_hosts.append(relation_get('private-address',
 
=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
--- hooks/charmhelpers/contrib/openstack/neutron.py	2013-11-06 03:57:51 +0000
+++ hooks/charmhelpers/contrib/openstack/neutron.py	2014-03-26 11:07:00 +0000
@@ -18,6 +18,22 @@
     return 'linux-headers-%s' % kver
 
 
+def kernel_version():
+    """ Retrieve the current major kernel version as a tuple e.g. (3, 13) """
+    kver = check_output(['uname', '-r']).strip()
+    kver = kver.split('.')
+    return (int(kver[0]), int(kver[1]))
+
+
+def determine_dkms_package():
+    """ Determine which DKMS package should be used based on kernel version """
+    # NOTE: 3.13 kernels have support for GRE and VXLAN native
+    if kernel_version() >= (3, 13):
+        return []
+    else:
+        return ['openvswitch-datapath-dkms']
+
+
 # legacy
 def quantum_plugins():
     from charmhelpers.contrib.openstack import context
@@ -32,7 +48,7 @@
                                         database=config('neutron-database'),
                                         relation_prefix='neutron')],
             'services': ['quantum-plugin-openvswitch-agent'],
-            'packages': [[headers_package(), 'openvswitch-datapath-dkms'],
+            'packages': [[headers_package()] + determine_dkms_package(),
                          ['quantum-plugin-openvswitch-agent']],
             'server_packages': ['quantum-server',
                                 'quantum-plugin-openvswitch'],
@@ -57,7 +73,8 @@
 
 def neutron_plugins():
     from charmhelpers.contrib.openstack import context
-    return {
+    release = os_release('nova-common')
+    plugins = {
         'ovs': {
             'config': '/etc/neutron/plugins/openvswitch/'
                       'ovs_neutron_plugin.ini',
@@ -68,8 +85,8 @@
                                         database=config('neutron-database'),
                                         relation_prefix='neutron')],
             'services': ['neutron-plugin-openvswitch-agent'],
-            'packages': [[headers_package(), 'openvswitch-datapath-dkms'],
-                         ['quantum-plugin-openvswitch-agent']],
+            'packages': [[headers_package()] + determine_dkms_package(),
+                         ['neutron-plugin-openvswitch-agent']],
             'server_packages': ['neutron-server',
                                 'neutron-plugin-openvswitch'],
             'server_services': ['neutron-server']
@@ -89,6 +106,13 @@
             'server_services': ['neutron-server']
         }
     }
+    # NOTE: patch in ml2 plugin for icehouse onwards
+    if release >= 'icehouse':
+        plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini'
+        plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin'
+        plugins['ovs']['server_packages'] = ['neutron-server',
+                                             'neutron-plugin-ml2']
+    return plugins
 
 
 def neutron_plugin_attribute(plugin, attr, net_manager=None):
 
=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	2014-03-14 13:24:39 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2014-03-26 11:07:00 +0000
@@ -65,6 +65,9 @@
     ('1.10.0', 'havana'),
     ('1.9.1', 'havana'),
     ('1.9.0', 'havana'),
+    ('1.13.0', 'icehouse'),
+    ('1.12.0', 'icehouse'),
+    ('1.11.0', 'icehouse'),
 ])
 
 DEFAULT_LOOPBACK_SIZE = '5G'
@@ -420,19 +423,19 @@
     Resolves hostname for given IP, or returns the input
     if it is already a hostname.
     """
-    if not is_ip(address):
-        return address
-
-    try:
-        import dns.reversename
-    except ImportError:
-        apt_install('python-dnspython')
-        import dns.reversename
-
-    rev = dns.reversename.from_address(address)
-    result = ns_query(rev)
-    if not result:
-        return None
+    if is_ip(address):
+        try:
+            import dns.reversename
+        except ImportError:
+            apt_install('python-dnspython')
+            import dns.reversename
+
+        rev = dns.reversename.from_address(address)
+        result = ns_query(rev)
+        if not result:
+            return None
+    else:
+        result = address
 
     if fqdn:
         # strip trailing .
 
=== modified file 'hooks/charmhelpers/contrib/storage/linux/utils.py'
--- hooks/charmhelpers/contrib/storage/linux/utils.py	2014-03-14 13:24:39 +0000
+++ hooks/charmhelpers/contrib/storage/linux/utils.py	2014-03-26 11:07:00 +0000
@@ -22,4 +22,5 @@
 
     :param block_device: str: Full path of block device to clean.
     '''
-    check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device])
+    check_call(['sgdisk', '--zap-all', '--clear',
+                '--mbrtogpt', block_device])
 
=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2014-03-14 13:24:39 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2014-03-26 11:07:00 +0000
@@ -97,6 +97,29 @@
         subprocess.call(cmd, env=env)
 
 
+def apt_upgrade(options=None, fatal=False, dist=False):
+    """Upgrade all packages"""
+    if options is None:
+        options = ['--option=Dpkg::Options::=--force-confold']
+
+    cmd = ['apt-get', '--assume-yes']
+    cmd.extend(options)
+    if dist:
+        cmd.append('dist-upgrade')
+    else:
+        cmd.append('upgrade')
+    log("Upgrading with options: {}".format(options))
+
+    env = os.environ.copy()
+    if 'DEBIAN_FRONTEND' not in env:
+        env['DEBIAN_FRONTEND'] = 'noninteractive'
+
+    if fatal:
+        subprocess.check_call(cmd, env=env)
+    else:
+        subprocess.call(cmd, env=env)
+
+
 def apt_update(fatal=False):
     """Update local apt cache"""
     cmd = ['apt-get', 'update']
 
=== modified file 'hooks/charmhelpers/fetch/archiveurl.py'
--- hooks/charmhelpers/fetch/archiveurl.py	2013-09-02 16:36:50 +0000
+++ hooks/charmhelpers/fetch/archiveurl.py	2014-03-26 11:07:00 +0000
@@ -1,5 +1,7 @@
 import os
 import urllib2
+import urlparse
+
 from charmhelpers.fetch import (
     BaseFetchHandler,
     UnhandledSource
@@ -24,6 +26,19 @@
     def download(self, source, dest):
         # propogate all exceptions
         # URLError, OSError, etc
+        proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
+        if proto in ('http', 'https'):
+            auth, barehost = urllib2.splituser(netloc)
+            if auth is not None:
+                source = urlparse.urlunparse((proto, barehost, path, params, query, fragment))
+                username, password = urllib2.splitpasswd(auth)
+                passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+                # Realm is set to None in add_password to force the username and password
+                # to be used whatever the realm
+                passman.add_password(None, source, username, password)
+                authhandler = urllib2.HTTPBasicAuthHandler(passman)
+                opener = urllib2.build_opener(authhandler)
+                urllib2.install_opener(opener)
         response = urllib2.urlopen(source)
         try:
             with open(dest, 'w') as dest_file:
 
=== modified file 'hooks/quantum_utils.py'
--- hooks/quantum_utils.py	2014-03-14 13:24:39 +0000
+++ hooks/quantum_utils.py	2014-03-26 11:07:00 +0000
@@ -25,6 +25,9 @@
 )
 
 import charmhelpers.contrib.openstack.context as context
+from charmhelpers.contrib.openstack.context import (
+    SyslogContext
+)
 import charmhelpers.contrib.openstack.templating as templating
 from charmhelpers.contrib.openstack.neutron import headers_package
 from quantum_contexts import (
@@ -150,7 +153,8 @@
         'hook_contexts': [context.AMQPContext(),
                           QuantumSharedDBContext(),
                           NetworkServiceContext(),
-                          QuantumGatewayContext()],
+                          QuantumGatewayContext(),
+                          SyslogContext()],
         'services': ['nova-api-metadata']
     },
 }
@@ -188,7 +192,8 @@
 QUANTUM_OVS_CONFIG_FILES = {
     QUANTUM_CONF: {
         'hook_contexts': [context.AMQPContext(),
-                          QuantumGatewayContext()],
+                          QuantumGatewayContext(),
+                          SyslogContext()],
         'services': ['quantum-l3-agent',
                      'quantum-dhcp-agent',
                      'quantum-metadata-agent',
@@ -214,7 +219,8 @@
 NEUTRON_OVS_CONFIG_FILES = {
     NEUTRON_CONF: {
         'hook_contexts': [context.AMQPContext(),
-                          QuantumGatewayContext()],
+                          QuantumGatewayContext(),
+                          SyslogContext()],
         'services': ['neutron-l3-agent',
                      'neutron-dhcp-agent',
                      'neutron-metadata-agent',
 
=== modified file 'revision'
--- revision	2014-03-14 13:24:39 +0000
+++ revision	2014-03-26 11:07:00 +0000
@@ -1,1 +1,1 @@
-63
+64
 \ No newline at end of file
 
=== modified file 'templates/folsom/metadata_agent.ini'
--- templates/folsom/metadata_agent.ini	2013-09-04 09:46:54 +0000
+++ templates/folsom/metadata_agent.ini	2014-03-26 11:07:00 +0000
@@ -14,4 +14,4 @@
 # shared secret to prevent spoofing. You may select any string for a secret,
 # but it must match here and in the configuration used by the Nova Metadata
 # Server. NOTE: Nova uses a different key: quantum_metadata_proxy_shared_secret
-metadata_proxy_shared_secret = {{ shared_secret }}
+metadata_proxy_shared_secret = {{ shared_secret }}
 \ No newline at end of file
 
=== modified file 'templates/folsom/nova.conf'
--- templates/folsom/nova.conf	2013-09-04 09:46:54 +0000
+++ templates/folsom/nova.conf	2014-03-26 11:07:00 +0000
@@ -4,6 +4,7 @@
 lock_path=/var/lock/nova
 root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
 verbose=True
+use_syslog = {{ use_syslog }}
 api_paste_config=/etc/nova/api-paste.ini
 enabled_apis=metadata
 multi_host=True
 
=== modified file 'templates/folsom/quantum.conf'
--- templates/folsom/quantum.conf	2013-09-04 09:46:54 +0000
+++ templates/folsom/quantum.conf	2014-03-26 11:07:00 +0000
@@ -5,6 +5,7 @@
 rabbit_host = {{ rabbitmq_host }}
 rabbit_password = {{ rabbitmq_password }}
 debug = True
+use_syslog = {{ use_syslog }}
 bind_host = 0.0.0.0
 bind_port = 9696
 core_plugin = {{ core_plugin }} 
 
=== modified file 'templates/havana/neutron.conf'
--- templates/havana/neutron.conf	2013-09-04 09:46:54 +0000
+++ templates/havana/neutron.conf	2014-03-26 11:07:00 +0000
@@ -5,6 +5,7 @@
 rabbit_host = {{ rabbitmq_host }}
 rabbit_password = {{ rabbitmq_password }}
 debug = True
+use_syslog = {{ use_syslog }}
 bind_host = 0.0.0.0
 bind_port = 9696
 core_plugin = {{ core_plugin }} 
 
=== modified file 'templates/havana/nova.conf'
--- templates/havana/nova.conf	2013-09-04 09:46:54 +0000
+++ templates/havana/nova.conf	2014-03-26 11:07:00 +0000
@@ -4,6 +4,7 @@
 lock_path=/var/lock/nova
 root_helper=sudo nova-rootwrap /etc/nova/rootwrap.conf
 verbose=True
+use_syslog = {{ use_syslog }}
 api_paste_config=/etc/nova/api-paste.ini
 enabled_apis=metadata
 multi_host=True
 
=== modified file 'unit_tests/test_quantum_contexts.py'
--- unit_tests/test_quantum_contexts.py	2014-03-14 13:24:39 +0000
+++ unit_tests/test_quantum_contexts.py	2014-03-26 11:07:00 +0000
@@ -1,5 +1,10 @@
-from mock import MagicMock, patch
+from mock import (
+    Mock,
+    MagicMock,
+    patch
+)
 import quantum_contexts
+import sys
 from contextlib import contextmanager
 
 from test_utils import (
@@ -199,6 +204,7 @@
     def setUp(self):
         super(TestQuantumGatewayContext, self).setUp(quantum_contexts,
                                                      TO_PATCH)
+        self.config.side_effect = self.test_config.get
 
     @patch.object(quantum_contexts, 'get_shared_secret')
     @patch.object(quantum_contexts, 'get_host_ip')
@@ -256,6 +262,19 @@
         super(TestHostIP, self).setUp(quantum_contexts,
                                       TO_PATCH)
         self.config.side_effect = self.test_config.get
+        # Save and inject
+        self.mods = {'dns': None, 'dns.resolver': None}
+        for mod in self.mods:
+            if mod not in sys.modules:
+                sys.modules[mod] = Mock()
+            else:
+                del self.mods[mod]
+
+    def tearDown(self):
+        super(TestHostIP, self).tearDown()
+        # Cleanup
+        for mod in self.mods.keys():
+            del sys.modules[mod]
 
     def test_get_host_ip_already_ip(self):
         self.assertEquals(quantum_contexts.get_host_ip('10.5.0.1'),
@@ -268,8 +287,7 @@
 
     @patch('dns.resolver.query')
     def test_get_host_ip_hostname_unresolvable(self, _query):
-        class NXDOMAIN(Exception):
-            pass
+        class NXDOMAIN(Exception): pass
         _query.side_effect = NXDOMAIN()
         self.assertRaises(NXDOMAIN, quantum_contexts.get_host_ip,
                           'missing.example.com')