Merge lp:~hopem/charms/precise/nova-compute/lp1273067 into lp:~openstack-charmers-archive/charms/precise/nova-compute/trunk

[Edward Hope-Morley]

[James Page (james-page)]

review: Approve

=== modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
--- hooks/charmhelpers/contrib/hahelpers/cluster.py	2013-08-14 16:13:38 +0000
+++ hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-03-26 11:39:18 +0000
@@ -126,17 +126,17 @@
     return public_port - (i * 10)
 
 
-def determine_haproxy_port(public_port):
+def determine_apache_port(public_port):
     '''
-    Description: Determine correct proxy listening port based on public IP +
-    existence of HTTPS reverse proxy.
+    Description: Determine correct apache listening port based on public IP +
+    state of the cluster.
 
     public_port: int: standard public port for given service
 
     returns: int: the correct listening port for the HAProxy service
     '''
     i = 0
-    if https():
+    if len(peer_units()) > 0 or is_clustered():
         i += 1
     return public_port - (i * 10)
 

=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2014-02-13 08:29:21 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2014-03-26 11:39:18 +0000
@@ -26,11 +26,10 @@
 )
 
 from charmhelpers.contrib.hahelpers.cluster import (
+    determine_apache_port,
     determine_api_port,
-    determine_haproxy_port,
     https,
-    is_clustered,
-    peer_units,
+    is_clustered
 )
 
 from charmhelpers.contrib.hahelpers.apache import (
@@ -200,6 +199,7 @@
 
         ctxt = {}
         for rid in relation_ids('amqp'):
+            ha_vip_only = False
             for unit in related_units(rid):
                 if relation_get('clustered', rid=rid, unit=unit):
                     ctxt['clustered'] = True
@@ -214,11 +214,18 @@
                                                       unit=unit),
                     'rabbitmq_virtual_host': vhost,
                 })
+                if relation_get('ha_queues', rid=rid, unit=unit) is not None:
+                    ctxt['rabbitmq_ha_queues'] = True
+
+                ha_vip_only = relation_get('ha-vip-only',
+                                           rid=rid, unit=unit) is not None
+
                 if context_complete(ctxt):
                     # Sufficient information found = break out!
                     break
             # Used for active/active rabbitmq >= grizzly
-            if 'clustered' not in ctxt and len(related_units(rid)) > 1:
+            if ('clustered' not in ctxt or ha_vip_only) \
+                    and len(related_units(rid)) > 1:
                 rabbitmq_hosts = []
                 for unit in related_units(rid):
                     rabbitmq_hosts.append(relation_get('private-address',
@@ -237,10 +244,13 @@
         '''This generates context for /etc/ceph/ceph.conf templates'''
         if not relation_ids('ceph'):
             return {}
+
         log('Generating template context for ceph')
+
         mon_hosts = []
         auth = None
         key = None
+        use_syslog = str(config('use-syslog')).lower()
         for rid in relation_ids('ceph'):
             for unit in related_units(rid):
                 mon_hosts.append(relation_get('private-address', rid=rid,
@@ -252,6 +262,7 @@
             'mon_hosts': ' '.join(mon_hosts),
             'auth': auth,
             'key': key,
+            'use_syslog': use_syslog
         }
 
         if not os.path.isdir('/etc/ceph'):
@@ -380,17 +391,15 @@
             'private_address': unit_get('private-address'),
             'endpoints': []
         }
-        for ext_port in self.external_ports:
-            if peer_units() or is_clustered():
-                int_port = determine_haproxy_port(ext_port)
-            else:
-                int_port = determine_api_port(ext_port)
+        for api_port in self.external_ports:
+            ext_port = determine_apache_port(api_port)
+            int_port = determine_api_port(api_port)
             portmap = (int(ext_port), int(int_port))
             ctxt['endpoints'].append(portmap)
         return ctxt
 
 
-class NeutronContext(object):
+class NeutronContext(OSContextGenerator):
     interfaces = []
 
     @property
@@ -451,6 +460,22 @@
 
         return nvp_ctxt
 
+    def neutron_ctxt(self):
+        if https():
+            proto = 'https'
+        else:
+            proto = 'http'
+        if is_clustered():
+            host = config('vip')
+        else:
+            host = unit_get('private-address')
+        url = '%s://%s:%s' % (proto, host, '9696')
+        ctxt = {
+            'network_manager': self.network_manager,
+            'neutron_url': url,
+        }
+        return ctxt
+
     def __call__(self):
         self._ensure_packages()
 
@@ -460,7 +485,7 @@
         if not self.plugin:
             return {}
 
-        ctxt = {'network_manager': self.network_manager}
+        ctxt = self.neutron_ctxt()
 
         if self.plugin == 'ovs':
             ctxt.update(self.ovs_ctxt())
@@ -586,6 +611,7 @@
 
 
 class SyslogContext(OSContextGenerator):
+
     def __call__(self):
         ctxt = {
             'use_syslog': config('use-syslog')

=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
--- hooks/charmhelpers/contrib/openstack/neutron.py	2013-10-22 23:02:45 +0000
+++ hooks/charmhelpers/contrib/openstack/neutron.py	2014-03-26 11:39:18 +0000
@@ -18,6 +18,22 @@
     return 'linux-headers-%s' % kver
 
 
+def kernel_version():
+    """ Retrieve the current major kernel version as a tuple e.g. (3, 13) """
+    kver = check_output(['uname', '-r']).strip()
+    kver = kver.split('.')
+    return (int(kver[0]), int(kver[1]))
+
+
+def determine_dkms_package():
+    """ Determine which DKMS package should be used based on kernel version """
+    # NOTE: 3.13 kernels have support for GRE and VXLAN native
+    if kernel_version() >= (3, 13):
+        return []
+    else:
+        return ['openvswitch-datapath-dkms']
+
+
 # legacy
 def quantum_plugins():
     from charmhelpers.contrib.openstack import context
@@ -32,7 +48,7 @@
                                         database=config('neutron-database'),
                                         relation_prefix='neutron')],
             'services': ['quantum-plugin-openvswitch-agent'],
-            'packages': [[headers_package(), 'openvswitch-datapath-dkms'],
+            'packages': [[headers_package()] + determine_dkms_package(),
                          ['quantum-plugin-openvswitch-agent']],
             'server_packages': ['quantum-server',
                                 'quantum-plugin-openvswitch'],
@@ -57,7 +73,8 @@
 
 def neutron_plugins():
     from charmhelpers.contrib.openstack import context
-    return {
+    release = os_release('nova-common')
+    plugins = {
         'ovs': {
             'config': '/etc/neutron/plugins/openvswitch/'
                       'ovs_neutron_plugin.ini',
@@ -68,8 +85,8 @@
                                         database=config('neutron-database'),
                                         relation_prefix='neutron')],
             'services': ['neutron-plugin-openvswitch-agent'],
-            'packages': [[headers_package(), 'openvswitch-datapath-dkms'],
-                         ['quantum-plugin-openvswitch-agent']],
+            'packages': [[headers_package()] + determine_dkms_package(),
+                         ['neutron-plugin-openvswitch-agent']],
             'server_packages': ['neutron-server',
                                 'neutron-plugin-openvswitch'],
             'server_services': ['neutron-server']
@@ -89,6 +106,13 @@
             'server_services': ['neutron-server']
         }
     }
+    # NOTE: patch in ml2 plugin for icehouse onwards
+    if release >= 'icehouse':
+        plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini'
+        plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin'
+        plugins['ovs']['server_packages'] = ['neutron-server',
+                                             'neutron-plugin-ml2']
+    return plugins
 
 
 def neutron_plugin_attribute(plugin, attr, net_manager=None):

=== modified file 'hooks/charmhelpers/contrib/openstack/templates/ceph.conf'
--- hooks/charmhelpers/contrib/openstack/templates/ceph.conf	2013-08-12 21:48:24 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/ceph.conf	2014-03-26 11:39:18 +0000
@@ -3,9 +3,13 @@
 # cinder configuration file maintained by Juju
 # local changes may be overwritten.
 ###############################################################################
+[global]
 {% if auth -%}
-[global]
  auth_supported = {{ auth }}
  keyring = /etc/ceph/$cluster.$name.keyring
  mon host = {{ mon_hosts }}
 {% endif -%}
+ log to syslog = {{ use_syslog }}
+ err to syslog = {{ use_syslog }}
+ clog to syslog = {{ use_syslog }}
+

=== modified file 'hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg'
--- hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg	2013-08-12 21:48:24 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg	2014-03-26 11:39:18 +0000
@@ -8,8 +8,8 @@
 
 defaults
     log global
-    mode http
-    option httplog
+    mode tcp
+    option tcplog
     option dontlognull
     retries 3
     timeout queue 1000
@@ -29,7 +29,6 @@
 {% for service, ports in service_ports.iteritems() -%}
 listen {{ service }} 0.0.0.0:{{ ports[0] }}
     balance roundrobin
-    option tcplog
     {% for unit, address in units.iteritems() -%}
     server {{ unit }} {{ address }}:{{ ports[1] }} check
     {% endfor %}

=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	2014-02-13 08:29:21 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2014-03-26 11:39:18 +0000
@@ -65,6 +65,9 @@
     ('1.10.0', 'havana'),
     ('1.9.1', 'havana'),
     ('1.9.0', 'havana'),
+    ('1.13.0', 'icehouse'),
+    ('1.12.0', 'icehouse'),
+    ('1.11.0', 'icehouse'),
 ])
 
 DEFAULT_LOOPBACK_SIZE = '5G'
@@ -420,19 +423,19 @@
     Resolves hostname for given IP, or returns the input
     if it is already a hostname.
     """
-    if not is_ip(address):
-        return address
-
-    try:
-        import dns.reversename
-    except ImportError:
-        apt_install('python-dnspython')
-        import dns.reversename
-
-    rev = dns.reversename.from_address(address)
-    result = ns_query(rev)
-    if not result:
-        return None
+    if is_ip(address):
+        try:
+            import dns.reversename
+        except ImportError:
+            apt_install('python-dnspython')
+            import dns.reversename
+
+        rev = dns.reversename.from_address(address)
+        result = ns_query(rev)
+        if not result:
+            return None
+    else:
+        result = address
 
     if fqdn:
         # strip trailing .

=== modified file 'hooks/charmhelpers/contrib/storage/linux/ceph.py'
--- hooks/charmhelpers/contrib/storage/linux/ceph.py	2013-11-06 03:50:44 +0000
+++ hooks/charmhelpers/contrib/storage/linux/ceph.py	2014-03-26 11:39:18 +0000
@@ -49,6 +49,9 @@
  auth supported = {auth}
  keyring = {keyring}
  mon host = {mon_hosts}
+ log to syslog = {use_syslog}
+ err to syslog = {use_syslog}
+ clog to syslog = {use_syslog}
 """
 
 
@@ -194,7 +197,7 @@
     return hosts
 
 
-def configure(service, key, auth):
+def configure(service, key, auth, use_syslog):
     ''' Perform basic configuration of Ceph '''
     create_keyring(service, key)
     create_key_file(service, key)
@@ -202,7 +205,8 @@
     with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
         ceph_conf.write(CEPH_CONF.format(auth=auth,
                                          keyring=_keyring_path(service),
-                                         mon_hosts=",".join(map(str, hosts))))
+                                         mon_hosts=",".join(map(str, hosts)),
+                                         use_syslog=use_syslog))
     modprobe('rbd')
 
 

=== modified file 'hooks/charmhelpers/contrib/storage/linux/utils.py'
--- hooks/charmhelpers/contrib/storage/linux/utils.py	2014-02-13 08:29:21 +0000
+++ hooks/charmhelpers/contrib/storage/linux/utils.py	2014-03-26 11:39:18 +0000
@@ -22,4 +22,5 @@
 
     :param block_device: str: Full path of block device to clean.
     '''
-    check_call(['sgdisk', '--zap-all', '--mbrtogpt', block_device])
+    check_call(['sgdisk', '--zap-all', '--clear',
+                '--mbrtogpt', block_device])

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2014-02-13 08:29:21 +0000
+++ hooks/charmhelpers/core/host.py	2014-03-26 11:39:18 +0000
@@ -194,7 +194,7 @@
         return None
 
 
-def restart_on_change(restart_map):
+def restart_on_change(restart_map, stopstart=False):
     """Restart services based on configuration files changing
 
     This function is used a decorator, for example
@@ -219,8 +219,14 @@
             for path in restart_map:
                 if checksums[path] != file_hash(path):
                     restarts += restart_map[path]
-            for service_name in list(OrderedDict.fromkeys(restarts)):
-                service('restart', service_name)
+            services_list = list(OrderedDict.fromkeys(restarts))
+            if not stopstart:
+                for service_name in services_list:
+                    service('restart', service_name)
+            else:
+                for action in ['stop', 'start']:
+                    for service_name in services_list:
+                        service(action, service_name)
         return wrapped_f
     return wrap
 

=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2014-02-13 08:29:21 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2014-03-26 11:39:18 +0000
@@ -97,6 +97,29 @@
         subprocess.call(cmd, env=env)
 
 
+def apt_upgrade(options=None, fatal=False, dist=False):
+    """Upgrade all packages"""
+    if options is None:
+        options = ['--option=Dpkg::Options::=--force-confold']
+
+    cmd = ['apt-get', '--assume-yes']
+    cmd.extend(options)
+    if dist:
+        cmd.append('dist-upgrade')
+    else:
+        cmd.append('upgrade')
+    log("Upgrading with options: {}".format(options))
+
+    env = os.environ.copy()
+    if 'DEBIAN_FRONTEND' not in env:
+        env['DEBIAN_FRONTEND'] = 'noninteractive'
+
+    if fatal:
+        subprocess.check_call(cmd, env=env)
+    else:
+        subprocess.call(cmd, env=env)
+
+
 def apt_update(fatal=False):
     """Update local apt cache"""
     cmd = ['apt-get', 'update']
@@ -135,6 +158,10 @@
 
 
 def add_source(source, key=None):
+    if source is None:
+        log('Source is not present. Skipping')
+        return
+
     if (source.startswith('ppa:') or
         source.startswith('http') or
         source.startswith('deb ') or

=== modified file 'hooks/charmhelpers/fetch/archiveurl.py'
--- hooks/charmhelpers/fetch/archiveurl.py	2013-09-23 13:23:51 +0000
+++ hooks/charmhelpers/fetch/archiveurl.py	2014-03-26 11:39:18 +0000
@@ -1,5 +1,7 @@
 import os
 import urllib2
+import urlparse
+
 from charmhelpers.fetch import (
     BaseFetchHandler,
     UnhandledSource
@@ -24,6 +26,19 @@
     def download(self, source, dest):
         # propogate all exceptions
         # URLError, OSError, etc
+        proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
+        if proto in ('http', 'https'):
+            auth, barehost = urllib2.splituser(netloc)
+            if auth is not None:
+                source = urlparse.urlunparse((proto, barehost, path, params, query, fragment))
+                username, password = urllib2.splitpasswd(auth)
+                passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+                # Realm is set to None in add_password to force the username and password
+                # to be used whatever the realm
+                passman.add_password(None, source, username, password)
+                authhandler = urllib2.HTTPBasicAuthHandler(passman)
+                opener = urllib2.build_opener(authhandler)
+                urllib2.install_opener(opener)
         response = urllib2.urlopen(source)
         try:
             with open(dest, 'w') as dest_file: