Merge lp:~james-page/charms/trusty/neutron-agents-midonet/trunk into lp:~celebdor/charms/trusty/neutron-agents-midonet/split

[James Page]

Unmerged revisions

51. By James Page on 2016-02-16

Misc updates

50. By James Page on 2016-02-16

Tweak charm sources

49. By James Page on 2016-02-16

Fully mock out unit tests from underlying os

48. By Antoni Segura Puimedon <email address hidden> on 2016-02-12

amulet: remove old tests and add check for nova-api-metadata

47. By Antoni Segura Puimedon <email address hidden> on 2016-02-11

Fix unit tests

46. By Antoni Segura Puimedon <email address hidden> on 2016-02-11

linting fix

45. By Antoni Segura Puimedon <email address hidden> on 2016-02-08

use the right midonet plugin for Liberty and above

=== modified file 'hooks/services.py'
--- hooks/services.py	2016-02-04 11:42:36 +0000
+++ hooks/services.py	2016-02-16 14:28:40 +0000
@@ -18,9 +18,9 @@
 from charmhelpers.contrib.openstack import utils
 from charmhelpers.core import hookenv
 from charmhelpers.core import host
-from charmhelpers.core import unitdata
 from charmhelpers.core.services import base
 from charmhelpers.core.services import helpers
+from charmhelpers.core import unitdata
 from midonet_helpers import puppet
 from midonet_helpers import relations as common_relations
 from midonet_helpers import repositories
@@ -86,6 +86,10 @@
     amqp = context.AMQPContext()
     network_service = MidonetNetworkServiceContext(rel_name='neutron_agents')
     midonet = common_relations.MidonetApiRelation()
+    if ost_release in ('juno', 'kilo'):
+        midonet_plugin = 'python-neutron-plugin-midonet'
+    else:
+        midonet_plugin = 'python-networking-midonet'
 
     manager = base.ServiceManager([
         {
@@ -102,7 +106,7 @@
                     source='dhcp_agent.ini',
                     target='/etc/neutron/dhcp_agent.ini'),
                 puppet.InstallPackagesCallback(
-                    ('python-neutron-plugin-midonet',)),
+                    (midonet_plugin,)),
                 OSTRenderer('/etc/neutron/neutron.conf',
                             [amqp]),
             ],
@@ -127,8 +131,10 @@
             ],
         },
     ])
+
     manager.manage()
     if (host.service_running('neutron-dhcp-agent') and
             host.service_running('neutron-metadata-agent') and
             host.service_running('nova-api-metadata')):
-        hookenv.status_set('active', 'All nova and neutron agents up and running')
+        hookenv.status_set('active',
+                           'All nova and neutron agents up and running')

=== modified file 'tests/00-setup' (properties changed: +x to -x)
=== added file 'tests/006-minimal-trusty-kilo-mem'
--- tests/006-minimal-trusty-kilo-mem	1970-01-01 00:00:00 +0000
+++ tests/006-minimal-trusty-kilo-mem	2016-02-16 14:28:40 +0000
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2015 Midokura SARL, All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+import basic_deployment
+
+if __name__ == '__main__':
+    if None in (os.getenv('MEM_USERNAME'), os.getenv('MEM_PWD')):
+        print('Missing MEM_USERNAME and MEM_PWD environment variables for '
+              'running Midokura Enterprise MidoNet tests. Skipping test')
+    else:
+        deployment = basic_deployment.AgentMinimumDeployment(
+            ubuntu_series='trusty',
+            openstack_origin='cloud:trusty-kilo',
+            midonet_origin='mem-1.9')
+        deployment.run_tests()

=== modified file 'tests/010-basic-trusty-juno'
--- tests/010-basic-trusty-juno	2016-01-27 17:01:34 +0000
+++ tests/010-basic-trusty-juno	2016-02-16 14:28:40 +0000
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright (c) 2015 Midokura SARL, All Rights Reserved.
 #
@@ -13,13 +13,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-
-from basic_deployment import MidonetBasicDeployment
+import basic_deployment
 
 if __name__ == '__main__':
-    deployment = MidonetBasicDeployment(
+    deployment = basic_deployment.AgentMinimumDeployment(
         ubuntu_series='trusty',
         openstack_origin='cloud:trusty-juno',
         midonet_origin='midonet-2015.06')
-    deployment.run_tests()

=== modified file 'tests/011-basic-trusty-kilo'
--- tests/011-basic-trusty-kilo	2016-01-27 17:01:34 +0000
+++ tests/011-basic-trusty-kilo	2016-02-16 14:28:40 +0000
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright (c) 2015 Midokura SARL, All Rights Reserved.
 #
@@ -13,13 +13,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-
-from basic_deployment import MidonetBasicDeployment
+import basic_deployment
 
 if __name__ == '__main__':
-    deployment = MidonetBasicDeployment(
+    deployment = basic_deployment.AgentMinimumDeployment(
         ubuntu_series='trusty',
         openstack_origin='cloud:trusty-kilo',
         midonet_origin='midonet-2015.06')
-    deployment.run_tests()

=== added file 'tests/012-basic-trusty-liberty'
--- tests/012-basic-trusty-liberty	1970-01-01 00:00:00 +0000
+++ tests/012-basic-trusty-liberty	2016-02-16 14:28:40 +0000
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2015 Midokura SARL, All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import basic_deployment
+
+if __name__ == '__main__':
+    deployment = basic_deployment.AgentMinimumDeployment(
+        ubuntu_series='trusty',
+        openstack_origin='cloud:trusty-liberty',
+        midonet_origin='midonet-2015.06')

=== modified file 'tests/basic_deployment.py'
--- tests/basic_deployment.py	2016-02-04 11:42:36 +0000
+++ tests/basic_deployment.py	2016-02-16 14:28:40 +0000
@@ -31,7 +31,7 @@
 SETUP_TIMEOUT = 9600
 
 
-class MidonetBasicDeployment():
+class AgentMinimumDeployment():
     def __init__(self, ubuntu_series, openstack_origin=None,
                  midonet_origin=None):
         self.os_release = openstack_origin
@@ -43,6 +43,16 @@
         self.d = amulet.Deployment(series=self.series)
         self.d.charm_name = 'neutron-agents-midonet'
 
+        # Sentries
+        self.zookeeper_sentry = None
+        self.keystone_sentry = None
+        self.cassandra_sentry = None
+        self.nova_compute_sentry = None
+        self.mn_api_sentry = None
+        self.agents_mn_sentry = None
+        self.keystone_api_relation = None
+        self.zookeeper_api_relation = None
+
         self._add_services()
         self._add_relations()
         self._configure_services()
@@ -50,37 +60,28 @@
         self._initialize_tests()
 
     def _add_services(self):
-        self.d.add('mysql', charm='cs:trusty/mysql', units=1,
+        self.d.add('mysql', charm='mysql', units=1,
                    series=self.series)
-        self.d.add('rabbitmq-server', charm='cs:trusty/rabbitmq-server',
+        self.d.add('keystone', charm='keystone', units=1, series=self.series)
+        self.d.add('cassandra', charm='cassandra',
                    units=1, series=self.series)
-        self.d.add('cassandra', charm='cassandra', units=1,
-                   branch='cs:trusty/cassandra', series=self.series)
-        self.d.add('keystone', charm='keystone', units=1, series=self.series,
-                branch='cs:trusty/keystone')
-        self.d.add('compute', charm='nova-compute', units=1,
-                   branch='cs:trusty/nova-compute', series=self.series,
-                   constraints={'mem': '8G'})
-        self.d.add('neutron', charm='neutron-api', units=1,
-                   branch='lp:~celebdor/charms/trusty/neutron-api/liberty',
-                   series=self.series)
-        self.d.add('nova', charm='nova-cloud-controller',
-               units=1, series=self.series)
-        self.d.add('glance', charm='glance',
-               branch='cs:trusty/glance',
-               units=1, series=self.series)
-        self.d.add('cinder', charm='cinder',
-               branch='cs:trusty/glance',
-               units=1, series=self.series)
-        self.d.add('zookeeper', charm='cs:trusty/zookeeper', units=1,
-                   series=self.series)
-        self.d.add('midonet-api', charm='midonet-api', units=1,
+        self.d.add('zookeeper', charm='zookeeper', units=1,
+                   series=self.series)
+        self.d.add('rabbitmq', charm='rabbitmq-server', units=1,
+                   series=self.series)
+        self.d.add('nova', charm='nova-cloud-controller', units=1,
+                   series=self.series)
+        self.d.add('neutron',
+                   charm='cs:~openstack-charmers-next/trusty/neutron-api',
+                   units=1,
+                   series=self.series)
+        self.d.add('midonet-api', charm='midonet-api',
                    branch='lp:~celebdor/charms/trusty/midonet-api/trunk',
                    series=self.series)
-        self.d.add('midonet-agent', charm='midonet-agent', units=1,
+        self.d.add('midonet-agent', charm='midonet-agent',
                    branch='lp:~celebdor/charms/trusty/midonet-agent/trunk',
                    series=self.series)
-        self.d.add('neutron-agents-midonet', constraints={'mem': '4G'})
+        self.d.add('neutron-agents-midonet', constraints={'mem': "4G"})
 
     def _add_relations(self):
         # Keystone
@@ -89,65 +90,53 @@
         # Nova
         self.d.relate('mysql:shared-db', 'nova:shared-db')
         self.d.relate('keystone:identity-service', 'nova:identity-service')
-        self.d.relate('rabbitmq-server:amqp', 'nova:amqp')
+        self.d.relate('rabbitmq:amqp', 'nova:amqp')
         self.d.relate('neutron-agents-midonet:neutron_agents',
                       'nova:quantum-network-service')
         self.d.relate('neutron:neutron-api', 'nova:neutron-api')
-        self.d.relate('compute:cloud-compute', 'nova:cloud-compute')
-        self.d.relate('glance:image-service', 'nova:image-service')
-        self.d.relate('cinder:cinder-volume-service',
-                      'nova:cinder-volume-service')
-
-        # Cinder
-        self.d.relate('keystone:identity-service', 'cinder:identity-service')
-        self.d.relate('mysql:shared-db', 'cinder:shared-db')
-        self.d.relate('rabbitmq-server:amqp', 'cinder:amqp')
-        self.d.relate('glance:image-service', 'cinder:image-service')
-
-        # Glance
-        self.d.relate('keystone:identity-service', 'glance:identity-service')
-        self.d.relate('mysql:shared-db', 'glance:shared-db')
-        self.d.relate('rabbitmq-server:amqp', 'glance:amqp')
 
         # Neutron
         self.d.relate('mysql:shared-db', 'neutron:shared-db')
         self.d.relate('keystone:identity-service', 'neutron:identity-service')
-        self.d.relate('rabbitmq-server:amqp', 'neutron:amqp')
+        self.d.relate('rabbitmq:amqp', 'neutron:amqp')
         self.d.relate('neutron-agents-midonet:neutron-plugin-api',
                       'neutron:neutron-plugin-api')
+        self.d.relate('midonet-api:midonet-api', 'neutron:midonet')
 
         # MidoNet
         self.d.relate('keystone:identity-service', 'midonet-api:keystone')
         self.d.relate('zookeeper:zookeeper', 'midonet-api:zookeeper')
         self.d.relate('midonet-agent:host', 'midonet-api:host')
-        self.d.relate('neutron:midonet', 'midonet-api:midonet-api')
-        self.d.relate('rabbitmq-server:amqp', 'neutron-gents-midonet:amqp')
+        self.d.relate('rabbitmq:amqp', 'neutron-agents-midonet:amqp')
         self.d.relate('midonet-agent:host', 'neutron-agents-midonet:agent')
+        self.d.relate('midonet-api:midonet-api',
+                      'neutron-agents-midonet:midonet')
         self.d.relate('cassandra:database', 'midonet-agent:cassandra')
         self.d.relate('zookeeper:zookeeper', 'midonet-agent:zookeeper')
-        self.d.relate('midonet-api:midonet-api',
-                      'neutron-agents-midonet:midonet')
-
-        # Compute
-        self.d.relate('rabbitmq-server:amqp', 'compute:amqp')
-        self.d.relate('midonet-agent:neutron-plugin', 'compute:neutron-plugin')
-        self.d.relate('glance:image-service', 'compute:image-service')
 
     def _configure_services(self):
-        repo_config = {'midonet-origin': self.midonet_origin,
-                      'openstack-origin': self.os_release}
-        if self.mem:
-            repo_config.update(self.mem_credentials)
-
         self.d.configure('keystone', {
-                         'admin-user': 'admin',
-                         'admin-password': 'openstack',
-                         'admin-token': 'osttesting',
-                         'enable-pki': 'false',
-                         'openstack-origin': self.os_release})
-
-        self.d.configure('mysql', {
-                         'max-connections': 500})
+            'admin-user': 'admin',
+            'admin-password': 'openstack',
+            'admin-token': 'osttesting',
+            'enable-pki': 'false',
+            'openstack-origin': self.os_release})
+
+        self.d.configure('nova', {
+            'network-manager': 'Neutron',
+            'openstack-origin': self.os_release})
+
+        neutron_conf = {
+            'l2-population': 'false',
+            'neutron-security-groups': 'true',
+            'neutron-plugin': 'midonet',
+            'neutron-external-network': 'Public_Network',
+            'midonet-origin': self.midonet_origin,
+            'openstack-origin': self.os_release}
+        if self.mem:
+            neutron_conf.update(self.mem_credentials)
+
+        self.d.configure('neutron', neutron_conf)
 
         self.d.configure('cassandra', {
             'authenticator': 'AllowAllAuthenticator',
@@ -157,33 +146,16 @@
             'deb http://debian.datastax.com/community 2.0 main',
             'extra_packages': 'dsc20'})
 
-        self.d.configure('compute', {
-                         'openstack-origin': self.os_release,
-                         'virt-type': 'qemu',
-                         'flat-interface': 'eth0',
-                         'manage-neutron-plugin-legacy-mode': 'false'})
-
-        self.d.configure('nova', {
-                         'openstack-origin': self.os_release,
-                         'network-manager': 'Neutron',
-                         'shared-secret': 'secret'})
-
-        self.d.configure('neutron', {
-                         'neutron-plugin': 'midonet',
-                         'neutron-security-groups': 'True',
-                         'neutron-external-network': 'Public_Network',
-                         'l2-population': 'False',
-                         'openstack-origin': self.os_release,
-                         'midonet-origin': self.midonet_origin})
-
-
-        self.d.configure('midonet-api', repo_config)
-
-        neutron_agts_midonet_config = {'shared-secret': 'secret'}
-        neutron_agts_midonet_config.update(repo_config)
-        self.d.configure('neutron-agents-midonet', neutron_agts_midonet_config)
-
-        self.d.configure('midonet-agent', repo_config)
+        self.d.configure('mysql', {'max-connections': 500})
+
+        api_config = {'midonet-origin': self.midonet_origin,
+                      'openstack-origin': self.os_release}
+        if self.mem:
+            api_config.update(self.mem_credentials)
+        self.d.configure('midonet-api', api_config)
+        self.d.configure('midonet-agent', api_config)
+        self.d.configure('neutron-agents-midonet', api_config)
+
 
     def _deploy(self):
         try:
@@ -196,8 +168,6 @@
     def _initialize_tests(self):
         self.zookeeper_sentry = self.d.sentry.unit['zookeeper/0']
         self.keystone_sentry = self.d.sentry.unit['keystone/0']
-        self.cassandra_sentry = self.d.sentry.unit['cassandra/0']
-        self.nova_compute_sentry = self.d.sentry.unit['compute/0']
         self.mn_api_sentry = self.d.sentry.unit['midonet-api/0']
         self.mn_agent_sentry = self.d.sentry.unit['midonet-agent/0']
         self.agents_mn_sentry = self.d.sentry.unit['neutron-agents-midonet/0']
@@ -206,12 +176,6 @@
             'identity-service', 'midonet-api:keystone')
         self.zookeeper_api_relation = self.zookeeper_sentry.relation(
             'zookeeper', 'midonet-api:zookeeper')
-        self.zk_host_relation = self.zookeeper_sentry.relation(
-            'zookeeper', 'midonet-agent:zookeeper')
-        self.cs_host_relation = self.cassandra_sentry.relation(
-            'database', 'midonet-agent:cassandra')
-        self.api_host_relation = self.mn_api_sentry.relation(
-            'host', 'midonet-agent:host')
 
     def run_tests(self):
         for test in dir(self):
@@ -249,3 +213,10 @@
             amulet.raise_status(amulet.FAIL,
                                 msg="neutron-metadata-agent not running: %s" %
                                 meta_agent_out)
+        u.log.debug('Checking nova-api-metadata...')
+        meta_agent_out, _ = self.agents_mn_sentry.run(
+            'service nova-api-metadata status')
+        if 'running' not in meta_agent_out:
+            amulet.raise_status(amulet.FAIL,
+                                msg="nova-api-metadata not running: %s" %
+                                meta_agent_out)

=== modified file 'unit_tests/test_context.py'
--- unit_tests/test_context.py	2016-01-28 00:53:03 +0000
+++ unit_tests/test_context.py	2016-02-16 14:28:40 +0000
@@ -18,7 +18,7 @@
 import shutil
 import tempfile
 
-import mock
+import services
 
 from .midonet_helpers import test
 
@@ -55,38 +55,20 @@
                      '10.10.10.11']
         test.CharmTestCase.__init__(self, *args, **kwargs)
 
-    @mock.patch('charmhelpers.fetch.apt_install')
-    def setUp(self, apt_install):
-
-        import services
-
+    def setUp(self):
         self.relations = test.FakeRelation(self._full_relation_data())
-        self.services = services
         test.CharmTestCase.setUp(self, services.hookenv, HOOKENV_PATCHES)
 
         def lsb_release():
-            ubuntu_trusty = {
+            return {
                 'DISTRIB_ID': 'Ubuntu',
                 'DISTRIB_RELEASE': '14.04',
                 'DISTRIB_CODENAME': 'trusty',
                 'DISTRIB_DESCRIPTION': "Ubuntu 14.04.3 LTS",
             }
 
-            try:
-                data = services.host.lsb_release()
-            except Exception:
-                return ubuntu_trusty
-            if data.get('DISTRIB_ID') != 'Ubuntu':
-                return ubuntu_trusty
-            return data
-
         def get_os_codename_package(pkg, fatal=True):
-            version = 'liberty'
-            try:
-                ret_vers = services.utils.get_os_codename_package(pkg, fatal)
-            except Exception:
-                return version
-            return ret_vers
+            return 'liberty'
 
         self.patch_all(services.utils,
                        ('get_os_codename_package',
@@ -94,38 +76,21 @@
         self.lsb_release.side_effect = lsb_release
         self.get_os_codename_package.side_effect = get_os_codename_package
 
+        self.patch_all(services.base, ('service_restart', 'service_stop'))
+        self.patch_all(services.helpers, ('render_template',))
         self.patch_all(services.host, ('service_running',))
-        self.service_running.return_value = True
-
-        _templating = mock.patch.object(services.helpers, 'render_template')
-        setattr(self, 'render_template', _templating.start())
-        self.addCleanup(_templating.stop)
-
-        _check_call = mock.patch.object(services.hookenv.subprocess,
-                                        'check_call')
-        setattr(self, 'check_call', _check_call.start())
-        self.addCleanup(_check_call.stop)
-
-        _service_restart = mock.patch.object(services.base, 'service_restart')
-        setattr(self, 'service_restart', _service_restart.start())
-        self.addCleanup(_service_restart.stop)
-
-        _service_stop = mock.patch.object(services.base, 'service_stop')
-        setattr(self, 'service_stop', _service_stop.start())
-        self.addCleanup(_service_stop.stop)
-
-        _relation_helpers = mock.patch.object(
-            services.relations.helpers, 'hookenv', spec=[
-                'relation_ids',
-                'related_units',
-                'relation_get'])
-        setattr(self, 'relation_helpers', _relation_helpers.start())
-        self.addCleanup(_relation_helpers.stop)
+        self.patch_all(services.puppet, ('_apply',))
+        self.patch_all(services.templating.OSConfigRenderer, ('write_all',))
 
         self.test_config.config['openstack-origin'] = 'cloud:trusty-liberty'
         self.config.side_effect = self.test_config.get
-        self.relation_get.side_effect = self.test_relation.get
+        self.related_units.side_effect = self.relations.related_units
+        self.relation_get.side_effect = self.relations.get
+        self.relation_ids.side_effect = self.relations.relation_ids
 
+        services.context.related_units = self.related_units
+        services.context.relation_get = self.relation_get
+        services.context.relation_ids = self.relation_ids
         services.context.config = self.config
 
         self.charm_dir.return_value = tempfile.mkdtemp()
@@ -135,21 +100,7 @@
         test.CharmTestCase.tearDown(self)
 
     def test_config_change(self):
-
-        self.data = self._full_relation_data()
-
-        self.relation_helpers.relation_ids.side_effect = (
-            self.relations.relation_ids)
-
-        import services
-        services.context.relation_ids = self.relation_helpers.relation_ids
-
-        self.relation_helpers.related_units.side_effect = (
-            self.relations.related_units)
-        self.relation_helpers.relation_get.side_effect = (
-            self.relations.get)
-
-        self.services.manage()
+        services.manage()
 
         self.assertTrue(self.render_template.called)
         self.assertTrue(self.service_restart.called)
@@ -181,4 +132,11 @@
                     'port': 8080,
                 },
             },
+            'amqp:2': {
+                'rabbitmq/0': {
+                    'clustered': False,
+                    'private-address': '10.0.30.14',
+                    'password': 'foopass',
+                },
+            },
         }