Merge lp:~termie/nova/remove_ioloop into lp:~hudson-openstack/nova/trunk

[termie]

Removes most traces of tornado from the codebase, leaving only the things that gundlach's branch will remove later.

The tests were passing even while the code was wrong during part of this process so I updated the TrialTestCase tests to emulate the behavior that BaseTestCase was providing of automatically turning test cases that return generators into defer.inlineCallbacks.

[Joshua McKenty (joshua-mckenty)]

lgtm, with a bit of either hesitation or confusion about how you're storing originalAttach as an instance property (in _monkeyPatchAttach). Isn't there some risk that rpc.Consumer.attach_to_twisted will be called independently from some of the code during the test runs? (I suppose it's very unlikely...)

review: Approve

=== modified file 'nova/rpc.py'
--- nova/rpc.py	2010-08-18 15:44:24 +0000
+++ nova/rpc.py	2010-09-08 20:59:25 +0000
@@ -81,21 +81,6 @@
         self.failed_connection = False
         super(Consumer, self).__init__(*args, **kwargs)
 
-    # TODO(termie): it would be nice to give these some way of automatically
-    #               cleaning up after themselves
-    def attach_to_tornado(self, io_inst=None):
-        """Attach a callback to tornado that fires 10 times a second"""
-        from tornado import ioloop
-        if io_inst is None:
-            io_inst = ioloop.IOLoop.instance()
-
-        injected = ioloop.PeriodicCallback(
-            lambda: self.fetch(enable_callbacks=True), 100, io_loop=io_inst)
-        injected.start()
-        return injected
-
-    attachToTornado = attach_to_tornado
-
     def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False):
         """Wraps the parent fetch with some logic for failed connections"""
         # TODO(vish): the logic for failed connections and logging should be
@@ -123,6 +108,7 @@
         """Attach a callback to twisted that fires 10 times a second"""
         loop = task.LoopingCall(self.fetch, enable_callbacks=True)
         loop.start(interval=0.1)
+        return loop
 
 
 class Publisher(messaging.Publisher):
@@ -264,7 +250,6 @@
     msg_id = uuid.uuid4().hex
     msg.update({'_msg_id': msg_id})
     LOG.debug("MSG_ID is %s" % (msg_id))
-
     conn = Connection.instance()
     d = defer.Deferred()
     consumer = DirectConsumer(connection=conn, msg_id=msg_id)
@@ -278,7 +263,7 @@
             return d.callback(data['result'])
 
     consumer.register_callback(deferred_receive)
-    injected = consumer.attach_to_tornado()
+    injected = consumer.attach_to_twisted()
 
     # clean up after the injected listened and return x
     d.addCallback(lambda x: injected.stop() and x or x)

=== modified file 'nova/test.py'
--- nova/test.py	2010-08-18 15:44:24 +0000
+++ nova/test.py	2010-09-08 20:59:25 +0000
@@ -33,6 +33,7 @@
 
 from nova import fakerabbit
 from nova import flags
+from nova import rpc
 
 
 FLAGS = flags.FLAGS
@@ -62,19 +63,29 @@
         self.mox = mox.Mox()
         self.stubs = stubout.StubOutForTesting()
         self.flag_overrides = {}
+        self.injected = []
+        self._monkeyPatchAttach()
 
     def tearDown(self): # pylint: disable-msg=C0103
         """Runs after each test method to finalize/tear down test environment"""
-        super(TrialTestCase, self).tearDown()
         self.reset_flags()
         self.mox.UnsetStubs()
         self.stubs.UnsetAll()
         self.stubs.SmartUnsetAll()
         self.mox.VerifyAll()
+        
+        rpc.Consumer.attach_to_twisted = self.originalAttach
+        for x in self.injected:
+            try:
+                x.stop()
+            except AssertionError:
+                pass
 
         if FLAGS.fake_rabbit:
             fakerabbit.reset_all()
 
+        super(TrialTestCase, self).tearDown()
+
     def flags(self, **kw):
         """Override flag variables for a test"""
         for k, v in kw.iteritems():
@@ -90,16 +101,51 @@
         for k, v in self.flag_overrides.iteritems():
             setattr(FLAGS, k, v)
 
+    def run(self, result=None):
+        test_method = getattr(self, self._testMethodName)
+        setattr(self,
+                self._testMethodName,
+                self._maybeInlineCallbacks(test_method, result))
+        rv = super(TrialTestCase, self).run(result)
+        setattr(self, self._testMethodName, test_method)
+        return rv
+
+    def _maybeInlineCallbacks(self, func, result):
+        def _wrapped():
+            g = func()
+            if isinstance(g, defer.Deferred):
+                return g
+            if not hasattr(g, 'send'):
+                return defer.succeed(g)
+
+            inlined = defer.inlineCallbacks(func)
+            d = inlined()
+            return d
+        _wrapped.func_name = func.func_name
+        return _wrapped
+
+    def _monkeyPatchAttach(self):
+        self.originalAttach = rpc.Consumer.attach_to_twisted
+        def _wrapped(innerSelf):
+            rv = self.originalAttach(innerSelf)
+            self.injected.append(rv)
+            return rv
+
+        _wrapped.func_name = self.originalAttach.func_name
+        rpc.Consumer.attach_to_twisted = _wrapped
+
 
 class BaseTestCase(TrialTestCase):
     # TODO(jaypipes): Can this be moved into the TrialTestCase class?
-    """Base test case class for all unit tests."""
+    """Base test case class for all unit tests.
+    
+    DEPRECATED: This is being removed once Tornado is gone, use TrialTestCase.
+    """
     def setUp(self): # pylint: disable-msg=C0103
         """Run before each test method to initialize test environment"""
         super(BaseTestCase, self).setUp()
         # TODO(termie): we could possibly keep a more global registry of
         #               the injected listeners... this is fine for now though
-        self.injected = []
         self.ioloop = ioloop.IOLoop.instance()
 
         self._waiting = None
@@ -109,8 +155,6 @@
     def tearDown(self):# pylint: disable-msg=C0103
         """Runs after each test method to finalize/tear down test environment"""
         super(BaseTestCase, self).tearDown()
-        for x in self.injected:
-            x.stop()
         if FLAGS.fake_rabbit:
             fakerabbit.reset_all()
 

=== modified file 'nova/tests/access_unittest.py'
--- nova/tests/access_unittest.py	2010-07-28 23:11:02 +0000
+++ nova/tests/access_unittest.py	2010-09-08 20:59:25 +0000
@@ -30,7 +30,7 @@
 class Context(object):
     pass
 
-class AccessTestCase(test.BaseTestCase):
+class AccessTestCase(test.TrialTestCase):
     def setUp(self):
         super(AccessTestCase, self).setUp()
         FLAGS.connection_type = 'fake'

=== modified file 'nova/tests/auth_unittest.py'
--- nova/tests/auth_unittest.py	2010-08-11 01:04:23 +0000
+++ nova/tests/auth_unittest.py	2010-09-08 20:59:25 +0000
@@ -31,7 +31,7 @@
 FLAGS = flags.FLAGS
 
 
-class AuthTestCase(test.BaseTestCase):
+class AuthTestCase(test.TrialTestCase):
     flush_db = False
     def setUp(self):
         super(AuthTestCase, self).setUp()

=== modified file 'nova/tests/cloud_unittest.py'
--- nova/tests/cloud_unittest.py	2010-08-19 01:25:16 +0000
+++ nova/tests/cloud_unittest.py	2010-09-08 20:59:25 +0000
@@ -19,7 +19,6 @@
 import logging
 import StringIO
 import time
-from tornado import ioloop
 from twisted.internet import defer
 import unittest
 from xml.etree import ElementTree
@@ -36,7 +35,7 @@
 FLAGS = flags.FLAGS
 
 
-class CloudTestCase(test.BaseTestCase):
+class CloudTestCase(test.TrialTestCase):
     def setUp(self):
         super(CloudTestCase, self).setUp()
         self.flags(connection_type='fake',
@@ -51,18 +50,21 @@
         # set up a service
         self.compute = service.ComputeService()
         self.compute_consumer = rpc.AdapterConsumer(connection=self.conn,
-                                                     topic=FLAGS.compute_topic,
-                                                     proxy=self.compute)
-        self.injected.append(self.compute_consumer.attach_to_tornado(self.ioloop))
+                                                    topic=FLAGS.compute_topic,
+                                                    proxy=self.compute)
+        self.compute_consumer.attach_to_twisted()
 
         try:
             manager.AuthManager().create_user('admin', 'admin', 'admin')
         except: pass
         admin = manager.AuthManager().get_user('admin')
         project = manager.AuthManager().create_project('proj', 'admin', 'proj')
-        self.context = api.APIRequestContext(handler=None,project=project,user=admin)
+        self.context = api.APIRequestContext(handler=None,
+                                             project=project,
+                                             user=admin)
 
     def tearDown(self):
+        super(CloudTestCase, self).tearDown()
         manager.AuthManager().delete_project('proj')
         manager.AuthManager().delete_user('admin')
 

=== modified file 'nova/tests/objectstore_unittest.py'
--- nova/tests/objectstore_unittest.py	2010-08-18 15:44:24 +0000
+++ nova/tests/objectstore_unittest.py	2010-09-08 20:59:25 +0000
@@ -53,7 +53,7 @@
 os.makedirs(os.path.join(OSS_TEMPDIR, 'buckets'))
 
 
-class ObjectStoreTestCase(test.BaseTestCase):
+class ObjectStoreTestCase(test.TrialTestCase):
     """Test objectstore API directly."""
 
     def setUp(self): # pylint: disable-msg=C0103

=== modified file 'nova/tests/rpc_unittest.py'
--- nova/tests/rpc_unittest.py	2010-08-18 15:44:24 +0000
+++ nova/tests/rpc_unittest.py	2010-09-08 20:59:25 +0000
@@ -30,7 +30,7 @@
 FLAGS = flags.FLAGS
 
 
-class RpcTestCase(test.BaseTestCase):
+class RpcTestCase(test.TrialTestCase):
     """Test cases for rpc"""
     def setUp(self):  # pylint: disable-msg=C0103
         super(RpcTestCase, self).setUp()
@@ -39,8 +39,7 @@
         self.consumer = rpc.AdapterConsumer(connection=self.conn,
                                             topic='test',
                                             proxy=self.receiver)
-
-        self.injected.append(self.consumer.attach_to_tornado(self.ioloop))
+        self.consumer.attach_to_twisted()
 
     def test_call_succeed(self):
         """Get a value through rpc call"""