Merge lp:~cjwatson/lazr.jobrunner/py3 into lp:lazr.jobrunner

[Colin Watson]

Add Python 3 support (requires celery >= 4.3).

[Adam Collard (adam-collard)]

review: Approve

=== modified file 'NEWS.txt'
--- NEWS.txt	2019-03-08 18:24:33 +0000
+++ NEWS.txt	2019-05-02 14:16:27 +0000
@@ -1,6 +1,10 @@
 News
 ====
 
+UNRELEASED
+----------
+* Add Python 3 support (requires celery >= 4.3).
+
 0.14
 ----
 * Add tox testing support.
 
=== modified file 'setup.py'
--- setup.py	2019-03-08 18:24:33 +0000
+++ setup.py	2019-05-02 14:16:27 +0000
@@ -28,11 +28,13 @@
     # List your project dependencies here.
     # For more details, see:
     # http://packages.python.org/distribute/setuptools.html#declaring-dependencies
-    'celery>=3.0,<5.0',
+    'celery>=3.0,<5.0; python_version < "3"',
+    'celery>=4.3,<5.0; python_version >= "3"',
 ]
 
 tests_require = [
     'oops>=0.0.11',
+    'six',
     'unittest2',
     'zope.testing',
     ]
@@ -51,8 +53,8 @@
     url='https://launchpad.net/lazr.jobrunner',
     license='GPL v3',
     packages=find_packages('src'),
-    package_dir = {'': 'src'},
-    namespace_packages = ['lazr'],
+    package_dir={'': 'src'},
+    namespace_packages=['lazr'],
     include_package_data=True,
     zip_safe=False,
     install_requires=install_requires,
 
=== modified file 'src/lazr/__init__.py'
--- src/lazr/__init__.py	2012-02-23 10:32:17 +0000
+++ src/lazr/__init__.py	2019-05-02 14:16:27 +0000
@@ -1,1 +1,23 @@
-__import__('pkg_resources').declare_namespace(__name__)
+# Copyright 2019 Canonical Ltd.  All rights reserved.
+#
+# This file is part of lazr.jobrunner.
+#
+# lazr.jobrunner is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# lazr.jobrunner is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+# License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with lazr.jobrunner.  If not, see <http://www.gnu.org/licenses/>.
+
+# this is a namespace package
+try:
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+    __path__ = pkgutil.extend_path(__path__, __name__)
 
=== modified file 'src/lazr/jobrunner/bin/clear_queues.py'
--- src/lazr/jobrunner/bin/clear_queues.py	2019-03-05 17:06:13 +0000
+++ src/lazr/jobrunner/bin/clear_queues.py	2019-05-02 14:16:27 +0000
@@ -17,6 +17,8 @@
 
 """Inspect Celery result queues."""
 
+from __future__ import absolute_import, print_function
+
 __metaclass__ = type
 
 from argparse import ArgumentParser
@@ -28,9 +30,9 @@
 
 
 def show_queue_data(body, message):
-    print '%s: %s' % (
+    print('%s: %s' % (
         message.delivery_info['routing_key'],
-        json.dumps(body, sort_keys=True))
+        json.dumps(body, sort_keys=True)))
 
 
 def clear_queues(args):
@@ -56,7 +58,7 @@
                 RunJob.app, [queue], callbacks=[show_queue_data],
                 retain=True, passive_queues=True)
         except amqp.exceptions.NotFound as exc:
-            print >>sys.stderr, exc.reply_text
+            print(exc.reply_text, file=sys.stderr)
 
 
 def main():
 
=== modified file 'src/lazr/jobrunner/jobrunner.py'
--- src/lazr/jobrunner/jobrunner.py	2012-10-24 17:57:42 +0000
+++ src/lazr/jobrunner/jobrunner.py	2019-05-02 14:16:27 +0000
@@ -160,7 +160,7 @@
         try:
             try:
                 job.run()
-            except self.retryErrorTypes(job), e:
+            except self.retryErrorTypes(job) as e:
                 if job.attempt_count > job.max_retries:
                     raise
                 self.logger.warning(
@@ -192,7 +192,7 @@
             try:
                 try:
                     self.runJob(job, fallback)
-                except self.userErrorTypes(job), e:
+                except self.userErrorTypes(job) as e:
                     self.logger.info(
                         '%s failed with user error %r.'
                         % (self.job_str(job), e))
@@ -200,7 +200,7 @@
                 except Exception:
                     info = sys.exc_info()
                     return self._doOops(job, info)
-            except Exception, e:
+            except Exception as e:
                 # This only happens if _doOops() fails.
                 self.logger.exception("Failure in _doOops: %s" % e)
                 info = sys.exc_info()
 
=== modified file 'src/lazr/jobrunner/tests/config1.py'
--- src/lazr/jobrunner/tests/config1.py	2012-07-09 10:58:00 +0000
+++ src/lazr/jobrunner/tests/config1.py	2019-05-02 14:16:27 +0000
@@ -1,4 +1,4 @@
-from simple_config import *
+from lazr.jobrunner.tests.simple_config import *
 import os
 import oops
 CELERY_ANNOTATIONS = {
 
=== modified file 'src/lazr/jobrunner/tests/config_do_not_create_missing_queues.py'
--- src/lazr/jobrunner/tests/config_do_not_create_missing_queues.py	2012-07-09 10:58:00 +0000
+++ src/lazr/jobrunner/tests/config_do_not_create_missing_queues.py	2019-05-02 14:16:27 +0000
@@ -1,2 +1,2 @@
-from simple_config import *
+from lazr.jobrunner.tests.simple_config import *
 CELERY_CREATE_MISSING_QUEUES = False
 
=== modified file 'src/lazr/jobrunner/tests/test_celerytask.py'
--- src/lazr/jobrunner/tests/test_celerytask.py	2019-03-06 12:17:21 +0000
+++ src/lazr/jobrunner/tests/test_celerytask.py	2019-05-02 14:16:27 +0000
@@ -31,7 +31,12 @@
 import tempfile
 from time import sleep
 from unittest import TestCase
-import urllib2
+
+from six.moves.urllib.error import URLError
+from six.moves.urllib.request import (
+    build_opener,
+    HTTPBasicAuthHandler,
+    )
 
 os.environ.setdefault('CELERY_CONFIG_MODULE', 'lazr.jobrunner.celeryconfig')
 
@@ -143,7 +148,7 @@
         def ensure_dir(path):
             try:
                 os.mkdir(path)
-            except OSError, e:
+            except OSError as e:
                 if e.errno != errno.EEXIST:
                     raise
         ensure_dir(self.job_root)
@@ -176,7 +181,7 @@
         try:
             with self._job_output_file(job.job_id, 'r') as job_output_file:
                     return job_output_file.read()
-        except IOError, e:
+        except IOError as e:
             if e.errno == errno.ENOENT:
                 return None
             raise
@@ -269,7 +274,7 @@
         # There is exactly one OOPS report.
         self.assertEqual(1, len(oops_repository.oopses))
         # This OOPS describes a MemoryError.
-        oops_report = oops_repository.oopses.values()[0]
+        oops_report = list(oops_repository.oopses.values())[0]
         self.assertEqual('MemoryError', oops_report['type'])
 
     def test_acquires_lease(self):
@@ -288,11 +293,11 @@
 class TestCeleryD(TestCase):
 
     def getQueueInfo(self):
-        auth_handler = urllib2.HTTPBasicAuthHandler()
+        auth_handler = HTTPBasicAuthHandler()
         auth_handler.add_password(
             realm='Management: Web UI', user='guest', passwd='guest',
             uri='http://localhost:55672/api/queues')
-        opener = urllib2.build_opener(auth_handler)
+        opener = build_opener(auth_handler)
         info = opener.open('http://localhost:55672/api/queues').read()
         info = json.loads(info)
         # info is a list of dictionaries with details about the queues.
@@ -305,7 +310,7 @@
         super(TestCeleryD, self).setUp()
         try:
             self.queue_status_during_setup = self.getQueueInfo()
-        except urllib2.URLError:
+        except URLError:
             # The rabbitmq-management package is currently broken
             # on Precise, so the RabbitMQ management interface may
             # not be available.
@@ -314,7 +319,7 @@
     def tearDown(self):
         try:
             current_queue_status = self.getQueueInfo()
-        except urllib2.URLError:
+        except URLError:
             # See setUp()
             return
         bad_queues = []
@@ -390,9 +395,10 @@
             err = proc.stderr.read()
             self.assertEqual(JobStatus.FAILED, job.status)
             self.assertIs(None, job.job_source.get_output(job))
-            self.assertIn(
-                "OOPS while executing job 10: [] Exception(u'Catch me if you"
-                " can!',)", err)
+            expected_message = (
+                "OOPS while executing job 10: [] Exception(%r,)" %
+                u'Catch me if you can!').encode('UTF-8')
+            self.assertIn(expected_message, err)
 
     def test_timeout_long(self):
         """Raises exception when a job exceeds the configured time limit."""
@@ -404,7 +410,7 @@
         self.assertEqual(JobStatus.FAILED, job.status)
         err = proc.stderr.read()
         self.assertIn(
-            'OOPS while executing job 10: [] SoftTimeLimitExceeded', err)
+            b'OOPS while executing job 10: [] SoftTimeLimitExceeded', err)
 
     def test_timeout_in_fast_lane_passes_in_slow_lane(self):
         # If a fast and a slow lane are configured, jobs which time out
@@ -526,12 +532,12 @@
         return (
             '%s: {"children": [], "result": null, "status": "SUCCESS", '
             '"task_id": "%s", "traceback": null}\n'
-            % (self.queueName(task_id), task_id))
+            % (self.queueName(task_id), task_id)).encode('UTF-8')
 
     def noQueueMessage(self, task_id):
         return (
             "NOT_FOUND - no queue '%s' in vhost '/'\n"
-            % self.queueName(task_id))
+            % self.queueName(task_id)).encode('UTF-8')
 
     def test_clear_queues__result_not_consumed(self):
         """When a Celery task is started so that a result is returned
@@ -546,12 +552,12 @@
         clear_queue_config = 'lazr.jobrunner.tests.simple_config'
         stdout, stderr = self.runClearQueues(clear_queue_config, [task_id])
         self.assertEqual(self.successMessage(task_id), stdout)
-        self.assertEqual('', stderr)
+        self.assertEqual(b'', stderr)
 
         # Reading a queue is destructive. An attempt to read again from
         # a queue results in an error.
         stdout, stderr = self.runClearQueues(clear_queue_config, [task_id])
-        self.assertEqual('', stdout)
+        self.assertEqual(b'', stdout)
         self.assertEqual(self.noQueueMessage(task_id), stderr)
 
     def test_clear_queues__two_queues(self):
@@ -567,7 +573,7 @@
         expected_stdout = (
             self.successMessage(task_id_1) + self.successMessage(task_id_2))
         self.assertEqual(expected_stdout, stdout)
-        self.assertEqual('', stderr)
+        self.assertEqual(b'', stderr)
 
     def test_clear_queues__task_without_result(self):
         """A Celery task which was started so that no result is returned
@@ -577,7 +583,7 @@
         task_id = self.invokeJob(celery_config_jobrunner, RunFileJobNoResult)
         clear_queue_config = 'lazr.jobrunner.tests.simple_config'
         stdout, stderr = self.runClearQueues(clear_queue_config, [task_id])
-        self.assertEqual('', stdout)
+        self.assertEqual(b'', stdout)
         self.assertEqual(self.noQueueMessage(task_id), stderr)
 
     def test_clear_queues__config_create_missing_queues_false(self):
@@ -596,5 +602,5 @@
         # not have the desired effect
         task_id = 'this-queue-does-not-exist'
         stdout, stderr = self.runClearQueues(celery_config, [task_id])
-        self.assertEqual('', stdout)
+        self.assertEqual(b'', stdout)
         self.assertEqual(self.noQueueMessage(task_id), stderr)
 
=== modified file 'src/lazr/jobrunner/tests/time_limit_config_fast_lane.py'
--- src/lazr/jobrunner/tests/time_limit_config_fast_lane.py	2012-04-10 12:44:00 +0000
+++ src/lazr/jobrunner/tests/time_limit_config_fast_lane.py	2019-05-02 14:16:27 +0000
@@ -1,3 +1,3 @@
-from config_two_queues import *
+from lazr.jobrunner.tests.config_two_queues import *
 CELERYD_TASK_SOFT_TIME_LIMIT = 1
 FALLBACK_QUEUE = 'standard_slow'
 
=== modified file 'src/lazr/jobrunner/tests/time_limit_config_slow_lane.py'
--- src/lazr/jobrunner/tests/time_limit_config_slow_lane.py	2012-04-10 12:44:00 +0000
+++ src/lazr/jobrunner/tests/time_limit_config_slow_lane.py	2019-05-02 14:16:27 +0000
@@ -1,2 +1,2 @@
-from config_two_queues import *
+from lazr.jobrunner.tests.config_two_queues import *
 CELERYD_TASK_SOFT_TIME_LIMIT = 5
 
=== modified file 'tox.ini'
--- tox.ini	2019-05-02 14:16:27 +0000
+++ tox.ini	2019-05-02 14:16:27 +0000
@@ -1,6 +1,6 @@
 [tox]
 envlist =
-    py27-celery{31,40,41,42,43}
+    py27-celery{31,40,41,42,43}, py35-celery43
 
 [testenv]
 commands =