lazr.jobrunner

Merge lp:~cjwatson/lazr.jobrunner/py3 into lp:lazr.jobrunner

  1. py3
  2. Merge into trunk
Proposed by Colin Watson
Status: Merged
Merged at revision: 59
Proposed branch: lp:~cjwatson/lazr.jobrunner/py3
Merge into: lp:lazr.jobrunner
Prerequisite: lp:~cjwatson/lazr.jobrunner/celery-4.3
Diff against target: 340 lines (+70/-34)
11 files modified
NEWS.txt (+4/-0)
setup.py (+5/-3)
src/lazr/__init__.py (+23/-1)
src/lazr/jobrunner/bin/clear_queues.py (+5/-3)
src/lazr/jobrunner/jobrunner.py (+3/-3)
src/lazr/jobrunner/tests/config1.py (+1/-1)
src/lazr/jobrunner/tests/config_do_not_create_missing_queues.py (+1/-1)
src/lazr/jobrunner/tests/test_celerytask.py (+25/-19)
src/lazr/jobrunner/tests/time_limit_config_fast_lane.py (+1/-1)
src/lazr/jobrunner/tests/time_limit_config_slow_lane.py (+1/-1)
tox.ini (+1/-1)
To merge this branch: bzr merge lp:~cjwatson/lazr.jobrunner/py3
Reviewer Review Type Date Requested Status
Adam Collard (community) Approve
Launchpad code reviewers Pending
Review via email: mp+366179@code.launchpad.net

Commit message

Add Python 3 support (requires celery >= 4.3).

To post a comment you must log in.
Revision history for this message
Adam Collard (adam-collard) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'NEWS.txt'
2--- NEWS.txt 2019-03-08 18:24:33 +0000
3+++ NEWS.txt 2019-05-02 14:16:27 +0000
4@@ -1,6 +1,10 @@
5 News
6 ====
7
8+UNRELEASED
9+----------
10+* Add Python 3 support (requires celery >= 4.3).
11+
12 0.14
13 ----
14 * Add tox testing support.
15
16=== modified file 'setup.py'
17--- setup.py 2019-03-08 18:24:33 +0000
18+++ setup.py 2019-05-02 14:16:27 +0000
19@@ -28,11 +28,13 @@
20 # List your project dependencies here.
21 # For more details, see:
22 # http://packages.python.org/distribute/setuptools.html#declaring-dependencies
23- 'celery>=3.0,<5.0',
24+ 'celery>=3.0,<5.0; python_version < "3"',
25+ 'celery>=4.3,<5.0; python_version >= "3"',
26 ]
27
28 tests_require = [
29 'oops>=0.0.11',
30+ 'six',
31 'unittest2',
32 'zope.testing',
33 ]
34@@ -51,8 +53,8 @@
35 url='https://launchpad.net/lazr.jobrunner',
36 license='GPL v3',
37 packages=find_packages('src'),
38- package_dir = {'': 'src'},
39- namespace_packages = ['lazr'],
40+ package_dir={'': 'src'},
41+ namespace_packages=['lazr'],
42 include_package_data=True,
43 zip_safe=False,
44 install_requires=install_requires,
45
46=== modified file 'src/lazr/__init__.py'
47--- src/lazr/__init__.py 2012-02-23 10:32:17 +0000
48+++ src/lazr/__init__.py 2019-05-02 14:16:27 +0000
49@@ -1,1 +1,23 @@
50-__import__('pkg_resources').declare_namespace(__name__)
51+# Copyright 2019 Canonical Ltd. All rights reserved.
52+#
53+# This file is part of lazr.jobrunner.
54+#
55+# lazr.jobrunner is free software: you can redistribute it and/or modify it
56+# under the terms of the GNU Lesser General Public License as published by
57+# the Free Software Foundation, version 3 of the License.
58+#
59+# lazr.jobrunner is distributed in the hope that it will be useful, but WITHOUT
60+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
61+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
62+# License for more details.
63+#
64+# You should have received a copy of the GNU Lesser General Public License
65+# along with lazr.jobrunner. If not, see <http://www.gnu.org/licenses/>.
66+
67+# this is a namespace package
68+try:
69+ import pkg_resources
70+ pkg_resources.declare_namespace(__name__)
71+except ImportError:
72+ import pkgutil
73+ __path__ = pkgutil.extend_path(__path__, __name__)
74
75=== modified file 'src/lazr/jobrunner/bin/clear_queues.py'
76--- src/lazr/jobrunner/bin/clear_queues.py 2019-03-05 17:06:13 +0000
77+++ src/lazr/jobrunner/bin/clear_queues.py 2019-05-02 14:16:27 +0000
78@@ -17,6 +17,8 @@
79
80 """Inspect Celery result queues."""
81
82+from __future__ import absolute_import, print_function
83+
84 __metaclass__ = type
85
86 from argparse import ArgumentParser
87@@ -28,9 +30,9 @@
88
89
90 def show_queue_data(body, message):
91- print '%s: %s' % (
92+ print('%s: %s' % (
93 message.delivery_info['routing_key'],
94- json.dumps(body, sort_keys=True))
95+ json.dumps(body, sort_keys=True)))
96
97
98 def clear_queues(args):
99@@ -56,7 +58,7 @@
100 RunJob.app, [queue], callbacks=[show_queue_data],
101 retain=True, passive_queues=True)
102 except amqp.exceptions.NotFound as exc:
103- print >>sys.stderr, exc.reply_text
104+ print(exc.reply_text, file=sys.stderr)
105
106
107 def main():
108
109=== modified file 'src/lazr/jobrunner/jobrunner.py'
110--- src/lazr/jobrunner/jobrunner.py 2012-10-24 17:57:42 +0000
111+++ src/lazr/jobrunner/jobrunner.py 2019-05-02 14:16:27 +0000
112@@ -160,7 +160,7 @@
113 try:
114 try:
115 job.run()
116- except self.retryErrorTypes(job), e:
117+ except self.retryErrorTypes(job) as e:
118 if job.attempt_count > job.max_retries:
119 raise
120 self.logger.warning(
121@@ -192,7 +192,7 @@
122 try:
123 try:
124 self.runJob(job, fallback)
125- except self.userErrorTypes(job), e:
126+ except self.userErrorTypes(job) as e:
127 self.logger.info(
128 '%s failed with user error %r.'
129 % (self.job_str(job), e))
130@@ -200,7 +200,7 @@
131 except Exception:
132 info = sys.exc_info()
133 return self._doOops(job, info)
134- except Exception, e:
135+ except Exception as e:
136 # This only happens if _doOops() fails.
137 self.logger.exception("Failure in _doOops: %s" % e)
138 info = sys.exc_info()
139
140=== modified file 'src/lazr/jobrunner/tests/config1.py'
141--- src/lazr/jobrunner/tests/config1.py 2012-07-09 10:58:00 +0000
142+++ src/lazr/jobrunner/tests/config1.py 2019-05-02 14:16:27 +0000
143@@ -1,4 +1,4 @@
144-from simple_config import *
145+from lazr.jobrunner.tests.simple_config import *
146 import os
147 import oops
148 CELERY_ANNOTATIONS = {
149
150=== modified file 'src/lazr/jobrunner/tests/config_do_not_create_missing_queues.py'
151--- src/lazr/jobrunner/tests/config_do_not_create_missing_queues.py 2012-07-09 10:58:00 +0000
152+++ src/lazr/jobrunner/tests/config_do_not_create_missing_queues.py 2019-05-02 14:16:27 +0000
153@@ -1,2 +1,2 @@
154-from simple_config import *
155+from lazr.jobrunner.tests.simple_config import *
156 CELERY_CREATE_MISSING_QUEUES = False
157
158=== modified file 'src/lazr/jobrunner/tests/test_celerytask.py'
159--- src/lazr/jobrunner/tests/test_celerytask.py 2019-03-06 12:17:21 +0000
160+++ src/lazr/jobrunner/tests/test_celerytask.py 2019-05-02 14:16:27 +0000
161@@ -31,7 +31,12 @@
162 import tempfile
163 from time import sleep
164 from unittest import TestCase
165-import urllib2
166+
167+from six.moves.urllib.error import URLError
168+from six.moves.urllib.request import (
169+ build_opener,
170+ HTTPBasicAuthHandler,
171+ )
172
173 os.environ.setdefault('CELERY_CONFIG_MODULE', 'lazr.jobrunner.celeryconfig')
174
175@@ -143,7 +148,7 @@
176 def ensure_dir(path):
177 try:
178 os.mkdir(path)
179- except OSError, e:
180+ except OSError as e:
181 if e.errno != errno.EEXIST:
182 raise
183 ensure_dir(self.job_root)
184@@ -176,7 +181,7 @@
185 try:
186 with self._job_output_file(job.job_id, 'r') as job_output_file:
187 return job_output_file.read()
188- except IOError, e:
189+ except IOError as e:
190 if e.errno == errno.ENOENT:
191 return None
192 raise
193@@ -269,7 +274,7 @@
194 # There is exactly one OOPS report.
195 self.assertEqual(1, len(oops_repository.oopses))
196 # This OOPS describes a MemoryError.
197- oops_report = oops_repository.oopses.values()[0]
198+ oops_report = list(oops_repository.oopses.values())[0]
199 self.assertEqual('MemoryError', oops_report['type'])
200
201 def test_acquires_lease(self):
202@@ -288,11 +293,11 @@
203 class TestCeleryD(TestCase):
204
205 def getQueueInfo(self):
206- auth_handler = urllib2.HTTPBasicAuthHandler()
207+ auth_handler = HTTPBasicAuthHandler()
208 auth_handler.add_password(
209 realm='Management: Web UI', user='guest', passwd='guest',
210 uri='http://localhost:55672/api/queues')
211- opener = urllib2.build_opener(auth_handler)
212+ opener = build_opener(auth_handler)
213 info = opener.open('http://localhost:55672/api/queues').read()
214 info = json.loads(info)
215 # info is a list of dictionaries with details about the queues.
216@@ -305,7 +310,7 @@
217 super(TestCeleryD, self).setUp()
218 try:
219 self.queue_status_during_setup = self.getQueueInfo()
220- except urllib2.URLError:
221+ except URLError:
222 # The rabbitmq-management package is currently broken
223 # on Precise, so the RabbitMQ management interface may
224 # not be available.
225@@ -314,7 +319,7 @@
226 def tearDown(self):
227 try:
228 current_queue_status = self.getQueueInfo()
229- except urllib2.URLError:
230+ except URLError:
231 # See setUp()
232 return
233 bad_queues = []
234@@ -390,9 +395,10 @@
235 err = proc.stderr.read()
236 self.assertEqual(JobStatus.FAILED, job.status)
237 self.assertIs(None, job.job_source.get_output(job))
238- self.assertIn(
239- "OOPS while executing job 10: [] Exception(u'Catch me if you"
240- " can!',)", err)
241+ expected_message = (
242+ "OOPS while executing job 10: [] Exception(%r,)" %
243+ u'Catch me if you can!').encode('UTF-8')
244+ self.assertIn(expected_message, err)
245
246 def test_timeout_long(self):
247 """Raises exception when a job exceeds the configured time limit."""
248@@ -404,7 +410,7 @@
249 self.assertEqual(JobStatus.FAILED, job.status)
250 err = proc.stderr.read()
251 self.assertIn(
252- 'OOPS while executing job 10: [] SoftTimeLimitExceeded', err)
253+ b'OOPS while executing job 10: [] SoftTimeLimitExceeded', err)
254
255 def test_timeout_in_fast_lane_passes_in_slow_lane(self):
256 # If a fast and a slow lane are configured, jobs which time out
257@@ -526,12 +532,12 @@
258 return (
259 '%s: {"children": [], "result": null, "status": "SUCCESS", '
260 '"task_id": "%s", "traceback": null}\n'
261- % (self.queueName(task_id), task_id))
262+ % (self.queueName(task_id), task_id)).encode('UTF-8')
263
264 def noQueueMessage(self, task_id):
265 return (
266 "NOT_FOUND - no queue '%s' in vhost '/'\n"
267- % self.queueName(task_id))
268+ % self.queueName(task_id)).encode('UTF-8')
269
270 def test_clear_queues__result_not_consumed(self):
271 """When a Celery task is started so that a result is returned
272@@ -546,12 +552,12 @@
273 clear_queue_config = 'lazr.jobrunner.tests.simple_config'
274 stdout, stderr = self.runClearQueues(clear_queue_config, [task_id])
275 self.assertEqual(self.successMessage(task_id), stdout)
276- self.assertEqual('', stderr)
277+ self.assertEqual(b'', stderr)
278
279 # Reading a queue is destructive. An attempt to read again from
280 # a queue results in an error.
281 stdout, stderr = self.runClearQueues(clear_queue_config, [task_id])
282- self.assertEqual('', stdout)
283+ self.assertEqual(b'', stdout)
284 self.assertEqual(self.noQueueMessage(task_id), stderr)
285
286 def test_clear_queues__two_queues(self):
287@@ -567,7 +573,7 @@
288 expected_stdout = (
289 self.successMessage(task_id_1) + self.successMessage(task_id_2))
290 self.assertEqual(expected_stdout, stdout)
291- self.assertEqual('', stderr)
292+ self.assertEqual(b'', stderr)
293
294 def test_clear_queues__task_without_result(self):
295 """A Celery task which was started so that no result is returned
296@@ -577,7 +583,7 @@
297 task_id = self.invokeJob(celery_config_jobrunner, RunFileJobNoResult)
298 clear_queue_config = 'lazr.jobrunner.tests.simple_config'
299 stdout, stderr = self.runClearQueues(clear_queue_config, [task_id])
300- self.assertEqual('', stdout)
301+ self.assertEqual(b'', stdout)
302 self.assertEqual(self.noQueueMessage(task_id), stderr)
303
304 def test_clear_queues__config_create_missing_queues_false(self):
305@@ -596,5 +602,5 @@
306 # not have the desired effect
307 task_id = 'this-queue-does-not-exist'
308 stdout, stderr = self.runClearQueues(celery_config, [task_id])
309- self.assertEqual('', stdout)
310+ self.assertEqual(b'', stdout)
311 self.assertEqual(self.noQueueMessage(task_id), stderr)
312
313=== modified file 'src/lazr/jobrunner/tests/time_limit_config_fast_lane.py'
314--- src/lazr/jobrunner/tests/time_limit_config_fast_lane.py 2012-04-10 12:44:00 +0000
315+++ src/lazr/jobrunner/tests/time_limit_config_fast_lane.py 2019-05-02 14:16:27 +0000
316@@ -1,3 +1,3 @@
317-from config_two_queues import *
318+from lazr.jobrunner.tests.config_two_queues import *
319 CELERYD_TASK_SOFT_TIME_LIMIT = 1
320 FALLBACK_QUEUE = 'standard_slow'
321
322=== modified file 'src/lazr/jobrunner/tests/time_limit_config_slow_lane.py'
323--- src/lazr/jobrunner/tests/time_limit_config_slow_lane.py 2012-04-10 12:44:00 +0000
324+++ src/lazr/jobrunner/tests/time_limit_config_slow_lane.py 2019-05-02 14:16:27 +0000
325@@ -1,2 +1,2 @@
326-from config_two_queues import *
327+from lazr.jobrunner.tests.config_two_queues import *
328 CELERYD_TASK_SOFT_TIME_LIMIT = 5
329
330=== modified file 'tox.ini'
331--- tox.ini 2019-05-02 14:16:27 +0000
332+++ tox.ini 2019-05-02 14:16:27 +0000
333@@ -1,6 +1,6 @@
334 [tox]
335 envlist =
336- py27-celery{31,40,41,42,43}
337+ py27-celery{31,40,41,42,43}, py35-celery43
338
339 [testenv]
340 commands =

Subscribers

People subscribed via source and target branches

to all changes: