Merge lp:~chris.macnaughton/mojo/py3002 into lp:~ost-maintainers/mojo/py3

Proposed by Chris MacNaughton on 2018-01-19
Status: Merged
Approved by: Ryan Beisner on 2018-01-26
Approved revision: 474
Merged at revision: 465
Proposed branch: lp:~chris.macnaughton/mojo/py3002
Merge into: lp:~ost-maintainers/mojo/py3
Diff against target: 337 lines (+71/-33)
9 files modified
contrib/test/utils.py (+1/-1)
mojo/cli.py (+13/-15)
mojo/juju/status.py (+3/-1)
mojo/juju/wait.py (+3/-0)
mojo/phase.py (+4/-1)
mojo/tests/test_juju2.py (+19/-9)
mojo/utils.py (+3/-0)
setup.py (+1/-0)
tox.ini (+24/-6)
To merge this branch: bzr merge lp:~chris.macnaughton/mojo/py3002
Reviewer Review Type Date Requested Status
OpenStack Charm Testing Maintainers 2018-01-19 Pending
Review via email: mp+336352@code.launchpad.net
To post a comment you must log in.
lp:~chris.macnaughton/mojo/py3002 updated on 2018-01-22
470. By Chris MacNaughton on 2018-01-22

add list() back around map() to make it work on py2+3

Have confirmed with a basic mojo run that this works on py2 still, commencing a test on py3

Not quite py3 ready:

2018-01-24 11:10:49 [INFO] Rsyncing /home/ubuntu/mojo-openstack-specs to /srv/mojo/openstack/xenial/_20181024_111047/spec
2018-01-24 11:10:50 [INFO] Retrieve the spec's manifest
2018-01-24 11:10:50 [INFO] Manifest comment:

#############################################################################
Collect the charm branches from Launchpad
#############################################################################

2018-01-24 11:10:50 [INFO] Building resource tree
2018-01-24 11:10:50 [ERROR] Unknown error
Traceback (most recent call last):
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/cli.py", line 628, in run_with_args
    args.func(args)
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/utils.py", line 305, in wrapped
    return method(*args, **kwargs)
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/cli.py", line 346, in run_from_manifest
    manifest.run(project, workspace, args.stage)
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/manifest.py", line 100, in run
    auto_repo=auto_repo)
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/phase.py", line 221, in run
    rendered_collect.write(Template(inp.read()).render(**tmpl_vars))
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/tempfile.py", line 622, in func_wrapper
    return func(*args, **kwargs)
TypeError: a bytes-like object is required, not 'str'

lp:~chris.macnaughton/mojo/py3002 updated on 2018-01-25
471. By Chris MacNaughton on 2018-01-25

decode bytes to str

472. By Chris MacNaughton on 2018-01-25

import six

473. By Chris MacNaughton on 2018-01-25

add encoding to popen

474. By Chris MacNaughton on 2018-01-25

ensure we decode command line output if needed

Ok, have run the openstack-mojo-specs, ceph base spec with both py2 and py3, and have identical output ;)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'contrib/test/utils.py'
2--- contrib/test/utils.py 2018-01-18 10:23:15 +0000
3+++ contrib/test/utils.py 2018-01-25 14:10:22 +0000
4@@ -44,6 +44,6 @@
5 """
6 try:
7 subprocess.check_output(['juju', 'switch'])
8- except subprocess.CalledProcessError as OSError:
9+ except subprocess.CalledProcessError, OSError:
10 print('Error checking active Juju controller')
11 sys.exit(1)
12
13=== modified file 'mojo/cli.py'
14--- mojo/cli.py 2018-01-18 10:23:15 +0000
15+++ mojo/cli.py 2018-01-25 14:10:22 +0000
16@@ -1,6 +1,6 @@
17 # Copyright 2014, 2015 Canonical Ltd. This software is licensed under the
18 # GNU General Public License version 3 (see the file LICENSE).
19-
20+from __future__ import print_function
21 import argparse
22 import errno
23 import logging
24@@ -76,7 +76,7 @@
25
26
27 def exit_with_usage(parser, message):
28- print(message + "\n", file=sys.stderr)
29+ sys.stderr.write(message + "\n")
30 parser.print_help(file=sys.stderr)
31 sys.exit(1)
32
33@@ -268,7 +268,7 @@
34 try:
35 project.create_workspace(spec_url, workspace, stage=args.stage, logfile=args.logfile)
36 except OSError:
37- print("Are you sure the project exists?", file=sys.stderr)
38+ sys.stderr.write("Are you sure the project exists?")
39 if args.blunt:
40 raise
41 sys.exit(2)
42@@ -649,9 +649,9 @@
43 sys.exit(1)
44 except mojo.exceptions.ConfigNotFoundException as e:
45 logging.error(e.message)
46- print("Searched:", file=sys.stderr)
47+ sys.stderr.write("Searched:")
48 for path in e.search_list:
49- print(" {}".format(path), file=sys.stderr)
50+ sys.stderr.write(" {}".format(path))
51 sys.exit(1)
52 except (mojo.exceptions.InvalidManifestException,
53 mojo.phase.FileDoesNotExistException,
54@@ -672,14 +672,14 @@
55 if ioe.errno != errno.EACCES:
56 raise
57
58- print('ERROR: Unable to read container configuration '
59- 'from {}'.format(filename), file=sys.stderr)
60+ sys.stderr.write('ERROR: Unable to read container configuration '
61+ 'from {}'.format(filename))
62
63 try:
64 os.listdir('/var/lib/lxc')
65 except OSError as ose:
66 if ose.errno == errno.EACCES:
67- print(dedent(
68+ sys.stderr.write(dedent(
69 '''\
70 ERROR: /var/lib/lxc is not readable; consider one of the following:
71
72@@ -688,8 +688,7 @@
73 sudo chmod 0750 /var/lib/lxc
74
75 Option 2:
76- sudo chmod 0755 /var/lib/lxc'''),
77- file=sys.stderr)
78+ sudo chmod 0755 /var/lib/lxc'''))
79 sys.exit(1)
80
81 try:
82@@ -699,18 +698,17 @@
83 # The problem isn't /var/lib/lxc; it's the container!
84 if ose.errno == errno.EACCES:
85 # We know it's not /var/lib/lxc; that was checked above.
86- print(dedent(
87+ sys.stderr.write(dedent(
88 '''\
89 ERROR: {container} exists but is not readable; consider doing the following:
90
91 sudo chmod 0755 {container}''').format(
92- container=container),
93- file=sys.stderr)
94+ container=container))
95 sys.exit(1)
96 if ose.errno == errno.ENOENT:
97 # Mojo checks this itself, so this would be very strange.
98- print('ERROR: {} does not exist; was the Mojo project '
99- 'created properly?!'.format(container), file=sys.stderr)
100+ sys.stderr.write('ERROR: {} does not exist; was the Mojo project '
101+ 'created properly?!'.format(container))
102 sys.exit(1)
103
104 # Some things are beyond even our powers to diagnose.
105
106=== modified file 'mojo/juju/status.py'
107--- mojo/juju/status.py 2018-01-18 10:23:15 +0000
108+++ mojo/juju/status.py 2018-01-25 14:10:22 +0000
109@@ -4,6 +4,7 @@
110 import subprocess
111 import time
112 import yaml
113+import six
114
115 from collections import Counter
116
117@@ -12,6 +13,8 @@
118 try:
119 with open(os.devnull, 'w') as devnull:
120 version = subprocess.check_output(['juju', 'version'], stderr=devnull)
121+ if six.PY3:
122+ version = version.decode('utf-8')
123 major_version = int(version[0])
124 except subprocess.CalledProcessError:
125 # In cases where the Juju Status yaml is provided by a file this may legitimately fail
126@@ -463,7 +466,6 @@
127 # self.ready() will raise exceptions on error states
128 if self.ready() or not wait_for_steady:
129 return
130-
131 logging.info("Waiting for environment to reach steady state")
132 wait(max_wait=timeout)
133 logging.info("Environment has reached steady state")
134
135=== modified file 'mojo/juju/wait.py'
136--- mojo/juju/wait.py 2018-01-18 10:23:15 +0000
137+++ mojo/juju/wait.py 2018-01-25 14:10:22 +0000
138@@ -31,6 +31,7 @@
139 from textwrap import dedent
140 import time
141 import yaml
142+import six
143
144
145 __version__ = '2.5.0'
146@@ -76,6 +77,8 @@
147 logging.error(err)
148 logging.error("{} failed: {}".format(' '.join(cmd), p.returncode))
149 raise JujuWaitException(p.returncode or 43)
150+ if six.PY3:
151+ out = out.decode('utf-8')
152 return out
153
154
155
156=== modified file 'mojo/phase.py'
157--- mojo/phase.py 2018-01-18 11:04:39 +0000
158+++ mojo/phase.py 2018-01-25 14:10:22 +0000
159@@ -25,6 +25,7 @@
160 import mojo.juju
161 import mojo.juju.utils
162 import codetree
163+import six
164 from deployer.config import ConfigStack
165
166
167@@ -215,7 +216,7 @@
168 with chdir(workspace.build_dir):
169 try:
170 tmpl_vars = {'series': project.series}
171- with tempfile.NamedTemporaryFile() as rendered_collect:
172+ with tempfile.NamedTemporaryFile(mode='wt') as rendered_collect:
173 with open(config) as inp:
174 try:
175 rendered_collect.write(Template(inp.read()).render(**tmpl_vars))
176@@ -363,6 +364,8 @@
177 command = env_vars + [script]
178 try:
179 output = subprocess.check_output(command, stderr=subprocess.STDOUT)
180+ if six.PY3:
181+ output = output.decode('utf-8')
182 except subprocess.CalledProcessError as e:
183 if gather_debug_logs:
184 exception_output = self._gather_debug_logs(e, workspace, stage)
185
186=== modified file 'mojo/tests/test_juju2.py'
187--- mojo/tests/test_juju2.py 2018-01-18 10:23:15 +0000
188+++ mojo/tests/test_juju2.py 2018-01-25 14:10:22 +0000
189@@ -128,19 +128,22 @@
190 def test_model_version(self):
191 self.assertEqual(self.status.model_version(), '2.0-beta18')
192
193- def test_status(self):
194+ @mock.patch('subprocess.check_output')
195+ def test_status(self, mock_check_output):
196 """Test the status function"""
197 self.assertEqual(self.status.status(), self.status._raw_status)
198
199 with mock.patch('subprocess.check_output') as mock_check_output:
200 # First check with no environment set
201- no_env_status = mojo.juju.status.Juju2Status(command_timeout=None)
202- no_env_status.status()
203+ with mock.patch('mojo.juju.status.major_version', new=2):
204+ no_env_status = mojo.juju.status.Juju2Status(command_timeout=None)
205+ no_env_status.status()
206 mock_check_output.assert_called_with(
207 ['juju', 'status', '--format=yaml'])
208 # Second test with an environment set
209- env_status = mojo.juju.status.Juju2Status(environment="test-env")
210- env_status.status()
211+ with mock.patch('mojo.juju.status.major_version', new=2):
212+ env_status = mojo.juju.status.Juju2Status(environment="test-env")
213+ env_status.status()
214 mock_check_output.assert_called_with(
215 ['/usr/bin/timeout', '--kill-after', '5', '600',
216 'juju', 'status', '--format=yaml', '-m', 'test-env'])
217@@ -240,7 +243,12 @@
218 'since': '16 Sep 2016 15:23:55Z'},
219 'relations': {'juju-info': ['apache2', 'prometheus']},
220 'subordinate-to': ['apache2', 'prometheus']}}}
221- self.assertEqual(self.status.yaml_status(force_update=True), expected)
222+ with mock.patch('subprocess.check_output') as mock_check_output:
223+ mock_check_output.return_value = self._testdata_from_file('juju2-idle.yaml')
224+ self.assertEqual(self.status.yaml_status(force_update=True), expected)
225+ # mock_check_output.assert_called_with(
226+ # ['/usr/bin/timeout', '--kill-after', '5', '600',
227+ # 'juju', 'status', '--format=yaml'])
228
229
230 class Juju2CheckOutputWithTimeoutTestException(Exception):
231@@ -305,7 +313,7 @@
232 timeout_exception=Juju2CheckOutputWithTimeoutTestException)
233
234 self.assertTrue(
235- arcm.exception.message.endswith(' timed out after 5 seconds'))
236+ str(arcm.exception).endswith(' timed out after 5 seconds'))
237
238 @mock.patch('subprocess.check_output')
239 def test_command_times_out_and_is_killed(self, check_output_mock):
240@@ -323,7 +331,7 @@
241 timeout_exception=Juju2CheckOutputWithTimeoutTestException)
242
243 self.assertTrue(
244- arcm.exception.message.endswith(' timed out after 5 seconds (killed)'))
245+ str(arcm.exception).endswith(' timed out after 5 seconds (killed)'))
246
247
248 class Juju2ChecksTest(TestCase):
249@@ -386,7 +394,9 @@
250 sys.argv = ['self', '-f', self.status_path, 'get_ips', 'all']
251 rc = parse_status()
252 self.assertEqual(rc, 0)
253- self.assertEqual(mock_stdout.getvalue(), '10.25.2.111\n10.242.242.242\n10.25.2.109\n10.25.2.112\n10.25.2.110\n')
254+ ips = mock_stdout.getvalue().split("\n")
255+ ips.sort()
256+ self.assertEqual("{}\n".format("\n".join(ips).lstrip()), '10.242.242.242\n10.25.2.109\n10.25.2.110\n10.25.2.111\n10.25.2.112\n')
257
258 @mock.patch('sys.stdout', new_callable=StringIO)
259 def test_GetIPs_app(self, mock_stdout):
260
261=== modified file 'mojo/utils.py'
262--- mojo/utils.py 2017-12-22 10:31:56 +0000
263+++ mojo/utils.py 2018-01-25 14:10:22 +0000
264@@ -13,6 +13,7 @@
265 import time
266
267 from hashlib import sha1
268+import six
269 from .exceptions import Unprivileged
270
271
272@@ -262,6 +263,8 @@
273 eof = True
274 for stream in to_read:
275 line = stream.read(read_bytes)
276+ if six.PY3:
277+ line = line.decode('utf-8')
278 if line != "":
279 eof = False
280 output += line
281
282=== modified file 'setup.py'
283--- setup.py 2017-05-10 03:58:59 +0000
284+++ setup.py 2018-01-25 14:10:22 +0000
285@@ -21,6 +21,7 @@
286 "jujuclient >= 0.0.7",
287 "juju-deployer >= 0.6.4",
288 "pylxd < 2.1.0",
289+ "six >= 1.11.0",
290 "python-cinderclient >= 1.0.0"],
291 packages=find_packages(),
292 classifiers=[
293
294=== modified file 'tox.ini'
295--- tox.ini 2018-01-18 10:23:15 +0000
296+++ tox.ini 2018-01-25 14:10:22 +0000
297@@ -1,6 +1,6 @@
298 # mojo_exec: build a py3 virtualenv and execute a basic mojo command to confirm py3-only functionality.
299 [tox]
300-envlist = mojo_exec
301+envlist = mojo2_exec,mojo3_exec,py27,py36
302 skipsdist = True
303
304 [testenv]
305@@ -10,8 +10,26 @@
306 install_command =
307 pip install --allow-unverified python-apt {opts} {packages}
308
309-[testenv:mojo_exec]
310-basepython = python3
311-deps = -r{toxinidir}/requirements.txt
312-commands = pip install --upgrade .
313- mojo --version
314+[testenv:mojo3_exec]
315+basepython = python3
316+deps = -r{toxinidir}/requirements.txt
317+commands = pip install --upgrade .
318+ mojo --version
319+
320+[testenv:mojo2_exec]
321+basepython = python2
322+deps = -r{toxinidir}/requirements.txt
323+commands = pip install --upgrade .
324+ mojo --version
325+
326+[testenv:py3]
327+basepython = python3
328+deps = -r{toxinidir}/requirements.txt
329+ -r{toxinidir}/test-requirements.txt
330+commands = nosetests {posargs} {toxinidir}/mojo/tests
331+
332+[testenv:py27]
333+basepython = python2.7
334+deps = -r{toxinidir}/requirements.txt
335+ -r{toxinidir}/test-requirements.txt
336+commands = nosetests {posargs} {toxinidir}/mojo/tests
337\ No newline at end of file

Subscribers

People subscribed via source and target branches