Merge lp:~chris.macnaughton/mojo/py3002 into lp:~ost-maintainers/mojo/py3

Proposed by Chris MacNaughton on 2018-01-19
Status: Merged
Approved by: Ryan Beisner on 2018-01-26
Approved revision: 474
Merged at revision: 465
Proposed branch: lp:~chris.macnaughton/mojo/py3002
Merge into: lp:~ost-maintainers/mojo/py3
Diff against target: 337 lines (+71/-33)
9 files modified
contrib/test/utils.py (+1/-1)
mojo/cli.py (+13/-15)
mojo/juju/status.py (+3/-1)
mojo/juju/wait.py (+3/-0)
mojo/phase.py (+4/-1)
mojo/tests/test_juju2.py (+19/-9)
mojo/utils.py (+3/-0)
setup.py (+1/-0)
tox.ini (+24/-6)
To merge this branch: bzr merge lp:~chris.macnaughton/mojo/py3002
Reviewer Review Type Date Requested Status
OpenStack Charm Testing Maintainers 2018-01-19 Pending
Review via email: mp+336352@code.launchpad.net
To post a comment you must log in.
lp:~chris.macnaughton/mojo/py3002 updated on 2018-01-22
470. By Chris MacNaughton on 2018-01-22

add list() back around map() to make it work on py2+3

Have confirmed with a basic mojo run that this works on py2 still, commencing a test on py3

Not quite py3 ready:

2018-01-24 11:10:49 [INFO] Rsyncing /home/ubuntu/mojo-openstack-specs to /srv/mojo/openstack/xenial/_20181024_111047/spec
2018-01-24 11:10:50 [INFO] Retrieve the spec's manifest
2018-01-24 11:10:50 [INFO] Manifest comment:

#############################################################################
Collect the charm branches from Launchpad
#############################################################################

2018-01-24 11:10:50 [INFO] Building resource tree
2018-01-24 11:10:50 [ERROR] Unknown error
Traceback (most recent call last):
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/cli.py", line 628, in run_with_args
    args.func(args)
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/utils.py", line 305, in wrapped
    return method(*args, **kwargs)
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/cli.py", line 346, in run_from_manifest
    manifest.run(project, workspace, args.stage)
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/manifest.py", line 100, in run
    auto_repo=auto_repo)
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/phase.py", line 221, in run
    rendered_collect.write(Template(inp.read()).render(**tmpl_vars))
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/tempfile.py", line 622, in func_wrapper
    return func(*args, **kwargs)
TypeError: a bytes-like object is required, not 'str'

lp:~chris.macnaughton/mojo/py3002 updated on 2018-01-25
471. By Chris MacNaughton on 2018-01-25

decode bytes to str

472. By Chris MacNaughton on 2018-01-25

import six

473. By Chris MacNaughton on 2018-01-25

add encoding to popen

474. By Chris MacNaughton on 2018-01-25

ensure we decode command line output if needed

Ok, have run the openstack-mojo-specs, ceph base spec with both py2 and py3, and have identical output ;)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'contrib/test/utils.py'
--- contrib/test/utils.py 2018-01-18 10:23:15 +0000
+++ contrib/test/utils.py 2018-01-25 14:10:22 +0000
@@ -44,6 +44,6 @@
44 """44 """
45 try:45 try:
46 subprocess.check_output(['juju', 'switch'])46 subprocess.check_output(['juju', 'switch'])
47 except subprocess.CalledProcessError as OSError:47 except subprocess.CalledProcessError, OSError:
48 print('Error checking active Juju controller')48 print('Error checking active Juju controller')
49 sys.exit(1)49 sys.exit(1)
5050
=== modified file 'mojo/cli.py'
--- mojo/cli.py 2018-01-18 10:23:15 +0000
+++ mojo/cli.py 2018-01-25 14:10:22 +0000
@@ -1,6 +1,6 @@
1# Copyright 2014, 2015 Canonical Ltd. This software is licensed under the1# Copyright 2014, 2015 Canonical Ltd. This software is licensed under the
2# GNU General Public License version 3 (see the file LICENSE).2# GNU General Public License version 3 (see the file LICENSE).
33from __future__ import print_function
4import argparse4import argparse
5import errno5import errno
6import logging6import logging
@@ -76,7 +76,7 @@
7676
7777
78def exit_with_usage(parser, message):78def exit_with_usage(parser, message):
79 print(message + "\n", file=sys.stderr)79 sys.stderr.write(message + "\n")
80 parser.print_help(file=sys.stderr)80 parser.print_help(file=sys.stderr)
81 sys.exit(1)81 sys.exit(1)
8282
@@ -268,7 +268,7 @@
268 try:268 try:
269 project.create_workspace(spec_url, workspace, stage=args.stage, logfile=args.logfile)269 project.create_workspace(spec_url, workspace, stage=args.stage, logfile=args.logfile)
270 except OSError:270 except OSError:
271 print("Are you sure the project exists?", file=sys.stderr)271 sys.stderr.write("Are you sure the project exists?")
272 if args.blunt:272 if args.blunt:
273 raise273 raise
274 sys.exit(2)274 sys.exit(2)
@@ -649,9 +649,9 @@
649 sys.exit(1)649 sys.exit(1)
650 except mojo.exceptions.ConfigNotFoundException as e:650 except mojo.exceptions.ConfigNotFoundException as e:
651 logging.error(e.message)651 logging.error(e.message)
652 print("Searched:", file=sys.stderr)652 sys.stderr.write("Searched:")
653 for path in e.search_list:653 for path in e.search_list:
654 print(" {}".format(path), file=sys.stderr)654 sys.stderr.write(" {}".format(path))
655 sys.exit(1)655 sys.exit(1)
656 except (mojo.exceptions.InvalidManifestException,656 except (mojo.exceptions.InvalidManifestException,
657 mojo.phase.FileDoesNotExistException,657 mojo.phase.FileDoesNotExistException,
@@ -672,14 +672,14 @@
672 if ioe.errno != errno.EACCES:672 if ioe.errno != errno.EACCES:
673 raise673 raise
674674
675 print('ERROR: Unable to read container configuration '675 sys.stderr.write('ERROR: Unable to read container configuration '
676 'from {}'.format(filename), file=sys.stderr)676 'from {}'.format(filename))
677677
678 try:678 try:
679 os.listdir('/var/lib/lxc')679 os.listdir('/var/lib/lxc')
680 except OSError as ose:680 except OSError as ose:
681 if ose.errno == errno.EACCES:681 if ose.errno == errno.EACCES:
682 print(dedent(682 sys.stderr.write(dedent(
683 '''\683 '''\
684 ERROR: /var/lib/lxc is not readable; consider one of the following:684 ERROR: /var/lib/lxc is not readable; consider one of the following:
685685
@@ -688,8 +688,7 @@
688 sudo chmod 0750 /var/lib/lxc688 sudo chmod 0750 /var/lib/lxc
689689
690 Option 2:690 Option 2:
691 sudo chmod 0755 /var/lib/lxc'''),691 sudo chmod 0755 /var/lib/lxc'''))
692 file=sys.stderr)
693 sys.exit(1)692 sys.exit(1)
694693
695 try:694 try:
@@ -699,18 +698,17 @@
699 # The problem isn't /var/lib/lxc; it's the container!698 # The problem isn't /var/lib/lxc; it's the container!
700 if ose.errno == errno.EACCES:699 if ose.errno == errno.EACCES:
701 # We know it's not /var/lib/lxc; that was checked above.700 # We know it's not /var/lib/lxc; that was checked above.
702 print(dedent(701 sys.stderr.write(dedent(
703 '''\702 '''\
704 ERROR: {container} exists but is not readable; consider doing the following:703 ERROR: {container} exists but is not readable; consider doing the following:
705704
706 sudo chmod 0755 {container}''').format(705 sudo chmod 0755 {container}''').format(
707 container=container),706 container=container))
708 file=sys.stderr)
709 sys.exit(1)707 sys.exit(1)
710 if ose.errno == errno.ENOENT:708 if ose.errno == errno.ENOENT:
711 # Mojo checks this itself, so this would be very strange.709 # Mojo checks this itself, so this would be very strange.
712 print('ERROR: {} does not exist; was the Mojo project '710 sys.stderr.write('ERROR: {} does not exist; was the Mojo project '
713 'created properly?!'.format(container), file=sys.stderr)711 'created properly?!'.format(container))
714 sys.exit(1)712 sys.exit(1)
715713
716 # Some things are beyond even our powers to diagnose.714 # Some things are beyond even our powers to diagnose.
717715
=== modified file 'mojo/juju/status.py'
--- mojo/juju/status.py 2018-01-18 10:23:15 +0000
+++ mojo/juju/status.py 2018-01-25 14:10:22 +0000
@@ -4,6 +4,7 @@
4import subprocess4import subprocess
5import time5import time
6import yaml6import yaml
7import six
78
8from collections import Counter9from collections import Counter
910
@@ -12,6 +13,8 @@
12try:13try:
13 with open(os.devnull, 'w') as devnull:14 with open(os.devnull, 'w') as devnull:
14 version = subprocess.check_output(['juju', 'version'], stderr=devnull)15 version = subprocess.check_output(['juju', 'version'], stderr=devnull)
16 if six.PY3:
17 version = version.decode('utf-8')
15 major_version = int(version[0])18 major_version = int(version[0])
16except subprocess.CalledProcessError:19except subprocess.CalledProcessError:
17 # In cases where the Juju Status yaml is provided by a file this may legitimately fail20 # In cases where the Juju Status yaml is provided by a file this may legitimately fail
@@ -463,7 +466,6 @@
463 # self.ready() will raise exceptions on error states466 # self.ready() will raise exceptions on error states
464 if self.ready() or not wait_for_steady:467 if self.ready() or not wait_for_steady:
465 return468 return
466
467 logging.info("Waiting for environment to reach steady state")469 logging.info("Waiting for environment to reach steady state")
468 wait(max_wait=timeout)470 wait(max_wait=timeout)
469 logging.info("Environment has reached steady state")471 logging.info("Environment has reached steady state")
470472
=== modified file 'mojo/juju/wait.py'
--- mojo/juju/wait.py 2018-01-18 10:23:15 +0000
+++ mojo/juju/wait.py 2018-01-25 14:10:22 +0000
@@ -31,6 +31,7 @@
31from textwrap import dedent31from textwrap import dedent
32import time32import time
33import yaml33import yaml
34import six
3435
3536
36__version__ = '2.5.0'37__version__ = '2.5.0'
@@ -76,6 +77,8 @@
76 logging.error(err)77 logging.error(err)
77 logging.error("{} failed: {}".format(' '.join(cmd), p.returncode))78 logging.error("{} failed: {}".format(' '.join(cmd), p.returncode))
78 raise JujuWaitException(p.returncode or 43)79 raise JujuWaitException(p.returncode or 43)
80 if six.PY3:
81 out = out.decode('utf-8')
79 return out82 return out
8083
8184
8285
=== modified file 'mojo/phase.py'
--- mojo/phase.py 2018-01-18 11:04:39 +0000
+++ mojo/phase.py 2018-01-25 14:10:22 +0000
@@ -25,6 +25,7 @@
25import mojo.juju25import mojo.juju
26import mojo.juju.utils26import mojo.juju.utils
27import codetree27import codetree
28import six
28from deployer.config import ConfigStack29from deployer.config import ConfigStack
2930
3031
@@ -215,7 +216,7 @@
215 with chdir(workspace.build_dir):216 with chdir(workspace.build_dir):
216 try:217 try:
217 tmpl_vars = {'series': project.series}218 tmpl_vars = {'series': project.series}
218 with tempfile.NamedTemporaryFile() as rendered_collect:219 with tempfile.NamedTemporaryFile(mode='wt') as rendered_collect:
219 with open(config) as inp:220 with open(config) as inp:
220 try:221 try:
221 rendered_collect.write(Template(inp.read()).render(**tmpl_vars))222 rendered_collect.write(Template(inp.read()).render(**tmpl_vars))
@@ -363,6 +364,8 @@
363 command = env_vars + [script]364 command = env_vars + [script]
364 try:365 try:
365 output = subprocess.check_output(command, stderr=subprocess.STDOUT)366 output = subprocess.check_output(command, stderr=subprocess.STDOUT)
367 if six.PY3:
368 output = output.decode('utf-8')
366 except subprocess.CalledProcessError as e:369 except subprocess.CalledProcessError as e:
367 if gather_debug_logs:370 if gather_debug_logs:
368 exception_output = self._gather_debug_logs(e, workspace, stage)371 exception_output = self._gather_debug_logs(e, workspace, stage)
369372
=== modified file 'mojo/tests/test_juju2.py'
--- mojo/tests/test_juju2.py 2018-01-18 10:23:15 +0000
+++ mojo/tests/test_juju2.py 2018-01-25 14:10:22 +0000
@@ -128,19 +128,22 @@
128 def test_model_version(self):128 def test_model_version(self):
129 self.assertEqual(self.status.model_version(), '2.0-beta18')129 self.assertEqual(self.status.model_version(), '2.0-beta18')
130130
131 def test_status(self):131 @mock.patch('subprocess.check_output')
132 def test_status(self, mock_check_output):
132 """Test the status function"""133 """Test the status function"""
133 self.assertEqual(self.status.status(), self.status._raw_status)134 self.assertEqual(self.status.status(), self.status._raw_status)
134135
135 with mock.patch('subprocess.check_output') as mock_check_output:136 with mock.patch('subprocess.check_output') as mock_check_output:
136 # First check with no environment set137 # First check with no environment set
137 no_env_status = mojo.juju.status.Juju2Status(command_timeout=None)138 with mock.patch('mojo.juju.status.major_version', new=2):
138 no_env_status.status()139 no_env_status = mojo.juju.status.Juju2Status(command_timeout=None)
140 no_env_status.status()
139 mock_check_output.assert_called_with(141 mock_check_output.assert_called_with(
140 ['juju', 'status', '--format=yaml'])142 ['juju', 'status', '--format=yaml'])
141 # Second test with an environment set143 # Second test with an environment set
142 env_status = mojo.juju.status.Juju2Status(environment="test-env")144 with mock.patch('mojo.juju.status.major_version', new=2):
143 env_status.status()145 env_status = mojo.juju.status.Juju2Status(environment="test-env")
146 env_status.status()
144 mock_check_output.assert_called_with(147 mock_check_output.assert_called_with(
145 ['/usr/bin/timeout', '--kill-after', '5', '600',148 ['/usr/bin/timeout', '--kill-after', '5', '600',
146 'juju', 'status', '--format=yaml', '-m', 'test-env'])149 'juju', 'status', '--format=yaml', '-m', 'test-env'])
@@ -240,7 +243,12 @@
240 'since': '16 Sep 2016 15:23:55Z'},243 'since': '16 Sep 2016 15:23:55Z'},
241 'relations': {'juju-info': ['apache2', 'prometheus']},244 'relations': {'juju-info': ['apache2', 'prometheus']},
242 'subordinate-to': ['apache2', 'prometheus']}}}245 'subordinate-to': ['apache2', 'prometheus']}}}
243 self.assertEqual(self.status.yaml_status(force_update=True), expected)246 with mock.patch('subprocess.check_output') as mock_check_output:
247 mock_check_output.return_value = self._testdata_from_file('juju2-idle.yaml')
248 self.assertEqual(self.status.yaml_status(force_update=True), expected)
249 # mock_check_output.assert_called_with(
250 # ['/usr/bin/timeout', '--kill-after', '5', '600',
251 # 'juju', 'status', '--format=yaml'])
244252
245253
246class Juju2CheckOutputWithTimeoutTestException(Exception):254class Juju2CheckOutputWithTimeoutTestException(Exception):
@@ -305,7 +313,7 @@
305 timeout_exception=Juju2CheckOutputWithTimeoutTestException)313 timeout_exception=Juju2CheckOutputWithTimeoutTestException)
306314
307 self.assertTrue(315 self.assertTrue(
308 arcm.exception.message.endswith(' timed out after 5 seconds'))316 str(arcm.exception).endswith(' timed out after 5 seconds'))
309317
310 @mock.patch('subprocess.check_output')318 @mock.patch('subprocess.check_output')
311 def test_command_times_out_and_is_killed(self, check_output_mock):319 def test_command_times_out_and_is_killed(self, check_output_mock):
@@ -323,7 +331,7 @@
323 timeout_exception=Juju2CheckOutputWithTimeoutTestException)331 timeout_exception=Juju2CheckOutputWithTimeoutTestException)
324332
325 self.assertTrue(333 self.assertTrue(
326 arcm.exception.message.endswith(' timed out after 5 seconds (killed)'))334 str(arcm.exception).endswith(' timed out after 5 seconds (killed)'))
327335
328336
329class Juju2ChecksTest(TestCase):337class Juju2ChecksTest(TestCase):
@@ -386,7 +394,9 @@
386 sys.argv = ['self', '-f', self.status_path, 'get_ips', 'all']394 sys.argv = ['self', '-f', self.status_path, 'get_ips', 'all']
387 rc = parse_status()395 rc = parse_status()
388 self.assertEqual(rc, 0)396 self.assertEqual(rc, 0)
389 self.assertEqual(mock_stdout.getvalue(), '10.25.2.111\n10.242.242.242\n10.25.2.109\n10.25.2.112\n10.25.2.110\n')397 ips = mock_stdout.getvalue().split("\n")
398 ips.sort()
399 self.assertEqual("{}\n".format("\n".join(ips).lstrip()), '10.242.242.242\n10.25.2.109\n10.25.2.110\n10.25.2.111\n10.25.2.112\n')
390400
391 @mock.patch('sys.stdout', new_callable=StringIO)401 @mock.patch('sys.stdout', new_callable=StringIO)
392 def test_GetIPs_app(self, mock_stdout):402 def test_GetIPs_app(self, mock_stdout):
393403
=== modified file 'mojo/utils.py'
--- mojo/utils.py 2017-12-22 10:31:56 +0000
+++ mojo/utils.py 2018-01-25 14:10:22 +0000
@@ -13,6 +13,7 @@
13import time13import time
1414
15from hashlib import sha115from hashlib import sha1
16import six
16from .exceptions import Unprivileged17from .exceptions import Unprivileged
1718
1819
@@ -262,6 +263,8 @@
262 eof = True263 eof = True
263 for stream in to_read:264 for stream in to_read:
264 line = stream.read(read_bytes)265 line = stream.read(read_bytes)
266 if six.PY3:
267 line = line.decode('utf-8')
265 if line != "":268 if line != "":
266 eof = False269 eof = False
267 output += line270 output += line
268271
=== modified file 'setup.py'
--- setup.py 2017-05-10 03:58:59 +0000
+++ setup.py 2018-01-25 14:10:22 +0000
@@ -21,6 +21,7 @@
21 "jujuclient >= 0.0.7",21 "jujuclient >= 0.0.7",
22 "juju-deployer >= 0.6.4",22 "juju-deployer >= 0.6.4",
23 "pylxd < 2.1.0",23 "pylxd < 2.1.0",
24 "six >= 1.11.0",
24 "python-cinderclient >= 1.0.0"],25 "python-cinderclient >= 1.0.0"],
25 packages=find_packages(),26 packages=find_packages(),
26 classifiers=[27 classifiers=[
2728
=== modified file 'tox.ini'
--- tox.ini 2018-01-18 10:23:15 +0000
+++ tox.ini 2018-01-25 14:10:22 +0000
@@ -1,6 +1,6 @@
1# mojo_exec: build a py3 virtualenv and execute a basic mojo command to confirm py3-only functionality.1# mojo_exec: build a py3 virtualenv and execute a basic mojo command to confirm py3-only functionality.
2[tox]2[tox]
3envlist = mojo_exec3envlist = mojo2_exec,mojo3_exec,py27,py36
4skipsdist = True4skipsdist = True
55
6[testenv]6[testenv]
@@ -10,8 +10,26 @@
10install_command =10install_command =
11 pip install --allow-unverified python-apt {opts} {packages}11 pip install --allow-unverified python-apt {opts} {packages}
1212
13[testenv:mojo_exec]13[testenv:mojo3_exec]
14basepython = python314basepython = python3
15deps = -r{toxinidir}/requirements.txt15deps = -r{toxinidir}/requirements.txt
16commands = pip install --upgrade .16commands = pip install --upgrade .
17 mojo --version17 mojo --version
18
19[testenv:mojo2_exec]
20basepython = python2
21deps = -r{toxinidir}/requirements.txt
22commands = pip install --upgrade .
23 mojo --version
24
25[testenv:py3]
26basepython = python3
27deps = -r{toxinidir}/requirements.txt
28 -r{toxinidir}/test-requirements.txt
29commands = nosetests {posargs} {toxinidir}/mojo/tests
30
31[testenv:py27]
32basepython = python2.7
33deps = -r{toxinidir}/requirements.txt
34 -r{toxinidir}/test-requirements.txt
35commands = nosetests {posargs} {toxinidir}/mojo/tests
18\ No newline at end of file36\ No newline at end of file

Subscribers

People subscribed via source and target branches