Merge lp:~chris.macnaughton/mojo/py3002 into lp:~ost-maintainers/mojo/py3

[Chris MacNaughton]

[lp:~chris.macnaughton/mojo/py3002]

470. By Chris MacNaughton on 2018-01-22

add list() back around map() to make it work on py2+3

[Chris MacNaughton (chris.macnaughton)]

Have confirmed with a basic mojo run that this works on py2 still, commencing a test on py3

[Chris MacNaughton (chris.macnaughton)]

Not quite py3 ready:

2018-01-24 11:10:49 [INFO] Rsyncing /home/ubuntu/mojo-openstack-specs to /srv/mojo/openstack/xenial/_20181024_111047/spec
2018-01-24 11:10:50 [INFO] Retrieve the spec's manifest
2018-01-24 11:10:50 [INFO] Manifest comment:

#############################################################################
Collect the charm branches from Launchpad
#############################################################################

2018-01-24 11:10:50 [INFO] Building resource tree
2018-01-24 11:10:50 [ERROR] Unknown error
Traceback (most recent call last):
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/cli.py", line 628, in run_with_args
    args.func(args)
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/utils.py", line 305, in wrapped
    return method(*args, **kwargs)
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/cli.py", line 346, in run_from_manifest
    manifest.run(project, workspace, args.stage)
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/manifest.py", line 100, in run
    auto_repo=auto_repo)
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/site-packages/mojo/phase.py", line 221, in run
    rendered_collect.write(Template(inp.read()).render(**tmpl_vars))
  File "/home/ubuntu/charm-test-infra/.tox/clients/lib/python3.5/tempfile.py", line 622, in func_wrapper
    return func(*args, **kwargs)
TypeError: a bytes-like object is required, not 'str'

[lp:~chris.macnaughton/mojo/py3002]

471. By Chris MacNaughton on 2018-01-25

decode bytes to str

472. By Chris MacNaughton on 2018-01-25

import six

473. By Chris MacNaughton on 2018-01-25

add encoding to popen

474. By Chris MacNaughton on 2018-01-25

ensure we decode command line output if needed

[Chris MacNaughton (chris.macnaughton)]

Ok, have run the openstack-mojo-specs, ceph base spec with both py2 and py3, and have identical output ;)

=== modified file 'contrib/test/utils.py'
--- contrib/test/utils.py	2018-01-18 10:23:15 +0000
+++ contrib/test/utils.py	2018-01-25 14:10:22 +0000
@@ -44,6 +44,6 @@
     """
     try:
         subprocess.check_output(['juju', 'switch'])
-    except subprocess.CalledProcessError as OSError:
+    except subprocess.CalledProcessError, OSError:
         print('Error checking active Juju controller')
         sys.exit(1)

=== modified file 'mojo/cli.py'
--- mojo/cli.py	2018-01-18 10:23:15 +0000
+++ mojo/cli.py	2018-01-25 14:10:22 +0000
@@ -1,6 +1,6 @@
 # Copyright 2014, 2015 Canonical Ltd.  This software is licensed under the
 # GNU General Public License version 3 (see the file LICENSE).
-
+from __future__ import print_function
 import argparse
 import errno
 import logging
@@ -76,7 +76,7 @@
 
 
 def exit_with_usage(parser, message):
-    print(message + "\n", file=sys.stderr)
+    sys.stderr.write(message + "\n")
     parser.print_help(file=sys.stderr)
     sys.exit(1)
 
@@ -268,7 +268,7 @@
     try:
         project.create_workspace(spec_url, workspace, stage=args.stage, logfile=args.logfile)
     except OSError:
-        print("Are you sure the project exists?", file=sys.stderr)
+        sys.stderr.write("Are you sure the project exists?")
         if args.blunt:
             raise
         sys.exit(2)
@@ -649,9 +649,9 @@
         sys.exit(1)
     except mojo.exceptions.ConfigNotFoundException as e:
         logging.error(e.message)
-        print("Searched:", file=sys.stderr)
+        sys.stderr.write("Searched:")
         for path in e.search_list:
-            print("  {}".format(path), file=sys.stderr)
+            sys.stderr.write("  {}".format(path))
         sys.exit(1)
     except (mojo.exceptions.InvalidManifestException,
             mojo.phase.FileDoesNotExistException,
@@ -672,14 +672,14 @@
         if ioe.errno != errno.EACCES:
             raise
 
-        print('ERROR: Unable to read container configuration '
-              'from {}'.format(filename), file=sys.stderr)
+        sys.stderr.write('ERROR: Unable to read container configuration '
+              'from {}'.format(filename))
 
         try:
             os.listdir('/var/lib/lxc')
         except OSError as ose:
             if ose.errno == errno.EACCES:
-                print(dedent(
+                sys.stderr.write(dedent(
                     '''\
                     ERROR: /var/lib/lxc is not readable; consider one of the following:
 
@@ -688,8 +688,7 @@
                         sudo chmod 0750 /var/lib/lxc
 
                     Option 2:
-                        sudo chmod 0755 /var/lib/lxc'''),
-                      file=sys.stderr)
+                        sudo chmod 0755 /var/lib/lxc'''))
                 sys.exit(1)
 
         try:
@@ -699,18 +698,17 @@
             # The problem isn't /var/lib/lxc; it's the container!
             if ose.errno == errno.EACCES:
                 # We know it's not /var/lib/lxc; that was checked above.
-                print(dedent(
+                sys.stderr.write(dedent(
                     '''\
                     ERROR: {container} exists but is not readable; consider doing the following:
 
                         sudo chmod 0755 {container}''').format(
-                            container=container),
-                      file=sys.stderr)
+                            container=container))
                 sys.exit(1)
             if ose.errno == errno.ENOENT:
                 # Mojo checks this itself, so this would be very strange.
-                print('ERROR: {} does not exist; was the Mojo project '
-                      'created properly?!'.format(container), file=sys.stderr)
+                sys.stderr.write('ERROR: {} does not exist; was the Mojo project '
+                      'created properly?!'.format(container))
                 sys.exit(1)
 
         # Some things are beyond even our powers to diagnose.

=== modified file 'mojo/juju/status.py'
--- mojo/juju/status.py	2018-01-18 10:23:15 +0000
+++ mojo/juju/status.py	2018-01-25 14:10:22 +0000
@@ -4,6 +4,7 @@
 import subprocess
 import time
 import yaml
+import six
 
 from collections import Counter
 
@@ -12,6 +13,8 @@
 try:
     with open(os.devnull, 'w') as devnull:
         version = subprocess.check_output(['juju', 'version'], stderr=devnull)
+        if six.PY3:
+            version = version.decode('utf-8')
     major_version = int(version[0])
 except subprocess.CalledProcessError:
     # In cases where the Juju Status yaml is provided by a file this may legitimately fail
@@ -463,7 +466,6 @@
         # self.ready() will raise exceptions on error states
         if self.ready() or not wait_for_steady:
             return
-
         logging.info("Waiting for environment to reach steady state")
         wait(max_wait=timeout)
         logging.info("Environment has reached steady state")

=== modified file 'mojo/juju/wait.py'
--- mojo/juju/wait.py	2018-01-18 10:23:15 +0000
+++ mojo/juju/wait.py	2018-01-25 14:10:22 +0000
@@ -31,6 +31,7 @@
 from textwrap import dedent
 import time
 import yaml
+import six
 
 
 __version__ = '2.5.0'
@@ -76,6 +77,8 @@
         logging.error(err)
         logging.error("{} failed: {}".format(' '.join(cmd), p.returncode))
         raise JujuWaitException(p.returncode or 43)
+    if six.PY3:
+        out = out.decode('utf-8')
     return out
 
 

=== modified file 'mojo/phase.py'
--- mojo/phase.py	2018-01-18 11:04:39 +0000
+++ mojo/phase.py	2018-01-25 14:10:22 +0000
@@ -25,6 +25,7 @@
 import mojo.juju
 import mojo.juju.utils
 import codetree
+import six
 from deployer.config import ConfigStack
 
 
@@ -215,7 +216,7 @@
         with chdir(workspace.build_dir):
             try:
                 tmpl_vars = {'series': project.series}
-                with tempfile.NamedTemporaryFile() as rendered_collect:
+                with tempfile.NamedTemporaryFile(mode='wt') as rendered_collect:
                     with open(config) as inp:
                         try:
                             rendered_collect.write(Template(inp.read()).render(**tmpl_vars))
@@ -363,6 +364,8 @@
             command = env_vars + [script]
             try:
                 output = subprocess.check_output(command, stderr=subprocess.STDOUT)
+                if six.PY3:
+                    output = output.decode('utf-8')
             except subprocess.CalledProcessError as e:
                 if gather_debug_logs:
                     exception_output = self._gather_debug_logs(e, workspace, stage)

=== modified file 'mojo/tests/test_juju2.py'
--- mojo/tests/test_juju2.py	2018-01-18 10:23:15 +0000
+++ mojo/tests/test_juju2.py	2018-01-25 14:10:22 +0000
@@ -128,19 +128,22 @@
     def test_model_version(self):
         self.assertEqual(self.status.model_version(), '2.0-beta18')
 
-    def test_status(self):
+    @mock.patch('subprocess.check_output')
+    def test_status(self, mock_check_output):
         """Test the status function"""
         self.assertEqual(self.status.status(), self.status._raw_status)
 
         with mock.patch('subprocess.check_output') as mock_check_output:
             # First check with no environment set
-            no_env_status = mojo.juju.status.Juju2Status(command_timeout=None)
-            no_env_status.status()
+            with mock.patch('mojo.juju.status.major_version', new=2):
+                no_env_status = mojo.juju.status.Juju2Status(command_timeout=None)
+                no_env_status.status()
             mock_check_output.assert_called_with(
                 ['juju', 'status', '--format=yaml'])
             # Second test with an environment set
-            env_status = mojo.juju.status.Juju2Status(environment="test-env")
-            env_status.status()
+            with mock.patch('mojo.juju.status.major_version', new=2):
+                env_status = mojo.juju.status.Juju2Status(environment="test-env")
+                env_status.status()
             mock_check_output.assert_called_with(
                 ['/usr/bin/timeout', '--kill-after', '5', '600',
                  'juju', 'status', '--format=yaml', '-m', 'test-env'])
@@ -240,7 +243,12 @@
                                                                  'since': '16 Sep 2016 15:23:55Z'},
                                           'relations': {'juju-info': ['apache2', 'prometheus']},
                                           'subordinate-to': ['apache2', 'prometheus']}}}
-        self.assertEqual(self.status.yaml_status(force_update=True), expected)
+        with mock.patch('subprocess.check_output') as mock_check_output:
+            mock_check_output.return_value = self._testdata_from_file('juju2-idle.yaml')
+            self.assertEqual(self.status.yaml_status(force_update=True), expected)
+            # mock_check_output.assert_called_with(
+            #     ['/usr/bin/timeout', '--kill-after', '5', '600',
+            #      'juju', 'status', '--format=yaml'])
 
 
 class Juju2CheckOutputWithTimeoutTestException(Exception):
@@ -305,7 +313,7 @@
                 timeout_exception=Juju2CheckOutputWithTimeoutTestException)
 
         self.assertTrue(
-            arcm.exception.message.endswith(' timed out after 5 seconds'))
+            str(arcm.exception).endswith(' timed out after 5 seconds'))
 
     @mock.patch('subprocess.check_output')
     def test_command_times_out_and_is_killed(self, check_output_mock):
@@ -323,7 +331,7 @@
                 timeout_exception=Juju2CheckOutputWithTimeoutTestException)
 
         self.assertTrue(
-            arcm.exception.message.endswith(' timed out after 5 seconds (killed)'))
+            str(arcm.exception).endswith(' timed out after 5 seconds (killed)'))
 
 
 class Juju2ChecksTest(TestCase):
@@ -386,7 +394,9 @@
         sys.argv = ['self', '-f', self.status_path, 'get_ips', 'all']
         rc = parse_status()
         self.assertEqual(rc, 0)
-        self.assertEqual(mock_stdout.getvalue(), '10.25.2.111\n10.242.242.242\n10.25.2.109\n10.25.2.112\n10.25.2.110\n')
+        ips = mock_stdout.getvalue().split("\n")
+        ips.sort()
+        self.assertEqual("{}\n".format("\n".join(ips).lstrip()), '10.242.242.242\n10.25.2.109\n10.25.2.110\n10.25.2.111\n10.25.2.112\n')
 
     @mock.patch('sys.stdout', new_callable=StringIO)
     def test_GetIPs_app(self, mock_stdout):

=== modified file 'mojo/utils.py'
--- mojo/utils.py	2017-12-22 10:31:56 +0000
+++ mojo/utils.py	2018-01-25 14:10:22 +0000
@@ -13,6 +13,7 @@
 import time
 
 from hashlib import sha1
+import six
 from .exceptions import Unprivileged
 
 
@@ -262,6 +263,8 @@
         eof = True
         for stream in to_read:
             line = stream.read(read_bytes)
+            if six.PY3:
+                line = line.decode('utf-8')
             if line != "":
                 eof = False
             output += line

=== modified file 'setup.py'
--- setup.py	2017-05-10 03:58:59 +0000
+++ setup.py	2018-01-25 14:10:22 +0000
@@ -21,6 +21,7 @@
                       "jujuclient >= 0.0.7",
                       "juju-deployer >= 0.6.4",
                       "pylxd < 2.1.0",
+                      "six >= 1.11.0",
                       "python-cinderclient >= 1.0.0"],
     packages=find_packages(),
     classifiers=[

=== modified file 'tox.ini'
--- tox.ini	2018-01-18 10:23:15 +0000
+++ tox.ini	2018-01-25 14:10:22 +0000
@@ -1,6 +1,6 @@
 # mojo_exec: build a py3 virtualenv and execute a basic mojo command to confirm py3-only functionality.
 [tox]
-envlist = mojo_exec
+envlist = mojo2_exec,mojo3_exec,py27,py36
 skipsdist = True
 
 [testenv]
@@ -10,8 +10,26 @@
 install_command =
   pip install --allow-unverified python-apt {opts} {packages}
 
-[testenv:mojo_exec]
-basepython = python3
-deps = -r{toxinidir}/requirements.txt
-commands = pip install --upgrade .
-           mojo --version
+[testenv:mojo3_exec]
+basepython = python3
+deps = -r{toxinidir}/requirements.txt
+commands = pip install --upgrade .
+           mojo --version
+
+[testenv:mojo2_exec]
+basepython = python2
+deps = -r{toxinidir}/requirements.txt
+commands = pip install --upgrade .
+           mojo --version
+
+[testenv:py3]
+basepython = python3
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands = nosetests {posargs} {toxinidir}/mojo/tests
+
+[testenv:py27]
+basepython = python2.7
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands = nosetests {posargs} {toxinidir}/mojo/tests
\ No newline at end of file