Merge lp:~sseman/juju-ci-tools/py3-pkg into lp:juju-ci-tools

[Seman]

This branch updates pipdeps.py so that when installing packages, it also installs packages needed for Python3 from the requirements_py3.txt file.

Currently, the main package needed for Python 3 is https://pypi.python.org/pypi/juju/0.1.2. I have uploaded this package and the needed dependencies to S3 pip archive bucket. Before installing the packages for Python 3, first it checked if the machine have Python 3 (ver 3.5 or above) and pip3 installed. If they are not installed, it skips installing the packages.

Also, if pipdeps.py is running inside virtualenv, the "--user" option for pip causes an issue. I have added a code to detect and remove "--user" option when running within virtualenv.

Finally, I updated Makefile to run pipdeps.py in verbose mode. When I was having issues, I had to manually run the command to see why pip was failing.

[Curtis Hovey (sinzui)]

Thank you. I wonder how precise will handle this given it does have an old version of py3 installed.

review: Approve (code)

=== modified file 'Makefile'
--- Makefile	2016-12-06 04:54:17 +0000
+++ Makefile	2017-01-09 20:34:36 +0000
@@ -20,7 +20,7 @@
 install-deps: juju-ci-tools.common_0.1.4-0_all.deb apt-update
 	sudo dpkg -i juju-ci-tools.common_0.1.4-0_all.deb || true
 	sudo apt-get install -y -f
-	./pipdeps.py install
+	./pipdeps.py -v install
 name=NAMEHERE
 assess_file=assess_$(name).py
 test_assess_file=tests/test_assess_$(name).py

=== modified file 'pipdeps.py'
--- pipdeps.py	2016-06-23 02:50:43 +0000
+++ pipdeps.py	2017-01-09 20:34:36 +0000
@@ -4,6 +4,7 @@
 from __future__ import print_function
 
 import argparse
+from distutils.version import StrictVersion
 import os
 import platform
 import subprocess
@@ -19,16 +20,23 @@
 PREFIX = "juju-ci-tools/"
 REQUIREMENTS = os.path.join(os.path.realpath(os.path.dirname(__file__)),
                             "requirements.txt")
+REQUIREMENTS_PY3 = os.path.join(os.path.realpath(os.path.dirname(__file__)),
+                                "requirements_py3.txt")
 MAC_WIN_REQS = os.path.join(os.path.realpath(os.path.dirname(__file__)),
                             "mac-win-requirements.txt")
 OBSOLETE = os.path.join(os.path.realpath(os.path.dirname(__file__)),
                         "obsolete-requirements.txt")
 
 
-def get_requirements():
+def get_requirements(python3=False):
+    """Return requirements file path."""
     if platform.dist()[0] in ('Ubuntu', 'debian'):
+        if python3:
+            return REQUIREMENTS_PY3
         return REQUIREMENTS
     else:
+        if python3:
+            return None
         return MAC_WIN_REQS
 
 
@@ -50,9 +58,34 @@
     return boto.s3.connection.S3Connection(access_key, secret_key)
 
 
-def run_pip_install(extra_args, requirements, verbose=False):
+def is_py3_supported():
+    """Determine if Python3 and pip3 are installed."""
+    try:
+        version = subprocess.check_output(
+            ['python3', '--version'], stderr=subprocess.STDOUT)
+    except OSError:
+        return False
+    version = version.strip().split()[-1]
+    if StrictVersion(version) < StrictVersion('3.5'):
+        return False
+    try:
+        subprocess.check_call(['pip3', '--version'])
+    except OSError:
+        return False
+    return True
+
+
+def run_pip3_install(extra_args, requirements, verbose=False):
+    if requirements and is_py3_supported():
+        run_pip_install(extra_args, requirements, verbose=verbose,
+                        cmd=['pip3'])
+    else:
+        print('Python 3 is not installed.')
+
+
+def run_pip_install(extra_args, requirements, verbose=False, cmd=None):
     """Run pip install in a subprocess with given additional arguments."""
-    cmd = ["pip"]
+    cmd = cmd or ['pip']
     if not verbose:
         cmd.append("-q")
     cmd.extend(["install", "-r", requirements])
@@ -85,15 +118,29 @@
         subprocess.check_call(uninstall_cmd)
 
 
-def command_install(bucket, requirements, verbose=False):
+def get_pip_args(archives_url):
+    args = ["--no-index", "--find-links", archives_url]
+    # --user option is invalid when running inside virtualenv. Check
+    # sys.real_prefix to determine if it is executing inside virtualenv.
+    # Inside virtualenv, the sys.prefix points to the virtualenv directory
+    # and sys.real_prefix points to the real prefix.
+    # Running outside a virtualenv, sys should not have real_prefix attribute.
+    if not hasattr(sys, 'real_prefix'):
+        args.append("--user")
+    return args
+
+
+def command_install(bucket, requirements, verbose=False,
+                    requirements_py3=None):
     with utility.temp_dir() as archives_dir:
         for key in bucket.list(prefix=PREFIX):
             archive = key.name[len(PREFIX):]
             key.get_contents_to_filename(os.path.join(archives_dir, archive))
         archives_url = "file://" + archives_dir
+        pip_args = get_pip_args(archives_url)
         run_pip_uninstall(OBSOLETE)
-        run_pip_install(["--user", "--no-index", "--find-links", archives_url],
-                        requirements, verbose=verbose)
+        run_pip_install(pip_args, requirements, verbose=verbose)
+        run_pip3_install(pip_args, requirements_py3, verbose=verbose)
 
 
 def command_update(s3, requirements, verbose=False):
@@ -136,6 +183,10 @@
     parser.add_argument(
         "--requirements", default=get_requirements(), type=os.path.expanduser,
         help="Location requirements file to use.")
+    parser.add_argument(
+        "--requirements_py3", default=get_requirements(python3=True),
+        type=os.path.expanduser,
+        help="Location requirements file to use for Python 3.")
     subparsers = parser.add_subparsers(dest="command")
     subparsers.add_parser("install", help="Download deps from S3 and install.")
     subparsers.add_parser(
@@ -157,7 +208,8 @@
     else:
         bucket = s3.get_bucket(BUCKET)
         if args.command == "install":
-            command_install(bucket, args.requirements, args.verbose)
+            command_install(
+                bucket, args.requirements, args.verbose, args.requirements_py3)
         elif args.command == "list":
             command_list(bucket, args.verbose)
         elif args.command == "delete":

=== modified file 'requirements_py3.txt'
--- requirements_py3.txt	2016-12-10 00:47:37 +0000
+++ requirements_py3.txt	2017-01-09 20:34:36 +0000
@@ -1,1 +1,2 @@
-juju>=0.0.1
\ No newline at end of file
+juju>=0.1.2
+

=== modified file 'tests/test_pipdeps.py'
--- tests/test_pipdeps.py	2016-06-23 02:50:43 +0000
+++ tests/test_pipdeps.py	2017-01-09 20:34:36 +0000
@@ -2,10 +2,12 @@
 
 import mock
 import os
+import subprocess
 import unittest
 
 
 import pipdeps
+import tests
 import utility
 
 
@@ -69,6 +71,10 @@
         os.path.realpath(os.path.dirname(pipdeps.__file__)),
         "requirements.txt")
 
+    req_path_py3 = os.path.join(
+        os.path.realpath(os.path.dirname(pipdeps.__file__)),
+        "requirements_py3.txt")
+
     def test_added_args(self):
         with mock.patch("subprocess.check_call", autospec=True) as cc_mock:
             pipdeps.run_pip_install(["--user"], self.req_path)
@@ -82,6 +88,18 @@
         cc_mock.assert_called_once_with([
             "pip", "install", "-r", self.req_path, "--download", "/tmp/pip"])
 
+    def test_pip3_install(self):
+        with mock.patch("subprocess.check_output", autospec=True,
+                        return_value='3.5.0') as co_mock:
+            with mock.patch("subprocess.check_call", autospec=True) as cc_mock:
+                pipdeps.run_pip3_install(["--user"], self.req_path_py3)
+        co_mock.assert_called_once_with(
+            ['python3', '--version'], stderr=subprocess.STDOUT)
+        calls = [mock.call(['pip3', '--version']),
+                 mock.call(['pip3', '-q', 'install', '-r', self.req_path_py3,
+                            '--user'])]
+        self.assertEqual(cc_mock.call_args_list, calls)
+
 
 class TestRunPipUninstall(unittest.TestCase):
 
@@ -117,3 +135,43 @@
             self.assertEqual(pipdeps.MAC_WIN_REQS, pipdeps.get_requirements())
 
             self.assertEqual(pipdeps.MAC_WIN_REQS, pipdeps.get_requirements())
+
+
+class TestIsPy3Supported(tests.TestCase):
+
+    def test_is_py3_supported(self):
+        with mock.patch("subprocess.check_output", autospec=True,
+                        return_value='3.5.0') as co_mock:
+            with mock.patch("subprocess.check_call", autospec=True) as cc_mock:
+                self.assertTrue(pipdeps.is_py3_supported())
+        co_mock.assert_called_once_with(
+            ['python3', '--version'], stderr=subprocess.STDOUT)
+        cc_mock.assert_called_once_with(['pip3', '--version'])
+
+    def test_is_py3_supported_older_python3_version(self):
+        with mock.patch("subprocess.check_output", autospec=True,
+                        return_value='3.4') as co_mock:
+            with mock.patch("subprocess.check_call", autospec=True) as cc_mock:
+                self.assertFalse(pipdeps.is_py3_supported())
+        co_mock.assert_called_once_with(
+            ['python3', '--version'], stderr=subprocess.STDOUT)
+        self.assertEqual(cc_mock.call_count, 0)
+
+    def test_is_pyt3_supported_python3_not_installed(self):
+        with mock.patch("subprocess.check_output", autospec=True,
+                        side_effect=OSError(2, 'No such file')) as co_mock:
+            with mock.patch("subprocess.check_call", autospec=True) as cc_mock:
+                self.assertFalse(pipdeps.is_py3_supported())
+        co_mock.assert_called_once_with(
+            ['python3', '--version'], stderr=subprocess.STDOUT)
+        self.assertEqual(cc_mock.call_count, 0)
+
+    def test_is_pyt3_supported_pip3_not_installed(self):
+        with mock.patch("subprocess.check_output", autospec=True,
+                        return_value='3.5.2') as co_mock:
+            with mock.patch("subprocess.check_call", autospec=True,
+                            side_effect=OSError(2, 'No such file')) as cc_mock:
+                self.assertFalse(pipdeps.is_py3_supported())
+        co_mock.assert_called_once_with(
+            ['python3', '--version'], stderr=subprocess.STDOUT)
+        cc_mock.assert_called_once_with(['pip3', '--version'])