Merge lp:~dobey/ubuntuone-dev-tools/update-4-0 into lp:ubuntuone-dev-tools/stable-4-0

[dobey]

[Brian Curtin]

    - Store a list for ignore keys rather than a list on Python 2 and an iterator on Python 3
    - Add support for a -3 flag to enable the tests to run against Python 3
    - Enable unicode_literals for accurate text/bytes distinction to work with Python 3
    - Prepare u1lint for Python 3 usage. Update imports for 3x and use the print function.
    - Convert unicode type usage to use the bytes/text compatibility layer.
    - Update octal literal use for Python 3
    - Import StringIO from the io module for Python 3 compatibility
    - Use items instead of iteritems for Python 3 compatibility.
    - Adjust exception handling to work on Python 3
    - Change to Python 3's print function usage via "from __future__ import print_function"
    - Move long usage to int.
    - Import urllib names by trying Python 3 first and falling back to Python 2
    - Add a small compatibility module for Python 2 and 3 Unicode/bytes distinction
    - Remove xrange usage, instead moving to range regardless of version.

[Eric Casteleijn (thisfred)]

review: Approve

[lp:~dobey/ubuntuone-dev-tools/update-4-0]

81. By Brian Curtin on 2012-08-28

[Brian Curtin]

    - Store a list for ignore keys rather than a list on Python 2 and an iterator on Python 3
    - Add support for a -3 flag to enable the tests to run against Python 3
    - Enable unicode_literals for accurate text/bytes distinction to work with Python 3
    - Prepare u1lint for Python 3 usage. Update imports for 3x and use the print function.
    - Convert unicode type usage to use the bytes/text compatibility layer.
    - Update octal literal use for Python 3
    - Import StringIO from the io module for Python 3 compatibility
    - Use items instead of iteritems for Python 3 compatibility.
    - Adjust exception handling to work on Python 3
    - Change to Python 3's print function usage via "from __future__ import print_function"
    - Move long usage to int.
    - Import urllib names by trying Python 3 first and falling back to Python 2
    - Add a small compatibility module for Python 2 and 3 Unicode/bytes distinction
    - Remove xrange usage, instead moving to range regardless of version.

=== modified file 'bin/u1lint'
--- bin/u1lint	2012-06-08 11:55:53 +0000
+++ bin/u1lint	2012-08-28 20:35:21 +0000
@@ -28,7 +28,15 @@
 # files in the program, then also delete it here.
 """Wrapper script for pylint command."""
 
-import ConfigParser
+from __future__ import print_function
+
+# pylint: disable=F0401
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+# pylint: disable=F0401
+
 import os
 import subprocess
 import sys
@@ -54,31 +62,34 @@
             return path
 
     # pylint: disable=F0401
-    import _winreg
+    try:
+        import winreg
+    except ImportError:
+        import _winreg as winreg
     # pylint: enable=F0401
-    software_key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, 'Software')
+    software_key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, 'Software')
     python_key = None
     try:
-        python_key = _winreg.OpenKey(software_key, 'Python')
+        python_key = winreg.OpenKey(software_key, 'Python')
     # pylint: disable=E0602
     except WindowsError:
     # pylint: enable=E0602
         try:
             # look in the WoW6432node, we are running python
             # 32 on a 64 machine
-            wow6432node_key = _winreg.OpenKey(software_key, 'WoW6432Node')
-            python_key = _winreg.OpenKey(wow6432node_key, 'Python')
+            wow6432node_key = winreg.OpenKey(software_key, 'WoW6432Node')
+            python_key = winreg.OpenKey(wow6432node_key, 'Python')
         # pylint: disable=E0602
         except WindowsError:
         # pylint: enable=E0602
             raise InvalidSetupException(
                 'Could not located python installation path.')
     try:
-        core_key = _winreg.OpenKey(python_key, 'PythonCore')
+        core_key = winreg.OpenKey(python_key, 'PythonCore')
         # pylint: disable=E1101
-        version_key = _winreg.OpenKey(core_key, sys.winver)
+        version_key = winreg.OpenKey(core_key, sys.winver)
         # pylint: enable=E1101
-        return _winreg.QueryValue(version_key, 'InstallPath')
+        return winreg.QueryValue(version_key, 'InstallPath')
     # pylint: disable=E0602
     except WindowsError:
     # pylint: enable=E0602
@@ -139,13 +150,13 @@
 def _read_pylintrc_ignored():
     """Get the ignored files list from pylintrc"""
     try:
-        config = ConfigParser.ConfigParser()
+        config = configparser.ConfigParser()
         config.read([PYLINTRC])
 
         # pylint: disable=E1103
         return [os.path.join(SRCDIR, item) for item in
                     config.get("MASTER", "ignore").split(",")]
-    except (TypeError, ConfigParser.NoOptionError):
+    except (TypeError, configparser.NoOptionError):
         return []
     # pylint: enable=E1103
 
@@ -245,10 +256,9 @@
 
     output = "".join(notices.readlines())
     if output != "":
-        print "== Python Lint Notices =="
+        print("== Python Lint Notices ==")
         (failed, grouped) = _group_lines_by_file(output)
-        print grouped
-        print ""
+        print(grouped, end="\n\n")
 
     returncode = sp.wait()
     # XXX Testing that W0511 does not cause a failure

=== modified file 'run-tests'
--- run-tests	2012-07-27 19:02:53 +0000
+++ run-tests	2012-08-28 20:35:21 +0000
@@ -17,6 +17,11 @@
 
 PYTHON="python"
 
+if [ "$1" = "-3" ]
+then
+    PYTHON="python3"
+fi
+
 $PYTHON bin/u1trial -i "test_squid_windows.py" -c ubuntuone
 $PYTHON bin/u1trial --reactor=twisted -i "test_squid_windows.py" ubuntuone
 echo "Running style checks..."

=== added file 'ubuntuone/devtools/compat.py'
--- ubuntuone/devtools/compat.py	1970-01-01 00:00:00 +0000
+++ ubuntuone/devtools/compat.py	2012-08-28 20:35:21 +0000
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2012 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# In addition, as a special exception, the copyright holders give
+# permission to link the code of portions of this program with the
+# OpenSSL library under certain conditions as described in each
+# individual source file, and distribute linked combinations
+# including the two.
+# You must obey the GNU General Public License in all respects
+# for all of the code used other than OpenSSL.  If you modify
+# file(s) with this exception, you may extend this exception to your
+# version of the file(s), but you are not obligated to do so.  If you
+# do not wish to do so, delete this exception statement from your
+# version.  If you delete this exception statement from all source
+# files in the program, then also delete it here.
+"""Python 2 and 3 compatibility."""
+
+from __future__ import unicode_literals
+
+# The following approach was outlined in Lennart Regebro's
+# "Porting to Python 3" book.
+# http://python3porting.com/noconv.html#more-bytes-strings-and-unicode
+
+import sys
+
+# Disable redefined builtin, invalid name warning
+# pylint: disable=W0622,C0103
+
+if sys.version_info < (3,):
+    text_type = unicode
+    binary_type = str
+    basestring = basestring
+else:
+    text_type = str
+    binary_type = bytes
+    basestring = str

=== modified file 'ubuntuone/devtools/handlers.py'
--- ubuntuone/devtools/handlers.py	2012-04-04 19:13:46 +0000
+++ ubuntuone/devtools/handlers.py	2012-08-28 20:35:21 +0000
@@ -31,6 +31,8 @@
 # files in the program, then also delete it here.
 """Set of helpers handlers."""
 
+from __future__ import print_function
+
 import logging
 
 
@@ -51,15 +53,15 @@
     def dump_contents(self, msgs):
         """Dumps the contents of the MementoHandler."""
         if self.debug:
-            print "Expecting:"
+            print("Expecting:")
             for msg in msgs:
-                print "\t", msg
-            print "MementoHandler contents:"
+                print("\t", msg)
+            print("MementoHandler contents:")
             for rec in self.records:
-                print "\t", rec.exc_info
-                print "\t", logging.getLevelName(rec.levelno)
-                print "\t\t", rec.message
-                print "\t\t", rec.exc_text
+                print("\t", rec.exc_info)
+                print("\t", logging.getLevelName(rec.levelno))
+                print("\t\t", rec.message)
+                print("\t\t", rec.exc_text)
 
     def check(self, level, *msgs):
         """Verifies that the msgs are logged in the specified level"""

=== modified file 'ubuntuone/devtools/runners/__init__.py'
--- ubuntuone/devtools/runners/__init__.py	2012-08-13 21:07:54 +0000
+++ ubuntuone/devtools/runners/__init__.py	2012-08-28 20:35:21 +0000
@@ -39,6 +39,7 @@
 from ubuntuone.devtools.errors import TestError, UsageError
 from ubuntuone.devtools.testing.txcheck import TXCheckSuite
 from ubuntuone.devtools.utils import OptionParser
+from ubuntuone.devtools.compat import text_type
 
 __all__ = ['BaseTestOptions', 'BaseTestRunner', 'main']
 
@@ -173,7 +174,7 @@
         if config['loop']:
             old_suite = suite
             suite = unittest.TestSuite()
-            for _ in xrange(config['loop']):
+            for _ in range(config['loop']):
                 suite.addTest(old_suite)
 
         return suite
@@ -212,18 +213,18 @@
         """Comma-separate list of test modules to ignore,
            e.g: test_gtk.py, test_account.py
            """
-        self['ignore-modules'] = map(unicode.strip, option.split(','))
+        self['ignore-modules'] = list(map(text_type.strip, option.split(',')))
 
     def opt_ignore_paths(self, option):
         """Comma-separated list of relative paths to ignore,
            e.g: tests/platform/windows, tests/platform/macosx
            """
-        self['ignore-paths'] = map(unicode.strip, option.split(','))
+        self['ignore-paths'] = list(map(text_type.strip, option.split(',')))
 
     def opt_loop(self, option):
         """Loop tests the specified number of times."""
         try:
-            self['loop'] = long(option)
+            self['loop'] = int(option)
         except ValueError:
             raise UsageError('A positive integer value must be specified.')
 

=== modified file 'ubuntuone/devtools/services/dbus.py'
--- ubuntuone/devtools/services/dbus.py	2012-03-30 17:44:03 +0000
+++ ubuntuone/devtools/services/dbus.py	2012-08-28 20:35:21 +0000
@@ -34,7 +34,13 @@
 import subprocess
 
 from distutils.spawn import find_executable
-from urllib import quote
+
+# pylint: disable=F0401,E0611
+try:
+    from urllib.parse import quote
+except ImportError:
+    from urllib import quote
+# pylint: enable=F0401,E0611
 
 from ubuntuone.devtools.services import find_config_file
 DBUS_CONFIG_FILE = 'dbus-session.conf.in'

=== modified file 'ubuntuone/devtools/services/squid.py'
--- ubuntuone/devtools/services/squid.py	2012-06-05 20:52:54 +0000
+++ ubuntuone/devtools/services/squid.py	2012-08-28 20:35:21 +0000
@@ -27,6 +27,8 @@
 # files in the program, then also delete it here.
 """Utilities for finding and running a squid proxy for testing."""
 
+from __future__ import print_function
+
 import random
 import signal
 # pylint:disable=W0402
@@ -263,7 +265,7 @@
     def _is_squid_running(self):
         """Return if squid is running."""
         squid_args = ['-k', 'check', '-f', self.config_file]
-        print 'Starting squid version...'
+        print('Starting squid version...')
         message = 'Waiting for squid to start...'
         for timeout in (0.4, 0.1, 0.1, 0.2, 0.5, 1, 3, 5):
             try:
@@ -274,7 +276,7 @@
                 return True
             except subprocess.CalledProcessError:
                 message += '.'
-                print message
+                print(message)
                 time.sleep(timeout)
         return False
 

=== modified file 'ubuntuone/devtools/services/tests/test_dbus.py'
--- ubuntuone/devtools/services/tests/test_dbus.py	2012-03-30 17:44:03 +0000
+++ ubuntuone/devtools/services/tests/test_dbus.py	2012-08-28 20:35:21 +0000
@@ -32,7 +32,13 @@
 
 from ubuntuone.devtools.testcases.dbus import DBusTestCase
 from ubuntuone.devtools.services.dbus import DBusRunner
-from urllib import unquote
+
+# pylint: disable=F0401,E0611
+try:
+    from urllib.parse import unquote
+except ImportError:
+    from urllib import unquote
+# pylint: enable=F0401,E0611
 
 
 class TestWithDBus(DBusTestCase):

=== modified file 'ubuntuone/devtools/services/tests/test_squid.py'
--- ubuntuone/devtools/services/tests/test_squid.py	2012-04-24 17:58:16 +0000
+++ ubuntuone/devtools/services/tests/test_squid.py	2012-08-28 20:35:21 +0000
@@ -30,7 +30,7 @@
 import json
 import os
 
-from StringIO import StringIO
+from io import BytesIO
 
 from twisted.internet import defer
 
@@ -237,7 +237,7 @@
         self._assert_missing_binary('htpasswd')
 
 
-class Pipe(StringIO):
+class Pipe(BytesIO):
     """A read write pipe."""
 
     def read(self):
@@ -364,8 +364,8 @@
     def test_start_error(self):
         """Test that we do raise an exception correctly."""
         # set the error in the pipes
-        out = 'Normal out'
-        err = 'Error goes here'
+        out = b'Normal out'
+        err = b'Error goes here'
         self.subprocess.stdout.write(out)
         self.subprocess.stderr.write(err)
         no_op = lambda _: None

=== modified file 'ubuntuone/devtools/testcases/__init__.py'
--- ubuntuone/devtools/testcases/__init__.py	2012-06-08 11:55:53 +0000
+++ ubuntuone/devtools/testcases/__init__.py	2012-08-28 20:35:21 +0000
@@ -169,19 +169,19 @@
             return
         # change perms to rw, so we can delete the temp dir
         if path != getattr(self, '__root', None):
-            os.chmod(os.path.dirname(path), 0755)
+            os.chmod(os.path.dirname(path), 0o755)
         if not os.access(path, os.W_OK):
-            os.chmod(path, 0755)
+            os.chmod(path, 0o755)
         # pylint: disable=W0612
         for dirpath, dirs, files in os.walk(path):
             for dirname in dirs:
                 if not os.access(os.path.join(dirpath, dirname), os.W_OK):
-                    os.chmod(os.path.join(dirpath, dirname), 0777)
+                    os.chmod(os.path.join(dirpath, dirname), 0o777)
         shutil.rmtree(path)
 
     def makedirs(self, path):
         """Custom makedirs that handle ro parent."""
         parent = os.path.dirname(path)
         if os.path.exists(parent):
-            os.chmod(parent, 0755)
+            os.chmod(parent, 0o755)
         os.makedirs(path)

=== modified file 'ubuntuone/devtools/testcases/dbus.py'
--- ubuntuone/devtools/testcases/dbus.py	2012-07-27 19:02:53 +0000
+++ ubuntuone/devtools/testcases/dbus.py	2012-08-28 20:35:21 +0000
@@ -45,7 +45,7 @@
 # pylint: disable=F0401,C0103,W0406,E0611
 try:
     import dbus
-except ImportError, e:
+except ImportError as e:
     dbus = None
 
 try:
@@ -93,7 +93,13 @@
         yield super(DBusTestCase, self).setUp()
 
         # We need to ensure DBUS_SESSION_BUS_ADDRESS is private here
-        from urllib import unquote
+        # pylint: disable=F0401,E0611
+        try:
+            from urllib.parse import unquote
+        except ImportError:
+            from urllib import unquote
+        # pylint: enable=F0401,E0611
+
         bus_address = os.environ.get('DBUS_SESSION_BUS_ADDRESS', None)
         if os.path.dirname(unquote(bus_address.split(',')[0].split('=')[1])) \
                 != os.path.dirname(os.getcwd()):

=== modified file 'ubuntuone/devtools/testing/tests/test_txwebserver.py'
--- ubuntuone/devtools/testing/tests/test_txwebserver.py	2012-04-17 15:53:14 +0000
+++ ubuntuone/devtools/testing/tests/test_txwebserver.py	2012-08-28 20:35:21 +0000
@@ -28,6 +28,7 @@
 
 """Test the twisted webserver."""
 
+from __future__ import unicode_literals
 
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
@@ -36,13 +37,13 @@
 from ubuntuone.devtools.testing.txwebserver import HTTPWebServer
 
 
-SAMPLE_KEY = "result"
-SAMPLE_VALUE = "sample result"
-SAMPLE_RESOURCE = '{{"{0}": "{1}"}}'.format(SAMPLE_KEY, SAMPLE_VALUE)
-SIMPLERESOURCE = "simpleresource"
-OTHER_SIMPLERESOURCE = "othersimpleresource"
-THROWERROR = "throwerror"
-UNAUTHORIZED = "unauthorized"
+SAMPLE_KEY = b"result"
+SAMPLE_VALUE = b"sample result"
+SAMPLE_RESOURCE = b'{{"{0}": "{1}"}}'.format(SAMPLE_KEY, SAMPLE_VALUE)
+SIMPLERESOURCE = b"simpleresource"
+OTHER_SIMPLERESOURCE = b"othersimpleresource"
+THROWERROR = b"throwerror"
+UNAUTHORIZED = b"unauthorized"
 
 # no init
 # pylint: disable=W0232
@@ -76,7 +77,7 @@
         root.putChild(UNAUTHORIZED, unauthorized_resource)
         self.server = HTTPWebServer(root)
         self.server.start()
-        self.uri = "http://127.0.0.1:{port}/".format(
+        self.uri = b"http://127.0.0.1:{port}/".format(
                                                    port=self.server.get_port())
         self.addCleanup(self.server.stop)
 
@@ -104,7 +105,7 @@
 
     def test_get_iri(self):
         """Test getting the iri from the server."""
-        expected = u"http://127.0.0.1:{port}/"
+        expected = "http://127.0.0.1:{port}/"
         self.assert_url(expected)
 
     def test_get_port(self):
@@ -133,7 +134,7 @@
 
     def get_uri(self, server):
         """Return the uri for the server."""
-        url = "http://127.0.0.1:{port}/"
+        url = b"http://127.0.0.1:{port}/"
         return url.format(port=server.get_port())
 
     @defer.inlineCallbacks

=== modified file 'ubuntuone/devtools/testing/txcheck.py'
--- ubuntuone/devtools/testing/txcheck.py	2012-03-30 17:44:03 +0000
+++ ubuntuone/devtools/testing/txcheck.py	2012-08-28 20:35:21 +0000
@@ -70,7 +70,7 @@
     def __hash__(self):
         """Return hash."""
         member_hash = 0
-        for (key, value) in self.__dict__.iteritems():
+        for (key, value) in self.__dict__.items():
             member_hash ^= hash(key) ^ hash(value)
         return hash(type(self)) ^ member_hash
 

=== modified file 'ubuntuone/devtools/testing/txwebserver.py'
--- ubuntuone/devtools/testing/txwebserver.py	2012-04-26 11:23:19 +0000
+++ ubuntuone/devtools/testing/txwebserver.py	2012-08-28 20:35:21 +0000
@@ -28,6 +28,8 @@
 
 """A tx based web server."""
 
+from __future__ import unicode_literals
+
 from twisted.internet import defer, reactor, ssl
 from twisted.protocols.policies import WrappingFactory
 from twisted.web import server
@@ -65,7 +67,7 @@
 
     def get_iri(self):
         """Build the iri for this mock server."""
-        return u"{scheme}://127.0.0.1:{port}/".format(scheme=self.scheme,
+        return "{scheme}://127.0.0.1:{port}/".format(scheme=self.scheme,
                                                    port=self.get_port())
 
     def get_port(self):