Merge lp:~jelmer/brz/some-more into lp:brz

[Jelmer Vernooij]

Fix a couple more tests on Python 3.

Fix a couple more UI-y tests on Python 3.

[Martin Packman (gz)]

I think I'd looked at some of these changes before, all seems reasonable to me. There's messiness but it's not new messiness.

review: Approve

[The Breezy Bot (the-breezy-bot)]

Merging failed
https://ci.breezy-vcs.org/job/land-brz/451/

=== modified file 'breezy/cache_utf8.py'
--- breezy/cache_utf8.py	2018-08-13 01:38:21 +0000
+++ breezy/cache_utf8.py	2018-09-11 03:12:36 +0000
@@ -108,8 +108,9 @@
     # real Unicode string. Unicode and plain strings of this type will have the
     # same hash, so we can just use it as the key in _uni_to_utf8, but we need
     # the return value to be different in _utf8_to_uni
-    ascii_str = _uni_to_utf8.setdefault(ascii_str, ascii_str)
-    _utf8_to_uni.setdefault(ascii_str, ascii_str.decode('ascii'))
+    uni_str = ascii_str.decode('ascii')
+    ascii_str = _uni_to_utf8.setdefault(uni_str, ascii_str)
+    _utf8_to_uni.setdefault(ascii_str, uni_str)
     return ascii_str
 
 
 
=== modified file 'breezy/commands.py'
--- breezy/commands.py	2018-07-26 01:01:49 +0000
+++ breezy/commands.py	2018-09-11 03:12:36 +0000
@@ -901,7 +901,7 @@
     else:
         args = argv
 
-    # for python 2.5 and later, optparse raises this exception if a non-ascii
+    # python 2's optparse raises this exception if a non-ascii
     # option name is given.  See http://bugs.python.org/issue2931
     try:
         options, args = parser.parse_args(args)
 
=== modified file 'breezy/tests/blackbox/test_exceptions.py'
--- breezy/tests/blackbox/test_exceptions.py	2018-08-26 11:21:16 +0000
+++ breezy/tests/blackbox/test_exceptions.py	2018-09-11 03:12:36 +0000
@@ -54,7 +54,7 @@
             raise tests.TestNotApplicable("Needs system beholden to C locales")
         if PY3:
             raise tests.TestNotApplicable("Unable to pass argv to subprocess as bytes")
-        out, err = self.run_bzr_subprocess(["\xa0"],
+        out, err = self.run_bzr_subprocess([b"\xa0"],
             env_changes={"LANG": "C", "LC_ALL": "C"},
             universal_newlines=True,
             retcode=errors.EXIT_ERROR)
@@ -75,8 +75,11 @@
     "Test that we handle http://bugs.python.org/issue2931"
 
     def test_nonascii_optparse(self):
-        """Reasonable error raised when non-ascii in option name"""
-        error_re = 'Only ASCII permitted in option names'
+        """Reasonable error raised when non-ascii in option name on Python 2"""
+        if PY3:
+            error_re = u'no such option: -\xe4'
+        else:
+            error_re = 'Only ASCII permitted in option names'
         out = self.run_bzr_error([error_re], ['st', u'-\xe4'])
 
 
 
=== modified file 'breezy/tests/test_urlutils.py'
--- breezy/tests/test_urlutils.py	2018-08-05 22:33:53 +0000
+++ breezy/tests/test_urlutils.py	2018-09-11 03:12:36 +0000
@@ -126,11 +126,12 @@
         eq('http://host/~bob%2525-._',
                 normalize_url(u'http://host/%7Ebob%2525%2D%2E%5F'))
 
-        # Normalize verifies URLs when they are not unicode
-        # (indicating they did not come from the user)
-        self.assertRaises(urlutils.InvalidURL, normalize_url,
-                'http://host/\xb5')
-        self.assertRaises(urlutils.InvalidURL, normalize_url, 'http://host/ ')
+        if not PY3:
+            # On Python 2, normalize verifies URLs when they are not unicode
+            # (indicating they did not come from the user)
+            self.assertRaises(urlutils.InvalidURL, normalize_url,
+                    b'http://host/\xb5')
+            self.assertRaises(urlutils.InvalidURL, normalize_url, b'http://host/ ')
 
     def test_url_scheme_re(self):
         # Test paths that may be URLs
 
=== modified file 'breezy/urlutils.py'
--- breezy/urlutils.py	2018-08-19 12:24:17 +0000
+++ breezy/urlutils.py	2018-09-11 03:12:36 +0000
@@ -38,6 +38,7 @@
 """)
 
 from .sixish import (
+    int2byte,
     PY3,
     text_type,
     )
@@ -700,10 +701,10 @@
 _no_decode_ords = [ord(c) for c in _no_decode_chars]
 _no_decode_hex = (['%02x' % o for o in _no_decode_ords]
                 + ['%02X' % o for o in _no_decode_ords])
-_hex_display_map = dict(([('%02x' % o, chr(o)) for o in range(256)]
-                    + [('%02X' % o, chr(o)) for o in range(256)]))
+_hex_display_map = dict(([('%02x' % o, int2byte(o)) for o in range(256)]
+                    + [('%02X' % o, int2byte(o)) for o in range(256)]))
 #These entries get mapped to themselves
-_hex_display_map.update((hex, '%'+hex) for hex in _no_decode_hex)
+_hex_display_map.update((hex, b'%'+hex.encode('ascii')) for hex in _no_decode_hex)
 
 # These characters shouldn't be percent-encoded, and it's always safe to
 # unencode them if they are.
@@ -724,6 +725,49 @@
    "%#"         # Extra reserved characters
 )
 
+
+def _unescape_segment_for_display(segment, encoding):
+    """Unescape a segment for display.
+
+    Helper for unescape_for_display
+
+    :param url: A 7-bit ASCII URL
+    :param encoding: The final output encoding
+
+    :return: A unicode string which can be safely encoded into the
+         specified encoding.
+    """
+    escaped_chunks = segment.split('%')
+    escaped_chunks[0] = escaped_chunks[0].encode('utf-8')
+    for j in range(1, len(escaped_chunks)):
+        item = escaped_chunks[j]
+        try:
+            escaped_chunks[j] = _hex_display_map[item[:2]]
+        except KeyError:
+            # Put back the percent symbol
+            escaped_chunks[j] = b'%' + (item[:2].encode('utf-8') if PY3 else item[:2])
+        except UnicodeDecodeError:
+            escaped_chunks[j] = unichr(int(item[:2], 16)).encode('utf-8')
+        escaped_chunks[j] +=  (item[2:].encode('utf-8') if PY3 else item[2:])
+    unescaped = b''.join(escaped_chunks)
+    try:
+        decoded = unescaped.decode('utf-8')
+    except UnicodeDecodeError:
+        # If this path segment cannot be properly utf-8 decoded
+        # after doing unescaping we will just leave it alone
+        return segment
+    else:
+        try:
+            decoded.encode(encoding)
+        except UnicodeEncodeError:
+            # If this chunk cannot be encoded in the local
+            # encoding, then we should leave it alone
+            return segment
+        else:
+            # Otherwise take the url decoded one
+            return decoded
+
+
 def unescape_for_display(url, encoding):
     """Decode what you can for a URL, so that we get a nice looking path.
 
@@ -752,36 +796,7 @@
     # Split into sections to try to decode utf-8
     res = url.split('/')
     for i in range(1, len(res)):
-        escaped_chunks = res[i].split('%')
-        for j in range(1, len(escaped_chunks)):
-            item = escaped_chunks[j]
-            try:
-                escaped_chunks[j] = _hex_display_map[item[:2]] + item[2:]
-            except KeyError:
-                # Put back the percent symbol
-                escaped_chunks[j] = '%' + item
-            except UnicodeDecodeError:
-                escaped_chunks[j] = unichr(int(item[:2], 16)) + item[2:]
-        unescaped = ''.join(escaped_chunks)
-        if sys.version_info[0] == 2:
-            try:
-                decoded = unescaped.decode('utf-8')
-            except UnicodeDecodeError:
-                # If this path segment cannot be properly utf-8 decoded
-                # after doing unescaping we will just leave it alone
-                pass
-            else:
-                try:
-                    decoded.encode(encoding)
-                except UnicodeEncodeError:
-                    # If this chunk cannot be encoded in the local
-                    # encoding, then we should leave it alone
-                    pass
-                else:
-                    # Otherwise take the url decoded one
-                    res[i] = decoded
-        else:
-            res[i] = unescaped
+        res[i] = _unescape_segment_for_display(res[i], encoding)
     return u'/'.join(res)
 
 
 
=== modified file 'python3.passing'
--- python3.passing	2018-09-11 02:50:43 +0000
+++ python3.passing	2018-09-11 03:12:36 +0000
@@ -2013,6 +2013,7 @@
 breezy.tests.blackbox.test_exceptions.TestExceptionReporting.test_exception_exitcode
 breezy.tests.blackbox.test_exceptions.TestExceptionReporting.test_undecodable_argv
 breezy.tests.blackbox.test_exceptions.TestExceptionReporting.test_utf8_default_fs_enc
+breezy.tests.blackbox.test_exceptions.TestOptParseBugHandling.test_nonascii_optparse
 breezy.tests.blackbox.test_export_pot.TestExportPot.test_export_pot
 breezy.tests.blackbox.test_export_pot.TestExportPot.test_export_pot_plugin
 breezy.tests.blackbox.test_export_pot.TestExportPot.test_export_pot_plugin_unknown
@@ -8485,7 +8486,9 @@
 breezy.tests.per_intertree.test_compare.TestCompare.test_content_modification(InterTree(PreviewTree))
 breezy.tests.per_intertree.test_compare.TestCompare.test_dangling(InterDirStateTree(C))
 breezy.tests.per_intertree.test_compare.TestCompare.test_dangling(InterDirStateTree(PY))
+breezy.tests.per_intertree.test_compare.TestCompare.test_dangling(InterTree)
 breezy.tests.per_intertree.test_compare.TestCompare.test_dangling(InterTree(CHKInventory))
+breezy.tests.per_intertree.test_compare.TestCompare.test_dangling(InterTree(PreviewTree))
 breezy.tests.per_intertree.test_compare.TestCompare.test_default_ignores_unversioned_files(InterDirStateTree(C))
 breezy.tests.per_intertree.test_compare.TestCompare.test_default_ignores_unversioned_files(InterDirStateTree(PY))
 breezy.tests.per_intertree.test_compare.TestCompare.test_default_ignores_unversioned_files(InterTree)
@@ -8597,7 +8600,10 @@
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_default_ignores_unversioned_files(InterTree(CHKInventory))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_default_ignores_unversioned_files(InterTree(PreviewTree))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_deleted_and_unknown(InterDirStateTree(C))
+breezy.tests.per_intertree.test_compare.TestIterChanges.test_deleted_and_unknown(InterDirStateTree(PY))
+breezy.tests.per_intertree.test_compare.TestIterChanges.test_deleted_and_unknown(InterTree)
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_deleted_and_unknown(InterTree(CHKInventory))
+breezy.tests.per_intertree.test_compare.TestIterChanges.test_deleted_and_unknown(InterTree(PreviewTree))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_deleted_unicode(InterDirStateTree(C))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_deleted_unicode(InterDirStateTree(PY))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_deleted_unicode(InterTree)
@@ -8699,7 +8705,10 @@
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_renamed_and_added(InterTree(CHKInventory))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_renamed_and_added(InterTree(PreviewTree))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_renamed_and_unknown(InterDirStateTree(C))
+breezy.tests.per_intertree.test_compare.TestIterChanges.test_renamed_and_unknown(InterDirStateTree(PY))
+breezy.tests.per_intertree.test_compare.TestIterChanges.test_renamed_and_unknown(InterTree)
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_renamed_and_unknown(InterTree(CHKInventory))
+breezy.tests.per_intertree.test_compare.TestIterChanges.test_renamed_and_unknown(InterTree(PreviewTree))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_renamed_unicode(InterDirStateTree(C))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_renamed_unicode(InterDirStateTree(PY))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_renamed_unicode(InterTree)
@@ -8796,7 +8805,10 @@
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_unknown_unicode(InterTree(CHKInventory))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_unknown_unicode(InterTree(PreviewTree))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_unversioned_paths_in_target_matching_source_old_names(InterDirStateTree(C))
+breezy.tests.per_intertree.test_compare.TestIterChanges.test_unversioned_paths_in_target_matching_source_old_names(InterDirStateTree(PY))
+breezy.tests.per_intertree.test_compare.TestIterChanges.test_unversioned_paths_in_target_matching_source_old_names(InterTree)
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_unversioned_paths_in_target_matching_source_old_names(InterTree(CHKInventory))
+breezy.tests.per_intertree.test_compare.TestIterChanges.test_unversioned_paths_in_target_matching_source_old_names(InterTree(PreviewTree))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_unversioned_paths_in_tree(InterDirStateTree(C))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_unversioned_paths_in_tree(InterDirStateTree(PY))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_unversioned_paths_in_tree(InterTree)
@@ -8813,7 +8825,10 @@
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_unversioned_subtree_only_emits_root(InterTree(CHKInventory))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_unversioned_subtree_only_emits_root(InterTree(PreviewTree))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_versioned_symlinks(InterDirStateTree(C))
+breezy.tests.per_intertree.test_compare.TestIterChanges.test_versioned_symlinks(InterDirStateTree(PY))
+breezy.tests.per_intertree.test_compare.TestIterChanges.test_versioned_symlinks(InterTree)
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_versioned_symlinks(InterTree(CHKInventory))
+breezy.tests.per_intertree.test_compare.TestIterChanges.test_versioned_symlinks(InterTree(PreviewTree))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_versioned_symlinks_specific_files(InterDirStateTree(C))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_versioned_symlinks_specific_files(InterDirStateTree(PY))
 breezy.tests.per_intertree.test_compare.TestIterChanges.test_versioned_symlinks_specific_files(InterTree)
@@ -16291,6 +16306,8 @@
 breezy.tests.per_transport.TransportTests.test_connection_error(ChrootTransport,TestingChrootServer)
 breezy.tests.per_transport.TransportTests.test_connection_error(FakeNFSTransportDecorator,FakeNFSServer)
 breezy.tests.per_transport.TransportTests.test_connection_error(FakeVFATTransportDecorator,FakeVFATServer)
+breezy.tests.per_transport.TransportTests.test_connection_error(HTTPS_transport,HTTPSServer)
+breezy.tests.per_transport.TransportTests.test_connection_error(HttpTransport,HttpServer)
 breezy.tests.per_transport.TransportTests.test_connection_error(LocalTransport,LocalURLServer)
 breezy.tests.per_transport.TransportTests.test_connection_error(MemoryTransport,MemoryServer)
 breezy.tests.per_transport.TransportTests.test_connection_error(NoSmartTransportDecorator,NoSmartTransportServer)
@@ -30677,6 +30694,7 @@
 breezy.tests.test_urlutils.TestUrlToPath.test_join_segment_parameters
 breezy.tests.test_urlutils.TestUrlToPath.test_join_segment_parameters_raw
 breezy.tests.test_urlutils.TestUrlToPath.test_normalize_url_files
+breezy.tests.test_urlutils.TestUrlToPath.test_normalize_url_hybrid
 breezy.tests.test_urlutils.TestUrlToPath.test_posix_local_path_from_url
 breezy.tests.test_urlutils.TestUrlToPath.test_posix_local_path_to_url
 breezy.tests.test_urlutils.TestUrlToPath.test_relative_url
@@ -30685,6 +30703,7 @@
 breezy.tests.test_urlutils.TestUrlToPath.test_split_segment_parameters_raw
 breezy.tests.test_urlutils.TestUrlToPath.test_strip_trailing_slash
 breezy.tests.test_urlutils.TestUrlToPath.test_unescape
+breezy.tests.test_urlutils.TestUrlToPath.test_unescape_for_display_utf8
 breezy.tests.test_urlutils.TestUrlToPath.test_url_scheme_re
 breezy.tests.test_urlutils.TestUrlToPath.test_win32_extract_drive_letter
 breezy.tests.test_urlutils.TestUrlToPath.test_win32_local_path_from_url