Merge lp:~gz/brz/buffer_to_memoryview into lp:brz

[Martin Packman]

Use memoryview over buffer for compatibility

The 'buffer' 2to3 fixer was especially unhelpful here, creating invalid code. Pretty simple to adapt our uses though.

Branch also makes the classes TestSendAll, TestPumpFile, and TestPumpStringFile pass on Python 3.

[Jelmer Vernooij (jelmer)]

review: Approve

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
http://10.242.247.184:8080/job/brz-dev/4/

[Martin Packman (gz)]

Paramiko is failing me, have filed and proposed fixes upstream:

<https://github.com/paramiko/paramiko/issues/967>
<https://github.com/paramiko/paramiko/issues/968>

Choice here of monkey patching their py3compat module or dropping the optimisation (which is aimed at other transports).

[Martin Packman (gz)]

Removed an old paramiko compatiblity hack and introduced a new one.

=== modified file 'breezy/osutils.py'
--- breezy/osutils.py	2017-05-24 19:44:00 +0000
+++ breezy/osutils.py	2017-05-26 01:01:46 +0000
@@ -746,11 +746,11 @@
     # drives).
     if not segment_size:
         segment_size = 5242880 # 5MB
-    segments = range(len(bytes) / segment_size + 1)
+    offsets = range(0, len(bytes), segment_size)
+    view = memoryview(bytes)
     write = file_handle.write
-    for segment_index in segments:
-        segment = buffer(bytes, segment_index * segment_size, segment_size)
-        write(segment)
+    for offset in offsets:
+        write(view[offset:offset+segment_size])
 
 
 def file_iterator(input_file, readsize=32768):
@@ -2176,9 +2176,10 @@
     """
     sent_total = 0
     byte_count = len(bytes)
+    view = memoryview(bytes)
     while sent_total < byte_count:
         try:
-            sent = sock.send(buffer(bytes, sent_total, MAX_SOCKET_CHUNK))
+            sent = sock.send(view[sent_total:sent_total+MAX_SOCKET_CHUNK])
         except (socket.error, IOError) as e:
             if e.args[0] in _end_of_stream_errors:
                 raise errors.ConnectionReset(

=== modified file 'breezy/tests/file_utils.py'
--- breezy/tests/file_utils.py	2017-05-22 00:56:52 +0000
+++ breezy/tests/file_utils.py	2017-05-26 01:01:46 +0000
@@ -33,7 +33,7 @@
         """Reads size characters from the input (or the rest of the string if
         size is -1)."""
         data = self.data.read(size)
-        self.max_read_size = max(self.max_read_size, len(data))
+        self.max_read_size = max(self.max_read_size or 0, len(data))
         self.read_count += 1
         return data
 

=== modified file 'breezy/tests/test_osutils.py'
--- breezy/tests/test_osutils.py	2017-05-22 00:56:52 +0000
+++ breezy/tests/test_osutils.py	2017-05-26 01:01:46 +0000
@@ -16,6 +16,8 @@
 
 """Tests for the osutils wrapper."""
 
+from __future__ import absolute_import, division
+
 import errno
 import os
 import re
@@ -603,8 +605,8 @@
         super(TestPumpFile, self).setUp()
         # create a test datablock
         self.block_size = 512
-        pattern = '0123456789ABCDEF'
-        self.test_data = pattern * (3 * self.block_size / len(pattern))
+        pattern = b'0123456789ABCDEF'
+        self.test_data = pattern * (3 * self.block_size // len(pattern))
         self.test_data_len = len(self.test_data)
 
     def test_bracket_block_size(self):
@@ -616,8 +618,8 @@
         from_file = file_utils.FakeReadFile(self.test_data)
         to_file = BytesIO()
 
-        # read (max / 2) bytes and verify read size wasn't affected
-        num_bytes_to_read = self.block_size / 2
+        # read (max // 2) bytes and verify read size wasn't affected
+        num_bytes_to_read = self.block_size // 2
         osutils.pumpfile(from_file, to_file, num_bytes_to_read, self.block_size)
         self.assertEqual(from_file.get_max_read_size(), num_bytes_to_read)
         self.assertEqual(from_file.get_read_count(), 1)
@@ -742,23 +744,23 @@
 
     def test_empty(self):
         output = BytesIO()
-        osutils.pump_string_file("", output)
-        self.assertEqual("", output.getvalue())
+        osutils.pump_string_file(b"", output)
+        self.assertEqual(b"", output.getvalue())
 
     def test_more_than_segment_size(self):
         output = BytesIO()
-        osutils.pump_string_file("123456789", output, 2)
-        self.assertEqual("123456789", output.getvalue())
+        osutils.pump_string_file(b"123456789", output, 2)
+        self.assertEqual(b"123456789", output.getvalue())
 
     def test_segment_size(self):
         output = BytesIO()
-        osutils.pump_string_file("12", output, 2)
-        self.assertEqual("12", output.getvalue())
+        osutils.pump_string_file(b"12", output, 2)
+        self.assertEqual(b"12", output.getvalue())
 
     def test_segment_size_multiple(self):
         output = BytesIO()
-        osutils.pump_string_file("1234", output, 2)
-        self.assertEqual("1234", output.getvalue())
+        osutils.pump_string_file(b"1234", output, 2)
+        self.assertEqual(b"1234", output.getvalue())
 
 
 class TestRelpath(tests.TestCase):
@@ -892,7 +894,7 @@
         for err in errs:
             sock = DisconnectedSocket(err)
             self.assertRaises(errors.ConnectionReset,
-                osutils.send_all, sock, 'some more content')
+                osutils.send_all, sock, b'some more content')
 
     def test_send_with_no_progress(self):
         # See https://bugs.launchpad.net/bzr/+bug/1047309
@@ -909,7 +911,7 @@
                 return 0
         sock = NoSendingSocket()
         self.assertRaises(errors.ConnectionReset,
-                          osutils.send_all, sock, 'content')
+                          osutils.send_all, sock, b'content')
         self.assertEqual(1, sock.call_count)
 
 

=== modified file 'breezy/transport/sftp.py'
--- breezy/transport/sftp.py	2017-05-25 00:04:21 +0000
+++ breezy/transport/sftp.py	2017-05-26 01:01:46 +0000
@@ -85,9 +85,41 @@
     from paramiko.sftp_file import SFTPFile
 
 
-_paramiko_version = getattr(paramiko, '__version_info__', (0, 0, 0))
-# don't use prefetch unless paramiko version >= 1.5.5 (there were bugs earlier)
-_default_do_prefetch = (_paramiko_version >= (1, 5, 5))
+# GZ 2017-05-25: Some dark hackery to monkeypatch out issues with paramiko's
+# Python 3 compatibility code. Replace broken b() and asbytes() code.
+try:
+    from paramiko.py3compat import b as _bad
+    from paramiko.common import asbytes as _bad_asbytes
+except ImportError:
+    pass
+else:
+    def _b_for_broken_paramiko(s, encoding='utf8'):
+        """Hacked b() that does not raise TypeError."""
+        # https://github.com/paramiko/paramiko/issues/967
+        if not isinstance(s, bytes):
+            encode = getattr(s, 'encode', None)
+            if encode is not None:
+                return encode(encoding)
+            # Would like to pass buffer objects along, but have to realise.
+            tobytes = getattr(s, 'tobytes', None)
+            if tobytes is not None:
+                return tobytes()
+        return s
+
+    def _asbytes_for_broken_paramiko(s):
+        """Hacked asbytes() that does not raise Exception."""
+        # https://github.com/paramiko/paramiko/issues/968
+        if not isinstance(s, bytes):
+            encode = getattr(s, 'encode', None)
+            if encode is not None:
+                return encode('utf8')
+            asbytes = getattr(s, 'asbytes', None)
+            if asbytes is not None:
+                return asbytes()
+        return s
+
+    _bad.func_code = _b_for_broken_paramiko.func_code
+    _bad_asbytes.func_code = _asbytes_for_broken_paramiko.func_code
 
 
 class SFTPLock(object):
@@ -317,7 +349,6 @@
 class SFTPTransport(ConnectedTransport):
     """Transport implementation for SFTP access."""
 
-    _do_prefetch = _default_do_prefetch
     # TODO: jam 20060717 Conceivably these could be configurable, either
     #       by auto-tuning at run-time, or by a configuration (per host??)
     #       but the performance curve is pretty flat, so just going with
@@ -414,7 +445,7 @@
             path = self._remote_path(relpath)
             f = self._get_sftp().file(path, mode='rb')
             size = f.stat().st_size
-            if self._do_prefetch and (getattr(f, 'prefetch', None) is not None):
+            if getattr(f, 'prefetch', None) is not None:
                 f.prefetch(size)
             return f
         except (IOError, paramiko.SSHException) as e: