Merge lp:~gz/brz/buffer_to_memoryview into lp:brz

Proposed by Martin Packman
Status: Merged
Approved by: Jelmer Vernooij
Approved revision: 6636
Merge reported by: The Breezy Bot
Merged at revision: not available
Proposed branch: lp:~gz/brz/buffer_to_memoryview
Merge into: lp:brz
Diff against target: 195 lines (+59/-25)
4 files modified
breezy/osutils.py (+6/-5)
breezy/tests/file_utils.py (+1/-1)
breezy/tests/test_osutils.py (+16/-14)
breezy/transport/sftp.py (+36/-5)
To merge this branch: bzr merge lp:~gz/brz/buffer_to_memoryview
Reviewer Review Type Date Requested Status
Jelmer Vernooij Approve
Review via email: mp+324573@code.launchpad.net

Commit message

Use memoryview over buffer for compatibility

Description of the change

The 'buffer' 2to3 fixer was especially unhelpful here, creating invalid code. Pretty simple to adapt our uses though.

Branch also makes the classes TestSendAll, TestPumpFile, and TestPumpStringFile pass on Python 3.

To post a comment you must log in.
Revision history for this message
Jelmer Vernooij (jelmer) :
review: Approve
Revision history for this message
The Breezy Bot (the-breezy-bot) wrote :

Running landing tests failed
http://10.242.247.184:8080/job/brz-dev/4/

Revision history for this message
Martin Packman (gz) wrote :

Paramiko is failing me, have filed and proposed fixes upstream:

<https://github.com/paramiko/paramiko/issues/967>
<https://github.com/paramiko/paramiko/issues/968>

Choice here of monkey patching their py3compat module or dropping the optimisation (which is aimed at other transports).

lp:~gz/brz/buffer_to_memoryview updated
6635. By Martin Packman

Remove old compatibility hack for pre 1.5.5 paramiko

6636. By Martin Packman

Dark hackery to get around paramiko Python 3 compatibility mistakes

Revision history for this message
Martin Packman (gz) wrote :

Removed an old paramiko compatiblity hack and introduced a new one.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'breezy/osutils.py'
--- breezy/osutils.py 2017-05-24 19:44:00 +0000
+++ breezy/osutils.py 2017-05-26 01:01:46 +0000
@@ -746,11 +746,11 @@
746 # drives).746 # drives).
747 if not segment_size:747 if not segment_size:
748 segment_size = 5242880 # 5MB748 segment_size = 5242880 # 5MB
749 segments = range(len(bytes) / segment_size + 1)749 offsets = range(0, len(bytes), segment_size)
750 view = memoryview(bytes)
750 write = file_handle.write751 write = file_handle.write
751 for segment_index in segments:752 for offset in offsets:
752 segment = buffer(bytes, segment_index * segment_size, segment_size)753 write(view[offset:offset+segment_size])
753 write(segment)
754754
755755
756def file_iterator(input_file, readsize=32768):756def file_iterator(input_file, readsize=32768):
@@ -2176,9 +2176,10 @@
2176 """2176 """
2177 sent_total = 02177 sent_total = 0
2178 byte_count = len(bytes)2178 byte_count = len(bytes)
2179 view = memoryview(bytes)
2179 while sent_total < byte_count:2180 while sent_total < byte_count:
2180 try:2181 try:
2181 sent = sock.send(buffer(bytes, sent_total, MAX_SOCKET_CHUNK))2182 sent = sock.send(view[sent_total:sent_total+MAX_SOCKET_CHUNK])
2182 except (socket.error, IOError) as e:2183 except (socket.error, IOError) as e:
2183 if e.args[0] in _end_of_stream_errors:2184 if e.args[0] in _end_of_stream_errors:
2184 raise errors.ConnectionReset(2185 raise errors.ConnectionReset(
21852186
=== modified file 'breezy/tests/file_utils.py'
--- breezy/tests/file_utils.py 2017-05-22 00:56:52 +0000
+++ breezy/tests/file_utils.py 2017-05-26 01:01:46 +0000
@@ -33,7 +33,7 @@
33 """Reads size characters from the input (or the rest of the string if33 """Reads size characters from the input (or the rest of the string if
34 size is -1)."""34 size is -1)."""
35 data = self.data.read(size)35 data = self.data.read(size)
36 self.max_read_size = max(self.max_read_size, len(data))36 self.max_read_size = max(self.max_read_size or 0, len(data))
37 self.read_count += 137 self.read_count += 1
38 return data38 return data
3939
4040
=== modified file 'breezy/tests/test_osutils.py'
--- breezy/tests/test_osutils.py 2017-05-22 00:56:52 +0000
+++ breezy/tests/test_osutils.py 2017-05-26 01:01:46 +0000
@@ -16,6 +16,8 @@
1616
17"""Tests for the osutils wrapper."""17"""Tests for the osutils wrapper."""
1818
19from __future__ import absolute_import, division
20
19import errno21import errno
20import os22import os
21import re23import re
@@ -603,8 +605,8 @@
603 super(TestPumpFile, self).setUp()605 super(TestPumpFile, self).setUp()
604 # create a test datablock606 # create a test datablock
605 self.block_size = 512607 self.block_size = 512
606 pattern = '0123456789ABCDEF'608 pattern = b'0123456789ABCDEF'
607 self.test_data = pattern * (3 * self.block_size / len(pattern))609 self.test_data = pattern * (3 * self.block_size // len(pattern))
608 self.test_data_len = len(self.test_data)610 self.test_data_len = len(self.test_data)
609611
610 def test_bracket_block_size(self):612 def test_bracket_block_size(self):
@@ -616,8 +618,8 @@
616 from_file = file_utils.FakeReadFile(self.test_data)618 from_file = file_utils.FakeReadFile(self.test_data)
617 to_file = BytesIO()619 to_file = BytesIO()
618620
619 # read (max / 2) bytes and verify read size wasn't affected621 # read (max // 2) bytes and verify read size wasn't affected
620 num_bytes_to_read = self.block_size / 2622 num_bytes_to_read = self.block_size // 2
621 osutils.pumpfile(from_file, to_file, num_bytes_to_read, self.block_size)623 osutils.pumpfile(from_file, to_file, num_bytes_to_read, self.block_size)
622 self.assertEqual(from_file.get_max_read_size(), num_bytes_to_read)624 self.assertEqual(from_file.get_max_read_size(), num_bytes_to_read)
623 self.assertEqual(from_file.get_read_count(), 1)625 self.assertEqual(from_file.get_read_count(), 1)
@@ -742,23 +744,23 @@
742744
743 def test_empty(self):745 def test_empty(self):
744 output = BytesIO()746 output = BytesIO()
745 osutils.pump_string_file("", output)747 osutils.pump_string_file(b"", output)
746 self.assertEqual("", output.getvalue())748 self.assertEqual(b"", output.getvalue())
747749
748 def test_more_than_segment_size(self):750 def test_more_than_segment_size(self):
749 output = BytesIO()751 output = BytesIO()
750 osutils.pump_string_file("123456789", output, 2)752 osutils.pump_string_file(b"123456789", output, 2)
751 self.assertEqual("123456789", output.getvalue())753 self.assertEqual(b"123456789", output.getvalue())
752754
753 def test_segment_size(self):755 def test_segment_size(self):
754 output = BytesIO()756 output = BytesIO()
755 osutils.pump_string_file("12", output, 2)757 osutils.pump_string_file(b"12", output, 2)
756 self.assertEqual("12", output.getvalue())758 self.assertEqual(b"12", output.getvalue())
757759
758 def test_segment_size_multiple(self):760 def test_segment_size_multiple(self):
759 output = BytesIO()761 output = BytesIO()
760 osutils.pump_string_file("1234", output, 2)762 osutils.pump_string_file(b"1234", output, 2)
761 self.assertEqual("1234", output.getvalue())763 self.assertEqual(b"1234", output.getvalue())
762764
763765
764class TestRelpath(tests.TestCase):766class TestRelpath(tests.TestCase):
@@ -892,7 +894,7 @@
892 for err in errs:894 for err in errs:
893 sock = DisconnectedSocket(err)895 sock = DisconnectedSocket(err)
894 self.assertRaises(errors.ConnectionReset,896 self.assertRaises(errors.ConnectionReset,
895 osutils.send_all, sock, 'some more content')897 osutils.send_all, sock, b'some more content')
896898
897 def test_send_with_no_progress(self):899 def test_send_with_no_progress(self):
898 # See https://bugs.launchpad.net/bzr/+bug/1047309900 # See https://bugs.launchpad.net/bzr/+bug/1047309
@@ -909,7 +911,7 @@
909 return 0911 return 0
910 sock = NoSendingSocket()912 sock = NoSendingSocket()
911 self.assertRaises(errors.ConnectionReset,913 self.assertRaises(errors.ConnectionReset,
912 osutils.send_all, sock, 'content')914 osutils.send_all, sock, b'content')
913 self.assertEqual(1, sock.call_count)915 self.assertEqual(1, sock.call_count)
914916
915917
916918
=== modified file 'breezy/transport/sftp.py'
--- breezy/transport/sftp.py 2017-05-25 00:04:21 +0000
+++ breezy/transport/sftp.py 2017-05-26 01:01:46 +0000
@@ -85,9 +85,41 @@
85 from paramiko.sftp_file import SFTPFile85 from paramiko.sftp_file import SFTPFile
8686
8787
88_paramiko_version = getattr(paramiko, '__version_info__', (0, 0, 0))88# GZ 2017-05-25: Some dark hackery to monkeypatch out issues with paramiko's
89# don't use prefetch unless paramiko version >= 1.5.5 (there were bugs earlier)89# Python 3 compatibility code. Replace broken b() and asbytes() code.
90_default_do_prefetch = (_paramiko_version >= (1, 5, 5))90try:
91 from paramiko.py3compat import b as _bad
92 from paramiko.common import asbytes as _bad_asbytes
93except ImportError:
94 pass
95else:
96 def _b_for_broken_paramiko(s, encoding='utf8'):
97 """Hacked b() that does not raise TypeError."""
98 # https://github.com/paramiko/paramiko/issues/967
99 if not isinstance(s, bytes):
100 encode = getattr(s, 'encode', None)
101 if encode is not None:
102 return encode(encoding)
103 # Would like to pass buffer objects along, but have to realise.
104 tobytes = getattr(s, 'tobytes', None)
105 if tobytes is not None:
106 return tobytes()
107 return s
108
109 def _asbytes_for_broken_paramiko(s):
110 """Hacked asbytes() that does not raise Exception."""
111 # https://github.com/paramiko/paramiko/issues/968
112 if not isinstance(s, bytes):
113 encode = getattr(s, 'encode', None)
114 if encode is not None:
115 return encode('utf8')
116 asbytes = getattr(s, 'asbytes', None)
117 if asbytes is not None:
118 return asbytes()
119 return s
120
121 _bad.func_code = _b_for_broken_paramiko.func_code
122 _bad_asbytes.func_code = _asbytes_for_broken_paramiko.func_code
91123
92124
93class SFTPLock(object):125class SFTPLock(object):
@@ -317,7 +349,6 @@
317class SFTPTransport(ConnectedTransport):349class SFTPTransport(ConnectedTransport):
318 """Transport implementation for SFTP access."""350 """Transport implementation for SFTP access."""
319351
320 _do_prefetch = _default_do_prefetch
321 # TODO: jam 20060717 Conceivably these could be configurable, either352 # TODO: jam 20060717 Conceivably these could be configurable, either
322 # by auto-tuning at run-time, or by a configuration (per host??)353 # by auto-tuning at run-time, or by a configuration (per host??)
323 # but the performance curve is pretty flat, so just going with354 # but the performance curve is pretty flat, so just going with
@@ -414,7 +445,7 @@
414 path = self._remote_path(relpath)445 path = self._remote_path(relpath)
415 f = self._get_sftp().file(path, mode='rb')446 f = self._get_sftp().file(path, mode='rb')
416 size = f.stat().st_size447 size = f.stat().st_size
417 if self._do_prefetch and (getattr(f, 'prefetch', None) is not None):448 if getattr(f, 'prefetch', None) is not None:
418 f.prefetch(size)449 f.prefetch(size)
419 return f450 return f
420 except (IOError, paramiko.SSHException) as e:451 except (IOError, paramiko.SSHException) as e:

Subscribers

People subscribed via source and target branches