Merge lp:~gz/brz/i_unzipping into lp:brz

[Martin Packman]

Make use of zip Python 3 compatible

Fix uses of zip and itertools.izip to work across Python 2 and 3.

Using future_builtins in preference to conditional import of izip name.

Did some small refactoring of ancient code while touching things, stuff like knit wants to go away anyway.

[Jelmer Vernooij (jelmer)]

review: Approve

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
http://10.242.247.184:8080/job/brz-dev/3/

[Martin Packman (gz)]

Fixed test failures from per_versionedfile tests also really needing an iterator. Also reshuffled comments and moved zip and map to commonly importable location.

=== modified file 'breezy/config.py'
--- breezy/config.py	2017-05-22 00:56:52 +0000
+++ breezy/config.py	2017-05-24 23:37:37 +0000
@@ -1191,8 +1191,7 @@
         else:
             # Rely on zip truncating in length to the length of the shortest
             # argument sequence.
-            names = zip(location_parts, section_parts)
-            for name in names:
+            for name in zip(location_parts, section_parts):
                 if not fnmatch.fnmatch(name[0], name[1]):
                     matched = False
                     break
 
=== modified file 'breezy/index.py'
--- breezy/index.py	2017-05-22 00:56:52 +0000
+++ breezy/index.py	2017-05-24 23:37:37 +0000
@@ -1449,6 +1449,7 @@
         """
         indices_info = zip(self._index_names, self._indices)
         if 'index' in debug.debug_flags:
+            indices_info = list(indices_info)
             trace.mutter('CombinedGraphIndex reordering: currently %r, '
                          'promoting %r', indices_info, hit_indices)
         hit_names = []
 
=== modified file 'breezy/knit.py'
--- breezy/knit.py	2017-05-22 00:56:52 +0000
+++ breezy/knit.py	2017-05-24 23:37:37 +0000
@@ -53,8 +53,6 @@
 
 from __future__ import absolute_import
 
-
-from itertools import izip
 import operator
 import os
 
@@ -471,7 +469,7 @@
 
     def __init__(self, lines):
         KnitContent.__init__(self)
-        self._lines = lines
+        self._lines = list(lines)
 
     def annotate(self):
         """Return a list of (origin, text) for each content line."""
@@ -504,7 +502,7 @@
         return lines
 
     def copy(self):
-        return AnnotatedKnitContent(self._lines[:])
+        return AnnotatedKnitContent(self._lines)
 
 
 class PlainKnitContent(KnitContent):
@@ -599,7 +597,7 @@
         #       but the code itself doesn't really depend on that.
         #       Figure out a way to not require the overhead of turning the
         #       list back into tuples.
-        lines = [tuple(line.split(' ', 1)) for line in content]
+        lines = (tuple(line.split(' ', 1)) for line in content)
         return AnnotatedKnitContent(lines)
 
     def parse_line_delta_iter(self, lines):
@@ -1933,8 +1931,7 @@
         raw_data = self._access.get_raw_records(
             [index_memo for key, index_memo in needed_records])
 
-        for (key, index_memo), data in \
-                izip(iter(needed_records), raw_data):
+        for (key, index_memo), data in zip(needed_records, raw_data):
             content, digest = self._parse_record(key[-1], data)
             yield key, content, digest
 
 
=== modified file 'breezy/log.py'
--- breezy/log.py	2017-05-22 00:56:52 +0000
+++ breezy/log.py	2017-05-24 23:37:37 +0000
@@ -87,16 +87,10 @@
     )
 from breezy.sixish import (
     BytesIO,
-    PY3,
+    zip,
     )
 
 
-if PY3:
-    izip = zip
-else:
-    izip = itertools.izip
-
-
 def find_touching_revisions(branch, file_id):
     """Yield a description of revisions which affect the file_id.
 
@@ -834,7 +828,7 @@
         # A single batch conversion is faster than many incremental ones.
         # As we have all the data, do a batch conversion.
         nones = [None] * len(view_revisions)
-        log_rev_iterator = iter([zip(view_revisions, nones, nones)])
+        log_rev_iterator = iter([list(zip(view_revisions, nones, nones))])
     else:
         def _convert():
             for view in view_revisions:
@@ -945,11 +939,11 @@
         new_revs = []
         if delta_type == 'full' and not check_fileids:
             deltas = repository.get_deltas_for_revisions(revisions)
-            for rev, delta in izip(revs, deltas):
+            for rev, delta in zip(revs, deltas):
                 new_revs.append((rev[0], rev[1], delta))
         else:
             deltas = repository.get_deltas_for_revisions(revisions, fileid_set)
-            for rev, delta in izip(revs, deltas):
+            for rev, delta in zip(revs, deltas):
                 if check_fileids:
                     if delta is None or not delta.has_changed():
                         continue
@@ -1005,7 +999,7 @@
         revision_ids = [view[0] for view, _, _ in revs]
         revisions = repository.get_revisions(revision_ids)
         revs = [(rev[0], revision, rev[2]) for rev, revision in
-            izip(revs, revisions)]
+            zip(revs, revisions)]
         yield revs
 
 
 
=== modified file 'breezy/repofmt/knitpack_repo.py'
--- breezy/repofmt/knitpack_repo.py	2017-05-22 00:56:52 +0000
+++ breezy/repofmt/knitpack_repo.py	2017-05-24 23:37:37 +0000
@@ -20,7 +20,6 @@
 
 from ..lazy_import import lazy_import
 lazy_import(globals(), """
-from itertools import izip
 import time
 
 from breezy import (
@@ -68,6 +67,9 @@
     PackRootCommitBuilder,
     RepositoryPackCollection,
     )
+from ..sixish import (
+    zip
+    )
 from ..vf_repository import (
     StreamSource,
     )
@@ -659,8 +661,8 @@
                 if self._reload_func is not None:
                     self._reload_func()
                 raise
-            for (names, read_func), (_1, _2, (key, eol_flag)) in \
-                izip(reader.iter_records(), pack_readv_requests):
+            for (names, read_func), (_1, _2, (key, eol_flag)) in zip(
+                    reader.iter_records(), pack_readv_requests):
                 raw_data = read_func(None)
                 # check the header only
                 if output_lines is not None:
@@ -711,8 +713,8 @@
                 if self._reload_func is not None:
                     self._reload_func()
                 raise
-            for (names, read_func), (key, eol_flag, references) in \
-                izip(reader.iter_records(), node_vector):
+            for (names, read_func), (key, eol_flag, references) in zip(
+                    reader.iter_records(), node_vector):
                 raw_data = read_func(None)
                 if output_lines:
                     # read the entire thing
 
=== modified file 'breezy/sixish.py'
--- breezy/sixish.py	2017-05-21 18:16:32 +0000
+++ breezy/sixish.py	2017-05-24 23:37:37 +0000
@@ -24,7 +24,6 @@
 
 from six import (
     binary_type,
-    PY2,
     PY3,
     reraise,
     string_types,
@@ -40,6 +39,8 @@
     import io as _io
     BytesIO = _io.BytesIO
     StringIO = _io.StringIO
+    from builtins import zip, map
 else:
     from cStringIO import StringIO as BytesIO
     from StringIO import StringIO
+    from future_builtins import zip, map
 
=== modified file 'breezy/tests/__init__.py'
--- breezy/tests/__init__.py	2017-05-22 00:56:52 +0000
+++ breezy/tests/__init__.py	2017-05-24 23:37:37 +0000
@@ -3456,7 +3456,7 @@
     # than the fastest.
     partitions = [list() for i in range(count)]
     tests = iter_suite_tests(suite)
-    for partition, test in itertools.izip(itertools.cycle(partitions), tests):
+    for partition, test in zip(itertools.cycle(partitions), tests):
         partition.append(test)
     return partitions
 
 
=== modified file 'breezy/tests/blackbox/test_log.py'
--- breezy/tests/blackbox/test_log.py	2017-05-21 18:10:28 +0000
+++ breezy/tests/blackbox/test_log.py	2017-05-24 23:37:37 +0000
@@ -17,7 +17,6 @@
 
 """Black-box tests for brz log."""
 
-from itertools import izip
 import os
 
 from breezy import (
@@ -693,8 +692,8 @@
                            for r in self.get_captured_revisions()])
         # Now check the diffs, adding the revno  in case of failure
         fmt = 'In revno %s\n%s'
-        for expected_rev, actual_rev in izip(expected,
-                                             self.get_captured_revisions()):
+        for expected_rev, actual_rev in zip(expected,
+                                            self.get_captured_revisions()):
             revno, depth, expected_diff = expected_rev
             actual_diff = actual_rev.diff
             self.assertEqualDiff(fmt % (revno, expected_diff),
 
=== modified file 'breezy/tests/per_controldir/test_controldir.py'
--- breezy/tests/per_controldir/test_controldir.py	2017-05-22 00:56:52 +0000
+++ breezy/tests/per_controldir/test_controldir.py	2017-05-24 23:37:37 +0000
@@ -16,8 +16,6 @@
 
 """Tests for control directory implementations - tests a controldir format."""
 
-from itertools import izip
-
 import breezy.branch
 from breezy import (
     bzrdir as _mod_bzrdir,
@@ -1560,7 +1558,7 @@
         self.assertPathExists(old_path)
         self.assertPathExists(new_path)
         for (((dir_relpath1, _), entries1),
-             ((dir_relpath2, _), entries2)) in izip(
+             ((dir_relpath2, _), entries2)) in zip(
                 osutils.walkdirs(old_path),
                 osutils.walkdirs(new_path)):
             self.assertEqual(dir_relpath1, dir_relpath2)
 
=== modified file 'breezy/tests/per_repository/test_repository.py'
--- breezy/tests/per_repository/test_repository.py	2017-05-22 00:56:52 +0000
+++ breezy/tests/per_repository/test_repository.py	2017-05-24 23:37:37 +0000
@@ -421,7 +421,7 @@
         revision_ids = ['a-rev', 'b-rev', 'c-rev']
         revisions = repo.get_revisions(revision_ids)
         self.assertEqual(len(revisions), 3)
-        zipped = zip(revisions, revision_ids)
+        zipped = list(zip(revisions, revision_ids))
         self.assertEqual(len(zipped), 3)
         for revision, revision_id in zipped:
             self.assertEqual(revision.revision_id, revision_id)
 
=== modified file 'breezy/tests/per_transport.py'
--- breezy/tests/per_transport.py	2017-05-23 14:08:03 +0000
+++ breezy/tests/per_transport.py	2017-05-24 23:37:37 +0000
@@ -20,7 +20,6 @@
 TransportTestProviderAdapter.
 """
 
-import itertools
 import os
 import stat
 import sys
@@ -43,6 +42,7 @@
 from ..osutils import getcwd
 from ..sixish import (
     BytesIO,
+    zip,
     )
 from ..smart import medium
 from . import (
@@ -191,15 +191,15 @@
         self.build_tree(files, transport=t, line_endings='binary')
         self.check_transport_contents('contents of a\n', t, 'a')
         content_f = t.get_multi(files)
-        # Use itertools.izip() instead of use zip() or map(), since they fully
-        # evaluate their inputs, the transport requests should be issued and
+        # Must use iter zip() from future not old version which will fully
+        # evaluate its inputs, the transport requests should be issued and
         # handled sequentially (we don't want to force transport to buffer).
-        for content, f in itertools.izip(contents, content_f):
+        for content, f in zip(contents, content_f):
             self.assertEqual(content, f.read())
 
         content_f = t.get_multi(iter(files))
-        # Use itertools.izip() for the same reason
-        for content, f in itertools.izip(contents, content_f):
+        # Again this zip() must come from the future
+        for content, f in zip(contents, content_f):
             self.assertEqual(content, f.read())
 
     def test_get_unknown_file(self):
 
=== modified file 'breezy/tests/per_versionedfile.py'
--- breezy/tests/per_versionedfile.py	2017-05-22 00:56:52 +0000
+++ breezy/tests/per_versionedfile.py	2017-05-24 23:37:37 +0000
@@ -22,7 +22,7 @@
 # considered typical and check that it can be detected/corrected.
 
 from gzip import GzipFile
-from itertools import chain, izip
+import itertools
 
 from .. import (
     errors,
@@ -45,6 +45,7 @@
     )
 from ..sixish import (
     BytesIO,
+    zip,
     )
 from . import (
     TestCase,
@@ -2028,9 +2029,9 @@
         :param records: A list to collect the seen records.
         :return: A generator of the records in stream.
         """
-        # We make assertions during copying to catch things early for
-        # easier debugging.
-        for record, ref_record in izip(stream, expected):
+        # We make assertions during copying to catch things early for easier
+        # debugging. This must use the iterating zip() from the future.
+        for record, ref_record in zip(stream, expected):
             records.append(record)
             self.assertEqual(ref_record.key, record.key)
             self.assertEqual(ref_record.storage_kind, record.storage_kind)
@@ -2444,7 +2445,7 @@
         origin_entries = source.get_record_stream(origin_keys, 'unordered', False)
         end_entries = source.get_record_stream(end_keys, 'topological', False)
         start_entries = source.get_record_stream(start_keys, 'topological', False)
-        entries = chain(origin_entries, end_entries, start_entries)
+        entries = itertools.chain(origin_entries, end_entries, start_entries)
         try:
             files.insert_record_stream(entries)
         except RevisionNotPresent:
@@ -2476,7 +2477,7 @@
         streams = []
         for key in reversed(keys):
             streams.append(source.get_record_stream([key], 'unordered', False))
-        deltas = chain(*streams[:-1])
+        deltas = itertools.chain.from_iterable(streams[:-1])
         files = self.get_versionedfiles()
         try:
             files.insert_record_stream(deltas)
 
=== modified file 'breezy/transport/sftp.py'
--- breezy/transport/sftp.py	2017-05-22 00:56:52 +0000
+++ breezy/transport/sftp.py	2017-05-24 23:37:37 +0000
@@ -49,6 +49,9 @@
                            ParamikoNotPresent,
                            )
 from ..osutils import fancy_rename
+from ..sixish import (
+    zip,
+    )
 from ..trace import mutter, warning
 from ..transport import (
     FileFileStream,
@@ -202,7 +205,7 @@
         # short readv.
         data_stream = itertools.chain(fp.readv(requests),
                                       itertools.repeat(None))
-        for (start, length), data in itertools.izip(requests, data_stream):
+        for (start, length), data in zip(requests, data_stream):
             if data is None:
                 if cur_coalesced is not None:
                     raise errors.ShortReadvError(self.relpath,
 
=== modified file 'breezy/versionedfile.py'
--- breezy/versionedfile.py	2017-05-22 00:56:52 +0000
+++ breezy/versionedfile.py	2017-05-24 23:37:37 +0000
@@ -43,6 +43,7 @@
 from .registry import Registry
 from .sixish import (
     BytesIO,
+    zip,
     )
 from .textmerge import TextMerge
 
@@ -535,10 +536,10 @@
                                   if not mpvf.has_version(p))
         present_parents = set(self.get_parent_map(needed_parents).keys())
         for parent_id, lines in zip(present_parents,
-                                 self._get_lf_split_line_list(present_parents)):
+                self._get_lf_split_line_list(present_parents)):
             mpvf.add_version(lines, parent_id, [])
-        for (version, parent_ids, expected_sha1, mpdiff), lines in\
-            zip(records, mpvf.get_line_list(versions)):
+        for (version, parent_ids, expected_sha1, mpdiff), lines in zip(
+                records, mpvf.get_line_list(versions)):
             if len(parent_ids) == 1:
                 left_matching_blocks = list(mpdiff.get_matching_blocks(0,
                     mpvf.get_diff(parent_ids[0]).num_lines()))
@@ -1027,8 +1028,8 @@
                 continue
             mpvf.add_version(chunks_to_lines(record.get_bytes_as('chunked')),
                 record.key, [])
-        for (key, parent_keys, expected_sha1, mpdiff), lines in\
-            zip(records, mpvf.get_line_list(versions)):
+        for (key, parent_keys, expected_sha1, mpdiff), lines in zip(
+                records, mpvf.get_line_list(versions)):
             if len(parent_keys) == 1:
                 left_matching_blocks = list(mpdiff.get_matching_blocks(0,
                     mpvf.get_diff(parent_keys[0]).num_lines()))
@@ -1318,7 +1319,7 @@
             prefix_keys.append(key[-1])
         return result
 
-    def _get_all_prefixes(self):
+    def _iter_all_prefixes(self):
         # Identify all key prefixes.
         # XXX: A bit hacky, needs polish.
         if isinstance(self._mapper, ConstantMapper):
@@ -1413,7 +1414,7 @@
                 yield line, prefix + (version,)
 
     def _iter_all_components(self):
-        for path, prefix in self._get_all_prefixes():
+        for path, prefix in self._iter_all_prefixes():
             yield prefix, self._get_vf(path)
 
     def keys(self):
 
=== modified file 'breezy/workingtree_4.py'
--- breezy/workingtree_4.py	2017-05-22 00:56:52 +0000
+++ breezy/workingtree_4.py	2017-05-24 23:37:37 +0000
@@ -1353,8 +1353,8 @@
                 _mod_revision.NULL_REVISION)
             trees = []
         else:
-            trees = zip(revision_ids,
-                        self.branch.repository.revision_trees(revision_ids))
+            trees = list(zip(revision_ids,
+                        self.branch.repository.revision_trees(revision_ids)))
             base_tree = trees[0][1]
         state = self.current_dirstate()
         # We don't support ghosts yet