Merge lp:~gz/brz/unmapped into lp:brz

[Martin Packman]

Make use of map Python 3 compatible

Cope with the builtin map function becoming imap in Python 3.

Worse case the fixer throws in an extra pointless list copy, where it looked like that might matter I refactored or pulled in the iterator version from future_builtins.

There are a few changes on somewhat hot paths, where larger refactors/rewrites are probably called for, but avoided doing so for now.

The static tuple intern change is somewhat driveby, we can't use sys.intern on bytestrings in Python 3 anyway, and a better interface would be constructing the tuple with a new method like from_bytes_interned or something to mega-intern the result.

Same as before just with prereq branch set correctly.

[Jelmer Vernooij (jelmer)]

review: Approve

[Jelmer Vernooij (jelmer)]

review: Approve

=== modified file 'breezy/btree_index.py'
--- breezy/btree_index.py	2017-05-22 00:56:52 +0000
+++ breezy/btree_index.py	2017-05-24 23:42:41 +0000
@@ -42,6 +42,7 @@
 from .index import _OPTION_NODE_REFS, _OPTION_KEY_ELEMENTS, _OPTION_LEN
 from .sixish import (
     BytesIO,
+    map,
     )
 
 
@@ -662,7 +663,9 @@
         for line in lines[2:]:
             if line == '':
                 break
-            nodes.append(as_st(map(intern, line.split('\0'))).intern())
+            # GZ 2017-05-24: Used to intern() each chunk of line as well, need
+            # to recheck performance and perhaps adapt StaticTuple to adjust.
+            nodes.append(as_st(line.split(b'\0')).intern())
         return nodes
 
 
@@ -1497,9 +1500,9 @@
         if not options_line.startswith(_OPTION_ROW_LENGTHS):
             raise errors.BadIndexOptions(self)
         try:
-            self._row_lengths = map(int, [length for length in
+            self._row_lengths = [int(length) for length in
                 options_line[len(_OPTION_ROW_LENGTHS):].split(',')
-                if len(length)])
+                if length]
         except ValueError:
             raise errors.BadIndexOptions(self)
         self._compute_row_offsets()
 
=== modified file 'breezy/dirstate.py'
--- breezy/dirstate.py	2017-05-22 11:22:28 +0000
+++ breezy/dirstate.py	2017-05-24 23:42:41 +0000
@@ -1953,7 +1953,7 @@
         lines = []
         lines.append(self._get_parents_line(self.get_parent_ids()))
         lines.append(self._get_ghosts_line(self._ghosts))
-        lines.extend(self._get_entry_lines())
+        lines.extend(self._iter_entry_lines())
         return self._get_output_lines(lines)
 
     def _get_ghosts_line(self, ghost_ids):
@@ -1964,7 +1964,7 @@
         """Create a line for the state file for parents information."""
         return '\0'.join([str(len(parent_ids))] + parent_ids)
 
-    def _get_entry_lines(self):
+    def _iter_entry_lines(self):
         """Create lines for entries."""
         return map(self._entry_to_line, self._iter_entries())
 
 
=== modified file 'breezy/export_pot.py'
--- breezy/export_pot.py	2017-05-22 00:56:52 +0000
+++ breezy/export_pot.py	2017-05-24 23:42:41 +0000
@@ -65,9 +65,8 @@
         if not lines[-1]:
             del lines[-1]
             lines[-1] = lines[-1] + '\n'
-        lines = map(_escape, lines)
         lineterm = '\\n"\n"'
-        s = '""\n"' + lineterm.join(lines) + '"'
+        s = '""\n"' + lineterm.join(map(_escape, lines)) + '"'
     return s
 
 
 
=== modified file 'breezy/groupcompress.py'
--- breezy/groupcompress.py	2017-05-22 00:56:52 +0000
+++ breezy/groupcompress.py	2017-05-24 23:42:41 +0000
@@ -42,6 +42,9 @@
 
 from .btree_index import BTreeBuilder
 from .lru_cache import LRUSizeCache
+from .sixish import (
+    map,
+    )
 from .versionedfile import (
     _KeyRefs,
     adapter_registry,
@@ -300,7 +303,7 @@
         compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION)
         # Peak in this point is 1 fulltext, 1 compressed text, + zlib overhead
         # (measured peak is maybe 30MB over the above...)
-        compressed_chunks = map(compressor.compress, chunks)
+        compressed_chunks = list(map(compressor.compress, chunks))
         compressed_chunks.append(compressor.flush())
         # Ignore empty chunks
         self._z_content_chunks = [c for c in compressed_chunks if c]
 
=== modified file 'breezy/mutabletree.py'
--- breezy/mutabletree.py	2017-05-22 00:56:52 +0000
+++ breezy/mutabletree.py	2017-05-24 23:42:41 +0000
@@ -667,7 +667,7 @@
         # filename alone
         # only expanding if symlinks are supported avoids windows path bugs
         if osutils.has_symlinks():
-            file_list = map(osutils.normalizepath, file_list)
+            file_list = list(map(osutils.normalizepath, file_list))
 
         user_dirs = {}
         # validate user file paths and convert all paths to tree
 
=== modified file 'breezy/osutils.py'
--- breezy/osutils.py	2017-05-22 00:56:52 +0000
+++ breezy/osutils.py	2017-05-24 23:42:41 +0000
@@ -811,7 +811,8 @@
 def sha_strings(strings, _factory=sha):
     """Return the sha-1 of concatenation of strings"""
     s = _factory()
-    map(s.update, strings)
+    for string in strings:
+        s.update(string)
     return s.hexdigest()
 
 
 
=== modified file 'breezy/plugin.py'
--- breezy/plugin.py	2017-05-22 00:56:52 +0000
+++ breezy/plugin.py	2017-05-24 23:42:41 +0000
@@ -266,7 +266,7 @@
 
     # Get rid of trailing slashes, since Python can't handle them when
     # it tries to import modules.
-    paths = map(_strip_trailing_sep, paths)
+    paths = list(map(_strip_trailing_sep, paths))
     return paths
 
 
@@ -317,7 +317,7 @@
     # this function, and since it sets plugins.__path__, it should set it to
     # something that will be valid for Python to use (in case people try to
     # run "import breezy.plugins.PLUGINNAME" after calling this function).
-    _mod_plugins.__path__ = map(_strip_trailing_sep, dirs)
+    _mod_plugins.__path__ = list(map(_strip_trailing_sep, dirs))
     for d in dirs:
         if not d:
             continue
 
=== modified file 'breezy/plugins/changelog_merge/changelog_merge.py'
--- breezy/plugins/changelog_merge/changelog_merge.py	2017-05-22 00:56:52 +0000
+++ breezy/plugins/changelog_merge/changelog_merge.py	2017-05-24 23:42:41 +0000
@@ -48,7 +48,7 @@
                 entries.append([])
                 entry = entries[-1]
             entry.append(line)
-    return map(tuple, entries)
+    return list(map(tuple, entries))
 
 
 def entries_to_lines(entries):
@@ -106,8 +106,8 @@
     This algorithm does O(N^2 * logN) SequenceMatcher.ratio() calls, which is
     pretty bad, but it shouldn't be used very often.
     """
-    deleted_entries_as_strs = map(entry_as_str, deleted_entries)
-    new_entries_as_strs = map(entry_as_str, new_entries)
+    deleted_entries_as_strs = list(map(entry_as_str, deleted_entries))
+    new_entries_as_strs = list(map(entry_as_str, new_entries))
     result_new = list(new_entries)
     result_deleted = list(deleted_entries)
     result_edits = []
 
=== modified file 'breezy/plugins/weave_fmt/bzrdir.py'
--- breezy/plugins/weave_fmt/bzrdir.py	2017-05-22 00:56:52 +0000
+++ breezy/plugins/weave_fmt/bzrdir.py	2017-05-24 23:42:41 +0000
@@ -415,7 +415,7 @@
         Also upgrade the inventory to refer to the text revision ids."""
         rev_id = rev.revision_id
         trace.mutter('converting texts of revision {%s}', rev_id)
-        parent_invs = map(self._load_updated_inventory, present_parents)
+        parent_invs = list(map(self._load_updated_inventory, present_parents))
         entries = inv.iter_entries()
         entries.next()
         for path, ie in entries:
 
=== modified file 'breezy/repository.py'
--- breezy/repository.py	2017-05-22 00:56:52 +0000
+++ breezy/repository.py	2017-05-24 23:42:41 +0000
@@ -924,8 +924,8 @@
             not part of revision_ids themselves
         """
         parent_map = self.get_parent_map(revision_ids)
-        parent_ids = set()
-        map(parent_ids.update, parent_map.itervalues())
+        parent_ids = set(itertools.chain.from_iterable(
+            parent_map.itervalues()))
         parent_ids.difference_update(revision_ids)
         parent_ids.discard(_mod_revision.NULL_REVISION)
         return parent_ids
 
=== modified file 'breezy/smart/repository.py'
--- breezy/smart/repository.py	2017-05-22 00:56:52 +0000
+++ breezy/smart/repository.py	2017-05-24 23:42:41 +0000
@@ -19,6 +19,7 @@
 from __future__ import absolute_import
 
 import bz2
+import itertools
 import os
 try:
     import queue
@@ -307,8 +308,7 @@
         else:
             search_ids = repository.all_revision_ids()
         search = graph._make_breadth_first_searcher(search_ids)
-        transitive_ids = set()
-        map(transitive_ids.update, list(search))
+        transitive_ids = set(itertools.chain.from_iterable(search))
         parent_map = graph.get_parent_map(transitive_ids)
         revision_graph = _strip_NULL_ghosts(parent_map)
         if revision_id and revision_id not in revision_graph:
 
=== modified file 'breezy/tests/per_pack_repository.py'
--- breezy/tests/per_pack_repository.py	2017-05-23 14:08:03 +0000
+++ breezy/tests/per_pack_repository.py	2017-05-24 23:42:41 +0000
@@ -358,7 +358,7 @@
         for _1, key, val, refs in pack.revision_index.iter_all_entries():
             if isinstance(format.repository_format, RepositoryFormat2a):
                 # group_start, group_len, internal_start, internal_len
-                pos = map(int, val.split())
+                pos = list(map(int, val.split()))
             else:
                 # eol_flag, start, len
                 pos = int(val[1:].split()[0])
 
=== modified file 'breezy/tests/per_versionedfile.py'
--- breezy/tests/per_versionedfile.py	2017-05-24 23:30:47 +0000
+++ breezy/tests/per_versionedfile.py	2017-05-24 23:42:41 +0000
@@ -945,9 +945,9 @@
             return x + '\n'
 
         w = self.get_file()
-        w.add_lines('text0', [], map(addcrlf, base))
-        w.add_lines('text1', ['text0'], map(addcrlf, a))
-        w.add_lines('text2', ['text0'], map(addcrlf, b))
+        w.add_lines('text0', [], list(map(addcrlf, base)))
+        w.add_lines('text1', ['text0'], list(map(addcrlf, a)))
+        w.add_lines('text2', ['text0'], list(map(addcrlf, b)))
 
         self.log_contents(w)
 
@@ -963,7 +963,7 @@
         mt.seek(0)
         self.log(mt.getvalue())
 
-        mp = map(addcrlf, mp)
+        mp = list(map(addcrlf, mp))
         self.assertEqual(mt.readlines(), mp)
 
 
 
=== modified file 'breezy/tests/per_workingtree/test_paths2ids.py'
--- breezy/tests/per_workingtree/test_paths2ids.py	2017-05-21 18:10:28 +0000
+++ breezy/tests/per_workingtree/test_paths2ids.py	2017-05-24 23:42:41 +0000
@@ -21,8 +21,6 @@
 find_ids_across_trees.
 """
 
-from operator import attrgetter
-
 from breezy import errors
 from breezy.tests import features
 from breezy.tests.per_workingtree import TestCaseWithWorkingTree
@@ -47,10 +45,12 @@
         """Run paths2ids for tree, and check the result."""
         tree.lock_read()
         if trees:
-            map(apply, map(attrgetter('lock_read'), trees))
+            for t in trees:
+                t.lock_read()
             result = tree.paths2ids(paths, trees,
                 require_versioned=require_versioned)
-            map(apply, map(attrgetter('unlock'), trees))
+            for t in trees:
+                t.unlock()
         else:
             result = tree.paths2ids(paths,
                 require_versioned=require_versioned)
 
=== modified file 'breezy/tests/test_diff.py'
--- breezy/tests/test_diff.py	2017-05-22 00:56:52 +0000
+++ breezy/tests/test_diff.py	2017-05-24 23:42:41 +0000
@@ -1194,8 +1194,8 @@
                           ]
                           , list(unified_diff(txt_a, txt_b,
                                  sequencematcher=psm)))
-        txt_a = map(lambda x: x+'\n', 'abcdefghijklmnop')
-        txt_b = map(lambda x: x+'\n', 'abcdefxydefghijklmnop')
+        txt_a = [x+'\n' for x in 'abcdefghijklmnop']
+        txt_b = [x+'\n' for x in 'abcdefxydefghijklmnop']
         # This is the result with LongestCommonSubstring matching
         self.assertEqual(['--- \n',
                            '+++ \n',
@@ -1307,8 +1307,8 @@
                           , list(unified_diff_files('a1', 'b1',
                                  sequencematcher=psm)))
 
-        txt_a = map(lambda x: x+'\n', 'abcdefghijklmnop')
-        txt_b = map(lambda x: x+'\n', 'abcdefxydefghijklmnop')
+        txt_a = [x+'\n' for x in 'abcdefghijklmnop']
+        txt_b = [x+'\n' for x in 'abcdefxydefghijklmnop']
         with open('a2', 'wb') as f: f.writelines(txt_a)
         with open('b2', 'wb') as f: f.writelines(txt_b)
 
 
=== modified file 'breezy/tests/test_http.py'
--- breezy/tests/test_http.py	2017-05-22 00:56:52 +0000
+++ breezy/tests/test_http.py	2017-05-24 23:42:41 +0000
@@ -1391,8 +1391,8 @@
 
     def test_range_header(self):
         # Valid ranges
-        map(self.assertEqual,['0', '234'],
-            list(self._file_contents('a', [(0,0), (2,4)])),)
+        self.assertEqual(
+            ['0', '234'], list(self._file_contents('a', [(0,0), (2,4)])))
 
     def test_range_header_tail(self):
         self.assertEqual('789', self._file_tail('a', 3))
 
=== modified file 'breezy/tests/test_rio.py'
--- breezy/tests/test_rio.py	2017-05-21 18:10:28 +0000
+++ breezy/tests/test_rio.py	2017-05-24 23:42:41 +0000
@@ -130,8 +130,8 @@
             s.add(k, v)
         s2 = read_stanza(s.to_lines())
         self.assertEqual(s, s2)
-        self.assertEqual(s.get_all('a'), map(str, [10, 100, 1000]))
-        self.assertEqual(s.get_all('b'), map(str, [20, 200, 2000]))
+        self.assertEqual(s.get_all('a'), ['10', '100', '1000'])
+        self.assertEqual(s.get_all('b'), ['20', '200', '2000'])
 
     def test_backslash(self):
         s = Stanza(q='\\')
 
=== modified file 'breezy/transport/memory.py'
--- breezy/transport/memory.py	2017-05-22 00:56:52 +0000
+++ breezy/transport/memory.py	2017-05-24 23:42:41 +0000
@@ -193,8 +193,8 @@
                 if path.startswith(_abspath):
                     trailing = path[len(_abspath):]
                     if trailing and '/' not in trailing:
-                        result.append(trailing)
-        return map(urlutils.escape, result)
+                        result.append(urlutils.escape(trailing))
+        return result
 
     def rename(self, rel_from, rel_to):
         """Rename a file or directory; fail if the destination exists"""
 
=== modified file 'breezy/transport/sftp.py'
--- breezy/transport/sftp.py	2017-05-24 23:30:47 +0000
+++ breezy/transport/sftp.py	2017-05-24 23:42:41 +0000
@@ -282,7 +282,7 @@
         if data_chunks:
             if 'sftp' in debug.debug_flags:
                 mutter('SFTP readv left with %d out-of-order bytes',
-                    sum(map(lambda x: len(x[1]), data_chunks)))
+                    sum(len(x[1]) for x in data_chunks))
             # We've processed all the readv data, at this point, anything we
             # couldn't process is in data_chunks. This doesn't happen often, so
             # this code path isn't optimized
 
=== modified file 'breezy/util/simplemapi.py'
--- breezy/util/simplemapi.py	2017-02-05 16:35:58 +0000
+++ breezy/util/simplemapi.py	2017-05-24 23:42:41 +0000
@@ -233,10 +233,9 @@
 
     attach = []
     AttachWork = attachfiles.split(';')
-    for file in AttachWork:
-        if os.path.exists(file):
-            attach.append(file)
-    attach = map(os.path.abspath, attach)
+    for f in AttachWork:
+        if os.path.exists(f):
+            attach.append(os.path.abspath(f))
 
     restore = os.getcwd()
     try:
 
=== modified file 'breezy/versionedfile.py'
--- breezy/versionedfile.py	2017-05-24 23:42:40 +0000
+++ breezy/versionedfile.py	2017-05-24 23:42:41 +0000
@@ -1093,9 +1093,9 @@
         while pending:
             this_parent_map = self.get_parent_map(pending)
             parent_map.update(this_parent_map)
-            pending = set()
-            map(pending.update, this_parent_map.itervalues())
-            pending = pending.difference(parent_map)
+            pending = set(itertools.chain.from_iterable(
+                this_parent_map.itervalues()))
+            pending.difference_update(parent_map)
         kg = _mod_graph.KnownGraph(parent_map)
         return kg
 
 
=== modified file 'breezy/vf_repository.py'
--- breezy/vf_repository.py	2017-05-22 00:56:52 +0000
+++ breezy/vf_repository.py	2017-05-24 23:42:41 +0000
@@ -1516,8 +1516,8 @@
             revision_keys
         """
         parent_map = self.revisions.get_parent_map(revision_keys)
-        parent_keys = set()
-        map(parent_keys.update, parent_map.itervalues())
+        parent_keys = set(itertools.chain.from_iterable(
+            parent_map.itervalues()))
         parent_keys.difference_update(revision_keys)
         parent_keys.discard(_mod_revision.NULL_REVISION)
         return parent_keys
 
=== modified file 'breezy/weave.py'
--- breezy/weave.py	2017-05-22 00:56:52 +0000
+++ breezy/weave.py	2017-05-24 23:42:41 +0000
@@ -393,7 +393,7 @@
     def _add_lines(self, version_id, parents, lines, parent_texts,
        left_matching_blocks, nostore_sha, random_id, check_content):
         """See VersionedFile.add_lines."""
-        idx = self._add(version_id, lines, map(self._lookup, parents),
+        idx = self._add(version_id, lines, list(map(self._lookup, parents)),
             nostore_sha=nostore_sha)
         return sha_strings(lines), sum(map(len, lines)), idx
 
 
=== modified file 'breezy/weavefile.py'
--- breezy/weavefile.py	2017-05-22 00:56:52 +0000
+++ breezy/weavefile.py	2017-05-24 23:42:41 +0000
@@ -135,7 +135,7 @@
         l = lines.next()
         if l[0] == 'i':
             if len(l) > 2:
-                w._parents.append(map(int, l[2:].split(' ')))
+                w._parents.append(list(map(int, l[2:].split(' '))))
             else:
                 w._parents.append([])
             l = lines.next()[:-1]