Merge lp:~jelmer/brz/follow-tree-references-iter-entries-by-dir into lp:brz

[Jelmer Vernooij]

Add recurse_nested argument to Tree.iter_entries_by_dir.

Add follow_tree_references argument to Tree.iter_entries_by_dir.

This is another step towards nested tree support.

[The Breezy Bot (the-breezy-bot)]

Voting criteria not met
https://ci.breezy-vcs.org/job/brz/job/brz-land/579/

[Jelmer Vernooij (jelmer)]

review: Approve

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/580/

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/581/

=== modified file 'breezy/bzr/inventorytree.py'
--- breezy/bzr/inventorytree.py	2020-01-11 02:39:34 +0000
+++ breezy/bzr/inventorytree.py	2020-01-11 02:39:34 +0000
@@ -213,7 +213,8 @@
     def all_versioned_paths(self):
         return {path for path, entry in self.iter_entries_by_dir()}
 
-    def iter_entries_by_dir(self, specific_files=None):
+    def iter_entries_by_dir(self, specific_files=None,
+                            recurse_nested=False):
         """Walk the tree in 'by_dir' order.
 
         This will yield each entry in the tree as a (path, entry) tuple.
@@ -226,17 +227,21 @@
                 inventory_file_ids = []
                 for path in specific_files:
                     inventory, inv_file_id = self._path2inv_file_id(path)
-                    if inv_file_id is None:
-                        continue
-                    if inventory is not self.root_inventory:  # for now
+                    if inventory and inventory is not self.root_inventory:
                         raise AssertionError("%r != %r" % (
                             inventory, self.root_inventory))
                     inventory_file_ids.append(inv_file_id)
             else:
                 inventory_file_ids = None
-            # FIXME: Handle nested trees
-            return self.root_inventory.iter_entries_by_dir(
-                specific_file_ids=inventory_file_ids)
+            def iter_entries(inv):
+                for p, e in inv.iter_entries_by_dir(specific_file_ids=inventory_file_ids):
+                    if e.kind == 'tree-reference' and recurse_nested:
+                        subinv = self._get_nested_tree(p, e.file_id, e.reference_revision).root_inventory
+                        for subp, e in iter_entries(subinv):
+                            yield (osutils.pathjoin(p, subp) if subp else p), e
+                    else:
+                        yield p, e
+            return iter_entries(self.root_inventory)
 
     def iter_child_entries(self, path):
         with self.lock_read():

=== modified file 'breezy/bzr/workingtree.py'
--- breezy/bzr/workingtree.py	2020-01-11 02:39:34 +0000
+++ breezy/bzr/workingtree.py	2020-01-11 02:39:34 +0000
@@ -1728,7 +1728,7 @@
             self.control_transport.put_bytes(
                 'format', self._format.as_string())
 
-    def _check_for_tree_references(self, iterator):
+    def _check_for_tree_references(self, iterator, recurse_nested, specific_files=None):
         """See if directories have become tree-references."""
         blocked_parent_ids = set()
         for path, ie in iterator:
@@ -1740,12 +1740,25 @@
                 continue
             if (ie.kind == 'directory' and
                     self._directory_is_tree_reference(path)):
+
                 # This InventoryDirectory needs to be a TreeReference
                 ie = inventory.TreeReference(ie.file_id, ie.name, ie.parent_id)
                 blocked_parent_ids.add(ie.file_id)
-            yield path, ie
-
-    def iter_entries_by_dir(self, specific_files=None):
+
+            if ie.kind == 'tree-reference' and recurse_nested:
+                subtree = self.get_nested_tree(path)
+                for subpath, ie in subtree.iter_entries_by_dir(
+                        recurse_nested=recurse_nested,
+                        specific_files=specific_files):
+                    if subpath:
+                        full_subpath = osutils.pathjoin(path, subpath)
+                    else:
+                        full_subpath = path
+                    yield full_subpath, ie
+            else:
+                yield path, ie
+
+    def iter_entries_by_dir(self, specific_files=None, recurse_nested=False):
         """See Tree.iter_entries_by_dir()"""
         # The only trick here is that if we supports_tree_reference then we
         # need to detect if a directory becomes a tree-reference.
@@ -1754,7 +1767,9 @@
         if not self.supports_tree_reference():
             return iterator
         else:
-            return self._check_for_tree_references(iterator)
+            return self._check_for_tree_references(
+                iterator, recurse_nested=recurse_nested,
+                specific_files=specific_files)
 
     def get_canonical_paths(self, paths):
         """Look up canonical paths for multiple items.

=== modified file 'breezy/bzr/workingtree_4.py'
--- breezy/bzr/workingtree_4.py	2020-01-11 02:39:34 +0000
+++ breezy/bzr/workingtree_4.py	2020-01-11 02:39:34 +0000
@@ -374,10 +374,6 @@
                     # add this entry to the parent map.
                     parent_ies[(dirname + b'/' + name).strip(b'/')] = inv_entry
                 elif kind == 'tree-reference':
-                    if not self._repo_supports_tree_reference:
-                        raise errors.UnsupportedOperation(
-                            self._generate_inventory,
-                            self.branch.repository)
                     inv_entry.reference_revision = link_or_sha1 or None
                 elif kind != 'symlink':
                     raise AssertionError("unknown kind %r" % kind)

=== modified file 'breezy/filter_tree.py'
--- breezy/filter_tree.py	2018-11-16 18:33:17 +0000
+++ breezy/filter_tree.py	2020-01-11 02:39:34 +0000
@@ -63,14 +63,14 @@
     def is_executable(self, path):
         return self.backing_tree.is_executable(path)
 
-    def iter_entries_by_dir(self, specific_files=None):
+    def iter_entries_by_dir(self, specific_files=None, recurse_nested=False):
         # NB: This simply returns the parent tree's entries; the length may be
         # wrong but it can't easily be calculated without filtering the whole
         # text.  Currently all callers cope with this; perhaps they should be
         # updated to a narrower interface that only provides things guaranteed
         # cheaply available across all trees. -- mbp 20110705
         return self.backing_tree.iter_entries_by_dir(
-            specific_files=specific_files)
+            specific_files=specific_files, recurse_nested=recurse_nested)
 
     def lock_read(self):
         return self.backing_tree.lock_read()

=== modified file 'breezy/git/tree.py'
--- breezy/git/tree.py	2020-01-11 02:39:34 +0000
+++ breezy/git/tree.py	2020-01-11 02:39:34 +0000
@@ -526,7 +526,8 @@
                 yield self._get_file_ie(store, child_path, name, mode, hexsha,
                                         file_id)
 
-    def iter_entries_by_dir(self, specific_files=None):
+    def iter_entries_by_dir(self, specific_files=None,
+                            recurse_nested=False):
         if self.tree is None:
             return
         if specific_files is not None:
@@ -547,6 +548,10 @@
                     continue
                 child_path = posixpath.join(path, name)
                 child_path_decoded = child_path.decode('utf-8')
+                if recurse_nested and S_ISGITLINK(mode):
+                    mode = stat.S_IFDIR
+                    store = self._get_submodule_store(child_path)
+                    hexsha = store[hexsha].tree
                 if stat.S_ISDIR(mode):
                     if (specific_files is None or
                             any([p for p in specific_files if p.startswith(
@@ -1217,7 +1222,8 @@
                 else:
                     yield (posixpath.join(basepath, path), value)
 
-    def iter_entries_by_dir(self, specific_files=None):
+    def iter_entries_by_dir(self, specific_files=None,
+                            recurse_nested=False):
         with self.lock_read():
             if specific_files is not None:
                 specific_files = set(specific_files)
@@ -1228,7 +1234,8 @@
             if specific_files is None or u"" in specific_files:
                 ret[(u"", u"")] = root_ie
             dir_ids = {u"": root_ie.file_id}
-            for path, value in self._recurse_index_entries():
+            for path, value in self._recurse_index_entries(
+                    recurse_nested=recurse_nested):
                 if self.mapping.is_special_file(path):
                     continue
                 path = path.decode("utf-8")

=== modified file 'breezy/tests/blackbox/test_join.py'
--- breezy/tests/blackbox/test_join.py	2020-01-11 02:39:34 +0000
+++ breezy/tests/blackbox/test_join.py	2020-01-11 02:39:34 +0000
@@ -76,7 +76,8 @@
             self.assertEqual('file1', sub_tree.id2path(b'file1-id'))
             self.assertEqual(subtree_root_id, sub_tree.path2id(''))
             self.assertEqual('', sub_tree.id2path(subtree_root_id))
-            self.assertEqual(b'file1-id', base_tree.path2id('subtree/file1'))
+            self.assertEqual(
+                sub_tree.path2id('file1'), base_tree.path2id('subtree/file1'))
 
         base_tree.lock_read()
         self.addCleanup(base_tree.unlock)

=== modified file 'breezy/tests/per_tree/test_test_trees.py'
--- breezy/tests/per_tree/test_test_trees.py	2019-10-13 14:28:07 +0000
+++ breezy/tests/per_tree/test_test_trees.py	2020-01-11 02:39:34 +0000
@@ -410,3 +410,43 @@
         last_revision = getattr(tree, 'last_revision', None)
         if last_revision is not None:
             self.assertIsInstance(last_revision(), bytes)
+
+    def skip_if_no_reference(self, tree):
+        if not getattr(tree, 'supports_tree_reference', lambda: False)():
+            raise TestNotApplicable('Tree references not supported')
+
+    def create_nested(self):
+        work_tree = self.make_branch_and_tree('wt')
+        with work_tree.lock_write():
+            self.skip_if_no_reference(work_tree)
+            subtree = self.make_branch_and_tree('wt/subtree')
+            self.build_tree(['wt/subtree/a'])
+            subtree.add(['a'])
+            subtree.commit('foo')
+            work_tree.add_reference(subtree)
+        tree = self._convert_tree(work_tree)
+        self.skip_if_no_reference(tree)
+        return tree, subtree
+
+    def test_iter_entries_with_unfollowed_reference(self):
+        tree, subtree = self.create_nested()
+        expected = [
+            ('', 'directory'),
+            ('subtree', 'tree-reference')]
+        with tree.lock_read():
+            path_entries = list(tree.iter_entries_by_dir(recurse_nested=False))
+            actual = [(path, ie.kind)
+                      for path, ie in path_entries]
+        self.assertEqual(expected, actual)
+
+    def test_iter_entries_with_followed_reference(self):
+        tree, subtree = self.create_nested()
+        expected = [
+            ('', 'directory'),
+            ('subtree', 'directory'),
+            ('subtree/a', 'file')]
+        with tree.lock_read():
+            path_entries = list(tree.iter_entries_by_dir(recurse_nested=True))
+            actual = [(path, ie.kind)
+                      for path, ie in path_entries]
+        self.assertEqual(expected, actual)

=== modified file 'breezy/tests/test_dirstate.py'
--- breezy/tests/test_dirstate.py	2020-01-11 02:39:34 +0000
+++ breezy/tests/test_dirstate.py	2020-01-11 02:39:34 +0000
@@ -2564,8 +2564,8 @@
             basis_tree.root_inventory)
         state.update_basis_by_delta(delta, b'target')
         state._validate()
-        dirstate_tree = workingtree_4.DirStateRevisionTree(state,
-                                                           b'target', _Repo(), None)
+        dirstate_tree = workingtree_4.DirStateRevisionTree(
+            state, b'target', _Repo(), None)
         # The target now that delta has been applied should match the
         # RevisionTree
         self.assertEqual([], list(dirstate_tree.iter_changes(target_tree)))

=== modified file 'breezy/transform.py'
--- breezy/transform.py	2019-09-29 12:45:26 +0000
+++ breezy/transform.py	2020-01-11 02:39:34 +0000
@@ -2218,7 +2218,11 @@
         for entry, trans_id in self._make_inv_entries(todo):
             yield entry
 
-    def iter_entries_by_dir(self, specific_files=None):
+    def iter_entries_by_dir(self, specific_files=None, recurse_nested=False):
+        if recurse_nested:
+            raise NotImplementedError(
+                'follow tree references not yet supported')
+
         # This may not be a maximally efficient implementation, but it is
         # reasonably straightforward.  An implementation that grafts the
         # TreeTransform changes onto the tree's iter_entries_by_dir results
@@ -2242,8 +2246,13 @@
         path_entries.sort()
         return path_entries
 
-    def list_files(self, include_root=False, from_dir=None, recursive=True):
+    def list_files(self, include_root=False, from_dir=None, recursive=True,
+                   recurse_nested=False):
         """See WorkingTree.list_files."""
+        if recurse_nested:
+            raise NotImplementedError(
+                'follow tree references not yet supported')
+
         # XXX This should behave like WorkingTree.list_files, but is really
         # more like RevisionTree.list_files.
         if from_dir == '.':

=== modified file 'breezy/tree.py'
--- breezy/tree.py	2020-01-11 02:39:34 +0000
+++ breezy/tree.py	2020-01-11 02:39:34 +0000
@@ -317,7 +317,7 @@
         """
         raise NotImplementedError(self.id2path)
 
-    def iter_entries_by_dir(self, specific_files=None):
+    def iter_entries_by_dir(self, specific_files=None, recurse_nested=False):
         """Walk the tree in 'by_dir' order.
 
         This will yield each entry in the tree as a (path, entry) tuple.
@@ -341,6 +341,10 @@
         The yield order (ignoring root) would be::
 
           a, f, a/b, a/d, a/b/c, a/d/e, f/g
+
+        If recurse_nested is enabled then nested trees are included as if
+        they were a part of the tree. If is disabled then TreeReference
+        objects (without any children) are yielded.
         """
         raise NotImplementedError(self.iter_entries_by_dir)