Merge lp:~jelmer/brz/export-nested into lp:brz

[Jelmer Vernooij]

Add support for --recurse-nested argument to 'brz export'.

Add support for --recurse-nested argument to 'brz export'.

[Jelmer Vernooij (jelmer)]

review: Approve

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/882/

[The Breezy Bot (the-breezy-bot)]

Merging failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/884/

=== modified file 'breezy/archive/__init__.py'
--- breezy/archive/__init__.py	2020-02-18 01:57:45 +0000
+++ breezy/archive/__init__.py	2022-01-03 23:15:42 +0000
@@ -77,13 +77,14 @@
 
 
 def create_archive(format, tree, name, root=None, subdir=None,
-                   force_mtime=None):
+                   force_mtime=None, recurse_nested=False):
     try:
         archive_fn = format_registry.get(format)
     except KeyError:
         raise errors.NoSuchExportFormat(format)
     return archive_fn(tree, name, root=root, subdir=subdir,
-                      force_mtime=force_mtime)
+                      force_mtime=force_mtime,
+                      recurse_nested=recurse_nested)
 
 
 format_registry = ArchiveFormatRegistry()

=== modified file 'breezy/archive/tar.py'
--- breezy/archive/tar.py	2020-02-18 01:57:45 +0000
+++ breezy/archive/tar.py	2022-01-03 23:15:42 +0000
@@ -79,21 +79,20 @@
     return (item, fileobj)
 
 
-def tarball_generator(tree, root, subdir=None, force_mtime=None, format=''):
+def tarball_generator(tree, root, subdir=None, force_mtime=None, format='', recurse_nested=False):
     """Export tree contents to a tarball.
 
-    :returns: A generator that will produce file content chunks.
-
-    :param tree: Tree to export
-
-    :param subdir: Sub directory to export
-
-    :param force_mtime: Option mtime to force, instead of using tree
+    Args:
+      tree: Tree to export
+      subdir: Sub directory to export
+      force_mtime: Option mtime to force, instead of using tree
         timestamps.
+    Returns: A generator that will produce file content chunks.
     """
     buf = BytesIO()
     with closing(tarfile.open(None, "w:%s" % format, buf)) as ball, tree.lock_read():
-        for final_path, tree_path, entry in _export_iter_entries(tree, subdir):
+        for final_path, tree_path, entry in _export_iter_entries(
+                tree, subdir, recurse_nested=recurse_nested):
             (item, fileobj) = prepare_tarball_item(
                 tree, root, final_path, tree_path, entry, force_mtime)
             ball.addfile(item, fileobj)
@@ -104,7 +103,7 @@
     yield buf.getvalue()
 
 
-def tgz_generator(tree, dest, root, subdir, force_mtime=None):
+def tgz_generator(tree, dest, root, subdir, force_mtime=None, recurse_nested=False):
     """Export this tree to a new tar file.
 
     `dest` will be created holding the contents of this tree; if it
@@ -133,7 +132,9 @@
         buf = BytesIO()
         zipstream = gzip.GzipFile(basename, 'w', fileobj=buf,
                                   mtime=root_mtime)
-        for chunk in tarball_generator(tree, root, subdir, force_mtime):
+        for chunk in tarball_generator(
+                tree, root, subdir, force_mtime,
+                recurse_nested=recurse_nested):
             zipstream.write(chunk)
             # Yield the data that was written so far, rinse, repeat.
             yield buf.getvalue()
@@ -144,33 +145,37 @@
         yield buf.getvalue()
 
 
-def tbz_generator(tree, dest, root, subdir, force_mtime=None):
+def tbz_generator(tree, dest, root, subdir, force_mtime=None, recurse_nested=False):
     """Export this tree to a new tar file.
 
     `dest` will be created holding the contents of this tree; if it
     already exists, it will be clobbered, like with "tar -c".
     """
     return tarball_generator(
-        tree, root, subdir, force_mtime, format='bz2')
+        tree, root, subdir, force_mtime, format='bz2',
+        recurse_nested=recurse_nested)
 
 
 def plain_tar_generator(tree, dest, root, subdir,
-                        force_mtime=None):
+                        force_mtime=None, recurse_nested=False):
     """Export this tree to a new tar file.
 
     `dest` will be created holding the contents of this tree; if it
     already exists, it will be clobbered, like with "tar -c".
     """
     return tarball_generator(
-        tree, root, subdir, force_mtime, format='')
-
-
-def tar_xz_generator(tree, dest, root, subdir, force_mtime=None):
-    return tar_lzma_generator(tree, dest, root, subdir, force_mtime, "xz")
+        tree, root, subdir, force_mtime, format='',
+        recurse_nested=recurse_nested)
+
+
+def tar_xz_generator(tree, dest, root, subdir, force_mtime=None, recurse_nested=False):
+    return tar_lzma_generator(
+        tree, dest, root, subdir, force_mtime, "xz",
+        recurse_nested=recurse_nested)
 
 
 def tar_lzma_generator(tree, dest, root, subdir, force_mtime=None,
-                       compression_format="alone"):
+                       compression_format="alone", recurse_nested=False):
     """Export this tree to a new .tar.lzma file.
 
     `dest` will be created holding the contents of this tree; if it
@@ -189,7 +194,8 @@
             }[compression_format])
 
     for chunk in tarball_generator(
-            tree, root, subdir, force_mtime=force_mtime):
+            tree, root, subdir, force_mtime=force_mtime,
+            recurse_nested=recurse_nested):
         yield compressor.compress(chunk)
 
     yield compressor.flush()

=== modified file 'breezy/archive/zip.py'
--- breezy/archive/zip.py	2020-02-18 01:57:45 +0000
+++ breezy/archive/zip.py	2022-01-03 23:15:42 +0000
@@ -43,7 +43,7 @@
 
 
 def zip_archive_generator(tree, dest, root, subdir=None,
-                          force_mtime=None):
+                          force_mtime=None, recurse_nested=False):
     """ Export this tree to a new zip file.
 
     `dest` will be created holding the contents of this tree; if it
@@ -53,7 +53,8 @@
     with tempfile.SpooledTemporaryFile() as buf:
         with closing(zipfile.ZipFile(buf, "w", compression)) as zipf, \
                 tree.lock_read():
-            for dp, tp, ie in _export_iter_entries(tree, subdir):
+            for dp, tp, ie in _export_iter_entries(
+                    tree, subdir, recurse_nested=recurse_nested):
                 mutter("  export {%s} kind %s to %s", tp, ie.kind, dest)
 
                 # zipfile.ZipFile switches all paths to forward

=== modified file 'breezy/builtins.py'
--- breezy/builtins.py	2021-12-20 23:40:21 +0000
+++ breezy/builtins.py	2022-01-03 23:15:42 +0000
@@ -3368,11 +3368,13 @@
                      Option('uncommitted',
                             help='Export the working tree contents rather than that of the '
                             'last revision.'),
+                     Option('recurse-nested',
+                            help='Include contents of nested trees.'),
                      ]
 
     def run(self, dest, branch_or_subdir=None, revision=None, format=None,
             root=None, filters=False, per_file_timestamps=False, uncommitted=False,
-            directory=u'.'):
+            directory=u'.', recurse_nested=False):
         from .export import export, guess_format, get_root_name
 
         if branch_or_subdir is None:
@@ -3411,7 +3413,8 @@
 
         try:
             export(export_tree, dest, format, root, subdir,
-                   per_file_timestamps=per_file_timestamps)
+                   per_file_timestamps=per_file_timestamps,
+                   recurse_nested=recurse_nested)
         except errors.NoSuchExportFormat as e:
             raise errors.CommandError(
                 gettext('Unsupported export format: %s') % e.format)

=== modified file 'breezy/bzr/inventorytree.py'
--- breezy/bzr/inventorytree.py	2021-12-25 12:35:22 +0000
+++ breezy/bzr/inventorytree.py	2022-01-03 23:15:42 +0000
@@ -302,9 +302,11 @@
             def iter_entries(inv):
                 for p, e in inv.iter_entries_by_dir(specific_file_ids=inventory_file_ids):
                     if e.kind == 'tree-reference' and recurse_nested:
-                        subinv = self._get_nested_tree(p, e.file_id, e.reference_revision).root_inventory
-                        for subp, e in iter_entries(subinv):
-                            yield (osutils.pathjoin(p, subp) if subp else p), e
+                        subtree = self._get_nested_tree(p, e.file_id, e.reference_revision)
+                        with subtree.lock_read():
+                            subinv = subtree.root_inventory
+                            for subp, e in iter_entries(subinv):
+                                yield (osutils.pathjoin(p, subp) if subp else p), e
                     else:
                         yield p, e
             return iter_entries(self.root_inventory)

=== modified file 'breezy/bzr/remote.py'
--- breezy/bzr/remote.py	2021-08-18 22:38:08 +0000
+++ breezy/bzr/remote.py	2022-01-03 23:15:42 +0000
@@ -962,13 +962,19 @@
     def __init__(self, repository, inv, revision_id):
         super(RemoteInventoryTree, self).__init__(repository, inv, revision_id)
 
-    def archive(self, format, name, root=None, subdir=None, force_mtime=None):
+    def archive(self, format, name, root=None, subdir=None, force_mtime=None, recurse_nested=False):
+        if recurse_nested:
+            # For now, just fall back to non-HPSS mode if nested trees are involved.
+            return super(RemoteInventoryTree, self).archive(
+                format, name, root, subdir, force_mtime=force_mtime,
+                recurse_nested=recurse_nested)
         ret = self._repository._revision_archive(
             self.get_revision_id(), format, name, root, subdir,
             force_mtime=force_mtime)
         if ret is None:
             return super(RemoteInventoryTree, self).archive(
-                format, name, root, subdir, force_mtime=force_mtime)
+                format, name, root, subdir, force_mtime=force_mtime,
+                recurse_nested=recurse_nested)
         return ret
 
     def annotate_iter(self, path,

=== modified file 'breezy/bzr/workingtree.py'
--- breezy/bzr/workingtree.py	2021-11-13 13:01:39 +0000
+++ breezy/bzr/workingtree.py	2022-01-03 23:15:42 +0000
@@ -73,6 +73,7 @@
     errors,
     osutils,
     )
+from ..controldir import ControlDir
 from ..lock import LogicalLockResult
 from .inventorytree import InventoryRevisionTree, MutableInventoryTree
 from ..trace import mutter, note
@@ -509,6 +510,9 @@
     def get_nested_tree(self, path):
         return WorkingTree.open(self.abspath(path))
 
+    def _get_nested_tree(self, path, file_id, reference_revision):
+        return self.get_nested_tree(path)
+
     def set_parent_trees(self, parents_list, allow_leftmost_as_ghost=False):
         """See MutableTree.set_parent_trees."""
         parent_ids = [rev for (rev, tree) in parents_list]
@@ -1756,7 +1760,7 @@
         # The only trick here is that if we supports_tree_reference then we
         # need to detect if a directory becomes a tree-reference.
         iterator = super(WorkingTree, self).iter_entries_by_dir(
-            specific_files=specific_files)
+            specific_files=specific_files, recurse_nested=recurse_nested)
         if not self.supports_tree_reference():
             return iterator
         else:

=== modified file 'breezy/export.py'
--- breezy/export.py	2022-01-03 13:33:13 +0000
+++ breezy/export.py	2022-01-03 23:15:42 +0000
@@ -31,27 +31,29 @@
 
 
 def export(tree, dest, format=None, root=None, subdir=None,
-           per_file_timestamps=False, fileobj=None):
+           per_file_timestamps=False, fileobj=None,
+           recurse_nested=False):
     """Export the given Tree to the specific destination.
 
-    :param tree: A Tree (such as RevisionTree) to export
-    :param dest: The destination where the files,etc should be put
-    :param format: The format (dir, zip, etc), if None, it will check the
-                   extension on dest, looking for a match
-    :param root: The root location inside the format.
-                 It is common practise to have zipfiles and tarballs
-                 extract into a subdirectory, rather than into the
-                 current working directory.
-                 If root is None, the default root will be
-                 selected as the destination without its
-                 extension.
-    :param subdir: A starting directory within the tree. None means to export
-        the entire tree, and anything else should specify the relative path to
-        a directory to start exporting from.
-    :param per_file_timestamps: Whether to use the timestamp stored in the
-        tree rather than now(). This will do a revision lookup
-        for every file so will be significantly slower.
-    :param fileobj: Optional file object to use
+    Args:
+      tree: A Tree (such as RevisionTree) to export
+      dest: The destination where the files,etc should be put
+      format: The format (dir, zip, etc), if None, it will check the
+              extension on dest, looking for a match
+      root: The root location inside the format.
+            It is common practise to have zipfiles and tarballs
+            extract into a subdirectory, rather than into the
+            current working directory.
+            If root is None, the default root will be
+            selected as the destination without its
+            extension.
+      subdir: A starting directory within the tree. None means to export the
+              entire tree, and anything else should specify the relative path
+              to a directory to start exporting from.
+      per_file_timestamps: Whether to use the timestamp stored in the tree
+          rather than now(). This will do a revision lookup for every file so will
+          be significantly slower.
+      fileobj: Optional file object to use
     """
     if format is None and dest is not None:
         format = guess_format(dest)
@@ -79,13 +81,15 @@
         # then we should stream a tar file and unpack that on the fly.
         with tree.lock_read():
             for unused in dir_exporter_generator(tree, dest, root, subdir,
-                                                 force_mtime):
+                                                 force_mtime,
+                                                 recurse_nested=recurse_nested):
                 pass
         return
 
     with tree.lock_read():
         chunks = tree.archive(format, dest, root=root,
-                              subdir=subdir, force_mtime=force_mtime)
+                              subdir=subdir, force_mtime=force_mtime,
+                              recurse_nested=recurse_nested)
         if dest == '-':
             for chunk in chunks:
                 getattr(sys.stdout, 'buffer', sys.stdout).write(chunk)
@@ -126,7 +130,7 @@
     return dest
 
 
-def _export_iter_entries(tree, subdir, skip_special=True):
+def _export_iter_entries(tree, subdir, skip_special=True, recurse_nested=False):
     """Iter the entries for tree suitable for exporting.
 
     :param tree: A tree object.
@@ -139,7 +143,7 @@
         subdir = None
     if subdir is not None:
         subdir = subdir.rstrip('/')
-    entries = tree.iter_entries_by_dir()
+    entries = tree.iter_entries_by_dir(recurse_nested=recurse_nested)
     for path, entry in entries:
         if path == '':
             continue
@@ -166,7 +170,8 @@
 
 
 def dir_exporter_generator(tree, dest, root, subdir=None,
-                           force_mtime=None, fileobj=None):
+                           force_mtime=None, fileobj=None,
+                           recurse_nested=False):
     """Return a generator that exports this tree to a new directory.
 
     `dest` should either not exist or should be empty. If it does not exist it
@@ -192,7 +197,8 @@
     # Note in the case of revision trees, this does trigger a double inventory
     # lookup, hopefully it isn't too expensive.
     to_fetch = []
-    for dp, tp, ie in _export_iter_entries(tree, subdir):
+    for dp, tp, ie in _export_iter_entries(
+            tree, subdir, recurse_nested=recurse_nested):
         fullpath = osutils.pathjoin(dest, dp)
         if ie.kind == "file":
             to_fetch.append((tp, (dp, tp, None)))

=== modified file 'breezy/git/remote.py'
--- breezy/git/remote.py	2021-11-13 13:27:11 +0000
+++ breezy/git/remote.py	2022-01-03 23:15:42 +0000
@@ -421,7 +421,9 @@
         return RemoteGitRepository
 
     def archive(self, format, committish, write_data, progress=None,
-                write_error=None, subdirs=None, prefix=None):
+                write_error=None, subdirs=None, prefix=None, recurse_nested=False):
+        if recurse_nested:
+            raise NotImplementedError('recurse_nested is not yet supported')
         if progress is None:
             pb = ui.ui_factory.nested_progress_bar()
             progress = DefaultProgressReporter(pb).progress
@@ -882,7 +884,7 @@
 
 class GitRemoteRevisionTree(RevisionTree):
 
-    def archive(self, format, name, root=None, subdir=None, force_mtime=None):
+    def archive(self, format, name, root=None, subdir=None, force_mtime=None, recurse_nested=False):
         """Create an archive of this tree.
 
         :param format: Format name (e.g. 'tar')
@@ -891,6 +893,8 @@
         :param subdir: Subdirectory to export (or None)
         :return: Iterator over archive chunks
         """
+        if recurse_nested:
+            raise NotImplementedError('recurse_nested is not yet supported')
         commit = self._repository.lookup_bzr_revision_id(
             self.get_revision_id())[0]
         import tempfile

=== modified file 'breezy/git/tree.py'
--- breezy/git/tree.py	2021-12-25 12:35:22 +0000
+++ breezy/git/tree.py	2022-01-03 23:15:42 +0000
@@ -351,6 +351,9 @@
         else:
             nested_repo_transport = self._repository.controldir.control_transport.clone(
                 posixpath.join('modules', decode_git_path(info[1])))
+            if not nested_repo_transport.has('.'):
+                nested_repo_transport = self._repository.controldir.user_transport.clone(
+                    posixpath.join(decode_git_path(info[1]), '.git'))
         nested_controldir = _mod_controldir.ControlDir.open_from_transport(
             nested_repo_transport)
         return nested_controldir.find_repository()
@@ -574,14 +577,16 @@
                 child_path_decoded = decode_git_path(child_path)
                 if recurse_nested and S_ISGITLINK(mode):
                     mode = stat.S_IFDIR
-                    store = self._get_submodule_store(child_path)
-                    hexsha = store[hexsha].tree
+                    substore = self._get_submodule_store(child_path)
+                    hexsha = substore[hexsha].tree
+                else:
+                    substore = store
                 if stat.S_ISDIR(mode):
                     if (specific_files is None or
                             any([p for p in specific_files if p.startswith(
                                 child_path)])):
                         extradirs.append(
-                            (store, child_path, hexsha,
+                            (substore, child_path, hexsha,
                              self.path2id(child_path_decoded)))
                 if specific_files is None or child_path in specific_files:
                     if stat.S_ISDIR(mode):
@@ -589,7 +594,7 @@
                                self._get_dir_ie(child_path, parent_id))
                     else:
                         yield (child_path_decoded,
-                               self._get_file_ie(store, child_path, name, mode,
+                               self._get_file_ie(substore, child_path, name, mode,
                                                  hexsha, parent_id))
             todo.extendleft(reversed(extradirs))
 

=== modified file 'breezy/tests/blackbox/test_export.py'
--- breezy/tests/blackbox/test_export.py	2020-06-19 21:26:53 +0000
+++ breezy/tests/blackbox/test_export.py	2022-01-03 23:15:42 +0000
@@ -241,6 +241,23 @@
         d_info = zfile.getinfo(names[3])
         self.assertEqual(dir_attr, d_info.external_attr)
 
+    def test_dir_export_nested(self):
+        tree = self.make_branch_and_tree('dir')
+        self.build_tree(['dir/a'])
+        tree.add('a')
+
+        subtree = self.make_branch_and_tree('dir/subdir')
+        tree.add_reference(subtree)
+
+        self.build_tree(['dir/subdir/b'])
+        subtree.add('b')
+
+        self.run_bzr('export --uncommitted direxport1 dir')
+        self.assertFalse(os.path.exists('direxport1/subdir/b'))
+
+        self.run_bzr('export --recurse-nested --uncommitted direxport2 dir')
+        self.assertTrue(os.path.exists('direxport2/subdir/b'))
+
     def test_dir_export(self):
         tree = self.make_branch_and_tree('dir')
         self.build_tree(['dir/a'])

=== modified file 'breezy/tests/per_tree/test_export.py'
--- breezy/tests/per_tree/test_export.py	2021-12-08 18:46:11 +0000
+++ breezy/tests/per_tree/test_export.py	2022-01-03 23:15:42 +0000
@@ -18,12 +18,14 @@
 import tarfile
 import zipfile
 
+from breezy.errors import UnsupportedOperation
 from breezy.export import export
 from breezy import osutils
 from breezy import tests
 from breezy.tests.per_tree import TestCaseWithTree
 from breezy.tests import (
     features,
+    TestNotApplicable,
     )
 
 
@@ -59,6 +61,35 @@
         names = self.get_export_names()
         self.assertIn('output/link', names)
 
+    def prepare_nested_export(self, recurse_nested):
+        tree = self.make_branch_and_tree('dir')
+        self.build_tree(['dir/a'])
+        tree.add('a')
+        tree.commit('1')
+
+        subtree = self.make_branch_and_tree('dir/subdir')
+
+        self.build_tree(['dir/subdir/b'])
+        subtree.add('b')
+        subtree.commit('1a')
+
+        try:
+            tree.add_reference(subtree)
+        except UnsupportedOperation:
+            raise TestNotApplicable('format does not supported nested trees')
+        tree.commit('2')
+        export(tree, 'output', self.exporter, recurse_nested=recurse_nested)
+
+    def test_export_nested_recurse(self):
+        self.prepare_nested_export(True)
+        names = self.get_export_names()
+        self.assertIn('output/subdir/b', names)
+
+    def test_export_nested_nonrecurse(self):
+        self.prepare_nested_export(False)
+        names = self.get_export_names()
+        self.assertNotIn('output/subdir/b', names)
+
 
 class TestTar(ExportTest, TestCaseWithTree):
 
@@ -94,5 +125,10 @@
     exporter = 'dir'
 
     def get_export_names(self):
-        return [osutils.pathjoin('output', name)
-                for name in os.listdir('output')]
+        ret = []
+        for (dirpath, dirnames, filenames) in os.walk('output'):
+            for dirname in dirnames:
+                ret.append(osutils.pathjoin(dirpath, dirname))
+            for filename in filenames:
+                ret.append(osutils.pathjoin(dirpath, filename))
+        return ret

=== modified file 'breezy/tree.py'
--- breezy/tree.py	2021-12-25 12:35:22 +0000
+++ breezy/tree.py	2022-01-03 23:15:42 +0000
@@ -713,19 +713,21 @@
         return searcher
 
     def archive(self, format, name, root='', subdir=None,
-                force_mtime=None):
+                force_mtime=None, recurse_nested=False):
         """Create an archive of this tree.
 
-        :param format: Format name (e.g. 'tar')
-        :param name: target file name
-        :param root: Root directory name (or None)
-        :param subdir: Subdirectory to export (or None)
-        :return: Iterator over archive chunks
+        Args:
+          format: Format name (e.g. 'tar')
+          name: target file name
+          root: Root directory name (or None)
+          subdir: Subdirectory to export (or None)
+        Returns: Iterator over archive chunks
         """
         from .archive import create_archive
         with self.lock_read():
             return create_archive(format, self, name, root,
-                                  subdir, force_mtime=force_mtime)
+                                  subdir, force_mtime=force_mtime,
+                                  recurse_nested=recurse_nested)
 
     @classmethod
     def versionable_kind(cls, kind):