Merge lp:~jelmer/brz/hpss-archive into lp:brz

[Jelmer Vernooij]

Add a HPSS implementation of Tree.archive.

Add a HPSS implementation of Tree.archive.

This very significantly speeds up 'bzr export' operations of remote trees.
This is useful for e.g. 'brz merge-upstream', especially on large trees.

[The Breezy Bot (the-breezy-bot)]

Voting criteria not met
https://ci.breezy-vcs.org/job/land-brz/177/

[Martin Packman (gz)]

One inline comment right at the end.

review: Approve

=== added directory 'breezy/archive'
=== added file 'breezy/archive/__init__.py'
--- breezy/archive/__init__.py	1970-01-01 00:00:00 +0000
+++ breezy/archive/__init__.py	2018-05-20 16:34:17 +0000
@@ -0,0 +1,103 @@
+# Copyright (C) 2018 Breezy Developers
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+"""Export trees to tarballs, zipfiles, etc.
+"""
+
+from __future__ import absolute_import
+
+import os
+import time
+import warnings
+
+from .. import (
+    errors,
+    pyutils,
+    registry,
+    trace,
+    )
+
+
+class ArchiveFormatInfo(object):
+
+    def __init__(self, extensions):
+        self.extensions = extensions
+
+
+class ArchiveFormatRegistry(registry.Registry):
+    """Registry of archive formats."""
+
+    def __init__(self):
+        self._extension_map = {}
+        super(ArchiveFormatRegistry, self).__init__()
+
+    @property
+    def extensions(self):
+        return self._extension_map.keys()
+
+    def register(self, key, factory, extensions, help=None):
+        """Register an archive format.
+        """
+        registry.Registry.register(self, key, factory, help,
+            ArchiveFormatInfo(extensions))
+        self._register_extensions(key, extensions)
+
+    def register_lazy(self, key, module_name, member_name, extensions,
+                      help=None):
+        registry.Registry.register_lazy(self, key, module_name, member_name,
+            help, ArchiveFormatInfo(extensions))
+        self._register_extensions(key, extensions)
+
+    def _register_extensions(self, name, extensions):
+        for ext in extensions:
+            self._extension_map[ext] = name
+
+    def get_format_from_filename(self, filename):
+        """Determine the archive format from an extension.
+
+        :param filename: Filename to guess from
+        :return: A format name, or None
+        """
+        for ext, format in self._extension_map.items():
+            if filename.endswith(ext):
+                return format
+        else:
+            return None
+
+
+def create_archive(format, tree, name, root=None, subdir=None,
+                   force_mtime=None):
+    try:
+        archive_fn = format_registry.get(format)
+    except KeyError:
+        raise errors.NoSuchExportFormat(format)
+    return archive_fn(tree, name, root=root, subdir=subdir,
+                      force_mtime=force_mtime)
+
+
+format_registry = ArchiveFormatRegistry()
+format_registry.register_lazy('tar', 'breezy.archive.tar',
+                              'plain_tar_generator', ['.tar'], )
+format_registry.register_lazy('tgz', 'breezy.archive.tar',
+                              'tgz_generator', ['.tar.gz', '.tgz'])
+format_registry.register_lazy('tbz2', 'breezy.archive.tar',
+                              'tbz_generator',  ['.tar.bz2', '.tbz2'])
+format_registry.register_lazy('tlzma', 'breezy.archive.tar',
+                              'tar_lzma_generator', ['.tar.lzma'])
+format_registry.register_lazy('txz', 'breezy.archive.tar',
+                              'tar_xz_generator', ['.tar.xz'])
+format_registry.register_lazy('zip', 'breezy.archive.zip',
+                              'zip_archive_generator', ['.zip'])

=== renamed file 'breezy/export/tar_exporter.py' => 'breezy/archive/tar.py'
--- breezy/export/tar_exporter.py	2018-05-20 16:34:17 +0000
+++ breezy/archive/tar.py	2018-05-20 16:34:17 +0000
@@ -18,6 +18,7 @@
 
 from __future__ import absolute_import
 
+from contextlib import closing
 import os
 import sys
 import tarfile
@@ -81,141 +82,114 @@
     return (item, fileobj)
 
 
-def export_tarball_generator(tree, ball, root, subdir=None, force_mtime=None):
+def tarball_generator(tree, root, subdir=None, force_mtime=None, format=''):
     """Export tree contents to a tarball.
 
-    :returns: A generator that will repeatedly produce None as each file is
-        emitted.  The entire generator must be consumed to complete writing
-        the file.
+    :returns: A generator that will produce file content chunks.
 
     :param tree: Tree to export
 
-    :param ball: Tarball to export to; it will be closed when writing is
-        complete.
-
     :param subdir: Sub directory to export
 
     :param force_mtime: Option mtime to force, instead of using tree
         timestamps.
     """
-    try:
+    buf = BytesIO()
+    with closing(tarfile.open(None, "w:%s" % format, buf)) as ball, tree.lock_read():
         for final_path, tree_path, entry in _export_iter_entries(tree, subdir):
             (item, fileobj) = prepare_tarball_item(
                 tree, root, final_path, tree_path, entry, force_mtime)
             ball.addfile(item, fileobj)
-            yield
-    finally:
-        ball.close()
-
-
-def tgz_exporter_generator(tree, dest, root, subdir, force_mtime=None,
-    fileobj=None):
-    """Export this tree to a new tar file.
-
-    `dest` will be created holding the contents of this tree; if it
-    already exists, it will be clobbered, like with "tar -c".
-    """
-    import gzip
-    if force_mtime is not None:
-        root_mtime = force_mtime
-    elif (getattr(tree, "repository", None) and
-          getattr(tree, "get_revision_id", None)):
-        # If this is a revision tree, use the revisions' timestamp
-        rev = tree.repository.get_revision(tree.get_revision_id())
-        root_mtime = rev.timestamp
-    elif tree.get_root_id() is not None:
-        root_mtime = tree.get_file_mtime('', tree.get_root_id())
-    else:
-        root_mtime = None
-
-    is_stdout = False
-    basename = None
-    if fileobj is not None:
-        stream = fileobj
-    elif dest == '-':
-        stream = sys.stdout
-        is_stdout = True
-    else:
-        stream = open(dest, 'wb')
-    # gzip file is used with an explicit fileobj so that
-    # the basename can be stored in the gzip file rather than
-    # dest. (bug 102234)
-    basename = os.path.basename(dest)
-    zipstream = gzip.GzipFile(basename, 'w', fileobj=stream,
-                              mtime=root_mtime)
-    ball = tarfile.open(None, 'w|', fileobj=zipstream)
-    for _ in export_tarball_generator(
-        tree, ball, root, subdir, force_mtime):
-        yield
-    # Closing zipstream may trigger writes to stream
-    zipstream.close()
-    if not is_stdout:
-        # Now we can safely close the stream
-        stream.close()
-
-
-def tbz_exporter_generator(tree, dest, root, subdir,
-                           force_mtime=None, fileobj=None):
-    """Export this tree to a new tar file.
-
-    `dest` will be created holding the contents of this tree; if it
-    already exists, it will be clobbered, like with "tar -c".
-    """
-    if fileobj is not None:
-        ball = tarfile.open(None, 'w|bz2', fileobj)
-    elif dest == '-':
-        ball = tarfile.open(None, 'w|bz2', sys.stdout)
-    else:
-        ball = tarfile.open(dest, 'w:bz2')
-    return export_tarball_generator(
-        tree, ball, root, subdir, force_mtime)
-
-
-def plain_tar_exporter_generator(tree, dest, root, subdir, compression=None,
-    force_mtime=None, fileobj=None):
-    """Export this tree to a new tar file.
-
-    `dest` will be created holding the contents of this tree; if it
-    already exists, it will be clobbered, like with "tar -c".
-    """
-    if fileobj is not None:
-        stream = fileobj
-    elif dest == '-':
-        stream = sys.stdout
-    else:
-        stream = open(dest, 'wb')
-    ball = tarfile.open(None, 'w|', stream)
-    return export_tarball_generator(
-        tree, ball, root, subdir, force_mtime)
-
-
-def tar_xz_exporter_generator(tree, dest, root, subdir,
-                              force_mtime=None, fileobj=None):
-    return tar_lzma_exporter_generator(tree, dest, root, subdir,
-                                       force_mtime, fileobj, "xz")
-
-
-def tar_lzma_exporter_generator(tree, dest, root, subdir,
-                      force_mtime=None, fileobj=None,
-                                compression_format="alone"):
+            # Yield the data that was written so far, rinse, repeat.
+            yield buf.getvalue()
+            buf.truncate(0)
+            buf.seek(0)
+    yield buf.getvalue()
+
+
+def tgz_generator(tree, dest, root, subdir, force_mtime=None):
+    """Export this tree to a new tar file.
+
+    `dest` will be created holding the contents of this tree; if it
+    already exists, it will be clobbered, like with "tar -c".
+    """
+    with tree.lock_read():
+        import gzip
+        if force_mtime is not None:
+            root_mtime = force_mtime
+        elif (getattr(tree, "repository", None) and
+              getattr(tree, "get_revision_id", None)):
+            # If this is a revision tree, use the revisions' timestamp
+            rev = tree.repository.get_revision(tree.get_revision_id())
+            root_mtime = rev.timestamp
+        elif tree.get_root_id() is not None:
+            root_mtime = tree.get_file_mtime('', tree.get_root_id())
+        else:
+            root_mtime = None
+
+        is_stdout = False
+        basename = None
+        # gzip file is used with an explicit fileobj so that
+        # the basename can be stored in the gzip file rather than
+        # dest. (bug 102234)
+        basename = os.path.basename(dest)
+        buf = BytesIO()
+        zipstream = gzip.GzipFile(basename, 'w', fileobj=buf,
+                                  mtime=root_mtime)
+        for chunk in tarball_generator(
+            tree, root, subdir, force_mtime):
+            zipstream.write(chunk)
+            # Yield the data that was written so far, rinse, repeat.
+            yield buf.getvalue()
+            buf.truncate(0)
+            buf.seek(0)
+        # Closing zipstream may trigger writes to stream
+        zipstream.close()
+        yield buf.getvalue()
+
+
+def tbz_generator(tree, dest, root, subdir, force_mtime=None):
+    """Export this tree to a new tar file.
+
+    `dest` will be created holding the contents of this tree; if it
+    already exists, it will be clobbered, like with "tar -c".
+    """
+    return tarball_generator(
+        tree, root, subdir, force_mtime, format='bz2')
+
+
+def plain_tar_generator(tree, dest, root, subdir,
+    force_mtime=None):
+    """Export this tree to a new tar file.
+
+    `dest` will be created holding the contents of this tree; if it
+    already exists, it will be clobbered, like with "tar -c".
+    """
+    return tarball_generator(
+        tree, root, subdir, force_mtime, format='')
+
+
+def tar_xz_generator(tree, dest, root, subdir, force_mtime=None):
+    return tar_lzma_generator(tree, dest, root, subdir, force_mtime, "xz")
+
+
+def tar_lzma_generator(tree, dest, root, subdir, force_mtime=None,
+                       compression_format="alone"):
     """Export this tree to a new .tar.lzma file.
 
     `dest` will be created holding the contents of this tree; if it
     already exists, it will be clobbered, like with "tar -c".
     """
-    if dest == '-':
-        raise errors.BzrError("Writing to stdout not supported for .tar.lzma")
-
-    if fileobj is not None:
-        raise errors.BzrError(
-            "Writing to fileobject not supported for .tar.lzma")
     try:
         import lzma
     except ImportError as e:
         raise errors.DependencyNotPresent('lzma', e)
 
-    stream = lzma.LZMAFile(dest.encode(osutils._fs_enc), 'w',
-        options={"format": compression_format})
-    ball = tarfile.open(None, 'w:', fileobj=stream)
-    return export_tarball_generator(
-        tree, ball, root, subdir, force_mtime=force_mtime)
+    compressor = lzma.LZMACompressor(
+            options={"format": compression_format})
+
+    for chunk in tarball_generator(
+            tree, root, subdir, force_mtime=force_mtime):
+        yield compressor.compress(chunk)
+
+    yield compressor.flush()

=== renamed file 'breezy/export/zip_exporter.py' => 'breezy/archive/zip.py'
--- breezy/export/zip_exporter.py	2018-05-14 20:49:13 +0000
+++ breezy/archive/zip.py	2018-05-20 16:34:17 +0000
@@ -19,9 +19,11 @@
 
 from __future__ import absolute_import
 
+from contextlib import closing
 import os
 import stat
 import sys
+import tempfile
 import time
 import zipfile
 
@@ -42,64 +44,56 @@
 _DIR_ATTR = stat.S_IFDIR | ZIP_DIRECTORY_BIT | DIR_PERMISSIONS
 
 
-def zip_exporter_generator(tree, dest, root, subdir=None,
-    force_mtime=None, fileobj=None):
+def zip_archive_generator(tree, dest, root, subdir=None,
+    force_mtime=None):
     """ Export this tree to a new zip file.
 
     `dest` will be created holding the contents of this tree; if it
     already exists, it will be overwritten".
     """
-
     compression = zipfile.ZIP_DEFLATED
-    if fileobj is not None:
-        dest = fileobj
-    elif dest == "-":
-        dest = sys.stdout
-    zipf = zipfile.ZipFile(dest, "w", compression)
-    try:
-        for dp, tp, ie in _export_iter_entries(tree, subdir):
-            file_id = ie.file_id
-            mutter("  export {%s} kind %s to %s", file_id, ie.kind, dest)
-
-            # zipfile.ZipFile switches all paths to forward
-            # slashes anyway, so just stick with that.
-            if force_mtime is not None:
-                mtime = force_mtime
-            else:
-                mtime = tree.get_file_mtime(tp, ie.file_id)
-            date_time = time.localtime(mtime)[:6]
-            filename = osutils.pathjoin(root, dp).encode('utf8')
-            if ie.kind == "file":
-                zinfo = zipfile.ZipInfo(
-                            filename=filename,
-                            date_time=date_time)
-                zinfo.compress_type = compression
-                zinfo.external_attr = _FILE_ATTR
-                content = tree.get_file_text(tp, file_id)
-                zipf.writestr(zinfo, content)
-            elif ie.kind in ("directory", "tree-reference"):
-                # Directories must contain a trailing slash, to indicate
-                # to the zip routine that they are really directories and
-                # not just empty files.
-                zinfo = zipfile.ZipInfo(
-                            filename=filename + '/',
-                            date_time=date_time)
-                zinfo.compress_type = compression
-                zinfo.external_attr = _DIR_ATTR
-                zipf.writestr(zinfo, '')
-            elif ie.kind == "symlink":
-                zinfo = zipfile.ZipInfo(
-                            filename=(filename + '.lnk'),
-                            date_time=date_time)
-                zinfo.compress_type = compression
-                zinfo.external_attr = _FILE_ATTR
-                zipf.writestr(zinfo, tree.get_symlink_target(tp, file_id))
-            yield
-
-        zipf.close()
-
-    except UnicodeEncodeError:
-        zipf.close()
-        os.remove(dest)
-        from breezy.errors import BzrError
-        raise BzrError("Can't export non-ascii filenames to zip")
+    with tempfile.SpooledTemporaryFile() as buf:
+        with closing(zipfile.ZipFile(buf, "w", compression)) as zipf, \
+             tree.lock_read():
+            for dp, tp, ie in _export_iter_entries(tree, subdir):
+                file_id = ie.file_id
+                mutter("  export {%s} kind %s to %s", file_id, ie.kind, dest)
+
+                # zipfile.ZipFile switches all paths to forward
+                # slashes anyway, so just stick with that.
+                if force_mtime is not None:
+                    mtime = force_mtime
+                else:
+                    mtime = tree.get_file_mtime(tp, ie.file_id)
+                date_time = time.localtime(mtime)[:6]
+                filename = osutils.pathjoin(root, dp).encode('utf8')
+                if ie.kind == "file":
+                    zinfo = zipfile.ZipInfo(
+                                filename=filename,
+                                date_time=date_time)
+                    zinfo.compress_type = compression
+                    zinfo.external_attr = _FILE_ATTR
+                    content = tree.get_file_text(tp, file_id)
+                    zipf.writestr(zinfo, content)
+                elif ie.kind in ("directory", "tree-reference"):
+                    # Directories must contain a trailing slash, to indicate
+                    # to the zip routine that they are really directories and
+                    # not just empty files.
+                    zinfo = zipfile.ZipInfo(
+                                filename=filename + '/',
+                                date_time=date_time)
+                    zinfo.compress_type = compression
+                    zinfo.external_attr = _DIR_ATTR
+                    zipf.writestr(zinfo, '')
+                elif ie.kind == "symlink":
+                    zinfo = zipfile.ZipInfo(
+                                filename=(filename + '.lnk'),
+                                date_time=date_time)
+                    zinfo.compress_type = compression
+                    zinfo.external_attr = _FILE_ATTR
+                    zipf.writestr(zinfo, tree.get_symlink_target(tp, file_id))
+        # Urgh, headers are written last since they include e.g. file size.
+        # So we have to buffer it all :(
+        buf.seek(0)
+        for chunk in osutils.file_iterator(buf):
+            yield chunk

=== modified file 'breezy/builtins.py'
--- breezy/builtins.py	2018-05-20 16:34:17 +0000
+++ breezy/builtins.py	2018-05-20 16:34:17 +0000
@@ -3311,7 +3311,7 @@
     def run(self, dest, branch_or_subdir=None, revision=None, format=None,
         root=None, filters=False, per_file_timestamps=False, uncommitted=False,
         directory=u'.'):
-        from .export import export
+        from .export import export, guess_format, get_root_name
 
         if branch_or_subdir is None:
             branch_or_subdir = directory
@@ -3331,27 +3331,22 @@
                     'export', revision, branch=b,
                     tree=tree)
 
+        if format is None:
+            format = guess_format(dest)
+
+        if root is None:
+            root = get_root_name(dest)
+
+        if not per_file_timestamps:
+            force_mtime = time.time()
+        else:
+            force_mtime = None
+
         if filters:
             from breezy.filter_tree import ContentFilterTree
             export_tree = ContentFilterTree(
                     export_tree, export_tree._content_filter_stack)
 
-        # Try asking the tree first..
-        if not per_file_timestamps:
-            chunks = export_tree.archive(
-                dest, format, root=root, subdir=subdir)
-            try:
-                if dest == '-':
-                    self.outf.writelines(chunks)
-                else:
-                    import tempfile
-                    with tempfile.NamedTemporaryFile(delete=False) as temp:
-                        temp.writelines(chunks)
-                    os.rename(temp.name, dest)
-            except errors.NoSuchExportFormat:
-                pass
-            else:
-                return
         try:
             export(export_tree, dest, format, root, subdir,
                    per_file_timestamps=per_file_timestamps)

=== modified file 'breezy/bzr/remote.py'
--- breezy/bzr/remote.py	2018-05-19 17:31:48 +0000
+++ breezy/bzr/remote.py	2018-05-20 16:34:17 +0000
@@ -17,6 +17,7 @@
 from __future__ import absolute_import
 
 import bz2
+import os
 import zlib
 
 from .. import (
@@ -921,6 +922,20 @@
         return RemoteControlStore(self)
 
 
+class RemoteInventoryTree(InventoryRevisionTree):
+
+    def __init__(self, repository, inv, revision_id):
+        super(RemoteInventoryTree, self).__init__(repository, inv, revision_id)
+
+    def archive(self, name, format=None, root=None, subdir=None):
+        ret = self._repository._revision_archive(
+                self.get_revision_id(), name, format, root, subdir)
+        if ret is None:
+            return super(RemoteInventoryTree, self).archive(
+                name, format, root, subdir)
+        return ret
+
+
 class RemoteRepositoryFormat(vf_repository.VersionedFileRepositoryFormat):
     """Format for repositories accessed over a _SmartClient.
 
@@ -2505,7 +2520,7 @@
         with self.lock_read():
             inventories = self.iter_inventories(revision_ids)
             for inv in inventories:
-                yield InventoryRevisionTree(self, inv, inv.revision_id)
+                yield RemoteInventoryTree(self, inv, inv.revision_id)
 
     def get_revision_reconcile(self, revision_id):
         with self.lock_read():
@@ -2795,6 +2810,25 @@
         if response[0] != b'ok':
             raise errors.UnexpectedSmartServerResponse(response)
 
+    def _revision_archive(self, revision_id, name, format, root, subdir):
+        path = self.controldir._path_for_remote_call(self._client)
+        format = format or ''
+        root = root or ''
+        subdir = subdir or ''
+        try:
+            response, protocol = self._call_expecting_body(
+                b'Repository.revision_archive', path,
+                revision_id,
+                os.path.basename(name).encode('utf-8'),
+                format.encode('ascii'),
+                root.encode('utf-8'),
+                subdir.encode('utf-8'))
+        except errors.UnknownSmartMethod:
+            return None
+        if response[0] == b'ok':
+            return iter([protocol.read_body_bytes()])
+        raise errors.UnexpectedSmartServerResponse(response)
+
 
 class RemoteStreamSink(vf_repository.StreamSink):
 

=== modified file 'breezy/bzr/smart/repository.py'
--- breezy/bzr/smart/repository.py	2018-05-13 02:18:13 +0000
+++ breezy/bzr/smart/repository.py	2018-05-20 16:34:17 +0000
@@ -1292,3 +1292,27 @@
         self._ordering = ordering
         # Signal that we want a body
         return None
+
+
+class SmartServerRepositoryRevisionArchive(SmartServerRepositoryRequest):
+
+    def do_repository_request(self, repository, revision_id, name, format,
+                             root, subdir=None, force_mtime=None):
+        """Stream an archive file for a specific revision.
+
+        :param repository: The repository to stream from.
+        :param revision_id: Revision for which to export the tree
+        :param name: Target file name
+        :param format: Format (tar, tgz, tbz2, etc)
+        :param root: Name of root directory (or '')
+        :param subdir: Subdirectory to export, if not the root
+        """
+        tree = repository.revision_tree(revision_id)
+        return SuccessfulSmartServerResponse((b'ok',),
+            body_stream=self.body_stream(
+                tree, os.path.basename(name), format, root, subdir,
+                force_mtime))
+
+    def body_stream(self, tree, name, format, root, subdir=None, force_mtime=None):
+        with tree.lock_read():
+            return tree.archive(format, name, root, subdir, force_mtime)

=== modified file 'breezy/bzr/smart/request.py'
--- breezy/bzr/smart/request.py	2018-05-19 17:31:48 +0000
+++ breezy/bzr/smart/request.py	2018-05-20 16:34:17 +0000
@@ -778,6 +778,9 @@
     b'Repository.reconcile', 'breezy.bzr.smart.repository',
     'SmartServerRepositoryReconcile', info='idem')
 request_handlers.register_lazy(
+    b'Repository.revision_archive', 'breezy.bzr.smart.repository',
+    'SmartServerRepositoryRevisionArchive', info='read')
+request_handlers.register_lazy(
     b'Repository.tarball', 'breezy.bzr.smart.repository',
     'SmartServerRepositoryTarball', info='read')
 request_handlers.register_lazy(

=== removed directory 'breezy/export'
=== renamed file 'breezy/export/__init__.py' => 'breezy/export.py'
--- breezy/export/__init__.py	2018-05-20 16:34:17 +0000
+++ breezy/export.py	2018-05-20 16:34:17 +0000
@@ -19,197 +19,19 @@
 
 from __future__ import absolute_import
 
+import errno
 import os
+import sys
 import time
 import warnings
 
-from .. import (
+from . import (
+    archive,
     errors,
-    pyutils,
+    osutils,
     trace,
     )
 
-# Maps format name => export function
-_exporters = {}
-# Maps filename extensions => export format name
-_exporter_extensions = {}
-
-
-def register_exporter(format, extensions, func, override=False):
-    """Register an exporter.
-
-    :param format: This is the name of the format, such as 'tgz' or 'zip'
-    :param extensions: Extensions which should be used in the case that a
-                       format was not explicitly specified.
-    :type extensions: List
-    :param func: The function. It will be called with (tree, dest, root)
-    :param override: Whether to override an object which already exists.
-                     Frequently plugins will want to provide functionality
-                     until it shows up in mainline, so the default is False.
-    """
-    global _exporters, _exporter_extensions
-
-    if (format not in _exporters) or override:
-        _exporters[format] = func
-
-    for ext in extensions:
-        if (ext not in _exporter_extensions) or override:
-            _exporter_extensions[ext] = format
-
-
-def register_lazy_exporter(scheme, extensions, module, funcname):
-    """Register lazy-loaded exporter function.
-
-    When requesting a specific type of export, load the respective path.
-    """
-    def _loader(tree, dest, root, subdir, force_mtime, fileobj):
-        func = pyutils.get_named_object(module, funcname)
-        return func(tree, dest, root, subdir, force_mtime=force_mtime,
-            fileobj=fileobj)
-
-    register_exporter(scheme, extensions, _loader)
-
-
-def get_stream_export_generator(tree, name=None, format=None, root=None,
-        subdir=None, per_file_timestamps=False):
-    """Returns a generator that exports the given tree as a stream.
-
-    The generator is expected to yield None while exporting the tree while the
-    actual export is written to ``fileobj``.
-
-    :param tree: A Tree (such as RevisionTree) to export
-
-    :param dest: The destination where the files, etc should be put
-
-    :param format: The format (dir, zip, etc), if None, it will check the
-        extension on dest, looking for a match
-
-    :param root: The root location inside the format.  It is common practise to
-        have zipfiles and tarballs extract into a subdirectory, rather than
-        into the current working directory.  If root is None, the default root
-        will be selected as the destination without its extension.
-
-    :param subdir: A starting directory within the tree. None means to export
-        the entire tree, and anything else should specify the relative path to
-        a directory to start exporting from.
-
-    :param per_file_timestamps: Whether to use the timestamp stored in the tree
-        rather than now(). This will do a revision lookup for every file so
-        will be significantly slower.
-    """
-    global _exporters
-
-    if format is None and name is not None:
-        format = get_format_from_filename(name)
-
-    if format is None:
-        # Default to tar
-        format = 'dir'
-
-    if format in ('dir', 'tlzma', 'txz', 'tbz2'):
-        # formats that don't support streaming
-        raise errors.NoSuchExportFormat(format)
-
-    if format not in _exporters:
-        raise errors.NoSuchExportFormat(format)
-
-    # Most of the exporters will just have to call
-    # this function anyway, so why not do it for them
-    if root is None:
-        root = get_root_name(name)
-
-    if not per_file_timestamps:
-        force_mtime = time.time()
-    else:
-        force_mtime = None
-
-    oldpos = 0
-    import tempfile
-    with tempfile.NamedTemporaryFile() as temp:
-        with tree.lock_read():
-            for _ in _exporters[format](
-                tree, name, root, subdir,
-                force_mtime=force_mtime, fileobj=temp.file):
-                pos = temp.tell()
-                temp.seek(oldpos)
-                data = temp.read()
-                oldpos = pos
-                temp.seek(pos)
-                yield data
-            # FIXME(JRV): urgh, some exporters close the file for us so we need to reopen
-            # it here.
-            with open(temp.name, 'rb') as temp:
-                temp.seek(oldpos)
-                yield temp.read()
-
-
-def get_format_from_filename(name):
-    global _exporter_extensions
-
-    for ext in _exporter_extensions:
-        if name.endswith(ext):
-            return _exporter_extensions[ext]
-
-
-def get_export_generator(tree, dest=None, format=None, root=None, subdir=None,
-                         per_file_timestamps=False, fileobj=None):
-    """Returns a generator that exports the given tree.
-
-    The generator is expected to yield None while exporting the tree while the
-    actual export is written to ``fileobj``.
-
-    :param tree: A Tree (such as RevisionTree) to export
-
-    :param dest: The destination where the files, etc should be put
-
-    :param format: The format (dir, zip, etc), if None, it will check the
-        extension on dest, looking for a match
-
-    :param root: The root location inside the format.  It is common practise to
-        have zipfiles and tarballs extract into a subdirectory, rather than
-        into the current working directory.  If root is None, the default root
-        will be selected as the destination without its extension.
-
-    :param subdir: A starting directory within the tree. None means to export
-        the entire tree, and anything else should specify the relative path to
-        a directory to start exporting from.
-
-    :param per_file_timestamps: Whether to use the timestamp stored in the tree
-        rather than now(). This will do a revision lookup for every file so
-        will be significantly slower.
-
-    :param fileobj: Optional file object to use
-    """
-    global _exporters
-
-    if format is None and dest is not None:
-        format = get_format_from_filename(dest)
-
-    if format is None:
-        # Default to 'dir'
-        format = 'dir'
-
-    # Most of the exporters will just have to call
-    # this function anyway, so why not do it for them
-    if root is None:
-        root = get_root_name(dest)
-
-    if format not in _exporters:
-        raise errors.NoSuchExportFormat(format)
-
-    if not per_file_timestamps:
-        force_mtime = time.time()
-    else:
-        force_mtime = None
-
-    trace.mutter('export version %r', tree)
-
-    with tree.lock_read():
-        for _ in _exporters[format](
-            tree, dest, root, subdir,
-            force_mtime=force_mtime, fileobj=fileobj):
-            yield
-
 
 def export(tree, dest, format=None, root=None, subdir=None,
            per_file_timestamps=False, fileobj=None):
@@ -234,9 +56,56 @@
         for every file so will be significantly slower.
     :param fileobj: Optional file object to use
     """
-    for _ in get_export_generator(tree, dest, format, root, subdir,
-                                  per_file_timestamps, fileobj):
-        pass
+    if format is None and dest is not None:
+        format = guess_format(dest)
+
+    # Most of the exporters will just have to call
+    # this function anyway, so why not do it for them
+    if root is None:
+        root = get_root_name(dest)
+
+    if not per_file_timestamps:
+        force_mtime = time.time()
+    else:
+        force_mtime = None
+
+    trace.mutter('export version %r', tree)
+
+    if format == 'dir':
+        # TODO(jelmer): If the tree is remote (e.g. HPSS, Git Remote),
+        # then we should stream a tar file and unpack that on the fly.
+        with tree.lock_read():
+            for unused in dir_exporter_generator(tree, dest, root, subdir,
+                    force_mtime):
+                pass
+        return
+
+    with tree.lock_read():
+        chunks = archive.create_archive(format, tree, dest, root, subdir,
+                                        force_mtime)
+        if dest == '-':
+            for chunk in chunks:
+                 sys.stdout.write(chunk)
+        elif fileobj is not None:
+            for chunk in chunks:
+                fileobj.write(chunk)
+        else:
+            with open(dest, 'wb') as f:
+                for chunk in chunks:
+                    f.writelines(chunk)
+
+
+def guess_format(filename, default='dir'):
+    """Guess the export format based on a file name.
+
+    :param filename: Filename to guess from
+    :param default: Default format to fall back to
+    :return: format name
+    """
+    format = archive.format_registry.get_format_from_filename(filename)
+    if format is None:
+        format = default
+    return format
 
 
 def get_root_name(dest):
@@ -248,7 +117,7 @@
         # Exporting to -/foo doesn't make sense so use relative paths.
         return ''
     dest = os.path.basename(dest)
-    for ext in _exporter_extensions:
+    for ext in archive.format_registry.extensions:
         if dest.endswith(ext):
             return dest[:-len(ext)]
     return dest
@@ -293,18 +162,67 @@
         yield final_path, path, entry
 
 
-register_lazy_exporter('dir', [], 'breezy.export.dir_exporter',
-                       'dir_exporter_generator')
-register_lazy_exporter('tar', ['.tar'], 'breezy.export.tar_exporter',
-                       'plain_tar_exporter_generator')
-register_lazy_exporter('tgz', ['.tar.gz', '.tgz'],
-                       'breezy.export.tar_exporter',
-                       'tgz_exporter_generator')
-register_lazy_exporter('tbz2', ['.tar.bz2', '.tbz2'],
-                       'breezy.export.tar_exporter', 'tbz_exporter_generator')
-register_lazy_exporter('tlzma', ['.tar.lzma'], 'breezy.export.tar_exporter',
-                       'tar_lzma_exporter_generator')
-register_lazy_exporter('txz', ['.tar.xz'], 'breezy.export.tar_exporter',
-                       'tar_xz_exporter_generator')
-register_lazy_exporter('zip', ['.zip'], 'breezy.export.zip_exporter',
-                       'zip_exporter_generator')
+def dir_exporter_generator(tree, dest, root, subdir=None,
+                           force_mtime=None, fileobj=None):
+    """Return a generator that exports this tree to a new directory.
+
+    `dest` should either not exist or should be empty. If it does not exist it
+    will be created holding the contents of this tree.
+
+    :note: If the export fails, the destination directory will be
+           left in an incompletely exported state: export is not transactional.
+    """
+    try:
+        os.mkdir(dest)
+    except OSError as e:
+        if e.errno == errno.EEXIST:
+            # check if directory empty
+            if os.listdir(dest) != []:
+                raise errors.BzrError(
+                    "Can't export tree to non-empty directory.")
+        else:
+            raise
+    # Iterate everything, building up the files we will want to export, and
+    # creating the directories and symlinks that we need.
+    # This tracks (file_id, (destination_path, executable))
+    # This matches the api that tree.iter_files_bytes() wants
+    # Note in the case of revision trees, this does trigger a double inventory
+    # lookup, hopefully it isn't too expensive.
+    to_fetch = []
+    for dp, tp, ie in _export_iter_entries(tree, subdir):
+        fullpath = osutils.pathjoin(dest, dp)
+        if ie.kind == "file":
+            to_fetch.append((tp, (dp, tp, ie.file_id)))
+        elif ie.kind in ("directory", "tree-reference"):
+            os.mkdir(fullpath)
+        elif ie.kind == "symlink":
+            try:
+                symlink_target = tree.get_symlink_target(tp, ie.file_id)
+                os.symlink(symlink_target, fullpath)
+            except OSError as e:
+                raise errors.BzrError(
+                    "Failed to create symlink %r -> %r, error: %s"
+                    % (fullpath, symlink_target, e))
+        else:
+            raise errors.BzrError("don't know how to export {%s} of kind %r" %
+               (tp, ie.kind))
+
+        yield
+    # The data returned here can be in any order, but we've already created all
+    # the directories
+    flags = os.O_CREAT | os.O_TRUNC | os.O_WRONLY | getattr(os, 'O_BINARY', 0)
+    for (relpath, treepath, file_id), chunks in tree.iter_files_bytes(to_fetch):
+        fullpath = osutils.pathjoin(dest, relpath)
+        # We set the mode and let the umask sort out the file info
+        mode = 0o666
+        if tree.is_executable(treepath, file_id):
+            mode = 0o777
+        with os.fdopen(os.open(fullpath, flags, mode), 'wb') as out:
+            out.writelines(chunks)
+        if force_mtime is not None:
+            mtime = force_mtime
+        else:
+            mtime = tree.get_file_mtime(treepath, file_id)
+        os.utime(fullpath, (mtime, mtime))
+
+        yield

=== removed file 'breezy/export/dir_exporter.py'
--- breezy/export/dir_exporter.py	2018-05-14 20:49:13 +0000
+++ breezy/export/dir_exporter.py	1970-01-01 00:00:00 +0000
@@ -1,93 +0,0 @@
-# Copyright (C) 2005-2011 Canonical Ltd
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-"""Export a breezy.tree.Tree to a new or empty directory."""
-
-from __future__ import absolute_import
-
-import errno
-import os
-
-from .. import errors, osutils
-from ..export import _export_iter_entries
-
-
-def dir_exporter_generator(tree, dest, root, subdir=None,
-                           force_mtime=None, fileobj=None):
-    """Return a generator that exports this tree to a new directory.
-
-    `dest` should either not exist or should be empty. If it does not exist it
-    will be created holding the contents of this tree.
-
-    :param fileobj: Is not used in this exporter
-
-    :note: If the export fails, the destination directory will be
-           left in an incompletely exported state: export is not transactional.
-    """
-    try:
-        os.mkdir(dest)
-    except OSError as e:
-        if e.errno == errno.EEXIST:
-            # check if directory empty
-            if os.listdir(dest) != []:
-                raise errors.BzrError(
-                    "Can't export tree to non-empty directory.")
-        else:
-            raise
-    # Iterate everything, building up the files we will want to export, and
-    # creating the directories and symlinks that we need.
-    # This tracks (file_id, (destination_path, executable))
-    # This matches the api that tree.iter_files_bytes() wants
-    # Note in the case of revision trees, this does trigger a double inventory
-    # lookup, hopefully it isn't too expensive.
-    to_fetch = []
-    for dp, tp, ie in _export_iter_entries(tree, subdir):
-        fullpath = osutils.pathjoin(dest, dp)
-        if ie.kind == "file":
-            to_fetch.append((tp, (dp, tp, ie.file_id)))
-        elif ie.kind in ("directory", "tree-reference"):
-            os.mkdir(fullpath)
-        elif ie.kind == "symlink":
-            try:
-                symlink_target = tree.get_symlink_target(tp, ie.file_id)
-                os.symlink(symlink_target, fullpath)
-            except OSError as e:
-                raise errors.BzrError(
-                    "Failed to create symlink %r -> %r, error: %s"
-                    % (fullpath, symlink_target, e))
-        else:
-            raise errors.BzrError("don't know how to export {%s} of kind %r" %
-               (tp, ie.kind))
-
-        yield
-    # The data returned here can be in any order, but we've already created all
-    # the directories
-    flags = os.O_CREAT | os.O_TRUNC | os.O_WRONLY | getattr(os, 'O_BINARY', 0)
-    for (relpath, treepath, file_id), chunks in tree.iter_files_bytes(to_fetch):
-        fullpath = osutils.pathjoin(dest, relpath)
-        # We set the mode and let the umask sort out the file info
-        mode = 0o666
-        if tree.is_executable(treepath, file_id):
-            mode = 0o777
-        with os.fdopen(os.open(fullpath, flags, mode), 'wb') as out:
-            out.writelines(chunks)
-        if force_mtime is not None:
-            mtime = force_mtime
-        else:
-            mtime = tree.get_file_mtime(treepath, file_id)
-        os.utime(fullpath, (mtime, mtime))
-
-        yield

=== modified file 'breezy/tests/blackbox/test_export.py'
--- breezy/tests/blackbox/test_export.py	2018-02-18 15:21:06 +0000
+++ breezy/tests/blackbox/test_export.py	2018-05-20 16:34:17 +0000
@@ -25,6 +25,7 @@
 import zipfile
 
 
+from ...archive import zip
 from ... import (
     export,
     )
@@ -224,9 +225,9 @@
         # forward slashes
         self.assertEqual(['test/a', 'test/b/', 'test/b/c', 'test/d/'], names)
 
-        file_attr = stat.S_IFREG | export.zip_exporter.FILE_PERMISSIONS
-        dir_attr = (stat.S_IFDIR | export.zip_exporter.ZIP_DIRECTORY_BIT |
-                    export.zip_exporter.DIR_PERMISSIONS)
+        file_attr = stat.S_IFREG | zip.FILE_PERMISSIONS
+        dir_attr = (stat.S_IFDIR | zip.ZIP_DIRECTORY_BIT |
+                    zip.DIR_PERMISSIONS)
 
         a_info = zfile.getinfo(names[0])
         self.assertEqual(file_attr, a_info.external_attr)

=== modified file 'breezy/tests/per_tree/test_archive.py'
--- breezy/tests/per_tree/test_archive.py	2018-05-20 16:34:17 +0000
+++ breezy/tests/per_tree/test_archive.py	2018-05-20 16:34:17 +0000
@@ -41,10 +41,10 @@
         tree_a = self.workingtree_to_test_tree(work_a)
         output_path = 'output'
         with open(output_path, 'wb') as f:
-            f.writelines(tree_a.archive(output_path, format=self.format))
+            f.writelines(tree_a.archive(self.format, output_path))
         names = self.get_export_names(output_path)
-        self.assertIn('output/file', names)
-        self.assertIn('output/dir', names)
+        self.assertIn('file', names)
+        self.assertIn('dir', names)
 
     def test_export_symlink(self):
         self.requireFeature(features.SymlinkFeature)
@@ -55,9 +55,9 @@
         tree_a = self.workingtree_to_test_tree(work_a)
         output_path = 'output'
         with open(output_path, 'wb') as f:
-            f.writelines(tree_a.archive(output_path, format=self.format))
+            f.writelines(tree_a.archive(self.format, output_path))
         names = self.get_export_names(output_path)
-        self.assertIn('output/link', names)
+        self.assertIn('link', names)
 
     def get_output_names(self, path):
         raise NotImplementedError(self.get_output_names)
@@ -107,9 +107,9 @@
         tree_a = self.workingtree_to_test_tree(work_a)
         output_path = 'output'
         with open(output_path, 'wb') as f:
-            f.writelines(tree_a.archive(output_path, format=self.format))
+            f.writelines(tree_a.archive(self.format, output_path))
         names = self.get_export_names(output_path)
-        self.assertIn('output/link.lnk', names)
+        self.assertIn('link.lnk', names)
 
 
 class GenericArchiveTests(TestCaseWithTree):
@@ -125,5 +125,4 @@
 
         self.assertRaises(
                 errors.NoSuchExportFormat,
-                list,
-                tree_a.archive('foo', format='dir'))
+                tree_a.archive, 'dir', 'foo')

=== modified file 'breezy/tests/test_export.py'
--- breezy/tests/test_export.py	2018-05-14 20:49:13 +0000
+++ breezy/tests/test_export.py	2018-05-20 16:34:17 +0000
@@ -27,7 +27,7 @@
     tests,
     )
 from ..export import get_root_name
-from ..export.tar_exporter import export_tarball_generator
+from ..archive.tar import tarball_generator
 from ..sixish import (
     BytesIO,
     )
@@ -250,24 +250,14 @@
         tf = tarfile.open('target.tar.bz2')
         self.assertEqual(["target/a"], tf.getnames())
 
-    def test_xz_stdout(self):
-        wt = self.make_branch_and_tree('.')
-        self.assertRaises(errors.BzrError, export.export, wt, '-',
-            format="txz")
-
     def test_export_tarball_generator(self):
         wt = self.make_branch_and_tree('.')
         self.build_tree(['a'])
         wt.add(["a"])
         wt.commit("1", timestamp=42)
         target = BytesIO()
-        ball = tarfile.open(None, "w|", target)
-        wt.lock_read()
-        try:
-            for _ in export_tarball_generator(wt, ball, "bar"):
-                pass
-        finally:
-            wt.unlock()
+        with wt.lock_read():
+            target.writelines(tarball_generator(wt, "bar"))
         # Ball should now be closed.
         target.seek(0)
         ball2 = tarfile.open(None, "r", target)

=== modified file 'breezy/tests/test_remote.py'
--- breezy/tests/test_remote.py	2018-05-19 17:31:48 +0000
+++ breezy/tests/test_remote.py	2018-05-20 16:34:17 +0000
@@ -25,6 +25,7 @@
 
 import base64
 import bz2
+import tarfile
 import zlib
 
 from .. import (
@@ -4305,3 +4306,39 @@
             b'success', (b'ok', ), iter([]))
         self.assertRaises(errors.NoSuchRevision, list, repo.iter_inventories(
             [b"somerevid"]))
+
+
+class TestRepositoryRevisionTreeArchive(TestRemoteRepository):
+    """Test Repository.iter_inventories."""
+
+    def _serialize_inv_delta(self, old_name, new_name, delta):
+        serializer = inventory_delta.InventoryDeltaSerializer(True, False)
+        return b"".join(serializer.delta_to_lines(old_name, new_name, delta))
+
+    def test_simple(self):
+        transport_path = 'quack'
+        repo, client = self.setup_fake_client_and_repository(transport_path)
+        fmt = controldir.format_registry.get('2a')().repository_format
+        repo._format = fmt
+        stream = [('inventory-deltas', [
+            versionedfile.FulltextContentFactory(b'somerevid', None, None,
+                self._serialize_inv_delta(b'null:', b'somerevid', []))])]
+        client.add_expected_call(
+            b'VersionedFileRepository.get_inventories', (b'quack/', b'unordered'),
+            b'success', (b'ok', ),
+            _stream_to_byte_stream(stream, fmt))
+        f = BytesIO()
+        with tarfile.open(mode='w', fileobj=f) as tf:
+            info = tarfile.TarInfo('somefile')
+            info.mtime = 432432
+            contents = b'some data'
+            info.type = tarfile.REGTYPE
+            info.mode = 0o644
+            info.size = len(contents)
+            tf.addfile(info, BytesIO(contents))
+        client.add_expected_call(
+            b'Repository.revision_archive', (b'quack/', b'somerevid', b'tar', b'', b'', b''),
+            b'success', (b'ok', ),
+            f.getvalue())
+        tree = repo.revision_tree(b'somerevid')
+        self.assertEqual(f.getvalue(), b''.join(tree.archive('tar')))

=== modified file 'breezy/tests/test_smart.py'
--- breezy/tests/test_smart.py	2018-05-19 17:31:48 +0000
+++ breezy/tests/test_smart.py	2018-05-20 16:34:17 +0000
@@ -25,6 +25,8 @@
 """
 
 import bz2
+from io import BytesIO
+import tarfile
 import zlib
 
 from breezy import (
@@ -2713,3 +2715,21 @@
         self.assertEqual(response.args, (b"ok", ))
         self.assertEqual(b"".join(response.body_stream),
             b"Bazaar pack format 1 (introduced in 0.18)\nB54\n\nBazaar repository format 2a (needs bzr 1.16 or later)\nE")
+
+
+class TestSmartServerRepositoryRevisionArchive(tests.TestCaseWithTransport):
+
+    def test_get(self):
+        backing = self.get_transport()
+        request = smart_repo.SmartServerRepositoryRevisionArchive(backing)
+        t = self.make_branch_and_tree('.')
+        self.addCleanup(t.lock_write().unlock)
+        self.build_tree_contents([("file", b"somecontents")])
+        t.add(["file"], [b"thefileid"])
+        t.commit(rev_id=b'somerev', message="add file")
+        response = request.execute(b'', b"somerev", "foo.tar", "tar", "foo")
+        self.assertTrue(response.is_successful())
+        self.assertEqual(response.args, (b"ok", ))
+        b = BytesIO(b"".join(response.body_stream))
+        with tarfile.open(mode='r', fileobj=b) as tf:
+            self.assertEqual(['foo/file'], tf.getnames())

=== modified file 'breezy/tree.py'
--- breezy/tree.py	2018-05-20 16:34:17 +0000
+++ breezy/tree.py	2018-05-20 16:34:17 +0000
@@ -653,19 +653,22 @@
         searcher = default_searcher
         return searcher
 
-    def archive(self, name, format=None, root=None, subdir=None):
+    def archive(self, format, name, root='', subdir=None,
+                force_mtime=None):
         """Create an archive of this tree.
 
         :param name: target file name
         :param format: Format name (e.g. 'tar')
         :param root: Root directory name (or None)
         :param subdir: Subdirectory to export (or None)
+        :param per_file_timestamps: Whether to set the timestamp
+            for each file to the last changed time.
         :return: Iterator over archive chunks
         """
-        from .export import get_stream_export_generator
+        from .archive import create_archive
         with self.lock_read():
-            return get_stream_export_generator(self, name, format, root,
-                    subdir)
+            return create_archive(format, self, name, root,
+                    subdir, force_mtime=force_mtime)
 
     @classmethod
     def versionable_kind(cls, kind):