Merge lp:~jelmer/brz/git-archive into lp:brz

[Jelmer Vernooij]

Implement GitRevisionTree.archive for remote git trees.

Implement GitRevisionTree.archive for remote git trees.

[Martin Packman (gz)]

Looks good, see one inline comment.

review: Approve

=== added directory 'breezy/archive'
=== added file 'breezy/archive/__init__.py'
--- breezy/archive/__init__.py	1970-01-01 00:00:00 +0000
+++ breezy/archive/__init__.py	2018-05-20 16:08:56 +0000
@@ -0,0 +1,103 @@
+# Copyright (C) 2018 Breezy Developers
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+"""Export trees to tarballs, zipfiles, etc.
+"""
+
+from __future__ import absolute_import
+
+import os
+import time
+import warnings
+
+from .. import (
+    errors,
+    pyutils,
+    registry,
+    trace,
+    )
+
+
+class ArchiveFormatInfo(object):
+
+    def __init__(self, extensions):
+        self.extensions = extensions
+
+
+class ArchiveFormatRegistry(registry.Registry):
+    """Registry of archive formats."""
+
+    def __init__(self):
+        self._extension_map = {}
+        super(ArchiveFormatRegistry, self).__init__()
+
+    @property
+    def extensions(self):
+        return self._extension_map.keys()
+
+    def register(self, key, factory, extensions, help=None):
+        """Register an archive format.
+        """
+        registry.Registry.register(self, key, factory, help,
+            ArchiveFormatInfo(extensions))
+        self._register_extensions(key, extensions)
+
+    def register_lazy(self, key, module_name, member_name, extensions,
+                      help=None):
+        registry.Registry.register_lazy(self, key, module_name, member_name,
+            help, ArchiveFormatInfo(extensions))
+        self._register_extensions(key, extensions)
+
+    def _register_extensions(self, name, extensions):
+        for ext in extensions:
+            self._extension_map[ext] = name
+
+    def get_format_from_filename(self, filename):
+        """Determine the archive format from an extension.
+
+        :param filename: Filename to guess from
+        :return: A format name, or None
+        """
+        for ext, format in self._extension_map.items():
+            if filename.endswith(ext):
+                return format
+        else:
+            return None
+
+
+def create_archive(format, tree, name, root=None, subdir=None,
+                   force_mtime=None):
+    try:
+        archive_fn = format_registry.get(format)
+    except KeyError:
+        raise errors.NoSuchExportFormat(format)
+    return archive_fn(tree, name, root=root, subdir=subdir,
+                      force_mtime=force_mtime)
+
+
+format_registry = ArchiveFormatRegistry()
+format_registry.register_lazy('tar', 'breezy.archive.tar',
+                              'plain_tar_generator', ['.tar'], )
+format_registry.register_lazy('tgz', 'breezy.archive.tar',
+                              'tgz_generator', ['.tar.gz', '.tgz'])
+format_registry.register_lazy('tbz2', 'breezy.archive.tar',
+                              'tbz_generator',  ['.tar.bz2', '.tbz2'])
+format_registry.register_lazy('tlzma', 'breezy.archive.tar',
+                              'tar_lzma_generator', ['.tar.lzma'])
+format_registry.register_lazy('txz', 'breezy.archive.tar',
+                              'tar_xz_generator', ['.tar.xz'])
+format_registry.register_lazy('zip', 'breezy.archive.zip',
+                              'zip_archive_generator', ['.zip'])

=== renamed file 'breezy/export/tar_exporter.py' => 'breezy/archive/tar.py'
--- breezy/export/tar_exporter.py	2018-05-20 16:08:56 +0000
+++ breezy/archive/tar.py	2018-05-20 16:08:56 +0000
@@ -18,6 +18,7 @@
 
 from __future__ import absolute_import
 
+from contextlib import closing
 import os
 import sys
 import tarfile
@@ -81,141 +82,114 @@
     return (item, fileobj)
 
 
-def export_tarball_generator(tree, ball, root, subdir=None, force_mtime=None):
+def tarball_generator(tree, root, subdir=None, force_mtime=None, format=''):
     """Export tree contents to a tarball.
 
-    :returns: A generator that will repeatedly produce None as each file is
-        emitted.  The entire generator must be consumed to complete writing
-        the file.
+    :returns: A generator that will produce file content chunks.
 
     :param tree: Tree to export
 
-    :param ball: Tarball to export to; it will be closed when writing is
-        complete.
-
     :param subdir: Sub directory to export
 
     :param force_mtime: Option mtime to force, instead of using tree
         timestamps.
     """
-    try:
+    buf = BytesIO()
+    with closing(tarfile.open(None, "w:%s" % format, buf)) as ball, tree.lock_read():
         for final_path, tree_path, entry in _export_iter_entries(tree, subdir):
             (item, fileobj) = prepare_tarball_item(
                 tree, root, final_path, tree_path, entry, force_mtime)
             ball.addfile(item, fileobj)
-            yield
-    finally:
-        ball.close()
-
-
-def tgz_exporter_generator(tree, dest, root, subdir, force_mtime=None,
-    fileobj=None):
-    """Export this tree to a new tar file.
-
-    `dest` will be created holding the contents of this tree; if it
-    already exists, it will be clobbered, like with "tar -c".
-    """
-    import gzip
-    if force_mtime is not None:
-        root_mtime = force_mtime
-    elif (getattr(tree, "repository", None) and
-          getattr(tree, "get_revision_id", None)):
-        # If this is a revision tree, use the revisions' timestamp
-        rev = tree.repository.get_revision(tree.get_revision_id())
-        root_mtime = rev.timestamp
-    elif tree.get_root_id() is not None:
-        root_mtime = tree.get_file_mtime('', tree.get_root_id())
-    else:
-        root_mtime = None
-
-    is_stdout = False
-    basename = None
-    if fileobj is not None:
-        stream = fileobj
-    elif dest == '-':
-        stream = sys.stdout
-        is_stdout = True
-    else:
-        stream = open(dest, 'wb')
-    # gzip file is used with an explicit fileobj so that
-    # the basename can be stored in the gzip file rather than
-    # dest. (bug 102234)
-    basename = os.path.basename(dest)
-    zipstream = gzip.GzipFile(basename, 'w', fileobj=stream,
-                              mtime=root_mtime)
-    ball = tarfile.open(None, 'w|', fileobj=zipstream)
-    for _ in export_tarball_generator(
-        tree, ball, root, subdir, force_mtime):
-        yield
-    # Closing zipstream may trigger writes to stream
-    zipstream.close()
-    if not is_stdout:
-        # Now we can safely close the stream
-        stream.close()
-
-
-def tbz_exporter_generator(tree, dest, root, subdir,
-                           force_mtime=None, fileobj=None):
-    """Export this tree to a new tar file.
-
-    `dest` will be created holding the contents of this tree; if it
-    already exists, it will be clobbered, like with "tar -c".
-    """
-    if fileobj is not None:
-        ball = tarfile.open(None, 'w|bz2', fileobj)
-    elif dest == '-':
-        ball = tarfile.open(None, 'w|bz2', sys.stdout)
-    else:
-        ball = tarfile.open(dest, 'w:bz2')
-    return export_tarball_generator(
-        tree, ball, root, subdir, force_mtime)
-
-
-def plain_tar_exporter_generator(tree, dest, root, subdir, compression=None,
-    force_mtime=None, fileobj=None):
-    """Export this tree to a new tar file.
-
-    `dest` will be created holding the contents of this tree; if it
-    already exists, it will be clobbered, like with "tar -c".
-    """
-    if fileobj is not None:
-        stream = fileobj
-    elif dest == '-':
-        stream = sys.stdout
-    else:
-        stream = open(dest, 'wb')
-    ball = tarfile.open(None, 'w|', stream)
-    return export_tarball_generator(
-        tree, ball, root, subdir, force_mtime)
-
-
-def tar_xz_exporter_generator(tree, dest, root, subdir,
-                              force_mtime=None, fileobj=None):
-    return tar_lzma_exporter_generator(tree, dest, root, subdir,
-                                       force_mtime, fileobj, "xz")
-
-
-def tar_lzma_exporter_generator(tree, dest, root, subdir,
-                      force_mtime=None, fileobj=None,
-                                compression_format="alone"):
+            # Yield the data that was written so far, rinse, repeat.
+            yield buf.getvalue()
+            buf.truncate(0)
+            buf.seek(0)
+    yield buf.getvalue()
+
+
+def tgz_generator(tree, dest, root, subdir, force_mtime=None):
+    """Export this tree to a new tar file.
+
+    `dest` will be created holding the contents of this tree; if it
+    already exists, it will be clobbered, like with "tar -c".
+    """
+    with tree.lock_read():
+        import gzip
+        if force_mtime is not None:
+            root_mtime = force_mtime
+        elif (getattr(tree, "repository", None) and
+              getattr(tree, "get_revision_id", None)):
+            # If this is a revision tree, use the revisions' timestamp
+            rev = tree.repository.get_revision(tree.get_revision_id())
+            root_mtime = rev.timestamp
+        elif tree.get_root_id() is not None:
+            root_mtime = tree.get_file_mtime('', tree.get_root_id())
+        else:
+            root_mtime = None
+
+        is_stdout = False
+        basename = None
+        # gzip file is used with an explicit fileobj so that
+        # the basename can be stored in the gzip file rather than
+        # dest. (bug 102234)
+        basename = os.path.basename(dest)
+        buf = BytesIO()
+        zipstream = gzip.GzipFile(basename, 'w', fileobj=buf,
+                                  mtime=root_mtime)
+        for chunk in tarball_generator(
+            tree, root, subdir, force_mtime):
+            zipstream.write(chunk)
+            # Yield the data that was written so far, rinse, repeat.
+            yield buf.getvalue()
+            buf.truncate(0)
+            buf.seek(0)
+        # Closing zipstream may trigger writes to stream
+        zipstream.close()
+        yield buf.getvalue()
+
+
+def tbz_generator(tree, dest, root, subdir, force_mtime=None):
+    """Export this tree to a new tar file.
+
+    `dest` will be created holding the contents of this tree; if it
+    already exists, it will be clobbered, like with "tar -c".
+    """
+    return tarball_generator(
+        tree, root, subdir, force_mtime, format='bz2')
+
+
+def plain_tar_generator(tree, dest, root, subdir,
+    force_mtime=None):
+    """Export this tree to a new tar file.
+
+    `dest` will be created holding the contents of this tree; if it
+    already exists, it will be clobbered, like with "tar -c".
+    """
+    return tarball_generator(
+        tree, root, subdir, force_mtime, format='')
+
+
+def tar_xz_generator(tree, dest, root, subdir, force_mtime=None):
+    return tar_lzma_generator(tree, dest, root, subdir, force_mtime, "xz")
+
+
+def tar_lzma_generator(tree, dest, root, subdir, force_mtime=None,
+                       compression_format="alone"):
     """Export this tree to a new .tar.lzma file.
 
     `dest` will be created holding the contents of this tree; if it
     already exists, it will be clobbered, like with "tar -c".
     """
-    if dest == '-':
-        raise errors.BzrError("Writing to stdout not supported for .tar.lzma")
-
-    if fileobj is not None:
-        raise errors.BzrError(
-            "Writing to fileobject not supported for .tar.lzma")
     try:
         import lzma
     except ImportError as e:
         raise errors.DependencyNotPresent('lzma', e)
 
-    stream = lzma.LZMAFile(dest.encode(osutils._fs_enc), 'w',
-        options={"format": compression_format})
-    ball = tarfile.open(None, 'w:', fileobj=stream)
-    return export_tarball_generator(
-        tree, ball, root, subdir, force_mtime=force_mtime)
+    compressor = lzma.LZMACompressor(
+            options={"format": compression_format})
+
+    for chunk in tarball_generator(
+            tree, root, subdir, force_mtime=force_mtime):
+        yield compressor.compress(chunk)
+
+    yield compressor.flush()

=== renamed file 'breezy/export/zip_exporter.py' => 'breezy/archive/zip.py'
--- breezy/export/zip_exporter.py	2018-05-14 20:49:13 +0000
+++ breezy/archive/zip.py	2018-05-20 16:08:56 +0000
@@ -19,9 +19,11 @@
 
 from __future__ import absolute_import
 
+from contextlib import closing
 import os
 import stat
 import sys
+import tempfile
 import time
 import zipfile
 
@@ -42,64 +44,56 @@
 _DIR_ATTR = stat.S_IFDIR | ZIP_DIRECTORY_BIT | DIR_PERMISSIONS
 
 
-def zip_exporter_generator(tree, dest, root, subdir=None,
-    force_mtime=None, fileobj=None):
+def zip_archive_generator(tree, dest, root, subdir=None,
+    force_mtime=None):
     """ Export this tree to a new zip file.
 
     `dest` will be created holding the contents of this tree; if it
     already exists, it will be overwritten".
     """
-
     compression = zipfile.ZIP_DEFLATED
-    if fileobj is not None:
-        dest = fileobj
-    elif dest == "-":
-        dest = sys.stdout
-    zipf = zipfile.ZipFile(dest, "w", compression)
-    try:
-        for dp, tp, ie in _export_iter_entries(tree, subdir):
-            file_id = ie.file_id
-            mutter("  export {%s} kind %s to %s", file_id, ie.kind, dest)
-
-            # zipfile.ZipFile switches all paths to forward
-            # slashes anyway, so just stick with that.
-            if force_mtime is not None:
-                mtime = force_mtime
-            else:
-                mtime = tree.get_file_mtime(tp, ie.file_id)
-            date_time = time.localtime(mtime)[:6]
-            filename = osutils.pathjoin(root, dp).encode('utf8')
-            if ie.kind == "file":
-                zinfo = zipfile.ZipInfo(
-                            filename=filename,
-                            date_time=date_time)
-                zinfo.compress_type = compression
-                zinfo.external_attr = _FILE_ATTR
-                content = tree.get_file_text(tp, file_id)
-                zipf.writestr(zinfo, content)
-            elif ie.kind in ("directory", "tree-reference"):
-                # Directories must contain a trailing slash, to indicate
-                # to the zip routine that they are really directories and
-                # not just empty files.
-                zinfo = zipfile.ZipInfo(
-                            filename=filename + '/',
-                            date_time=date_time)
-                zinfo.compress_type = compression
-                zinfo.external_attr = _DIR_ATTR
-                zipf.writestr(zinfo, '')
-            elif ie.kind == "symlink":
-                zinfo = zipfile.ZipInfo(
-                            filename=(filename + '.lnk'),
-                            date_time=date_time)
-                zinfo.compress_type = compression
-                zinfo.external_attr = _FILE_ATTR
-                zipf.writestr(zinfo, tree.get_symlink_target(tp, file_id))
-            yield
-
-        zipf.close()
-
-    except UnicodeEncodeError:
-        zipf.close()
-        os.remove(dest)
-        from breezy.errors import BzrError
-        raise BzrError("Can't export non-ascii filenames to zip")
+    with tempfile.SpooledTemporaryFile() as buf:
+        with closing(zipfile.ZipFile(buf, "w", compression)) as zipf, \
+             tree.lock_read():
+            for dp, tp, ie in _export_iter_entries(tree, subdir):
+                file_id = ie.file_id
+                mutter("  export {%s} kind %s to %s", file_id, ie.kind, dest)
+
+                # zipfile.ZipFile switches all paths to forward
+                # slashes anyway, so just stick with that.
+                if force_mtime is not None:
+                    mtime = force_mtime
+                else:
+                    mtime = tree.get_file_mtime(tp, ie.file_id)
+                date_time = time.localtime(mtime)[:6]
+                filename = osutils.pathjoin(root, dp).encode('utf8')
+                if ie.kind == "file":
+                    zinfo = zipfile.ZipInfo(
+                                filename=filename,
+                                date_time=date_time)
+                    zinfo.compress_type = compression
+                    zinfo.external_attr = _FILE_ATTR
+                    content = tree.get_file_text(tp, file_id)
+                    zipf.writestr(zinfo, content)
+                elif ie.kind in ("directory", "tree-reference"):
+                    # Directories must contain a trailing slash, to indicate
+                    # to the zip routine that they are really directories and
+                    # not just empty files.
+                    zinfo = zipfile.ZipInfo(
+                                filename=filename + '/',
+                                date_time=date_time)
+                    zinfo.compress_type = compression
+                    zinfo.external_attr = _DIR_ATTR
+                    zipf.writestr(zinfo, '')
+                elif ie.kind == "symlink":
+                    zinfo = zipfile.ZipInfo(
+                                filename=(filename + '.lnk'),
+                                date_time=date_time)
+                    zinfo.compress_type = compression
+                    zinfo.external_attr = _FILE_ATTR
+                    zipf.writestr(zinfo, tree.get_symlink_target(tp, file_id))
+        # Urgh, headers are written last since they include e.g. file size.
+        # So we have to buffer it all :(
+        buf.seek(0)
+        for chunk in osutils.file_iterator(buf):
+            yield chunk

=== modified file 'breezy/builtins.py'
--- breezy/builtins.py	2018-05-20 16:08:56 +0000
+++ breezy/builtins.py	2018-05-20 16:08:56 +0000
@@ -3311,7 +3311,7 @@
     def run(self, dest, branch_or_subdir=None, revision=None, format=None,
         root=None, filters=False, per_file_timestamps=False, uncommitted=False,
         directory=u'.'):
-        from .export import export
+        from .export import export, guess_format, get_root_name
 
         if branch_or_subdir is None:
             branch_or_subdir = directory
@@ -3331,27 +3331,22 @@
                     'export', revision, branch=b,
                     tree=tree)
 
+        if format is None:
+            format = guess_format(dest)
+
+        if root is None:
+            root = get_root_name(dest)
+
+        if not per_file_timestamps:
+            force_mtime = time.time()
+        else:
+            force_mtime = None
+
         if filters:
             from breezy.filter_tree import ContentFilterTree
             export_tree = ContentFilterTree(
                     export_tree, export_tree._content_filter_stack)
 
-        # Try asking the tree first..
-        if not per_file_timestamps:
-            chunks = export_tree.archive(
-                dest, format, root=root, subdir=subdir)
-            try:
-                if dest == '-':
-                    self.outf.writelines(chunks)
-                else:
-                    import tempfile
-                    with tempfile.NamedTemporaryFile(delete=False) as temp:
-                        temp.writelines(chunks)
-                    os.rename(temp.name, dest)
-            except errors.NoSuchExportFormat:
-                pass
-            else:
-                return
         try:
             export(export_tree, dest, format, root, subdir,
                    per_file_timestamps=per_file_timestamps)

=== removed directory 'breezy/export'
=== renamed file 'breezy/export/__init__.py' => 'breezy/export.py'
--- breezy/export/__init__.py	2018-05-20 16:08:56 +0000
+++ breezy/export.py	2018-05-20 16:08:56 +0000
@@ -19,197 +19,19 @@
 
 from __future__ import absolute_import
 
+import errno
 import os
+import sys
 import time
 import warnings
 
-from .. import (
+from . import (
+    archive,
     errors,
-    pyutils,
+    osutils,
     trace,
     )
 
-# Maps format name => export function
-_exporters = {}
-# Maps filename extensions => export format name
-_exporter_extensions = {}
-
-
-def register_exporter(format, extensions, func, override=False):
-    """Register an exporter.
-
-    :param format: This is the name of the format, such as 'tgz' or 'zip'
-    :param extensions: Extensions which should be used in the case that a
-                       format was not explicitly specified.
-    :type extensions: List
-    :param func: The function. It will be called with (tree, dest, root)
-    :param override: Whether to override an object which already exists.
-                     Frequently plugins will want to provide functionality
-                     until it shows up in mainline, so the default is False.
-    """
-    global _exporters, _exporter_extensions
-
-    if (format not in _exporters) or override:
-        _exporters[format] = func
-
-    for ext in extensions:
-        if (ext not in _exporter_extensions) or override:
-            _exporter_extensions[ext] = format
-
-
-def register_lazy_exporter(scheme, extensions, module, funcname):
-    """Register lazy-loaded exporter function.
-
-    When requesting a specific type of export, load the respective path.
-    """
-    def _loader(tree, dest, root, subdir, force_mtime, fileobj):
-        func = pyutils.get_named_object(module, funcname)
-        return func(tree, dest, root, subdir, force_mtime=force_mtime,
-            fileobj=fileobj)
-
-    register_exporter(scheme, extensions, _loader)
-
-
-def get_stream_export_generator(tree, name=None, format=None, root=None,
-        subdir=None, per_file_timestamps=False):
-    """Returns a generator that exports the given tree as a stream.
-
-    The generator is expected to yield None while exporting the tree while the
-    actual export is written to ``fileobj``.
-
-    :param tree: A Tree (such as RevisionTree) to export
-
-    :param dest: The destination where the files, etc should be put
-
-    :param format: The format (dir, zip, etc), if None, it will check the
-        extension on dest, looking for a match
-
-    :param root: The root location inside the format.  It is common practise to
-        have zipfiles and tarballs extract into a subdirectory, rather than
-        into the current working directory.  If root is None, the default root
-        will be selected as the destination without its extension.
-
-    :param subdir: A starting directory within the tree. None means to export
-        the entire tree, and anything else should specify the relative path to
-        a directory to start exporting from.
-
-    :param per_file_timestamps: Whether to use the timestamp stored in the tree
-        rather than now(). This will do a revision lookup for every file so
-        will be significantly slower.
-    """
-    global _exporters
-
-    if format is None and name is not None:
-        format = get_format_from_filename(name)
-
-    if format is None:
-        # Default to tar
-        format = 'dir'
-
-    if format in ('dir', 'tlzma', 'txz', 'tbz2'):
-        # formats that don't support streaming
-        raise errors.NoSuchExportFormat(format)
-
-    if format not in _exporters:
-        raise errors.NoSuchExportFormat(format)
-
-    # Most of the exporters will just have to call
-    # this function anyway, so why not do it for them
-    if root is None:
-        root = get_root_name(name)
-
-    if not per_file_timestamps:
-        force_mtime = time.time()
-    else:
-        force_mtime = None
-
-    oldpos = 0
-    import tempfile
-    with tempfile.NamedTemporaryFile() as temp:
-        with tree.lock_read():
-            for _ in _exporters[format](
-                tree, name, root, subdir,
-                force_mtime=force_mtime, fileobj=temp.file):
-                pos = temp.tell()
-                temp.seek(oldpos)
-                data = temp.read()
-                oldpos = pos
-                temp.seek(pos)
-                yield data
-            # FIXME(JRV): urgh, some exporters close the file for us so we need to reopen
-            # it here.
-            with open(temp.name, 'rb') as temp:
-                temp.seek(oldpos)
-                yield temp.read()
-
-
-def get_format_from_filename(name):
-    global _exporter_extensions
-
-    for ext in _exporter_extensions:
-        if name.endswith(ext):
-            return _exporter_extensions[ext]
-
-
-def get_export_generator(tree, dest=None, format=None, root=None, subdir=None,
-                         per_file_timestamps=False, fileobj=None):
-    """Returns a generator that exports the given tree.
-
-    The generator is expected to yield None while exporting the tree while the
-    actual export is written to ``fileobj``.
-
-    :param tree: A Tree (such as RevisionTree) to export
-
-    :param dest: The destination where the files, etc should be put
-
-    :param format: The format (dir, zip, etc), if None, it will check the
-        extension on dest, looking for a match
-
-    :param root: The root location inside the format.  It is common practise to
-        have zipfiles and tarballs extract into a subdirectory, rather than
-        into the current working directory.  If root is None, the default root
-        will be selected as the destination without its extension.
-
-    :param subdir: A starting directory within the tree. None means to export
-        the entire tree, and anything else should specify the relative path to
-        a directory to start exporting from.
-
-    :param per_file_timestamps: Whether to use the timestamp stored in the tree
-        rather than now(). This will do a revision lookup for every file so
-        will be significantly slower.
-
-    :param fileobj: Optional file object to use
-    """
-    global _exporters
-
-    if format is None and dest is not None:
-        format = get_format_from_filename(dest)
-
-    if format is None:
-        # Default to 'dir'
-        format = 'dir'
-
-    # Most of the exporters will just have to call
-    # this function anyway, so why not do it for them
-    if root is None:
-        root = get_root_name(dest)
-
-    if format not in _exporters:
-        raise errors.NoSuchExportFormat(format)
-
-    if not per_file_timestamps:
-        force_mtime = time.time()
-    else:
-        force_mtime = None
-
-    trace.mutter('export version %r', tree)
-
-    with tree.lock_read():
-        for _ in _exporters[format](
-            tree, dest, root, subdir,
-            force_mtime=force_mtime, fileobj=fileobj):
-            yield
-
 
 def export(tree, dest, format=None, root=None, subdir=None,
            per_file_timestamps=False, fileobj=None):
@@ -234,9 +56,55 @@
         for every file so will be significantly slower.
     :param fileobj: Optional file object to use
     """
-    for _ in get_export_generator(tree, dest, format, root, subdir,
-                                  per_file_timestamps, fileobj):
-        pass
+    if format is None and dest is not None:
+        format = guess_format(dest)
+
+    # Most of the exporters will just have to call
+    # this function anyway, so why not do it for them
+    if root is None:
+        root = get_root_name(dest)
+
+    if not per_file_timestamps:
+        force_mtime = time.time()
+    else:
+        force_mtime = None
+
+    trace.mutter('export version %r', tree)
+
+    if format == 'dir':
+        # TODO(jelmer): If the tree is remote (e.g. HPSS, Git Remote),
+        # then we should stream a tar file and unpack that on the fly.
+        with tree.lock_read():
+            for unused in dir_exporter_generator(tree, dest, root, subdir,
+                    force_mtime):
+                pass
+        return
+
+    with tree.lock_read():
+        chunks = tree.archive(format, dest, root=root, subdir=subdir, force_mtime=force_mtime)
+        if dest == '-':
+            for chunk in chunks:
+                 sys.stdout.write(chunk)
+        elif fileobj is not None:
+            for chunk in chunks:
+                fileobj.write(chunk)
+        else:
+            with open(dest, 'wb') as f:
+                for chunk in chunks:
+                    f.writelines(chunk)
+
+
+def guess_format(filename, default='dir'):
+    """Guess the export format based on a file name.
+
+    :param filename: Filename to guess from
+    :param default: Default format to fall back to
+    :return: format name
+    """
+    format = archive.format_registry.get_format_from_filename(filename)
+    if format is None:
+        format = default
+    return format
 
 
 def get_root_name(dest):
@@ -248,7 +116,7 @@
         # Exporting to -/foo doesn't make sense so use relative paths.
         return ''
     dest = os.path.basename(dest)
-    for ext in _exporter_extensions:
+    for ext in archive.format_registry.extensions:
         if dest.endswith(ext):
             return dest[:-len(ext)]
     return dest
@@ -293,18 +161,67 @@
         yield final_path, path, entry
 
 
-register_lazy_exporter('dir', [], 'breezy.export.dir_exporter',
-                       'dir_exporter_generator')
-register_lazy_exporter('tar', ['.tar'], 'breezy.export.tar_exporter',
-                       'plain_tar_exporter_generator')
-register_lazy_exporter('tgz', ['.tar.gz', '.tgz'],
-                       'breezy.export.tar_exporter',
-                       'tgz_exporter_generator')
-register_lazy_exporter('tbz2', ['.tar.bz2', '.tbz2'],
-                       'breezy.export.tar_exporter', 'tbz_exporter_generator')
-register_lazy_exporter('tlzma', ['.tar.lzma'], 'breezy.export.tar_exporter',
-                       'tar_lzma_exporter_generator')
-register_lazy_exporter('txz', ['.tar.xz'], 'breezy.export.tar_exporter',
-                       'tar_xz_exporter_generator')
-register_lazy_exporter('zip', ['.zip'], 'breezy.export.zip_exporter',
-                       'zip_exporter_generator')
+def dir_exporter_generator(tree, dest, root, subdir=None,
+                           force_mtime=None, fileobj=None):
+    """Return a generator that exports this tree to a new directory.
+
+    `dest` should either not exist or should be empty. If it does not exist it
+    will be created holding the contents of this tree.
+
+    :note: If the export fails, the destination directory will be
+           left in an incompletely exported state: export is not transactional.
+    """
+    try:
+        os.mkdir(dest)
+    except OSError as e:
+        if e.errno == errno.EEXIST:
+            # check if directory empty
+            if os.listdir(dest) != []:
+                raise errors.BzrError(
+                    "Can't export tree to non-empty directory.")
+        else:
+            raise
+    # Iterate everything, building up the files we will want to export, and
+    # creating the directories and symlinks that we need.
+    # This tracks (file_id, (destination_path, executable))
+    # This matches the api that tree.iter_files_bytes() wants
+    # Note in the case of revision trees, this does trigger a double inventory
+    # lookup, hopefully it isn't too expensive.
+    to_fetch = []
+    for dp, tp, ie in _export_iter_entries(tree, subdir):
+        fullpath = osutils.pathjoin(dest, dp)
+        if ie.kind == "file":
+            to_fetch.append((tp, (dp, tp, ie.file_id)))
+        elif ie.kind in ("directory", "tree-reference"):
+            os.mkdir(fullpath)
+        elif ie.kind == "symlink":
+            try:
+                symlink_target = tree.get_symlink_target(tp, ie.file_id)
+                os.symlink(symlink_target, fullpath)
+            except OSError as e:
+                raise errors.BzrError(
+                    "Failed to create symlink %r -> %r, error: %s"
+                    % (fullpath, symlink_target, e))
+        else:
+            raise errors.BzrError("don't know how to export {%s} of kind %r" %
+               (tp, ie.kind))
+
+        yield
+    # The data returned here can be in any order, but we've already created all
+    # the directories
+    flags = os.O_CREAT | os.O_TRUNC | os.O_WRONLY | getattr(os, 'O_BINARY', 0)
+    for (relpath, treepath, file_id), chunks in tree.iter_files_bytes(to_fetch):
+        fullpath = osutils.pathjoin(dest, relpath)
+        # We set the mode and let the umask sort out the file info
+        mode = 0o666
+        if tree.is_executable(treepath, file_id):
+            mode = 0o777
+        with os.fdopen(os.open(fullpath, flags, mode), 'wb') as out:
+            out.writelines(chunks)
+        if force_mtime is not None:
+            mtime = force_mtime
+        else:
+            mtime = tree.get_file_mtime(treepath, file_id)
+        os.utime(fullpath, (mtime, mtime))
+
+        yield

=== removed file 'breezy/export/dir_exporter.py'
--- breezy/export/dir_exporter.py	2018-05-14 20:49:13 +0000
+++ breezy/export/dir_exporter.py	1970-01-01 00:00:00 +0000
@@ -1,93 +0,0 @@
-# Copyright (C) 2005-2011 Canonical Ltd
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
-
-"""Export a breezy.tree.Tree to a new or empty directory."""
-
-from __future__ import absolute_import
-
-import errno
-import os
-
-from .. import errors, osutils
-from ..export import _export_iter_entries
-
-
-def dir_exporter_generator(tree, dest, root, subdir=None,
-                           force_mtime=None, fileobj=None):
-    """Return a generator that exports this tree to a new directory.
-
-    `dest` should either not exist or should be empty. If it does not exist it
-    will be created holding the contents of this tree.
-
-    :param fileobj: Is not used in this exporter
-
-    :note: If the export fails, the destination directory will be
-           left in an incompletely exported state: export is not transactional.
-    """
-    try:
-        os.mkdir(dest)
-    except OSError as e:
-        if e.errno == errno.EEXIST:
-            # check if directory empty
-            if os.listdir(dest) != []:
-                raise errors.BzrError(
-                    "Can't export tree to non-empty directory.")
-        else:
-            raise
-    # Iterate everything, building up the files we will want to export, and
-    # creating the directories and symlinks that we need.
-    # This tracks (file_id, (destination_path, executable))
-    # This matches the api that tree.iter_files_bytes() wants
-    # Note in the case of revision trees, this does trigger a double inventory
-    # lookup, hopefully it isn't too expensive.
-    to_fetch = []
-    for dp, tp, ie in _export_iter_entries(tree, subdir):
-        fullpath = osutils.pathjoin(dest, dp)
-        if ie.kind == "file":
-            to_fetch.append((tp, (dp, tp, ie.file_id)))
-        elif ie.kind in ("directory", "tree-reference"):
-            os.mkdir(fullpath)
-        elif ie.kind == "symlink":
-            try:
-                symlink_target = tree.get_symlink_target(tp, ie.file_id)
-                os.symlink(symlink_target, fullpath)
-            except OSError as e:
-                raise errors.BzrError(
-                    "Failed to create symlink %r -> %r, error: %s"
-                    % (fullpath, symlink_target, e))
-        else:
-            raise errors.BzrError("don't know how to export {%s} of kind %r" %
-               (tp, ie.kind))
-
-        yield
-    # The data returned here can be in any order, but we've already created all
-    # the directories
-    flags = os.O_CREAT | os.O_TRUNC | os.O_WRONLY | getattr(os, 'O_BINARY', 0)
-    for (relpath, treepath, file_id), chunks in tree.iter_files_bytes(to_fetch):
-        fullpath = osutils.pathjoin(dest, relpath)
-        # We set the mode and let the umask sort out the file info
-        mode = 0o666
-        if tree.is_executable(treepath, file_id):
-            mode = 0o777
-        with os.fdopen(os.open(fullpath, flags, mode), 'wb') as out:
-            out.writelines(chunks)
-        if force_mtime is not None:
-            mtime = force_mtime
-        else:
-            mtime = tree.get_file_mtime(treepath, file_id)
-        os.utime(fullpath, (mtime, mtime))
-
-        yield

=== modified file 'breezy/plugins/git/remote.py'
--- breezy/plugins/git/remote.py	2018-05-13 22:54:28 +0000
+++ breezy/plugins/git/remote.py	2018-05-20 16:08:56 +0000
@@ -18,12 +18,14 @@
 
 from __future__ import absolute_import
 
+from io import BytesIO
 import re
 
 from ... import (
     config,
     debug,
     errors,
+    osutils,
     trace,
     ui,
     urlutils,
@@ -45,6 +47,7 @@
     NoWorkingTree,
     UninitializableFormat,
     )
+from ...revisiontree import RevisionTree
 from ...transport import (
     Transport,
     register_urlparse_netloc_protocol,
@@ -364,6 +367,25 @@
     def _gitrepository_class(self):
         return RemoteGitRepository
 
+    def archive(self, format, committish, write_data, progress=None, write_error=None,
+                subdirs=None, prefix=None):
+        if format not in ('tar', 'zip'):
+            raise errors.NoSuchExportFormat(format)
+        if progress is None:
+            pb = ui.ui_factory.nested_progress_bar()
+            progress = DefaultProgressReporter(pb).progress
+        else:
+            pb = None
+        try:
+            self._client.archive(self._client_path, committish,
+                write_data, progress, write_error, format=format,
+                subdirs=subdirs, prefix=prefix)
+        except GitProtocolError as e:
+            raise parse_git_error(self.transport.external_url(), e)
+        finally:
+            if pb is not None:
+                pb.finished()
+
     def fetch_pack(self, determine_wants, graph_walker, pack_data, progress=None):
         if progress is None:
             pb = ui.ui_factory.nested_progress_bar()
@@ -699,6 +721,39 @@
                 external_url.startswith("git:"))
 
 
+class GitRemoteRevisionTree(RevisionTree):
+
+    def archive(self, format, name, root=None, subdir=None, force_mtime=None):
+        """Create an archive of this tree.
+
+        :param format: Format name (e.g. 'tar')
+        :param name: target file name
+        :param root: Root directory name (or None)
+        :param subdir: Subdirectory to export (or None)
+        :return: Iterator over archive chunks
+        """
+        commit = self._repository.lookup_bzr_revision_id(
+            self.get_revision_id())[0]
+        f = tempfile.SpooledTemporaryFile()
+        # git-upload-archive(1) generaly only supports refs. So let's see if we
+        # can find one.
+        reverse_refs = {
+                v: k for (k, v) in
+                self._repository.controldir.get_refs_container().as_dict().items()}
+        try:
+            committish = reverse_refs[commit]
+        except KeyError:
+            # No? Maybe the user has uploadArchive.allowUnreachable enabled.
+            # Let's hope for the best.
+            committish = commit
+        self._repository.archive(
+                format, committish, f.write,
+                subdirs=([subdir] if subdir else None),
+                prefix=(root+'/') if root else '')
+        f.seek(0)
+        return osutils.file_iterator(f)
+
+
 class RemoteGitRepository(GitRepository):
 
     @property
@@ -708,6 +763,9 @@
     def get_parent_map(self, revids):
         raise GitSmartRemoteNotSupported(self.get_parent_map, self)
 
+    def archive(self, *args, **kwargs):
+        return self.controldir.archive(*args, **kwargs)
+
     def fetch_pack(self, determine_wants, graph_walker, pack_data,
                    progress=None):
         return self.controldir.fetch_pack(determine_wants, graph_walker,
@@ -745,7 +803,7 @@
         return mapping.revision_id_foreign_to_bzr(foreign_revid)
 
     def revision_tree(self, revid):
-        raise GitSmartRemoteNotSupported(self.revision_tree, self)
+        return GitRemoteRevisionTree(self, revid)
 
     def get_revisions(self, revids):
         raise GitSmartRemoteNotSupported(self.get_revisions, self)

=== modified file 'breezy/tests/blackbox/test_export.py'
--- breezy/tests/blackbox/test_export.py	2018-02-18 15:21:06 +0000
+++ breezy/tests/blackbox/test_export.py	2018-05-20 16:08:56 +0000
@@ -25,6 +25,7 @@
 import zipfile
 
 
+from ...archive import zip
 from ... import (
     export,
     )
@@ -224,9 +225,9 @@
         # forward slashes
         self.assertEqual(['test/a', 'test/b/', 'test/b/c', 'test/d/'], names)
 
-        file_attr = stat.S_IFREG | export.zip_exporter.FILE_PERMISSIONS
-        dir_attr = (stat.S_IFDIR | export.zip_exporter.ZIP_DIRECTORY_BIT |
-                    export.zip_exporter.DIR_PERMISSIONS)
+        file_attr = stat.S_IFREG | zip.FILE_PERMISSIONS
+        dir_attr = (stat.S_IFDIR | zip.ZIP_DIRECTORY_BIT |
+                    zip.DIR_PERMISSIONS)
 
         a_info = zfile.getinfo(names[0])
         self.assertEqual(file_attr, a_info.external_attr)

=== modified file 'breezy/tests/per_tree/test_archive.py'
--- breezy/tests/per_tree/test_archive.py	2018-05-20 16:08:56 +0000
+++ breezy/tests/per_tree/test_archive.py	2018-05-20 16:08:56 +0000
@@ -41,10 +41,10 @@
         tree_a = self.workingtree_to_test_tree(work_a)
         output_path = 'output'
         with open(output_path, 'wb') as f:
-            f.writelines(tree_a.archive(output_path, format=self.format))
+            f.writelines(tree_a.archive(self.format, output_path))
         names = self.get_export_names(output_path)
-        self.assertIn('output/file', names)
-        self.assertIn('output/dir', names)
+        self.assertIn('file', names)
+        self.assertIn('dir', names)
 
     def test_export_symlink(self):
         self.requireFeature(features.SymlinkFeature)
@@ -55,9 +55,9 @@
         tree_a = self.workingtree_to_test_tree(work_a)
         output_path = 'output'
         with open(output_path, 'wb') as f:
-            f.writelines(tree_a.archive(output_path, format=self.format))
+            f.writelines(tree_a.archive(self.format, output_path))
         names = self.get_export_names(output_path)
-        self.assertIn('output/link', names)
+        self.assertIn('link', names)
 
     def get_output_names(self, path):
         raise NotImplementedError(self.get_output_names)
@@ -107,9 +107,9 @@
         tree_a = self.workingtree_to_test_tree(work_a)
         output_path = 'output'
         with open(output_path, 'wb') as f:
-            f.writelines(tree_a.archive(output_path, format=self.format))
+            f.writelines(tree_a.archive(self.format, output_path))
         names = self.get_export_names(output_path)
-        self.assertIn('output/link.lnk', names)
+        self.assertIn('link.lnk', names)
 
 
 class GenericArchiveTests(TestCaseWithTree):
@@ -125,5 +125,4 @@
 
         self.assertRaises(
                 errors.NoSuchExportFormat,
-                list,
-                tree_a.archive('foo', format='dir'))
+                tree_a.archive, 'dir', 'foo')

=== modified file 'breezy/tests/test_export.py'
--- breezy/tests/test_export.py	2018-05-14 20:49:13 +0000
+++ breezy/tests/test_export.py	2018-05-20 16:08:56 +0000
@@ -27,7 +27,7 @@
     tests,
     )
 from ..export import get_root_name
-from ..export.tar_exporter import export_tarball_generator
+from ..archive.tar import tarball_generator
 from ..sixish import (
     BytesIO,
     )
@@ -250,24 +250,14 @@
         tf = tarfile.open('target.tar.bz2')
         self.assertEqual(["target/a"], tf.getnames())
 
-    def test_xz_stdout(self):
-        wt = self.make_branch_and_tree('.')
-        self.assertRaises(errors.BzrError, export.export, wt, '-',
-            format="txz")
-
     def test_export_tarball_generator(self):
         wt = self.make_branch_and_tree('.')
         self.build_tree(['a'])
         wt.add(["a"])
         wt.commit("1", timestamp=42)
         target = BytesIO()
-        ball = tarfile.open(None, "w|", target)
-        wt.lock_read()
-        try:
-            for _ in export_tarball_generator(wt, ball, "bar"):
-                pass
-        finally:
-            wt.unlock()
+        with wt.lock_read():
+            target.writelines(tarball_generator(wt, "bar"))
         # Ball should now be closed.
         target.seek(0)
         ball2 = tarfile.open(None, "r", target)

=== modified file 'breezy/tree.py'
--- breezy/tree.py	2018-05-20 16:08:56 +0000
+++ breezy/tree.py	2018-05-20 16:08:56 +0000
@@ -653,19 +653,22 @@
         searcher = default_searcher
         return searcher
 
-    def archive(self, name, format=None, root=None, subdir=None):
+    def archive(self, format, name, root='', subdir=None,
+                force_mtime=None):
         """Create an archive of this tree.
 
         :param name: target file name
         :param format: Format name (e.g. 'tar')
         :param root: Root directory name (or None)
         :param subdir: Subdirectory to export (or None)
+        :param per_file_timestamps: Whether to set the timestamp
+            for each file to the last changed time.
         :return: Iterator over archive chunks
         """
+        from .archive import create_archive
         with self.lock_read():
-            from .export import get_stream_export_generator
-            return get_stream_export_generator(self, name, format, root,
-                    subdir)
+            return create_archive(format, self, name, root,
+                    subdir, force_mtime=force_mtime)
 
     @classmethod
     def versionable_kind(cls, kind):