Merge lp:~jelmer/brz/depth into lp:brz

[Jelmer Vernooij]

Add a depth argument to ControlDir.sprout() and Repository.fetch().

[Martin Packman (gz)]

Thanks!

review: Approve

[lp:~jelmer/brz/depth]

7154. By Jelmer Vernooij on 2019-03-04

Fix tests.

7155. By Jelmer Vernooij on 2020-06-01

Merge lp:brz/3.1.

7156. By Jelmer Vernooij on 2022-07-08

Merge lp:brz/3.2.

7157. By Jelmer Vernooij on 2022-09-16

Merge trunk.

7158. By Jelmer Vernooij on 2022-10-27

Merge lp:brz/3.3

Unmerged revisions

7158. By Jelmer Vernooij on 2022-10-27

Merge lp:brz/3.3

7157. By Jelmer Vernooij on 2022-09-16

Merge trunk.

7156. By Jelmer Vernooij on 2022-07-08

Merge lp:brz/3.2.

7155. By Jelmer Vernooij on 2020-06-01

Merge lp:brz/3.1.

7154. By Jelmer Vernooij on 2019-03-04

Fix tests.

7153. By Jelmer Vernooij on 2019-02-10

add depth argument to ControlDir.sprout.

7152. By Jelmer Vernooij on 2019-02-10

Enable depth fetching for Git repositories.

7151. By Jelmer Vernooij on 2019-02-09

For the moment, Dulwich doesn't support depth fetching locally.

7150. By Jelmer Vernooij on 2019-02-09

Add Repository.support_fetch_depth.

7149. By Jelmer Vernooij on 2019-02-09

Merge dulwich-compat.

=== modified file 'breezy/branch.py'
--- breezy/branch.py	2018-11-30 12:39:04 +0000
+++ breezy/branch.py	2019-02-11 01:23:45 +0000
@@ -680,18 +680,19 @@
         """Get the tree_path and branch_location for a tree reference."""
         raise errors.UnsupportedOperation(self.get_reference_info, self)
 
-    def fetch(self, from_branch, last_revision=None, limit=None):
+    def fetch(self, from_branch, last_revision=None, limit=None, depth=None):
         """Copy revisions from from_branch into this branch.
 
         :param from_branch: Where to copy from.
         :param last_revision: What revision to stop at (None for at the end
                               of the branch.
         :param limit: Optional rough limit of revisions to fetch
+        :param depth: Revision depth
         :return: None
         """
         with self.lock_write():
             return InterBranch.get(from_branch, self).fetch(
-                last_revision, limit=limit)
+                last_revision, limit=limit, depth=depth)
 
     def get_bound_location(self):
         """Return the URL of the branch we are bound to.
@@ -2098,11 +2099,12 @@
         """
         raise NotImplementedError(self.copy_content_into)
 
-    def fetch(self, stop_revision=None, limit=None):
+    def fetch(self, stop_revision=None, limit=None, depth=None):
         """Fetch revisions.
 
         :param stop_revision: Last revision to fetch
         :param limit: Optional rough limit of revisions to fetch
+        :param depth: Optional revision depth
         """
         raise NotImplementedError(self.fetch)
 
@@ -2154,9 +2156,11 @@
             if self.source._push_should_merge_tags():
                 self.source.tags.merge_to(self.target.tags)
 
-    def fetch(self, stop_revision=None, limit=None):
+    def fetch(self, stop_revision=None, limit=None, depth=None):
         if self.target.base == self.source.base:
             return (0, [])
+        if depth is not None:
+            raise errors.FetchDepthUnsupported(self)
         with self.source.lock_read(), self.target.lock_write():
             fetch_spec_factory = fetch.FetchSpecFactory()
             fetch_spec_factory.source_branch = self.source
 
=== modified file 'breezy/bzr/bzrdir.py'
--- breezy/bzr/bzrdir.py	2018-11-18 19:48:57 +0000
+++ breezy/bzr/bzrdir.py	2019-02-11 01:23:45 +0000
@@ -350,7 +350,7 @@
                recurse='down', possible_transports=None,
                accelerator_tree=None, hardlink=False, stacked=False,
                source_branch=None, create_tree_if_local=True,
-               lossy=False):
+               depth=None):
         """Create a copy of this controldir prepared for use as a new line of
         development.
 
@@ -374,6 +374,7 @@
             location of this control directory.
         :param create_tree_if_local: If true, a working-tree will be created
             when working locally.
+        :param depth: Optional fetch depth
         :return: The created control directory
         """
         operation = cleanup.OperationWithCleanups(self._sprout)
@@ -382,12 +383,15 @@
             recurse=recurse, possible_transports=possible_transports,
             accelerator_tree=accelerator_tree, hardlink=hardlink,
             stacked=stacked, source_branch=source_branch,
-            create_tree_if_local=create_tree_if_local)
+            create_tree_if_local=create_tree_if_local, depth=depth)
 
     def _sprout(self, op, url, revision_id=None, force_new_repo=False,
                 recurse='down', possible_transports=None,
                 accelerator_tree=None, hardlink=False, stacked=False,
-                source_branch=None, create_tree_if_local=True, lossy=False):
+                source_branch=None, create_tree_if_local=True,
+                depth=None):
+        if depth is not None:
+            raise errors.FetchDepthUnsupported(self)
         add_cleanup = op.add_cleanup
         fetch_spec_factory = fetch.FetchSpecFactory()
         if revision_id is not None:
 
=== modified file 'breezy/controldir.py'
--- breezy/controldir.py	2018-11-12 01:41:38 +0000
+++ breezy/controldir.py	2019-02-11 01:23:45 +0000
@@ -373,7 +373,7 @@
                recurse='down', possible_transports=None,
                accelerator_tree=None, hardlink=False, stacked=False,
                source_branch=None, create_tree_if_local=True,
-               lossy=False):
+               depth=None):
         """Create a copy of this controldir prepared for use as a new line of
         development.
 
@@ -396,6 +396,7 @@
             location of this control directory.
         :param create_tree_if_local: If true, a working-tree will be created
             when working locally.
+        :param depth: Possible fetch depth
         """
         raise NotImplementedError(self.sprout)
 
 
=== modified file 'breezy/errors.py'
--- breezy/errors.py	2018-11-11 04:08:32 +0000
+++ breezy/errors.py	2019-02-11 01:23:45 +0000
@@ -1851,6 +1851,14 @@
         self.tname = type(method_self).__name__
 
 
+class FetchDepthUnsupported(UnsupportedOperation):
+
+    fmt = ("InterBranch %(interbranch)r does not support fetching depths.")
+
+    def __init__(self, interbranch):
+        BzrError.__init__(self, interbranch=interbranch)
+
+
 class FetchLimitUnsupported(UnsupportedOperation):
 
     fmt = ("InterBranch %(interbranch)r does not support fetching limits.")
 
=== modified file 'breezy/git/branch.py'
--- breezy/git/branch.py	2019-01-19 17:13:53 +0000
+++ breezy/git/branch.py	2019-02-11 01:23:45 +0000
@@ -656,9 +656,9 @@
     def break_lock(self):
         self.repository._git.refs.unlock_ref(self.ref)
 
-    def fetch(self, from_branch, last_revision=None, limit=None):
+    def fetch(self, from_branch, last_revision=None, limit=None, depth=None):
         return branch.InterBranch.get(from_branch, self).fetch(
-            stop_revision=last_revision, limit=limit)
+            stop_revision=last_revision, limit=limit, depth=depth)
 
     def _gen_revision_history(self):
         if self.head is None:
@@ -885,10 +885,10 @@
             return False
         return True
 
-    def fetch(self, stop_revision=None, fetch_tags=None, limit=None):
-        self.fetch_objects(stop_revision, fetch_tags=fetch_tags, limit=limit)
+    def fetch(self, stop_revision=None, fetch_tags=None, limit=None, depth=None):
+        self.fetch_objects(stop_revision, fetch_tags=fetch_tags, limit=limit, depth=depth)
 
-    def fetch_objects(self, stop_revision, fetch_tags, limit=None):
+    def fetch_objects(self, stop_revision, fetch_tags, limit=None, depth=None):
         interrepo = self._get_interrepo(self.source, self.target)
         if fetch_tags is None:
             c = self.source.get_config_stack()
@@ -909,7 +909,7 @@
                 [self._last_revid], include_tags=fetch_tags)
             return real(heads)
         pack_hint, head, refs = interrepo.fetch_objects(
-            determine_wants, self.source.mapping, limit=limit)
+            determine_wants, self.source.mapping, limit=limit, depth=depth)
         if (pack_hint is not None and
                 self.target.repository._format.pack_compresses):
             self.target.repository.pack(hint=pack_hint)
@@ -1034,7 +1034,7 @@
 class InterGitBranch(branch.GenericInterBranch):
     """InterBranch implementation that pulls between Git branches."""
 
-    def fetch(self, stop_revision=None, fetch_tags=None, limit=None):
+    def fetch(self, stop_revision=None, fetch_tags=None, limit=None, depth=None):
         raise NotImplementedError(self.fetch)
 
 
@@ -1101,14 +1101,14 @@
         return (isinstance(source, GitBranch) and
                 isinstance(target, LocalGitBranch))
 
-    def fetch(self, stop_revision=None, fetch_tags=None, limit=None):
+    def fetch(self, stop_revision=None, fetch_tags=None, limit=None, depth=None):
         interrepo = _mod_repository.InterRepository.get(self.source.repository,
                                                         self.target.repository)
         if stop_revision is None:
             stop_revision = self.source.last_revision()
         determine_wants = interrepo.get_determine_wants_revids(
             [stop_revision], include_tags=fetch_tags)
-        interrepo.fetch_objects(determine_wants, limit=limit)
+        interrepo.fetch_objects(determine_wants, limit=limit, depth=depth)
 
     def _basic_push(self, overwrite=False, stop_revision=None):
         if overwrite is True:
@@ -1298,7 +1298,7 @@
         return ret
 
     def fetch(self, stop_revision=None, fetch_tags=None, lossy=False,
-              limit=None):
+              limit=None, depth=None):
         if stop_revision is None:
             stop_revision = self.source.last_revision()
         ret = []
@@ -1307,7 +1307,7 @@
                 ret.append((None, v))
         ret.append((None, stop_revision))
         try:
-            self.interrepo.fetch_objects(ret, lossy=lossy, limit=limit)
+            self.interrepo.fetch_objects(ret, lossy=lossy, limit=limit, depth=depth)
         except NoPushSupport:
             raise errors.NoRoundtrippingSupport(self.source, self.target)
 
 
=== modified file 'breezy/git/dir.py'
--- breezy/git/dir.py	2018-11-16 11:37:47 +0000
+++ breezy/git/dir.py	2019-02-11 01:23:45 +0000
@@ -149,7 +149,7 @@
     def sprout(self, url, revision_id=None, force_new_repo=False,
                recurse='down', possible_transports=None,
                accelerator_tree=None, hardlink=False, stacked=False,
-               source_branch=None, create_tree_if_local=True):
+               source_branch=None, create_tree_if_local=True, depth=None):
         from ..repository import InterRepository
         from ..transport.local import LocalTransport
         from ..transport import get_transport
@@ -178,7 +178,8 @@
         else:
             determine_wants = interrepo.determine_wants_all
         interrepo.fetch_objects(determine_wants=determine_wants,
-                                mapping=source_branch.mapping)
+                                mapping=source_branch.mapping,
+                                depth=depth)
         result_branch = source_branch.sprout(
             result, revision_id=revision_id, repository=result_repo)
         if (create_tree_if_local and
 
=== modified file 'breezy/git/interrepo.py'
--- breezy/git/interrepo.py	2018-11-16 23:15:15 +0000
+++ breezy/git/interrepo.py	2019-02-11 01:23:45 +0000
@@ -39,6 +39,7 @@
 
 from ..errors import (
     DivergedBranches,
+    FetchDepthUnsupported,
     FetchLimitUnsupported,
     InvalidRevisionId,
     LossyPushToSameVCS,
@@ -53,6 +54,7 @@
     )
 from ..sixish import (
     viewitems,
+    viewkeys,
     viewvalues,
     )
 from .. import (
@@ -124,9 +126,9 @@
         """
         raise NotImplementedError(self.fetch_refs)
 
-    def search_missing_revision_ids(self,
-                                    find_ghosts=True, revision_ids=None,
-                                    if_present_ids=None, limit=None):
+    def search_missing_revision_ids(
+            self, find_ghosts=True, revision_ids=None, if_present_ids=None,
+            limit=None, depth=None):
         if limit is not None:
             raise FetchLimitUnsupported(self)
         git_shas = []
@@ -190,7 +192,7 @@
                 return False
         return self._commit_needs_fetching(sha_id)
 
-    def missing_revisions(self, stop_revisions):
+    def _missing_revisions(self, stop_revisions, depth=None):
         """Find the revisions that are missing from the target repository.
 
         :param stop_revisions: Revisions to check for (tuples with
@@ -204,32 +206,35 @@
         for (sha1, revid) in stop_revisions:
             if sha1 is not None and revid is not None:
                 revid_sha_map[revid] = sha1
-                stop_revids.append(revid)
+                stop_revids.append((revid, 1))
             elif sha1 is not None:
                 if self._commit_needs_fetching(sha1):
                     for (kind, (revid, tree_sha, verifiers)) in self.source_store.lookup_git_sha(sha1):
                         revid_sha_map[revid] = sha1
-                        stop_revids.append(revid)
+                        stop_revids.append((revid, 1))
             else:
                 if revid is None:
                     raise AssertionError
-                stop_revids.append(revid)
+                stop_revids.append((revid, 1))
         missing = set()
         graph = self.source.get_graph()
         pb = ui.ui_factory.nested_progress_bar()
         try:
             while stop_revids:
-                new_stop_revids = []
-                for revid in stop_revids:
+                new_stop_revids = {}
+                for revid, revid_depth in stop_revids:
                     sha1 = revid_sha_map.get(revid)
                     if (revid not in missing and
                             self._revision_needs_fetching(sha1, revid)):
                         missing.add(revid)
-                        new_stop_revids.append(revid)
+                        if depth is None or revid_depth < depth:
+                            new_stop_revids[revid] = revid_depth
                 stop_revids = set()
-                parent_map = graph.get_parent_map(new_stop_revids)
-                for parent_revids in viewvalues(parent_map):
-                    stop_revids.update(parent_revids)
+                parent_map = graph.get_parent_map(viewkeys(new_stop_revids))
+                for revid, parent_revids in viewvalues(parent_map):
+                    stop_revids.update(
+                        [(parent_revid, new_stop_revids[revid] + 1)
+                         for parent_revid in parent_revids])
                 pb.update("determining revisions to fetch", len(missing))
         finally:
             pb.finished()
@@ -292,7 +297,7 @@
                     result_refs[name] = (gitid, revid if not lossy else self.mapping.revision_id_foreign_to_bzr(gitid))
         return revidmap, old_refs, result_refs
 
-    def fetch_objects(self, revs, lossy, limit=None):
+    def fetch_objects(self, revs, lossy, limit=None, depth=None):
         if not lossy and not self.mapping.roundtripping:
             for git_sha, bzr_revid in revs:
                 if (bzr_revid is not None and
@@ -300,7 +305,7 @@
                     raise NoPushSupport(self.source, self.target, self.mapping,
                                         bzr_revid)
         with self.source_store.lock_read():
-            todo = list(self.missing_revisions(revs))[:limit]
+            todo = list(self._missing_revisions(revs, depth=depth))[:limit]
             revidmap = {}
             pb = ui.ui_factory.nested_progress_bar()
             try:
@@ -325,7 +330,7 @@
                 pb.finished()
 
     def fetch(self, revision_id=None, pb=None, find_ghosts=False,
-              fetch_spec=None, mapped_refs=None):
+              fetch_spec=None, mapped_refs=None, depth=None):
         if mapped_refs is not None:
             stop_revisions = mapped_refs
         elif revision_id is not None:
@@ -342,7 +347,7 @@
                               for revid in self.source.all_revision_ids()]
         self._warn_slow()
         try:
-            self.fetch_objects(stop_revisions, lossy=False)
+            self.fetch_objects(stop_revisions, lossy=False, depth=depth)
         except NoPushSupport:
             raise NoRoundtrippingSupport(self.source, self.target)
 
@@ -501,7 +506,8 @@
                 'Fetching from Git to Bazaar repository. '
                 'For better performance, fetch into a Git repository.')
 
-    def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
+    def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False,
+                      depth=None):
         """Fetch objects from a remote server.
 
         :param determine_wants: determine_wants callback
@@ -522,7 +528,8 @@
         return self.get_determine_wants_heads(wants, include_tags=include_tags)
 
     def fetch(self, revision_id=None, find_ghosts=False,
-              mapping=None, fetch_spec=None, include_tags=False):
+              mapping=None, fetch_spec=None, include_tags=False,
+              depth=None):
         if mapping is None:
             mapping = self.source.get_mapping()
         if revision_id is not None:
@@ -543,8 +550,8 @@
         else:
             determine_wants = self.determine_wants_all
 
-        (pack_hint, _, remote_refs) = self.fetch_objects(determine_wants,
-                                                         mapping)
+        (pack_hint, _, remote_refs) = self.fetch_objects(
+            determine_wants, mapping, depth=depth)
         if pack_hint is not None and self.target._format.pack_compresses:
             self.target.pack(hint=pack_hint)
         return remote_refs
@@ -563,7 +570,8 @@
             all_parents.update(values)
         return set(all_revs) - all_parents
 
-    def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
+    def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False,
+                      depth=None):
         """See `InterGitNonGitRepository`."""
         self._warn_slow()
         store = get_object_store(self.target, mapping)
@@ -577,7 +585,8 @@
             pb = ui.ui_factory.nested_progress_bar()
             try:
                 objects_iter = self.source.fetch_objects(
-                    wants_recorder, graph_walker, store.get_raw)
+                    wants_recorder, graph_walker, store.get_raw,
+                    depth=depth)
                 trace.mutter("Importing %d new revisions",
                              len(wants_recorder.wants))
                 (pack_hint, last_rev) = import_git_objects(
@@ -605,22 +614,22 @@
     """InterRepository that copies revisions from a local Git into a non-Git
     repository."""
 
-    def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False):
+    def fetch_objects(self, determine_wants, mapping, limit=None, lossy=False,
+                      depth=None):
         """See `InterGitNonGitRepository`."""
+        if depth is not None:
+            raise FetchDepthUnsupported(self)
         self._warn_slow()
         remote_refs = self.source.controldir.get_refs_container().as_dict()
         wants = determine_wants(remote_refs)
         pb = ui.ui_factory.nested_progress_bar()
         target_git_object_retriever = get_object_store(self.target, mapping)
         try:
-            target_git_object_retriever.lock_write()
-            try:
+            with target_git_object_retriever.lock_write():
                 (pack_hint, last_rev) = import_git_objects(
                     self.target, mapping, self.source._git.object_store,
                     target_git_object_retriever, wants, pb, limit)
                 return (pack_hint, last_rev, remote_refs)
-            finally:
-                target_git_object_retriever.unlock()
         finally:
             pb.finished()
 
@@ -660,7 +669,7 @@
         return None, old_refs, new_refs
 
     def fetch_objects(self, determine_wants, mapping=None, limit=None,
-                      lossy=False):
+                      lossy=False, depth=None):
         raise NotImplementedError(self.fetch_objects)
 
     def _target_has_shas(self, shas):
@@ -669,7 +678,7 @@
 
     def fetch(self, revision_id=None, find_ghosts=False,
               mapping=None, fetch_spec=None, branches=None, limit=None,
-              include_tags=False):
+              include_tags=False, depth=None):
         if mapping is None:
             mapping = self.source.get_mapping()
         if revision_id is not None:
@@ -698,7 +707,7 @@
             determine_wants = self.get_determine_wants_revids(
                 args, include_tags=include_tags)
         wants_recorder = DetermineWantsRecorder(determine_wants)
-        self.fetch_objects(wants_recorder, mapping, limit=limit)
+        self.fetch_objects(wants_recorder, mapping, limit=limit, depth=depth)
         return wants_recorder.remote_refs
 
     def get_determine_wants_revids(self, revids, include_tags=False):
@@ -720,7 +729,7 @@
 class InterLocalGitLocalGitRepository(InterGitGitRepository):
 
     def fetch_objects(self, determine_wants, mapping=None, limit=None,
-                      lossy=False):
+                      lossy=False, depth=None):
         if lossy:
             raise LossyPushToSameVCS(self.source, self.target)
         if limit is not None:
@@ -731,7 +740,7 @@
         try:
             refs = self.source._git.fetch(
                 self.target._git, determine_wants,
-                progress=progress)
+                progress=progress, depth=depth)
         finally:
             pb.finished()
         return (None, None, refs)
@@ -746,7 +755,7 @@
 class InterRemoteGitLocalGitRepository(InterGitGitRepository):
 
     def fetch_objects(self, determine_wants, mapping=None, limit=None,
-                      lossy=False):
+                      lossy=False, depth=None):
         if lossy:
             raise LossyPushToSameVCS(self.source, self.target)
         if limit is not None:
@@ -768,10 +777,12 @@
         else:
             f, commit, abort = self.target._git.object_store.add_pack()
         try:
-            refs = self.source.controldir.fetch_pack(
-                determine_wants, graphwalker, f.write)
+            fetch_result = self.source.controldir.fetch_pack(
+                determine_wants, graphwalker, f.write, depth=depth)
             commit()
-            return (None, None, refs)
+            self.target._git.update_shallow(
+                fetch_result.new_shallow, fetch_result.new_unshallow)
+            return (None, None, fetch_result.refs)
         except BaseException:
             abort()
             raise
 
=== modified file 'breezy/git/remote.py'
--- breezy/git/remote.py	2019-02-01 16:56:56 +0000
+++ breezy/git/remote.py	2019-02-11 01:23:45 +0000
@@ -416,8 +416,8 @@
             if pb is not None:
                 pb.finished()
 
-    def fetch_pack(self, determine_wants, graph_walker, pack_data,
-                   progress=None):
+    def fetch_pack(self, determine_wants, graph_walker, pack_data, progress=None,
+                   depth=None):
         if progress is None:
             pb = ui.ui_factory.nested_progress_bar()
             progress = DefaultProgressReporter(pb).progress
@@ -426,7 +426,7 @@
         try:
             result = self._client.fetch_pack(
                 self._client_path, determine_wants, graph_walker, pack_data,
-                progress)
+                progress, depth=depth)
             if result.refs is None:
                 result.refs = {}
             self._refs = remote_refs_dict_to_container(
@@ -539,9 +539,10 @@
     def get_refs_container(self):
         if self._refs is not None:
             return self._refs
-        result = self.fetch_pack(lambda x: None, None,
-                                 lambda x: None,
-                                 lambda x: trace.mutter("git: %s" % x))
+        result = self.fetch_pack(
+            lambda x: None, None,
+            lambda x: None, lambda x: trace.mutter("git: %s" % x),
+            depth=None)
         self._refs = remote_refs_dict_to_container(
             result.refs, result.symrefs)
         return self._refs
@@ -841,19 +842,20 @@
         return self.controldir.archive(*args, **kwargs)
 
     def fetch_pack(self, determine_wants, graph_walker, pack_data,
-                   progress=None):
+                   progress=None, depth=None):
         return self.controldir.fetch_pack(
-            determine_wants, graph_walker, pack_data, progress)
+            determine_wants, graph_walker, pack_data, progress, depth=depth)
 
     def send_pack(self, get_changed_refs, generate_pack_data):
         return self.controldir.send_pack(get_changed_refs, generate_pack_data)
 
     def fetch_objects(self, determine_wants, graph_walker, resolve_ext_ref,
-                      progress=None):
+                      progress=None, depth=None):
         fd, path = tempfile.mkstemp(suffix=".pack")
         try:
-            self.fetch_pack(determine_wants, graph_walker,
-                            lambda x: os.write(fd, x), progress)
+            self.fetch_pack(
+                determine_wants, graph_walker,
+                lambda x: os.write(fd, x), progress, depth=depth)
         finally:
             os.close(fd)
         if os.path.getsize(path) == 0:
 
=== modified file 'breezy/git/repository.py'
--- breezy/git/repository.py	2018-11-30 12:39:04 +0000
+++ breezy/git/repository.py	2019-02-11 01:23:45 +0000
@@ -131,6 +131,7 @@
     _serializer = None
     vcs = foreign_vcs_git
     chk_bytes = None
+    supports_fetch_depth = True
 
     def __init__(self, gitdir):
         self._transport = gitdir.root_transport
 
=== modified file 'breezy/git/tests/test_blackbox.py'
--- breezy/git/tests/test_blackbox.py	2019-01-02 18:49:15 +0000
+++ breezy/git/tests/test_blackbox.py	2019-02-11 01:23:45 +0000
@@ -308,6 +308,7 @@
         self.repo.stage("foo")
         self.repo.do_commit(
             b"message", committer=b"Somebody <[e-mail]>",
+            author=b"Somebody <[e-mail]>",
             commit_timestamp=1526330165, commit_timezone=0,
             author_timestamp=1526330165, author_timezone=0,
             merge_heads=[b'aa' * 20])
 
=== modified file 'breezy/git/transportgit.py'
--- breezy/git/transportgit.py	2018-11-11 14:23:06 +0000
+++ breezy/git/transportgit.py	2019-02-11 01:23:45 +0000
@@ -36,6 +36,7 @@
 from dulwich.object_store import (
     PackBasedObjectStore,
     PACKDIR,
+    read_packs_file,
     )
 from dulwich.pack import (
     MemoryPackIndex,
@@ -587,16 +588,41 @@
                 ret.append(l)
             return ret
 
-    @property
-    def packs(self):
-        # FIXME: Never invalidates.
-        if not self._pack_cache:
-            self._update_pack_cache()
-        return self._pack_cache.values()
-
     def _update_pack_cache(self):
-        for pack in self._load_packs():
-            self._pack_cache[pack._basename] = pack
+        pack_files = set()
+        pack_dir_contents = self._pack_names()
+        for name in pack_dir_contents:
+            if name.startswith("pack-") and name.endswith(".pack"):
+                # verify that idx exists first (otherwise the pack was not yet
+                # fully written)
+                idx_name = os.path.splitext(name)[0] + ".idx"
+                if idx_name in pack_dir_contents:
+                    pack_files.add(os.path.splitext(name)[0])
+
+        new_packs = []
+        for basename in pack_files:
+            pack_name = basename + ".pack"
+            if basename not in self._pack_cache:
+                try:
+                    size = self.pack_transport.stat(pack_name).st_size
+                except TransportNotPossible:
+                    f = self.pack_transport.get(pack_name)
+                    pd = PackData(pack_name, f)
+                else:
+                    pd = PackData(
+                        pack_name, self.pack_transport.get(pack_name),
+                        size=size)
+                idxname = basename + ".idx"
+                idx = load_pack_index_file(
+                    idxname, self.pack_transport.get(idxname))
+                pack = Pack.from_objects(pd, idx)
+                pack._basename = basename
+                self._pack_cache[basename] = pack
+                new_packs.append(pack)
+        # Remove disappeared pack files
+        for f in set(self._pack_cache) - pack_files:
+            self._pack_cache.pop(f).close()
+        return new_packs
 
     def _pack_names(self):
         try:
@@ -608,9 +634,6 @@
                 # Hmm, warn about running 'git update-server-info' ?
                 return iter([])
             else:
-                # TODO(jelmer): Move to top-level after dulwich
-                # 0.19.7 is released.
-                from dulwich.object_store import read_packs_file
                 with f:
                     return read_packs_file(f)
         except NoSuchFile:
@@ -619,26 +642,10 @@
     def _remove_pack(self, pack):
         self.pack_transport.delete(os.path.basename(pack.index.path))
         self.pack_transport.delete(pack.data.filename)
-
-    def _load_packs(self):
-        ret = []
-        for name in self._pack_names():
-            if name.startswith("pack-") and name.endswith(".pack"):
-                try:
-                    size = self.pack_transport.stat(name).st_size
-                except TransportNotPossible:
-                    f = self.pack_transport.get(name)
-                    pd = PackData(name, f)
-                else:
-                    pd = PackData(name, self.pack_transport.get(name),
-                                  size=size)
-                idxname = name.replace(".pack", ".idx")
-                idx = load_pack_index_file(
-                    idxname, self.pack_transport.get(idxname))
-                pack = Pack.from_objects(pd, idx)
-                pack._basename = idxname[:-4]
-                ret.append(pack)
-        return ret
+        try:
+            del self._pack_cache[os.path.basename(pack._basename)]
+        except KeyError:
+            pass
 
     def _iter_loose_objects(self):
         for base in self.transport.list_dir('.'):
@@ -702,7 +709,7 @@
         idx = load_pack_index_file(basename + ".idx", idxfile)
         final_pack = Pack.from_objects(p, idx)
         final_pack._basename = basename
-        self._add_known_pack(basename, final_pack)
+        self._add_cached_pack(basename, final_pack)
         return final_pack
 
     def move_in_thin_pack(self, f):
@@ -735,8 +742,6 @@
             write_pack_index_v2(idxfile, entries, data_sum)
         finally:
             idxfile.close()
-        # TODO(jelmer): Just add new pack to the cache
-        self._flush_pack_cache()
 
     def add_pack(self):
         """Add a new pack to this object store.
 
=== modified file 'breezy/plugins/weave_fmt/bzrdir.py'
--- breezy/plugins/weave_fmt/bzrdir.py	2018-11-11 04:08:32 +0000
+++ breezy/plugins/weave_fmt/bzrdir.py	2019-02-11 01:23:45 +0000
@@ -895,8 +895,10 @@
     def sprout(self, url, revision_id=None, force_new_repo=False,
                possible_transports=None, accelerator_tree=None,
                hardlink=False, stacked=False, create_tree_if_local=True,
-               source_branch=None):
+               source_branch=None, depth=None):
         """See ControlDir.sprout()."""
+        if depth is not None:
+            raise errors.FetchDepthUnsupported(self)
         if source_branch is not None:
             my_branch = self.open_branch()
             if source_branch.base != my_branch.base:
 
=== modified file 'breezy/repository.py'
--- breezy/repository.py	2019-02-01 16:56:56 +0000
+++ breezy/repository.py	2019-02-11 01:23:45 +0000
@@ -260,6 +260,10 @@
     # items in the tree, or just bulk fetching/pushing of data?
     supports_random_access = True
 
+    # Does this repository implementation support fetching with
+    # a certain graph depth?
+    supports_fetch_depth = False
+
     def abort_write_group(self, suppress_errors=False):
         """Commit the contents accrued within the current write group.
 
 
=== modified file 'breezy/tests/per_controldir/test_controldir.py'
--- breezy/tests/per_controldir/test_controldir.py	2018-11-11 04:08:32 +0000
+++ breezy/tests/per_controldir/test_controldir.py	2019-02-11 01:23:45 +0000
@@ -86,7 +86,7 @@
 
     def sproutOrSkip(self, from_bzrdir, to_url, revision_id=None,
                      force_new_repo=False, accelerator_tree=None,
-                     create_tree_if_local=True):
+                     create_tree_if_local=True, depth=None):
         """Sprout from_bzrdir into to_url, or raise TestSkipped.
 
         A simple wrapper for from_bzrdir.sprout that translates NotLocalUrl into
@@ -99,7 +99,8 @@
                                     force_new_repo=force_new_repo,
                                     possible_transports=[to_transport],
                                     accelerator_tree=accelerator_tree,
-                                    create_tree_if_local=create_tree_if_local)
+                                    create_tree_if_local=create_tree_if_local,
+                                    depth=depth)
         return target
 
     def test_uninitializable(self):
@@ -1050,6 +1051,20 @@
         self.addCleanup(repo.lock_read().unlock)
         self.assertEqual(None, repo.get_parent_map([rev1]).get(rev1))
 
+    def test_sprout_with_depth(self):
+        tree = self.make_branch_and_tree('source')
+        self.build_tree(['source/foo'])
+        tree.add('foo')
+        tree.commit('revision 1')
+        rev2 = tree.commit('revision 2', allow_pointless=True)
+        dir = tree.controldir
+        try:
+            target = self.sproutOrSkip(dir, self.get_url('target'), depth=1)
+        except errors.FetchDepthUnsupported:
+            self.assertFalse(tree.branch.repository.supports_fetch_depth)
+        else:
+            self.assertEqual({rev2}, target.open_repository().all_revision_ids())
+
     def test_format_initialize_find_open(self):
         # loopback test to check the current format initializes to itself.
         if not self.bzrdir_format.is_initializable():
 
=== modified file 'breezy/tests/per_interbranch/test_fetch.py'
--- breezy/tests/per_interbranch/test_fetch.py	2018-11-11 04:08:32 +0000
+++ breezy/tests/per_interbranch/test_fetch.py	2019-02-11 01:23:45 +0000
@@ -16,7 +16,11 @@
 
 """Tests for InterBranch.fetch."""
 
-from breezy.errors import FetchLimitUnsupported, NoRoundtrippingSupport
+from breezy.errors import (
+    FetchDepthUnsupported,
+    FetchLimitUnsupported,
+    NoRoundtrippingSupport,
+    )
 from breezy.revision import NULL_REVISION
 from breezy.tests import TestNotApplicable
 from breezy.tests.per_interbranch import (
@@ -113,3 +117,39 @@
         self.assertEqual(
             {rev1, rev2},
             b2.repository.has_revisions([rev1, rev2, rev3]))
+
+    def test_fetch_revisions_depth(self):
+        """Test fetch-revision operation."""
+        builder = self.make_branch_builder(
+            'b1', format=self.branch_format_from._matchingcontroldir)
+        builder.start_series()
+        rev1 = builder.build_commit()
+        rev2 = builder.build_commit()
+        rev3 = builder.build_commit()
+        builder.finish_series()
+        b1 = builder.get_branch()
+        b2 = self.make_to_branch('b2')
+        try:
+            if b2.repository.supports_fetch_depth:
+                b2.fetch(b1, depth=1)
+            else:
+                self.assertRaises(FetchDepthUnsupported, b2.fetch, b1, depth=1)
+                raise TestNotApplicable(
+                    'interbranch does not support fetch depths')
+        except NoRoundtrippingSupport:
+            raise TestNotApplicable(
+                'lossless cross-vcs fetch %r to %r not supported' %
+                (b1, b2))
+
+        self.assertEqual(
+            {rev3},
+            b2.repository.has_revisions([rev1, rev2, rev3]))
+
+        # fetch does not update the last revision
+        self.assertEqual(NULL_REVISION, b2.last_revision())
+
+        # Incrementally fetch one more
+        b2.fetch(b1, depth=2)
+
+        self.assertEqual(
+            {rev2, rev3}, b2.repository.has_revisions([rev1, rev2, rev3]))
 
=== modified file 'breezy/tests/per_repository/test_repository.py'
--- breezy/tests/per_repository/test_repository.py	2019-02-01 16:56:56 +0000
+++ breezy/tests/per_repository/test_repository.py	2019-02-11 01:23:45 +0000
@@ -125,6 +125,9 @@
     def test_attribute_format_supports_random_access(self):
         self.assertRepositoryAttribute('supports_random_access', (True, False))
 
+    def test_attribute_format_supports_fetch_depth(self):
+        self.assertRepositoryAttribute('supports_fetch_depth', (True, False))
+
     def test_attribute_format_supports_setting_revision_ids(self):
         self.assertFormatAttribute('supports_setting_revision_ids',
                                    (True, False))
 
=== modified file 'breezy/tests/test_foreign.py'
--- breezy/tests/test_foreign.py	2018-11-25 20:44:56 +0000
+++ breezy/tests/test_foreign.py	2019-02-11 01:23:45 +0000
@@ -339,14 +339,15 @@
     def sprout(self, url, revision_id=None, force_new_repo=False,
                recurse='down', possible_transports=None,
                accelerator_tree=None, hardlink=False, stacked=False,
-               source_branch=None):
+               source_branch=None, depth=None):
         # dirstate doesn't cope with accelerator_trees well
         # that have a different control dir
         return super(DummyForeignVcsDir, self).sprout(
             url=url,
             revision_id=revision_id, force_new_repo=force_new_repo,
             recurse=recurse, possible_transports=possible_transports,
-            hardlink=hardlink, stacked=stacked, source_branch=source_branch)
+            hardlink=hardlink, stacked=stacked, source_branch=source_branch,
+            depth=depth)
 
 
 def register_dummy_foreign_for_test(testcase):
 
=== modified file 'setup.py'
--- setup.py	2019-01-24 00:23:01 +0000
+++ setup.py	2019-02-11 01:23:45 +0000
@@ -60,7 +60,7 @@
         # Technically, Breezy works without these two dependencies too. But there's
         # no way to enable them by default and let users opt out.
         'fastimport>=0.9.8',
-        'dulwich>=0.19.1',
+        'dulwich>=0.19.11',
         ],
     'extras_require': {
         'fastimport': [],