Merge lp:~jelmer/brz/iterobjectss into lp:brz/3.1

[Jelmer Vernooij]

Simplify InterTree handling for Git trees.

Simplify InterTree handling for Git trees.

[Jelmer Vernooij (jelmer)]

review: Approve

=== modified file 'breezy/git/tests/test_workingtree.py'
--- breezy/git/tests/test_workingtree.py	2020-08-06 01:40:59 +0000
+++ breezy/git/tests/test_workingtree.py	2020-08-08 18:37:30 +0000
@@ -23,8 +23,11 @@
 import stat
 
 from dulwich import __version__ as dulwich_version
-from dulwich.diff_tree import RenameDetector
+from dulwich.diff_tree import RenameDetector, tree_changes
 from dulwich.index import IndexEntry
+from dulwich.object_store import (
+    OverlayObjectStore,
+    )
 from dulwich.objects import (
     S_IFGITLINK,
     Blob,
@@ -42,7 +45,6 @@
     default_mapping,
     )
 from ..tree import (
-    changes_between_git_tree_and_working_copy,
     tree_delta_from_git_changes,
     )
 from ..workingtree import (
@@ -54,6 +56,22 @@
     )
 
 
+def changes_between_git_tree_and_working_copy(source_store, from_tree_sha, target,
+                                              want_unchanged=False,
+                                              want_unversioned=False,
+                                              rename_detector=None,
+                                              include_trees=True):
+    """Determine the changes between a git tree and a working tree with index.
+
+    """
+    to_tree_sha, extras = target.git_snapshot(want_unversioned=want_unversioned)
+    store = OverlayObjectStore([source_store, target.store])
+    return tree_changes(
+        store, from_tree_sha, to_tree_sha, include_trees=include_trees,
+        rename_detector=rename_detector,
+        want_unchanged=want_unchanged, change_type_same=True), extras
+
+
 class GitWorkingTreeTests(TestCaseWithTransport):
 
     def setUp(self):

=== modified file 'breezy/git/tree.py'
--- breezy/git/tree.py	2020-08-07 00:29:47 +0000
+++ breezy/git/tree.py	2020-08-08 18:37:30 +0000
@@ -264,7 +264,24 @@
     return path
 
 
-class GitRevisionTree(revisiontree.RevisionTree):
+class GitTree(_mod_tree.Tree):
+
+    def iter_git_objects(self):
+        """Iterate over all the objects in the tree.
+
+        :return :Yields tuples with (path, sha, mode)
+        """
+        raise NotImplementedError(self.iter_git_objects)
+
+    def git_snapshot(self):
+        """Snapshot a tree, and return tree object.
+
+        :return: Tree sha and set of extras
+        """
+        raise NotImplementedError(self.snapshot)
+
+
+class GitRevisionTree(revisiontree.RevisionTree, GitTree):
     """Revision tree implementation based on Git objects."""
 
     def __init__(self, repository, revision_id):
@@ -286,6 +303,9 @@
                 raise errors.NoSuchRevision(repository, revision_id)
             self.tree = commit.tree
 
+    def git_snapshot(self, want_unversioned=False):
+        return self.tree, set()
+
     def _submodule_info(self):
         if self._submodules is None:
             try:
@@ -977,10 +997,18 @@
     _matching_to_tree_format = None
     _test_mutable_trees_to_test_trees = None
 
+    def __init__(self, source, target):
+        super(InterGitTrees, self).__init__(source, target)
+        if self.source.store == self.target.store:
+            self.store = self.source.store
+        else:
+            self.store = OverlayObjectStore(
+                [self.source.store, self.target.store])
+        self.rename_detector = RenameDetector(self.store)
+
     @classmethod
     def is_compatible(cls, source, target):
-        return (isinstance(source, GitRevisionTree) and
-                isinstance(target, GitRevisionTree))
+        return isinstance(source, GitTree) and isinstance(target, GitTree)
 
     def compare(self, want_unchanged=False, specific_files=None,
                 extra_trees=None, require_versioned=False, include_root=False,
@@ -1018,7 +1046,25 @@
     def _iter_git_changes(self, want_unchanged=False, specific_files=None,
                           require_versioned=False, extra_trees=None,
                           want_unversioned=False, include_trees=True):
-        raise NotImplementedError(self._iter_git_changes)
+        trees = [self.source]
+        if extra_trees is not None:
+            trees.extend(extra_trees)
+        if specific_files is not None:
+            specific_files = self.target.find_related_paths_across_trees(
+                specific_files, trees,
+                require_versioned=require_versioned)
+        # TODO(jelmer): Restrict to specific_files, for performance reasons.
+        with self.lock_read():
+            from_tree_sha, from_extras = self.source.git_snapshot(
+                want_unversioned=want_unversioned)
+            to_tree_sha, to_extras = self.target.git_snapshot(
+                want_unversioned=want_unversioned)
+            changes = tree_changes(
+                self.store, from_tree_sha, to_tree_sha,
+                include_trees=include_trees,
+                rename_detector=self.rename_detector,
+                want_unchanged=want_unchanged, change_type_same=True)
+            return changes, from_extras, to_extras
 
     def find_target_path(self, path, recurse='none'):
         ret = self.find_target_paths([path], recurse=recurse)
@@ -1073,48 +1119,10 @@
         return ret
 
 
-class InterGitRevisionTrees(InterGitTrees):
-    """InterTree that works between two git revision trees."""
-
-    _matching_from_tree_format = None
-    _matching_to_tree_format = None
-    _test_mutable_trees_to_test_trees = None
-
-    @classmethod
-    def is_compatible(cls, source, target):
-        return (isinstance(source, GitRevisionTree) and
-                isinstance(target, GitRevisionTree))
-
-    def _iter_git_changes(self, want_unchanged=False, specific_files=None,
-                          require_versioned=True, extra_trees=None,
-                          want_unversioned=False, include_trees=True):
-        trees = [self.source]
-        if extra_trees is not None:
-            trees.extend(extra_trees)
-        if specific_files is not None:
-            specific_files = self.target.find_related_paths_across_trees(
-                specific_files, trees,
-                require_versioned=require_versioned)
-
-        if (self.source._repository._git.object_store !=
-                self.target._repository._git.object_store):
-            store = OverlayObjectStore(
-                [self.source._repository._git.object_store,
-                    self.target._repository._git.object_store])
-        else:
-            store = self.source._repository._git.object_store
-        rename_detector = RenameDetector(store)
-        changes = tree_changes(
-            store, self.source.tree, self.target.tree,
-            want_unchanged=want_unchanged, include_trees=include_trees,
-            change_type_same=True, rename_detector=rename_detector)
-        return changes, set(), set()
-
-
-_mod_tree.InterTree.register_optimiser(InterGitRevisionTrees)
-
-
-class MutableGitIndexTree(mutabletree.MutableTree):
+_mod_tree.InterTree.register_optimiser(InterGitTrees)
+
+
+class MutableGitIndexTree(mutabletree.MutableTree, GitTree):
 
     def __init__(self):
         self._lock_mode = None
@@ -1123,6 +1131,9 @@
         self._index_dirty = False
         self._submodules = None
 
+    def git_snapshot(self, want_unversioned=False):
+        return snapshot_workingtree(self, want_unversioned=want_unversioned)
+
     def is_versioned(self, path):
         with self.lock_read():
             path = encode_git_path(path.rstrip('/'))
@@ -1142,7 +1153,7 @@
         if self._lock_mode is None:
             raise errors.ObjectNotLocked(self)
         self._versioned_dirs = set()
-        for p, i in self._recurse_index_entries():
+        for p, sha, mode in self.iter_git_objects():
             self._ensure_versioned_dir(posixpath.dirname(p))
 
     def _ensure_versioned_dir(self, dirname):
@@ -1301,6 +1312,10 @@
         if self._versioned_dirs is not None:
             self._ensure_versioned_dir(index_path)
 
+    def iter_git_objects(self):
+        for p, entry in self._recurse_index_entries():
+            yield p, entry.sha, entry.mode
+
     def _recurse_index_entries(self, index=None, basepath=b"",
                                recurse_nested=False):
         # Iterate over all index entries
@@ -1398,7 +1413,7 @@
             if data is None:
                 data = self.branch.repository._git.object_store[sha].data
             ie.text_sha1 = osutils.sha_string(data)
-            ie.text_size = len(data)
+            ie.text_size = size
             ie.executable = bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode)
         return ie
 
@@ -1687,131 +1702,6 @@
                 return True
 
 
-class InterToIndexGitTree(InterGitTrees):
-    """InterTree that works between a Git revision tree and an index."""
-
-    def __init__(self, source, target):
-        super(InterToIndexGitTree, self).__init__(source, target)
-        if self.source.store == self.target.store:
-            self.store = self.source.store
-        else:
-            self.store = OverlayObjectStore(
-                [self.source.store, self.target.store])
-        self.rename_detector = RenameDetector(self.store)
-
-    @classmethod
-    def is_compatible(cls, source, target):
-        return (isinstance(source, GitRevisionTree) and
-                isinstance(target, MutableGitIndexTree))
-
-    def _iter_git_changes(self, want_unchanged=False, specific_files=None,
-                          require_versioned=False, extra_trees=None,
-                          want_unversioned=False, include_trees=True):
-        trees = [self.source]
-        if extra_trees is not None:
-            trees.extend(extra_trees)
-        if specific_files is not None:
-            specific_files = self.target.find_related_paths_across_trees(
-                specific_files, trees,
-                require_versioned=require_versioned)
-        # TODO(jelmer): Restrict to specific_files, for performance reasons.
-        with self.lock_read():
-            changes, target_extras = changes_between_git_tree_and_working_copy(
-                self.source.store, self.source.tree,
-                self.target, want_unchanged=want_unchanged,
-                want_unversioned=want_unversioned,
-                rename_detector=self.rename_detector,
-                include_trees=include_trees)
-            return changes, set(), target_extras
-
-
-_mod_tree.InterTree.register_optimiser(InterToIndexGitTree)
-
-
-class InterFromIndexGitTree(InterGitTrees):
-    """InterTree that works between a Git revision tree and an index."""
-
-    def __init__(self, source, target):
-        super(InterFromIndexGitTree, self).__init__(source, target)
-        if self.source.store == self.target.store:
-            self.store = self.source.store
-        else:
-            self.store = OverlayObjectStore(
-                [self.source.store, self.target.store])
-        self.rename_detector = RenameDetector(self.store)
-
-    @classmethod
-    def is_compatible(cls, source, target):
-        return (isinstance(target, GitRevisionTree) and
-                isinstance(source, MutableGitIndexTree))
-
-    def _iter_git_changes(self, want_unchanged=False, specific_files=None,
-                          require_versioned=False, extra_trees=None,
-                          want_unversioned=False, include_trees=True):
-        trees = [self.source]
-        if extra_trees is not None:
-            trees.extend(extra_trees)
-        if specific_files is not None:
-            specific_files = self.target.find_related_paths_across_trees(
-                specific_files, trees,
-                require_versioned=require_versioned)
-        # TODO(jelmer): Restrict to specific_files, for performance reasons.
-        with self.lock_read():
-            from_tree_sha, extras = snapshot_workingtree(self.source, want_unversioned=want_unversioned)
-            return tree_changes(
-                self.store, from_tree_sha, self.target.tree,
-                include_trees=include_trees,
-                rename_detector=self.rename_detector,
-                want_unchanged=want_unchanged, change_type_same=True), extras
-
-
-_mod_tree.InterTree.register_optimiser(InterFromIndexGitTree)
-
-
-class InterIndexGitTree(InterGitTrees):
-    """InterTree that works between a Git revision tree and an index."""
-
-    def __init__(self, source, target):
-        super(InterIndexGitTree, self).__init__(source, target)
-        if self.source.store == self.target.store:
-            self.store = self.source.store
-        else:
-            self.store = OverlayObjectStore(
-                [self.source.store, self.target.store])
-        self.rename_detector = RenameDetector(self.store)
-
-    @classmethod
-    def is_compatible(cls, source, target):
-        return (isinstance(target, MutableGitIndexTree) and
-                isinstance(source, MutableGitIndexTree))
-
-    def _iter_git_changes(self, want_unchanged=False, specific_files=None,
-                          require_versioned=False, extra_trees=None,
-                          want_unversioned=False, include_trees=True):
-        trees = [self.source]
-        if extra_trees is not None:
-            trees.extend(extra_trees)
-        if specific_files is not None:
-            specific_files = self.target.find_related_paths_across_trees(
-                specific_files, trees,
-                require_versioned=require_versioned)
-        # TODO(jelmer): Restrict to specific_files, for performance reasons.
-        with self.lock_read():
-            from_tree_sha, from_extras = snapshot_workingtree(
-                self.source, want_unversioned=want_unversioned)
-            to_tree_sha, to_extras = snapshot_workingtree(
-                self.target, want_unversioned=want_unversioned)
-            changes = tree_changes(
-                self.store, from_tree_sha, to_tree_sha,
-                include_trees=include_trees,
-                rename_detector=self.rename_detector,
-                want_unchanged=want_unchanged, change_type_same=True)
-            return changes, from_extras, to_extras
-
-
-_mod_tree.InterTree.register_optimiser(InterIndexGitTree)
-
-
 def snapshot_workingtree(target, want_unversioned=False):
     extras = set()
     blobs = {}
@@ -1859,21 +1749,21 @@
                         target.store.add_object(blob)
                 blobs[path] = (live_entry.sha, cleanup_mode(live_entry.mode))
     if want_unversioned:
-        for e in target._iter_files_recursive(include_dirs=False):
+        for extra in target._iter_files_recursive(include_dirs=False):
             try:
-                e, accessible = osutils.normalized_filename(e)
+                extra, accessible = osutils.normalized_filename(extra)
             except UnicodeDecodeError:
                 raise errors.BadFilenameEncoding(
-                    e, osutils._fs_enc)
-            np = encode_git_path(e)
+                    extra, osutils._fs_enc)
+            np = encode_git_path(extra)
             if np in blobs:
                 continue
-            st = target._lstat(e)
+            st = target._lstat(extra)
             if stat.S_ISDIR(st.st_mode):
                 blob = Tree()
             elif stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode):
                 blob = blob_from_path_and_stat(
-                    target.abspath(e).encode(osutils._fs_enc), st)
+                    target.abspath(extra).encode(osutils._fs_enc), st)
             else:
                 continue
             target.store.add_object(blob)
@@ -1881,19 +1771,3 @@
             extras.add(np)
     return commit_tree(
         target.store, dirified + [(p, s, m) for (p, (s, m)) in blobs.items()]), extras
-
-
-def changes_between_git_tree_and_working_copy(source_store, from_tree_sha, target,
-                                              want_unchanged=False,
-                                              want_unversioned=False,
-                                              rename_detector=None,
-                                              include_trees=True):
-    """Determine the changes between a git tree and a working tree with index.
-
-    """
-    to_tree_sha, extras = snapshot_workingtree(target, want_unversioned=want_unversioned)
-    store = OverlayObjectStore([source_store, target.store])
-    return tree_changes(
-        store, from_tree_sha, to_tree_sha, include_trees=include_trees,
-        rename_detector=rename_detector,
-        want_unchanged=want_unchanged, change_type_same=True), extras

=== modified file 'breezy/git/workingtree.py'
--- breezy/git/workingtree.py	2020-08-06 22:01:46 +0000
+++ breezy/git/workingtree.py	2020-08-08 18:37:30 +0000
@@ -572,7 +572,7 @@
         """
         with self.lock_read():
             index_paths = set(
-                [decode_git_path(p) for p, i in self._recurse_index_entries()])
+                [decode_git_path(p) for p, sha, mode in self.iter_git_objects()])
             all_paths = set(self._iter_files_recursive(include_dirs=False))
             return iter(all_paths - index_paths)