Merge lp:~jelmer/brz/iterobjectss into lp:brz/3.1

Proposed by Jelmer Vernooij
Status: Merged
Approved by: Jelmer Vernooij
Approved revision: no longer in the source branch.
Merge reported by: The Breezy Bot
Merged at revision: not available
Proposed branch: lp:~jelmer/brz/iterobjectss
Merge into: lp:brz/3.1
Diff against target: 419 lines (+90/-198)
3 files modified
breezy/git/tests/test_workingtree.py (+20/-2)
breezy/git/tree.py (+69/-195)
breezy/git/workingtree.py (+1/-1)
To merge this branch: bzr merge lp:~jelmer/brz/iterobjectss
Reviewer Review Type Date Requested Status
Jelmer Vernooij Approve
Review via email: mp+388958@code.launchpad.net

Commit message

Simplify InterTree handling for Git trees.

Description of the change

Simplify InterTree handling for Git trees.

To post a comment you must log in.
Revision history for this message
Jelmer Vernooij (jelmer) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'breezy/git/tests/test_workingtree.py'
--- breezy/git/tests/test_workingtree.py 2020-08-06 01:40:59 +0000
+++ breezy/git/tests/test_workingtree.py 2020-08-08 18:37:30 +0000
@@ -23,8 +23,11 @@
23import stat23import stat
2424
25from dulwich import __version__ as dulwich_version25from dulwich import __version__ as dulwich_version
26from dulwich.diff_tree import RenameDetector26from dulwich.diff_tree import RenameDetector, tree_changes
27from dulwich.index import IndexEntry27from dulwich.index import IndexEntry
28from dulwich.object_store import (
29 OverlayObjectStore,
30 )
28from dulwich.objects import (31from dulwich.objects import (
29 S_IFGITLINK,32 S_IFGITLINK,
30 Blob,33 Blob,
@@ -42,7 +45,6 @@
42 default_mapping,45 default_mapping,
43 )46 )
44from ..tree import (47from ..tree import (
45 changes_between_git_tree_and_working_copy,
46 tree_delta_from_git_changes,48 tree_delta_from_git_changes,
47 )49 )
48from ..workingtree import (50from ..workingtree import (
@@ -54,6 +56,22 @@
54 )56 )
5557
5658
59def changes_between_git_tree_and_working_copy(source_store, from_tree_sha, target,
60 want_unchanged=False,
61 want_unversioned=False,
62 rename_detector=None,
63 include_trees=True):
64 """Determine the changes between a git tree and a working tree with index.
65
66 """
67 to_tree_sha, extras = target.git_snapshot(want_unversioned=want_unversioned)
68 store = OverlayObjectStore([source_store, target.store])
69 return tree_changes(
70 store, from_tree_sha, to_tree_sha, include_trees=include_trees,
71 rename_detector=rename_detector,
72 want_unchanged=want_unchanged, change_type_same=True), extras
73
74
57class GitWorkingTreeTests(TestCaseWithTransport):75class GitWorkingTreeTests(TestCaseWithTransport):
5876
59 def setUp(self):77 def setUp(self):
6078
=== modified file 'breezy/git/tree.py'
--- breezy/git/tree.py 2020-08-07 00:29:47 +0000
+++ breezy/git/tree.py 2020-08-08 18:37:30 +0000
@@ -264,7 +264,24 @@
264 return path264 return path
265265
266266
267class GitRevisionTree(revisiontree.RevisionTree):267class GitTree(_mod_tree.Tree):
268
269 def iter_git_objects(self):
270 """Iterate over all the objects in the tree.
271
272 :return :Yields tuples with (path, sha, mode)
273 """
274 raise NotImplementedError(self.iter_git_objects)
275
276 def git_snapshot(self):
277 """Snapshot a tree, and return tree object.
278
279 :return: Tree sha and set of extras
280 """
281 raise NotImplementedError(self.snapshot)
282
283
284class GitRevisionTree(revisiontree.RevisionTree, GitTree):
268 """Revision tree implementation based on Git objects."""285 """Revision tree implementation based on Git objects."""
269286
270 def __init__(self, repository, revision_id):287 def __init__(self, repository, revision_id):
@@ -286,6 +303,9 @@
286 raise errors.NoSuchRevision(repository, revision_id)303 raise errors.NoSuchRevision(repository, revision_id)
287 self.tree = commit.tree304 self.tree = commit.tree
288305
306 def git_snapshot(self, want_unversioned=False):
307 return self.tree, set()
308
289 def _submodule_info(self):309 def _submodule_info(self):
290 if self._submodules is None:310 if self._submodules is None:
291 try:311 try:
@@ -977,10 +997,18 @@
977 _matching_to_tree_format = None997 _matching_to_tree_format = None
978 _test_mutable_trees_to_test_trees = None998 _test_mutable_trees_to_test_trees = None
979999
1000 def __init__(self, source, target):
1001 super(InterGitTrees, self).__init__(source, target)
1002 if self.source.store == self.target.store:
1003 self.store = self.source.store
1004 else:
1005 self.store = OverlayObjectStore(
1006 [self.source.store, self.target.store])
1007 self.rename_detector = RenameDetector(self.store)
1008
980 @classmethod1009 @classmethod
981 def is_compatible(cls, source, target):1010 def is_compatible(cls, source, target):
982 return (isinstance(source, GitRevisionTree) and1011 return isinstance(source, GitTree) and isinstance(target, GitTree)
983 isinstance(target, GitRevisionTree))
9841012
985 def compare(self, want_unchanged=False, specific_files=None,1013 def compare(self, want_unchanged=False, specific_files=None,
986 extra_trees=None, require_versioned=False, include_root=False,1014 extra_trees=None, require_versioned=False, include_root=False,
@@ -1018,7 +1046,25 @@
1018 def _iter_git_changes(self, want_unchanged=False, specific_files=None,1046 def _iter_git_changes(self, want_unchanged=False, specific_files=None,
1019 require_versioned=False, extra_trees=None,1047 require_versioned=False, extra_trees=None,
1020 want_unversioned=False, include_trees=True):1048 want_unversioned=False, include_trees=True):
1021 raise NotImplementedError(self._iter_git_changes)1049 trees = [self.source]
1050 if extra_trees is not None:
1051 trees.extend(extra_trees)
1052 if specific_files is not None:
1053 specific_files = self.target.find_related_paths_across_trees(
1054 specific_files, trees,
1055 require_versioned=require_versioned)
1056 # TODO(jelmer): Restrict to specific_files, for performance reasons.
1057 with self.lock_read():
1058 from_tree_sha, from_extras = self.source.git_snapshot(
1059 want_unversioned=want_unversioned)
1060 to_tree_sha, to_extras = self.target.git_snapshot(
1061 want_unversioned=want_unversioned)
1062 changes = tree_changes(
1063 self.store, from_tree_sha, to_tree_sha,
1064 include_trees=include_trees,
1065 rename_detector=self.rename_detector,
1066 want_unchanged=want_unchanged, change_type_same=True)
1067 return changes, from_extras, to_extras
10221068
1023 def find_target_path(self, path, recurse='none'):1069 def find_target_path(self, path, recurse='none'):
1024 ret = self.find_target_paths([path], recurse=recurse)1070 ret = self.find_target_paths([path], recurse=recurse)
@@ -1073,48 +1119,10 @@
1073 return ret1119 return ret
10741120
10751121
1076class InterGitRevisionTrees(InterGitTrees):1122_mod_tree.InterTree.register_optimiser(InterGitTrees)
1077 """InterTree that works between two git revision trees."""1123
10781124
1079 _matching_from_tree_format = None1125class MutableGitIndexTree(mutabletree.MutableTree, GitTree):
1080 _matching_to_tree_format = None
1081 _test_mutable_trees_to_test_trees = None
1082
1083 @classmethod
1084 def is_compatible(cls, source, target):
1085 return (isinstance(source, GitRevisionTree) and
1086 isinstance(target, GitRevisionTree))
1087
1088 def _iter_git_changes(self, want_unchanged=False, specific_files=None,
1089 require_versioned=True, extra_trees=None,
1090 want_unversioned=False, include_trees=True):
1091 trees = [self.source]
1092 if extra_trees is not None:
1093 trees.extend(extra_trees)
1094 if specific_files is not None:
1095 specific_files = self.target.find_related_paths_across_trees(
1096 specific_files, trees,
1097 require_versioned=require_versioned)
1098
1099 if (self.source._repository._git.object_store !=
1100 self.target._repository._git.object_store):
1101 store = OverlayObjectStore(
1102 [self.source._repository._git.object_store,
1103 self.target._repository._git.object_store])
1104 else:
1105 store = self.source._repository._git.object_store
1106 rename_detector = RenameDetector(store)
1107 changes = tree_changes(
1108 store, self.source.tree, self.target.tree,
1109 want_unchanged=want_unchanged, include_trees=include_trees,
1110 change_type_same=True, rename_detector=rename_detector)
1111 return changes, set(), set()
1112
1113
1114_mod_tree.InterTree.register_optimiser(InterGitRevisionTrees)
1115
1116
1117class MutableGitIndexTree(mutabletree.MutableTree):
11181126
1119 def __init__(self):1127 def __init__(self):
1120 self._lock_mode = None1128 self._lock_mode = None
@@ -1123,6 +1131,9 @@
1123 self._index_dirty = False1131 self._index_dirty = False
1124 self._submodules = None1132 self._submodules = None
11251133
1134 def git_snapshot(self, want_unversioned=False):
1135 return snapshot_workingtree(self, want_unversioned=want_unversioned)
1136
1126 def is_versioned(self, path):1137 def is_versioned(self, path):
1127 with self.lock_read():1138 with self.lock_read():
1128 path = encode_git_path(path.rstrip('/'))1139 path = encode_git_path(path.rstrip('/'))
@@ -1142,7 +1153,7 @@
1142 if self._lock_mode is None:1153 if self._lock_mode is None:
1143 raise errors.ObjectNotLocked(self)1154 raise errors.ObjectNotLocked(self)
1144 self._versioned_dirs = set()1155 self._versioned_dirs = set()
1145 for p, i in self._recurse_index_entries():1156 for p, sha, mode in self.iter_git_objects():
1146 self._ensure_versioned_dir(posixpath.dirname(p))1157 self._ensure_versioned_dir(posixpath.dirname(p))
11471158
1148 def _ensure_versioned_dir(self, dirname):1159 def _ensure_versioned_dir(self, dirname):
@@ -1301,6 +1312,10 @@
1301 if self._versioned_dirs is not None:1312 if self._versioned_dirs is not None:
1302 self._ensure_versioned_dir(index_path)1313 self._ensure_versioned_dir(index_path)
13031314
1315 def iter_git_objects(self):
1316 for p, entry in self._recurse_index_entries():
1317 yield p, entry.sha, entry.mode
1318
1304 def _recurse_index_entries(self, index=None, basepath=b"",1319 def _recurse_index_entries(self, index=None, basepath=b"",
1305 recurse_nested=False):1320 recurse_nested=False):
1306 # Iterate over all index entries1321 # Iterate over all index entries
@@ -1398,7 +1413,7 @@
1398 if data is None:1413 if data is None:
1399 data = self.branch.repository._git.object_store[sha].data1414 data = self.branch.repository._git.object_store[sha].data
1400 ie.text_sha1 = osutils.sha_string(data)1415 ie.text_sha1 = osutils.sha_string(data)
1401 ie.text_size = len(data)1416 ie.text_size = size
1402 ie.executable = bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode)1417 ie.executable = bool(stat.S_ISREG(mode) and stat.S_IEXEC & mode)
1403 return ie1418 return ie
14041419
@@ -1687,131 +1702,6 @@
1687 return True1702 return True
16881703
16891704
1690class InterToIndexGitTree(InterGitTrees):
1691 """InterTree that works between a Git revision tree and an index."""
1692
1693 def __init__(self, source, target):
1694 super(InterToIndexGitTree, self).__init__(source, target)
1695 if self.source.store == self.target.store:
1696 self.store = self.source.store
1697 else:
1698 self.store = OverlayObjectStore(
1699 [self.source.store, self.target.store])
1700 self.rename_detector = RenameDetector(self.store)
1701
1702 @classmethod
1703 def is_compatible(cls, source, target):
1704 return (isinstance(source, GitRevisionTree) and
1705 isinstance(target, MutableGitIndexTree))
1706
1707 def _iter_git_changes(self, want_unchanged=False, specific_files=None,
1708 require_versioned=False, extra_trees=None,
1709 want_unversioned=False, include_trees=True):
1710 trees = [self.source]
1711 if extra_trees is not None:
1712 trees.extend(extra_trees)
1713 if specific_files is not None:
1714 specific_files = self.target.find_related_paths_across_trees(
1715 specific_files, trees,
1716 require_versioned=require_versioned)
1717 # TODO(jelmer): Restrict to specific_files, for performance reasons.
1718 with self.lock_read():
1719 changes, target_extras = changes_between_git_tree_and_working_copy(
1720 self.source.store, self.source.tree,
1721 self.target, want_unchanged=want_unchanged,
1722 want_unversioned=want_unversioned,
1723 rename_detector=self.rename_detector,
1724 include_trees=include_trees)
1725 return changes, set(), target_extras
1726
1727
1728_mod_tree.InterTree.register_optimiser(InterToIndexGitTree)
1729
1730
1731class InterFromIndexGitTree(InterGitTrees):
1732 """InterTree that works between a Git revision tree and an index."""
1733
1734 def __init__(self, source, target):
1735 super(InterFromIndexGitTree, self).__init__(source, target)
1736 if self.source.store == self.target.store:
1737 self.store = self.source.store
1738 else:
1739 self.store = OverlayObjectStore(
1740 [self.source.store, self.target.store])
1741 self.rename_detector = RenameDetector(self.store)
1742
1743 @classmethod
1744 def is_compatible(cls, source, target):
1745 return (isinstance(target, GitRevisionTree) and
1746 isinstance(source, MutableGitIndexTree))
1747
1748 def _iter_git_changes(self, want_unchanged=False, specific_files=None,
1749 require_versioned=False, extra_trees=None,
1750 want_unversioned=False, include_trees=True):
1751 trees = [self.source]
1752 if extra_trees is not None:
1753 trees.extend(extra_trees)
1754 if specific_files is not None:
1755 specific_files = self.target.find_related_paths_across_trees(
1756 specific_files, trees,
1757 require_versioned=require_versioned)
1758 # TODO(jelmer): Restrict to specific_files, for performance reasons.
1759 with self.lock_read():
1760 from_tree_sha, extras = snapshot_workingtree(self.source, want_unversioned=want_unversioned)
1761 return tree_changes(
1762 self.store, from_tree_sha, self.target.tree,
1763 include_trees=include_trees,
1764 rename_detector=self.rename_detector,
1765 want_unchanged=want_unchanged, change_type_same=True), extras
1766
1767
1768_mod_tree.InterTree.register_optimiser(InterFromIndexGitTree)
1769
1770
1771class InterIndexGitTree(InterGitTrees):
1772 """InterTree that works between a Git revision tree and an index."""
1773
1774 def __init__(self, source, target):
1775 super(InterIndexGitTree, self).__init__(source, target)
1776 if self.source.store == self.target.store:
1777 self.store = self.source.store
1778 else:
1779 self.store = OverlayObjectStore(
1780 [self.source.store, self.target.store])
1781 self.rename_detector = RenameDetector(self.store)
1782
1783 @classmethod
1784 def is_compatible(cls, source, target):
1785 return (isinstance(target, MutableGitIndexTree) and
1786 isinstance(source, MutableGitIndexTree))
1787
1788 def _iter_git_changes(self, want_unchanged=False, specific_files=None,
1789 require_versioned=False, extra_trees=None,
1790 want_unversioned=False, include_trees=True):
1791 trees = [self.source]
1792 if extra_trees is not None:
1793 trees.extend(extra_trees)
1794 if specific_files is not None:
1795 specific_files = self.target.find_related_paths_across_trees(
1796 specific_files, trees,
1797 require_versioned=require_versioned)
1798 # TODO(jelmer): Restrict to specific_files, for performance reasons.
1799 with self.lock_read():
1800 from_tree_sha, from_extras = snapshot_workingtree(
1801 self.source, want_unversioned=want_unversioned)
1802 to_tree_sha, to_extras = snapshot_workingtree(
1803 self.target, want_unversioned=want_unversioned)
1804 changes = tree_changes(
1805 self.store, from_tree_sha, to_tree_sha,
1806 include_trees=include_trees,
1807 rename_detector=self.rename_detector,
1808 want_unchanged=want_unchanged, change_type_same=True)
1809 return changes, from_extras, to_extras
1810
1811
1812_mod_tree.InterTree.register_optimiser(InterIndexGitTree)
1813
1814
1815def snapshot_workingtree(target, want_unversioned=False):1705def snapshot_workingtree(target, want_unversioned=False):
1816 extras = set()1706 extras = set()
1817 blobs = {}1707 blobs = {}
@@ -1859,21 +1749,21 @@
1859 target.store.add_object(blob)1749 target.store.add_object(blob)
1860 blobs[path] = (live_entry.sha, cleanup_mode(live_entry.mode))1750 blobs[path] = (live_entry.sha, cleanup_mode(live_entry.mode))
1861 if want_unversioned:1751 if want_unversioned:
1862 for e in target._iter_files_recursive(include_dirs=False):1752 for extra in target._iter_files_recursive(include_dirs=False):
1863 try:1753 try:
1864 e, accessible = osutils.normalized_filename(e)1754 extra, accessible = osutils.normalized_filename(extra)
1865 except UnicodeDecodeError:1755 except UnicodeDecodeError:
1866 raise errors.BadFilenameEncoding(1756 raise errors.BadFilenameEncoding(
1867 e, osutils._fs_enc)1757 extra, osutils._fs_enc)
1868 np = encode_git_path(e)1758 np = encode_git_path(extra)
1869 if np in blobs:1759 if np in blobs:
1870 continue1760 continue
1871 st = target._lstat(e)1761 st = target._lstat(extra)
1872 if stat.S_ISDIR(st.st_mode):1762 if stat.S_ISDIR(st.st_mode):
1873 blob = Tree()1763 blob = Tree()
1874 elif stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode):1764 elif stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode):
1875 blob = blob_from_path_and_stat(1765 blob = blob_from_path_and_stat(
1876 target.abspath(e).encode(osutils._fs_enc), st)1766 target.abspath(extra).encode(osutils._fs_enc), st)
1877 else:1767 else:
1878 continue1768 continue
1879 target.store.add_object(blob)1769 target.store.add_object(blob)
@@ -1881,19 +1771,3 @@
1881 extras.add(np)1771 extras.add(np)
1882 return commit_tree(1772 return commit_tree(
1883 target.store, dirified + [(p, s, m) for (p, (s, m)) in blobs.items()]), extras1773 target.store, dirified + [(p, s, m) for (p, (s, m)) in blobs.items()]), extras
1884
1885
1886def changes_between_git_tree_and_working_copy(source_store, from_tree_sha, target,
1887 want_unchanged=False,
1888 want_unversioned=False,
1889 rename_detector=None,
1890 include_trees=True):
1891 """Determine the changes between a git tree and a working tree with index.
1892
1893 """
1894 to_tree_sha, extras = snapshot_workingtree(target, want_unversioned=want_unversioned)
1895 store = OverlayObjectStore([source_store, target.store])
1896 return tree_changes(
1897 store, from_tree_sha, to_tree_sha, include_trees=include_trees,
1898 rename_detector=rename_detector,
1899 want_unchanged=want_unchanged, change_type_same=True), extras
19001774
=== modified file 'breezy/git/workingtree.py'
--- breezy/git/workingtree.py 2020-08-06 22:01:46 +0000
+++ breezy/git/workingtree.py 2020-08-08 18:37:30 +0000
@@ -572,7 +572,7 @@
572 """572 """
573 with self.lock_read():573 with self.lock_read():
574 index_paths = set(574 index_paths = set(
575 [decode_git_path(p) for p, i in self._recurse_index_entries()])575 [decode_git_path(p) for p, sha, mode in self.iter_git_objects()])
576 all_paths = set(self._iter_files_recursive(include_dirs=False))576 all_paths = set(self._iter_files_recursive(include_dirs=False))
577 return iter(all_paths - index_paths)577 return iter(all_paths - index_paths)
578578

Subscribers

People subscribed via source and target branches