Merge lp:~jelmer/brz/git-submodule into lp:brz

[Jelmer Vernooij]

Fix cloning of git repositories with submodules.

Fix cloning of git repositories with submodules.

[Jelmer Vernooij (jelmer)]

review: Approve

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/613/

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/614/

Unmerged revisions

7454. By Jelmer Vernooij on 2020-01-17

Use strip_segment_parameters.

7453. By Jelmer Vernooij on 2020-01-17

Ignore missing nested trees.

7452. By Jelmer Vernooij on 2020-01-17

Fix typo.

7451. By Jelmer Vernooij on 2020-01-17

Import from gitmodules.

7450. By Jelmer Vernooij on 2020-01-17

Use find_previous_path rather than id2path.

7449. By Jelmer Vernooij on 2020-01-17

Fix tree reference listing.

7448. By Jelmer Vernooij on 2020-01-17

Move reference info to Git.

7447. By Jelmer Vernooij on 2020-01-17

Fix cloning of git repositories with submodules.

=== modified file 'breezy/branch.py'
--- breezy/branch.py	2019-10-19 23:18:56 +0000
+++ breezy/branch.py	2020-01-17 02:30:07 +0000
@@ -51,7 +51,7 @@
     text_type,
     viewitems,
     )
-from .trace import mutter, mutter_callsite, note, is_quiet
+from .trace import mutter, mutter_callsite, note, is_quiet, warning
 
 
 class UnstackableBranchFormat(errors.BzrError):
@@ -676,14 +676,6 @@
             raise errors.UpgradeRequired(self.user_url)
         self.get_config_stack().set('append_revisions_only', enabled)
 
-    def set_reference_info(self, tree_path, branch_location, file_id=None):
-        """Set the branch location to use for a tree reference."""
-        raise errors.UnsupportedOperation(self.set_reference_info, self)
-
-    def get_reference_info(self, path):
-        """Get the tree_path and branch_location for a tree reference."""
-        raise errors.UnsupportedOperation(self.get_reference_info, self)
-
     def fetch(self, from_branch, last_revision=None, limit=None):
         """Copy revisions from from_branch into this branch.
 
@@ -1412,7 +1404,10 @@
         basis_tree = tree.basis_tree()
         with basis_tree.lock_read():
             for path in basis_tree.iter_references():
-                reference_parent = self.reference_parent(path)
+                reference_parent = tree.reference_parent(path)
+                if reference_parent is None:
+                    warning('Branch location for %s unknown.', path)
+                    continue
                 reference_parent.create_checkout(
                     tree.abspath(path),
                     basis_tree.get_reference_revision(path), lightweight)
@@ -1432,8 +1427,11 @@
         :return: A branch associated with the nested tree
         """
         # FIXME should provide multiple branches, based on config
-        return Branch.open(self.controldir.root_transport.clone(path).base,
-                           possible_transports=possible_transports)
+        try:
+            return Branch.open(self.controldir.root_transport.clone(path).base,
+                               possible_transports=possible_transports)
+        except errors.NotBranchError:
+            return None
 
     def supports_tags(self):
         return self._format.supports_tags()

=== modified file 'breezy/builtins.py'
--- breezy/builtins.py	2020-01-12 14:46:56 +0000
+++ breezy/builtins.py	2020-01-17 02:30:07 +0000
@@ -6738,21 +6738,22 @@
         if tree is None:
             tree = branch.basis_tree()
         if path is None:
-            info = viewitems(branch._get_all_reference_info())
+            info = [
+                (path, tree.get_reference_info(path))
+                for path in tree.iter_references()]
             self._display_reference_info(tree, branch, info)
         else:
             if not tree.is_versioned(path) and not force_unversioned:
                 raise errors.NotVersionedError(path)
             if location is None:
-                info = [(path, branch.get_reference_info(path))]
+                info = [(path, tree.get_reference_info(path))]
                 self._display_reference_info(tree, branch, info)
             else:
-                branch.set_reference_info(
-                    path, location, file_id=tree.path2id(path))
+                tree.set_reference_info(path, location)
 
     def _display_reference_info(self, tree, branch, info):
         ref_list = []
-        for path, (location, file_id) in info:
+        for path, location in info:
             ref_list.append((path, location))
         for path, location in sorted(ref_list):
             self.outf.write('%s %s\n' % (path, location))

=== modified file 'breezy/bzr/branch.py'
--- breezy/bzr/branch.py	2020-01-12 13:56:10 +0000
+++ breezy/bzr/branch.py	2020-01-17 02:30:07 +0000
@@ -450,6 +450,14 @@
             reconciler = BranchReconciler(self, thorough=thorough)
             return reconciler.reconcile()
 
+    def set_reference_info(self, tree_path, branch_location, file_id=None):
+        """Set the branch location to use for a tree reference."""
+        raise errors.UnsupportedOperation(self.set_reference_info, self)
+
+    def get_reference_info(self, path):
+        """Get the tree_path and branch_location for a tree reference."""
+        raise errors.UnsupportedOperation(self.get_reference_info, self)
+
 
 class BzrBranch8(BzrBranch):
     """A branch that stores tree-reference locations."""

=== modified file 'breezy/bzr/bzrdir.py'
--- breezy/bzr/bzrdir.py	2019-10-20 00:11:04 +0000
+++ breezy/bzr/bzrdir.py	2020-01-17 02:30:07 +0000
@@ -65,6 +65,7 @@
 from ..trace import (
     mutter,
     note,
+    warning,
     )
 
 from .. import (
@@ -467,8 +468,13 @@
                     subtrees = []
                 for path in subtrees:
                     target = urlutils.join(url, urlutils.escape(path))
-                    sublocation = source_branch.reference_parent(
+                    sublocation = wt.reference_parent(
                         path, possible_transports=possible_transports)
+                    if sublocation is None:
+                        warning(
+                            'Ignoring nested tree %s, parent location unknown.',
+                            path)
+                        continue
                     sublocation.controldir.sprout(
                         target, basis.get_reference_revision(path),
                         force_new_repo=force_new_repo, recurse=recurse,

=== modified file 'breezy/bzr/workingtree.py'
--- breezy/bzr/workingtree.py	2020-01-11 02:39:17 +0000
+++ breezy/bzr/workingtree.py	2020-01-17 02:30:07 +0000
@@ -1796,6 +1796,13 @@
                 else:
                     yield get_canonical_path(self, path, normalize)
 
+    def get_reference_info(self, path):
+        return self.branch.get_reference_info(path)[0]
+
+    def set_reference_info(self, tree_path, branch_location):
+        self.branch.set_reference_info(
+            tree_path, branch_location, file_id=self.path2id(tree_path))
+
 
 class WorkingTreeFormatMetaDir(bzrdir.BzrFormat, WorkingTreeFormat):
     """Base class for working trees that live in bzr meta directories."""

=== modified file 'breezy/bzr/workingtree_4.py'
--- breezy/bzr/workingtree_4.py	2020-01-11 22:33:18 +0000
+++ breezy/bzr/workingtree_4.py	2020-01-17 02:30:07 +0000
@@ -130,6 +130,9 @@
         self.views = self._make_views()
         # --- allow tests to select the dirstate iter_changes implementation
         self._iter_changes = dirstate._process_entry
+        self._repo_supports_tree_reference = getattr(
+            self._branch.repository._format, "supports_tree_reference",
+            False)
 
     def _add(self, files, ids, kinds):
         """See MutableTree._add."""
@@ -550,20 +553,21 @@
             # When the repo doesn't support references, we will have nothing to
             # return
             return
-        for key, tree_details in self.current_dirstate()._iter_entries():
-            if tree_details[0][0] in (b'a', b'r'):  # absent, relocated
-                # not relevant to the working tree
-                continue
-            if not key[1]:
-                # the root is not a reference.
-                continue
-            relpath = pathjoin(key[0].decode('utf8'), key[1].decode('utf8'))
-            try:
-                if self.kind(relpath) == 'tree-reference':
-                    yield relpath
-            except errors.NoSuchFile:
-                # path is missing on disk.
-                continue
+        with self.lock_read():
+            for key, tree_details in self.current_dirstate()._iter_entries():
+                if tree_details[0][0] in (b'a', b'r'):  # absent, relocated
+                    # not relevant to the working tree
+                    continue
+                if not key[1]:
+                    # the root is not a reference.
+                    continue
+                relpath = pathjoin(key[0].decode('utf8'), key[1].decode('utf8'))
+                try:
+                    if self.kind(relpath) == 'tree-reference':
+                        yield relpath
+                except errors.NoSuchFile:
+                    # path is missing on disk.
+                    continue
 
     def _observed_sha1(self, path, sha_and_stat):
         """See MutableTree._observed_sha1."""

=== modified file 'breezy/git/branch.py'
--- breezy/git/branch.py	2020-01-16 23:24:40 +0000
+++ breezy/git/branch.py	2020-01-17 02:30:07 +0000
@@ -22,6 +22,10 @@
 from io import BytesIO
 from collections import defaultdict
 
+from dulwich.config import (
+    ConfigFile as GitConfigFile,
+    parse_submodules,
+    )
 from dulwich.objects import (
     NotCommitError,
     ZERO_SHA,
@@ -980,6 +984,15 @@
             other_branch=self.source)
         return head, refs
 
+    def _import_tree_references(self, revid):
+        tree = self.target.repository.revision_tree(revid)
+        with tree.get_file('.gitmodules') as f:
+            for path, url, section in parse_submodules(
+                    GitConfigFile.from_file(f)):
+                self.target.set_reference_info(
+                    path.decode('utf-8'), url.decode('utf-8'),
+                    tree.path2id(path.decode('utf-8')))
+
     def _basic_pull(self, stop_revision, overwrite, run_hooks,
                     _override_hook_target, _hook_master):
         if overwrite is True:
@@ -1007,6 +1020,7 @@
                 result.tag_conflicts = tags_ret
             (result.new_revno, result.new_revid) = \
                 self.target.last_revision_info()
+            self._import_tree_references(result.new_revid)
             if _hook_master:
                 result.master_branch = _hook_master
                 result.local_branch = result.target_branch
@@ -1075,6 +1089,7 @@
             self.target.tags, "tags" in overwrite, ignore_master=True)
         (result.tag_updates, result.tag_conflicts) = tags_ret
         result.new_revno, result.new_revid = self.target.last_revision_info()
+        self._import_tree_references(result.new_revid)
         return result
 
 

=== modified file 'breezy/git/dir.py'
--- breezy/git/dir.py	2020-01-12 14:46:56 +0000
+++ breezy/git/dir.py	2020-01-17 02:30:07 +0000
@@ -207,8 +207,13 @@
                     subtrees = []
                 for path in subtrees:
                     target = urlutils.join(url, urlutils.escape(path))
-                    sublocation = source_branch.reference_parent(
+                    sublocation = wt.reference_parent(
                         path, possible_transports=possible_transports)
+                    if sublocation is None:
+                        trace.warning(
+                            'Ignoring nested tree %s, parent location unknown.',
+                            path)
+                        continue
                     sublocation.controldir.sprout(
                         target, basis.get_reference_revision(path),
                         force_new_repo=force_new_repo, recurse=recurse,

=== modified file 'breezy/git/fetch.py'
--- breezy/git/fetch.py	2019-10-28 01:38:39 +0000
+++ breezy/git/fetch.py	2020-01-17 02:30:07 +0000
@@ -55,6 +55,7 @@
 from ..bzr.testament import (
     StrictTestament3,
     )
+from ..tree import find_previous_path
 from ..tsort import (
     topo_sort,
     )
@@ -125,9 +126,8 @@
     # Check what revision we should store
     parent_keys = []
     for ptree in parent_bzr_trees:
-        try:
-            ppath = ptree.id2path(file_id)
-        except errors.NoSuchId:
+        ppath = find_previous_path(base_bzr_tree, ptree, decoded_path, file_id)
+        if ppath is None:
             continue
         pkind = ptree.kind(ppath)
         if (pkind == ie.kind and

=== modified file 'breezy/git/workingtree.py'
--- breezy/git/workingtree.py	2020-01-10 01:37:30 +0000
+++ breezy/git/workingtree.py	2020-01-17 02:30:07 +0000
@@ -25,6 +25,7 @@
 from dulwich.ignore import (
     IgnoreFilterManager,
     )
+from dulwich.config import ConfigFile as GitConfigFile
 from dulwich.file import GitFile, FileLocked
 from dulwich.index import (
     Index,
@@ -49,6 +50,7 @@
 import sys
 
 from .. import (
+    branch as _mod_branch,
     conflicts as _mod_conflicts,
     errors,
     controldir as _mod_controldir,
@@ -1325,6 +1327,37 @@
                     new_parents = [revision_id]
                 tree.set_parent_ids(new_parents)
 
+    def reference_parent(self, path, possible_transports=None):
+        remote_url = self.get_reference_info(path)
+        if remote_url is None:
+            trace.warning("Unable to find submodule info for %s", path)
+            return None
+        return _mod_branch.Branch.open(remote_url, possible_transports=possible_transports)
+
+    def get_reference_info(self, path):
+        submodule_info = self._submodule_info()
+        info = submodule_info.get(path.encode('utf-8'))
+        if info is None:
+            return None
+        return info[0].decode('utf-8')
+
+    def set_reference_info(self, tree_path, branch_location):
+        path = self.abspath('.gitmodules')
+        try:
+            config = GitConfigFile.from_path(path)
+        except EnvironmentError as e:
+            if e.errno == errno.ENOENT:
+                config = GitConfigFile()
+            else:
+                raise
+        config.set(
+            (b'submodule', tree_path.encode('utf-8')),
+            b'path', tree_path.encode('utf-8'))
+        config.set(
+            (b'submodule', tree_path.encode('utf-8')),
+            b'url', branch_location.encode('utf-8'))
+        config.write_to_path(path)
+
 
 class GitWorkingTreeFormat(workingtree.WorkingTreeFormat):
 

=== modified file 'breezy/merge.py'
--- breezy/merge.py	2019-10-13 17:31:55 +0000
+++ breezy/merge.py	2020-01-17 02:30:07 +0000
@@ -641,7 +641,7 @@
                     continue
                 sub_merge = Merger(sub_tree.branch, this_tree=sub_tree)
                 sub_merge.merge_type = self.merge_type
-                other_branch = self.other_branch.reference_parent(relpath)
+                other_branch = self.other_tree.reference_parent(relpath)
                 sub_merge.set_other_revision(other_revision, other_branch)
                 base_tree_path = _mod_tree.find_previous_path(
                     self.this_tree, self.base_tree, relpath)

=== modified file 'breezy/tests/blackbox/test_branch.py'
--- breezy/tests/blackbox/test_branch.py	2019-10-19 23:18:56 +0000
+++ breezy/tests/blackbox/test_branch.py	2020-01-17 02:30:07 +0000
@@ -367,6 +367,7 @@
         subtree.add(['a'])
         subtree.commit('add subtree contents')
         orig.add_reference(subtree)
+        orig.branch.set_reference_info('subtree', subtree.branch.user_url)
         orig.commit('add subtree')
 
         self.run_bzr('branch source target')
@@ -376,6 +377,20 @@
         self.assertTreesEqual(orig, target)
         self.assertTreesEqual(subtree, target_subtree)
 
+    def test_branch_with_nested_trees_reference_unset(self):
+        orig = self.make_branch_and_tree('source', format='development-subtree')
+        subtree = self.make_branch_and_tree('source/subtree')
+        self.build_tree(['source/subtree/a'])
+        subtree.add(['a'])
+        subtree.commit('add subtree contents')
+        orig.add_reference(subtree)
+        orig.commit('add subtree')
+
+        self.run_bzr('branch source target')
+
+        target = WorkingTree.open('target')
+        self.assertRaises(errors.NotBranchError, WorkingTree.open, 'target/subtree')
+
     def test_branch_with_nested_trees_no_recurse(self):
         orig = self.make_branch_and_tree('source', format='development-subtree')
         subtree = self.make_branch_and_tree('source/subtree')

=== modified file 'breezy/tests/blackbox/test_reference.py'
--- breezy/tests/blackbox/test_reference.py	2018-11-16 11:37:47 +0000
+++ breezy/tests/blackbox/test_reference.py	2020-01-17 02:30:07 +0000
@@ -76,6 +76,8 @@
         tree.add('file', b'file-id')
         out, err = self.run_bzr('reference tree/file http://example.org')
         location, file_id = tree.branch.get_reference_info('file')
+        tree_location = tree.get_reference_info('file')
+        self.assertEqual('http://example.org', tree_location)
         self.assertEqual('http://example.org', location)
         self.assertEqual(b'file-id', file_id)
         self.assertEqual('', out)

=== modified file 'breezy/tests/per_branch/test_branch.py'
--- breezy/tests/per_branch/test_branch.py	2020-01-12 15:16:27 +0000
+++ breezy/tests/per_branch/test_branch.py	2020-01-17 02:30:07 +0000
@@ -858,8 +858,8 @@
             raise tests.TestNotApplicable('Tree cannot hold references.')
         reference_parent = tree.branch.reference_parent(
             urlutils.relative_url(
-                urlutils.split_segment_parameters(tree.branch.user_url)[0],
-                urlutils.split_segment_parameters(subtree.branch.user_url)[0]))
+                urlutils.strip_segment_parameters(tree.branch.user_url),
+                urlutils.strip_segment_parameters(subtree.branch.user_url)))
         self.assertEqual(subtree.branch.user_url, reference_parent.user_url)
 
     def test_reference_parent_accepts_possible_transports(self):
@@ -872,8 +872,8 @@
             raise tests.TestNotApplicable('Tree cannot hold references.')
         reference_parent = tree.branch.reference_parent(
             urlutils.relative_url(
-                urlutils.split_segment_parameters(tree.branch.user_url)[0],
-                urlutils.split_segment_parameters(subtree.branch.user_url)[0]),
+                urlutils.strip_segment_parameters(tree.branch.user_url),
+                urlutils.strip_segment_parameters(subtree.branch.user_url)),
             possible_transports=[subtree.controldir.root_transport])
 
     def test_get_reference_info(self):
@@ -1014,8 +1014,7 @@
             'branch/new-branch').open_branch()
         new_branch.set_reference_info('../foo', '../foo', b'file-id2')
         new_branch.push(branch)
-        self.assertEqual('foo',
-                         branch.get_reference_info('../foo')[0])
+        self.assertEqual('foo', branch.get_reference_info('../foo')[0])
 
     def test_merge_updates_references(self):
         branch = self.make_branch_with_reference('branch', 'reference')

=== modified file 'breezy/workingtree.py'
--- breezy/workingtree.py	2020-01-11 17:50:28 +0000
+++ breezy/workingtree.py	2020-01-17 02:30:07 +0000
@@ -1325,6 +1325,16 @@
         with self.lock_read():
             return next(self.get_canonical_paths([path]))
 
+    def reference_parent(self, path, possible_transports=None):
+        return self.branch.reference_parent(
+            path, possible_transports=possible_transports)
+
+    def get_reference_info(self, path):
+        raise errors.UnsupportedOperation(self.get_reference_info, self)
+
+    def set_reference_info(self, tree_path, branch_location):
+        raise errors.UnsupportedOperation(self.set_reference_info, self)
+
 
 class WorkingTreeFormatRegistry(ControlComponentFormatRegistry):
     """Registry for working tree formats."""