Merge lp:~jelmer/brz/branch-reference-remote into lp:brz

[Jelmer Vernooij]

Support fetching / pushing reference information to remote branches.

Support fetching / pushing reference information to remote branches.

[Jelmer Vernooij (jelmer)]

review: Approve

[The Breezy Bot (the-breezy-bot)]

Running landing tests failed
https://ci.breezy-vcs.org/job/brz/job/brz-land/621/

=== modified file 'breezy/branch.py'
--- breezy/branch.py	2020-01-19 03:22:04 +0000
+++ breezy/branch.py	2020-01-19 15:24:24 +0000
@@ -1271,7 +1271,7 @@
             revision_id=revision_id)
 
     def update_references(self, target):
-        if not getattr(self._format, 'supports_reference_locations', False):
+        if not self._format.supports_reference_locations:
             return
         return InterBranch.get(self, target).update_references()
 
@@ -2391,8 +2391,12 @@
         new_base = self.target.base
         target_reference_dict = self.target._get_all_reference_info()
         for tree_path, (branch_location, file_id) in viewitems(reference_dict):
-            branch_location = urlutils.rebase_url(branch_location,
-                                                  old_base, new_base)
+            try:
+                branch_location = urlutils.rebase_url(branch_location,
+                                                      old_base, new_base)
+            except urlutils.InvalidRebaseURLs:
+                # Fall back to absolute URL
+                branch_location = urlutils.join(old_base, branch_location)
             target_reference_dict.setdefault(
                 tree_path, (branch_location, file_id))
         self.target._set_all_reference_info(target_reference_dict)
 
=== modified file 'breezy/bzr/branch.py'
--- breezy/bzr/branch.py	2020-01-19 03:22:04 +0000
+++ breezy/bzr/branch.py	2020-01-19 15:24:24 +0000
@@ -465,11 +465,20 @@
         :return: A branch associated with the nested tree
         """
         try:
-            return Branch.open_from_transport(
-                self.controldir.root_transport.clone(path),
-                possible_transports=possible_transports)
-        except errors.NotBranchError:
-            return None
+            branch_location = self.get_reference_info(file_id)[0]
+        except errors.UnsupportedOperation:
+            branch_location = None
+        if branch_location is None:
+            try:
+                return Branch.open_from_transport(
+                    self.controldir.root_transport.clone(path),
+                    possible_transports=possible_transports)
+            except errors.NotBranchError:
+                return None
+        return Branch.open(
+            urlutils.join(
+                urlutils.strip_segment_parameters(self.user_url), branch_location),
+            possible_transports=possible_transports)
 
 
 class BzrBranch8(BzrBranch):
@@ -565,7 +574,7 @@
                 with self._transport.get('references') as rio_file:
                     stanzas = rio.read_stanzas(rio_file)
                     info_dict = {
-                        s['file_id'].encode('ascii'): (
+                        s['file_id'].encode('utf-8'): (
                             s['branch_location'],
                             s['tree_path'] if 'tree_path' in s else None)
                         for s in stanzas}
@@ -595,25 +604,6 @@
         """
         return self._get_all_reference_info().get(file_id, (None, None))
 
-    def reference_parent(self, file_id, path, possible_transports=None):
-        """Return the parent branch for a tree-reference.
-
-        :param path: The path of the nested tree in the tree
-        :return: A branch associated with the nested tree
-        """
-        branch_location = self.get_reference_info(file_id)[0]
-        if branch_location is None:
-            try:
-                return Branch.open_from_transport(
-                    self.controldir.root_transport.clone(path),
-                    possible_transports=possible_transports)
-            except errors.NotBranchError:
-                return None
-        else:
-            branch_location = urlutils.join(self.user_url, branch_location)
-            return Branch.open(
-                branch_location, possible_transports=possible_transports)
-
     def set_push_location(self, location):
         """See Branch.set_push_location."""
         self._set_config_location('push_location', location)
 
=== modified file 'breezy/bzr/fullhistory.py'
--- breezy/bzr/fullhistory.py	2018-11-11 04:08:32 +0000
+++ breezy/bzr/fullhistory.py	2020-01-19 15:24:24 +0000
@@ -173,3 +173,5 @@
 
     def supports_tags(self):
         return False
+
+    supports_reference_locations = False
 
=== modified file 'breezy/bzr/remote.py'
--- breezy/bzr/remote.py	2020-01-18 22:02:05 +0000
+++ breezy/bzr/remote.py	2020-01-19 15:24:24 +0000
@@ -3397,6 +3397,11 @@
         self._ensure_real()
         return self._custom_format.supports_set_append_revisions_only()
 
+    @property
+    def supports_reference_locations(self):
+        self._ensure_real()
+        return self._custom_format.supports_reference_locations
+
     def stores_revno(self):
         return True
 
@@ -3415,8 +3420,6 @@
                 return True
         return False
 
-    supports_reference_locations = False
-
 
 class RemoteBranchStore(_mod_config.IniFileStore):
     """Branch store which attempts to use HPSS calls to retrieve branch store.
@@ -4179,28 +4182,60 @@
             reconciler = BranchReconciler(self, thorough=thorough)
             return reconciler.reconcile()
 
-    def set_reference_info(self, tree_path, branch_location, file_id=None):
-        raise errors.UnsupportedOperation(self.set_reference_info, self)
-
-    def get_reference_info(self, tree_path):
-        raise errors.UnsupportedOperation(self.get_reference_info, self)
+    def get_reference_info(self, file_id):
+        """Get the tree_path and branch_location for a tree reference."""
+        if not self._format.supports_reference_locations:
+            raise errors.UnsupportedOperation(self.get_reference_info, self)
+        return self._get_all_reference_info().get(file_id, (None, None))
+
+    def set_reference_info(self, file_id, branch_location, tree_path=None):
+        """Set the branch location to use for a tree reference."""
+        if not self._format.supports_reference_locations:
+            raise errors.UnsupportedOperation(self.set_reference_info, self)
+        self._ensure_real()
+        self._real_branch.set_reference_info(
+            file_id, branch_location, tree_path)
+
+    def _set_all_reference_info(self, reference_info):
+        if not self._format.supports_reference_locations:
+            raise errors.UnsupportedOperation(self.set_reference_info, self)
+        self._ensure_real()
+        self._real_branch._set_all_reference_info(reference_info)
 
     def _get_all_reference_info(self):
-        self._ensure_real()
-        return self._real_branch._get_all_reference_info()
+        if not self._format.supports_reference_locations:
+            return {}
+        try:
+            response, handler = self._call_expecting_body(
+                b'Branch.get_all_reference_info', self._remote_path())
+        except errors.UnknownSmartMethod:
+            self._ensure_real()
+            return self._real_branch._get_all_reference_info()
+        if len(response) and response[0] != b'ok':
+            raise errors.UnexpectedSmartServerResponse(response)
+        ret = {}
+        for (f, u, p) in bencode.bdecode(handler.read_body_bytes()):
+            ret[f] = (u.decode('utf-8'), p.decode('utf-8') if p else None)
+        return ret
 
-    def reference_parent(self, path, possible_transports=None):
+    def reference_parent(self, file_id, path, possible_transports=None):
         """Return the parent branch for a tree-reference.
 
         :param path: The path of the nested tree in the tree
         :return: A branch associated with the nested tree
         """
-        branch_location = self.get_reference_info(path)[0]
+        branch_location = self.get_reference_info(file_id)[0]
         if branch_location is None:
-            return BzrBranch.reference_parent(self, path, possible_transports)
-        branch_location = urlutils.join(self.user_url, branch_location)
-        return Branch.open(branch_location,
-                           possible_transports=possible_transports)
+            try:
+                return branch.Branch.open_from_transport(
+                    self.controldir.root_transport.clone(path),
+                    possible_transports=possible_transports)
+            except errors.NotBranchError:
+                return None
+        return branch.Branch.open(
+            urlutils.join(
+                urlutils.strip_segment_parameters(self.user_url), branch_location),
+            possible_transports=possible_transports)
 
 
 class RemoteConfig(object):
 
=== modified file 'breezy/bzr/smart/branch.py'
--- breezy/bzr/smart/branch.py	2019-03-06 21:27:22 +0000
+++ breezy/bzr/smart/branch.py	2020-01-19 15:24:24 +0000
@@ -439,3 +439,17 @@
             return SuccessfulSmartServerResponse((b'yes',))
         else:
             return SuccessfulSmartServerResponse((b'no',))
+
+
+class SmartServerBranchRequestGetAllReferenceInfo(SmartServerBranchRequest):
+    """Get the reference information.
+
+    New in 3.1.
+    """
+
+    def do_with_branch(self, branch):
+        all_reference_info = branch._get_all_reference_info()
+        content = bencode.bencode([
+            (key, value[0].encode('utf-8'), value[1].encode('utf-8') if value[1] else b'')
+            for (key, value) in all_reference_info.items()])
+        return SuccessfulSmartServerResponse((b'ok', ), content)
 
=== modified file 'breezy/bzr/smart/request.py'
--- breezy/bzr/smart/request.py	2019-06-30 10:50:40 +0000
+++ breezy/bzr/smart/request.py	2020-01-19 15:24:24 +0000
@@ -609,6 +609,9 @@
     b'Branch.revision_id_to_revno', 'breezy.bzr.smart.branch',
     'SmartServerBranchRequestRevisionIdToRevno', info='read')
 request_handlers.register_lazy(
+    b'Branch.get_all_reference_info', 'breezy.bzr.smart.branch',
+    'SmartServerBranchRequestGetAllReferenceInfo', info='read')
+request_handlers.register_lazy(
     b'BzrDir.checkout_metadir', 'breezy.bzr.smart.bzrdir',
     'SmartServerBzrDirRequestCheckoutMetaDir', info='read')
 request_handlers.register_lazy(
 
=== modified file 'breezy/git/branch.py'
--- breezy/git/branch.py	2020-01-18 16:15:37 +0000
+++ breezy/git/branch.py	2020-01-19 15:24:24 +0000
@@ -348,6 +348,8 @@
         """True if this branch format store revision numbers."""
         return False
 
+    supports_reference_locations = False
+
 
 class LocalGitBranchFormat(GitBranchFormat):
 
 
=== modified file 'breezy/plugins/weave_fmt/branch.py'
--- breezy/plugins/weave_fmt/branch.py	2018-11-11 04:08:32 +0000
+++ breezy/plugins/weave_fmt/branch.py	2020-01-19 15:24:24 +0000
@@ -215,3 +215,5 @@
 
     def supports_leaving_lock(self):
         return False
+
+    supports_reference_locations = False
 
=== modified file 'breezy/tests/blackbox/test_branch.py'
--- breezy/tests/blackbox/test_branch.py	2020-01-19 03:22:04 +0000
+++ breezy/tests/blackbox/test_branch.py	2020-01-19 15:24:24 +0000
@@ -557,9 +557,10 @@
         # become necessary for this use case. Please do not adjust this number
         # upwards without agreement from bzr's network support maintainers.
         self.assertLength(2, self.hpss_connections)
-        self.assertLength(33, self.hpss_calls)
-        self.expectFailure("branching to the same branch requires VFS access",
-                           self.assertThat, self.hpss_calls, ContainsNoVfsCalls)
+        self.assertLength(34, self.hpss_calls)
+        self.expectFailure(
+            "branching to the same branch requires VFS access",
+            self.assertThat, self.hpss_calls, ContainsNoVfsCalls)
 
     def test_branch_from_trivial_branch_streaming_acceptance(self):
         self.setup_smart_server_with_call_log()
@@ -575,7 +576,7 @@
         # become necessary for this use case. Please do not adjust this number
         # upwards without agreement from bzr's network support maintainers.
         self.assertThat(self.hpss_calls, ContainsNoVfsCalls)
-        self.assertLength(10, self.hpss_calls)
+        self.assertLength(11, self.hpss_calls)
         self.assertLength(1, self.hpss_connections)
 
     def test_branch_from_trivial_stacked_branch_streaming_acceptance(self):
@@ -597,7 +598,7 @@
         # being too low. If rpc_count increases, more network roundtrips have
         # become necessary for this use case. Please do not adjust this number
         # upwards without agreement from bzr's network support maintainers.
-        self.assertLength(15, self.hpss_calls)
+        self.assertLength(16, self.hpss_calls)
         self.assertLength(1, self.hpss_connections)
         self.assertThat(self.hpss_calls, ContainsNoVfsCalls)
 
@@ -617,7 +618,7 @@
         # being too low. If rpc_count increases, more network roundtrips have
         # become necessary for this use case. Please do not adjust this number
         # upwards without agreement from bzr's network support maintainers.
-        self.assertLength(10, self.hpss_calls)
+        self.assertLength(11, self.hpss_calls)
         self.assertThat(self.hpss_calls, ContainsNoVfsCalls)
         self.assertLength(1, self.hpss_connections)
 
@@ -659,7 +660,7 @@
         # become necessary for this use case. Please do not adjust this number
         # upwards without agreement from bzr's network support maintainers.
         self.assertThat(self.hpss_calls, ContainsNoVfsCalls)
-        self.assertLength(11, self.hpss_calls)
+        self.assertLength(12, self.hpss_calls)
         self.assertLength(1, self.hpss_connections)
 
 
 
=== modified file 'breezy/tests/blackbox/test_checkout.py'
--- breezy/tests/blackbox/test_checkout.py	2018-11-11 04:08:32 +0000
+++ breezy/tests/blackbox/test_checkout.py	2020-01-19 15:24:24 +0000
@@ -211,7 +211,7 @@
         # being too low. If rpc_count increases, more network roundtrips have
         # become necessary for this use case. Please do not adjust this number
         # upwards without agreement from bzr's network support maintainers.
-        self.assertLength(10, self.hpss_calls)
+        self.assertLength(11, self.hpss_calls)
         self.assertLength(1, self.hpss_connections)
         self.assertThat(self.hpss_calls, ContainsNoVfsCalls)
 
 
=== modified file 'breezy/tests/blackbox/test_pull.py'
--- breezy/tests/blackbox/test_pull.py	2019-02-09 02:59:15 +0000
+++ breezy/tests/blackbox/test_pull.py	2020-01-19 15:24:24 +0000
@@ -413,7 +413,7 @@
         # being too low. If rpc_count increases, more network roundtrips have
         # become necessary for this use case. Please do not adjust this number
         # upwards without agreement from bzr's network support maintainers.
-        self.assertLength(19, self.hpss_calls)
+        self.assertLength(20, self.hpss_calls)
         self.assertLength(1, self.hpss_connections)
         remote = branch.Branch.open('stacked')
         self.assertEndsWith(remote.get_stacked_on_url(), '/parent')
 
=== modified file 'breezy/tests/per_workingtree/test_workingtree.py'
--- breezy/tests/per_workingtree/test_workingtree.py	2020-01-19 03:22:04 +0000
+++ breezy/tests/per_workingtree/test_workingtree.py	2020-01-19 15:24:24 +0000
@@ -1385,8 +1385,8 @@
         tree = WorkingTree.open('branch')
         branch_location = tree.get_reference_info('path/to/file')
         self.assertEqual(
-            urlutils.local_path_to_url('branch/path/to/location'),
-            urlutils.join(tree.branch.user_url, branch_location))
+            urlutils.join(urlutils.strip_segment_parameters(tree.branch.user_url), 'path/to/location'),
+            urlutils.join(urlutils.strip_segment_parameters(tree.branch.user_url), branch_location))
 
     def test_set_null_reference_info(self):
         tree = self.make_branch_and_tree('branch')
@@ -1443,7 +1443,8 @@
         tree = self.make_tree_with_reference('branch', '../reference')
         new_tree = tree.branch.controldir.sprout('new-branch').open_workingtree()
         self.assertEqual(
-            urlutils.local_path_to_url('reference'),
+            urlutils.join(urlutils.strip_segment_parameters(tree.branch.user_url),
+                          '../reference'),
             urlutils.join(urlutils.strip_segment_parameters(new_tree.branch.user_url),
                           new_tree.get_reference_info('path/to/file')))
 
@@ -1451,7 +1452,7 @@
         tree = self.make_tree_with_reference('branch', '../reference')
         new_tree = tree.controldir.clone('new-branch').open_workingtree()
         self.assertEqual(
-            urlutils.local_path_to_url('reference'),
+            urlutils.join(urlutils.strip_segment_parameters(tree.branch.user_url), '../reference'),
             urlutils.join(urlutils.strip_segment_parameters(new_tree.branch.user_url),
                           new_tree.get_reference_info('path/to/file')))
 
@@ -1460,7 +1461,8 @@
         new_tree = tree.controldir.sprout(
             'branch/new-branch').open_workingtree()
         self.assertEqual(
-            urlutils.local_path_to_url('branch/reference'),
+            urlutils.join(urlutils.strip_segment_parameters(tree.branch.user_url),
+                          'reference'),
             urlutils.join(urlutils.strip_segment_parameters(new_tree.branch.user_url),
                           new_tree.get_reference_info('path/to/file')))
 
@@ -1470,7 +1472,9 @@
             'new_branch', 'reference2')
         new_tree.branch.update_references(tree.branch)
         self.assertEqual(
-            urlutils.local_path_to_url('branch/reference'),
+            urlutils.join(
+                urlutils.strip_segment_parameters(tree.branch.user_url),
+                'reference'),
             urlutils.join(
                 urlutils.strip_segment_parameters(tree.branch.user_url),
                 tree.get_reference_info('path/to/file')))
@@ -1481,7 +1485,9 @@
             'new_branch', 'reference2')
         new_tree.branch.update_references(tree.branch)
         self.assertEqual(
-            urlutils.local_path_to_url('branch/reference'),
+            urlutils.join(
+                urlutils.strip_segment_parameters(tree.branch.user_url),
+                'reference'),
             urlutils.join(
                 urlutils.strip_segment_parameters(tree.branch.user_url),
                 tree.get_reference_info('path/to/file')))
@@ -1495,7 +1501,9 @@
         new_tree.set_reference_info('foo', '../foo')
         new_tree.branch.update_references(tree.branch)
         self.assertEqual(
-            urlutils.local_path_to_url('branch/reference'),
+            urlutils.join(
+                urlutils.strip_segment_parameters(tree.branch.user_url),
+                'reference'),
             urlutils.join(
                 urlutils.strip_segment_parameters(tree.branch.user_url),
                 tree.get_reference_info('path/to/file')))
@@ -1510,7 +1518,7 @@
         new_tree.commit('set reference')
         tree.pull(new_tree.branch)
         self.assertEqual(
-            urlutils.local_path_to_url('branch/foo'),
+            urlutils.join(urlutils.strip_segment_parameters(new_tree.branch.user_url), '../foo'),
             urlutils.join(tree.branch.user_url, tree.get_reference_info('foo')))
 
     def test_push_updates_references(self):
@@ -1524,8 +1532,8 @@
         tree.pull(new_tree.branch)
         tree.update()
         self.assertEqual(
-            urlutils.local_path_to_url('branch/foo'),
-            urlutils.join(tree.branch.user_url, tree.get_reference_info('foo')))
+            urlutils.join(urlutils.strip_segment_parameters(new_tree.branch.user_url), '../foo'),
+            urlutils.join(urlutils.strip_segment_parameters(tree.branch.user_url), tree.get_reference_info('foo')))
 
     def test_merge_updates_references(self):
         orig_tree = self.make_tree_with_reference('branch', 'reference')
@@ -1542,5 +1550,5 @@
         merger.merge_type = merge.Merge3Merger
         merger.do_merge()
         self.assertEqual(
-            urlutils.local_path_to_url('branch/reference'),
+            urlutils.join(urlutils.strip_segment_parameters(orig_tree.branch.user_url), 'reference'),
             urlutils.join(urlutils.strip_segment_parameters(tree.branch.user_url), tree.get_reference_info('path/to/file')))
 
=== modified file 'breezy/tests/test_remote.py'
--- breezy/tests/test_remote.py	2019-11-06 01:31:41 +0000
+++ breezy/tests/test_remote.py	2020-01-19 15:24:24 +0000
@@ -4529,3 +4529,23 @@
             (b'baserevid', b'line 1\n'),
             (b'somerevid', b'line2\n')],
             list(tree.annotate_iter('filename')))
+
+
+class TestBranchGetAllReferenceInfo(RemoteBranchTestCase):
+
+    def test_get_all_reference_info(self):
+        transport = MemoryTransport()
+        client = FakeClient(transport.base)
+        client.add_expected_call(
+            b'Branch.get_stacked_on_url', (b'quack/',),
+            b'error', (b'NotStacked',))
+        client.add_expected_call(
+            b'Branch.get_all_reference_info', (b'quack/',),
+            b'success', (b'ok',), bencode.bencode([
+                (b'file-id', b'https://www.example.com/', b'')]))
+        transport.mkdir('quack')
+        transport = transport.clone('quack')
+        branch = self.make_remote_branch(transport, client)
+        result = branch._get_all_reference_info()
+        self.assertFinished(client)
+        self.assertEqual({b'file-id': ('https://www.example.com/', None)}, result)
 
=== modified file 'breezy/tests/test_smart.py'
--- breezy/tests/test_smart.py	2019-12-23 03:04:00 +0000
+++ breezy/tests/test_smart.py	2020-01-19 15:24:24 +0000
@@ -2819,3 +2819,18 @@
         self.assertEqual(
             [[b'somerev', b'somecontents\n'], [b'somerev', b'morecontents\n']],
             bencode.bdecode(response.body))
+
+
+class TestSmartServerBranchRequestGetAllReferenceInfo(TestLockedBranch):
+
+    def test_get_some(self):
+        backing = self.get_transport()
+        request = smart_branch.SmartServerBranchRequestGetAllReferenceInfo(backing)
+        branch = self.make_branch('.')
+        branch.set_reference_info('some/path', 'http://www.example.com/')
+        response = request.execute(b'')
+        self.assertTrue(response.is_successful())
+        self.assertEqual(response.args, (b"ok", ))
+        self.assertEqual(
+            [[b'some/path', b'http://www.example.com/', b'']],
+            bencode.bdecode(response.body))
 
=== modified file 'doc/en/release-notes/brz-3.1.txt'
--- doc/en/release-notes/brz-3.1.txt	2020-01-12 23:41:10 +0000
+++ doc/en/release-notes/brz-3.1.txt	2020-01-19 15:24:24 +0000
@@ -46,6 +46,9 @@
    have installed and speeds up import time since psutil brings in
    various other modules. (Jelmer Vernooij)
 
+ * Information about tree references can now be updated on remote
+   branches. (Jelmer Vernooij)
+
  * Warn the user when they attempt to use Breezy in a Subversion
    working copy. (Jelmer Vernooij)