Merge lp:~jelmer/brz/hpss-working-tree-update into lp:brz

[Jelmer Vernooij]

Add a HPSS call for updating the remote working tree.

Add a HPSS call for updating the remote working tree.

[Jelmer Vernooij (jelmer)]

All the plumbing for this is in place, but the trouble is that this requires breaking out of the chroot since WorkingTree requires a filesystem path.

Unmerged revisions

6324. By Jelmer Vernooij on 2018-05-19

Pass along remote argument.

6323. By Jelmer Vernooij on 2018-05-19

Fix some tests.

6322. By Jelmer Vernooij on 2018-05-18

merge trunk.

6321. By Jelmer Vernooij on 2011-11-29

Update NEWS.

6320. By Jelmer Vernooij on 2011-11-28

Add test for push updating remote working tree.

6319. By Jelmer Vernooij on 2011-11-28

Add server side.

6318. By Jelmer Vernooij on 2011-11-28

Merge hpss-working-tree.

6317. By Jelmer Vernooij on 2011-11-28

Add basic RemoteWorkingTree.update.

6316. By Jelmer Vernooij on 2011-11-28

Support opening remote working trees.

=== modified file 'breezy/bzr/bzrdir.py'
--- breezy/bzr/bzrdir.py	2018-05-07 14:35:05 +0000
+++ breezy/bzr/bzrdir.py	2018-05-19 02:48:08 +0000
@@ -1103,13 +1103,13 @@
         return format.open(self, _found=True)
 
     def open_workingtree(self, unsupported=False,
-            recommend_upgrade=True):
+            recommend_upgrade=True, remote=False):
         """See BzrDir.open_workingtree."""
         from .workingtree import WorkingTreeFormatMetaDir
         format = WorkingTreeFormatMetaDir.find_format(self)
         format.check_support_status(unsupported, recommend_upgrade,
             basedir=self.root_transport.base)
-        return format.open(self, _found=True)
+        return format.open(self, remote=remote, _found=True)
 
     def _get_config(self):
         return config.TransportConfig(self.transport, 'control.conf')

=== modified file 'breezy/bzr/remote.py'
--- breezy/bzr/remote.py	2018-05-13 16:23:17 +0000
+++ breezy/bzr/remote.py	2018-05-19 02:48:08 +0000
@@ -45,6 +45,7 @@
     inventory_delta,
     vf_repository,
     vf_search,
+    workingtree,
     )
 from .branch import BranchReferenceFormat
 from ..branch import BranchWriteLockResult
@@ -883,9 +884,12 @@
                 self._has_working_tree = (response[0] == b'yes')
         return self._has_working_tree
 
-    def open_workingtree(self, recommend_upgrade=True):
+    def open_workingtree(self, recommend_upgrade=True, remote=False):
         if self.has_workingtree():
-            raise errors.NotLocalUrl(self.root_transport)
+            if remote:
+                return RemoteWorkingTree(self, self.open_branch())
+            else:
+                raise errors.NotLocalUrl(self.root_transport.base)
         else:
             raise errors.NoWorkingTree(self.root_transport.base)
 
@@ -4127,6 +4131,168 @@
         return self._bzrdir._real_bzrdir
 
 
+class RemoteWorkingTreeFormat(workingtree.WorkingTreeFormat):
+
+    def __init__(self):
+        super(RemoteWorkingTreeFormat, self).__init__()
+
+    def get_format_description(self):
+        return "Remote Working Tree"
+
+    def __eq__(self, other):
+        return (isinstance(other, RemoteWorkingTreeFormat) and
+            self.__dict__ == other.__dict__)
+
+
+class RemoteWorkingTree(workingtree.WorkingTree, _RpcHelper, lock._RelockDebugMixin):
+    """Working tree stored on a server accessed by HPSS RPC.
+
+    At the moment this only implements .update(). All other methods
+    raise _ensure_real().
+    """
+
+    def __init__(self, remote_controldir, remote_branch, _client=None):
+        self.controldir = remote_controldir
+        self._branch = remote_branch
+        self._format = RemoteWorkingTreeFormat()
+        self._real_workingtree = None
+        self._lock_mode = None
+        self._lock_count = 0
+        if _client is None:
+            self._client = remote_controldir._client
+        else:
+            self._client = _client
+
+    @property
+    def user_transport(self):
+        return self.controldir.user_transport
+
+    @property
+    def control_transport(self):
+        # XXX: Normally you shouldn't directly get at the remote repository
+        # transport, but I'm not sure it's worth making this method
+        # optional -- mbp 2010-04-21
+        return self.controldir.get_repository_transport(None)
+
+    def _translate_error(self, err, **context):
+        _translate_error(err, workingtree=self, **context)
+
+    def _ensure_real(self):
+        # Most working tree operations require local access at the moment.
+        # When Bazaar working trees can be accessed over
+        # transports, set self._real_workingtree here.
+        raise errors.NotLocalUrl(self.controldir.root_transport)
+
+    def is_locked(self):
+        return self._lock_count >= 1
+
+    def lock_read(self):
+        """Lock the working tree for read operations.
+
+        :return: A bzrlib.lock.LogicalLockResult.
+        """
+        # wrong eventually - want a local lock cache context
+        if not self._lock_mode:
+            self._note_lock('r')
+            self._lock_mode = 'r'
+            self._lock_count = 1
+            if self._real_workingtree is not None:
+                self._real_workingtree.lock_read()
+            self.branch.lock_read()
+        else:
+            self._lock_count += 1
+        return lock.LogicalLockResult(self.unlock)
+
+    def lock_write(self):
+        if not self._lock_mode:
+            self._note_lock('w')
+            if self._real_workingtree is not None:
+                self._real_workingtree.lock_write()
+            self._lock_mode = 'w'
+            self._lock_count = 1
+            self.branch.lock_write()
+        elif self._lock_mode == 'r':
+            raise errors.ReadOnlyError(self)
+        else:
+            self._lock_count += 1
+        return lock.LogicalLockResult(
+            lambda: self.unlock())
+
+    @only_raises(errors.LockNotHeld, errors.LockBroken)
+    def unlock(self):
+        if not self._lock_count:
+            return lock.cant_unlock_not_held(self)
+        self._lock_count -= 1
+        if self._lock_count > 0:
+            return
+        old_mode = self._lock_mode
+        self._lock_mode = None
+        try:
+            if self._real_workingtree is not None:
+                self._real_workingtree.unlock()
+        finally:
+            self.branch.unlock()
+
+    def get_physical_lock_status(self):
+        """See WorkingTree.get_physical_lock_status()."""
+        path = self.controldir._path_for_remote_call(self._client)
+        try:
+            response = self._call(b'WorkingTree.get_physical_lock_status', path)
+        except errors.UnknownSmartMethod:
+            self._ensure_real()
+            return self._real_workingtree.get_physical_lock_status()
+        if response[0] not in (b'yes', b'no'):
+            raise errors.UnexpectedSmartServerResponse(response)
+        return (response[0] == b'yes')
+
+    def _update_rpc(self, old_tip=None, change_reporter=None,
+               possible_transports=None, revision=None, show_base=False):
+        medium = self.controldir._client._medium
+        path = self.controldir._path_for_remote_call(self.controldir._client)
+        if revision is None:
+            revision = ''
+        if old_tip is None:
+            old_tip = ''
+        response, response_handler = self._call_expecting_body(
+            'WorkingTree.update', path, revision, old_tip, show_base)
+        if response[0] != 'ok':
+            raise errors.UnexpectedSmartServerResponse(response)
+        if change_reporter is None:
+            response_handler.cancel_read_body()
+        else:
+            byte_stream = response_handler.read_streamed_body()
+            data = ""
+            for bytes in byte_stream:
+                data += bytes
+                lines = bytes.split("\n")
+                data = lines.pop()
+                for line in lines:
+                    if not line.startswith("change: "):
+                        raise errors.UnexpectedSmartServerResponse(response)
+                    change = bencode.bdecode_as_tuple(line[8:])
+                    # FIXME: More post-processing
+                    change_reporter.report(*change)
+        return int(response[1])
+
+    def update(self, old_tip=None, change_reporter=None,
+               possible_transports=None, revision=None, show_base=False):
+        try:
+            return self._update_rpc(old_tip=old_tip,
+                change_reporter=change_reporter,
+                possible_transports=possible_transports, revision=revision,
+                show_base=show_base)
+        except errors.UnknownSmartMethod:
+            self._ensure_real()
+            return self._real_workingtree.update(old_tip=old_tip,
+                change_reporter=change_reporter,
+               possible_transports=possible_transports,
+               revision=revision, show_base=show_base)
+
+    def get_parent_ids(self):
+        self._ensure_real()
+        return self._real_workingtree.get_parent_ids()
+
+
 error_translators = registry.Registry()
 no_context_error_translators = registry.Registry()
 

=== modified file 'breezy/bzr/smart/branch.py'
--- breezy/bzr/smart/branch.py	2018-05-13 02:18:13 +0000
+++ breezy/bzr/smart/branch.py	2018-05-19 02:48:08 +0000
@@ -14,7 +14,7 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
-"""Server-side branch related request implmentations."""
+"""Server-side branch related request implementations."""
 
 from __future__ import absolute_import
 

=== modified file 'breezy/bzr/smart/request.py'
--- breezy/bzr/smart/request.py	2018-05-13 02:18:13 +0000
+++ breezy/bzr/smart/request.py	2018-05-19 02:48:08 +0000
@@ -791,3 +791,9 @@
 request_handlers.register_lazy(
     b'Transport.is_readonly', 'breezy.bzr.smart.request',
     'SmartServerIsReadonly', info='read')
+request_handlers.register_lazy(
+    b'WorkingTree.update', 'breezy.bzr.smart.workingtree',
+    'SmartServerWorkingTreeUpdate', info='mutate')
+request_handlers.register_lazy(
+    b'WorkingTree.get_physical_lock_status', 'breezy.bzr.smart.workingtree',
+    'SmartServerWorkingTreeGetPhysicalLockStatus', info='read')

=== added file 'breezy/bzr/smart/workingtree.py'
--- breezy/bzr/smart/workingtree.py	1970-01-01 00:00:00 +0000
+++ breezy/bzr/smart/workingtree.py	2018-05-19 02:48:08 +0000
@@ -0,0 +1,115 @@
+# Copyright (C) 2011 Canonical Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+"""Server-side working tree related request implementations."""
+
+from __future__ import absolute_import
+
+
+from breezy import (
+    bencode,
+    delta as _mod_delta,
+    errors,
+    revision as _mod_revision,
+    )
+from breezy.controldir import ControlDir
+from breezy.bzr.smart.request import (
+    FailedSmartServerResponse,
+    SmartServerRequest,
+    SuccessfulSmartServerResponse,
+    )
+
+
+class SmartServerWorkingTreeRequest(SmartServerRequest):
+    """Base class for handling common working tree request logic.
+    """
+
+    def do(self, path, *args):
+        """Execute a request for a working tree at path.
+
+        All working tree requests take a path to the tree as their first
+        argument.
+
+        :param path: The path for the repository as received from the
+            client.
+        :return: A SmartServerResponse from self.do_with_workingtree().
+        """
+        transport = self.transport_from_client_path(path)
+        controldir = ControlDir.open_from_transport(transport)
+        workingtree = controldir.open_workingtree(remote=True)
+        return self.do_with_workingtree(workingtree, *args)
+
+    def do_with_workingtree(self, workingtree, *args):
+        raise NotImplementedError(self.do_with_workingtree)
+
+
+class SerializingChangeReporter(_mod_delta._ChangeReporter):
+    """Change reporter that serializes."""
+
+    def __init__(self):
+        self.chunks = []
+
+    def _format_path(self, path):
+        if path is None:
+            return ""
+        elif path == "":
+            return "."
+        else:
+            return path.encode("utf-8")
+
+    def _format_kind(self, kind):
+        if kind is None:
+            return ""
+        return kind
+
+    def report(self, file_id, paths, versioned, renamed, modified, exe_change,
+            kind):
+        self.chunks.append("report: %s\n" % bencode.bencode((file_id,
+            self._format_path(paths[0]), self._format_path(paths[1]),
+            versioned, renamed, modified, exe_change,
+            self._format_kind(kind[0]), self._format_kind(kind[1]))))
+
+
+class SmartServerWorkingTreeUpdate(SmartServerWorkingTreeRequest):
+    """Update a working tree.
+
+    New in 3.0.
+    """
+
+    def do_with_workingtree(self, workingtree, revision, old_tip, show_base):
+        if revision == '':
+            revision = None
+        if old_tip == '':
+            old_tip = None
+        reporter = SerializingChangeReporter()
+        with workingtree.lock_write(): # TODO(jelmer): Accept token
+            nconflicts = workingtree.update(change_reporter=reporter,
+                    revision=revision, old_tip=old_tip, show_base=show_base)
+        return SuccessfulSmartServerResponse(('ok', nconflicts),
+                body_stream=iter(reporter.chunks))
+
+
+class SmartServerWorkingTreeGetPhysicalLockStatus(SmartServerWorkingTreeRequest):
+    """Get the physical lock status for a working tree.
+
+    New in 3.0.
+    """
+
+    def do_with_workingtree(self, workingtree):
+        if workingtree.get_physical_lock_status():
+            return SuccessfulSmartServerResponse((b'yes', ))
+        else:
+            return SuccessfulSmartServerResponse((b'no', ))

=== modified file 'breezy/bzr/workingtree_4.py'
--- breezy/bzr/workingtree_4.py	2018-03-28 01:17:12 +0000
+++ breezy/bzr/workingtree_4.py	2018-05-19 02:48:08 +0000
@@ -1567,7 +1567,7 @@
         :param wt: the WorkingTree object
         """
 
-    def open(self, a_controldir, _found=False):
+    def open(self, a_controldir, remote=False, _found=False):
         """Return the WorkingTree object for a_controldir
 
         _found is a private parameter, do not use it. It is used to indicate
@@ -1576,7 +1576,7 @@
         if not _found:
             # we are being called directly and must probe.
             raise NotImplementedError
-        if not isinstance(a_controldir.transport, LocalTransport):
+        if not remote and not isinstance(a_controldir.transport, LocalTransport):
             raise errors.NotLocalUrl(a_controldir.transport.base)
         wt = self._open(a_controldir, self._open_control_files(a_controldir))
         return wt

=== modified file 'breezy/controldir.py'
--- breezy/controldir.py	2018-05-07 14:35:05 +0000
+++ breezy/controldir.py	2018-05-19 02:48:08 +0000
@@ -306,6 +306,7 @@
             (but still fully supported).
         :param from_branch: override controldir branch (for lightweight
             checkouts)
+        :param remote: open working tree even if it's remote
         """
         raise NotImplementedError(self.open_workingtree)
 
@@ -444,9 +445,9 @@
                 # FIXME: Should be done only if we succeed ? -- vila 2012-01-18
                 source.set_push_location(br_to.base)
             try:
-                tree_to = self.open_workingtree()
+                tree_to = self.open_workingtree(remote=True)
             except errors.NotLocalUrl:
-                push_result.branch_push_result = source.push(br_to, 
+                push_result.branch_push_result = source.push(br_to,
                     overwrite, stop_revision=revision_id, lossy=lossy)
                 push_result.workingtree_updated = False
             except errors.NoWorkingTree:

=== modified file 'breezy/tests/blackbox/test_push.py'
--- breezy/tests/blackbox/test_push.py	2018-03-26 00:54:10 +0000
+++ breezy/tests/blackbox/test_push.py	2018-05-19 02:48:08 +0000
@@ -266,6 +266,23 @@
         # ancestry
         self.assertEqual([('A',), ('C',)], sorted(target_repo.revisions.keys()))
 
+    def test_push_smart_updates_tree(self):
+        self.setup_smart_server_with_call_log()
+        t = self.make_branch_and_tree('from')
+        self.build_tree(['from/afile'])
+        t.add(['afile'])
+        self.make_branch_and_tree('to')
+        t.commit(allow_pointless=True, message='first commit')
+        self.reset_smart_call_log()
+        self.run_bzr(['push', self.get_url('to')], working_dir='from')
+        # This figure represent the amount of work to perform this use case. It
+        # is entirely ok to reduce this number if a test fails due to rpc_count
+        # being too low. If rpc_count increases, more network roundtrips have
+        # become necessary for this use case. Please do not adjust this number
+        # upwards without agreement from bzr's network support maintainers.
+        self.assertLength(11, self.hpss_calls)
+        self.assertPathExists('to/afile')
+
     def test_push_smart_non_stacked_streaming_acceptance(self):
         self.setup_smart_server_with_call_log()
         t = self.make_branch_and_tree('from')

=== modified file 'breezy/tests/test_remote.py'
--- breezy/tests/test_remote.py	2018-05-13 02:45:58 +0000
+++ breezy/tests/test_remote.py	2018-05-19 02:48:08 +0000
@@ -60,6 +60,7 @@
     RemoteBranchFormat,
     RemoteBzrDir,
     RemoteBzrDirFormat,
+    RemoteWorkingTree,
     RemoteRepository,
     RemoteRepositoryFormat,
     )
@@ -659,11 +660,24 @@
         client, transport = self.make_fake_client_and_transport()
         client.add_expected_call(
             b'BzrDir.open_2.1', (b'quack/',), b'success', (b'yes', b'yes'))
+        branch_network_name = self.get_branch_format().network_name()
+        repo_network_name = self.get_repo_format().network_name()
+        client.add_expected_call(
+            'BzrDir.open_branchV3', ('quack/',),
+            'success', ('branch', branch_network_name))
+        client.add_expected_call(
+            'BzrDir.find_repositoryV3', ('quack/',),
+            'success', ('ok', '', 'no', 'no', 'no', repo_network_name))
+        client.add_expected_call(
+            'Branch.get_stacked_on_url', ('quack/',),
+            'error', ('NotStacked',))
         bd = RemoteBzrDir(transport, RemoteBzrDirFormat(),
             _client=client, _force_probe=True)
         self.assertIsInstance(bd, RemoteBzrDir)
         self.assertTrue(bd.has_workingtree())
         self.assertRaises(errors.NotLocalUrl, bd.open_workingtree)
+        tree = bd.open_workingtree(remote=True)
+        self.assertIsInstance(tree, RemoteWorkingTree)
         self.assertFinished(client)
 
     def test_backwards_compat(self):
@@ -4261,6 +4275,46 @@
         repo.pack([b'hinta', b'hintb'])
 
 
+class RemoteWorkingTreeTestCase(RemoteBranchTestCase):
+
+    def make_remote_working_tree(self, transport, client):
+        """Make a RemoteWorkingTree using 'client' as its _SmartClient.
+        """
+        bzrdir = self.make_remote_bzrdir(transport, client)
+        return RemoteWorkingTree(bzrdir, bzrdir.open_branch())
+
+
+class TestWorkingTreeLocking(RemoteWorkingTreeTestCase):
+
+    def test_lock(self):
+        transport = MemoryTransport()
+        client = FakeClient(transport.base)
+        branch_network_name = self.get_branch_format().network_name()
+        repo_network_name = self.get_repo_format().network_name()
+        client.add_expected_call(
+            'BzrDir.open_branchV3', ('.',),
+            'success', ('branch', branch_network_name))
+        client.add_expected_call(
+            'BzrDir.find_repositoryV3', ('.',),
+            'success', ('ok', '', 'no', 'no', 'no', repo_network_name))
+        client.add_expected_call(
+            'Branch.get_stacked_on_url', ('.',),
+            'error', ('NotStacked',))
+        client.add_expected_call(
+            'Branch.lock_write', ('.', '', ''),
+            'success', ('ok', 'repo token', 'branch token'))
+        client.add_expected_call(
+            'Branch.unlock', ('.', 'repo token', 'branch token'),
+            'success', ('ok', ))
+        tree = self.make_remote_working_tree(transport, client)
+        self.assertFalse(tree.is_locked())
+        tree.lock_write()
+        self.assertTrue(tree.is_locked())
+        tree.unlock()
+        self.assertFalse(tree.is_locked())
+        self.assertFinished(client)
+
+
 class TestRepositoryIterInventories(TestRemoteRepository):
     """Test Repository.iter_inventories."""
 

=== modified file 'breezy/workingtree.py'
--- breezy/workingtree.py	2018-03-25 00:39:16 +0000
+++ breezy/workingtree.py	2018-05-19 02:48:08 +0000
@@ -406,7 +406,7 @@
         else:
             parents = [last_rev]
         try:
-            merges_bytes = self._transport.get_bytes('pending-merges')
+            merges_bytes = self.control_transport.get_bytes('pending-merges')
         except errors.NoSuchFile:
             pass
         else:
@@ -584,7 +584,7 @@
 
     def _set_merges_from_parent_ids(self, parent_ids):
         merges = parent_ids[1:]
-        self._transport.put_bytes('pending-merges', b'\n'.join(merges),
+        self.control_transport.put_bytes('pending-merges', b'\n'.join(merges),
             mode=self.controldir._get_file_mode())
 
     def _filter_parent_ids_by_ancestry(self, revision_ids):

=== modified file 'doc/en/release-notes/brz-3.0.txt'
--- doc/en/release-notes/brz-3.0.txt	2018-03-24 03:03:03 +0000
+++ doc/en/release-notes/brz-3.0.txt	2018-05-19 02:48:08 +0000
@@ -126,6 +126,9 @@
  * New ``bzr cp`` command which copies files (but does not currently track
    history). (Jelmer Vernooij, start towards #269095)
 
+* ``bzr push`` will now update the remote working tree when it pushes to
+  a smart server running Breezy 3.0 or later. (Jelmer Vernooij, #325355)
+
 Bug Fixes
 *********