Merge ~cjwatson/launchpad:sync-branches into launchpad:master

[Colin Watson]

Add script to sync branches from production to staging

We currently have a "codehosting_branch_mirror.py" script on (qa)staging that rsyncs a few branches from production for testing. This is a Python script that uses Launchpad's virtualenv, imports our modules, and talks to our database. As a result, it can easily be broken by changes in Launchpad: in particular, it was broken by moving to Python 3.

Bring a version of this script into our tree so that we can make sure it stays working. I modified it considerably, mainly to allow selecting branches on the command line, and added tests.

[Ioana Lasc (ilasc)]

looks good!

review: Approve

diff --git a/lib/lp/codehosting/scripts/sync_branches.py b/lib/lp/codehosting/scripts/sync_branches.py
new file mode 100644
index 0000000..d138a28
--- /dev/null
+++ b/lib/lp/codehosting/scripts/sync_branches.py
@@ -0,0 +1,86 @@
+# Copyright 2007-2021 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Sync branches from production to a staging environment."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = ['SyncBranchesScript']
+
+import os.path
+try:
+    from shlex import quote as shell_quote
+except ImportError:
+    from pipes import quote as shell_quote
+import subprocess
+
+from zope.component import getUtility
+
+from lp.code.interfaces.branch import IBranchSet
+from lp.codehosting.vfs import branch_id_to_path
+from lp.services.config import config
+from lp.services.scripts.base import (
+    LaunchpadScript,
+    LaunchpadScriptFailure,
+    )
+
+
+# We don't want to spend too long syncing branches.
+BRANCH_LIMIT = 35
+REMOTE_SERVER = "bazaar.launchpad.net"
+
+
+class SyncBranchesScript(LaunchpadScript):
+    """Sync branches from production to a staging environment."""
+
+    usage = "%prog [options] BRANCH_NAME [...]"
+    description = __doc__ + "\n" + (
+        'Branch names may be given in any of the forms accepted for lp: '
+        'URLs, but without the leading "lp:".')
+
+    def _syncBranch(self, branch):
+        branch_path = branch_id_to_path(branch.id)
+        branch_dir = os.path.join(
+            config.codehosting.mirrored_branches_root, branch_path)
+        if not os.path.exists(branch_dir):
+            os.makedirs(branch_dir)
+        args = [
+            "rsync", "-a", "--delete-after",
+            "%s::mirrors/%s/" % (REMOTE_SERVER, branch_path),
+            "%s/" % branch_dir,
+            ]
+        try:
+            subprocess.check_output(args, universal_newlines=True)
+        except subprocess.CalledProcessError as e:
+            if "No such file or directory" in e.output:
+                self.logger.warning(
+                    "Branch %s (%s) not found, ignoring",
+                    branch.identity, branch_path)
+            else:
+                raise LaunchpadScriptFailure(
+                    "There was an error running: %s\n"
+                    "Status: %s\n"
+                    "Output: %s" % (
+                        " ".join(shell_quote(arg) for arg in args),
+                        e.returncode, e.output.rstrip("\n")))
+        else:
+            self.logger.info("Rsynced %s (%s)", branch.identity, branch_path)
+
+    def main(self):
+        branches = []
+        for branch_name in self.args:
+            branch = getUtility(IBranchSet).getByPath(branch_name)
+            if branch is not None:
+                branches.append(branch)
+            else:
+                self.logger.warning("Branch %s does not exist", branch_name)
+
+        self.logger.info("There are %d branches to rsync", len(branches))
+
+        if len(branches) > BRANCH_LIMIT:
+            raise LaunchpadScriptFailure(
+                "Refusing to rsync more than %d branches" % BRANCH_LIMIT)
+
+        for branch in branches:
+            self._syncBranch(branch)
diff --git a/lib/lp/codehosting/scripts/tests/test_sync_branches.py b/lib/lp/codehosting/scripts/tests/test_sync_branches.py
new file mode 100644
index 0000000..9340e88
--- /dev/null
+++ b/lib/lp/codehosting/scripts/tests/test_sync_branches.py
@@ -0,0 +1,177 @@
+# Copyright 2021 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Test syncing branches from production to a staging environment."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+
+import os.path
+import subprocess
+from textwrap import dedent
+
+from fixtures import (
+    MockPatch,
+    TempDir,
+    )
+from testtools.matchers import (
+    DirExists,
+    Equals,
+    Matcher,
+    MatchesListwise,
+    Not,
+    )
+
+from lp.codehosting.scripts.sync_branches import SyncBranchesScript
+from lp.codehosting.vfs import branch_id_to_path
+from lp.services.config import config
+from lp.services.log.logger import BufferLogger
+from lp.services.scripts.base import LaunchpadScriptFailure
+from lp.testing import TestCaseWithFactory
+from lp.testing.layers import ZopelessDatabaseLayer
+
+
+class BranchDirectoryCreated(Matcher):
+
+    def match(self, branch):
+        return DirExists().match(
+            os.path.join(
+                config.codehosting.mirrored_branches_root,
+                branch_id_to_path(branch.id)))
+
+
+class BranchSyncProcessMatches(MatchesListwise):
+
+    def __init__(self, branch):
+        branch_path = branch_id_to_path(branch.id)
+        super(BranchSyncProcessMatches, self).__init__([
+            Equals(([
+                "rsync", "-a", "--delete-after",
+                "bazaar.launchpad.net::mirrors/%s/" % branch_path,
+                "%s/" % os.path.join(
+                    config.codehosting.mirrored_branches_root, branch_path),
+                ],)),
+            Equals({"universal_newlines": True}),
+            ])
+
+
+class TestSyncBranches(TestCaseWithFactory):
+
+    layer = ZopelessDatabaseLayer
+
+    def setUp(self):
+        super(TestSyncBranches, self).setUp()
+        self.tempdir = self.useFixture(TempDir()).path
+        self.pushConfig("codehosting", mirrored_branches_root=self.tempdir)
+        self.mock_check_output = self.useFixture(
+            MockPatch("subprocess.check_output")).mock
+        self.logger = BufferLogger()
+
+    def _runScript(self, branch_names):
+        script = SyncBranchesScript(
+            "sync-branches", test_args=branch_names, logger=self.logger)
+        script.main()
+
+    def test_unknown_branch(self):
+        branch = self.factory.makeBranch()
+        self._runScript(
+            [branch.unique_name, branch.unique_name + "-nonexistent"])
+        self.assertIn(
+            "WARNING Branch %s-nonexistent does not exist\n" % (
+                branch.unique_name),
+            self.logger.getLogBuffer())
+        # Other branches are synced anyway.
+        self.assertThat(branch, BranchDirectoryCreated())
+        self.assertThat(
+            self.mock_check_output.call_args_list,
+            MatchesListwise([
+                BranchSyncProcessMatches(branch),
+                ]))
+
+    def test_too_many_branches(self):
+        branches = [self.factory.makeBranch() for _ in range(36)]
+        self.assertRaisesWithContent(
+            LaunchpadScriptFailure, "Refusing to rsync more than 35 branches",
+            self._runScript, [branch.unique_name for branch in branches])
+        for branch in branches:
+            self.assertThat(branch, Not(BranchDirectoryCreated()))
+        self.assertEqual([], self.mock_check_output.call_args_list)
+
+    def test_branch_storage_missing(self):
+        branches = [self.factory.makeBranch() for _ in range(2)]
+        branch_paths = [branch_id_to_path(branch.id) for branch in branches]
+
+        def check_output_side_effect(args, **kwargs):
+            if "%s/%s/" % (self.tempdir, branch_paths[0]) in args:
+                raise subprocess.CalledProcessError(
+                    23, args,
+                    output=(
+                        'rsync: change_dir "/%s" (in mirrors) failed: '
+                        'No such file or directory (2)' % branch_paths[0]))
+            else:
+                return None
+
+        self.mock_check_output.side_effect = check_output_side_effect
+        self._runScript([branch.unique_name for branch in branches])
+        branch_displays = [
+            "%s (%s)" % (branch.identity, branch_path)
+            for branch, branch_path in zip(branches, branch_paths)]
+        self.assertEqual(
+            dedent("""\
+                INFO There are 2 branches to rsync
+                WARNING Branch {} not found, ignoring
+                INFO Rsynced {}
+                """).format(*branch_displays),
+            self.logger.getLogBuffer())
+        self.assertThat(
+            branches,
+            MatchesListwise([BranchDirectoryCreated() for _ in branches]))
+        self.assertThat(
+            self.mock_check_output.call_args_list,
+            MatchesListwise([
+                BranchSyncProcessMatches(branch) for branch in branches
+                ]))
+
+    def test_branch_other_rsync_error(self):
+        branch = self.factory.makeBranch()
+        self.mock_check_output.side_effect = subprocess.CalledProcessError(
+            1, [], output="rsync exploded\n")
+        self.assertRaisesWithContent(
+            LaunchpadScriptFailure,
+            "There was an error running: "
+            "rsync -a --delete-after "
+            "bazaar.launchpad.net::mirrors/{}/ {}/{}/\n"
+            "Status: 1\n"
+            "Output: rsync exploded".format(
+                branch_id_to_path(branch.id),
+                self.tempdir, branch_id_to_path(branch.id)),
+            self._runScript, [branch.unique_name])
+        self.assertThat(branch, BranchDirectoryCreated())
+        self.assertThat(
+            self.mock_check_output.call_args_list,
+            MatchesListwise([BranchSyncProcessMatches(branch)]))
+
+    def test_success(self):
+        branches = [self.factory.makeBranch() for _ in range(3)]
+        branch_paths = [branch_id_to_path(branch.id) for branch in branches]
+        self._runScript([branch.unique_name for branch in branches])
+        branch_displays = [
+            "%s (%s)" % (branch.identity, branch_path)
+            for branch, branch_path in zip(branches, branch_paths)]
+        self.assertEqual(
+            dedent("""\
+                INFO There are 3 branches to rsync
+                INFO Rsynced {}
+                INFO Rsynced {}
+                INFO Rsynced {}
+                """).format(*branch_displays),
+            self.logger.getLogBuffer())
+        self.assertThat(
+            branches,
+            MatchesListwise([BranchDirectoryCreated() for _ in branches]))
+        self.assertThat(
+            self.mock_check_output.call_args_list,
+            MatchesListwise([
+                BranchSyncProcessMatches(branch) for branch in branches
+                ]))
diff --git a/scripts/sync-branches.py b/scripts/sync-branches.py
new file mode 100755
index 0000000..b370bd1
--- /dev/null
+++ b/scripts/sync-branches.py
@@ -0,0 +1,13 @@
+#!/usr/bin/python2 -S
+# Copyright 2021 Canonical Ltd.  All rights reserved.
+
+"""Sync branches from production to a staging environment."""
+
+import _pythonpath
+
+from lp.codehosting.scripts.sync_branches import SyncBranchesScript
+
+
+if __name__ == "__main__":
+    script = SyncBranchesScript("sync-branches", dbuser="ro")
+    script.lock_and_run()