Merge ~pappacena/turnip:copy-ref-helper into turnip:master

[Thiago F. Pappacena]

Celery task to copy refs between repositories and delete them

[Colin Watson (cjwatson)]

review: Needs Fixing

[Thiago F. Pappacena (pappacena)]

Pushed the requested changes.

[Colin Watson (cjwatson)]

review: Approve

[Otto Co-Pilot (otto-copilot)]

Merging failed
https://jenkins.ols.canonical.com/online-services/job/turnip/190/

diff --git a/turnip/api/store.py b/turnip/api/store.py
index f7909c7..d3e3985 100644
--- a/turnip/api/store.py
+++ b/turnip/api/store.py
@@ -9,6 +9,7 @@ import re
 import shutil
 import subprocess
 import uuid
+from collections import defaultdict
 
 from contextlib2 import (
     contextmanager,
@@ -120,6 +121,59 @@ def write_alternates(repo_path, alternate_repo_paths):
 object_dir_re = re.compile(r'\A[0-9a-f][0-9a-f]\Z')
 
 
[e-mail]
+def fetch_refs(operations):
+    """Copy a set of refs from one git repository to another.
+
+    This is implemented now using git client's "git fetch" command,
+    since it's way easier than trying to copy the refs, commits and objects
+    manually using pygit.
+
+    :param operations: A list of tuples describing the copy operations,
+        in the format (from_root, from_ref, to_root, to_ref). If "to_ref" is
+        None, the target ref will have the same name as the source ref.
+    """
+    # Group copy operations by source/dest repositories pairs.
+    grouped_refs = defaultdict(set)
+    for from_root, from_ref, to_root, to_ref in operations:
+        grouped_refs[(from_root, to_root)].add((from_ref, to_ref))
+
+    # A pair of (cmd, stderr) errors happened during the copy.
+    errors = []
+    for repo_pair, refs_pairs in grouped_refs.items():
+        from_root, to_root = repo_pair
+        cmd = [b'git', b'fetch', b'--no-tags', from_root]
+        cmd += [b"%s:%s" % (a, b if b else a) for a, b in refs_pairs]
+
+        # XXX pappacena: On Python3, this could be replaced with
+        # stdout=subprocess.DEVNULL.
+        with open(os.devnull, 'wb') as devnull:
+            proc = subprocess.Popen(
+                cmd, cwd=to_root,
+                stdout=devnull, stderr=subprocess.PIPE)
+            _, stderr = proc.communicate()
+        if proc.returncode != 0:
+            errors.append(cmd, stderr)
+
+    if errors:
+        details = b"\n ".join(b"%s = %s" % (cmd, err) for cmd, err in errors)
+        raise GitError(b"Error copying refs: %s" % details)
+
+
[e-mail]
+def delete_refs(operations):
+    """Remove refs from repositories.
+
+    :param operations: A list of tuples (repo_root, ref_name) to be deleted.
+    """
+    repos = {}
+    for repo_root, ref_name in operations:
+        if repo_root not in repos:
+            repos[repo_root] = Repository(repo_root)
+        repo = repos[repo_root]
+        repo.references[ref_name].delete()
+
+
 def copy_refs(from_root, to_root):
     """Copy refs from one .git directory to another.
 
diff --git a/turnip/api/tests/test_store.py b/turnip/api/tests/test_store.py
index 02ed3fd..d12216b 100644
--- a/turnip/api/tests/test_store.py
+++ b/turnip/api/tests/test_store.py
@@ -26,6 +26,7 @@ from turnip.api.tests.test_helpers import (
     open_repo,
     RepoFactory,
     )
+from turnip.tests.tasks import CeleryWorkerFixture
 
 
 class InitTestCase(TestCase):
@@ -95,8 +96,9 @@ class InitTestCase(TestCase):
 
     def makeOrig(self):
         self.orig_path = os.path.join(self.repo_store, 'orig/')
-        orig = RepoFactory(
-            self.orig_path, num_branches=3, num_commits=2, num_tags=2).build()
+        self.orig_factory = RepoFactory(
+            self.orig_path, num_branches=3, num_commits=2, num_tags=2)
+        orig = self.orig_factory.build()
         self.orig_refs = {}
         for ref in orig.references.objects:
             obj = orig[ref.target]
@@ -356,3 +358,91 @@ class InitTestCase(TestCase):
             self.orig_refs, os.path.join(to_path, 'turnip-subordinate'))
         self.assertPackedRefs(
             packed_refs, os.path.join(too_path, 'turnip-subordinate'))
+
+    def test_fetch_refs(self):
+        celery_fixture = CeleryWorkerFixture()
+        self.useFixture(celery_fixture)
+
+        self.makeOrig()
+        # Creates a new branch in the orig repository.
+        orig_path = self.orig_path
+        orig = self.orig_factory.repo
+        master_tip = orig.references[b'refs/heads/master'].target.hex
+
+        orig_branch_name = b'new-branch'
+        orig_ref_name = b'refs/heads/new-branch'
+        orig.create_branch(orig_branch_name, orig[master_tip])
+        orig_commit_oid = self.orig_factory.add_commit(
+            b'foobar file content', 'foobar.txt', parents=[master_tip],
+            ref=orig_ref_name)
+        orig_blob_id = orig[orig_commit_oid].tree[0].id
+
+        dest_path = os.path.join(self.repo_store, 'to/')
+        store.init_repo(dest_path, clone_from=self.orig_path)
+
+        dest = pygit2.Repository(dest_path)
+        self.assertEqual([], dest.references.objects)
+
+        dest_ref_name = b'refs/merge/123'
+        store.fetch_refs.apply_async(args=([
+            (orig_path, orig_commit_oid.hex, dest_path, dest_ref_name)], ))
+        celery_fixture.waitUntil(5, lambda: len(dest.references.objects) == 1)
+
+        self.assertEqual(1, len(dest.references.objects))
+        copied_ref = dest.references.objects[0]
+        self.assertEqual(dest_ref_name, copied_ref.name)
+        self.assertEqual(
+            orig.references[orig_ref_name].target,
+            dest.references[dest_ref_name].target)
+        self.assertEqual(b'foobar file content', dest[orig_blob_id].data)
+
+        # Updating and copying again should work too, and it should be
+        # compatible with using the ref name instead of the commit ID too.
+        orig_commit_oid = self.orig_factory.add_commit(
+            b'changed foobar content', 'foobar.txt', parents=[orig_commit_oid],
+            ref=orig_ref_name)
+        orig_blob_id = orig[orig_commit_oid].tree[0].id
+
+        store.fetch_refs.apply_async(args=([
+            (orig_path, orig_ref_name, dest_path, dest_ref_name)], ))
+
+        def waitForNewCommit():
+            try:
+                return dest[orig_blob_id].data == b'changed foobar content'
+            except KeyError:
+                return False
+        celery_fixture.waitUntil(5, waitForNewCommit)
+
+        self.assertEqual(1, len(dest.references.objects))
+        copied_ref = dest.references.objects[0]
+        self.assertEqual(dest_ref_name, copied_ref.name)
+        self.assertEqual(
+            orig.references[orig_ref_name].target,
+            dest.references[dest_ref_name].target)
+        self.assertEqual(b'changed foobar content', dest[orig_blob_id].data)
+
+    def test_delete_ref(self):
+        celery_fixture = CeleryWorkerFixture()
+        self.useFixture(celery_fixture)
+
+        self.makeOrig()
+        orig_path = self.orig_path
+        orig = self.orig_factory.repo
+
+        master_tip = orig.references[b'refs/heads/master'].target.hex
+        new_branch_name = b'new-branch'
+        new_ref_name = b'refs/heads/new-branch'
+        orig.create_branch(new_branch_name, orig[master_tip])
+        self.orig_factory.add_commit(
+            b'foobar file content', 'foobar.txt', parents=[master_tip],
+            ref=new_ref_name)
+
+        before_refs_len = len(orig.references.objects)
+        operations = [(orig_path, new_ref_name)]
+        store.delete_refs.apply_async((operations, ))
+        celery_fixture.waitUntil(
+            5, lambda: len(orig.references.objects) < before_refs_len)
+
+        self.assertEqual(before_refs_len - 1, len(orig.references.objects))
+        self.assertNotIn(
+            new_branch_name, [i.name for i in orig.references.objects])