Merge ~pappacena/turnip:copy-and-delete-ref-api into turnip:master

[Thiago F. Pappacena]

API to copy refs between repositories and delete refs

[Colin Watson (cjwatson)]

review: Approve

[Thiago F. Pappacena (pappacena)]

Pushed the requested changes.

[Otto Co-Pilot (otto-copilot)]

Running landing tests failed
https://jenkins.ols.canonical.com/online-services/job/turnip/199/

[Otto Co-Pilot (otto-copilot)]

Running landing tests failed
https://jenkins.ols.canonical.com/online-services/job/turnip/200/

7c3c5f2... by Thiago F. Pappacena on 2020-10-09

Test fix: assertRepositoryCreatedAsynchronously should retry even if API crashes

diff --git a/turnip/api/tests/test_api.py b/turnip/api/tests/test_api.py
index dbdf77a..15c366d 100644
--- a/turnip/api/tests/test_api.py
+++ b/turnip/api/tests/test_api.py
@@ -316,6 +316,120 @@ class ApiTestCase(TestCase, ApiRepoStoreMixin):
         resp = self.get_ref(tag)
         self.assertTrue(tag in resp)
 
+    def test_delete_ref(self):
+        celery_fixture = CeleryWorkerFixture()
+        self.useFixture(celery_fixture)
+
+        repo = RepoFactory(
+            self.repo_store, num_branches=5, num_commits=1, num_tags=1).build()
+        self.assertEqual(7, len(repo.references.objects))
+
+        ref = 'refs/heads/branch-0'
+        url = '/repo/{}/{}'.format(self.repo_path, ref)
+        resp = self.app.delete(quote(url))
+
+        self.assertEqual(6, len(repo.references.objects))
+        self.assertEqual(200, resp.status_code)
+        self.assertEqual('', resp.body)
+
+    def test_delete_non_existing_ref(self):
+        celery_fixture = CeleryWorkerFixture()
+        self.useFixture(celery_fixture)
+
+        repo = RepoFactory(
+            self.repo_store, num_branches=5, num_commits=1, num_tags=1).build()
+        self.assertEqual(7, len(repo.references.objects))
+
+        ref = 'refs/heads/fake-branch'
+        url = '/repo/{}/{}'.format(self.repo_path, ref)
+        resp = self.app.delete(quote(url), expect_errors=True)
+        self.assertEqual(404, resp.status_code)
+        self.assertEqual({
+            'status': 'error',
+            'errors': [{
+                'description': 'Ref refs/heads/fake-branch does not exist.',
+                'location': 'body',
+                'name': u'operations'
+            }]}, resp.json)
+
+    def test_copy_ref_api(self):
+        celery_fixture = CeleryWorkerFixture()
+        self.useFixture(celery_fixture)
+        repo1_path = os.path.join(self.repo_root, 'repo1')
+        repo2_path = os.path.join(self.repo_root, 'repo2')
+        repo3_path = os.path.join(self.repo_root, 'repo3')
+
+        repo1_factory = RepoFactory(
+            repo1_path, num_branches=5, num_commits=1, num_tags=1)
+        repo1 = repo1_factory.build()
+        self.assertEqual(7, len(repo1.references.objects))
+
+        repo2_factory = RepoFactory(
+            repo2_path, num_branches=1, num_commits=1, num_tags=1)
+        repo2 = repo2_factory.build()
+        self.assertEqual(3, len(repo2.references.objects))
+
+        repo3_factory = RepoFactory(
+            repo3_path, num_branches=1, num_commits=1, num_tags=1)
+        repo3 = repo3_factory.build()
+        self.assertEqual(3, len(repo3.references.objects))
+
+        url = '/repo/repo1/refs-copy'
+        body = {
+            "operations": [
+                {
+                    b"from": b"refs/heads/branch-4",
+                    b"to": {b"repo": b'repo2', b"ref": b"refs/merge/123/head"}
+                }, {
+                    b"from": b"refs/heads/branch-4",
+                    b"to": {b"repo": b'repo3', b"ref": b"refs/merge/987/head"}
+                }]}
+        resp = self.app.post_json(quote(url), body)
+        self.assertEqual(202, resp.status_code)
+
+        def branchCreated():
+            repo2_refs = [i.name for i in repo2.references.objects]
+            repo3_refs = [i.name for i in repo3.references.objects]
+            return (b'refs/merge/123/head' in repo2_refs and
+                    b'refs/merge/987/head' in repo3_refs)
+
+        celery_fixture.waitUntil(5, branchCreated)
+        self.assertEqual(4, len(repo2.references.objects))
+        self.assertEqual(202, resp.status_code)
+        self.assertEqual('', resp.body)
+
+    def test_copy_non_existing_ref(self):
+        celery_fixture = CeleryWorkerFixture()
+        self.useFixture(celery_fixture)
+
+        repo_path = os.path.join(self.repo_root, 'repo1')
+        repo = RepoFactory(
+            repo_path, num_branches=5, num_commits=1, num_tags=1).build()
+        self.assertEqual(7, len(repo.references.objects))
+
+        body = {
+            "operations": [{
+                b"from": b"refs/heads/nope",
+                b"to": {b"repo": b'repo2', b"ref": b"refs/merge/123/head"}
+            }, {
+                b"from": b"refs/heads/no-ref",
+                b"to": {b"repo": b'repo2', b"ref": b"refs/merge/123/head"}
+            }]}
+
+        url = '/repo/repo1/refs-copy'
+        resp = self.app.post_json(quote(url), body, expect_errors=True)
+        self.assertEqual(404, resp.status_code)
+        self.assertEqual({
+            'status': 'error',
+            'errors': [{
+                'description': 'Ref refs/heads/nope does not exist.',
+                'location': 'body',
+                'name': u'operations'
+            }, {
+                'description': 'Ref refs/heads/no-ref does not exist.',
+                'location': 'body', 'name': u'operations'
+            }]}, resp.json)
+
     def test_repo_compare_commits(self):
         """Ensure expected changes exist in diff patch."""
         repo = RepoFactory(self.repo_store)
@@ -929,10 +1043,14 @@ class AsyncRepoCreationAPI(TestCase, ApiRepoStoreMixin):
         start = datetime.now()
         while datetime.now() <= (start + timeout):
             self._doReactorIteration()
-            resp = self.app.get('/repo/{}'.format(repo_path),
-                                expect_errors=True)
-            if resp.status_code == 200 and resp.json['is_available']:
-                return
+            try:
+                resp = self.app.get('/repo/{}'.format(repo_path),
+                                    expect_errors=True)
+                if resp.status_code == 200 and resp.json['is_available']:
+                    return
+            except Exception:
+                # If we have any unexpected error, wait a bit and retry.
+                pass
             time.sleep(0.1)
         self.fail(
             "Repository %s was not created after %s secs"
diff --git a/turnip/api/views.py b/turnip/api/views.py
index de6bb4e..049f3ee 100644
--- a/turnip/api/views.py
+++ b/turnip/api/views.py
@@ -9,6 +9,7 @@ from cornice.resource import resource
 from cornice.util import extract_json_data
 from pygit2 import GitError
 import pyramid.httpexceptions as exc
+from pyramid.response import Response
 
 from turnip.config import config
 from turnip.api import store
@@ -162,6 +163,65 @@ class RepackAPI(BaseAPI):
         return
 
 
+@resource(path='/repo/{name}/refs-copy')
+class RefCopyAPI(BaseAPI):
+    """Provides HTTP API for git references copy operations."""
+
+    def __init__(self, request, context=None):
+        super(RefCopyAPI, self).__init__()
+        self.request = request
+
+    def _validate_refs(self, repo_store, repo_name, refs_or_commits):
+        """Checks if a given list of ref names or commits ID exists in
+        repo. If not, raises 404 exception.
+
+        Note that the API copy runs async, in a celery job. So,
+        this validation does not guarantee that the copy operation will
+        actually be done since someone could delete the ref_or_commit
+        between the check and the actual execution of the copy task.
+        """
+        for ref_or_commit in refs_or_commits:
+            # Checks if it's a commit.
+            try:
+                store.get_commit(repo_store, repo_name, ref_or_commit)
+                return
+            except GitError:
+                pass
+            # Checks if it's a ref name.
+            try:
+                store.get_ref(repo_store, repo_name, ref_or_commit)
+                return
+            except KeyError:
+                self.request.errors.add(
+                    'body', 'operations',
+                    'Ref %s does not exist.' % ref_or_commit)
+
+        if len(self.request.errors):
+            self.request.errors.status = 404
+
+    @validate_path
+    def post(self, repo_store, repo_name):
+        orig_path = os.path.join(repo_store, repo_name)
+        copy_refs_args = []
+        operations = self.request.json.get('operations', [])
+        self._validate_refs(
+            repo_store, repo_name, [i["from"] for i in operations])
+        if len(self.request.errors):
+            return
+
+        for operation in operations:
+            source = operation["from"]
+            dest = operation["to"]
+            dest_repo = dest.get('repo')
+            dest_ref_name = dest.get('ref')
+            dest_path = os.path.join(repo_store, dest_repo)
+            copy_refs_args.append(
+                (orig_path, source, dest_path, dest_ref_name))
+
+        store.fetch_refs.apply_async(args=(copy_refs_args, ))
+        return Response(status=202)
+
+
 @resource(collection_path='/repo/{name}/refs',
           path='/repo/{name}/refs/{ref:.*}')
 class RefAPI(BaseAPI):
@@ -188,6 +248,21 @@ class RefAPI(BaseAPI):
         except (KeyError, GitError):
             return exc.HTTPNotFound()
 
+    @validate_path
+    def delete(self, repo_store, repo_name):
+        ref = 'refs/' + self.request.matchdict['ref']
+        # Make sure the ref actually exists. Otherwise, raise a 404.
+        try:
+            store.get_ref(repo_store, repo_name, ref)
+        except (KeyError, GitError):
+            self.request.errors.add(
+                'body', 'operations', 'Ref %s does not exist.' % ref)
+            self.request.errors.status = 404
+            return
+        repo_path = os.path.join(repo_store, repo_name)
+        store.delete_refs([(repo_path, ref)])
+        return Response(status=200)
+
 
 @resource(path='/repo/{name}/compare/{commits}')
 class DiffAPI(BaseAPI):