Merge lp:~cjwatson/launchpad/git-ref-remote-lp-non-production into lp:launchpad

[Colin Watson]

Allow fetching blobs from repositories hosted on git.launchpad.net from non-production Launchpad instances.

This makes life easier when doing e.g. snap build QA on dogfood.

[William Grant (wgrant)]

review: Approve (code)

=== modified file 'lib/lp/code/model/gitref.py'
--- lib/lp/code/model/gitref.py	2018-10-22 12:43:55 +0000
+++ lib/lp/code/model/gitref.py	2018-10-25 15:18:55 +0000
@@ -702,6 +702,25 @@
     return response.content
 
 
+def _fetch_blob_from_launchpad(repository_url, ref_path, filename):
+    repo_path = urlsplit(repository_url).path.strip("/")
+    try:
+        response = urlfetch(
+            "https://git.launchpad.net/%s/plain/%s" % (
+                repo_path, quote(filename)),
+            params={"h": ref_path})
+    except requests.RequestException as e:
+        if (e.response is not None and
+                e.response.status_code == requests.codes.NOT_FOUND):
+            raise GitRepositoryBlobNotFound(
+                repository_url, filename, rev=ref_path)
+        else:
+            raise GitRepositoryScanFault(
+                "Failed to get file from Git repository at %s: %s" %
+                (repository_url, str(e)))
+    return response.content
+
+
 @implementer(IGitRef)
 @provider(IGitRefRemoteSet)
 class GitRefRemote(GitRefMixin):
@@ -800,17 +819,29 @@
         # dispatch a build job or a code import or something like that to do
         # so.  For now, we just special-case some providers where we know
         # how to fetch a blob on its own.
-        repository = getUtility(IGitLookup).getByUrl(self.repository_url)
-        if repository is not None:
-            # This is one of our own repositories.  Doing this by URL seems
-            # gratuitously complex, but apparently we already have some
-            # examples of this on production.
-            return repository.getBlob(filename, rev=self.path)
         url = urlsplit(self.repository_url)
         if (url.hostname == "github.com" and
                 len(url.path.strip("/").split("/")) == 2):
             return _fetch_blob_from_github(
                 self.repository_url, self.path, filename)
+        if (url.hostname == "git.launchpad.net" and
+                config.vhost.mainsite.hostname != "launchpad.net"):
+            # Even if this isn't launchpad.net, we can still retrieve files
+            # from git.launchpad.net by URL, as a QA convenience.  (We check
+            # config.vhost.mainsite.hostname rather than
+            # config.codehosting.git_*_root because the dogfood instance
+            # points git_*_root to git.launchpad.net but doesn't share the
+            # production database.)
+            return _fetch_blob_from_launchpad(
+                self.repository_url, self.path, filename)
+        codehosting_host = urlsplit(config.codehosting.git_anon_root).hostname
+        if url.hostname == codehosting_host:
+            repository = getUtility(IGitLookup).getByUrl(self.repository_url)
+            if repository is not None:
+                # This is one of our own repositories.  Doing this by URL
+                # seems gratuitously complex, but apparently we already have
+                # some examples of this on production.
+                return repository.getBlob(filename, rev=self.path)
         raise GitRepositoryBlobUnsupportedRemote(self.repository_url)
 
     @property

=== modified file 'lib/lp/code/model/tests/test_gitref.py'
--- lib/lp/code/model/tests/test_gitref.py	2018-10-16 15:29:37 +0000
+++ lib/lp/code/model/tests/test_gitref.py	2018-10-25 15:18:55 +0000
@@ -344,6 +344,51 @@
         self.assertEqual(expected_calls, hosting_fixture.getBlob.calls)
 
     @responses.activate
+    def test_remote_launchpad_production_branch(self):
+        ref = self.factory.makeGitRefRemote(
+            repository_url="https://git.launchpad.net/~owner/+git/name",
+            path="refs/heads/path")
+        responses.add(
+            "GET",
+            "https://git.launchpad.net/~owner/+git/name/plain/dir/file"
+            "?h=refs%2Fheads%2Fpath",
+            body=b"foo")
+        self.assertEqual(b"foo", ref.getBlob("dir/file"))
+
+    @responses.activate
+    def test_remote_launchpad_production_HEAD(self):
+        ref = self.factory.makeGitRefRemote(
+            repository_url="https://git.launchpad.net/~owner/+git/name",
+            path="HEAD")
+        responses.add(
+            "GET",
+            "https://git.launchpad.net/~owner/+git/name/plain/dir/file?h=HEAD",
+            body=b"foo")
+        self.assertEqual(b"foo", ref.getBlob("dir/file"))
+
+    @responses.activate
+    def test_remote_launchpad_production_404(self):
+        ref = self.factory.makeGitRefRemote(
+            repository_url="https://git.launchpad.net/~owner/+git/name",
+            path="HEAD")
+        responses.add(
+            "GET",
+            "https://git.launchpad.net/~owner/+git/name/plain/dir/file?h=HEAD",
+            status=404)
+        self.assertRaises(GitRepositoryBlobNotFound, ref.getBlob, "dir/file")
+
+    @responses.activate
+    def test_remote_launchpad_production_error(self):
+        ref = self.factory.makeGitRefRemote(
+            repository_url="https://git.launchpad.net/~owner/+git/name",
+            path="HEAD")
+        responses.add(
+            "GET",
+            "https://git.launchpad.net/~owner/+git/name/plain/dir/file?h=HEAD",
+            status=500)
+        self.assertRaises(GitRepositoryScanFault, ref.getBlob, "dir/file")
+
+    @responses.activate
     def test_remote_github_branch(self):
         ref = self.factory.makeGitRefRemote(
             repository_url="https://github.com/owner/name",