Merge ~cjwatson/launchpad-buildd:charm-snap-build-proxy into launchpad-buildd:master

[Colin Watson]

Honour proxy arguments when building charms

charmcraft essentially always runs pip3 as part of building the charm, and has no obvious way to provide a local package index. It looks as though we'll need to regard charms as requiring internet access to build.

[Cristian Gonzalez (cristiangsp)]

Looks good!

review: Approve

diff --git a/debian/changelog b/debian/changelog
index e66175d..0dcf32e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,6 +1,7 @@
 launchpad-buildd (198) UNRELEASED; urgency=medium
 
   * Run charmcraft in verbose mode.
+  * Honour proxy arguments when building charms.
 
  -- Colin Watson <[e-mail]>  Fri, 09 Jul 2021 14:08:58 +0100
 
diff --git a/lpbuildd/charm.py b/lpbuildd/charm.py
index e5c5fe9..0838fe7 100644
--- a/lpbuildd/charm.py
+++ b/lpbuildd/charm.py
@@ -11,6 +11,7 @@ from lpbuildd.debian import (
     DebianBuildState,
     DebianBuildManager,
     )
+from lpbuildd.snap import SnapBuildProxyMixin
 
 
 RETCODE_SUCCESS = 0
@@ -22,7 +23,7 @@ class CharmBuildState(DebianBuildState):
     BUILD_CHARM = "BUILD_CHARM"
 
 
-class CharmBuildManager(DebianBuildManager):
+class CharmBuildManager(SnapBuildProxyMixin, DebianBuildManager):
     """Build a charm."""
 
     backend_name = "lxd"
@@ -40,6 +41,9 @@ class CharmBuildManager(DebianBuildManager):
         self.git_path = extra_args.get("git_path")
         self.build_path = extra_args.get("build_path")
         self.channels = extra_args.get("channels", {})
+        self.proxy_url = extra_args.get("proxy_url")
+        self.revocation_endpoint = extra_args.get("revocation_endpoint")
+        self.proxy_service = None
 
         super(CharmBuildManager, self).initiate(files, chroot, extra_args)
 
diff --git a/lpbuildd/target/build_charm.py b/lpbuildd/target/build_charm.py
index 95da85e..7610a8b 100644
--- a/lpbuildd/target/build_charm.py
+++ b/lpbuildd/target/build_charm.py
@@ -2,7 +2,6 @@
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 from __future__ import print_function
-import functools
 
 __metaclass__ = type
 
@@ -14,6 +13,7 @@ import sys
 from lpbuildd.target.backend import check_path_escape
 from lpbuildd.target.build_snap import SnapChannelsAction
 from lpbuildd.target.operation import Operation
+from lpbuildd.target.snapbuildproxy import SnapBuildProxyOperationMixin
 from lpbuildd.target.snapstore import SnapStoreOperationMixin
 from lpbuildd.target.vcs import VCSOperationMixin
 
@@ -25,7 +25,8 @@ RETCODE_FAILURE_BUILD = 201
 logger = logging.getLogger(__name__)
 
 
-class BuildCharm(VCSOperationMixin, SnapStoreOperationMixin, Operation):
+class BuildCharm(SnapBuildProxyOperationMixin, VCSOperationMixin,
+                 SnapStoreOperationMixin, Operation):
 
     description = "Build a charm."
 
@@ -69,6 +70,9 @@ class BuildCharm(VCSOperationMixin, SnapStoreOperationMixin, Operation):
     def install(self):
         logger.info("Running install phase")
         deps = []
+        if self.args.proxy_url:
+            deps.extend(self.proxy_deps)
+            self.install_git_proxy()
         if self.args.backend == "lxd":
             # udev is installed explicitly to work around
             # https://bugs.launchpad.net/snapd/+bug/1731519.
@@ -99,7 +103,8 @@ class BuildCharm(VCSOperationMixin, SnapStoreOperationMixin, Operation):
     def repo(self):
         """Collect git or bzr branch."""
         logger.info("Running repo phase...")
-        self.vcs_fetch(self.args.name, cwd="/home/buildd")
+        env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
+        self.vcs_fetch(self.args.name, cwd="/home/buildd", env=env)
         self.save_status(self.buildd_path)
 
     def build(self):
@@ -109,8 +114,13 @@ class BuildCharm(VCSOperationMixin, SnapStoreOperationMixin, Operation):
             self.args.name,
             self.args.build_path)
         check_path_escape(self.buildd_path, build_context_path)
+        env = OrderedDict()
+        if self.args.proxy_url:
+            env["http_proxy"] = self.args.proxy_url
+            env["https_proxy"] = self.args.proxy_url
+            env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
         args = ["charmcraft", "build", "-v", "-f", build_context_path]
-        self.run_build_command(args)
+        self.run_build_command(args, env=env)
 
     def run(self):
         try:
diff --git a/lpbuildd/target/tests/test_build_charm.py b/lpbuildd/target/tests/test_build_charm.py
index 4e77943..8be840e 100644
--- a/lpbuildd/target/tests/test_build_charm.py
+++ b/lpbuildd/target/tests/test_build_charm.py
@@ -5,6 +5,7 @@ __metaclass__ = type
 
 import json
 import os
+import stat
 import subprocess
 from textwrap import dedent
 
@@ -13,6 +14,7 @@ from fixtures import (
     TempDir,
     )
 import responses
+from systemfixtures import FakeFilesystem
 from testtools.matchers import (
     AnyMatch,
     Equals,
@@ -171,11 +173,11 @@ class TestBuildCharm(TestCase):
             "--backend=fake", "--series=xenial", "--arch=amd64", "1",
             "--git-repository", "lp:foo",
             "--snap-store-proxy-url", "http://snap-store-proxy.example/",
-            "test-snap",
+            "test-image",
             ]
-        build_snap = parse_args(args=args).operation
-        build_snap.install()
-        self.assertThat(build_snap.backend.run.calls, MatchesListwise([
+        build_charm = parse_args(args=args).operation
+        build_charm.install()
+        self.assertThat(build_charm.backend.run.calls, MatchesListwise([
             RanAptGet("install", "git"),
             RanCommand(
                 ["snap", "ack", "/dev/stdin"], input_text=store_assertion),
@@ -184,6 +186,31 @@ class TestBuildCharm(TestCase):
             RanCommand(["mkdir", "-p", "/home/buildd"]),
             ]))
 
+    def test_install_proxy(self):
+        args = [
+            "build-charm",
+            "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+            "--git-repository", "lp:foo",
+            "--proxy-url", "http://proxy.example:3128/",
+            "test-image",
+            ]
+        build_charm = parse_args(args=args).operation
+        build_charm.bin = "/builderbin"
+        self.useFixture(FakeFilesystem()).add("/builderbin")
+        os.mkdir("/builderbin")
+        with open("/builderbin/snap-git-proxy", "w") as proxy_script:
+            proxy_script.write("proxy script\n")
+            os.fchmod(proxy_script.fileno(), 0o755)
+        build_charm.install()
+        self.assertThat(build_charm.backend.run.calls, MatchesListwise([
+            RanAptGet("install", "python3", "socat", "git"),
+            RanSnap("install", "charmcraft"),
+            RanCommand(["mkdir", "-p", "/home/buildd"]),
+            ]))
+        self.assertEqual(
+            (b"proxy script\n", stat.S_IFREG | 0o755),
+            build_charm.backend.backend_fs["/usr/local/bin/snap-git-proxy"])
+
     def test_repo_bzr(self):
         args = [
             "build-charm",
@@ -284,6 +311,39 @@ class TestBuildCharm(TestCase):
         with open(status_path) as status:
             self.assertEqual({"revision_id": "0" * 40}, json.load(status))
 
+    def test_repo_proxy(self):
+        args = [
+            "build-charm",
+            "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+            "--git-repository", "lp:foo",
+            "--proxy-url", "http://proxy.example:3128/",
+            "test-image",
+            ]
+        build_charm = parse_args(args=args).operation
+        build_charm.backend.build_path = self.useFixture(TempDir()).path
+        build_charm.backend.run = FakeRevisionID("0" * 40)
+        build_charm.repo()
+        env = {
+            "http_proxy": "http://proxy.example:3128/",
+            "https_proxy": "http://proxy.example:3128/",
+            "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy",
+            }
+        self.assertThat(build_charm.backend.run.calls, MatchesListwise([
+            RanBuildCommand(
+                ["git", "clone", "lp:foo", "test-image"],
+                cwd="/home/buildd", **env),
+            RanBuildCommand(
+                ["git", "submodule", "update", "--init", "--recursive"],
+                cwd="/home/buildd/test-image", **env),
+            RanBuildCommand(
+                ["git", "rev-parse", "HEAD^{}"],
+                cwd="/home/buildd/test-image", get_output=True,
+                universal_newlines=True),
+            ]))
+        status_path = os.path.join(build_charm.backend.build_path, "status")
+        with open(status_path) as status:
+            self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+
     def test_build(self):
         args = [
             "build-charm",
@@ -317,6 +377,27 @@ class TestBuildCharm(TestCase):
                 cwd="/home/buildd/test-image"),
             ]))
 
+    def test_build_proxy(self):
+        args = [
+            "build-charm",
+            "--backend=fake", "--series=xenial", "--arch=amd64", "1",
+            "--branch", "lp:foo", "--proxy-url", "http://proxy.example:3128/",
+            "test-image",
+            ]
+        build_charm = parse_args(args=args).operation
+        build_charm.build()
+        env = {
+            "http_proxy": "http://proxy.example:3128/",
+            "https_proxy": "http://proxy.example:3128/",
+            "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy",
+            }
+        self.assertThat(build_charm.backend.run.calls, MatchesListwise([
+            RanBuildCommand(
+                ["charmcraft", "build", "-v", "-f",
+                 "/home/buildd/test-image/."],
+                cwd="/home/buildd/test-image", **env),
+            ]))
+
     def test_run_succeeds(self):
         args = [
             "build-charm",
diff --git a/lpbuildd/tests/test_charm.py b/lpbuildd/tests/test_charm.py
index 407f732..8610bad 100644
--- a/lpbuildd/tests/test_charm.py
+++ b/lpbuildd/tests/test_charm.py
@@ -4,6 +4,10 @@
 __metaclass__ = type
 
 import os
+try:
+    from unittest import mock
+except ImportError:
+    import mock
 
 from fixtures import (
     EnvironmentVariable,
@@ -137,3 +141,15 @@ class TestCharmBuildManagerIteration(TestCase):
         self.assertEqual(
             self.buildmanager.iterate, self.buildmanager.iterators[-1])
         self.assertFalse(self.builder.wasCalled("buildFail"))
+
+    @mock.patch('lpbuildd.snap.urlopen')
+    def test_revokeProxyToken(self, urlopen_mock):
+        self.buildmanager.revocation_endpoint = "http://revoke_endpoint"
+        self.buildmanager.proxy_url = "http://username:password@proxy_url"
+        self.buildmanager.revokeProxyToken()
+        self.assertEqual(1, urlopen_mock.call_count)
+        request = urlopen_mock.call_args[0][0]
+        self.assertEqual(
+            {'Authorization': "Basic dXNlcm5hbWU6cGFzc3dvcmQ="},
+            request.headers)
+        self.assertEqual('http://revoke_endpoint', request.get_full_url())