Merge lp:~cjwatson/launchpad-buildd/snap-git-proxy into lp:launchpad-buildd

[Colin Watson]

Configure a git:// proxy for snap builds (LP: #1663920).

[William Grant (wgrant)]

review: Approve (code)

[lp:~cjwatson/launchpad-buildd/snap-git-proxy]

217. By Colin Watson on 2017-05-11

Sanitise snap proxy auth from logs.

[lp:~cjwatson/launchpad-buildd/snap-git-proxy]

218. By Colin Watson on 2017-05-11

Merge trunk.

=== modified file 'MANIFEST.in'
--- MANIFEST.in	2017-04-26 12:24:32 +0000
+++ MANIFEST.in	2017-05-11 11:45:36 +0000
@@ -15,6 +15,7 @@
 include sbuildrc
 include scan-for-processes
 include slave-prep
+include snap-git-proxy
 include sudo-wrapper
 include template-buildd-slave.conf
 include test_buildd_generatetranslationtemplates
 
=== modified file 'buildsnap'
--- buildsnap	2017-04-03 12:30:15 +0000
+++ buildsnap	2017-05-11 11:45:36 +0000
@@ -117,7 +117,16 @@
             deps.append("bzr")
         else:
             deps.append("git")
+        if self.options.proxy_url:
+            deps.extend(["python3", "socat"])
         self.chroot(["apt-get", "-y", "install"] + deps)
+        if self.options.proxy_url:
+            subprocess.check_call([
+                "sudo", "cp", "-a",
+                os.path.join(os.path.dirname(__file__), "snap-git-proxy"),
+                os.path.join(
+                    self.chroot_path, "usr", "local", "bin", "snap-git-proxy"),
+                ])
 
     def repo(self):
         """Collect git or bzr branch."""
@@ -126,6 +135,7 @@
         if self.options.proxy_url:
             env["http_proxy"] = self.options.proxy_url
             env["https_proxy"] = self.options.proxy_url
+            env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
         if self.options.branch is not None:
             self.run_build_command(['ls', '/build'])
             cmd = ["bzr", "branch", self.options.branch, self.name]
@@ -161,6 +171,7 @@
         if self.options.proxy_url:
             env["http_proxy"] = self.options.proxy_url
             env["https_proxy"] = self.options.proxy_url
+            env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
         self.run_build_command(
             ["snapcraft", "pull"],
             path=os.path.join("/build", self.name),
@@ -173,6 +184,7 @@
         if self.options.proxy_url:
             env["http_proxy"] = self.options.proxy_url
             env["https_proxy"] = self.options.proxy_url
+            env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
         self.run_build_command(
             ["snapcraft"], path=os.path.join("/build", self.name), env=env)
 
 
=== modified file 'debian/changelog'
--- debian/changelog	2017-05-11 08:52:10 +0000
+++ debian/changelog	2017-05-11 11:45:36 +0000
@@ -4,6 +4,7 @@
     other XML-RPC status information (LP: #1679157).
   * Write out trusted keys sent by buildd-manager (LP: #1626739).
   * Add tests for lpbuildd.pottery, extracted from Launchpad.
+  * Configure a git:// proxy for snap builds (LP: #1663920).
 
  -- Colin Watson <[e-mail]>  Fri, 31 Mar 2017 15:31:10 +0100
 
 
=== modified file 'debian/launchpad-buildd.install'
--- debian/launchpad-buildd.install	2017-04-26 12:24:32 +0000
+++ debian/launchpad-buildd.install	2017-05-11 11:45:36 +0000
@@ -14,6 +14,7 @@
 sbuild-package			usr/share/launchpad-buildd/slavebin
 scan-for-processes		usr/share/launchpad-buildd/slavebin
 slave-prep			usr/share/launchpad-buildd/slavebin
+snap-git-proxy			usr/share/launchpad-buildd/slavebin
 sudo-wrapper			usr/share/launchpad-buildd/slavebin
 umount-chroot			usr/share/launchpad-buildd/slavebin
 unpack-chroot			usr/share/launchpad-buildd/slavebin
 
=== modified file 'lpbuildd/slave.py'
--- lpbuildd/slave.py	2017-05-11 08:42:49 +0000
+++ lpbuildd/slave.py	2017-05-11 11:45:36 +0000
@@ -37,9 +37,12 @@
     # This regular expression will be used to remove authentication
     # credentials from URLs.
     password_re = re.compile('://([^:]+:[^@]+@)(\S+)')
+    # Snap proxy passwords are UUIDs.
+    snap_proxy_auth_re = re.compile(',proxyauth=[^:]+:[A-Za-z0-9-]+')
 
     for line in text_seq:
         sanitized_line = password_re.sub(r'://\2', line)
+        sanitized_line = snap_proxy_auth_re.sub('', sanitized_line)
         yield sanitized_line
 
 
 
=== modified file 'lpbuildd/tests/buildlog'
--- lpbuildd/tests/buildlog	2011-11-09 07:50:56 +0000
+++ lpbuildd/tests/buildlog	2017-05-11 11:45:36 +0000
@@ -21,3 +21,4 @@
 Need to get 0B/2832kB of archives.
 After unpacking 94.2kB of additional disk space will be used.
 (Reading database ... 8942 files and directories currently installed.)
+socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
 
=== modified file 'lpbuildd/tests/buildlog.long'
--- lpbuildd/tests/buildlog.long	2011-11-09 07:50:56 +0000
+++ lpbuildd/tests/buildlog.long	2017-05-11 11:45:36 +0000
@@ -21,6 +21,7 @@
 Need to get 0B/2832kB of archives.
 After unpacking 94.2kB of additional disk space will be used.
 (Reading database ... 8942 files and directories currently installed.)
+socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
 RUN: /usr/share/launchpad-buildd/slavebin/unpack-chroot ['unpack-chroot', '370614-896976', '/home/buildd/filecache-default/a40e3c410938399b35051833fe5244f9ac6f3774']
 Unpacking chroot for build 370614-896976
 RUN: /usr/share/launchpad-buildd/slavebin/mount-chroot ['mount-chroot', '370614-896976']
@@ -80,3 +81,4 @@
 Get:3 http://scrub:[e-mail] gutsy/main Packages [1085kB]
 Get:4 http://ftpmaster.internal gutsy/universe Packages [3991kB]
 Fetched 5142kB in 5s (1012kB/s)
+socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
 
=== modified file 'lpbuildd/tests/test_1.diff'
--- lpbuildd/tests/test_1.diff	2011-11-09 07:50:56 +0000
+++ lpbuildd/tests/test_1.diff	2017-05-11 11:45:36 +0000
@@ -14,4 +14,11 @@
 +Get:3 http://ftpmaster.internal gutsy/main Packages [1085kB]
  Get:4 http://ftpmaster.internal gutsy/universe Packages [3991kB]
  Fetched 5142kB in 5s (1012kB/s)
- Reading package lists...
 \ No newline at end of file
+ Reading package lists...
+@@ -21,4 +21,4 @@
+
+ Need to get 0B/2832kB of archives.
+ After unpacking 94.2kB of additional disk space will be used.
+ (Reading database ... 8942 files and directories currently installed.)
+-socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
++socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128
 \ No newline at end of file
 
=== modified file 'lpbuildd/tests/test_2.diff'
--- lpbuildd/tests/test_2.diff	2011-11-09 07:50:56 +0000
+++ lpbuildd/tests/test_2.diff	2017-05-11 11:45:36 +0000
@@ -2,10 +2,9 @@
 
 +++  
 
-@@ -1,11 +1,10 @@
+@@ -1,10 +1,9 @@
 
--ting chroot for build 370614-896976
- RUN: /usr/share/launchpad-buildd/slavebin/apply-ogre-model ['apply-ogre-model', '370614-896976', 'universe']
+-l ['apply-ogre-model', '370614-896976', 'universe']
  Attempting OGRE for universe in build-370614-896976
  RUN: /usr/share/launchpad-buildd/slavebin/update-debian-chroot ['update-debian-chroot', '370614-896976']
  Updating debian chroot for build 370614-896976
@@ -17,7 +16,7 @@
  Get:4 http://ftpmaster.internal gutsy/universe Packages [3991kB]
  Fetched 5142kB in 5s (1012kB/s)
  Reading package lists...
-@@ -26,8 +25,8 @@
+@@ -25,9 +24,9 @@
 
  Attempting OGRE for universe in build-370614-896976
  RUN: /usr/share/launchpad-buildd/slavebin/update-debian-chroot ['update-debian-chroot', '370614-896976']
@@ -29,4 +28,6 @@
 +Get:2 http://ftpmaster.internal gutsy Release [65.9kB]
 +Get:3 http://ftpmaster.internal gutsy/main Packages [1085kB]
  Get:4 http://ftpmaster.internal gutsy/universe Packages [3991kB]
- Fetched 5142kB in 5s (1012kB/s)
 \ No newline at end of file
+ Fetched 5142kB in 5s (1012kB/s)
+-socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
++socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128
 \ No newline at end of file
 
=== modified file 'lpbuildd/tests/test_buildd_slave.py'
--- lpbuildd/tests/test_buildd_slave.py	2014-05-10 17:40:29 +0000
+++ lpbuildd/tests/test_buildd_slave.py	2017-05-11 11:45:36 +0000
@@ -3,7 +3,7 @@
 
 """Buildd Slave tests.
 
-This file contains the follwoing tests:
+This file contains the following tests:
 
  * Basic authentication handling (used to download private sources);
  * Build log sanitization (removal of passwords from private buildlog);
 
=== added file 'snap-git-proxy'
--- snap-git-proxy	1970-01-01 00:00:00 +0000
+++ snap-git-proxy	2017-05-11 11:45:36 +0000
@@ -0,0 +1,28 @@
+#! /usr/bin/python3
+# Copyright 2017 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Proxy the git protocol via http_proxy.
+
+Note that this is copied into the build chroot and run there.
+"""
+
+import os
+import sys
+from urllib.parse import urlparse
+
+
+def main():
+    proxy_url = urlparse(os.environ["http_proxy"])
+    proxy_arg = "PROXY:%s:%s:%s" % (
+        proxy_url.hostname, sys.argv[1], sys.argv[2])
+    if proxy_url.port:
+        proxy_arg += ",proxyport=%s" % proxy_url.port
+    if proxy_url.username:
+        proxy_arg += ",proxyauth=%s:%s" % (
+            proxy_url.username, proxy_url.password)
+    os.execvp("socat", ["socat", "STDIO", proxy_arg])
+
+
+if __name__ == "__main__":
+    main()