Merge lp:~cjwatson/launchpad-buildd/snap-git-proxy into lp:launchpad-buildd

Proposed by Colin Watson
Status: Merged
Merged at revision: 219
Proposed branch: lp:~cjwatson/launchpad-buildd/snap-git-proxy
Merge into: lp:launchpad-buildd
Diff against target: 224 lines (+64/-7)
11 files modified
MANIFEST.in (+1/-0)
buildsnap (+12/-0)
debian/changelog (+1/-0)
debian/launchpad-buildd.install (+1/-0)
lpbuildd/slave.py (+3/-0)
lpbuildd/tests/buildlog (+1/-0)
lpbuildd/tests/buildlog.long (+2/-0)
lpbuildd/tests/test_1.diff (+8/-1)
lpbuildd/tests/test_2.diff (+6/-5)
lpbuildd/tests/test_buildd_slave.py (+1/-1)
snap-git-proxy (+28/-0)
To merge this branch: bzr merge lp:~cjwatson/launchpad-buildd/snap-git-proxy
Reviewer Review Type Date Requested Status
William Grant code Approve
Review via email: mp+323691@code.launchpad.net

Commit message

Configure a git:// proxy for snap builds (LP: #1663920).

To post a comment you must log in.
Revision history for this message
William Grant (wgrant) :
review: Approve (code)
Revision history for this message
Colin Watson (cjwatson) :
217. By Colin Watson

Sanitise snap proxy auth from logs.

Revision history for this message
Colin Watson (cjwatson) :
218. By Colin Watson

Merge trunk.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'MANIFEST.in'
2--- MANIFEST.in 2017-04-26 12:24:32 +0000
3+++ MANIFEST.in 2017-05-11 11:45:36 +0000
4@@ -15,6 +15,7 @@
5 include sbuildrc
6 include scan-for-processes
7 include slave-prep
8+include snap-git-proxy
9 include sudo-wrapper
10 include template-buildd-slave.conf
11 include test_buildd_generatetranslationtemplates
12
13=== modified file 'buildsnap'
14--- buildsnap 2017-04-03 12:30:15 +0000
15+++ buildsnap 2017-05-11 11:45:36 +0000
16@@ -117,7 +117,16 @@
17 deps.append("bzr")
18 else:
19 deps.append("git")
20+ if self.options.proxy_url:
21+ deps.extend(["python3", "socat"])
22 self.chroot(["apt-get", "-y", "install"] + deps)
23+ if self.options.proxy_url:
24+ subprocess.check_call([
25+ "sudo", "cp", "-a",
26+ os.path.join(os.path.dirname(__file__), "snap-git-proxy"),
27+ os.path.join(
28+ self.chroot_path, "usr", "local", "bin", "snap-git-proxy"),
29+ ])
30
31 def repo(self):
32 """Collect git or bzr branch."""
33@@ -126,6 +135,7 @@
34 if self.options.proxy_url:
35 env["http_proxy"] = self.options.proxy_url
36 env["https_proxy"] = self.options.proxy_url
37+ env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
38 if self.options.branch is not None:
39 self.run_build_command(['ls', '/build'])
40 cmd = ["bzr", "branch", self.options.branch, self.name]
41@@ -161,6 +171,7 @@
42 if self.options.proxy_url:
43 env["http_proxy"] = self.options.proxy_url
44 env["https_proxy"] = self.options.proxy_url
45+ env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
46 self.run_build_command(
47 ["snapcraft", "pull"],
48 path=os.path.join("/build", self.name),
49@@ -173,6 +184,7 @@
50 if self.options.proxy_url:
51 env["http_proxy"] = self.options.proxy_url
52 env["https_proxy"] = self.options.proxy_url
53+ env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy"
54 self.run_build_command(
55 ["snapcraft"], path=os.path.join("/build", self.name), env=env)
56
57
58=== modified file 'debian/changelog'
59--- debian/changelog 2017-05-11 08:52:10 +0000
60+++ debian/changelog 2017-05-11 11:45:36 +0000
61@@ -4,6 +4,7 @@
62 other XML-RPC status information (LP: #1679157).
63 * Write out trusted keys sent by buildd-manager (LP: #1626739).
64 * Add tests for lpbuildd.pottery, extracted from Launchpad.
65+ * Configure a git:// proxy for snap builds (LP: #1663920).
66
67 -- Colin Watson <cjwatson@ubuntu.com> Fri, 31 Mar 2017 15:31:10 +0100
68
69
70=== modified file 'debian/launchpad-buildd.install'
71--- debian/launchpad-buildd.install 2017-04-26 12:24:32 +0000
72+++ debian/launchpad-buildd.install 2017-05-11 11:45:36 +0000
73@@ -14,6 +14,7 @@
74 sbuild-package usr/share/launchpad-buildd/slavebin
75 scan-for-processes usr/share/launchpad-buildd/slavebin
76 slave-prep usr/share/launchpad-buildd/slavebin
77+snap-git-proxy usr/share/launchpad-buildd/slavebin
78 sudo-wrapper usr/share/launchpad-buildd/slavebin
79 umount-chroot usr/share/launchpad-buildd/slavebin
80 unpack-chroot usr/share/launchpad-buildd/slavebin
81
82=== modified file 'lpbuildd/slave.py'
83--- lpbuildd/slave.py 2017-05-11 08:42:49 +0000
84+++ lpbuildd/slave.py 2017-05-11 11:45:36 +0000
85@@ -37,9 +37,12 @@
86 # This regular expression will be used to remove authentication
87 # credentials from URLs.
88 password_re = re.compile('://([^:]+:[^@]+@)(\S+)')
89+ # Snap proxy passwords are UUIDs.
90+ snap_proxy_auth_re = re.compile(',proxyauth=[^:]+:[A-Za-z0-9-]+')
91
92 for line in text_seq:
93 sanitized_line = password_re.sub(r'://\2', line)
94+ sanitized_line = snap_proxy_auth_re.sub('', sanitized_line)
95 yield sanitized_line
96
97
98
99=== modified file 'lpbuildd/tests/buildlog'
100--- lpbuildd/tests/buildlog 2011-11-09 07:50:56 +0000
101+++ lpbuildd/tests/buildlog 2017-05-11 11:45:36 +0000
102@@ -21,3 +21,4 @@
103 Need to get 0B/2832kB of archives.
104 After unpacking 94.2kB of additional disk space will be used.
105 (Reading database ... 8942 files and directories currently installed.)
106+socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
107
108=== modified file 'lpbuildd/tests/buildlog.long'
109--- lpbuildd/tests/buildlog.long 2011-11-09 07:50:56 +0000
110+++ lpbuildd/tests/buildlog.long 2017-05-11 11:45:36 +0000
111@@ -21,6 +21,7 @@
112 Need to get 0B/2832kB of archives.
113 After unpacking 94.2kB of additional disk space will be used.
114 (Reading database ... 8942 files and directories currently installed.)
115+socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
116 RUN: /usr/share/launchpad-buildd/slavebin/unpack-chroot ['unpack-chroot', '370614-896976', '/home/buildd/filecache-default/a40e3c410938399b35051833fe5244f9ac6f3774']
117 Unpacking chroot for build 370614-896976
118 RUN: /usr/share/launchpad-buildd/slavebin/mount-chroot ['mount-chroot', '370614-896976']
119@@ -80,3 +81,4 @@
120 Get:3 http://scrub:this@ftpmaster.internal gutsy/main Packages [1085kB]
121 Get:4 http://ftpmaster.internal gutsy/universe Packages [3991kB]
122 Fetched 5142kB in 5s (1012kB/s)
123+socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
124
125=== modified file 'lpbuildd/tests/test_1.diff'
126--- lpbuildd/tests/test_1.diff 2011-11-09 07:50:56 +0000
127+++ lpbuildd/tests/test_1.diff 2017-05-11 11:45:36 +0000
128@@ -14,4 +14,11 @@
129 +Get:3 http://ftpmaster.internal gutsy/main Packages [1085kB]
130 Get:4 http://ftpmaster.internal gutsy/universe Packages [3991kB]
131 Fetched 5142kB in 5s (1012kB/s)
132- Reading package lists...
133\ No newline at end of file
134+ Reading package lists...
135+@@ -21,4 +21,4 @@
136+
137+ Need to get 0B/2832kB of archives.
138+ After unpacking 94.2kB of additional disk space will be used.
139+ (Reading database ... 8942 files and directories currently installed.)
140+-socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
141++socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128
142\ No newline at end of file
143
144=== modified file 'lpbuildd/tests/test_2.diff'
145--- lpbuildd/tests/test_2.diff 2011-11-09 07:50:56 +0000
146+++ lpbuildd/tests/test_2.diff 2017-05-11 11:45:36 +0000
147@@ -2,10 +2,9 @@
148
149 +++
150
151-@@ -1,11 +1,10 @@
152+@@ -1,10 +1,9 @@
153
154--ting chroot for build 370614-896976
155- RUN: /usr/share/launchpad-buildd/slavebin/apply-ogre-model ['apply-ogre-model', '370614-896976', 'universe']
156+-l ['apply-ogre-model', '370614-896976', 'universe']
157 Attempting OGRE for universe in build-370614-896976
158 RUN: /usr/share/launchpad-buildd/slavebin/update-debian-chroot ['update-debian-chroot', '370614-896976']
159 Updating debian chroot for build 370614-896976
160@@ -17,7 +16,7 @@
161 Get:4 http://ftpmaster.internal gutsy/universe Packages [3991kB]
162 Fetched 5142kB in 5s (1012kB/s)
163 Reading package lists...
164-@@ -26,8 +25,8 @@
165+@@ -25,9 +24,9 @@
166
167 Attempting OGRE for universe in build-370614-896976
168 RUN: /usr/share/launchpad-buildd/slavebin/update-debian-chroot ['update-debian-chroot', '370614-896976']
169@@ -29,4 +28,6 @@
170 +Get:2 http://ftpmaster.internal gutsy Release [65.9kB]
171 +Get:3 http://ftpmaster.internal gutsy/main Packages [1085kB]
172 Get:4 http://ftpmaster.internal gutsy/universe Packages [3991kB]
173- Fetched 5142kB in 5s (1012kB/s)
174\ No newline at end of file
175+ Fetched 5142kB in 5s (1012kB/s)
176+-socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah
177++socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128
178\ No newline at end of file
179
180=== modified file 'lpbuildd/tests/test_buildd_slave.py'
181--- lpbuildd/tests/test_buildd_slave.py 2014-05-10 17:40:29 +0000
182+++ lpbuildd/tests/test_buildd_slave.py 2017-05-11 11:45:36 +0000
183@@ -3,7 +3,7 @@
184
185 """Buildd Slave tests.
186
187-This file contains the follwoing tests:
188+This file contains the following tests:
189
190 * Basic authentication handling (used to download private sources);
191 * Build log sanitization (removal of passwords from private buildlog);
192
193=== added file 'snap-git-proxy'
194--- snap-git-proxy 1970-01-01 00:00:00 +0000
195+++ snap-git-proxy 2017-05-11 11:45:36 +0000
196@@ -0,0 +1,28 @@
197+#! /usr/bin/python3
198+# Copyright 2017 Canonical Ltd. This software is licensed under the
199+# GNU Affero General Public License version 3 (see the file LICENSE).
200+
201+"""Proxy the git protocol via http_proxy.
202+
203+Note that this is copied into the build chroot and run there.
204+"""
205+
206+import os
207+import sys
208+from urllib.parse import urlparse
209+
210+
211+def main():
212+ proxy_url = urlparse(os.environ["http_proxy"])
213+ proxy_arg = "PROXY:%s:%s:%s" % (
214+ proxy_url.hostname, sys.argv[1], sys.argv[2])
215+ if proxy_url.port:
216+ proxy_arg += ",proxyport=%s" % proxy_url.port
217+ if proxy_url.username:
218+ proxy_arg += ",proxyauth=%s:%s" % (
219+ proxy_url.username, proxy_url.password)
220+ os.execvp("socat", ["socat", "STDIO", proxy_arg])
221+
222+
223+if __name__ == "__main__":
224+ main()

Subscribers

People subscribed via source and target branches

to all changes: