Merge ~cjwatson/launchpad-buildd:rename-builder-proxy into launchpad-buildd:master
- Git
- lp:~cjwatson/launchpad-buildd
- rename-builder-proxy
- Merge into master
Proposed by
Colin Watson
Status: | Merged |
---|---|
Approved by: | Colin Watson |
Approved revision: | 5e8e02d5b4d63e8420397000ca8992eca646b94a |
Merge reported by: | Otto Co-Pilot |
Merged at revision: | not available |
Proposed branch: | ~cjwatson/launchpad-buildd:rename-builder-proxy |
Merge into: | launchpad-buildd:master |
Diff against target: |
1165 lines (+375/-309) 25 files modified
MANIFEST.in (+1/-1) buildd-genconfig (+3/-3) debian/changelog (+2/-0) debian/launchpad-buildd.install (+1/-1) debian/postinst (+1/-1) debian/upgrade-config (+56/-0) lpbuildd/builder.py (+3/-3) lpbuildd/charm.py (+2/-2) lpbuildd/oci.py (+2/-2) lpbuildd/proxy.py (+261/-0) lpbuildd/snap.py (+2/-254) lpbuildd/target/build_charm.py (+3/-3) lpbuildd/target/build_oci.py (+2/-2) lpbuildd/target/build_snap.py (+4/-4) lpbuildd/target/proxy.py (+6/-6) lpbuildd/target/tests/test_build_charm.py (+5/-4) lpbuildd/target/tests/test_build_oci.py (+3/-3) lpbuildd/target/tests/test_build_snap.py (+5/-5) lpbuildd/tests/buildlog (+1/-1) lpbuildd/tests/buildlog.long (+2/-2) lpbuildd/tests/test_1.diff (+2/-2) lpbuildd/tests/test_2.diff (+3/-3) lpbuildd/tests/test_charm.py (+1/-1) lpbuildd/tests/test_snap.py (+3/-3) template-buildd-slave.conf (+1/-3) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Tom Wardill (community) | Approve | ||
Cristian Gonzalez (community) | Approve | ||
Review via email: mp+406005@code.launchpad.net |
Commit message
Rename "snap proxy" to "builder proxy"
Description of the change
It's used for several different build types nowadays, and is too easily confused with the snap store proxy.
The XML-RPC protocol between Launchpad and launchpad-buildd remains unchanged, but the launchpad-buildd configuration file needs modifying (done for typical cases by debian/
To post a comment you must log in.
Revision history for this message
Tom Wardill (twom) : | # |
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/MANIFEST.in b/MANIFEST.in |
2 | index 44e8248..f061118 100644 |
3 | --- a/MANIFEST.in |
4 | +++ b/MANIFEST.in |
5 | @@ -3,8 +3,8 @@ include Makefile |
6 | include bin/builder-prep |
7 | include bin/buildrecipe |
8 | include bin/in-target |
9 | +include bin/lpbuildd-git-proxy |
10 | include bin/sbuild-package |
11 | -include bin/snap-git-proxy |
12 | include bin/test_buildd_generatetranslationtemplates |
13 | include bin/test_buildd_recipe |
14 | include buildd-genconfig |
15 | diff --git a/bin/snap-git-proxy b/bin/lpbuildd-git-proxy |
16 | similarity index 100% |
17 | rename from bin/snap-git-proxy |
18 | rename to bin/lpbuildd-git-proxy |
19 | diff --git a/buildd-genconfig b/buildd-genconfig |
20 | index 467dee4..2615b88 100755 |
21 | --- a/buildd-genconfig |
22 | +++ b/buildd-genconfig |
23 | @@ -37,8 +37,8 @@ parser.add_option("-t", "--template", dest="TEMPLATE", |
24 | metavar="FILE", |
25 | default="/usr/share/launchpad-buildd/template-buildd-slave.conf") |
26 | |
27 | -parser.add_option("--snap-proxy-port", dest="SNAPPROXYPORT", |
28 | - help="the port the local snap proxy binds to", |
29 | +parser.add_option("--proxy-port", dest="PROXYPORT", |
30 | + help="the port the local builder proxy binds to", |
31 | metavar="PORT", |
32 | default="8222") |
33 | |
34 | @@ -51,7 +51,7 @@ replacements = { |
35 | "@BINDHOST@": options.BINDHOST, |
36 | "@ARCHTAG@": options.ARCHTAG, |
37 | "@BINDPORT@": options.BINDPORT, |
38 | - "@SNAPPROXYPORT@": options.SNAPPROXYPORT, |
39 | + "@PROXYPORT@": options.PROXYPORT, |
40 | } |
41 | |
42 | for replacement_key in replacements: |
43 | diff --git a/debian/changelog b/debian/changelog |
44 | index 5668969..33c356e 100644 |
45 | --- a/debian/changelog |
46 | +++ b/debian/changelog |
47 | @@ -1,6 +1,8 @@ |
48 | launchpad-buildd (200) UNRELEASED; urgency=medium |
49 | |
50 | * Add core22 to the list of supported core snap names. |
51 | + * Rename "snap proxy" to "builder proxy", since it's used for several |
52 | + different build types nowadays. |
53 | |
54 | -- Colin Watson <cjwatson@ubuntu.com> Fri, 13 Aug 2021 14:05:55 +0100 |
55 | |
56 | diff --git a/debian/launchpad-buildd.install b/debian/launchpad-buildd.install |
57 | index 981c763..95db295 100644 |
58 | --- a/debian/launchpad-buildd.install |
59 | +++ b/debian/launchpad-buildd.install |
60 | @@ -2,8 +2,8 @@ bin/builder-prep usr/share/launchpad-buildd/bin |
61 | bin/buildrecipe usr/share/launchpad-buildd/bin |
62 | bin/check-implicit-pointer-functions usr/share/launchpad-buildd/bin |
63 | bin/in-target usr/share/launchpad-buildd/bin |
64 | +bin/lpbuildd-git-proxy usr/share/launchpad-buildd/bin |
65 | bin/sbuild-package usr/share/launchpad-buildd/bin |
66 | -bin/snap-git-proxy usr/share/launchpad-buildd/bin |
67 | buildd-genconfig usr/share/launchpad-buildd |
68 | debian/upgrade-config usr/share/launchpad-buildd |
69 | default/launchpad-buildd etc/default |
70 | diff --git a/debian/postinst b/debian/postinst |
71 | index 7118bb1..1ad6123 100644 |
72 | --- a/debian/postinst |
73 | +++ b/debian/postinst |
74 | @@ -14,7 +14,7 @@ BUILDDUID=${BUILDD_UID:-2001} |
75 | |
76 | make_buildd() |
77 | { |
78 | - /usr/share/launchpad-buildd/buildd-genconfig --name=default --host=0.0.0.0 --port=8221 --snap-proxy-port=8222 > \ |
79 | + /usr/share/launchpad-buildd/buildd-genconfig --name=default --host=0.0.0.0 --port=8221 --proxy-port=8222 > \ |
80 | /etc/launchpad-buildd/default |
81 | echo Default buildd created. |
82 | } |
83 | diff --git a/debian/upgrade-config b/debian/upgrade-config |
84 | index 3c7b9d4..3145a89 100755 |
85 | --- a/debian/upgrade-config |
86 | +++ b/debian/upgrade-config |
87 | @@ -7,6 +7,18 @@ |
88 | |
89 | from __future__ import print_function |
90 | |
91 | +try: |
92 | + from configparser import ( |
93 | + ConfigParser as SafeConfigParser, |
94 | + NoOptionError, |
95 | + NoSectionError, |
96 | + ) |
97 | +except ImportError: |
98 | + from ConfigParser import ( |
99 | + SafeConfigParser, |
100 | + NoOptionError, |
101 | + NoSectionError, |
102 | + ) |
103 | import os |
104 | import re |
105 | import sys |
106 | @@ -219,6 +231,48 @@ def upgrade_to_190(): |
107 | line = "[builder]\n" |
108 | out_file.write(line) |
109 | |
110 | +def upgrade_to_200(): |
111 | + print("Upgrading %s to version 200" % conf_file) |
112 | + |
113 | + # We need to move snapmanager.proxyport to builder.proxyport, so start |
114 | + # by parsing the existing file to find the current value. |
115 | + conf = SafeConfigParser() |
116 | + conf.read(conf_file) |
117 | + try: |
118 | + proxyport = conf.get("snapmanager", "proxyport") |
119 | + except (NoOptionError, NoSectionError): |
120 | + proxyport = None |
121 | + |
122 | + os.rename(conf_file, conf_file + "-prev200~") |
123 | + with open(conf_file + "-prev200~", "r") as in_file: |
124 | + with open(conf_file, "w") as out_file: |
125 | + in_builder = False |
126 | + in_snapmanager = False |
127 | + wrote_blank = False |
128 | + for line in in_file: |
129 | + if line.startswith("[builder]"): |
130 | + in_builder = True |
131 | + elif in_builder and (line.startswith("[") or not line.strip()): |
132 | + if proxyport is not None: |
133 | + out_file.write("proxyport = %s\n" % proxyport) |
134 | + in_builder = False |
135 | + elif line.startswith("[snapmanager]"): |
136 | + # Delete this section. |
137 | + in_snapmanager = True |
138 | + |
139 | + if not line.strip(): |
140 | + wrote_blank = True |
141 | + elif not in_snapmanager: |
142 | + if wrote_blank: |
143 | + out_file.write("\n") |
144 | + out_file.write(line) |
145 | + wrote_blank = False |
146 | + |
147 | + if (in_snapmanager and |
148 | + not line.startswith("[snapmanager]") and |
149 | + (line.startswith("[") or not line.strip())): |
150 | + in_snapmanager = False |
151 | + |
152 | if __name__ == "__main__": |
153 | old_version = re.sub(r'[~-].*', '', old_version) |
154 | if apt_pkg.version_compare(old_version, "12") < 0: |
155 | @@ -249,3 +303,5 @@ if __name__ == "__main__": |
156 | upgrade_to_162() |
157 | if apt_pkg.version_compare(old_version, "190") < 0: |
158 | upgrade_to_190() |
159 | + if apt_pkg.version_compare(old_version, "200") < 0: |
160 | + upgrade_to_200() |
161 | diff --git a/lpbuildd/builder.py b/lpbuildd/builder.py |
162 | index 5e459be..d83c10f 100644 |
163 | --- a/lpbuildd/builder.py |
164 | +++ b/lpbuildd/builder.py |
165 | @@ -49,12 +49,12 @@ def _sanitizeURLs(bytes_seq): |
166 | # This regular expression will be used to remove authentication |
167 | # credentials from URLs. |
168 | password_re = re.compile(br'://([^:]+:[^@]+@)(\S+)') |
169 | - # Snap proxy passwords are UUIDs. |
170 | - snap_proxy_auth_re = re.compile(br',proxyauth=[^:]+:[A-Za-z0-9-]+') |
171 | + # Builder proxy passwords are UUIDs. |
172 | + proxy_auth_re = re.compile(br',proxyauth=[^:]+:[A-Za-z0-9-]+') |
173 | |
174 | for line in bytes_seq: |
175 | sanitized_line = password_re.sub(br'://\2', line) |
176 | - sanitized_line = snap_proxy_auth_re.sub(b'', sanitized_line) |
177 | + sanitized_line = proxy_auth_re.sub(b'', sanitized_line) |
178 | yield sanitized_line |
179 | |
180 | |
181 | diff --git a/lpbuildd/charm.py b/lpbuildd/charm.py |
182 | index dcfc546..520b97c 100644 |
183 | --- a/lpbuildd/charm.py |
184 | +++ b/lpbuildd/charm.py |
185 | @@ -11,7 +11,7 @@ from lpbuildd.debian import ( |
186 | DebianBuildState, |
187 | DebianBuildManager, |
188 | ) |
189 | -from lpbuildd.snap import SnapBuildProxyMixin |
190 | +from lpbuildd.proxy import BuildManagerProxyMixin |
191 | |
192 | |
193 | RETCODE_SUCCESS = 0 |
194 | @@ -23,7 +23,7 @@ class CharmBuildState(DebianBuildState): |
195 | BUILD_CHARM = "BUILD_CHARM" |
196 | |
197 | |
198 | -class CharmBuildManager(SnapBuildProxyMixin, DebianBuildManager): |
199 | +class CharmBuildManager(BuildManagerProxyMixin, DebianBuildManager): |
200 | """Build a charm.""" |
201 | |
202 | backend_name = "lxd" |
203 | diff --git a/lpbuildd/oci.py b/lpbuildd/oci.py |
204 | index 15bf5b8..aa28a03 100644 |
205 | --- a/lpbuildd/oci.py |
206 | +++ b/lpbuildd/oci.py |
207 | @@ -22,7 +22,7 @@ from lpbuildd.debian import ( |
208 | DebianBuildManager, |
209 | DebianBuildState, |
210 | ) |
211 | -from lpbuildd.snap import SnapBuildProxyMixin |
212 | +from lpbuildd.proxy import BuildManagerProxyMixin |
213 | |
214 | |
215 | RETCODE_SUCCESS = 0 |
216 | @@ -34,7 +34,7 @@ class OCIBuildState(DebianBuildState): |
217 | BUILD_OCI = "BUILD_OCI" |
218 | |
219 | |
220 | -class OCIBuildManager(SnapBuildProxyMixin, DebianBuildManager): |
221 | +class OCIBuildManager(BuildManagerProxyMixin, DebianBuildManager): |
222 | """Build an OCI Image.""" |
223 | |
224 | backend_name = "lxd" |
225 | diff --git a/lpbuildd/proxy.py b/lpbuildd/proxy.py |
226 | new file mode 100644 |
227 | index 0000000..908770d |
228 | --- /dev/null |
229 | +++ b/lpbuildd/proxy.py |
230 | @@ -0,0 +1,261 @@ |
231 | +# Copyright 2015-2021 Canonical Ltd. This software is licensed under the |
232 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
233 | + |
234 | +from __future__ import print_function |
235 | + |
236 | +__metaclass__ = type |
237 | + |
238 | +import base64 |
239 | +import io |
240 | + |
241 | +from six.moves.urllib.error import ( |
242 | + HTTPError, |
243 | + URLError, |
244 | + ) |
245 | +from six.moves.urllib.parse import urlparse |
246 | +from six.moves.urllib.request import ( |
247 | + Request, |
248 | + urlopen, |
249 | + ) |
250 | +from twisted.application import strports |
251 | +from twisted.internet import reactor |
252 | +from twisted.internet.interfaces import IHalfCloseableProtocol |
253 | +from twisted.python.compat import intToBytes |
254 | +from twisted.web import ( |
255 | + http, |
256 | + proxy, |
257 | + ) |
258 | +from zope.interface import implementer |
259 | + |
260 | + |
261 | +class BuilderProxyClient(proxy.ProxyClient): |
262 | + |
263 | + def __init__(self, command, rest, version, headers, data, father): |
264 | + proxy.ProxyClient.__init__( |
265 | + self, command, rest, version, headers, data, father) |
266 | + # Why doesn't ProxyClient at least store this? |
267 | + self.version = version |
268 | + # We must avoid calling self.father.finish in the event that its |
269 | + # connection was already lost, i.e. if the original client |
270 | + # disconnects first (which is particularly likely in the case of |
271 | + # CONNECT). |
272 | + d = self.father.notifyFinish() |
273 | + d.addBoth(self.requestFinished) |
274 | + |
275 | + def connectionMade(self): |
276 | + proxy.ProxyClient.connectionMade(self) |
277 | + self.father.setChildClient(self) |
278 | + |
279 | + def sendCommand(self, command, path): |
280 | + # For some reason, HTTPClient.sendCommand doesn't preserve the |
281 | + # protocol version. |
282 | + self.transport.writeSequence( |
283 | + [command, b' ', path, b' ', self.version, b'\r\n']) |
284 | + |
285 | + def handleEndHeaders(self): |
286 | + self.father.handleEndHeaders() |
287 | + |
288 | + def sendData(self, data): |
289 | + self.transport.write(data) |
290 | + |
291 | + def endData(self): |
292 | + if self.transport is not None: |
293 | + self.transport.loseWriteConnection() |
294 | + |
295 | + def requestFinished(self, result): |
296 | + self._finished = True |
297 | + self.transport.loseConnection() |
298 | + |
299 | + |
300 | +class BuilderProxyClientFactory(proxy.ProxyClientFactory): |
301 | + |
302 | + protocol = BuilderProxyClient |
303 | + |
304 | + |
305 | +class BuilderProxyRequest(http.Request): |
306 | + |
307 | + child_client = None |
308 | + _request_buffer = None |
309 | + _request_data_done = False |
310 | + |
311 | + def setChildClient(self, child_client): |
312 | + self.child_client = child_client |
313 | + if self._request_buffer is not None: |
314 | + self.child_client.sendData(self._request_buffer.getvalue()) |
315 | + self._request_buffer = None |
316 | + if self._request_data_done: |
317 | + self.child_client.endData() |
318 | + |
319 | + def allHeadersReceived(self, command, path, version): |
320 | + # Normally done in `requestReceived`, but we disable that since it |
321 | + # does other things we don't want. |
322 | + self.method, self.uri, self.clientproto = command, path, version |
323 | + self.client = self.channel.transport.getPeer() |
324 | + self.host = self.channel.transport.getHost() |
325 | + |
326 | + remote_parsed = urlparse(self.channel.factory.remote_url) |
327 | + request_parsed = urlparse(path) |
328 | + headers = self.getAllHeaders().copy() |
329 | + if b"host" not in headers and request_parsed.netloc: |
330 | + headers[b"host"] = request_parsed.netloc |
331 | + if remote_parsed.username: |
332 | + auth = (remote_parsed.username + ":" + |
333 | + remote_parsed.password).encode("ASCII") |
334 | + authHeader = b"Basic " + base64.b64encode(auth) |
335 | + headers[b"proxy-authorization"] = authHeader |
336 | + self.client_factory = BuilderProxyClientFactory( |
337 | + command, path, version, headers, b"", self) |
338 | + reactor.connectTCP( |
339 | + remote_parsed.hostname, remote_parsed.port, self.client_factory) |
340 | + |
341 | + def requestReceived(self, command, path, version): |
342 | + # We do most of our work in `allHeadersReceived` instead. |
343 | + pass |
344 | + |
345 | + def rawDataReceived(self, data): |
346 | + if self.child_client is not None: |
347 | + if not self._request_data_done: |
348 | + self.child_client.sendData(data) |
349 | + else: |
350 | + if self._request_buffer is None: |
351 | + self._request_buffer = io.BytesIO() |
352 | + self._request_buffer.write(data) |
353 | + |
354 | + def handleEndHeaders(self): |
355 | + # Cut-down version of Request.write. We must avoid switching to |
356 | + # chunked encoding for the sake of CONNECT; since our actual |
357 | + # response data comes from another proxy, we can cut some corners. |
358 | + if self.startedWriting: |
359 | + return |
360 | + self.startedWriting = 1 |
361 | + l = [] |
362 | + l.append( |
363 | + self.clientproto + b" " + intToBytes(self.code) + b" " + |
364 | + self.code_message + b"\r\n") |
365 | + for name, values in self.responseHeaders.getAllRawHeaders(): |
366 | + for value in values: |
367 | + l.extend([name, b": ", value, b"\r\n"]) |
368 | + l.append(b"\r\n") |
369 | + self.transport.writeSequence(l) |
370 | + |
371 | + def write(self, data): |
372 | + if self.channel is not None: |
373 | + self.channel.resetTimeout() |
374 | + http.Request.write(self, data) |
375 | + |
376 | + def endData(self): |
377 | + if self.child_client is not None: |
378 | + self.child_client.endData() |
379 | + self._request_data_done = True |
380 | + |
381 | + |
382 | +@implementer(IHalfCloseableProtocol) |
383 | +class BuilderProxy(http.HTTPChannel): |
384 | + """A channel that streams request data. |
385 | + |
386 | + The stock HTTPChannel isn't quite suitable for our needs, because it |
387 | + expects to read the entire request data before passing control to the |
388 | + request. This doesn't work well for CONNECT. |
389 | + """ |
390 | + |
391 | + requestFactory = BuilderProxyRequest |
392 | + |
393 | + def checkPersistence(self, request, version): |
394 | + # ProxyClient.__init__ forces "Connection: close". |
395 | + return False |
396 | + |
397 | + def allHeadersReceived(self): |
398 | + http.HTTPChannel.allHeadersReceived(self) |
399 | + self.requests[-1].allHeadersReceived( |
400 | + self._command, self._path, self._version) |
401 | + if self._command == b"CONNECT": |
402 | + # This is a lie, but we don't want HTTPChannel to decide that |
403 | + # the request is finished just because a CONNECT request |
404 | + # (naturally) has no Content-Length. |
405 | + self.length = -1 |
406 | + |
407 | + def rawDataReceived(self, data): |
408 | + self.resetTimeout() |
409 | + if self.requests: |
410 | + self.requests[-1].rawDataReceived(data) |
411 | + |
412 | + def readConnectionLost(self): |
413 | + for request in self.requests: |
414 | + request.endData() |
415 | + |
416 | + def writeConnectionLost(self): |
417 | + pass |
418 | + |
419 | + |
420 | +class BuilderProxyFactory(http.HTTPFactory): |
421 | + |
422 | + protocol = BuilderProxy |
423 | + |
424 | + def __init__(self, manager, remote_url, *args, **kwargs): |
425 | + http.HTTPFactory.__init__(self, *args, **kwargs) |
426 | + self.manager = manager |
427 | + self.remote_url = remote_url |
428 | + |
429 | + def log(self, request): |
430 | + # Log requests to the build log rather than to Twisted. |
431 | + # Reimplement log formatting because there's no point logging the IP |
432 | + # here. |
433 | + referrer = http._escape(request.getHeader(b"referer") or b"-") |
434 | + agent = http._escape(request.getHeader(b"user-agent") or b"-") |
435 | + line = ( |
436 | + u'%(timestamp)s "%(method)s %(uri)s %(protocol)s" ' |
437 | + u'%(code)d %(length)s "%(referrer)s" "%(agent)s"\n' % { |
438 | + 'timestamp': self._logDateTime, |
439 | + 'method': http._escape(request.method), |
440 | + 'uri': http._escape(request.uri), |
441 | + 'protocol': http._escape(request.clientproto), |
442 | + 'code': request.code, |
443 | + 'length': request.sentLength or "-", |
444 | + 'referrer': referrer, |
445 | + 'agent': agent, |
446 | + }) |
447 | + self.manager._builder.log(line.encode("UTF-8")) |
448 | + |
449 | + |
450 | +class BuildManagerProxyMixin: |
451 | + |
452 | + def startProxy(self): |
453 | + """Start the local builder proxy, if necessary.""" |
454 | + if not self.proxy_url: |
455 | + return [] |
456 | + proxy_port = self._builder._config.get("builder", "proxyport") |
457 | + proxy_factory = BuilderProxyFactory(self, self.proxy_url, timeout=60) |
458 | + self.proxy_service = strports.service( |
459 | + "tcp:%s" % proxy_port, proxy_factory) |
460 | + self.proxy_service.setServiceParent(self._builder.service) |
461 | + if self.backend_name == "lxd": |
462 | + proxy_host = self.backend.ipv4_network.ip |
463 | + else: |
464 | + proxy_host = "localhost" |
465 | + return ["--proxy-url", "http://{}:{}/".format(proxy_host, proxy_port)] |
466 | + |
467 | + def stopProxy(self): |
468 | + """Stop the local builder proxy, if necessary.""" |
469 | + if self.proxy_service is None: |
470 | + return |
471 | + self.proxy_service.disownServiceParent() |
472 | + self.proxy_service = None |
473 | + |
474 | + def revokeProxyToken(self): |
475 | + """Revoke builder proxy token.""" |
476 | + if not self.revocation_endpoint: |
477 | + return |
478 | + self._builder.log("Revoking proxy token...\n") |
479 | + url = urlparse(self.proxy_url) |
480 | + auth = "{}:{}".format(url.username, url.password) |
481 | + headers = { |
482 | + "Authorization": "Basic {}".format( |
483 | + base64.b64encode(auth.encode('utf-8')).decode('utf-8')) |
484 | + } |
485 | + req = Request(self.revocation_endpoint, None, headers) |
486 | + req.get_method = lambda: "DELETE" |
487 | + try: |
488 | + urlopen(req) |
489 | + except (HTTPError, URLError) as e: |
490 | + self._builder.log( |
491 | + "Unable to revoke token for %s: %s" % (url.username, e)) |
492 | diff --git a/lpbuildd/snap.py b/lpbuildd/snap.py |
493 | index d720d49..2a5ffc4 100644 |
494 | --- a/lpbuildd/snap.py |
495 | +++ b/lpbuildd/snap.py |
496 | @@ -5,8 +5,6 @@ from __future__ import print_function |
497 | |
498 | __metaclass__ = type |
499 | |
500 | -import base64 |
501 | -import io |
502 | import json |
503 | import os |
504 | import sys |
505 | @@ -15,30 +13,13 @@ from six.moves.configparser import ( |
506 | NoOptionError, |
507 | NoSectionError, |
508 | ) |
509 | -from six.moves.urllib.error import ( |
510 | - HTTPError, |
511 | - URLError, |
512 | - ) |
513 | -from six.moves.urllib.parse import urlparse |
514 | -from six.moves.urllib.request import ( |
515 | - Request, |
516 | - urlopen, |
517 | - ) |
518 | -from twisted.application import strports |
519 | -from twisted.internet import reactor |
520 | -from twisted.internet.interfaces import IHalfCloseableProtocol |
521 | -from twisted.python.compat import intToBytes |
522 | -from twisted.web import ( |
523 | - http, |
524 | - proxy, |
525 | - ) |
526 | -from zope.interface import implementer |
527 | |
528 | from lpbuildd.debian import ( |
529 | DebianBuildManager, |
530 | DebianBuildState, |
531 | get_build_path, |
532 | ) |
533 | +from lpbuildd.proxy import BuildManagerProxyMixin |
534 | |
535 | |
536 | RETCODE_SUCCESS = 0 |
537 | @@ -46,244 +27,11 @@ RETCODE_FAILURE_INSTALL = 200 |
538 | RETCODE_FAILURE_BUILD = 201 |
539 | |
540 | |
541 | -class SnapProxyClient(proxy.ProxyClient): |
542 | - |
543 | - def __init__(self, command, rest, version, headers, data, father): |
544 | - proxy.ProxyClient.__init__( |
545 | - self, command, rest, version, headers, data, father) |
546 | - # Why doesn't ProxyClient at least store this? |
547 | - self.version = version |
548 | - # We must avoid calling self.father.finish in the event that its |
549 | - # connection was already lost, i.e. if the original client |
550 | - # disconnects first (which is particularly likely in the case of |
551 | - # CONNECT). |
552 | - d = self.father.notifyFinish() |
553 | - d.addBoth(self.requestFinished) |
554 | - |
555 | - def connectionMade(self): |
556 | - proxy.ProxyClient.connectionMade(self) |
557 | - self.father.setChildClient(self) |
558 | - |
559 | - def sendCommand(self, command, path): |
560 | - # For some reason, HTTPClient.sendCommand doesn't preserve the |
561 | - # protocol version. |
562 | - self.transport.writeSequence( |
563 | - [command, b' ', path, b' ', self.version, b'\r\n']) |
564 | - |
565 | - def handleEndHeaders(self): |
566 | - self.father.handleEndHeaders() |
567 | - |
568 | - def sendData(self, data): |
569 | - self.transport.write(data) |
570 | - |
571 | - def endData(self): |
572 | - if self.transport is not None: |
573 | - self.transport.loseWriteConnection() |
574 | - |
575 | - def requestFinished(self, result): |
576 | - self._finished = True |
577 | - self.transport.loseConnection() |
578 | - |
579 | - |
580 | -class SnapProxyClientFactory(proxy.ProxyClientFactory): |
581 | - |
582 | - protocol = SnapProxyClient |
583 | - |
584 | - |
585 | -class SnapProxyRequest(http.Request): |
586 | - |
587 | - child_client = None |
588 | - _request_buffer = None |
589 | - _request_data_done = False |
590 | - |
591 | - def setChildClient(self, child_client): |
592 | - self.child_client = child_client |
593 | - if self._request_buffer is not None: |
594 | - self.child_client.sendData(self._request_buffer.getvalue()) |
595 | - self._request_buffer = None |
596 | - if self._request_data_done: |
597 | - self.child_client.endData() |
598 | - |
599 | - def allHeadersReceived(self, command, path, version): |
600 | - # Normally done in `requestReceived`, but we disable that since it |
601 | - # does other things we don't want. |
602 | - self.method, self.uri, self.clientproto = command, path, version |
603 | - self.client = self.channel.transport.getPeer() |
604 | - self.host = self.channel.transport.getHost() |
605 | - |
606 | - remote_parsed = urlparse(self.channel.factory.remote_url) |
607 | - request_parsed = urlparse(path) |
608 | - headers = self.getAllHeaders().copy() |
609 | - if b"host" not in headers and request_parsed.netloc: |
610 | - headers[b"host"] = request_parsed.netloc |
611 | - if remote_parsed.username: |
612 | - auth = (remote_parsed.username + ":" + |
613 | - remote_parsed.password).encode("ASCII") |
614 | - authHeader = b"Basic " + base64.b64encode(auth) |
615 | - headers[b"proxy-authorization"] = authHeader |
616 | - self.client_factory = SnapProxyClientFactory( |
617 | - command, path, version, headers, b"", self) |
618 | - reactor.connectTCP( |
619 | - remote_parsed.hostname, remote_parsed.port, self.client_factory) |
620 | - |
621 | - def requestReceived(self, command, path, version): |
622 | - # We do most of our work in `allHeadersReceived` instead. |
623 | - pass |
624 | - |
625 | - def rawDataReceived(self, data): |
626 | - if self.child_client is not None: |
627 | - if not self._request_data_done: |
628 | - self.child_client.sendData(data) |
629 | - else: |
630 | - if self._request_buffer is None: |
631 | - self._request_buffer = io.BytesIO() |
632 | - self._request_buffer.write(data) |
633 | - |
634 | - def handleEndHeaders(self): |
635 | - # Cut-down version of Request.write. We must avoid switching to |
636 | - # chunked encoding for the sake of CONNECT; since our actual |
637 | - # response data comes from another proxy, we can cut some corners. |
638 | - if self.startedWriting: |
639 | - return |
640 | - self.startedWriting = 1 |
641 | - l = [] |
642 | - l.append( |
643 | - self.clientproto + b" " + intToBytes(self.code) + b" " + |
644 | - self.code_message + b"\r\n") |
645 | - for name, values in self.responseHeaders.getAllRawHeaders(): |
646 | - for value in values: |
647 | - l.extend([name, b": ", value, b"\r\n"]) |
648 | - l.append(b"\r\n") |
649 | - self.transport.writeSequence(l) |
650 | - |
651 | - def write(self, data): |
652 | - if self.channel is not None: |
653 | - self.channel.resetTimeout() |
654 | - http.Request.write(self, data) |
655 | - |
656 | - def endData(self): |
657 | - if self.child_client is not None: |
658 | - self.child_client.endData() |
659 | - self._request_data_done = True |
660 | - |
661 | - |
662 | -@implementer(IHalfCloseableProtocol) |
663 | -class SnapProxy(http.HTTPChannel): |
664 | - """A channel that streams request data. |
665 | - |
666 | - The stock HTTPChannel isn't quite suitable for our needs, because it |
667 | - expects to read the entire request data before passing control to the |
668 | - request. This doesn't work well for CONNECT. |
669 | - """ |
670 | - |
671 | - requestFactory = SnapProxyRequest |
672 | - |
673 | - def checkPersistence(self, request, version): |
674 | - # ProxyClient.__init__ forces "Connection: close". |
675 | - return False |
676 | - |
677 | - def allHeadersReceived(self): |
678 | - http.HTTPChannel.allHeadersReceived(self) |
679 | - self.requests[-1].allHeadersReceived( |
680 | - self._command, self._path, self._version) |
681 | - if self._command == b"CONNECT": |
682 | - # This is a lie, but we don't want HTTPChannel to decide that |
683 | - # the request is finished just because a CONNECT request |
684 | - # (naturally) has no Content-Length. |
685 | - self.length = -1 |
686 | - |
687 | - def rawDataReceived(self, data): |
688 | - self.resetTimeout() |
689 | - if self.requests: |
690 | - self.requests[-1].rawDataReceived(data) |
691 | - |
692 | - def readConnectionLost(self): |
693 | - for request in self.requests: |
694 | - request.endData() |
695 | - |
696 | - def writeConnectionLost(self): |
697 | - pass |
698 | - |
699 | - |
700 | -class SnapProxyFactory(http.HTTPFactory): |
701 | - |
702 | - protocol = SnapProxy |
703 | - |
704 | - def __init__(self, manager, remote_url, *args, **kwargs): |
705 | - http.HTTPFactory.__init__(self, *args, **kwargs) |
706 | - self.manager = manager |
707 | - self.remote_url = remote_url |
708 | - |
709 | - def log(self, request): |
710 | - # Log requests to the build log rather than to Twisted. |
711 | - # Reimplement log formatting because there's no point logging the IP |
712 | - # here. |
713 | - referrer = http._escape(request.getHeader(b"referer") or b"-") |
714 | - agent = http._escape(request.getHeader(b"user-agent") or b"-") |
715 | - line = ( |
716 | - u'%(timestamp)s "%(method)s %(uri)s %(protocol)s" ' |
717 | - u'%(code)d %(length)s "%(referrer)s" "%(agent)s"\n' % { |
718 | - 'timestamp': self._logDateTime, |
719 | - 'method': http._escape(request.method), |
720 | - 'uri': http._escape(request.uri), |
721 | - 'protocol': http._escape(request.clientproto), |
722 | - 'code': request.code, |
723 | - 'length': request.sentLength or "-", |
724 | - 'referrer': referrer, |
725 | - 'agent': agent, |
726 | - }) |
727 | - self.manager._builder.log(line.encode("UTF-8")) |
728 | - |
729 | - |
730 | class SnapBuildState(DebianBuildState): |
731 | BUILD_SNAP = "BUILD_SNAP" |
732 | |
733 | |
734 | -class SnapBuildProxyMixin(): |
735 | - |
736 | - def startProxy(self): |
737 | - """Start the local snap proxy, if necessary.""" |
738 | - if not self.proxy_url: |
739 | - return [] |
740 | - proxy_port = self._builder._config.get("snapmanager", "proxyport") |
741 | - proxy_factory = SnapProxyFactory(self, self.proxy_url, timeout=60) |
742 | - self.proxy_service = strports.service( |
743 | - "tcp:%s" % proxy_port, proxy_factory) |
744 | - self.proxy_service.setServiceParent(self._builder.service) |
745 | - if self.backend_name == "lxd": |
746 | - proxy_host = self.backend.ipv4_network.ip |
747 | - else: |
748 | - proxy_host = "localhost" |
749 | - return ["--proxy-url", "http://{}:{}/".format(proxy_host, proxy_port)] |
750 | - |
751 | - def stopProxy(self): |
752 | - """Stop the local snap proxy, if necessary.""" |
753 | - if self.proxy_service is None: |
754 | - return |
755 | - self.proxy_service.disownServiceParent() |
756 | - self.proxy_service = None |
757 | - |
758 | - def revokeProxyToken(self): |
759 | - """Revoke builder proxy token.""" |
760 | - if not self.revocation_endpoint: |
761 | - return |
762 | - self._builder.log("Revoking proxy token...\n") |
763 | - url = urlparse(self.proxy_url) |
764 | - auth = "{}:{}".format(url.username, url.password) |
765 | - headers = { |
766 | - "Authorization": "Basic {}".format( |
767 | - base64.b64encode(auth.encode('utf-8')).decode('utf-8')) |
768 | - } |
769 | - req = Request(self.revocation_endpoint, None, headers) |
770 | - req.get_method = lambda: "DELETE" |
771 | - try: |
772 | - urlopen(req) |
773 | - except (HTTPError, URLError) as e: |
774 | - self._builder.log( |
775 | - "Unable to revoke token for %s: %s" % (url.username, e)) |
776 | - |
777 | - |
778 | -class SnapBuildManager(SnapBuildProxyMixin, DebianBuildManager): |
779 | +class SnapBuildManager(BuildManagerProxyMixin, DebianBuildManager): |
780 | """Build a snap.""" |
781 | |
782 | backend_name = "lxd" |
783 | diff --git a/lpbuildd/target/build_charm.py b/lpbuildd/target/build_charm.py |
784 | index 28f9d0b..a6c6278 100644 |
785 | --- a/lpbuildd/target/build_charm.py |
786 | +++ b/lpbuildd/target/build_charm.py |
787 | @@ -13,7 +13,7 @@ import sys |
788 | from lpbuildd.target.backend import check_path_escape |
789 | from lpbuildd.target.build_snap import SnapChannelsAction |
790 | from lpbuildd.target.operation import Operation |
791 | -from lpbuildd.target.snapbuildproxy import SnapBuildProxyOperationMixin |
792 | +from lpbuildd.target.proxy import BuilderProxyOperationMixin |
793 | from lpbuildd.target.snapstore import SnapStoreOperationMixin |
794 | from lpbuildd.target.vcs import VCSOperationMixin |
795 | |
796 | @@ -25,7 +25,7 @@ RETCODE_FAILURE_BUILD = 201 |
797 | logger = logging.getLogger(__name__) |
798 | |
799 | |
800 | -class BuildCharm(SnapBuildProxyOperationMixin, VCSOperationMixin, |
801 | +class BuildCharm(BuilderProxyOperationMixin, VCSOperationMixin, |
802 | SnapStoreOperationMixin, Operation): |
803 | |
804 | description = "Build a charm." |
805 | @@ -124,7 +124,7 @@ class BuildCharm(SnapBuildProxyOperationMixin, VCSOperationMixin, |
806 | if self.args.proxy_url: |
807 | env["http_proxy"] = self.args.proxy_url |
808 | env["https_proxy"] = self.args.proxy_url |
809 | - env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy" |
810 | + env["GIT_PROXY_COMMAND"] = "/usr/local/bin/lpbuildd-git-proxy" |
811 | args = [ |
812 | "charmcraft", "build", "-v", |
813 | "-p", build_context_path, |
814 | diff --git a/lpbuildd/target/build_oci.py b/lpbuildd/target/build_oci.py |
815 | index 5615325..437932c 100644 |
816 | --- a/lpbuildd/target/build_oci.py |
817 | +++ b/lpbuildd/target/build_oci.py |
818 | @@ -14,7 +14,7 @@ from textwrap import dedent |
819 | |
820 | from lpbuildd.target.backend import check_path_escape |
821 | from lpbuildd.target.operation import Operation |
822 | -from lpbuildd.target.snapbuildproxy import SnapBuildProxyOperationMixin |
823 | +from lpbuildd.target.proxy import BuilderProxyOperationMixin |
824 | from lpbuildd.target.snapstore import SnapStoreOperationMixin |
825 | from lpbuildd.target.vcs import VCSOperationMixin |
826 | |
827 | @@ -26,7 +26,7 @@ RETCODE_FAILURE_BUILD = 201 |
828 | logger = logging.getLogger(__name__) |
829 | |
830 | |
831 | -class BuildOCI(SnapBuildProxyOperationMixin, VCSOperationMixin, |
832 | +class BuildOCI(BuilderProxyOperationMixin, VCSOperationMixin, |
833 | SnapStoreOperationMixin, Operation): |
834 | |
835 | description = "Build an OCI image." |
836 | diff --git a/lpbuildd/target/build_snap.py b/lpbuildd/target/build_snap.py |
837 | index be9d655..bdbc760 100644 |
838 | --- a/lpbuildd/target/build_snap.py |
839 | +++ b/lpbuildd/target/build_snap.py |
840 | @@ -17,7 +17,7 @@ from textwrap import dedent |
841 | from six.moves.urllib.parse import urlparse |
842 | |
843 | from lpbuildd.target.operation import Operation |
844 | -from lpbuildd.target.snapbuildproxy import SnapBuildProxyOperationMixin |
845 | +from lpbuildd.target.proxy import BuilderProxyOperationMixin |
846 | from lpbuildd.target.snapstore import SnapStoreOperationMixin |
847 | from lpbuildd.target.vcs import VCSOperationMixin |
848 | |
849 | @@ -47,7 +47,7 @@ class SnapChannelsAction(argparse.Action): |
850 | getattr(namespace, self.dest)[snap] = channel |
851 | |
852 | |
853 | -class BuildSnap(SnapBuildProxyOperationMixin, VCSOperationMixin, |
854 | +class BuildSnap(BuilderProxyOperationMixin, VCSOperationMixin, |
855 | SnapStoreOperationMixin, Operation): |
856 | |
857 | description = "Build a snap." |
858 | @@ -189,7 +189,7 @@ class BuildSnap(SnapBuildProxyOperationMixin, VCSOperationMixin, |
859 | if self.args.proxy_url: |
860 | env["http_proxy"] = self.args.proxy_url |
861 | env["https_proxy"] = self.args.proxy_url |
862 | - env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy" |
863 | + env["GIT_PROXY_COMMAND"] = "/usr/local/bin/lpbuildd-git-proxy" |
864 | self.run_build_command( |
865 | ["snapcraft", "pull"], |
866 | cwd=os.path.join("/build", self.args.name), |
867 | @@ -213,7 +213,7 @@ class BuildSnap(SnapBuildProxyOperationMixin, VCSOperationMixin, |
868 | if self.args.proxy_url: |
869 | env["http_proxy"] = self.args.proxy_url |
870 | env["https_proxy"] = self.args.proxy_url |
871 | - env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy" |
872 | + env["GIT_PROXY_COMMAND"] = "/usr/local/bin/lpbuildd-git-proxy" |
873 | self.run_build_command( |
874 | ["snapcraft"], |
875 | cwd=os.path.join("/build", self.args.name), |
876 | diff --git a/lpbuildd/target/snapbuildproxy.py b/lpbuildd/target/proxy.py |
877 | similarity index 81% |
878 | rename from lpbuildd/target/snapbuildproxy.py |
879 | rename to lpbuildd/target/proxy.py |
880 | index 6ab8097..4834e46 100644 |
881 | --- a/lpbuildd/target/snapbuildproxy.py |
882 | +++ b/lpbuildd/target/proxy.py |
883 | @@ -9,12 +9,12 @@ from collections import OrderedDict |
884 | import os |
885 | |
886 | |
887 | -class SnapBuildProxyOperationMixin: |
888 | - """Methods supporting the build time HTTP proxy for snap and OCI builds.""" |
889 | +class BuilderProxyOperationMixin: |
890 | + """Methods supporting the build time HTTP proxy for certain build types.""" |
891 | |
892 | @classmethod |
893 | def add_arguments(cls, parser): |
894 | - super(SnapBuildProxyOperationMixin, cls).add_arguments(parser) |
895 | + super(BuilderProxyOperationMixin, cls).add_arguments(parser) |
896 | parser.add_argument("--proxy-url", help="builder proxy url") |
897 | parser.add_argument( |
898 | "--revocation-endpoint", |
899 | @@ -26,8 +26,8 @@ class SnapBuildProxyOperationMixin: |
900 | |
901 | def install_git_proxy(self): |
902 | self.backend.copy_in( |
903 | - os.path.join(self.bin, "snap-git-proxy"), |
904 | - "/usr/local/bin/snap-git-proxy") |
905 | + os.path.join(self.bin, "lpbuildd-git-proxy"), |
906 | + "/usr/local/bin/lpbuildd-git-proxy") |
907 | |
908 | def build_proxy_environment(self, proxy_url=None, env=None): |
909 | """Extend a command environment to include http proxy variables.""" |
910 | @@ -37,5 +37,5 @@ class SnapBuildProxyOperationMixin: |
911 | if proxy_url: |
912 | full_env["http_proxy"] = self.args.proxy_url |
913 | full_env["https_proxy"] = self.args.proxy_url |
914 | - full_env["GIT_PROXY_COMMAND"] = "/usr/local/bin/snap-git-proxy" |
915 | + full_env["GIT_PROXY_COMMAND"] = "/usr/local/bin/lpbuildd-git-proxy" |
916 | return full_env |
917 | diff --git a/lpbuildd/target/tests/test_build_charm.py b/lpbuildd/target/tests/test_build_charm.py |
918 | index ae1da6a..61e8602 100644 |
919 | --- a/lpbuildd/target/tests/test_build_charm.py |
920 | +++ b/lpbuildd/target/tests/test_build_charm.py |
921 | @@ -198,7 +198,7 @@ class TestBuildCharm(TestCase): |
922 | build_charm.bin = "/builderbin" |
923 | self.useFixture(FakeFilesystem()).add("/builderbin") |
924 | os.mkdir("/builderbin") |
925 | - with open("/builderbin/snap-git-proxy", "w") as proxy_script: |
926 | + with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script: |
927 | proxy_script.write("proxy script\n") |
928 | os.fchmod(proxy_script.fileno(), 0o755) |
929 | build_charm.install() |
930 | @@ -211,7 +211,8 @@ class TestBuildCharm(TestCase): |
931 | ])) |
932 | self.assertEqual( |
933 | (b"proxy script\n", stat.S_IFREG | 0o755), |
934 | - build_charm.backend.backend_fs["/usr/local/bin/snap-git-proxy"]) |
935 | + build_charm.backend.backend_fs[ |
936 | + "/usr/local/bin/lpbuildd-git-proxy"]) |
937 | |
938 | def test_repo_bzr(self): |
939 | args = [ |
940 | @@ -328,7 +329,7 @@ class TestBuildCharm(TestCase): |
941 | env = { |
942 | "http_proxy": "http://proxy.example:3128/", |
943 | "https_proxy": "http://proxy.example:3128/", |
944 | - "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy", |
945 | + "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy", |
946 | } |
947 | self.assertThat(build_charm.backend.run.calls, MatchesListwise([ |
948 | RanBuildCommand( |
949 | @@ -394,7 +395,7 @@ class TestBuildCharm(TestCase): |
950 | "CHARMCRAFT_MANAGED_MODE": "1", |
951 | "http_proxy": "http://proxy.example:3128/", |
952 | "https_proxy": "http://proxy.example:3128/", |
953 | - "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy", |
954 | + "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy", |
955 | } |
956 | self.assertThat(build_charm.backend.run.calls, MatchesListwise([ |
957 | RanBuildCommand( |
958 | diff --git a/lpbuildd/target/tests/test_build_oci.py b/lpbuildd/target/tests/test_build_oci.py |
959 | index 3b9966a..4b95221 100644 |
960 | --- a/lpbuildd/target/tests/test_build_oci.py |
961 | +++ b/lpbuildd/target/tests/test_build_oci.py |
962 | @@ -177,7 +177,7 @@ class TestBuildOCI(TestCase): |
963 | build_oci.bin = "/builderbin" |
964 | self.useFixture(FakeFilesystem()).add("/builderbin") |
965 | os.mkdir("/builderbin") |
966 | - with open("/builderbin/snap-git-proxy", "w") as proxy_script: |
967 | + with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script: |
968 | proxy_script.write("proxy script\n") |
969 | os.fchmod(proxy_script.fileno(), 0o755) |
970 | build_oci.install() |
971 | @@ -190,7 +190,7 @@ class TestBuildOCI(TestCase): |
972 | ])) |
973 | self.assertEqual( |
974 | (b"proxy script\n", stat.S_IFREG | 0o755), |
975 | - build_oci.backend.backend_fs["/usr/local/bin/snap-git-proxy"]) |
976 | + build_oci.backend.backend_fs["/usr/local/bin/lpbuildd-git-proxy"]) |
977 | |
978 | def test_repo_bzr(self): |
979 | args = [ |
980 | @@ -279,7 +279,7 @@ class TestBuildOCI(TestCase): |
981 | env = { |
982 | "http_proxy": "http://proxy.example:3128/", |
983 | "https_proxy": "http://proxy.example:3128/", |
984 | - "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy", |
985 | + "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy", |
986 | } |
987 | self.assertThat(build_oci.backend.run.calls, MatchesListwise([ |
988 | RanBuildCommand( |
989 | diff --git a/lpbuildd/target/tests/test_build_snap.py b/lpbuildd/target/tests/test_build_snap.py |
990 | index 3e9f320..1a529ea 100644 |
991 | --- a/lpbuildd/target/tests/test_build_snap.py |
992 | +++ b/lpbuildd/target/tests/test_build_snap.py |
993 | @@ -183,7 +183,7 @@ class TestBuildSnap(TestCase): |
994 | build_snap.bin = "/builderbin" |
995 | self.useFixture(FakeFilesystem()).add("/builderbin") |
996 | os.mkdir("/builderbin") |
997 | - with open("/builderbin/snap-git-proxy", "w") as proxy_script: |
998 | + with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script: |
999 | proxy_script.write("proxy script\n") |
1000 | os.fchmod(proxy_script.fileno(), 0o755) |
1001 | build_snap.install() |
1002 | @@ -193,7 +193,7 @@ class TestBuildSnap(TestCase): |
1003 | ])) |
1004 | self.assertEqual( |
1005 | (b"proxy script\n", stat.S_IFREG | 0o755), |
1006 | - build_snap.backend.backend_fs["/usr/local/bin/snap-git-proxy"]) |
1007 | + build_snap.backend.backend_fs["/usr/local/bin/lpbuildd-git-proxy"]) |
1008 | self.assertEqual( |
1009 | (b"[global]\n" |
1010 | b"http-proxy-host = proxy.example\n" |
1011 | @@ -333,7 +333,7 @@ class TestBuildSnap(TestCase): |
1012 | env = { |
1013 | "http_proxy": "http://proxy.example:3128/", |
1014 | "https_proxy": "http://proxy.example:3128/", |
1015 | - "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy", |
1016 | + "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy", |
1017 | } |
1018 | self.assertThat(build_snap.backend.run.calls, MatchesListwise([ |
1019 | RanBuildCommand( |
1020 | @@ -389,7 +389,7 @@ class TestBuildSnap(TestCase): |
1021 | "SNAPCRAFT_BUILD_ENVIRONMENT": "host", |
1022 | "http_proxy": "http://proxy.example:3128/", |
1023 | "https_proxy": "http://proxy.example:3128/", |
1024 | - "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy", |
1025 | + "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy", |
1026 | } |
1027 | self.assertThat(build_snap.backend.run.calls, MatchesListwise([ |
1028 | RanBuildCommand( |
1029 | @@ -473,7 +473,7 @@ class TestBuildSnap(TestCase): |
1030 | "SNAPCRAFT_BUILD_ENVIRONMENT": "host", |
1031 | "http_proxy": "http://proxy.example:3128/", |
1032 | "https_proxy": "http://proxy.example:3128/", |
1033 | - "GIT_PROXY_COMMAND": "/usr/local/bin/snap-git-proxy", |
1034 | + "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy", |
1035 | } |
1036 | self.assertThat(build_snap.backend.run.calls, MatchesListwise([ |
1037 | RanBuildCommand(["snapcraft"], cwd="/build/test-snap", **env), |
1038 | diff --git a/lpbuildd/tests/buildlog b/lpbuildd/tests/buildlog |
1039 | index bc033f6..4b64e85 100644 |
1040 | --- a/lpbuildd/tests/buildlog |
1041 | +++ b/lpbuildd/tests/buildlog |
1042 | @@ -21,4 +21,4 @@ The following packages will be upgraded: |
1043 | Need to get 0B/2832kB of archives. |
1044 | After unpacking 94.2kB of additional disk space will be used. |
1045 | (Reading database ... 8942 files and directories currently installed.) |
1046 | -socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah |
1047 | +socat STDIO PROXY:builder-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah |
1048 | diff --git a/lpbuildd/tests/buildlog.long b/lpbuildd/tests/buildlog.long |
1049 | index 8de21c7..9ca5464 100644 |
1050 | --- a/lpbuildd/tests/buildlog.long |
1051 | +++ b/lpbuildd/tests/buildlog.long |
1052 | @@ -21,7 +21,7 @@ The following packages will be upgraded: |
1053 | Need to get 0B/2832kB of archives. |
1054 | After unpacking 94.2kB of additional disk space will be used. |
1055 | (Reading database ... 8942 files and directories currently installed.) |
1056 | -socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah |
1057 | +socat STDIO PROXY:builder-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah |
1058 | RUN: /usr/share/launchpad-buildd/bin/unpack-chroot 370614-896976 /home/buildd/filecache-default/a40e3c410938399b35051833fe5244f9ac6f3774 |
1059 | Unpacking chroot for build 370614-896976 |
1060 | RUN: /usr/share/launchpad-buildd/bin/mount-chroot 370614-896976 |
1061 | @@ -81,4 +81,4 @@ Get:2 http://must:go@ftpmaster.internal gutsy Release [65.9kB] |
1062 | Get:3 http://scrub:this@ftpmaster.internal gutsy/main Packages [1085kB] |
1063 | Get:4 http://ftpmaster.internal gutsy/universe Packages [3991kB] |
1064 | Fetched 5142kB in 5s (1012kB/s) |
1065 | -socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah |
1066 | +socat STDIO PROXY:builder-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah |
1067 | diff --git a/lpbuildd/tests/test_1.diff b/lpbuildd/tests/test_1.diff |
1068 | index 249b2af..d4f9b70 100644 |
1069 | --- a/lpbuildd/tests/test_1.diff |
1070 | +++ b/lpbuildd/tests/test_1.diff |
1071 | @@ -20,5 +20,5 @@ |
1072 | Need to get 0B/2832kB of archives. |
1073 | After unpacking 94.2kB of additional disk space will be used. |
1074 | (Reading database ... 8942 files and directories currently installed.) |
1075 | --socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah |
1076 | -+socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128 |
1077 | \ No newline at end of file |
1078 | +-socat STDIO PROXY:builder-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah |
1079 | ++socat STDIO PROXY:builder-proxy.launchpad.dev:github.com:443,proxyport=3128 |
1080 | \ No newline at end of file |
1081 | diff --git a/lpbuildd/tests/test_2.diff b/lpbuildd/tests/test_2.diff |
1082 | index 8cccd43..77362dd 100644 |
1083 | --- a/lpbuildd/tests/test_2.diff |
1084 | +++ b/lpbuildd/tests/test_2.diff |
1085 | @@ -4,7 +4,7 @@ |
1086 | |
1087 | @@ -1,13 +1,12 @@ |
1088 | |
1089 | --or build 370614-896976 |
1090 | +-build 370614-896976 |
1091 | RUN: /usr/share/launchpad-buildd/bin/mount-chroot 370614-896976 |
1092 | Mounting chroot for build 370614-896976 |
1093 | RUN: /usr/share/launchpad-buildd/bin/apply-ogre-model 370614-896976 universe |
1094 | @@ -32,5 +32,5 @@ |
1095 | +Get:3 http://ftpmaster.internal gutsy/main Packages [1085kB] |
1096 | Get:4 http://ftpmaster.internal gutsy/universe Packages [3991kB] |
1097 | Fetched 5142kB in 5s (1012kB/s) |
1098 | --socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah |
1099 | -+socat STDIO PROXY:snap-proxy.launchpad.dev:github.com:443,proxyport=3128 |
1100 | \ No newline at end of file |
1101 | +-socat STDIO PROXY:builder-proxy.launchpad.dev:github.com:443,proxyport=3128,proxyauth=user:blah |
1102 | ++socat STDIO PROXY:builder-proxy.launchpad.dev:github.com:443,proxyport=3128 |
1103 | \ No newline at end of file |
1104 | diff --git a/lpbuildd/tests/test_charm.py b/lpbuildd/tests/test_charm.py |
1105 | index 8610bad..18e01a1 100644 |
1106 | --- a/lpbuildd/tests/test_charm.py |
1107 | +++ b/lpbuildd/tests/test_charm.py |
1108 | @@ -142,7 +142,7 @@ class TestCharmBuildManagerIteration(TestCase): |
1109 | self.buildmanager.iterate, self.buildmanager.iterators[-1]) |
1110 | self.assertFalse(self.builder.wasCalled("buildFail")) |
1111 | |
1112 | - @mock.patch('lpbuildd.snap.urlopen') |
1113 | + @mock.patch('lpbuildd.proxy.urlopen') |
1114 | def test_revokeProxyToken(self, urlopen_mock): |
1115 | self.buildmanager.revocation_endpoint = "http://revoke_endpoint" |
1116 | self.buildmanager.proxy_url = "http://username:password@proxy_url" |
1117 | diff --git a/lpbuildd/tests/test_snap.py b/lpbuildd/tests/test_snap.py |
1118 | index 5b7053d..b018a53 100644 |
1119 | --- a/lpbuildd/tests/test_snap.py |
1120 | +++ b/lpbuildd/tests/test_snap.py |
1121 | @@ -29,10 +29,10 @@ from twisted.web import ( |
1122 | static, |
1123 | ) |
1124 | |
1125 | +from lpbuildd.proxy import BuilderProxyFactory |
1126 | from lpbuildd.snap import ( |
1127 | SnapBuildManager, |
1128 | SnapBuildState, |
1129 | - SnapProxyFactory, |
1130 | ) |
1131 | from lpbuildd.tests.fakebuilder import FakeBuilder |
1132 | from lpbuildd.tests.matchers import HasWaitingFiles |
1133 | @@ -411,7 +411,7 @@ class TestSnapBuildManagerIteration(TestCase): |
1134 | return remote_proxy_listener |
1135 | |
1136 | def startLocalProxy(self, remote_url): |
1137 | - proxy_factory = SnapProxyFactory( |
1138 | + proxy_factory = BuilderProxyFactory( |
1139 | self.buildmanager, remote_url, timeout=60) |
1140 | proxy_listener = reactor.listenTCP(0, proxy_factory) |
1141 | self.addCleanup(proxy_listener.stopListening) |
1142 | @@ -453,7 +453,7 @@ class TestSnapBuildManagerIteration(TestCase): |
1143 | # the code under test since the stock twisted.web.proxy doesn't support |
1144 | # CONNECT. |
1145 | |
1146 | - @mock.patch('lpbuildd.snap.urlopen') |
1147 | + @mock.patch('lpbuildd.proxy.urlopen') |
1148 | def test_revokeProxyToken(self, urlopen_mock): |
1149 | self.buildmanager.revocation_endpoint = "http://revoke_endpoint" |
1150 | self.buildmanager.proxy_url = "http://username:password@proxy_url" |
1151 | diff --git a/template-buildd-slave.conf b/template-buildd-slave.conf |
1152 | index b195c82..4c389e6 100644 |
1153 | --- a/template-buildd-slave.conf |
1154 | +++ b/template-buildd-slave.conf |
1155 | @@ -9,9 +9,7 @@ bindhost = @BINDHOST@ |
1156 | bindport = @BINDPORT@ |
1157 | ntphost = ntp.buildd |
1158 | sharepath = /usr/share/launchpad-buildd |
1159 | +proxyport = @PROXYPORT@ |
1160 | |
1161 | [translationtemplatesmanager] |
1162 | resultarchive = translation-templates.tar.gz |
1163 | - |
1164 | -[snapmanager] |
1165 | -proxyport = @SNAPPROXYPORT@ |
Looks good!