Merge lp:~cjwatson/launchpad-buildd/proxy-configs into lp:launchpad-buildd
- proxy-configs
- Merge into trunk
Proposed by
Colin Watson
Status: | Merged |
---|---|
Merged at revision: | 382 |
Proposed branch: | lp:~cjwatson/launchpad-buildd/proxy-configs |
Merge into: | lp:launchpad-buildd |
Diff against target: |
685 lines (+293/-23) 17 files modified
debian/changelog (+7/-0) debian/control (+3/-0) lpbuildd/debian.py (+13/-2) lpbuildd/livefs.py (+12/-1) lpbuildd/snap.py (+11/-1) lpbuildd/target/apt.py (+13/-1) lpbuildd/target/build_livefs.py (+5/-2) lpbuildd/target/build_snap.py (+5/-2) lpbuildd/target/snapstore.py (+42/-0) lpbuildd/target/tests/test_apt.py (+22/-1) lpbuildd/target/tests/test_build_livefs.py (+39/-2) lpbuildd/target/tests/test_build_snap.py (+41/-4) lpbuildd/tests/fakebuilder.py (+22/-2) lpbuildd/tests/test_debian.py (+31/-1) lpbuildd/tests/test_livefs.py (+13/-2) lpbuildd/tests/test_snap.py (+10/-1) setup.py (+4/-1) |
To merge this branch: | bzr merge lp:~cjwatson/launchpad-buildd/proxy-configs |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Tom Wardill (community) | Approve | ||
Review via email: mp+362332@code.launchpad.net |
Commit message
Allow configuring APT or snap store proxies via a new [proxy] configuration file section.
Description of the change
This makes testing builds behind a slow connection 90% less annoying.
While python-requests and python-six are new direct dependencies in this branch, they were already indirect dependencies.
To post a comment you must log in.
Revision history for this message
Tom Wardill (twom) : | # |
review:
Approve
- 359. By Colin Watson
-
Simplify snap_store_
set_proxy using the X-Assertion- Store-Id header. - 360. By Colin Watson
-
Merge trunk.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'debian/changelog' |
2 | --- debian/changelog 2019-05-23 16:23:54 +0000 |
3 | +++ debian/changelog 2019-06-05 14:11:48 +0000 |
4 | @@ -1,3 +1,10 @@ |
5 | +launchpad-buildd (175) UNRELEASED; urgency=medium |
6 | + |
7 | + * Allow configuring APT or snap store proxies via a new [proxy] |
8 | + configuration file section. |
9 | + |
10 | + -- Colin Watson <cjwatson@ubuntu.com> Wed, 05 Jun 2019 15:06:54 +0100 |
11 | + |
12 | launchpad-buildd (174) xenial; urgency=medium |
13 | |
14 | * Fix a missing piece from the changes in launchpad-buildd 168 that were |
15 | |
16 | === modified file 'debian/control' |
17 | --- debian/control 2018-06-07 16:23:18 +0000 |
18 | +++ debian/control 2019-06-05 14:11:48 +0000 |
19 | @@ -17,7 +17,10 @@ |
20 | python-mock, |
21 | python-netaddr, |
22 | python-pylxd, |
23 | + python-requests, |
24 | + python-responses, |
25 | python-setuptools, |
26 | + python-six, |
27 | python-systemfixtures, |
28 | python-testtools, |
29 | python-twisted, |
30 | |
31 | === modified file 'lpbuildd/debian.py' |
32 | --- lpbuildd/debian.py 2019-02-12 10:35:12 +0000 |
33 | +++ lpbuildd/debian.py 2019-06-05 14:11:48 +0000 |
34 | @@ -1,4 +1,4 @@ |
35 | -# Copyright 2009-2018 Canonical Ltd. This software is licensed under the |
36 | +# Copyright 2009-2019 Canonical Ltd. This software is licensed under the |
37 | # GNU Affero General Public License version 3 (see the file LICENSE). |
38 | |
39 | # Authors: Daniel Silverstone <daniel.silverstone@canonical.com> |
40 | @@ -11,6 +11,10 @@ |
41 | import re |
42 | import signal |
43 | |
44 | +from six.moves.configparser import ( |
45 | + NoOptionError, |
46 | + NoSectionError, |
47 | + ) |
48 | from twisted.internet import ( |
49 | defer, |
50 | threads, |
51 | @@ -59,7 +63,14 @@ |
52 | |
53 | Mainly used for PPA builds. |
54 | """ |
55 | - self.runTargetSubProcess("override-sources-list", *self.sources_list) |
56 | + args = [] |
57 | + try: |
58 | + apt_proxy_url = self._builder._config.get("proxy", "apt") |
59 | + args.extend(["--apt-proxy-url", apt_proxy_url]) |
60 | + except (NoSectionError, NoOptionError): |
61 | + pass |
62 | + args.extend(self.sources_list) |
63 | + self.runTargetSubProcess("override-sources-list", *args) |
64 | |
65 | def doTrustedKeys(self): |
66 | """Add trusted keys.""" |
67 | |
68 | === modified file 'lpbuildd/livefs.py' |
69 | --- lpbuildd/livefs.py 2019-02-12 10:35:12 +0000 |
70 | +++ lpbuildd/livefs.py 2019-06-05 14:11:48 +0000 |
71 | @@ -1,10 +1,15 @@ |
72 | -# Copyright 2013-2018 Canonical Ltd. This software is licensed under the |
73 | +# Copyright 2013-2019 Canonical Ltd. This software is licensed under the |
74 | # GNU Affero General Public License version 3 (see the file LICENSE). |
75 | |
76 | __metaclass__ = type |
77 | |
78 | import os |
79 | |
80 | +from six.moves.configparser import ( |
81 | + NoOptionError, |
82 | + NoSectionError, |
83 | + ) |
84 | + |
85 | from lpbuildd.debian import ( |
86 | DebianBuildManager, |
87 | DebianBuildState, |
88 | @@ -71,6 +76,12 @@ |
89 | args.extend(["--repo-snapshot-stamp", self.repo_snapshot_stamp]) |
90 | if self.cohort_key: |
91 | args.extend(["--cohort-key", self.cohort_key]) |
92 | + try: |
93 | + snap_store_proxy_url = self._builder._config.get( |
94 | + "proxy", "snapstore") |
95 | + args.extend(["--snap-store-proxy-url", snap_store_proxy_url]) |
96 | + except (NoSectionError, NoOptionError): |
97 | + pass |
98 | if self.debug: |
99 | args.append("--debug") |
100 | self.runTargetSubProcess("buildlivefs", *args) |
101 | |
102 | === modified file 'lpbuildd/snap.py' |
103 | --- lpbuildd/snap.py 2019-05-22 17:59:14 +0000 |
104 | +++ lpbuildd/snap.py 2019-06-05 14:11:48 +0000 |
105 | @@ -1,4 +1,4 @@ |
106 | -# Copyright 2015-2018 Canonical Ltd. This software is licensed under the |
107 | +# Copyright 2015-2019 Canonical Ltd. This software is licensed under the |
108 | # GNU Affero General Public License version 3 (see the file LICENSE). |
109 | |
110 | from __future__ import print_function |
111 | @@ -29,6 +29,10 @@ |
112 | ) |
113 | from urlparse import urlparse |
114 | |
115 | +from six.moves.configparser import ( |
116 | + NoOptionError, |
117 | + NoSectionError, |
118 | + ) |
119 | from twisted.application import strports |
120 | from twisted.internet import reactor |
121 | from twisted.internet.interfaces import IHalfCloseableProtocol |
122 | @@ -351,6 +355,12 @@ |
123 | args.append("--build-source-tarball") |
124 | if self.private: |
125 | args.append("--private") |
126 | + try: |
127 | + snap_store_proxy_url = self._builder._config.get( |
128 | + "proxy", "snapstore") |
129 | + args.extend(["--snap-store-proxy-url", snap_store_proxy_url]) |
130 | + except (NoSectionError, NoOptionError): |
131 | + pass |
132 | args.append(self.name) |
133 | self.runTargetSubProcess("buildsnap", *args) |
134 | |
135 | |
136 | === modified file 'lpbuildd/target/apt.py' |
137 | --- lpbuildd/target/apt.py 2017-08-22 15:55:44 +0000 |
138 | +++ lpbuildd/target/apt.py 2019-06-05 14:11:48 +0000 |
139 | @@ -1,4 +1,4 @@ |
140 | -# Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
141 | +# Copyright 2009-2019 Canonical Ltd. This software is licensed under the |
142 | # GNU Affero General Public License version 3 (see the file LICENSE). |
143 | |
144 | from __future__ import print_function |
145 | @@ -26,6 +26,8 @@ |
146 | def add_arguments(cls, parser): |
147 | super(OverrideSourcesList, cls).add_arguments(parser) |
148 | parser.add_argument( |
149 | + "--apt-proxy-url", metavar="URL", help="APT proxy URL") |
150 | + parser.add_argument( |
151 | "archives", metavar="ARCHIVE", nargs="+", |
152 | help="sources.list lines") |
153 | |
154 | @@ -37,6 +39,16 @@ |
155 | sources_list.flush() |
156 | os.fchmod(sources_list.fileno(), 0o644) |
157 | self.backend.copy_in(sources_list.name, "/etc/apt/sources.list") |
158 | + if self.args.apt_proxy_url is not None: |
159 | + with tempfile.NamedTemporaryFile() as apt_proxy_conf: |
160 | + print( |
161 | + 'Acquire::http::Proxy "{}";'.format( |
162 | + self.args.apt_proxy_url), |
163 | + file=apt_proxy_conf) |
164 | + apt_proxy_conf.flush() |
165 | + os.fchmod(apt_proxy_conf.fileno(), 0o644) |
166 | + self.backend.copy_in( |
167 | + apt_proxy_conf.name, "/etc/apt/apt.conf.d/99proxy") |
168 | return 0 |
169 | |
170 | |
171 | |
172 | === modified file 'lpbuildd/target/build_livefs.py' |
173 | --- lpbuildd/target/build_livefs.py 2019-01-28 13:10:24 +0000 |
174 | +++ lpbuildd/target/build_livefs.py 2019-06-05 14:11:48 +0000 |
175 | @@ -1,4 +1,4 @@ |
176 | -# Copyright 2013-2017 Canonical Ltd. This software is licensed under the |
177 | +# Copyright 2013-2019 Canonical Ltd. This software is licensed under the |
178 | # GNU Affero General Public License version 3 (see the file LICENSE). |
179 | |
180 | from __future__ import print_function |
181 | @@ -10,6 +10,7 @@ |
182 | import os |
183 | |
184 | from lpbuildd.target.operation import Operation |
185 | +from lpbuildd.target.snapstore import SnapStoreOperationMixin |
186 | |
187 | |
188 | RETCODE_FAILURE_INSTALL = 200 |
189 | @@ -29,7 +30,7 @@ |
190 | return os.path.join(os.environ["HOME"], "build-" + build_id, *extra) |
191 | |
192 | |
193 | -class BuildLiveFS(Operation): |
194 | +class BuildLiveFS(SnapStoreOperationMixin, Operation): |
195 | |
196 | description = "Build a live file system." |
197 | |
198 | @@ -94,6 +95,8 @@ |
199 | if self.backend.is_package_available(dep): |
200 | deps.append(dep) |
201 | self.backend.run(["apt-get", "-y", "install"] + deps) |
202 | + if self.args.backend in ("lxd", "fake"): |
203 | + self.snap_store_set_proxy() |
204 | if self.args.arch == "i386": |
205 | self.backend.run([ |
206 | "apt-get", "-y", "--no-install-recommends", "install", |
207 | |
208 | === modified file 'lpbuildd/target/build_snap.py' |
209 | --- lpbuildd/target/build_snap.py 2019-05-08 14:52:28 +0000 |
210 | +++ lpbuildd/target/build_snap.py 2019-06-05 14:11:48 +0000 |
211 | @@ -1,4 +1,4 @@ |
212 | -# Copyright 2015-2017 Canonical Ltd. This software is licensed under the |
213 | +# Copyright 2015-2019 Canonical Ltd. This software is licensed under the |
214 | # GNU Affero General Public License version 3 (see the file LICENSE). |
215 | |
216 | from __future__ import print_function |
217 | @@ -19,6 +19,7 @@ |
218 | from urlparse import urlparse |
219 | |
220 | from lpbuildd.target.operation import Operation |
221 | +from lpbuildd.target.snapstore import SnapStoreOperationMixin |
222 | from lpbuildd.target.vcs import VCSOperationMixin |
223 | |
224 | |
225 | @@ -47,7 +48,7 @@ |
226 | getattr(namespace, self.dest)[snap] = channel |
227 | |
228 | |
229 | -class BuildSnap(VCSOperationMixin, Operation): |
230 | +class BuildSnap(VCSOperationMixin, SnapStoreOperationMixin, Operation): |
231 | |
232 | description = "Build a snap." |
233 | |
234 | @@ -154,6 +155,8 @@ |
235 | else: |
236 | deps.append("snapcraft") |
237 | self.backend.run(["apt-get", "-y", "install"] + deps) |
238 | + if self.args.backend in ("lxd", "fake"): |
239 | + self.snap_store_set_proxy() |
240 | for snap_name in self.core_snap_names: |
241 | if snap_name in self.args.channels: |
242 | self.backend.run( |
243 | |
244 | === added file 'lpbuildd/target/snapstore.py' |
245 | --- lpbuildd/target/snapstore.py 1970-01-01 00:00:00 +0000 |
246 | +++ lpbuildd/target/snapstore.py 2019-06-05 14:11:48 +0000 |
247 | @@ -0,0 +1,42 @@ |
248 | +# Copyright 2019 Canonical Ltd. This software is licensed under the |
249 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
250 | + |
251 | +from __future__ import print_function |
252 | + |
253 | +__metaclass__ = type |
254 | + |
255 | +import requests |
256 | +from six.moves.urllib.parse import ( |
257 | + urljoin, |
258 | + urlparse, |
259 | + urlunparse, |
260 | + ) |
261 | + |
262 | + |
263 | +class SnapStoreOperationMixin: |
264 | + """Methods supporting operations that interact with the snap store.""" |
265 | + |
266 | + @classmethod |
267 | + def add_arguments(cls, parser): |
268 | + super(SnapStoreOperationMixin, cls).add_arguments(parser) |
269 | + parser.add_argument( |
270 | + "--snap-store-proxy-url", metavar="URL", |
271 | + help="snap store proxy URL") |
272 | + |
273 | + def snap_store_set_proxy(self): |
274 | + if self.args.snap_store_proxy_url is None: |
275 | + return |
276 | + # Canonicalise: proxy registration always sends only the scheme and |
277 | + # domain. |
278 | + parsed_url = urlparse(self.args.snap_store_proxy_url) |
279 | + canonical_url = urlunparse( |
280 | + [parsed_url.scheme, parsed_url.netloc, "", "", "", ""]) |
281 | + assertions_response = requests.get( |
282 | + urljoin(canonical_url, "v2/auth/store/assertions")) |
283 | + assertions_response.raise_for_status() |
284 | + self.backend.run( |
285 | + ["snap", "ack", "/dev/stdin"], input_text=assertions_response.text) |
286 | + store_id = assertions_response.headers.get("X-Assertion-Store-Id") |
287 | + if store_id is not None: |
288 | + self.backend.run( |
289 | + ["snap", "set", "core", "proxy.store={}".format(store_id)]) |
290 | |
291 | === modified file 'lpbuildd/target/tests/test_apt.py' |
292 | --- lpbuildd/target/tests/test_apt.py 2019-02-12 10:41:20 +0000 |
293 | +++ lpbuildd/target/tests/test_apt.py 2019-06-05 14:11:48 +0000 |
294 | @@ -1,4 +1,4 @@ |
295 | -# Copyright 2017 Canonical Ltd. This software is licensed under the |
296 | +# Copyright 2017-2019 Canonical Ltd. This software is licensed under the |
297 | # GNU Affero General Public License version 3 (see the file LICENSE). |
298 | |
299 | __metaclass__ = type |
300 | @@ -53,6 +53,27 @@ |
301 | """).encode("UTF-8"), stat.S_IFREG | 0o644), |
302 | override_sources_list.backend.backend_fs["/etc/apt/sources.list"]) |
303 | |
304 | + def test_apt_proxy(self): |
305 | + args = [ |
306 | + "override-sources-list", |
307 | + "--backend=fake", "--series=xenial", "--arch=amd64", "1", |
308 | + "--apt-proxy-url", "http://apt-proxy.example:3128/", |
309 | + "deb http://archive.ubuntu.com/ubuntu xenial main", |
310 | + ] |
311 | + override_sources_list = parse_args(args=args).operation |
312 | + self.assertEqual(0, override_sources_list.run()) |
313 | + self.assertEqual( |
314 | + (dedent("""\ |
315 | + deb http://archive.ubuntu.com/ubuntu xenial main |
316 | + """).encode("UTF-8"), stat.S_IFREG | 0o644), |
317 | + override_sources_list.backend.backend_fs["/etc/apt/sources.list"]) |
318 | + self.assertEqual( |
319 | + (dedent("""\ |
320 | + Acquire::http::Proxy "http://apt-proxy.example:3128/"; |
321 | + """).encode("UTF-8"), stat.S_IFREG | 0o644), |
322 | + override_sources_list.backend.backend_fs[ |
323 | + "/etc/apt/apt.conf.d/99proxy"]) |
324 | + |
325 | |
326 | class TestAddTrustedKeys(TestCase): |
327 | |
328 | |
329 | === modified file 'lpbuildd/target/tests/test_build_livefs.py' |
330 | --- lpbuildd/target/tests/test_build_livefs.py 2019-02-12 10:41:20 +0000 |
331 | +++ lpbuildd/target/tests/test_build_livefs.py 2019-06-05 14:11:48 +0000 |
332 | @@ -1,11 +1,13 @@ |
333 | -# Copyright 2017 Canonical Ltd. This software is licensed under the |
334 | +# Copyright 2017-2019 Canonical Ltd. This software is licensed under the |
335 | # GNU Affero General Public License version 3 (see the file LICENSE). |
336 | |
337 | __metaclass__ = type |
338 | |
339 | import subprocess |
340 | +from textwrap import dedent |
341 | |
342 | from fixtures import FakeLogger |
343 | +import responses |
344 | from testtools import TestCase |
345 | from testtools.matchers import ( |
346 | AnyMatch, |
347 | @@ -26,12 +28,17 @@ |
348 | |
349 | class RanCommand(MatchesListwise): |
350 | |
351 | - def __init__(self, args, echo=None, cwd=None, **env): |
352 | + def __init__(self, args, echo=None, cwd=None, input_text=None, |
353 | + get_output=None, **env): |
354 | kwargs_matcher = {} |
355 | if echo is not None: |
356 | kwargs_matcher["echo"] = Is(echo) |
357 | if cwd: |
358 | kwargs_matcher["cwd"] = Equals(cwd) |
359 | + if input_text: |
360 | + kwargs_matcher["input_text"] = Equals(input_text) |
361 | + if get_output is not None: |
362 | + kwargs_matcher["get_output"] = Is(get_output) |
363 | if env: |
364 | kwargs_matcher["env"] = MatchesDict( |
365 | {key: Equals(value) for key, value in env.items()}) |
366 | @@ -113,6 +120,36 @@ |
367 | "--install-recommends", "install", "ubuntu-defaults-builder"), |
368 | ])) |
369 | |
370 | + @responses.activate |
371 | + def test_install_snap_store_proxy(self): |
372 | + store_assertion = dedent("""\ |
373 | + type: store |
374 | + store: store-id |
375 | + url: http://snap-store-proxy.example |
376 | + |
377 | + body |
378 | + """) |
379 | + |
380 | + def respond(request): |
381 | + return 200, {"X-Assertion-Store-Id": "store-id"}, store_assertion |
382 | + |
383 | + responses.add_callback( |
384 | + "GET", "http://snap-store-proxy.example/v2/auth/store/assertions", |
385 | + callback=respond) |
386 | + args = [ |
387 | + "buildlivefs", |
388 | + "--backend=fake", "--series=xenial", "--arch=amd64", "1", |
389 | + "--snap-store-proxy-url", "http://snap-store-proxy.example/", |
390 | + ] |
391 | + build_livefs = parse_args(args=args).operation |
392 | + build_livefs.install() |
393 | + self.assertThat(build_livefs.backend.run.calls, MatchesListwise([ |
394 | + RanAptGet("install", "livecd-rootfs"), |
395 | + RanCommand( |
396 | + ["snap", "ack", "/dev/stdin"], input_text=store_assertion), |
397 | + RanCommand(["snap", "set", "core", "proxy.store=store-id"]), |
398 | + ])) |
399 | + |
400 | def test_build(self): |
401 | args = [ |
402 | "buildlivefs", |
403 | |
404 | === modified file 'lpbuildd/target/tests/test_build_snap.py' |
405 | --- lpbuildd/target/tests/test_build_snap.py 2019-05-08 14:52:28 +0000 |
406 | +++ lpbuildd/target/tests/test_build_snap.py 2019-06-05 14:11:48 +0000 |
407 | @@ -1,4 +1,4 @@ |
408 | -# Copyright 2017 Canonical Ltd. This software is licensed under the |
409 | +# Copyright 2017-2019 Canonical Ltd. This software is licensed under the |
410 | # GNU Affero General Public License version 3 (see the file LICENSE). |
411 | |
412 | __metaclass__ = type |
413 | @@ -7,11 +7,13 @@ |
414 | import os.path |
415 | import stat |
416 | import subprocess |
417 | +from textwrap import dedent |
418 | |
419 | from fixtures import ( |
420 | FakeLogger, |
421 | TempDir, |
422 | ) |
423 | +import responses |
424 | from systemfixtures import FakeFilesystem |
425 | from testtools import TestCase |
426 | from testtools.matchers import ( |
427 | @@ -33,14 +35,17 @@ |
428 | |
429 | class RanCommand(MatchesListwise): |
430 | |
431 | - def __init__(self, args, get_output=None, echo=None, cwd=None, **env): |
432 | + def __init__(self, args, echo=None, cwd=None, input_text=None, |
433 | + get_output=None, **env): |
434 | kwargs_matcher = {} |
435 | - if get_output is not None: |
436 | - kwargs_matcher["get_output"] = Is(get_output) |
437 | if echo is not None: |
438 | kwargs_matcher["echo"] = Is(echo) |
439 | if cwd: |
440 | kwargs_matcher["cwd"] = Equals(cwd) |
441 | + if input_text: |
442 | + kwargs_matcher["input_text"] = Equals(input_text) |
443 | + if get_output is not None: |
444 | + kwargs_matcher["get_output"] = Is(get_output) |
445 | if env: |
446 | kwargs_matcher["env"] = MatchesDict( |
447 | {key: Equals(value) for key, value in env.items()}) |
448 | @@ -132,6 +137,38 @@ |
449 | RanAptGet("install", "git", "snapcraft"), |
450 | ])) |
451 | |
452 | + @responses.activate |
453 | + def test_install_snap_store_proxy(self): |
454 | + store_assertion = dedent("""\ |
455 | + type: store |
456 | + store: store-id |
457 | + url: http://snap-store-proxy.example |
458 | + |
459 | + body |
460 | + """) |
461 | + |
462 | + def respond(request): |
463 | + return 200, {"X-Assertion-Store-Id": "store-id"}, store_assertion |
464 | + |
465 | + responses.add_callback( |
466 | + "GET", "http://snap-store-proxy.example/v2/auth/store/assertions", |
467 | + callback=respond) |
468 | + args = [ |
469 | + "buildsnap", |
470 | + "--backend=fake", "--series=xenial", "--arch=amd64", "1", |
471 | + "--git-repository", "lp:foo", |
472 | + "--snap-store-proxy-url", "http://snap-store-proxy.example/", |
473 | + "test-snap", |
474 | + ] |
475 | + build_snap = parse_args(args=args).operation |
476 | + build_snap.install() |
477 | + self.assertThat(build_snap.backend.run.calls, MatchesListwise([ |
478 | + RanAptGet("install", "git", "snapcraft"), |
479 | + RanCommand( |
480 | + ["snap", "ack", "/dev/stdin"], input_text=store_assertion), |
481 | + RanCommand(["snap", "set", "core", "proxy.store=store-id"]), |
482 | + ])) |
483 | + |
484 | def test_install_proxy(self): |
485 | args = [ |
486 | "buildsnap", |
487 | |
488 | === modified file 'lpbuildd/tests/fakebuilder.py' |
489 | --- lpbuildd/tests/fakebuilder.py 2019-02-12 10:41:20 +0000 |
490 | +++ lpbuildd/tests/fakebuilder.py 2019-06-05 14:11:48 +0000 |
491 | @@ -1,4 +1,4 @@ |
492 | -# Copyright 2013 Canonical Ltd. This software is licensed under the |
493 | +# Copyright 2013-2019 Canonical Ltd. This software is licensed under the |
494 | # GNU Affero General Public License version 3 (see the file LICENSE). |
495 | |
496 | __metaclass__ = type |
497 | @@ -9,12 +9,18 @@ |
498 | 'UncontainedBackend', |
499 | ] |
500 | |
501 | +from collections import defaultdict |
502 | import hashlib |
503 | import os |
504 | import shutil |
505 | import stat |
506 | import subprocess |
507 | |
508 | +from six.moves.configparser import ( |
509 | + NoOptionError, |
510 | + NoSectionError, |
511 | + ) |
512 | + |
513 | from lpbuildd.target.backend import Backend |
514 | from lpbuildd.util import ( |
515 | set_personality, |
516 | @@ -78,8 +84,22 @@ |
517 | |
518 | |
519 | class FakeConfig: |
520 | + def __init__(self): |
521 | + self._overrides = defaultdict(dict) |
522 | + |
523 | def get(self, section, key): |
524 | - return key |
525 | + if key in self._overrides[section]: |
526 | + return self._overrides[section][key] |
527 | + elif section == "proxy": |
528 | + if not self._overrides[section]: |
529 | + raise NoSectionError(section) |
530 | + else: |
531 | + raise NoOptionError(section, key) |
532 | + else: |
533 | + return key |
534 | + |
535 | + def set(self, section, key, value): |
536 | + self._overrides[section][key] = value |
537 | |
538 | |
539 | class FakeBuilder: |
540 | |
541 | === modified file 'lpbuildd/tests/test_debian.py' |
542 | --- lpbuildd/tests/test_debian.py 2019-02-12 10:41:20 +0000 |
543 | +++ lpbuildd/tests/test_debian.py 2019-06-05 14:11:48 +0000 |
544 | @@ -1,4 +1,4 @@ |
545 | -# Copyright 2017 Canonical Ltd. This software is licensed under the |
546 | +# Copyright 2017-2019 Canonical Ltd. This software is licensed under the |
547 | # GNU Affero General Public License version 3 (see the file LICENSE). |
548 | |
549 | __metaclass__ = type |
550 | @@ -404,6 +404,36 @@ |
551 | self.assertTrue(self.builder.wasCalled('buildOK')) |
552 | self.assertTrue(self.builder.wasCalled('buildComplete')) |
553 | |
554 | + def test_iterate_apt_proxy(self): |
555 | + # The build manager can be configured to use an APT proxy. |
556 | + self.builder._config.set( |
557 | + 'proxy', 'apt', 'http://apt-proxy.example:3128/') |
558 | + extra_args = { |
559 | + 'arch_tag': 'amd64', |
560 | + 'archives': [ |
561 | + 'deb http://ppa.launchpad.dev/owner/name/ubuntu xenial main', |
562 | + ], |
563 | + 'series': 'xenial', |
564 | + } |
565 | + self.startBuild(extra_args) |
566 | + |
567 | + self.buildmanager.iterate(0) |
568 | + self.assertEqual(DebianBuildState.UNPACK, self.getState()) |
569 | + self.buildmanager.iterate(0) |
570 | + self.assertEqual(DebianBuildState.MOUNT, self.getState()) |
571 | + self.buildmanager.iterate(0) |
572 | + self.assertEqual(DebianBuildState.SOURCES, self.getState()) |
573 | + self.assertEqual( |
574 | + (['sharepath/bin/in-target', 'in-target', 'override-sources-list', |
575 | + '--backend=chroot', '--series=xenial', '--arch=amd64', |
576 | + self.buildid, |
577 | + '--apt-proxy-url', 'http://apt-proxy.example:3128/', |
578 | + 'deb http://ppa.launchpad.dev/owner/name/ubuntu xenial main'], |
579 | + None), |
580 | + self.buildmanager.commands[-1]) |
581 | + self.assertEqual( |
582 | + self.buildmanager.iterate, self.buildmanager.iterators[-1]) |
583 | + |
584 | def test_iterate_lxd(self): |
585 | # The build manager passes the image_type argument through to |
586 | # unpack-chroot. |
587 | |
588 | === modified file 'lpbuildd/tests/test_livefs.py' |
589 | --- lpbuildd/tests/test_livefs.py 2019-02-12 10:41:20 +0000 |
590 | +++ lpbuildd/tests/test_livefs.py 2019-06-05 14:11:48 +0000 |
591 | @@ -1,4 +1,4 @@ |
592 | -# Copyright 2013-2018 Canonical Ltd. This software is licensed under the |
593 | +# Copyright 2013-2019 Canonical Ltd. This software is licensed under the |
594 | # GNU Affero General Public License version 3 (see the file LICENSE). |
595 | |
596 | __metaclass__ = type |
597 | @@ -58,7 +58,7 @@ |
598 | return self.buildmanager._state |
599 | |
600 | @defer.inlineCallbacks |
601 | - def startBuild(self): |
602 | + def startBuild(self, options=None): |
603 | # The build manager's iterate() kicks off the consecutive states |
604 | # after INIT. |
605 | extra_args = { |
606 | @@ -85,6 +85,8 @@ |
607 | "--backend=lxd", "--series=saucy", "--arch=i386", self.buildid, |
608 | "--project", "ubuntu", |
609 | ] |
610 | + if options is not None: |
611 | + expected_command.extend(options) |
612 | self.assertEqual(expected_command, self.buildmanager.commands[-1]) |
613 | self.assertEqual( |
614 | self.buildmanager.iterate, self.buildmanager.iterators[-1]) |
615 | @@ -131,6 +133,15 @@ |
616 | self.assertFalse(self.builder.wasCalled("buildFail")) |
617 | |
618 | @defer.inlineCallbacks |
619 | + def test_iterate_snap_store_proxy(self): |
620 | + # The build manager can be told to use a snap store proxy. |
621 | + self.builder._config.set( |
622 | + "proxy", "snapstore", "http://snap-store-proxy.example/") |
623 | + expected_options = [ |
624 | + "--snap-store-proxy-url", "http://snap-store-proxy.example/"] |
625 | + yield self.startBuild(options=expected_options) |
626 | + |
627 | + @defer.inlineCallbacks |
628 | def test_omits_symlinks(self): |
629 | # Symlinks in the build output are not included in gathered results. |
630 | yield self.startBuild() |
631 | |
632 | === modified file 'lpbuildd/tests/test_snap.py' |
633 | --- lpbuildd/tests/test_snap.py 2019-05-22 17:59:14 +0000 |
634 | +++ lpbuildd/tests/test_snap.py 2019-06-05 14:11:48 +0000 |
635 | @@ -1,4 +1,4 @@ |
636 | -# Copyright 2015-2018 Canonical Ltd. This software is licensed under the |
637 | +# Copyright 2015-2019 Canonical Ltd. This software is licensed under the |
638 | # GNU Affero General Public License version 3 (see the file LICENSE). |
639 | |
640 | __metaclass__ = type |
641 | @@ -377,6 +377,15 @@ |
642 | self.buildmanager.iterate, self.buildmanager.iterators[-1]) |
643 | self.assertFalse(self.builder.wasCalled("buildFail")) |
644 | |
645 | + @defer.inlineCallbacks |
646 | + def test_iterate_snap_store_proxy(self): |
647 | + # The build manager can be told to use a snap store proxy. |
648 | + self.builder._config.set( |
649 | + "proxy", "snapstore", "http://snap-store-proxy.example/") |
650 | + expected_options = [ |
651 | + "--snap-store-proxy-url", "http://snap-store-proxy.example/"] |
652 | + yield self.startBuild(options=expected_options) |
653 | + |
654 | def getListenerURL(self, listener): |
655 | port = listener.getHost().port |
656 | return b"http://localhost:%d/" % port |
657 | |
658 | === modified file 'setup.py' |
659 | --- setup.py 2018-10-25 09:04:12 +0000 |
660 | +++ setup.py 2019-06-05 14:11:48 +0000 |
661 | @@ -1,6 +1,6 @@ |
662 | #!/usr/bin/env python |
663 | |
664 | -# Copyright 2015 Canonical Ltd. All rights reserved. |
665 | +# Copyright 2015-2019 Canonical Ltd. All rights reserved. |
666 | # |
667 | # This file is part of launchpad-buildd. |
668 | # |
669 | @@ -59,6 +59,8 @@ |
670 | # causes problems for Launchpad's build system. |
671 | #'python-apt', |
672 | 'python-debian>=0.1.23', |
673 | + 'requests', |
674 | + 'six', |
675 | 'Twisted', |
676 | 'zope.interface', |
677 | ], |
678 | @@ -69,6 +71,7 @@ |
679 | tests_require=[ |
680 | 'fixtures', |
681 | 'mock', |
682 | + 'responses', |
683 | 'systemfixtures', |
684 | 'testtools', |
685 | 'txfixtures', |