Merge lp:~cjwatson/launchpad-buildd/start-stop-python into lp:launchpad-buildd
- start-stop-python
- Merge into trunk
Proposed by
Colin Watson
Status: | Merged |
---|---|
Merged at revision: | 254 |
Proposed branch: | lp:~cjwatson/launchpad-buildd/start-stop-python |
Merge into: | lp:launchpad-buildd |
Prerequisite: | lp:~cjwatson/launchpad-buildd/create-remove-python |
Diff against target: |
824 lines (+382/-94) 20 files modified
MANIFEST.in (+0/-2) bin/mount-chroot (+0/-25) bin/umount-chroot (+0/-40) debian/changelog (+2/-0) debian/control (+1/-1) debian/launchpad-buildd.install (+0/-2) lpbuildd/slave.py (+2/-6) lpbuildd/target/backend.py (+11/-0) lpbuildd/target/chroot.py (+50/-1) lpbuildd/target/cli.py (+4/-0) lpbuildd/target/lifecycle.py (+25/-0) lpbuildd/target/tests/test_chroot.py (+131/-1) lpbuildd/target/tests/test_lifecycle.py (+59/-1) lpbuildd/target/tests/testfixtures.py (+38/-0) lpbuildd/tests/test_binarypackage.py (+6/-2) lpbuildd/tests/test_debian.py (+14/-4) lpbuildd/tests/test_livefs.py (+5/-1) lpbuildd/tests/test_snap.py (+10/-2) lpbuildd/tests/test_sourcepackagerecipe.py (+12/-3) lpbuildd/tests/test_translationtemplatesbuildmanager.py (+12/-3) |
To merge this branch: | bzr merge lp:~cjwatson/launchpad-buildd/start-stop-python |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
William Grant | code | Approve | |
Review via email: mp+328570@code.launchpad.net |
Commit message
Rewrite mount-chroot and umount-chroot in Python, allowing them to have
unit tests.
Description of the change
To post a comment you must log in.
- 254. By Colin Watson
-
Merge trunk.
- 255. By Colin Watson
-
Merge Start and Stop into lpbuildd.
target. lifecycle.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'MANIFEST.in' |
2 | --- MANIFEST.in 2017-08-22 14:23:16 +0000 |
3 | +++ MANIFEST.in 2017-08-22 14:36:57 +0000 |
4 | @@ -5,14 +5,12 @@ |
5 | include bin/buildsnap |
6 | include bin/generate-translation-templates |
7 | include bin/in-target |
8 | -include bin/mount-chroot |
9 | include bin/sbuild-package |
10 | include bin/scan-for-processes |
11 | include bin/slave-prep |
12 | include bin/snap-git-proxy |
13 | include bin/test_buildd_generatetranslationtemplates |
14 | include bin/test_buildd_recipe |
15 | -include bin/umount-chroot |
16 | include buildd-genconfig |
17 | include buildd-slave.tac |
18 | include debian/changelog |
19 | |
20 | === removed file 'bin/mount-chroot' |
21 | --- bin/mount-chroot 2017-07-25 22:11:19 +0000 |
22 | +++ bin/mount-chroot 1970-01-01 00:00:00 +0000 |
23 | @@ -1,25 +0,0 @@ |
24 | -#!/bin/sh |
25 | -# |
26 | -# Copyright 2009 Canonical Ltd. This software is licensed under the |
27 | -# GNU Affero General Public License version 3 (see the file LICENSE). |
28 | - |
29 | -# Buildd Slave tool to mount a chroot |
30 | - |
31 | -# Expects build id as arg 1, makes build-id to contain the build |
32 | - |
33 | -# Needs SUDO to be set to a sudo instance for passwordless access |
34 | - |
35 | -SUDO=/usr/bin/sudo |
36 | -BUILDID="$1" |
37 | - |
38 | -set -e |
39 | - |
40 | -exec 2>&1 |
41 | - |
42 | -echo "Mounting chroot for build $BUILDID" |
43 | - |
44 | -$SUDO mount -t proc none "$HOME/build-$BUILDID/chroot-autobuild/proc" |
45 | -$SUDO mount -t devpts -o gid=5,mode=620 none "$HOME/build-$BUILDID/chroot-autobuild/dev/pts" |
46 | -$SUDO mount -t sysfs none "$HOME/build-$BUILDID/chroot-autobuild/sys" |
47 | -$SUDO mount -t tmpfs none "$HOME/build-$BUILDID/chroot-autobuild/dev/shm" |
48 | -$SUDO cp /etc/hosts /etc/hostname /etc/resolv.conf $HOME/build-$BUILDID/chroot-autobuild/etc/ |
49 | |
50 | === removed file 'bin/umount-chroot' |
51 | --- bin/umount-chroot 2017-07-25 22:11:19 +0000 |
52 | +++ bin/umount-chroot 1970-01-01 00:00:00 +0000 |
53 | @@ -1,40 +0,0 @@ |
54 | -#!/bin/sh |
55 | -# |
56 | -# Copyright 2009-2011 Canonical Ltd. This software is licensed under the |
57 | -# GNU Affero General Public License version 3 (see the file LICENSE). |
58 | - |
59 | -# Buildd Slave tool to mount a chroot |
60 | - |
61 | -# Expects build id as arg 1, makes build-id to contain the build |
62 | - |
63 | -# Needs SUDO to be set to a sudo instance for passwordless access |
64 | - |
65 | -SUDO=/usr/bin/sudo |
66 | -BUILDID="$1" |
67 | -GREP=/bin/grep |
68 | -CUT=/usr/bin/cut |
69 | -XARGS=/usr/bin/xargs |
70 | -SORT=/usr/bin/sort |
71 | - |
72 | -set -e |
73 | - |
74 | -exec 2>&1 |
75 | - |
76 | -echo "Unmounting chroot for build $BUILDID..." |
77 | - |
78 | -# binfmt-support adds a mount under /proc, which means that our first |
79 | -# pass at umounting fails unless we reverse the list. Leave the while |
80 | -# loop in just to handle pathological cases, too. |
81 | -COUNT=0 |
82 | -while $GREP -q "$HOME/build-$BUILDID/chroot-autobuild" /proc/mounts; do |
83 | - COUNT=$(($COUNT+1)) |
84 | - if [ $COUNT -ge 20 ]; then |
85 | - echo "failed to umount $HOME/build-$BUILDID/chroot-autobuild" |
86 | - if [ -x /usr/bin/lsof ]; then |
87 | - /usr/bin/lsof "$HOME/build-$BUILDID/chroot-autobuild" |
88 | - fi |
89 | - exit 1 |
90 | - fi |
91 | - $GREP "$HOME/build-$BUILDID/chroot-autobuild" /proc/mounts | \ |
92 | - $CUT -d\ -f2 | LANG=C $SORT -r | $XARGS -r -n 1 $SUDO umount || sleep 1 |
93 | -done |
94 | |
95 | === modified file 'debian/changelog' |
96 | --- debian/changelog 2017-08-04 02:27:56 +0000 |
97 | +++ debian/changelog 2017-08-22 14:36:57 +0000 |
98 | @@ -17,6 +17,8 @@ |
99 | * Configure sbuild to use schroot sessions rather than sudo. |
100 | * Rewrite unpack-chroot and remove-build in Python, allowing them to have |
101 | unit tests. |
102 | + * Rewrite mount-chroot and umount-chroot in Python, allowing them to have |
103 | + unit tests. |
104 | |
105 | -- Colin Watson <cjwatson@ubuntu.com> Tue, 25 Jul 2017 23:07:58 +0100 |
106 | |
107 | |
108 | === modified file 'debian/control' |
109 | --- debian/control 2017-08-03 13:13:08 +0000 |
110 | +++ debian/control 2017-08-22 14:36:57 +0000 |
111 | @@ -10,7 +10,7 @@ |
112 | Architecture: all |
113 | Depends: python-lpbuildd (=${source:Version}), python, debootstrap, dpkg-dev, |
114 | file, bzip2, sudo, ntpdate, adduser, apt-transport-https, lsb-release, |
115 | - pristine-tar, python-apt, sbuild, schroot, ${misc:Depends} |
116 | + pristine-tar, python-apt, sbuild, schroot, lsof, ${misc:Depends} |
117 | Description: Launchpad buildd slave |
118 | This is the launchpad buildd slave package. It contains everything needed to |
119 | get a launchpad buildd going apart from the database manipulation required to |
120 | |
121 | === modified file 'debian/launchpad-buildd.install' |
122 | --- debian/launchpad-buildd.install 2017-08-22 14:23:16 +0000 |
123 | +++ debian/launchpad-buildd.install 2017-08-22 14:36:57 +0000 |
124 | @@ -8,9 +8,7 @@ |
125 | bin/buildsnap usr/share/launchpad-buildd/slavebin |
126 | bin/generate-translation-templates usr/share/launchpad-buildd/slavebin |
127 | bin/in-target usr/share/launchpad-buildd/slavebin |
128 | -bin/mount-chroot usr/share/launchpad-buildd/slavebin |
129 | bin/sbuild-package usr/share/launchpad-buildd/slavebin |
130 | bin/scan-for-processes usr/share/launchpad-buildd/slavebin |
131 | bin/slave-prep usr/share/launchpad-buildd/slavebin |
132 | bin/snap-git-proxy usr/share/launchpad-buildd/slavebin |
133 | -bin/umount-chroot usr/share/launchpad-buildd/slavebin |
134 | |
135 | === modified file 'lpbuildd/slave.py' |
136 | --- lpbuildd/slave.py 2017-08-22 14:23:16 +0000 |
137 | +++ lpbuildd/slave.py 2017-08-22 14:36:57 +0000 |
138 | @@ -122,8 +122,6 @@ |
139 | self._slavebin = os.path.join(self._sharepath, "slavebin") |
140 | self._preppath = os.path.join(self._slavebin, "slave-prep") |
141 | self._intargetpath = os.path.join(self._slavebin, "in-target") |
142 | - self._mountpath = os.path.join(self._slavebin, "mount-chroot") |
143 | - self._umountpath = os.path.join(self._slavebin, "umount-chroot") |
144 | self._scanpath = os.path.join(self._slavebin, "scan-for-processes") |
145 | self._subprocess = None |
146 | self._reaped_states = set() |
147 | @@ -197,13 +195,11 @@ |
148 | |
149 | def doMounting(self): |
150 | """Mount things in the chroot, e.g. proc.""" |
151 | - self.runSubProcess( self._mountpath, |
152 | - ["mount-chroot", self._buildid]) |
153 | + self.runTargetSubProcess("mount-chroot") |
154 | |
155 | def doUnmounting(self): |
156 | """Unmount the chroot.""" |
157 | - self.runSubProcess( self._umountpath, |
158 | - ["umount-chroot", self._buildid]) |
159 | + self.runTargetSubProcess("umount-chroot") |
160 | |
161 | def initiate(self, files, chroot, extra_args): |
162 | """Initiate a build given the input files. |
163 | |
164 | === modified file 'lpbuildd/target/backend.py' |
165 | --- lpbuildd/target/backend.py 2017-08-22 14:23:16 +0000 |
166 | +++ lpbuildd/target/backend.py 2017-08-22 14:36:57 +0000 |
167 | @@ -29,6 +29,13 @@ |
168 | """ |
169 | raise NotImplementedError |
170 | |
171 | + def start(self): |
172 | + """Start the backend. |
173 | + |
174 | + This puts the backend into a state where it can run commands. |
175 | + """ |
176 | + raise NotImplementedError |
177 | + |
178 | def run(self, args, env=None, input_text=None, **kwargs): |
179 | """Run a command in the target environment. |
180 | |
181 | @@ -53,6 +60,10 @@ |
182 | """ |
183 | raise NotImplementedError |
184 | |
185 | + def stop(self): |
186 | + """Stop the backend.""" |
187 | + raise NotImplementedError |
188 | + |
189 | def remove(self): |
190 | """Remove the backend.""" |
191 | subprocess.check_call(["sudo", "rm", "-rf", self.build_path]) |
192 | |
193 | === modified file 'lpbuildd/target/chroot.py' |
194 | --- lpbuildd/target/chroot.py 2017-08-04 02:27:56 +0000 |
195 | +++ lpbuildd/target/chroot.py 2017-08-22 14:36:57 +0000 |
196 | @@ -8,8 +8,12 @@ |
197 | import os.path |
198 | import stat |
199 | import subprocess |
200 | +import time |
201 | |
202 | -from lpbuildd.target.backend import Backend |
203 | +from lpbuildd.target.backend import ( |
204 | + Backend, |
205 | + BackendException, |
206 | + ) |
207 | from lpbuildd.util import ( |
208 | set_personality, |
209 | shell_escape, |
210 | @@ -28,6 +32,25 @@ |
211 | subprocess.check_call( |
212 | ["sudo", "tar", "-C", self.build_path, "-xf", tarball_path]) |
213 | |
214 | + def start(self): |
215 | + """See `Backend`.""" |
216 | + mounts = ( |
217 | + ("proc", None, "none", "proc"), |
218 | + ("devpts", "gid=5,mode=620", "none", "dev/pts"), |
219 | + ("sysfs", None, "none", "sys"), |
220 | + ("tmpfs", None, "none", "dev/shm"), |
221 | + ) |
222 | + for mount in mounts: |
223 | + cmd = ["sudo", "mount", "-t", mount[0]] |
224 | + if mount[1]: |
225 | + cmd.extend(["-o", mount[1]]) |
226 | + cmd.append(mount[2]) |
227 | + cmd.append(os.path.join(self.chroot_path, mount[3])) |
228 | + subprocess.check_call(cmd) |
229 | + |
230 | + for path in ("/etc/hosts", "/etc/hostname", "/etc/resolv.conf"): |
231 | + self.copy_in(path, path) |
232 | + |
233 | def run(self, args, env=None, input_text=None, **kwargs): |
234 | """See `Backend`.""" |
235 | if env: |
236 | @@ -57,3 +80,29 @@ |
237 | subprocess.check_call( |
238 | ["sudo", "install", "-o", "root", "-g", "root", "-m", "%o" % mode, |
239 | source_path, full_target_path]) |
240 | + |
241 | + def _get_chroot_mounts(self): |
242 | + with open("/proc/mounts") as mounts_file: |
243 | + for line in mounts_file: |
244 | + mount_path = line.split()[1] |
245 | + if mount_path.startswith(self.chroot_path): |
246 | + yield mount_path |
247 | + |
248 | + def stop(self): |
249 | + """See `Backend`.""" |
250 | + for _ in range(20): |
251 | + # Reverse the list, since we must unmount subdirectories before |
252 | + # parent directories. |
253 | + mounts = reversed(list(self._get_chroot_mounts())) |
254 | + if not mounts: |
255 | + break |
256 | + retcodes = [ |
257 | + subprocess.call(["sudo", "umount", mount]) |
258 | + for mount in mounts] |
259 | + if any(retcodes): |
260 | + time.sleep(1) |
261 | + else: |
262 | + if list(self._get_chroot_mounts()): |
263 | + subprocess.check_call(["lsof", self.chroot_path]) |
264 | + raise BackendException( |
265 | + "Failed to unmount %s" % self.chroot_path) |
266 | |
267 | === modified file 'lpbuildd/target/cli.py' |
268 | --- lpbuildd/target/cli.py 2017-08-22 14:25:03 +0000 |
269 | +++ lpbuildd/target/cli.py 2017-08-22 14:36:57 +0000 |
270 | @@ -17,6 +17,8 @@ |
271 | from lpbuildd.target.lifecycle import ( |
272 | Create, |
273 | Remove, |
274 | + Start, |
275 | + Stop, |
276 | ) |
277 | |
278 | |
279 | @@ -42,7 +44,9 @@ |
280 | operations = { |
281 | "add-trusted-keys": AddTrustedKeys, |
282 | "override-sources-list": OverrideSourcesList, |
283 | + "mount-chroot": Start, |
284 | "remove-build": Remove, |
285 | + "umount-chroot": Stop, |
286 | "unpack-chroot": Create, |
287 | "update-debian-chroot": Update, |
288 | } |
289 | |
290 | === modified file 'lpbuildd/target/lifecycle.py' |
291 | --- lpbuildd/target/lifecycle.py 2017-08-22 14:25:03 +0000 |
292 | +++ lpbuildd/target/lifecycle.py 2017-08-22 14:36:57 +0000 |
293 | @@ -7,6 +7,7 @@ |
294 | |
295 | import logging |
296 | |
297 | +from lpbuildd.target.backend import BackendException |
298 | from lpbuildd.target.operation import Operation |
299 | |
300 | |
301 | @@ -28,6 +29,30 @@ |
302 | return 0 |
303 | |
304 | |
305 | +class Start(Operation): |
306 | + |
307 | + description = "Start the target environment." |
308 | + |
309 | + def run(self): |
310 | + logger.info("Starting target for build %s", self.args.build_id) |
311 | + self.backend.start() |
312 | + return 0 |
313 | + |
314 | + |
315 | +class Stop(Operation): |
316 | + |
317 | + description = "Stop the target environment." |
318 | + |
319 | + def run(self): |
320 | + logger.info("Stopping target for build %s", self.args.build_id) |
321 | + try: |
322 | + self.backend.stop() |
323 | + except BackendException: |
324 | + logger.exception('Failed to stop target') |
325 | + return 1 |
326 | + return 0 |
327 | + |
328 | + |
329 | class Remove(Operation): |
330 | |
331 | description = "Remove the target environment." |
332 | |
333 | === modified file 'lpbuildd/target/tests/test_chroot.py' |
334 | --- lpbuildd/target/tests/test_chroot.py 2017-08-04 02:27:56 +0000 |
335 | +++ lpbuildd/target/tests/test_chroot.py 2017-08-22 14:36:57 +0000 |
336 | @@ -4,15 +4,23 @@ |
337 | __metaclass__ = type |
338 | |
339 | import os.path |
340 | +from textwrap import dedent |
341 | +import time |
342 | |
343 | from fixtures import ( |
344 | EnvironmentVariable, |
345 | TempDir, |
346 | ) |
347 | -from systemfixtures import FakeProcesses |
348 | +from systemfixtures import ( |
349 | + FakeFilesystem, |
350 | + FakeProcesses, |
351 | + FakeTime, |
352 | + ) |
353 | from testtools import TestCase |
354 | |
355 | +from lpbuildd.target.backend import BackendException |
356 | from lpbuildd.target.chroot import Chroot |
357 | +from lpbuildd.target.tests.testfixtures import SudoUmount |
358 | |
359 | |
360 | class TestChroot(TestCase): |
361 | @@ -31,6 +39,43 @@ |
362 | expected_args, |
363 | [proc._args["args"] for proc in processes_fixture.procs]) |
364 | |
365 | + def test_start(self): |
366 | + self.useFixture(EnvironmentVariable("HOME", "/expected/home")) |
367 | + processes_fixture = self.useFixture(FakeProcesses()) |
368 | + processes_fixture.add(lambda _: {}, name="sudo") |
369 | + fs_fixture = self.useFixture(FakeFilesystem()) |
370 | + fs_fixture.add("/etc") |
371 | + os.mkdir("/etc") |
372 | + for etc_name in ("hosts", "hostname", "resolv.conf.real"): |
373 | + with open(os.path.join("/etc", etc_name), "w") as etc_file: |
374 | + etc_file.write("%s\n" % etc_name) |
375 | + os.chmod(os.path.join("/etc", etc_name), 0o644) |
376 | + os.symlink("resolv.conf.real", "/etc/resolv.conf") |
377 | + Chroot("1", "xenial", "amd64").start() |
378 | + |
379 | + expected_args = [ |
380 | + ["sudo", "mount", "-t", "proc", "none", |
381 | + "/expected/home/build-1/chroot-autobuild/proc"], |
382 | + ["sudo", "mount", "-t", "devpts", "-o", "gid=5,mode=620", "none", |
383 | + "/expected/home/build-1/chroot-autobuild/dev/pts"], |
384 | + ["sudo", "mount", "-t", "sysfs", "none", |
385 | + "/expected/home/build-1/chroot-autobuild/sys"], |
386 | + ["sudo", "mount", "-t", "tmpfs", "none", |
387 | + "/expected/home/build-1/chroot-autobuild/dev/shm"], |
388 | + ["sudo", "install", "-o", "root", "-g", "root", "-m", "644", |
389 | + "/etc/hosts", |
390 | + "/expected/home/build-1/chroot-autobuild/etc/hosts"], |
391 | + ["sudo", "install", "-o", "root", "-g", "root", "-m", "644", |
392 | + "/etc/hostname", |
393 | + "/expected/home/build-1/chroot-autobuild/etc/hostname"], |
394 | + ["sudo", "install", "-o", "root", "-g", "root", "-m", "644", |
395 | + "/etc/resolv.conf", |
396 | + "/expected/home/build-1/chroot-autobuild/etc/resolv.conf"], |
397 | + ] |
398 | + self.assertEqual( |
399 | + expected_args, |
400 | + [proc._args["args"] for proc in processes_fixture.procs]) |
401 | + |
402 | def test_run(self): |
403 | self.useFixture(EnvironmentVariable("HOME", "/expected/home")) |
404 | processes_fixture = self.useFixture(FakeProcesses()) |
405 | @@ -69,6 +114,91 @@ |
406 | expected_args, |
407 | [proc._args["args"] for proc in processes_fixture.procs]) |
408 | |
409 | + def _make_initial_proc_mounts(self): |
410 | + fs_fixture = self.useFixture(FakeFilesystem()) |
411 | + fs_fixture.add("/proc") |
412 | + os.mkdir("/proc") |
413 | + with open("/proc/mounts", "w") as mounts_file: |
414 | + mounts_file.write(dedent("""\ |
415 | + sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0 |
416 | + proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 |
417 | + none {chroot}/proc proc rw,relatime 0 0 |
418 | + none {chroot}/dev/pts devpts rw,relative,gid=5,mode=620 0 0 |
419 | + none {chroot}/sys sysfs rw,relatime 0 0 |
420 | + none {chroot}/dev/shm tmpfs rw,relatime 0 0 |
421 | + """.format(chroot="/expected/home/build-1/chroot-autobuild"))) |
422 | + |
423 | + def test_stop(self): |
424 | + self.useFixture(EnvironmentVariable("HOME", "/expected/home")) |
425 | + processes_fixture = self.useFixture(FakeProcesses()) |
426 | + processes_fixture.add(SudoUmount(), name="sudo") |
427 | + self._make_initial_proc_mounts() |
428 | + self.useFixture(FakeTime()) |
429 | + start_time = time.time() |
430 | + Chroot("1", "xenial", "amd64").stop() |
431 | + |
432 | + expected_chroot_path = "/expected/home/build-1/chroot-autobuild" |
433 | + expected_args = [ |
434 | + ["sudo", "umount", expected_chroot_path + "/dev/shm"], |
435 | + ["sudo", "umount", expected_chroot_path + "/sys"], |
436 | + ["sudo", "umount", expected_chroot_path + "/dev/pts"], |
437 | + ["sudo", "umount", expected_chroot_path + "/proc"], |
438 | + ] |
439 | + self.assertEqual( |
440 | + expected_args, |
441 | + [proc._args["args"] for proc in processes_fixture.procs]) |
442 | + self.assertEqual(start_time, time.time()) |
443 | + |
444 | + def test_stop_retries(self): |
445 | + self.useFixture(EnvironmentVariable("HOME", "/expected/home")) |
446 | + processes_fixture = self.useFixture(FakeProcesses()) |
447 | + delays = {"/expected/home/build-1/chroot-autobuild/sys": 1} |
448 | + processes_fixture.add(SudoUmount(delays=delays), name="sudo") |
449 | + self._make_initial_proc_mounts() |
450 | + self.useFixture(FakeTime()) |
451 | + start_time = time.time() |
452 | + Chroot("1", "xenial", "amd64").stop() |
453 | + |
454 | + expected_chroot_path = "/expected/home/build-1/chroot-autobuild" |
455 | + expected_args = [ |
456 | + ["sudo", "umount", expected_chroot_path + "/dev/shm"], |
457 | + ["sudo", "umount", expected_chroot_path + "/sys"], |
458 | + ["sudo", "umount", expected_chroot_path + "/dev/pts"], |
459 | + ["sudo", "umount", expected_chroot_path + "/proc"], |
460 | + ["sudo", "umount", expected_chroot_path + "/sys"], |
461 | + ] |
462 | + self.assertEqual( |
463 | + expected_args, |
464 | + [proc._args["args"] for proc in processes_fixture.procs]) |
465 | + self.assertEqual(start_time + 1, time.time()) |
466 | + |
467 | + def test_stop_too_many_retries(self): |
468 | + self.useFixture(EnvironmentVariable("HOME", "/expected/home")) |
469 | + processes_fixture = self.useFixture(FakeProcesses()) |
470 | + delays = {"/expected/home/build-1/chroot-autobuild/sys": 20} |
471 | + processes_fixture.add(SudoUmount(delays=delays), name="sudo") |
472 | + processes_fixture.add(lambda _: {}, name="lsof") |
473 | + self._make_initial_proc_mounts() |
474 | + self.useFixture(FakeTime()) |
475 | + start_time = time.time() |
476 | + self.assertRaises( |
477 | + BackendException, Chroot("1", "xenial", "amd64").stop) |
478 | + |
479 | + expected_chroot_path = "/expected/home/build-1/chroot-autobuild" |
480 | + expected_args = [ |
481 | + ["sudo", "umount", expected_chroot_path + "/dev/shm"], |
482 | + ["sudo", "umount", expected_chroot_path + "/sys"], |
483 | + ["sudo", "umount", expected_chroot_path + "/dev/pts"], |
484 | + ["sudo", "umount", expected_chroot_path + "/proc"], |
485 | + ] |
486 | + expected_args.extend( |
487 | + [["sudo", "umount", expected_chroot_path + "/sys"]] * 19) |
488 | + expected_args.append(["lsof", expected_chroot_path]) |
489 | + self.assertEqual( |
490 | + expected_args, |
491 | + [proc._args["args"] for proc in processes_fixture.procs]) |
492 | + self.assertEqual(start_time + 20, time.time()) |
493 | + |
494 | def test_remove(self): |
495 | self.useFixture(EnvironmentVariable("HOME", "/expected/home")) |
496 | processes_fixture = self.useFixture(FakeProcesses()) |
497 | |
498 | === modified file 'lpbuildd/target/tests/test_lifecycle.py' |
499 | --- lpbuildd/target/tests/test_lifecycle.py 2017-08-22 14:25:03 +0000 |
500 | +++ lpbuildd/target/tests/test_lifecycle.py 2017-08-22 14:36:57 +0000 |
501 | @@ -3,11 +3,17 @@ |
502 | |
503 | __metaclass__ = type |
504 | |
505 | +import os.path |
506 | + |
507 | from fixtures import EnvironmentVariable |
508 | -from systemfixtures import FakeProcesses |
509 | +from systemfixtures import ( |
510 | + FakeFilesystem, |
511 | + FakeProcesses, |
512 | + ) |
513 | from testtools import TestCase |
514 | |
515 | from lpbuildd.target.cli import parse_args |
516 | +from lpbuildd.target.tests.testfixtures import SudoUmount |
517 | |
518 | |
519 | class TestCreate(TestCase): |
520 | @@ -32,6 +38,58 @@ |
521 | [proc._args["args"] for proc in processes_fixture.procs]) |
522 | |
523 | |
524 | +class TestStart(TestCase): |
525 | + |
526 | + def test_succeeds(self): |
527 | + self.useFixture(EnvironmentVariable("HOME", "/expected/home")) |
528 | + processes_fixture = self.useFixture(FakeProcesses()) |
529 | + processes_fixture.add(lambda _: {}, name="sudo") |
530 | + fs_fixture = self.useFixture(FakeFilesystem()) |
531 | + fs_fixture.add("/etc") |
532 | + os.mkdir("/etc") |
533 | + for etc_name in ("hosts", "hostname", "resolv.conf.real"): |
534 | + with open(os.path.join("/etc", etc_name), "w") as etc_file: |
535 | + etc_file.write("%s\n" % etc_name) |
536 | + os.chmod(os.path.join("/etc", etc_name), 0o644) |
537 | + os.symlink("resolv.conf.real", "/etc/resolv.conf") |
538 | + args = [ |
539 | + "mount-chroot", |
540 | + "--backend=chroot", "--series=xenial", "--arch=amd64", "1", |
541 | + ] |
542 | + parse_args(args=args).operation.run() |
543 | + |
544 | + # Tested in more detail in lpbuildd.target.tests.test_chroot. |
545 | + self.assertIn( |
546 | + ["sudo", "mount", "-t", "proc", "none", |
547 | + "/expected/home/build-1/chroot-autobuild/proc"], |
548 | + [proc._args["args"] for proc in processes_fixture.procs]) |
549 | + |
550 | + |
551 | +class TestStop(TestCase): |
552 | + |
553 | + def test_succeeds(self): |
554 | + self.useFixture(EnvironmentVariable("HOME", "/expected/home")) |
555 | + processes_fixture = self.useFixture(FakeProcesses()) |
556 | + processes_fixture.add(SudoUmount(), name="sudo") |
557 | + fs_fixture = self.useFixture(FakeFilesystem()) |
558 | + fs_fixture.add("/proc") |
559 | + os.mkdir("/proc") |
560 | + with open("/proc/mounts", "w") as mounts_file: |
561 | + mounts_file.write( |
562 | + "none {chroot}/proc proc rw,relatime 0 0".format( |
563 | + chroot="/expected/home/build-1/chroot-autobuild")) |
564 | + args = [ |
565 | + "umount-chroot", |
566 | + "--backend=chroot", "--series=xenial", "--arch=amd64", "1", |
567 | + ] |
568 | + parse_args(args=args).operation.run() |
569 | + |
570 | + # Tested in more detail in lpbuildd.target.tests.test_chroot. |
571 | + self.assertIn( |
572 | + ["sudo", "umount", "/expected/home/build-1/chroot-autobuild/proc"], |
573 | + [proc._args["args"] for proc in processes_fixture.procs]) |
574 | + |
575 | + |
576 | class TestRemove(TestCase): |
577 | |
578 | def test_succeeds(self): |
579 | |
580 | === added file 'lpbuildd/target/tests/testfixtures.py' |
581 | --- lpbuildd/target/tests/testfixtures.py 1970-01-01 00:00:00 +0000 |
582 | +++ lpbuildd/target/tests/testfixtures.py 2017-08-22 14:36:57 +0000 |
583 | @@ -0,0 +1,38 @@ |
584 | +# Copyright 2017 Canonical Ltd. This software is licensed under the |
585 | +# GNU Affero General Public License version 3 (see the file LICENSE). |
586 | + |
587 | +__metaclass__ = type |
588 | + |
589 | +import argparse |
590 | + |
591 | + |
592 | +class SudoUmount: |
593 | + |
594 | + name = "sudo" |
595 | + |
596 | + def __init__(self, delays=None): |
597 | + self.delays = delays or {} |
598 | + |
599 | + def __call__(self, proc_args): |
600 | + parser = argparse.ArgumentParser() |
601 | + parser.add_argument("command", choices=["umount"]) |
602 | + parser.add_argument("mount_path") |
603 | + args = parser.parse_args(proc_args["args"][1:]) |
604 | + if self.delays.get(args.mount_path, 0) > 0: |
605 | + self.delays[args.mount_path] -= 1 |
606 | + return {'returncode': 1} |
607 | + with open("/proc/mounts") as mounts_file: |
608 | + mounts = mounts_file.readlines() |
609 | + to_remove = None |
610 | + for i, mount in reversed(list(enumerate(mounts))): |
611 | + if mount.split()[1] == args.mount_path: |
612 | + to_remove = i |
613 | + break |
614 | + if to_remove is None: |
615 | + return {'returncode': 1} |
616 | + else: |
617 | + del mounts[to_remove] |
618 | + with open("/proc/mounts", "w") as mounts_file: |
619 | + for mount in mounts: |
620 | + mounts_file.write(mount) |
621 | + return {} |
622 | |
623 | === modified file 'lpbuildd/tests/test_binarypackage.py' |
624 | --- lpbuildd/tests/test_binarypackage.py 2017-08-22 14:23:16 +0000 |
625 | +++ lpbuildd/tests/test_binarypackage.py 2017-08-22 14:36:57 +0000 |
626 | @@ -158,7 +158,9 @@ |
627 | self.buildmanager.iterateReap(self.getState(), 0) |
628 | self.assertState( |
629 | BinaryPackageBuildState.UMOUNT, |
630 | - ['sharepath/slavebin/umount-chroot', 'umount-chroot', |
631 | + ['sharepath/slavebin/in-target', 'in-target', |
632 | + 'umount-chroot', |
633 | + '--backend=chroot', '--series=warty', '--arch=i386', |
634 | self.buildid], final=True) |
635 | |
636 | def test_iterate(self): |
637 | @@ -293,7 +295,9 @@ |
638 | self.buildmanager.iterate(128 + 9) # SIGKILL |
639 | self.assertState( |
640 | BinaryPackageBuildState.UMOUNT, |
641 | - ['sharepath/slavebin/umount-chroot', 'umount-chroot', |
642 | + ['sharepath/slavebin/in-target', 'in-target', |
643 | + 'umount-chroot', |
644 | + '--backend=chroot', '--series=warty', '--arch=i386', |
645 | self.buildid], final=True) |
646 | |
647 | def test_abort_between_subprocesses(self): |
648 | |
649 | === modified file 'lpbuildd/tests/test_debian.py' |
650 | --- lpbuildd/tests/test_debian.py 2017-08-22 14:23:16 +0000 |
651 | +++ lpbuildd/tests/test_debian.py 2017-08-22 14:36:57 +0000 |
652 | @@ -118,7 +118,10 @@ |
653 | self.buildmanager.iterate(0) |
654 | self.assertEqual(DebianBuildState.MOUNT, self.getState()) |
655 | self.assertEqual( |
656 | - (['sharepath/slavebin/mount-chroot', 'mount-chroot', self.buildid], |
657 | + (['sharepath/slavebin/in-target', 'in-target', |
658 | + 'mount-chroot', |
659 | + '--backend=chroot', '--series=xenial', '--arch=amd64', |
660 | + self.buildid], |
661 | None), |
662 | self.buildmanager.commands[-1]) |
663 | self.assertEqual( |
664 | @@ -169,7 +172,9 @@ |
665 | self.buildmanager.iterateReap(self.getState(), 0) |
666 | self.assertEqual(DebianBuildState.UMOUNT, self.getState()) |
667 | self.assertEqual( |
668 | - (['sharepath/slavebin/umount-chroot', 'umount-chroot', |
669 | + (['sharepath/slavebin/in-target', 'in-target', |
670 | + 'umount-chroot', |
671 | + '--backend=chroot', '--series=xenial', '--arch=amd64', |
672 | self.buildid], |
673 | None), |
674 | self.buildmanager.commands[-1]) |
675 | @@ -225,7 +230,10 @@ |
676 | self.buildmanager.iterate(0) |
677 | self.assertEqual(DebianBuildState.MOUNT, self.getState()) |
678 | self.assertEqual( |
679 | - (['sharepath/slavebin/mount-chroot', 'mount-chroot', self.buildid], |
680 | + (['sharepath/slavebin/in-target', 'in-target', |
681 | + 'mount-chroot', |
682 | + '--backend=chroot', '--series=xenial', '--arch=amd64', |
683 | + self.buildid], |
684 | None), |
685 | self.buildmanager.commands[-1]) |
686 | self.assertEqual( |
687 | @@ -288,7 +296,9 @@ |
688 | self.buildmanager.iterateReap(self.getState(), 0) |
689 | self.assertEqual(DebianBuildState.UMOUNT, self.getState()) |
690 | self.assertEqual( |
691 | - (['sharepath/slavebin/umount-chroot', 'umount-chroot', |
692 | + (['sharepath/slavebin/in-target', 'in-target', |
693 | + 'umount-chroot', |
694 | + '--backend=chroot', '--series=xenial', '--arch=amd64', |
695 | self.buildid], |
696 | None), |
697 | self.buildmanager.commands[-1]) |
698 | |
699 | === modified file 'lpbuildd/tests/test_livefs.py' |
700 | --- lpbuildd/tests/test_livefs.py 2015-05-11 05:55:24 +0000 |
701 | +++ lpbuildd/tests/test_livefs.py 2017-08-22 14:36:57 +0000 |
702 | @@ -114,7 +114,11 @@ |
703 | # Control returns to the DebianBuildManager in the UMOUNT state. |
704 | self.buildmanager.iterateReap(self.getState(), 0) |
705 | expected_command = [ |
706 | - "sharepath/slavebin/umount-chroot", "umount-chroot", self.buildid] |
707 | + "sharepath/slavebin/in-target", "in-target", |
708 | + "umount-chroot", |
709 | + "--backend=chroot", "--series=saucy", "--arch=i386", |
710 | + self.buildid, |
711 | + ] |
712 | self.assertEqual(LiveFilesystemBuildState.UMOUNT, self.getState()) |
713 | self.assertEqual(expected_command, self.buildmanager.commands[-1]) |
714 | self.assertEqual( |
715 | |
716 | === modified file 'lpbuildd/tests/test_snap.py' |
717 | --- lpbuildd/tests/test_snap.py 2017-07-28 13:57:47 +0000 |
718 | +++ lpbuildd/tests/test_snap.py 2017-08-22 14:36:57 +0000 |
719 | @@ -124,7 +124,11 @@ |
720 | # Control returns to the DebianBuildManager in the UMOUNT state. |
721 | self.buildmanager.iterateReap(self.getState(), 0) |
722 | expected_command = [ |
723 | - "sharepath/slavebin/umount-chroot", "umount-chroot", self.buildid] |
724 | + "sharepath/slavebin/in-target", "in-target", |
725 | + "umount-chroot", |
726 | + "--backend=chroot", "--series=xenial", "--arch=i386", |
727 | + self.buildid, |
728 | + ] |
729 | self.assertEqual(SnapBuildState.UMOUNT, self.getState()) |
730 | self.assertEqual(expected_command, self.buildmanager.commands[-1]) |
731 | self.assertEqual( |
732 | @@ -168,7 +172,11 @@ |
733 | # Control returns to the DebianBuildManager in the UMOUNT state. |
734 | self.buildmanager.iterateReap(self.getState(), 0) |
735 | expected_command = [ |
736 | - "sharepath/slavebin/umount-chroot", "umount-chroot", self.buildid] |
737 | + "sharepath/slavebin/in-target", "in-target", |
738 | + "umount-chroot", |
739 | + "--backend=chroot", "--series=xenial", "--arch=i386", |
740 | + self.buildid, |
741 | + ] |
742 | self.assertEqual(SnapBuildState.UMOUNT, self.getState()) |
743 | self.assertEqual(expected_command, self.buildmanager.commands[-1]) |
744 | self.assertEqual( |
745 | |
746 | === modified file 'lpbuildd/tests/test_sourcepackagerecipe.py' |
747 | --- lpbuildd/tests/test_sourcepackagerecipe.py 2017-07-28 13:57:47 +0000 |
748 | +++ lpbuildd/tests/test_sourcepackagerecipe.py 2017-08-22 14:36:57 +0000 |
749 | @@ -141,7 +141,10 @@ |
750 | # Control returns to the DebianBuildManager in the UMOUNT state. |
751 | self.buildmanager.iterateReap(self.getState(), 0) |
752 | expected_command = [ |
753 | - 'sharepath/slavebin/umount-chroot', 'umount-chroot', self.buildid |
754 | + 'sharepath/slavebin/in-target', 'in-target', |
755 | + 'umount-chroot', |
756 | + '--backend=chroot', '--series=maverick', '--arch=i386', |
757 | + self.buildid, |
758 | ] |
759 | self.assertEqual(SourcePackageRecipeBuildState.UMOUNT, self.getState()) |
760 | self.assertEqual(expected_command, self.buildmanager.commands[-1]) |
761 | @@ -179,7 +182,10 @@ |
762 | # Control returns to the DebianBuildManager in the UMOUNT state. |
763 | self.buildmanager.iterateReap(self.getState(), 0) |
764 | expected_command = [ |
765 | - 'sharepath/slavebin/umount-chroot', 'umount-chroot', self.buildid, |
766 | + 'sharepath/slavebin/in-target', 'in-target', |
767 | + 'umount-chroot', |
768 | + '--backend=chroot', '--series=maverick', '--arch=i386', |
769 | + self.buildid, |
770 | ] |
771 | self.assertEqual(SourcePackageRecipeBuildState.UMOUNT, self.getState()) |
772 | self.assertEqual(expected_command, self.buildmanager.commands[-1]) |
773 | @@ -214,7 +220,10 @@ |
774 | # Control returns to the DebianBuildManager in the UMOUNT state. |
775 | self.buildmanager.iterateReap(self.getState(), 0) |
776 | expected_command = [ |
777 | - 'sharepath/slavebin/umount-chroot', 'umount-chroot', self.buildid, |
778 | + 'sharepath/slavebin/in-target', 'in-target', |
779 | + 'umount-chroot', |
780 | + '--backend=chroot', '--series=maverick', '--arch=i386', |
781 | + self.buildid, |
782 | ] |
783 | self.assertEqual(SourcePackageRecipeBuildState.UMOUNT, self.getState()) |
784 | self.assertEqual(expected_command, self.buildmanager.commands[-1]) |
785 | |
786 | === modified file 'lpbuildd/tests/test_translationtemplatesbuildmanager.py' |
787 | --- lpbuildd/tests/test_translationtemplatesbuildmanager.py 2017-07-28 13:57:47 +0000 |
788 | +++ lpbuildd/tests/test_translationtemplatesbuildmanager.py 2017-08-22 14:36:57 +0000 |
789 | @@ -119,7 +119,10 @@ |
790 | # The control returns to the DebianBuildManager in the UMOUNT state. |
791 | self.buildmanager.iterateReap(self.getState(), 0) |
792 | expected_command = [ |
793 | - 'sharepath/slavebin/umount-chroot', 'umount-chroot', self.buildid, |
794 | + 'sharepath/slavebin/in-target', 'in-target', |
795 | + 'umount-chroot', |
796 | + '--backend=chroot', '--series=xenial', '--arch=i386', |
797 | + self.buildid, |
798 | ] |
799 | self.assertEqual( |
800 | TranslationTemplatesBuildState.UMOUNT, self.getState()) |
801 | @@ -144,7 +147,10 @@ |
802 | self.assertEqual( |
803 | TranslationTemplatesBuildState.UMOUNT, self.getState()) |
804 | expected_command = [ |
805 | - 'sharepath/slavebin/umount-chroot', 'umount-chroot', self.buildid, |
806 | + 'sharepath/slavebin/in-target', 'in-target', |
807 | + 'umount-chroot', |
808 | + '--backend=chroot', '--series=xenial', '--arch=i386', |
809 | + self.buildid, |
810 | ] |
811 | self.assertEqual(expected_command, self.buildmanager.commands[-1]) |
812 | self.assertEqual( |
813 | @@ -180,7 +186,10 @@ |
814 | self.assertEqual( |
815 | TranslationTemplatesBuildState.UMOUNT, self.getState()) |
816 | expected_command = [ |
817 | - 'sharepath/slavebin/umount-chroot', 'umount-chroot', self.buildid |
818 | + 'sharepath/slavebin/in-target', 'in-target', |
819 | + 'umount-chroot', |
820 | + '--backend=chroot', '--series=xenial', '--arch=i386', |
821 | + self.buildid, |
822 | ] |
823 | self.assertEqual(expected_command, self.buildmanager.commands[-1]) |
824 | self.assertEqual( |
start/stop/ create/ remove don't seem like they belong apart. Can you merge them into lpbuildd. target. lifecycle or similar?