Merge lp:~cjwatson/launchpad-buildd/rename-slave-classes into lp:launchpad-buildd
- rename-slave-classes
- Merge into trunk
Proposed by
Colin Watson
Status: | Merged |
---|---|
Merged at revision: | 366 |
Proposed branch: | lp:~cjwatson/launchpad-buildd/rename-slave-classes |
Merge into: | lp:launchpad-buildd |
Prerequisite: | lp:~cjwatson/launchpad-buildd/rename-register-builder |
Diff against target: |
914 lines (+144/-146) 10 files modified
debian/changelog (+1/-0) lpbuildd/binarypackage.py (+8/-8) lpbuildd/buildd-slave.tac (+19/-20) lpbuildd/builder.py (+66/-66) lpbuildd/debian.py (+22/-25) lpbuildd/livefs.py (+2/-2) lpbuildd/snap.py (+7/-7) lpbuildd/sourcepackagerecipe.py (+8/-8) lpbuildd/tests/harness.py (+3/-3) lpbuildd/translationtemplates.py (+8/-7) |
To merge this branch: | bzr merge lp:~cjwatson/launchpad-buildd/rename-slave-classes |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
William Grant | code | Approve | |
Review via email: mp+363046@code.launchpad.net |
Commit message
Rename BuildDSlave to Builder and XMLRPCBuildDSlave to XMLRPCBuilder.
Description of the change
To post a comment you must log in.
Revision history for this message
William Grant (wgrant) : | # |
review:
Approve
(code)
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'debian/changelog' | |||
2 | --- debian/changelog 2019-02-11 12:22:42 +0000 | |||
3 | +++ debian/changelog 2019-02-12 10:36:12 +0000 | |||
4 | @@ -6,6 +6,7 @@ | |||
5 | 6 | for core16 and core18. | 6 | for core16 and core18. |
6 | 7 | * Move /usr/share/launchpad-buildd/slavebin to | 7 | * Move /usr/share/launchpad-buildd/slavebin to |
7 | 8 | /usr/share/launchpad-buildd/bin. | 8 | /usr/share/launchpad-buildd/bin. |
8 | 9 | * Rename BuildDSlave to Builder and XMLRPCBuildDSlave to XMLRPCBuilder. | ||
9 | 9 | 10 | ||
10 | 10 | -- Colin Watson <cjwatson@ubuntu.com> Fri, 08 Feb 2019 15:09:35 +0000 | 11 | -- Colin Watson <cjwatson@ubuntu.com> Fri, 08 Feb 2019 15:09:35 +0000 |
11 | 11 | 12 | ||
12 | 12 | 13 | ||
13 | === modified file 'lpbuildd/binarypackage.py' | |||
14 | --- lpbuildd/binarypackage.py 2019-02-11 12:22:42 +0000 | |||
15 | +++ lpbuildd/binarypackage.py 2019-02-12 10:36:12 +0000 | |||
16 | @@ -92,8 +92,8 @@ | |||
17 | 92 | 92 | ||
18 | 93 | initial_build_state = BinaryPackageBuildState.SBUILD | 93 | initial_build_state = BinaryPackageBuildState.SBUILD |
19 | 94 | 94 | ||
22 | 95 | def __init__(self, slave, buildid, **kwargs): | 95 | def __init__(self, builder, buildid, **kwargs): |
23 | 96 | DebianBuildManager.__init__(self, slave, buildid, **kwargs) | 96 | DebianBuildManager.__init__(self, builder, buildid, **kwargs) |
24 | 97 | self._sbuildpath = os.path.join(self._bin, "sbuild-package") | 97 | self._sbuildpath = os.path.join(self._bin, "sbuild-package") |
25 | 98 | 98 | ||
26 | 99 | @property | 99 | @property |
27 | @@ -334,9 +334,9 @@ | |||
28 | 334 | unsat_deps.append(or_dep) | 334 | unsat_deps.append(or_dep) |
29 | 335 | return self.stripDependencies(unsat_deps) | 335 | return self.stripDependencies(unsat_deps) |
30 | 336 | except Exception: | 336 | except Exception: |
32 | 337 | self._slave.log("Failed to analyse dep-wait:\n") | 337 | self._builder.log("Failed to analyse dep-wait:\n") |
33 | 338 | for line in traceback.format_exc().splitlines(True): | 338 | for line in traceback.format_exc().splitlines(True): |
35 | 339 | self._slave.log(line) | 339 | self._builder.log(line) |
36 | 340 | return None | 340 | return None |
37 | 341 | 341 | ||
38 | 342 | def iterate_SBUILD(self, success): | 342 | def iterate_SBUILD(self, success): |
39 | @@ -400,17 +400,17 @@ | |||
40 | 400 | if missing_dep is not None: | 400 | if missing_dep is not None: |
41 | 401 | print("Returning build status: DEPFAIL") | 401 | print("Returning build status: DEPFAIL") |
42 | 402 | print("Dependencies: " + missing_dep) | 402 | print("Dependencies: " + missing_dep) |
44 | 403 | self._slave.depFail(missing_dep) | 403 | self._builder.depFail(missing_dep) |
45 | 404 | elif success == SBuildExitCodes.GIVENBACK: | 404 | elif success == SBuildExitCodes.GIVENBACK: |
46 | 405 | print("Returning build status: GIVENBACK") | 405 | print("Returning build status: GIVENBACK") |
48 | 406 | self._slave.giveBack() | 406 | self._builder.giveBack() |
49 | 407 | elif success == SBuildExitCodes.FAILED: | 407 | elif success == SBuildExitCodes.FAILED: |
50 | 408 | print("Returning build status: PACKAGEFAIL") | 408 | print("Returning build status: PACKAGEFAIL") |
52 | 409 | self._slave.buildFail() | 409 | self._builder.buildFail() |
53 | 410 | elif success >= SBuildExitCodes.BUILDERFAIL: | 410 | elif success >= SBuildExitCodes.BUILDERFAIL: |
54 | 411 | # anything else is assumed to be a buildd failure | 411 | # anything else is assumed to be a buildd failure |
55 | 412 | print("Returning build status: BUILDERFAIL") | 412 | print("Returning build status: BUILDERFAIL") |
57 | 413 | self._slave.builderFail() | 413 | self._builder.builderFail() |
58 | 414 | self.alreadyfailed = True | 414 | self.alreadyfailed = True |
59 | 415 | self.doReapProcesses(self._state) | 415 | self.doReapProcesses(self._state) |
60 | 416 | 416 | ||
61 | 417 | 417 | ||
62 | === modified file 'lpbuildd/buildd-slave.tac' | |||
63 | --- lpbuildd/buildd-slave.tac 2019-02-12 10:36:11 +0000 | |||
64 | +++ lpbuildd/buildd-slave.tac 2019-02-12 10:36:12 +0000 | |||
65 | @@ -1,8 +1,7 @@ | |||
66 | 1 | # Copyright 2009-2011 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2011 Canonical Ltd. This software is licensed under the |
67 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
68 | 3 | 3 | ||
71 | 4 | # Buildd Slave implementation | 4 | # XXX: dsilvers: 2005/01/21: Currently everything logged in the builder gets |
70 | 5 | # XXX: dsilvers: 2005/01/21: Currently everything logged in the slave gets | ||
72 | 6 | # passed through to the twistd log too. this could get dangerous/big | 5 | # passed through to the twistd log too. this could get dangerous/big |
73 | 7 | 6 | ||
74 | 8 | try: | 7 | try: |
75 | @@ -23,9 +22,9 @@ | |||
76 | 23 | ) | 22 | ) |
77 | 24 | 23 | ||
78 | 25 | from lpbuildd.binarypackage import BinaryPackageBuildManager | 24 | from lpbuildd.binarypackage import BinaryPackageBuildManager |
79 | 25 | from lpbuildd.builder import XMLRPCBuilder | ||
80 | 26 | from lpbuildd.livefs import LiveFilesystemBuildManager | 26 | from lpbuildd.livefs import LiveFilesystemBuildManager |
81 | 27 | from lpbuildd.log import RotatableFileLogObserver | 27 | from lpbuildd.log import RotatableFileLogObserver |
82 | 28 | from lpbuildd.slave import XMLRPCBuildDSlave | ||
83 | 29 | from lpbuildd.snap import SnapBuildManager | 28 | from lpbuildd.snap import SnapBuildManager |
84 | 30 | from lpbuildd.sourcepackagerecipe import SourcePackageRecipeBuildManager | 29 | from lpbuildd.sourcepackagerecipe import SourcePackageRecipeBuildManager |
85 | 31 | from lpbuildd.translationtemplates import TranslationTemplatesBuildManager | 30 | from lpbuildd.translationtemplates import TranslationTemplatesBuildManager |
86 | @@ -38,31 +37,31 @@ | |||
87 | 38 | 37 | ||
88 | 39 | conf = SafeConfigParser() | 38 | conf = SafeConfigParser() |
89 | 40 | conf.read(conffile) | 39 | conf.read(conffile) |
91 | 41 | slave = XMLRPCBuildDSlave(conf) | 40 | builder = XMLRPCBuilder(conf) |
92 | 42 | 41 | ||
96 | 43 | slave.registerManager(BinaryPackageBuildManager, "binarypackage") | 42 | builder.registerManager(BinaryPackageBuildManager, "binarypackage") |
97 | 44 | slave.registerManager(SourcePackageRecipeBuildManager, "sourcepackagerecipe") | 43 | builder.registerManager(SourcePackageRecipeBuildManager, "sourcepackagerecipe") |
98 | 45 | slave.registerManager( | 44 | builder.registerManager( |
99 | 46 | TranslationTemplatesBuildManager, 'translation-templates') | 45 | TranslationTemplatesBuildManager, 'translation-templates') |
102 | 47 | slave.registerManager(LiveFilesystemBuildManager, "livefs") | 46 | builder.registerManager(LiveFilesystemBuildManager, "livefs") |
103 | 48 | slave.registerManager(SnapBuildManager, "snap") | 47 | builder.registerManager(SnapBuildManager, "snap") |
104 | 49 | 48 | ||
106 | 50 | application = service.Application('BuildDSlave') | 49 | application = service.Application('Builder') |
107 | 51 | application.addComponent( | 50 | application.addComponent( |
108 | 52 | RotatableFileLogObserver(options.get('logfile')), ignoreClass=1) | 51 | RotatableFileLogObserver(options.get('logfile')), ignoreClass=1) |
111 | 53 | builddslaveService = service.IServiceCollection(application) | 52 | builderService = service.IServiceCollection(application) |
112 | 54 | slave.slave.service = builddslaveService | 53 | builder.builder.service = builderService |
113 | 55 | 54 | ||
114 | 56 | root = resource.Resource() | 55 | root = resource.Resource() |
116 | 57 | root.putChild('rpc', slave) | 56 | root.putChild('rpc', builder) |
117 | 58 | root.putChild('filecache', static.File(conf.get('slave', 'filecache'))) | 57 | root.putChild('filecache', static.File(conf.get('slave', 'filecache'))) |
125 | 59 | slavesite = server.Site(root) | 58 | buildersite = server.Site(root) |
126 | 60 | 59 | ||
127 | 61 | strports.service("tcp:%s" % slave.slave._config.get("slave", "bindport"), | 60 | strports.service("tcp:%s" % builder.builder._config.get("slave", "bindport"), |
128 | 62 | slavesite).setServiceParent(builddslaveService) | 61 | buildersite).setServiceParent(builderService) |
129 | 63 | 62 | ||
130 | 64 | # You can interact with a running slave like this: | 63 | # You can interact with a running builder like this: |
131 | 65 | # (assuming the slave is on localhost:8221) | 64 | # (assuming the builder is on localhost:8221) |
132 | 66 | # | 65 | # |
133 | 67 | # python3 | 66 | # python3 |
134 | 68 | # from xmlrpc.client import ServerProxy | 67 | # from xmlrpc.client import ServerProxy |
135 | 69 | 68 | ||
136 | === renamed file 'lpbuildd/slave.py' => 'lpbuildd/builder.py' | |||
137 | --- lpbuildd/slave.py 2019-02-12 10:36:11 +0000 | |||
138 | +++ lpbuildd/builder.py 2019-02-12 10:36:12 +0000 | |||
139 | @@ -75,10 +75,10 @@ | |||
140 | 75 | # | 75 | # |
141 | 76 | # when we start using Twisted 2.0. | 76 | # when we start using Twisted 2.0. |
142 | 77 | class RunCapture(protocol.ProcessProtocol): | 77 | class RunCapture(protocol.ProcessProtocol): |
144 | 78 | """Run a command and capture its output to a slave's log""" | 78 | """Run a command and capture its output to a builder's log.""" |
145 | 79 | 79 | ||
148 | 80 | def __init__(self, slave, callback, stdin=None): | 80 | def __init__(self, builder, callback, stdin=None): |
149 | 81 | self.slave = slave | 81 | self.builder = builder |
150 | 82 | self.notify = callback | 82 | self.notify = callback |
151 | 83 | self.stdin = stdin | 83 | self.stdin = stdin |
152 | 84 | self.builderFailCall = None | 84 | self.builderFailCall = None |
153 | @@ -92,13 +92,13 @@ | |||
154 | 92 | 92 | ||
155 | 93 | def outReceived(self, data): | 93 | def outReceived(self, data): |
156 | 94 | """Pass on stdout data to the log.""" | 94 | """Pass on stdout data to the log.""" |
158 | 95 | self.slave.log(data) | 95 | self.builder.log(data) |
159 | 96 | 96 | ||
160 | 97 | def errReceived(self, data): | 97 | def errReceived(self, data): |
161 | 98 | """Pass on stderr data to the log. | 98 | """Pass on stderr data to the log. |
162 | 99 | 99 | ||
163 | 100 | With a bit of luck we won't interleave horribly.""" | 100 | With a bit of luck we won't interleave horribly.""" |
165 | 101 | self.slave.log(data) | 101 | self.builder.log(data) |
166 | 102 | 102 | ||
167 | 103 | def processEnded(self, statusobject): | 103 | def processEnded(self, statusobject): |
168 | 104 | """This method is called when a child process got terminated. | 104 | """This method is called when a child process got terminated. |
169 | @@ -115,29 +115,29 @@ | |||
170 | 115 | if self.builderFailCall and self.builderFailCall.active(): | 115 | if self.builderFailCall and self.builderFailCall.active(): |
171 | 116 | self.builderFailCall.cancel() | 116 | self.builderFailCall.cancel() |
172 | 117 | 117 | ||
174 | 118 | # notify the slave, it'll perform the required actions | 118 | # notify the builder, it'll perform the required actions |
175 | 119 | if self.notify is not None: | 119 | if self.notify is not None: |
176 | 120 | self.notify(statusobject.value.exitCode) | 120 | self.notify(statusobject.value.exitCode) |
177 | 121 | 121 | ||
178 | 122 | 122 | ||
179 | 123 | class BuildManager(object): | 123 | class BuildManager(object): |
181 | 124 | """Build Daemon slave build manager abstract parent""" | 124 | """Build manager abstract parent.""" |
182 | 125 | 125 | ||
183 | 126 | backend_name = "chroot" | 126 | backend_name = "chroot" |
184 | 127 | 127 | ||
186 | 128 | def __init__(self, slave, buildid, reactor=None): | 128 | def __init__(self, builder, buildid, reactor=None): |
187 | 129 | """Create a BuildManager. | 129 | """Create a BuildManager. |
188 | 130 | 130 | ||
190 | 131 | :param slave: A `BuildDSlave`. | 131 | :param builder: A `Builder`. |
191 | 132 | :param buildid: Identifying string for this build. | 132 | :param buildid: Identifying string for this build. |
192 | 133 | """ | 133 | """ |
193 | 134 | object.__init__(self) | 134 | object.__init__(self) |
194 | 135 | self._buildid = buildid | 135 | self._buildid = buildid |
196 | 136 | self._slave = slave | 136 | self._builder = builder |
197 | 137 | if reactor is None: | 137 | if reactor is None: |
198 | 138 | reactor = default_reactor | 138 | reactor = default_reactor |
199 | 139 | self._reactor = reactor | 139 | self._reactor = reactor |
201 | 140 | self._sharepath = slave._config.get("slave", "sharepath") | 140 | self._sharepath = builder._config.get("slave", "sharepath") |
202 | 141 | self._bin = os.path.join(self._sharepath, "bin") | 141 | self._bin = os.path.join(self._sharepath, "bin") |
203 | 142 | self._preppath = os.path.join(self._bin, "builder-prep") | 142 | self._preppath = os.path.join(self._bin, "builder-prep") |
204 | 143 | self._intargetpath = os.path.join(self._bin, "in-target") | 143 | self._intargetpath = os.path.join(self._bin, "in-target") |
205 | @@ -155,8 +155,8 @@ | |||
206 | 155 | """Run a subprocess capturing the results in the log.""" | 155 | """Run a subprocess capturing the results in the log.""" |
207 | 156 | if iterate is None: | 156 | if iterate is None: |
208 | 157 | iterate = self.iterate | 157 | iterate = self.iterate |
211 | 158 | self._subprocess = RunCapture(self._slave, iterate, stdin=stdin) | 158 | self._subprocess = RunCapture(self._builder, iterate, stdin=stdin) |
212 | 159 | self._slave.log("RUN: %s %s\n" % ( | 159 | self._builder.log("RUN: %s %s\n" % ( |
213 | 160 | command, " ".join(shell_escape(arg) for arg in args[1:]))) | 160 | command, " ".join(shell_escape(arg) for arg in args[1:]))) |
214 | 161 | childfds = { | 161 | childfds = { |
215 | 162 | 0: devnull.fileno() if stdin is None else "w", | 162 | 0: devnull.fileno() if stdin is None else "w", |
216 | @@ -191,7 +191,7 @@ | |||
217 | 191 | if state is not None and state in self._reaped_states: | 191 | if state is not None and state in self._reaped_states: |
218 | 192 | # We've already reaped this state. To avoid a loop, proceed | 192 | # We've already reaped this state. To avoid a loop, proceed |
219 | 193 | # immediately to the next iterator. | 193 | # immediately to the next iterator. |
221 | 194 | self._slave.log("Already reaped from state %s...\n" % state) | 194 | self._builder.log("Already reaped from state %s...\n" % state) |
222 | 195 | if notify: | 195 | if notify: |
223 | 196 | self.iterateReap(state, 0) | 196 | self.iterateReap(state, 0) |
224 | 197 | else: | 197 | else: |
225 | @@ -211,7 +211,7 @@ | |||
226 | 211 | # Sanitize the URLs in the buildlog file if this is a build | 211 | # Sanitize the URLs in the buildlog file if this is a build |
227 | 212 | # in a private archive. | 212 | # in a private archive. |
228 | 213 | if self.needs_sanitized_logs: | 213 | if self.needs_sanitized_logs: |
230 | 214 | self._slave.sanitizeBuildlog(self._slave.cachePath("buildlog")) | 214 | self._builder.sanitizeBuildlog(self._builder.cachePath("buildlog")) |
231 | 215 | 215 | ||
232 | 216 | if self.fast_cleanup: | 216 | if self.fast_cleanup: |
233 | 217 | self.iterate(0, quiet=True) | 217 | self.iterate(0, quiet=True) |
234 | @@ -235,18 +235,18 @@ | |||
235 | 235 | evaluates to True the build at hand is for a private archive. | 235 | evaluates to True the build at hand is for a private archive. |
236 | 236 | """ | 236 | """ |
237 | 237 | if 'build_url' in extra_args: | 237 | if 'build_url' in extra_args: |
239 | 238 | self._slave.log("%s\n" % extra_args['build_url']) | 238 | self._builder.log("%s\n" % extra_args['build_url']) |
240 | 239 | 239 | ||
241 | 240 | os.mkdir("%s/build-%s" % (self.home, self._buildid)) | 240 | os.mkdir("%s/build-%s" % (self.home, self._buildid)) |
242 | 241 | for f in files: | 241 | for f in files: |
244 | 242 | os.symlink( self._slave.cachePath(files[f]), | 242 | os.symlink( self._builder.cachePath(files[f]), |
245 | 243 | "%s/build-%s/%s" % (self.home, | 243 | "%s/build-%s/%s" % (self.home, |
246 | 244 | self._buildid, f)) | 244 | self._buildid, f)) |
248 | 245 | self._chroottarfile = self._slave.cachePath(chroot) | 245 | self._chroottarfile = self._builder.cachePath(chroot) |
249 | 246 | 246 | ||
250 | 247 | self.image_type = extra_args.get('image_type', 'chroot') | 247 | self.image_type = extra_args.get('image_type', 'chroot') |
251 | 248 | self.series = extra_args['series'] | 248 | self.series = extra_args['series'] |
253 | 249 | self.arch_tag = extra_args.get('arch_tag', self._slave.getArch()) | 249 | self.arch_tag = extra_args.get('arch_tag', self._builder.getArch()) |
254 | 250 | self.fast_cleanup = extra_args.get('fast_cleanup', False) | 250 | self.fast_cleanup = extra_args.get('fast_cleanup', False) |
255 | 251 | 251 | ||
256 | 252 | # Check whether this is a build in a private archive and | 252 | # Check whether this is a build in a private archive and |
257 | @@ -271,7 +271,7 @@ | |||
258 | 271 | return {} | 271 | return {} |
259 | 272 | 272 | ||
260 | 273 | def iterate(self, success, quiet=False): | 273 | def iterate(self, success, quiet=False): |
262 | 274 | """Perform an iteration of the slave. | 274 | """Perform an iteration of the builder. |
263 | 275 | 275 | ||
264 | 276 | The BuildManager tends to work by invoking several | 276 | The BuildManager tends to work by invoking several |
265 | 277 | subprocesses in order. the iterate method is called by the | 277 | subprocesses in order. the iterate method is called by the |
266 | @@ -282,7 +282,7 @@ | |||
267 | 282 | "used") | 282 | "used") |
268 | 283 | 283 | ||
269 | 284 | def iterateReap(self, state, success): | 284 | def iterateReap(self, state, success): |
271 | 285 | """Perform an iteration of the slave following subprocess reaping. | 285 | """Perform an iteration of the builder following subprocess reaping. |
272 | 286 | 286 | ||
273 | 287 | Subprocess reaping is special, typically occurring at several | 287 | Subprocess reaping is special, typically occurring at several |
274 | 288 | positions in a build manager's state machine. We therefore keep | 288 | positions in a build manager's state machine. We therefore keep |
275 | @@ -321,9 +321,9 @@ | |||
276 | 321 | 321 | ||
277 | 322 | def builderFail(self, reason, primary_subprocess): | 322 | def builderFail(self, reason, primary_subprocess): |
278 | 323 | """Mark the builder as failed.""" | 323 | """Mark the builder as failed.""" |
280 | 324 | self._slave.log("ABORTING: %s\n" % reason) | 324 | self._builder.log("ABORTING: %s\n" % reason) |
281 | 325 | self._subprocess.builderFailCall = None | 325 | self._subprocess.builderFailCall = None |
283 | 326 | self._slave.builderFail() | 326 | self._builder.builderFail() |
284 | 327 | self.alreadyfailed = True | 327 | self.alreadyfailed = True |
285 | 328 | # If we failed to kill all processes in the chroot, then the primary | 328 | # If we failed to kill all processes in the chroot, then the primary |
286 | 329 | # subprocess (i.e. the one running immediately before | 329 | # subprocess (i.e. the one running immediately before |
287 | @@ -332,7 +332,7 @@ | |||
288 | 332 | try: | 332 | try: |
289 | 333 | primary_subprocess.transport.signalProcess('KILL') | 333 | primary_subprocess.transport.signalProcess('KILL') |
290 | 334 | except process.ProcessExitedAlready: | 334 | except process.ProcessExitedAlready: |
292 | 335 | self._slave.log("ABORTING: Process Exited Already\n") | 335 | self._builder.log("ABORTING: Process Exited Already\n") |
293 | 336 | primary_subprocess.transport.loseConnection() | 336 | primary_subprocess.transport.loseConnection() |
294 | 337 | # Leave the reaper running, but disconnect it from our state | 337 | # Leave the reaper running, but disconnect it from our state |
295 | 338 | # machine. Perhaps an admin can make something of it, and in any | 338 | # machine. Perhaps an admin can make something of it, and in any |
296 | @@ -346,7 +346,7 @@ | |||
297 | 346 | try: | 346 | try: |
298 | 347 | fetched_path = os.path.join(fetched_dir, os.path.basename(path)) | 347 | fetched_path = os.path.join(fetched_dir, os.path.basename(path)) |
299 | 348 | self.backend.copy_out(path, fetched_path) | 348 | self.backend.copy_out(path, fetched_path) |
301 | 349 | self._slave.addWaitingFile(fetched_path) | 349 | self._builder.addWaitingFile(fetched_path) |
302 | 350 | finally: | 350 | finally: |
303 | 351 | shutil.rmtree(fetched_dir) | 351 | shutil.rmtree(fetched_dir) |
304 | 352 | 352 | ||
305 | @@ -375,10 +375,8 @@ | |||
306 | 375 | ABORTED = "BuildStatus.ABORTED" | 375 | ABORTED = "BuildStatus.ABORTED" |
307 | 376 | 376 | ||
308 | 377 | 377 | ||
313 | 378 | class BuildDSlave(object): | 378 | class Builder(object): |
314 | 379 | """Build Daemon slave. Implementation of most needed functions | 379 | """The core of a builder.""" |
311 | 380 | for a Build-Slave device. | ||
312 | 381 | """ | ||
315 | 382 | 380 | ||
316 | 383 | def __init__(self, config): | 381 | def __init__(self, config): |
317 | 384 | object.__init__(self) | 382 | object.__init__(self) |
318 | @@ -395,7 +393,7 @@ | |||
319 | 395 | raise ValueError("FileCache path is not a dir") | 393 | raise ValueError("FileCache path is not a dir") |
320 | 396 | 394 | ||
321 | 397 | def getArch(self): | 395 | def getArch(self): |
323 | 398 | """Return the Architecture tag for the slave.""" | 396 | """Return the Architecture tag for the builder.""" |
324 | 399 | return self._config.get("slave","architecturetag") | 397 | return self._config.get("slave","architecturetag") |
325 | 400 | 398 | ||
326 | 401 | def cachePath(self, file): | 399 | def cachePath(self, file): |
327 | @@ -423,7 +421,7 @@ | |||
328 | 423 | """Ensure we have the file with the checksum specified. | 421 | """Ensure we have the file with the checksum specified. |
329 | 424 | 422 | ||
330 | 425 | Optionally you can provide the librarian URL and | 423 | Optionally you can provide the librarian URL and |
332 | 426 | the build slave will fetch the file if it doesn't have it. | 424 | the builder will fetch the file if it doesn't have it. |
333 | 427 | Return a tuple containing: (<present>, <info>) | 425 | Return a tuple containing: (<present>, <info>) |
334 | 428 | """ | 426 | """ |
335 | 429 | extra_info = 'No URL' | 427 | extra_info = 'No URL' |
336 | @@ -497,19 +495,19 @@ | |||
337 | 497 | if self.builderstatus == BuilderStatus.ABORTING: | 495 | if self.builderstatus == BuilderStatus.ABORTING: |
338 | 498 | # This might happen if the master side restarts in the middle of | 496 | # This might happen if the master side restarts in the middle of |
339 | 499 | # an abort cycle. | 497 | # an abort cycle. |
341 | 500 | self.log("Slave already ABORTING when asked to abort") | 498 | self.log("Builder already ABORTING when asked to abort") |
342 | 501 | return | 499 | return |
343 | 502 | if self.builderstatus != BuilderStatus.BUILDING: | 500 | if self.builderstatus != BuilderStatus.BUILDING: |
344 | 503 | # XXX: Should raise a known Fault so that the client can make | 501 | # XXX: Should raise a known Fault so that the client can make |
345 | 504 | # useful decisions about the error! | 502 | # useful decisions about the error! |
347 | 505 | raise ValueError("Slave is not BUILDING when asked to abort") | 503 | raise ValueError("Builder is not BUILDING when asked to abort") |
348 | 506 | self.manager.abort() | 504 | self.manager.abort() |
349 | 507 | self.builderstatus = BuilderStatus.ABORTING | 505 | self.builderstatus = BuilderStatus.ABORTING |
350 | 508 | 506 | ||
351 | 509 | def clean(self): | 507 | def clean(self): |
352 | 510 | """Clean up pending files and reset the internal build state.""" | 508 | """Clean up pending files and reset the internal build state.""" |
353 | 511 | if self.builderstatus != BuilderStatus.WAITING: | 509 | if self.builderstatus != BuilderStatus.WAITING: |
355 | 512 | raise ValueError('Slave is not WAITING when asked to clean') | 510 | raise ValueError('Builder is not WAITING when asked to clean') |
356 | 513 | for f in set(self.waitingfiles.values()): | 511 | for f in set(self.waitingfiles.values()): |
357 | 514 | os.remove(self.cachePath(f)) | 512 | os.remove(self.cachePath(f)) |
358 | 515 | self.builderstatus = BuilderStatus.IDLE | 513 | self.builderstatus = BuilderStatus.IDLE |
359 | @@ -587,7 +585,8 @@ | |||
360 | 587 | def startBuild(self, manager): | 585 | def startBuild(self, manager): |
361 | 588 | """Start a build with the provided BuildManager instance.""" | 586 | """Start a build with the provided BuildManager instance.""" |
362 | 589 | if self.builderstatus != BuilderStatus.IDLE: | 587 | if self.builderstatus != BuilderStatus.IDLE: |
364 | 590 | raise ValueError("Slave is not IDLE when asked to start building") | 588 | raise ValueError( |
365 | 589 | "Builder is not IDLE when asked to start building") | ||
366 | 591 | self.manager = manager | 590 | self.manager = manager |
367 | 592 | self.builderstatus = BuilderStatus.BUILDING | 591 | self.builderstatus = BuilderStatus.BUILDING |
368 | 593 | self.emptyLog() | 592 | self.emptyLog() |
369 | @@ -602,8 +601,8 @@ | |||
370 | 602 | """Cease building because the builder has a problem.""" | 601 | """Cease building because the builder has a problem.""" |
371 | 603 | if self.builderstatus not in (BuilderStatus.BUILDING, | 602 | if self.builderstatus not in (BuilderStatus.BUILDING, |
372 | 604 | BuilderStatus.ABORTING): | 603 | BuilderStatus.ABORTING): |
375 | 605 | raise ValueError("Slave is not BUILDING|ABORTING when set to " | 604 | raise ValueError( |
376 | 606 | "BUILDERFAIL") | 605 | "Builder is not BUILDING|ABORTING when set to BUILDERFAIL") |
377 | 607 | self.buildstatus = BuildStatus.BUILDERFAIL | 606 | self.buildstatus = BuildStatus.BUILDERFAIL |
378 | 608 | 607 | ||
379 | 609 | def chrootFail(self): | 608 | def chrootFail(self): |
380 | @@ -613,38 +612,38 @@ | |||
381 | 613 | package. | 612 | package. |
382 | 614 | """ | 613 | """ |
383 | 615 | if self.builderstatus != BuilderStatus.BUILDING: | 614 | if self.builderstatus != BuilderStatus.BUILDING: |
385 | 616 | raise ValueError("Slave is not BUILDING when set to CHROOTFAIL") | 615 | raise ValueError("Builder is not BUILDING when set to CHROOTFAIL") |
386 | 617 | self.buildstatus = BuildStatus.CHROOTFAIL | 616 | self.buildstatus = BuildStatus.CHROOTFAIL |
387 | 618 | 617 | ||
388 | 619 | def buildFail(self): | 618 | def buildFail(self): |
389 | 620 | """Cease building because the package failed to build.""" | 619 | """Cease building because the package failed to build.""" |
390 | 621 | if self.builderstatus != BuilderStatus.BUILDING: | 620 | if self.builderstatus != BuilderStatus.BUILDING: |
392 | 622 | raise ValueError("Slave is not BUILDING when set to PACKAGEFAIL") | 621 | raise ValueError("Builder is not BUILDING when set to PACKAGEFAIL") |
393 | 623 | self.buildstatus = BuildStatus.PACKAGEFAIL | 622 | self.buildstatus = BuildStatus.PACKAGEFAIL |
394 | 624 | 623 | ||
395 | 625 | def buildOK(self): | 624 | def buildOK(self): |
396 | 626 | """Having passed all possible failure states, mark a build as OK.""" | 625 | """Having passed all possible failure states, mark a build as OK.""" |
397 | 627 | if self.builderstatus != BuilderStatus.BUILDING: | 626 | if self.builderstatus != BuilderStatus.BUILDING: |
399 | 628 | raise ValueError("Slave is not BUILDING when set to OK") | 627 | raise ValueError("Builder is not BUILDING when set to OK") |
400 | 629 | self.buildstatus = BuildStatus.OK | 628 | self.buildstatus = BuildStatus.OK |
401 | 630 | 629 | ||
402 | 631 | def depFail(self, dependencies): | 630 | def depFail(self, dependencies): |
403 | 632 | """Cease building due to a dependency issue.""" | 631 | """Cease building due to a dependency issue.""" |
404 | 633 | if self.builderstatus != BuilderStatus.BUILDING: | 632 | if self.builderstatus != BuilderStatus.BUILDING: |
406 | 634 | raise ValueError("Slave is not BUILDING when set to DEPFAIL") | 633 | raise ValueError("Builder is not BUILDING when set to DEPFAIL") |
407 | 635 | self.buildstatus = BuildStatus.DEPFAIL | 634 | self.buildstatus = BuildStatus.DEPFAIL |
408 | 636 | self.builddependencies = dependencies | 635 | self.builddependencies = dependencies |
409 | 637 | 636 | ||
410 | 638 | def giveBack(self): | 637 | def giveBack(self): |
411 | 639 | """Give-back package due to a transient buildd/archive issue.""" | 638 | """Give-back package due to a transient buildd/archive issue.""" |
412 | 640 | if self.builderstatus != BuilderStatus.BUILDING: | 639 | if self.builderstatus != BuilderStatus.BUILDING: |
414 | 641 | raise ValueError("Slave is not BUILDING when set to GIVENBACK") | 640 | raise ValueError("Builder is not BUILDING when set to GIVENBACK") |
415 | 642 | self.buildstatus = BuildStatus.GIVENBACK | 641 | self.buildstatus = BuildStatus.GIVENBACK |
416 | 643 | 642 | ||
417 | 644 | def buildAborted(self): | 643 | def buildAborted(self): |
418 | 645 | """Mark a build as aborted.""" | 644 | """Mark a build as aborted.""" |
419 | 646 | if self.builderstatus != BuilderStatus.ABORTING: | 645 | if self.builderstatus != BuilderStatus.ABORTING: |
421 | 647 | raise ValueError("Slave is not ABORTING when set to ABORTED") | 646 | raise ValueError("Builder is not ABORTING when set to ABORTED") |
422 | 648 | if self.buildstatus != BuildStatus.BUILDERFAIL: | 647 | if self.buildstatus != BuildStatus.BUILDERFAIL: |
423 | 649 | self.buildstatus = BuildStatus.ABORTED | 648 | self.buildstatus = BuildStatus.ABORTED |
424 | 650 | 649 | ||
425 | @@ -658,8 +657,8 @@ | |||
426 | 658 | self.buildAborted() | 657 | self.buildAborted() |
427 | 659 | self.builderstatus = BuilderStatus.WAITING | 658 | self.builderstatus = BuilderStatus.WAITING |
428 | 660 | else: | 659 | else: |
431 | 661 | raise ValueError("Slave is not BUILDING|ABORTING when told build " | 660 | raise ValueError( |
432 | 662 | "is complete") | 661 | "Builder is not BUILDING|ABORTING when told build is complete") |
433 | 663 | 662 | ||
434 | 664 | def sanitizeBuildlog(self, log_path): | 663 | def sanitizeBuildlog(self, log_path): |
435 | 665 | """Removes passwords from buildlog URLs. | 664 | """Removes passwords from buildlog URLs. |
436 | @@ -699,8 +698,8 @@ | |||
437 | 699 | sanitized_file.close() | 698 | sanitized_file.close() |
438 | 700 | 699 | ||
439 | 701 | 700 | ||
442 | 702 | class XMLRPCBuildDSlave(xmlrpc.XMLRPC): | 701 | class XMLRPCBuilder(xmlrpc.XMLRPC): |
443 | 703 | """XMLRPC build daemon slave management interface""" | 702 | """XMLRPC builder management interface.""" |
444 | 704 | 703 | ||
445 | 705 | def __init__(self, config): | 704 | def __init__(self, config): |
446 | 706 | xmlrpc.XMLRPC.__init__(self, allowNone=True) | 705 | xmlrpc.XMLRPC.__init__(self, allowNone=True) |
447 | @@ -709,7 +708,7 @@ | |||
448 | 709 | # implying the presence of /filecache/ /filecache/buildlog and | 708 | # implying the presence of /filecache/ /filecache/buildlog and |
449 | 710 | # the reduced and optimised XMLRPC interface. | 709 | # the reduced and optimised XMLRPC interface. |
450 | 711 | self.protocolversion = '1.0' | 710 | self.protocolversion = '1.0' |
452 | 712 | self.slave = BuildDSlave(config) | 711 | self.builder = Builder(config) |
453 | 713 | self._managers = {} | 712 | self._managers = {} |
454 | 714 | cache = apt.Cache() | 713 | cache = apt.Cache() |
455 | 715 | try: | 714 | try: |
456 | @@ -728,7 +727,7 @@ | |||
457 | 728 | 727 | ||
458 | 729 | def xmlrpc_info(self): | 728 | def xmlrpc_info(self): |
459 | 730 | """Return the protocol version and the manager methods supported.""" | 729 | """Return the protocol version and the manager methods supported.""" |
461 | 731 | return (self.protocolversion, self.slave.getArch(), | 730 | return (self.protocolversion, self.builder.getArch(), |
462 | 732 | list(self._managers)) | 731 | list(self._managers)) |
463 | 733 | 732 | ||
464 | 734 | def xmlrpc_status(self): | 733 | def xmlrpc_status(self): |
465 | @@ -737,7 +736,7 @@ | |||
466 | 737 | Depending on the builder status we return differing amounts of data, | 736 | Depending on the builder status we return differing amounts of data, |
467 | 738 | but this always includes the builder status itself. | 737 | but this always includes the builder status itself. |
468 | 739 | """ | 738 | """ |
470 | 740 | status = self.slave.builderstatus | 739 | status = self.builder.builderstatus |
471 | 741 | statusname = status.split('.')[-1] | 740 | statusname = status.split('.')[-1] |
472 | 742 | func = getattr(self, "status_" + statusname, None) | 741 | func = getattr(self, "status_" + statusname, None) |
473 | 743 | if func is None: | 742 | if func is None: |
474 | @@ -746,8 +745,8 @@ | |||
475 | 746 | if self._version is not None: | 745 | if self._version is not None: |
476 | 747 | ret["builder_version"] = self._version | 746 | ret["builder_version"] = self._version |
477 | 748 | ret.update(func()) | 747 | ret.update(func()) |
480 | 749 | if self.slave.manager is not None: | 748 | if self.builder.manager is not None: |
481 | 750 | ret.update(self.slave.manager.status()) | 749 | ret.update(self.builder.manager.status()) |
482 | 751 | return ret | 750 | return ret |
483 | 752 | 751 | ||
484 | 753 | def status_IDLE(self): | 752 | def status_IDLE(self): |
485 | @@ -759,7 +758,7 @@ | |||
486 | 759 | 758 | ||
487 | 760 | Returns the build id and up to one kilobyte of log tail. | 759 | Returns the build id and up to one kilobyte of log tail. |
488 | 761 | """ | 760 | """ |
490 | 762 | tail = self.slave.getLogTail() | 761 | tail = self.builder.getLogTail() |
491 | 763 | return {"build_id": self.buildid, "logtail": Binary(tail)} | 762 | return {"build_id": self.buildid, "logtail": Binary(tail)} |
492 | 764 | 763 | ||
493 | 765 | def status_WAITING(self): | 764 | def status_WAITING(self): |
494 | @@ -770,13 +769,13 @@ | |||
495 | 770 | and the build id but no file set. | 769 | and the build id but no file set. |
496 | 771 | """ | 770 | """ |
497 | 772 | ret = { | 771 | ret = { |
499 | 773 | "build_status": self.slave.buildstatus, | 772 | "build_status": self.builder.buildstatus, |
500 | 774 | "build_id": self.buildid, | 773 | "build_id": self.buildid, |
501 | 775 | } | 774 | } |
506 | 776 | if self.slave.buildstatus in (BuildStatus.OK, BuildStatus.PACKAGEFAIL, | 775 | if self.builder.buildstatus in ( |
507 | 777 | BuildStatus.DEPFAIL): | 776 | BuildStatus.OK, BuildStatus.PACKAGEFAIL, BuildStatus.DEPFAIL): |
508 | 778 | ret["filemap"] = self.slave.waitingfiles | 777 | ret["filemap"] = self.builder.waitingfiles |
509 | 779 | ret["dependencies"] = self.slave.builddependencies | 778 | ret["dependencies"] = self.builder.builddependencies |
510 | 780 | return ret | 779 | return ret |
511 | 781 | 780 | ||
512 | 782 | def status_ABORTING(self): | 781 | def status_ABORTING(self): |
513 | @@ -790,16 +789,16 @@ | |||
514 | 790 | 789 | ||
515 | 791 | def xmlrpc_ensurepresent(self, sha1sum, url, username, password): | 790 | def xmlrpc_ensurepresent(self, sha1sum, url, username, password): |
516 | 792 | """Attempt to ensure the given file is present.""" | 791 | """Attempt to ensure the given file is present.""" |
518 | 793 | return self.slave.ensurePresent(sha1sum, url, username, password) | 792 | return self.builder.ensurePresent(sha1sum, url, username, password) |
519 | 794 | 793 | ||
520 | 795 | def xmlrpc_abort(self): | 794 | def xmlrpc_abort(self): |
521 | 796 | """Abort the current build.""" | 795 | """Abort the current build.""" |
523 | 797 | self.slave.abort() | 796 | self.builder.abort() |
524 | 798 | return BuilderStatus.ABORTING | 797 | return BuilderStatus.ABORTING |
525 | 799 | 798 | ||
526 | 800 | def xmlrpc_clean(self): | 799 | def xmlrpc_clean(self): |
529 | 801 | """Clean up the waiting files and reset the slave's internal state.""" | 800 | """Clean up the waiting files and reset the builder's state.""" |
530 | 802 | self.slave.clean() | 801 | self.builder.clean() |
531 | 803 | return BuilderStatus.IDLE | 802 | return BuilderStatus.IDLE |
532 | 804 | 803 | ||
533 | 805 | def xmlrpc_build(self, buildid, managertag, chrootsum, filemap, args): | 804 | def xmlrpc_build(self, buildid, managertag, chrootsum, filemap, args): |
534 | @@ -813,7 +812,7 @@ | |||
535 | 813 | extra_info = "%s not in %r" % (managertag, list(self._managers)) | 812 | extra_info = "%s not in %r" % (managertag, list(self._managers)) |
536 | 814 | return (BuilderStatus.UNKNOWNBUILDER, extra_info) | 813 | return (BuilderStatus.UNKNOWNBUILDER, extra_info) |
537 | 815 | # check requested chroot availability | 814 | # check requested chroot availability |
539 | 816 | chroot_present, info = self.slave.ensurePresent(chrootsum) | 815 | chroot_present, info = self.builder.ensurePresent(chrootsum) |
540 | 817 | if not chroot_present: | 816 | if not chroot_present: |
541 | 818 | extra_info = """CHROOTSUM -> %s | 817 | extra_info = """CHROOTSUM -> %s |
542 | 819 | ***** INFO ***** | 818 | ***** INFO ***** |
543 | @@ -823,7 +822,7 @@ | |||
544 | 823 | return (BuilderStatus.UNKNOWNSUM, extra_info) | 822 | return (BuilderStatus.UNKNOWNSUM, extra_info) |
545 | 824 | # check requested files availability | 823 | # check requested files availability |
546 | 825 | for filesum in filemap.values(): | 824 | for filesum in filemap.values(): |
548 | 826 | file_present, info = self.slave.ensurePresent(filesum) | 825 | file_present, info = self.builder.ensurePresent(filesum) |
549 | 827 | if not file_present: | 826 | if not file_present: |
550 | 828 | extra_info = """FILESUM -> %s | 827 | extra_info = """FILESUM -> %s |
551 | 829 | ***** INFO ***** | 828 | ***** INFO ***** |
552 | @@ -838,6 +837,7 @@ | |||
553 | 838 | # builder is available, buildd is non empty, | 837 | # builder is available, buildd is non empty, |
554 | 839 | # filelist is consistent, chrootsum is available, let's initiate... | 838 | # filelist is consistent, chrootsum is available, let's initiate... |
555 | 840 | self.buildid = buildid | 839 | self.buildid = buildid |
558 | 841 | self.slave.startBuild(self._managers[managertag](self.slave, buildid)) | 840 | self.builder.startBuild( |
559 | 842 | self.slave.manager.initiate(filemap, chrootsum, args) | 841 | self._managers[managertag](self.builder, buildid)) |
560 | 842 | self.builder.manager.initiate(filemap, chrootsum, args) | ||
561 | 843 | return (BuilderStatus.BUILDING, buildid) | 843 | return (BuilderStatus.BUILDING, buildid) |
562 | 844 | 844 | ||
563 | === modified file 'lpbuildd/debian.py' | |||
564 | --- lpbuildd/debian.py 2018-10-19 06:41:14 +0000 | |||
565 | +++ lpbuildd/debian.py 2019-02-12 10:36:12 +0000 | |||
566 | @@ -4,8 +4,6 @@ | |||
567 | 4 | # Authors: Daniel Silverstone <daniel.silverstone@canonical.com> | 4 | # Authors: Daniel Silverstone <daniel.silverstone@canonical.com> |
568 | 5 | # and Adam Conrad <adam.conrad@canonical.com> | 5 | # and Adam Conrad <adam.conrad@canonical.com> |
569 | 6 | 6 | ||
570 | 7 | # Buildd Slave sbuild manager implementation | ||
571 | 8 | |||
572 | 9 | __metaclass__ = type | 7 | __metaclass__ = type |
573 | 10 | 8 | ||
574 | 11 | import base64 | 9 | import base64 |
575 | @@ -19,9 +17,7 @@ | |||
576 | 19 | ) | 17 | ) |
577 | 20 | from twisted.python import log | 18 | from twisted.python import log |
578 | 21 | 19 | ||
582 | 22 | from lpbuildd.slave import ( | 20 | from lpbuildd.builder import BuildManager |
580 | 23 | BuildManager, | ||
581 | 24 | ) | ||
583 | 25 | 21 | ||
584 | 26 | 22 | ||
585 | 27 | class DebianBuildState: | 23 | class DebianBuildState: |
586 | @@ -39,11 +35,11 @@ | |||
587 | 39 | class DebianBuildManager(BuildManager): | 35 | class DebianBuildManager(BuildManager): |
588 | 40 | """Base behaviour for Debian chrooted builds.""" | 36 | """Base behaviour for Debian chrooted builds.""" |
589 | 41 | 37 | ||
593 | 42 | def __init__(self, slave, buildid, **kwargs): | 38 | def __init__(self, builder, buildid, **kwargs): |
594 | 43 | BuildManager.__init__(self, slave, buildid, **kwargs) | 39 | BuildManager.__init__(self, builder, buildid, **kwargs) |
595 | 44 | self._cachepath = slave._config.get("slave", "filecache") | 40 | self._cachepath = builder._config.get("slave", "filecache") |
596 | 45 | self._state = DebianBuildState.INIT | 41 | self._state = DebianBuildState.INIT |
598 | 46 | slave.emptyLog() | 42 | builder.emptyLog() |
599 | 47 | self.alreadyfailed = False | 43 | self.alreadyfailed = False |
600 | 48 | self._iterator = None | 44 | self._iterator = None |
601 | 49 | 45 | ||
602 | @@ -110,12 +106,12 @@ | |||
603 | 110 | The primary file we care about is the .changes file. We key from there. | 106 | The primary file we care about is the .changes file. We key from there. |
604 | 111 | """ | 107 | """ |
605 | 112 | path = self.getChangesFilename() | 108 | path = self.getChangesFilename() |
607 | 113 | self._slave.addWaitingFile(path) | 109 | self._builder.addWaitingFile(path) |
608 | 114 | 110 | ||
609 | 115 | chfile = open(path, "r") | 111 | chfile = open(path, "r") |
610 | 116 | try: | 112 | try: |
611 | 117 | for fn in self._parseChangesFile(chfile): | 113 | for fn in self._parseChangesFile(chfile): |
613 | 118 | self._slave.addWaitingFile( | 114 | self._builder.addWaitingFile( |
614 | 119 | get_build_path(self.home, self._buildid, fn)) | 115 | get_build_path(self.home, self._buildid, fn)) |
615 | 120 | finally: | 116 | finally: |
616 | 121 | chfile.close() | 117 | chfile.close() |
617 | @@ -127,11 +123,12 @@ | |||
618 | 127 | def failed_to_gather(failure): | 123 | def failed_to_gather(failure): |
619 | 128 | if failure.check(defer.CancelledError): | 124 | if failure.check(defer.CancelledError): |
620 | 129 | if not self.alreadyfailed: | 125 | if not self.alreadyfailed: |
623 | 130 | self._slave.log("Build cancelled unexpectedly!") | 126 | self._builder.log("Build cancelled unexpectedly!") |
624 | 131 | self._slave.buildFail() | 127 | self._builder.buildFail() |
625 | 132 | else: | 128 | else: |
628 | 133 | self._slave.log("Failed to gather results: %s" % failure.value) | 129 | self._builder.log( |
629 | 134 | self._slave.buildFail() | 130 | "Failed to gather results: %s" % failure.value) |
630 | 131 | self._builder.buildFail() | ||
631 | 135 | self.alreadyfailed = True | 132 | self.alreadyfailed = True |
632 | 136 | 133 | ||
633 | 137 | def reap(ignored): | 134 | def reap(ignored): |
634 | @@ -173,7 +170,7 @@ | |||
635 | 173 | if success != 0: | 170 | if success != 0: |
636 | 174 | if not self.alreadyfailed: | 171 | if not self.alreadyfailed: |
637 | 175 | # The init failed, can't fathom why that would be... | 172 | # The init failed, can't fathom why that would be... |
639 | 176 | self._slave.builderFail() | 173 | self._builder.builderFail() |
640 | 177 | self.alreadyfailed = True | 174 | self.alreadyfailed = True |
641 | 178 | self._state = DebianBuildState.CLEANUP | 175 | self._state = DebianBuildState.CLEANUP |
642 | 179 | self.doCleanup() | 176 | self.doCleanup() |
643 | @@ -186,7 +183,7 @@ | |||
644 | 186 | if success != 0: | 183 | if success != 0: |
645 | 187 | if not self.alreadyfailed: | 184 | if not self.alreadyfailed: |
646 | 188 | # The unpack failed for some reason... | 185 | # The unpack failed for some reason... |
648 | 189 | self._slave.chrootFail() | 186 | self._builder.chrootFail() |
649 | 190 | self.alreadyfailed = True | 187 | self.alreadyfailed = True |
650 | 191 | self._state = DebianBuildState.CLEANUP | 188 | self._state = DebianBuildState.CLEANUP |
651 | 192 | self.doCleanup() | 189 | self.doCleanup() |
652 | @@ -198,7 +195,7 @@ | |||
653 | 198 | """Just finished doing the mounts.""" | 195 | """Just finished doing the mounts.""" |
654 | 199 | if success != 0: | 196 | if success != 0: |
655 | 200 | if not self.alreadyfailed: | 197 | if not self.alreadyfailed: |
657 | 201 | self._slave.chrootFail() | 198 | self._builder.chrootFail() |
658 | 202 | self.alreadyfailed = True | 199 | self.alreadyfailed = True |
659 | 203 | self._state = DebianBuildState.UMOUNT | 200 | self._state = DebianBuildState.UMOUNT |
660 | 204 | self.doUnmounting() | 201 | self.doUnmounting() |
661 | @@ -258,7 +255,7 @@ | |||
662 | 258 | """Just finished overwriting sources.list.""" | 255 | """Just finished overwriting sources.list.""" |
663 | 259 | if success != 0: | 256 | if success != 0: |
664 | 260 | if not self.alreadyfailed: | 257 | if not self.alreadyfailed: |
666 | 261 | self._slave.chrootFail() | 258 | self._builder.chrootFail() |
667 | 262 | self.alreadyfailed = True | 259 | self.alreadyfailed = True |
668 | 263 | self.doReapProcesses(self._state) | 260 | self.doReapProcesses(self._state) |
669 | 264 | elif self.trusted_keys: | 261 | elif self.trusted_keys: |
670 | @@ -277,7 +274,7 @@ | |||
671 | 277 | """Just finished adding trusted keys.""" | 274 | """Just finished adding trusted keys.""" |
672 | 278 | if success != 0: | 275 | if success != 0: |
673 | 279 | if not self.alreadyfailed: | 276 | if not self.alreadyfailed: |
675 | 280 | self._slave.chrootFail() | 277 | self._builder.chrootFail() |
676 | 281 | self.alreadyfailed = True | 278 | self.alreadyfailed = True |
677 | 282 | self.doReapProcesses(self._state) | 279 | self.doReapProcesses(self._state) |
678 | 283 | else: | 280 | else: |
679 | @@ -293,7 +290,7 @@ | |||
680 | 293 | """Just finished updating the chroot.""" | 290 | """Just finished updating the chroot.""" |
681 | 294 | if success != 0: | 291 | if success != 0: |
682 | 295 | if not self.alreadyfailed: | 292 | if not self.alreadyfailed: |
684 | 296 | self._slave.chrootFail() | 293 | self._builder.chrootFail() |
685 | 297 | self.alreadyfailed = True | 294 | self.alreadyfailed = True |
686 | 298 | self.doReapProcesses(self._state) | 295 | self.doReapProcesses(self._state) |
687 | 299 | else: | 296 | else: |
688 | @@ -309,7 +306,7 @@ | |||
689 | 309 | """Just finished doing the unmounting.""" | 306 | """Just finished doing the unmounting.""" |
690 | 310 | if success != 0: | 307 | if success != 0: |
691 | 311 | if not self.alreadyfailed: | 308 | if not self.alreadyfailed: |
693 | 312 | self._slave.builderFail() | 309 | self._builder.builderFail() |
694 | 313 | self.alreadyfailed = True | 310 | self.alreadyfailed = True |
695 | 314 | self._state = DebianBuildState.CLEANUP | 311 | self._state = DebianBuildState.CLEANUP |
696 | 315 | self.doCleanup() | 312 | self.doCleanup() |
697 | @@ -318,13 +315,13 @@ | |||
698 | 318 | """Just finished the cleanup.""" | 315 | """Just finished the cleanup.""" |
699 | 319 | if success != 0: | 316 | if success != 0: |
700 | 320 | if not self.alreadyfailed: | 317 | if not self.alreadyfailed: |
702 | 321 | self._slave.builderFail() | 318 | self._builder.builderFail() |
703 | 322 | self.alreadyfailed = True | 319 | self.alreadyfailed = True |
704 | 323 | else: | 320 | else: |
705 | 324 | # Successful clean | 321 | # Successful clean |
706 | 325 | if not self.alreadyfailed: | 322 | if not self.alreadyfailed: |
709 | 326 | self._slave.buildOK() | 323 | self._builder.buildOK() |
710 | 327 | self._slave.buildComplete() | 324 | self._builder.buildComplete() |
711 | 328 | 325 | ||
712 | 329 | def abortReap(self): | 326 | def abortReap(self): |
713 | 330 | """Abort by killing all processes in the chroot, as hard as we can. | 327 | """Abort by killing all processes in the chroot, as hard as we can. |
714 | 331 | 328 | ||
715 | === modified file 'lpbuildd/livefs.py' | |||
716 | --- lpbuildd/livefs.py 2019-01-28 13:07:53 +0000 | |||
717 | +++ lpbuildd/livefs.py 2019-02-12 10:36:12 +0000 | |||
718 | @@ -83,12 +83,12 @@ | |||
719 | 83 | elif (retcode >= RETCODE_FAILURE_INSTALL and | 83 | elif (retcode >= RETCODE_FAILURE_INSTALL and |
720 | 84 | retcode <= RETCODE_FAILURE_BUILD): | 84 | retcode <= RETCODE_FAILURE_BUILD): |
721 | 85 | if not self.alreadyfailed: | 85 | if not self.alreadyfailed: |
723 | 86 | self._slave.buildFail() | 86 | self._builder.buildFail() |
724 | 87 | print("Returning build status: Build failed.") | 87 | print("Returning build status: Build failed.") |
725 | 88 | self.alreadyfailed = True | 88 | self.alreadyfailed = True |
726 | 89 | else: | 89 | else: |
727 | 90 | if not self.alreadyfailed: | 90 | if not self.alreadyfailed: |
729 | 91 | self._slave.builderFail() | 91 | self._builder.builderFail() |
730 | 92 | print("Returning build status: Builder failed.") | 92 | print("Returning build status: Builder failed.") |
731 | 93 | self.alreadyfailed = True | 93 | self.alreadyfailed = True |
732 | 94 | self.doReapProcesses(self._state) | 94 | self.doReapProcesses(self._state) |
733 | 95 | 95 | ||
734 | === modified file 'lpbuildd/snap.py' | |||
735 | --- lpbuildd/snap.py 2019-02-07 11:52:09 +0000 | |||
736 | +++ lpbuildd/snap.py 2019-02-12 10:36:12 +0000 | |||
737 | @@ -237,7 +237,7 @@ | |||
738 | 237 | 'referrer': referrer, | 237 | 'referrer': referrer, |
739 | 238 | 'agent': agent, | 238 | 'agent': agent, |
740 | 239 | }) | 239 | }) |
742 | 240 | self.manager._slave.log(line.encode("UTF-8")) | 240 | self.manager._builder.log(line.encode("UTF-8")) |
743 | 241 | 241 | ||
744 | 242 | 242 | ||
745 | 243 | class SnapBuildState(DebianBuildState): | 243 | class SnapBuildState(DebianBuildState): |
746 | @@ -274,10 +274,10 @@ | |||
747 | 274 | """Start the local snap proxy, if necessary.""" | 274 | """Start the local snap proxy, if necessary.""" |
748 | 275 | if not self.proxy_url: | 275 | if not self.proxy_url: |
749 | 276 | return [] | 276 | return [] |
751 | 277 | proxy_port = self._slave._config.get("snapmanager", "proxyport") | 277 | proxy_port = self._builder._config.get("snapmanager", "proxyport") |
752 | 278 | proxy_factory = SnapProxyFactory(self, self.proxy_url, timeout=60) | 278 | proxy_factory = SnapProxyFactory(self, self.proxy_url, timeout=60) |
753 | 279 | self.proxy_service = strports.service(proxy_port, proxy_factory) | 279 | self.proxy_service = strports.service(proxy_port, proxy_factory) |
755 | 280 | self.proxy_service.setServiceParent(self._slave.service) | 280 | self.proxy_service.setServiceParent(self._builder.service) |
756 | 281 | if self.backend_name == "lxd": | 281 | if self.backend_name == "lxd": |
757 | 282 | proxy_host = self.backend.ipv4_network.ip | 282 | proxy_host = self.backend.ipv4_network.ip |
758 | 283 | else: | 283 | else: |
759 | @@ -295,7 +295,7 @@ | |||
760 | 295 | """Revoke builder proxy token.""" | 295 | """Revoke builder proxy token.""" |
761 | 296 | if not self.revocation_endpoint: | 296 | if not self.revocation_endpoint: |
762 | 297 | return | 297 | return |
764 | 298 | self._slave.log("Revoking proxy token...\n") | 298 | self._builder.log("Revoking proxy token...\n") |
765 | 299 | url = urlparse(self.proxy_url) | 299 | url = urlparse(self.proxy_url) |
766 | 300 | auth = "{}:{}".format(url.username, url.password) | 300 | auth = "{}:{}".format(url.username, url.password) |
767 | 301 | headers = { | 301 | headers = { |
768 | @@ -306,7 +306,7 @@ | |||
769 | 306 | try: | 306 | try: |
770 | 307 | urlopen(req) | 307 | urlopen(req) |
771 | 308 | except (HTTPError, URLError) as e: | 308 | except (HTTPError, URLError) as e: |
773 | 309 | self._slave.log( | 309 | self._builder.log( |
774 | 310 | "Unable to revoke token for %s: %s" % (url.username, e)) | 310 | "Unable to revoke token for %s: %s" % (url.username, e)) |
775 | 311 | 311 | ||
776 | 312 | def status(self): | 312 | def status(self): |
777 | @@ -362,12 +362,12 @@ | |||
778 | 362 | elif (retcode >= RETCODE_FAILURE_INSTALL and | 362 | elif (retcode >= RETCODE_FAILURE_INSTALL and |
779 | 363 | retcode <= RETCODE_FAILURE_BUILD): | 363 | retcode <= RETCODE_FAILURE_BUILD): |
780 | 364 | if not self.alreadyfailed: | 364 | if not self.alreadyfailed: |
782 | 365 | self._slave.buildFail() | 365 | self._builder.buildFail() |
783 | 366 | print("Returning build status: Build failed.") | 366 | print("Returning build status: Build failed.") |
784 | 367 | self.alreadyfailed = True | 367 | self.alreadyfailed = True |
785 | 368 | else: | 368 | else: |
786 | 369 | if not self.alreadyfailed: | 369 | if not self.alreadyfailed: |
788 | 370 | self._slave.builderFail() | 370 | self._builder.builderFail() |
789 | 371 | print("Returning build status: Builder failed.") | 371 | print("Returning build status: Builder failed.") |
790 | 372 | self.alreadyfailed = True | 372 | self.alreadyfailed = True |
791 | 373 | self.doReapProcesses(self._state) | 373 | self.doReapProcesses(self._state) |
792 | 374 | 374 | ||
793 | === modified file 'lpbuildd/sourcepackagerecipe.py' | |||
794 | --- lpbuildd/sourcepackagerecipe.py 2019-02-11 12:22:42 +0000 | |||
795 | +++ lpbuildd/sourcepackagerecipe.py 2019-02-12 10:36:12 +0000 | |||
796 | @@ -55,13 +55,13 @@ | |||
797 | 55 | 55 | ||
798 | 56 | initial_build_state = SourcePackageRecipeBuildState.BUILD_RECIPE | 56 | initial_build_state = SourcePackageRecipeBuildState.BUILD_RECIPE |
799 | 57 | 57 | ||
801 | 58 | def __init__(self, slave, buildid): | 58 | def __init__(self, builder, buildid): |
802 | 59 | """Constructor. | 59 | """Constructor. |
803 | 60 | 60 | ||
805 | 61 | :param slave: A build slave device. | 61 | :param builder: A builder. |
806 | 62 | :param buildid: The id of the build (a str). | 62 | :param buildid: The id of the build (a str). |
807 | 63 | """ | 63 | """ |
809 | 64 | DebianBuildManager.__init__(self, slave, buildid) | 64 | DebianBuildManager.__init__(self, builder, buildid) |
810 | 65 | self.build_recipe_path = os.path.join(self._bin, "buildrecipe") | 65 | self.build_recipe_path = os.path.join(self._bin, "buildrecipe") |
811 | 66 | 66 | ||
812 | 67 | def initiate(self, files, chroot, extra_args): | 67 | def initiate(self, files, chroot, extra_args): |
813 | @@ -109,24 +109,24 @@ | |||
814 | 109 | '.*: Depends: ([^ ]*( \([^)]*\))?)') | 109 | '.*: Depends: ([^ ]*( \([^)]*\))?)') |
815 | 110 | _, mo = self.searchLogContents([[rx, re.M]]) | 110 | _, mo = self.searchLogContents([[rx, re.M]]) |
816 | 111 | if mo: | 111 | if mo: |
818 | 112 | self._slave.depFail(mo.group(1)) | 112 | self._builder.depFail(mo.group(1)) |
819 | 113 | print("Returning build status: DEPFAIL") | 113 | print("Returning build status: DEPFAIL") |
820 | 114 | print("Dependencies: " + mo.group(1)) | 114 | print("Dependencies: " + mo.group(1)) |
821 | 115 | else: | 115 | else: |
822 | 116 | print("Returning build status: Build failed") | 116 | print("Returning build status: Build failed") |
824 | 117 | self._slave.buildFail() | 117 | self._builder.buildFail() |
825 | 118 | self.alreadyfailed = True | 118 | self.alreadyfailed = True |
826 | 119 | elif ( | 119 | elif ( |
827 | 120 | retcode >= RETCODE_FAILURE_INSTALL and | 120 | retcode >= RETCODE_FAILURE_INSTALL and |
828 | 121 | retcode <= RETCODE_FAILURE_BUILD_SOURCE_PACKAGE): | 121 | retcode <= RETCODE_FAILURE_BUILD_SOURCE_PACKAGE): |
829 | 122 | # XXX AaronBentley 2009-01-13: We should handle depwait separately | 122 | # XXX AaronBentley 2009-01-13: We should handle depwait separately |
830 | 123 | if not self.alreadyfailed: | 123 | if not self.alreadyfailed: |
832 | 124 | self._slave.buildFail() | 124 | self._builder.buildFail() |
833 | 125 | print("Returning build status: Build failed.") | 125 | print("Returning build status: Build failed.") |
834 | 126 | self.alreadyfailed = True | 126 | self.alreadyfailed = True |
835 | 127 | else: | 127 | else: |
836 | 128 | if not self.alreadyfailed: | 128 | if not self.alreadyfailed: |
838 | 129 | self._slave.builderFail() | 129 | self._builder.builderFail() |
839 | 130 | print("Returning build status: Builder failed.") | 130 | print("Returning build status: Builder failed.") |
840 | 131 | self.alreadyfailed = True | 131 | self.alreadyfailed = True |
841 | 132 | self.doReapProcesses(self._state) | 132 | self.doReapProcesses(self._state) |
842 | @@ -150,5 +150,5 @@ | |||
843 | 150 | The manifest is also a useful record. | 150 | The manifest is also a useful record. |
844 | 151 | """ | 151 | """ |
845 | 152 | DebianBuildManager.gatherResults(self) | 152 | DebianBuildManager.gatherResults(self) |
847 | 153 | self._slave.addWaitingFile(get_build_path( | 153 | self._builder.addWaitingFile(get_build_path( |
848 | 154 | self.home, self._buildid, 'manifest')) | 154 | self.home, self._buildid, 'manifest')) |
849 | 155 | 155 | ||
850 | === modified file 'lpbuildd/tests/harness.py' | |||
851 | --- lpbuildd/tests/harness.py 2017-11-22 15:25:33 +0000 | |||
852 | +++ lpbuildd/tests/harness.py 2019-02-12 10:36:12 +0000 | |||
853 | @@ -18,7 +18,7 @@ | |||
854 | 18 | from fixtures import EnvironmentVariable | 18 | from fixtures import EnvironmentVariable |
855 | 19 | from txfixtures.tachandler import TacTestFixture | 19 | from txfixtures.tachandler import TacTestFixture |
856 | 20 | 20 | ||
858 | 21 | from lpbuildd.slave import BuildDSlave | 21 | from lpbuildd.builder import Builder |
859 | 22 | 22 | ||
860 | 23 | 23 | ||
861 | 24 | test_conffile = os.path.join( | 24 | test_conffile = os.path.join( |
862 | @@ -41,12 +41,12 @@ | |||
863 | 41 | """Unit tests for logtail mechanisms.""" | 41 | """Unit tests for logtail mechanisms.""" |
864 | 42 | 42 | ||
865 | 43 | def setUp(self): | 43 | def setUp(self): |
867 | 44 | """Setup a BuildDSlave using the test config.""" | 44 | """Setup a Builder using the test config.""" |
868 | 45 | conf = SafeConfigParser() | 45 | conf = SafeConfigParser() |
869 | 46 | conf.read(test_conffile) | 46 | conf.read(test_conffile) |
870 | 47 | conf.set("slave", "filecache", tempfile.mkdtemp()) | 47 | conf.set("slave", "filecache", tempfile.mkdtemp()) |
871 | 48 | 48 | ||
873 | 49 | self.slave = BuildDSlave(conf) | 49 | self.slave = Builder(conf) |
874 | 50 | self.slave._log = True | 50 | self.slave._log = True |
875 | 51 | self.slave.manager = MockBuildManager() | 51 | self.slave.manager = MockBuildManager() |
876 | 52 | 52 | ||
877 | 53 | 53 | ||
878 | === modified file 'lpbuildd/translationtemplates.py' | |||
879 | --- lpbuildd/translationtemplates.py 2018-10-19 06:41:14 +0000 | |||
880 | +++ lpbuildd/translationtemplates.py 2019-02-12 10:36:12 +0000 | |||
881 | @@ -24,14 +24,15 @@ | |||
882 | 24 | 24 | ||
883 | 25 | This is the implementation of `TranslationTemplatesBuildJob`. The | 25 | This is the implementation of `TranslationTemplatesBuildJob`. The |
884 | 26 | latter runs on the master server; TranslationTemplatesBuildManager | 26 | latter runs on the master server; TranslationTemplatesBuildManager |
886 | 27 | runs on the build slave. | 27 | runs on the builder. |
887 | 28 | """ | 28 | """ |
888 | 29 | 29 | ||
889 | 30 | initial_build_state = TranslationTemplatesBuildState.GENERATE | 30 | initial_build_state = TranslationTemplatesBuildState.GENERATE |
890 | 31 | 31 | ||
894 | 32 | def __init__(self, slave, buildid): | 32 | def __init__(self, builder, buildid): |
895 | 33 | super(TranslationTemplatesBuildManager, self).__init__(slave, buildid) | 33 | super(TranslationTemplatesBuildManager, self).__init__( |
896 | 34 | self._resultname = slave._config.get( | 34 | builder, buildid) |
897 | 35 | self._resultname = builder._config.get( | ||
898 | 35 | "translationtemplatesmanager", "resultarchive") | 36 | "translationtemplatesmanager", "resultarchive") |
899 | 36 | 37 | ||
900 | 37 | def initiate(self, files, chroot, extra_args): | 38 | def initiate(self, files, chroot, extra_args): |
901 | @@ -67,11 +68,11 @@ | |||
902 | 67 | else: | 68 | else: |
903 | 68 | if not self.alreadyfailed: | 69 | if not self.alreadyfailed: |
904 | 69 | if retcode == RETCODE_FAILURE_INSTALL: | 70 | if retcode == RETCODE_FAILURE_INSTALL: |
906 | 70 | self._slave.chrootFail() | 71 | self._builder.chrootFail() |
907 | 71 | elif retcode == RETCODE_FAILURE_BUILD: | 72 | elif retcode == RETCODE_FAILURE_BUILD: |
909 | 72 | self._slave.buildFail() | 73 | self._builder.buildFail() |
910 | 73 | else: | 74 | else: |
912 | 74 | self._slave.builderFail() | 75 | self._builder.builderFail() |
913 | 75 | self.alreadyfailed = True | 76 | self.alreadyfailed = True |
914 | 76 | self.doReapProcesses(self._state) | 77 | self.doReapProcesses(self._state) |
915 | 77 | 78 |