Merge lp:~wgrant/launchpad/ppa-reset-2.0-basics into lp:launchpad
- ppa-reset-2.0-basics
- Merge into devel
Proposed by
William Grant
Status: | Merged |
---|---|
Merged at revision: | 17050 |
Proposed branch: | lp:~wgrant/launchpad/ppa-reset-2.0-basics |
Merge into: | lp:launchpad |
Prerequisite: | lp:~wgrant/launchpad/ppa-reset-2.0-model |
Diff against target: |
985 lines (+361/-297) 6 files modified
lib/lp/buildmaster/interactor.py (+56/-54) lib/lp/buildmaster/manager.py (+76/-54) lib/lp/buildmaster/tests/mock_slaves.py (+6/-1) lib/lp/buildmaster/tests/test_interactor.py (+96/-100) lib/lp/buildmaster/tests/test_manager.py (+118/-72) lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py (+9/-16) |
To merge this branch: | bzr merge lp:~wgrant/launchpad/ppa-reset-2.0-basics |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Colin Watson (community) | Approve | ||
Review via email: mp+223397@code.launchpad.net |
Commit message
Reset virtual builders a cycle before any dispatch attempt, and persist the clean status in the DB. Preparation for the new Scalingstack reset protocol.
Description of the change
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'lib/lp/buildmaster/interactor.py' | |||
2 | --- lib/lp/buildmaster/interactor.py 2014-05-10 18:40:36 +0000 | |||
3 | +++ lib/lp/buildmaster/interactor.py 2014-06-18 08:49:42 +0000 | |||
4 | @@ -21,6 +21,7 @@ | |||
5 | 21 | removeSecurityProxy, | 21 | removeSecurityProxy, |
6 | 22 | ) | 22 | ) |
7 | 23 | 23 | ||
8 | 24 | from lp.buildmaster.enums import BuilderCleanStatus | ||
9 | 24 | from lp.buildmaster.interfaces.builder import ( | 25 | from lp.buildmaster.interfaces.builder import ( |
10 | 25 | BuildDaemonError, | 26 | BuildDaemonError, |
11 | 26 | CannotFetchFile, | 27 | CannotFetchFile, |
12 | @@ -216,7 +217,7 @@ | |||
13 | 216 | BuilderVitals = namedtuple( | 217 | BuilderVitals = namedtuple( |
14 | 217 | 'BuilderVitals', | 218 | 'BuilderVitals', |
15 | 218 | ('name', 'url', 'virtualized', 'vm_host', 'builderok', 'manual', | 219 | ('name', 'url', 'virtualized', 'vm_host', 'builderok', 'manual', |
17 | 219 | 'build_queue', 'version')) | 220 | 'build_queue', 'version', 'clean_status')) |
18 | 220 | 221 | ||
19 | 221 | _BQ_UNSPECIFIED = object() | 222 | _BQ_UNSPECIFIED = object() |
20 | 222 | 223 | ||
21 | @@ -226,7 +227,8 @@ | |||
22 | 226 | build_queue = builder.currentjob | 227 | build_queue = builder.currentjob |
23 | 227 | return BuilderVitals( | 228 | return BuilderVitals( |
24 | 228 | builder.name, builder.url, builder.virtualized, builder.vm_host, | 229 | builder.name, builder.url, builder.virtualized, builder.vm_host, |
26 | 229 | builder.builderok, builder.manual, build_queue, builder.version) | 230 | builder.builderok, builder.manual, build_queue, builder.version, |
27 | 231 | builder.clean_status) | ||
28 | 230 | 232 | ||
29 | 231 | 233 | ||
30 | 232 | class BuilderInteractor(object): | 234 | class BuilderInteractor(object): |
31 | @@ -249,44 +251,6 @@ | |||
32 | 249 | return behaviour | 251 | return behaviour |
33 | 250 | 252 | ||
34 | 251 | @classmethod | 253 | @classmethod |
35 | 252 | @defer.inlineCallbacks | ||
36 | 253 | def rescueIfLost(cls, vitals, slave, slave_status, expected_cookie, | ||
37 | 254 | logger=None): | ||
38 | 255 | """Reset the slave if its job information doesn't match the DB. | ||
39 | 256 | |||
40 | 257 | This checks the build ID reported in the slave status against | ||
41 | 258 | the given cookie. If it isn't building what we think it should | ||
42 | 259 | be, the current build will be aborted and the slave cleaned in | ||
43 | 260 | preparation for a new task. | ||
44 | 261 | |||
45 | 262 | :return: A Deferred that fires when the dialog with the slave is | ||
46 | 263 | finished. Its return value is True if the slave is lost, | ||
47 | 264 | False otherwise. | ||
48 | 265 | """ | ||
49 | 266 | # Determine the slave's current build cookie. | ||
50 | 267 | status = slave_status['builder_status'] | ||
51 | 268 | slave_cookie = slave_status.get('build_id') | ||
52 | 269 | |||
53 | 270 | if slave_cookie == expected_cookie: | ||
54 | 271 | # The master and slave agree about the current job. Continue. | ||
55 | 272 | defer.returnValue(False) | ||
56 | 273 | else: | ||
57 | 274 | # The master and slave disagree. The master is our master, | ||
58 | 275 | # so try to rescue the slave. | ||
59 | 276 | # An IDLE slave doesn't need rescuing (SlaveScanner.scan | ||
60 | 277 | # will rescue the DB side instead), and we just have to wait | ||
61 | 278 | # out an ABORTING one. | ||
62 | 279 | if status == 'BuilderStatus.WAITING': | ||
63 | 280 | yield slave.clean() | ||
64 | 281 | elif status == 'BuilderStatus.BUILDING': | ||
65 | 282 | yield slave.abort() | ||
66 | 283 | if logger: | ||
67 | 284 | logger.info( | ||
68 | 285 | "Builder slave '%s' rescued from %r (expected %r)." % | ||
69 | 286 | (vitals.name, slave_cookie, expected_cookie)) | ||
70 | 287 | defer.returnValue(True) | ||
71 | 288 | |||
72 | 289 | @classmethod | ||
73 | 290 | def resumeSlaveHost(cls, vitals, slave): | 254 | def resumeSlaveHost(cls, vitals, slave): |
74 | 291 | """Resume the slave host to a known good condition. | 255 | """Resume the slave host to a known good condition. |
75 | 292 | 256 | ||
76 | @@ -323,6 +287,49 @@ | |||
77 | 323 | 287 | ||
78 | 324 | @classmethod | 288 | @classmethod |
79 | 325 | @defer.inlineCallbacks | 289 | @defer.inlineCallbacks |
80 | 290 | def cleanSlave(cls, vitals, slave): | ||
81 | 291 | """Prepare a slave for a new build. | ||
82 | 292 | |||
83 | 293 | :return: A Deferred that fires when this stage of the resume | ||
84 | 294 | operations finishes. If the value is True, the slave is now clean. | ||
85 | 295 | If it's False, the clean is still in progress and this must be | ||
86 | 296 | called again later. | ||
87 | 297 | """ | ||
88 | 298 | if vitals.virtualized: | ||
89 | 299 | # If we are building a virtual build, resume the virtual | ||
90 | 300 | # machine. Before we try and contact the resumed slave, | ||
91 | 301 | # we're going to send it a message. This is to ensure | ||
92 | 302 | # it's accepting packets from the outside world, because | ||
93 | 303 | # testing has shown that the first packet will randomly | ||
94 | 304 | # fail for no apparent reason. This could be a quirk of | ||
95 | 305 | # the Xen guest, we're not sure. We also don't care | ||
96 | 306 | # about the result from this message, just that it's | ||
97 | 307 | # sent, hence the "addBoth". See bug 586359. | ||
98 | 308 | yield cls.resumeSlaveHost(vitals, slave) | ||
99 | 309 | yield slave.echo("ping") | ||
100 | 310 | defer.returnValue(True) | ||
101 | 311 | else: | ||
102 | 312 | slave_status = yield slave.status() | ||
103 | 313 | status = slave_status.get('builder_status', None) | ||
104 | 314 | if status == 'BuilderStatus.IDLE': | ||
105 | 315 | # This is as clean as we can get it. | ||
106 | 316 | defer.returnValue(True) | ||
107 | 317 | elif status == 'BuilderStatus.BUILDING': | ||
108 | 318 | # Asynchronously abort() the slave and wait until WAITING. | ||
109 | 319 | yield slave.abort() | ||
110 | 320 | defer.returnValue(False) | ||
111 | 321 | elif status == 'BuilderStatus.ABORTING': | ||
112 | 322 | # Wait it out until WAITING. | ||
113 | 323 | defer.returnValue(False) | ||
114 | 324 | elif status == 'BuilderStatus.WAITING': | ||
115 | 325 | # Just a synchronous clean() call and we'll be idle. | ||
116 | 326 | yield slave.clean() | ||
117 | 327 | defer.returnValue(True) | ||
118 | 328 | raise BuildDaemonError( | ||
119 | 329 | "Invalid status during clean: %r" % status) | ||
120 | 330 | |||
121 | 331 | @classmethod | ||
122 | 332 | @defer.inlineCallbacks | ||
123 | 326 | def _startBuild(cls, build_queue_item, vitals, builder, slave, behaviour, | 333 | def _startBuild(cls, build_queue_item, vitals, builder, slave, behaviour, |
124 | 327 | logger): | 334 | logger): |
125 | 328 | """Start a build on this builder. | 335 | """Start a build on this builder. |
126 | @@ -342,17 +349,12 @@ | |||
127 | 342 | raise BuildDaemonError( | 349 | raise BuildDaemonError( |
128 | 343 | "Attempted to start a build on a known-bad builder.") | 350 | "Attempted to start a build on a known-bad builder.") |
129 | 344 | 351 | ||
141 | 345 | # If we are building a virtual build, resume the virtual | 352 | if builder.clean_status != BuilderCleanStatus.CLEAN: |
142 | 346 | # machine. Before we try and contact the resumed slave, we're | 353 | raise BuildDaemonError( |
143 | 347 | # going to send it a message. This is to ensure it's accepting | 354 | "Attempted to start build on a dirty slave.") |
144 | 348 | # packets from the outside world, because testing has shown that | 355 | |
145 | 349 | # the first packet will randomly fail for no apparent reason. | 356 | builder.setCleanStatus(BuilderCleanStatus.DIRTY) |
146 | 350 | # This could be a quirk of the Xen guest, we're not sure. We | 357 | transaction.commit() |
136 | 351 | # also don't care about the result from this message, just that | ||
137 | 352 | # it's sent, hence the "addBoth". See bug 586359. | ||
138 | 353 | if builder.virtualized: | ||
139 | 354 | yield cls.resumeSlaveHost(vitals, slave) | ||
140 | 355 | yield slave.echo("ping") | ||
147 | 356 | 358 | ||
148 | 357 | yield behaviour.dispatchBuildToSlave(build_queue_item.id, logger) | 359 | yield behaviour.dispatchBuildToSlave(build_queue_item.id, logger) |
149 | 358 | 360 | ||
150 | @@ -448,9 +450,9 @@ | |||
151 | 448 | :return: A Deferred that fires when the slave dialog is finished. | 450 | :return: A Deferred that fires when the slave dialog is finished. |
152 | 449 | """ | 451 | """ |
153 | 450 | # IDLE is deliberately not handled here, because it should be | 452 | # IDLE is deliberately not handled here, because it should be |
157 | 451 | # impossible to get past rescueIfLost unless the slave matches | 453 | # impossible to get past the cookie check unless the slave |
158 | 452 | # the DB, and this method isn't called unless the DB says | 454 | # matches the DB, and this method isn't called unless the DB |
159 | 453 | # there's a job. | 455 | # says there's a job. |
160 | 454 | builder_status = slave_status['builder_status'] | 456 | builder_status = slave_status['builder_status'] |
161 | 455 | if builder_status == 'BuilderStatus.BUILDING': | 457 | if builder_status == 'BuilderStatus.BUILDING': |
162 | 456 | # Build still building, collect the logtail. | 458 | # Build still building, collect the logtail. |
163 | 457 | 459 | ||
164 | === modified file 'lib/lp/buildmaster/manager.py' | |||
165 | --- lib/lp/buildmaster/manager.py 2014-02-05 23:22:30 +0000 | |||
166 | +++ lib/lp/buildmaster/manager.py 2014-06-18 08:49:42 +0000 | |||
167 | @@ -26,6 +26,7 @@ | |||
168 | 26 | from zope.component import getUtility | 26 | from zope.component import getUtility |
169 | 27 | 27 | ||
170 | 28 | from lp.buildmaster.enums import ( | 28 | from lp.buildmaster.enums import ( |
171 | 29 | BuilderCleanStatus, | ||
172 | 29 | BuildQueueStatus, | 30 | BuildQueueStatus, |
173 | 30 | BuildStatus, | 31 | BuildStatus, |
174 | 31 | ) | 32 | ) |
175 | @@ -413,70 +414,91 @@ | |||
176 | 413 | vitals = self.builder_factory.getVitals(self.builder_name) | 414 | vitals = self.builder_factory.getVitals(self.builder_name) |
177 | 414 | interactor = self.interactor_factory() | 415 | interactor = self.interactor_factory() |
178 | 415 | slave = self.slave_factory(vitals) | 416 | slave = self.slave_factory(vitals) |
208 | 416 | if vitals.builderok: | 417 | |
209 | 417 | self.logger.debug("Scanning %s." % self.builder_name) | 418 | if vitals.build_queue is not None: |
210 | 418 | slave_status = yield slave.status() | 419 | if vitals.clean_status != BuilderCleanStatus.DIRTY: |
211 | 419 | else: | 420 | # This is probably a grave bug with security implications, |
212 | 420 | slave_status = None | 421 | # as a slave that has a job must be cleaned afterwards. |
213 | 421 | 422 | raise BuildDaemonError("Non-dirty builder allegedly building.") | |
214 | 422 | # Confirm that the DB and slave sides are in a valid, mutually | 423 | |
215 | 423 | # agreeable state. | 424 | lost_reason = None |
216 | 424 | lost_reason = None | 425 | if not vitals.builderok: |
217 | 425 | if not vitals.builderok: | 426 | lost_reason = '%s is disabled' % vitals.name |
218 | 426 | lost_reason = '%s is disabled' % vitals.name | 427 | else: |
219 | 427 | else: | 428 | slave_status = yield slave.status() |
220 | 428 | self.updateVersion(vitals, slave_status) | 429 | # Ensure that the slave has the job that we think it |
221 | 429 | cancelled = yield self.checkCancellation(vitals, slave, interactor) | 430 | # should. |
222 | 430 | if cancelled: | 431 | slave_cookie = slave_status.get('build_id') |
223 | 431 | return | 432 | expected_cookie = self.getExpectedCookie(vitals) |
224 | 432 | assert slave_status is not None | 433 | if slave_cookie != expected_cookie: |
225 | 433 | lost = yield interactor.rescueIfLost( | 434 | lost_reason = ( |
226 | 434 | vitals, slave, slave_status, self.getExpectedCookie(vitals), | 435 | '%s is lost (expected %r, got %r)' % ( |
227 | 435 | self.logger) | 436 | vitals.name, expected_cookie, slave_cookie)) |
228 | 436 | if lost: | 437 | |
229 | 437 | lost_reason = '%s is lost' % vitals.name | 438 | if lost_reason is not None: |
230 | 438 | 439 | # The slave is either confused or disabled, so reset and | |
231 | 439 | # The slave is lost or the builder is disabled. We can't | 440 | # requeue the job. The next scan cycle will clean up the |
232 | 440 | # continue to update the job status or dispatch a new job, so | 441 | # slave if appropriate. |
204 | 441 | # just rescue the assigned job, if any, so it can be dispatched | ||
205 | 442 | # to another slave. | ||
206 | 443 | if lost_reason is not None: | ||
207 | 444 | if vitals.build_queue is not None: | ||
233 | 445 | self.logger.warn( | 442 | self.logger.warn( |
234 | 446 | "%s. Resetting BuildQueue %d.", lost_reason, | 443 | "%s. Resetting BuildQueue %d.", lost_reason, |
235 | 447 | vitals.build_queue.id) | 444 | vitals.build_queue.id) |
236 | 448 | vitals.build_queue.reset() | 445 | vitals.build_queue.reset() |
237 | 449 | transaction.commit() | 446 | transaction.commit() |
246 | 450 | return | 447 | return |
247 | 451 | 448 | ||
248 | 452 | # We've confirmed that the slave state matches the DB. Continue | 449 | cancelled = yield self.checkCancellation( |
249 | 453 | # with updating the job status, or dispatching a new job if the | 450 | vitals, slave, interactor) |
250 | 454 | # builder is idle. | 451 | if cancelled: |
251 | 455 | if vitals.build_queue is not None: | 452 | return |
252 | 456 | # Scan the slave and get the logtail, or collect the build | 453 | |
253 | 457 | # if it's ready. Yes, "updateBuild" is a bad name. | 454 | # The slave and DB agree on the builder's state. Scan the |
254 | 455 | # slave and get the logtail, or collect the build if it's | ||
255 | 456 | # ready. Yes, "updateBuild" is a bad name. | ||
256 | 458 | assert slave_status is not None | 457 | assert slave_status is not None |
257 | 459 | yield interactor.updateBuild( | 458 | yield interactor.updateBuild( |
258 | 460 | vitals, slave, slave_status, self.builder_factory, | 459 | vitals, slave, slave_status, self.builder_factory, |
259 | 461 | self.behaviour_factory) | 460 | self.behaviour_factory) |
260 | 462 | elif vitals.manual: | ||
261 | 463 | # If the builder is in manual mode, don't dispatch anything. | ||
262 | 464 | self.logger.debug( | ||
263 | 465 | '%s is in manual mode, not dispatching.' % vitals.name) | ||
264 | 466 | else: | 461 | else: |
278 | 467 | # See if there is a job we can dispatch to the builder slave. | 462 | if not vitals.builderok: |
279 | 468 | builder = self.builder_factory[self.builder_name] | 463 | return |
280 | 469 | # Try to dispatch the job. If it fails, don't attempt to | 464 | # We think the builder is idle. If it's clean, dispatch. If |
281 | 470 | # just retry the scan; we need to reset the job so the | 465 | # it's dirty, clean. |
282 | 471 | # dispatch will be reattempted. | 466 | if vitals.clean_status == BuilderCleanStatus.CLEAN: |
283 | 472 | d = interactor.findAndStartJob(vitals, builder, slave) | 467 | slave_status = yield slave.status() |
284 | 473 | d.addErrback(functools.partial(self._scanFailed, False)) | 468 | if slave_status.get('builder_status') != 'BuilderStatus.IDLE': |
285 | 474 | yield d | 469 | raise BuildDaemonError( |
286 | 475 | if builder.currentjob is not None: | 470 | 'Allegedly clean slave not idle (%r instead)' |
287 | 476 | # After a successful dispatch we can reset the | 471 | % slave_status.get('builder_status')) |
288 | 477 | # failure_count. | 472 | self.updateVersion(vitals, slave_status) |
289 | 478 | builder.resetFailureCount() | 473 | if vitals.manual: |
290 | 479 | transaction.commit() | 474 | # If the builder is in manual mode, don't dispatch |
291 | 475 | # anything. | ||
292 | 476 | self.logger.debug( | ||
293 | 477 | '%s is in manual mode, not dispatching.', vitals.name) | ||
294 | 478 | return | ||
295 | 479 | # Try to find and dispatch a job. If it fails, don't | ||
296 | 480 | # attempt to just retry the scan; we need to reset | ||
297 | 481 | # the job so the dispatch will be reattempted. | ||
298 | 482 | builder = self.builder_factory[self.builder_name] | ||
299 | 483 | d = interactor.findAndStartJob(vitals, builder, slave) | ||
300 | 484 | d.addErrback(functools.partial(self._scanFailed, False)) | ||
301 | 485 | yield d | ||
302 | 486 | if builder.currentjob is not None: | ||
303 | 487 | # After a successful dispatch we can reset the | ||
304 | 488 | # failure_count. | ||
305 | 489 | builder.resetFailureCount() | ||
306 | 490 | transaction.commit() | ||
307 | 491 | else: | ||
308 | 492 | # Ask the BuilderInteractor to clean the slave. It might | ||
309 | 493 | # be immediately cleaned on return, in which case we go | ||
310 | 494 | # straight back to CLEAN, or we might have to spin | ||
311 | 495 | # through another few cycles. | ||
312 | 496 | done = yield interactor.cleanSlave(vitals, slave) | ||
313 | 497 | if done: | ||
314 | 498 | builder = self.builder_factory[self.builder_name] | ||
315 | 499 | builder.setCleanStatus(BuilderCleanStatus.CLEAN) | ||
316 | 500 | self.logger.debug('%s has been cleaned.', vitals.name) | ||
317 | 501 | transaction.commit() | ||
318 | 480 | 502 | ||
319 | 481 | 503 | ||
320 | 482 | class NewBuildersScanner: | 504 | class NewBuildersScanner: |
321 | 483 | 505 | ||
322 | === modified file 'lib/lp/buildmaster/tests/mock_slaves.py' | |||
323 | --- lib/lp/buildmaster/tests/mock_slaves.py 2014-06-06 10:32:55 +0000 | |||
324 | +++ lib/lp/buildmaster/tests/mock_slaves.py 2014-06-18 08:49:42 +0000 | |||
325 | @@ -30,6 +30,7 @@ | |||
326 | 30 | from twisted.internet import defer | 30 | from twisted.internet import defer |
327 | 31 | from twisted.web import xmlrpc | 31 | from twisted.web import xmlrpc |
328 | 32 | 32 | ||
329 | 33 | from lp.buildmaster.enums import BuilderCleanStatus | ||
330 | 33 | from lp.buildmaster.interactor import BuilderSlave | 34 | from lp.buildmaster.interactor import BuilderSlave |
331 | 34 | from lp.buildmaster.interfaces.builder import CannotFetchFile | 35 | from lp.buildmaster.interfaces.builder import CannotFetchFile |
332 | 35 | from lp.services.config import config | 36 | from lp.services.config import config |
333 | @@ -48,7 +49,7 @@ | |||
334 | 48 | 49 | ||
335 | 49 | def __init__(self, name='mock-builder', builderok=True, manual=False, | 50 | def __init__(self, name='mock-builder', builderok=True, manual=False, |
336 | 50 | virtualized=True, vm_host=None, url='http://fake:0000', | 51 | virtualized=True, vm_host=None, url='http://fake:0000', |
338 | 51 | version=None): | 52 | version=None, clean_status=BuilderCleanStatus.DIRTY): |
339 | 52 | self.currentjob = None | 53 | self.currentjob = None |
340 | 53 | self.builderok = builderok | 54 | self.builderok = builderok |
341 | 54 | self.manual = manual | 55 | self.manual = manual |
342 | @@ -58,6 +59,10 @@ | |||
343 | 58 | self.vm_host = vm_host | 59 | self.vm_host = vm_host |
344 | 59 | self.failnotes = None | 60 | self.failnotes = None |
345 | 60 | self.version = version | 61 | self.version = version |
346 | 62 | self.clean_status = clean_status | ||
347 | 63 | |||
348 | 64 | def setCleanStatus(self, clean_status): | ||
349 | 65 | self.clean_status = clean_status | ||
350 | 61 | 66 | ||
351 | 62 | def failBuilder(self, reason): | 67 | def failBuilder(self, reason): |
352 | 63 | self.builderok = False | 68 | self.builderok = False |
353 | 64 | 69 | ||
354 | === modified file 'lib/lp/buildmaster/tests/test_interactor.py' | |||
355 | --- lib/lp/buildmaster/tests/test_interactor.py 2014-06-06 11:15:29 +0000 | |||
356 | +++ lib/lp/buildmaster/tests/test_interactor.py 2014-06-18 08:49:42 +0000 | |||
357 | @@ -16,19 +16,24 @@ | |||
358 | 16 | SynchronousDeferredRunTest, | 16 | SynchronousDeferredRunTest, |
359 | 17 | ) | 17 | ) |
360 | 18 | from testtools.matchers import ContainsAll | 18 | from testtools.matchers import ContainsAll |
361 | 19 | from testtools.testcase import ExpectedException | ||
362 | 19 | from twisted.internet import defer | 20 | from twisted.internet import defer |
363 | 20 | from twisted.internet.task import Clock | 21 | from twisted.internet.task import Clock |
364 | 21 | from twisted.python.failure import Failure | 22 | from twisted.python.failure import Failure |
365 | 22 | from twisted.web.client import getPage | 23 | from twisted.web.client import getPage |
366 | 23 | from zope.security.proxy import removeSecurityProxy | 24 | from zope.security.proxy import removeSecurityProxy |
367 | 24 | 25 | ||
369 | 25 | from lp.buildmaster.enums import BuildStatus | 26 | from lp.buildmaster.enums import ( |
370 | 27 | BuilderCleanStatus, | ||
371 | 28 | BuildStatus, | ||
372 | 29 | ) | ||
373 | 26 | from lp.buildmaster.interactor import ( | 30 | from lp.buildmaster.interactor import ( |
374 | 27 | BuilderInteractor, | 31 | BuilderInteractor, |
375 | 28 | BuilderSlave, | 32 | BuilderSlave, |
376 | 29 | extract_vitals_from_db, | 33 | extract_vitals_from_db, |
377 | 30 | ) | 34 | ) |
378 | 31 | from lp.buildmaster.interfaces.builder import ( | 35 | from lp.buildmaster.interfaces.builder import ( |
379 | 36 | BuildDaemonError, | ||
380 | 32 | CannotFetchFile, | 37 | CannotFetchFile, |
381 | 33 | CannotResumeHost, | 38 | CannotResumeHost, |
382 | 34 | ) | 39 | ) |
383 | @@ -150,94 +155,75 @@ | |||
384 | 150 | slave = BuilderInteractor.makeSlaveFromVitals(vitals) | 155 | slave = BuilderInteractor.makeSlaveFromVitals(vitals) |
385 | 151 | self.assertEqual(5, slave.timeout) | 156 | self.assertEqual(5, slave.timeout) |
386 | 152 | 157 | ||
391 | 153 | @defer.inlineCallbacks | 158 | |
392 | 154 | def test_rescueIfLost_aborts_lost_and_broken_slave(self): | 159 | class TestBuilderInteractorCleanSlave(TestCase): |
393 | 155 | # A slave that's 'lost' should be aborted; when the slave is | 160 | |
394 | 156 | # broken then abort() should also throw a fault. | 161 | run_tests_with = AsynchronousDeferredRunTest |
395 | 162 | |||
396 | 163 | @defer.inlineCallbacks | ||
397 | 164 | def assertCleanCalls(self, builder, slave, calls, done): | ||
398 | 165 | actually_done = yield BuilderInteractor.cleanSlave( | ||
399 | 166 | extract_vitals_from_db(builder), slave) | ||
400 | 167 | self.assertEqual(done, actually_done) | ||
401 | 168 | self.assertEqual(calls, slave.method_log) | ||
402 | 169 | |||
403 | 170 | @defer.inlineCallbacks | ||
404 | 171 | def test_virtual(self): | ||
405 | 172 | # We don't care what the status of a virtual builder is; we just | ||
406 | 173 | # reset its VM and check that it's back. | ||
407 | 174 | yield self.assertCleanCalls( | ||
408 | 175 | MockBuilder( | ||
409 | 176 | virtualized=True, vm_host='lol', | ||
410 | 177 | clean_status=BuilderCleanStatus.DIRTY), | ||
411 | 178 | OkSlave(), ['resume', 'echo'], True) | ||
412 | 179 | |||
413 | 180 | @defer.inlineCallbacks | ||
414 | 181 | def test_nonvirtual_idle(self): | ||
415 | 182 | # An IDLE non-virtual slave is already as clean as we can get it. | ||
416 | 183 | yield self.assertCleanCalls( | ||
417 | 184 | MockBuilder( | ||
418 | 185 | virtualized=False, clean_status=BuilderCleanStatus.DIRTY), | ||
419 | 186 | OkSlave(), ['status'], True) | ||
420 | 187 | |||
421 | 188 | @defer.inlineCallbacks | ||
422 | 189 | def test_nonvirtual_building(self): | ||
423 | 190 | # A BUILDING non-virtual slave needs to be aborted. It'll go | ||
424 | 191 | # through ABORTING and eventually be picked up from WAITING. | ||
425 | 192 | yield self.assertCleanCalls( | ||
426 | 193 | MockBuilder( | ||
427 | 194 | virtualized=False, clean_status=BuilderCleanStatus.DIRTY), | ||
428 | 195 | BuildingSlave(), ['status', 'abort'], False) | ||
429 | 196 | |||
430 | 197 | @defer.inlineCallbacks | ||
431 | 198 | def test_nonvirtual_aborting(self): | ||
432 | 199 | # An ABORTING non-virtual slave must be waited out. It should | ||
433 | 200 | # hit WAITING eventually. | ||
434 | 201 | yield self.assertCleanCalls( | ||
435 | 202 | MockBuilder( | ||
436 | 203 | virtualized=False, clean_status=BuilderCleanStatus.DIRTY), | ||
437 | 204 | AbortingSlave(), ['status'], False) | ||
438 | 205 | |||
439 | 206 | @defer.inlineCallbacks | ||
440 | 207 | def test_nonvirtual_waiting(self): | ||
441 | 208 | # A WAITING non-virtual slave just needs clean() called. | ||
442 | 209 | yield self.assertCleanCalls( | ||
443 | 210 | MockBuilder( | ||
444 | 211 | virtualized=False, clean_status=BuilderCleanStatus.DIRTY), | ||
445 | 212 | WaitingSlave(), ['status', 'clean'], True) | ||
446 | 213 | |||
447 | 214 | @defer.inlineCallbacks | ||
448 | 215 | def test_nonvirtual_broken(self): | ||
449 | 216 | # A broken non-virtual builder is probably unrecoverable, so the | ||
450 | 217 | # method just crashes. | ||
451 | 218 | vitals = extract_vitals_from_db(MockBuilder( | ||
452 | 219 | virtualized=False, clean_status=BuilderCleanStatus.DIRTY)) | ||
453 | 157 | slave = LostBuildingBrokenSlave() | 220 | slave = LostBuildingBrokenSlave() |
454 | 158 | slave_status = yield slave.status() | ||
455 | 159 | try: | 221 | try: |
459 | 160 | yield BuilderInteractor.rescueIfLost( | 222 | yield BuilderInteractor.cleanSlave(vitals, slave) |
457 | 161 | extract_vitals_from_db(MockBuilder()), slave, slave_status, | ||
458 | 162 | 'trivial') | ||
460 | 163 | except xmlrpclib.Fault: | 223 | except xmlrpclib.Fault: |
462 | 164 | self.assertIn('abort', slave.call_log) | 224 | self.assertEqual(['status', 'abort'], slave.call_log) |
463 | 165 | else: | 225 | else: |
539 | 166 | self.fail("xmlrpclib.Fault not raised") | 226 | self.fail("abort() should crash.") |
465 | 167 | |||
466 | 168 | @defer.inlineCallbacks | ||
467 | 169 | def test_recover_idle_slave(self): | ||
468 | 170 | # An idle slave is not rescued, even if it's not meant to be | ||
469 | 171 | # idle. SlaveScanner.scan() will clean up the DB side, because | ||
470 | 172 | # we still report that it's lost. | ||
471 | 173 | slave = OkSlave() | ||
472 | 174 | slave_status = yield slave.status() | ||
473 | 175 | lost = yield BuilderInteractor.rescueIfLost( | ||
474 | 176 | extract_vitals_from_db(MockBuilder()), slave, slave_status, | ||
475 | 177 | 'trivial') | ||
476 | 178 | self.assertTrue(lost) | ||
477 | 179 | self.assertEqual(['status'], slave.call_log) | ||
478 | 180 | |||
479 | 181 | @defer.inlineCallbacks | ||
480 | 182 | def test_recover_ok_slave(self): | ||
481 | 183 | # An idle slave that's meant to be idle is not rescued. | ||
482 | 184 | slave = OkSlave() | ||
483 | 185 | slave_status = yield slave.status() | ||
484 | 186 | lost = yield BuilderInteractor.rescueIfLost( | ||
485 | 187 | extract_vitals_from_db(MockBuilder()), slave, slave_status, None) | ||
486 | 188 | self.assertFalse(lost) | ||
487 | 189 | self.assertEqual(['status'], slave.call_log) | ||
488 | 190 | |||
489 | 191 | @defer.inlineCallbacks | ||
490 | 192 | def test_recover_waiting_slave_with_good_id(self): | ||
491 | 193 | # rescueIfLost does not attempt to abort or clean a builder that is | ||
492 | 194 | # WAITING. | ||
493 | 195 | waiting_slave = WaitingSlave(build_id='trivial') | ||
494 | 196 | slave_status = yield waiting_slave.status() | ||
495 | 197 | lost = yield BuilderInteractor.rescueIfLost( | ||
496 | 198 | extract_vitals_from_db(MockBuilder()), waiting_slave, slave_status, | ||
497 | 199 | 'trivial') | ||
498 | 200 | self.assertFalse(lost) | ||
499 | 201 | self.assertEqual(['status'], waiting_slave.call_log) | ||
500 | 202 | |||
501 | 203 | @defer.inlineCallbacks | ||
502 | 204 | def test_recover_waiting_slave_with_bad_id(self): | ||
503 | 205 | # If a slave is WAITING with a build for us to get, and the build | ||
504 | 206 | # cookie cannot be verified, which means we don't recognize the build, | ||
505 | 207 | # then rescueBuilderIfLost should attempt to abort it, so that the | ||
506 | 208 | # builder is reset for a new build, and the corrupt build is | ||
507 | 209 | # discarded. | ||
508 | 210 | waiting_slave = WaitingSlave(build_id='non-trivial') | ||
509 | 211 | slave_status = yield waiting_slave.status() | ||
510 | 212 | lost = yield BuilderInteractor.rescueIfLost( | ||
511 | 213 | extract_vitals_from_db(MockBuilder()), waiting_slave, slave_status, | ||
512 | 214 | 'trivial') | ||
513 | 215 | self.assertTrue(lost) | ||
514 | 216 | self.assertEqual(['status', 'clean'], waiting_slave.call_log) | ||
515 | 217 | |||
516 | 218 | @defer.inlineCallbacks | ||
517 | 219 | def test_recover_building_slave_with_good_id(self): | ||
518 | 220 | # rescueIfLost does not attempt to abort or clean a builder that is | ||
519 | 221 | # BUILDING. | ||
520 | 222 | building_slave = BuildingSlave(build_id='trivial') | ||
521 | 223 | slave_status = yield building_slave.status() | ||
522 | 224 | lost = yield BuilderInteractor.rescueIfLost( | ||
523 | 225 | extract_vitals_from_db(MockBuilder()), building_slave, | ||
524 | 226 | slave_status, 'trivial') | ||
525 | 227 | self.assertFalse(lost) | ||
526 | 228 | self.assertEqual(['status'], building_slave.call_log) | ||
527 | 229 | |||
528 | 230 | @defer.inlineCallbacks | ||
529 | 231 | def test_recover_building_slave_with_bad_id(self): | ||
530 | 232 | # If a slave is BUILDING with a build id we don't recognize, then we | ||
531 | 233 | # abort the build, thus stopping it in its tracks. | ||
532 | 234 | building_slave = BuildingSlave(build_id='non-trivial') | ||
533 | 235 | slave_status = yield building_slave.status() | ||
534 | 236 | lost = yield BuilderInteractor.rescueIfLost( | ||
535 | 237 | extract_vitals_from_db(MockBuilder()), building_slave, | ||
536 | 238 | slave_status, 'trivial') | ||
537 | 239 | self.assertTrue(lost) | ||
538 | 240 | self.assertEqual(['status', 'abort'], building_slave.call_log) | ||
540 | 241 | 227 | ||
541 | 242 | 228 | ||
542 | 243 | class TestBuilderSlaveStatus(TestCase): | 229 | class TestBuilderSlaveStatus(TestCase): |
543 | @@ -321,6 +307,7 @@ | |||
544 | 321 | processor = self.factory.makeProcessor(name="i386") | 307 | processor = self.factory.makeProcessor(name="i386") |
545 | 322 | builder = self.factory.makeBuilder( | 308 | builder = self.factory.makeBuilder( |
546 | 323 | processors=[processor], virtualized=True, vm_host="bladh") | 309 | processors=[processor], virtualized=True, vm_host="bladh") |
547 | 310 | builder.setCleanStatus(BuilderCleanStatus.CLEAN) | ||
548 | 324 | self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave())) | 311 | self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave())) |
549 | 325 | distroseries = self.factory.makeDistroSeries() | 312 | distroseries = self.factory.makeDistroSeries() |
550 | 326 | das = self.factory.makeDistroArchSeries( | 313 | das = self.factory.makeDistroArchSeries( |
551 | @@ -377,21 +364,30 @@ | |||
552 | 377 | 364 | ||
553 | 378 | return d.addCallback(check_build_started) | 365 | return d.addCallback(check_build_started) |
554 | 379 | 366 | ||
570 | 380 | def test_virtual_job_dispatch_pings_before_building(self): | 367 | @defer.inlineCallbacks |
571 | 381 | # We need to send a ping to the builder to work around a bug | 368 | def test_findAndStartJob_requires_clean_slave(self): |
572 | 382 | # where sometimes the first network packet sent is dropped. | 369 | # findAndStartJob ensures that its slave starts CLEAN. |
573 | 383 | builder, build = self._setupBinaryBuildAndBuilder() | 370 | builder, build = self._setupBinaryBuildAndBuilder() |
574 | 384 | candidate = build.queueBuild() | 371 | builder.setCleanStatus(BuilderCleanStatus.DIRTY) |
575 | 385 | removeSecurityProxy(builder)._findBuildCandidate = FakeMethod( | 372 | candidate = build.queueBuild() |
576 | 386 | result=candidate) | 373 | removeSecurityProxy(builder)._findBuildCandidate = FakeMethod( |
577 | 387 | vitals = extract_vitals_from_db(builder) | 374 | result=candidate) |
578 | 388 | slave = OkSlave() | 375 | vitals = extract_vitals_from_db(builder) |
579 | 389 | d = BuilderInteractor.findAndStartJob(vitals, builder, slave) | 376 | with ExpectedException( |
580 | 390 | 377 | BuildDaemonError, | |
581 | 391 | def check_build_started(candidate): | 378 | "Attempted to start build on a dirty slave."): |
582 | 392 | self.assertIn(('echo', 'ping'), slave.call_log) | 379 | yield BuilderInteractor.findAndStartJob(vitals, builder, OkSlave()) |
583 | 393 | 380 | ||
584 | 394 | return d.addCallback(check_build_started) | 381 | @defer.inlineCallbacks |
585 | 382 | def test_findAndStartJob_dirties_slave(self): | ||
586 | 383 | # findAndStartJob marks its builder DIRTY before dispatching. | ||
587 | 384 | builder, build = self._setupBinaryBuildAndBuilder() | ||
588 | 385 | candidate = build.queueBuild() | ||
589 | 386 | removeSecurityProxy(builder)._findBuildCandidate = FakeMethod( | ||
590 | 387 | result=candidate) | ||
591 | 388 | vitals = extract_vitals_from_db(builder) | ||
592 | 389 | yield BuilderInteractor.findAndStartJob(vitals, builder, OkSlave()) | ||
593 | 390 | self.assertEqual(BuilderCleanStatus.DIRTY, builder.clean_status) | ||
594 | 395 | 391 | ||
595 | 396 | 392 | ||
596 | 397 | class TestSlave(TestCase): | 393 | class TestSlave(TestCase): |
597 | 398 | 394 | ||
598 | === modified file 'lib/lp/buildmaster/tests/test_manager.py' | |||
599 | --- lib/lp/buildmaster/tests/test_manager.py 2014-06-06 10:32:55 +0000 | |||
600 | +++ lib/lp/buildmaster/tests/test_manager.py 2014-06-18 08:49:42 +0000 | |||
601 | @@ -13,6 +13,7 @@ | |||
602 | 13 | AsynchronousDeferredRunTest, | 13 | AsynchronousDeferredRunTest, |
603 | 14 | ) | 14 | ) |
604 | 15 | from testtools.matchers import Equals | 15 | from testtools.matchers import Equals |
605 | 16 | from testtools.testcase import ExpectedException | ||
606 | 16 | import transaction | 17 | import transaction |
607 | 17 | from twisted.internet import ( | 18 | from twisted.internet import ( |
608 | 18 | defer, | 19 | defer, |
609 | @@ -25,6 +26,7 @@ | |||
610 | 25 | from zope.security.proxy import removeSecurityProxy | 26 | from zope.security.proxy import removeSecurityProxy |
611 | 26 | 27 | ||
612 | 27 | from lp.buildmaster.enums import ( | 28 | from lp.buildmaster.enums import ( |
613 | 29 | BuilderCleanStatus, | ||
614 | 28 | BuildQueueStatus, | 30 | BuildQueueStatus, |
615 | 29 | BuildStatus, | 31 | BuildStatus, |
616 | 30 | ) | 32 | ) |
617 | @@ -33,7 +35,10 @@ | |||
618 | 33 | BuilderSlave, | 35 | BuilderSlave, |
619 | 34 | extract_vitals_from_db, | 36 | extract_vitals_from_db, |
620 | 35 | ) | 37 | ) |
622 | 36 | from lp.buildmaster.interfaces.builder import IBuilderSet | 38 | from lp.buildmaster.interfaces.builder import ( |
623 | 39 | BuildDaemonError, | ||
624 | 40 | IBuilderSet, | ||
625 | 41 | ) | ||
626 | 37 | from lp.buildmaster.interfaces.buildfarmjobbehaviour import ( | 42 | from lp.buildmaster.interfaces.buildfarmjobbehaviour import ( |
627 | 38 | IBuildFarmJobBehaviour, | 43 | IBuildFarmJobBehaviour, |
628 | 39 | ) | 44 | ) |
629 | @@ -113,6 +118,7 @@ | |||
630 | 113 | job = builder.currentjob | 118 | job = builder.currentjob |
631 | 114 | if job is not None: | 119 | if job is not None: |
632 | 115 | job.reset() | 120 | job.reset() |
633 | 121 | builder.setCleanStatus(BuilderCleanStatus.CLEAN) | ||
634 | 116 | 122 | ||
635 | 117 | transaction.commit() | 123 | transaction.commit() |
636 | 118 | 124 | ||
637 | @@ -155,6 +161,7 @@ | |||
638 | 155 | # Set this to 1 here so that _checkDispatch can make sure it's | 161 | # Set this to 1 here so that _checkDispatch can make sure it's |
639 | 156 | # reset to 0 after a successful dispatch. | 162 | # reset to 0 after a successful dispatch. |
640 | 157 | builder.failure_count = 1 | 163 | builder.failure_count = 1 |
641 | 164 | builder.setCleanStatus(BuilderCleanStatus.CLEAN) | ||
642 | 158 | 165 | ||
643 | 159 | # Run 'scan' and check its result. | 166 | # Run 'scan' and check its result. |
644 | 160 | switch_dbuser(config.builddmaster.dbuser) | 167 | switch_dbuser(config.builddmaster.dbuser) |
645 | @@ -258,6 +265,7 @@ | |||
646 | 258 | def test_scan_with_nothing_to_dispatch(self): | 265 | def test_scan_with_nothing_to_dispatch(self): |
647 | 259 | factory = LaunchpadObjectFactory() | 266 | factory = LaunchpadObjectFactory() |
648 | 260 | builder = factory.makeBuilder() | 267 | builder = factory.makeBuilder() |
649 | 268 | builder.setCleanStatus(BuilderCleanStatus.CLEAN) | ||
650 | 261 | self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave())) | 269 | self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(OkSlave())) |
651 | 262 | transaction.commit() | 270 | transaction.commit() |
652 | 263 | scanner = self._getScanner(builder_name=builder.name) | 271 | scanner = self._getScanner(builder_name=builder.name) |
653 | @@ -422,9 +430,9 @@ | |||
654 | 422 | self.assertFalse(builder.builderok) | 430 | self.assertFalse(builder.builderok) |
655 | 423 | 431 | ||
656 | 424 | @defer.inlineCallbacks | 432 | @defer.inlineCallbacks |
660 | 425 | def test_fail_to_resume_slave_resets_job(self): | 433 | def test_fail_to_resume_leaves_it_dirty(self): |
661 | 426 | # If an attempt to resume and dispatch a slave fails, it should | 434 | # If an attempt to resume a slave fails, its failure count is |
662 | 427 | # reset the job via job.reset() | 435 | # incremented and it is left DIRTY. |
663 | 428 | 436 | ||
664 | 429 | # Make a slave with a failing resume() method. | 437 | # Make a slave with a failing resume() method. |
665 | 430 | slave = OkSlave() | 438 | slave = OkSlave() |
666 | @@ -435,26 +443,21 @@ | |||
667 | 435 | builder = removeSecurityProxy( | 443 | builder = removeSecurityProxy( |
668 | 436 | getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME]) | 444 | getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME]) |
669 | 437 | self._resetBuilder(builder) | 445 | self._resetBuilder(builder) |
670 | 446 | builder.setCleanStatus(BuilderCleanStatus.DIRTY) | ||
671 | 447 | builder.virtualized = True | ||
672 | 438 | self.assertEqual(0, builder.failure_count) | 448 | self.assertEqual(0, builder.failure_count) |
673 | 439 | self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(slave)) | 449 | self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(slave)) |
674 | 440 | builder.vm_host = "fake_vm_host" | 450 | builder.vm_host = "fake_vm_host" |
675 | 441 | |||
676 | 442 | scanner = self._getScanner() | ||
677 | 443 | |||
678 | 444 | # Get the next job that will be dispatched. | ||
679 | 445 | job = removeSecurityProxy(builder._findBuildCandidate()) | ||
680 | 446 | job.virtualized = True | ||
681 | 447 | builder.virtualized = True | ||
682 | 448 | transaction.commit() | 451 | transaction.commit() |
687 | 449 | yield scanner.singleCycle() | 452 | |
688 | 450 | 453 | # A spin of the scanner will see the DIRTY builder and reset it. | |
689 | 451 | # The failure_count will have been incremented on the builder, we | 454 | # Our patched reset will fail. |
690 | 452 | # can check that to see that a dispatch attempt did indeed occur. | 455 | yield self._getScanner().singleCycle() |
691 | 456 | |||
692 | 457 | # The failure_count will have been incremented on the builder, | ||
693 | 458 | # and it will be left DIRTY. | ||
694 | 453 | self.assertEqual(1, builder.failure_count) | 459 | self.assertEqual(1, builder.failure_count) |
699 | 454 | # There should also be no builder set on the job. | 460 | self.assertEqual(BuilderCleanStatus.DIRTY, builder.clean_status) |
696 | 455 | self.assertIsNone(job.builder) | ||
697 | 456 | build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(job) | ||
698 | 457 | self.assertEqual(build.status, BuildStatus.NEEDSBUILD) | ||
700 | 458 | 461 | ||
701 | 459 | @defer.inlineCallbacks | 462 | @defer.inlineCallbacks |
702 | 460 | def test_update_slave_version(self): | 463 | def test_update_slave_version(self): |
703 | @@ -569,15 +572,25 @@ | |||
704 | 569 | builder.name, BuilderFactory(), BufferLogger(), | 572 | builder.name, BuilderFactory(), BufferLogger(), |
705 | 570 | slave_factory=get_slave, clock=clock) | 573 | slave_factory=get_slave, clock=clock) |
706 | 571 | 574 | ||
709 | 572 | # The slave is idle and there's a build candidate, so the first | 575 | # The slave is idle and dirty, so the first scan will clean it |
710 | 573 | # scan will reset the builder and dispatch the build. | 576 | # with a reset. |
711 | 577 | self.assertEqual(BuilderCleanStatus.DIRTY, builder.clean_status) | ||
712 | 578 | yield scanner.scan() | ||
713 | 579 | self.assertEqual(['resume', 'echo'], get_slave.result.method_log) | ||
714 | 580 | self.assertEqual(BuilderCleanStatus.CLEAN, builder.clean_status) | ||
715 | 581 | self.assertIs(None, builder.currentjob) | ||
716 | 582 | |||
717 | 583 | # The slave is idle and clean, and there's a build candidate, so | ||
718 | 584 | # the next scan will dispatch the build. | ||
719 | 585 | get_slave.result = OkSlave() | ||
720 | 574 | yield scanner.scan() | 586 | yield scanner.scan() |
721 | 575 | self.assertEqual( | 587 | self.assertEqual( |
723 | 576 | ['status', 'resume', 'echo', 'ensurepresent', 'build'], | 588 | ['status', 'ensurepresent', 'build'], |
724 | 577 | get_slave.result.method_log) | 589 | get_slave.result.method_log) |
725 | 578 | self.assertEqual(bq, builder.currentjob) | 590 | self.assertEqual(bq, builder.currentjob) |
726 | 579 | self.assertEqual(BuildQueueStatus.RUNNING, bq.status) | 591 | self.assertEqual(BuildQueueStatus.RUNNING, bq.status) |
727 | 580 | self.assertEqual(BuildStatus.BUILDING, build.status) | 592 | self.assertEqual(BuildStatus.BUILDING, build.status) |
728 | 593 | self.assertEqual(BuilderCleanStatus.DIRTY, builder.clean_status) | ||
729 | 581 | 594 | ||
730 | 582 | # build() has been called, so switch in a BUILDING slave. | 595 | # build() has been called, so switch in a BUILDING slave. |
731 | 583 | # Scans will now just do a status() each, as the logtail is | 596 | # Scans will now just do a status() each, as the logtail is |
732 | @@ -596,7 +609,8 @@ | |||
733 | 596 | # When the build finishes, the scanner will notice, call | 609 | # When the build finishes, the scanner will notice, call |
734 | 597 | # handleStatus(), and then clean the builder. Our fake | 610 | # handleStatus(), and then clean the builder. Our fake |
735 | 598 | # _handleStatus_OK doesn't do anything special, but there'd | 611 | # _handleStatus_OK doesn't do anything special, but there'd |
737 | 599 | # usually be file retrievals in the middle. | 612 | # usually be file retrievals in the middle. The builder remains |
738 | 613 | # dirty afterward. | ||
739 | 600 | get_slave.result = WaitingSlave( | 614 | get_slave.result = WaitingSlave( |
740 | 601 | build_id=IBuildFarmJobBehaviour(build).getBuildCookie()) | 615 | build_id=IBuildFarmJobBehaviour(build).getBuildCookie()) |
741 | 602 | yield scanner.scan() | 616 | yield scanner.scan() |
742 | @@ -604,13 +618,15 @@ | |||
743 | 604 | self.assertIs(None, builder.currentjob) | 618 | self.assertIs(None, builder.currentjob) |
744 | 605 | self.assertEqual(BuildStatus.UPLOADING, build.status) | 619 | self.assertEqual(BuildStatus.UPLOADING, build.status) |
745 | 606 | self.assertEqual(builder, build.builder) | 620 | self.assertEqual(builder, build.builder) |
746 | 621 | self.assertEqual(BuilderCleanStatus.DIRTY, builder.clean_status) | ||
747 | 607 | 622 | ||
750 | 608 | # We're clean, so let's flip back to an idle slave and | 623 | # We're idle and dirty, so let's flip back to an idle slave and |
751 | 609 | # confirm that a scan does nothing special. | 624 | # confirm that the slave gets cleaned. |
752 | 610 | get_slave.result = OkSlave() | 625 | get_slave.result = OkSlave() |
753 | 611 | yield scanner.scan() | 626 | yield scanner.scan() |
755 | 612 | self.assertEqual(['status'], get_slave.result.method_log) | 627 | self.assertEqual(['resume', 'echo'], get_slave.result.method_log) |
756 | 613 | self.assertIs(None, builder.currentjob) | 628 | self.assertIs(None, builder.currentjob) |
757 | 629 | self.assertEqual(BuilderCleanStatus.CLEAN, builder.clean_status) | ||
758 | 614 | 630 | ||
759 | 615 | 631 | ||
760 | 616 | class TestPrefetchedBuilderFactory(TestCaseWithFactory): | 632 | class TestPrefetchedBuilderFactory(TestCaseWithFactory): |
761 | @@ -740,74 +756,104 @@ | |||
762 | 740 | 756 | ||
763 | 741 | run_tests_with = AsynchronousDeferredRunTest | 757 | run_tests_with = AsynchronousDeferredRunTest |
764 | 742 | 758 | ||
765 | 759 | def getScanner(self, builder_factory=None, interactor=None, slave=None, | ||
766 | 760 | behaviour=None): | ||
767 | 761 | if builder_factory is None: | ||
768 | 762 | builder_factory = MockBuilderFactory( | ||
769 | 763 | MockBuilder(virtualized=False), None) | ||
770 | 764 | if interactor is None: | ||
771 | 765 | interactor = BuilderInteractor() | ||
772 | 766 | interactor.updateBuild = FakeMethod() | ||
773 | 767 | if slave is None: | ||
774 | 768 | slave = OkSlave() | ||
775 | 769 | if behaviour is None: | ||
776 | 770 | behaviour = TrivialBehaviour() | ||
777 | 771 | return SlaveScanner( | ||
778 | 772 | 'mock', builder_factory, BufferLogger(), | ||
779 | 773 | interactor_factory=FakeMethod(interactor), | ||
780 | 774 | slave_factory=FakeMethod(slave), | ||
781 | 775 | behaviour_factory=FakeMethod(behaviour)) | ||
782 | 776 | |||
783 | 743 | @defer.inlineCallbacks | 777 | @defer.inlineCallbacks |
784 | 744 | def test_scan_with_job(self): | 778 | def test_scan_with_job(self): |
785 | 745 | # SlaveScanner.scan calls updateBuild() when a job is building. | 779 | # SlaveScanner.scan calls updateBuild() when a job is building. |
786 | 746 | slave = BuildingSlave('trivial') | 780 | slave = BuildingSlave('trivial') |
787 | 747 | bq = FakeBuildQueue() | 781 | bq = FakeBuildQueue() |
798 | 748 | 782 | scanner = self.getScanner( | |
799 | 749 | # Instrument updateBuild. | 783 | builder_factory=MockBuilderFactory(MockBuilder(), bq), |
800 | 750 | interactor = BuilderInteractor() | 784 | slave=slave) |
791 | 751 | interactor.updateBuild = FakeMethod() | ||
792 | 752 | |||
793 | 753 | scanner = SlaveScanner( | ||
794 | 754 | 'mock', MockBuilderFactory(MockBuilder(), bq), BufferLogger(), | ||
795 | 755 | interactor_factory=FakeMethod(interactor), | ||
796 | 756 | slave_factory=FakeMethod(slave), | ||
797 | 757 | behaviour_factory=FakeMethod(TrivialBehaviour())) | ||
801 | 758 | 785 | ||
802 | 759 | yield scanner.scan() | 786 | yield scanner.scan() |
803 | 760 | self.assertEqual(['status'], slave.call_log) | 787 | self.assertEqual(['status'], slave.call_log) |
805 | 761 | self.assertEqual(1, interactor.updateBuild.call_count) | 788 | self.assertEqual( |
806 | 789 | 1, scanner.interactor_factory.result.updateBuild.call_count) | ||
807 | 762 | self.assertEqual(0, bq.reset.call_count) | 790 | self.assertEqual(0, bq.reset.call_count) |
808 | 763 | 791 | ||
809 | 764 | @defer.inlineCallbacks | 792 | @defer.inlineCallbacks |
813 | 765 | def test_scan_aborts_lost_slave_with_job(self): | 793 | def test_scan_recovers_lost_slave_with_job(self): |
814 | 766 | # SlaveScanner.scan uses BuilderInteractor.rescueIfLost to abort | 794 | # SlaveScanner.scan identifies slaves that aren't building what |
815 | 767 | # slaves that don't have the expected job. | 795 | # they should be, resets the jobs, and then aborts the slaves. |
816 | 768 | slave = BuildingSlave('nontrivial') | 796 | slave = BuildingSlave('nontrivial') |
817 | 769 | bq = FakeBuildQueue() | 797 | bq = FakeBuildQueue() |
828 | 770 | 798 | builder = MockBuilder(virtualized=False) | |
829 | 771 | # Instrument updateBuild. | 799 | scanner = self.getScanner( |
830 | 772 | interactor = BuilderInteractor() | 800 | builder_factory=MockBuilderFactory(builder, bq), |
831 | 773 | interactor.updateBuild = FakeMethod() | 801 | slave=slave) |
822 | 774 | |||
823 | 775 | scanner = SlaveScanner( | ||
824 | 776 | 'mock', MockBuilderFactory(MockBuilder(), bq), BufferLogger(), | ||
825 | 777 | interactor_factory=FakeMethod(interactor), | ||
826 | 778 | slave_factory=FakeMethod(slave), | ||
827 | 779 | behaviour_factory=FakeMethod(TrivialBehaviour())) | ||
832 | 780 | 802 | ||
833 | 781 | # A single scan will call status(), notice that the slave is lost, | 803 | # A single scan will call status(), notice that the slave is lost, |
836 | 782 | # abort() the slave, then reset() the job without calling | 804 | # and reset() the job without calling updateBuild(). |
835 | 783 | # updateBuild(). | ||
837 | 784 | yield scanner.scan() | 805 | yield scanner.scan() |
840 | 785 | self.assertEqual(['status', 'abort'], slave.call_log) | 806 | self.assertEqual(['status'], slave.call_log) |
841 | 786 | self.assertEqual(0, interactor.updateBuild.call_count) | 807 | self.assertEqual( |
842 | 808 | 0, scanner.interactor_factory.result.updateBuild.call_count) | ||
843 | 787 | self.assertEqual(1, bq.reset.call_count) | 809 | self.assertEqual(1, bq.reset.call_count) |
844 | 810 | # The reset would normally have unset build_queue. | ||
845 | 811 | scanner.builder_factory.updateTestData(builder, None) | ||
846 | 812 | |||
847 | 813 | # The next scan will see a dirty idle builder with a BUILDING | ||
848 | 814 | # slave, and abort() it. | ||
849 | 815 | yield scanner.scan() | ||
850 | 816 | self.assertEqual(['status', 'status', 'abort'], slave.call_log) | ||
851 | 788 | 817 | ||
852 | 789 | @defer.inlineCallbacks | 818 | @defer.inlineCallbacks |
856 | 790 | def test_scan_aborts_lost_slave_when_idle(self): | 819 | def test_scan_recovers_lost_slave_when_idle(self): |
857 | 791 | # SlaveScanner.scan uses BuilderInteractor.rescueIfLost to abort | 820 | # SlaveScanner.scan identifies slaves that are building when |
858 | 792 | # slaves that aren't meant to have a job. | 821 | # they shouldn't be and aborts them. |
859 | 793 | slave = BuildingSlave() | 822 | slave = BuildingSlave() |
874 | 794 | 823 | scanner = self.getScanner(slave=slave) | |
861 | 795 | # Instrument updateBuild. | ||
862 | 796 | interactor = BuilderInteractor() | ||
863 | 797 | interactor.updateBuild = FakeMethod() | ||
864 | 798 | |||
865 | 799 | scanner = SlaveScanner( | ||
866 | 800 | 'mock', MockBuilderFactory(MockBuilder(), None), BufferLogger(), | ||
867 | 801 | interactor_factory=FakeMethod(interactor), | ||
868 | 802 | slave_factory=FakeMethod(slave), | ||
869 | 803 | behaviour_factory=FakeMethod(None)) | ||
870 | 804 | |||
871 | 805 | # A single scan will call status(), notice that the slave is lost, | ||
872 | 806 | # abort() the slave, then reset() the job without calling | ||
873 | 807 | # updateBuild(). | ||
875 | 808 | yield scanner.scan() | 824 | yield scanner.scan() |
876 | 809 | self.assertEqual(['status', 'abort'], slave.call_log) | 825 | self.assertEqual(['status', 'abort'], slave.call_log) |
878 | 810 | self.assertEqual(0, interactor.updateBuild.call_count) | 826 | |
879 | 827 | @defer.inlineCallbacks | ||
880 | 828 | def test_scan_building_but_not_dirty_builder_explodes(self): | ||
881 | 829 | # Builders with a build assigned must be dirty for safety | ||
882 | 830 | # reasons. If we run into one that's clean, we blow up. | ||
883 | 831 | slave = BuildingSlave() | ||
884 | 832 | builder = MockBuilder(clean_status=BuilderCleanStatus.CLEAN) | ||
885 | 833 | bq = FakeBuildQueue() | ||
886 | 834 | scanner = self.getScanner( | ||
887 | 835 | slave=slave, builder_factory=MockBuilderFactory(builder, bq)) | ||
888 | 836 | |||
889 | 837 | with ExpectedException( | ||
890 | 838 | BuildDaemonError, "Non-dirty builder allegedly building."): | ||
891 | 839 | yield scanner.scan() | ||
892 | 840 | self.assertEqual([], slave.call_log) | ||
893 | 841 | |||
894 | 842 | @defer.inlineCallbacks | ||
895 | 843 | def test_scan_clean_but_not_idle_slave_explodes(self): | ||
896 | 844 | # Clean builders by definition have slaves that are idle. If | ||
897 | 845 | # an ostensibly clean slave isn't idle, blow up. | ||
898 | 846 | slave = BuildingSlave() | ||
899 | 847 | builder = MockBuilder(clean_status=BuilderCleanStatus.CLEAN) | ||
900 | 848 | scanner = self.getScanner( | ||
901 | 849 | slave=slave, builder_factory=MockBuilderFactory(builder, None)) | ||
902 | 850 | |||
903 | 851 | with ExpectedException( | ||
904 | 852 | BuildDaemonError, | ||
905 | 853 | "Allegedly clean slave not idle " | ||
906 | 854 | "\('BuilderStatus.BUILDING' instead\)"): | ||
907 | 855 | yield scanner.scan() | ||
908 | 856 | self.assertEqual(['status'], slave.call_log) | ||
909 | 811 | 857 | ||
910 | 812 | def test_getExpectedCookie_caches(self): | 858 | def test_getExpectedCookie_caches(self): |
911 | 813 | bf = MockBuilderFactory(MockBuilder(), FakeBuildQueue()) | 859 | bf = MockBuilderFactory(MockBuilder(), FakeBuildQueue()) |
912 | 814 | 860 | ||
913 | === modified file 'lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py' | |||
914 | --- lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py 2014-06-06 10:32:55 +0000 | |||
915 | +++ lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py 2014-06-18 08:49:42 +0000 | |||
916 | @@ -19,6 +19,7 @@ | |||
917 | 19 | from zope.security.proxy import removeSecurityProxy | 19 | from zope.security.proxy import removeSecurityProxy |
918 | 20 | 20 | ||
919 | 21 | from lp.buildmaster.enums import ( | 21 | from lp.buildmaster.enums import ( |
920 | 22 | BuilderCleanStatus, | ||
921 | 22 | BuildQueueStatus, | 23 | BuildQueueStatus, |
922 | 23 | BuildStatus, | 24 | BuildStatus, |
923 | 24 | ) | 25 | ) |
924 | @@ -131,10 +132,7 @@ | |||
925 | 131 | build_log = [ | 132 | build_log = [ |
926 | 132 | ('build', cookie, 'binarypackage', chroot.content.sha1, | 133 | ('build', cookie, 'binarypackage', chroot.content.sha1, |
927 | 133 | filemap_names, extra_args)] | 134 | filemap_names, extra_args)] |
932 | 134 | if builder.virtualized: | 135 | result = upload_logs + build_log |
929 | 135 | result = [('echo', 'ping')] + upload_logs + build_log | ||
930 | 136 | else: | ||
931 | 137 | result = upload_logs + build_log | ||
933 | 138 | return result | 136 | return result |
934 | 139 | 137 | ||
935 | 140 | def test_non_virtual_ppa_dispatch(self): | 138 | def test_non_virtual_ppa_dispatch(self): |
936 | @@ -146,6 +144,7 @@ | |||
937 | 146 | archive = self.factory.makeArchive(virtualized=False) | 144 | archive = self.factory.makeArchive(virtualized=False) |
938 | 147 | slave = OkSlave() | 145 | slave = OkSlave() |
939 | 148 | builder = self.factory.makeBuilder(virtualized=False) | 146 | builder = self.factory.makeBuilder(virtualized=False) |
940 | 147 | builder.setCleanStatus(BuilderCleanStatus.CLEAN) | ||
941 | 149 | vitals = extract_vitals_from_db(builder) | 148 | vitals = extract_vitals_from_db(builder) |
942 | 150 | build = self.factory.makeBinaryPackageBuild( | 149 | build = self.factory.makeBinaryPackageBuild( |
943 | 151 | builder=builder, archive=archive) | 150 | builder=builder, archive=archive) |
944 | @@ -164,12 +163,11 @@ | |||
945 | 164 | return d | 163 | return d |
946 | 165 | 164 | ||
947 | 166 | def test_virtual_ppa_dispatch(self): | 165 | def test_virtual_ppa_dispatch(self): |
948 | 167 | # Make sure the builder slave gets reset before a build is | ||
949 | 168 | # dispatched to it. | ||
950 | 169 | archive = self.factory.makeArchive(virtualized=True) | 166 | archive = self.factory.makeArchive(virtualized=True) |
951 | 170 | slave = OkSlave() | 167 | slave = OkSlave() |
952 | 171 | builder = self.factory.makeBuilder( | 168 | builder = self.factory.makeBuilder( |
953 | 172 | virtualized=True, vm_host="foohost") | 169 | virtualized=True, vm_host="foohost") |
954 | 170 | builder.setCleanStatus(BuilderCleanStatus.CLEAN) | ||
955 | 173 | vitals = extract_vitals_from_db(builder) | 171 | vitals = extract_vitals_from_db(builder) |
956 | 174 | build = self.factory.makeBinaryPackageBuild( | 172 | build = self.factory.makeBinaryPackageBuild( |
957 | 175 | builder=builder, archive=archive) | 173 | builder=builder, archive=archive) |
958 | @@ -182,22 +180,17 @@ | |||
959 | 182 | d = interactor._startBuild( | 180 | d = interactor._startBuild( |
960 | 183 | bq, vitals, builder, slave, | 181 | bq, vitals, builder, slave, |
961 | 184 | interactor.getBuildBehaviour(bq, builder, slave), BufferLogger()) | 182 | interactor.getBuildBehaviour(bq, builder, slave), BufferLogger()) |
972 | 185 | 183 | d.addCallback( | |
973 | 186 | def check_build(ignored): | 184 | self.assertExpectedInteraction, slave.call_log, builder, build, |
974 | 187 | # We expect the first call to the slave to be a resume call, | 185 | lf, archive, ArchivePurpose.PPA) |
975 | 188 | # followed by the rest of the usual calls we expect. | 186 | return d |
966 | 189 | expected_resume_call = slave.call_log.pop(0) | ||
967 | 190 | self.assertEqual('resume', expected_resume_call) | ||
968 | 191 | self.assertExpectedInteraction( | ||
969 | 192 | ignored, slave.call_log, builder, build, lf, archive, | ||
970 | 193 | ArchivePurpose.PPA) | ||
971 | 194 | return d.addCallback(check_build) | ||
976 | 195 | 187 | ||
977 | 196 | def test_partner_dispatch_no_publishing_history(self): | 188 | def test_partner_dispatch_no_publishing_history(self): |
978 | 197 | archive = self.factory.makeArchive( | 189 | archive = self.factory.makeArchive( |
979 | 198 | virtualized=False, purpose=ArchivePurpose.PARTNER) | 190 | virtualized=False, purpose=ArchivePurpose.PARTNER) |
980 | 199 | slave = OkSlave() | 191 | slave = OkSlave() |
981 | 200 | builder = self.factory.makeBuilder(virtualized=False) | 192 | builder = self.factory.makeBuilder(virtualized=False) |
982 | 193 | builder.setCleanStatus(BuilderCleanStatus.CLEAN) | ||
983 | 201 | vitals = extract_vitals_from_db(builder) | 194 | vitals = extract_vitals_from_db(builder) |
984 | 202 | build = self.factory.makeBinaryPackageBuild( | 195 | build = self.factory.makeBinaryPackageBuild( |
985 | 203 | builder=builder, archive=archive) | 196 | builder=builder, archive=archive) |
Looks good to me; just one minor suggestion.