Merge lp:~allenap/launchpad/handle-status-ro-crash-bug-905853 into lp:launchpad
- handle-status-ro-crash-bug-905853
- Merge into devel
Proposed by
Gavin Panella
Status: | Merged |
---|---|
Approved by: | Julian Edwards |
Approved revision: | no longer in the source branch. |
Merged at revision: | 14641 |
Proposed branch: | lp:~allenap/launchpad/handle-status-ro-crash-bug-905853 |
Merge into: | lp:launchpad |
Diff against target: |
1657 lines (+555/-227) 16 files modified
buildout-templates/bin/retest.in (+11/-2) lib/lp/archiveuploader/tests/test_uploadprocessor.py (+4/-2) lib/lp/buildmaster/interfaces/builder.py (+3/-1) lib/lp/buildmaster/manager.py (+74/-46) lib/lp/buildmaster/model/builder.py (+28/-15) lib/lp/buildmaster/model/buildfarmjobbehavior.py (+63/-32) lib/lp/buildmaster/model/packagebuild.py (+94/-54) lib/lp/buildmaster/testing.py (+59/-0) lib/lp/buildmaster/tests/test_builder.py (+25/-2) lib/lp/buildmaster/tests/test_manager.py (+120/-29) lib/lp/buildmaster/tests/test_packagebuild.py (+18/-16) lib/lp/code/model/tests/test_sourcepackagerecipebuild.py (+6/-8) lib/lp/services/database/tests/test_transaction_policy.py (+14/-4) lib/lp/services/database/transaction_policy.py (+16/-4) lib/lp/soyuz/tests/test_binarypackagebuild.py (+7/-4) lib/lp/translations/model/translationtemplatesbuildbehavior.py (+13/-8) |
To merge this branch: | bzr merge lp:~allenap/launchpad/handle-status-ro-crash-bug-905853 |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Graham Binns (community) | code | Approve | |
Julian Edwards (community) | Approve | ||
Review via email: mp+86298@code.launchpad.net |
Commit message
[r=gmb,
Description of the change
This fixes the problem described in the bug, but it also fixes the tests to (a) run correctly (i.e. with AsynchronousDef
To post a comment you must log in.
Revision history for this message
Graham Binns (gmb) wrote : | # |
Nice branch; I've no complaints.
review:
Approve
(code)
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'buildout-templates/bin/retest.in' | |||
2 | --- buildout-templates/bin/retest.in 2011-12-22 14:35:39 +0000 | |||
3 | +++ buildout-templates/bin/retest.in 2012-01-03 12:26:30 +0000 | |||
4 | @@ -28,7 +28,7 @@ | |||
5 | 28 | import os | 28 | import os |
6 | 29 | import re | 29 | import re |
7 | 30 | import sys | 30 | import sys |
9 | 31 | from itertools import takewhile | 31 | from itertools import takewhile, imap |
10 | 32 | 32 | ||
11 | 33 | ${python-relative-path-setup} | 33 | ${python-relative-path-setup} |
12 | 34 | 34 | ||
13 | @@ -38,6 +38,14 @@ | |||
14 | 38 | # Regular expression to match numbered stories. | 38 | # Regular expression to match numbered stories. |
15 | 39 | STORY_RE = re.compile("(.*)/\d{2}-.*") | 39 | STORY_RE = re.compile("(.*)/\d{2}-.*") |
16 | 40 | 40 | ||
17 | 41 | # Regular expression to remove terminal color escapes. | ||
18 | 42 | COLOR_RE = re.compile("\x1b[[][0-9;]+m") | ||
19 | 43 | |||
20 | 44 | |||
21 | 45 | def decolorize(text): | ||
22 | 46 | """Remove all ANSI terminal color escapes from `text`.""" | ||
23 | 47 | return COLOR_RE.sub("", text) | ||
24 | 48 | |||
25 | 41 | 49 | ||
26 | 42 | def get_test_name(test): | 50 | def get_test_name(test): |
27 | 43 | """Get the test name of a failed test. | 51 | """Get the test name of a failed test. |
28 | @@ -91,7 +99,8 @@ | |||
29 | 91 | 99 | ||
30 | 92 | 100 | ||
31 | 93 | if __name__ == '__main__': | 101 | if __name__ == '__main__': |
33 | 94 | tests = extract_tests(fileinput.input()) | 102 | lines = imap(decolorize, fileinput.input()) |
34 | 103 | tests = extract_tests(lines) | ||
35 | 95 | if len(tests) >= 1: | 104 | if len(tests) >= 1: |
36 | 96 | run_tests(tests) | 105 | run_tests(tests) |
37 | 97 | else: | 106 | else: |
38 | 98 | 107 | ||
39 | === modified file 'lib/lp/archiveuploader/tests/test_uploadprocessor.py' | |||
40 | --- lib/lp/archiveuploader/tests/test_uploadprocessor.py 2011-12-30 06:14:56 +0000 | |||
41 | +++ lib/lp/archiveuploader/tests/test_uploadprocessor.py 2012-01-03 12:26:30 +0000 | |||
42 | @@ -629,7 +629,8 @@ | |||
43 | 629 | from_addr, to_addrs, raw_msg = stub.test_emails.pop() | 629 | from_addr, to_addrs, raw_msg = stub.test_emails.pop() |
44 | 630 | foo_bar = "Foo Bar <foo.bar@canonical.com>" | 630 | foo_bar = "Foo Bar <foo.bar@canonical.com>" |
45 | 631 | daniel = "Daniel Silverstone <daniel.silverstone@canonical.com>" | 631 | daniel = "Daniel Silverstone <daniel.silverstone@canonical.com>" |
47 | 632 | self.assertEqual([e.strip() for e in to_addrs], [foo_bar, daniel]) | 632 | self.assertContentEqual( |
48 | 633 | [foo_bar, daniel], [e.strip() for e in to_addrs]) | ||
49 | 633 | self.assertTrue( | 634 | self.assertTrue( |
50 | 634 | "NEW" in raw_msg, "Expected email containing 'NEW', got:\n%s" | 635 | "NEW" in raw_msg, "Expected email containing 'NEW', got:\n%s" |
51 | 635 | % raw_msg) | 636 | % raw_msg) |
52 | @@ -663,7 +664,8 @@ | |||
53 | 663 | from_addr, to_addrs, raw_msg = stub.test_emails.pop() | 664 | from_addr, to_addrs, raw_msg = stub.test_emails.pop() |
54 | 664 | daniel = "Daniel Silverstone <daniel.silverstone@canonical.com>" | 665 | daniel = "Daniel Silverstone <daniel.silverstone@canonical.com>" |
55 | 665 | foo_bar = "Foo Bar <foo.bar@canonical.com>" | 666 | foo_bar = "Foo Bar <foo.bar@canonical.com>" |
57 | 666 | self.assertEqual([e.strip() for e in to_addrs], [foo_bar, daniel]) | 667 | self.assertContentEqual( |
58 | 668 | [foo_bar, daniel], [e.strip() for e in to_addrs]) | ||
59 | 667 | self.assertTrue("Waiting for approval" in raw_msg, | 669 | self.assertTrue("Waiting for approval" in raw_msg, |
60 | 668 | "Expected an 'upload awaits approval' email.\n" | 670 | "Expected an 'upload awaits approval' email.\n" |
61 | 669 | "Got:\n%s" % raw_msg) | 671 | "Got:\n%s" % raw_msg) |
62 | 670 | 672 | ||
63 | === modified file 'lib/lp/buildmaster/interfaces/builder.py' | |||
64 | --- lib/lp/buildmaster/interfaces/builder.py 2012-01-01 02:58:52 +0000 | |||
65 | +++ lib/lp/buildmaster/interfaces/builder.py 2012-01-03 12:26:30 +0000 | |||
66 | @@ -1,4 +1,4 @@ | |||
68 | 1 | # Copyright 2009 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2011 Canonical Ltd. This software is licensed under the |
69 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
70 | 3 | 3 | ||
71 | 4 | # pylint: disable-msg=E0211,E0213 | 4 | # pylint: disable-msg=E0211,E0213 |
72 | @@ -195,6 +195,8 @@ | |||
73 | 195 | 195 | ||
74 | 196 | def setSlaveForTesting(proxy): | 196 | def setSlaveForTesting(proxy): |
75 | 197 | """Sets the RPC proxy through which to operate the build slave.""" | 197 | """Sets the RPC proxy through which to operate the build slave.""" |
76 | 198 | # XXX JeroenVermeulen 2011-11-09, bug=888010: Don't use this. | ||
77 | 199 | # It's a trap. See bug for details. | ||
78 | 198 | 200 | ||
79 | 199 | def verifySlaveBuildCookie(slave_build_id): | 201 | def verifySlaveBuildCookie(slave_build_id): |
80 | 200 | """Verify that a slave's build cookie is consistent. | 202 | """Verify that a slave's build cookie is consistent. |
81 | 201 | 203 | ||
82 | === modified file 'lib/lp/buildmaster/manager.py' | |||
83 | --- lib/lp/buildmaster/manager.py 2012-01-01 02:58:52 +0000 | |||
84 | +++ lib/lp/buildmaster/manager.py 2012-01-03 12:26:30 +0000 | |||
85 | @@ -1,4 +1,4 @@ | |||
87 | 1 | # Copyright 2009 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2011 Canonical Ltd. This software is licensed under the |
88 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
89 | 3 | 3 | ||
90 | 4 | """Soyuz buildd slave manager logic.""" | 4 | """Soyuz buildd slave manager logic.""" |
91 | @@ -34,6 +34,7 @@ | |||
92 | 34 | BuildBehaviorMismatch, | 34 | BuildBehaviorMismatch, |
93 | 35 | ) | 35 | ) |
94 | 36 | from lp.buildmaster.model.builder import Builder | 36 | from lp.buildmaster.model.builder import Builder |
95 | 37 | from lp.services.database.transaction_policy import DatabaseTransactionPolicy | ||
96 | 37 | 38 | ||
97 | 38 | 39 | ||
98 | 39 | BUILDD_MANAGER_LOG_NAME = "slave-scanner" | 40 | BUILDD_MANAGER_LOG_NAME = "slave-scanner" |
99 | @@ -111,13 +112,17 @@ | |||
100 | 111 | # algorithm for polling. | 112 | # algorithm for polling. |
101 | 112 | SCAN_INTERVAL = 15 | 113 | SCAN_INTERVAL = 15 |
102 | 113 | 114 | ||
104 | 114 | def __init__(self, builder_name, logger): | 115 | def __init__(self, builder_name, logger, clock=None): |
105 | 115 | self.builder_name = builder_name | 116 | self.builder_name = builder_name |
106 | 116 | self.logger = logger | 117 | self.logger = logger |
107 | 118 | if clock is None: | ||
108 | 119 | clock = reactor | ||
109 | 120 | self._clock = clock | ||
110 | 117 | 121 | ||
111 | 118 | def startCycle(self): | 122 | def startCycle(self): |
112 | 119 | """Scan the builder and dispatch to it or deal with failures.""" | 123 | """Scan the builder and dispatch to it or deal with failures.""" |
113 | 120 | self.loop = LoopingCall(self.singleCycle) | 124 | self.loop = LoopingCall(self.singleCycle) |
114 | 125 | self.loop.clock = self._clock | ||
115 | 121 | self.stopping_deferred = self.loop.start(self.SCAN_INTERVAL) | 126 | self.stopping_deferred = self.loop.start(self.SCAN_INTERVAL) |
116 | 122 | return self.stopping_deferred | 127 | return self.stopping_deferred |
117 | 123 | 128 | ||
118 | @@ -138,51 +143,58 @@ | |||
119 | 138 | 1. Print the error in the log | 143 | 1. Print the error in the log |
120 | 139 | 2. Increment and assess failure counts on the builder and job. | 144 | 2. Increment and assess failure counts on the builder and job. |
121 | 140 | """ | 145 | """ |
125 | 141 | # Make sure that pending database updates are removed as it | 146 | # Since this is a failure path, we could be in a broken |
126 | 142 | # could leave the database in an inconsistent state (e.g. The | 147 | # transaction. Get us a fresh one. |
124 | 143 | # job says it's running but the buildqueue has no builder set). | ||
127 | 144 | transaction.abort() | 148 | transaction.abort() |
128 | 145 | 149 | ||
129 | 146 | # If we don't recognise the exception include a stack trace with | 150 | # If we don't recognise the exception include a stack trace with |
130 | 147 | # the error. | 151 | # the error. |
131 | 148 | error_message = failure.getErrorMessage() | 152 | error_message = failure.getErrorMessage() |
133 | 149 | if failure.check( | 153 | familiar_error = failure.check( |
134 | 150 | BuildSlaveFailure, CannotBuild, BuildBehaviorMismatch, | 154 | BuildSlaveFailure, CannotBuild, BuildBehaviorMismatch, |
138 | 151 | CannotResumeHost, BuildDaemonError, CannotFetchFile): | 155 | CannotResumeHost, BuildDaemonError, CannotFetchFile) |
139 | 152 | self.logger.info("Scanning %s failed with: %s" % ( | 156 | if familiar_error: |
140 | 153 | self.builder_name, error_message)) | 157 | self.logger.info( |
141 | 158 | "Scanning %s failed with: %s", | ||
142 | 159 | self.builder_name, error_message) | ||
143 | 154 | else: | 160 | else: |
145 | 155 | self.logger.info("Scanning %s failed with: %s\n%s" % ( | 161 | self.logger.info( |
146 | 162 | "Scanning %s failed with: %s\n%s", | ||
147 | 156 | self.builder_name, failure.getErrorMessage(), | 163 | self.builder_name, failure.getErrorMessage(), |
149 | 157 | failure.getTraceback())) | 164 | failure.getTraceback()) |
150 | 158 | 165 | ||
151 | 159 | # Decide if we need to terminate the job or fail the | 166 | # Decide if we need to terminate the job or fail the |
152 | 160 | # builder. | 167 | # builder. |
153 | 161 | try: | 168 | try: |
154 | 162 | builder = get_builder(self.builder_name) | 169 | builder = get_builder(self.builder_name) |
162 | 163 | builder.gotFailure() | 170 | transaction.commit() |
163 | 164 | if builder.currentjob is not None: | 171 | |
164 | 165 | build_farm_job = builder.getCurrentBuildFarmJob() | 172 | with DatabaseTransactionPolicy(read_only=False): |
165 | 166 | build_farm_job.gotFailure() | 173 | builder.gotFailure() |
166 | 167 | self.logger.info( | 174 | |
167 | 168 | "builder %s failure count: %s, " | 175 | if builder.currentjob is None: |
168 | 169 | "job '%s' failure count: %s" % ( | 176 | self.logger.info( |
169 | 177 | "Builder %s failed a probe, count: %s", | ||
170 | 178 | self.builder_name, builder.failure_count) | ||
171 | 179 | else: | ||
172 | 180 | build_farm_job = builder.getCurrentBuildFarmJob() | ||
173 | 181 | build_farm_job.gotFailure() | ||
174 | 182 | self.logger.info( | ||
175 | 183 | "builder %s failure count: %s, " | ||
176 | 184 | "job '%s' failure count: %s", | ||
177 | 170 | self.builder_name, | 185 | self.builder_name, |
178 | 171 | builder.failure_count, | 186 | builder.failure_count, |
179 | 172 | build_farm_job.title, | 187 | build_farm_job.title, |
187 | 173 | build_farm_job.failure_count)) | 188 | build_farm_job.failure_count) |
188 | 174 | else: | 189 | |
189 | 175 | self.logger.info( | 190 | assessFailureCounts(builder, failure.getErrorMessage()) |
190 | 176 | "Builder %s failed a probe, count: %s" % ( | 191 | transaction.commit() |
184 | 177 | self.builder_name, builder.failure_count)) | ||
185 | 178 | assessFailureCounts(builder, failure.getErrorMessage()) | ||
186 | 179 | transaction.commit() | ||
191 | 180 | except: | 192 | except: |
192 | 181 | # Catastrophic code failure! Not much we can do. | 193 | # Catastrophic code failure! Not much we can do. |
193 | 194 | transaction.abort() | ||
194 | 182 | self.logger.error( | 195 | self.logger.error( |
195 | 183 | "Miserable failure when trying to examine failure counts:\n", | 196 | "Miserable failure when trying to examine failure counts:\n", |
196 | 184 | exc_info=True) | 197 | exc_info=True) |
197 | 185 | transaction.abort() | ||
198 | 186 | 198 | ||
199 | 187 | def checkCancellation(self, builder): | 199 | def checkCancellation(self, builder): |
200 | 188 | """See if there is a pending cancellation request. | 200 | """See if there is a pending cancellation request. |
201 | @@ -236,14 +248,9 @@ | |||
202 | 236 | """ | 248 | """ |
203 | 237 | # We need to re-fetch the builder object on each cycle as the | 249 | # We need to re-fetch the builder object on each cycle as the |
204 | 238 | # Storm store is invalidated over transaction boundaries. | 250 | # Storm store is invalidated over transaction boundaries. |
205 | 239 | |||
206 | 240 | self.builder = get_builder(self.builder_name) | 251 | self.builder = get_builder(self.builder_name) |
207 | 241 | 252 | ||
208 | 242 | def status_updated(ignored): | 253 | def status_updated(ignored): |
209 | 243 | # Commit the changes done while possibly rescuing jobs, to | ||
210 | 244 | # avoid holding table locks. | ||
211 | 245 | transaction.commit() | ||
212 | 246 | |||
213 | 247 | # See if we think there's an active build on the builder. | 254 | # See if we think there's an active build on the builder. |
214 | 248 | buildqueue = self.builder.getBuildQueue() | 255 | buildqueue = self.builder.getBuildQueue() |
215 | 249 | 256 | ||
216 | @@ -253,14 +260,10 @@ | |||
217 | 253 | return self.builder.updateBuild(buildqueue) | 260 | return self.builder.updateBuild(buildqueue) |
218 | 254 | 261 | ||
219 | 255 | def build_updated(ignored): | 262 | def build_updated(ignored): |
220 | 256 | # Commit changes done while updating the build, to avoid | ||
221 | 257 | # holding table locks. | ||
222 | 258 | transaction.commit() | ||
223 | 259 | |||
224 | 260 | # If the builder is in manual mode, don't dispatch anything. | 263 | # If the builder is in manual mode, don't dispatch anything. |
225 | 261 | if self.builder.manual: | 264 | if self.builder.manual: |
226 | 262 | self.logger.debug( | 265 | self.logger.debug( |
228 | 263 | '%s is in manual mode, not dispatching.' % | 266 | '%s is in manual mode, not dispatching.', |
229 | 264 | self.builder.name) | 267 | self.builder.name) |
230 | 265 | return | 268 | return |
231 | 266 | 269 | ||
232 | @@ -278,22 +281,33 @@ | |||
233 | 278 | job = self.builder.currentjob | 281 | job = self.builder.currentjob |
234 | 279 | if job is not None and not self.builder.builderok: | 282 | if job is not None and not self.builder.builderok: |
235 | 280 | self.logger.info( | 283 | self.logger.info( |
239 | 281 | "%s was made unavailable, resetting attached " | 284 | "%s was made unavailable; resetting attached job.", |
240 | 282 | "job" % self.builder.name) | 285 | self.builder.name) |
238 | 283 | job.reset() | ||
241 | 284 | transaction.commit() | 286 | transaction.commit() |
242 | 287 | with DatabaseTransactionPolicy(read_only=False): | ||
243 | 288 | job.reset() | ||
244 | 289 | transaction.commit() | ||
245 | 285 | return | 290 | return |
246 | 286 | 291 | ||
247 | 287 | # See if there is a job we can dispatch to the builder slave. | 292 | # See if there is a job we can dispatch to the builder slave. |
248 | 288 | 293 | ||
249 | 294 | # XXX JeroenVermeulen 2011-10-11, bug=872112: The job's | ||
250 | 295 | # failure count will be reset once the job has started | ||
251 | 296 | # successfully. Because of intervening commits, you may see | ||
252 | 297 | # a build with a nonzero failure count that's actually going | ||
253 | 298 | # to succeed later (and have a failure count of zero). Or | ||
254 | 299 | # it may fail yet end up with a lower failure count than you | ||
255 | 300 | # saw earlier. | ||
256 | 289 | d = self.builder.findAndStartJob() | 301 | d = self.builder.findAndStartJob() |
257 | 290 | 302 | ||
258 | 291 | def job_started(candidate): | 303 | def job_started(candidate): |
259 | 292 | if self.builder.currentjob is not None: | 304 | if self.builder.currentjob is not None: |
260 | 293 | # After a successful dispatch we can reset the | 305 | # After a successful dispatch we can reset the |
261 | 294 | # failure_count. | 306 | # failure_count. |
262 | 295 | self.builder.resetFailureCount() | ||
263 | 296 | transaction.commit() | 307 | transaction.commit() |
264 | 308 | with DatabaseTransactionPolicy(read_only=False): | ||
265 | 309 | self.builder.resetFailureCount() | ||
266 | 310 | transaction.commit() | ||
267 | 297 | return self.builder.slave | 311 | return self.builder.slave |
268 | 298 | else: | 312 | else: |
269 | 299 | return None | 313 | return None |
270 | @@ -372,6 +386,7 @@ | |||
271 | 372 | self.logger = self._setupLogger() | 386 | self.logger = self._setupLogger() |
272 | 373 | self.new_builders_scanner = NewBuildersScanner( | 387 | self.new_builders_scanner = NewBuildersScanner( |
273 | 374 | manager=self, clock=clock) | 388 | manager=self, clock=clock) |
274 | 389 | self.transaction_policy = DatabaseTransactionPolicy(read_only=True) | ||
275 | 375 | 390 | ||
276 | 376 | def _setupLogger(self): | 391 | def _setupLogger(self): |
277 | 377 | """Set up a 'slave-scanner' logger that redirects to twisted. | 392 | """Set up a 'slave-scanner' logger that redirects to twisted. |
278 | @@ -390,16 +405,28 @@ | |||
279 | 390 | logger.setLevel(level) | 405 | logger.setLevel(level) |
280 | 391 | return logger | 406 | return logger |
281 | 392 | 407 | ||
282 | 408 | def enterReadOnlyDatabasePolicy(self): | ||
283 | 409 | """Set the database transaction policy to read-only. | ||
284 | 410 | |||
285 | 411 | Any previously pending changes are committed first. | ||
286 | 412 | """ | ||
287 | 413 | transaction.commit() | ||
288 | 414 | self.transaction_policy.__enter__() | ||
289 | 415 | |||
290 | 416 | def exitReadOnlyDatabasePolicy(self, *args): | ||
291 | 417 | """Reset database transaction policy to the default read-write.""" | ||
292 | 418 | self.transaction_policy.__exit__(None, None, None) | ||
293 | 419 | |||
294 | 393 | def startService(self): | 420 | def startService(self): |
295 | 394 | """Service entry point, called when the application starts.""" | 421 | """Service entry point, called when the application starts.""" |
296 | 422 | # Avoiding circular imports. | ||
297 | 423 | from lp.buildmaster.interfaces.builder import IBuilderSet | ||
298 | 424 | |||
299 | 425 | self.enterReadOnlyDatabasePolicy() | ||
300 | 395 | 426 | ||
301 | 396 | # Get a list of builders and set up scanners on each one. | 427 | # Get a list of builders and set up scanners on each one. |
308 | 397 | 428 | self.addScanForBuilders( | |
309 | 398 | # Avoiding circular imports. | 429 | [builder.name for builder in getUtility(IBuilderSet)]) |
304 | 399 | from lp.buildmaster.interfaces.builder import IBuilderSet | ||
305 | 400 | builder_set = getUtility(IBuilderSet) | ||
306 | 401 | builders = [builder.name for builder in builder_set] | ||
307 | 402 | self.addScanForBuilders(builders) | ||
310 | 403 | self.new_builders_scanner.scheduleScan() | 430 | self.new_builders_scanner.scheduleScan() |
311 | 404 | 431 | ||
312 | 405 | # Events will now fire in the SlaveScanner objects to scan each | 432 | # Events will now fire in the SlaveScanner objects to scan each |
313 | @@ -420,6 +447,7 @@ | |||
314 | 420 | # stopped, so we can wait on them all at once here before | 447 | # stopped, so we can wait on them all at once here before |
315 | 421 | # exiting. | 448 | # exiting. |
316 | 422 | d = defer.DeferredList(deferreds, consumeErrors=True) | 449 | d = defer.DeferredList(deferreds, consumeErrors=True) |
317 | 450 | d.addCallback(self.exitReadOnlyDatabasePolicy) | ||
318 | 423 | return d | 451 | return d |
319 | 424 | 452 | ||
320 | 425 | def addScanForBuilders(self, builders): | 453 | def addScanForBuilders(self, builders): |
321 | 426 | 454 | ||
322 | === modified file 'lib/lp/buildmaster/model/builder.py' | |||
323 | --- lib/lp/buildmaster/model/builder.py 2012-01-02 11:21:14 +0000 | |||
324 | +++ lib/lp/buildmaster/model/builder.py 2012-01-03 12:26:30 +0000 | |||
325 | @@ -1,4 +1,4 @@ | |||
327 | 1 | # Copyright 2009,2011 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2011 Canonical Ltd. This software is licensed under the |
328 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
329 | 3 | 3 | ||
330 | 4 | # pylint: disable-msg=E0611,W0212 | 4 | # pylint: disable-msg=E0611,W0212 |
331 | @@ -66,6 +66,7 @@ | |||
332 | 66 | SQLBase, | 66 | SQLBase, |
333 | 67 | sqlvalues, | 67 | sqlvalues, |
334 | 68 | ) | 68 | ) |
335 | 69 | from lp.services.database.transaction_policy import DatabaseTransactionPolicy | ||
336 | 69 | from lp.services.helpers import filenameToContentType | 70 | from lp.services.helpers import filenameToContentType |
337 | 70 | from lp.services.job.interfaces.job import JobStatus | 71 | from lp.services.job.interfaces.job import JobStatus |
338 | 71 | from lp.services.job.model.job import Job | 72 | from lp.services.job.model.job import Job |
339 | @@ -545,6 +546,8 @@ | |||
340 | 545 | 546 | ||
341 | 546 | def setSlaveForTesting(self, proxy): | 547 | def setSlaveForTesting(self, proxy): |
342 | 547 | """See IBuilder.""" | 548 | """See IBuilder.""" |
343 | 549 | # XXX JeroenVermeulen 2011-11-09, bug=888010: Don't use this. | ||
344 | 550 | # It's a trap. See bug for details. | ||
345 | 548 | self._testing_slave = proxy | 551 | self._testing_slave = proxy |
346 | 549 | del get_property_cache(self).slave | 552 | del get_property_cache(self).slave |
347 | 550 | 553 | ||
348 | @@ -673,10 +676,13 @@ | |||
349 | 673 | bytes_written = out_file.tell() | 676 | bytes_written = out_file.tell() |
350 | 674 | out_file.seek(0) | 677 | out_file.seek(0) |
351 | 675 | 678 | ||
356 | 676 | library_file = getUtility(ILibraryFileAliasSet).create( | 679 | transaction.commit() |
357 | 677 | filename, bytes_written, out_file, | 680 | with DatabaseTransactionPolicy(read_only=False): |
358 | 678 | contentType=filenameToContentType(filename), | 681 | library_file = getUtility(ILibraryFileAliasSet).create( |
359 | 679 | restricted=private) | 682 | filename, bytes_written, out_file, |
360 | 683 | contentType=filenameToContentType(filename), | ||
361 | 684 | restricted=private) | ||
362 | 685 | transaction.commit() | ||
363 | 680 | finally: | 686 | finally: |
364 | 681 | # Remove the temporary file. getFile() closes the file | 687 | # Remove the temporary file. getFile() closes the file |
365 | 682 | # object. | 688 | # object. |
366 | @@ -714,7 +720,7 @@ | |||
367 | 714 | def acquireBuildCandidate(self): | 720 | def acquireBuildCandidate(self): |
368 | 715 | """Acquire a build candidate in an atomic fashion. | 721 | """Acquire a build candidate in an atomic fashion. |
369 | 716 | 722 | ||
371 | 717 | When retrieiving a candidate we need to mark it as building | 723 | When retrieving a candidate we need to mark it as building |
372 | 718 | immediately so that it is not dispatched by another builder in the | 724 | immediately so that it is not dispatched by another builder in the |
373 | 719 | build manager. | 725 | build manager. |
374 | 720 | 726 | ||
375 | @@ -724,12 +730,15 @@ | |||
376 | 724 | can be in this code at the same time. | 730 | can be in this code at the same time. |
377 | 725 | 731 | ||
378 | 726 | If there's ever more than one build manager running at once, then | 732 | If there's ever more than one build manager running at once, then |
380 | 727 | this code will need some sort of mutex. | 733 | this code will need some sort of mutex, or run in a single |
381 | 734 | transaction. | ||
382 | 728 | """ | 735 | """ |
383 | 729 | candidate = self._findBuildCandidate() | 736 | candidate = self._findBuildCandidate() |
384 | 730 | if candidate is not None: | 737 | if candidate is not None: |
385 | 731 | candidate.markAsBuilding(self) | ||
386 | 732 | transaction.commit() | 738 | transaction.commit() |
387 | 739 | with DatabaseTransactionPolicy(read_only=False): | ||
388 | 740 | candidate.markAsBuilding(self) | ||
389 | 741 | transaction.commit() | ||
390 | 733 | return candidate | 742 | return candidate |
391 | 734 | 743 | ||
392 | 735 | def _findBuildCandidate(self): | 744 | def _findBuildCandidate(self): |
393 | @@ -792,13 +801,17 @@ | |||
394 | 792 | store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR) | 801 | store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR) |
395 | 793 | candidate_jobs = store.execute(query).get_all() | 802 | candidate_jobs = store.execute(query).get_all() |
396 | 794 | 803 | ||
404 | 795 | for (candidate_id,) in candidate_jobs: | 804 | transaction.commit() |
405 | 796 | candidate = getUtility(IBuildQueueSet).get(candidate_id) | 805 | with DatabaseTransactionPolicy(read_only=False): |
406 | 797 | job_class = job_classes[candidate.job_type] | 806 | for (candidate_id,) in candidate_jobs: |
407 | 798 | candidate_approved = job_class.postprocessCandidate( | 807 | candidate = getUtility(IBuildQueueSet).get(candidate_id) |
408 | 799 | candidate, logger) | 808 | job_class = job_classes[candidate.job_type] |
409 | 800 | if candidate_approved: | 809 | candidate_approved = job_class.postprocessCandidate( |
410 | 801 | return candidate | 810 | candidate, logger) |
411 | 811 | if candidate_approved: | ||
412 | 812 | transaction.commit() | ||
413 | 813 | return candidate | ||
414 | 814 | transaction.commit() | ||
415 | 802 | 815 | ||
416 | 803 | return None | 816 | return None |
417 | 804 | 817 | ||
418 | 805 | 818 | ||
419 | === modified file 'lib/lp/buildmaster/model/buildfarmjobbehavior.py' | |||
420 | --- lib/lp/buildmaster/model/buildfarmjobbehavior.py 2011-12-30 02:24:09 +0000 | |||
421 | +++ lib/lp/buildmaster/model/buildfarmjobbehavior.py 2012-01-03 12:26:30 +0000 | |||
422 | @@ -1,4 +1,4 @@ | |||
424 | 1 | # Copyright 2009-2010 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2011 Canonical Ltd. This software is licensed under the |
425 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
426 | 3 | 3 | ||
427 | 4 | # pylint: disable-msg=E0211,E0213 | 4 | # pylint: disable-msg=E0211,E0213 |
428 | @@ -16,6 +16,7 @@ | |||
429 | 16 | import socket | 16 | import socket |
430 | 17 | import xmlrpclib | 17 | import xmlrpclib |
431 | 18 | 18 | ||
432 | 19 | import transaction | ||
433 | 19 | from twisted.internet import defer | 20 | from twisted.internet import defer |
434 | 20 | from zope.component import getUtility | 21 | from zope.component import getUtility |
435 | 21 | from zope.interface import implements | 22 | from zope.interface import implements |
436 | @@ -30,6 +31,7 @@ | |||
437 | 30 | IBuildFarmJobBehavior, | 31 | IBuildFarmJobBehavior, |
438 | 31 | ) | 32 | ) |
439 | 32 | from lp.services import encoding | 33 | from lp.services import encoding |
440 | 34 | from lp.services.database.transaction_policy import DatabaseTransactionPolicy | ||
441 | 33 | from lp.services.job.interfaces.job import JobStatus | 35 | from lp.services.job.interfaces.job import JobStatus |
442 | 34 | from lp.services.librarian.interfaces.client import ILibrarianClient | 36 | from lp.services.librarian.interfaces.client import ILibrarianClient |
443 | 35 | 37 | ||
444 | @@ -69,6 +71,25 @@ | |||
445 | 69 | if slave_build_cookie != expected_cookie: | 71 | if slave_build_cookie != expected_cookie: |
446 | 70 | raise CorruptBuildCookie("Invalid slave build cookie.") | 72 | raise CorruptBuildCookie("Invalid slave build cookie.") |
447 | 71 | 73 | ||
448 | 74 | def _getBuilderStatusHandler(self, status_text, logger): | ||
449 | 75 | """Look up the handler method for a given builder status. | ||
450 | 76 | |||
451 | 77 | If status is not a known one, logs an error and returns None. | ||
452 | 78 | """ | ||
453 | 79 | builder_status_handlers = { | ||
454 | 80 | 'BuilderStatus.IDLE': self.updateBuild_IDLE, | ||
455 | 81 | 'BuilderStatus.BUILDING': self.updateBuild_BUILDING, | ||
456 | 82 | 'BuilderStatus.ABORTING': self.updateBuild_ABORTING, | ||
457 | 83 | 'BuilderStatus.ABORTED': self.updateBuild_ABORTED, | ||
458 | 84 | 'BuilderStatus.WAITING': self.updateBuild_WAITING, | ||
459 | 85 | } | ||
460 | 86 | handler = builder_status_handlers.get(status_text) | ||
461 | 87 | if handler is None: | ||
462 | 88 | logger.critical( | ||
463 | 89 | "Builder on %s returned unknown status %s; failing it.", | ||
464 | 90 | self._builder.url, status_text) | ||
465 | 91 | return handler | ||
466 | 92 | |||
467 | 72 | def updateBuild(self, queueItem): | 93 | def updateBuild(self, queueItem): |
468 | 73 | """See `IBuildFarmJobBehavior`.""" | 94 | """See `IBuildFarmJobBehavior`.""" |
469 | 74 | logger = logging.getLogger('slave-scanner') | 95 | logger = logging.getLogger('slave-scanner') |
470 | @@ -76,6 +97,7 @@ | |||
471 | 76 | d = self._builder.slaveStatus() | 97 | d = self._builder.slaveStatus() |
472 | 77 | 98 | ||
473 | 78 | def got_failure(failure): | 99 | def got_failure(failure): |
474 | 100 | transaction.abort() | ||
475 | 79 | failure.trap(xmlrpclib.Fault, socket.error) | 101 | failure.trap(xmlrpclib.Fault, socket.error) |
476 | 80 | info = failure.value | 102 | info = failure.value |
477 | 81 | info = ("Could not contact the builder %s, caught a (%s)" | 103 | info = ("Could not contact the builder %s, caught a (%s)" |
478 | @@ -83,27 +105,22 @@ | |||
479 | 83 | raise BuildSlaveFailure(info) | 105 | raise BuildSlaveFailure(info) |
480 | 84 | 106 | ||
481 | 85 | def got_status(slave_status): | 107 | def got_status(slave_status): |
482 | 86 | builder_status_handlers = { | ||
483 | 87 | 'BuilderStatus.IDLE': self.updateBuild_IDLE, | ||
484 | 88 | 'BuilderStatus.BUILDING': self.updateBuild_BUILDING, | ||
485 | 89 | 'BuilderStatus.ABORTING': self.updateBuild_ABORTING, | ||
486 | 90 | 'BuilderStatus.ABORTED': self.updateBuild_ABORTED, | ||
487 | 91 | 'BuilderStatus.WAITING': self.updateBuild_WAITING, | ||
488 | 92 | } | ||
489 | 93 | |||
490 | 94 | builder_status = slave_status['builder_status'] | 108 | builder_status = slave_status['builder_status'] |
496 | 95 | if builder_status not in builder_status_handlers: | 109 | status_handler = self._getBuilderStatusHandler( |
497 | 96 | logger.critical( | 110 | builder_status, logger) |
498 | 97 | "Builder on %s returned unknown status %s, failing it" | 111 | if status_handler is None: |
499 | 98 | % (self._builder.url, builder_status)) | 112 | error = ( |
495 | 99 | self._builder.failBuilder( | ||
500 | 100 | "Unknown status code (%s) returned from status() probe." | 113 | "Unknown status code (%s) returned from status() probe." |
501 | 101 | % builder_status) | 114 | % builder_status) |
507 | 102 | # XXX: This will leave the build and job in a bad state, but | 115 | transaction.commit() |
508 | 103 | # should never be possible, since our builder statuses are | 116 | with DatabaseTransactionPolicy(read_only=False): |
509 | 104 | # known. | 117 | self._builder.failBuilder(error) |
510 | 105 | queueItem._builder = None | 118 | # XXX: This will leave the build and job in a bad |
511 | 106 | queueItem.setDateStarted(None) | 119 | # state, but should never be possible since our |
512 | 120 | # builder statuses are known. | ||
513 | 121 | queueItem._builder = None | ||
514 | 122 | queueItem.setDateStarted(None) | ||
515 | 123 | transaction.commit() | ||
516 | 107 | return | 124 | return |
517 | 108 | 125 | ||
518 | 109 | # Since logtail is a xmlrpclib.Binary container and it is | 126 | # Since logtail is a xmlrpclib.Binary container and it is |
519 | @@ -113,9 +130,8 @@ | |||
520 | 113 | # will simply remove the proxy. | 130 | # will simply remove the proxy. |
521 | 114 | logtail = removeSecurityProxy(slave_status.get('logtail')) | 131 | logtail = removeSecurityProxy(slave_status.get('logtail')) |
522 | 115 | 132 | ||
523 | 116 | method = builder_status_handlers[builder_status] | ||
524 | 117 | return defer.maybeDeferred( | 133 | return defer.maybeDeferred( |
526 | 118 | method, queueItem, slave_status, logtail, logger) | 134 | status_handler, queueItem, slave_status, logtail, logger) |
527 | 119 | 135 | ||
528 | 120 | d.addErrback(got_failure) | 136 | d.addErrback(got_failure) |
529 | 121 | d.addCallback(got_status) | 137 | d.addCallback(got_status) |
530 | @@ -127,22 +143,32 @@ | |||
531 | 127 | Log this and reset the record. | 143 | Log this and reset the record. |
532 | 128 | """ | 144 | """ |
533 | 129 | logger.warn( | 145 | logger.warn( |
537 | 130 | "Builder %s forgot about buildqueue %d -- resetting buildqueue " | 146 | "Builder %s forgot about buildqueue %d -- " |
538 | 131 | "record" % (queueItem.builder.url, queueItem.id)) | 147 | "resetting buildqueue record.", |
539 | 132 | queueItem.reset() | 148 | queueItem.builder.url, queueItem.id) |
540 | 149 | transaction.commit() | ||
541 | 150 | with DatabaseTransactionPolicy(read_only=False): | ||
542 | 151 | queueItem.reset() | ||
543 | 152 | transaction.commit() | ||
544 | 133 | 153 | ||
545 | 134 | def updateBuild_BUILDING(self, queueItem, slave_status, logtail, logger): | 154 | def updateBuild_BUILDING(self, queueItem, slave_status, logtail, logger): |
546 | 135 | """Build still building, collect the logtail""" | 155 | """Build still building, collect the logtail""" |
550 | 136 | if queueItem.job.status != JobStatus.RUNNING: | 156 | transaction.commit() |
551 | 137 | queueItem.job.start() | 157 | with DatabaseTransactionPolicy(read_only=False): |
552 | 138 | queueItem.logtail = encoding.guess(str(logtail)) | 158 | if queueItem.job.status != JobStatus.RUNNING: |
553 | 159 | queueItem.job.start() | ||
554 | 160 | queueItem.logtail = encoding.guess(str(logtail)) | ||
555 | 161 | transaction.commit() | ||
556 | 139 | 162 | ||
557 | 140 | def updateBuild_ABORTING(self, queueItem, slave_status, logtail, logger): | 163 | def updateBuild_ABORTING(self, queueItem, slave_status, logtail, logger): |
558 | 141 | """Build was ABORTED. | 164 | """Build was ABORTED. |
559 | 142 | 165 | ||
560 | 143 | Master-side should wait until the slave finish the process correctly. | 166 | Master-side should wait until the slave finish the process correctly. |
561 | 144 | """ | 167 | """ |
563 | 145 | queueItem.logtail = "Waiting for slave process to be terminated" | 168 | transaction.commit() |
564 | 169 | with DatabaseTransactionPolicy(read_only=False): | ||
565 | 170 | queueItem.logtail = "Waiting for slave process to be terminated" | ||
566 | 171 | transaction.commit() | ||
567 | 146 | 172 | ||
568 | 147 | def updateBuild_ABORTED(self, queueItem, slave_status, logtail, logger): | 173 | def updateBuild_ABORTED(self, queueItem, slave_status, logtail, logger): |
569 | 148 | """ABORTING process has successfully terminated. | 174 | """ABORTING process has successfully terminated. |
570 | @@ -150,11 +176,16 @@ | |||
571 | 150 | Clean the builder for another jobs. | 176 | Clean the builder for another jobs. |
572 | 151 | """ | 177 | """ |
573 | 152 | d = queueItem.builder.cleanSlave() | 178 | d = queueItem.builder.cleanSlave() |
574 | 179 | |||
575 | 153 | def got_cleaned(ignored): | 180 | def got_cleaned(ignored): |
580 | 154 | queueItem.builder = None | 181 | transaction.commit() |
581 | 155 | if queueItem.job.status != JobStatus.FAILED: | 182 | with DatabaseTransactionPolicy(read_only=False): |
582 | 156 | queueItem.job.fail() | 183 | queueItem.builder = None |
583 | 157 | queueItem.specific_job.jobAborted() | 184 | if queueItem.job.status != JobStatus.FAILED: |
584 | 185 | queueItem.job.fail() | ||
585 | 186 | queueItem.specific_job.jobAborted() | ||
586 | 187 | transaction.commit() | ||
587 | 188 | |||
588 | 158 | return d.addCallback(got_cleaned) | 189 | return d.addCallback(got_cleaned) |
589 | 159 | 190 | ||
590 | 160 | def extractBuildStatus(self, slave_status): | 191 | def extractBuildStatus(self, slave_status): |
591 | 161 | 192 | ||
592 | === modified file 'lib/lp/buildmaster/model/packagebuild.py' | |||
593 | --- lib/lp/buildmaster/model/packagebuild.py 2011-12-30 06:14:56 +0000 | |||
594 | +++ lib/lp/buildmaster/model/packagebuild.py 2012-01-03 12:26:30 +0000 | |||
595 | @@ -1,4 +1,4 @@ | |||
597 | 1 | # Copyright 2010 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2010-2011 Canonical Ltd. This software is licensed under the |
598 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
599 | 3 | 3 | ||
600 | 4 | __metaclass__ = type | 4 | __metaclass__ = type |
601 | @@ -24,6 +24,7 @@ | |||
602 | 24 | Storm, | 24 | Storm, |
603 | 25 | Unicode, | 25 | Unicode, |
604 | 26 | ) | 26 | ) |
605 | 27 | import transaction | ||
606 | 27 | from zope.component import getUtility | 28 | from zope.component import getUtility |
607 | 28 | from zope.interface import ( | 29 | from zope.interface import ( |
608 | 29 | classProvides, | 30 | classProvides, |
609 | @@ -50,6 +51,7 @@ | |||
610 | 50 | from lp.services.config import config | 51 | from lp.services.config import config |
611 | 51 | from lp.services.database.enumcol import DBEnum | 52 | from lp.services.database.enumcol import DBEnum |
612 | 52 | from lp.services.database.lpstorm import IMasterStore | 53 | from lp.services.database.lpstorm import IMasterStore |
613 | 54 | from lp.services.database.transaction_policy import DatabaseTransactionPolicy | ||
614 | 53 | from lp.services.helpers import filenameToContentType | 55 | from lp.services.helpers import filenameToContentType |
615 | 54 | from lp.services.librarian.browser import ProxiedLibraryFileAlias | 56 | from lp.services.librarian.browser import ProxiedLibraryFileAlias |
616 | 55 | from lp.services.librarian.interfaces import ILibraryFileAliasSet | 57 | from lp.services.librarian.interfaces import ILibraryFileAliasSet |
617 | @@ -177,19 +179,24 @@ | |||
618 | 177 | def storeBuildInfo(build, librarian, slave_status): | 179 | def storeBuildInfo(build, librarian, slave_status): |
619 | 178 | """See `IPackageBuild`.""" | 180 | """See `IPackageBuild`.""" |
620 | 179 | def got_log(lfa_id): | 181 | def got_log(lfa_id): |
621 | 182 | dependencies = slave_status.get('dependencies') | ||
622 | 183 | if dependencies is not None: | ||
623 | 184 | dependencies = unicode(dependencies) | ||
624 | 185 | |||
625 | 180 | # log, builder and date_finished are read-only, so we must | 186 | # log, builder and date_finished are read-only, so we must |
626 | 181 | # currently remove the security proxy to set them. | 187 | # currently remove the security proxy to set them. |
627 | 182 | naked_build = removeSecurityProxy(build) | 188 | naked_build = removeSecurityProxy(build) |
638 | 183 | naked_build.log = lfa_id | 189 | |
639 | 184 | naked_build.builder = build.buildqueue_record.builder | 190 | transaction.commit() |
640 | 185 | # XXX cprov 20060615 bug=120584: Currently buildduration includes | 191 | with DatabaseTransactionPolicy(read_only=False): |
641 | 186 | # the scanner latency, it should really be asking the slave for | 192 | naked_build.log = lfa_id |
642 | 187 | # the duration spent building locally. | 193 | naked_build.builder = build.buildqueue_record.builder |
643 | 188 | naked_build.date_finished = datetime.datetime.now(pytz.UTC) | 194 | # XXX cprov 20060615 bug=120584: Currently buildduration |
644 | 189 | if slave_status.get('dependencies') is not None: | 195 | # includes the scanner latency. It should really be asking |
645 | 190 | build.dependencies = unicode(slave_status.get('dependencies')) | 196 | # the slave for the duration spent building locally. |
646 | 191 | else: | 197 | naked_build.date_finished = datetime.datetime.now(pytz.UTC) |
647 | 192 | build.dependencies = None | 198 | build.dependencies = dependencies |
648 | 199 | transaction.commit() | ||
649 | 193 | 200 | ||
650 | 194 | d = build.getLogFromSlave(build) | 201 | d = build.getLogFromSlave(build) |
651 | 195 | return d.addCallback(got_log) | 202 | return d.addCallback(got_log) |
652 | @@ -290,22 +297,41 @@ | |||
653 | 290 | 297 | ||
654 | 291 | def handleStatus(self, status, librarian, slave_status): | 298 | def handleStatus(self, status, librarian, slave_status): |
655 | 292 | """See `IPackageBuild`.""" | 299 | """See `IPackageBuild`.""" |
656 | 300 | # Avoid circular imports. | ||
657 | 293 | from lp.buildmaster.manager import BUILDD_MANAGER_LOG_NAME | 301 | from lp.buildmaster.manager import BUILDD_MANAGER_LOG_NAME |
658 | 302 | |||
659 | 294 | logger = logging.getLogger(BUILDD_MANAGER_LOG_NAME) | 303 | logger = logging.getLogger(BUILDD_MANAGER_LOG_NAME) |
660 | 295 | send_notification = status in self.ALLOWED_STATUS_NOTIFICATIONS | 304 | send_notification = status in self.ALLOWED_STATUS_NOTIFICATIONS |
661 | 296 | method = getattr(self, '_handleStatus_' + status, None) | 305 | method = getattr(self, '_handleStatus_' + status, None) |
662 | 297 | if method is None: | 306 | if method is None: |
666 | 298 | logger.critical("Unknown BuildStatus '%s' for builder '%s'" | 307 | logger.critical( |
667 | 299 | % (status, self.buildqueue_record.builder.url)) | 308 | "Unknown BuildStatus '%s' for builder '%s'", |
668 | 300 | return | 309 | status, self.buildqueue_record.builder.url) |
669 | 310 | return None | ||
670 | 311 | |||
671 | 301 | d = method(librarian, slave_status, logger, send_notification) | 312 | d = method(librarian, slave_status, logger, send_notification) |
672 | 302 | return d | 313 | return d |
673 | 303 | 314 | ||
674 | 315 | def _destroy_buildqueue_record(self, unused_arg): | ||
675 | 316 | """Destroy this build's `BuildQueue` record.""" | ||
676 | 317 | transaction.commit() | ||
677 | 318 | with DatabaseTransactionPolicy(read_only=False): | ||
678 | 319 | self.buildqueue_record.destroySelf() | ||
679 | 320 | transaction.commit() | ||
680 | 321 | |||
681 | 304 | def _release_builder_and_remove_queue_item(self): | 322 | def _release_builder_and_remove_queue_item(self): |
682 | 305 | # Release the builder for another job. | 323 | # Release the builder for another job. |
683 | 306 | d = self.buildqueue_record.builder.cleanSlave() | 324 | d = self.buildqueue_record.builder.cleanSlave() |
684 | 307 | # Remove BuildQueue record. | 325 | # Remove BuildQueue record. |
686 | 308 | return d.addCallback(lambda x: self.buildqueue_record.destroySelf()) | 326 | return d.addCallback(self._destroy_buildqueue_record) |
687 | 327 | |||
688 | 328 | def _notify_if_appropriate(self, appropriate=True, extra_info=None): | ||
689 | 329 | """If `appropriate`, call `self.notify` in a write transaction.""" | ||
690 | 330 | if appropriate: | ||
691 | 331 | transaction.commit() | ||
692 | 332 | with DatabaseTransactionPolicy(read_only=False): | ||
693 | 333 | self.notify(extra_info=extra_info) | ||
694 | 334 | transaction.commit() | ||
695 | 309 | 335 | ||
696 | 310 | def _handleStatus_OK(self, librarian, slave_status, logger, | 336 | def _handleStatus_OK(self, librarian, slave_status, logger, |
697 | 311 | send_notification): | 337 | send_notification): |
698 | @@ -321,16 +347,19 @@ | |||
699 | 321 | self.buildqueue_record.specific_job.build.title, | 347 | self.buildqueue_record.specific_job.build.title, |
700 | 322 | self.buildqueue_record.builder.name)) | 348 | self.buildqueue_record.builder.name)) |
701 | 323 | 349 | ||
704 | 324 | # If this is a binary package build, discard it if its source is | 350 | # If this is a binary package build for a source that is no |
705 | 325 | # no longer published. | 351 | # longer published, discard it. |
706 | 326 | if self.build_farm_job_type == BuildFarmJobType.PACKAGEBUILD: | 352 | if self.build_farm_job_type == BuildFarmJobType.PACKAGEBUILD: |
707 | 327 | build = self.buildqueue_record.specific_job.build | 353 | build = self.buildqueue_record.specific_job.build |
708 | 328 | if not build.current_source_publication: | 354 | if not build.current_source_publication: |
710 | 329 | build.status = BuildStatus.SUPERSEDED | 355 | transaction.commit() |
711 | 356 | with DatabaseTransactionPolicy(read_only=False): | ||
712 | 357 | build.status = BuildStatus.SUPERSEDED | ||
713 | 358 | transaction.commit() | ||
714 | 330 | return self._release_builder_and_remove_queue_item() | 359 | return self._release_builder_and_remove_queue_item() |
715 | 331 | 360 | ||
718 | 332 | # Explode before collect a binary that is denied in this | 361 | # Explode rather than collect a binary that is denied in this |
719 | 333 | # distroseries/pocket | 362 | # distroseries/pocket. |
720 | 334 | if not self.archive.allowUpdatesToReleasePocket(): | 363 | if not self.archive.allowUpdatesToReleasePocket(): |
721 | 335 | assert self.distro_series.canUploadToPocket(self.pocket), ( | 364 | assert self.distro_series.canUploadToPocket(self.pocket), ( |
722 | 336 | "%s (%s) can not be built for pocket %s: illegal status" | 365 | "%s (%s) can not be built for pocket %s: illegal status" |
723 | @@ -375,18 +404,26 @@ | |||
724 | 375 | # files from the slave. | 404 | # files from the slave. |
725 | 376 | if successful_copy_from_slave: | 405 | if successful_copy_from_slave: |
726 | 377 | logger.info( | 406 | logger.info( |
729 | 378 | "Gathered %s %d completely. Moving %s to uploader queue." | 407 | "Gathered %s %d completely. " |
730 | 379 | % (self.__class__.__name__, self.id, upload_leaf)) | 408 | "Moving %s to uploader queue.", |
731 | 409 | self.__class__.__name__, self.id, upload_leaf) | ||
732 | 380 | target_dir = os.path.join(root, "incoming") | 410 | target_dir = os.path.join(root, "incoming") |
734 | 381 | self.status = BuildStatus.UPLOADING | 411 | resulting_status = BuildStatus.UPLOADING |
735 | 382 | else: | 412 | else: |
736 | 383 | logger.warning( | 413 | logger.warning( |
742 | 384 | "Copy from slave for build %s was unsuccessful.", self.id) | 414 | "Copy from slave for build %s was unsuccessful.", |
743 | 385 | self.status = BuildStatus.FAILEDTOUPLOAD | 415 | self.id) |
739 | 386 | if send_notification: | ||
740 | 387 | self.notify( | ||
741 | 388 | extra_info='Copy from slave was unsuccessful.') | ||
744 | 389 | target_dir = os.path.join(root, "failed") | 416 | target_dir = os.path.join(root, "failed") |
745 | 417 | resulting_status = BuildStatus.FAILEDTOUPLOAD | ||
746 | 418 | |||
747 | 419 | transaction.commit() | ||
748 | 420 | with DatabaseTransactionPolicy(read_only=False): | ||
749 | 421 | self.status = resulting_status | ||
750 | 422 | transaction.commit() | ||
751 | 423 | |||
752 | 424 | if not successful_copy_from_slave: | ||
753 | 425 | self._notify_if_appropriate( | ||
754 | 426 | send_notification, "Copy from slave was unsuccessful.") | ||
755 | 390 | 427 | ||
756 | 391 | if not os.path.exists(target_dir): | 428 | if not os.path.exists(target_dir): |
757 | 392 | os.mkdir(target_dir) | 429 | os.mkdir(target_dir) |
758 | @@ -394,10 +431,6 @@ | |||
759 | 394 | # Release the builder for another job. | 431 | # Release the builder for another job. |
760 | 395 | d = self._release_builder_and_remove_queue_item() | 432 | d = self._release_builder_and_remove_queue_item() |
761 | 396 | 433 | ||
762 | 397 | # Commit so there are no race conditions with archiveuploader | ||
763 | 398 | # about self.status. | ||
764 | 399 | Store.of(self).commit() | ||
765 | 400 | |||
766 | 401 | # Move the directory used to grab the binaries into | 434 | # Move the directory used to grab the binaries into |
767 | 402 | # the incoming directory so the upload processor never | 435 | # the incoming directory so the upload processor never |
768 | 403 | # sees half-finished uploads. | 436 | # sees half-finished uploads. |
769 | @@ -421,14 +454,15 @@ | |||
770 | 421 | set the job status as FAILEDTOBUILD, store available info and | 454 | set the job status as FAILEDTOBUILD, store available info and |
771 | 422 | remove Buildqueue entry. | 455 | remove Buildqueue entry. |
772 | 423 | """ | 456 | """ |
774 | 424 | self.status = BuildStatus.FAILEDTOBUILD | 457 | transaction.commit() |
775 | 458 | with DatabaseTransactionPolicy(read_only=False): | ||
776 | 459 | self.status = BuildStatus.FAILEDTOBUILD | ||
777 | 460 | transaction.commit() | ||
778 | 425 | 461 | ||
779 | 426 | def build_info_stored(ignored): | 462 | def build_info_stored(ignored): |
782 | 427 | if send_notification: | 463 | self._notify_if_appropriate(send_notification) |
781 | 428 | self.notify() | ||
783 | 429 | d = self.buildqueue_record.builder.cleanSlave() | 464 | d = self.buildqueue_record.builder.cleanSlave() |
786 | 430 | return d.addCallback( | 465 | return d.addCallback(self._destroy_buildqueue_record) |
785 | 431 | lambda x: self.buildqueue_record.destroySelf()) | ||
787 | 432 | 466 | ||
788 | 433 | d = self.storeBuildInfo(self, librarian, slave_status) | 467 | d = self.storeBuildInfo(self, librarian, slave_status) |
789 | 434 | return d.addCallback(build_info_stored) | 468 | return d.addCallback(build_info_stored) |
790 | @@ -441,16 +475,16 @@ | |||
791 | 441 | MANUALDEPWAIT, store available information, remove BuildQueue | 475 | MANUALDEPWAIT, store available information, remove BuildQueue |
792 | 442 | entry and release builder slave for another job. | 476 | entry and release builder slave for another job. |
793 | 443 | """ | 477 | """ |
795 | 444 | self.status = BuildStatus.MANUALDEPWAIT | 478 | with DatabaseTransactionPolicy(read_only=False): |
796 | 479 | self.status = BuildStatus.MANUALDEPWAIT | ||
797 | 480 | transaction.commit() | ||
798 | 445 | 481 | ||
799 | 446 | def build_info_stored(ignored): | 482 | def build_info_stored(ignored): |
800 | 447 | logger.critical("***** %s is MANUALDEPWAIT *****" | 483 | logger.critical("***** %s is MANUALDEPWAIT *****" |
801 | 448 | % self.buildqueue_record.builder.name) | 484 | % self.buildqueue_record.builder.name) |
804 | 449 | if send_notification: | 485 | self._notify_if_appropriate(send_notification) |
803 | 450 | self.notify() | ||
805 | 451 | d = self.buildqueue_record.builder.cleanSlave() | 486 | d = self.buildqueue_record.builder.cleanSlave() |
808 | 452 | return d.addCallback( | 487 | return d.addCallback(self._destroy_buildqueue_record) |
807 | 453 | lambda x: self.buildqueue_record.destroySelf()) | ||
809 | 454 | 488 | ||
810 | 455 | d = self.storeBuildInfo(self, librarian, slave_status) | 489 | d = self.storeBuildInfo(self, librarian, slave_status) |
811 | 456 | return d.addCallback(build_info_stored) | 490 | return d.addCallback(build_info_stored) |
812 | @@ -463,20 +497,29 @@ | |||
813 | 463 | job as CHROOTFAIL, store available information, remove BuildQueue | 497 | job as CHROOTFAIL, store available information, remove BuildQueue |
814 | 464 | and release the builder. | 498 | and release the builder. |
815 | 465 | """ | 499 | """ |
817 | 466 | self.status = BuildStatus.CHROOTWAIT | 500 | with DatabaseTransactionPolicy(read_only=False): |
818 | 501 | self.status = BuildStatus.CHROOTWAIT | ||
819 | 502 | transaction.commit() | ||
820 | 467 | 503 | ||
821 | 468 | def build_info_stored(ignored): | 504 | def build_info_stored(ignored): |
826 | 469 | logger.critical("***** %s is CHROOTWAIT *****" % | 505 | logger.critical( |
827 | 470 | self.buildqueue_record.builder.name) | 506 | "***** %s is CHROOTWAIT *****", |
828 | 471 | if send_notification: | 507 | self.buildqueue_record.builder.name) |
829 | 472 | self.notify() | 508 | |
830 | 509 | self._notify_if_appropriate(send_notification) | ||
831 | 473 | d = self.buildqueue_record.builder.cleanSlave() | 510 | d = self.buildqueue_record.builder.cleanSlave() |
834 | 474 | return d.addCallback( | 511 | return d.addCallback(self._destroy_buildqueue_record) |
833 | 475 | lambda x: self.buildqueue_record.destroySelf()) | ||
835 | 476 | 512 | ||
836 | 477 | d = self.storeBuildInfo(self, librarian, slave_status) | 513 | d = self.storeBuildInfo(self, librarian, slave_status) |
837 | 478 | return d.addCallback(build_info_stored) | 514 | return d.addCallback(build_info_stored) |
838 | 479 | 515 | ||
839 | 516 | def _reset_buildqueue_record(self, ignored_arg=None): | ||
840 | 517 | """Reset the `BuildQueue` record, in a write transaction.""" | ||
841 | 518 | transaction.commit() | ||
842 | 519 | with DatabaseTransactionPolicy(read_only=False): | ||
843 | 520 | self.buildqueue_record.reset() | ||
844 | 521 | transaction.commit() | ||
845 | 522 | |||
846 | 480 | def _handleStatus_BUILDERFAIL(self, librarian, slave_status, logger, | 523 | def _handleStatus_BUILDERFAIL(self, librarian, slave_status, logger, |
847 | 481 | send_notification): | 524 | send_notification): |
848 | 482 | """Handle builder failures. | 525 | """Handle builder failures. |
849 | @@ -490,11 +533,8 @@ | |||
850 | 490 | self.buildqueue_record.builder.failBuilder( | 533 | self.buildqueue_record.builder.failBuilder( |
851 | 491 | "Builder returned BUILDERFAIL when asked for its status") | 534 | "Builder returned BUILDERFAIL when asked for its status") |
852 | 492 | 535 | ||
853 | 493 | def build_info_stored(ignored): | ||
854 | 494 | # simply reset job | ||
855 | 495 | self.buildqueue_record.reset() | ||
856 | 496 | d = self.storeBuildInfo(self, librarian, slave_status) | 536 | d = self.storeBuildInfo(self, librarian, slave_status) |
858 | 497 | return d.addCallback(build_info_stored) | 537 | return d.addCallback(self._reset_buildqueue_record) |
859 | 498 | 538 | ||
860 | 499 | def _handleStatus_GIVENBACK(self, librarian, slave_status, logger, | 539 | def _handleStatus_GIVENBACK(self, librarian, slave_status, logger, |
861 | 500 | send_notification): | 540 | send_notification): |
862 | @@ -514,7 +554,7 @@ | |||
863 | 514 | # the next Paris Summit, infinity has some ideas about how | 554 | # the next Paris Summit, infinity has some ideas about how |
864 | 515 | # to use this content. For now we just ensure it's stored. | 555 | # to use this content. For now we just ensure it's stored. |
865 | 516 | d = self.buildqueue_record.builder.cleanSlave() | 556 | d = self.buildqueue_record.builder.cleanSlave() |
867 | 517 | self.buildqueue_record.reset() | 557 | self._reset_buildqueue_record() |
868 | 518 | return d | 558 | return d |
869 | 519 | 559 | ||
870 | 520 | d = self.storeBuildInfo(self, librarian, slave_status) | 560 | d = self.storeBuildInfo(self, librarian, slave_status) |
871 | 521 | 561 | ||
872 | === added file 'lib/lp/buildmaster/testing.py' | |||
873 | --- lib/lp/buildmaster/testing.py 1970-01-01 00:00:00 +0000 | |||
874 | +++ lib/lp/buildmaster/testing.py 2012-01-03 12:26:30 +0000 | |||
875 | @@ -0,0 +1,59 @@ | |||
876 | 1 | # Copyright 2011 Canonical Ltd. This software is licensed under the | ||
877 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | ||
878 | 3 | |||
879 | 4 | """Testing helpers for buildmaster code.""" | ||
880 | 5 | |||
881 | 6 | __metaclass__ = type | ||
882 | 7 | __all__ = [ | ||
883 | 8 | "BuilddManagerTestFixture", | ||
884 | 9 | ] | ||
885 | 10 | |||
886 | 11 | from contextlib import contextmanager | ||
887 | 12 | |||
888 | 13 | import fixtures | ||
889 | 14 | import transaction | ||
890 | 15 | |||
891 | 16 | from lp.services.database.transaction_policy import DatabaseTransactionPolicy | ||
892 | 17 | |||
893 | 18 | |||
894 | 19 | class BuilddManagerTestFixture(fixtures.Fixture): | ||
895 | 20 | """Helps provide an environment more like `BuilddManager` provides. | ||
896 | 21 | |||
897 | 22 | This mimics the default transaction access policy of `BuilddManager`, | ||
898 | 23 | though it can be configured with a different policy by passing in `store` | ||
899 | 24 | and/or `read_only`. See `BuilddManager.enterReadOnlyDatabasePolicy`. | ||
900 | 25 | |||
901 | 26 | Because this will shift into a read-only database transaction access mode, | ||
902 | 27 | individual tests that need to do more setup can use the `extraSetUp()` | ||
903 | 28 | context manager to temporarily shift back to a read-write mode. | ||
904 | 29 | """ | ||
905 | 30 | |||
906 | 31 | def __init__(self, store=None, read_only=True): | ||
907 | 32 | super(BuilddManagerTestFixture, self).__init__() | ||
908 | 33 | self.policy = DatabaseTransactionPolicy( | ||
909 | 34 | store=store, read_only=read_only) | ||
910 | 35 | |||
911 | 36 | def setUp(self): | ||
912 | 37 | # Commit everything done so far then shift into a read-only | ||
913 | 38 | # transaction access mode by default. | ||
914 | 39 | super(BuilddManagerTestFixture, self).setUp() | ||
915 | 40 | transaction.commit() | ||
916 | 41 | self.policy.__enter__() | ||
917 | 42 | self.addCleanup(self.policy.__exit__, None, None, None) | ||
918 | 43 | |||
919 | 44 | @staticmethod | ||
920 | 45 | @contextmanager | ||
921 | 46 | def extraSetUp(): | ||
922 | 47 | """Temporarily enter a read-write transaction to do extra setup. | ||
923 | 48 | |||
924 | 49 | For example: | ||
925 | 50 | |||
926 | 51 | with self.extraSetUp(): | ||
927 | 52 | removeSecurityProxy(self.build).date_finished = None | ||
928 | 53 | |||
929 | 54 | On exit it will commit the changes and restore the previous | ||
930 | 55 | transaction access mode. | ||
931 | 56 | """ | ||
932 | 57 | with DatabaseTransactionPolicy(read_only=False): | ||
933 | 58 | yield | ||
934 | 59 | transaction.commit() | ||
935 | 0 | 60 | ||
936 | === modified file 'lib/lp/buildmaster/tests/test_builder.py' | |||
937 | --- lib/lp/buildmaster/tests/test_builder.py 2011-12-30 06:14:56 +0000 | |||
938 | +++ lib/lp/buildmaster/tests/test_builder.py 2012-01-03 12:26:30 +0000 | |||
939 | @@ -1,4 +1,4 @@ | |||
941 | 1 | # Copyright 2009-2010 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2011 Canonical Ltd. This software is licensed under the |
942 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
943 | 3 | 3 | ||
944 | 4 | """Test Builder features.""" | 4 | """Test Builder features.""" |
945 | @@ -15,6 +15,7 @@ | |||
946 | 15 | AsynchronousDeferredRunTestForBrokenTwisted, | 15 | AsynchronousDeferredRunTestForBrokenTwisted, |
947 | 16 | SynchronousDeferredRunTest, | 16 | SynchronousDeferredRunTest, |
948 | 17 | ) | 17 | ) |
949 | 18 | import transaction | ||
950 | 18 | from twisted.internet.defer import ( | 19 | from twisted.internet.defer import ( |
951 | 19 | CancelledError, | 20 | CancelledError, |
952 | 20 | DeferredList, | 21 | DeferredList, |
953 | @@ -60,8 +61,10 @@ | |||
954 | 60 | TrivialBehavior, | 61 | TrivialBehavior, |
955 | 61 | WaitingSlave, | 62 | WaitingSlave, |
956 | 62 | ) | 63 | ) |
957 | 64 | from lp.registry.interfaces.pocket import PackagePublishingPocket | ||
958 | 63 | from lp.services.config import config | 65 | from lp.services.config import config |
959 | 64 | from lp.services.database.sqlbase import flush_database_updates | 66 | from lp.services.database.sqlbase import flush_database_updates |
960 | 67 | from lp.services.database.transaction_policy import DatabaseTransactionPolicy | ||
961 | 65 | from lp.services.job.interfaces.job import JobStatus | 68 | from lp.services.job.interfaces.job import JobStatus |
962 | 66 | from lp.services.log.logger import BufferLogger | 69 | from lp.services.log.logger import BufferLogger |
963 | 67 | from lp.services.webapp.interfaces import ( | 70 | from lp.services.webapp.interfaces import ( |
964 | @@ -152,7 +155,7 @@ | |||
965 | 152 | d = lostbuilding_builder.updateStatus(BufferLogger()) | 155 | d = lostbuilding_builder.updateStatus(BufferLogger()) |
966 | 153 | def check_slave_status(failure): | 156 | def check_slave_status(failure): |
967 | 154 | self.assertIn('abort', slave.call_log) | 157 | self.assertIn('abort', slave.call_log) |
969 | 155 | # 'Fault' comes from the LostBuildingBrokenSlave, this is | 158 | # 'Fault' comes from the LostBuildingBrokenSlave. This is |
970 | 156 | # just testing that the value is passed through. | 159 | # just testing that the value is passed through. |
971 | 157 | self.assertIsInstance(failure.value, xmlrpclib.Fault) | 160 | self.assertIsInstance(failure.value, xmlrpclib.Fault) |
972 | 158 | return d.addBoth(check_slave_status) | 161 | return d.addBoth(check_slave_status) |
973 | @@ -531,6 +534,26 @@ | |||
974 | 531 | # And the old_candidate is superseded: | 534 | # And the old_candidate is superseded: |
975 | 532 | self.assertEqual(BuildStatus.SUPERSEDED, build.status) | 535 | self.assertEqual(BuildStatus.SUPERSEDED, build.status) |
976 | 533 | 536 | ||
977 | 537 | def test_findBuildCandidate_postprocesses_in_read_write_policy(self): | ||
978 | 538 | # _findBuildCandidate invokes BuildFarmJob.postprocessCandidate, | ||
979 | 539 | # which may modify the database. This happens in a read-write | ||
980 | 540 | # transaction even if _findBuildCandidate itself runs in a | ||
981 | 541 | # read-only transaction policy. | ||
982 | 542 | |||
983 | 543 | # PackageBuildJob.postprocessCandidate will attempt to delete | ||
984 | 544 | # security builds. | ||
985 | 545 | pub = self.publisher.getPubSource( | ||
986 | 546 | sourcename="gedit", status=PackagePublishingStatus.PUBLISHED, | ||
987 | 547 | archive=self.factory.makeArchive(), | ||
988 | 548 | pocket=PackagePublishingPocket.SECURITY) | ||
989 | 549 | pub.createMissingBuilds() | ||
990 | 550 | transaction.commit() | ||
991 | 551 | with DatabaseTransactionPolicy(read_only=True): | ||
992 | 552 | removeSecurityProxy(self.frog_builder)._findBuildCandidate() | ||
993 | 553 | # The test is that this passes without a "transaction is | ||
994 | 554 | # read-only" error. | ||
995 | 555 | transaction.commit() | ||
996 | 556 | |||
997 | 534 | def test_acquireBuildCandidate_marks_building(self): | 557 | def test_acquireBuildCandidate_marks_building(self): |
998 | 535 | # acquireBuildCandidate() should call _findBuildCandidate and | 558 | # acquireBuildCandidate() should call _findBuildCandidate and |
999 | 536 | # mark the build as building. | 559 | # mark the build as building. |
1000 | 537 | 560 | ||
1001 | === modified file 'lib/lp/buildmaster/tests/test_manager.py' | |||
1002 | --- lib/lp/buildmaster/tests/test_manager.py 2012-01-01 02:58:52 +0000 | |||
1003 | +++ lib/lp/buildmaster/tests/test_manager.py 2012-01-03 12:26:30 +0000 | |||
1004 | @@ -1,8 +1,9 @@ | |||
1006 | 1 | # Copyright 2009-2010 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2011 Canonical Ltd. This software is licensed under the |
1007 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
1008 | 3 | 3 | ||
1009 | 4 | """Tests for the renovated slave scanner aka BuilddManager.""" | 4 | """Tests for the renovated slave scanner aka BuilddManager.""" |
1010 | 5 | 5 | ||
1011 | 6 | from collections import namedtuple | ||
1012 | 6 | import os | 7 | import os |
1013 | 7 | import signal | 8 | import signal |
1014 | 8 | import time | 9 | import time |
1015 | @@ -34,15 +35,18 @@ | |||
1016 | 34 | SlaveScanner, | 35 | SlaveScanner, |
1017 | 35 | ) | 36 | ) |
1018 | 36 | from lp.buildmaster.model.builder import Builder | 37 | from lp.buildmaster.model.builder import Builder |
1019 | 38 | from lp.buildmaster.model.packagebuild import PackageBuild | ||
1020 | 37 | from lp.buildmaster.tests.harness import BuilddManagerTestSetup | 39 | from lp.buildmaster.tests.harness import BuilddManagerTestSetup |
1021 | 38 | from lp.buildmaster.tests.mock_slaves import ( | 40 | from lp.buildmaster.tests.mock_slaves import ( |
1022 | 39 | BrokenSlave, | 41 | BrokenSlave, |
1023 | 40 | BuildingSlave, | 42 | BuildingSlave, |
1024 | 41 | make_publisher, | 43 | make_publisher, |
1025 | 42 | OkSlave, | 44 | OkSlave, |
1026 | 45 | WaitingSlave, | ||
1027 | 43 | ) | 46 | ) |
1028 | 44 | from lp.registry.interfaces.distribution import IDistributionSet | 47 | from lp.registry.interfaces.distribution import IDistributionSet |
1029 | 45 | from lp.services.config import config | 48 | from lp.services.config import config |
1030 | 49 | from lp.services.database.transaction_policy import DatabaseTransactionPolicy | ||
1031 | 46 | from lp.services.log.logger import BufferLogger | 50 | from lp.services.log.logger import BufferLogger |
1032 | 47 | from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet | 51 | from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet |
1033 | 48 | from lp.testing import ( | 52 | from lp.testing import ( |
1034 | @@ -58,7 +62,10 @@ | |||
1035 | 58 | LaunchpadZopelessLayer, | 62 | LaunchpadZopelessLayer, |
1036 | 59 | ZopelessDatabaseLayer, | 63 | ZopelessDatabaseLayer, |
1037 | 60 | ) | 64 | ) |
1039 | 61 | from lp.testing.sampledata import BOB_THE_BUILDER_NAME | 65 | from lp.testing.sampledata import ( |
1040 | 66 | BOB_THE_BUILDER_NAME, | ||
1041 | 67 | FROG_THE_BUILDER_NAME, | ||
1042 | 68 | ) | ||
1043 | 62 | 69 | ||
1044 | 63 | 70 | ||
1045 | 64 | class TestSlaveScannerScan(TestCase): | 71 | class TestSlaveScannerScan(TestCase): |
1046 | @@ -76,6 +83,8 @@ | |||
1047 | 76 | 'bob' builder. | 83 | 'bob' builder. |
1048 | 77 | """ | 84 | """ |
1049 | 78 | super(TestSlaveScannerScan, self).setUp() | 85 | super(TestSlaveScannerScan, self).setUp() |
1050 | 86 | self.read_only = DatabaseTransactionPolicy(read_only=True) | ||
1051 | 87 | |||
1052 | 79 | # Creating the required chroots needed for dispatching. | 88 | # Creating the required chroots needed for dispatching. |
1053 | 80 | test_publisher = make_publisher() | 89 | test_publisher = make_publisher() |
1054 | 81 | ubuntu = getUtility(IDistributionSet).getByName('ubuntu') | 90 | ubuntu = getUtility(IDistributionSet).getByName('ubuntu') |
1055 | @@ -83,6 +92,15 @@ | |||
1056 | 83 | test_publisher.setUpDefaultDistroSeries(hoary) | 92 | test_publisher.setUpDefaultDistroSeries(hoary) |
1057 | 84 | test_publisher.addFakeChroots() | 93 | test_publisher.addFakeChroots() |
1058 | 85 | 94 | ||
1059 | 95 | def _enterReadOnly(self): | ||
1060 | 96 | """Go into read-only transaction policy.""" | ||
1061 | 97 | self.read_only.__enter__() | ||
1062 | 98 | self.addCleanup(self._exitReadOnly) | ||
1063 | 99 | |||
1064 | 100 | def _exitReadOnly(self): | ||
1065 | 101 | """Leave read-only transaction policy.""" | ||
1066 | 102 | self.read_only.__exit__(None, None, None) | ||
1067 | 103 | |||
1068 | 86 | def _resetBuilder(self, builder): | 104 | def _resetBuilder(self, builder): |
1069 | 87 | """Reset the given builder and its job.""" | 105 | """Reset the given builder and its job.""" |
1070 | 88 | 106 | ||
1071 | @@ -93,6 +111,23 @@ | |||
1072 | 93 | 111 | ||
1073 | 94 | transaction.commit() | 112 | transaction.commit() |
1074 | 95 | 113 | ||
1075 | 114 | def getFreshBuilder(self, slave=None, name=BOB_THE_BUILDER_NAME, | ||
1076 | 115 | failure_count=0): | ||
1077 | 116 | """Return a builder. | ||
1078 | 117 | |||
1079 | 118 | The builder is taken from sample data, but reset to a usable state. | ||
1080 | 119 | Be careful: this is not a proper factory method. Identical calls | ||
1081 | 120 | return (and reset) the same builder. Don't rely on that though; | ||
1082 | 121 | maybe someday we'll have a proper factory here. | ||
1083 | 122 | """ | ||
1084 | 123 | if slave is None: | ||
1085 | 124 | slave = OkSlave() | ||
1086 | 125 | builder = getUtility(IBuilderSet)[name] | ||
1087 | 126 | self._resetBuilder(builder) | ||
1088 | 127 | builder.setSlaveForTesting(slave) | ||
1089 | 128 | builder.failure_count = failure_count | ||
1090 | 129 | return builder | ||
1091 | 130 | |||
1092 | 96 | def assertBuildingJob(self, job, builder, logtail=None): | 131 | def assertBuildingJob(self, job, builder, logtail=None): |
1093 | 97 | """Assert the given job is building on the given builder.""" | 132 | """Assert the given job is building on the given builder.""" |
1094 | 98 | from lp.services.job.interfaces.job import JobStatus | 133 | from lp.services.job.interfaces.job import JobStatus |
1095 | @@ -107,14 +142,14 @@ | |||
1096 | 107 | self.assertEqual(build.status, BuildStatus.BUILDING) | 142 | self.assertEqual(build.status, BuildStatus.BUILDING) |
1097 | 108 | self.assertEqual(job.logtail, logtail) | 143 | self.assertEqual(job.logtail, logtail) |
1098 | 109 | 144 | ||
1100 | 110 | def _getScanner(self, builder_name=None): | 145 | def _getScanner(self, builder_name=None, clock=None): |
1101 | 111 | """Instantiate a SlaveScanner object. | 146 | """Instantiate a SlaveScanner object. |
1102 | 112 | 147 | ||
1103 | 113 | Replace its default logging handler by a testing version. | 148 | Replace its default logging handler by a testing version. |
1104 | 114 | """ | 149 | """ |
1105 | 115 | if builder_name is None: | 150 | if builder_name is None: |
1106 | 116 | builder_name = BOB_THE_BUILDER_NAME | 151 | builder_name = BOB_THE_BUILDER_NAME |
1108 | 117 | scanner = SlaveScanner(builder_name, BufferLogger()) | 152 | scanner = SlaveScanner(builder_name, BufferLogger(), clock=clock) |
1109 | 118 | scanner.logger.name = 'slave-scanner' | 153 | scanner.logger.name = 'slave-scanner' |
1110 | 119 | 154 | ||
1111 | 120 | return scanner | 155 | return scanner |
1112 | @@ -130,17 +165,15 @@ | |||
1113 | 130 | def testScanDispatchForResetBuilder(self): | 165 | def testScanDispatchForResetBuilder(self): |
1114 | 131 | # A job gets dispatched to the sampledata builder after it's reset. | 166 | # A job gets dispatched to the sampledata builder after it's reset. |
1115 | 132 | 167 | ||
1123 | 133 | # Reset sampledata builder. | 168 | # Obtain a builder. Initialize failure count to 1 so that |
1124 | 134 | builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME] | 169 | # _checkDispatch can make sure that a successful dispatch resets |
1125 | 135 | self._resetBuilder(builder) | 170 | # the count to 0. |
1126 | 136 | builder.setSlaveForTesting(OkSlave()) | 171 | builder = self.getFreshBuilder(failure_count=1) |
1120 | 137 | # Set this to 1 here so that _checkDispatch can make sure it's | ||
1121 | 138 | # reset to 0 after a successful dispatch. | ||
1122 | 139 | builder.failure_count = 1 | ||
1127 | 140 | 172 | ||
1128 | 141 | # Run 'scan' and check its result. | 173 | # Run 'scan' and check its result. |
1129 | 142 | self.layer.txn.commit() | 174 | self.layer.txn.commit() |
1130 | 143 | self.layer.switchDbUser(config.builddmaster.dbuser) | 175 | self.layer.switchDbUser(config.builddmaster.dbuser) |
1131 | 176 | self._enterReadOnly() | ||
1132 | 144 | scanner = self._getScanner() | 177 | scanner = self._getScanner() |
1133 | 145 | d = defer.maybeDeferred(scanner.scan) | 178 | d = defer.maybeDeferred(scanner.scan) |
1134 | 146 | d.addCallback(self._checkDispatch, builder) | 179 | d.addCallback(self._checkDispatch, builder) |
1135 | @@ -153,20 +186,18 @@ | |||
1136 | 153 | to the asynchonous dispatcher and the builder remained active | 186 | to the asynchonous dispatcher and the builder remained active |
1137 | 154 | and IDLE. | 187 | and IDLE. |
1138 | 155 | """ | 188 | """ |
1140 | 156 | self.assertTrue(slave is None, "Unexpected slave.") | 189 | self.assertIs(None, slave, "Unexpected slave.") |
1141 | 157 | 190 | ||
1142 | 158 | builder = getUtility(IBuilderSet).get(builder.id) | 191 | builder = getUtility(IBuilderSet).get(builder.id) |
1143 | 159 | self.assertTrue(builder.builderok) | 192 | self.assertTrue(builder.builderok) |
1145 | 160 | self.assertTrue(builder.currentjob is None) | 193 | self.assertIs(None, builder.currentjob) |
1146 | 161 | 194 | ||
1147 | 162 | def testNoDispatchForMissingChroots(self): | 195 | def testNoDispatchForMissingChroots(self): |
1148 | 163 | # When a required chroot is not present the `scan` method | 196 | # When a required chroot is not present the `scan` method |
1149 | 164 | # should not return any `RecordingSlaves` to be processed | 197 | # should not return any `RecordingSlaves` to be processed |
1150 | 165 | # and the builder used should remain active and IDLE. | 198 | # and the builder used should remain active and IDLE. |
1151 | 166 | 199 | ||
1155 | 167 | # Reset sampledata builder. | 200 | builder = self.getFreshBuilder() |
1153 | 168 | builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME] | ||
1154 | 169 | self._resetBuilder(builder) | ||
1156 | 170 | 201 | ||
1157 | 171 | # Remove hoary/i386 chroot. | 202 | # Remove hoary/i386 chroot. |
1158 | 172 | login('foo.bar@canonical.com') | 203 | login('foo.bar@canonical.com') |
1159 | @@ -179,6 +210,7 @@ | |||
1160 | 179 | 210 | ||
1161 | 180 | # Run 'scan' and check its result. | 211 | # Run 'scan' and check its result. |
1162 | 181 | self.layer.switchDbUser(config.builddmaster.dbuser) | 212 | self.layer.switchDbUser(config.builddmaster.dbuser) |
1163 | 213 | self._enterReadOnly() | ||
1164 | 182 | scanner = self._getScanner() | 214 | scanner = self._getScanner() |
1165 | 183 | d = defer.maybeDeferred(scanner.singleCycle) | 215 | d = defer.maybeDeferred(scanner.singleCycle) |
1166 | 184 | d.addCallback(self._checkNoDispatch, builder) | 216 | d.addCallback(self._checkNoDispatch, builder) |
1167 | @@ -220,6 +252,7 @@ | |||
1168 | 220 | 252 | ||
1169 | 221 | # Run 'scan' and check its result. | 253 | # Run 'scan' and check its result. |
1170 | 222 | self.layer.switchDbUser(config.builddmaster.dbuser) | 254 | self.layer.switchDbUser(config.builddmaster.dbuser) |
1171 | 255 | self._enterReadOnly() | ||
1172 | 223 | scanner = self._getScanner() | 256 | scanner = self._getScanner() |
1173 | 224 | d = defer.maybeDeferred(scanner.scan) | 257 | d = defer.maybeDeferred(scanner.scan) |
1174 | 225 | d.addCallback(self._checkJobRescued, builder, job) | 258 | d.addCallback(self._checkJobRescued, builder, job) |
1175 | @@ -255,25 +288,27 @@ | |||
1176 | 255 | 288 | ||
1177 | 256 | # Run 'scan' and check its result. | 289 | # Run 'scan' and check its result. |
1178 | 257 | self.layer.switchDbUser(config.builddmaster.dbuser) | 290 | self.layer.switchDbUser(config.builddmaster.dbuser) |
1179 | 291 | self._enterReadOnly() | ||
1180 | 258 | scanner = self._getScanner() | 292 | scanner = self._getScanner() |
1181 | 259 | d = defer.maybeDeferred(scanner.scan) | 293 | d = defer.maybeDeferred(scanner.scan) |
1182 | 260 | d.addCallback(self._checkJobUpdated, builder, job) | 294 | d.addCallback(self._checkJobUpdated, builder, job) |
1183 | 261 | return d | 295 | return d |
1184 | 262 | 296 | ||
1185 | 263 | def test_scan_with_nothing_to_dispatch(self): | 297 | def test_scan_with_nothing_to_dispatch(self): |
1188 | 264 | factory = LaunchpadObjectFactory() | 298 | builder = self.factory.makeBuilder() |
1187 | 265 | builder = factory.makeBuilder() | ||
1189 | 266 | builder.setSlaveForTesting(OkSlave()) | 299 | builder.setSlaveForTesting(OkSlave()) |
1190 | 300 | transaction.commit() | ||
1191 | 301 | self._enterReadOnly() | ||
1192 | 267 | scanner = self._getScanner(builder_name=builder.name) | 302 | scanner = self._getScanner(builder_name=builder.name) |
1193 | 268 | d = scanner.scan() | 303 | d = scanner.scan() |
1194 | 269 | return d.addCallback(self._checkNoDispatch, builder) | 304 | return d.addCallback(self._checkNoDispatch, builder) |
1195 | 270 | 305 | ||
1196 | 271 | def test_scan_with_manual_builder(self): | 306 | def test_scan_with_manual_builder(self): |
1197 | 272 | # Reset sampledata builder. | 307 | # Reset sampledata builder. |
1201 | 273 | builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME] | 308 | builder = self.getFreshBuilder() |
1199 | 274 | self._resetBuilder(builder) | ||
1200 | 275 | builder.setSlaveForTesting(OkSlave()) | ||
1202 | 276 | builder.manual = True | 309 | builder.manual = True |
1203 | 310 | transaction.commit() | ||
1204 | 311 | self._enterReadOnly() | ||
1205 | 277 | scanner = self._getScanner() | 312 | scanner = self._getScanner() |
1206 | 278 | d = scanner.scan() | 313 | d = scanner.scan() |
1207 | 279 | d.addCallback(self._checkNoDispatch, builder) | 314 | d.addCallback(self._checkNoDispatch, builder) |
1208 | @@ -281,10 +316,10 @@ | |||
1209 | 281 | 316 | ||
1210 | 282 | def test_scan_with_not_ok_builder(self): | 317 | def test_scan_with_not_ok_builder(self): |
1211 | 283 | # Reset sampledata builder. | 318 | # Reset sampledata builder. |
1215 | 284 | builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME] | 319 | builder = self.getFreshBuilder() |
1213 | 285 | self._resetBuilder(builder) | ||
1214 | 286 | builder.setSlaveForTesting(OkSlave()) | ||
1216 | 287 | builder.builderok = False | 320 | builder.builderok = False |
1217 | 321 | transaction.commit() | ||
1218 | 322 | self._enterReadOnly() | ||
1219 | 288 | scanner = self._getScanner() | 323 | scanner = self._getScanner() |
1220 | 289 | d = scanner.scan() | 324 | d = scanner.scan() |
1221 | 290 | # Because the builder is not ok, we can't use _checkNoDispatch. | 325 | # Because the builder is not ok, we can't use _checkNoDispatch. |
1222 | @@ -293,25 +328,27 @@ | |||
1223 | 293 | return d | 328 | return d |
1224 | 294 | 329 | ||
1225 | 295 | def test_scan_of_broken_slave(self): | 330 | def test_scan_of_broken_slave(self): |
1230 | 296 | builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME] | 331 | builder = self.getFreshBuilder(slave=BrokenSlave()) |
1231 | 297 | self._resetBuilder(builder) | 332 | transaction.commit() |
1232 | 298 | builder.setSlaveForTesting(BrokenSlave()) | 333 | self._enterReadOnly() |
1229 | 299 | builder.failure_count = 0 | ||
1233 | 300 | scanner = self._getScanner(builder_name=builder.name) | 334 | scanner = self._getScanner(builder_name=builder.name) |
1234 | 301 | d = scanner.scan() | 335 | d = scanner.scan() |
1235 | 302 | return assert_fails_with(d, xmlrpclib.Fault) | 336 | return assert_fails_with(d, xmlrpclib.Fault) |
1236 | 303 | 337 | ||
1237 | 304 | def _assertFailureCounting(self, builder_count, job_count, | 338 | def _assertFailureCounting(self, builder_count, job_count, |
1238 | 305 | expected_builder_count, expected_job_count): | 339 | expected_builder_count, expected_job_count): |
1239 | 340 | # Avoid circular imports. | ||
1240 | 341 | from lp.buildmaster import manager as manager_module | ||
1241 | 342 | |||
1242 | 306 | # If scan() fails with an exception, failure_counts should be | 343 | # If scan() fails with an exception, failure_counts should be |
1243 | 307 | # incremented. What we do with the results of the failure | 344 | # incremented. What we do with the results of the failure |
1244 | 308 | # counts is tested below separately, this test just makes sure that | 345 | # counts is tested below separately, this test just makes sure that |
1245 | 309 | # scan() is setting the counts. | 346 | # scan() is setting the counts. |
1246 | 310 | def failing_scan(): | 347 | def failing_scan(): |
1247 | 311 | return defer.fail(Exception("fake exception")) | 348 | return defer.fail(Exception("fake exception")) |
1248 | 349 | |||
1249 | 312 | scanner = self._getScanner() | 350 | scanner = self._getScanner() |
1250 | 313 | scanner.scan = failing_scan | 351 | scanner.scan = failing_scan |
1251 | 314 | from lp.buildmaster import manager as manager_module | ||
1252 | 315 | self.patch(manager_module, 'assessFailureCounts', FakeMethod()) | 352 | self.patch(manager_module, 'assessFailureCounts', FakeMethod()) |
1253 | 316 | builder = getUtility(IBuilderSet)[scanner.builder_name] | 353 | builder = getUtility(IBuilderSet)[scanner.builder_name] |
1254 | 317 | 354 | ||
1255 | @@ -459,6 +496,60 @@ | |||
1256 | 459 | d.addCallback(check_cancelled, builder, buildqueue) | 496 | d.addCallback(check_cancelled, builder, buildqueue) |
1257 | 460 | return d | 497 | return d |
1258 | 461 | 498 | ||
1259 | 499 | def makeFakeFailure(self): | ||
1260 | 500 | """Produce a fake failure for use with SlaveScanner._scanFailed.""" | ||
1261 | 501 | FakeFailure = namedtuple('FakeFailure', ['getErrorMessage', 'check']) | ||
1262 | 502 | return FakeFailure( | ||
1263 | 503 | FakeMethod(self.factory.getUniqueString()), | ||
1264 | 504 | FakeMethod(True)) | ||
1265 | 505 | |||
1266 | 506 | def test_interleaved_success_and_failure_do_not_interfere(self): | ||
1267 | 507 | # It's possible for one builder to fail while another continues | ||
1268 | 508 | # to function properly. When that happens, the failed builder | ||
1269 | 509 | # may cause database changes to be rolled back. But that does | ||
1270 | 510 | # not affect the functioning builder. | ||
1271 | 511 | clock = task.Clock() | ||
1272 | 512 | |||
1273 | 513 | broken_builder = self.getFreshBuilder( | ||
1274 | 514 | slave=BrokenSlave(), name=BOB_THE_BUILDER_NAME) | ||
1275 | 515 | broken_scanner = self._getScanner(builder_name=broken_builder.name) | ||
1276 | 516 | good_builder = self.getFreshBuilder( | ||
1277 | 517 | slave=WaitingSlave(), name=FROG_THE_BUILDER_NAME) | ||
1278 | 518 | good_build = self.factory.makeBinaryPackageBuild( | ||
1279 | 519 | distroarchseries=self.factory.makeDistroArchSeries()) | ||
1280 | 520 | |||
1281 | 521 | # The good build is being handled by the good builder. | ||
1282 | 522 | buildqueue = good_build.queueBuild() | ||
1283 | 523 | buildqueue.builder = good_builder | ||
1284 | 524 | |||
1285 | 525 | removeSecurityProxy(good_build.build_farm_job).date_started = UTC_NOW | ||
1286 | 526 | |||
1287 | 527 | # The good builder requests information from a successful build, | ||
1288 | 528 | # and up receiving it, updates the build's metadata. | ||
1289 | 529 | # Our dependencies string goes into the build, and its | ||
1290 | 530 | # date_finished will be set. | ||
1291 | 531 | dependencies = self.factory.getUniqueString() | ||
1292 | 532 | PackageBuild.storeBuildInfo( | ||
1293 | 533 | good_build, None, {'dependencies': dependencies}) | ||
1294 | 534 | clock.advance(1) | ||
1295 | 535 | |||
1296 | 536 | # The broken scanner experiences a failure before the good | ||
1297 | 537 | # scanner is receiving its data. This aborts the ongoing | ||
1298 | 538 | # transaction. | ||
1299 | 539 | # As a somewhat weird example, if the builder changed its own | ||
1300 | 540 | # title, that change will be rolled back. | ||
1301 | 541 | original_broken_builder_title = broken_builder.title | ||
1302 | 542 | broken_builder.title = self.factory.getUniqueString() | ||
1303 | 543 | broken_scanner._scanFailed(self.makeFakeFailure()) | ||
1304 | 544 | |||
1305 | 545 | # The work done by the good scanner is retained. The | ||
1306 | 546 | # storeBuildInfo code committed it. | ||
1307 | 547 | self.assertEqual(dependencies, good_build.dependencies) | ||
1308 | 548 | self.assertIsNot(None, good_build.date_finished) | ||
1309 | 549 | |||
1310 | 550 | # The work done by the broken scanner is rolled back. | ||
1311 | 551 | self.assertEqual(original_broken_builder_title, broken_builder.title) | ||
1312 | 552 | |||
1313 | 462 | 553 | ||
1314 | 463 | class TestCancellationChecking(TestCaseWithFactory): | 554 | class TestCancellationChecking(TestCaseWithFactory): |
1315 | 464 | """Unit tests for the checkCancellation method.""" | 555 | """Unit tests for the checkCancellation method.""" |
1316 | 465 | 556 | ||
1317 | === modified file 'lib/lp/buildmaster/tests/test_packagebuild.py' | |||
1318 | --- lib/lp/buildmaster/tests/test_packagebuild.py 2011-12-30 06:14:56 +0000 | |||
1319 | +++ lib/lp/buildmaster/tests/test_packagebuild.py 2012-01-03 12:26:30 +0000 | |||
1320 | @@ -1,4 +1,4 @@ | |||
1322 | 1 | # Copyright 2010 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2010-2011 Canonical Ltd. This software is licensed under the |
1323 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
1324 | 3 | 3 | ||
1325 | 4 | """Tests for `IPackageBuild`.""" | 4 | """Tests for `IPackageBuild`.""" |
1326 | @@ -12,6 +12,7 @@ | |||
1327 | 12 | import tempfile | 12 | import tempfile |
1328 | 13 | 13 | ||
1329 | 14 | from storm.store import Store | 14 | from storm.store import Store |
1330 | 15 | from testtools.deferredruntest import AsynchronousDeferredRunTest | ||
1331 | 15 | from zope.component import getUtility | 16 | from zope.component import getUtility |
1332 | 16 | from zope.security.interfaces import Unauthorized | 17 | from zope.security.interfaces import Unauthorized |
1333 | 17 | from zope.security.proxy import removeSecurityProxy | 18 | from zope.security.proxy import removeSecurityProxy |
1334 | @@ -26,8 +27,10 @@ | |||
1335 | 26 | IPackageBuildSet, | 27 | IPackageBuildSet, |
1336 | 27 | IPackageBuildSource, | 28 | IPackageBuildSource, |
1337 | 28 | ) | 29 | ) |
1338 | 30 | from lp.buildmaster.model.builder import BuilderSlave | ||
1339 | 29 | from lp.buildmaster.model.buildfarmjob import BuildFarmJob | 31 | from lp.buildmaster.model.buildfarmjob import BuildFarmJob |
1340 | 30 | from lp.buildmaster.model.packagebuild import PackageBuild | 32 | from lp.buildmaster.model.packagebuild import PackageBuild |
1341 | 33 | from lp.buildmaster.testing import BuilddManagerTestFixture | ||
1342 | 31 | from lp.buildmaster.tests.mock_slaves import WaitingSlave | 34 | from lp.buildmaster.tests.mock_slaves import WaitingSlave |
1343 | 32 | from lp.registry.interfaces.pocket import PackagePublishingPocket | 35 | from lp.registry.interfaces.pocket import PackagePublishingPocket |
1344 | 33 | from lp.services.config import config | 36 | from lp.services.config import config |
1345 | @@ -281,12 +284,10 @@ | |||
1346 | 281 | 284 | ||
1347 | 282 | 285 | ||
1348 | 283 | class TestHandleStatusMixin: | 286 | class TestHandleStatusMixin: |
1353 | 284 | """Tests for `IPackageBuild`s handleStatus method. | 287 | """Tests for `IPackageBuild`s handleStatus method.""" |
1350 | 285 | |||
1351 | 286 | This should be run with a Trial TestCase. | ||
1352 | 287 | """ | ||
1354 | 288 | 288 | ||
1355 | 289 | layer = LaunchpadZopelessLayer | 289 | layer = LaunchpadZopelessLayer |
1356 | 290 | run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=20) | ||
1357 | 290 | 291 | ||
1358 | 291 | def makeBuild(self): | 292 | def makeBuild(self): |
1359 | 292 | """Allow classes to override the build with which the test runs.""" | 293 | """Allow classes to override the build with which the test runs.""" |
1360 | @@ -303,7 +304,7 @@ | |||
1361 | 303 | self.build.buildqueue_record.setDateStarted(UTC_NOW) | 304 | self.build.buildqueue_record.setDateStarted(UTC_NOW) |
1362 | 304 | self.slave = WaitingSlave('BuildStatus.OK') | 305 | self.slave = WaitingSlave('BuildStatus.OK') |
1363 | 305 | self.slave.valid_file_hashes.append('test_file_hash') | 306 | self.slave.valid_file_hashes.append('test_file_hash') |
1365 | 306 | builder.setSlaveForTesting(self.slave) | 307 | self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(self.slave)) |
1366 | 307 | 308 | ||
1367 | 308 | # We overwrite the buildmaster root to use a temp directory. | 309 | # We overwrite the buildmaster root to use a temp directory. |
1368 | 309 | tempdir = tempfile.mkdtemp() | 310 | tempdir = tempfile.mkdtemp() |
1369 | @@ -321,6 +322,8 @@ | |||
1370 | 321 | removeSecurityProxy(self.build).verifySuccessfulUpload = FakeMethod( | 322 | removeSecurityProxy(self.build).verifySuccessfulUpload = FakeMethod( |
1371 | 322 | result=True) | 323 | result=True) |
1372 | 323 | 324 | ||
1373 | 325 | self.useFixture(BuilddManagerTestFixture()) | ||
1374 | 326 | |||
1375 | 324 | def assertResultCount(self, count, result): | 327 | def assertResultCount(self, count, result): |
1376 | 325 | self.assertEquals( | 328 | self.assertEquals( |
1377 | 326 | 1, len(os.listdir(os.path.join(self.upload_root, result)))) | 329 | 1, len(os.listdir(os.path.join(self.upload_root, result)))) |
1378 | @@ -344,7 +347,7 @@ | |||
1379 | 344 | def got_status(ignored): | 347 | def got_status(ignored): |
1380 | 345 | self.assertEqual(BuildStatus.FAILEDTOUPLOAD, self.build.status) | 348 | self.assertEqual(BuildStatus.FAILEDTOUPLOAD, self.build.status) |
1381 | 346 | self.assertResultCount(0, "failed") | 349 | self.assertResultCount(0, "failed") |
1383 | 347 | self.assertIdentical(None, self.build.buildqueue_record) | 350 | self.assertIs(None, self.build.buildqueue_record) |
1384 | 348 | 351 | ||
1385 | 349 | d = self.build.handleStatus('OK', None, { | 352 | d = self.build.handleStatus('OK', None, { |
1386 | 350 | 'filemap': {'/tmp/myfile.py': 'test_file_hash'}, | 353 | 'filemap': {'/tmp/myfile.py': 'test_file_hash'}, |
1387 | @@ -365,7 +368,8 @@ | |||
1388 | 365 | 368 | ||
1389 | 366 | def test_handleStatus_OK_sets_build_log(self): | 369 | def test_handleStatus_OK_sets_build_log(self): |
1390 | 367 | # The build log is set during handleStatus. | 370 | # The build log is set during handleStatus. |
1392 | 368 | removeSecurityProxy(self.build).log = None | 371 | with BuilddManagerTestFixture.extraSetUp(): |
1393 | 372 | removeSecurityProxy(self.build).log = None | ||
1394 | 369 | self.assertEqual(None, self.build.log) | 373 | self.assertEqual(None, self.build.log) |
1395 | 370 | d = self.build.handleStatus('OK', None, { | 374 | d = self.build.handleStatus('OK', None, { |
1396 | 371 | 'filemap': {'myfile.py': 'test_file_hash'}, | 375 | 'filemap': {'myfile.py': 'test_file_hash'}, |
1397 | @@ -386,14 +390,10 @@ | |||
1398 | 386 | 390 | ||
1399 | 387 | def got_status(ignored): | 391 | def got_status(ignored): |
1400 | 388 | if expected_notification: | 392 | if expected_notification: |
1404 | 389 | self.failIf( | 393 | self.assertNotEqual( |
1405 | 390 | len(pop_notifications()) == 0, | 394 | 0, len(pop_notifications()), "No notifications received.") |
1403 | 391 | "No notifications received") | ||
1406 | 392 | else: | 395 | else: |
1411 | 393 | self.failIf( | 396 | self.assertContentEqual([], pop_notifications()) |
1408 | 394 | len(pop_notifications()) > 0, | ||
1409 | 395 | "Notifications received") | ||
1410 | 396 | |||
1412 | 397 | d = self.build.handleStatus(status, None, {}) | 397 | d = self.build.handleStatus(status, None, {}) |
1413 | 398 | return d.addCallback(got_status) | 398 | return d.addCallback(got_status) |
1414 | 399 | 399 | ||
1415 | @@ -408,7 +408,9 @@ | |||
1416 | 408 | 408 | ||
1417 | 409 | def test_date_finished_set(self): | 409 | def test_date_finished_set(self): |
1418 | 410 | # The date finished is updated during handleStatus_OK. | 410 | # The date finished is updated during handleStatus_OK. |
1420 | 411 | removeSecurityProxy(self.build).date_finished = None | 411 | with BuilddManagerTestFixture.extraSetUp(): |
1421 | 412 | removeSecurityProxy(self.build).date_finished = None | ||
1422 | 413 | |||
1423 | 412 | self.assertEqual(None, self.build.date_finished) | 414 | self.assertEqual(None, self.build.date_finished) |
1424 | 413 | d = self.build.handleStatus('OK', None, { | 415 | d = self.build.handleStatus('OK', None, { |
1425 | 414 | 'filemap': {'myfile.py': 'test_file_hash'}, | 416 | 'filemap': {'myfile.py': 'test_file_hash'}, |
1426 | 415 | 417 | ||
1427 | === modified file 'lib/lp/code/model/tests/test_sourcepackagerecipebuild.py' | |||
1428 | --- lib/lp/code/model/tests/test_sourcepackagerecipebuild.py 2012-01-01 02:58:52 +0000 | |||
1429 | +++ lib/lp/code/model/tests/test_sourcepackagerecipebuild.py 2012-01-03 12:26:30 +0000 | |||
1430 | @@ -13,14 +13,15 @@ | |||
1431 | 13 | 13 | ||
1432 | 14 | from pytz import utc | 14 | from pytz import utc |
1433 | 15 | from storm.locals import Store | 15 | from storm.locals import Store |
1434 | 16 | from testtools.deferredruntest import AsynchronousDeferredRunTest | ||
1435 | 16 | import transaction | 17 | import transaction |
1436 | 17 | from twisted.trial.unittest import TestCase as TrialTestCase | ||
1437 | 18 | from zope.component import getUtility | 18 | from zope.component import getUtility |
1438 | 19 | from zope.security.proxy import removeSecurityProxy | 19 | from zope.security.proxy import removeSecurityProxy |
1439 | 20 | 20 | ||
1440 | 21 | from lp.app.errors import NotFoundError | 21 | from lp.app.errors import NotFoundError |
1441 | 22 | from lp.buildmaster.enums import BuildStatus | 22 | from lp.buildmaster.enums import BuildStatus |
1442 | 23 | from lp.buildmaster.interfaces.buildqueue import IBuildQueue | 23 | from lp.buildmaster.interfaces.buildqueue import IBuildQueue |
1443 | 24 | from lp.buildmaster.model.builder import BuilderSlave | ||
1444 | 24 | from lp.buildmaster.model.buildfarmjob import BuildFarmJob | 25 | from lp.buildmaster.model.buildfarmjob import BuildFarmJob |
1445 | 25 | from lp.buildmaster.model.packagebuild import PackageBuild | 26 | from lp.buildmaster.model.packagebuild import PackageBuild |
1446 | 26 | from lp.buildmaster.tests.mock_slaves import WaitingSlave | 27 | from lp.buildmaster.tests.mock_slaves import WaitingSlave |
1447 | @@ -588,14 +589,11 @@ | |||
1448 | 588 | self.assertEquals(0, len(notifications)) | 589 | self.assertEquals(0, len(notifications)) |
1449 | 589 | 590 | ||
1450 | 590 | 591 | ||
1452 | 591 | class TestBuildNotifications(TrialTestCase): | 592 | class TestBuildNotifications(TestCaseWithFactory): |
1453 | 592 | 593 | ||
1454 | 593 | layer = LaunchpadZopelessLayer | 594 | layer = LaunchpadZopelessLayer |
1455 | 594 | 595 | ||
1460 | 595 | def setUp(self): | 596 | run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=20) |
1457 | 596 | super(TestBuildNotifications, self).setUp() | ||
1458 | 597 | from lp.testing.factory import LaunchpadObjectFactory | ||
1459 | 598 | self.factory = LaunchpadObjectFactory() | ||
1461 | 599 | 597 | ||
1462 | 600 | def prepare_build(self, fake_successful_upload=False): | 598 | def prepare_build(self, fake_successful_upload=False): |
1463 | 601 | queue_record = self.factory.makeSourcePackageRecipeBuildJob() | 599 | queue_record = self.factory.makeSourcePackageRecipeBuildJob() |
1464 | @@ -608,7 +606,7 @@ | |||
1465 | 608 | result=True) | 606 | result=True) |
1466 | 609 | queue_record.builder = self.factory.makeBuilder() | 607 | queue_record.builder = self.factory.makeBuilder() |
1467 | 610 | slave = WaitingSlave('BuildStatus.OK') | 608 | slave = WaitingSlave('BuildStatus.OK') |
1469 | 611 | queue_record.builder.setSlaveForTesting(slave) | 609 | self.patch(BuilderSlave, 'makeBuilderSlave', FakeMethod(slave)) |
1470 | 612 | return build | 610 | return build |
1471 | 613 | 611 | ||
1472 | 614 | def assertDeferredNotifyCount(self, status, build, expected_count): | 612 | def assertDeferredNotifyCount(self, status, build, expected_count): |
1473 | @@ -666,5 +664,5 @@ | |||
1474 | 666 | 664 | ||
1475 | 667 | 665 | ||
1476 | 668 | class TestHandleStatusForSPRBuild( | 666 | class TestHandleStatusForSPRBuild( |
1478 | 669 | MakeSPRecipeBuildMixin, TestHandleStatusMixin, TrialTestCase): | 667 | MakeSPRecipeBuildMixin, TestHandleStatusMixin, TestCaseWithFactory): |
1479 | 670 | """IPackageBuild.handleStatus works with SPRecipe builds.""" | 668 | """IPackageBuild.handleStatus works with SPRecipe builds.""" |
1480 | 671 | 669 | ||
1481 | === modified file 'lib/lp/services/database/tests/test_transaction_policy.py' | |||
1482 | --- lib/lp/services/database/tests/test_transaction_policy.py 2012-01-01 02:58:52 +0000 | |||
1483 | +++ lib/lp/services/database/tests/test_transaction_policy.py 2012-01-03 12:26:30 +0000 | |||
1484 | @@ -76,16 +76,26 @@ | |||
1485 | 76 | 76 | ||
1486 | 77 | self.assertRaises(InternalError, make_forbidden_update) | 77 | self.assertRaises(InternalError, make_forbidden_update) |
1487 | 78 | 78 | ||
1491 | 79 | def test_will_not_start_in_ongoing_transaction(self): | 79 | def test_will_not_go_read_only_when_read_write_transaction_ongoing(self): |
1492 | 80 | # You cannot enter a DatabaseTransactionPolicy while already in | 80 | # You cannot enter a read-only DatabaseTransactionPolicy while in an |
1493 | 81 | # a transaction. | 81 | # active read-write transaction. |
1494 | 82 | def enter_policy(): | 82 | def enter_policy(): |
1496 | 83 | with DatabaseTransactionPolicy(): | 83 | with DatabaseTransactionPolicy(read_only=True): |
1497 | 84 | pass | 84 | pass |
1498 | 85 | 85 | ||
1499 | 86 | self.writeToDatabase() | 86 | self.writeToDatabase() |
1500 | 87 | self.assertRaises(TransactionInProgress, enter_policy) | 87 | self.assertRaises(TransactionInProgress, enter_policy) |
1501 | 88 | 88 | ||
1502 | 89 | def test_will_go_read_write_when_read_write_transaction_ongoing(self): | ||
1503 | 90 | # You can enter a read-write DatabaseTransactionPolicy while in an | ||
1504 | 91 | # active read-write transaction. | ||
1505 | 92 | def enter_policy(): | ||
1506 | 93 | with DatabaseTransactionPolicy(read_only=False): | ||
1507 | 94 | pass | ||
1508 | 95 | |||
1509 | 96 | self.writeToDatabase() | ||
1510 | 97 | enter_policy() # No exception. | ||
1511 | 98 | |||
1512 | 89 | def test_successful_exit_requires_commit_or_abort(self): | 99 | def test_successful_exit_requires_commit_or_abort(self): |
1513 | 90 | # If a read-write policy exits normally (which would probably | 100 | # If a read-write policy exits normally (which would probably |
1514 | 91 | # indicate successful completion of its code), it requires that | 101 | # indicate successful completion of its code), it requires that |
1515 | 92 | 102 | ||
1516 | === modified file 'lib/lp/services/database/transaction_policy.py' | |||
1517 | --- lib/lp/services/database/transaction_policy.py 2011-12-30 06:14:56 +0000 | |||
1518 | +++ lib/lp/services/database/transaction_policy.py 2012-01-03 12:26:30 +0000 | |||
1519 | @@ -92,9 +92,18 @@ | |||
1520 | 92 | 92 | ||
1521 | 93 | :raise TransactionInProgress: if a transaction was already ongoing. | 93 | :raise TransactionInProgress: if a transaction was already ongoing. |
1522 | 94 | """ | 94 | """ |
1525 | 95 | self._checkNoTransaction( | 95 | # We must check the transaction status before checking the current |
1526 | 96 | "Entered DatabaseTransactionPolicy while in a transaction.") | 96 | # policy because getting the policy causes a status change. |
1527 | 97 | in_transaction = self._isInTransaction() | ||
1528 | 97 | self.previous_policy = self._getCurrentPolicy() | 98 | self.previous_policy = self._getCurrentPolicy() |
1529 | 99 | # If the current transaction is read-write and we're moving to | ||
1530 | 100 | # read-only then we should check for a transaction. If the current | ||
1531 | 101 | # policy is read-only then we don't care if a transaction is in | ||
1532 | 102 | # progress. | ||
1533 | 103 | if in_transaction and self.read_only and not self.previous_policy: | ||
1534 | 104 | raise TransactionInProgress( | ||
1535 | 105 | "Attempting to enter a read-only transaction while holding " | ||
1536 | 106 | "open a read-write transaction.") | ||
1537 | 98 | self._setPolicy(self.read_only) | 107 | self._setPolicy(self.read_only) |
1538 | 99 | # Commit should include the policy itself. If this breaks | 108 | # Commit should include the policy itself. If this breaks |
1539 | 100 | # because the transaction was already in a failed state before | 109 | # because the transaction was already in a failed state before |
1540 | @@ -133,8 +142,11 @@ | |||
1541 | 133 | def _isInTransaction(self): | 142 | def _isInTransaction(self): |
1542 | 134 | """Is our store currently in a transaction?""" | 143 | """Is our store currently in a transaction?""" |
1543 | 135 | pg_connection = self.store._connection._raw_connection | 144 | pg_connection = self.store._connection._raw_connection |
1546 | 136 | status = pg_connection.get_transaction_status() | 145 | if pg_connection is None: |
1547 | 137 | return status != TRANSACTION_STATUS_IDLE | 146 | return False |
1548 | 147 | else: | ||
1549 | 148 | status = pg_connection.get_transaction_status() | ||
1550 | 149 | return status != TRANSACTION_STATUS_IDLE | ||
1551 | 138 | 150 | ||
1552 | 139 | def _checkNoTransaction(self, error_msg): | 151 | def _checkNoTransaction(self, error_msg): |
1553 | 140 | """Verify that no transaction is ongoing. | 152 | """Verify that no transaction is ongoing. |
1554 | 141 | 153 | ||
1555 | === modified file 'lib/lp/soyuz/tests/test_binarypackagebuild.py' | |||
1556 | --- lib/lp/soyuz/tests/test_binarypackagebuild.py 2012-01-01 02:58:52 +0000 | |||
1557 | +++ lib/lp/soyuz/tests/test_binarypackagebuild.py 2012-01-03 12:26:30 +0000 | |||
1558 | @@ -1,4 +1,4 @@ | |||
1560 | 1 | # Copyright 2009-2010 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2011 Canonical Ltd. This software is licensed under the |
1561 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
1562 | 3 | 3 | ||
1563 | 4 | """Test Build features.""" | 4 | """Test Build features.""" |
1564 | @@ -10,7 +10,6 @@ | |||
1565 | 10 | 10 | ||
1566 | 11 | import pytz | 11 | import pytz |
1567 | 12 | from storm.store import Store | 12 | from storm.store import Store |
1568 | 13 | from twisted.trial.unittest import TestCase as TrialTestCase | ||
1569 | 14 | from zope.component import getUtility | 13 | from zope.component import getUtility |
1570 | 15 | from zope.security.proxy import removeSecurityProxy | 14 | from zope.security.proxy import removeSecurityProxy |
1571 | 16 | 15 | ||
1572 | @@ -18,6 +17,7 @@ | |||
1573 | 18 | from lp.buildmaster.interfaces.builder import IBuilderSet | 17 | from lp.buildmaster.interfaces.builder import IBuilderSet |
1574 | 19 | from lp.buildmaster.interfaces.buildqueue import IBuildQueue | 18 | from lp.buildmaster.interfaces.buildqueue import IBuildQueue |
1575 | 20 | from lp.buildmaster.interfaces.packagebuild import IPackageBuild | 19 | from lp.buildmaster.interfaces.packagebuild import IPackageBuild |
1576 | 20 | from lp.buildmaster.model.builder import BuilderSlave | ||
1577 | 21 | from lp.buildmaster.model.buildqueue import BuildQueue | 21 | from lp.buildmaster.model.buildqueue import BuildQueue |
1578 | 22 | from lp.buildmaster.tests.mock_slaves import WaitingSlave | 22 | from lp.buildmaster.tests.mock_slaves import WaitingSlave |
1579 | 23 | from lp.buildmaster.tests.test_packagebuild import ( | 23 | from lp.buildmaster.tests.test_packagebuild import ( |
1580 | @@ -48,6 +48,7 @@ | |||
1581 | 48 | logout, | 48 | logout, |
1582 | 49 | TestCaseWithFactory, | 49 | TestCaseWithFactory, |
1583 | 50 | ) | 50 | ) |
1584 | 51 | from lp.testing.fakemethod import FakeMethod | ||
1585 | 51 | from lp.testing.layers import ( | 52 | from lp.testing.layers import ( |
1586 | 52 | DatabaseFunctionalLayer, | 53 | DatabaseFunctionalLayer, |
1587 | 53 | LaunchpadZopelessLayer, | 54 | LaunchpadZopelessLayer, |
1588 | @@ -522,7 +523,9 @@ | |||
1589 | 522 | self.build = gedit_src_hist.createMissingBuilds()[0] | 523 | self.build = gedit_src_hist.createMissingBuilds()[0] |
1590 | 523 | 524 | ||
1591 | 524 | self.builder = self.factory.makeBuilder() | 525 | self.builder = self.factory.makeBuilder() |
1593 | 525 | self.builder.setSlaveForTesting(WaitingSlave('BuildStatus.OK')) | 526 | self.patch( |
1594 | 527 | BuilderSlave, 'makeBuilderSlave', | ||
1595 | 528 | FakeMethod(WaitingSlave('BuildStatus.OK'))) | ||
1596 | 526 | self.build.buildqueue_record.markAsBuilding(self.builder) | 529 | self.build.buildqueue_record.markAsBuilding(self.builder) |
1597 | 527 | 530 | ||
1598 | 528 | def testDependencies(self): | 531 | def testDependencies(self): |
1599 | @@ -568,7 +571,7 @@ | |||
1600 | 568 | 571 | ||
1601 | 569 | 572 | ||
1602 | 570 | class TestHandleStatusForBinaryPackageBuild( | 573 | class TestHandleStatusForBinaryPackageBuild( |
1604 | 571 | MakeBinaryPackageBuildMixin, TestHandleStatusMixin, TrialTestCase): | 574 | MakeBinaryPackageBuildMixin, TestHandleStatusMixin, TestCaseWithFactory): |
1605 | 572 | """IPackageBuild.handleStatus works with binary builds.""" | 575 | """IPackageBuild.handleStatus works with binary builds.""" |
1606 | 573 | 576 | ||
1607 | 574 | 577 | ||
1608 | 575 | 578 | ||
1609 | === modified file 'lib/lp/translations/model/translationtemplatesbuildbehavior.py' | |||
1610 | --- lib/lp/translations/model/translationtemplatesbuildbehavior.py 2012-01-01 02:58:52 +0000 | |||
1611 | +++ lib/lp/translations/model/translationtemplatesbuildbehavior.py 2012-01-03 12:26:30 +0000 | |||
1612 | @@ -1,4 +1,4 @@ | |||
1614 | 1 | # Copyright 2010 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2010-2011 Canonical Ltd. This software is licensed under the |
1615 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
1616 | 3 | 3 | ||
1617 | 4 | """An `IBuildFarmJobBehavior` for `TranslationTemplatesBuildJob`. | 4 | """An `IBuildFarmJobBehavior` for `TranslationTemplatesBuildJob`. |
1618 | @@ -16,6 +16,7 @@ | |||
1619 | 16 | import tempfile | 16 | import tempfile |
1620 | 17 | 17 | ||
1621 | 18 | import pytz | 18 | import pytz |
1622 | 19 | import transaction | ||
1623 | 19 | from twisted.internet import defer | 20 | from twisted.internet import defer |
1624 | 20 | from zope.component import getUtility | 21 | from zope.component import getUtility |
1625 | 21 | from zope.interface import implements | 22 | from zope.interface import implements |
1626 | @@ -28,6 +29,7 @@ | |||
1627 | 28 | ) | 29 | ) |
1628 | 29 | from lp.buildmaster.model.buildfarmjobbehavior import BuildFarmJobBehaviorBase | 30 | from lp.buildmaster.model.buildfarmjobbehavior import BuildFarmJobBehaviorBase |
1629 | 30 | from lp.registry.interfaces.productseries import IProductSeriesSet | 31 | from lp.registry.interfaces.productseries import IProductSeriesSet |
1630 | 32 | from lp.services.database.transaction_policy import DatabaseTransactionPolicy | ||
1631 | 31 | from lp.translations.interfaces.translationimportqueue import ( | 33 | from lp.translations.interfaces.translationimportqueue import ( |
1632 | 32 | ITranslationImportQueue, | 34 | ITranslationImportQueue, |
1633 | 33 | ) | 35 | ) |
1634 | @@ -132,13 +134,16 @@ | |||
1635 | 132 | def storeBuildInfo(build, queue_item, build_status): | 134 | def storeBuildInfo(build, queue_item, build_status): |
1636 | 133 | """See `IPackageBuild`.""" | 135 | """See `IPackageBuild`.""" |
1637 | 134 | def got_log(lfa_id): | 136 | def got_log(lfa_id): |
1645 | 135 | build.build.log = lfa_id | 137 | transaction.commit() |
1646 | 136 | build.build.builder = queue_item.builder | 138 | with DatabaseTransactionPolicy(read_only=False): |
1647 | 137 | build.build.date_started = queue_item.date_started | 139 | build.build.log = lfa_id |
1648 | 138 | # XXX cprov 20060615 bug=120584: Currently buildduration includes | 140 | build.build.builder = queue_item.builder |
1649 | 139 | # the scanner latency, it should really be asking the slave for | 141 | build.build.date_started = queue_item.date_started |
1650 | 140 | # the duration spent building locally. | 142 | # XXX cprov 20060615 bug=120584: Currently buildduration |
1651 | 141 | build.build.date_finished = datetime.datetime.now(pytz.UTC) | 143 | # includes the scanner latency. It should really be |
1652 | 144 | # asking the slave for the duration spent building locally. | ||
1653 | 145 | build.build.date_finished = datetime.datetime.now(pytz.UTC) | ||
1654 | 146 | transaction.commit() | ||
1655 | 142 | 147 | ||
1656 | 143 | d = build.getLogFromSlave(build, queue_item) | 148 | d = build.getLogFromSlave(build, queue_item) |
1657 | 144 | return d.addCallback(got_log) | 149 | return d.addCallback(got_log) |
<bigjools> allenap: in the tests in your branch, it's probably worth refactoring the bit that sets properties on objects in a r/w transaction us_OK_sets_ build_log?
<allenap> bigjools: Erm, which bit?
<allenap> bigjools: Like in test_handleStat
<bigjools> allenap: line 72/83 of the diff
<bigjools> allenap: I suspect we'll need to do that a lot more in the future
<allenap> bigjools: I don't know what a better way would be. I could instead enter read-only mode in each test individually (via a fixture) I guess.
<bigjools> allenap: I was thinking just a test helper
<bigjools> like setattr
<bigjools> but does the whole transactionny thing
<allenap> bigjools: With the removeSecurityProxy thing too I assume.
<bigjools> allenap: no, the caller can do that
<allenap> bigjools: Okay, I think I have a cool way to do that.