Merge lp:~mars/launchpad/test-ghost-update-2 into lp:~launchpad/launchpad/ghost-line
- test-ghost-update-2
- Merge into ghost-line
Proposed by
Māris Fogels
Status: | Merged |
---|---|
Approved by: | Māris Fogels |
Approved revision: | 11787 |
Merged at revision: | 11784 |
Proposed branch: | lp:~mars/launchpad/test-ghost-update-2 |
Merge into: | lp:~launchpad/launchpad/ghost-line |
Diff against target: |
7763 lines (+3923/-2238) 36 files modified
lib/canonical/launchpad/icing/style-3-0.css.in (+6/-0) lib/lp/app/browser/configure.zcml (+6/-0) lib/lp/app/browser/linkchecker.py (+77/-0) lib/lp/app/browser/stringformatter.py (+3/-1) lib/lp/app/browser/tests/test_linkchecker.py (+83/-0) lib/lp/app/configure.zcml (+0/-14) lib/lp/app/doc/displaying-paragraphs-of-text.txt (+11/-11) lib/lp/app/javascript/lp-links.js (+105/-0) lib/lp/app/templates/base-layout-macros.pt (+9/-0) lib/lp/bugs/windmill/tests/test_bug_commenting.py (+1/-1) lib/lp/buildmaster/doc/builder.txt (+118/-2) lib/lp/buildmaster/interfaces/builder.py (+62/-83) lib/lp/buildmaster/manager.py (+468/-204) lib/lp/buildmaster/model/builder.py (+224/-240) lib/lp/buildmaster/model/buildfarmjobbehavior.py (+52/-60) lib/lp/buildmaster/model/packagebuild.py (+0/-6) lib/lp/buildmaster/tests/mock_slaves.py (+32/-157) lib/lp/buildmaster/tests/test_builder.py (+154/-582) lib/lp/buildmaster/tests/test_manager.py (+782/-248) lib/lp/buildmaster/tests/test_packagebuild.py (+0/-12) lib/lp/code/model/recipebuilder.py (+28/-32) lib/lp/code/windmill/tests/test_branch_broken_links.py (+113/-0) lib/lp/code/windmill/tests/test_branchmergeproposal_review.py (+1/-1) lib/lp/soyuz/browser/tests/test_builder_views.py (+1/-1) lib/lp/soyuz/doc/buildd-dispatching.txt (+371/-0) lib/lp/soyuz/doc/buildd-slavescanner.txt (+876/-0) lib/lp/soyuz/model/binarypackagebuildbehavior.py (+41/-59) lib/lp/soyuz/tests/test_binarypackagebuildbehavior.py (+8/-290) lib/lp/soyuz/tests/test_doc.py (+6/-0) lib/lp/testing/factory.py (+2/-8) lib/lp/translations/doc/translationtemplatesbuildbehavior.txt (+114/-0) lib/lp/translations/model/translationtemplatesbuildbehavior.py (+14/-20) lib/lp/translations/stories/buildfarm/xx-build-summary.txt (+1/-1) lib/lp/translations/tests/test_translationtemplatesbuildbehavior.py (+153/-202) lib/lp_sitecustomize.py (+0/-3) utilities/migrater/file-ownership.txt (+1/-0) |
To merge this branch: | bzr merge lp:~mars/launchpad/test-ghost-update-2 |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Māris Fogels (community) | Approve | ||
Review via email: mp+42515@code.launchpad.net |
Commit message
Testing the bundle-merge tarmac command
Description of the change
Testing the bundle-merge Tarmac command
To post a comment you must log in.
Revision history for this message
Māris Fogels (mars) : | # |
review:
Approve
Revision history for this message
Launchpad PQM Bot (launchpad-pqm) wrote : | # |
- 11786. By Māris Fogels
-
Merged r11806
- 11787. By Māris Fogels
-
Added a file for testing
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'lib/canonical/launchpad/icing/style-3-0.css.in' | |||
2 | --- lib/canonical/launchpad/icing/style-3-0.css.in 2010-09-23 11:17:45 +0000 | |||
3 | +++ lib/canonical/launchpad/icing/style-3-0.css.in 2010-12-07 16:29:13 +0000 | |||
4 | @@ -284,6 +284,12 @@ | |||
5 | 284 | a.help.icon, a.sprite.maybe.help { | 284 | a.help.icon, a.sprite.maybe.help { |
6 | 285 | border: none; | 285 | border: none; |
7 | 286 | } | 286 | } |
8 | 287 | a.invalid-link { | ||
9 | 288 | disabled: True; | ||
10 | 289 | color: #909090; | ||
11 | 290 | text-decoration: none; | ||
12 | 291 | cursor: default; | ||
13 | 292 | } | ||
14 | 287 | img, a img { | 293 | img, a img { |
15 | 288 | /* No border on images that are links. */ | 294 | /* No border on images that are links. */ |
16 | 289 | border: none; | 295 | border: none; |
17 | 290 | 296 | ||
18 | === modified file 'lib/lp/app/browser/configure.zcml' | |||
19 | --- lib/lp/app/browser/configure.zcml 2010-10-15 01:27:04 +0000 | |||
20 | +++ lib/lp/app/browser/configure.zcml 2010-12-07 16:29:13 +0000 | |||
21 | @@ -98,6 +98,12 @@ | |||
22 | 98 | template="../templates/launchpad-search-form.pt" | 98 | template="../templates/launchpad-search-form.pt" |
23 | 99 | permission="zope.Public" /> | 99 | permission="zope.Public" /> |
24 | 100 | 100 | ||
25 | 101 | <browser:page | ||
26 | 102 | for="*" | ||
27 | 103 | name="+check-links" | ||
28 | 104 | class="lp.app.browser.linkchecker.LinkCheckerAPI" | ||
29 | 105 | permission="zope.Public"/> | ||
30 | 106 | |||
31 | 101 | <!-- TALES namespaces. --> | 107 | <!-- TALES namespaces. --> |
32 | 102 | 108 | ||
33 | 103 | <!-- TALES lp: namespace (should be deprecated) --> | 109 | <!-- TALES lp: namespace (should be deprecated) --> |
34 | 104 | 110 | ||
35 | === added file 'lib/lp/app/browser/linkchecker.py' | |||
36 | --- lib/lp/app/browser/linkchecker.py 1970-01-01 00:00:00 +0000 | |||
37 | +++ lib/lp/app/browser/linkchecker.py 2010-12-07 16:29:13 +0000 | |||
38 | @@ -0,0 +1,77 @@ | |||
39 | 1 | # Copyright 2009 Canonical Ltd. This software is licensed under the | ||
40 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | ||
41 | 3 | |||
42 | 4 | # pylint: disable-msg=E0211,E0213 | ||
43 | 5 | |||
44 | 6 | __metaclass__ = type | ||
45 | 7 | __all__ = [ | ||
46 | 8 | 'LinkCheckerAPI', | ||
47 | 9 | ] | ||
48 | 10 | |||
49 | 11 | import simplejson | ||
50 | 12 | from zope.component import getUtility | ||
51 | 13 | |||
52 | 14 | from lp.app.errors import NotFoundError | ||
53 | 15 | from lp.code.errors import ( | ||
54 | 16 | CannotHaveLinkedBranch, | ||
55 | 17 | InvalidNamespace, | ||
56 | 18 | NoLinkedBranch, | ||
57 | 19 | NoSuchBranch, | ||
58 | 20 | ) | ||
59 | 21 | from lp.code.interfaces.branchlookup import IBranchLookup | ||
60 | 22 | from lp.registry.interfaces.product import InvalidProductName | ||
61 | 23 | |||
62 | 24 | |||
63 | 25 | class LinkCheckerAPI: | ||
64 | 26 | """Validates Launchpad shortcut links. | ||
65 | 27 | |||
66 | 28 | This class provides the endpoint of an Ajax call to .../+check-links. | ||
67 | 29 | When invoked with a collection of links harvested from a page, it will | ||
68 | 30 | check the validity of each one and send a response containing those that | ||
69 | 31 | are invalid. Javascript on the page will set the style of invalid links to | ||
70 | 32 | something appropriate. | ||
71 | 33 | |||
72 | 34 | This initial implementation supports processing links like the following: | ||
73 | 35 | /+branch/foo/bar | ||
74 | 36 | |||
75 | 37 | The implementation can easily be extended to handle other forms by | ||
76 | 38 | providing a method to handle the link type extracted from the json | ||
77 | 39 | request. | ||
78 | 40 | """ | ||
79 | 41 | |||
80 | 42 | def __init__(self, context, request): | ||
81 | 43 | # We currently only use the request. | ||
82 | 44 | # self.context = context | ||
83 | 45 | self.request = request | ||
84 | 46 | |||
85 | 47 | # Each link type has it's own validation method. | ||
86 | 48 | self.link_checkers = dict( | ||
87 | 49 | branch_links=self.check_branch_links, | ||
88 | 50 | ) | ||
89 | 51 | |||
90 | 52 | def __call__(self): | ||
91 | 53 | result = {} | ||
92 | 54 | links_to_check_data = self.request.get('link_hrefs') | ||
93 | 55 | links_to_check = simplejson.loads(links_to_check_data) | ||
94 | 56 | |||
95 | 57 | for link_type in links_to_check: | ||
96 | 58 | links = links_to_check[link_type] | ||
97 | 59 | invalid_links = self.link_checkers[link_type](links) | ||
98 | 60 | result['invalid_'+link_type] = invalid_links | ||
99 | 61 | |||
100 | 62 | self.request.response.setHeader('Content-type', 'application/json') | ||
101 | 63 | return simplejson.dumps(result) | ||
102 | 64 | |||
103 | 65 | def check_branch_links(self, links): | ||
104 | 66 | """Check links of the form /+branch/foo/bar""" | ||
105 | 67 | invalid_links = [] | ||
106 | 68 | branch_lookup = getUtility(IBranchLookup) | ||
107 | 69 | for link in links: | ||
108 | 70 | path = link.strip('/')[len('+branch/'):] | ||
109 | 71 | try: | ||
110 | 72 | branch_lookup.getByLPPath(path) | ||
111 | 73 | except (CannotHaveLinkedBranch, InvalidNamespace, | ||
112 | 74 | InvalidProductName, NoLinkedBranch, NoSuchBranch, | ||
113 | 75 | NotFoundError): | ||
114 | 76 | invalid_links.append(link) | ||
115 | 77 | return invalid_links | ||
116 | 0 | 78 | ||
117 | === modified file 'lib/lp/app/browser/stringformatter.py' | |||
118 | --- lib/lp/app/browser/stringformatter.py 2010-09-25 14:29:32 +0000 | |||
119 | +++ lib/lp/app/browser/stringformatter.py 2010-12-07 16:29:13 +0000 | |||
120 | @@ -274,7 +274,9 @@ | |||
121 | 274 | return FormattersAPI._linkify_bug_number( | 274 | return FormattersAPI._linkify_bug_number( |
122 | 275 | lp_url, path, trailers) | 275 | lp_url, path, trailers) |
123 | 276 | url = '/+branch/%s' % path | 276 | url = '/+branch/%s' % path |
125 | 277 | return '<a href="%s">%s</a>%s' % ( | 277 | # Mark the links with a 'branch-short-link' class so they can be |
126 | 278 | # harvested and validated when the page is rendered. | ||
127 | 279 | return '<a href="%s" class="branch-short-link">%s</a>%s' % ( | ||
128 | 278 | cgi.escape(url, quote=True), | 280 | cgi.escape(url, quote=True), |
129 | 279 | cgi.escape(lp_url), | 281 | cgi.escape(lp_url), |
130 | 280 | cgi.escape(trailers)) | 282 | cgi.escape(trailers)) |
131 | 281 | 283 | ||
132 | === added file 'lib/lp/app/browser/tests/test_linkchecker.py' | |||
133 | --- lib/lp/app/browser/tests/test_linkchecker.py 1970-01-01 00:00:00 +0000 | |||
134 | +++ lib/lp/app/browser/tests/test_linkchecker.py 2010-12-07 16:29:13 +0000 | |||
135 | @@ -0,0 +1,83 @@ | |||
136 | 1 | # Copyright 2010 Canonical Ltd. This software is licensed under the | ||
137 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | ||
138 | 3 | |||
139 | 4 | """Unit tests for the LinkCheckerAPI.""" | ||
140 | 5 | |||
141 | 6 | __metaclass__ = type | ||
142 | 7 | |||
143 | 8 | from random import shuffle | ||
144 | 9 | |||
145 | 10 | import simplejson | ||
146 | 11 | from zope.security.proxy import removeSecurityProxy | ||
147 | 12 | |||
148 | 13 | from canonical.launchpad.webapp.servers import LaunchpadTestRequest | ||
149 | 14 | from canonical.testing.layers import DatabaseFunctionalLayer | ||
150 | 15 | from lp.app.browser.linkchecker import LinkCheckerAPI | ||
151 | 16 | from lp.testing import TestCaseWithFactory | ||
152 | 17 | |||
153 | 18 | |||
154 | 19 | class TestLinkCheckerAPI(TestCaseWithFactory): | ||
155 | 20 | |||
156 | 21 | layer = DatabaseFunctionalLayer | ||
157 | 22 | |||
158 | 23 | BRANCH_URL_TEMPLATE = '/+branch/%s' | ||
159 | 24 | |||
160 | 25 | def check_invalid_links(self, result_json, link_type, invalid_links): | ||
161 | 26 | link_dict = simplejson.loads(result_json) | ||
162 | 27 | links_to_check = link_dict[link_type] | ||
163 | 28 | self.assertEqual(len(invalid_links), len(links_to_check)) | ||
164 | 29 | self.assertEqual(set(invalid_links), set(links_to_check)) | ||
165 | 30 | |||
166 | 31 | def make_valid_links(self): | ||
167 | 32 | branch = self.factory.makeProductBranch() | ||
168 | 33 | valid_branch_url = self.BRANCH_URL_TEMPLATE % branch.unique_name | ||
169 | 34 | product = self.factory.makeProduct() | ||
170 | 35 | product_branch = self.factory.makeProductBranch(product=product) | ||
171 | 36 | removeSecurityProxy(product).development_focus.branch = product_branch | ||
172 | 37 | valid_product_url = self.BRANCH_URL_TEMPLATE % product.name | ||
173 | 38 | |||
174 | 39 | return [ | ||
175 | 40 | valid_branch_url, | ||
176 | 41 | valid_product_url, | ||
177 | 42 | ] | ||
178 | 43 | |||
179 | 44 | def make_invalid_links(self): | ||
180 | 45 | return [ | ||
181 | 46 | self.BRANCH_URL_TEMPLATE % 'foo', | ||
182 | 47 | self.BRANCH_URL_TEMPLATE % 'bar', | ||
183 | 48 | ] | ||
184 | 49 | |||
185 | 50 | def invoke_branch_link_checker( | ||
186 | 51 | self, valid_branch_urls=None, invalid_branch_urls=None): | ||
187 | 52 | if valid_branch_urls is None: | ||
188 | 53 | valid_branch_urls = {} | ||
189 | 54 | if invalid_branch_urls is None: | ||
190 | 55 | invalid_branch_urls = {} | ||
191 | 56 | |||
192 | 57 | branch_urls = list(valid_branch_urls) | ||
193 | 58 | branch_urls.extend(invalid_branch_urls) | ||
194 | 59 | shuffle(branch_urls) | ||
195 | 60 | |||
196 | 61 | links_to_check = dict(branch_links=branch_urls) | ||
197 | 62 | link_json = simplejson.dumps(links_to_check) | ||
198 | 63 | |||
199 | 64 | request = LaunchpadTestRequest(link_hrefs=link_json) | ||
200 | 65 | link_checker = LinkCheckerAPI(object(), request) | ||
201 | 66 | result_json = link_checker() | ||
202 | 67 | self.check_invalid_links( | ||
203 | 68 | result_json, 'invalid_branch_links', invalid_branch_urls) | ||
204 | 69 | |||
205 | 70 | def test_only_valid_branchlinks(self): | ||
206 | 71 | branch_urls = self.make_valid_links() | ||
207 | 72 | self.invoke_branch_link_checker(valid_branch_urls=branch_urls) | ||
208 | 73 | |||
209 | 74 | def test_only_invalid_branchlinks(self): | ||
210 | 75 | branch_urls = self.make_invalid_links() | ||
211 | 76 | self.invoke_branch_link_checker(invalid_branch_urls=branch_urls) | ||
212 | 77 | |||
213 | 78 | def test_valid_and_invald_branchlinks(self): | ||
214 | 79 | valid_branch_urls = self.make_valid_links() | ||
215 | 80 | invalid_branch_urls = self.make_invalid_links() | ||
216 | 81 | self.invoke_branch_link_checker( | ||
217 | 82 | valid_branch_urls=valid_branch_urls, | ||
218 | 83 | invalid_branch_urls=invalid_branch_urls) | ||
219 | 0 | 84 | ||
220 | === added file 'lib/lp/app/configure.zcml' | |||
221 | --- lib/lp/app/configure.zcml 1970-01-01 00:00:00 +0000 | |||
222 | +++ lib/lp/app/configure.zcml 2010-12-07 16:29:13 +0000 | |||
223 | @@ -0,0 +1,14 @@ | |||
224 | 1 | <!-- Copyright 2009 Canonical Ltd. This software is licensed under the | ||
225 | 2 | GNU Affero General Public License version 3 (see the file LICENSE). | ||
226 | 3 | --> | ||
227 | 4 | |||
228 | 5 | <configure | ||
229 | 6 | xmlns="http://namespaces.zope.org/zope" | ||
230 | 7 | xmlns:browser="http://namespaces.zope.org/browser" | ||
231 | 8 | xmlns:i18n="http://namespaces.zope.org/i18n" | ||
232 | 9 | xmlns:xmlrpc="http://namespaces.zope.org/xmlrpc" | ||
233 | 10 | xmlns:lp="http://namespaces.canonical.com/lp" | ||
234 | 11 | i18n_domain="launchpad"> | ||
235 | 12 | <include | ||
236 | 13 | package=".browser"/> | ||
237 | 14 | </configure> | ||
238 | 0 | 15 | ||
239 | === removed file 'lib/lp/app/configure.zcml' | |||
240 | --- lib/lp/app/configure.zcml 2009-07-17 02:25:09 +0000 | |||
241 | +++ lib/lp/app/configure.zcml 1970-01-01 00:00:00 +0000 | |||
242 | @@ -1,14 +0,0 @@ | |||
243 | 1 | <!-- Copyright 2009 Canonical Ltd. This software is licensed under the | ||
244 | 2 | GNU Affero General Public License version 3 (see the file LICENSE). | ||
245 | 3 | --> | ||
246 | 4 | |||
247 | 5 | <configure | ||
248 | 6 | xmlns="http://namespaces.zope.org/zope" | ||
249 | 7 | xmlns:browser="http://namespaces.zope.org/browser" | ||
250 | 8 | xmlns:i18n="http://namespaces.zope.org/i18n" | ||
251 | 9 | xmlns:xmlrpc="http://namespaces.zope.org/xmlrpc" | ||
252 | 10 | xmlns:lp="http://namespaces.canonical.com/lp" | ||
253 | 11 | i18n_domain="launchpad"> | ||
254 | 12 | <include | ||
255 | 13 | package=".browser"/> | ||
256 | 14 | </configure> | ||
257 | 15 | 0 | ||
258 | === modified file 'lib/lp/app/doc/displaying-paragraphs-of-text.txt' | |||
259 | --- lib/lp/app/doc/displaying-paragraphs-of-text.txt 2010-10-09 16:36:22 +0000 | |||
260 | +++ lib/lp/app/doc/displaying-paragraphs-of-text.txt 2010-12-07 16:29:13 +0000 | |||
261 | @@ -357,17 +357,17 @@ | |||
262 | 357 | ... 'lp:///foo\n' | 357 | ... 'lp:///foo\n' |
263 | 358 | ... 'lp:/foo\n') | 358 | ... 'lp:/foo\n') |
264 | 359 | >>> print test_tales('foo/fmt:text-to-html', foo=text) | 359 | >>> print test_tales('foo/fmt:text-to-html', foo=text) |
276 | 360 | <p><a href="/+branch/~foo/bar/baz">lp:~foo/bar/baz</a><br /> | 360 | <p><a href="/+branch/~foo/bar/baz" class="...">lp:~foo/bar/baz</a><br /> |
277 | 361 | <a href="/+branch/~foo/bar/bug-123">lp:~foo/bar/bug-123</a><br /> | 361 | <a href="/+branch/~foo/bar/bug-123" class="...">lp:~foo/bar/bug-123</a><br /> |
278 | 362 | <a href="/+branch/~foo/+junk/baz">lp:~foo/+junk/baz</a><br /> | 362 | <a href="/+branch/~foo/+junk/baz" class="...">lp:~foo/+junk/baz</a><br /> |
279 | 363 | <a href="/+branch/~foo/ubuntu/jaunty/evolution/baz">lp:~foo/ubuntu/jaunty/evolution/baz</a><br /> | 363 | <a href="/+branch/~foo/ubuntu/jaunty/evolution/baz" class="...">lp:~foo/ubuntu/jaunty/evolution/baz</a><br /> |
280 | 364 | <a href="/+branch/foo/bar">lp:foo/bar</a><br /> | 364 | <a href="/+branch/foo/bar" class="...">lp:foo/bar</a><br /> |
281 | 365 | <a href="/+branch/foo">lp:foo</a><br /> | 365 | <a href="/+branch/foo" class="...">lp:foo</a><br /> |
282 | 366 | <a href="/+branch/foo">lp:foo</a>,<br /> | 366 | <a href="/+branch/foo" class="...">lp:foo</a>,<br /> |
283 | 367 | <a href="/+branch/foo/bar">lp:foo/bar</a>.<br /> | 367 | <a href="/+branch/foo/bar" class="...">lp:foo/bar</a>.<br /> |
284 | 368 | <a href="/+branch/foo/bar/baz">lp:foo/bar/baz</a><br /> | 368 | <a href="/+branch/foo/bar/baz" class="...">lp:foo/bar/baz</a><br /> |
285 | 369 | <a href="/+branch/foo">lp:///foo</a><br /> | 369 | <a href="/+branch/foo" class="...">lp:///foo</a><br /> |
286 | 370 | <a href="/+branch/foo">lp:/foo</a></p> | 370 | <a href="/+branch/foo" class="...">lp:/foo</a></p> |
287 | 371 | 371 | ||
288 | 372 | Text that looks like a branch reference, but is followed only by digits is | 372 | Text that looks like a branch reference, but is followed only by digits is |
289 | 373 | treated as a link to a bug. | 373 | treated as a link to a bug. |
290 | 374 | 374 | ||
291 | === added file 'lib/lp/app/javascript/lp-links.js' | |||
292 | --- lib/lp/app/javascript/lp-links.js 1970-01-01 00:00:00 +0000 | |||
293 | +++ lib/lp/app/javascript/lp-links.js 2010-12-07 16:29:13 +0000 | |||
294 | @@ -0,0 +1,105 @@ | |||
295 | 1 | /** | ||
296 | 2 | * Launchpad utilities for manipulating links. | ||
297 | 3 | * | ||
298 | 4 | * @module app | ||
299 | 5 | * @submodule links | ||
300 | 6 | */ | ||
301 | 7 | |||
302 | 8 | YUI.add('lp.app.links', function(Y) { | ||
303 | 9 | |||
304 | 10 | function harvest_links(Y, links_holder, link_class, link_type) { | ||
305 | 11 | // Get any links of the specified link_class and store them as the | ||
306 | 12 | // specified link_type in the specified links_holder | ||
307 | 13 | var link_info = new Array(); | ||
308 | 14 | Y.all('.'+link_class).each(function(link) { | ||
309 | 15 | var href = link.getAttribute('href'); | ||
310 | 16 | if( link_info.indexOf(href)<0 ) { | ||
311 | 17 | link_info.push(href); | ||
312 | 18 | } | ||
313 | 19 | }); | ||
314 | 20 | if( link_info.length > 0 ) { | ||
315 | 21 | links_holder[link_type] = link_info; | ||
316 | 22 | } | ||
317 | 23 | } | ||
318 | 24 | |||
319 | 25 | function process_invalid_links( | ||
320 | 26 | Y, link_info, link_class, link_type, title) { | ||
321 | 27 | // We have a collection of invalid links possibly containing links of | ||
322 | 28 | // type link_type, so we need to remove the existing link_class, | ||
323 | 29 | // replace it with an invalid-link class, and set the link title. | ||
324 | 30 | var invalid_links = Y.Array(link_info['invalid_'+link_type]); | ||
325 | 31 | |||
326 | 32 | if( invalid_links.length > 0) { | ||
327 | 33 | Y.all('.'+link_class).each(function(link) { | ||
328 | 34 | var href = link.getAttribute('href'); | ||
329 | 35 | if( invalid_links.indexOf(href)>=0 ) { | ||
330 | 36 | var msg = title + href; | ||
331 | 37 | link.removeClass(link_class); | ||
332 | 38 | link.addClass('invalid-link'); | ||
333 | 39 | link.title = msg | ||
334 | 40 | link.on('click', function(e) { | ||
335 | 41 | e.halt(); | ||
336 | 42 | alert(msg); | ||
337 | 43 | }); | ||
338 | 44 | } | ||
339 | 45 | }); | ||
340 | 46 | } | ||
341 | 47 | } | ||
342 | 48 | |||
343 | 49 | var links = Y.namespace('lp.app.links'); | ||
344 | 50 | |||
345 | 51 | links.check_valid_lp_links = function() { | ||
346 | 52 | // Grabs any lp: style links on the page and checks that they are | ||
347 | 53 | // valid. Invalid ones have their class changed to "invalid-link". | ||
348 | 54 | // ATM, we only handle +branch links. | ||
349 | 55 | |||
350 | 56 | var links_to_check = {} | ||
351 | 57 | |||
352 | 58 | // We get all the links with defined css classes. | ||
353 | 59 | // At the moment, we just handle branch links, but in future... | ||
354 | 60 | harvest_links(Y, links_to_check, 'branch-short-link', 'branch_links'); | ||
355 | 61 | |||
356 | 62 | // Do we have anything to do? | ||
357 | 63 | if( Y.Object.size(links_to_check) == 0 ) { | ||
358 | 64 | return; | ||
359 | 65 | } | ||
360 | 66 | |||
361 | 67 | // Get the final json to send | ||
362 | 68 | var json_link_info = Y.JSON.stringify(links_to_check); | ||
363 | 69 | var qs = ''; | ||
364 | 70 | qs = LP.client.append_qs(qs, 'link_hrefs', json_link_info); | ||
365 | 71 | |||
366 | 72 | var config = { | ||
367 | 73 | on: { | ||
368 | 74 | failure: function(id, response, args) { | ||
369 | 75 | // If we have firebug installed, log the error. | ||
370 | 76 | if( console != undefined ) { | ||
371 | 77 | console.log("Link Check Error: " + args + ': ' | ||
372 | 78 | + response.status + ' - ' + | ||
373 | 79 | response.statusText + ' - ' | ||
374 | 80 | + response.responseXML); | ||
375 | 81 | } | ||
376 | 82 | }, | ||
377 | 83 | success: function(id, response) { | ||
378 | 84 | var link_info = Y.JSON.parse(response.responseText) | ||
379 | 85 | // ATM, we just handle branch links, but in future... | ||
380 | 86 | process_invalid_links(Y, link_info, 'branch-short-link', | ||
381 | 87 | 'branch_links', "Invalid branch: "); | ||
382 | 88 | } | ||
383 | 89 | } | ||
384 | 90 | } | ||
385 | 91 | var uri = '+check-links'; | ||
386 | 92 | var on = Y.merge(config.on); | ||
387 | 93 | var client = this; | ||
388 | 94 | var y_config = { method: "POST", | ||
389 | 95 | headers: {'Accept': 'application/json'}, | ||
390 | 96 | on: on, | ||
391 | 97 | 'arguments': [client, uri], | ||
392 | 98 | data: qs}; | ||
393 | 99 | Y.io(uri, y_config); | ||
394 | 100 | }; | ||
395 | 101 | |||
396 | 102 | }, "0.1", {"requires": [ | ||
397 | 103 | "base", "node", "io", "dom", "json" | ||
398 | 104 | ]}); | ||
399 | 105 | |||
400 | 0 | 106 | ||
401 | === modified file 'lib/lp/app/templates/base-layout-macros.pt' | |||
402 | --- lib/lp/app/templates/base-layout-macros.pt 2010-10-25 13:16:10 +0000 | |||
403 | +++ lib/lp/app/templates/base-layout-macros.pt 2010-12-07 16:29:13 +0000 | |||
404 | @@ -175,6 +175,8 @@ | |||
405 | 175 | <script type="text/javascript" | 175 | <script type="text/javascript" |
406 | 176 | tal:attributes="src string:${lp_js}/app/lp-mochi.js"></script> | 176 | tal:attributes="src string:${lp_js}/app/lp-mochi.js"></script> |
407 | 177 | <script type="text/javascript" | 177 | <script type="text/javascript" |
408 | 178 | tal:attributes="src string:${lp_js}/app/lp-links.js"></script> | ||
409 | 179 | <script type="text/javascript" | ||
410 | 178 | tal:attributes="src string:${lp_js}/app/dragscroll.js"></script> | 180 | tal:attributes="src string:${lp_js}/app/dragscroll.js"></script> |
411 | 179 | <script type="text/javascript" | 181 | <script type="text/javascript" |
412 | 180 | tal:attributes="src string:${lp_js}/app/picker.js"></script> | 182 | tal:attributes="src string:${lp_js}/app/picker.js"></script> |
413 | @@ -304,6 +306,13 @@ | |||
414 | 304 | // anywhere outside of it. | 306 | // anywhere outside of it. |
415 | 305 | Y.on('click', handleClickOnPage, window); | 307 | Y.on('click', handleClickOnPage, window); |
416 | 306 | }); | 308 | }); |
417 | 309 | |||
418 | 310 | LPS.use('lp.app.links', | ||
419 | 311 | function(Y) { | ||
420 | 312 | Y.on('load', function(e) { | ||
421 | 313 | Y.lp.app.links.check_valid_lp_links(); | ||
422 | 314 | }, window); | ||
423 | 315 | }); | ||
424 | 307 | </script> | 316 | </script> |
425 | 308 | </metal:page-javascript> | 317 | </metal:page-javascript> |
426 | 309 | 318 | ||
427 | 310 | 319 | ||
428 | === modified file 'lib/lp/bugs/windmill/tests/test_bug_commenting.py' | |||
429 | --- lib/lp/bugs/windmill/tests/test_bug_commenting.py 2010-08-20 20:31:18 +0000 | |||
430 | +++ lib/lp/bugs/windmill/tests/test_bug_commenting.py 2010-12-07 16:29:13 +0000 | |||
431 | @@ -18,7 +18,7 @@ | |||
432 | 18 | WAIT_ELEMENT_COMPLETE = u'30000' | 18 | WAIT_ELEMENT_COMPLETE = u'30000' |
433 | 19 | WAIT_CHECK_CHANGE = u'1000' | 19 | WAIT_CHECK_CHANGE = u'1000' |
434 | 20 | ADD_COMMENT_BUTTON = ( | 20 | ADD_COMMENT_BUTTON = ( |
436 | 21 | u'//input[@id="field.actions.save" and @class="button js-action"]') | 21 | u'//input[@id="field.actions.save" and contains(@class, "button")]') |
437 | 22 | 22 | ||
438 | 23 | 23 | ||
439 | 24 | class TestBugCommenting(WindmillTestCase): | 24 | class TestBugCommenting(WindmillTestCase): |
440 | 25 | 25 | ||
441 | === modified file 'lib/lp/buildmaster/doc/builder.txt' | |||
442 | --- lib/lp/buildmaster/doc/builder.txt 2010-09-24 12:10:52 +0000 | |||
443 | +++ lib/lp/buildmaster/doc/builder.txt 2010-12-07 16:29:13 +0000 | |||
444 | @@ -19,6 +19,9 @@ | |||
445 | 19 | As expected, it implements IBuilder. | 19 | As expected, it implements IBuilder. |
446 | 20 | 20 | ||
447 | 21 | >>> from canonical.launchpad.webapp.testing import verifyObject | 21 | >>> from canonical.launchpad.webapp.testing import verifyObject |
448 | 22 | >>> from lp.buildmaster.interfaces.builder import IBuilder | ||
449 | 23 | >>> verifyObject(IBuilder, builder) | ||
450 | 24 | True | ||
451 | 22 | 25 | ||
452 | 23 | >>> print builder.name | 26 | >>> print builder.name |
453 | 24 | bob | 27 | bob |
454 | @@ -83,7 +86,7 @@ | |||
455 | 83 | The 'new' method will create a new builder in the database. | 86 | The 'new' method will create a new builder in the database. |
456 | 84 | 87 | ||
457 | 85 | >>> bnew = builderset.new(1, 'http://dummy.com:8221/', 'dummy', | 88 | >>> bnew = builderset.new(1, 'http://dummy.com:8221/', 'dummy', |
459 | 86 | ... 'Dummy Title', 'eh ?', 1) | 89 | ... 'Dummy Title', 'eh ?', 1) |
460 | 87 | >>> bnew.name | 90 | >>> bnew.name |
461 | 88 | u'dummy' | 91 | u'dummy' |
462 | 89 | 92 | ||
463 | @@ -167,7 +170,7 @@ | |||
464 | 167 | >>> recipe_bq.processor = i386_family.processors[0] | 170 | >>> recipe_bq.processor = i386_family.processors[0] |
465 | 168 | >>> recipe_bq.virtualized = True | 171 | >>> recipe_bq.virtualized = True |
466 | 169 | >>> transaction.commit() | 172 | >>> transaction.commit() |
468 | 170 | 173 | ||
469 | 171 | >>> queue_sizes = builderset.getBuildQueueSizes() | 174 | >>> queue_sizes = builderset.getBuildQueueSizes() |
470 | 172 | >>> print queue_sizes['virt']['386'] | 175 | >>> print queue_sizes['virt']['386'] |
471 | 173 | (1L, datetime.timedelta(0, 64)) | 176 | (1L, datetime.timedelta(0, 64)) |
472 | @@ -185,3 +188,116 @@ | |||
473 | 185 | 188 | ||
474 | 186 | >>> print queue_sizes['virt']['386'] | 189 | >>> print queue_sizes['virt']['386'] |
475 | 187 | (2L, datetime.timedelta(0, 128)) | 190 | (2L, datetime.timedelta(0, 128)) |
476 | 191 | |||
477 | 192 | |||
478 | 193 | Resuming buildd slaves | ||
479 | 194 | ====================== | ||
480 | 195 | |||
481 | 196 | Virtual slaves are resumed using a command specified in the | ||
482 | 197 | configuration profile. Production configuration uses a SSH trigger | ||
483 | 198 | account accessed via a private key available in the builddmaster | ||
484 | 199 | machine (which used ftpmaster configuration profile) as in: | ||
485 | 200 | |||
486 | 201 | {{{ | ||
487 | 202 | ssh ~/.ssh/ppa-reset-key ppa@%(vm_host)s | ||
488 | 203 | }}} | ||
489 | 204 | |||
490 | 205 | The test configuration uses a fake command that can be performed in | ||
491 | 206 | development machine and allow us to tests the important features used | ||
492 | 207 | in production, as 'vm_host' variable replacement. | ||
493 | 208 | |||
494 | 209 | >>> from canonical.config import config | ||
495 | 210 | >>> config.builddmaster.vm_resume_command | ||
496 | 211 | 'echo %(vm_host)s' | ||
497 | 212 | |||
498 | 213 | Before performing the command, it checks if the builder is indeed | ||
499 | 214 | virtual and raises CannotResumeHost if it isn't. | ||
500 | 215 | |||
501 | 216 | >>> bob = getUtility(IBuilderSet)['bob'] | ||
502 | 217 | >>> bob.resumeSlaveHost() | ||
503 | 218 | Traceback (most recent call last): | ||
504 | 219 | ... | ||
505 | 220 | CannotResumeHost: Builder is not virtualized. | ||
506 | 221 | |||
507 | 222 | For testing purposes resumeSlaveHost returns the stdout and stderr | ||
508 | 223 | buffer resulted from the command. | ||
509 | 224 | |||
510 | 225 | >>> frog = getUtility(IBuilderSet)['frog'] | ||
511 | 226 | >>> out, err = frog.resumeSlaveHost() | ||
512 | 227 | >>> print out.strip() | ||
513 | 228 | localhost-host.ppa | ||
514 | 229 | |||
515 | 230 | If the specified command fails, resumeSlaveHost also raises | ||
516 | 231 | CannotResumeHost exception with the results stdout and stderr. | ||
517 | 232 | |||
518 | 233 | # The command must have a vm_host dict key and when executed, | ||
519 | 234 | # have a returncode that is not 0. | ||
520 | 235 | >>> vm_resume_command = """ | ||
521 | 236 | ... [builddmaster] | ||
522 | 237 | ... vm_resume_command: test "%(vm_host)s = 'false'" | ||
523 | 238 | ... """ | ||
524 | 239 | >>> config.push('vm_resume_command', vm_resume_command) | ||
525 | 240 | >>> frog.resumeSlaveHost() | ||
526 | 241 | Traceback (most recent call last): | ||
527 | 242 | ... | ||
528 | 243 | CannotResumeHost: Resuming failed: | ||
529 | 244 | OUT: | ||
530 | 245 | <BLANKLINE> | ||
531 | 246 | ERR: | ||
532 | 247 | <BLANKLINE> | ||
533 | 248 | |||
534 | 249 | Restore default value for resume command. | ||
535 | 250 | |||
536 | 251 | >>> config_data = config.pop('vm_resume_command') | ||
537 | 252 | |||
538 | 253 | |||
539 | 254 | Rescuing lost slaves | ||
540 | 255 | ==================== | ||
541 | 256 | |||
542 | 257 | Builder.rescueIfLost() checks the build ID reported in the slave status | ||
543 | 258 | against the database. If it isn't building what we think it should be, | ||
544 | 259 | the current build will be aborted and the slave cleaned in preparation | ||
545 | 260 | for a new task. The decision about the slave's correctness is left up | ||
546 | 261 | to IBuildFarmJobBehavior.verifySlaveBuildCookie -- for these examples we | ||
547 | 262 | will use a special behavior that just checks if the cookie reads 'good'. | ||
548 | 263 | |||
549 | 264 | >>> import logging | ||
550 | 265 | >>> from lp.buildmaster.interfaces.builder import CorruptBuildCookie | ||
551 | 266 | >>> from lp.buildmaster.tests.mock_slaves import ( | ||
552 | 267 | ... BuildingSlave, MockBuilder, OkSlave, WaitingSlave) | ||
553 | 268 | |||
554 | 269 | >>> class TestBuildBehavior: | ||
555 | 270 | ... def verifySlaveBuildCookie(self, cookie): | ||
556 | 271 | ... if cookie != 'good': | ||
557 | 272 | ... raise CorruptBuildCookie('Bad value') | ||
558 | 273 | |||
559 | 274 | >>> def rescue_slave_if_lost(slave): | ||
560 | 275 | ... builder = MockBuilder('mock', slave, TestBuildBehavior()) | ||
561 | 276 | ... builder.rescueIfLost(logging.getLogger()) | ||
562 | 277 | |||
563 | 278 | An idle slave is not rescued. | ||
564 | 279 | |||
565 | 280 | >>> rescue_slave_if_lost(OkSlave()) | ||
566 | 281 | |||
567 | 282 | Slaves building or having built the correct build are not rescued | ||
568 | 283 | either. | ||
569 | 284 | |||
570 | 285 | >>> rescue_slave_if_lost(BuildingSlave(build_id='good')) | ||
571 | 286 | >>> rescue_slave_if_lost(WaitingSlave(build_id='good')) | ||
572 | 287 | |||
573 | 288 | But if a slave is building the wrong ID, it is declared lost and | ||
574 | 289 | an abort is attempted. MockSlave prints out a message when it is aborted | ||
575 | 290 | or cleaned. | ||
576 | 291 | |||
577 | 292 | >>> rescue_slave_if_lost(BuildingSlave(build_id='bad')) | ||
578 | 293 | Aborting slave | ||
579 | 294 | INFO:root:Builder 'mock' rescued from 'bad': 'Bad value' | ||
580 | 295 | |||
581 | 296 | Slaves having completed an incorrect build are also declared lost, | ||
582 | 297 | but there's no need to abort a completed build. Such builders are | ||
583 | 298 | instead simply cleaned, ready for the next build. | ||
584 | 299 | |||
585 | 300 | >>> rescue_slave_if_lost(WaitingSlave(build_id='bad')) | ||
586 | 301 | Cleaning slave | ||
587 | 302 | INFO:root:Builder 'mock' rescued from 'bad': 'Bad value' | ||
588 | 303 | |||
589 | 188 | 304 | ||
590 | === modified file 'lib/lp/buildmaster/interfaces/builder.py' | |||
591 | --- lib/lp/buildmaster/interfaces/builder.py 2010-10-18 11:57:09 +0000 | |||
592 | +++ lib/lp/buildmaster/interfaces/builder.py 2010-12-07 16:29:13 +0000 | |||
593 | @@ -154,6 +154,11 @@ | |||
594 | 154 | 154 | ||
595 | 155 | currentjob = Attribute("BuildQueue instance for job being processed.") | 155 | currentjob = Attribute("BuildQueue instance for job being processed.") |
596 | 156 | 156 | ||
597 | 157 | is_available = Bool( | ||
598 | 158 | title=_("Whether or not a builder is available for building " | ||
599 | 159 | "new jobs. "), | ||
600 | 160 | required=False) | ||
601 | 161 | |||
602 | 157 | failure_count = Int( | 162 | failure_count = Int( |
603 | 158 | title=_('Failure Count'), required=False, default=0, | 163 | title=_('Failure Count'), required=False, default=0, |
604 | 159 | description=_("Number of consecutive failures for this builder.")) | 164 | description=_("Number of consecutive failures for this builder.")) |
605 | @@ -168,74 +173,32 @@ | |||
606 | 168 | def resetFailureCount(): | 173 | def resetFailureCount(): |
607 | 169 | """Set the failure_count back to zero.""" | 174 | """Set the failure_count back to zero.""" |
608 | 170 | 175 | ||
649 | 171 | def failBuilder(reason): | 176 | def checkSlaveAlive(): |
650 | 172 | """Mark builder as failed for a given reason.""" | 177 | """Check that the buildd slave is alive. |
651 | 173 | 178 | ||
652 | 174 | def setSlaveForTesting(proxy): | 179 | This pings the slave over the network via the echo method and looks |
653 | 175 | """Sets the RPC proxy through which to operate the build slave.""" | 180 | for the sent message as the reply. |
654 | 176 | 181 | ||
655 | 177 | def verifySlaveBuildCookie(slave_build_id): | 182 | :raises BuildDaemonError: When the slave is down. |
616 | 178 | """Verify that a slave's build cookie is consistent. | ||
617 | 179 | |||
618 | 180 | This should delegate to the current `IBuildFarmJobBehavior`. | ||
619 | 181 | """ | ||
620 | 182 | |||
621 | 183 | def transferSlaveFileToLibrarian(file_sha1, filename, private): | ||
622 | 184 | """Transfer a file from the slave to the librarian. | ||
623 | 185 | |||
624 | 186 | :param file_sha1: The file's sha1, which is how the file is addressed | ||
625 | 187 | in the slave XMLRPC protocol. Specially, the file_sha1 'buildlog' | ||
626 | 188 | will cause the build log to be retrieved and gzipped. | ||
627 | 189 | :param filename: The name of the file to be given to the librarian file | ||
628 | 190 | alias. | ||
629 | 191 | :param private: True if the build is for a private archive. | ||
630 | 192 | :return: A librarian file alias. | ||
631 | 193 | """ | ||
632 | 194 | |||
633 | 195 | def getBuildQueue(): | ||
634 | 196 | """Return a `BuildQueue` if there's an active job on this builder. | ||
635 | 197 | |||
636 | 198 | :return: A BuildQueue, or None. | ||
637 | 199 | """ | ||
638 | 200 | |||
639 | 201 | def getCurrentBuildFarmJob(): | ||
640 | 202 | """Return a `BuildFarmJob` for this builder.""" | ||
641 | 203 | |||
642 | 204 | # All methods below here return Deferred. | ||
643 | 205 | |||
644 | 206 | def isAvailable(): | ||
645 | 207 | """Whether or not a builder is available for building new jobs. | ||
646 | 208 | |||
647 | 209 | :return: A Deferred that fires with True or False, depending on | ||
648 | 210 | whether the builder is available or not. | ||
656 | 211 | """ | 183 | """ |
657 | 212 | 184 | ||
658 | 213 | def rescueIfLost(logger=None): | 185 | def rescueIfLost(logger=None): |
659 | 214 | """Reset the slave if its job information doesn't match the DB. | 186 | """Reset the slave if its job information doesn't match the DB. |
660 | 215 | 187 | ||
669 | 216 | This checks the build ID reported in the slave status against the | 188 | If the builder is BUILDING or WAITING but has a build ID string |
670 | 217 | database. If it isn't building what we think it should be, the current | 189 | that doesn't match what is stored in the DB, we have to dismiss |
671 | 218 | build will be aborted and the slave cleaned in preparation for a new | 190 | its current actions and clean the slave for another job, assuming |
672 | 219 | task. The decision about the slave's correctness is left up to | 191 | the XMLRPC is working properly at this point. |
665 | 220 | `IBuildFarmJobBehavior.verifySlaveBuildCookie`. | ||
666 | 221 | |||
667 | 222 | :return: A Deferred that fires when the dialog with the slave is | ||
668 | 223 | finished. It does not have a return value. | ||
673 | 224 | """ | 192 | """ |
674 | 225 | 193 | ||
675 | 226 | def updateStatus(logger=None): | 194 | def updateStatus(logger=None): |
681 | 227 | """Update the builder's status by probing it. | 195 | """Update the builder's status by probing it.""" |
677 | 228 | |||
678 | 229 | :return: A Deferred that fires when the dialog with the slave is | ||
679 | 230 | finished. It does not have a return value. | ||
680 | 231 | """ | ||
682 | 232 | 196 | ||
683 | 233 | def cleanSlave(): | 197 | def cleanSlave(): |
689 | 234 | """Clean any temporary files from the slave. | 198 | """Clean any temporary files from the slave.""" |
690 | 235 | 199 | ||
691 | 236 | :return: A Deferred that fires when the dialog with the slave is | 200 | def failBuilder(reason): |
692 | 237 | finished. It does not have a return value. | 201 | """Mark builder as failed for a given reason.""" |
688 | 238 | """ | ||
693 | 239 | 202 | ||
694 | 240 | def requestAbort(): | 203 | def requestAbort(): |
695 | 241 | """Ask that a build be aborted. | 204 | """Ask that a build be aborted. |
696 | @@ -243,9 +206,6 @@ | |||
697 | 243 | This takes place asynchronously: Actually killing everything running | 206 | This takes place asynchronously: Actually killing everything running |
698 | 244 | can take some time so the slave status should be queried again to | 207 | can take some time so the slave status should be queried again to |
699 | 245 | detect when the abort has taken effect. (Look for status ABORTED). | 208 | detect when the abort has taken effect. (Look for status ABORTED). |
700 | 246 | |||
701 | 247 | :return: A Deferred that fires when the dialog with the slave is | ||
702 | 248 | finished. It does not have a return value. | ||
703 | 249 | """ | 209 | """ |
704 | 250 | 210 | ||
705 | 251 | def resumeSlaveHost(): | 211 | def resumeSlaveHost(): |
706 | @@ -257,35 +217,37 @@ | |||
707 | 257 | :raises: CannotResumeHost: if builder is not virtual or if the | 217 | :raises: CannotResumeHost: if builder is not virtual or if the |
708 | 258 | configuration command has failed. | 218 | configuration command has failed. |
709 | 259 | 219 | ||
713 | 260 | :return: A Deferred that fires when the resume operation finishes, | 220 | :return: command stdout and stderr buffers as a tuple. |
711 | 261 | whose value is a (stdout, stderr) tuple for success, or a Failure | ||
712 | 262 | whose value is a CannotResumeHost exception. | ||
714 | 263 | """ | 221 | """ |
715 | 264 | 222 | ||
716 | 223 | def setSlaveForTesting(proxy): | ||
717 | 224 | """Sets the RPC proxy through which to operate the build slave.""" | ||
718 | 225 | |||
719 | 265 | def slaveStatus(): | 226 | def slaveStatus(): |
720 | 266 | """Get the slave status for this builder. | 227 | """Get the slave status for this builder. |
721 | 267 | 228 | ||
726 | 268 | :return: A Deferred which fires when the slave dialog is complete. | 229 | :return: a dict containing at least builder_status, but potentially |
727 | 269 | Its value is a dict containing at least builder_status, but | 230 | other values included by the current build behavior. |
724 | 270 | potentially other values included by the current build | ||
725 | 271 | behavior. | ||
728 | 272 | """ | 231 | """ |
729 | 273 | 232 | ||
730 | 274 | def slaveStatusSentence(): | 233 | def slaveStatusSentence(): |
731 | 275 | """Get the slave status sentence for this builder. | 234 | """Get the slave status sentence for this builder. |
732 | 276 | 235 | ||
737 | 277 | :return: A Deferred which fires when the slave dialog is complete. | 236 | :return: A tuple with the first element containing the slave status, |
738 | 278 | Its value is a tuple with the first element containing the | 237 | build_id-queue-id and then optionally more elements depending on |
739 | 279 | slave status, build_id-queue-id and then optionally more | 238 | the status. |
740 | 280 | elements depending on the status. | 239 | """ |
741 | 240 | |||
742 | 241 | def verifySlaveBuildCookie(slave_build_id): | ||
743 | 242 | """Verify that a slave's build cookie is consistent. | ||
744 | 243 | |||
745 | 244 | This should delegate to the current `IBuildFarmJobBehavior`. | ||
746 | 281 | """ | 245 | """ |
747 | 282 | 246 | ||
748 | 283 | def updateBuild(queueItem): | 247 | def updateBuild(queueItem): |
749 | 284 | """Verify the current build job status. | 248 | """Verify the current build job status. |
750 | 285 | 249 | ||
751 | 286 | Perform the required actions for each state. | 250 | Perform the required actions for each state. |
752 | 287 | |||
753 | 288 | :return: A Deferred that fires when the slave dialog is finished. | ||
754 | 289 | """ | 251 | """ |
755 | 290 | 252 | ||
756 | 291 | def startBuild(build_queue_item, logger): | 253 | def startBuild(build_queue_item, logger): |
757 | @@ -293,10 +255,21 @@ | |||
758 | 293 | 255 | ||
759 | 294 | :param build_queue_item: A BuildQueueItem to build. | 256 | :param build_queue_item: A BuildQueueItem to build. |
760 | 295 | :param logger: A logger to be used to log diagnostic information. | 257 | :param logger: A logger to be used to log diagnostic information. |
765 | 296 | 258 | :raises BuildSlaveFailure: When the build slave fails. | |
766 | 297 | :return: A Deferred that fires after the dispatch has completed whose | 259 | :raises CannotBuild: When a build cannot be started for some reason |
767 | 298 | value is None, or a Failure that contains an exception | 260 | other than the build slave failing. |
768 | 299 | explaining what went wrong. | 261 | """ |
769 | 262 | |||
770 | 263 | def transferSlaveFileToLibrarian(file_sha1, filename, private): | ||
771 | 264 | """Transfer a file from the slave to the librarian. | ||
772 | 265 | |||
773 | 266 | :param file_sha1: The file's sha1, which is how the file is addressed | ||
774 | 267 | in the slave XMLRPC protocol. Specially, the file_sha1 'buildlog' | ||
775 | 268 | will cause the build log to be retrieved and gzipped. | ||
776 | 269 | :param filename: The name of the file to be given to the librarian file | ||
777 | 270 | alias. | ||
778 | 271 | :param private: True if the build is for a private archive. | ||
779 | 272 | :return: A librarian file alias. | ||
780 | 300 | """ | 273 | """ |
781 | 301 | 274 | ||
782 | 302 | def handleTimeout(logger, error_message): | 275 | def handleTimeout(logger, error_message): |
783 | @@ -311,8 +284,6 @@ | |||
784 | 311 | 284 | ||
785 | 312 | :param logger: The logger object to be used for logging. | 285 | :param logger: The logger object to be used for logging. |
786 | 313 | :param error_message: The error message to be used for logging. | 286 | :param error_message: The error message to be used for logging. |
787 | 314 | :return: A Deferred that fires after the virtual slave was resumed | ||
788 | 315 | or immediately if it's a non-virtual slave. | ||
789 | 316 | """ | 287 | """ |
790 | 317 | 288 | ||
791 | 318 | def findAndStartJob(buildd_slave=None): | 289 | def findAndStartJob(buildd_slave=None): |
792 | @@ -320,9 +291,17 @@ | |||
793 | 320 | 291 | ||
794 | 321 | :param buildd_slave: An optional buildd slave that this builder should | 292 | :param buildd_slave: An optional buildd slave that this builder should |
795 | 322 | talk to. | 293 | talk to. |
799 | 323 | :return: A Deferred whose value is the `IBuildQueue` instance | 294 | :return: the `IBuildQueue` instance found or None if no job was found. |
800 | 324 | found or None if no job was found. | 295 | """ |
801 | 325 | """ | 296 | |
802 | 297 | def getBuildQueue(): | ||
803 | 298 | """Return a `BuildQueue` if there's an active job on this builder. | ||
804 | 299 | |||
805 | 300 | :return: A BuildQueue, or None. | ||
806 | 301 | """ | ||
807 | 302 | |||
808 | 303 | def getCurrentBuildFarmJob(): | ||
809 | 304 | """Return a `BuildFarmJob` for this builder.""" | ||
810 | 326 | 305 | ||
811 | 327 | 306 | ||
812 | 328 | class IBuilderSet(Interface): | 307 | class IBuilderSet(Interface): |
813 | 329 | 308 | ||
814 | === modified file 'lib/lp/buildmaster/manager.py' | |||
815 | --- lib/lp/buildmaster/manager.py 2010-10-20 12:28:46 +0000 | |||
816 | +++ lib/lp/buildmaster/manager.py 2010-12-07 16:29:13 +0000 | |||
817 | @@ -10,10 +10,13 @@ | |||
818 | 10 | 'BuilddManager', | 10 | 'BuilddManager', |
819 | 11 | 'BUILDD_MANAGER_LOG_NAME', | 11 | 'BUILDD_MANAGER_LOG_NAME', |
820 | 12 | 'FailDispatchResult', | 12 | 'FailDispatchResult', |
821 | 13 | 'RecordingSlave', | ||
822 | 13 | 'ResetDispatchResult', | 14 | 'ResetDispatchResult', |
823 | 15 | 'buildd_success_result_map', | ||
824 | 14 | ] | 16 | ] |
825 | 15 | 17 | ||
826 | 16 | import logging | 18 | import logging |
827 | 19 | import os | ||
828 | 17 | 20 | ||
829 | 18 | import transaction | 21 | import transaction |
830 | 19 | from twisted.application import service | 22 | from twisted.application import service |
831 | @@ -21,27 +24,129 @@ | |||
832 | 21 | defer, | 24 | defer, |
833 | 22 | reactor, | 25 | reactor, |
834 | 23 | ) | 26 | ) |
836 | 24 | from twisted.internet.task import LoopingCall | 27 | from twisted.protocols.policies import TimeoutMixin |
837 | 25 | from twisted.python import log | 28 | from twisted.python import log |
838 | 29 | from twisted.python.failure import Failure | ||
839 | 30 | from twisted.web import xmlrpc | ||
840 | 26 | from zope.component import getUtility | 31 | from zope.component import getUtility |
841 | 27 | 32 | ||
842 | 33 | from canonical.config import config | ||
843 | 34 | from canonical.launchpad.webapp import urlappend | ||
844 | 35 | from lp.services.database import write_transaction | ||
845 | 28 | from lp.buildmaster.enums import BuildStatus | 36 | from lp.buildmaster.enums import BuildStatus |
857 | 29 | from lp.buildmaster.interfaces.buildfarmjobbehavior import ( | 37 | from lp.services.twistedsupport.processmonitor import ProcessWithTimeout |
847 | 30 | BuildBehaviorMismatch, | ||
848 | 31 | ) | ||
849 | 32 | from lp.buildmaster.model.builder import Builder | ||
850 | 33 | from lp.buildmaster.interfaces.builder import ( | ||
851 | 34 | BuildDaemonError, | ||
852 | 35 | BuildSlaveFailure, | ||
853 | 36 | CannotBuild, | ||
854 | 37 | CannotFetchFile, | ||
855 | 38 | CannotResumeHost, | ||
856 | 39 | ) | ||
858 | 40 | 38 | ||
859 | 41 | 39 | ||
860 | 42 | BUILDD_MANAGER_LOG_NAME = "slave-scanner" | 40 | BUILDD_MANAGER_LOG_NAME = "slave-scanner" |
861 | 43 | 41 | ||
862 | 44 | 42 | ||
863 | 43 | buildd_success_result_map = { | ||
864 | 44 | 'ensurepresent': True, | ||
865 | 45 | 'build': 'BuilderStatus.BUILDING', | ||
866 | 46 | } | ||
867 | 47 | |||
868 | 48 | |||
869 | 49 | class QueryWithTimeoutProtocol(xmlrpc.QueryProtocol, TimeoutMixin): | ||
870 | 50 | """XMLRPC query protocol with a configurable timeout. | ||
871 | 51 | |||
872 | 52 | XMLRPC queries using this protocol will be unconditionally closed | ||
873 | 53 | when the timeout is elapsed. The timeout is fetched from the context | ||
874 | 54 | Launchpad configuration file (`config.builddmaster.socket_timeout`). | ||
875 | 55 | """ | ||
876 | 56 | def connectionMade(self): | ||
877 | 57 | xmlrpc.QueryProtocol.connectionMade(self) | ||
878 | 58 | self.setTimeout(config.builddmaster.socket_timeout) | ||
879 | 59 | |||
880 | 60 | |||
881 | 61 | class QueryFactoryWithTimeout(xmlrpc._QueryFactory): | ||
882 | 62 | """XMLRPC client factory with timeout support.""" | ||
883 | 63 | # Make this factory quiet. | ||
884 | 64 | noisy = False | ||
885 | 65 | # Use the protocol with timeout support. | ||
886 | 66 | protocol = QueryWithTimeoutProtocol | ||
887 | 67 | |||
888 | 68 | |||
889 | 69 | class RecordingSlave: | ||
890 | 70 | """An RPC proxy for buildd slaves that records instructions to the latter. | ||
891 | 71 | |||
892 | 72 | The idea here is to merely record the instructions that the slave-scanner | ||
893 | 73 | issues to the buildd slaves and "replay" them a bit later in asynchronous | ||
894 | 74 | and parallel fashion. | ||
895 | 75 | |||
896 | 76 | By dealing with a number of buildd slaves in parallel we remove *the* | ||
897 | 77 | major slave-scanner throughput issue while avoiding large-scale changes to | ||
898 | 78 | its code base. | ||
899 | 79 | """ | ||
900 | 80 | |||
901 | 81 | def __init__(self, name, url, vm_host): | ||
902 | 82 | self.name = name | ||
903 | 83 | self.url = url | ||
904 | 84 | self.vm_host = vm_host | ||
905 | 85 | |||
906 | 86 | self.resume_requested = False | ||
907 | 87 | self.calls = [] | ||
908 | 88 | |||
909 | 89 | def __repr__(self): | ||
910 | 90 | return '<%s:%s>' % (self.name, self.url) | ||
911 | 91 | |||
912 | 92 | def cacheFile(self, logger, libraryfilealias): | ||
913 | 93 | """Cache the file on the server.""" | ||
914 | 94 | self.ensurepresent( | ||
915 | 95 | libraryfilealias.content.sha1, libraryfilealias.http_url, '', '') | ||
916 | 96 | |||
917 | 97 | def sendFileToSlave(self, *args): | ||
918 | 98 | """Helper to send a file to this builder.""" | ||
919 | 99 | return self.ensurepresent(*args) | ||
920 | 100 | |||
921 | 101 | def ensurepresent(self, *args): | ||
922 | 102 | """Download files needed for the build.""" | ||
923 | 103 | self.calls.append(('ensurepresent', args)) | ||
924 | 104 | result = buildd_success_result_map.get('ensurepresent') | ||
925 | 105 | return [result, 'Download'] | ||
926 | 106 | |||
927 | 107 | def build(self, *args): | ||
928 | 108 | """Perform the build.""" | ||
929 | 109 | # XXX: This method does not appear to be used. | ||
930 | 110 | self.calls.append(('build', args)) | ||
931 | 111 | result = buildd_success_result_map.get('build') | ||
932 | 112 | return [result, args[0]] | ||
933 | 113 | |||
934 | 114 | def resume(self): | ||
935 | 115 | """Record the request to resume the builder.. | ||
936 | 116 | |||
937 | 117 | Always succeed. | ||
938 | 118 | |||
939 | 119 | :return: a (stdout, stderr, subprocess exitcode) triple | ||
940 | 120 | """ | ||
941 | 121 | self.resume_requested = True | ||
942 | 122 | return ['', '', 0] | ||
943 | 123 | |||
944 | 124 | def resumeSlave(self, clock=None): | ||
945 | 125 | """Resume the builder in a asynchronous fashion. | ||
946 | 126 | |||
947 | 127 | Used the configuration command-line in the same way | ||
948 | 128 | `BuilddSlave.resume` does. | ||
949 | 129 | |||
950 | 130 | Also use the builddmaster configuration 'socket_timeout' as | ||
951 | 131 | the process timeout. | ||
952 | 132 | |||
953 | 133 | :param clock: An optional twisted.internet.task.Clock to override | ||
954 | 134 | the default clock. For use in tests. | ||
955 | 135 | |||
956 | 136 | :return: a Deferred | ||
957 | 137 | """ | ||
958 | 138 | resume_command = config.builddmaster.vm_resume_command % { | ||
959 | 139 | 'vm_host': self.vm_host} | ||
960 | 140 | # Twisted API require string and the configuration provides unicode. | ||
961 | 141 | resume_argv = [str(term) for term in resume_command.split()] | ||
962 | 142 | |||
963 | 143 | d = defer.Deferred() | ||
964 | 144 | p = ProcessWithTimeout( | ||
965 | 145 | d, config.builddmaster.socket_timeout, clock=clock) | ||
966 | 146 | p.spawnProcess(resume_argv[0], tuple(resume_argv)) | ||
967 | 147 | return d | ||
968 | 148 | |||
969 | 149 | |||
970 | 45 | def get_builder(name): | 150 | def get_builder(name): |
971 | 46 | """Helper to return the builder given the slave for this request.""" | 151 | """Helper to return the builder given the slave for this request.""" |
972 | 47 | # Avoiding circular imports. | 152 | # Avoiding circular imports. |
973 | @@ -54,12 +159,9 @@ | |||
974 | 54 | # builder.currentjob hides a complicated query, don't run it twice. | 159 | # builder.currentjob hides a complicated query, don't run it twice. |
975 | 55 | # See bug 623281. | 160 | # See bug 623281. |
976 | 56 | current_job = builder.currentjob | 161 | current_job = builder.currentjob |
981 | 57 | if current_job is None: | 162 | build_job = current_job.specific_job.build |
978 | 58 | job_failure_count = 0 | ||
979 | 59 | else: | ||
980 | 60 | job_failure_count = current_job.specific_job.build.failure_count | ||
982 | 61 | 163 | ||
984 | 62 | if builder.failure_count == job_failure_count and current_job is not None: | 164 | if builder.failure_count == build_job.failure_count: |
985 | 63 | # If the failure count for the builder is the same as the | 165 | # If the failure count for the builder is the same as the |
986 | 64 | # failure count for the job being built, then we cannot | 166 | # failure count for the job being built, then we cannot |
987 | 65 | # tell whether the job or the builder is at fault. The best | 167 | # tell whether the job or the builder is at fault. The best |
988 | @@ -68,28 +170,17 @@ | |||
989 | 68 | current_job.reset() | 170 | current_job.reset() |
990 | 69 | return | 171 | return |
991 | 70 | 172 | ||
993 | 71 | if builder.failure_count > job_failure_count: | 173 | if builder.failure_count > build_job.failure_count: |
994 | 72 | # The builder has failed more than the jobs it's been | 174 | # The builder has failed more than the jobs it's been |
1008 | 73 | # running. | 175 | # running, so let's disable it and re-schedule the build. |
1009 | 74 | 176 | builder.failBuilder(fail_notes) | |
1010 | 75 | # Re-schedule the build if there is one. | 177 | current_job.reset() |
998 | 76 | if current_job is not None: | ||
999 | 77 | current_job.reset() | ||
1000 | 78 | |||
1001 | 79 | # We are a little more tolerant with failing builders than | ||
1002 | 80 | # failing jobs because sometimes they get unresponsive due to | ||
1003 | 81 | # human error, flaky networks etc. We expect the builder to get | ||
1004 | 82 | # better, whereas jobs are very unlikely to get better. | ||
1005 | 83 | if builder.failure_count >= Builder.FAILURE_THRESHOLD: | ||
1006 | 84 | # It's also gone over the threshold so let's disable it. | ||
1007 | 85 | builder.failBuilder(fail_notes) | ||
1011 | 86 | else: | 178 | else: |
1012 | 87 | # The job is the culprit! Override its status to 'failed' | 179 | # The job is the culprit! Override its status to 'failed' |
1013 | 88 | # to make sure it won't get automatically dispatched again, | 180 | # to make sure it won't get automatically dispatched again, |
1014 | 89 | # and remove the buildqueue request. The failure should | 181 | # and remove the buildqueue request. The failure should |
1015 | 90 | # have already caused any relevant slave data to be stored | 182 | # have already caused any relevant slave data to be stored |
1016 | 91 | # on the build record so don't worry about that here. | 183 | # on the build record so don't worry about that here. |
1017 | 92 | build_job = current_job.specific_job.build | ||
1018 | 93 | build_job.status = BuildStatus.FAILEDTOBUILD | 184 | build_job.status = BuildStatus.FAILEDTOBUILD |
1019 | 94 | builder.currentjob.destroySelf() | 185 | builder.currentjob.destroySelf() |
1020 | 95 | 186 | ||
1021 | @@ -99,108 +190,133 @@ | |||
1022 | 99 | # next buildd scan. | 190 | # next buildd scan. |
1023 | 100 | 191 | ||
1024 | 101 | 192 | ||
1025 | 193 | class BaseDispatchResult: | ||
1026 | 194 | """Base class for *DispatchResult variations. | ||
1027 | 195 | |||
1028 | 196 | It will be extended to represent dispatching results and allow | ||
1029 | 197 | homogeneous processing. | ||
1030 | 198 | """ | ||
1031 | 199 | |||
1032 | 200 | def __init__(self, slave, info=None): | ||
1033 | 201 | self.slave = slave | ||
1034 | 202 | self.info = info | ||
1035 | 203 | |||
1036 | 204 | def _cleanJob(self, job): | ||
1037 | 205 | """Clean up in case of builder reset or dispatch failure.""" | ||
1038 | 206 | if job is not None: | ||
1039 | 207 | job.reset() | ||
1040 | 208 | |||
1041 | 209 | def assessFailureCounts(self): | ||
1042 | 210 | """View builder/job failure_count and work out which needs to die. | ||
1043 | 211 | |||
1044 | 212 | :return: True if we disabled something, False if we did not. | ||
1045 | 213 | """ | ||
1046 | 214 | builder = get_builder(self.slave.name) | ||
1047 | 215 | assessFailureCounts(builder, self.info) | ||
1048 | 216 | |||
1049 | 217 | def ___call__(self): | ||
1050 | 218 | raise NotImplementedError( | ||
1051 | 219 | "Call sites must define an evaluation method.") | ||
1052 | 220 | |||
1053 | 221 | |||
1054 | 222 | class FailDispatchResult(BaseDispatchResult): | ||
1055 | 223 | """Represents a communication failure while dispatching a build job.. | ||
1056 | 224 | |||
1057 | 225 | When evaluated this object mark the corresponding `IBuilder` as | ||
1058 | 226 | 'NOK' with the given text as 'failnotes'. It also cleans up the running | ||
1059 | 227 | job (`IBuildQueue`). | ||
1060 | 228 | """ | ||
1061 | 229 | |||
1062 | 230 | def __repr__(self): | ||
1063 | 231 | return '%r failure (%s)' % (self.slave, self.info) | ||
1064 | 232 | |||
1065 | 233 | @write_transaction | ||
1066 | 234 | def __call__(self): | ||
1067 | 235 | self.assessFailureCounts() | ||
1068 | 236 | |||
1069 | 237 | |||
1070 | 238 | class ResetDispatchResult(BaseDispatchResult): | ||
1071 | 239 | """Represents a failure to reset a builder. | ||
1072 | 240 | |||
1073 | 241 | When evaluated this object simply cleans up the running job | ||
1074 | 242 | (`IBuildQueue`) and marks the builder down. | ||
1075 | 243 | """ | ||
1076 | 244 | |||
1077 | 245 | def __repr__(self): | ||
1078 | 246 | return '%r reset failure' % self.slave | ||
1079 | 247 | |||
1080 | 248 | @write_transaction | ||
1081 | 249 | def __call__(self): | ||
1082 | 250 | builder = get_builder(self.slave.name) | ||
1083 | 251 | # Builders that fail to reset should be disabled as per bug | ||
1084 | 252 | # 563353. | ||
1085 | 253 | # XXX Julian bug=586362 | ||
1086 | 254 | # This is disabled until this code is not also used for dispatch | ||
1087 | 255 | # failures where we *don't* want to disable the builder. | ||
1088 | 256 | # builder.failBuilder(self.info) | ||
1089 | 257 | self._cleanJob(builder.currentjob) | ||
1090 | 258 | |||
1091 | 259 | |||
1092 | 102 | class SlaveScanner: | 260 | class SlaveScanner: |
1093 | 103 | """A manager for a single builder.""" | 261 | """A manager for a single builder.""" |
1094 | 104 | 262 | ||
1095 | 105 | # The interval between each poll cycle, in seconds. We'd ideally | ||
1096 | 106 | # like this to be lower but 5 seems a reasonable compromise between | ||
1097 | 107 | # responsivity and load on the database server, since in each cycle | ||
1098 | 108 | # we can run quite a few queries. | ||
1099 | 109 | SCAN_INTERVAL = 5 | 263 | SCAN_INTERVAL = 5 |
1100 | 110 | 264 | ||
1101 | 265 | # These are for the benefit of tests; see `TestingSlaveScanner`. | ||
1102 | 266 | # It pokes fake versions in here so that it can verify methods were | ||
1103 | 267 | # called. The tests should really be using FakeMethod() though. | ||
1104 | 268 | reset_result = ResetDispatchResult | ||
1105 | 269 | fail_result = FailDispatchResult | ||
1106 | 270 | |||
1107 | 111 | def __init__(self, builder_name, logger): | 271 | def __init__(self, builder_name, logger): |
1108 | 112 | self.builder_name = builder_name | 272 | self.builder_name = builder_name |
1109 | 113 | self.logger = logger | 273 | self.logger = logger |
1110 | 274 | self._deferred_list = [] | ||
1111 | 275 | |||
1112 | 276 | def scheduleNextScanCycle(self): | ||
1113 | 277 | """Schedule another scan of the builder some time in the future.""" | ||
1114 | 278 | self._deferred_list = [] | ||
1115 | 279 | # XXX: Change this to use LoopingCall. | ||
1116 | 280 | reactor.callLater(self.SCAN_INTERVAL, self.startCycle) | ||
1117 | 114 | 281 | ||
1118 | 115 | def startCycle(self): | 282 | def startCycle(self): |
1119 | 116 | """Scan the builder and dispatch to it or deal with failures.""" | 283 | """Scan the builder and dispatch to it or deal with failures.""" |
1120 | 117 | self.loop = LoopingCall(self.singleCycle) | ||
1121 | 118 | self.stopping_deferred = self.loop.start(self.SCAN_INTERVAL) | ||
1122 | 119 | return self.stopping_deferred | ||
1123 | 120 | |||
1124 | 121 | def stopCycle(self): | ||
1125 | 122 | """Terminate the LoopingCall.""" | ||
1126 | 123 | self.loop.stop() | ||
1127 | 124 | |||
1128 | 125 | def singleCycle(self): | ||
1129 | 126 | self.logger.debug("Scanning builder: %s" % self.builder_name) | 284 | self.logger.debug("Scanning builder: %s" % self.builder_name) |
1154 | 127 | d = self.scan() | 285 | |
1155 | 128 | 286 | try: | |
1156 | 129 | d.addErrback(self._scanFailed) | 287 | slave = self.scan() |
1157 | 130 | return d | 288 | if slave is None: |
1158 | 131 | 289 | self.scheduleNextScanCycle() | |
1159 | 132 | def _scanFailed(self, failure): | 290 | else: |
1160 | 133 | """Deal with failures encountered during the scan cycle. | 291 | # XXX: Ought to return Deferred. |
1161 | 134 | 292 | self.resumeAndDispatch(slave) | |
1162 | 135 | 1. Print the error in the log | 293 | except: |
1163 | 136 | 2. Increment and assess failure counts on the builder and job. | 294 | error = Failure() |
1140 | 137 | """ | ||
1141 | 138 | # Make sure that pending database updates are removed as it | ||
1142 | 139 | # could leave the database in an inconsistent state (e.g. The | ||
1143 | 140 | # job says it's running but the buildqueue has no builder set). | ||
1144 | 141 | transaction.abort() | ||
1145 | 142 | |||
1146 | 143 | # If we don't recognise the exception include a stack trace with | ||
1147 | 144 | # the error. | ||
1148 | 145 | error_message = failure.getErrorMessage() | ||
1149 | 146 | if failure.check( | ||
1150 | 147 | BuildSlaveFailure, CannotBuild, BuildBehaviorMismatch, | ||
1151 | 148 | CannotResumeHost, BuildDaemonError, CannotFetchFile): | ||
1152 | 149 | self.logger.info("Scanning failed with: %s" % error_message) | ||
1153 | 150 | else: | ||
1164 | 151 | self.logger.info("Scanning failed with: %s\n%s" % | 295 | self.logger.info("Scanning failed with: %s\n%s" % |
1166 | 152 | (failure.getErrorMessage(), failure.getTraceback())) | 296 | (error.getErrorMessage(), error.getTraceback())) |
1167 | 153 | 297 | ||
1168 | 154 | # Decide if we need to terminate the job or fail the | ||
1169 | 155 | # builder. | ||
1170 | 156 | try: | ||
1171 | 157 | builder = get_builder(self.builder_name) | 298 | builder = get_builder(self.builder_name) |
1188 | 158 | builder.gotFailure() | 299 | |
1189 | 159 | if builder.currentjob is not None: | 300 | # Decide if we need to terminate the job or fail the |
1190 | 160 | build_farm_job = builder.getCurrentBuildFarmJob() | 301 | # builder. |
1191 | 161 | build_farm_job.gotFailure() | 302 | self._incrementFailureCounts(builder) |
1192 | 162 | self.logger.info( | 303 | self.logger.info( |
1193 | 163 | "builder %s failure count: %s, " | 304 | "builder failure count: %s, job failure count: %s" % ( |
1194 | 164 | "job '%s' failure count: %s" % ( | 305 | builder.failure_count, |
1195 | 165 | self.builder_name, | 306 | builder.getCurrentBuildFarmJob().failure_count)) |
1196 | 166 | builder.failure_count, | 307 | assessFailureCounts(builder, error.getErrorMessage()) |
1181 | 167 | build_farm_job.title, | ||
1182 | 168 | build_farm_job.failure_count)) | ||
1183 | 169 | else: | ||
1184 | 170 | self.logger.info( | ||
1185 | 171 | "Builder %s failed a probe, count: %s" % ( | ||
1186 | 172 | self.builder_name, builder.failure_count)) | ||
1187 | 173 | assessFailureCounts(builder, failure.getErrorMessage()) | ||
1197 | 174 | transaction.commit() | 308 | transaction.commit() |
1205 | 175 | except: | 309 | |
1206 | 176 | # Catastrophic code failure! Not much we can do. | 310 | self.scheduleNextScanCycle() |
1207 | 177 | self.logger.error( | 311 | |
1208 | 178 | "Miserable failure when trying to examine failure counts:\n", | 312 | @write_transaction |
1202 | 179 | exc_info=True) | ||
1203 | 180 | transaction.abort() | ||
1204 | 181 | |||
1209 | 182 | def scan(self): | 313 | def scan(self): |
1210 | 183 | """Probe the builder and update/dispatch/collect as appropriate. | 314 | """Probe the builder and update/dispatch/collect as appropriate. |
1211 | 184 | 315 | ||
1231 | 185 | There are several steps to scanning: | 316 | The whole method is wrapped in a transaction, but we do partial |
1232 | 186 | 317 | commits to avoid holding locks on tables. | |
1233 | 187 | 1. If the builder is marked as "ok" then probe it to see what state | 318 | |
1234 | 188 | it's in. This is where lost jobs are rescued if we think the | 319 | :return: A `RecordingSlave` if we dispatched a job to it, or None. |
1216 | 189 | builder is doing something that it later tells us it's not, | ||
1217 | 190 | and also where the multi-phase abort procedure happens. | ||
1218 | 191 | See IBuilder.rescueIfLost, which is called by | ||
1219 | 192 | IBuilder.updateStatus(). | ||
1220 | 193 | 2. If the builder is still happy, we ask it if it has an active build | ||
1221 | 194 | and then either update the build in Launchpad or collect the | ||
1222 | 195 | completed build. (builder.updateBuild) | ||
1223 | 196 | 3. If the builder is not happy or it was marked as unavailable | ||
1224 | 197 | mid-build, we need to reset the job that we thought it had, so | ||
1225 | 198 | that the job is dispatched elsewhere. | ||
1226 | 199 | 4. If the builder is idle and we have another build ready, dispatch | ||
1227 | 200 | it. | ||
1228 | 201 | |||
1229 | 202 | :return: A Deferred that fires when the scan is complete, whose | ||
1230 | 203 | value is A `BuilderSlave` if we dispatched a job to it, or None. | ||
1235 | 204 | """ | 320 | """ |
1236 | 205 | # We need to re-fetch the builder object on each cycle as the | 321 | # We need to re-fetch the builder object on each cycle as the |
1237 | 206 | # Storm store is invalidated over transaction boundaries. | 322 | # Storm store is invalidated over transaction boundaries. |
1238 | @@ -208,72 +324,240 @@ | |||
1239 | 208 | self.builder = get_builder(self.builder_name) | 324 | self.builder = get_builder(self.builder_name) |
1240 | 209 | 325 | ||
1241 | 210 | if self.builder.builderok: | 326 | if self.builder.builderok: |
1243 | 211 | d = self.builder.updateStatus(self.logger) | 327 | self.builder.updateStatus(self.logger) |
1244 | 328 | transaction.commit() | ||
1245 | 329 | |||
1246 | 330 | # See if we think there's an active build on the builder. | ||
1247 | 331 | buildqueue = self.builder.getBuildQueue() | ||
1248 | 332 | |||
1249 | 333 | # XXX Julian 2010-07-29 bug=611258 | ||
1250 | 334 | # We're not using the RecordingSlave until dispatching, which | ||
1251 | 335 | # means that this part blocks until we've received a response | ||
1252 | 336 | # from the builder. updateBuild() needs to be made | ||
1253 | 337 | # asyncronous. | ||
1254 | 338 | |||
1255 | 339 | # Scan the slave and get the logtail, or collect the build if | ||
1256 | 340 | # it's ready. Yes, "updateBuild" is a bad name. | ||
1257 | 341 | if buildqueue is not None: | ||
1258 | 342 | self.builder.updateBuild(buildqueue) | ||
1259 | 343 | transaction.commit() | ||
1260 | 344 | |||
1261 | 345 | # If the builder is in manual mode, don't dispatch anything. | ||
1262 | 346 | if self.builder.manual: | ||
1263 | 347 | self.logger.debug( | ||
1264 | 348 | '%s is in manual mode, not dispatching.' % self.builder.name) | ||
1265 | 349 | return None | ||
1266 | 350 | |||
1267 | 351 | # If the builder is marked unavailable, don't dispatch anything. | ||
1268 | 352 | # Additionaly, because builders can be removed from the pool at | ||
1269 | 353 | # any time, we need to see if we think there was a build running | ||
1270 | 354 | # on it before it was marked unavailable. In this case we reset | ||
1271 | 355 | # the build thusly forcing it to get re-dispatched to another | ||
1272 | 356 | # builder. | ||
1273 | 357 | if not self.builder.is_available: | ||
1274 | 358 | job = self.builder.currentjob | ||
1275 | 359 | if job is not None and not self.builder.builderok: | ||
1276 | 360 | self.logger.info( | ||
1277 | 361 | "%s was made unavailable, resetting attached " | ||
1278 | 362 | "job" % self.builder.name) | ||
1279 | 363 | job.reset() | ||
1280 | 364 | transaction.commit() | ||
1281 | 365 | return None | ||
1282 | 366 | |||
1283 | 367 | # See if there is a job we can dispatch to the builder slave. | ||
1284 | 368 | |||
1285 | 369 | # XXX: Rather than use the slave actually associated with the builder | ||
1286 | 370 | # (which, incidentally, shouldn't be a property anyway), we make a new | ||
1287 | 371 | # RecordingSlave so we can get access to its asynchronous | ||
1288 | 372 | # "resumeSlave" method. Blech. | ||
1289 | 373 | slave = RecordingSlave( | ||
1290 | 374 | self.builder.name, self.builder.url, self.builder.vm_host) | ||
1291 | 375 | # XXX: Passing buildd_slave=slave overwrites the 'slave' property of | ||
1292 | 376 | # self.builder. Not sure why this is needed yet. | ||
1293 | 377 | self.builder.findAndStartJob(buildd_slave=slave) | ||
1294 | 378 | if self.builder.currentjob is not None: | ||
1295 | 379 | # After a successful dispatch we can reset the | ||
1296 | 380 | # failure_count. | ||
1297 | 381 | self.builder.resetFailureCount() | ||
1298 | 382 | transaction.commit() | ||
1299 | 383 | return slave | ||
1300 | 384 | |||
1301 | 385 | return None | ||
1302 | 386 | |||
1303 | 387 | def resumeAndDispatch(self, slave): | ||
1304 | 388 | """Chain the resume and dispatching Deferreds.""" | ||
1305 | 389 | # XXX: resumeAndDispatch makes Deferreds without returning them. | ||
1306 | 390 | if slave.resume_requested: | ||
1307 | 391 | # The slave needs to be reset before we can dispatch to | ||
1308 | 392 | # it (e.g. a virtual slave) | ||
1309 | 393 | |||
1310 | 394 | # XXX: Two problems here. The first is that 'resumeSlave' only | ||
1311 | 395 | # exists on RecordingSlave (BuilderSlave calls it 'resume'). | ||
1312 | 396 | d = slave.resumeSlave() | ||
1313 | 397 | d.addBoth(self.checkResume, slave) | ||
1314 | 212 | else: | 398 | else: |
1315 | 399 | # No resume required, build dispatching can commence. | ||
1316 | 213 | d = defer.succeed(None) | 400 | d = defer.succeed(None) |
1317 | 214 | 401 | ||
1374 | 215 | def status_updated(ignored): | 402 | # Dispatch the build to the slave asynchronously. |
1375 | 216 | # Commit the changes done while possibly rescuing jobs, to | 403 | d.addCallback(self.initiateDispatch, slave) |
1376 | 217 | # avoid holding table locks. | 404 | # Store this deferred so we can wait for it along with all |
1377 | 218 | transaction.commit() | 405 | # the others that will be generated by RecordingSlave during |
1378 | 219 | 406 | # the dispatch process, and chain a callback after they've | |
1379 | 220 | # See if we think there's an active build on the builder. | 407 | # all fired. |
1380 | 221 | buildqueue = self.builder.getBuildQueue() | 408 | self._deferred_list.append(d) |
1381 | 222 | 409 | ||
1382 | 223 | # Scan the slave and get the logtail, or collect the build if | 410 | def initiateDispatch(self, resume_result, slave): |
1383 | 224 | # it's ready. Yes, "updateBuild" is a bad name. | 411 | """Start dispatching a build to a slave. |
1384 | 225 | if buildqueue is not None: | 412 | |
1385 | 226 | return self.builder.updateBuild(buildqueue) | 413 | If the previous task in chain (slave resuming) has failed it will |
1386 | 227 | 414 | receive a `ResetBuilderRequest` instance as 'resume_result' and | |
1387 | 228 | def build_updated(ignored): | 415 | will immediately return that so the subsequent callback can collect |
1388 | 229 | # Commit changes done while updating the build, to avoid | 416 | it. |
1389 | 230 | # holding table locks. | 417 | |
1390 | 231 | transaction.commit() | 418 | If the slave resuming succeeded, it starts the XMLRPC dialogue. The |
1391 | 232 | 419 | dialogue may consist of many calls to the slave before the build | |
1392 | 233 | # If the builder is in manual mode, don't dispatch anything. | 420 | starts. Each call is done via a Deferred event, where slave calls |
1393 | 234 | if self.builder.manual: | 421 | are sent in callSlave(), and checked in checkDispatch() which will |
1394 | 235 | self.logger.debug( | 422 | keep firing events via callSlave() until all the events are done or |
1395 | 236 | '%s is in manual mode, not dispatching.' % | 423 | an error occurs. |
1396 | 237 | self.builder.name) | 424 | """ |
1397 | 238 | return | 425 | if resume_result is not None: |
1398 | 239 | 426 | self.slaveConversationEnded() | |
1399 | 240 | # If the builder is marked unavailable, don't dispatch anything. | 427 | return resume_result |
1400 | 241 | # Additionaly, because builders can be removed from the pool at | 428 | |
1401 | 242 | # any time, we need to see if we think there was a build running | 429 | self.logger.info('Dispatching: %s' % slave) |
1402 | 243 | # on it before it was marked unavailable. In this case we reset | 430 | self.callSlave(slave) |
1403 | 244 | # the build thusly forcing it to get re-dispatched to another | 431 | |
1404 | 245 | # builder. | 432 | def _getProxyForSlave(self, slave): |
1405 | 246 | 433 | """Return a twisted.web.xmlrpc.Proxy for the buildd slave. | |
1406 | 247 | return self.builder.isAvailable().addCallback(got_available) | 434 | |
1407 | 248 | 435 | Uses a protocol with timeout support, See QueryFactoryWithTimeout. | |
1408 | 249 | def got_available(available): | 436 | """ |
1409 | 250 | if not available: | 437 | proxy = xmlrpc.Proxy(str(urlappend(slave.url, 'rpc'))) |
1410 | 251 | job = self.builder.currentjob | 438 | proxy.queryFactory = QueryFactoryWithTimeout |
1411 | 252 | if job is not None and not self.builder.builderok: | 439 | return proxy |
1412 | 253 | self.logger.info( | 440 | |
1413 | 254 | "%s was made unavailable, resetting attached " | 441 | def callSlave(self, slave): |
1414 | 255 | "job" % self.builder.name) | 442 | """Dispatch the next XMLRPC for the given slave.""" |
1415 | 256 | job.reset() | 443 | if len(slave.calls) == 0: |
1416 | 257 | transaction.commit() | 444 | # That's the end of the dialogue with the slave. |
1417 | 258 | return | 445 | self.slaveConversationEnded() |
1418 | 259 | 446 | return | |
1419 | 260 | # See if there is a job we can dispatch to the builder slave. | 447 | |
1420 | 261 | 448 | # Get an XMLRPC proxy for the buildd slave. | |
1421 | 262 | d = self.builder.findAndStartJob() | 449 | proxy = self._getProxyForSlave(slave) |
1422 | 263 | def job_started(candidate): | 450 | method, args = slave.calls.pop(0) |
1423 | 264 | if self.builder.currentjob is not None: | 451 | d = proxy.callRemote(method, *args) |
1424 | 265 | # After a successful dispatch we can reset the | 452 | d.addBoth(self.checkDispatch, method, slave) |
1425 | 266 | # failure_count. | 453 | self._deferred_list.append(d) |
1426 | 267 | self.builder.resetFailureCount() | 454 | self.logger.debug('%s -> %s(%s)' % (slave, method, args)) |
1427 | 268 | transaction.commit() | 455 | |
1428 | 269 | return self.builder.slave | 456 | def slaveConversationEnded(self): |
1429 | 270 | else: | 457 | """After all the Deferreds are set up, chain a callback on them.""" |
1430 | 458 | dl = defer.DeferredList(self._deferred_list, consumeErrors=True) | ||
1431 | 459 | dl.addBoth(self.evaluateDispatchResult) | ||
1432 | 460 | return dl | ||
1433 | 461 | |||
1434 | 462 | def evaluateDispatchResult(self, deferred_list_results): | ||
1435 | 463 | """Process the DispatchResult for this dispatch chain. | ||
1436 | 464 | |||
1437 | 465 | After waiting for the Deferred chain to finish, we'll have a | ||
1438 | 466 | DispatchResult to evaluate, which deals with the result of | ||
1439 | 467 | dispatching. | ||
1440 | 468 | """ | ||
1441 | 469 | # The `deferred_list_results` is what we get when waiting on a | ||
1442 | 470 | # DeferredList. It's a list of tuples of (status, result) where | ||
1443 | 471 | # result is what the last callback in that chain returned. | ||
1444 | 472 | |||
1445 | 473 | # If the result is an instance of BaseDispatchResult we need to | ||
1446 | 474 | # evaluate it, as there's further action required at the end of | ||
1447 | 475 | # the dispatch chain. None, resulting from successful chains, | ||
1448 | 476 | # are discarded. | ||
1449 | 477 | |||
1450 | 478 | dispatch_results = [ | ||
1451 | 479 | result for status, result in deferred_list_results | ||
1452 | 480 | if isinstance(result, BaseDispatchResult)] | ||
1453 | 481 | |||
1454 | 482 | for result in dispatch_results: | ||
1455 | 483 | self.logger.info("%r" % result) | ||
1456 | 484 | result() | ||
1457 | 485 | |||
1458 | 486 | # At this point, we're done dispatching, so we can schedule the | ||
1459 | 487 | # next scan cycle. | ||
1460 | 488 | self.scheduleNextScanCycle() | ||
1461 | 489 | |||
1462 | 490 | # For the test suite so that it can chain callback results. | ||
1463 | 491 | return deferred_list_results | ||
1464 | 492 | |||
1465 | 493 | def checkResume(self, response, slave): | ||
1466 | 494 | """Check the result of resuming a slave. | ||
1467 | 495 | |||
1468 | 496 | If there's a problem resuming, we return a ResetDispatchResult which | ||
1469 | 497 | will get evaluated at the end of the scan, or None if the resume | ||
1470 | 498 | was OK. | ||
1471 | 499 | |||
1472 | 500 | :param response: the tuple that's constructed in | ||
1473 | 501 | ProcessWithTimeout.processEnded(), or a Failure that | ||
1474 | 502 | contains the tuple. | ||
1475 | 503 | :param slave: the slave object we're talking to | ||
1476 | 504 | """ | ||
1477 | 505 | if isinstance(response, Failure): | ||
1478 | 506 | out, err, code = response.value | ||
1479 | 507 | else: | ||
1480 | 508 | out, err, code = response | ||
1481 | 509 | if code == os.EX_OK: | ||
1482 | 510 | return None | ||
1483 | 511 | |||
1484 | 512 | error_text = '%s\n%s' % (out, err) | ||
1485 | 513 | self.logger.error('%s resume failure: %s' % (slave, error_text)) | ||
1486 | 514 | return self.reset_result(slave, error_text) | ||
1487 | 515 | |||
1488 | 516 | def _incrementFailureCounts(self, builder): | ||
1489 | 517 | builder.gotFailure() | ||
1490 | 518 | builder.getCurrentBuildFarmJob().gotFailure() | ||
1491 | 519 | |||
1492 | 520 | def checkDispatch(self, response, method, slave): | ||
1493 | 521 | """Verify the results of a slave xmlrpc call. | ||
1494 | 522 | |||
1495 | 523 | If it failed and it compromises the slave then return a corresponding | ||
1496 | 524 | `FailDispatchResult`, if it was a communication failure, simply | ||
1497 | 525 | reset the slave by returning a `ResetDispatchResult`. | ||
1498 | 526 | """ | ||
1499 | 527 | from lp.buildmaster.interfaces.builder import IBuilderSet | ||
1500 | 528 | builder = getUtility(IBuilderSet)[slave.name] | ||
1501 | 529 | |||
1502 | 530 | # XXX these DispatchResult classes are badly named and do the | ||
1503 | 531 | # same thing. We need to fix that. | ||
1504 | 532 | self.logger.debug( | ||
1505 | 533 | '%s response for "%s": %s' % (slave, method, response)) | ||
1506 | 534 | |||
1507 | 535 | if isinstance(response, Failure): | ||
1508 | 536 | self.logger.warn( | ||
1509 | 537 | '%s communication failed (%s)' % | ||
1510 | 538 | (slave, response.getErrorMessage())) | ||
1511 | 539 | self.slaveConversationEnded() | ||
1512 | 540 | self._incrementFailureCounts(builder) | ||
1513 | 541 | return self.fail_result(slave) | ||
1514 | 542 | |||
1515 | 543 | if isinstance(response, list) and len(response) == 2: | ||
1516 | 544 | if method in buildd_success_result_map: | ||
1517 | 545 | expected_status = buildd_success_result_map.get(method) | ||
1518 | 546 | status, info = response | ||
1519 | 547 | if status == expected_status: | ||
1520 | 548 | self.callSlave(slave) | ||
1521 | 271 | return None | 549 | return None |
1527 | 272 | return d.addCallback(job_started) | 550 | else: |
1528 | 273 | 551 | info = 'Unknown slave method: %s' % method | |
1529 | 274 | d.addCallback(status_updated) | 552 | else: |
1530 | 275 | d.addCallback(build_updated) | 553 | info = 'Unexpected response: %s' % repr(response) |
1531 | 276 | return d | 554 | |
1532 | 555 | self.logger.error( | ||
1533 | 556 | '%s failed to dispatch (%s)' % (slave, info)) | ||
1534 | 557 | |||
1535 | 558 | self.slaveConversationEnded() | ||
1536 | 559 | self._incrementFailureCounts(builder) | ||
1537 | 560 | return self.fail_result(slave, info) | ||
1538 | 277 | 561 | ||
1539 | 278 | 562 | ||
1540 | 279 | class NewBuildersScanner: | 563 | class NewBuildersScanner: |
1541 | @@ -294,21 +578,15 @@ | |||
1542 | 294 | self.current_builders = [ | 578 | self.current_builders = [ |
1543 | 295 | builder.name for builder in getUtility(IBuilderSet)] | 579 | builder.name for builder in getUtility(IBuilderSet)] |
1544 | 296 | 580 | ||
1545 | 297 | def stop(self): | ||
1546 | 298 | """Terminate the LoopingCall.""" | ||
1547 | 299 | self.loop.stop() | ||
1548 | 300 | |||
1549 | 301 | def scheduleScan(self): | 581 | def scheduleScan(self): |
1550 | 302 | """Schedule a callback SCAN_INTERVAL seconds later.""" | 582 | """Schedule a callback SCAN_INTERVAL seconds later.""" |
1555 | 303 | self.loop = LoopingCall(self.scan) | 583 | return self._clock.callLater(self.SCAN_INTERVAL, self.scan) |
1552 | 304 | self.loop.clock = self._clock | ||
1553 | 305 | self.stopping_deferred = self.loop.start(self.SCAN_INTERVAL) | ||
1554 | 306 | return self.stopping_deferred | ||
1556 | 307 | 584 | ||
1557 | 308 | def scan(self): | 585 | def scan(self): |
1558 | 309 | """If a new builder appears, create a SlaveScanner for it.""" | 586 | """If a new builder appears, create a SlaveScanner for it.""" |
1559 | 310 | new_builders = self.checkForNewBuilders() | 587 | new_builders = self.checkForNewBuilders() |
1560 | 311 | self.manager.addScanForBuilders(new_builders) | 588 | self.manager.addScanForBuilders(new_builders) |
1561 | 589 | self.scheduleScan() | ||
1562 | 312 | 590 | ||
1563 | 313 | def checkForNewBuilders(self): | 591 | def checkForNewBuilders(self): |
1564 | 314 | """See if any new builders were added.""" | 592 | """See if any new builders were added.""" |
1565 | @@ -331,7 +609,10 @@ | |||
1566 | 331 | manager=self, clock=clock) | 609 | manager=self, clock=clock) |
1567 | 332 | 610 | ||
1568 | 333 | def _setupLogger(self): | 611 | def _setupLogger(self): |
1570 | 334 | """Set up a 'slave-scanner' logger that redirects to twisted. | 612 | """Setup a 'slave-scanner' logger that redirects to twisted. |
1571 | 613 | |||
1572 | 614 | It is going to be used locally and within the thread running | ||
1573 | 615 | the scan() method. | ||
1574 | 335 | 616 | ||
1575 | 336 | Make it less verbose to avoid messing too much with the old code. | 617 | Make it less verbose to avoid messing too much with the old code. |
1576 | 337 | """ | 618 | """ |
1577 | @@ -362,29 +643,12 @@ | |||
1578 | 362 | # Events will now fire in the SlaveScanner objects to scan each | 643 | # Events will now fire in the SlaveScanner objects to scan each |
1579 | 363 | # builder. | 644 | # builder. |
1580 | 364 | 645 | ||
1581 | 365 | def stopService(self): | ||
1582 | 366 | """Callback for when we need to shut down.""" | ||
1583 | 367 | # XXX: lacks unit tests | ||
1584 | 368 | # All the SlaveScanner objects need to be halted gracefully. | ||
1585 | 369 | deferreds = [slave.stopping_deferred for slave in self.builder_slaves] | ||
1586 | 370 | deferreds.append(self.new_builders_scanner.stopping_deferred) | ||
1587 | 371 | |||
1588 | 372 | self.new_builders_scanner.stop() | ||
1589 | 373 | for slave in self.builder_slaves: | ||
1590 | 374 | slave.stopCycle() | ||
1591 | 375 | |||
1592 | 376 | # The 'stopping_deferred's are called back when the loops are | ||
1593 | 377 | # stopped, so we can wait on them all at once here before | ||
1594 | 378 | # exiting. | ||
1595 | 379 | d = defer.DeferredList(deferreds, consumeErrors=True) | ||
1596 | 380 | return d | ||
1597 | 381 | |||
1598 | 382 | def addScanForBuilders(self, builders): | 646 | def addScanForBuilders(self, builders): |
1599 | 383 | """Set up scanner objects for the builders specified.""" | 647 | """Set up scanner objects for the builders specified.""" |
1600 | 384 | for builder in builders: | 648 | for builder in builders: |
1601 | 385 | slave_scanner = SlaveScanner(builder, self.logger) | 649 | slave_scanner = SlaveScanner(builder, self.logger) |
1602 | 386 | self.builder_slaves.append(slave_scanner) | 650 | self.builder_slaves.append(slave_scanner) |
1604 | 387 | slave_scanner.startCycle() | 651 | slave_scanner.scheduleNextScanCycle() |
1605 | 388 | 652 | ||
1606 | 389 | # Return the slave list for the benefit of tests. | 653 | # Return the slave list for the benefit of tests. |
1607 | 390 | return self.builder_slaves | 654 | return self.builder_slaves |
1608 | 391 | 655 | ||
1609 | === modified file 'lib/lp/buildmaster/model/builder.py' | |||
1610 | --- lib/lp/buildmaster/model/builder.py 2010-10-20 11:54:27 +0000 | |||
1611 | +++ lib/lp/buildmaster/model/builder.py 2010-12-07 16:29:13 +0000 | |||
1612 | @@ -13,11 +13,12 @@ | |||
1613 | 13 | ] | 13 | ] |
1614 | 14 | 14 | ||
1615 | 15 | import gzip | 15 | import gzip |
1616 | 16 | import httplib | ||
1617 | 16 | import logging | 17 | import logging |
1618 | 17 | import os | 18 | import os |
1619 | 18 | import socket | 19 | import socket |
1620 | 20 | import subprocess | ||
1621 | 19 | import tempfile | 21 | import tempfile |
1622 | 20 | import transaction | ||
1623 | 21 | import urllib2 | 22 | import urllib2 |
1624 | 22 | import xmlrpclib | 23 | import xmlrpclib |
1625 | 23 | 24 | ||
1626 | @@ -33,13 +34,6 @@ | |||
1627 | 33 | Count, | 34 | Count, |
1628 | 34 | Sum, | 35 | Sum, |
1629 | 35 | ) | 36 | ) |
1630 | 36 | |||
1631 | 37 | from twisted.internet import ( | ||
1632 | 38 | defer, | ||
1633 | 39 | reactor as default_reactor, | ||
1634 | 40 | ) | ||
1635 | 41 | from twisted.web import xmlrpc | ||
1636 | 42 | |||
1637 | 43 | from zope.component import getUtility | 37 | from zope.component import getUtility |
1638 | 44 | from zope.interface import implements | 38 | from zope.interface import implements |
1639 | 45 | 39 | ||
1640 | @@ -64,6 +58,7 @@ | |||
1641 | 64 | from lp.buildmaster.interfaces.builder import ( | 58 | from lp.buildmaster.interfaces.builder import ( |
1642 | 65 | BuildDaemonError, | 59 | BuildDaemonError, |
1643 | 66 | BuildSlaveFailure, | 60 | BuildSlaveFailure, |
1644 | 61 | CannotBuild, | ||
1645 | 67 | CannotFetchFile, | 62 | CannotFetchFile, |
1646 | 68 | CannotResumeHost, | 63 | CannotResumeHost, |
1647 | 69 | CorruptBuildCookie, | 64 | CorruptBuildCookie, |
1648 | @@ -71,6 +66,9 @@ | |||
1649 | 71 | IBuilderSet, | 66 | IBuilderSet, |
1650 | 72 | ) | 67 | ) |
1651 | 73 | from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJobSet | 68 | from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJobSet |
1652 | 69 | from lp.buildmaster.interfaces.buildfarmjobbehavior import ( | ||
1653 | 70 | BuildBehaviorMismatch, | ||
1654 | 71 | ) | ||
1655 | 74 | from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet | 72 | from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet |
1656 | 75 | from lp.buildmaster.model.buildfarmjobbehavior import IdleBuildBehavior | 73 | from lp.buildmaster.model.buildfarmjobbehavior import IdleBuildBehavior |
1657 | 76 | from lp.buildmaster.model.buildqueue import ( | 74 | from lp.buildmaster.model.buildqueue import ( |
1658 | @@ -80,9 +78,9 @@ | |||
1659 | 80 | from lp.registry.interfaces.person import validate_public_person | 78 | from lp.registry.interfaces.person import validate_public_person |
1660 | 81 | from lp.services.job.interfaces.job import JobStatus | 79 | from lp.services.job.interfaces.job import JobStatus |
1661 | 82 | from lp.services.job.model.job import Job | 80 | from lp.services.job.model.job import Job |
1662 | 81 | from lp.services.osutils import until_no_eintr | ||
1663 | 83 | from lp.services.propertycache import cachedproperty | 82 | from lp.services.propertycache import cachedproperty |
1666 | 84 | from lp.services.twistedsupport.processmonitor import ProcessWithTimeout | 83 | from lp.services.twistedsupport.xmlrpc import BlockingProxy |
1665 | 85 | from lp.services.twistedsupport import cancel_on_timeout | ||
1667 | 86 | # XXX Michael Nelson 2010-01-13 bug=491330 | 84 | # XXX Michael Nelson 2010-01-13 bug=491330 |
1668 | 87 | # These dependencies on soyuz will be removed when getBuildRecords() | 85 | # These dependencies on soyuz will be removed when getBuildRecords() |
1669 | 88 | # is moved. | 86 | # is moved. |
1670 | @@ -94,9 +92,25 @@ | |||
1671 | 94 | from lp.soyuz.model.processor import Processor | 92 | from lp.soyuz.model.processor import Processor |
1672 | 95 | 93 | ||
1673 | 96 | 94 | ||
1677 | 97 | class QuietQueryFactory(xmlrpc._QueryFactory): | 95 | class TimeoutHTTPConnection(httplib.HTTPConnection): |
1678 | 98 | """XMLRPC client factory that doesn't splatter the log with junk.""" | 96 | |
1679 | 99 | noisy = False | 97 | def connect(self): |
1680 | 98 | """Override the standard connect() methods to set a timeout""" | ||
1681 | 99 | ret = httplib.HTTPConnection.connect(self) | ||
1682 | 100 | self.sock.settimeout(config.builddmaster.socket_timeout) | ||
1683 | 101 | return ret | ||
1684 | 102 | |||
1685 | 103 | |||
1686 | 104 | class TimeoutHTTP(httplib.HTTP): | ||
1687 | 105 | _connection_class = TimeoutHTTPConnection | ||
1688 | 106 | |||
1689 | 107 | |||
1690 | 108 | class TimeoutTransport(xmlrpclib.Transport): | ||
1691 | 109 | """XMLRPC Transport to setup a socket with defined timeout""" | ||
1692 | 110 | |||
1693 | 111 | def make_connection(self, host): | ||
1694 | 112 | host, extra_headers, x509 = self.get_host_info(host) | ||
1695 | 113 | return TimeoutHTTP(host) | ||
1696 | 100 | 114 | ||
1697 | 101 | 115 | ||
1698 | 102 | class BuilderSlave(object): | 116 | class BuilderSlave(object): |
1699 | @@ -111,7 +125,24 @@ | |||
1700 | 111 | # many false positives in your test run and will most likely break | 125 | # many false positives in your test run and will most likely break |
1701 | 112 | # production. | 126 | # production. |
1702 | 113 | 127 | ||
1704 | 114 | def __init__(self, proxy, builder_url, vm_host, reactor=None): | 128 | # XXX: This (BuilderSlave) should use composition, rather than |
1705 | 129 | # inheritance. | ||
1706 | 130 | |||
1707 | 131 | # XXX: Have a documented interface for the XML-RPC server: | ||
1708 | 132 | # - what methods | ||
1709 | 133 | # - what return values expected | ||
1710 | 134 | # - what faults | ||
1711 | 135 | # (see XMLRPCBuildDSlave in lib/canonical/buildd/slave.py). | ||
1712 | 136 | |||
1713 | 137 | # XXX: Arguably, this interface should be asynchronous | ||
1714 | 138 | # (i.e. Deferred-returning). This would mean that Builder (see below) | ||
1715 | 139 | # would have to expect Deferreds. | ||
1716 | 140 | |||
1717 | 141 | # XXX: Once we have a client object with a defined, tested interface, we | ||
1718 | 142 | # should make a test double that doesn't do any XML-RPC and can be used to | ||
1719 | 143 | # make testing easier & tests faster. | ||
1720 | 144 | |||
1721 | 145 | def __init__(self, proxy, builder_url, vm_host): | ||
1722 | 115 | """Initialize a BuilderSlave. | 146 | """Initialize a BuilderSlave. |
1723 | 116 | 147 | ||
1724 | 117 | :param proxy: An XML-RPC proxy, implementing 'callRemote'. It must | 148 | :param proxy: An XML-RPC proxy, implementing 'callRemote'. It must |
1725 | @@ -124,87 +155,63 @@ | |||
1726 | 124 | self._file_cache_url = urlappend(builder_url, 'filecache') | 155 | self._file_cache_url = urlappend(builder_url, 'filecache') |
1727 | 125 | self._server = proxy | 156 | self._server = proxy |
1728 | 126 | 157 | ||
1729 | 127 | if reactor is None: | ||
1730 | 128 | self.reactor = default_reactor | ||
1731 | 129 | else: | ||
1732 | 130 | self.reactor = reactor | ||
1733 | 131 | |||
1734 | 132 | @classmethod | 158 | @classmethod |
1755 | 133 | def makeBuilderSlave(cls, builder_url, vm_host, reactor=None, proxy=None): | 159 | def makeBlockingSlave(cls, builder_url, vm_host): |
1756 | 134 | """Create and return a `BuilderSlave`. | 160 | rpc_url = urlappend(builder_url, 'rpc') |
1757 | 135 | 161 | server_proxy = xmlrpclib.ServerProxy( | |
1758 | 136 | :param builder_url: The URL of the slave buildd machine, | 162 | rpc_url, transport=TimeoutTransport(), allow_none=True) |
1759 | 137 | e.g. http://localhost:8221 | 163 | return cls(BlockingProxy(server_proxy), builder_url, vm_host) |
1740 | 138 | :param vm_host: If the slave is virtual, specify its host machine here. | ||
1741 | 139 | :param reactor: Used by tests to override the Twisted reactor. | ||
1742 | 140 | :param proxy: Used By tests to override the xmlrpc.Proxy. | ||
1743 | 141 | """ | ||
1744 | 142 | rpc_url = urlappend(builder_url.encode('utf-8'), 'rpc') | ||
1745 | 143 | if proxy is None: | ||
1746 | 144 | server_proxy = xmlrpc.Proxy(rpc_url, allowNone=True) | ||
1747 | 145 | server_proxy.queryFactory = QuietQueryFactory | ||
1748 | 146 | else: | ||
1749 | 147 | server_proxy = proxy | ||
1750 | 148 | return cls(server_proxy, builder_url, vm_host, reactor) | ||
1751 | 149 | |||
1752 | 150 | def _with_timeout(self, d): | ||
1753 | 151 | TIMEOUT = config.builddmaster.socket_timeout | ||
1754 | 152 | return cancel_on_timeout(d, TIMEOUT, self.reactor) | ||
1760 | 153 | 164 | ||
1761 | 154 | def abort(self): | 165 | def abort(self): |
1762 | 155 | """Abort the current build.""" | 166 | """Abort the current build.""" |
1764 | 156 | return self._with_timeout(self._server.callRemote('abort')) | 167 | return self._server.callRemote('abort') |
1765 | 157 | 168 | ||
1766 | 158 | def clean(self): | 169 | def clean(self): |
1767 | 159 | """Clean up the waiting files and reset the slave's internal state.""" | 170 | """Clean up the waiting files and reset the slave's internal state.""" |
1769 | 160 | return self._with_timeout(self._server.callRemote('clean')) | 171 | return self._server.callRemote('clean') |
1770 | 161 | 172 | ||
1771 | 162 | def echo(self, *args): | 173 | def echo(self, *args): |
1772 | 163 | """Echo the arguments back.""" | 174 | """Echo the arguments back.""" |
1774 | 164 | return self._with_timeout(self._server.callRemote('echo', *args)) | 175 | return self._server.callRemote('echo', *args) |
1775 | 165 | 176 | ||
1776 | 166 | def info(self): | 177 | def info(self): |
1777 | 167 | """Return the protocol version and the builder methods supported.""" | 178 | """Return the protocol version and the builder methods supported.""" |
1779 | 168 | return self._with_timeout(self._server.callRemote('info')) | 179 | return self._server.callRemote('info') |
1780 | 169 | 180 | ||
1781 | 170 | def status(self): | 181 | def status(self): |
1782 | 171 | """Return the status of the build daemon.""" | 182 | """Return the status of the build daemon.""" |
1784 | 172 | return self._with_timeout(self._server.callRemote('status')) | 183 | return self._server.callRemote('status') |
1785 | 173 | 184 | ||
1786 | 174 | def ensurepresent(self, sha1sum, url, username, password): | 185 | def ensurepresent(self, sha1sum, url, username, password): |
1787 | 175 | # XXX: Nothing external calls this. Make it private. | ||
1788 | 176 | """Attempt to ensure the given file is present.""" | 186 | """Attempt to ensure the given file is present.""" |
1791 | 177 | return self._with_timeout(self._server.callRemote( | 187 | return self._server.callRemote( |
1792 | 178 | 'ensurepresent', sha1sum, url, username, password)) | 188 | 'ensurepresent', sha1sum, url, username, password) |
1793 | 179 | 189 | ||
1794 | 180 | def getFile(self, sha_sum): | 190 | def getFile(self, sha_sum): |
1795 | 181 | """Construct a file-like object to return the named file.""" | 191 | """Construct a file-like object to return the named file.""" |
1796 | 182 | # XXX 2010-10-18 bug=662631 | ||
1797 | 183 | # Change this to do non-blocking IO. | ||
1798 | 184 | file_url = urlappend(self._file_cache_url, sha_sum) | 192 | file_url = urlappend(self._file_cache_url, sha_sum) |
1799 | 185 | return urllib2.urlopen(file_url) | 193 | return urllib2.urlopen(file_url) |
1800 | 186 | 194 | ||
1812 | 187 | def resume(self, clock=None): | 195 | def resume(self): |
1813 | 188 | """Resume the builder in an asynchronous fashion. | 196 | """Resume a virtual builder. |
1814 | 189 | 197 | ||
1815 | 190 | We use the builddmaster configuration 'socket_timeout' as | 198 | It uses the configuration command-line (replacing 'vm_host') and |
1816 | 191 | the process timeout. | 199 | return its output. |
1817 | 192 | 200 | ||
1818 | 193 | :param clock: An optional twisted.internet.task.Clock to override | 201 | :return: a (stdout, stderr, subprocess exitcode) triple |
1808 | 194 | the default clock. For use in tests. | ||
1809 | 195 | |||
1810 | 196 | :return: a Deferred that returns a | ||
1811 | 197 | (stdout, stderr, subprocess exitcode) triple | ||
1819 | 198 | """ | 202 | """ |
1820 | 203 | # XXX: This executes the vm_resume_command | ||
1821 | 204 | # synchronously. RecordingSlave does so asynchronously. Since we | ||
1822 | 205 | # always want to do this asynchronously, there's no need for the | ||
1823 | 206 | # duplication. | ||
1824 | 199 | resume_command = config.builddmaster.vm_resume_command % { | 207 | resume_command = config.builddmaster.vm_resume_command % { |
1825 | 200 | 'vm_host': self._vm_host} | 208 | 'vm_host': self._vm_host} |
1833 | 201 | # Twisted API requires string but the configuration provides unicode. | 209 | resume_argv = resume_command.split() |
1834 | 202 | resume_argv = [term.encode('utf-8') for term in resume_command.split()] | 210 | resume_process = subprocess.Popen( |
1835 | 203 | d = defer.Deferred() | 211 | resume_argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
1836 | 204 | p = ProcessWithTimeout( | 212 | stdout, stderr = resume_process.communicate() |
1837 | 205 | d, config.builddmaster.socket_timeout, clock=clock) | 213 | |
1838 | 206 | p.spawnProcess(resume_argv[0], tuple(resume_argv)) | 214 | return (stdout, stderr, resume_process.returncode) |
1832 | 207 | return d | ||
1839 | 208 | 215 | ||
1840 | 209 | def cacheFile(self, logger, libraryfilealias): | 216 | def cacheFile(self, logger, libraryfilealias): |
1841 | 210 | """Make sure that the file at 'libraryfilealias' is on the slave. | 217 | """Make sure that the file at 'libraryfilealias' is on the slave. |
1842 | @@ -217,15 +224,13 @@ | |||
1843 | 217 | "Asking builder on %s to ensure it has file %s (%s, %s)" % ( | 224 | "Asking builder on %s to ensure it has file %s (%s, %s)" % ( |
1844 | 218 | self._file_cache_url, libraryfilealias.filename, url, | 225 | self._file_cache_url, libraryfilealias.filename, url, |
1845 | 219 | libraryfilealias.content.sha1)) | 226 | libraryfilealias.content.sha1)) |
1847 | 220 | return self.sendFileToSlave(libraryfilealias.content.sha1, url) | 227 | self.sendFileToSlave(libraryfilealias.content.sha1, url) |
1848 | 221 | 228 | ||
1849 | 222 | def sendFileToSlave(self, sha1, url, username="", password=""): | 229 | def sendFileToSlave(self, sha1, url, username="", password=""): |
1850 | 223 | """Helper to send the file at 'url' with 'sha1' to this builder.""" | 230 | """Helper to send the file at 'url' with 'sha1' to this builder.""" |
1856 | 224 | d = self.ensurepresent(sha1, url, username, password) | 231 | present, info = self.ensurepresent(sha1, url, username, password) |
1857 | 225 | def check_present((present, info)): | 232 | if not present: |
1858 | 226 | if not present: | 233 | raise CannotFetchFile(url, info) |
1854 | 227 | raise CannotFetchFile(url, info) | ||
1855 | 228 | return d.addCallback(check_present) | ||
1859 | 229 | 234 | ||
1860 | 230 | def build(self, buildid, builder_type, chroot_sha1, filemap, args): | 235 | def build(self, buildid, builder_type, chroot_sha1, filemap, args): |
1861 | 231 | """Build a thing on this build slave. | 236 | """Build a thing on this build slave. |
1862 | @@ -238,18 +243,19 @@ | |||
1863 | 238 | :param args: A dictionary of extra arguments. The contents depend on | 243 | :param args: A dictionary of extra arguments. The contents depend on |
1864 | 239 | the build job type. | 244 | the build job type. |
1865 | 240 | """ | 245 | """ |
1872 | 241 | d = self._with_timeout(self._server.callRemote( | 246 | try: |
1873 | 242 | 'build', buildid, builder_type, chroot_sha1, filemap, args)) | 247 | return self._server.callRemote( |
1874 | 243 | def got_fault(failure): | 248 | 'build', buildid, builder_type, chroot_sha1, filemap, args) |
1875 | 244 | failure.trap(xmlrpclib.Fault) | 249 | except xmlrpclib.Fault, info: |
1876 | 245 | raise BuildSlaveFailure(failure.value) | 250 | raise BuildSlaveFailure(info) |
1871 | 246 | return d.addErrback(got_fault) | ||
1877 | 247 | 251 | ||
1878 | 248 | 252 | ||
1879 | 249 | # This is a separate function since MockBuilder needs to use it too. | 253 | # This is a separate function since MockBuilder needs to use it too. |
1880 | 250 | # Do not use it -- (Mock)Builder.rescueIfLost should be used instead. | 254 | # Do not use it -- (Mock)Builder.rescueIfLost should be used instead. |
1881 | 251 | def rescueBuilderIfLost(builder, logger=None): | 255 | def rescueBuilderIfLost(builder, logger=None): |
1882 | 252 | """See `IBuilder`.""" | 256 | """See `IBuilder`.""" |
1883 | 257 | status_sentence = builder.slaveStatusSentence() | ||
1884 | 258 | |||
1885 | 253 | # 'ident_position' dict relates the position of the job identifier | 259 | # 'ident_position' dict relates the position of the job identifier |
1886 | 254 | # token in the sentence received from status(), according the | 260 | # token in the sentence received from status(), according the |
1887 | 255 | # two status we care about. See see lib/canonical/buildd/slave.py | 261 | # two status we care about. See see lib/canonical/buildd/slave.py |
1888 | @@ -259,58 +265,61 @@ | |||
1889 | 259 | 'BuilderStatus.WAITING': 2 | 265 | 'BuilderStatus.WAITING': 2 |
1890 | 260 | } | 266 | } |
1891 | 261 | 267 | ||
1918 | 262 | d = builder.slaveStatusSentence() | 268 | # Isolate the BuilderStatus string, always the first token in |
1919 | 263 | 269 | # see lib/canonical/buildd/slave.py and | |
1920 | 264 | def got_status(status_sentence): | 270 | # IBuilder.slaveStatusSentence(). |
1921 | 265 | """After we get the status, clean if we have to. | 271 | status = status_sentence[0] |
1922 | 266 | 272 | ||
1923 | 267 | Always return status_sentence. | 273 | # If the cookie test below fails, it will request an abort of the |
1924 | 268 | """ | 274 | # builder. This will leave the builder in the aborted state and |
1925 | 269 | # Isolate the BuilderStatus string, always the first token in | 275 | # with no assigned job, and we should now "clean" the slave which |
1926 | 270 | # see lib/canonical/buildd/slave.py and | 276 | # will reset its state back to IDLE, ready to accept new builds. |
1927 | 271 | # IBuilder.slaveStatusSentence(). | 277 | # This situation is usually caused by a temporary loss of |
1928 | 272 | status = status_sentence[0] | 278 | # communications with the slave and the build manager had to reset |
1929 | 273 | 279 | # the job. | |
1930 | 274 | # If the cookie test below fails, it will request an abort of the | 280 | if status == 'BuilderStatus.ABORTED' and builder.currentjob is None: |
1931 | 275 | # builder. This will leave the builder in the aborted state and | 281 | builder.cleanSlave() |
1932 | 276 | # with no assigned job, and we should now "clean" the slave which | 282 | if logger is not None: |
1933 | 277 | # will reset its state back to IDLE, ready to accept new builds. | 283 | logger.info( |
1934 | 278 | # This situation is usually caused by a temporary loss of | 284 | "Builder '%s' cleaned up from ABORTED" % builder.name) |
1935 | 279 | # communications with the slave and the build manager had to reset | 285 | return |
1936 | 280 | # the job. | 286 | |
1937 | 281 | if status == 'BuilderStatus.ABORTED' and builder.currentjob is None: | 287 | # If slave is not building nor waiting, it's not in need of rescuing. |
1938 | 282 | if logger is not None: | 288 | if status not in ident_position.keys(): |
1939 | 283 | logger.info( | 289 | return |
1940 | 284 | "Builder '%s' being cleaned up from ABORTED" % | 290 | |
1941 | 285 | (builder.name,)) | 291 | slave_build_id = status_sentence[ident_position[status]] |
1942 | 286 | d = builder.cleanSlave() | 292 | |
1943 | 287 | return d.addCallback(lambda ignored: status_sentence) | 293 | try: |
1944 | 294 | builder.verifySlaveBuildCookie(slave_build_id) | ||
1945 | 295 | except CorruptBuildCookie, reason: | ||
1946 | 296 | if status == 'BuilderStatus.WAITING': | ||
1947 | 297 | builder.cleanSlave() | ||
1948 | 288 | else: | 298 | else: |
1974 | 289 | return status_sentence | 299 | builder.requestAbort() |
1975 | 290 | 300 | if logger: | |
1976 | 291 | def rescue_slave(status_sentence): | 301 | logger.info( |
1977 | 292 | # If slave is not building nor waiting, it's not in need of rescuing. | 302 | "Builder '%s' rescued from '%s': '%s'" % |
1978 | 293 | status = status_sentence[0] | 303 | (builder.name, slave_build_id, reason)) |
1979 | 294 | if status not in ident_position.keys(): | 304 | |
1980 | 295 | return | 305 | |
1981 | 296 | slave_build_id = status_sentence[ident_position[status]] | 306 | def _update_builder_status(builder, logger=None): |
1982 | 297 | try: | 307 | """Really update the builder status.""" |
1983 | 298 | builder.verifySlaveBuildCookie(slave_build_id) | 308 | try: |
1984 | 299 | except CorruptBuildCookie, reason: | 309 | builder.checkSlaveAlive() |
1985 | 300 | if status == 'BuilderStatus.WAITING': | 310 | builder.rescueIfLost(logger) |
1986 | 301 | d = builder.cleanSlave() | 311 | # Catch only known exceptions. |
1987 | 302 | else: | 312 | # XXX cprov 2007-06-15 bug=120571: ValueError & TypeError catching is |
1988 | 303 | d = builder.requestAbort() | 313 | # disturbing in this context. We should spend sometime sanitizing the |
1989 | 304 | def log_rescue(ignored): | 314 | # exceptions raised in the Builder API since we already started the |
1990 | 305 | if logger: | 315 | # main refactoring of this area. |
1991 | 306 | logger.info( | 316 | except (ValueError, TypeError, xmlrpclib.Fault, |
1992 | 307 | "Builder '%s' rescued from '%s': '%s'" % | 317 | BuildDaemonError), reason: |
1993 | 308 | (builder.name, slave_build_id, reason)) | 318 | builder.failBuilder(str(reason)) |
1994 | 309 | return d.addCallback(log_rescue) | 319 | if logger: |
1995 | 310 | 320 | logger.warn( | |
1996 | 311 | d.addCallback(got_status) | 321 | "%s (%s) marked as failed due to: %s", |
1997 | 312 | d.addCallback(rescue_slave) | 322 | builder.name, builder.url, builder.failnotes, exc_info=True) |
1973 | 313 | return d | ||
1998 | 314 | 323 | ||
1999 | 315 | 324 | ||
2000 | 316 | def updateBuilderStatus(builder, logger=None): | 325 | def updateBuilderStatus(builder, logger=None): |
2001 | @@ -318,7 +327,16 @@ | |||
2002 | 318 | if logger: | 327 | if logger: |
2003 | 319 | logger.debug('Checking %s' % builder.name) | 328 | logger.debug('Checking %s' % builder.name) |
2004 | 320 | 329 | ||
2006 | 321 | return builder.rescueIfLost(logger) | 330 | MAX_EINTR_RETRIES = 42 # pulling a number out of my a$$ here |
2007 | 331 | try: | ||
2008 | 332 | return until_no_eintr( | ||
2009 | 333 | MAX_EINTR_RETRIES, _update_builder_status, builder, logger=logger) | ||
2010 | 334 | except socket.error, reason: | ||
2011 | 335 | # In Python 2.6 we can use IOError instead. It also has | ||
2012 | 336 | # reason.errno but we might be using 2.5 here so use the | ||
2013 | 337 | # index hack. | ||
2014 | 338 | error_message = str(reason) | ||
2015 | 339 | builder.handleTimeout(logger, error_message) | ||
2016 | 322 | 340 | ||
2017 | 323 | 341 | ||
2018 | 324 | class Builder(SQLBase): | 342 | class Builder(SQLBase): |
2019 | @@ -346,10 +364,6 @@ | |||
2020 | 346 | active = BoolCol(dbName='active', notNull=True, default=True) | 364 | active = BoolCol(dbName='active', notNull=True, default=True) |
2021 | 347 | failure_count = IntCol(dbName='failure_count', default=0, notNull=True) | 365 | failure_count = IntCol(dbName='failure_count', default=0, notNull=True) |
2022 | 348 | 366 | ||
2023 | 349 | # The number of times a builder can consecutively fail before we | ||
2024 | 350 | # give up and mark it builderok=False. | ||
2025 | 351 | FAILURE_THRESHOLD = 5 | ||
2026 | 352 | |||
2027 | 353 | def _getCurrentBuildBehavior(self): | 367 | def _getCurrentBuildBehavior(self): |
2028 | 354 | """Return the current build behavior.""" | 368 | """Return the current build behavior.""" |
2029 | 355 | if not safe_hasattr(self, '_current_build_behavior'): | 369 | if not safe_hasattr(self, '_current_build_behavior'): |
2030 | @@ -395,13 +409,18 @@ | |||
2031 | 395 | """See `IBuilder`.""" | 409 | """See `IBuilder`.""" |
2032 | 396 | self.failure_count = 0 | 410 | self.failure_count = 0 |
2033 | 397 | 411 | ||
2034 | 412 | def checkSlaveAlive(self): | ||
2035 | 413 | """See IBuilder.""" | ||
2036 | 414 | if self.slave.echo("Test")[0] != "Test": | ||
2037 | 415 | raise BuildDaemonError("Failed to echo OK") | ||
2038 | 416 | |||
2039 | 398 | def rescueIfLost(self, logger=None): | 417 | def rescueIfLost(self, logger=None): |
2040 | 399 | """See `IBuilder`.""" | 418 | """See `IBuilder`.""" |
2042 | 400 | return rescueBuilderIfLost(self, logger) | 419 | rescueBuilderIfLost(self, logger) |
2043 | 401 | 420 | ||
2044 | 402 | def updateStatus(self, logger=None): | 421 | def updateStatus(self, logger=None): |
2045 | 403 | """See `IBuilder`.""" | 422 | """See `IBuilder`.""" |
2047 | 404 | return updateBuilderStatus(self, logger) | 423 | updateBuilderStatus(self, logger) |
2048 | 405 | 424 | ||
2049 | 406 | def cleanSlave(self): | 425 | def cleanSlave(self): |
2050 | 407 | """See IBuilder.""" | 426 | """See IBuilder.""" |
2051 | @@ -421,23 +440,20 @@ | |||
2052 | 421 | def resumeSlaveHost(self): | 440 | def resumeSlaveHost(self): |
2053 | 422 | """See IBuilder.""" | 441 | """See IBuilder.""" |
2054 | 423 | if not self.virtualized: | 442 | if not self.virtualized: |
2056 | 424 | return defer.fail(CannotResumeHost('Builder is not virtualized.')) | 443 | raise CannotResumeHost('Builder is not virtualized.') |
2057 | 425 | 444 | ||
2058 | 426 | if not self.vm_host: | 445 | if not self.vm_host: |
2060 | 427 | return defer.fail(CannotResumeHost('Undefined vm_host.')) | 446 | raise CannotResumeHost('Undefined vm_host.') |
2061 | 428 | 447 | ||
2062 | 429 | logger = self._getSlaveScannerLogger() | 448 | logger = self._getSlaveScannerLogger() |
2063 | 430 | logger.debug("Resuming %s (%s)" % (self.name, self.url)) | 449 | logger.debug("Resuming %s (%s)" % (self.name, self.url)) |
2064 | 431 | 450 | ||
2070 | 432 | d = self.slave.resume() | 451 | stdout, stderr, returncode = self.slave.resume() |
2071 | 433 | def got_resume_ok((stdout, stderr, returncode)): | 452 | if returncode != 0: |
2067 | 434 | return stdout, stderr | ||
2068 | 435 | def got_resume_bad(failure): | ||
2069 | 436 | stdout, stderr, code = failure.value | ||
2072 | 437 | raise CannotResumeHost( | 453 | raise CannotResumeHost( |
2073 | 438 | "Resuming failed:\nOUT:\n%s\nERR:\n%s\n" % (stdout, stderr)) | 454 | "Resuming failed:\nOUT:\n%s\nERR:\n%s\n" % (stdout, stderr)) |
2074 | 439 | 455 | ||
2076 | 440 | return d.addCallback(got_resume_ok).addErrback(got_resume_bad) | 456 | return stdout, stderr |
2077 | 441 | 457 | ||
2078 | 442 | @cachedproperty | 458 | @cachedproperty |
2079 | 443 | def slave(self): | 459 | def slave(self): |
2080 | @@ -446,7 +462,7 @@ | |||
2081 | 446 | # the slave object, which is usually an XMLRPC client, with a | 462 | # the slave object, which is usually an XMLRPC client, with a |
2082 | 447 | # stub object that removes the need to actually create a buildd | 463 | # stub object that removes the need to actually create a buildd |
2083 | 448 | # slave in various states - which can be hard to create. | 464 | # slave in various states - which can be hard to create. |
2085 | 449 | return BuilderSlave.makeBuilderSlave(self.url, self.vm_host) | 465 | return BuilderSlave.makeBlockingSlave(self.url, self.vm_host) |
2086 | 450 | 466 | ||
2087 | 451 | def setSlaveForTesting(self, proxy): | 467 | def setSlaveForTesting(self, proxy): |
2088 | 452 | """See IBuilder.""" | 468 | """See IBuilder.""" |
2089 | @@ -467,23 +483,18 @@ | |||
2090 | 467 | 483 | ||
2091 | 468 | # If we are building a virtual build, resume the virtual machine. | 484 | # If we are building a virtual build, resume the virtual machine. |
2092 | 469 | if self.virtualized: | 485 | if self.virtualized: |
2096 | 470 | d = self.resumeSlaveHost() | 486 | self.resumeSlaveHost() |
2094 | 471 | else: | ||
2095 | 472 | d = defer.succeed(None) | ||
2097 | 473 | 487 | ||
2100 | 474 | def resume_done(ignored): | 488 | # Do it. |
2101 | 475 | return self.current_build_behavior.dispatchBuildToSlave( | 489 | build_queue_item.markAsBuilding(self) |
2102 | 490 | try: | ||
2103 | 491 | self.current_build_behavior.dispatchBuildToSlave( | ||
2104 | 476 | build_queue_item.id, logger) | 492 | build_queue_item.id, logger) |
2109 | 477 | 493 | except BuildSlaveFailure, e: | |
2110 | 478 | def eb_slave_failure(failure): | 494 | logger.debug("Disabling builder: %s" % self.url, exc_info=1) |
2107 | 479 | failure.trap(BuildSlaveFailure) | ||
2108 | 480 | e = failure.value | ||
2111 | 481 | self.failBuilder( | 495 | self.failBuilder( |
2112 | 482 | "Exception (%s) when setting up to new job" % (e,)) | 496 | "Exception (%s) when setting up to new job" % (e,)) |
2117 | 483 | 497 | except CannotFetchFile, e: | |
2114 | 484 | def eb_cannot_fetch_file(failure): | ||
2115 | 485 | failure.trap(CannotFetchFile) | ||
2116 | 486 | e = failure.value | ||
2118 | 487 | message = """Slave '%s' (%s) was unable to fetch file. | 498 | message = """Slave '%s' (%s) was unable to fetch file. |
2119 | 488 | ****** URL ******** | 499 | ****** URL ******** |
2120 | 489 | %s | 500 | %s |
2121 | @@ -492,19 +503,10 @@ | |||
2122 | 492 | ******************* | 503 | ******************* |
2123 | 493 | """ % (self.name, self.url, e.file_url, e.error_information) | 504 | """ % (self.name, self.url, e.file_url, e.error_information) |
2124 | 494 | raise BuildDaemonError(message) | 505 | raise BuildDaemonError(message) |
2129 | 495 | 506 | except socket.error, e: | |
2126 | 496 | def eb_socket_error(failure): | ||
2127 | 497 | failure.trap(socket.error) | ||
2128 | 498 | e = failure.value | ||
2130 | 499 | error_message = "Exception (%s) when setting up new job" % (e,) | 507 | error_message = "Exception (%s) when setting up new job" % (e,) |
2139 | 500 | d = self.handleTimeout(logger, error_message) | 508 | self.handleTimeout(logger, error_message) |
2140 | 501 | return d.addBoth(lambda ignored: failure) | 509 | raise BuildSlaveFailure |
2133 | 502 | |||
2134 | 503 | d.addCallback(resume_done) | ||
2135 | 504 | d.addErrback(eb_slave_failure) | ||
2136 | 505 | d.addErrback(eb_cannot_fetch_file) | ||
2137 | 506 | d.addErrback(eb_socket_error) | ||
2138 | 507 | return d | ||
2141 | 508 | 510 | ||
2142 | 509 | def failBuilder(self, reason): | 511 | def failBuilder(self, reason): |
2143 | 510 | """See IBuilder""" | 512 | """See IBuilder""" |
2144 | @@ -532,24 +534,22 @@ | |||
2145 | 532 | 534 | ||
2146 | 533 | def slaveStatus(self): | 535 | def slaveStatus(self): |
2147 | 534 | """See IBuilder.""" | 536 | """See IBuilder.""" |
2166 | 535 | d = self.slave.status() | 537 | builder_version, builder_arch, mechanisms = self.slave.info() |
2167 | 536 | def got_status(status_sentence): | 538 | status_sentence = self.slave.status() |
2168 | 537 | status = {'builder_status': status_sentence[0]} | 539 | |
2169 | 538 | 540 | status = {'builder_status': status_sentence[0]} | |
2170 | 539 | # Extract detailed status and log information if present. | 541 | |
2171 | 540 | # Although build_id is also easily extractable here, there is no | 542 | # Extract detailed status and log information if present. |
2172 | 541 | # valid reason for anything to use it, so we exclude it. | 543 | # Although build_id is also easily extractable here, there is no |
2173 | 542 | if status['builder_status'] == 'BuilderStatus.WAITING': | 544 | # valid reason for anything to use it, so we exclude it. |
2174 | 543 | status['build_status'] = status_sentence[1] | 545 | if status['builder_status'] == 'BuilderStatus.WAITING': |
2175 | 544 | else: | 546 | status['build_status'] = status_sentence[1] |
2176 | 545 | if status['builder_status'] == 'BuilderStatus.BUILDING': | 547 | else: |
2177 | 546 | status['logtail'] = status_sentence[2] | 548 | if status['builder_status'] == 'BuilderStatus.BUILDING': |
2178 | 547 | 549 | status['logtail'] = status_sentence[2] | |
2179 | 548 | self.current_build_behavior.updateSlaveStatus( | 550 | |
2180 | 549 | status_sentence, status) | 551 | self.current_build_behavior.updateSlaveStatus(status_sentence, status) |
2181 | 550 | return status | 552 | return status |
2164 | 551 | |||
2165 | 552 | return d.addCallback(got_status) | ||
2182 | 553 | 553 | ||
2183 | 554 | def slaveStatusSentence(self): | 554 | def slaveStatusSentence(self): |
2184 | 555 | """See IBuilder.""" | 555 | """See IBuilder.""" |
2185 | @@ -562,15 +562,13 @@ | |||
2186 | 562 | 562 | ||
2187 | 563 | def updateBuild(self, queueItem): | 563 | def updateBuild(self, queueItem): |
2188 | 564 | """See `IBuilder`.""" | 564 | """See `IBuilder`.""" |
2190 | 565 | return self.current_build_behavior.updateBuild(queueItem) | 565 | self.current_build_behavior.updateBuild(queueItem) |
2191 | 566 | 566 | ||
2192 | 567 | def transferSlaveFileToLibrarian(self, file_sha1, filename, private): | 567 | def transferSlaveFileToLibrarian(self, file_sha1, filename, private): |
2193 | 568 | """See IBuilder.""" | 568 | """See IBuilder.""" |
2194 | 569 | out_file_fd, out_file_name = tempfile.mkstemp(suffix=".buildlog") | 569 | out_file_fd, out_file_name = tempfile.mkstemp(suffix=".buildlog") |
2195 | 570 | out_file = os.fdopen(out_file_fd, "r+") | 570 | out_file = os.fdopen(out_file_fd, "r+") |
2196 | 571 | try: | 571 | try: |
2197 | 572 | # XXX 2010-10-18 bug=662631 | ||
2198 | 573 | # Change this to do non-blocking IO. | ||
2199 | 574 | slave_file = self.slave.getFile(file_sha1) | 572 | slave_file = self.slave.getFile(file_sha1) |
2200 | 575 | copy_and_close(slave_file, out_file) | 573 | copy_and_close(slave_file, out_file) |
2201 | 576 | # If the requested file is the 'buildlog' compress it using gzip | 574 | # If the requested file is the 'buildlog' compress it using gzip |
2202 | @@ -601,17 +599,18 @@ | |||
2203 | 601 | 599 | ||
2204 | 602 | return library_file.id | 600 | return library_file.id |
2205 | 603 | 601 | ||
2207 | 604 | def isAvailable(self): | 602 | @property |
2208 | 603 | def is_available(self): | ||
2209 | 605 | """See `IBuilder`.""" | 604 | """See `IBuilder`.""" |
2210 | 606 | if not self.builderok: | 605 | if not self.builderok: |
2219 | 607 | return defer.succeed(False) | 606 | return False |
2220 | 608 | d = self.slaveStatusSentence() | 607 | try: |
2221 | 609 | def catch_fault(failure): | 608 | slavestatus = self.slaveStatusSentence() |
2222 | 610 | failure.trap(xmlrpclib.Fault, socket.error) | 609 | except (xmlrpclib.Fault, socket.error): |
2223 | 611 | return False | 610 | return False |
2224 | 612 | def check_available(status): | 611 | if slavestatus[0] != BuilderStatus.IDLE: |
2225 | 613 | return status[0] == BuilderStatus.IDLE | 612 | return False |
2226 | 614 | return d.addCallbacks(check_available, catch_fault) | 613 | return True |
2227 | 615 | 614 | ||
2228 | 616 | def _getSlaveScannerLogger(self): | 615 | def _getSlaveScannerLogger(self): |
2229 | 617 | """Return the logger instance from buildd-slave-scanner.py.""" | 616 | """Return the logger instance from buildd-slave-scanner.py.""" |
2230 | @@ -622,27 +621,6 @@ | |||
2231 | 622 | logger = logging.getLogger('slave-scanner') | 621 | logger = logging.getLogger('slave-scanner') |
2232 | 623 | return logger | 622 | return logger |
2233 | 624 | 623 | ||
2234 | 625 | def acquireBuildCandidate(self): | ||
2235 | 626 | """Acquire a build candidate in an atomic fashion. | ||
2236 | 627 | |||
2237 | 628 | When retrieiving a candidate we need to mark it as building | ||
2238 | 629 | immediately so that it is not dispatched by another builder in the | ||
2239 | 630 | build manager. | ||
2240 | 631 | |||
2241 | 632 | We can consider this to be atomic because although the build manager | ||
2242 | 633 | is a Twisted app and gives the appearance of doing lots of things at | ||
2243 | 634 | once, it's still single-threaded so no more than one builder scan | ||
2244 | 635 | can be in this code at the same time. | ||
2245 | 636 | |||
2246 | 637 | If there's ever more than one build manager running at once, then | ||
2247 | 638 | this code will need some sort of mutex. | ||
2248 | 639 | """ | ||
2249 | 640 | candidate = self._findBuildCandidate() | ||
2250 | 641 | if candidate is not None: | ||
2251 | 642 | candidate.markAsBuilding(self) | ||
2252 | 643 | transaction.commit() | ||
2253 | 644 | return candidate | ||
2254 | 645 | |||
2255 | 646 | def _findBuildCandidate(self): | 624 | def _findBuildCandidate(self): |
2256 | 647 | """Find a candidate job for dispatch to an idle buildd slave. | 625 | """Find a candidate job for dispatch to an idle buildd slave. |
2257 | 648 | 626 | ||
2258 | @@ -722,46 +700,52 @@ | |||
2259 | 722 | :param candidate: The job to dispatch. | 700 | :param candidate: The job to dispatch. |
2260 | 723 | """ | 701 | """ |
2261 | 724 | logger = self._getSlaveScannerLogger() | 702 | logger = self._getSlaveScannerLogger() |
2266 | 725 | # Using maybeDeferred ensures that any exceptions are also | 703 | try: |
2267 | 726 | # wrapped up and caught later. | 704 | self.startBuild(candidate, logger) |
2268 | 727 | d = defer.maybeDeferred(self.startBuild, candidate, logger) | 705 | except (BuildSlaveFailure, CannotBuild, BuildBehaviorMismatch), err: |
2269 | 728 | return d | 706 | logger.warn('Could not build: %s' % err) |
2270 | 729 | 707 | ||
2271 | 730 | def handleTimeout(self, logger, error_message): | 708 | def handleTimeout(self, logger, error_message): |
2272 | 731 | """See IBuilder.""" | 709 | """See IBuilder.""" |
2273 | 710 | builder_should_be_failed = True | ||
2274 | 711 | |||
2275 | 732 | if self.virtualized: | 712 | if self.virtualized: |
2276 | 733 | # Virtualized/PPA builder: attempt a reset. | 713 | # Virtualized/PPA builder: attempt a reset. |
2277 | 734 | logger.warn( | 714 | logger.warn( |
2278 | 735 | "Resetting builder: %s -- %s" % (self.url, error_message), | 715 | "Resetting builder: %s -- %s" % (self.url, error_message), |
2279 | 736 | exc_info=True) | 716 | exc_info=True) |
2285 | 737 | d = self.resumeSlaveHost() | 717 | try: |
2286 | 738 | return d | 718 | self.resumeSlaveHost() |
2287 | 739 | else: | 719 | except CannotResumeHost, err: |
2288 | 740 | # XXX: This should really let the failure bubble up to the | 720 | # Failed to reset builder. |
2289 | 741 | # scan() method that does the failure counting. | 721 | logger.warn( |
2290 | 722 | "Failed to reset builder: %s -- %s" % | ||
2291 | 723 | (self.url, str(err)), exc_info=True) | ||
2292 | 724 | else: | ||
2293 | 725 | # Builder was reset, do *not* mark it as failed. | ||
2294 | 726 | builder_should_be_failed = False | ||
2295 | 727 | |||
2296 | 728 | if builder_should_be_failed: | ||
2297 | 742 | # Mark builder as 'failed'. | 729 | # Mark builder as 'failed'. |
2298 | 743 | logger.warn( | 730 | logger.warn( |
2300 | 744 | "Disabling builder: %s -- %s" % (self.url, error_message)) | 731 | "Disabling builder: %s -- %s" % (self.url, error_message), |
2301 | 732 | exc_info=True) | ||
2302 | 745 | self.failBuilder(error_message) | 733 | self.failBuilder(error_message) |
2303 | 746 | return defer.succeed(None) | ||
2304 | 747 | 734 | ||
2305 | 748 | def findAndStartJob(self, buildd_slave=None): | 735 | def findAndStartJob(self, buildd_slave=None): |
2306 | 749 | """See IBuilder.""" | 736 | """See IBuilder.""" |
2307 | 750 | # XXX This method should be removed in favour of two separately | ||
2308 | 751 | # called methods that find and dispatch the job. It will | ||
2309 | 752 | # require a lot of test fixing. | ||
2310 | 753 | logger = self._getSlaveScannerLogger() | 737 | logger = self._getSlaveScannerLogger() |
2312 | 754 | candidate = self.acquireBuildCandidate() | 738 | candidate = self._findBuildCandidate() |
2313 | 755 | 739 | ||
2314 | 756 | if candidate is None: | 740 | if candidate is None: |
2315 | 757 | logger.debug("No build candidates available for builder.") | 741 | logger.debug("No build candidates available for builder.") |
2317 | 758 | return defer.succeed(None) | 742 | return None |
2318 | 759 | 743 | ||
2319 | 760 | if buildd_slave is not None: | 744 | if buildd_slave is not None: |
2320 | 761 | self.setSlaveForTesting(buildd_slave) | 745 | self.setSlaveForTesting(buildd_slave) |
2321 | 762 | 746 | ||
2324 | 763 | d = self._dispatchBuildCandidate(candidate) | 747 | self._dispatchBuildCandidate(candidate) |
2325 | 764 | return d.addCallback(lambda ignored: candidate) | 748 | return candidate |
2326 | 765 | 749 | ||
2327 | 766 | def getBuildQueue(self): | 750 | def getBuildQueue(self): |
2328 | 767 | """See `IBuilder`.""" | 751 | """See `IBuilder`.""" |
2329 | 768 | 752 | ||
2330 | === modified file 'lib/lp/buildmaster/model/buildfarmjobbehavior.py' | |||
2331 | --- lib/lp/buildmaster/model/buildfarmjobbehavior.py 2010-10-20 11:54:27 +0000 | |||
2332 | +++ lib/lp/buildmaster/model/buildfarmjobbehavior.py 2010-12-07 16:29:13 +0000 | |||
2333 | @@ -16,18 +16,13 @@ | |||
2334 | 16 | import socket | 16 | import socket |
2335 | 17 | import xmlrpclib | 17 | import xmlrpclib |
2336 | 18 | 18 | ||
2337 | 19 | from twisted.internet import defer | ||
2338 | 20 | |||
2339 | 21 | from zope.component import getUtility | 19 | from zope.component import getUtility |
2340 | 22 | from zope.interface import implements | 20 | from zope.interface import implements |
2341 | 23 | from zope.security.proxy import removeSecurityProxy | 21 | from zope.security.proxy import removeSecurityProxy |
2342 | 24 | 22 | ||
2343 | 25 | from canonical import encoding | 23 | from canonical import encoding |
2344 | 26 | from canonical.librarian.interfaces import ILibrarianClient | 24 | from canonical.librarian.interfaces import ILibrarianClient |
2349 | 27 | from lp.buildmaster.interfaces.builder import ( | 25 | from lp.buildmaster.interfaces.builder import CorruptBuildCookie |
2346 | 28 | BuildSlaveFailure, | ||
2347 | 29 | CorruptBuildCookie, | ||
2348 | 30 | ) | ||
2350 | 31 | from lp.buildmaster.interfaces.buildfarmjobbehavior import ( | 26 | from lp.buildmaster.interfaces.buildfarmjobbehavior import ( |
2351 | 32 | BuildBehaviorMismatch, | 27 | BuildBehaviorMismatch, |
2352 | 33 | IBuildFarmJobBehavior, | 28 | IBuildFarmJobBehavior, |
2353 | @@ -74,53 +69,54 @@ | |||
2354 | 74 | """See `IBuildFarmJobBehavior`.""" | 69 | """See `IBuildFarmJobBehavior`.""" |
2355 | 75 | logger = logging.getLogger('slave-scanner') | 70 | logger = logging.getLogger('slave-scanner') |
2356 | 76 | 71 | ||
2362 | 77 | d = self._builder.slaveStatus() | 72 | try: |
2363 | 78 | 73 | slave_status = self._builder.slaveStatus() | |
2364 | 79 | def got_failure(failure): | 74 | except (xmlrpclib.Fault, socket.error), info: |
2365 | 80 | failure.trap(xmlrpclib.Fault, socket.error) | 75 | # XXX cprov 2005-06-29: |
2366 | 81 | info = failure.value | 76 | # Hmm, a problem with the xmlrpc interface, |
2367 | 77 | # disable the builder ?? or simple notice the failure | ||
2368 | 78 | # with a timestamp. | ||
2369 | 82 | info = ("Could not contact the builder %s, caught a (%s)" | 79 | info = ("Could not contact the builder %s, caught a (%s)" |
2370 | 83 | % (queueItem.builder.url, info)) | 80 | % (queueItem.builder.url, info)) |
2411 | 84 | raise BuildSlaveFailure(info) | 81 | logger.debug(info, exc_info=True) |
2412 | 85 | 82 | # keep the job for scan | |
2413 | 86 | def got_status(slave_status): | 83 | return |
2414 | 87 | builder_status_handlers = { | 84 | |
2415 | 88 | 'BuilderStatus.IDLE': self.updateBuild_IDLE, | 85 | builder_status_handlers = { |
2416 | 89 | 'BuilderStatus.BUILDING': self.updateBuild_BUILDING, | 86 | 'BuilderStatus.IDLE': self.updateBuild_IDLE, |
2417 | 90 | 'BuilderStatus.ABORTING': self.updateBuild_ABORTING, | 87 | 'BuilderStatus.BUILDING': self.updateBuild_BUILDING, |
2418 | 91 | 'BuilderStatus.ABORTED': self.updateBuild_ABORTED, | 88 | 'BuilderStatus.ABORTING': self.updateBuild_ABORTING, |
2419 | 92 | 'BuilderStatus.WAITING': self.updateBuild_WAITING, | 89 | 'BuilderStatus.ABORTED': self.updateBuild_ABORTED, |
2420 | 93 | } | 90 | 'BuilderStatus.WAITING': self.updateBuild_WAITING, |
2421 | 94 | 91 | } | |
2422 | 95 | builder_status = slave_status['builder_status'] | 92 | |
2423 | 96 | if builder_status not in builder_status_handlers: | 93 | builder_status = slave_status['builder_status'] |
2424 | 97 | logger.critical( | 94 | if builder_status not in builder_status_handlers: |
2425 | 98 | "Builder on %s returned unknown status %s, failing it" | 95 | logger.critical( |
2426 | 99 | % (self._builder.url, builder_status)) | 96 | "Builder on %s returned unknown status %s, failing it" |
2427 | 100 | self._builder.failBuilder( | 97 | % (self._builder.url, builder_status)) |
2428 | 101 | "Unknown status code (%s) returned from status() probe." | 98 | self._builder.failBuilder( |
2429 | 102 | % builder_status) | 99 | "Unknown status code (%s) returned from status() probe." |
2430 | 103 | # XXX: This will leave the build and job in a bad state, but | 100 | % builder_status) |
2431 | 104 | # should never be possible, since our builder statuses are | 101 | # XXX: This will leave the build and job in a bad state, but |
2432 | 105 | # known. | 102 | # should never be possible, since our builder statuses are |
2433 | 106 | queueItem._builder = None | 103 | # known. |
2434 | 107 | queueItem.setDateStarted(None) | 104 | queueItem._builder = None |
2435 | 108 | return | 105 | queueItem.setDateStarted(None) |
2436 | 109 | 106 | return | |
2437 | 110 | # Since logtail is a xmlrpclib.Binary container and it is | 107 | |
2438 | 111 | # returned from the IBuilder content class, it arrives | 108 | # Since logtail is a xmlrpclib.Binary container and it is returned |
2439 | 112 | # protected by a Zope Security Proxy, which is not declared, | 109 | # from the IBuilder content class, it arrives protected by a Zope |
2440 | 113 | # thus empty. Before passing it to the status handlers we | 110 | # Security Proxy, which is not declared, thus empty. Before passing |
2441 | 114 | # will simply remove the proxy. | 111 | # it to the status handlers we will simply remove the proxy. |
2442 | 115 | logtail = removeSecurityProxy(slave_status.get('logtail')) | 112 | logtail = removeSecurityProxy(slave_status.get('logtail')) |
2443 | 116 | 113 | ||
2444 | 117 | method = builder_status_handlers[builder_status] | 114 | method = builder_status_handlers[builder_status] |
2445 | 118 | return defer.maybeDeferred( | 115 | try: |
2446 | 119 | method, queueItem, slave_status, logtail, logger) | 116 | method(queueItem, slave_status, logtail, logger) |
2447 | 120 | 117 | except TypeError, e: | |
2448 | 121 | d.addErrback(got_failure) | 118 | logger.critical("Received wrong number of args in response.") |
2449 | 122 | d.addCallback(got_status) | 119 | logger.exception(e) |
2410 | 123 | return d | ||
2450 | 124 | 120 | ||
2451 | 125 | def updateBuild_IDLE(self, queueItem, slave_status, logtail, logger): | 121 | def updateBuild_IDLE(self, queueItem, slave_status, logtail, logger): |
2452 | 126 | """Somehow the builder forgot about the build job. | 122 | """Somehow the builder forgot about the build job. |
2453 | @@ -150,13 +146,11 @@ | |||
2454 | 150 | 146 | ||
2455 | 151 | Clean the builder for another jobs. | 147 | Clean the builder for another jobs. |
2456 | 152 | """ | 148 | """ |
2464 | 153 | d = queueItem.builder.cleanSlave() | 149 | queueItem.builder.cleanSlave() |
2465 | 154 | def got_cleaned(ignored): | 150 | queueItem.builder = None |
2466 | 155 | queueItem.builder = None | 151 | if queueItem.job.status != JobStatus.FAILED: |
2467 | 156 | if queueItem.job.status != JobStatus.FAILED: | 152 | queueItem.job.fail() |
2468 | 157 | queueItem.job.fail() | 153 | queueItem.specific_job.jobAborted() |
2462 | 158 | queueItem.specific_job.jobAborted() | ||
2463 | 159 | return d.addCallback(got_cleaned) | ||
2469 | 160 | 154 | ||
2470 | 161 | def extractBuildStatus(self, slave_status): | 155 | def extractBuildStatus(self, slave_status): |
2471 | 162 | """Read build status name. | 156 | """Read build status name. |
2472 | @@ -191,8 +185,6 @@ | |||
2473 | 191 | # XXX: dsilvers 2005-03-02: Confirm the builder has the right build? | 185 | # XXX: dsilvers 2005-03-02: Confirm the builder has the right build? |
2474 | 192 | 186 | ||
2475 | 193 | build = queueItem.specific_job.build | 187 | build = queueItem.specific_job.build |
2476 | 194 | # XXX 2010-10-18 bug=662631 | ||
2477 | 195 | # Change this to do non-blocking IO. | ||
2478 | 196 | build.handleStatus(build_status, librarian, slave_status) | 188 | build.handleStatus(build_status, librarian, slave_status) |
2479 | 197 | 189 | ||
2480 | 198 | 190 | ||
2481 | 199 | 191 | ||
2482 | === modified file 'lib/lp/buildmaster/model/packagebuild.py' | |||
2483 | --- lib/lp/buildmaster/model/packagebuild.py 2010-10-26 20:43:50 +0000 | |||
2484 | +++ lib/lp/buildmaster/model/packagebuild.py 2010-12-07 16:29:13 +0000 | |||
2485 | @@ -163,8 +163,6 @@ | |||
2486 | 163 | def getLogFromSlave(package_build): | 163 | def getLogFromSlave(package_build): |
2487 | 164 | """See `IPackageBuild`.""" | 164 | """See `IPackageBuild`.""" |
2488 | 165 | builder = package_build.buildqueue_record.builder | 165 | builder = package_build.buildqueue_record.builder |
2489 | 166 | # XXX 2010-10-18 bug=662631 | ||
2490 | 167 | # Change this to do non-blocking IO. | ||
2491 | 168 | return builder.transferSlaveFileToLibrarian( | 166 | return builder.transferSlaveFileToLibrarian( |
2492 | 169 | SLAVE_LOG_FILENAME, | 167 | SLAVE_LOG_FILENAME, |
2493 | 170 | package_build.buildqueue_record.getLogFileName(), | 168 | package_build.buildqueue_record.getLogFileName(), |
2494 | @@ -180,8 +178,6 @@ | |||
2495 | 180 | # log, builder and date_finished are read-only, so we must | 178 | # log, builder and date_finished are read-only, so we must |
2496 | 181 | # currently remove the security proxy to set them. | 179 | # currently remove the security proxy to set them. |
2497 | 182 | naked_build = removeSecurityProxy(build) | 180 | naked_build = removeSecurityProxy(build) |
2498 | 183 | # XXX 2010-10-18 bug=662631 | ||
2499 | 184 | # Change this to do non-blocking IO. | ||
2500 | 185 | naked_build.log = build.getLogFromSlave(build) | 181 | naked_build.log = build.getLogFromSlave(build) |
2501 | 186 | naked_build.builder = build.buildqueue_record.builder | 182 | naked_build.builder = build.buildqueue_record.builder |
2502 | 187 | # XXX cprov 20060615 bug=120584: Currently buildduration includes | 183 | # XXX cprov 20060615 bug=120584: Currently buildduration includes |
2503 | @@ -278,8 +274,6 @@ | |||
2504 | 278 | logger.critical("Unknown BuildStatus '%s' for builder '%s'" | 274 | logger.critical("Unknown BuildStatus '%s' for builder '%s'" |
2505 | 279 | % (status, self.buildqueue_record.builder.url)) | 275 | % (status, self.buildqueue_record.builder.url)) |
2506 | 280 | return | 276 | return |
2507 | 281 | # XXX 2010-10-18 bug=662631 | ||
2508 | 282 | # Change this to do non-blocking IO. | ||
2509 | 283 | method(librarian, slave_status, logger) | 277 | method(librarian, slave_status, logger) |
2510 | 284 | 278 | ||
2511 | 285 | def _handleStatus_OK(self, librarian, slave_status, logger): | 279 | def _handleStatus_OK(self, librarian, slave_status, logger): |
2512 | 286 | 280 | ||
2513 | === modified file 'lib/lp/buildmaster/tests/mock_slaves.py' | |||
2514 | --- lib/lp/buildmaster/tests/mock_slaves.py 2010-10-14 15:37:56 +0000 | |||
2515 | +++ lib/lp/buildmaster/tests/mock_slaves.py 2010-12-07 16:29:13 +0000 | |||
2516 | @@ -6,40 +6,21 @@ | |||
2517 | 6 | __metaclass__ = type | 6 | __metaclass__ = type |
2518 | 7 | 7 | ||
2519 | 8 | __all__ = [ | 8 | __all__ = [ |
2522 | 9 | 'AbortedSlave', | 9 | 'MockBuilder', |
2523 | 10 | 'AbortingSlave', | 10 | 'LostBuildingBrokenSlave', |
2524 | 11 | 'BrokenSlave', | 11 | 'BrokenSlave', |
2525 | 12 | 'OkSlave', | ||
2526 | 12 | 'BuildingSlave', | 13 | 'BuildingSlave', |
2534 | 13 | 'CorruptBehavior', | 14 | 'AbortedSlave', |
2528 | 14 | 'DeadProxy', | ||
2529 | 15 | 'LostBuildingBrokenSlave', | ||
2530 | 16 | 'MockBuilder', | ||
2531 | 17 | 'OkSlave', | ||
2532 | 18 | 'SlaveTestHelpers', | ||
2533 | 19 | 'TrivialBehavior', | ||
2535 | 20 | 'WaitingSlave', | 15 | 'WaitingSlave', |
2536 | 16 | 'AbortingSlave', | ||
2537 | 21 | ] | 17 | ] |
2538 | 22 | 18 | ||
2539 | 23 | import fixtures | ||
2540 | 24 | import os | ||
2541 | 25 | |||
2542 | 26 | from StringIO import StringIO | 19 | from StringIO import StringIO |
2543 | 27 | import xmlrpclib | 20 | import xmlrpclib |
2544 | 28 | 21 | ||
2557 | 29 | from testtools.content import Content | 22 | from lp.buildmaster.interfaces.builder import CannotFetchFile |
2546 | 30 | from testtools.content_type import UTF8_TEXT | ||
2547 | 31 | |||
2548 | 32 | from twisted.internet import defer | ||
2549 | 33 | from twisted.web import xmlrpc | ||
2550 | 34 | |||
2551 | 35 | from canonical.buildd.tests.harness import BuilddSlaveTestSetup | ||
2552 | 36 | |||
2553 | 37 | from lp.buildmaster.interfaces.builder import ( | ||
2554 | 38 | CannotFetchFile, | ||
2555 | 39 | CorruptBuildCookie, | ||
2556 | 40 | ) | ||
2558 | 41 | from lp.buildmaster.model.builder import ( | 23 | from lp.buildmaster.model.builder import ( |
2559 | 42 | BuilderSlave, | ||
2560 | 43 | rescueBuilderIfLost, | 24 | rescueBuilderIfLost, |
2561 | 44 | updateBuilderStatus, | 25 | updateBuilderStatus, |
2562 | 45 | ) | 26 | ) |
2563 | @@ -78,9 +59,15 @@ | |||
2564 | 78 | slave_build_id) | 59 | slave_build_id) |
2565 | 79 | 60 | ||
2566 | 80 | def cleanSlave(self): | 61 | def cleanSlave(self): |
2567 | 62 | # XXX: This should not print anything. The print is only here to make | ||
2568 | 63 | # doc/builder.txt a meaningful test. | ||
2569 | 64 | print 'Cleaning slave' | ||
2570 | 81 | return self.slave.clean() | 65 | return self.slave.clean() |
2571 | 82 | 66 | ||
2572 | 83 | def requestAbort(self): | 67 | def requestAbort(self): |
2573 | 68 | # XXX: This should not print anything. The print is only here to make | ||
2574 | 69 | # doc/builder.txt a meaningful test. | ||
2575 | 70 | print 'Aborting slave' | ||
2576 | 84 | return self.slave.abort() | 71 | return self.slave.abort() |
2577 | 85 | 72 | ||
2578 | 86 | def resumeSlave(self, logger): | 73 | def resumeSlave(self, logger): |
2579 | @@ -90,10 +77,10 @@ | |||
2580 | 90 | pass | 77 | pass |
2581 | 91 | 78 | ||
2582 | 92 | def rescueIfLost(self, logger=None): | 79 | def rescueIfLost(self, logger=None): |
2584 | 93 | return rescueBuilderIfLost(self, logger) | 80 | rescueBuilderIfLost(self, logger) |
2585 | 94 | 81 | ||
2586 | 95 | def updateStatus(self, logger=None): | 82 | def updateStatus(self, logger=None): |
2588 | 96 | return defer.maybeDeferred(updateBuilderStatus, self, logger) | 83 | updateBuilderStatus(self, logger) |
2589 | 97 | 84 | ||
2590 | 98 | 85 | ||
2591 | 99 | # XXX: It would be *really* nice to run some set of tests against the real | 86 | # XXX: It would be *really* nice to run some set of tests against the real |
2592 | @@ -108,44 +95,36 @@ | |||
2593 | 108 | self.arch_tag = arch_tag | 95 | self.arch_tag = arch_tag |
2594 | 109 | 96 | ||
2595 | 110 | def status(self): | 97 | def status(self): |
2597 | 111 | return defer.succeed(('BuilderStatus.IDLE', '')) | 98 | return ('BuilderStatus.IDLE', '') |
2598 | 112 | 99 | ||
2599 | 113 | def ensurepresent(self, sha1, url, user=None, password=None): | 100 | def ensurepresent(self, sha1, url, user=None, password=None): |
2600 | 114 | self.call_log.append(('ensurepresent', url, user, password)) | 101 | self.call_log.append(('ensurepresent', url, user, password)) |
2602 | 115 | return defer.succeed((True, None)) | 102 | return True, None |
2603 | 116 | 103 | ||
2604 | 117 | def build(self, buildid, buildtype, chroot, filemap, args): | 104 | def build(self, buildid, buildtype, chroot, filemap, args): |
2605 | 118 | self.call_log.append( | 105 | self.call_log.append( |
2606 | 119 | ('build', buildid, buildtype, chroot, filemap.keys(), args)) | 106 | ('build', buildid, buildtype, chroot, filemap.keys(), args)) |
2607 | 120 | info = 'OkSlave BUILDING' | 107 | info = 'OkSlave BUILDING' |
2609 | 121 | return defer.succeed(('BuildStatus.Building', info)) | 108 | return ('BuildStatus.Building', info) |
2610 | 122 | 109 | ||
2611 | 123 | def echo(self, *args): | 110 | def echo(self, *args): |
2612 | 124 | self.call_log.append(('echo',) + args) | 111 | self.call_log.append(('echo',) + args) |
2614 | 125 | return defer.succeed(args) | 112 | return args |
2615 | 126 | 113 | ||
2616 | 127 | def clean(self): | 114 | def clean(self): |
2617 | 128 | self.call_log.append('clean') | 115 | self.call_log.append('clean') |
2618 | 129 | return defer.succeed(None) | ||
2619 | 130 | 116 | ||
2620 | 131 | def abort(self): | 117 | def abort(self): |
2621 | 132 | self.call_log.append('abort') | 118 | self.call_log.append('abort') |
2622 | 133 | return defer.succeed(None) | ||
2623 | 134 | 119 | ||
2624 | 135 | def info(self): | 120 | def info(self): |
2625 | 136 | self.call_log.append('info') | 121 | self.call_log.append('info') |
2631 | 137 | return defer.succeed(('1.0', self.arch_tag, 'debian')) | 122 | return ('1.0', self.arch_tag, 'debian') |
2627 | 138 | |||
2628 | 139 | def resume(self): | ||
2629 | 140 | self.call_log.append('resume') | ||
2630 | 141 | return defer.succeed(("", "", 0)) | ||
2632 | 142 | 123 | ||
2633 | 143 | def sendFileToSlave(self, sha1, url, username="", password=""): | 124 | def sendFileToSlave(self, sha1, url, username="", password=""): |
2639 | 144 | d = self.ensurepresent(sha1, url, username, password) | 125 | present, info = self.ensurepresent(sha1, url, username, password) |
2640 | 145 | def check_present((present, info)): | 126 | if not present: |
2641 | 146 | if not present: | 127 | raise CannotFetchFile(url, info) |
2637 | 147 | raise CannotFetchFile(url, info) | ||
2638 | 148 | return d.addCallback(check_present) | ||
2642 | 149 | 128 | ||
2643 | 150 | def cacheFile(self, logger, libraryfilealias): | 129 | def cacheFile(self, logger, libraryfilealias): |
2644 | 151 | return self.sendFileToSlave( | 130 | return self.sendFileToSlave( |
2645 | @@ -162,11 +141,9 @@ | |||
2646 | 162 | def status(self): | 141 | def status(self): |
2647 | 163 | self.call_log.append('status') | 142 | self.call_log.append('status') |
2648 | 164 | buildlog = xmlrpclib.Binary("This is a build log") | 143 | buildlog = xmlrpclib.Binary("This is a build log") |
2651 | 165 | return defer.succeed( | 144 | return ('BuilderStatus.BUILDING', self.build_id, buildlog) |
2650 | 166 | ('BuilderStatus.BUILDING', self.build_id, buildlog)) | ||
2652 | 167 | 145 | ||
2653 | 168 | def getFile(self, sum): | 146 | def getFile(self, sum): |
2654 | 169 | # XXX: This needs to be updated to return a Deferred. | ||
2655 | 170 | self.call_log.append('getFile') | 147 | self.call_log.append('getFile') |
2656 | 171 | if sum == "buildlog": | 148 | if sum == "buildlog": |
2657 | 172 | s = StringIO("This is a build log") | 149 | s = StringIO("This is a build log") |
2658 | @@ -178,15 +155,11 @@ | |||
2659 | 178 | """A mock slave that looks like it's currently waiting.""" | 155 | """A mock slave that looks like it's currently waiting.""" |
2660 | 179 | 156 | ||
2661 | 180 | def __init__(self, state='BuildStatus.OK', dependencies=None, | 157 | def __init__(self, state='BuildStatus.OK', dependencies=None, |
2663 | 181 | build_id='1-1', filemap=None): | 158 | build_id='1-1'): |
2664 | 182 | super(WaitingSlave, self).__init__() | 159 | super(WaitingSlave, self).__init__() |
2665 | 183 | self.state = state | 160 | self.state = state |
2666 | 184 | self.dependencies = dependencies | 161 | self.dependencies = dependencies |
2667 | 185 | self.build_id = build_id | 162 | self.build_id = build_id |
2668 | 186 | if filemap is None: | ||
2669 | 187 | self.filemap = {} | ||
2670 | 188 | else: | ||
2671 | 189 | self.filemap = filemap | ||
2672 | 190 | 163 | ||
2673 | 191 | # By default, the slave only has a buildlog, but callsites | 164 | # By default, the slave only has a buildlog, but callsites |
2674 | 192 | # can update this list as needed. | 165 | # can update this list as needed. |
2675 | @@ -194,12 +167,10 @@ | |||
2676 | 194 | 167 | ||
2677 | 195 | def status(self): | 168 | def status(self): |
2678 | 196 | self.call_log.append('status') | 169 | self.call_log.append('status') |
2682 | 197 | return defer.succeed(( | 170 | return ('BuilderStatus.WAITING', self.state, self.build_id, {}, |
2683 | 198 | 'BuilderStatus.WAITING', self.state, self.build_id, self.filemap, | 171 | self.dependencies) |
2681 | 199 | self.dependencies)) | ||
2684 | 200 | 172 | ||
2685 | 201 | def getFile(self, hash): | 173 | def getFile(self, hash): |
2686 | 202 | # XXX: This needs to be updated to return a Deferred. | ||
2687 | 203 | self.call_log.append('getFile') | 174 | self.call_log.append('getFile') |
2688 | 204 | if hash in self.valid_file_hashes: | 175 | if hash in self.valid_file_hashes: |
2689 | 205 | content = "This is a %s" % hash | 176 | content = "This is a %s" % hash |
2690 | @@ -213,19 +184,15 @@ | |||
2691 | 213 | 184 | ||
2692 | 214 | def status(self): | 185 | def status(self): |
2693 | 215 | self.call_log.append('status') | 186 | self.call_log.append('status') |
2695 | 216 | return defer.succeed(('BuilderStatus.ABORTING', '1-1')) | 187 | return ('BuilderStatus.ABORTING', '1-1') |
2696 | 217 | 188 | ||
2697 | 218 | 189 | ||
2698 | 219 | class AbortedSlave(OkSlave): | 190 | class AbortedSlave(OkSlave): |
2699 | 220 | """A mock slave that looks like it's aborted.""" | 191 | """A mock slave that looks like it's aborted.""" |
2700 | 221 | 192 | ||
2702 | 222 | def clean(self): | 193 | def status(self): |
2703 | 223 | self.call_log.append('status') | 194 | self.call_log.append('status') |
2709 | 224 | return defer.succeed(None) | 195 | return ('BuilderStatus.ABORTED', '1-1') |
2705 | 225 | |||
2706 | 226 | def status(self): | ||
2707 | 227 | self.call_log.append('clean') | ||
2708 | 228 | return defer.succeed(('BuilderStatus.ABORTED', '1-1')) | ||
2710 | 229 | 196 | ||
2711 | 230 | 197 | ||
2712 | 231 | class LostBuildingBrokenSlave: | 198 | class LostBuildingBrokenSlave: |
2713 | @@ -239,108 +206,16 @@ | |||
2714 | 239 | 206 | ||
2715 | 240 | def status(self): | 207 | def status(self): |
2716 | 241 | self.call_log.append('status') | 208 | self.call_log.append('status') |
2718 | 242 | return defer.succeed(('BuilderStatus.BUILDING', '1000-10000')) | 209 | return ('BuilderStatus.BUILDING', '1000-10000') |
2719 | 243 | 210 | ||
2720 | 244 | def abort(self): | 211 | def abort(self): |
2721 | 245 | self.call_log.append('abort') | 212 | self.call_log.append('abort') |
2723 | 246 | return defer.fail(xmlrpclib.Fault(8002, "Could not abort")) | 213 | raise xmlrpclib.Fault(8002, "Could not abort") |
2724 | 247 | 214 | ||
2725 | 248 | 215 | ||
2726 | 249 | class BrokenSlave: | 216 | class BrokenSlave: |
2727 | 250 | """A mock slave that reports that it is broken.""" | 217 | """A mock slave that reports that it is broken.""" |
2728 | 251 | 218 | ||
2729 | 252 | def __init__(self): | ||
2730 | 253 | self.call_log = [] | ||
2731 | 254 | |||
2732 | 255 | def status(self): | 219 | def status(self): |
2733 | 256 | self.call_log.append('status') | 220 | self.call_log.append('status') |
2824 | 257 | return defer.fail(xmlrpclib.Fault(8001, "Broken slave")) | 221 | raise xmlrpclib.Fault(8001, "Broken slave") |
2735 | 258 | |||
2736 | 259 | |||
2737 | 260 | class CorruptBehavior: | ||
2738 | 261 | |||
2739 | 262 | def verifySlaveBuildCookie(self, cookie): | ||
2740 | 263 | raise CorruptBuildCookie("Bad value: %r" % (cookie,)) | ||
2741 | 264 | |||
2742 | 265 | |||
2743 | 266 | class TrivialBehavior: | ||
2744 | 267 | |||
2745 | 268 | def verifySlaveBuildCookie(self, cookie): | ||
2746 | 269 | pass | ||
2747 | 270 | |||
2748 | 271 | |||
2749 | 272 | class DeadProxy(xmlrpc.Proxy): | ||
2750 | 273 | """An xmlrpc.Proxy that doesn't actually send any messages. | ||
2751 | 274 | |||
2752 | 275 | Used when you want to test timeouts, for example. | ||
2753 | 276 | """ | ||
2754 | 277 | |||
2755 | 278 | def callRemote(self, *args, **kwargs): | ||
2756 | 279 | return defer.Deferred() | ||
2757 | 280 | |||
2758 | 281 | |||
2759 | 282 | class SlaveTestHelpers(fixtures.Fixture): | ||
2760 | 283 | |||
2761 | 284 | # The URL for the XML-RPC service set up by `BuilddSlaveTestSetup`. | ||
2762 | 285 | BASE_URL = 'http://localhost:8221' | ||
2763 | 286 | TEST_URL = '%s/rpc/' % (BASE_URL,) | ||
2764 | 287 | |||
2765 | 288 | def getServerSlave(self): | ||
2766 | 289 | """Set up a test build slave server. | ||
2767 | 290 | |||
2768 | 291 | :return: A `BuilddSlaveTestSetup` object. | ||
2769 | 292 | """ | ||
2770 | 293 | tachandler = BuilddSlaveTestSetup() | ||
2771 | 294 | tachandler.setUp() | ||
2772 | 295 | # Basically impossible to do this w/ TrialTestCase. But it would be | ||
2773 | 296 | # really nice to keep it. | ||
2774 | 297 | # | ||
2775 | 298 | # def addLogFile(exc_info): | ||
2776 | 299 | # self.addDetail( | ||
2777 | 300 | # 'xmlrpc-log-file', | ||
2778 | 301 | # Content(UTF8_TEXT, lambda: open(tachandler.logfile, 'r').read())) | ||
2779 | 302 | # self.addOnException(addLogFile) | ||
2780 | 303 | self.addCleanup(tachandler.tearDown) | ||
2781 | 304 | return tachandler | ||
2782 | 305 | |||
2783 | 306 | def getClientSlave(self, reactor=None, proxy=None): | ||
2784 | 307 | """Return a `BuilderSlave` for use in testing. | ||
2785 | 308 | |||
2786 | 309 | Points to a fixed URL that is also used by `BuilddSlaveTestSetup`. | ||
2787 | 310 | """ | ||
2788 | 311 | return BuilderSlave.makeBuilderSlave( | ||
2789 | 312 | self.TEST_URL, 'vmhost', reactor, proxy) | ||
2790 | 313 | |||
2791 | 314 | def makeCacheFile(self, tachandler, filename): | ||
2792 | 315 | """Make a cache file available on the remote slave. | ||
2793 | 316 | |||
2794 | 317 | :param tachandler: The TacTestSetup object used to start the remote | ||
2795 | 318 | slave. | ||
2796 | 319 | :param filename: The name of the file to create in the file cache | ||
2797 | 320 | area. | ||
2798 | 321 | """ | ||
2799 | 322 | path = os.path.join(tachandler.root, 'filecache', filename) | ||
2800 | 323 | fd = open(path, 'w') | ||
2801 | 324 | fd.write('something') | ||
2802 | 325 | fd.close() | ||
2803 | 326 | self.addCleanup(os.unlink, path) | ||
2804 | 327 | |||
2805 | 328 | def triggerGoodBuild(self, slave, build_id=None): | ||
2806 | 329 | """Trigger a good build on 'slave'. | ||
2807 | 330 | |||
2808 | 331 | :param slave: A `BuilderSlave` instance to trigger the build on. | ||
2809 | 332 | :param build_id: The build identifier. If not specified, defaults to | ||
2810 | 333 | an arbitrary string. | ||
2811 | 334 | :type build_id: str | ||
2812 | 335 | :return: The build id returned by the slave. | ||
2813 | 336 | """ | ||
2814 | 337 | if build_id is None: | ||
2815 | 338 | build_id = 'random-build-id' | ||
2816 | 339 | tachandler = self.getServerSlave() | ||
2817 | 340 | chroot_file = 'fake-chroot' | ||
2818 | 341 | dsc_file = 'thing' | ||
2819 | 342 | self.makeCacheFile(tachandler, chroot_file) | ||
2820 | 343 | self.makeCacheFile(tachandler, dsc_file) | ||
2821 | 344 | return slave.build( | ||
2822 | 345 | build_id, 'debian', chroot_file, {'.dsc': dsc_file}, | ||
2823 | 346 | {'ogrecomponent': 'main'}) | ||
2825 | 347 | 222 | ||
2826 | === modified file 'lib/lp/buildmaster/tests/test_builder.py' | |||
2827 | --- lib/lp/buildmaster/tests/test_builder.py 2010-10-18 16:44:22 +0000 | |||
2828 | +++ lib/lp/buildmaster/tests/test_builder.py 2010-12-07 16:29:13 +0000 | |||
2829 | @@ -3,24 +3,20 @@ | |||
2830 | 3 | 3 | ||
2831 | 4 | """Test Builder features.""" | 4 | """Test Builder features.""" |
2832 | 5 | 5 | ||
2833 | 6 | import errno | ||
2834 | 6 | import os | 7 | import os |
2836 | 7 | import signal | 8 | import socket |
2837 | 8 | import xmlrpclib | 9 | import xmlrpclib |
2838 | 9 | 10 | ||
2845 | 10 | from twisted.web.client import getPage | 11 | from testtools.content import Content |
2846 | 11 | 12 | from testtools.content_type import UTF8_TEXT | |
2841 | 12 | from twisted.internet.defer import CancelledError | ||
2842 | 13 | from twisted.internet.task import Clock | ||
2843 | 14 | from twisted.python.failure import Failure | ||
2844 | 15 | from twisted.trial.unittest import TestCase as TrialTestCase | ||
2847 | 16 | 13 | ||
2848 | 17 | from zope.component import getUtility | 14 | from zope.component import getUtility |
2849 | 18 | from zope.security.proxy import removeSecurityProxy | 15 | from zope.security.proxy import removeSecurityProxy |
2850 | 19 | 16 | ||
2851 | 20 | from canonical.buildd.slave import BuilderStatus | 17 | from canonical.buildd.slave import BuilderStatus |
2853 | 21 | from canonical.config import config | 18 | from canonical.buildd.tests.harness import BuilddSlaveTestSetup |
2854 | 22 | from canonical.database.sqlbase import flush_database_updates | 19 | from canonical.database.sqlbase import flush_database_updates |
2855 | 23 | from canonical.launchpad.scripts import QuietFakeLogger | ||
2856 | 24 | from canonical.launchpad.webapp.interfaces import ( | 20 | from canonical.launchpad.webapp.interfaces import ( |
2857 | 25 | DEFAULT_FLAVOR, | 21 | DEFAULT_FLAVOR, |
2858 | 26 | IStoreSelector, | 22 | IStoreSelector, |
2859 | @@ -28,38 +24,21 @@ | |||
2860 | 28 | ) | 24 | ) |
2861 | 29 | from canonical.testing.layers import ( | 25 | from canonical.testing.layers import ( |
2862 | 30 | DatabaseFunctionalLayer, | 26 | DatabaseFunctionalLayer, |
2866 | 31 | LaunchpadZopelessLayer, | 27 | LaunchpadZopelessLayer |
2864 | 32 | TwistedLaunchpadZopelessLayer, | ||
2865 | 33 | TwistedLayer, | ||
2867 | 34 | ) | 28 | ) |
2868 | 35 | from lp.buildmaster.enums import BuildStatus | 29 | from lp.buildmaster.enums import BuildStatus |
2874 | 36 | from lp.buildmaster.interfaces.builder import ( | 30 | from lp.buildmaster.interfaces.builder import IBuilder, IBuilderSet |
2870 | 37 | CannotFetchFile, | ||
2871 | 38 | IBuilder, | ||
2872 | 39 | IBuilderSet, | ||
2873 | 40 | ) | ||
2875 | 41 | from lp.buildmaster.interfaces.buildfarmjobbehavior import ( | 31 | from lp.buildmaster.interfaces.buildfarmjobbehavior import ( |
2876 | 42 | IBuildFarmJobBehavior, | 32 | IBuildFarmJobBehavior, |
2877 | 43 | ) | 33 | ) |
2878 | 44 | from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet | 34 | from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet |
2880 | 45 | from lp.buildmaster.interfaces.builder import CannotResumeHost | 35 | from lp.buildmaster.model.builder import BuilderSlave |
2881 | 46 | from lp.buildmaster.model.buildfarmjobbehavior import IdleBuildBehavior | 36 | from lp.buildmaster.model.buildfarmjobbehavior import IdleBuildBehavior |
2882 | 47 | from lp.buildmaster.model.buildqueue import BuildQueue | 37 | from lp.buildmaster.model.buildqueue import BuildQueue |
2883 | 48 | from lp.buildmaster.tests.mock_slaves import ( | 38 | from lp.buildmaster.tests.mock_slaves import ( |
2884 | 49 | AbortedSlave, | 39 | AbortedSlave, |
2885 | 50 | AbortingSlave, | ||
2886 | 51 | BrokenSlave, | ||
2887 | 52 | BuildingSlave, | ||
2888 | 53 | CorruptBehavior, | ||
2889 | 54 | DeadProxy, | ||
2890 | 55 | LostBuildingBrokenSlave, | ||
2891 | 56 | MockBuilder, | 40 | MockBuilder, |
2892 | 57 | OkSlave, | ||
2893 | 58 | SlaveTestHelpers, | ||
2894 | 59 | TrivialBehavior, | ||
2895 | 60 | WaitingSlave, | ||
2896 | 61 | ) | 41 | ) |
2897 | 62 | from lp.services.job.interfaces.job import JobStatus | ||
2898 | 63 | from lp.soyuz.enums import ( | 42 | from lp.soyuz.enums import ( |
2899 | 64 | ArchivePurpose, | 43 | ArchivePurpose, |
2900 | 65 | PackagePublishingStatus, | 44 | PackagePublishingStatus, |
2901 | @@ -70,12 +49,9 @@ | |||
2902 | 70 | ) | 49 | ) |
2903 | 71 | from lp.soyuz.tests.test_publishing import SoyuzTestPublisher | 50 | from lp.soyuz.tests.test_publishing import SoyuzTestPublisher |
2904 | 72 | from lp.testing import ( | 51 | from lp.testing import ( |
2908 | 73 | ANONYMOUS, | 52 | TestCase, |
2906 | 74 | login_as, | ||
2907 | 75 | logout, | ||
2909 | 76 | TestCaseWithFactory, | 53 | TestCaseWithFactory, |
2910 | 77 | ) | 54 | ) |
2911 | 78 | from lp.testing.factory import LaunchpadObjectFactory | ||
2912 | 79 | from lp.testing.fakemethod import FakeMethod | 55 | from lp.testing.fakemethod import FakeMethod |
2913 | 80 | 56 | ||
2914 | 81 | 57 | ||
2915 | @@ -116,121 +92,42 @@ | |||
2916 | 116 | bq = builder.getBuildQueue() | 92 | bq = builder.getBuildQueue() |
2917 | 117 | self.assertIs(None, bq) | 93 | self.assertIs(None, bq) |
2918 | 118 | 94 | ||
3034 | 119 | 95 | def test_updateBuilderStatus_catches_repeated_EINTR(self): | |
3035 | 120 | class TestBuilderWithTrial(TrialTestCase): | 96 | # A single EINTR return from a socket operation should cause the |
3036 | 121 | 97 | # operation to be retried, not fail/reset the builder. | |
3037 | 122 | layer = TwistedLaunchpadZopelessLayer | 98 | builder = removeSecurityProxy(self.factory.makeBuilder()) |
3038 | 123 | 99 | builder.handleTimeout = FakeMethod() | |
3039 | 124 | def setUp(self): | 100 | builder.rescueIfLost = FakeMethod() |
3040 | 125 | super(TestBuilderWithTrial, self) | 101 | |
3041 | 126 | self.slave_helper = SlaveTestHelpers() | 102 | def _fake_checkSlaveAlive(): |
3042 | 127 | self.slave_helper.setUp() | 103 | # Raise an EINTR error for all invocations. |
3043 | 128 | self.addCleanup(self.slave_helper.cleanUp) | 104 | raise socket.error(errno.EINTR, "fake eintr") |
3044 | 129 | self.factory = LaunchpadObjectFactory() | 105 | |
3045 | 130 | login_as(ANONYMOUS) | 106 | builder.checkSlaveAlive = _fake_checkSlaveAlive |
3046 | 131 | self.addCleanup(logout) | 107 | builder.updateStatus() |
3047 | 132 | 108 | ||
3048 | 133 | def test_updateStatus_aborts_lost_and_broken_slave(self): | 109 | # builder.updateStatus should eventually have called |
3049 | 134 | # A slave that's 'lost' should be aborted; when the slave is | 110 | # handleTimeout() |
3050 | 135 | # broken then abort() should also throw a fault. | 111 | self.assertEqual(1, builder.handleTimeout.call_count) |
3051 | 136 | slave = LostBuildingBrokenSlave() | 112 | |
3052 | 137 | lostbuilding_builder = MockBuilder( | 113 | def test_updateBuilderStatus_catches_single_EINTR(self): |
3053 | 138 | 'Lost Building Broken Slave', slave, behavior=CorruptBehavior()) | 114 | builder = removeSecurityProxy(self.factory.makeBuilder()) |
3054 | 139 | d = lostbuilding_builder.updateStatus(QuietFakeLogger()) | 115 | builder.handleTimeout = FakeMethod() |
3055 | 140 | def check_slave_status(failure): | 116 | builder.rescueIfLost = FakeMethod() |
3056 | 141 | self.assertIn('abort', slave.call_log) | 117 | self.eintr_returned = False |
3057 | 142 | # 'Fault' comes from the LostBuildingBrokenSlave, this is | 118 | |
3058 | 143 | # just testing that the value is passed through. | 119 | def _fake_checkSlaveAlive(): |
3059 | 144 | self.assertIsInstance(failure.value, xmlrpclib.Fault) | 120 | # raise an EINTR error for the first invocation only. |
3060 | 145 | return d.addBoth(check_slave_status) | 121 | if not self.eintr_returned: |
3061 | 146 | 122 | self.eintr_returned = True | |
3062 | 147 | def test_resumeSlaveHost_nonvirtual(self): | 123 | raise socket.error(errno.EINTR, "fake eintr") |
3063 | 148 | builder = self.factory.makeBuilder(virtualized=False) | 124 | |
3064 | 149 | d = builder.resumeSlaveHost() | 125 | builder.checkSlaveAlive = _fake_checkSlaveAlive |
3065 | 150 | return self.assertFailure(d, CannotResumeHost) | 126 | builder.updateStatus() |
3066 | 151 | 127 | ||
3067 | 152 | def test_resumeSlaveHost_no_vmhost(self): | 128 | # builder.updateStatus should never call handleTimeout() for a |
3068 | 153 | builder = self.factory.makeBuilder(virtualized=True, vm_host=None) | 129 | # single EINTR. |
3069 | 154 | d = builder.resumeSlaveHost() | 130 | self.assertEqual(0, builder.handleTimeout.call_count) |
2955 | 155 | return self.assertFailure(d, CannotResumeHost) | ||
2956 | 156 | |||
2957 | 157 | def test_resumeSlaveHost_success(self): | ||
2958 | 158 | reset_config = """ | ||
2959 | 159 | [builddmaster] | ||
2960 | 160 | vm_resume_command: /bin/echo -n parp""" | ||
2961 | 161 | config.push('reset', reset_config) | ||
2962 | 162 | self.addCleanup(config.pop, 'reset') | ||
2963 | 163 | |||
2964 | 164 | builder = self.factory.makeBuilder(virtualized=True, vm_host="pop") | ||
2965 | 165 | d = builder.resumeSlaveHost() | ||
2966 | 166 | def got_resume(output): | ||
2967 | 167 | self.assertEqual(('parp', ''), output) | ||
2968 | 168 | return d.addCallback(got_resume) | ||
2969 | 169 | |||
2970 | 170 | def test_resumeSlaveHost_command_failed(self): | ||
2971 | 171 | reset_fail_config = """ | ||
2972 | 172 | [builddmaster] | ||
2973 | 173 | vm_resume_command: /bin/false""" | ||
2974 | 174 | config.push('reset fail', reset_fail_config) | ||
2975 | 175 | self.addCleanup(config.pop, 'reset fail') | ||
2976 | 176 | builder = self.factory.makeBuilder(virtualized=True, vm_host="pop") | ||
2977 | 177 | d = builder.resumeSlaveHost() | ||
2978 | 178 | return self.assertFailure(d, CannotResumeHost) | ||
2979 | 179 | |||
2980 | 180 | def test_handleTimeout_resume_failure(self): | ||
2981 | 181 | reset_fail_config = """ | ||
2982 | 182 | [builddmaster] | ||
2983 | 183 | vm_resume_command: /bin/false""" | ||
2984 | 184 | config.push('reset fail', reset_fail_config) | ||
2985 | 185 | self.addCleanup(config.pop, 'reset fail') | ||
2986 | 186 | builder = self.factory.makeBuilder(virtualized=True, vm_host="pop") | ||
2987 | 187 | builder.builderok = True | ||
2988 | 188 | d = builder.handleTimeout(QuietFakeLogger(), 'blah') | ||
2989 | 189 | return self.assertFailure(d, CannotResumeHost) | ||
2990 | 190 | |||
2991 | 191 | def _setupRecipeBuildAndBuilder(self): | ||
2992 | 192 | # Helper function to make a builder capable of building a | ||
2993 | 193 | # recipe, returning both. | ||
2994 | 194 | processor = self.factory.makeProcessor(name="i386") | ||
2995 | 195 | builder = self.factory.makeBuilder( | ||
2996 | 196 | processor=processor, virtualized=True, vm_host="bladh") | ||
2997 | 197 | builder.setSlaveForTesting(OkSlave()) | ||
2998 | 198 | distroseries = self.factory.makeDistroSeries() | ||
2999 | 199 | das = self.factory.makeDistroArchSeries( | ||
3000 | 200 | distroseries=distroseries, architecturetag="i386", | ||
3001 | 201 | processorfamily=processor.family) | ||
3002 | 202 | chroot = self.factory.makeLibraryFileAlias() | ||
3003 | 203 | das.addOrUpdateChroot(chroot) | ||
3004 | 204 | distroseries.nominatedarchindep = das | ||
3005 | 205 | build = self.factory.makeSourcePackageRecipeBuild( | ||
3006 | 206 | distroseries=distroseries) | ||
3007 | 207 | return builder, build | ||
3008 | 208 | |||
3009 | 209 | def test_findAndStartJob_returns_candidate(self): | ||
3010 | 210 | # findAndStartJob finds the next queued job using _findBuildCandidate. | ||
3011 | 211 | # We don't care about the type of build at all. | ||
3012 | 212 | builder, build = self._setupRecipeBuildAndBuilder() | ||
3013 | 213 | candidate = build.queueBuild() | ||
3014 | 214 | # _findBuildCandidate is tested elsewhere, we just make sure that | ||
3015 | 215 | # findAndStartJob delegates to it. | ||
3016 | 216 | removeSecurityProxy(builder)._findBuildCandidate = FakeMethod( | ||
3017 | 217 | result=candidate) | ||
3018 | 218 | d = builder.findAndStartJob() | ||
3019 | 219 | return d.addCallback(self.assertEqual, candidate) | ||
3020 | 220 | |||
3021 | 221 | def test_findAndStartJob_starts_job(self): | ||
3022 | 222 | # findAndStartJob finds the next queued job using _findBuildCandidate | ||
3023 | 223 | # and then starts it. | ||
3024 | 224 | # We don't care about the type of build at all. | ||
3025 | 225 | builder, build = self._setupRecipeBuildAndBuilder() | ||
3026 | 226 | candidate = build.queueBuild() | ||
3027 | 227 | removeSecurityProxy(builder)._findBuildCandidate = FakeMethod( | ||
3028 | 228 | result=candidate) | ||
3029 | 229 | d = builder.findAndStartJob() | ||
3030 | 230 | def check_build_started(candidate): | ||
3031 | 231 | self.assertEqual(candidate.builder, builder) | ||
3032 | 232 | self.assertEqual(BuildStatus.BUILDING, build.status) | ||
3033 | 233 | return d.addCallback(check_build_started) | ||
3070 | 234 | 131 | ||
3071 | 235 | def test_slave(self): | 132 | def test_slave(self): |
3072 | 236 | # Builder.slave is a BuilderSlave that points at the actual Builder. | 133 | # Builder.slave is a BuilderSlave that points at the actual Builder. |
3073 | @@ -239,147 +136,25 @@ | |||
3074 | 239 | builder = removeSecurityProxy(self.factory.makeBuilder()) | 136 | builder = removeSecurityProxy(self.factory.makeBuilder()) |
3075 | 240 | self.assertEqual(builder.url, builder.slave.url) | 137 | self.assertEqual(builder.url, builder.slave.url) |
3076 | 241 | 138 | ||
3077 | 139 | |||
3078 | 140 | class Test_rescueBuilderIfLost(TestCaseWithFactory): | ||
3079 | 141 | """Tests for lp.buildmaster.model.builder.rescueBuilderIfLost.""" | ||
3080 | 142 | |||
3081 | 143 | layer = LaunchpadZopelessLayer | ||
3082 | 144 | |||
3083 | 242 | def test_recovery_of_aborted_slave(self): | 145 | def test_recovery_of_aborted_slave(self): |
3084 | 243 | # If a slave is in the ABORTED state, rescueBuilderIfLost should | 146 | # If a slave is in the ABORTED state, rescueBuilderIfLost should |
3085 | 244 | # clean it if we don't think it's currently building anything. | 147 | # clean it if we don't think it's currently building anything. |
3086 | 245 | # See bug 463046. | 148 | # See bug 463046. |
3087 | 246 | aborted_slave = AbortedSlave() | 149 | aborted_slave = AbortedSlave() |
3088 | 150 | # The slave's clean() method is normally an XMLRPC call, so we | ||
3089 | 151 | # can just stub it out and check that it got called. | ||
3090 | 152 | aborted_slave.clean = FakeMethod() | ||
3091 | 247 | builder = MockBuilder("mock_builder", aborted_slave) | 153 | builder = MockBuilder("mock_builder", aborted_slave) |
3092 | 248 | builder.currentjob = None | 154 | builder.currentjob = None |
3227 | 249 | d = builder.rescueIfLost() | 155 | builder.rescueIfLost() |
3228 | 250 | def check_slave_calls(ignored): | 156 | |
3229 | 251 | self.assertIn('clean', aborted_slave.call_log) | 157 | self.assertEqual(1, aborted_slave.clean.call_count) |
3096 | 252 | return d.addCallback(check_slave_calls) | ||
3097 | 253 | |||
3098 | 254 | def test_recover_ok_slave(self): | ||
3099 | 255 | # An idle slave is not rescued. | ||
3100 | 256 | slave = OkSlave() | ||
3101 | 257 | builder = MockBuilder("mock_builder", slave, TrivialBehavior()) | ||
3102 | 258 | d = builder.rescueIfLost() | ||
3103 | 259 | def check_slave_calls(ignored): | ||
3104 | 260 | self.assertNotIn('abort', slave.call_log) | ||
3105 | 261 | self.assertNotIn('clean', slave.call_log) | ||
3106 | 262 | return d.addCallback(check_slave_calls) | ||
3107 | 263 | |||
3108 | 264 | def test_recover_waiting_slave_with_good_id(self): | ||
3109 | 265 | # rescueIfLost does not attempt to abort or clean a builder that is | ||
3110 | 266 | # WAITING. | ||
3111 | 267 | waiting_slave = WaitingSlave() | ||
3112 | 268 | builder = MockBuilder("mock_builder", waiting_slave, TrivialBehavior()) | ||
3113 | 269 | d = builder.rescueIfLost() | ||
3114 | 270 | def check_slave_calls(ignored): | ||
3115 | 271 | self.assertNotIn('abort', waiting_slave.call_log) | ||
3116 | 272 | self.assertNotIn('clean', waiting_slave.call_log) | ||
3117 | 273 | return d.addCallback(check_slave_calls) | ||
3118 | 274 | |||
3119 | 275 | def test_recover_waiting_slave_with_bad_id(self): | ||
3120 | 276 | # If a slave is WAITING with a build for us to get, and the build | ||
3121 | 277 | # cookie cannot be verified, which means we don't recognize the build, | ||
3122 | 278 | # then rescueBuilderIfLost should attempt to abort it, so that the | ||
3123 | 279 | # builder is reset for a new build, and the corrupt build is | ||
3124 | 280 | # discarded. | ||
3125 | 281 | waiting_slave = WaitingSlave() | ||
3126 | 282 | builder = MockBuilder("mock_builder", waiting_slave, CorruptBehavior()) | ||
3127 | 283 | d = builder.rescueIfLost() | ||
3128 | 284 | def check_slave_calls(ignored): | ||
3129 | 285 | self.assertNotIn('abort', waiting_slave.call_log) | ||
3130 | 286 | self.assertIn('clean', waiting_slave.call_log) | ||
3131 | 287 | return d.addCallback(check_slave_calls) | ||
3132 | 288 | |||
3133 | 289 | def test_recover_building_slave_with_good_id(self): | ||
3134 | 290 | # rescueIfLost does not attempt to abort or clean a builder that is | ||
3135 | 291 | # BUILDING. | ||
3136 | 292 | building_slave = BuildingSlave() | ||
3137 | 293 | builder = MockBuilder("mock_builder", building_slave, TrivialBehavior()) | ||
3138 | 294 | d = builder.rescueIfLost() | ||
3139 | 295 | def check_slave_calls(ignored): | ||
3140 | 296 | self.assertNotIn('abort', building_slave.call_log) | ||
3141 | 297 | self.assertNotIn('clean', building_slave.call_log) | ||
3142 | 298 | return d.addCallback(check_slave_calls) | ||
3143 | 299 | |||
3144 | 300 | def test_recover_building_slave_with_bad_id(self): | ||
3145 | 301 | # If a slave is BUILDING with a build id we don't recognize, then we | ||
3146 | 302 | # abort the build, thus stopping it in its tracks. | ||
3147 | 303 | building_slave = BuildingSlave() | ||
3148 | 304 | builder = MockBuilder("mock_builder", building_slave, CorruptBehavior()) | ||
3149 | 305 | d = builder.rescueIfLost() | ||
3150 | 306 | def check_slave_calls(ignored): | ||
3151 | 307 | self.assertIn('abort', building_slave.call_log) | ||
3152 | 308 | self.assertNotIn('clean', building_slave.call_log) | ||
3153 | 309 | return d.addCallback(check_slave_calls) | ||
3154 | 310 | |||
3155 | 311 | |||
3156 | 312 | class TestBuilderSlaveStatus(TestBuilderWithTrial): | ||
3157 | 313 | |||
3158 | 314 | # Verify what IBuilder.slaveStatus returns with slaves in different | ||
3159 | 315 | # states. | ||
3160 | 316 | |||
3161 | 317 | def assertStatus(self, slave, builder_status=None, | ||
3162 | 318 | build_status=None, logtail=False, filemap=None, | ||
3163 | 319 | dependencies=None): | ||
3164 | 320 | builder = self.factory.makeBuilder() | ||
3165 | 321 | builder.setSlaveForTesting(slave) | ||
3166 | 322 | d = builder.slaveStatus() | ||
3167 | 323 | |||
3168 | 324 | def got_status(status_dict): | ||
3169 | 325 | expected = {} | ||
3170 | 326 | if builder_status is not None: | ||
3171 | 327 | expected["builder_status"] = builder_status | ||
3172 | 328 | if build_status is not None: | ||
3173 | 329 | expected["build_status"] = build_status | ||
3174 | 330 | if dependencies is not None: | ||
3175 | 331 | expected["dependencies"] = dependencies | ||
3176 | 332 | |||
3177 | 333 | # We don't care so much about the content of the logtail, | ||
3178 | 334 | # just that it's there. | ||
3179 | 335 | if logtail: | ||
3180 | 336 | tail = status_dict.pop("logtail") | ||
3181 | 337 | self.assertIsInstance(tail, xmlrpclib.Binary) | ||
3182 | 338 | |||
3183 | 339 | self.assertEqual(expected, status_dict) | ||
3184 | 340 | |||
3185 | 341 | return d.addCallback(got_status) | ||
3186 | 342 | |||
3187 | 343 | def test_slaveStatus_idle_slave(self): | ||
3188 | 344 | self.assertStatus( | ||
3189 | 345 | OkSlave(), builder_status='BuilderStatus.IDLE') | ||
3190 | 346 | |||
3191 | 347 | def test_slaveStatus_building_slave(self): | ||
3192 | 348 | self.assertStatus( | ||
3193 | 349 | BuildingSlave(), builder_status='BuilderStatus.BUILDING', | ||
3194 | 350 | logtail=True) | ||
3195 | 351 | |||
3196 | 352 | def test_slaveStatus_waiting_slave(self): | ||
3197 | 353 | self.assertStatus( | ||
3198 | 354 | WaitingSlave(), builder_status='BuilderStatus.WAITING', | ||
3199 | 355 | build_status='BuildStatus.OK', filemap={}) | ||
3200 | 356 | |||
3201 | 357 | def test_slaveStatus_aborting_slave(self): | ||
3202 | 358 | self.assertStatus( | ||
3203 | 359 | AbortingSlave(), builder_status='BuilderStatus.ABORTING') | ||
3204 | 360 | |||
3205 | 361 | def test_slaveStatus_aborted_slave(self): | ||
3206 | 362 | self.assertStatus( | ||
3207 | 363 | AbortedSlave(), builder_status='BuilderStatus.ABORTED') | ||
3208 | 364 | |||
3209 | 365 | def test_isAvailable_with_not_builderok(self): | ||
3210 | 366 | # isAvailable() is a wrapper around slaveStatusSentence() | ||
3211 | 367 | builder = self.factory.makeBuilder() | ||
3212 | 368 | builder.builderok = False | ||
3213 | 369 | d = builder.isAvailable() | ||
3214 | 370 | return d.addCallback(self.assertFalse) | ||
3215 | 371 | |||
3216 | 372 | def test_isAvailable_with_slave_fault(self): | ||
3217 | 373 | builder = self.factory.makeBuilder() | ||
3218 | 374 | builder.setSlaveForTesting(BrokenSlave()) | ||
3219 | 375 | d = builder.isAvailable() | ||
3220 | 376 | return d.addCallback(self.assertFalse) | ||
3221 | 377 | |||
3222 | 378 | def test_isAvailable_with_slave_idle(self): | ||
3223 | 379 | builder = self.factory.makeBuilder() | ||
3224 | 380 | builder.setSlaveForTesting(OkSlave()) | ||
3225 | 381 | d = builder.isAvailable() | ||
3226 | 382 | return d.addCallback(self.assertTrue) | ||
3230 | 383 | 158 | ||
3231 | 384 | 159 | ||
3232 | 385 | class TestFindBuildCandidateBase(TestCaseWithFactory): | 160 | class TestFindBuildCandidateBase(TestCaseWithFactory): |
3233 | @@ -413,49 +188,6 @@ | |||
3234 | 413 | builder.manual = False | 188 | builder.manual = False |
3235 | 414 | 189 | ||
3236 | 415 | 190 | ||
3237 | 416 | class TestFindBuildCandidateGeneralCases(TestFindBuildCandidateBase): | ||
3238 | 417 | # Test usage of findBuildCandidate not specific to any archive type. | ||
3239 | 418 | |||
3240 | 419 | def test_findBuildCandidate_supersedes_builds(self): | ||
3241 | 420 | # IBuilder._findBuildCandidate identifies if there are builds | ||
3242 | 421 | # for superseded source package releases in the queue and marks | ||
3243 | 422 | # the corresponding build record as SUPERSEDED. | ||
3244 | 423 | archive = self.factory.makeArchive() | ||
3245 | 424 | self.publisher.getPubSource( | ||
3246 | 425 | sourcename="gedit", status=PackagePublishingStatus.PUBLISHED, | ||
3247 | 426 | archive=archive).createMissingBuilds() | ||
3248 | 427 | old_candidate = removeSecurityProxy( | ||
3249 | 428 | self.frog_builder)._findBuildCandidate() | ||
3250 | 429 | |||
3251 | 430 | # The candidate starts off as NEEDSBUILD: | ||
3252 | 431 | build = getUtility(IBinaryPackageBuildSet).getByQueueEntry( | ||
3253 | 432 | old_candidate) | ||
3254 | 433 | self.assertEqual(BuildStatus.NEEDSBUILD, build.status) | ||
3255 | 434 | |||
3256 | 435 | # Now supersede the source package: | ||
3257 | 436 | publication = build.current_source_publication | ||
3258 | 437 | publication.status = PackagePublishingStatus.SUPERSEDED | ||
3259 | 438 | |||
3260 | 439 | # The candidate returned is now a different one: | ||
3261 | 440 | new_candidate = removeSecurityProxy( | ||
3262 | 441 | self.frog_builder)._findBuildCandidate() | ||
3263 | 442 | self.assertNotEqual(new_candidate, old_candidate) | ||
3264 | 443 | |||
3265 | 444 | # And the old_candidate is superseded: | ||
3266 | 445 | self.assertEqual(BuildStatus.SUPERSEDED, build.status) | ||
3267 | 446 | |||
3268 | 447 | def test_acquireBuildCandidate_marks_building(self): | ||
3269 | 448 | # acquireBuildCandidate() should call _findBuildCandidate and | ||
3270 | 449 | # mark the build as building. | ||
3271 | 450 | archive = self.factory.makeArchive() | ||
3272 | 451 | self.publisher.getPubSource( | ||
3273 | 452 | sourcename="gedit", status=PackagePublishingStatus.PUBLISHED, | ||
3274 | 453 | archive=archive).createMissingBuilds() | ||
3275 | 454 | candidate = removeSecurityProxy( | ||
3276 | 455 | self.frog_builder).acquireBuildCandidate() | ||
3277 | 456 | self.assertEqual(JobStatus.RUNNING, candidate.job.status) | ||
3278 | 457 | |||
3279 | 458 | |||
3280 | 459 | class TestFindBuildCandidatePPAWithSingleBuilder(TestCaseWithFactory): | 191 | class TestFindBuildCandidatePPAWithSingleBuilder(TestCaseWithFactory): |
3281 | 460 | 192 | ||
3282 | 461 | layer = LaunchpadZopelessLayer | 193 | layer = LaunchpadZopelessLayer |
3283 | @@ -588,16 +320,6 @@ | |||
3284 | 588 | build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(next_job) | 320 | build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(next_job) |
3285 | 589 | self.failUnlessEqual('joesppa', build.archive.name) | 321 | self.failUnlessEqual('joesppa', build.archive.name) |
3286 | 590 | 322 | ||
3287 | 591 | def test_findBuildCandidate_with_disabled_archive(self): | ||
3288 | 592 | # Disabled archives should not be considered for dispatching | ||
3289 | 593 | # builds. | ||
3290 | 594 | disabled_job = removeSecurityProxy(self.builder4)._findBuildCandidate() | ||
3291 | 595 | build = getUtility(IBinaryPackageBuildSet).getByQueueEntry( | ||
3292 | 596 | disabled_job) | ||
3293 | 597 | build.archive.disable() | ||
3294 | 598 | next_job = removeSecurityProxy(self.builder4)._findBuildCandidate() | ||
3295 | 599 | self.assertNotEqual(disabled_job, next_job) | ||
3296 | 600 | |||
3297 | 601 | 323 | ||
3298 | 602 | class TestFindBuildCandidatePrivatePPA(TestFindBuildCandidatePPABase): | 324 | class TestFindBuildCandidatePrivatePPA(TestFindBuildCandidatePPABase): |
3299 | 603 | 325 | ||
3300 | @@ -610,14 +332,6 @@ | |||
3301 | 610 | build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(next_job) | 332 | build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(next_job) |
3302 | 611 | self.failUnlessEqual('joesppa', build.archive.name) | 333 | self.failUnlessEqual('joesppa', build.archive.name) |
3303 | 612 | 334 | ||
3304 | 613 | # If the source for the build is still pending, it won't be | ||
3305 | 614 | # dispatched because the builder has to fetch the source files | ||
3306 | 615 | # from the (password protected) repo area, not the librarian. | ||
3307 | 616 | pub = build.current_source_publication | ||
3308 | 617 | pub.status = PackagePublishingStatus.PENDING | ||
3309 | 618 | candidate = removeSecurityProxy(self.builder4)._findBuildCandidate() | ||
3310 | 619 | self.assertNotEqual(next_job.id, candidate.id) | ||
3311 | 620 | |||
3312 | 621 | 335 | ||
3313 | 622 | class TestFindBuildCandidateDistroArchive(TestFindBuildCandidateBase): | 336 | class TestFindBuildCandidateDistroArchive(TestFindBuildCandidateBase): |
3314 | 623 | 337 | ||
3315 | @@ -760,48 +474,97 @@ | |||
3316 | 760 | self.builder.current_build_behavior, BinaryPackageBuildBehavior) | 474 | self.builder.current_build_behavior, BinaryPackageBuildBehavior) |
3317 | 761 | 475 | ||
3318 | 762 | 476 | ||
3320 | 763 | class TestSlave(TrialTestCase): | 477 | class TestSlave(TestCase): |
3321 | 764 | """ | 478 | """ |
3322 | 765 | Integration tests for BuilderSlave that verify how it works against a | 479 | Integration tests for BuilderSlave that verify how it works against a |
3323 | 766 | real slave server. | 480 | real slave server. |
3324 | 767 | """ | 481 | """ |
3325 | 768 | 482 | ||
3326 | 769 | layer = TwistedLayer | ||
3327 | 770 | |||
3328 | 771 | def setUp(self): | ||
3329 | 772 | super(TestSlave, self).setUp() | ||
3330 | 773 | self.slave_helper = SlaveTestHelpers() | ||
3331 | 774 | self.slave_helper.setUp() | ||
3332 | 775 | self.addCleanup(self.slave_helper.cleanUp) | ||
3333 | 776 | |||
3334 | 777 | # XXX: JonathanLange 2010-09-20 bug=643521: There are also tests for | 483 | # XXX: JonathanLange 2010-09-20 bug=643521: There are also tests for |
3335 | 778 | # BuilderSlave in buildd-slave.txt and in other places. The tests here | 484 | # BuilderSlave in buildd-slave.txt and in other places. The tests here |
3336 | 779 | # ought to become the canonical tests for BuilderSlave vs running buildd | 485 | # ought to become the canonical tests for BuilderSlave vs running buildd |
3337 | 780 | # XML-RPC server interaction. | 486 | # XML-RPC server interaction. |
3338 | 781 | 487 | ||
3339 | 488 | # The URL for the XML-RPC service set up by `BuilddSlaveTestSetup`. | ||
3340 | 489 | TEST_URL = 'http://localhost:8221/rpc/' | ||
3341 | 490 | |||
3342 | 491 | def getServerSlave(self): | ||
3343 | 492 | """Set up a test build slave server. | ||
3344 | 493 | |||
3345 | 494 | :return: A `BuilddSlaveTestSetup` object. | ||
3346 | 495 | """ | ||
3347 | 496 | tachandler = BuilddSlaveTestSetup() | ||
3348 | 497 | tachandler.setUp() | ||
3349 | 498 | self.addCleanup(tachandler.tearDown) | ||
3350 | 499 | def addLogFile(exc_info): | ||
3351 | 500 | self.addDetail( | ||
3352 | 501 | 'xmlrpc-log-file', | ||
3353 | 502 | Content(UTF8_TEXT, lambda: open(tachandler.logfile, 'r').read())) | ||
3354 | 503 | self.addOnException(addLogFile) | ||
3355 | 504 | return tachandler | ||
3356 | 505 | |||
3357 | 506 | def getClientSlave(self): | ||
3358 | 507 | """Return a `BuilderSlave` for use in testing. | ||
3359 | 508 | |||
3360 | 509 | Points to a fixed URL that is also used by `BuilddSlaveTestSetup`. | ||
3361 | 510 | """ | ||
3362 | 511 | return BuilderSlave.makeBlockingSlave(self.TEST_URL, 'vmhost') | ||
3363 | 512 | |||
3364 | 513 | def makeCacheFile(self, tachandler, filename): | ||
3365 | 514 | """Make a cache file available on the remote slave. | ||
3366 | 515 | |||
3367 | 516 | :param tachandler: The TacTestSetup object used to start the remote | ||
3368 | 517 | slave. | ||
3369 | 518 | :param filename: The name of the file to create in the file cache | ||
3370 | 519 | area. | ||
3371 | 520 | """ | ||
3372 | 521 | path = os.path.join(tachandler.root, 'filecache', filename) | ||
3373 | 522 | fd = open(path, 'w') | ||
3374 | 523 | fd.write('something') | ||
3375 | 524 | fd.close() | ||
3376 | 525 | self.addCleanup(os.unlink, path) | ||
3377 | 526 | |||
3378 | 527 | def triggerGoodBuild(self, slave, build_id=None): | ||
3379 | 528 | """Trigger a good build on 'slave'. | ||
3380 | 529 | |||
3381 | 530 | :param slave: A `BuilderSlave` instance to trigger the build on. | ||
3382 | 531 | :param build_id: The build identifier. If not specified, defaults to | ||
3383 | 532 | an arbitrary string. | ||
3384 | 533 | :type build_id: str | ||
3385 | 534 | :return: The build id returned by the slave. | ||
3386 | 535 | """ | ||
3387 | 536 | if build_id is None: | ||
3388 | 537 | build_id = self.getUniqueString() | ||
3389 | 538 | tachandler = self.getServerSlave() | ||
3390 | 539 | chroot_file = 'fake-chroot' | ||
3391 | 540 | dsc_file = 'thing' | ||
3392 | 541 | self.makeCacheFile(tachandler, chroot_file) | ||
3393 | 542 | self.makeCacheFile(tachandler, dsc_file) | ||
3394 | 543 | return slave.build( | ||
3395 | 544 | build_id, 'debian', chroot_file, {'.dsc': dsc_file}, | ||
3396 | 545 | {'ogrecomponent': 'main'}) | ||
3397 | 546 | |||
3398 | 782 | # XXX 2010-10-06 Julian bug=655559 | 547 | # XXX 2010-10-06 Julian bug=655559 |
3399 | 783 | # This is failing on buildbot but not locally; it's trying to abort | 548 | # This is failing on buildbot but not locally; it's trying to abort |
3400 | 784 | # before the build has started. | 549 | # before the build has started. |
3401 | 785 | def disabled_test_abort(self): | 550 | def disabled_test_abort(self): |
3403 | 786 | slave = self.slave_helper.getClientSlave() | 551 | slave = self.getClientSlave() |
3404 | 787 | # We need to be in a BUILDING state before we can abort. | 552 | # We need to be in a BUILDING state before we can abort. |
3409 | 788 | d = self.slave_helper.triggerGoodBuild(slave) | 553 | self.triggerGoodBuild(slave) |
3410 | 789 | d.addCallback(lambda ignored: slave.abort()) | 554 | result = slave.abort() |
3411 | 790 | d.addCallback(self.assertEqual, BuilderStatus.ABORTING) | 555 | self.assertEqual(result, BuilderStatus.ABORTING) |
3408 | 791 | return d | ||
3412 | 792 | 556 | ||
3413 | 793 | def test_build(self): | 557 | def test_build(self): |
3414 | 794 | # Calling 'build' with an expected builder type, a good build id, | 558 | # Calling 'build' with an expected builder type, a good build id, |
3415 | 795 | # valid chroot & filemaps works and returns a BuilderStatus of | 559 | # valid chroot & filemaps works and returns a BuilderStatus of |
3416 | 796 | # BUILDING. | 560 | # BUILDING. |
3417 | 797 | build_id = 'some-id' | 561 | build_id = 'some-id' |
3422 | 798 | slave = self.slave_helper.getClientSlave() | 562 | slave = self.getClientSlave() |
3423 | 799 | d = self.slave_helper.triggerGoodBuild(slave, build_id) | 563 | result = self.triggerGoodBuild(slave, build_id) |
3424 | 800 | return d.addCallback( | 564 | self.assertEqual([BuilderStatus.BUILDING, build_id], result) |
3421 | 801 | self.assertEqual, [BuilderStatus.BUILDING, build_id]) | ||
3425 | 802 | 565 | ||
3426 | 803 | def test_clean(self): | 566 | def test_clean(self): |
3428 | 804 | slave = self.slave_helper.getClientSlave() | 567 | slave = self.getClientSlave() |
3429 | 805 | # XXX: JonathanLange 2010-09-21: Calling clean() on the slave requires | 568 | # XXX: JonathanLange 2010-09-21: Calling clean() on the slave requires |
3430 | 806 | # it to be in either the WAITING or ABORTED states, and both of these | 569 | # it to be in either the WAITING or ABORTED states, and both of these |
3431 | 807 | # states are very difficult to achieve in a test environment. For the | 570 | # states are very difficult to achieve in a test environment. For the |
3432 | @@ -811,248 +574,57 @@ | |||
3433 | 811 | def test_echo(self): | 574 | def test_echo(self): |
3434 | 812 | # Calling 'echo' contacts the server which returns the arguments we | 575 | # Calling 'echo' contacts the server which returns the arguments we |
3435 | 813 | # gave it. | 576 | # gave it. |
3440 | 814 | self.slave_helper.getServerSlave() | 577 | self.getServerSlave() |
3441 | 815 | slave = self.slave_helper.getClientSlave() | 578 | slave = self.getClientSlave() |
3442 | 816 | d = slave.echo('foo', 'bar', 42) | 579 | result = slave.echo('foo', 'bar', 42) |
3443 | 817 | return d.addCallback(self.assertEqual, ['foo', 'bar', 42]) | 580 | self.assertEqual(['foo', 'bar', 42], result) |
3444 | 818 | 581 | ||
3445 | 819 | def test_info(self): | 582 | def test_info(self): |
3446 | 820 | # Calling 'info' gets some information about the slave. | 583 | # Calling 'info' gets some information about the slave. |
3450 | 821 | self.slave_helper.getServerSlave() | 584 | self.getServerSlave() |
3451 | 822 | slave = self.slave_helper.getClientSlave() | 585 | slave = self.getClientSlave() |
3452 | 823 | d = slave.info() | 586 | result = slave.info() |
3453 | 824 | # We're testing the hard-coded values, since the version is hard-coded | 587 | # We're testing the hard-coded values, since the version is hard-coded |
3454 | 825 | # into the remote slave, the supported build managers are hard-coded | 588 | # into the remote slave, the supported build managers are hard-coded |
3455 | 826 | # into the tac file for the remote slave and config is returned from | 589 | # into the tac file for the remote slave and config is returned from |
3456 | 827 | # the configuration file. | 590 | # the configuration file. |
3459 | 828 | return d.addCallback( | 591 | self.assertEqual( |
3458 | 829 | self.assertEqual, | ||
3460 | 830 | ['1.0', | 592 | ['1.0', |
3461 | 831 | 'i386', | 593 | 'i386', |
3462 | 832 | ['sourcepackagerecipe', | 594 | ['sourcepackagerecipe', |
3464 | 833 | 'translation-templates', 'binarypackage', 'debian']]) | 595 | 'translation-templates', 'binarypackage', 'debian']], |
3465 | 596 | result) | ||
3466 | 834 | 597 | ||
3467 | 835 | def test_initial_status(self): | 598 | def test_initial_status(self): |
3468 | 836 | # Calling 'status' returns the current status of the slave. The | 599 | # Calling 'status' returns the current status of the slave. The |
3469 | 837 | # initial status is IDLE. | 600 | # initial status is IDLE. |
3474 | 838 | self.slave_helper.getServerSlave() | 601 | self.getServerSlave() |
3475 | 839 | slave = self.slave_helper.getClientSlave() | 602 | slave = self.getClientSlave() |
3476 | 840 | d = slave.status() | 603 | status = slave.status() |
3477 | 841 | return d.addCallback(self.assertEqual, [BuilderStatus.IDLE, '']) | 604 | self.assertEqual([BuilderStatus.IDLE, ''], status) |
3478 | 842 | 605 | ||
3479 | 843 | def test_status_after_build(self): | 606 | def test_status_after_build(self): |
3480 | 844 | # Calling 'status' returns the current status of the slave. After a | 607 | # Calling 'status' returns the current status of the slave. After a |
3481 | 845 | # build has been triggered, the status is BUILDING. | 608 | # build has been triggered, the status is BUILDING. |
3483 | 846 | slave = self.slave_helper.getClientSlave() | 609 | slave = self.getClientSlave() |
3484 | 847 | build_id = 'status-build-id' | 610 | build_id = 'status-build-id' |
3492 | 848 | d = self.slave_helper.triggerGoodBuild(slave, build_id) | 611 | self.triggerGoodBuild(slave, build_id) |
3493 | 849 | d.addCallback(lambda ignored: slave.status()) | 612 | status = slave.status() |
3494 | 850 | def check_status(status): | 613 | self.assertEqual([BuilderStatus.BUILDING, build_id], status[:2]) |
3495 | 851 | self.assertEqual([BuilderStatus.BUILDING, build_id], status[:2]) | 614 | [log_file] = status[2:] |
3496 | 852 | [log_file] = status[2:] | 615 | self.assertIsInstance(log_file, xmlrpclib.Binary) |
3490 | 853 | self.assertIsInstance(log_file, xmlrpclib.Binary) | ||
3491 | 854 | return d.addCallback(check_status) | ||
3497 | 855 | 616 | ||
3498 | 856 | def test_ensurepresent_not_there(self): | 617 | def test_ensurepresent_not_there(self): |
3499 | 857 | # ensurepresent checks to see if a file is there. | 618 | # ensurepresent checks to see if a file is there. |
3505 | 858 | self.slave_helper.getServerSlave() | 619 | self.getServerSlave() |
3506 | 859 | slave = self.slave_helper.getClientSlave() | 620 | slave = self.getClientSlave() |
3507 | 860 | d = slave.ensurepresent('blahblah', None, None, None) | 621 | result = slave.ensurepresent('blahblah', None, None, None) |
3508 | 861 | d.addCallback(self.assertEqual, [False, 'No URL']) | 622 | self.assertEqual([False, 'No URL'], result) |
3504 | 862 | return d | ||
3509 | 863 | 623 | ||
3510 | 864 | def test_ensurepresent_actually_there(self): | 624 | def test_ensurepresent_actually_there(self): |
3511 | 865 | # ensurepresent checks to see if a file is there. | 625 | # ensurepresent checks to see if a file is there. |
3705 | 866 | tachandler = self.slave_helper.getServerSlave() | 626 | tachandler = self.getServerSlave() |
3706 | 867 | slave = self.slave_helper.getClientSlave() | 627 | slave = self.getClientSlave() |
3707 | 868 | self.slave_helper.makeCacheFile(tachandler, 'blahblah') | 628 | self.makeCacheFile(tachandler, 'blahblah') |
3708 | 869 | d = slave.ensurepresent('blahblah', None, None, None) | 629 | result = slave.ensurepresent('blahblah', None, None, None) |
3709 | 870 | d.addCallback(self.assertEqual, [True, 'No URL']) | 630 | self.assertEqual([True, 'No URL'], result) |
3517 | 871 | return d | ||
3518 | 872 | |||
3519 | 873 | def test_sendFileToSlave_not_there(self): | ||
3520 | 874 | self.slave_helper.getServerSlave() | ||
3521 | 875 | slave = self.slave_helper.getClientSlave() | ||
3522 | 876 | d = slave.sendFileToSlave('blahblah', None, None, None) | ||
3523 | 877 | return self.assertFailure(d, CannotFetchFile) | ||
3524 | 878 | |||
3525 | 879 | def test_sendFileToSlave_actually_there(self): | ||
3526 | 880 | tachandler = self.slave_helper.getServerSlave() | ||
3527 | 881 | slave = self.slave_helper.getClientSlave() | ||
3528 | 882 | self.slave_helper.makeCacheFile(tachandler, 'blahblah') | ||
3529 | 883 | d = slave.sendFileToSlave('blahblah', None, None, None) | ||
3530 | 884 | def check_present(ignored): | ||
3531 | 885 | d = slave.ensurepresent('blahblah', None, None, None) | ||
3532 | 886 | return d.addCallback(self.assertEqual, [True, 'No URL']) | ||
3533 | 887 | d.addCallback(check_present) | ||
3534 | 888 | return d | ||
3535 | 889 | |||
3536 | 890 | def test_resumeHost_success(self): | ||
3537 | 891 | # On a successful resume resume() fires the returned deferred | ||
3538 | 892 | # callback with 'None'. | ||
3539 | 893 | self.slave_helper.getServerSlave() | ||
3540 | 894 | slave = self.slave_helper.getClientSlave() | ||
3541 | 895 | |||
3542 | 896 | # The configuration testing command-line. | ||
3543 | 897 | self.assertEqual( | ||
3544 | 898 | 'echo %(vm_host)s', config.builddmaster.vm_resume_command) | ||
3545 | 899 | |||
3546 | 900 | # On success the response is None. | ||
3547 | 901 | def check_resume_success(response): | ||
3548 | 902 | out, err, code = response | ||
3549 | 903 | self.assertEqual(os.EX_OK, code) | ||
3550 | 904 | # XXX: JonathanLange 2010-09-23: We should instead pass the | ||
3551 | 905 | # expected vm_host into the client slave. Not doing this now, | ||
3552 | 906 | # since the SlaveHelper is being moved around. | ||
3553 | 907 | self.assertEqual("%s\n" % slave._vm_host, out) | ||
3554 | 908 | d = slave.resume() | ||
3555 | 909 | d.addBoth(check_resume_success) | ||
3556 | 910 | return d | ||
3557 | 911 | |||
3558 | 912 | def test_resumeHost_failure(self): | ||
3559 | 913 | # On a failed resume, 'resumeHost' fires the returned deferred | ||
3560 | 914 | # errorback with the `ProcessTerminated` failure. | ||
3561 | 915 | self.slave_helper.getServerSlave() | ||
3562 | 916 | slave = self.slave_helper.getClientSlave() | ||
3563 | 917 | |||
3564 | 918 | # Override the configuration command-line with one that will fail. | ||
3565 | 919 | failed_config = """ | ||
3566 | 920 | [builddmaster] | ||
3567 | 921 | vm_resume_command: test "%(vm_host)s = 'no-sir'" | ||
3568 | 922 | """ | ||
3569 | 923 | config.push('failed_resume_command', failed_config) | ||
3570 | 924 | self.addCleanup(config.pop, 'failed_resume_command') | ||
3571 | 925 | |||
3572 | 926 | # On failures, the response is a twisted `Failure` object containing | ||
3573 | 927 | # a tuple. | ||
3574 | 928 | def check_resume_failure(failure): | ||
3575 | 929 | out, err, code = failure.value | ||
3576 | 930 | # The process will exit with a return code of "1". | ||
3577 | 931 | self.assertEqual(code, 1) | ||
3578 | 932 | d = slave.resume() | ||
3579 | 933 | d.addBoth(check_resume_failure) | ||
3580 | 934 | return d | ||
3581 | 935 | |||
3582 | 936 | def test_resumeHost_timeout(self): | ||
3583 | 937 | # On a resume timeouts, 'resumeHost' fires the returned deferred | ||
3584 | 938 | # errorback with the `TimeoutError` failure. | ||
3585 | 939 | self.slave_helper.getServerSlave() | ||
3586 | 940 | slave = self.slave_helper.getClientSlave() | ||
3587 | 941 | |||
3588 | 942 | # Override the configuration command-line with one that will timeout. | ||
3589 | 943 | timeout_config = """ | ||
3590 | 944 | [builddmaster] | ||
3591 | 945 | vm_resume_command: sleep 5 | ||
3592 | 946 | socket_timeout: 1 | ||
3593 | 947 | """ | ||
3594 | 948 | config.push('timeout_resume_command', timeout_config) | ||
3595 | 949 | self.addCleanup(config.pop, 'timeout_resume_command') | ||
3596 | 950 | |||
3597 | 951 | # On timeouts, the response is a twisted `Failure` object containing | ||
3598 | 952 | # a `TimeoutError` error. | ||
3599 | 953 | def check_resume_timeout(failure): | ||
3600 | 954 | self.assertIsInstance(failure, Failure) | ||
3601 | 955 | out, err, code = failure.value | ||
3602 | 956 | self.assertEqual(code, signal.SIGKILL) | ||
3603 | 957 | clock = Clock() | ||
3604 | 958 | d = slave.resume(clock=clock) | ||
3605 | 959 | # Move the clock beyond the socket_timeout but earlier than the | ||
3606 | 960 | # sleep 5. This stops the test having to wait for the timeout. | ||
3607 | 961 | # Fast tests FTW! | ||
3608 | 962 | clock.advance(2) | ||
3609 | 963 | d.addBoth(check_resume_timeout) | ||
3610 | 964 | return d | ||
3611 | 965 | |||
3612 | 966 | |||
3613 | 967 | class TestSlaveTimeouts(TrialTestCase): | ||
3614 | 968 | # Testing that the methods that call callRemote() all time out | ||
3615 | 969 | # as required. | ||
3616 | 970 | |||
3617 | 971 | layer = TwistedLayer | ||
3618 | 972 | |||
3619 | 973 | def setUp(self): | ||
3620 | 974 | super(TestSlaveTimeouts, self).setUp() | ||
3621 | 975 | self.slave_helper = SlaveTestHelpers() | ||
3622 | 976 | self.slave_helper.setUp() | ||
3623 | 977 | self.addCleanup(self.slave_helper.cleanUp) | ||
3624 | 978 | self.clock = Clock() | ||
3625 | 979 | self.proxy = DeadProxy("url") | ||
3626 | 980 | self.slave = self.slave_helper.getClientSlave( | ||
3627 | 981 | reactor=self.clock, proxy=self.proxy) | ||
3628 | 982 | |||
3629 | 983 | def assertCancelled(self, d): | ||
3630 | 984 | self.clock.advance(config.builddmaster.socket_timeout + 1) | ||
3631 | 985 | return self.assertFailure(d, CancelledError) | ||
3632 | 986 | |||
3633 | 987 | def test_timeout_abort(self): | ||
3634 | 988 | return self.assertCancelled(self.slave.abort()) | ||
3635 | 989 | |||
3636 | 990 | def test_timeout_clean(self): | ||
3637 | 991 | return self.assertCancelled(self.slave.clean()) | ||
3638 | 992 | |||
3639 | 993 | def test_timeout_echo(self): | ||
3640 | 994 | return self.assertCancelled(self.slave.echo()) | ||
3641 | 995 | |||
3642 | 996 | def test_timeout_info(self): | ||
3643 | 997 | return self.assertCancelled(self.slave.info()) | ||
3644 | 998 | |||
3645 | 999 | def test_timeout_status(self): | ||
3646 | 1000 | return self.assertCancelled(self.slave.status()) | ||
3647 | 1001 | |||
3648 | 1002 | def test_timeout_ensurepresent(self): | ||
3649 | 1003 | return self.assertCancelled( | ||
3650 | 1004 | self.slave.ensurepresent(None, None, None, None)) | ||
3651 | 1005 | |||
3652 | 1006 | def test_timeout_build(self): | ||
3653 | 1007 | return self.assertCancelled( | ||
3654 | 1008 | self.slave.build(None, None, None, None, None)) | ||
3655 | 1009 | |||
3656 | 1010 | |||
3657 | 1011 | class TestSlaveWithLibrarian(TrialTestCase): | ||
3658 | 1012 | """Tests that need more of Launchpad to run.""" | ||
3659 | 1013 | |||
3660 | 1014 | layer = TwistedLaunchpadZopelessLayer | ||
3661 | 1015 | |||
3662 | 1016 | def setUp(self): | ||
3663 | 1017 | super(TestSlaveWithLibrarian, self) | ||
3664 | 1018 | self.slave_helper = SlaveTestHelpers() | ||
3665 | 1019 | self.slave_helper.setUp() | ||
3666 | 1020 | self.addCleanup(self.slave_helper.cleanUp) | ||
3667 | 1021 | self.factory = LaunchpadObjectFactory() | ||
3668 | 1022 | login_as(ANONYMOUS) | ||
3669 | 1023 | self.addCleanup(logout) | ||
3670 | 1024 | |||
3671 | 1025 | def test_ensurepresent_librarian(self): | ||
3672 | 1026 | # ensurepresent, when given an http URL for a file will download the | ||
3673 | 1027 | # file from that URL and report that the file is present, and it was | ||
3674 | 1028 | # downloaded. | ||
3675 | 1029 | |||
3676 | 1030 | # Use the Librarian because it's a "convenient" web server. | ||
3677 | 1031 | lf = self.factory.makeLibraryFileAlias( | ||
3678 | 1032 | 'HelloWorld.txt', content="Hello World") | ||
3679 | 1033 | self.layer.txn.commit() | ||
3680 | 1034 | self.slave_helper.getServerSlave() | ||
3681 | 1035 | slave = self.slave_helper.getClientSlave() | ||
3682 | 1036 | d = slave.ensurepresent( | ||
3683 | 1037 | lf.content.sha1, lf.http_url, "", "") | ||
3684 | 1038 | d.addCallback(self.assertEqual, [True, 'Download']) | ||
3685 | 1039 | return d | ||
3686 | 1040 | |||
3687 | 1041 | def test_retrieve_files_from_filecache(self): | ||
3688 | 1042 | # Files that are present on the slave can be downloaded with a | ||
3689 | 1043 | # filename made from the sha1 of the content underneath the | ||
3690 | 1044 | # 'filecache' directory. | ||
3691 | 1045 | content = "Hello World" | ||
3692 | 1046 | lf = self.factory.makeLibraryFileAlias( | ||
3693 | 1047 | 'HelloWorld.txt', content=content) | ||
3694 | 1048 | self.layer.txn.commit() | ||
3695 | 1049 | expected_url = '%s/filecache/%s' % ( | ||
3696 | 1050 | self.slave_helper.BASE_URL, lf.content.sha1) | ||
3697 | 1051 | self.slave_helper.getServerSlave() | ||
3698 | 1052 | slave = self.slave_helper.getClientSlave() | ||
3699 | 1053 | d = slave.ensurepresent( | ||
3700 | 1054 | lf.content.sha1, lf.http_url, "", "") | ||
3701 | 1055 | def check_file(ignored): | ||
3702 | 1056 | d = getPage(expected_url.encode('utf8')) | ||
3703 | 1057 | return d.addCallback(self.assertEqual, content) | ||
3704 | 1058 | return d.addCallback(check_file) | ||
3710 | 1059 | 631 | ||
3711 | === modified file 'lib/lp/buildmaster/tests/test_manager.py' | |||
3712 | --- lib/lp/buildmaster/tests/test_manager.py 2010-10-19 13:58:21 +0000 | |||
3713 | +++ lib/lp/buildmaster/tests/test_manager.py 2010-12-07 16:29:13 +0000 | |||
3714 | @@ -6,7 +6,6 @@ | |||
3715 | 6 | import os | 6 | import os |
3716 | 7 | import signal | 7 | import signal |
3717 | 8 | import time | 8 | import time |
3718 | 9 | import xmlrpclib | ||
3719 | 10 | 9 | ||
3720 | 11 | import transaction | 10 | import transaction |
3721 | 12 | 11 | ||
3722 | @@ -15,7 +14,9 @@ | |||
3723 | 15 | reactor, | 14 | reactor, |
3724 | 16 | task, | 15 | task, |
3725 | 17 | ) | 16 | ) |
3726 | 17 | from twisted.internet.error import ConnectionClosed | ||
3727 | 18 | from twisted.internet.task import ( | 18 | from twisted.internet.task import ( |
3728 | 19 | Clock, | ||
3729 | 19 | deferLater, | 20 | deferLater, |
3730 | 20 | ) | 21 | ) |
3731 | 21 | from twisted.python.failure import Failure | 22 | from twisted.python.failure import Failure |
3732 | @@ -29,45 +30,577 @@ | |||
3733 | 29 | ANONYMOUS, | 30 | ANONYMOUS, |
3734 | 30 | login, | 31 | login, |
3735 | 31 | ) | 32 | ) |
3739 | 32 | from canonical.launchpad.scripts.logger import ( | 33 | from canonical.launchpad.scripts.logger import BufferLogger |
3737 | 33 | QuietFakeLogger, | ||
3738 | 34 | ) | ||
3740 | 35 | from canonical.testing.layers import ( | 34 | from canonical.testing.layers import ( |
3741 | 36 | LaunchpadScriptLayer, | 35 | LaunchpadScriptLayer, |
3743 | 37 | TwistedLaunchpadZopelessLayer, | 36 | LaunchpadZopelessLayer, |
3744 | 38 | TwistedLayer, | 37 | TwistedLayer, |
3745 | 39 | ZopelessDatabaseLayer, | ||
3746 | 40 | ) | 38 | ) |
3747 | 41 | from lp.buildmaster.enums import BuildStatus | 39 | from lp.buildmaster.enums import BuildStatus |
3748 | 42 | from lp.buildmaster.interfaces.builder import IBuilderSet | 40 | from lp.buildmaster.interfaces.builder import IBuilderSet |
3749 | 43 | from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet | 41 | from lp.buildmaster.interfaces.buildqueue import IBuildQueueSet |
3750 | 44 | from lp.buildmaster.manager import ( | 42 | from lp.buildmaster.manager import ( |
3752 | 45 | assessFailureCounts, | 43 | BaseDispatchResult, |
3753 | 44 | buildd_success_result_map, | ||
3754 | 46 | BuilddManager, | 45 | BuilddManager, |
3755 | 46 | FailDispatchResult, | ||
3756 | 47 | NewBuildersScanner, | 47 | NewBuildersScanner, |
3757 | 48 | RecordingSlave, | ||
3758 | 49 | ResetDispatchResult, | ||
3759 | 48 | SlaveScanner, | 50 | SlaveScanner, |
3760 | 49 | ) | 51 | ) |
3761 | 50 | from lp.buildmaster.model.builder import Builder | ||
3762 | 51 | from lp.buildmaster.tests.harness import BuilddManagerTestSetup | 52 | from lp.buildmaster.tests.harness import BuilddManagerTestSetup |
3768 | 52 | from lp.buildmaster.tests.mock_slaves import ( | 53 | from lp.buildmaster.tests.mock_slaves import BuildingSlave |
3764 | 53 | BrokenSlave, | ||
3765 | 54 | BuildingSlave, | ||
3766 | 55 | OkSlave, | ||
3767 | 56 | ) | ||
3769 | 57 | from lp.registry.interfaces.distribution import IDistributionSet | 54 | from lp.registry.interfaces.distribution import IDistributionSet |
3770 | 58 | from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet | 55 | from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet |
3772 | 59 | from lp.testing import TestCaseWithFactory | 56 | from lp.soyuz.tests.test_publishing import SoyuzTestPublisher |
3773 | 57 | from lp.testing import TestCase as LaunchpadTestCase | ||
3774 | 60 | from lp.testing.factory import LaunchpadObjectFactory | 58 | from lp.testing.factory import LaunchpadObjectFactory |
3775 | 61 | from lp.testing.fakemethod import FakeMethod | 59 | from lp.testing.fakemethod import FakeMethod |
3776 | 62 | from lp.testing.sampledata import BOB_THE_BUILDER_NAME | 60 | from lp.testing.sampledata import BOB_THE_BUILDER_NAME |
3777 | 63 | 61 | ||
3778 | 64 | 62 | ||
3779 | 63 | class TestRecordingSlaves(TrialTestCase): | ||
3780 | 64 | """Tests for the recording slave class.""" | ||
3781 | 65 | layer = TwistedLayer | ||
3782 | 66 | |||
3783 | 67 | def setUp(self): | ||
3784 | 68 | """Setup a fresh `RecordingSlave` for tests.""" | ||
3785 | 69 | TrialTestCase.setUp(self) | ||
3786 | 70 | self.slave = RecordingSlave( | ||
3787 | 71 | 'foo', 'http://foo:8221/rpc', 'foo.host') | ||
3788 | 72 | |||
3789 | 73 | def test_representation(self): | ||
3790 | 74 | """`RecordingSlave` has a custom representation. | ||
3791 | 75 | |||
3792 | 76 | It encloses builder name and xmlrpc url for debug purposes. | ||
3793 | 77 | """ | ||
3794 | 78 | self.assertEqual('<foo:http://foo:8221/rpc>', repr(self.slave)) | ||
3795 | 79 | |||
3796 | 80 | def assert_ensurepresent(self, func): | ||
3797 | 81 | """Helper function to test results from calling ensurepresent.""" | ||
3798 | 82 | self.assertEqual( | ||
3799 | 83 | [True, 'Download'], | ||
3800 | 84 | func('boing', 'bar', 'baz')) | ||
3801 | 85 | self.assertEqual( | ||
3802 | 86 | [('ensurepresent', ('boing', 'bar', 'baz'))], | ||
3803 | 87 | self.slave.calls) | ||
3804 | 88 | |||
3805 | 89 | def test_ensurepresent(self): | ||
3806 | 90 | """`RecordingSlave.ensurepresent` always succeeds. | ||
3807 | 91 | |||
3808 | 92 | It returns the expected succeed code and records the interaction | ||
3809 | 93 | information for later use. | ||
3810 | 94 | """ | ||
3811 | 95 | self.assert_ensurepresent(self.slave.ensurepresent) | ||
3812 | 96 | |||
3813 | 97 | def test_sendFileToSlave(self): | ||
3814 | 98 | """RecordingSlave.sendFileToSlave always succeeeds. | ||
3815 | 99 | |||
3816 | 100 | It calls ensurepresent() and hence returns the same results. | ||
3817 | 101 | """ | ||
3818 | 102 | self.assert_ensurepresent(self.slave.sendFileToSlave) | ||
3819 | 103 | |||
3820 | 104 | def test_build(self): | ||
3821 | 105 | """`RecordingSlave.build` always succeeds. | ||
3822 | 106 | |||
3823 | 107 | It returns the expected succeed code and records the interaction | ||
3824 | 108 | information for later use. | ||
3825 | 109 | """ | ||
3826 | 110 | self.assertEqual( | ||
3827 | 111 | ['BuilderStatus.BUILDING', 'boing'], | ||
3828 | 112 | self.slave.build('boing', 'bar', 'baz')) | ||
3829 | 113 | self.assertEqual( | ||
3830 | 114 | [('build', ('boing', 'bar', 'baz'))], | ||
3831 | 115 | self.slave.calls) | ||
3832 | 116 | |||
3833 | 117 | def test_resume(self): | ||
3834 | 118 | """`RecordingSlave.resume` always returns successs.""" | ||
3835 | 119 | # Resume isn't requested in a just-instantiated RecordingSlave. | ||
3836 | 120 | self.assertFalse(self.slave.resume_requested) | ||
3837 | 121 | |||
3838 | 122 | # When resume is called, it returns the success list and mark | ||
3839 | 123 | # the slave for resuming. | ||
3840 | 124 | self.assertEqual(['', '', os.EX_OK], self.slave.resume()) | ||
3841 | 125 | self.assertTrue(self.slave.resume_requested) | ||
3842 | 126 | |||
3843 | 127 | def test_resumeHost_success(self): | ||
3844 | 128 | # On a successful resume resumeHost() fires the returned deferred | ||
3845 | 129 | # callback with 'None'. | ||
3846 | 130 | |||
3847 | 131 | # The configuration testing command-line. | ||
3848 | 132 | self.assertEqual( | ||
3849 | 133 | 'echo %(vm_host)s', config.builddmaster.vm_resume_command) | ||
3850 | 134 | |||
3851 | 135 | # On success the response is None. | ||
3852 | 136 | def check_resume_success(response): | ||
3853 | 137 | out, err, code = response | ||
3854 | 138 | self.assertEqual(os.EX_OK, code) | ||
3855 | 139 | self.assertEqual("%s\n" % self.slave.vm_host, out) | ||
3856 | 140 | d = self.slave.resumeSlave() | ||
3857 | 141 | d.addBoth(check_resume_success) | ||
3858 | 142 | return d | ||
3859 | 143 | |||
3860 | 144 | def test_resumeHost_failure(self): | ||
3861 | 145 | # On a failed resume, 'resumeHost' fires the returned deferred | ||
3862 | 146 | # errorback with the `ProcessTerminated` failure. | ||
3863 | 147 | |||
3864 | 148 | # Override the configuration command-line with one that will fail. | ||
3865 | 149 | failed_config = """ | ||
3866 | 150 | [builddmaster] | ||
3867 | 151 | vm_resume_command: test "%(vm_host)s = 'no-sir'" | ||
3868 | 152 | """ | ||
3869 | 153 | config.push('failed_resume_command', failed_config) | ||
3870 | 154 | self.addCleanup(config.pop, 'failed_resume_command') | ||
3871 | 155 | |||
3872 | 156 | # On failures, the response is a twisted `Failure` object containing | ||
3873 | 157 | # a tuple. | ||
3874 | 158 | def check_resume_failure(failure): | ||
3875 | 159 | out, err, code = failure.value | ||
3876 | 160 | # The process will exit with a return code of "1". | ||
3877 | 161 | self.assertEqual(code, 1) | ||
3878 | 162 | d = self.slave.resumeSlave() | ||
3879 | 163 | d.addBoth(check_resume_failure) | ||
3880 | 164 | return d | ||
3881 | 165 | |||
3882 | 166 | def test_resumeHost_timeout(self): | ||
3883 | 167 | # On a resume timeouts, 'resumeHost' fires the returned deferred | ||
3884 | 168 | # errorback with the `TimeoutError` failure. | ||
3885 | 169 | |||
3886 | 170 | # Override the configuration command-line with one that will timeout. | ||
3887 | 171 | timeout_config = """ | ||
3888 | 172 | [builddmaster] | ||
3889 | 173 | vm_resume_command: sleep 5 | ||
3890 | 174 | socket_timeout: 1 | ||
3891 | 175 | """ | ||
3892 | 176 | config.push('timeout_resume_command', timeout_config) | ||
3893 | 177 | self.addCleanup(config.pop, 'timeout_resume_command') | ||
3894 | 178 | |||
3895 | 179 | # On timeouts, the response is a twisted `Failure` object containing | ||
3896 | 180 | # a `TimeoutError` error. | ||
3897 | 181 | def check_resume_timeout(failure): | ||
3898 | 182 | self.assertIsInstance(failure, Failure) | ||
3899 | 183 | out, err, code = failure.value | ||
3900 | 184 | self.assertEqual(code, signal.SIGKILL) | ||
3901 | 185 | clock = Clock() | ||
3902 | 186 | d = self.slave.resumeSlave(clock=clock) | ||
3903 | 187 | # Move the clock beyond the socket_timeout but earlier than the | ||
3904 | 188 | # sleep 5. This stops the test having to wait for the timeout. | ||
3905 | 189 | # Fast tests FTW! | ||
3906 | 190 | clock.advance(2) | ||
3907 | 191 | d.addBoth(check_resume_timeout) | ||
3908 | 192 | return d | ||
3909 | 193 | |||
3910 | 194 | |||
3911 | 195 | class TestingXMLRPCProxy: | ||
3912 | 196 | """This class mimics a twisted XMLRPC Proxy class.""" | ||
3913 | 197 | |||
3914 | 198 | def __init__(self, failure_info=None): | ||
3915 | 199 | self.calls = [] | ||
3916 | 200 | self.failure_info = failure_info | ||
3917 | 201 | self.works = failure_info is None | ||
3918 | 202 | |||
3919 | 203 | def callRemote(self, *args): | ||
3920 | 204 | self.calls.append(args) | ||
3921 | 205 | if self.works: | ||
3922 | 206 | result = buildd_success_result_map.get(args[0]) | ||
3923 | 207 | else: | ||
3924 | 208 | result = 'boing' | ||
3925 | 209 | return defer.succeed([result, self.failure_info]) | ||
3926 | 210 | |||
3927 | 211 | |||
3928 | 212 | class TestingResetDispatchResult(ResetDispatchResult): | ||
3929 | 213 | """Override the evaluation method to simply annotate the call.""" | ||
3930 | 214 | |||
3931 | 215 | def __init__(self, slave, info=None): | ||
3932 | 216 | ResetDispatchResult.__init__(self, slave, info) | ||
3933 | 217 | self.processed = False | ||
3934 | 218 | |||
3935 | 219 | def __call__(self): | ||
3936 | 220 | self.processed = True | ||
3937 | 221 | |||
3938 | 222 | |||
3939 | 223 | class TestingFailDispatchResult(FailDispatchResult): | ||
3940 | 224 | """Override the evaluation method to simply annotate the call.""" | ||
3941 | 225 | |||
3942 | 226 | def __init__(self, slave, info=None): | ||
3943 | 227 | FailDispatchResult.__init__(self, slave, info) | ||
3944 | 228 | self.processed = False | ||
3945 | 229 | |||
3946 | 230 | def __call__(self): | ||
3947 | 231 | self.processed = True | ||
3948 | 232 | |||
3949 | 233 | |||
3950 | 234 | class TestingSlaveScanner(SlaveScanner): | ||
3951 | 235 | """Override the dispatch result factories """ | ||
3952 | 236 | |||
3953 | 237 | reset_result = TestingResetDispatchResult | ||
3954 | 238 | fail_result = TestingFailDispatchResult | ||
3955 | 239 | |||
3956 | 240 | |||
3957 | 241 | class TestSlaveScanner(TrialTestCase): | ||
3958 | 242 | """Tests for the actual build slave manager.""" | ||
3959 | 243 | layer = LaunchpadZopelessLayer | ||
3960 | 244 | |||
3961 | 245 | def setUp(self): | ||
3962 | 246 | TrialTestCase.setUp(self) | ||
3963 | 247 | self.manager = TestingSlaveScanner( | ||
3964 | 248 | BOB_THE_BUILDER_NAME, BufferLogger()) | ||
3965 | 249 | |||
3966 | 250 | self.fake_builder_url = 'http://bob.buildd:8221/' | ||
3967 | 251 | self.fake_builder_host = 'bob.host' | ||
3968 | 252 | |||
3969 | 253 | # We will use an instrumented SlaveScanner instance for tests in | ||
3970 | 254 | # this context. | ||
3971 | 255 | |||
3972 | 256 | # Stop cyclic execution and record the end of the cycle. | ||
3973 | 257 | self.stopped = False | ||
3974 | 258 | |||
3975 | 259 | def testNextCycle(): | ||
3976 | 260 | self.stopped = True | ||
3977 | 261 | |||
3978 | 262 | self.manager.scheduleNextScanCycle = testNextCycle | ||
3979 | 263 | |||
3980 | 264 | # Return the testing Proxy version. | ||
3981 | 265 | self.test_proxy = TestingXMLRPCProxy() | ||
3982 | 266 | |||
3983 | 267 | def testGetProxyForSlave(slave): | ||
3984 | 268 | return self.test_proxy | ||
3985 | 269 | self.manager._getProxyForSlave = testGetProxyForSlave | ||
3986 | 270 | |||
3987 | 271 | # Deactivate the 'scan' method. | ||
3988 | 272 | def testScan(): | ||
3989 | 273 | pass | ||
3990 | 274 | self.manager.scan = testScan | ||
3991 | 275 | |||
3992 | 276 | # Stop automatic collection of dispatching results. | ||
3993 | 277 | def testslaveConversationEnded(): | ||
3994 | 278 | pass | ||
3995 | 279 | self._realslaveConversationEnded = self.manager.slaveConversationEnded | ||
3996 | 280 | self.manager.slaveConversationEnded = testslaveConversationEnded | ||
3997 | 281 | |||
3998 | 282 | def assertIsDispatchReset(self, result): | ||
3999 | 283 | self.assertTrue( | ||
4000 | 284 | isinstance(result, TestingResetDispatchResult), | ||
4001 | 285 | 'Dispatch failure did not result in a ResetBuildResult object') | ||
4002 | 286 | |||
4003 | 287 | def assertIsDispatchFail(self, result): | ||
4004 | 288 | self.assertTrue( | ||
4005 | 289 | isinstance(result, TestingFailDispatchResult), | ||
4006 | 290 | 'Dispatch failure did not result in a FailBuildResult object') | ||
4007 | 291 | |||
4008 | 292 | def test_checkResume(self): | ||
4009 | 293 | """`SlaveScanner.checkResume` is chained after resume requests. | ||
4010 | 294 | |||
4011 | 295 | If the resume request succeed it returns None, otherwise it returns | ||
4012 | 296 | a `ResetBuildResult` (the one in the test context) that will be | ||
4013 | 297 | collect and evaluated later. | ||
4014 | 298 | |||
4015 | 299 | See `RecordingSlave.resumeHost` for more information about the resume | ||
4016 | 300 | result contents. | ||
4017 | 301 | """ | ||
4018 | 302 | slave = RecordingSlave('foo', 'http://foo.buildd:8221/', 'foo.host') | ||
4019 | 303 | |||
4020 | 304 | successful_response = ['', '', os.EX_OK] | ||
4021 | 305 | result = self.manager.checkResume(successful_response, slave) | ||
4022 | 306 | self.assertEqual( | ||
4023 | 307 | None, result, 'Successful resume checks should return None') | ||
4024 | 308 | |||
4025 | 309 | failed_response = ['stdout', 'stderr', 1] | ||
4026 | 310 | result = self.manager.checkResume(failed_response, slave) | ||
4027 | 311 | self.assertIsDispatchReset(result) | ||
4028 | 312 | self.assertEqual( | ||
4029 | 313 | '<foo:http://foo.buildd:8221/> reset failure', repr(result)) | ||
4030 | 314 | self.assertEqual( | ||
4031 | 315 | result.info, "stdout\nstderr") | ||
4032 | 316 | |||
4033 | 317 | def test_fail_to_resume_slave_resets_slave(self): | ||
4034 | 318 | # If an attempt to resume and dispatch a slave fails, we reset the | ||
4035 | 319 | # slave by calling self.reset_result(slave)(). | ||
4036 | 320 | |||
4037 | 321 | reset_result_calls = [] | ||
4038 | 322 | |||
4039 | 323 | class LoggingResetResult(BaseDispatchResult): | ||
4040 | 324 | """A DispatchResult that logs calls to itself. | ||
4041 | 325 | |||
4042 | 326 | This *must* subclass BaseDispatchResult, otherwise finishCycle() | ||
4043 | 327 | won't treat it like a dispatch result. | ||
4044 | 328 | """ | ||
4045 | 329 | |||
4046 | 330 | def __init__(self, slave, info=None): | ||
4047 | 331 | self.slave = slave | ||
4048 | 332 | |||
4049 | 333 | def __call__(self): | ||
4050 | 334 | reset_result_calls.append(self.slave) | ||
4051 | 335 | |||
4052 | 336 | # Make a failing slave that is requesting a resume. | ||
4053 | 337 | slave = RecordingSlave('foo', 'http://foo.buildd:8221/', 'foo.host') | ||
4054 | 338 | slave.resume_requested = True | ||
4055 | 339 | slave.resumeSlave = lambda: deferLater( | ||
4056 | 340 | reactor, 0, defer.fail, Failure(('out', 'err', 1))) | ||
4057 | 341 | |||
4058 | 342 | # Make the manager log the reset result calls. | ||
4059 | 343 | self.manager.reset_result = LoggingResetResult | ||
4060 | 344 | |||
4061 | 345 | # We only care about this one slave. Reset the list of manager | ||
4062 | 346 | # deferreds in case setUp did something unexpected. | ||
4063 | 347 | self.manager._deferred_list = [] | ||
4064 | 348 | |||
4065 | 349 | # Here, we're patching the slaveConversationEnded method so we can | ||
4066 | 350 | # get an extra callback at the end of it, so we can | ||
4067 | 351 | # verify that the reset_result was really called. | ||
4068 | 352 | def _slaveConversationEnded(): | ||
4069 | 353 | d = self._realslaveConversationEnded() | ||
4070 | 354 | return d.addCallback( | ||
4071 | 355 | lambda ignored: self.assertEqual([slave], reset_result_calls)) | ||
4072 | 356 | self.manager.slaveConversationEnded = _slaveConversationEnded | ||
4073 | 357 | |||
4074 | 358 | self.manager.resumeAndDispatch(slave) | ||
4075 | 359 | |||
4076 | 360 | def test_failed_to_resume_slave_ready_for_reset(self): | ||
4077 | 361 | # When a slave fails to resume, the manager has a Deferred in its | ||
4078 | 362 | # Deferred list that is ready to fire with a ResetDispatchResult. | ||
4079 | 363 | |||
4080 | 364 | # Make a failing slave that is requesting a resume. | ||
4081 | 365 | slave = RecordingSlave('foo', 'http://foo.buildd:8221/', 'foo.host') | ||
4082 | 366 | slave.resume_requested = True | ||
4083 | 367 | slave.resumeSlave = lambda: defer.fail(Failure(('out', 'err', 1))) | ||
4084 | 368 | |||
4085 | 369 | # We only care about this one slave. Reset the list of manager | ||
4086 | 370 | # deferreds in case setUp did something unexpected. | ||
4087 | 371 | self.manager._deferred_list = [] | ||
4088 | 372 | # Restore the slaveConversationEnded method. It's very relevant to | ||
4089 | 373 | # this test. | ||
4090 | 374 | self.manager.slaveConversationEnded = self._realslaveConversationEnded | ||
4091 | 375 | self.manager.resumeAndDispatch(slave) | ||
4092 | 376 | [d] = self.manager._deferred_list | ||
4093 | 377 | |||
4094 | 378 | # The Deferred for our failing slave should be ready to fire | ||
4095 | 379 | # successfully with a ResetDispatchResult. | ||
4096 | 380 | def check_result(result): | ||
4097 | 381 | self.assertIsInstance(result, ResetDispatchResult) | ||
4098 | 382 | self.assertEqual(slave, result.slave) | ||
4099 | 383 | self.assertFalse(result.processed) | ||
4100 | 384 | return d.addCallback(check_result) | ||
4101 | 385 | |||
4102 | 386 | def _setUpSlaveAndBuilder(self, builder_failure_count=None, | ||
4103 | 387 | job_failure_count=None): | ||
4104 | 388 | # Helper function to set up a builder and its recording slave. | ||
4105 | 389 | if builder_failure_count is None: | ||
4106 | 390 | builder_failure_count = 0 | ||
4107 | 391 | if job_failure_count is None: | ||
4108 | 392 | job_failure_count = 0 | ||
4109 | 393 | slave = RecordingSlave( | ||
4110 | 394 | BOB_THE_BUILDER_NAME, self.fake_builder_url, | ||
4111 | 395 | self.fake_builder_host) | ||
4112 | 396 | bob_builder = getUtility(IBuilderSet)[slave.name] | ||
4113 | 397 | bob_builder.failure_count = builder_failure_count | ||
4114 | 398 | bob_builder.getCurrentBuildFarmJob().failure_count = job_failure_count | ||
4115 | 399 | return slave, bob_builder | ||
4116 | 400 | |||
4117 | 401 | def test_checkDispatch_success(self): | ||
4118 | 402 | # SlaveScanner.checkDispatch returns None for a successful | ||
4119 | 403 | # dispatch. | ||
4120 | 404 | |||
4121 | 405 | """ | ||
4122 | 406 | If the dispatch request fails or a unknown method is given, it | ||
4123 | 407 | returns a `FailDispatchResult` (in the test context) that will | ||
4124 | 408 | be evaluated later. | ||
4125 | 409 | |||
4126 | 410 | Builders will be marked as failed if the following responses | ||
4127 | 411 | categories are received. | ||
4128 | 412 | |||
4129 | 413 | * Legitimate slave failures: when the response is a list with 2 | ||
4130 | 414 | elements but the first element ('status') does not correspond to | ||
4131 | 415 | the expected 'success' result. See `buildd_success_result_map`. | ||
4132 | 416 | |||
4133 | 417 | * Unexpected (code) failures: when the given 'method' is unknown | ||
4134 | 418 | or the response isn't a 2-element list or Failure instance. | ||
4135 | 419 | |||
4136 | 420 | Communication failures (a twisted `Failure` instance) will simply | ||
4137 | 421 | cause the builder to be reset, a `ResetDispatchResult` object is | ||
4138 | 422 | returned. In other words, network failures are ignored in this | ||
4139 | 423 | stage, broken builders will be identified and marked as so | ||
4140 | 424 | during 'scan()' stage. | ||
4141 | 425 | |||
4142 | 426 | On success dispatching it returns None. | ||
4143 | 427 | """ | ||
4144 | 428 | slave, bob_builder = self._setUpSlaveAndBuilder( | ||
4145 | 429 | builder_failure_count=0, job_failure_count=0) | ||
4146 | 430 | |||
4147 | 431 | # Successful legitimate response, None is returned. | ||
4148 | 432 | successful_response = [ | ||
4149 | 433 | buildd_success_result_map.get('ensurepresent'), 'cool builder'] | ||
4150 | 434 | result = self.manager.checkDispatch( | ||
4151 | 435 | successful_response, 'ensurepresent', slave) | ||
4152 | 436 | self.assertEqual( | ||
4153 | 437 | None, result, 'Successful dispatch checks should return None') | ||
4154 | 438 | |||
4155 | 439 | def test_checkDispatch_first_fail(self): | ||
4156 | 440 | # Failed legitimate response, results in FailDispatchResult and | ||
4157 | 441 | # failure_count on the job and the builder are both incremented. | ||
4158 | 442 | slave, bob_builder = self._setUpSlaveAndBuilder( | ||
4159 | 443 | builder_failure_count=0, job_failure_count=0) | ||
4160 | 444 | |||
4161 | 445 | failed_response = [False, 'uncool builder'] | ||
4162 | 446 | result = self.manager.checkDispatch( | ||
4163 | 447 | failed_response, 'ensurepresent', slave) | ||
4164 | 448 | self.assertIsDispatchFail(result) | ||
4165 | 449 | self.assertEqual( | ||
4166 | 450 | repr(result), | ||
4167 | 451 | '<bob:%s> failure (uncool builder)' % self.fake_builder_url) | ||
4168 | 452 | self.assertEqual(1, bob_builder.failure_count) | ||
4169 | 453 | self.assertEqual( | ||
4170 | 454 | 1, bob_builder.getCurrentBuildFarmJob().failure_count) | ||
4171 | 455 | |||
4172 | 456 | def test_checkDispatch_second_reset_fail_by_builder(self): | ||
4173 | 457 | # Twisted Failure response, results in a `FailDispatchResult`. | ||
4174 | 458 | slave, bob_builder = self._setUpSlaveAndBuilder( | ||
4175 | 459 | builder_failure_count=1, job_failure_count=0) | ||
4176 | 460 | |||
4177 | 461 | twisted_failure = Failure(ConnectionClosed('Boom!')) | ||
4178 | 462 | result = self.manager.checkDispatch( | ||
4179 | 463 | twisted_failure, 'ensurepresent', slave) | ||
4180 | 464 | self.assertIsDispatchFail(result) | ||
4181 | 465 | self.assertEqual( | ||
4182 | 466 | '<bob:%s> failure (None)' % self.fake_builder_url, repr(result)) | ||
4183 | 467 | self.assertEqual(2, bob_builder.failure_count) | ||
4184 | 468 | self.assertEqual( | ||
4185 | 469 | 1, bob_builder.getCurrentBuildFarmJob().failure_count) | ||
4186 | 470 | |||
4187 | 471 | def test_checkDispatch_second_comms_fail_by_builder(self): | ||
4188 | 472 | # Unexpected response, results in a `FailDispatchResult`. | ||
4189 | 473 | slave, bob_builder = self._setUpSlaveAndBuilder( | ||
4190 | 474 | builder_failure_count=1, job_failure_count=0) | ||
4191 | 475 | |||
4192 | 476 | unexpected_response = [1, 2, 3] | ||
4193 | 477 | result = self.manager.checkDispatch( | ||
4194 | 478 | unexpected_response, 'build', slave) | ||
4195 | 479 | self.assertIsDispatchFail(result) | ||
4196 | 480 | self.assertEqual( | ||
4197 | 481 | '<bob:%s> failure ' | ||
4198 | 482 | '(Unexpected response: [1, 2, 3])' % self.fake_builder_url, | ||
4199 | 483 | repr(result)) | ||
4200 | 484 | self.assertEqual(2, bob_builder.failure_count) | ||
4201 | 485 | self.assertEqual( | ||
4202 | 486 | 1, bob_builder.getCurrentBuildFarmJob().failure_count) | ||
4203 | 487 | |||
4204 | 488 | def test_checkDispatch_second_comms_fail_by_job(self): | ||
4205 | 489 | # Unknown method was given, results in a `FailDispatchResult`. | ||
4206 | 490 | # This could be caused by a faulty job which would fail the job. | ||
4207 | 491 | slave, bob_builder = self._setUpSlaveAndBuilder( | ||
4208 | 492 | builder_failure_count=0, job_failure_count=1) | ||
4209 | 493 | |||
4210 | 494 | successful_response = [ | ||
4211 | 495 | buildd_success_result_map.get('ensurepresent'), 'cool builder'] | ||
4212 | 496 | result = self.manager.checkDispatch( | ||
4213 | 497 | successful_response, 'unknown-method', slave) | ||
4214 | 498 | self.assertIsDispatchFail(result) | ||
4215 | 499 | self.assertEqual( | ||
4216 | 500 | '<bob:%s> failure ' | ||
4217 | 501 | '(Unknown slave method: unknown-method)' % self.fake_builder_url, | ||
4218 | 502 | repr(result)) | ||
4219 | 503 | self.assertEqual(1, bob_builder.failure_count) | ||
4220 | 504 | self.assertEqual( | ||
4221 | 505 | 2, bob_builder.getCurrentBuildFarmJob().failure_count) | ||
4222 | 506 | |||
4223 | 507 | def test_initiateDispatch(self): | ||
4224 | 508 | """Check `dispatchBuild` in various scenarios. | ||
4225 | 509 | |||
4226 | 510 | When there are no recording slaves (i.e. no build got dispatched | ||
4227 | 511 | in scan()) it simply finishes the cycle. | ||
4228 | 512 | |||
4229 | 513 | When there is a recording slave with pending slave calls, they are | ||
4230 | 514 | performed and if they all succeed the cycle is finished with no | ||
4231 | 515 | errors. | ||
4232 | 516 | |||
4233 | 517 | On slave call failure the chain is stopped immediately and an | ||
4234 | 518 | FailDispatchResult is collected while finishing the cycle. | ||
4235 | 519 | """ | ||
4236 | 520 | def check_no_events(results): | ||
4237 | 521 | errors = [ | ||
4238 | 522 | r for s, r in results if isinstance(r, BaseDispatchResult)] | ||
4239 | 523 | self.assertEqual(0, len(errors)) | ||
4240 | 524 | |||
4241 | 525 | def check_events(results): | ||
4242 | 526 | [error] = [r for s, r in results if r is not None] | ||
4243 | 527 | self.assertEqual( | ||
4244 | 528 | '<bob:%s> failure (very broken slave)' | ||
4245 | 529 | % self.fake_builder_url, | ||
4246 | 530 | repr(error)) | ||
4247 | 531 | self.assertTrue(error.processed) | ||
4248 | 532 | |||
4249 | 533 | def _wait_on_deferreds_then_check_no_events(): | ||
4250 | 534 | dl = self._realslaveConversationEnded() | ||
4251 | 535 | dl.addCallback(check_no_events) | ||
4252 | 536 | |||
4253 | 537 | def _wait_on_deferreds_then_check_events(): | ||
4254 | 538 | dl = self._realslaveConversationEnded() | ||
4255 | 539 | dl.addCallback(check_events) | ||
4256 | 540 | |||
4257 | 541 | # A functional slave charged with some interactions. | ||
4258 | 542 | slave = RecordingSlave( | ||
4259 | 543 | BOB_THE_BUILDER_NAME, self.fake_builder_url, | ||
4260 | 544 | self.fake_builder_host) | ||
4261 | 545 | slave.ensurepresent('arg1', 'arg2', 'arg3') | ||
4262 | 546 | slave.build('arg1', 'arg2', 'arg3') | ||
4263 | 547 | |||
4264 | 548 | # If the previous step (resuming) has failed nothing gets dispatched. | ||
4265 | 549 | reset_result = ResetDispatchResult(slave) | ||
4266 | 550 | result = self.manager.initiateDispatch(reset_result, slave) | ||
4267 | 551 | self.assertTrue(result is reset_result) | ||
4268 | 552 | self.assertFalse(slave.resume_requested) | ||
4269 | 553 | self.assertEqual(0, len(self.manager._deferred_list)) | ||
4270 | 554 | |||
4271 | 555 | # Operation with the default (funcional slave), no resets or | ||
4272 | 556 | # failures results are triggered. | ||
4273 | 557 | slave.resume() | ||
4274 | 558 | result = self.manager.initiateDispatch(None, slave) | ||
4275 | 559 | self.assertEqual(None, result) | ||
4276 | 560 | self.assertTrue(slave.resume_requested) | ||
4277 | 561 | self.assertEqual( | ||
4278 | 562 | [('ensurepresent', 'arg1', 'arg2', 'arg3'), | ||
4279 | 563 | ('build', 'arg1', 'arg2', 'arg3')], | ||
4280 | 564 | self.test_proxy.calls) | ||
4281 | 565 | self.assertEqual(2, len(self.manager._deferred_list)) | ||
4282 | 566 | |||
4283 | 567 | # Monkey patch the slaveConversationEnded method so we can chain a | ||
4284 | 568 | # callback to check the end of the result chain. | ||
4285 | 569 | self.manager.slaveConversationEnded = \ | ||
4286 | 570 | _wait_on_deferreds_then_check_no_events | ||
4287 | 571 | events = self.manager.slaveConversationEnded() | ||
4288 | 572 | |||
4289 | 573 | # Create a broken slave and insert interaction that will | ||
4290 | 574 | # cause the builder to be marked as fail. | ||
4291 | 575 | self.test_proxy = TestingXMLRPCProxy('very broken slave') | ||
4292 | 576 | slave = RecordingSlave( | ||
4293 | 577 | BOB_THE_BUILDER_NAME, self.fake_builder_url, | ||
4294 | 578 | self.fake_builder_host) | ||
4295 | 579 | slave.ensurepresent('arg1', 'arg2', 'arg3') | ||
4296 | 580 | slave.build('arg1', 'arg2', 'arg3') | ||
4297 | 581 | |||
4298 | 582 | result = self.manager.initiateDispatch(None, slave) | ||
4299 | 583 | self.assertEqual(None, result) | ||
4300 | 584 | self.assertEqual(3, len(self.manager._deferred_list)) | ||
4301 | 585 | self.assertEqual( | ||
4302 | 586 | [('ensurepresent', 'arg1', 'arg2', 'arg3')], | ||
4303 | 587 | self.test_proxy.calls) | ||
4304 | 588 | |||
4305 | 589 | # Monkey patch the slaveConversationEnded method so we can chain a | ||
4306 | 590 | # callback to check the end of the result chain. | ||
4307 | 591 | self.manager.slaveConversationEnded = \ | ||
4308 | 592 | _wait_on_deferreds_then_check_events | ||
4309 | 593 | events = self.manager.slaveConversationEnded() | ||
4310 | 594 | |||
4311 | 595 | return events | ||
4312 | 596 | |||
4313 | 597 | |||
4314 | 65 | class TestSlaveScannerScan(TrialTestCase): | 598 | class TestSlaveScannerScan(TrialTestCase): |
4315 | 66 | """Tests `SlaveScanner.scan` method. | 599 | """Tests `SlaveScanner.scan` method. |
4316 | 67 | 600 | ||
4317 | 68 | This method uses the old framework for scanning and dispatching builds. | 601 | This method uses the old framework for scanning and dispatching builds. |
4318 | 69 | """ | 602 | """ |
4320 | 70 | layer = TwistedLaunchpadZopelessLayer | 603 | layer = LaunchpadZopelessLayer |
4321 | 71 | 604 | ||
4322 | 72 | def setUp(self): | 605 | def setUp(self): |
4323 | 73 | """Setup TwistedLayer, TrialTestCase and BuilddSlaveTest. | 606 | """Setup TwistedLayer, TrialTestCase and BuilddSlaveTest. |
4324 | @@ -75,18 +608,19 @@ | |||
4325 | 75 | Also adjust the sampledata in a way a build can be dispatched to | 608 | Also adjust the sampledata in a way a build can be dispatched to |
4326 | 76 | 'bob' builder. | 609 | 'bob' builder. |
4327 | 77 | """ | 610 | """ |
4328 | 78 | from lp.soyuz.tests.test_publishing import SoyuzTestPublisher | ||
4329 | 79 | TwistedLayer.testSetUp() | 611 | TwistedLayer.testSetUp() |
4330 | 80 | TrialTestCase.setUp(self) | 612 | TrialTestCase.setUp(self) |
4331 | 81 | self.slave = BuilddSlaveTestSetup() | 613 | self.slave = BuilddSlaveTestSetup() |
4332 | 82 | self.slave.setUp() | 614 | self.slave.setUp() |
4333 | 83 | 615 | ||
4334 | 84 | # Creating the required chroots needed for dispatching. | 616 | # Creating the required chroots needed for dispatching. |
4335 | 617 | login('foo.bar@canonical.com') | ||
4336 | 85 | test_publisher = SoyuzTestPublisher() | 618 | test_publisher = SoyuzTestPublisher() |
4337 | 86 | ubuntu = getUtility(IDistributionSet).getByName('ubuntu') | 619 | ubuntu = getUtility(IDistributionSet).getByName('ubuntu') |
4338 | 87 | hoary = ubuntu.getSeries('hoary') | 620 | hoary = ubuntu.getSeries('hoary') |
4339 | 88 | test_publisher.setUpDefaultDistroSeries(hoary) | 621 | test_publisher.setUpDefaultDistroSeries(hoary) |
4340 | 89 | test_publisher.addFakeChroots() | 622 | test_publisher.addFakeChroots() |
4341 | 623 | login(ANONYMOUS) | ||
4342 | 90 | 624 | ||
4343 | 91 | def tearDown(self): | 625 | def tearDown(self): |
4344 | 92 | self.slave.tearDown() | 626 | self.slave.tearDown() |
4345 | @@ -94,7 +628,8 @@ | |||
4346 | 94 | TwistedLayer.testTearDown() | 628 | TwistedLayer.testTearDown() |
4347 | 95 | 629 | ||
4348 | 96 | def _resetBuilder(self, builder): | 630 | def _resetBuilder(self, builder): |
4350 | 97 | """Reset the given builder and its job.""" | 631 | """Reset the given builder and it's job.""" |
4351 | 632 | login('foo.bar@canonical.com') | ||
4352 | 98 | 633 | ||
4353 | 99 | builder.builderok = True | 634 | builder.builderok = True |
4354 | 100 | job = builder.currentjob | 635 | job = builder.currentjob |
4355 | @@ -102,6 +637,7 @@ | |||
4356 | 102 | job.reset() | 637 | job.reset() |
4357 | 103 | 638 | ||
4358 | 104 | transaction.commit() | 639 | transaction.commit() |
4359 | 640 | login(ANONYMOUS) | ||
4360 | 105 | 641 | ||
4361 | 106 | def assertBuildingJob(self, job, builder, logtail=None): | 642 | def assertBuildingJob(self, job, builder, logtail=None): |
4362 | 107 | """Assert the given job is building on the given builder.""" | 643 | """Assert the given job is building on the given builder.""" |
4363 | @@ -117,25 +653,55 @@ | |||
4364 | 117 | self.assertEqual(build.status, BuildStatus.BUILDING) | 653 | self.assertEqual(build.status, BuildStatus.BUILDING) |
4365 | 118 | self.assertEqual(job.logtail, logtail) | 654 | self.assertEqual(job.logtail, logtail) |
4366 | 119 | 655 | ||
4368 | 120 | def _getScanner(self, builder_name=None): | 656 | def _getManager(self): |
4369 | 121 | """Instantiate a SlaveScanner object. | 657 | """Instantiate a SlaveScanner object. |
4370 | 122 | 658 | ||
4371 | 123 | Replace its default logging handler by a testing version. | 659 | Replace its default logging handler by a testing version. |
4372 | 124 | """ | 660 | """ |
4377 | 125 | if builder_name is None: | 661 | manager = SlaveScanner(BOB_THE_BUILDER_NAME, BufferLogger()) |
4378 | 126 | builder_name = BOB_THE_BUILDER_NAME | 662 | manager.logger.name = 'slave-scanner' |
4375 | 127 | scanner = SlaveScanner(builder_name, QuietFakeLogger()) | ||
4376 | 128 | scanner.logger.name = 'slave-scanner' | ||
4379 | 129 | 663 | ||
4381 | 130 | return scanner | 664 | return manager |
4382 | 131 | 665 | ||
4383 | 132 | def _checkDispatch(self, slave, builder): | 666 | def _checkDispatch(self, slave, builder): |
4388 | 133 | # SlaveScanner.scan returns a slave when a dispatch was | 667 | """`SlaveScanner.scan` returns a `RecordingSlave`. |
4389 | 134 | # successful. We also check that the builder has a job on it. | 668 | |
4390 | 135 | 669 | The single slave returned should match the given builder and | |
4391 | 136 | self.assertTrue(slave is not None, "Expected a slave.") | 670 | contain interactions that should be performed asynchronously for |
4392 | 671 | properly dispatching the sampledata job. | ||
4393 | 672 | """ | ||
4394 | 673 | self.assertFalse( | ||
4395 | 674 | slave is None, "Unexpected recording_slaves.") | ||
4396 | 675 | |||
4397 | 676 | self.assertEqual(slave.name, builder.name) | ||
4398 | 677 | self.assertEqual(slave.url, builder.url) | ||
4399 | 678 | self.assertEqual(slave.vm_host, builder.vm_host) | ||
4400 | 137 | self.assertEqual(0, builder.failure_count) | 679 | self.assertEqual(0, builder.failure_count) |
4402 | 138 | self.assertTrue(builder.currentjob is not None) | 680 | |
4403 | 681 | self.assertEqual( | ||
4404 | 682 | [('ensurepresent', | ||
4405 | 683 | ('0feca720e2c29dafb2c900713ba560e03b758711', | ||
4406 | 684 | 'http://localhost:58000/93/fake_chroot.tar.gz', | ||
4407 | 685 | '', '')), | ||
4408 | 686 | ('ensurepresent', | ||
4409 | 687 | ('4e3961baf4f56fdbc95d0dd47f3c5bc275da8a33', | ||
4410 | 688 | 'http://localhost:58000/43/alsa-utils_1.0.9a-4ubuntu1.dsc', | ||
4411 | 689 | '', '')), | ||
4412 | 690 | ('build', | ||
4413 | 691 | ('6358a89e2215e19b02bf91e2e4d009640fae5cf8', | ||
4414 | 692 | 'binarypackage', '0feca720e2c29dafb2c900713ba560e03b758711', | ||
4415 | 693 | {'alsa-utils_1.0.9a-4ubuntu1.dsc': | ||
4416 | 694 | '4e3961baf4f56fdbc95d0dd47f3c5bc275da8a33'}, | ||
4417 | 695 | {'arch_indep': True, | ||
4418 | 696 | 'arch_tag': 'i386', | ||
4419 | 697 | 'archive_private': False, | ||
4420 | 698 | 'archive_purpose': 'PRIMARY', | ||
4421 | 699 | 'archives': | ||
4422 | 700 | ['deb http://ftpmaster.internal/ubuntu hoary main'], | ||
4423 | 701 | 'build_debug_symbols': False, | ||
4424 | 702 | 'ogrecomponent': 'main', | ||
4425 | 703 | 'suite': u'hoary'}))], | ||
4426 | 704 | slave.calls, "Job was not properly dispatched.") | ||
4427 | 139 | 705 | ||
4428 | 140 | def testScanDispatchForResetBuilder(self): | 706 | def testScanDispatchForResetBuilder(self): |
4429 | 141 | # A job gets dispatched to the sampledata builder after it's reset. | 707 | # A job gets dispatched to the sampledata builder after it's reset. |
4430 | @@ -143,27 +709,26 @@ | |||
4431 | 143 | # Reset sampledata builder. | 709 | # Reset sampledata builder. |
4432 | 144 | builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME] | 710 | builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME] |
4433 | 145 | self._resetBuilder(builder) | 711 | self._resetBuilder(builder) |
4434 | 146 | builder.setSlaveForTesting(OkSlave()) | ||
4435 | 147 | # Set this to 1 here so that _checkDispatch can make sure it's | 712 | # Set this to 1 here so that _checkDispatch can make sure it's |
4436 | 148 | # reset to 0 after a successful dispatch. | 713 | # reset to 0 after a successful dispatch. |
4437 | 149 | builder.failure_count = 1 | 714 | builder.failure_count = 1 |
4438 | 150 | 715 | ||
4439 | 151 | # Run 'scan' and check its result. | 716 | # Run 'scan' and check its result. |
4444 | 152 | self.layer.txn.commit() | 717 | LaunchpadZopelessLayer.switchDbUser(config.builddmaster.dbuser) |
4445 | 153 | self.layer.switchDbUser(config.builddmaster.dbuser) | 718 | manager = self._getManager() |
4446 | 154 | scanner = self._getScanner() | 719 | d = defer.maybeDeferred(manager.scan) |
4443 | 155 | d = defer.maybeDeferred(scanner.scan) | ||
4447 | 156 | d.addCallback(self._checkDispatch, builder) | 720 | d.addCallback(self._checkDispatch, builder) |
4448 | 157 | return d | 721 | return d |
4449 | 158 | 722 | ||
4451 | 159 | def _checkNoDispatch(self, slave, builder): | 723 | def _checkNoDispatch(self, recording_slave, builder): |
4452 | 160 | """Assert that no dispatch has occurred. | 724 | """Assert that no dispatch has occurred. |
4453 | 161 | 725 | ||
4455 | 162 | 'slave' is None, so no interations would be passed | 726 | 'recording_slave' is None, so no interations would be passed |
4456 | 163 | to the asynchonous dispatcher and the builder remained active | 727 | to the asynchonous dispatcher and the builder remained active |
4457 | 164 | and IDLE. | 728 | and IDLE. |
4458 | 165 | """ | 729 | """ |
4460 | 166 | self.assertTrue(slave is None, "Unexpected slave.") | 730 | self.assertTrue( |
4461 | 731 | recording_slave is None, "Unexpected recording_slave.") | ||
4462 | 167 | 732 | ||
4463 | 168 | builder = getUtility(IBuilderSet).get(builder.id) | 733 | builder = getUtility(IBuilderSet).get(builder.id) |
4464 | 169 | self.assertTrue(builder.builderok) | 734 | self.assertTrue(builder.builderok) |
4465 | @@ -188,9 +753,9 @@ | |||
4466 | 188 | login(ANONYMOUS) | 753 | login(ANONYMOUS) |
4467 | 189 | 754 | ||
4468 | 190 | # Run 'scan' and check its result. | 755 | # Run 'scan' and check its result. |
4472 | 191 | self.layer.switchDbUser(config.builddmaster.dbuser) | 756 | LaunchpadZopelessLayer.switchDbUser(config.builddmaster.dbuser) |
4473 | 192 | scanner = self._getScanner() | 757 | manager = self._getManager() |
4474 | 193 | d = defer.maybeDeferred(scanner.singleCycle) | 758 | d = defer.maybeDeferred(manager.scan) |
4475 | 194 | d.addCallback(self._checkNoDispatch, builder) | 759 | d.addCallback(self._checkNoDispatch, builder) |
4476 | 195 | return d | 760 | return d |
4477 | 196 | 761 | ||
4478 | @@ -228,9 +793,9 @@ | |||
4479 | 228 | login(ANONYMOUS) | 793 | login(ANONYMOUS) |
4480 | 229 | 794 | ||
4481 | 230 | # Run 'scan' and check its result. | 795 | # Run 'scan' and check its result. |
4485 | 231 | self.layer.switchDbUser(config.builddmaster.dbuser) | 796 | LaunchpadZopelessLayer.switchDbUser(config.builddmaster.dbuser) |
4486 | 232 | scanner = self._getScanner() | 797 | manager = self._getManager() |
4487 | 233 | d = defer.maybeDeferred(scanner.scan) | 798 | d = defer.maybeDeferred(manager.scan) |
4488 | 234 | d.addCallback(self._checkJobRescued, builder, job) | 799 | d.addCallback(self._checkJobRescued, builder, job) |
4489 | 235 | return d | 800 | return d |
4490 | 236 | 801 | ||
4491 | @@ -249,6 +814,8 @@ | |||
4492 | 249 | self.assertBuildingJob(job, builder, logtail='This is a build log') | 814 | self.assertBuildingJob(job, builder, logtail='This is a build log') |
4493 | 250 | 815 | ||
4494 | 251 | def testScanUpdatesBuildingJobs(self): | 816 | def testScanUpdatesBuildingJobs(self): |
4495 | 817 | # The job assigned to a broken builder is rescued. | ||
4496 | 818 | |||
4497 | 252 | # Enable sampledata builder attached to an appropriate testing | 819 | # Enable sampledata builder attached to an appropriate testing |
4498 | 253 | # slave. It will respond as if it was building the sampledata job. | 820 | # slave. It will respond as if it was building the sampledata job. |
4499 | 254 | builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME] | 821 | builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME] |
4500 | @@ -263,174 +830,188 @@ | |||
4501 | 263 | self.assertBuildingJob(job, builder) | 830 | self.assertBuildingJob(job, builder) |
4502 | 264 | 831 | ||
4503 | 265 | # Run 'scan' and check its result. | 832 | # Run 'scan' and check its result. |
4507 | 266 | self.layer.switchDbUser(config.builddmaster.dbuser) | 833 | LaunchpadZopelessLayer.switchDbUser(config.builddmaster.dbuser) |
4508 | 267 | scanner = self._getScanner() | 834 | manager = self._getManager() |
4509 | 268 | d = defer.maybeDeferred(scanner.scan) | 835 | d = defer.maybeDeferred(manager.scan) |
4510 | 269 | d.addCallback(self._checkJobUpdated, builder, job) | 836 | d.addCallback(self._checkJobUpdated, builder, job) |
4511 | 270 | return d | 837 | return d |
4512 | 271 | 838 | ||
4556 | 272 | def test_scan_with_nothing_to_dispatch(self): | 839 | def test_scan_assesses_failure_exceptions(self): |
4514 | 273 | factory = LaunchpadObjectFactory() | ||
4515 | 274 | builder = factory.makeBuilder() | ||
4516 | 275 | builder.setSlaveForTesting(OkSlave()) | ||
4517 | 276 | scanner = self._getScanner(builder_name=builder.name) | ||
4518 | 277 | d = scanner.scan() | ||
4519 | 278 | return d.addCallback(self._checkNoDispatch, builder) | ||
4520 | 279 | |||
4521 | 280 | def test_scan_with_manual_builder(self): | ||
4522 | 281 | # Reset sampledata builder. | ||
4523 | 282 | builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME] | ||
4524 | 283 | self._resetBuilder(builder) | ||
4525 | 284 | builder.setSlaveForTesting(OkSlave()) | ||
4526 | 285 | builder.manual = True | ||
4527 | 286 | scanner = self._getScanner() | ||
4528 | 287 | d = scanner.scan() | ||
4529 | 288 | d.addCallback(self._checkNoDispatch, builder) | ||
4530 | 289 | return d | ||
4531 | 290 | |||
4532 | 291 | def test_scan_with_not_ok_builder(self): | ||
4533 | 292 | # Reset sampledata builder. | ||
4534 | 293 | builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME] | ||
4535 | 294 | self._resetBuilder(builder) | ||
4536 | 295 | builder.setSlaveForTesting(OkSlave()) | ||
4537 | 296 | builder.builderok = False | ||
4538 | 297 | scanner = self._getScanner() | ||
4539 | 298 | d = scanner.scan() | ||
4540 | 299 | # Because the builder is not ok, we can't use _checkNoDispatch. | ||
4541 | 300 | d.addCallback( | ||
4542 | 301 | lambda ignored: self.assertIdentical(None, builder.currentjob)) | ||
4543 | 302 | return d | ||
4544 | 303 | |||
4545 | 304 | def test_scan_of_broken_slave(self): | ||
4546 | 305 | builder = getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME] | ||
4547 | 306 | self._resetBuilder(builder) | ||
4548 | 307 | builder.setSlaveForTesting(BrokenSlave()) | ||
4549 | 308 | builder.failure_count = 0 | ||
4550 | 309 | scanner = self._getScanner(builder_name=builder.name) | ||
4551 | 310 | d = scanner.scan() | ||
4552 | 311 | return self.assertFailure(d, xmlrpclib.Fault) | ||
4553 | 312 | |||
4554 | 313 | def _assertFailureCounting(self, builder_count, job_count, | ||
4555 | 314 | expected_builder_count, expected_job_count): | ||
4557 | 315 | # If scan() fails with an exception, failure_counts should be | 840 | # If scan() fails with an exception, failure_counts should be |
4561 | 316 | # incremented. What we do with the results of the failure | 841 | # incremented and tested. |
4559 | 317 | # counts is tested below separately, this test just makes sure that | ||
4560 | 318 | # scan() is setting the counts. | ||
4562 | 319 | def failing_scan(): | 842 | def failing_scan(): |
4566 | 320 | return defer.fail(Exception("fake exception")) | 843 | raise Exception("fake exception") |
4567 | 321 | scanner = self._getScanner() | 844 | manager = self._getManager() |
4568 | 322 | scanner.scan = failing_scan | 845 | manager.scan = failing_scan |
4569 | 846 | manager.scheduleNextScanCycle = FakeMethod() | ||
4570 | 323 | from lp.buildmaster import manager as manager_module | 847 | from lp.buildmaster import manager as manager_module |
4571 | 324 | self.patch(manager_module, 'assessFailureCounts', FakeMethod()) | 848 | self.patch(manager_module, 'assessFailureCounts', FakeMethod()) |
4581 | 325 | builder = getUtility(IBuilderSet)[scanner.builder_name] | 849 | builder = getUtility(IBuilderSet)[manager.builder_name] |
4582 | 326 | 850 | ||
4583 | 327 | builder.failure_count = builder_count | 851 | # Failure counts start at zero. |
4584 | 328 | builder.currentjob.specific_job.build.failure_count = job_count | 852 | self.assertEqual(0, builder.failure_count) |
4585 | 329 | # The _scanFailed() calls abort, so make sure our existing | 853 | self.assertEqual( |
4586 | 330 | # failure counts are persisted. | 854 | 0, builder.currentjob.specific_job.build.failure_count) |
4587 | 331 | self.layer.txn.commit() | 855 | |
4588 | 332 | 856 | # startCycle() calls scan() which is our fake one that throws an | |
4580 | 333 | # singleCycle() calls scan() which is our fake one that throws an | ||
4589 | 334 | # exception. | 857 | # exception. |
4591 | 335 | d = scanner.singleCycle() | 858 | manager.startCycle() |
4592 | 336 | 859 | ||
4593 | 337 | # Failure counts should be updated, and the assessment method | 860 | # Failure counts should be updated, and the assessment method |
4680 | 338 | # should have been called. The actual behaviour is tested below | 861 | # should have been called. |
4681 | 339 | # in TestFailureAssessments. | 862 | self.assertEqual(1, builder.failure_count) |
4682 | 340 | def got_scan(ignored): | 863 | self.assertEqual( |
4683 | 341 | self.assertEqual(expected_builder_count, builder.failure_count) | 864 | 1, builder.currentjob.specific_job.build.failure_count) |
4684 | 342 | self.assertEqual( | 865 | |
4685 | 343 | expected_job_count, | 866 | self.assertEqual( |
4686 | 344 | builder.currentjob.specific_job.build.failure_count) | 867 | 1, manager_module.assessFailureCounts.call_count) |
4687 | 345 | self.assertEqual( | 868 | |
4688 | 346 | 1, manager_module.assessFailureCounts.call_count) | 869 | |
4689 | 347 | 870 | class TestDispatchResult(LaunchpadTestCase): | |
4690 | 348 | return d.addCallback(got_scan) | 871 | """Tests `BaseDispatchResult` variations. |
4691 | 349 | 872 | ||
4692 | 350 | def test_scan_first_fail(self): | 873 | Variations of `BaseDispatchResult` when evaluated update the database |
4693 | 351 | # The first failure of a job should result in the failure_count | 874 | information according to their purpose. |
4694 | 352 | # on the job and the builder both being incremented. | 875 | """ |
4695 | 353 | self._assertFailureCounting( | 876 | |
4696 | 354 | builder_count=0, job_count=0, expected_builder_count=1, | 877 | layer = LaunchpadZopelessLayer |
4697 | 355 | expected_job_count=1) | 878 | |
4698 | 356 | 879 | def _getBuilder(self, name): | |
4699 | 357 | def test_scan_second_builder_fail(self): | 880 | """Return a fixed `IBuilder` instance from the sampledata. |
4700 | 358 | # The first failure of a job should result in the failure_count | 881 | |
4701 | 359 | # on the job and the builder both being incremented. | 882 | Ensure it's active (builderok=True) and it has a in-progress job. |
4702 | 360 | self._assertFailureCounting( | 883 | """ |
4703 | 361 | builder_count=1, job_count=0, expected_builder_count=2, | 884 | login('foo.bar@canonical.com') |
4704 | 362 | expected_job_count=1) | 885 | |
4705 | 363 | 886 | builder = getUtility(IBuilderSet)[name] | |
4706 | 364 | def test_scan_second_job_fail(self): | 887 | builder.builderok = True |
4707 | 365 | # The first failure of a job should result in the failure_count | 888 | |
4708 | 366 | # on the job and the builder both being incremented. | 889 | job = builder.currentjob |
4709 | 367 | self._assertFailureCounting( | 890 | build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(job) |
4710 | 368 | builder_count=0, job_count=1, expected_builder_count=1, | 891 | self.assertEqual( |
4711 | 369 | expected_job_count=2) | 892 | 'i386 build of mozilla-firefox 0.9 in ubuntu hoary RELEASE', |
4712 | 370 | 893 | build.title) | |
4713 | 371 | def test_scanFailed_handles_lack_of_a_job_on_the_builder(self): | 894 | |
4714 | 372 | def failing_scan(): | 895 | self.assertEqual('BUILDING', build.status.name) |
4715 | 373 | return defer.fail(Exception("fake exception")) | 896 | self.assertNotEqual(None, job.builder) |
4716 | 374 | scanner = self._getScanner() | 897 | self.assertNotEqual(None, job.date_started) |
4717 | 375 | scanner.scan = failing_scan | 898 | self.assertNotEqual(None, job.logtail) |
4718 | 376 | builder = getUtility(IBuilderSet)[scanner.builder_name] | 899 | |
4719 | 377 | builder.failure_count = Builder.FAILURE_THRESHOLD | 900 | transaction.commit() |
4720 | 378 | builder.currentjob.reset() | 901 | |
4721 | 379 | self.layer.txn.commit() | 902 | return builder, job.id |
4722 | 380 | 903 | ||
4723 | 381 | d = scanner.singleCycle() | 904 | def assertBuildqueueIsClean(self, buildqueue): |
4724 | 382 | 905 | # Check that the buildqueue is reset. | |
4725 | 383 | def scan_finished(ignored): | 906 | self.assertEqual(None, buildqueue.builder) |
4726 | 384 | self.assertFalse(builder.builderok) | 907 | self.assertEqual(None, buildqueue.date_started) |
4727 | 385 | 908 | self.assertEqual(None, buildqueue.logtail) | |
4728 | 386 | return d.addCallback(scan_finished) | 909 | |
4729 | 387 | 910 | def assertBuilderIsClean(self, builder): | |
4730 | 388 | def test_fail_to_resume_slave_resets_job(self): | 911 | # Check that the builder is ready for a new build. |
4731 | 389 | # If an attempt to resume and dispatch a slave fails, it should | 912 | self.assertTrue(builder.builderok) |
4732 | 390 | # reset the job via job.reset() | 913 | self.assertIs(None, builder.failnotes) |
4733 | 391 | 914 | self.assertIs(None, builder.currentjob) | |
4734 | 392 | # Make a slave with a failing resume() method. | 915 | |
4735 | 393 | slave = OkSlave() | 916 | def testResetDispatchResult(self): |
4736 | 394 | slave.resume = lambda: deferLater( | 917 | # Test that `ResetDispatchResult` resets the builder and job. |
4737 | 395 | reactor, 0, defer.fail, Failure(('out', 'err', 1))) | 918 | builder, job_id = self._getBuilder(BOB_THE_BUILDER_NAME) |
4738 | 396 | 919 | buildqueue_id = builder.currentjob.id | |
4739 | 397 | # Reset sampledata builder. | 920 | builder.builderok = True |
4740 | 398 | builder = removeSecurityProxy( | 921 | builder.failure_count = 1 |
4741 | 399 | getUtility(IBuilderSet)[BOB_THE_BUILDER_NAME]) | 922 | |
4742 | 400 | self._resetBuilder(builder) | 923 | # Setup a interaction to satisfy 'write_transaction' decorator. |
4743 | 401 | self.assertEqual(0, builder.failure_count) | 924 | login(ANONYMOUS) |
4744 | 402 | builder.setSlaveForTesting(slave) | 925 | slave = RecordingSlave(builder.name, builder.url, builder.vm_host) |
4745 | 403 | builder.vm_host = "fake_vm_host" | 926 | result = ResetDispatchResult(slave) |
4746 | 404 | 927 | result() | |
4747 | 405 | scanner = self._getScanner() | 928 | |
4748 | 406 | 929 | buildqueue = getUtility(IBuildQueueSet).get(buildqueue_id) | |
4749 | 407 | # Get the next job that will be dispatched. | 930 | self.assertBuildqueueIsClean(buildqueue) |
4750 | 408 | job = removeSecurityProxy(builder._findBuildCandidate()) | 931 | |
4751 | 409 | job.virtualized = True | 932 | # XXX Julian |
4752 | 410 | builder.virtualized = True | 933 | # Disabled test until bug 586362 is fixed. |
4753 | 411 | d = scanner.singleCycle() | 934 | #self.assertFalse(builder.builderok) |
4754 | 412 | 935 | self.assertBuilderIsClean(builder) | |
4755 | 413 | def check(ignored): | 936 | |
4756 | 414 | # The failure_count will have been incremented on the | 937 | def testFailDispatchResult(self): |
4757 | 415 | # builder, we can check that to see that a dispatch attempt | 938 | # Test that `FailDispatchResult` calls assessFailureCounts() so |
4758 | 416 | # did indeed occur. | 939 | # that we know the builders and jobs are failed as necessary |
4759 | 417 | self.assertEqual(1, builder.failure_count) | 940 | # when a FailDispatchResult is called at the end of the dispatch |
4760 | 418 | # There should also be no builder set on the job. | 941 | # chain. |
4761 | 419 | self.assertTrue(job.builder is None) | 942 | builder, job_id = self._getBuilder(BOB_THE_BUILDER_NAME) |
4762 | 420 | build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(job) | 943 | |
4763 | 421 | self.assertEqual(build.status, BuildStatus.NEEDSBUILD) | 944 | # Setup a interaction to satisfy 'write_transaction' decorator. |
4764 | 422 | 945 | login(ANONYMOUS) | |
4765 | 423 | return d.addCallback(check) | 946 | slave = RecordingSlave(builder.name, builder.url, builder.vm_host) |
4766 | 947 | result = FailDispatchResult(slave, 'does not work!') | ||
4767 | 948 | result.assessFailureCounts = FakeMethod() | ||
4768 | 949 | self.assertEqual(0, result.assessFailureCounts.call_count) | ||
4769 | 950 | result() | ||
4770 | 951 | self.assertEqual(1, result.assessFailureCounts.call_count) | ||
4771 | 952 | |||
4772 | 953 | def _setup_failing_dispatch_result(self): | ||
4773 | 954 | # assessFailureCounts should fail jobs or builders depending on | ||
4774 | 955 | # whether it sees the failure_counts on each increasing. | ||
4775 | 956 | builder, job_id = self._getBuilder(BOB_THE_BUILDER_NAME) | ||
4776 | 957 | slave = RecordingSlave(builder.name, builder.url, builder.vm_host) | ||
4777 | 958 | result = FailDispatchResult(slave, 'does not work!') | ||
4778 | 959 | return builder, result | ||
4779 | 960 | |||
4780 | 961 | def test_assessFailureCounts_equal_failures(self): | ||
4781 | 962 | # Basic case where the failure counts are equal and the job is | ||
4782 | 963 | # reset to try again & the builder is not failed. | ||
4783 | 964 | builder, result = self._setup_failing_dispatch_result() | ||
4784 | 965 | buildqueue = builder.currentjob | ||
4785 | 966 | build = buildqueue.specific_job.build | ||
4786 | 967 | builder.failure_count = 2 | ||
4787 | 968 | build.failure_count = 2 | ||
4788 | 969 | result.assessFailureCounts() | ||
4789 | 970 | |||
4790 | 971 | self.assertBuilderIsClean(builder) | ||
4791 | 972 | self.assertEqual('NEEDSBUILD', build.status.name) | ||
4792 | 973 | self.assertBuildqueueIsClean(buildqueue) | ||
4793 | 974 | |||
4794 | 975 | def test_assessFailureCounts_job_failed(self): | ||
4795 | 976 | # Case where the job has failed more than the builder. | ||
4796 | 977 | builder, result = self._setup_failing_dispatch_result() | ||
4797 | 978 | buildqueue = builder.currentjob | ||
4798 | 979 | build = buildqueue.specific_job.build | ||
4799 | 980 | build.failure_count = 2 | ||
4800 | 981 | builder.failure_count = 1 | ||
4801 | 982 | result.assessFailureCounts() | ||
4802 | 983 | |||
4803 | 984 | self.assertBuilderIsClean(builder) | ||
4804 | 985 | self.assertEqual('FAILEDTOBUILD', build.status.name) | ||
4805 | 986 | # The buildqueue should have been removed entirely. | ||
4806 | 987 | self.assertEqual( | ||
4807 | 988 | None, getUtility(IBuildQueueSet).getByBuilder(builder), | ||
4808 | 989 | "Buildqueue was not removed when it should be.") | ||
4809 | 990 | |||
4810 | 991 | def test_assessFailureCounts_builder_failed(self): | ||
4811 | 992 | # Case where the builder has failed more than the job. | ||
4812 | 993 | builder, result = self._setup_failing_dispatch_result() | ||
4813 | 994 | buildqueue = builder.currentjob | ||
4814 | 995 | build = buildqueue.specific_job.build | ||
4815 | 996 | build.failure_count = 2 | ||
4816 | 997 | builder.failure_count = 3 | ||
4817 | 998 | result.assessFailureCounts() | ||
4818 | 999 | |||
4819 | 1000 | self.assertFalse(builder.builderok) | ||
4820 | 1001 | self.assertEqual('does not work!', builder.failnotes) | ||
4821 | 1002 | self.assertTrue(builder.currentjob is None) | ||
4822 | 1003 | self.assertEqual('NEEDSBUILD', build.status.name) | ||
4823 | 1004 | self.assertBuildqueueIsClean(buildqueue) | ||
4824 | 424 | 1005 | ||
4825 | 425 | 1006 | ||
4826 | 426 | class TestBuilddManager(TrialTestCase): | 1007 | class TestBuilddManager(TrialTestCase): |
4827 | 427 | 1008 | ||
4829 | 428 | layer = TwistedLaunchpadZopelessLayer | 1009 | layer = LaunchpadZopelessLayer |
4830 | 429 | 1010 | ||
4831 | 430 | def _stub_out_scheduleNextScanCycle(self): | 1011 | def _stub_out_scheduleNextScanCycle(self): |
4832 | 431 | # stub out the code that adds a callLater, so that later tests | 1012 | # stub out the code that adds a callLater, so that later tests |
4833 | 432 | # don't get surprises. | 1013 | # don't get surprises. |
4835 | 433 | self.patch(SlaveScanner, 'startCycle', FakeMethod()) | 1014 | self.patch(SlaveScanner, 'scheduleNextScanCycle', FakeMethod()) |
4836 | 434 | 1015 | ||
4837 | 435 | def test_addScanForBuilders(self): | 1016 | def test_addScanForBuilders(self): |
4838 | 436 | # Test that addScanForBuilders generates NewBuildersScanner objects. | 1017 | # Test that addScanForBuilders generates NewBuildersScanner objects. |
4839 | @@ -459,62 +1040,10 @@ | |||
4840 | 459 | self.assertNotEqual(0, manager.new_builders_scanner.scan.call_count) | 1040 | self.assertNotEqual(0, manager.new_builders_scanner.scan.call_count) |
4841 | 460 | 1041 | ||
4842 | 461 | 1042 | ||
4843 | 462 | class TestFailureAssessments(TestCaseWithFactory): | ||
4844 | 463 | |||
4845 | 464 | layer = ZopelessDatabaseLayer | ||
4846 | 465 | |||
4847 | 466 | def setUp(self): | ||
4848 | 467 | TestCaseWithFactory.setUp(self) | ||
4849 | 468 | self.builder = self.factory.makeBuilder() | ||
4850 | 469 | self.build = self.factory.makeSourcePackageRecipeBuild() | ||
4851 | 470 | self.buildqueue = self.build.queueBuild() | ||
4852 | 471 | self.buildqueue.markAsBuilding(self.builder) | ||
4853 | 472 | |||
4854 | 473 | def test_equal_failures_reset_job(self): | ||
4855 | 474 | self.builder.gotFailure() | ||
4856 | 475 | self.builder.getCurrentBuildFarmJob().gotFailure() | ||
4857 | 476 | |||
4858 | 477 | assessFailureCounts(self.builder, "failnotes") | ||
4859 | 478 | self.assertIs(None, self.builder.currentjob) | ||
4860 | 479 | self.assertEqual(self.build.status, BuildStatus.NEEDSBUILD) | ||
4861 | 480 | |||
4862 | 481 | def test_job_failing_more_than_builder_fails_job(self): | ||
4863 | 482 | self.builder.getCurrentBuildFarmJob().gotFailure() | ||
4864 | 483 | |||
4865 | 484 | assessFailureCounts(self.builder, "failnotes") | ||
4866 | 485 | self.assertIs(None, self.builder.currentjob) | ||
4867 | 486 | self.assertEqual(self.build.status, BuildStatus.FAILEDTOBUILD) | ||
4868 | 487 | |||
4869 | 488 | def test_builder_failing_more_than_job_but_under_fail_threshold(self): | ||
4870 | 489 | self.builder.failure_count = Builder.FAILURE_THRESHOLD - 1 | ||
4871 | 490 | |||
4872 | 491 | assessFailureCounts(self.builder, "failnotes") | ||
4873 | 492 | self.assertIs(None, self.builder.currentjob) | ||
4874 | 493 | self.assertEqual(self.build.status, BuildStatus.NEEDSBUILD) | ||
4875 | 494 | self.assertTrue(self.builder.builderok) | ||
4876 | 495 | |||
4877 | 496 | def test_builder_failing_more_than_job_but_over_fail_threshold(self): | ||
4878 | 497 | self.builder.failure_count = Builder.FAILURE_THRESHOLD | ||
4879 | 498 | |||
4880 | 499 | assessFailureCounts(self.builder, "failnotes") | ||
4881 | 500 | self.assertIs(None, self.builder.currentjob) | ||
4882 | 501 | self.assertEqual(self.build.status, BuildStatus.NEEDSBUILD) | ||
4883 | 502 | self.assertFalse(self.builder.builderok) | ||
4884 | 503 | self.assertEqual("failnotes", self.builder.failnotes) | ||
4885 | 504 | |||
4886 | 505 | def test_builder_failing_with_no_attached_job(self): | ||
4887 | 506 | self.buildqueue.reset() | ||
4888 | 507 | self.builder.failure_count = Builder.FAILURE_THRESHOLD | ||
4889 | 508 | |||
4890 | 509 | assessFailureCounts(self.builder, "failnotes") | ||
4891 | 510 | self.assertFalse(self.builder.builderok) | ||
4892 | 511 | self.assertEqual("failnotes", self.builder.failnotes) | ||
4893 | 512 | |||
4894 | 513 | |||
4895 | 514 | class TestNewBuilders(TrialTestCase): | 1043 | class TestNewBuilders(TrialTestCase): |
4896 | 515 | """Test detecting of new builders.""" | 1044 | """Test detecting of new builders.""" |
4897 | 516 | 1045 | ||
4899 | 517 | layer = TwistedLaunchpadZopelessLayer | 1046 | layer = LaunchpadZopelessLayer |
4900 | 518 | 1047 | ||
4901 | 519 | def _getScanner(self, manager=None, clock=None): | 1048 | def _getScanner(self, manager=None, clock=None): |
4902 | 520 | return NewBuildersScanner(manager=manager, clock=clock) | 1049 | return NewBuildersScanner(manager=manager, clock=clock) |
4903 | @@ -555,8 +1084,11 @@ | |||
4904 | 555 | new_builders, builder_scanner.checkForNewBuilders()) | 1084 | new_builders, builder_scanner.checkForNewBuilders()) |
4905 | 556 | 1085 | ||
4906 | 557 | def test_scan(self): | 1086 | def test_scan(self): |
4908 | 558 | # See if scan detects new builders. | 1087 | # See if scan detects new builders and schedules the next scan. |
4909 | 559 | 1088 | ||
4910 | 1089 | # stub out the addScanForBuilders and scheduleScan methods since | ||
4911 | 1090 | # they use callLater; we only want to assert that they get | ||
4912 | 1091 | # called. | ||
4913 | 560 | def fake_checkForNewBuilders(): | 1092 | def fake_checkForNewBuilders(): |
4914 | 561 | return "new_builders" | 1093 | return "new_builders" |
4915 | 562 | 1094 | ||
4916 | @@ -572,6 +1104,9 @@ | |||
4917 | 572 | builder_scanner.scan() | 1104 | builder_scanner.scan() |
4918 | 573 | advance = NewBuildersScanner.SCAN_INTERVAL + 1 | 1105 | advance = NewBuildersScanner.SCAN_INTERVAL + 1 |
4919 | 574 | clock.advance(advance) | 1106 | clock.advance(advance) |
4920 | 1107 | self.assertNotEqual( | ||
4921 | 1108 | 0, builder_scanner.scheduleScan.call_count, | ||
4922 | 1109 | "scheduleScan did not get called") | ||
4923 | 575 | 1110 | ||
4924 | 576 | 1111 | ||
4925 | 577 | def is_file_growing(filepath, poll_interval=1, poll_repeat=10): | 1112 | def is_file_growing(filepath, poll_interval=1, poll_repeat=10): |
4926 | @@ -612,7 +1147,7 @@ | |||
4927 | 612 | return False | 1147 | return False |
4928 | 613 | 1148 | ||
4929 | 614 | 1149 | ||
4931 | 615 | class TestBuilddManagerScript(TestCaseWithFactory): | 1150 | class TestBuilddManagerScript(LaunchpadTestCase): |
4932 | 616 | 1151 | ||
4933 | 617 | layer = LaunchpadScriptLayer | 1152 | layer = LaunchpadScriptLayer |
4934 | 618 | 1153 | ||
4935 | @@ -621,7 +1156,6 @@ | |||
4936 | 621 | fixture = BuilddManagerTestSetup() | 1156 | fixture = BuilddManagerTestSetup() |
4937 | 622 | fixture.setUp() | 1157 | fixture.setUp() |
4938 | 623 | fixture.tearDown() | 1158 | fixture.tearDown() |
4939 | 624 | self.layer.force_dirty_database() | ||
4940 | 625 | 1159 | ||
4941 | 626 | # XXX Julian 2010-08-06 bug=614275 | 1160 | # XXX Julian 2010-08-06 bug=614275 |
4942 | 627 | # These next 2 tests are in the wrong place, they should be near the | 1161 | # These next 2 tests are in the wrong place, they should be near the |
4943 | 628 | 1162 | ||
4944 | === modified file 'lib/lp/buildmaster/tests/test_packagebuild.py' | |||
4945 | --- lib/lp/buildmaster/tests/test_packagebuild.py 2010-10-26 20:43:50 +0000 | |||
4946 | +++ lib/lp/buildmaster/tests/test_packagebuild.py 2010-12-07 16:29:13 +0000 | |||
4947 | @@ -97,8 +97,6 @@ | |||
4948 | 97 | self.assertRaises( | 97 | self.assertRaises( |
4949 | 98 | NotImplementedError, self.package_build.verifySuccessfulUpload) | 98 | NotImplementedError, self.package_build.verifySuccessfulUpload) |
4950 | 99 | self.assertRaises(NotImplementedError, self.package_build.notify) | 99 | self.assertRaises(NotImplementedError, self.package_build.notify) |
4951 | 100 | # XXX 2010-10-18 bug=662631 | ||
4952 | 101 | # Change this to do non-blocking IO. | ||
4953 | 102 | self.assertRaises( | 100 | self.assertRaises( |
4954 | 103 | NotImplementedError, self.package_build.handleStatus, | 101 | NotImplementedError, self.package_build.handleStatus, |
4955 | 104 | None, None, None) | 102 | None, None, None) |
4956 | @@ -311,8 +309,6 @@ | |||
4957 | 311 | # A filemap with plain filenames should not cause a problem. | 309 | # A filemap with plain filenames should not cause a problem. |
4958 | 312 | # The call to handleStatus will attempt to get the file from | 310 | # The call to handleStatus will attempt to get the file from |
4959 | 313 | # the slave resulting in a URL error in this test case. | 311 | # the slave resulting in a URL error in this test case. |
4960 | 314 | # XXX 2010-10-18 bug=662631 | ||
4961 | 315 | # Change this to do non-blocking IO. | ||
4962 | 316 | self.build.handleStatus('OK', None, { | 312 | self.build.handleStatus('OK', None, { |
4963 | 317 | 'filemap': {'myfile.py': 'test_file_hash'}, | 313 | 'filemap': {'myfile.py': 'test_file_hash'}, |
4964 | 318 | }) | 314 | }) |
4965 | @@ -323,8 +319,6 @@ | |||
4966 | 323 | def test_handleStatus_OK_absolute_filepath(self): | 319 | def test_handleStatus_OK_absolute_filepath(self): |
4967 | 324 | # A filemap that tries to write to files outside of | 320 | # A filemap that tries to write to files outside of |
4968 | 325 | # the upload directory will result in a failed upload. | 321 | # the upload directory will result in a failed upload. |
4969 | 326 | # XXX 2010-10-18 bug=662631 | ||
4970 | 327 | # Change this to do non-blocking IO. | ||
4971 | 328 | self.build.handleStatus('OK', None, { | 322 | self.build.handleStatus('OK', None, { |
4972 | 329 | 'filemap': {'/tmp/myfile.py': 'test_file_hash'}, | 323 | 'filemap': {'/tmp/myfile.py': 'test_file_hash'}, |
4973 | 330 | }) | 324 | }) |
4974 | @@ -335,8 +329,6 @@ | |||
4975 | 335 | def test_handleStatus_OK_relative_filepath(self): | 329 | def test_handleStatus_OK_relative_filepath(self): |
4976 | 336 | # A filemap that tries to write to files outside of | 330 | # A filemap that tries to write to files outside of |
4977 | 337 | # the upload directory will result in a failed upload. | 331 | # the upload directory will result in a failed upload. |
4978 | 338 | # XXX 2010-10-18 bug=662631 | ||
4979 | 339 | # Change this to do non-blocking IO. | ||
4980 | 340 | self.build.handleStatus('OK', None, { | 332 | self.build.handleStatus('OK', None, { |
4981 | 341 | 'filemap': {'../myfile.py': 'test_file_hash'}, | 333 | 'filemap': {'../myfile.py': 'test_file_hash'}, |
4982 | 342 | }) | 334 | }) |
4983 | @@ -347,8 +339,6 @@ | |||
4984 | 347 | # The build log is set during handleStatus. | 339 | # The build log is set during handleStatus. |
4985 | 348 | removeSecurityProxy(self.build).log = None | 340 | removeSecurityProxy(self.build).log = None |
4986 | 349 | self.assertEqual(None, self.build.log) | 341 | self.assertEqual(None, self.build.log) |
4987 | 350 | # XXX 2010-10-18 bug=662631 | ||
4988 | 351 | # Change this to do non-blocking IO. | ||
4989 | 352 | self.build.handleStatus('OK', None, { | 342 | self.build.handleStatus('OK', None, { |
4990 | 353 | 'filemap': {'myfile.py': 'test_file_hash'}, | 343 | 'filemap': {'myfile.py': 'test_file_hash'}, |
4991 | 354 | }) | 344 | }) |
4992 | @@ -358,8 +348,6 @@ | |||
4993 | 358 | # The date finished is updated during handleStatus_OK. | 348 | # The date finished is updated during handleStatus_OK. |
4994 | 359 | removeSecurityProxy(self.build).date_finished = None | 349 | removeSecurityProxy(self.build).date_finished = None |
4995 | 360 | self.assertEqual(None, self.build.date_finished) | 350 | self.assertEqual(None, self.build.date_finished) |
4996 | 361 | # XXX 2010-10-18 bug=662631 | ||
4997 | 362 | # Change this to do non-blocking IO. | ||
4998 | 363 | self.build.handleStatus('OK', None, { | 351 | self.build.handleStatus('OK', None, { |
4999 | 364 | 'filemap': {'myfile.py': 'test_file_hash'}, | 352 | 'filemap': {'myfile.py': 'test_file_hash'}, |
5000 | 365 | }) | 353 | }) |
The diff has been truncated for viewing.
No approved revision specified.