Merge lp:~vila/uci-engine/1302474-nova-transient-failures into lp:uci-engine
- 1302474-nova-transient-failures
- Merge into trunk
Status: | Merged |
---|---|
Approved by: | Evan |
Approved revision: | 746 |
Merged at revision: | 758 |
Proposed branch: | lp:~vila/uci-engine/1302474-nova-transient-failures |
Merge into: | lp:uci-engine |
Diff against target: |
557 lines (+254/-80) 4 files modified
ci-utils/ci_utils/testing/features.py (+5/-0) test_runner/tstrun/testbed.py (+113/-69) test_runner/tstrun/tests/test_testbed.py (+134/-9) test_runner/tstrun/tests/test_worker.py (+2/-2) |
To merge this branch: | bzr merge lp:~vila/uci-engine/1302474-nova-transient-failures |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Evan (community) | Approve | ||
PS Jenkins bot (community) | continuous-integration | Approve | |
Review via email: mp+229776@code.launchpad.net |
Commit message
Handle nova transient failures by re-trying failed requests *once*.
Description of the change
This handles nova transient failures by re-trying a failed request *once*.
See https:/
Re-trying once should be enough to guard against most of the issues I've encountered during HP cloud hiccups and should significantly improve the test runner reliability.
Backstory at https:/
More details below.
I've tried a simpler approach with nova.http.
This proposal address that by wrapping all the used nova requests into a dedicated nova client (and goes into the right direction for the long term solution).
I also found out that the nova <manager>.find() pattern issues more requests than necessary.
Since I was working on making the requests more reliable, having less of them was a no brainer.
Finally, I've added the MISSINGTESTs and removed a good chunk of FIXMEs (the
remaining ones are unrelated to nova).
PS Jenkins bot (ps-jenkins) wrote : | # |
- 746. By Vincent Ladeuil
-
Fix pep8 issue.
PS Jenkins bot (ps-jenkins) wrote : | # |
PASSED: Continuous integration, rev:746
http://
Executed test runs:
Click here to trigger a rebuild:
http://
Preview Diff
1 | === modified file 'ci-utils/ci_utils/testing/features.py' |
2 | --- ci-utils/ci_utils/testing/features.py 2014-07-30 16:24:48 +0000 |
3 | +++ ci-utils/ci_utils/testing/features.py 2014-08-06 13:17:27 +0000 |
4 | @@ -67,6 +67,8 @@ |
5 | if client is None: |
6 | return False |
7 | try: |
8 | + # can transiently fail with requests.exceptions.ConnectionError |
9 | + # (converted from MaxRetryError). |
10 | client.authenticate() |
11 | except nova_exceptions.ClientException: |
12 | return False |
13 | @@ -84,6 +86,9 @@ |
14 | except KeyError: |
15 | # If we miss some env vars, we can't get a client |
16 | return None |
17 | + except nova_exceptions.Unauthorized: |
18 | + # If the credentials are wrong, we can't get a client |
19 | + return None |
20 | |
21 | |
22 | # The single instance shared by all tests |
23 | |
24 | === modified file 'test_runner/tstrun/testbed.py' |
25 | --- test_runner/tstrun/testbed.py 2014-07-31 08:46:00 +0000 |
26 | +++ test_runner/tstrun/testbed.py 2014-08-06 13:17:27 +0000 |
27 | @@ -17,19 +17,20 @@ |
28 | |
29 | import logging |
30 | import os |
31 | -import requests |
32 | import subprocess |
33 | import time |
34 | |
35 | +import requests |
36 | |
37 | -from novaclient import exceptions |
38 | -from novaclient.v1_1 import client |
39 | +from novaclient import ( |
40 | + client, |
41 | + exceptions, |
42 | +) |
43 | from uciconfig import options |
44 | from ucivms import ( |
45 | config, |
46 | vms, |
47 | ) |
48 | - |
49 | from ci_utils import unit_config |
50 | |
51 | |
52 | @@ -99,6 +100,18 @@ |
53 | a way to do so. This is intended to be fixed in uci-vms so vm.apt_sources can |
54 | be used again. |
55 | ''')) |
56 | +register(options.Option('vm.nova.boot_timeout', default='300', |
57 | + from_unicode=options.float_from_store, |
58 | + help_string='''\ |
59 | +Max time to boot a nova instance (in seconds).''')) |
60 | +register(options.Option('vm.nova.set_ip_timeout', default='300', |
61 | + from_unicode=options.float_from_store, |
62 | + help_string='''\ |
63 | +Max time for a nova instance to get an IP (in seconds).''')) |
64 | +register(options.Option('vm.nova.cloud_init_timeout', default='1200', |
65 | + from_unicode=options.float_from_store, |
66 | + help_string='''\ |
67 | +Max time for cloud-init to fisnish (in seconds).''')) |
68 | |
69 | |
70 | logging.basicConfig(level=logging.INFO) |
71 | @@ -134,8 +147,69 @@ |
72 | return conf |
73 | |
74 | |
75 | +class NovaClient(object): |
76 | + """A nova client re-trying requests on known transient failures.""" |
77 | + |
78 | + def __init__(self, *args, **kwargs): |
79 | + debug = kwargs.pop('debug', False) |
80 | + # Activating debug will output the http requests issued by nova and the |
81 | + # corresponding responses. |
82 | + if debug: |
83 | + logging.root.setLevel(logging.DEBUG) |
84 | + self.nova = client.Client('1.1', *args, http_log_debug=debug, |
85 | + **kwargs) |
86 | + |
87 | + def retry(self, func, *args, **kwargs): |
88 | + try: |
89 | + return func(*args, **kwargs) |
90 | + except requests.ConnectionError: |
91 | + # Most common transient failure: the API server is unreachable |
92 | + nap_time = 1 |
93 | + log.warn('Received connection error for {},' |
94 | + ' retrying)'.format(func.__name__)) |
95 | + except exceptions.OverLimit: |
96 | + # This happens rarely but breaks badly if not caught. elmo |
97 | + # recommended a 30 seconds nap in that case. |
98 | + nap_time = 30 |
99 | + msg = ('Rate limit reached for {},' |
100 | + ' will sleep for {} seconds') |
101 | + log.exception(msg.format(func.__name__, nap_time)) |
102 | + time.sleep(nap_time) |
103 | + return func(*args, **kwargs) # Retry once |
104 | + |
105 | + def flavors_list(self): |
106 | + return self.retry(self.nova.flavors.list) |
107 | + |
108 | + def images_list(self): |
109 | + return self.retry(self.nova.images.list) |
110 | + |
111 | + def create_server(self, name, flavor, image, user_data): |
112 | + return self.retry(self.nova.servers.create, name=name, |
113 | + flavor=flavor, image=image, userdata=user_data) |
114 | + |
115 | + def delete_server(self, server_id): |
116 | + return self.retry(self.nova.servers.delete, server_id) |
117 | + |
118 | + def create_floating_ip(self): |
119 | + return self.retry(self.nova.floating_ips.create) |
120 | + |
121 | + def delete_floating_ip(self, floating_ip): |
122 | + return self.retry(self.nova.floating_ips.delete, floating_ip) |
123 | + |
124 | + def add_floating_ip(self, instance, floating_ip): |
125 | + return self.retry(instance.add_floating_ip, floating_ip) |
126 | + |
127 | + def get_server_details(self, server_id): |
128 | + return self.retry(self.nova.servers.get, server_id) |
129 | + |
130 | + def get_server_console(self, server, length=None): |
131 | + return self.retry(server.get_console_output, length) |
132 | + |
133 | + |
134 | class TestBed(vms.VM): |
135 | |
136 | + nova_client_class = NovaClient |
137 | + |
138 | def __init__(self, conf): |
139 | super(TestBed, self).__init__(conf) |
140 | self.instance = None |
141 | @@ -155,7 +229,8 @@ |
142 | self.conf.get('os.auth_url')] |
143 | kwargs = {'region_name': self.conf.get('os.region_name'), |
144 | 'service_type': 'compute'} |
145 | - return client.Client(*args, **kwargs) |
146 | + nova_client = self.nova_client_class(*args, **kwargs) |
147 | + return nova_client |
148 | |
149 | def ensure_ssh_key_is_available(self): |
150 | self.test_bed_key_path = self.conf.get('vm.ssh_key_path') |
151 | @@ -173,23 +248,22 @@ |
152 | '-f', self.test_bed_key_path, '-N', '']) |
153 | |
154 | def find_flavor(self): |
155 | - for flavor in self.conf.get('vm.flavors'): |
156 | - try: |
157 | - return self.nova.flavors.find(name=flavor) |
158 | - except exceptions.NotFound: |
159 | - pass |
160 | - # MISSINGTEST: some cloud doesn't provide one of our expected flavors |
161 | - # -- vila 2014-01-06 |
162 | + flavors = self.conf.get('vm.flavors') |
163 | + existing_flavors = self.nova.flavors_list() |
164 | + for flavor in flavors: |
165 | + for existing in existing_flavors: |
166 | + if flavor == existing.name: |
167 | + return existing |
168 | raise TestBedException( |
169 | - 'None of {} can be found'.format(self.flavors)) |
170 | + 'None of [{}] can be found'.format(','.join(flavors))) |
171 | |
172 | def find_nova_image(self): |
173 | image_name = self.conf.get('vm.image') |
174 | - try: |
175 | - return self.nova.images.find(name=image_name) |
176 | - except exceptions.NotFound: |
177 | - raise TestBedException( |
178 | - 'Image "{}" cannot be found'.format(image_name)) |
179 | + existing_images = self.nova.images_list() |
180 | + for existing in existing_images: |
181 | + if image_name == existing.name: |
182 | + return existing |
183 | + raise TestBedException('Image "{}" cannot be found'.format(image_name)) |
184 | |
185 | def setup(self): |
186 | flavor = self.find_flavor() |
187 | @@ -198,13 +272,13 @@ |
188 | self.create_user_data() |
189 | with open(self._user_data_path) as f: |
190 | user_data = f.read() |
191 | - self.instance = self.nova.servers.create( |
192 | + self.instance = self.nova.create_server( |
193 | name=self.conf.get('vm.name'), flavor=flavor, image=image, |
194 | - userdata=user_data) |
195 | + user_data=user_data) |
196 | self.wait_for_active_instance() |
197 | if unit_config.is_hpcloud(self.conf.get('os.auth_url')): |
198 | - self.floating_ip = self.nova.floating_ips.create() |
199 | - self.instance.add_floating_ip(self.floating_ip) |
200 | + self.floating_ip = self.nova.create_floating_ip() |
201 | + self.nova.add_floating_ip(self.instance, self.floating_ip) |
202 | self.wait_for_ip() |
203 | self.wait_for_cloud_init() |
204 | ppas = self.conf.get('vm.ppas') |
205 | @@ -236,38 +310,29 @@ |
206 | raise TestBedException('apt-get update never succeeded') |
207 | |
208 | def update_instance(self): |
209 | - # MISSINGTEST: What if the instance disappear ? (could be approximated |
210 | - # by deleting the nova instance) -- vila 2014-06-05 |
211 | try: |
212 | # Always query nova to get updated data about the instance |
213 | - self.instance = self.nova.servers.get(self.instance.id) |
214 | + self.instance = self.nova.get_server_details(self.instance.id) |
215 | return True |
216 | - except requests.ConnectionError: |
217 | - # The reported status is the one known before the last attempt. |
218 | - log.warn('Received connection error for {},' |
219 | - ' retrying (status was: {})'.format( |
220 | - self.instance.id, self.instance.status)) |
221 | - return False |
222 | + except: |
223 | + # But catch exceptions if something goes wrong. Higher levels will |
224 | + # deal with the instance not replying. |
225 | + return False |
226 | |
227 | def wait_for_active_instance(self): |
228 | - # FIXME: get_active_instance should be a config option |
229 | - # -- vila 2014-05-13 |
230 | - get_active_instance = 300 # in seconds so 5 minutes |
231 | - timeout_limit = time.time() + get_active_instance |
232 | - while time.time() < timeout_limit and self.instance.status != 'ACTIVE': |
233 | - time.sleep(5) |
234 | + timeout_limit = time.time() + self.conf.get('vm.nova.boot_timeout') |
235 | + while (time.time() <= timeout_limit |
236 | + and self.instance.status != 'ACTIVE'): |
237 | self.update_instance() |
238 | + time.sleep(5) |
239 | if self.instance.status != 'ACTIVE': |
240 | - # MISSINGTEST: What if the instance doesn't come up ? |
241 | msg = 'Instance never came up (last status: {})'.format( |
242 | self.instance.status) |
243 | raise TestBedException(msg) |
244 | |
245 | def wait_for_ip(self): |
246 | - # FIXME: get_ip_timeout should be a config option -- vila 2014-01-30 |
247 | - get_ip_timeout = 300 # in seconds so 5 minutes |
248 | - timeout_limit = time.time() + get_ip_timeout |
249 | - while time.time() < timeout_limit: |
250 | + timeout_limit = time.time() + self.conf.get('vm.nova.set_ip_timeout') |
251 | + while time.time() <= timeout_limit: |
252 | if not self.update_instance(): |
253 | time.sleep(5) |
254 | continue |
255 | @@ -280,46 +345,25 @@ |
256 | # the floating one. In both cases that gives us a reachable IP. |
257 | self.ip = networks[0][-1] |
258 | log.info('Got IP {} for {}'.format(self.ip, self.instance.id)) |
259 | - # FIXME: Right place to report how long it took to spin up the |
260 | - # instance as far as nova is concerned. -- vila 2014-01-30 |
261 | return |
262 | else: |
263 | log.info( |
264 | 'IP not yet available for {}'.format(self.instance.id)) |
265 | time.sleep(5) |
266 | - # MISSINGTEST: What if the instance still doesn't have an ip ? |
267 | msg = 'Instance {} never provided an IP'.format(self.instance.id) |
268 | raise TestBedException(msg) |
269 | |
270 | def get_cloud_init_console(self, length=None): |
271 | - return self.instance.get_console_output(length) |
272 | + return self.nova.get_server_console(self.instance, length) |
273 | |
274 | def wait_for_cloud_init(self): |
275 | - # FIXME: cloud_init_timeout should be a config option (related to |
276 | - # get_ip_timeout and probably the two can be merged) -- vila 2014-01-30 |
277 | - cloud_init_timeout = 1200 # in seconds so 20 minutes |
278 | - timeout_limit = time.time() + cloud_init_timeout |
279 | + timeout_limit = (time.time() |
280 | + + self.conf.get('vm.nova.cloud_init_timeout')) |
281 | final_message = self.conf.get('vm.final_message') |
282 | while time.time() < timeout_limit: |
283 | # A relatively cheap way to catch cloud-init completion is to watch |
284 | # the console for the specific message we specified in user-data). |
285 | - try: |
286 | - console = self.get_cloud_init_console(10) |
287 | - except exceptions.OverLimit: |
288 | - # This happens rarely but breaks badly if not caught. elmo |
289 | - # recommended a 30 seconds nap in that case. |
290 | - nap_time = 30 |
291 | - msg = ('Rate limit while acquiring nova console for {},' |
292 | - ' will sleep for {} seconds') |
293 | - log.exception(msg.format(self.instance.id, nap_time)) |
294 | - time.sleep(nap_time) |
295 | - continue |
296 | - except requests.ConnectionError: |
297 | - # The reported status is the one known before the last attempt. |
298 | - log.warn('Received connection error for {},' |
299 | - ' retrying)'.format(self.instance.id)) |
300 | - time.sleep(5) |
301 | - continue |
302 | + console = self.get_cloud_init_console(10) |
303 | if final_message in console: |
304 | # We're good to go |
305 | log.info( |
306 | @@ -330,10 +374,10 @@ |
307 | |
308 | def teardown(self): |
309 | if self.instance is not None: |
310 | - self.nova.servers.delete(self.instance.id) |
311 | + self.nova.delete_server(self.instance.id) |
312 | self.instance = None |
313 | if self.floating_ip is not None: |
314 | - self.nova.floating_ips.delete(self.floating_ip) |
315 | + self.nova.delete_floating_ip(self.floating_ip) |
316 | self.floating_ip = None |
317 | # FIXME: Now we can remove the testbed key from known_hosts (see |
318 | # ssh()). -- vila 2014-01-30 |
319 | |
320 | === modified file 'test_runner/tstrun/tests/test_testbed.py' |
321 | --- test_runner/tstrun/tests/test_testbed.py 2014-07-30 15:41:08 +0000 |
322 | +++ test_runner/tstrun/tests/test_testbed.py 2014-08-06 13:17:27 +0000 |
323 | @@ -19,11 +19,14 @@ |
324 | import subprocess |
325 | import unittest |
326 | |
327 | - |
328 | +import requests |
329 | from uciconfig import options |
330 | +from ucitests import ( |
331 | + assertions, |
332 | + fixtures, |
333 | +) |
334 | from ucivms.tests import fixtures as vms_fixtures |
335 | |
336 | - |
337 | from ci_utils import unit_config |
338 | from ci_utils.testing import ( |
339 | features, |
340 | @@ -36,6 +39,97 @@ |
341 | |
342 | @features.requires(tests.nova_creds) |
343 | @features.requires(features.nova_compute) |
344 | +class TestNovaClient(unittest.TestCase): |
345 | + """Check the nova client behavior when it encounters exceptions. |
346 | + |
347 | + This is achieved by overriding specific methods from NovaClient and |
348 | + exercising it through the TestBed methods. |
349 | + """ |
350 | + |
351 | + def setUp(self): |
352 | + super(TestNovaClient, self).setUp() |
353 | + vms_fixtures.isolate_from_disk(self) |
354 | + self.tb_name = 'testing-nova-client' |
355 | + # Prepare a suitable config, importing the nova credentials |
356 | + conf = testbed.vms_config_from_auth_config( |
357 | + self.tb_name, unit_config.get_auth_config()) |
358 | + # Default to precise |
359 | + conf.set('vm.release', 'precise') |
360 | + # Avoid triggering the 'atexit' hook as the config files are long gone |
361 | + # at that point. |
362 | + self.addCleanup(conf.store.save_changes) |
363 | + self.conf = conf |
364 | + os.makedirs(self.conf.get('vm.vms_dir')) |
365 | + |
366 | + def get_image_id(self, series='precise'): |
367 | + if unit_config.is_hpcloud(self.conf.get('os.auth_url')): |
368 | + test_images = features.hpcloud_test_images |
369 | + else: |
370 | + test_images = features.canonistack_test_images |
371 | + return test_images[series] |
372 | + |
373 | + def test_retry_is_called(self): |
374 | + self.retry_calls = [] |
375 | + |
376 | + class RetryingNovaClient(testbed.NovaClient): |
377 | + |
378 | + def retry(inner, func, *args, **kwargs): |
379 | + self.retry_calls.append((func, args, kwargs)) |
380 | + return super(RetryingNovaClient, inner).retry( |
381 | + func, *args, **kwargs) |
382 | + |
383 | + image_id = self.get_image_id() |
384 | + self.conf.set('vm.image', image_id) |
385 | + fixtures.patch(self, testbed.TestBed, |
386 | + 'nova_client_class', RetryingNovaClient) |
387 | + tb = testbed.TestBed(self.conf) |
388 | + self.assertEqual(image_id, tb.find_nova_image().name) |
389 | + assertions.assertLength(self, 1, self.retry_calls) |
390 | + |
391 | + def test_known_failure_is_retried(self): |
392 | + self.nb_calls = 0 |
393 | + |
394 | + class FailingOnceNovaClient(testbed.NovaClient): |
395 | + |
396 | + def fail_once(inner): |
397 | + self.nb_calls += 1 |
398 | + if self.nb_calls == 1: |
399 | + raise requests.ConnectionError() |
400 | + else: |
401 | + return inner.nova.flavors.list() |
402 | + |
403 | + def flavors_list(inner): |
404 | + return inner.retry(inner.fail_once) |
405 | + |
406 | + fixtures.patch(self, testbed.TestBed, |
407 | + 'nova_client_class', FailingOnceNovaClient) |
408 | + tb = testbed.TestBed(self.conf) |
409 | + tb.find_flavor() |
410 | + self.assertEqual(2, self.nb_calls) |
411 | + |
412 | + def test_unknown_failure_is_raised(self): |
413 | + |
414 | + class FailingNovaClient(testbed.NovaClient): |
415 | + |
416 | + def fail(inner): |
417 | + raise AssertionError('Boom!') |
418 | + |
419 | + def flavors_list(inner): |
420 | + return inner.retry(inner.fail) |
421 | + |
422 | + fixtures.patch(self, testbed.TestBed, |
423 | + 'nova_client_class', FailingNovaClient) |
424 | + tb = testbed.TestBed(self.conf) |
425 | + # This mimics what will happen when we encounter unknown transient |
426 | + # failures we want to catch: an exception will bubble up and we'll have |
427 | + # to add it to NovaClient.retry(). |
428 | + with self.assertRaises(AssertionError) as cm: |
429 | + tb.find_flavor() |
430 | + self.assertEqual('Boom!', unicode(cm.exception)) |
431 | + |
432 | + |
433 | +@features.requires(tests.nova_creds) |
434 | +@features.requires(features.nova_compute) |
435 | class TestTestbed(unittest.TestCase): |
436 | |
437 | def setUp(self): |
438 | @@ -66,42 +160,51 @@ |
439 | tb.setup() |
440 | self.assertEqual('vm.image must be set.', unicode(cm.exception)) |
441 | |
442 | - def test_create_unknown(self): |
443 | - tb = testbed.TestBed(self.conf) |
444 | + def test_create_unknown_image(self): |
445 | image_name = "I don't exist and eat kittens" |
446 | self.conf.set('vm.image', image_name) |
447 | + tb = testbed.TestBed(self.conf) |
448 | with self.assertRaises(testbed.TestBedException) as cm: |
449 | tb.setup() |
450 | self.assertEqual('Image "{}" cannot be found'.format(image_name), |
451 | unicode(cm.exception)) |
452 | |
453 | + def test_create_unknown_flavor(self): |
454 | + flavors = "I don't exist and eat kittens" |
455 | + self.conf.set('vm.flavors', flavors) |
456 | + tb = testbed.TestBed(self.conf) |
457 | + with self.assertRaises(testbed.TestBedException) as cm: |
458 | + tb.setup() |
459 | + self.assertEqual('None of [{}] can be found'.format(flavors), |
460 | + unicode(cm.exception)) |
461 | + |
462 | def test_existing_home_ssh(self): |
463 | # The first request for the worker requires creating ~/.ssh if it |
464 | # doesn't exist, but it may happen that this directory already exists |
465 | # (see http://pad.lv/1334146). |
466 | - tb = testbed.TestBed(self.conf) |
467 | ssh_home = os.path.expanduser('~/sshkeys') |
468 | os.mkdir(ssh_home) |
469 | self.conf.set('vm.ssh_key_path', os.path.join(ssh_home, 'id_rsa')) |
470 | + tb = testbed.TestBed(self.conf) |
471 | tb.ensure_ssh_key_is_available() |
472 | self.assertTrue(os.path.exists(ssh_home)) |
473 | self.assertTrue(os.path.exists(os.path.join(ssh_home, 'id_rsa'))) |
474 | self.assertTrue(os.path.exists(os.path.join(ssh_home, 'id_rsa.pub'))) |
475 | |
476 | def test_create_new_ssh_key(self): |
477 | - tb = testbed.TestBed(self.conf) |
478 | self.conf.set('vm.image', self.get_image_id()) |
479 | # We use a '~' path to cover proper uci-vms user expansion |
480 | self.conf.set('vm.ssh_key_path', '~/sshkeys/id_rsa') |
481 | + tb = testbed.TestBed(self.conf) |
482 | tb.ensure_ssh_key_is_available() |
483 | self.assertTrue(os.path.exists(os.path.expanduser('~/sshkeys/id_rsa'))) |
484 | self.assertTrue( |
485 | os.path.exists(os.path.expanduser('~/sshkeys/id_rsa.pub'))) |
486 | |
487 | def test_create_usable_testbed(self): |
488 | - tb = testbed.TestBed(self.conf) |
489 | self.conf.set('vm.release', 'saucy') |
490 | self.conf.set('vm.image', self.get_image_id('saucy')) |
491 | + tb = testbed.TestBed(self.conf) |
492 | self.addCleanup(tb.teardown) |
493 | tb.setup() |
494 | # We should be able to ssh with the right user |
495 | @@ -111,9 +214,9 @@ |
496 | self.assertEqual('ubuntu\n', out) |
497 | |
498 | def test_apt_get_update_retries(self): |
499 | - tb = testbed.TestBed(self.conf) |
500 | self.conf.set('vm.image', self.get_image_id()) |
501 | self.conf.set('vm.apt_get.update.timeouts', '0.1, 0.1') |
502 | + tb = testbed.TestBed(self.conf) |
503 | self.nb_calls = 0 |
504 | |
505 | class Proc(object): |
506 | @@ -134,9 +237,9 @@ |
507 | self.assertEqual(2, self.nb_calls) |
508 | |
509 | def test_apt_get_update_fails(self): |
510 | - tb = testbed.TestBed(self.conf) |
511 | self.conf.set('vm.image', self.get_image_id()) |
512 | self.conf.set('vm.apt_get.update.timeouts', '0.1, 0.1, 0.1') |
513 | + tb = testbed.TestBed(self.conf) |
514 | |
515 | def failing_update(): |
516 | class Proc(object): |
517 | @@ -151,3 +254,25 @@ |
518 | tb.safe_apt_get_update() |
519 | self.assertEqual('apt-get update never succeeded', |
520 | unicode(cm.exception)) |
521 | + |
522 | + def test_wait_for_instance_fails(self): |
523 | + self.conf.set('vm.image', self.get_image_id()) |
524 | + # Force a 0 timeout so the instance can't finish booting |
525 | + self.conf.set('vm.nova.boot_timeout', '0') |
526 | + tb = testbed.TestBed(self.conf) |
527 | + self.addCleanup(tb.teardown) |
528 | + with self.assertRaises(testbed.TestBedException) as cm: |
529 | + tb.setup() |
530 | + self.assertEqual('Instance never came up (last status: BUILD)', |
531 | + unicode(cm.exception)) |
532 | + |
533 | + def test_wait_for_ip_fails(self): |
534 | + self.conf.set('vm.image', self.get_image_id()) |
535 | + # Force a 0 timeout so the instance never get an IP |
536 | + self.conf.set('vm.nova.set_ip_timeout', '0') |
537 | + tb = testbed.TestBed(self.conf) |
538 | + self.addCleanup(tb.teardown) |
539 | + with self.assertRaises(testbed.TestBedException) as cm: |
540 | + tb.setup() |
541 | + msg = 'Instance {} never provided an IP'.format(tb.instance.id) |
542 | + self.assertEqual(msg, unicode(cm.exception)) |
543 | |
544 | === modified file 'test_runner/tstrun/tests/test_worker.py' |
545 | --- test_runner/tstrun/tests/test_worker.py 2014-07-31 18:29:40 +0000 |
546 | +++ test_runner/tstrun/tests/test_worker.py 2014-08-06 13:17:27 +0000 |
547 | @@ -147,9 +147,9 @@ |
548 | worker = run_worker.TestRunnerWorker(self.ds_factory) |
549 | |
550 | def broken_teardown(test_bed): |
551 | - self.addCleanup(test_bed.nova.servers.delete, test_bed.instance.id) |
552 | + self.addCleanup(test_bed.nova.delete_server, test_bed.instance.id) |
553 | if test_bed.floating_ip is not None: |
554 | - self.addCleanup(test_bed.nova.floating_ips.delete, |
555 | + self.addCleanup(test_bed.nova.delete_floating_ip, |
556 | test_bed.floating_ip) |
557 | raise AssertionError('Boom !') |
558 |
FAILED: Continuous integration, rev:745 s-jenkins. ubuntu- ci:8080/ job/uci- engine- ci/1245/
http://
Executed test runs:
Click here to trigger a rebuild: s-jenkins. ubuntu- ci:8080/ job/uci- engine- ci/1245/ rebuild
http://