Merge lp:~vila/uci-engine/1302474-nova-transient-failures into lp:uci-engine

Proposed by Vincent Ladeuil
Status: Merged
Approved by: Evan
Approved revision: 746
Merged at revision: 758
Proposed branch: lp:~vila/uci-engine/1302474-nova-transient-failures
Merge into: lp:uci-engine
Diff against target: 557 lines (+254/-80)
4 files modified
ci-utils/ci_utils/testing/features.py (+5/-0)
test_runner/tstrun/testbed.py (+113/-69)
test_runner/tstrun/tests/test_testbed.py (+134/-9)
test_runner/tstrun/tests/test_worker.py (+2/-2)
To merge this branch: bzr merge lp:~vila/uci-engine/1302474-nova-transient-failures
Reviewer Review Type Date Requested Status
Evan (community) Approve
PS Jenkins bot (community) continuous-integration Approve
Review via email: mp+229776@code.launchpad.net

Commit message

Handle nova transient failures by re-trying failed requests *once*.

Description of the change

This handles nova transient failures by re-trying a failed request *once*.

See https://app.asana.com/0/8499154990155/15202707172731 Handle permanent nova failures for the planned long term solution.

Re-trying once should be enough to guard against most of the issues I've encountered during HP cloud hiccups and should significantly improve the test runner reliability.

Backstory at https://app.asana.com/0/8312550429058/15041456645310 Handle transient nova failures.

More details below.

I've tried a simpler approach with nova.http.adapters['https://'].max_retries = 1 but, apart from peeking far too deep under the covers of the nova client implementation, it didn't provide a way to control which exceptions we wanted to catch nor when nor a way to log them.

This proposal address that by wrapping all the used nova requests into a dedicated nova client (and goes into the right direction for the long term solution).

I also found out that the nova <manager>.find() pattern issues more requests than necessary.

Since I was working on making the requests more reliable, having less of them was a no brainer.

Finally, I've added the MISSINGTESTs and removed a good chunk of FIXMEs (the
remaining ones are unrelated to nova).

To post a comment you must log in.
Revision history for this message
PS Jenkins bot (ps-jenkins) wrote :

FAILED: Continuous integration, rev:745
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/1245/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/1245/rebuild

review: Needs Fixing (continuous-integration)
746. By Vincent Ladeuil

Fix pep8 issue.

Revision history for this message
PS Jenkins bot (ps-jenkins) wrote :

PASSED: Continuous integration, rev:746
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/1246/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/1246/rebuild

review: Approve (continuous-integration)
Revision history for this message
Evan (ev) wrote :

This looks good. Top approving as well, since Vincent is on holiday.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'ci-utils/ci_utils/testing/features.py'
--- ci-utils/ci_utils/testing/features.py 2014-07-30 16:24:48 +0000
+++ ci-utils/ci_utils/testing/features.py 2014-08-06 13:17:27 +0000
@@ -67,6 +67,8 @@
67 if client is None:67 if client is None:
68 return False68 return False
69 try:69 try:
70 # can transiently fail with requests.exceptions.ConnectionError
71 # (converted from MaxRetryError).
70 client.authenticate()72 client.authenticate()
71 except nova_exceptions.ClientException:73 except nova_exceptions.ClientException:
72 return False74 return False
@@ -84,6 +86,9 @@
84 except KeyError:86 except KeyError:
85 # If we miss some env vars, we can't get a client87 # If we miss some env vars, we can't get a client
86 return None88 return None
89 except nova_exceptions.Unauthorized:
90 # If the credentials are wrong, we can't get a client
91 return None
8792
8893
89# The single instance shared by all tests94# The single instance shared by all tests
9095
=== modified file 'test_runner/tstrun/testbed.py'
--- test_runner/tstrun/testbed.py 2014-07-31 08:46:00 +0000
+++ test_runner/tstrun/testbed.py 2014-08-06 13:17:27 +0000
@@ -17,19 +17,20 @@
1717
18import logging18import logging
19import os19import os
20import requests
21import subprocess20import subprocess
22import time21import time
2322
23import requests
2424
25from novaclient import exceptions25from novaclient import (
26from novaclient.v1_1 import client26 client,
27 exceptions,
28)
27from uciconfig import options29from uciconfig import options
28from ucivms import (30from ucivms import (
29 config,31 config,
30 vms,32 vms,
31)33)
32
33from ci_utils import unit_config34from ci_utils import unit_config
3435
3536
@@ -99,6 +100,18 @@
99a way to do so. This is intended to be fixed in uci-vms so vm.apt_sources can100a way to do so. This is intended to be fixed in uci-vms so vm.apt_sources can
100be used again.101be used again.
101'''))102'''))
103register(options.Option('vm.nova.boot_timeout', default='300',
104 from_unicode=options.float_from_store,
105 help_string='''\
106Max time to boot a nova instance (in seconds).'''))
107register(options.Option('vm.nova.set_ip_timeout', default='300',
108 from_unicode=options.float_from_store,
109 help_string='''\
110Max time for a nova instance to get an IP (in seconds).'''))
111register(options.Option('vm.nova.cloud_init_timeout', default='1200',
112 from_unicode=options.float_from_store,
113 help_string='''\
114Max time for cloud-init to fisnish (in seconds).'''))
102115
103116
104logging.basicConfig(level=logging.INFO)117logging.basicConfig(level=logging.INFO)
@@ -134,8 +147,69 @@
134 return conf147 return conf
135148
136149
150class NovaClient(object):
151 """A nova client re-trying requests on known transient failures."""
152
153 def __init__(self, *args, **kwargs):
154 debug = kwargs.pop('debug', False)
155 # Activating debug will output the http requests issued by nova and the
156 # corresponding responses.
157 if debug:
158 logging.root.setLevel(logging.DEBUG)
159 self.nova = client.Client('1.1', *args, http_log_debug=debug,
160 **kwargs)
161
162 def retry(self, func, *args, **kwargs):
163 try:
164 return func(*args, **kwargs)
165 except requests.ConnectionError:
166 # Most common transient failure: the API server is unreachable
167 nap_time = 1
168 log.warn('Received connection error for {},'
169 ' retrying)'.format(func.__name__))
170 except exceptions.OverLimit:
171 # This happens rarely but breaks badly if not caught. elmo
172 # recommended a 30 seconds nap in that case.
173 nap_time = 30
174 msg = ('Rate limit reached for {},'
175 ' will sleep for {} seconds')
176 log.exception(msg.format(func.__name__, nap_time))
177 time.sleep(nap_time)
178 return func(*args, **kwargs) # Retry once
179
180 def flavors_list(self):
181 return self.retry(self.nova.flavors.list)
182
183 def images_list(self):
184 return self.retry(self.nova.images.list)
185
186 def create_server(self, name, flavor, image, user_data):
187 return self.retry(self.nova.servers.create, name=name,
188 flavor=flavor, image=image, userdata=user_data)
189
190 def delete_server(self, server_id):
191 return self.retry(self.nova.servers.delete, server_id)
192
193 def create_floating_ip(self):
194 return self.retry(self.nova.floating_ips.create)
195
196 def delete_floating_ip(self, floating_ip):
197 return self.retry(self.nova.floating_ips.delete, floating_ip)
198
199 def add_floating_ip(self, instance, floating_ip):
200 return self.retry(instance.add_floating_ip, floating_ip)
201
202 def get_server_details(self, server_id):
203 return self.retry(self.nova.servers.get, server_id)
204
205 def get_server_console(self, server, length=None):
206 return self.retry(server.get_console_output, length)
207
208
137class TestBed(vms.VM):209class TestBed(vms.VM):
138210
211 nova_client_class = NovaClient
212
139 def __init__(self, conf):213 def __init__(self, conf):
140 super(TestBed, self).__init__(conf)214 super(TestBed, self).__init__(conf)
141 self.instance = None215 self.instance = None
@@ -155,7 +229,8 @@
155 self.conf.get('os.auth_url')]229 self.conf.get('os.auth_url')]
156 kwargs = {'region_name': self.conf.get('os.region_name'),230 kwargs = {'region_name': self.conf.get('os.region_name'),
157 'service_type': 'compute'}231 'service_type': 'compute'}
158 return client.Client(*args, **kwargs)232 nova_client = self.nova_client_class(*args, **kwargs)
233 return nova_client
159234
160 def ensure_ssh_key_is_available(self):235 def ensure_ssh_key_is_available(self):
161 self.test_bed_key_path = self.conf.get('vm.ssh_key_path')236 self.test_bed_key_path = self.conf.get('vm.ssh_key_path')
@@ -173,23 +248,22 @@
173 '-f', self.test_bed_key_path, '-N', ''])248 '-f', self.test_bed_key_path, '-N', ''])
174249
175 def find_flavor(self):250 def find_flavor(self):
176 for flavor in self.conf.get('vm.flavors'):251 flavors = self.conf.get('vm.flavors')
177 try:252 existing_flavors = self.nova.flavors_list()
178 return self.nova.flavors.find(name=flavor)253 for flavor in flavors:
179 except exceptions.NotFound:254 for existing in existing_flavors:
180 pass255 if flavor == existing.name:
181 # MISSINGTEST: some cloud doesn't provide one of our expected flavors256 return existing
182 # -- vila 2014-01-06
183 raise TestBedException(257 raise TestBedException(
184 'None of {} can be found'.format(self.flavors))258 'None of [{}] can be found'.format(','.join(flavors)))
185259
186 def find_nova_image(self):260 def find_nova_image(self):
187 image_name = self.conf.get('vm.image')261 image_name = self.conf.get('vm.image')
188 try:262 existing_images = self.nova.images_list()
189 return self.nova.images.find(name=image_name)263 for existing in existing_images:
190 except exceptions.NotFound:264 if image_name == existing.name:
191 raise TestBedException(265 return existing
192 'Image "{}" cannot be found'.format(image_name))266 raise TestBedException('Image "{}" cannot be found'.format(image_name))
193267
194 def setup(self):268 def setup(self):
195 flavor = self.find_flavor()269 flavor = self.find_flavor()
@@ -198,13 +272,13 @@
198 self.create_user_data()272 self.create_user_data()
199 with open(self._user_data_path) as f:273 with open(self._user_data_path) as f:
200 user_data = f.read()274 user_data = f.read()
201 self.instance = self.nova.servers.create(275 self.instance = self.nova.create_server(
202 name=self.conf.get('vm.name'), flavor=flavor, image=image,276 name=self.conf.get('vm.name'), flavor=flavor, image=image,
203 userdata=user_data)277 user_data=user_data)
204 self.wait_for_active_instance()278 self.wait_for_active_instance()
205 if unit_config.is_hpcloud(self.conf.get('os.auth_url')):279 if unit_config.is_hpcloud(self.conf.get('os.auth_url')):
206 self.floating_ip = self.nova.floating_ips.create()280 self.floating_ip = self.nova.create_floating_ip()
207 self.instance.add_floating_ip(self.floating_ip)281 self.nova.add_floating_ip(self.instance, self.floating_ip)
208 self.wait_for_ip()282 self.wait_for_ip()
209 self.wait_for_cloud_init()283 self.wait_for_cloud_init()
210 ppas = self.conf.get('vm.ppas')284 ppas = self.conf.get('vm.ppas')
@@ -236,38 +310,29 @@
236 raise TestBedException('apt-get update never succeeded')310 raise TestBedException('apt-get update never succeeded')
237311
238 def update_instance(self):312 def update_instance(self):
239 # MISSINGTEST: What if the instance disappear ? (could be approximated
240 # by deleting the nova instance) -- vila 2014-06-05
241 try:313 try:
242 # Always query nova to get updated data about the instance314 # Always query nova to get updated data about the instance
243 self.instance = self.nova.servers.get(self.instance.id)315 self.instance = self.nova.get_server_details(self.instance.id)
244 return True316 return True
245 except requests.ConnectionError:317 except:
246 # The reported status is the one known before the last attempt.318 # But catch exceptions if something goes wrong. Higher levels will
247 log.warn('Received connection error for {},'319 # deal with the instance not replying.
248 ' retrying (status was: {})'.format(320 return False
249 self.instance.id, self.instance.status))
250 return False
251321
252 def wait_for_active_instance(self):322 def wait_for_active_instance(self):
253 # FIXME: get_active_instance should be a config option323 timeout_limit = time.time() + self.conf.get('vm.nova.boot_timeout')
254 # -- vila 2014-05-13324 while (time.time() <= timeout_limit
255 get_active_instance = 300 # in seconds so 5 minutes325 and self.instance.status != 'ACTIVE'):
256 timeout_limit = time.time() + get_active_instance
257 while time.time() < timeout_limit and self.instance.status != 'ACTIVE':
258 time.sleep(5)
259 self.update_instance()326 self.update_instance()
327 time.sleep(5)
260 if self.instance.status != 'ACTIVE':328 if self.instance.status != 'ACTIVE':
261 # MISSINGTEST: What if the instance doesn't come up ?
262 msg = 'Instance never came up (last status: {})'.format(329 msg = 'Instance never came up (last status: {})'.format(
263 self.instance.status)330 self.instance.status)
264 raise TestBedException(msg)331 raise TestBedException(msg)
265332
266 def wait_for_ip(self):333 def wait_for_ip(self):
267 # FIXME: get_ip_timeout should be a config option -- vila 2014-01-30334 timeout_limit = time.time() + self.conf.get('vm.nova.set_ip_timeout')
268 get_ip_timeout = 300 # in seconds so 5 minutes335 while time.time() <= timeout_limit:
269 timeout_limit = time.time() + get_ip_timeout
270 while time.time() < timeout_limit:
271 if not self.update_instance():336 if not self.update_instance():
272 time.sleep(5)337 time.sleep(5)
273 continue338 continue
@@ -280,46 +345,25 @@
280 # the floating one. In both cases that gives us a reachable IP.345 # the floating one. In both cases that gives us a reachable IP.
281 self.ip = networks[0][-1]346 self.ip = networks[0][-1]
282 log.info('Got IP {} for {}'.format(self.ip, self.instance.id))347 log.info('Got IP {} for {}'.format(self.ip, self.instance.id))
283 # FIXME: Right place to report how long it took to spin up the
284 # instance as far as nova is concerned. -- vila 2014-01-30
285 return348 return
286 else:349 else:
287 log.info(350 log.info(
288 'IP not yet available for {}'.format(self.instance.id))351 'IP not yet available for {}'.format(self.instance.id))
289 time.sleep(5)352 time.sleep(5)
290 # MISSINGTEST: What if the instance still doesn't have an ip ?
291 msg = 'Instance {} never provided an IP'.format(self.instance.id)353 msg = 'Instance {} never provided an IP'.format(self.instance.id)
292 raise TestBedException(msg)354 raise TestBedException(msg)
293355
294 def get_cloud_init_console(self, length=None):356 def get_cloud_init_console(self, length=None):
295 return self.instance.get_console_output(length)357 return self.nova.get_server_console(self.instance, length)
296358
297 def wait_for_cloud_init(self):359 def wait_for_cloud_init(self):
298 # FIXME: cloud_init_timeout should be a config option (related to360 timeout_limit = (time.time()
299 # get_ip_timeout and probably the two can be merged) -- vila 2014-01-30361 + self.conf.get('vm.nova.cloud_init_timeout'))
300 cloud_init_timeout = 1200 # in seconds so 20 minutes
301 timeout_limit = time.time() + cloud_init_timeout
302 final_message = self.conf.get('vm.final_message')362 final_message = self.conf.get('vm.final_message')
303 while time.time() < timeout_limit:363 while time.time() < timeout_limit:
304 # A relatively cheap way to catch cloud-init completion is to watch364 # A relatively cheap way to catch cloud-init completion is to watch
305 # the console for the specific message we specified in user-data).365 # the console for the specific message we specified in user-data).
306 try:366 console = self.get_cloud_init_console(10)
307 console = self.get_cloud_init_console(10)
308 except exceptions.OverLimit:
309 # This happens rarely but breaks badly if not caught. elmo
310 # recommended a 30 seconds nap in that case.
311 nap_time = 30
312 msg = ('Rate limit while acquiring nova console for {},'
313 ' will sleep for {} seconds')
314 log.exception(msg.format(self.instance.id, nap_time))
315 time.sleep(nap_time)
316 continue
317 except requests.ConnectionError:
318 # The reported status is the one known before the last attempt.
319 log.warn('Received connection error for {},'
320 ' retrying)'.format(self.instance.id))
321 time.sleep(5)
322 continue
323 if final_message in console:367 if final_message in console:
324 # We're good to go368 # We're good to go
325 log.info(369 log.info(
@@ -330,10 +374,10 @@
330374
331 def teardown(self):375 def teardown(self):
332 if self.instance is not None:376 if self.instance is not None:
333 self.nova.servers.delete(self.instance.id)377 self.nova.delete_server(self.instance.id)
334 self.instance = None378 self.instance = None
335 if self.floating_ip is not None:379 if self.floating_ip is not None:
336 self.nova.floating_ips.delete(self.floating_ip)380 self.nova.delete_floating_ip(self.floating_ip)
337 self.floating_ip = None381 self.floating_ip = None
338 # FIXME: Now we can remove the testbed key from known_hosts (see382 # FIXME: Now we can remove the testbed key from known_hosts (see
339 # ssh()). -- vila 2014-01-30383 # ssh()). -- vila 2014-01-30
340384
=== modified file 'test_runner/tstrun/tests/test_testbed.py'
--- test_runner/tstrun/tests/test_testbed.py 2014-07-30 15:41:08 +0000
+++ test_runner/tstrun/tests/test_testbed.py 2014-08-06 13:17:27 +0000
@@ -19,11 +19,14 @@
19import subprocess19import subprocess
20import unittest20import unittest
2121
2222import requests
23from uciconfig import options23from uciconfig import options
24from ucitests import (
25 assertions,
26 fixtures,
27)
24from ucivms.tests import fixtures as vms_fixtures28from ucivms.tests import fixtures as vms_fixtures
2529
26
27from ci_utils import unit_config30from ci_utils import unit_config
28from ci_utils.testing import (31from ci_utils.testing import (
29 features,32 features,
@@ -36,6 +39,97 @@
3639
37@features.requires(tests.nova_creds)40@features.requires(tests.nova_creds)
38@features.requires(features.nova_compute)41@features.requires(features.nova_compute)
42class TestNovaClient(unittest.TestCase):
43 """Check the nova client behavior when it encounters exceptions.
44
45 This is achieved by overriding specific methods from NovaClient and
46 exercising it through the TestBed methods.
47 """
48
49 def setUp(self):
50 super(TestNovaClient, self).setUp()
51 vms_fixtures.isolate_from_disk(self)
52 self.tb_name = 'testing-nova-client'
53 # Prepare a suitable config, importing the nova credentials
54 conf = testbed.vms_config_from_auth_config(
55 self.tb_name, unit_config.get_auth_config())
56 # Default to precise
57 conf.set('vm.release', 'precise')
58 # Avoid triggering the 'atexit' hook as the config files are long gone
59 # at that point.
60 self.addCleanup(conf.store.save_changes)
61 self.conf = conf
62 os.makedirs(self.conf.get('vm.vms_dir'))
63
64 def get_image_id(self, series='precise'):
65 if unit_config.is_hpcloud(self.conf.get('os.auth_url')):
66 test_images = features.hpcloud_test_images
67 else:
68 test_images = features.canonistack_test_images
69 return test_images[series]
70
71 def test_retry_is_called(self):
72 self.retry_calls = []
73
74 class RetryingNovaClient(testbed.NovaClient):
75
76 def retry(inner, func, *args, **kwargs):
77 self.retry_calls.append((func, args, kwargs))
78 return super(RetryingNovaClient, inner).retry(
79 func, *args, **kwargs)
80
81 image_id = self.get_image_id()
82 self.conf.set('vm.image', image_id)
83 fixtures.patch(self, testbed.TestBed,
84 'nova_client_class', RetryingNovaClient)
85 tb = testbed.TestBed(self.conf)
86 self.assertEqual(image_id, tb.find_nova_image().name)
87 assertions.assertLength(self, 1, self.retry_calls)
88
89 def test_known_failure_is_retried(self):
90 self.nb_calls = 0
91
92 class FailingOnceNovaClient(testbed.NovaClient):
93
94 def fail_once(inner):
95 self.nb_calls += 1
96 if self.nb_calls == 1:
97 raise requests.ConnectionError()
98 else:
99 return inner.nova.flavors.list()
100
101 def flavors_list(inner):
102 return inner.retry(inner.fail_once)
103
104 fixtures.patch(self, testbed.TestBed,
105 'nova_client_class', FailingOnceNovaClient)
106 tb = testbed.TestBed(self.conf)
107 tb.find_flavor()
108 self.assertEqual(2, self.nb_calls)
109
110 def test_unknown_failure_is_raised(self):
111
112 class FailingNovaClient(testbed.NovaClient):
113
114 def fail(inner):
115 raise AssertionError('Boom!')
116
117 def flavors_list(inner):
118 return inner.retry(inner.fail)
119
120 fixtures.patch(self, testbed.TestBed,
121 'nova_client_class', FailingNovaClient)
122 tb = testbed.TestBed(self.conf)
123 # This mimics what will happen when we encounter unknown transient
124 # failures we want to catch: an exception will bubble up and we'll have
125 # to add it to NovaClient.retry().
126 with self.assertRaises(AssertionError) as cm:
127 tb.find_flavor()
128 self.assertEqual('Boom!', unicode(cm.exception))
129
130
131@features.requires(tests.nova_creds)
132@features.requires(features.nova_compute)
39class TestTestbed(unittest.TestCase):133class TestTestbed(unittest.TestCase):
40134
41 def setUp(self):135 def setUp(self):
@@ -66,42 +160,51 @@
66 tb.setup()160 tb.setup()
67 self.assertEqual('vm.image must be set.', unicode(cm.exception))161 self.assertEqual('vm.image must be set.', unicode(cm.exception))
68162
69 def test_create_unknown(self):163 def test_create_unknown_image(self):
70 tb = testbed.TestBed(self.conf)
71 image_name = "I don't exist and eat kittens"164 image_name = "I don't exist and eat kittens"
72 self.conf.set('vm.image', image_name)165 self.conf.set('vm.image', image_name)
166 tb = testbed.TestBed(self.conf)
73 with self.assertRaises(testbed.TestBedException) as cm:167 with self.assertRaises(testbed.TestBedException) as cm:
74 tb.setup()168 tb.setup()
75 self.assertEqual('Image "{}" cannot be found'.format(image_name),169 self.assertEqual('Image "{}" cannot be found'.format(image_name),
76 unicode(cm.exception))170 unicode(cm.exception))
77171
172 def test_create_unknown_flavor(self):
173 flavors = "I don't exist and eat kittens"
174 self.conf.set('vm.flavors', flavors)
175 tb = testbed.TestBed(self.conf)
176 with self.assertRaises(testbed.TestBedException) as cm:
177 tb.setup()
178 self.assertEqual('None of [{}] can be found'.format(flavors),
179 unicode(cm.exception))
180
78 def test_existing_home_ssh(self):181 def test_existing_home_ssh(self):
79 # The first request for the worker requires creating ~/.ssh if it182 # The first request for the worker requires creating ~/.ssh if it
80 # doesn't exist, but it may happen that this directory already exists183 # doesn't exist, but it may happen that this directory already exists
81 # (see http://pad.lv/1334146).184 # (see http://pad.lv/1334146).
82 tb = testbed.TestBed(self.conf)
83 ssh_home = os.path.expanduser('~/sshkeys')185 ssh_home = os.path.expanduser('~/sshkeys')
84 os.mkdir(ssh_home)186 os.mkdir(ssh_home)
85 self.conf.set('vm.ssh_key_path', os.path.join(ssh_home, 'id_rsa'))187 self.conf.set('vm.ssh_key_path', os.path.join(ssh_home, 'id_rsa'))
188 tb = testbed.TestBed(self.conf)
86 tb.ensure_ssh_key_is_available()189 tb.ensure_ssh_key_is_available()
87 self.assertTrue(os.path.exists(ssh_home))190 self.assertTrue(os.path.exists(ssh_home))
88 self.assertTrue(os.path.exists(os.path.join(ssh_home, 'id_rsa')))191 self.assertTrue(os.path.exists(os.path.join(ssh_home, 'id_rsa')))
89 self.assertTrue(os.path.exists(os.path.join(ssh_home, 'id_rsa.pub')))192 self.assertTrue(os.path.exists(os.path.join(ssh_home, 'id_rsa.pub')))
90193
91 def test_create_new_ssh_key(self):194 def test_create_new_ssh_key(self):
92 tb = testbed.TestBed(self.conf)
93 self.conf.set('vm.image', self.get_image_id())195 self.conf.set('vm.image', self.get_image_id())
94 # We use a '~' path to cover proper uci-vms user expansion196 # We use a '~' path to cover proper uci-vms user expansion
95 self.conf.set('vm.ssh_key_path', '~/sshkeys/id_rsa')197 self.conf.set('vm.ssh_key_path', '~/sshkeys/id_rsa')
198 tb = testbed.TestBed(self.conf)
96 tb.ensure_ssh_key_is_available()199 tb.ensure_ssh_key_is_available()
97 self.assertTrue(os.path.exists(os.path.expanduser('~/sshkeys/id_rsa')))200 self.assertTrue(os.path.exists(os.path.expanduser('~/sshkeys/id_rsa')))
98 self.assertTrue(201 self.assertTrue(
99 os.path.exists(os.path.expanduser('~/sshkeys/id_rsa.pub')))202 os.path.exists(os.path.expanduser('~/sshkeys/id_rsa.pub')))
100203
101 def test_create_usable_testbed(self):204 def test_create_usable_testbed(self):
102 tb = testbed.TestBed(self.conf)
103 self.conf.set('vm.release', 'saucy')205 self.conf.set('vm.release', 'saucy')
104 self.conf.set('vm.image', self.get_image_id('saucy'))206 self.conf.set('vm.image', self.get_image_id('saucy'))
207 tb = testbed.TestBed(self.conf)
105 self.addCleanup(tb.teardown)208 self.addCleanup(tb.teardown)
106 tb.setup()209 tb.setup()
107 # We should be able to ssh with the right user210 # We should be able to ssh with the right user
@@ -111,9 +214,9 @@
111 self.assertEqual('ubuntu\n', out)214 self.assertEqual('ubuntu\n', out)
112215
113 def test_apt_get_update_retries(self):216 def test_apt_get_update_retries(self):
114 tb = testbed.TestBed(self.conf)
115 self.conf.set('vm.image', self.get_image_id())217 self.conf.set('vm.image', self.get_image_id())
116 self.conf.set('vm.apt_get.update.timeouts', '0.1, 0.1')218 self.conf.set('vm.apt_get.update.timeouts', '0.1, 0.1')
219 tb = testbed.TestBed(self.conf)
117 self.nb_calls = 0220 self.nb_calls = 0
118221
119 class Proc(object):222 class Proc(object):
@@ -134,9 +237,9 @@
134 self.assertEqual(2, self.nb_calls)237 self.assertEqual(2, self.nb_calls)
135238
136 def test_apt_get_update_fails(self):239 def test_apt_get_update_fails(self):
137 tb = testbed.TestBed(self.conf)
138 self.conf.set('vm.image', self.get_image_id())240 self.conf.set('vm.image', self.get_image_id())
139 self.conf.set('vm.apt_get.update.timeouts', '0.1, 0.1, 0.1')241 self.conf.set('vm.apt_get.update.timeouts', '0.1, 0.1, 0.1')
242 tb = testbed.TestBed(self.conf)
140243
141 def failing_update():244 def failing_update():
142 class Proc(object):245 class Proc(object):
@@ -151,3 +254,25 @@
151 tb.safe_apt_get_update()254 tb.safe_apt_get_update()
152 self.assertEqual('apt-get update never succeeded',255 self.assertEqual('apt-get update never succeeded',
153 unicode(cm.exception))256 unicode(cm.exception))
257
258 def test_wait_for_instance_fails(self):
259 self.conf.set('vm.image', self.get_image_id())
260 # Force a 0 timeout so the instance can't finish booting
261 self.conf.set('vm.nova.boot_timeout', '0')
262 tb = testbed.TestBed(self.conf)
263 self.addCleanup(tb.teardown)
264 with self.assertRaises(testbed.TestBedException) as cm:
265 tb.setup()
266 self.assertEqual('Instance never came up (last status: BUILD)',
267 unicode(cm.exception))
268
269 def test_wait_for_ip_fails(self):
270 self.conf.set('vm.image', self.get_image_id())
271 # Force a 0 timeout so the instance never get an IP
272 self.conf.set('vm.nova.set_ip_timeout', '0')
273 tb = testbed.TestBed(self.conf)
274 self.addCleanup(tb.teardown)
275 with self.assertRaises(testbed.TestBedException) as cm:
276 tb.setup()
277 msg = 'Instance {} never provided an IP'.format(tb.instance.id)
278 self.assertEqual(msg, unicode(cm.exception))
154279
=== modified file 'test_runner/tstrun/tests/test_worker.py'
--- test_runner/tstrun/tests/test_worker.py 2014-07-31 18:29:40 +0000
+++ test_runner/tstrun/tests/test_worker.py 2014-08-06 13:17:27 +0000
@@ -147,9 +147,9 @@
147 worker = run_worker.TestRunnerWorker(self.ds_factory)147 worker = run_worker.TestRunnerWorker(self.ds_factory)
148148
149 def broken_teardown(test_bed):149 def broken_teardown(test_bed):
150 self.addCleanup(test_bed.nova.servers.delete, test_bed.instance.id)150 self.addCleanup(test_bed.nova.delete_server, test_bed.instance.id)
151 if test_bed.floating_ip is not None:151 if test_bed.floating_ip is not None:
152 self.addCleanup(test_bed.nova.floating_ips.delete,152 self.addCleanup(test_bed.nova.delete_floating_ip,
153 test_bed.floating_ip)153 test_bed.floating_ip)
154 raise AssertionError('Boom !')154 raise AssertionError('Boom !')
155155

Subscribers

People subscribed via source and target branches