Merge lp:~vila/uci-engine/1302474-nova-transient-failures into lp:uci-engine
- 1302474-nova-transient-failures
- Merge into trunk
Status: | Merged |
---|---|
Approved by: | Evan |
Approved revision: | 746 |
Merged at revision: | 758 |
Proposed branch: | lp:~vila/uci-engine/1302474-nova-transient-failures |
Merge into: | lp:uci-engine |
Diff against target: |
557 lines (+254/-80) 4 files modified
ci-utils/ci_utils/testing/features.py (+5/-0) test_runner/tstrun/testbed.py (+113/-69) test_runner/tstrun/tests/test_testbed.py (+134/-9) test_runner/tstrun/tests/test_worker.py (+2/-2) |
To merge this branch: | bzr merge lp:~vila/uci-engine/1302474-nova-transient-failures |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Evan (community) | Approve | ||
PS Jenkins bot (community) | continuous-integration | Approve | |
Review via email: mp+229776@code.launchpad.net |
Commit message
Handle nova transient failures by re-trying failed requests *once*.
Description of the change
This handles nova transient failures by re-trying a failed request *once*.
See https:/
Re-trying once should be enough to guard against most of the issues I've encountered during HP cloud hiccups and should significantly improve the test runner reliability.
Backstory at https:/
More details below.
I've tried a simpler approach with nova.http.
This proposal address that by wrapping all the used nova requests into a dedicated nova client (and goes into the right direction for the long term solution).
I also found out that the nova <manager>.find() pattern issues more requests than necessary.
Since I was working on making the requests more reliable, having less of them was a no brainer.
Finally, I've added the MISSINGTESTs and removed a good chunk of FIXMEs (the
remaining ones are unrelated to nova).
PS Jenkins bot (ps-jenkins) wrote : | # |
- 746. By Vincent Ladeuil
-
Fix pep8 issue.
PS Jenkins bot (ps-jenkins) wrote : | # |
PASSED: Continuous integration, rev:746
http://
Executed test runs:
Click here to trigger a rebuild:
http://
Preview Diff
1 | === modified file 'ci-utils/ci_utils/testing/features.py' | |||
2 | --- ci-utils/ci_utils/testing/features.py 2014-07-30 16:24:48 +0000 | |||
3 | +++ ci-utils/ci_utils/testing/features.py 2014-08-06 13:17:27 +0000 | |||
4 | @@ -67,6 +67,8 @@ | |||
5 | 67 | if client is None: | 67 | if client is None: |
6 | 68 | return False | 68 | return False |
7 | 69 | try: | 69 | try: |
8 | 70 | # can transiently fail with requests.exceptions.ConnectionError | ||
9 | 71 | # (converted from MaxRetryError). | ||
10 | 70 | client.authenticate() | 72 | client.authenticate() |
11 | 71 | except nova_exceptions.ClientException: | 73 | except nova_exceptions.ClientException: |
12 | 72 | return False | 74 | return False |
13 | @@ -84,6 +86,9 @@ | |||
14 | 84 | except KeyError: | 86 | except KeyError: |
15 | 85 | # If we miss some env vars, we can't get a client | 87 | # If we miss some env vars, we can't get a client |
16 | 86 | return None | 88 | return None |
17 | 89 | except nova_exceptions.Unauthorized: | ||
18 | 90 | # If the credentials are wrong, we can't get a client | ||
19 | 91 | return None | ||
20 | 87 | 92 | ||
21 | 88 | 93 | ||
22 | 89 | # The single instance shared by all tests | 94 | # The single instance shared by all tests |
23 | 90 | 95 | ||
24 | === modified file 'test_runner/tstrun/testbed.py' | |||
25 | --- test_runner/tstrun/testbed.py 2014-07-31 08:46:00 +0000 | |||
26 | +++ test_runner/tstrun/testbed.py 2014-08-06 13:17:27 +0000 | |||
27 | @@ -17,19 +17,20 @@ | |||
28 | 17 | 17 | ||
29 | 18 | import logging | 18 | import logging |
30 | 19 | import os | 19 | import os |
31 | 20 | import requests | ||
32 | 21 | import subprocess | 20 | import subprocess |
33 | 22 | import time | 21 | import time |
34 | 23 | 22 | ||
35 | 23 | import requests | ||
36 | 24 | 24 | ||
39 | 25 | from novaclient import exceptions | 25 | from novaclient import ( |
40 | 26 | from novaclient.v1_1 import client | 26 | client, |
41 | 27 | exceptions, | ||
42 | 28 | ) | ||
43 | 27 | from uciconfig import options | 29 | from uciconfig import options |
44 | 28 | from ucivms import ( | 30 | from ucivms import ( |
45 | 29 | config, | 31 | config, |
46 | 30 | vms, | 32 | vms, |
47 | 31 | ) | 33 | ) |
48 | 32 | |||
49 | 33 | from ci_utils import unit_config | 34 | from ci_utils import unit_config |
50 | 34 | 35 | ||
51 | 35 | 36 | ||
52 | @@ -99,6 +100,18 @@ | |||
53 | 99 | a way to do so. This is intended to be fixed in uci-vms so vm.apt_sources can | 100 | a way to do so. This is intended to be fixed in uci-vms so vm.apt_sources can |
54 | 100 | be used again. | 101 | be used again. |
55 | 101 | ''')) | 102 | ''')) |
56 | 103 | register(options.Option('vm.nova.boot_timeout', default='300', | ||
57 | 104 | from_unicode=options.float_from_store, | ||
58 | 105 | help_string='''\ | ||
59 | 106 | Max time to boot a nova instance (in seconds).''')) | ||
60 | 107 | register(options.Option('vm.nova.set_ip_timeout', default='300', | ||
61 | 108 | from_unicode=options.float_from_store, | ||
62 | 109 | help_string='''\ | ||
63 | 110 | Max time for a nova instance to get an IP (in seconds).''')) | ||
64 | 111 | register(options.Option('vm.nova.cloud_init_timeout', default='1200', | ||
65 | 112 | from_unicode=options.float_from_store, | ||
66 | 113 | help_string='''\ | ||
67 | 114 | Max time for cloud-init to fisnish (in seconds).''')) | ||
68 | 102 | 115 | ||
69 | 103 | 116 | ||
70 | 104 | logging.basicConfig(level=logging.INFO) | 117 | logging.basicConfig(level=logging.INFO) |
71 | @@ -134,8 +147,69 @@ | |||
72 | 134 | return conf | 147 | return conf |
73 | 135 | 148 | ||
74 | 136 | 149 | ||
75 | 150 | class NovaClient(object): | ||
76 | 151 | """A nova client re-trying requests on known transient failures.""" | ||
77 | 152 | |||
78 | 153 | def __init__(self, *args, **kwargs): | ||
79 | 154 | debug = kwargs.pop('debug', False) | ||
80 | 155 | # Activating debug will output the http requests issued by nova and the | ||
81 | 156 | # corresponding responses. | ||
82 | 157 | if debug: | ||
83 | 158 | logging.root.setLevel(logging.DEBUG) | ||
84 | 159 | self.nova = client.Client('1.1', *args, http_log_debug=debug, | ||
85 | 160 | **kwargs) | ||
86 | 161 | |||
87 | 162 | def retry(self, func, *args, **kwargs): | ||
88 | 163 | try: | ||
89 | 164 | return func(*args, **kwargs) | ||
90 | 165 | except requests.ConnectionError: | ||
91 | 166 | # Most common transient failure: the API server is unreachable | ||
92 | 167 | nap_time = 1 | ||
93 | 168 | log.warn('Received connection error for {},' | ||
94 | 169 | ' retrying)'.format(func.__name__)) | ||
95 | 170 | except exceptions.OverLimit: | ||
96 | 171 | # This happens rarely but breaks badly if not caught. elmo | ||
97 | 172 | # recommended a 30 seconds nap in that case. | ||
98 | 173 | nap_time = 30 | ||
99 | 174 | msg = ('Rate limit reached for {},' | ||
100 | 175 | ' will sleep for {} seconds') | ||
101 | 176 | log.exception(msg.format(func.__name__, nap_time)) | ||
102 | 177 | time.sleep(nap_time) | ||
103 | 178 | return func(*args, **kwargs) # Retry once | ||
104 | 179 | |||
105 | 180 | def flavors_list(self): | ||
106 | 181 | return self.retry(self.nova.flavors.list) | ||
107 | 182 | |||
108 | 183 | def images_list(self): | ||
109 | 184 | return self.retry(self.nova.images.list) | ||
110 | 185 | |||
111 | 186 | def create_server(self, name, flavor, image, user_data): | ||
112 | 187 | return self.retry(self.nova.servers.create, name=name, | ||
113 | 188 | flavor=flavor, image=image, userdata=user_data) | ||
114 | 189 | |||
115 | 190 | def delete_server(self, server_id): | ||
116 | 191 | return self.retry(self.nova.servers.delete, server_id) | ||
117 | 192 | |||
118 | 193 | def create_floating_ip(self): | ||
119 | 194 | return self.retry(self.nova.floating_ips.create) | ||
120 | 195 | |||
121 | 196 | def delete_floating_ip(self, floating_ip): | ||
122 | 197 | return self.retry(self.nova.floating_ips.delete, floating_ip) | ||
123 | 198 | |||
124 | 199 | def add_floating_ip(self, instance, floating_ip): | ||
125 | 200 | return self.retry(instance.add_floating_ip, floating_ip) | ||
126 | 201 | |||
127 | 202 | def get_server_details(self, server_id): | ||
128 | 203 | return self.retry(self.nova.servers.get, server_id) | ||
129 | 204 | |||
130 | 205 | def get_server_console(self, server, length=None): | ||
131 | 206 | return self.retry(server.get_console_output, length) | ||
132 | 207 | |||
133 | 208 | |||
134 | 137 | class TestBed(vms.VM): | 209 | class TestBed(vms.VM): |
135 | 138 | 210 | ||
136 | 211 | nova_client_class = NovaClient | ||
137 | 212 | |||
138 | 139 | def __init__(self, conf): | 213 | def __init__(self, conf): |
139 | 140 | super(TestBed, self).__init__(conf) | 214 | super(TestBed, self).__init__(conf) |
140 | 141 | self.instance = None | 215 | self.instance = None |
141 | @@ -155,7 +229,8 @@ | |||
142 | 155 | self.conf.get('os.auth_url')] | 229 | self.conf.get('os.auth_url')] |
143 | 156 | kwargs = {'region_name': self.conf.get('os.region_name'), | 230 | kwargs = {'region_name': self.conf.get('os.region_name'), |
144 | 157 | 'service_type': 'compute'} | 231 | 'service_type': 'compute'} |
146 | 158 | return client.Client(*args, **kwargs) | 232 | nova_client = self.nova_client_class(*args, **kwargs) |
147 | 233 | return nova_client | ||
148 | 159 | 234 | ||
149 | 160 | def ensure_ssh_key_is_available(self): | 235 | def ensure_ssh_key_is_available(self): |
150 | 161 | self.test_bed_key_path = self.conf.get('vm.ssh_key_path') | 236 | self.test_bed_key_path = self.conf.get('vm.ssh_key_path') |
151 | @@ -173,23 +248,22 @@ | |||
152 | 173 | '-f', self.test_bed_key_path, '-N', '']) | 248 | '-f', self.test_bed_key_path, '-N', '']) |
153 | 174 | 249 | ||
154 | 175 | def find_flavor(self): | 250 | def find_flavor(self): |
162 | 176 | for flavor in self.conf.get('vm.flavors'): | 251 | flavors = self.conf.get('vm.flavors') |
163 | 177 | try: | 252 | existing_flavors = self.nova.flavors_list() |
164 | 178 | return self.nova.flavors.find(name=flavor) | 253 | for flavor in flavors: |
165 | 179 | except exceptions.NotFound: | 254 | for existing in existing_flavors: |
166 | 180 | pass | 255 | if flavor == existing.name: |
167 | 181 | # MISSINGTEST: some cloud doesn't provide one of our expected flavors | 256 | return existing |
161 | 182 | # -- vila 2014-01-06 | ||
168 | 183 | raise TestBedException( | 257 | raise TestBedException( |
170 | 184 | 'None of {} can be found'.format(self.flavors)) | 258 | 'None of [{}] can be found'.format(','.join(flavors))) |
171 | 185 | 259 | ||
172 | 186 | def find_nova_image(self): | 260 | def find_nova_image(self): |
173 | 187 | image_name = self.conf.get('vm.image') | 261 | image_name = self.conf.get('vm.image') |
179 | 188 | try: | 262 | existing_images = self.nova.images_list() |
180 | 189 | return self.nova.images.find(name=image_name) | 263 | for existing in existing_images: |
181 | 190 | except exceptions.NotFound: | 264 | if image_name == existing.name: |
182 | 191 | raise TestBedException( | 265 | return existing |
183 | 192 | 'Image "{}" cannot be found'.format(image_name)) | 266 | raise TestBedException('Image "{}" cannot be found'.format(image_name)) |
184 | 193 | 267 | ||
185 | 194 | def setup(self): | 268 | def setup(self): |
186 | 195 | flavor = self.find_flavor() | 269 | flavor = self.find_flavor() |
187 | @@ -198,13 +272,13 @@ | |||
188 | 198 | self.create_user_data() | 272 | self.create_user_data() |
189 | 199 | with open(self._user_data_path) as f: | 273 | with open(self._user_data_path) as f: |
190 | 200 | user_data = f.read() | 274 | user_data = f.read() |
192 | 201 | self.instance = self.nova.servers.create( | 275 | self.instance = self.nova.create_server( |
193 | 202 | name=self.conf.get('vm.name'), flavor=flavor, image=image, | 276 | name=self.conf.get('vm.name'), flavor=flavor, image=image, |
195 | 203 | userdata=user_data) | 277 | user_data=user_data) |
196 | 204 | self.wait_for_active_instance() | 278 | self.wait_for_active_instance() |
197 | 205 | if unit_config.is_hpcloud(self.conf.get('os.auth_url')): | 279 | if unit_config.is_hpcloud(self.conf.get('os.auth_url')): |
200 | 206 | self.floating_ip = self.nova.floating_ips.create() | 280 | self.floating_ip = self.nova.create_floating_ip() |
201 | 207 | self.instance.add_floating_ip(self.floating_ip) | 281 | self.nova.add_floating_ip(self.instance, self.floating_ip) |
202 | 208 | self.wait_for_ip() | 282 | self.wait_for_ip() |
203 | 209 | self.wait_for_cloud_init() | 283 | self.wait_for_cloud_init() |
204 | 210 | ppas = self.conf.get('vm.ppas') | 284 | ppas = self.conf.get('vm.ppas') |
205 | @@ -236,38 +310,29 @@ | |||
206 | 236 | raise TestBedException('apt-get update never succeeded') | 310 | raise TestBedException('apt-get update never succeeded') |
207 | 237 | 311 | ||
208 | 238 | def update_instance(self): | 312 | def update_instance(self): |
209 | 239 | # MISSINGTEST: What if the instance disappear ? (could be approximated | ||
210 | 240 | # by deleting the nova instance) -- vila 2014-06-05 | ||
211 | 241 | try: | 313 | try: |
212 | 242 | # Always query nova to get updated data about the instance | 314 | # Always query nova to get updated data about the instance |
214 | 243 | self.instance = self.nova.servers.get(self.instance.id) | 315 | self.instance = self.nova.get_server_details(self.instance.id) |
215 | 244 | return True | 316 | return True |
222 | 245 | except requests.ConnectionError: | 317 | except: |
223 | 246 | # The reported status is the one known before the last attempt. | 318 | # But catch exceptions if something goes wrong. Higher levels will |
224 | 247 | log.warn('Received connection error for {},' | 319 | # deal with the instance not replying. |
225 | 248 | ' retrying (status was: {})'.format( | 320 | return False |
220 | 249 | self.instance.id, self.instance.status)) | ||
221 | 250 | return False | ||
226 | 251 | 321 | ||
227 | 252 | def wait_for_active_instance(self): | 322 | def wait_for_active_instance(self): |
234 | 253 | # FIXME: get_active_instance should be a config option | 323 | timeout_limit = time.time() + self.conf.get('vm.nova.boot_timeout') |
235 | 254 | # -- vila 2014-05-13 | 324 | while (time.time() <= timeout_limit |
236 | 255 | get_active_instance = 300 # in seconds so 5 minutes | 325 | and self.instance.status != 'ACTIVE'): |
231 | 256 | timeout_limit = time.time() + get_active_instance | ||
232 | 257 | while time.time() < timeout_limit and self.instance.status != 'ACTIVE': | ||
233 | 258 | time.sleep(5) | ||
237 | 259 | self.update_instance() | 326 | self.update_instance() |
238 | 327 | time.sleep(5) | ||
239 | 260 | if self.instance.status != 'ACTIVE': | 328 | if self.instance.status != 'ACTIVE': |
240 | 261 | # MISSINGTEST: What if the instance doesn't come up ? | ||
241 | 262 | msg = 'Instance never came up (last status: {})'.format( | 329 | msg = 'Instance never came up (last status: {})'.format( |
242 | 263 | self.instance.status) | 330 | self.instance.status) |
243 | 264 | raise TestBedException(msg) | 331 | raise TestBedException(msg) |
244 | 265 | 332 | ||
245 | 266 | def wait_for_ip(self): | 333 | def wait_for_ip(self): |
250 | 267 | # FIXME: get_ip_timeout should be a config option -- vila 2014-01-30 | 334 | timeout_limit = time.time() + self.conf.get('vm.nova.set_ip_timeout') |
251 | 268 | get_ip_timeout = 300 # in seconds so 5 minutes | 335 | while time.time() <= timeout_limit: |
248 | 269 | timeout_limit = time.time() + get_ip_timeout | ||
249 | 270 | while time.time() < timeout_limit: | ||
252 | 271 | if not self.update_instance(): | 336 | if not self.update_instance(): |
253 | 272 | time.sleep(5) | 337 | time.sleep(5) |
254 | 273 | continue | 338 | continue |
255 | @@ -280,46 +345,25 @@ | |||
256 | 280 | # the floating one. In both cases that gives us a reachable IP. | 345 | # the floating one. In both cases that gives us a reachable IP. |
257 | 281 | self.ip = networks[0][-1] | 346 | self.ip = networks[0][-1] |
258 | 282 | log.info('Got IP {} for {}'.format(self.ip, self.instance.id)) | 347 | log.info('Got IP {} for {}'.format(self.ip, self.instance.id)) |
259 | 283 | # FIXME: Right place to report how long it took to spin up the | ||
260 | 284 | # instance as far as nova is concerned. -- vila 2014-01-30 | ||
261 | 285 | return | 348 | return |
262 | 286 | else: | 349 | else: |
263 | 287 | log.info( | 350 | log.info( |
264 | 288 | 'IP not yet available for {}'.format(self.instance.id)) | 351 | 'IP not yet available for {}'.format(self.instance.id)) |
265 | 289 | time.sleep(5) | 352 | time.sleep(5) |
266 | 290 | # MISSINGTEST: What if the instance still doesn't have an ip ? | ||
267 | 291 | msg = 'Instance {} never provided an IP'.format(self.instance.id) | 353 | msg = 'Instance {} never provided an IP'.format(self.instance.id) |
268 | 292 | raise TestBedException(msg) | 354 | raise TestBedException(msg) |
269 | 293 | 355 | ||
270 | 294 | def get_cloud_init_console(self, length=None): | 356 | def get_cloud_init_console(self, length=None): |
272 | 295 | return self.instance.get_console_output(length) | 357 | return self.nova.get_server_console(self.instance, length) |
273 | 296 | 358 | ||
274 | 297 | def wait_for_cloud_init(self): | 359 | def wait_for_cloud_init(self): |
279 | 298 | # FIXME: cloud_init_timeout should be a config option (related to | 360 | timeout_limit = (time.time() |
280 | 299 | # get_ip_timeout and probably the two can be merged) -- vila 2014-01-30 | 361 | + self.conf.get('vm.nova.cloud_init_timeout')) |
277 | 300 | cloud_init_timeout = 1200 # in seconds so 20 minutes | ||
278 | 301 | timeout_limit = time.time() + cloud_init_timeout | ||
281 | 302 | final_message = self.conf.get('vm.final_message') | 362 | final_message = self.conf.get('vm.final_message') |
282 | 303 | while time.time() < timeout_limit: | 363 | while time.time() < timeout_limit: |
283 | 304 | # A relatively cheap way to catch cloud-init completion is to watch | 364 | # A relatively cheap way to catch cloud-init completion is to watch |
284 | 305 | # the console for the specific message we specified in user-data). | 365 | # the console for the specific message we specified in user-data). |
302 | 306 | try: | 366 | console = self.get_cloud_init_console(10) |
286 | 307 | console = self.get_cloud_init_console(10) | ||
287 | 308 | except exceptions.OverLimit: | ||
288 | 309 | # This happens rarely but breaks badly if not caught. elmo | ||
289 | 310 | # recommended a 30 seconds nap in that case. | ||
290 | 311 | nap_time = 30 | ||
291 | 312 | msg = ('Rate limit while acquiring nova console for {},' | ||
292 | 313 | ' will sleep for {} seconds') | ||
293 | 314 | log.exception(msg.format(self.instance.id, nap_time)) | ||
294 | 315 | time.sleep(nap_time) | ||
295 | 316 | continue | ||
296 | 317 | except requests.ConnectionError: | ||
297 | 318 | # The reported status is the one known before the last attempt. | ||
298 | 319 | log.warn('Received connection error for {},' | ||
299 | 320 | ' retrying)'.format(self.instance.id)) | ||
300 | 321 | time.sleep(5) | ||
301 | 322 | continue | ||
303 | 323 | if final_message in console: | 367 | if final_message in console: |
304 | 324 | # We're good to go | 368 | # We're good to go |
305 | 325 | log.info( | 369 | log.info( |
306 | @@ -330,10 +374,10 @@ | |||
307 | 330 | 374 | ||
308 | 331 | def teardown(self): | 375 | def teardown(self): |
309 | 332 | if self.instance is not None: | 376 | if self.instance is not None: |
311 | 333 | self.nova.servers.delete(self.instance.id) | 377 | self.nova.delete_server(self.instance.id) |
312 | 334 | self.instance = None | 378 | self.instance = None |
313 | 335 | if self.floating_ip is not None: | 379 | if self.floating_ip is not None: |
315 | 336 | self.nova.floating_ips.delete(self.floating_ip) | 380 | self.nova.delete_floating_ip(self.floating_ip) |
316 | 337 | self.floating_ip = None | 381 | self.floating_ip = None |
317 | 338 | # FIXME: Now we can remove the testbed key from known_hosts (see | 382 | # FIXME: Now we can remove the testbed key from known_hosts (see |
318 | 339 | # ssh()). -- vila 2014-01-30 | 383 | # ssh()). -- vila 2014-01-30 |
319 | 340 | 384 | ||
320 | === modified file 'test_runner/tstrun/tests/test_testbed.py' | |||
321 | --- test_runner/tstrun/tests/test_testbed.py 2014-07-30 15:41:08 +0000 | |||
322 | +++ test_runner/tstrun/tests/test_testbed.py 2014-08-06 13:17:27 +0000 | |||
323 | @@ -19,11 +19,14 @@ | |||
324 | 19 | import subprocess | 19 | import subprocess |
325 | 20 | import unittest | 20 | import unittest |
326 | 21 | 21 | ||
328 | 22 | 22 | import requests | |
329 | 23 | from uciconfig import options | 23 | from uciconfig import options |
330 | 24 | from ucitests import ( | ||
331 | 25 | assertions, | ||
332 | 26 | fixtures, | ||
333 | 27 | ) | ||
334 | 24 | from ucivms.tests import fixtures as vms_fixtures | 28 | from ucivms.tests import fixtures as vms_fixtures |
335 | 25 | 29 | ||
336 | 26 | |||
337 | 27 | from ci_utils import unit_config | 30 | from ci_utils import unit_config |
338 | 28 | from ci_utils.testing import ( | 31 | from ci_utils.testing import ( |
339 | 29 | features, | 32 | features, |
340 | @@ -36,6 +39,97 @@ | |||
341 | 36 | 39 | ||
342 | 37 | @features.requires(tests.nova_creds) | 40 | @features.requires(tests.nova_creds) |
343 | 38 | @features.requires(features.nova_compute) | 41 | @features.requires(features.nova_compute) |
344 | 42 | class TestNovaClient(unittest.TestCase): | ||
345 | 43 | """Check the nova client behavior when it encounters exceptions. | ||
346 | 44 | |||
347 | 45 | This is achieved by overriding specific methods from NovaClient and | ||
348 | 46 | exercising it through the TestBed methods. | ||
349 | 47 | """ | ||
350 | 48 | |||
351 | 49 | def setUp(self): | ||
352 | 50 | super(TestNovaClient, self).setUp() | ||
353 | 51 | vms_fixtures.isolate_from_disk(self) | ||
354 | 52 | self.tb_name = 'testing-nova-client' | ||
355 | 53 | # Prepare a suitable config, importing the nova credentials | ||
356 | 54 | conf = testbed.vms_config_from_auth_config( | ||
357 | 55 | self.tb_name, unit_config.get_auth_config()) | ||
358 | 56 | # Default to precise | ||
359 | 57 | conf.set('vm.release', 'precise') | ||
360 | 58 | # Avoid triggering the 'atexit' hook as the config files are long gone | ||
361 | 59 | # at that point. | ||
362 | 60 | self.addCleanup(conf.store.save_changes) | ||
363 | 61 | self.conf = conf | ||
364 | 62 | os.makedirs(self.conf.get('vm.vms_dir')) | ||
365 | 63 | |||
366 | 64 | def get_image_id(self, series='precise'): | ||
367 | 65 | if unit_config.is_hpcloud(self.conf.get('os.auth_url')): | ||
368 | 66 | test_images = features.hpcloud_test_images | ||
369 | 67 | else: | ||
370 | 68 | test_images = features.canonistack_test_images | ||
371 | 69 | return test_images[series] | ||
372 | 70 | |||
373 | 71 | def test_retry_is_called(self): | ||
374 | 72 | self.retry_calls = [] | ||
375 | 73 | |||
376 | 74 | class RetryingNovaClient(testbed.NovaClient): | ||
377 | 75 | |||
378 | 76 | def retry(inner, func, *args, **kwargs): | ||
379 | 77 | self.retry_calls.append((func, args, kwargs)) | ||
380 | 78 | return super(RetryingNovaClient, inner).retry( | ||
381 | 79 | func, *args, **kwargs) | ||
382 | 80 | |||
383 | 81 | image_id = self.get_image_id() | ||
384 | 82 | self.conf.set('vm.image', image_id) | ||
385 | 83 | fixtures.patch(self, testbed.TestBed, | ||
386 | 84 | 'nova_client_class', RetryingNovaClient) | ||
387 | 85 | tb = testbed.TestBed(self.conf) | ||
388 | 86 | self.assertEqual(image_id, tb.find_nova_image().name) | ||
389 | 87 | assertions.assertLength(self, 1, self.retry_calls) | ||
390 | 88 | |||
391 | 89 | def test_known_failure_is_retried(self): | ||
392 | 90 | self.nb_calls = 0 | ||
393 | 91 | |||
394 | 92 | class FailingOnceNovaClient(testbed.NovaClient): | ||
395 | 93 | |||
396 | 94 | def fail_once(inner): | ||
397 | 95 | self.nb_calls += 1 | ||
398 | 96 | if self.nb_calls == 1: | ||
399 | 97 | raise requests.ConnectionError() | ||
400 | 98 | else: | ||
401 | 99 | return inner.nova.flavors.list() | ||
402 | 100 | |||
403 | 101 | def flavors_list(inner): | ||
404 | 102 | return inner.retry(inner.fail_once) | ||
405 | 103 | |||
406 | 104 | fixtures.patch(self, testbed.TestBed, | ||
407 | 105 | 'nova_client_class', FailingOnceNovaClient) | ||
408 | 106 | tb = testbed.TestBed(self.conf) | ||
409 | 107 | tb.find_flavor() | ||
410 | 108 | self.assertEqual(2, self.nb_calls) | ||
411 | 109 | |||
412 | 110 | def test_unknown_failure_is_raised(self): | ||
413 | 111 | |||
414 | 112 | class FailingNovaClient(testbed.NovaClient): | ||
415 | 113 | |||
416 | 114 | def fail(inner): | ||
417 | 115 | raise AssertionError('Boom!') | ||
418 | 116 | |||
419 | 117 | def flavors_list(inner): | ||
420 | 118 | return inner.retry(inner.fail) | ||
421 | 119 | |||
422 | 120 | fixtures.patch(self, testbed.TestBed, | ||
423 | 121 | 'nova_client_class', FailingNovaClient) | ||
424 | 122 | tb = testbed.TestBed(self.conf) | ||
425 | 123 | # This mimics what will happen when we encounter unknown transient | ||
426 | 124 | # failures we want to catch: an exception will bubble up and we'll have | ||
427 | 125 | # to add it to NovaClient.retry(). | ||
428 | 126 | with self.assertRaises(AssertionError) as cm: | ||
429 | 127 | tb.find_flavor() | ||
430 | 128 | self.assertEqual('Boom!', unicode(cm.exception)) | ||
431 | 129 | |||
432 | 130 | |||
433 | 131 | @features.requires(tests.nova_creds) | ||
434 | 132 | @features.requires(features.nova_compute) | ||
435 | 39 | class TestTestbed(unittest.TestCase): | 133 | class TestTestbed(unittest.TestCase): |
436 | 40 | 134 | ||
437 | 41 | def setUp(self): | 135 | def setUp(self): |
438 | @@ -66,42 +160,51 @@ | |||
439 | 66 | tb.setup() | 160 | tb.setup() |
440 | 67 | self.assertEqual('vm.image must be set.', unicode(cm.exception)) | 161 | self.assertEqual('vm.image must be set.', unicode(cm.exception)) |
441 | 68 | 162 | ||
444 | 69 | def test_create_unknown(self): | 163 | def test_create_unknown_image(self): |
443 | 70 | tb = testbed.TestBed(self.conf) | ||
445 | 71 | image_name = "I don't exist and eat kittens" | 164 | image_name = "I don't exist and eat kittens" |
446 | 72 | self.conf.set('vm.image', image_name) | 165 | self.conf.set('vm.image', image_name) |
447 | 166 | tb = testbed.TestBed(self.conf) | ||
448 | 73 | with self.assertRaises(testbed.TestBedException) as cm: | 167 | with self.assertRaises(testbed.TestBedException) as cm: |
449 | 74 | tb.setup() | 168 | tb.setup() |
450 | 75 | self.assertEqual('Image "{}" cannot be found'.format(image_name), | 169 | self.assertEqual('Image "{}" cannot be found'.format(image_name), |
451 | 76 | unicode(cm.exception)) | 170 | unicode(cm.exception)) |
452 | 77 | 171 | ||
453 | 172 | def test_create_unknown_flavor(self): | ||
454 | 173 | flavors = "I don't exist and eat kittens" | ||
455 | 174 | self.conf.set('vm.flavors', flavors) | ||
456 | 175 | tb = testbed.TestBed(self.conf) | ||
457 | 176 | with self.assertRaises(testbed.TestBedException) as cm: | ||
458 | 177 | tb.setup() | ||
459 | 178 | self.assertEqual('None of [{}] can be found'.format(flavors), | ||
460 | 179 | unicode(cm.exception)) | ||
461 | 180 | |||
462 | 78 | def test_existing_home_ssh(self): | 181 | def test_existing_home_ssh(self): |
463 | 79 | # The first request for the worker requires creating ~/.ssh if it | 182 | # The first request for the worker requires creating ~/.ssh if it |
464 | 80 | # doesn't exist, but it may happen that this directory already exists | 183 | # doesn't exist, but it may happen that this directory already exists |
465 | 81 | # (see http://pad.lv/1334146). | 184 | # (see http://pad.lv/1334146). |
466 | 82 | tb = testbed.TestBed(self.conf) | ||
467 | 83 | ssh_home = os.path.expanduser('~/sshkeys') | 185 | ssh_home = os.path.expanduser('~/sshkeys') |
468 | 84 | os.mkdir(ssh_home) | 186 | os.mkdir(ssh_home) |
469 | 85 | self.conf.set('vm.ssh_key_path', os.path.join(ssh_home, 'id_rsa')) | 187 | self.conf.set('vm.ssh_key_path', os.path.join(ssh_home, 'id_rsa')) |
470 | 188 | tb = testbed.TestBed(self.conf) | ||
471 | 86 | tb.ensure_ssh_key_is_available() | 189 | tb.ensure_ssh_key_is_available() |
472 | 87 | self.assertTrue(os.path.exists(ssh_home)) | 190 | self.assertTrue(os.path.exists(ssh_home)) |
473 | 88 | self.assertTrue(os.path.exists(os.path.join(ssh_home, 'id_rsa'))) | 191 | self.assertTrue(os.path.exists(os.path.join(ssh_home, 'id_rsa'))) |
474 | 89 | self.assertTrue(os.path.exists(os.path.join(ssh_home, 'id_rsa.pub'))) | 192 | self.assertTrue(os.path.exists(os.path.join(ssh_home, 'id_rsa.pub'))) |
475 | 90 | 193 | ||
476 | 91 | def test_create_new_ssh_key(self): | 194 | def test_create_new_ssh_key(self): |
477 | 92 | tb = testbed.TestBed(self.conf) | ||
478 | 93 | self.conf.set('vm.image', self.get_image_id()) | 195 | self.conf.set('vm.image', self.get_image_id()) |
479 | 94 | # We use a '~' path to cover proper uci-vms user expansion | 196 | # We use a '~' path to cover proper uci-vms user expansion |
480 | 95 | self.conf.set('vm.ssh_key_path', '~/sshkeys/id_rsa') | 197 | self.conf.set('vm.ssh_key_path', '~/sshkeys/id_rsa') |
481 | 198 | tb = testbed.TestBed(self.conf) | ||
482 | 96 | tb.ensure_ssh_key_is_available() | 199 | tb.ensure_ssh_key_is_available() |
483 | 97 | self.assertTrue(os.path.exists(os.path.expanduser('~/sshkeys/id_rsa'))) | 200 | self.assertTrue(os.path.exists(os.path.expanduser('~/sshkeys/id_rsa'))) |
484 | 98 | self.assertTrue( | 201 | self.assertTrue( |
485 | 99 | os.path.exists(os.path.expanduser('~/sshkeys/id_rsa.pub'))) | 202 | os.path.exists(os.path.expanduser('~/sshkeys/id_rsa.pub'))) |
486 | 100 | 203 | ||
487 | 101 | def test_create_usable_testbed(self): | 204 | def test_create_usable_testbed(self): |
488 | 102 | tb = testbed.TestBed(self.conf) | ||
489 | 103 | self.conf.set('vm.release', 'saucy') | 205 | self.conf.set('vm.release', 'saucy') |
490 | 104 | self.conf.set('vm.image', self.get_image_id('saucy')) | 206 | self.conf.set('vm.image', self.get_image_id('saucy')) |
491 | 207 | tb = testbed.TestBed(self.conf) | ||
492 | 105 | self.addCleanup(tb.teardown) | 208 | self.addCleanup(tb.teardown) |
493 | 106 | tb.setup() | 209 | tb.setup() |
494 | 107 | # We should be able to ssh with the right user | 210 | # We should be able to ssh with the right user |
495 | @@ -111,9 +214,9 @@ | |||
496 | 111 | self.assertEqual('ubuntu\n', out) | 214 | self.assertEqual('ubuntu\n', out) |
497 | 112 | 215 | ||
498 | 113 | def test_apt_get_update_retries(self): | 216 | def test_apt_get_update_retries(self): |
499 | 114 | tb = testbed.TestBed(self.conf) | ||
500 | 115 | self.conf.set('vm.image', self.get_image_id()) | 217 | self.conf.set('vm.image', self.get_image_id()) |
501 | 116 | self.conf.set('vm.apt_get.update.timeouts', '0.1, 0.1') | 218 | self.conf.set('vm.apt_get.update.timeouts', '0.1, 0.1') |
502 | 219 | tb = testbed.TestBed(self.conf) | ||
503 | 117 | self.nb_calls = 0 | 220 | self.nb_calls = 0 |
504 | 118 | 221 | ||
505 | 119 | class Proc(object): | 222 | class Proc(object): |
506 | @@ -134,9 +237,9 @@ | |||
507 | 134 | self.assertEqual(2, self.nb_calls) | 237 | self.assertEqual(2, self.nb_calls) |
508 | 135 | 238 | ||
509 | 136 | def test_apt_get_update_fails(self): | 239 | def test_apt_get_update_fails(self): |
510 | 137 | tb = testbed.TestBed(self.conf) | ||
511 | 138 | self.conf.set('vm.image', self.get_image_id()) | 240 | self.conf.set('vm.image', self.get_image_id()) |
512 | 139 | self.conf.set('vm.apt_get.update.timeouts', '0.1, 0.1, 0.1') | 241 | self.conf.set('vm.apt_get.update.timeouts', '0.1, 0.1, 0.1') |
513 | 242 | tb = testbed.TestBed(self.conf) | ||
514 | 140 | 243 | ||
515 | 141 | def failing_update(): | 244 | def failing_update(): |
516 | 142 | class Proc(object): | 245 | class Proc(object): |
517 | @@ -151,3 +254,25 @@ | |||
518 | 151 | tb.safe_apt_get_update() | 254 | tb.safe_apt_get_update() |
519 | 152 | self.assertEqual('apt-get update never succeeded', | 255 | self.assertEqual('apt-get update never succeeded', |
520 | 153 | unicode(cm.exception)) | 256 | unicode(cm.exception)) |
521 | 257 | |||
522 | 258 | def test_wait_for_instance_fails(self): | ||
523 | 259 | self.conf.set('vm.image', self.get_image_id()) | ||
524 | 260 | # Force a 0 timeout so the instance can't finish booting | ||
525 | 261 | self.conf.set('vm.nova.boot_timeout', '0') | ||
526 | 262 | tb = testbed.TestBed(self.conf) | ||
527 | 263 | self.addCleanup(tb.teardown) | ||
528 | 264 | with self.assertRaises(testbed.TestBedException) as cm: | ||
529 | 265 | tb.setup() | ||
530 | 266 | self.assertEqual('Instance never came up (last status: BUILD)', | ||
531 | 267 | unicode(cm.exception)) | ||
532 | 268 | |||
533 | 269 | def test_wait_for_ip_fails(self): | ||
534 | 270 | self.conf.set('vm.image', self.get_image_id()) | ||
535 | 271 | # Force a 0 timeout so the instance never get an IP | ||
536 | 272 | self.conf.set('vm.nova.set_ip_timeout', '0') | ||
537 | 273 | tb = testbed.TestBed(self.conf) | ||
538 | 274 | self.addCleanup(tb.teardown) | ||
539 | 275 | with self.assertRaises(testbed.TestBedException) as cm: | ||
540 | 276 | tb.setup() | ||
541 | 277 | msg = 'Instance {} never provided an IP'.format(tb.instance.id) | ||
542 | 278 | self.assertEqual(msg, unicode(cm.exception)) | ||
543 | 154 | 279 | ||
544 | === modified file 'test_runner/tstrun/tests/test_worker.py' | |||
545 | --- test_runner/tstrun/tests/test_worker.py 2014-07-31 18:29:40 +0000 | |||
546 | +++ test_runner/tstrun/tests/test_worker.py 2014-08-06 13:17:27 +0000 | |||
547 | @@ -147,9 +147,9 @@ | |||
548 | 147 | worker = run_worker.TestRunnerWorker(self.ds_factory) | 147 | worker = run_worker.TestRunnerWorker(self.ds_factory) |
549 | 148 | 148 | ||
550 | 149 | def broken_teardown(test_bed): | 149 | def broken_teardown(test_bed): |
552 | 150 | self.addCleanup(test_bed.nova.servers.delete, test_bed.instance.id) | 150 | self.addCleanup(test_bed.nova.delete_server, test_bed.instance.id) |
553 | 151 | if test_bed.floating_ip is not None: | 151 | if test_bed.floating_ip is not None: |
555 | 152 | self.addCleanup(test_bed.nova.floating_ips.delete, | 152 | self.addCleanup(test_bed.nova.delete_floating_ip, |
556 | 153 | test_bed.floating_ip) | 153 | test_bed.floating_ip) |
557 | 154 | raise AssertionError('Boom !') | 154 | raise AssertionError('Boom !') |
558 | 155 | 155 |
FAILED: Continuous integration, rev:745 s-jenkins. ubuntu- ci:8080/ job/uci- engine- ci/1245/
http://
Executed test runs:
Click here to trigger a rebuild: s-jenkins. ubuntu- ci:8080/ job/uci- engine- ci/1245/ rebuild
http://