Merge lp:~tribaal/charms/trusty/nova-compute/sync-charm-helpers into lp:~openstack-charmers-archive/charms/trusty/nova-compute/trunk
- Trusty Tahr (14.04)
- sync-charm-helpers
- Merge into trunk
Proposed by
Chris Glass
Status: | Merged |
---|---|
Merged at revision: | 61 |
Proposed branch: | lp:~tribaal/charms/trusty/nova-compute/sync-charm-helpers |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/nova-compute/trunk |
Diff against target: |
541 lines (+263/-74) 8 files modified
hooks/charmhelpers/contrib/openstack/context.py (+1/-1) hooks/charmhelpers/contrib/openstack/neutron.py (+17/-1) hooks/charmhelpers/contrib/openstack/utils.py (+8/-1) hooks/charmhelpers/contrib/storage/linux/lvm.py (+1/-1) hooks/charmhelpers/contrib/storage/linux/utils.py (+28/-5) hooks/charmhelpers/core/hookenv.py (+98/-1) hooks/charmhelpers/core/host.py (+14/-0) hooks/charmhelpers/fetch/__init__.py (+96/-64) |
To merge this branch: | bzr merge lp:~tribaal/charms/trusty/nova-compute/sync-charm-helpers |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
OpenStack Charmers | Pending | ||
Review via email: mp+220028@code.launchpad.net |
Commit message
Description of the change
This branch syncs back the changes from charm-helpers as of http://
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'hooks/charmhelpers/contrib/openstack/context.py' |
2 | --- hooks/charmhelpers/contrib/openstack/context.py 2014-04-04 16:45:38 +0000 |
3 | +++ hooks/charmhelpers/contrib/openstack/context.py 2014-05-19 12:48:55 +0000 |
4 | @@ -570,7 +570,7 @@ |
5 | |
6 | if self.plugin == 'ovs': |
7 | ctxt.update(self.ovs_ctxt()) |
8 | - elif self.plugin == 'nvp': |
9 | + elif self.plugin in ['nvp', 'nsx']: |
10 | ctxt.update(self.nvp_ctxt()) |
11 | |
12 | alchemy_flags = config('neutron-alchemy-flags') |
13 | |
14 | === modified file 'hooks/charmhelpers/contrib/openstack/neutron.py' |
15 | --- hooks/charmhelpers/contrib/openstack/neutron.py 2014-04-04 16:45:38 +0000 |
16 | +++ hooks/charmhelpers/contrib/openstack/neutron.py 2014-05-19 12:48:55 +0000 |
17 | @@ -114,14 +114,30 @@ |
18 | 'server_packages': ['neutron-server', |
19 | 'neutron-plugin-nicira'], |
20 | 'server_services': ['neutron-server'] |
21 | + }, |
22 | + 'nsx': { |
23 | + 'config': '/etc/neutron/plugins/vmware/nsx.ini', |
24 | + 'driver': 'vmware', |
25 | + 'contexts': [ |
26 | + context.SharedDBContext(user=config('neutron-database-user'), |
27 | + database=config('neutron-database'), |
28 | + relation_prefix='neutron', |
29 | + ssl_dir=NEUTRON_CONF_DIR)], |
30 | + 'services': [], |
31 | + 'packages': [], |
32 | + 'server_packages': ['neutron-server', |
33 | + 'neutron-plugin-vmware'], |
34 | + 'server_services': ['neutron-server'] |
35 | } |
36 | } |
37 | - # NOTE: patch in ml2 plugin for icehouse onwards |
38 | if release >= 'icehouse': |
39 | + # NOTE: patch in ml2 plugin for icehouse onwards |
40 | plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini' |
41 | plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin' |
42 | plugins['ovs']['server_packages'] = ['neutron-server', |
43 | 'neutron-plugin-ml2'] |
44 | + # NOTE: patch in vmware renames nvp->nsx for icehouse onwards |
45 | + plugins['nvp'] = plugins['nsx'] |
46 | return plugins |
47 | |
48 | |
49 | |
50 | === modified file 'hooks/charmhelpers/contrib/openstack/utils.py' |
51 | --- hooks/charmhelpers/contrib/openstack/utils.py 2014-04-10 16:09:53 +0000 |
52 | +++ hooks/charmhelpers/contrib/openstack/utils.py 2014-05-19 12:48:55 +0000 |
53 | @@ -131,6 +131,11 @@ |
54 | def get_os_codename_package(package, fatal=True): |
55 | '''Derive OpenStack release codename from an installed package.''' |
56 | apt.init() |
57 | + |
58 | + # Tell apt to build an in-memory cache to prevent race conditions (if |
59 | + # another process is already building the cache). |
60 | + apt.config.set("Dir::Cache::pkgcache", "") |
61 | + |
62 | cache = apt.Cache() |
63 | |
64 | try: |
65 | @@ -183,7 +188,7 @@ |
66 | if cname == codename: |
67 | return version |
68 | #e = "Could not determine OpenStack version for package: %s" % pkg |
69 | - #error_out(e) |
70 | + # error_out(e) |
71 | |
72 | |
73 | os_rel = None |
74 | @@ -401,6 +406,8 @@ |
75 | rtype = 'PTR' |
76 | elif isinstance(address, basestring): |
77 | rtype = 'A' |
78 | + else: |
79 | + return None |
80 | |
81 | answers = dns.resolver.query(address, rtype) |
82 | if answers: |
83 | |
84 | === modified file 'hooks/charmhelpers/contrib/storage/linux/lvm.py' |
85 | --- hooks/charmhelpers/contrib/storage/linux/lvm.py 2013-07-19 02:37:30 +0000 |
86 | +++ hooks/charmhelpers/contrib/storage/linux/lvm.py 2014-05-19 12:48:55 +0000 |
87 | @@ -62,7 +62,7 @@ |
88 | pvd = check_output(['pvdisplay', block_device]).splitlines() |
89 | for l in pvd: |
90 | if l.strip().startswith('VG Name'): |
91 | - vg = ' '.join(l.split()).split(' ').pop() |
92 | + vg = ' '.join(l.strip().split()[2:]) |
93 | return vg |
94 | |
95 | |
96 | |
97 | === modified file 'hooks/charmhelpers/contrib/storage/linux/utils.py' |
98 | --- hooks/charmhelpers/contrib/storage/linux/utils.py 2014-04-04 16:45:38 +0000 |
99 | +++ hooks/charmhelpers/contrib/storage/linux/utils.py 2014-05-19 12:48:55 +0000 |
100 | @@ -1,8 +1,11 @@ |
101 | -from os import stat |
102 | +import os |
103 | +import re |
104 | from stat import S_ISBLK |
105 | |
106 | from subprocess import ( |
107 | - check_call |
108 | + check_call, |
109 | + check_output, |
110 | + call |
111 | ) |
112 | |
113 | |
114 | @@ -12,7 +15,9 @@ |
115 | |
116 | :returns: boolean: True if path is a block device, False if not. |
117 | ''' |
118 | - return S_ISBLK(stat(path).st_mode) |
119 | + if not os.path.exists(path): |
120 | + return False |
121 | + return S_ISBLK(os.stat(path).st_mode) |
122 | |
123 | |
124 | def zap_disk(block_device): |
125 | @@ -22,5 +27,23 @@ |
126 | |
127 | :param block_device: str: Full path of block device to clean. |
128 | ''' |
129 | - check_call(['sgdisk', '--zap-all', '--clear', |
130 | - '--mbrtogpt', block_device]) |
131 | + # sometimes sgdisk exits non-zero; this is OK, dd will clean up |
132 | + call(['sgdisk', '--zap-all', '--mbrtogpt', |
133 | + '--clear', block_device]) |
134 | + dev_end = check_output(['blockdev', '--getsz', block_device]) |
135 | + gpt_end = int(dev_end.split()[0]) - 100 |
136 | + check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), |
137 | + 'bs=1M', 'count=1']) |
138 | + check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), |
139 | + 'bs=512', 'count=100', 'seek=%s' % (gpt_end)]) |
140 | + |
141 | +def is_device_mounted(device): |
142 | + '''Given a device path, return True if that device is mounted, and False |
143 | + if it isn't. |
144 | + |
145 | + :param device: str: Full path of the device to check. |
146 | + :returns: boolean: True if the path represents a mounted device, False if |
147 | + it doesn't. |
148 | + ''' |
149 | + out = check_output(['mount']) |
150 | + return bool(re.search(device + r"[0-9]+\b", out)) |
151 | |
152 | === modified file 'hooks/charmhelpers/core/hookenv.py' |
153 | --- hooks/charmhelpers/core/hookenv.py 2014-03-27 11:08:20 +0000 |
154 | +++ hooks/charmhelpers/core/hookenv.py 2014-05-19 12:48:55 +0000 |
155 | @@ -155,6 +155,100 @@ |
156 | return os.path.basename(sys.argv[0]) |
157 | |
158 | |
159 | +class Config(dict): |
160 | + """A Juju charm config dictionary that can write itself to |
161 | + disk (as json) and track which values have changed since |
162 | + the previous hook invocation. |
163 | + |
164 | + Do not instantiate this object directly - instead call |
165 | + ``hookenv.config()`` |
166 | + |
167 | + Example usage:: |
168 | + |
169 | + >>> # inside a hook |
170 | + >>> from charmhelpers.core import hookenv |
171 | + >>> config = hookenv.config() |
172 | + >>> config['foo'] |
173 | + 'bar' |
174 | + >>> config['mykey'] = 'myval' |
175 | + >>> config.save() |
176 | + |
177 | + |
178 | + >>> # user runs `juju set mycharm foo=baz` |
179 | + >>> # now we're inside subsequent config-changed hook |
180 | + >>> config = hookenv.config() |
181 | + >>> config['foo'] |
182 | + 'baz' |
183 | + >>> # test to see if this val has changed since last hook |
184 | + >>> config.changed('foo') |
185 | + True |
186 | + >>> # what was the previous value? |
187 | + >>> config.previous('foo') |
188 | + 'bar' |
189 | + >>> # keys/values that we add are preserved across hooks |
190 | + >>> config['mykey'] |
191 | + 'myval' |
192 | + >>> # don't forget to save at the end of hook! |
193 | + >>> config.save() |
194 | + |
195 | + """ |
196 | + CONFIG_FILE_NAME = '.juju-persistent-config' |
197 | + |
198 | + def __init__(self, *args, **kw): |
199 | + super(Config, self).__init__(*args, **kw) |
200 | + self._prev_dict = None |
201 | + self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) |
202 | + if os.path.exists(self.path): |
203 | + self.load_previous() |
204 | + |
205 | + def load_previous(self, path=None): |
206 | + """Load previous copy of config from disk so that current values |
207 | + can be compared to previous values. |
208 | + |
209 | + :param path: |
210 | + |
211 | + File path from which to load the previous config. If `None`, |
212 | + config is loaded from the default location. If `path` is |
213 | + specified, subsequent `save()` calls will write to the same |
214 | + path. |
215 | + |
216 | + """ |
217 | + self.path = path or self.path |
218 | + with open(self.path) as f: |
219 | + self._prev_dict = json.load(f) |
220 | + |
221 | + def changed(self, key): |
222 | + """Return true if the value for this key has changed since |
223 | + the last save. |
224 | + |
225 | + """ |
226 | + if self._prev_dict is None: |
227 | + return True |
228 | + return self.previous(key) != self.get(key) |
229 | + |
230 | + def previous(self, key): |
231 | + """Return previous value for this key, or None if there |
232 | + is no "previous" value. |
233 | + |
234 | + """ |
235 | + if self._prev_dict: |
236 | + return self._prev_dict.get(key) |
237 | + return None |
238 | + |
239 | + def save(self): |
240 | + """Save this config to disk. |
241 | + |
242 | + Preserves items in _prev_dict that do not exist in self. |
243 | + |
244 | + """ |
245 | + if self._prev_dict: |
246 | + for k, v in self._prev_dict.iteritems(): |
247 | + if k not in self: |
248 | + self[k] = v |
249 | + with open(self.path, 'w') as f: |
250 | + json.dump(self, f) |
251 | + |
252 | + |
253 | @cached |
254 | def config(scope=None): |
255 | """Juju charm configuration""" |
256 | @@ -163,7 +257,10 @@ |
257 | config_cmd_line.append(scope) |
258 | config_cmd_line.append('--format=json') |
259 | try: |
260 | - return json.loads(subprocess.check_output(config_cmd_line)) |
261 | + config_data = json.loads(subprocess.check_output(config_cmd_line)) |
262 | + if scope is not None: |
263 | + return config_data |
264 | + return Config(config_data) |
265 | except ValueError: |
266 | return None |
267 | |
268 | |
269 | === modified file 'hooks/charmhelpers/core/host.py' |
270 | --- hooks/charmhelpers/core/host.py 2014-04-04 16:45:38 +0000 |
271 | +++ hooks/charmhelpers/core/host.py 2014-05-19 12:48:55 +0000 |
272 | @@ -12,6 +12,7 @@ |
273 | import string |
274 | import subprocess |
275 | import hashlib |
276 | +import apt_pkg |
277 | |
278 | from collections import OrderedDict |
279 | |
280 | @@ -295,3 +296,16 @@ |
281 | if 'link/ether' in words: |
282 | hwaddr = words[words.index('link/ether') + 1] |
283 | return hwaddr |
284 | + |
285 | + |
286 | +def cmp_pkgrevno(package, revno, pkgcache=None): |
287 | + '''Compare supplied revno with the revno of the installed package |
288 | + 1 => Installed revno is greater than supplied arg |
289 | + 0 => Installed revno is the same as supplied arg |
290 | + -1 => Installed revno is less than supplied arg |
291 | + ''' |
292 | + if not pkgcache: |
293 | + apt_pkg.init() |
294 | + pkgcache = apt_pkg.Cache() |
295 | + pkg = pkgcache[package] |
296 | + return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) |
297 | |
298 | === modified file 'hooks/charmhelpers/fetch/__init__.py' |
299 | --- hooks/charmhelpers/fetch/__init__.py 2014-04-24 17:34:11 +0000 |
300 | +++ hooks/charmhelpers/fetch/__init__.py 2014-05-19 12:48:55 +0000 |
301 | @@ -1,4 +1,5 @@ |
302 | import importlib |
303 | +import time |
304 | from yaml import safe_load |
305 | from charmhelpers.core.host import ( |
306 | lsb_release |
307 | @@ -15,6 +16,7 @@ |
308 | import apt_pkg |
309 | import os |
310 | |
311 | + |
312 | CLOUD_ARCHIVE = """# Ubuntu Cloud Archive |
313 | deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main |
314 | """ |
315 | @@ -56,10 +58,62 @@ |
316 | 'precise-proposed/icehouse': 'precise-proposed/icehouse', |
317 | } |
318 | |
319 | +# The order of this list is very important. Handlers should be listed in from |
320 | +# least- to most-specific URL matching. |
321 | +FETCH_HANDLERS = ( |
322 | + 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler', |
323 | + 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler', |
324 | +) |
325 | + |
326 | +APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. |
327 | +APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks. |
328 | +APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. |
329 | + |
330 | + |
331 | +class SourceConfigError(Exception): |
332 | + pass |
333 | + |
334 | + |
335 | +class UnhandledSource(Exception): |
336 | + pass |
337 | + |
338 | + |
339 | +class AptLockError(Exception): |
340 | + pass |
341 | + |
342 | + |
343 | +class BaseFetchHandler(object): |
344 | + |
345 | + """Base class for FetchHandler implementations in fetch plugins""" |
346 | + |
347 | + def can_handle(self, source): |
348 | + """Returns True if the source can be handled. Otherwise returns |
349 | + a string explaining why it cannot""" |
350 | + return "Wrong source type" |
351 | + |
352 | + def install(self, source): |
353 | + """Try to download and unpack the source. Return the path to the |
354 | + unpacked files or raise UnhandledSource.""" |
355 | + raise UnhandledSource("Wrong source type {}".format(source)) |
356 | + |
357 | + def parse_url(self, url): |
358 | + return urlparse(url) |
359 | + |
360 | + def base_url(self, url): |
361 | + """Return url without querystring or fragment""" |
362 | + parts = list(self.parse_url(url)) |
363 | + parts[4:] = ['' for i in parts[4:]] |
364 | + return urlunparse(parts) |
365 | + |
366 | |
367 | def filter_installed_packages(packages): |
368 | """Returns a list of packages that require installation""" |
369 | apt_pkg.init() |
370 | + |
371 | + # Tell apt to build an in-memory cache to prevent race conditions (if |
372 | + # another process is already building the cache). |
373 | + apt_pkg.config.set("Dir::Cache::pkgcache", "") |
374 | + |
375 | cache = apt_pkg.Cache() |
376 | _pkgs = [] |
377 | for package in packages: |
378 | @@ -87,14 +141,7 @@ |
379 | cmd.extend(packages) |
380 | log("Installing {} with options: {}".format(packages, |
381 | options)) |
382 | - env = os.environ.copy() |
383 | - if 'DEBIAN_FRONTEND' not in env: |
384 | - env['DEBIAN_FRONTEND'] = 'noninteractive' |
385 | - |
386 | - if fatal: |
387 | - subprocess.check_call(cmd, env=env) |
388 | - else: |
389 | - subprocess.call(cmd, env=env) |
390 | + _run_apt_command(cmd, fatal) |
391 | |
392 | |
393 | def apt_upgrade(options=None, fatal=False, dist=False): |
394 | @@ -109,24 +156,13 @@ |
395 | else: |
396 | cmd.append('upgrade') |
397 | log("Upgrading with options: {}".format(options)) |
398 | - |
399 | - env = os.environ.copy() |
400 | - if 'DEBIAN_FRONTEND' not in env: |
401 | - env['DEBIAN_FRONTEND'] = 'noninteractive' |
402 | - |
403 | - if fatal: |
404 | - subprocess.check_call(cmd, env=env) |
405 | - else: |
406 | - subprocess.call(cmd, env=env) |
407 | + _run_apt_command(cmd, fatal) |
408 | |
409 | |
410 | def apt_update(fatal=False): |
411 | """Update local apt cache""" |
412 | cmd = ['apt-get', 'update'] |
413 | - if fatal: |
414 | - subprocess.check_call(cmd) |
415 | - else: |
416 | - subprocess.call(cmd) |
417 | + _run_apt_command(cmd, fatal) |
418 | |
419 | |
420 | def apt_purge(packages, fatal=False): |
421 | @@ -137,10 +173,7 @@ |
422 | else: |
423 | cmd.extend(packages) |
424 | log("Purging {}".format(packages)) |
425 | - if fatal: |
426 | - subprocess.check_call(cmd) |
427 | - else: |
428 | - subprocess.call(cmd) |
429 | + _run_apt_command(cmd, fatal) |
430 | |
431 | |
432 | def apt_hold(packages, fatal=False): |
433 | @@ -151,6 +184,7 @@ |
434 | else: |
435 | cmd.extend(packages) |
436 | log("Holding {}".format(packages)) |
437 | + |
438 | if fatal: |
439 | subprocess.check_call(cmd) |
440 | else: |
441 | @@ -188,10 +222,6 @@ |
442 | key]) |
443 | |
444 | |
445 | -class SourceConfigError(Exception): |
446 | - pass |
447 | - |
448 | - |
449 | def configure_sources(update=False, |
450 | sources_var='install_sources', |
451 | keys_var='install_keys'): |
452 | @@ -224,17 +254,6 @@ |
453 | if update: |
454 | apt_update(fatal=True) |
455 | |
456 | -# The order of this list is very important. Handlers should be listed in from |
457 | -# least- to most-specific URL matching. |
458 | -FETCH_HANDLERS = ( |
459 | - 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler', |
460 | - 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler', |
461 | -) |
462 | - |
463 | - |
464 | -class UnhandledSource(Exception): |
465 | - pass |
466 | - |
467 | |
468 | def install_remote(source): |
469 | """ |
470 | @@ -265,30 +284,6 @@ |
471 | return install_remote(source) |
472 | |
473 | |
474 | -class BaseFetchHandler(object): |
475 | - |
476 | - """Base class for FetchHandler implementations in fetch plugins""" |
477 | - |
478 | - def can_handle(self, source): |
479 | - """Returns True if the source can be handled. Otherwise returns |
480 | - a string explaining why it cannot""" |
481 | - return "Wrong source type" |
482 | - |
483 | - def install(self, source): |
484 | - """Try to download and unpack the source. Return the path to the |
485 | - unpacked files or raise UnhandledSource.""" |
486 | - raise UnhandledSource("Wrong source type {}".format(source)) |
487 | - |
488 | - def parse_url(self, url): |
489 | - return urlparse(url) |
490 | - |
491 | - def base_url(self, url): |
492 | - """Return url without querystring or fragment""" |
493 | - parts = list(self.parse_url(url)) |
494 | - parts[4:] = ['' for i in parts[4:]] |
495 | - return urlunparse(parts) |
496 | - |
497 | - |
498 | def plugins(fetch_handlers=None): |
499 | if not fetch_handlers: |
500 | fetch_handlers = FETCH_HANDLERS |
501 | @@ -306,3 +301,40 @@ |
502 | log("FetchHandler {} not found, skipping plugin".format( |
503 | handler_name)) |
504 | return plugin_list |
505 | + |
506 | + |
507 | +def _run_apt_command(cmd, fatal=False): |
508 | + """ |
509 | + Run an APT command, checking output and retrying if the fatal flag is set |
510 | + to True. |
511 | + |
512 | + :param: cmd: str: The apt command to run. |
513 | + :param: fatal: bool: Whether the command's output should be checked and |
514 | + retried. |
515 | + """ |
516 | + env = os.environ.copy() |
517 | + |
518 | + if 'DEBIAN_FRONTEND' not in env: |
519 | + env['DEBIAN_FRONTEND'] = 'noninteractive' |
520 | + |
521 | + if fatal: |
522 | + retry_count = 0 |
523 | + result = None |
524 | + |
525 | + # If the command is considered "fatal", we need to retry if the apt |
526 | + # lock was not acquired. |
527 | + |
528 | + while result is None or result == APT_NO_LOCK: |
529 | + try: |
530 | + result = subprocess.check_call(cmd, env=env) |
531 | + except subprocess.CalledProcessError, e: |
532 | + retry_count = retry_count + 1 |
533 | + if retry_count > APT_NO_LOCK_RETRY_COUNT: |
534 | + raise |
535 | + result = e.returncode |
536 | + log("Couldn't acquire DPKG lock. Will retry in {} seconds." |
537 | + "".format(APT_NO_LOCK_RETRY_DELAY)) |
538 | + time.sleep(APT_NO_LOCK_RETRY_DELAY) |
539 | + |
540 | + else: |
541 | + subprocess.call(cmd, env=env) |