Merge lp:~gnuoy/charms/trusty/ceilometer-agent/stable-charm-sync into lp:~openstack-charmers-archive/charms/trusty/ceilometer-agent/trunk
- Trusty Tahr (14.04)
- stable-charm-sync
- Merge into trunk
Proposed by
Liam Young
Status: | Merged |
---|---|
Merged at revision: | 39 |
Proposed branch: | lp:~gnuoy/charms/trusty/ceilometer-agent/stable-charm-sync |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/ceilometer-agent/trunk |
Diff against target: |
843 lines (+430/-94) 13 files modified
.bzrignore (+2/-0) Makefile (+8/-2) charm-helpers.yaml (+1/-1) hooks/charmhelpers/contrib/openstack/context.py (+10/-5) hooks/charmhelpers/contrib/openstack/neutron.py (+17/-1) hooks/charmhelpers/contrib/openstack/utils.py (+6/-4) hooks/charmhelpers/contrib/storage/linux/lvm.py (+1/-1) hooks/charmhelpers/contrib/storage/linux/utils.py (+32/-5) hooks/charmhelpers/core/fstab.py (+116/-0) hooks/charmhelpers/core/hookenv.py (+98/-1) hooks/charmhelpers/core/host.py (+34/-6) hooks/charmhelpers/fetch/__init__.py (+103/-67) hooks/charmhelpers/fetch/bzrurl.py (+2/-1) |
To merge this branch: | bzr merge lp:~gnuoy/charms/trusty/ceilometer-agent/stable-charm-sync |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Chris Glass (community) | Approve | ||
charmers | Pending | ||
Review via email: mp+230672@code.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
- 40. By Liam Young
-
Resync charmhelpers for cache bug fixes
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === added file '.bzrignore' |
2 | --- .bzrignore 1970-01-01 00:00:00 +0000 |
3 | +++ .bzrignore 2014-08-27 07:08:22 +0000 |
4 | @@ -0,0 +1,2 @@ |
5 | +bin |
6 | +.coverage |
7 | |
8 | === modified file 'Makefile' |
9 | --- Makefile 2013-10-14 16:10:30 +0000 |
10 | +++ Makefile 2014-08-27 07:08:22 +0000 |
11 | @@ -1,11 +1,17 @@ |
12 | #!/usr/bin/make |
13 | +PYTHON := /usr/bin/env python |
14 | |
15 | lint: |
16 | @flake8 --exclude hooks/charmhelpers hooks |
17 | @charm proof |
18 | |
19 | -sync: |
20 | - @charm-helper-sync -c charm-helpers.yaml |
21 | +bin/charm_helpers_sync.py: |
22 | + @mkdir -p bin |
23 | + @bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \ |
24 | + > bin/charm_helpers_sync.py |
25 | + |
26 | +sync: bin/charm_helpers_sync.py |
27 | + @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers.yaml |
28 | |
29 | test: |
30 | @$(PYTHON) /usr/bin/nosetests --nologcapture --with-coverage unit_tests |
31 | |
32 | === modified file 'charm-helpers.yaml' |
33 | --- charm-helpers.yaml 2014-03-25 17:03:52 +0000 |
34 | +++ charm-helpers.yaml 2014-08-27 07:08:22 +0000 |
35 | @@ -1,4 +1,4 @@ |
36 | -branch: lp:charm-helpers |
37 | +branch: lp:~openstack-charmers/charm-helpers/stable |
38 | destination: hooks/charmhelpers |
39 | include: |
40 | - core |
41 | |
42 | === modified file 'hooks/charmhelpers/contrib/openstack/context.py' |
43 | --- hooks/charmhelpers/contrib/openstack/context.py 2014-04-10 15:36:48 +0000 |
44 | +++ hooks/charmhelpers/contrib/openstack/context.py 2014-08-27 07:08:22 +0000 |
45 | @@ -24,6 +24,7 @@ |
46 | unit_get, |
47 | unit_private_ip, |
48 | ERROR, |
49 | + INFO |
50 | ) |
51 | |
52 | from charmhelpers.contrib.hahelpers.cluster import ( |
53 | @@ -570,7 +571,7 @@ |
54 | |
55 | if self.plugin == 'ovs': |
56 | ctxt.update(self.ovs_ctxt()) |
57 | - elif self.plugin == 'nvp': |
58 | + elif self.plugin in ['nvp', 'nsx']: |
59 | ctxt.update(self.nvp_ctxt()) |
60 | |
61 | alchemy_flags = config('neutron-alchemy-flags') |
62 | @@ -657,7 +658,7 @@ |
63 | self.interface = interface |
64 | |
65 | def __call__(self): |
66 | - ctxt = {} |
67 | + ctxt = {'sections': {}} |
68 | for rid in relation_ids(self.interface): |
69 | for unit in related_units(rid): |
70 | sub_config = relation_get('subordinate_configuration', |
71 | @@ -683,10 +684,14 @@ |
72 | |
73 | sub_config = sub_config[self.config_file] |
74 | for k, v in sub_config.iteritems(): |
75 | - ctxt[k] = v |
76 | + if k == 'sections': |
77 | + for section, config_dict in v.iteritems(): |
78 | + log("adding section '%s'" % (section)) |
79 | + ctxt[k][section] = config_dict |
80 | + else: |
81 | + ctxt[k] = v |
82 | |
83 | - if not ctxt: |
84 | - ctxt['sections'] = {} |
85 | + log("%d section(s) found" % (len(ctxt['sections'])), level=INFO) |
86 | |
87 | return ctxt |
88 | |
89 | |
90 | === modified file 'hooks/charmhelpers/contrib/openstack/neutron.py' |
91 | --- hooks/charmhelpers/contrib/openstack/neutron.py 2014-03-27 10:19:40 +0000 |
92 | +++ hooks/charmhelpers/contrib/openstack/neutron.py 2014-08-27 07:08:22 +0000 |
93 | @@ -114,14 +114,30 @@ |
94 | 'server_packages': ['neutron-server', |
95 | 'neutron-plugin-nicira'], |
96 | 'server_services': ['neutron-server'] |
97 | + }, |
98 | + 'nsx': { |
99 | + 'config': '/etc/neutron/plugins/vmware/nsx.ini', |
100 | + 'driver': 'vmware', |
101 | + 'contexts': [ |
102 | + context.SharedDBContext(user=config('neutron-database-user'), |
103 | + database=config('neutron-database'), |
104 | + relation_prefix='neutron', |
105 | + ssl_dir=NEUTRON_CONF_DIR)], |
106 | + 'services': [], |
107 | + 'packages': [], |
108 | + 'server_packages': ['neutron-server', |
109 | + 'neutron-plugin-vmware'], |
110 | + 'server_services': ['neutron-server'] |
111 | } |
112 | } |
113 | - # NOTE: patch in ml2 plugin for icehouse onwards |
114 | if release >= 'icehouse': |
115 | + # NOTE: patch in ml2 plugin for icehouse onwards |
116 | plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini' |
117 | plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin' |
118 | plugins['ovs']['server_packages'] = ['neutron-server', |
119 | 'neutron-plugin-ml2'] |
120 | + # NOTE: patch in vmware renames nvp->nsx for icehouse onwards |
121 | + plugins['nvp'] = plugins['nsx'] |
122 | return plugins |
123 | |
124 | |
125 | |
126 | === modified file 'hooks/charmhelpers/contrib/openstack/utils.py' |
127 | --- hooks/charmhelpers/contrib/openstack/utils.py 2014-04-10 15:36:48 +0000 |
128 | +++ hooks/charmhelpers/contrib/openstack/utils.py 2014-08-27 07:08:22 +0000 |
129 | @@ -24,7 +24,7 @@ |
130 | ) |
131 | |
132 | from charmhelpers.core.host import lsb_release, mounts, umount |
133 | -from charmhelpers.fetch import apt_install |
134 | +from charmhelpers.fetch import apt_install, apt_cache |
135 | from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk |
136 | from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device |
137 | |
138 | @@ -130,8 +130,8 @@ |
139 | |
140 | def get_os_codename_package(package, fatal=True): |
141 | '''Derive OpenStack release codename from an installed package.''' |
142 | - apt.init() |
143 | - cache = apt.Cache() |
144 | + |
145 | + cache = apt_cache() |
146 | |
147 | try: |
148 | pkg = cache[package] |
149 | @@ -183,7 +183,7 @@ |
150 | if cname == codename: |
151 | return version |
152 | #e = "Could not determine OpenStack version for package: %s" % pkg |
153 | - #error_out(e) |
154 | + # error_out(e) |
155 | |
156 | |
157 | os_rel = None |
158 | @@ -401,6 +401,8 @@ |
159 | rtype = 'PTR' |
160 | elif isinstance(address, basestring): |
161 | rtype = 'A' |
162 | + else: |
163 | + return None |
164 | |
165 | answers = dns.resolver.query(address, rtype) |
166 | if answers: |
167 | |
168 | === modified file 'hooks/charmhelpers/contrib/storage/linux/lvm.py' |
169 | --- hooks/charmhelpers/contrib/storage/linux/lvm.py 2013-11-06 01:23:03 +0000 |
170 | +++ hooks/charmhelpers/contrib/storage/linux/lvm.py 2014-08-27 07:08:22 +0000 |
171 | @@ -62,7 +62,7 @@ |
172 | pvd = check_output(['pvdisplay', block_device]).splitlines() |
173 | for l in pvd: |
174 | if l.strip().startswith('VG Name'): |
175 | - vg = ' '.join(l.split()).split(' ').pop() |
176 | + vg = ' '.join(l.strip().split()[2:]) |
177 | return vg |
178 | |
179 | |
180 | |
181 | === modified file 'hooks/charmhelpers/contrib/storage/linux/utils.py' |
182 | --- hooks/charmhelpers/contrib/storage/linux/utils.py 2014-03-20 13:38:40 +0000 |
183 | +++ hooks/charmhelpers/contrib/storage/linux/utils.py 2014-08-27 07:08:22 +0000 |
184 | @@ -1,8 +1,11 @@ |
185 | -from os import stat |
186 | +import os |
187 | +import re |
188 | from stat import S_ISBLK |
189 | |
190 | from subprocess import ( |
191 | - check_call |
192 | + check_call, |
193 | + check_output, |
194 | + call |
195 | ) |
196 | |
197 | |
198 | @@ -12,7 +15,9 @@ |
199 | |
200 | :returns: boolean: True if path is a block device, False if not. |
201 | ''' |
202 | - return S_ISBLK(stat(path).st_mode) |
203 | + if not os.path.exists(path): |
204 | + return False |
205 | + return S_ISBLK(os.stat(path).st_mode) |
206 | |
207 | |
208 | def zap_disk(block_device): |
209 | @@ -22,5 +27,27 @@ |
210 | |
211 | :param block_device: str: Full path of block device to clean. |
212 | ''' |
213 | - check_call(['sgdisk', '--zap-all', '--clear', |
214 | - '--mbrtogpt', block_device]) |
215 | + # sometimes sgdisk exits non-zero; this is OK, dd will clean up |
216 | + call(['sgdisk', '--zap-all', '--mbrtogpt', |
217 | + '--clear', block_device]) |
218 | + dev_end = check_output(['blockdev', '--getsz', block_device]) |
219 | + gpt_end = int(dev_end.split()[0]) - 100 |
220 | + check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), |
221 | + 'bs=1M', 'count=1']) |
222 | + check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device), |
223 | + 'bs=512', 'count=100', 'seek=%s' % (gpt_end)]) |
224 | + |
225 | + |
226 | +def is_device_mounted(device): |
227 | + '''Given a device path, return True if that device is mounted, and False |
228 | + if it isn't. |
229 | + |
230 | + :param device: str: Full path of the device to check. |
231 | + :returns: boolean: True if the path represents a mounted device, False if |
232 | + it doesn't. |
233 | + ''' |
234 | + is_partition = bool(re.search(r".*[0-9]+\b", device)) |
235 | + out = check_output(['mount']) |
236 | + if is_partition: |
237 | + return bool(re.search(device + r"\b", out)) |
238 | + return bool(re.search(device + r"[0-9]+\b", out)) |
239 | |
240 | === added file 'hooks/charmhelpers/core/fstab.py' |
241 | --- hooks/charmhelpers/core/fstab.py 1970-01-01 00:00:00 +0000 |
242 | +++ hooks/charmhelpers/core/fstab.py 2014-08-27 07:08:22 +0000 |
243 | @@ -0,0 +1,116 @@ |
244 | +#!/usr/bin/env python |
245 | +# -*- coding: utf-8 -*- |
246 | + |
247 | +__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
248 | + |
249 | +import os |
250 | + |
251 | + |
252 | +class Fstab(file): |
253 | + """This class extends file in order to implement a file reader/writer |
254 | + for file `/etc/fstab` |
255 | + """ |
256 | + |
257 | + class Entry(object): |
258 | + """Entry class represents a non-comment line on the `/etc/fstab` file |
259 | + """ |
260 | + def __init__(self, device, mountpoint, filesystem, |
261 | + options, d=0, p=0): |
262 | + self.device = device |
263 | + self.mountpoint = mountpoint |
264 | + self.filesystem = filesystem |
265 | + |
266 | + if not options: |
267 | + options = "defaults" |
268 | + |
269 | + self.options = options |
270 | + self.d = d |
271 | + self.p = p |
272 | + |
273 | + def __eq__(self, o): |
274 | + return str(self) == str(o) |
275 | + |
276 | + def __str__(self): |
277 | + return "{} {} {} {} {} {}".format(self.device, |
278 | + self.mountpoint, |
279 | + self.filesystem, |
280 | + self.options, |
281 | + self.d, |
282 | + self.p) |
283 | + |
284 | + DEFAULT_PATH = os.path.join(os.path.sep, 'etc', 'fstab') |
285 | + |
286 | + def __init__(self, path=None): |
287 | + if path: |
288 | + self._path = path |
289 | + else: |
290 | + self._path = self.DEFAULT_PATH |
291 | + file.__init__(self, self._path, 'r+') |
292 | + |
293 | + def _hydrate_entry(self, line): |
294 | + # NOTE: use split with no arguments to split on any |
295 | + # whitespace including tabs |
296 | + return Fstab.Entry(*filter( |
297 | + lambda x: x not in ('', None), |
298 | + line.strip("\n").split())) |
299 | + |
300 | + @property |
301 | + def entries(self): |
302 | + self.seek(0) |
303 | + for line in self.readlines(): |
304 | + try: |
305 | + if not line.startswith("#"): |
306 | + yield self._hydrate_entry(line) |
307 | + except ValueError: |
308 | + pass |
309 | + |
310 | + def get_entry_by_attr(self, attr, value): |
311 | + for entry in self.entries: |
312 | + e_attr = getattr(entry, attr) |
313 | + if e_attr == value: |
314 | + return entry |
315 | + return None |
316 | + |
317 | + def add_entry(self, entry): |
318 | + if self.get_entry_by_attr('device', entry.device): |
319 | + return False |
320 | + |
321 | + self.write(str(entry) + '\n') |
322 | + self.truncate() |
323 | + return entry |
324 | + |
325 | + def remove_entry(self, entry): |
326 | + self.seek(0) |
327 | + |
328 | + lines = self.readlines() |
329 | + |
330 | + found = False |
331 | + for index, line in enumerate(lines): |
332 | + if not line.startswith("#"): |
333 | + if self._hydrate_entry(line) == entry: |
334 | + found = True |
335 | + break |
336 | + |
337 | + if not found: |
338 | + return False |
339 | + |
340 | + lines.remove(line) |
341 | + |
342 | + self.seek(0) |
343 | + self.write(''.join(lines)) |
344 | + self.truncate() |
345 | + return True |
346 | + |
347 | + @classmethod |
348 | + def remove_by_mountpoint(cls, mountpoint, path=None): |
349 | + fstab = cls(path=path) |
350 | + entry = fstab.get_entry_by_attr('mountpoint', mountpoint) |
351 | + if entry: |
352 | + return fstab.remove_entry(entry) |
353 | + return False |
354 | + |
355 | + @classmethod |
356 | + def add(cls, device, mountpoint, filesystem, options=None, path=None): |
357 | + return cls(path=path).add_entry(Fstab.Entry(device, |
358 | + mountpoint, filesystem, |
359 | + options=options)) |
360 | |
361 | === modified file 'hooks/charmhelpers/core/hookenv.py' |
362 | --- hooks/charmhelpers/core/hookenv.py 2014-03-20 13:38:40 +0000 |
363 | +++ hooks/charmhelpers/core/hookenv.py 2014-08-27 07:08:22 +0000 |
364 | @@ -155,6 +155,100 @@ |
365 | return os.path.basename(sys.argv[0]) |
366 | |
367 | |
368 | +class Config(dict): |
369 | + """A Juju charm config dictionary that can write itself to |
370 | + disk (as json) and track which values have changed since |
371 | + the previous hook invocation. |
372 | + |
373 | + Do not instantiate this object directly - instead call |
374 | + ``hookenv.config()`` |
375 | + |
376 | + Example usage:: |
377 | + |
378 | + >>> # inside a hook |
379 | + >>> from charmhelpers.core import hookenv |
380 | + >>> config = hookenv.config() |
381 | + >>> config['foo'] |
382 | + 'bar' |
383 | + >>> config['mykey'] = 'myval' |
384 | + >>> config.save() |
385 | + |
386 | + |
387 | + >>> # user runs `juju set mycharm foo=baz` |
388 | + >>> # now we're inside subsequent config-changed hook |
389 | + >>> config = hookenv.config() |
390 | + >>> config['foo'] |
391 | + 'baz' |
392 | + >>> # test to see if this val has changed since last hook |
393 | + >>> config.changed('foo') |
394 | + True |
395 | + >>> # what was the previous value? |
396 | + >>> config.previous('foo') |
397 | + 'bar' |
398 | + >>> # keys/values that we add are preserved across hooks |
399 | + >>> config['mykey'] |
400 | + 'myval' |
401 | + >>> # don't forget to save at the end of hook! |
402 | + >>> config.save() |
403 | + |
404 | + """ |
405 | + CONFIG_FILE_NAME = '.juju-persistent-config' |
406 | + |
407 | + def __init__(self, *args, **kw): |
408 | + super(Config, self).__init__(*args, **kw) |
409 | + self._prev_dict = None |
410 | + self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) |
411 | + if os.path.exists(self.path): |
412 | + self.load_previous() |
413 | + |
414 | + def load_previous(self, path=None): |
415 | + """Load previous copy of config from disk so that current values |
416 | + can be compared to previous values. |
417 | + |
418 | + :param path: |
419 | + |
420 | + File path from which to load the previous config. If `None`, |
421 | + config is loaded from the default location. If `path` is |
422 | + specified, subsequent `save()` calls will write to the same |
423 | + path. |
424 | + |
425 | + """ |
426 | + self.path = path or self.path |
427 | + with open(self.path) as f: |
428 | + self._prev_dict = json.load(f) |
429 | + |
430 | + def changed(self, key): |
431 | + """Return true if the value for this key has changed since |
432 | + the last save. |
433 | + |
434 | + """ |
435 | + if self._prev_dict is None: |
436 | + return True |
437 | + return self.previous(key) != self.get(key) |
438 | + |
439 | + def previous(self, key): |
440 | + """Return previous value for this key, or None if there |
441 | + is no "previous" value. |
442 | + |
443 | + """ |
444 | + if self._prev_dict: |
445 | + return self._prev_dict.get(key) |
446 | + return None |
447 | + |
448 | + def save(self): |
449 | + """Save this config to disk. |
450 | + |
451 | + Preserves items in _prev_dict that do not exist in self. |
452 | + |
453 | + """ |
454 | + if self._prev_dict: |
455 | + for k, v in self._prev_dict.iteritems(): |
456 | + if k not in self: |
457 | + self[k] = v |
458 | + with open(self.path, 'w') as f: |
459 | + json.dump(self, f) |
460 | + |
461 | + |
462 | @cached |
463 | def config(scope=None): |
464 | """Juju charm configuration""" |
465 | @@ -163,7 +257,10 @@ |
466 | config_cmd_line.append(scope) |
467 | config_cmd_line.append('--format=json') |
468 | try: |
469 | - return json.loads(subprocess.check_output(config_cmd_line)) |
470 | + config_data = json.loads(subprocess.check_output(config_cmd_line)) |
471 | + if scope is not None: |
472 | + return config_data |
473 | + return Config(config_data) |
474 | except ValueError: |
475 | return None |
476 | |
477 | |
478 | === modified file 'hooks/charmhelpers/core/host.py' |
479 | --- hooks/charmhelpers/core/host.py 2014-03-20 13:38:40 +0000 |
480 | +++ hooks/charmhelpers/core/host.py 2014-08-27 07:08:22 +0000 |
481 | @@ -12,10 +12,12 @@ |
482 | import string |
483 | import subprocess |
484 | import hashlib |
485 | +import apt_pkg |
486 | |
487 | from collections import OrderedDict |
488 | |
489 | from hookenv import log |
490 | +from fstab import Fstab |
491 | |
492 | |
493 | def service_start(service_name): |
494 | @@ -34,7 +36,8 @@ |
495 | |
496 | |
497 | def service_reload(service_name, restart_on_failure=False): |
498 | - """Reload a system service, optionally falling back to restart if reload fails""" |
499 | + """Reload a system service, optionally falling back to restart if |
500 | + reload fails""" |
501 | service_result = service('reload', service_name) |
502 | if not service_result and restart_on_failure: |
503 | service_result = service('restart', service_name) |
504 | @@ -143,7 +146,19 @@ |
505 | target.write(content) |
506 | |
507 | |
508 | -def mount(device, mountpoint, options=None, persist=False): |
509 | +def fstab_remove(mp): |
510 | + """Remove the given mountpoint entry from /etc/fstab |
511 | + """ |
512 | + return Fstab.remove_by_mountpoint(mp) |
513 | + |
514 | + |
515 | +def fstab_add(dev, mp, fs, options=None): |
516 | + """Adds the given device entry to the /etc/fstab file |
517 | + """ |
518 | + return Fstab.add(dev, mp, fs, options=options) |
519 | + |
520 | + |
521 | +def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"): |
522 | """Mount a filesystem at a particular mountpoint""" |
523 | cmd_args = ['mount'] |
524 | if options is not None: |
525 | @@ -154,9 +169,9 @@ |
526 | except subprocess.CalledProcessError, e: |
527 | log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output)) |
528 | return False |
529 | + |
530 | if persist: |
531 | - # TODO: update fstab |
532 | - pass |
533 | + return fstab_add(device, mountpoint, filesystem, options=options) |
534 | return True |
535 | |
536 | |
537 | @@ -168,9 +183,9 @@ |
538 | except subprocess.CalledProcessError, e: |
539 | log('Error unmounting {}\n{}'.format(mountpoint, e.output)) |
540 | return False |
541 | + |
542 | if persist: |
543 | - # TODO: update fstab |
544 | - pass |
545 | + return fstab_remove(mountpoint) |
546 | return True |
547 | |
548 | |
549 | @@ -295,3 +310,16 @@ |
550 | if 'link/ether' in words: |
551 | hwaddr = words[words.index('link/ether') + 1] |
552 | return hwaddr |
553 | + |
554 | + |
555 | +def cmp_pkgrevno(package, revno, pkgcache=None): |
556 | + '''Compare supplied revno with the revno of the installed package |
557 | + 1 => Installed revno is greater than supplied arg |
558 | + 0 => Installed revno is the same as supplied arg |
559 | + -1 => Installed revno is less than supplied arg |
560 | + ''' |
561 | + from charmhelpers.fetch import apt_cache |
562 | + if not pkgcache: |
563 | + pkgcache = apt_cache() |
564 | + pkg = pkgcache[package] |
565 | + return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) |
566 | |
567 | === modified file 'hooks/charmhelpers/fetch/__init__.py' |
568 | --- hooks/charmhelpers/fetch/__init__.py 2014-03-20 13:40:24 +0000 |
569 | +++ hooks/charmhelpers/fetch/__init__.py 2014-08-27 07:08:22 +0000 |
570 | @@ -1,4 +1,5 @@ |
571 | import importlib |
572 | +import time |
573 | from yaml import safe_load |
574 | from charmhelpers.core.host import ( |
575 | lsb_release |
576 | @@ -15,6 +16,7 @@ |
577 | import apt_pkg |
578 | import os |
579 | |
580 | + |
581 | CLOUD_ARCHIVE = """# Ubuntu Cloud Archive |
582 | deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main |
583 | """ |
584 | @@ -56,11 +58,58 @@ |
585 | 'precise-proposed/icehouse': 'precise-proposed/icehouse', |
586 | } |
587 | |
588 | +# The order of this list is very important. Handlers should be listed in from |
589 | +# least- to most-specific URL matching. |
590 | +FETCH_HANDLERS = ( |
591 | + 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler', |
592 | + 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler', |
593 | +) |
594 | + |
595 | +APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. |
596 | +APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks. |
597 | +APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. |
598 | + |
599 | + |
600 | +class SourceConfigError(Exception): |
601 | + pass |
602 | + |
603 | + |
604 | +class UnhandledSource(Exception): |
605 | + pass |
606 | + |
607 | + |
608 | +class AptLockError(Exception): |
609 | + pass |
610 | + |
611 | + |
612 | +class BaseFetchHandler(object): |
613 | + |
614 | + """Base class for FetchHandler implementations in fetch plugins""" |
615 | + |
616 | + def can_handle(self, source): |
617 | + """Returns True if the source can be handled. Otherwise returns |
618 | + a string explaining why it cannot""" |
619 | + return "Wrong source type" |
620 | + |
621 | + def install(self, source): |
622 | + """Try to download and unpack the source. Return the path to the |
623 | + unpacked files or raise UnhandledSource.""" |
624 | + raise UnhandledSource("Wrong source type {}".format(source)) |
625 | + |
626 | + def parse_url(self, url): |
627 | + return urlparse(url) |
628 | + |
629 | + def base_url(self, url): |
630 | + """Return url without querystring or fragment""" |
631 | + parts = list(self.parse_url(url)) |
632 | + parts[4:] = ['' for i in parts[4:]] |
633 | + return urlunparse(parts) |
634 | + |
635 | |
636 | def filter_installed_packages(packages): |
637 | """Returns a list of packages that require installation""" |
638 | - apt_pkg.init() |
639 | - cache = apt_pkg.Cache() |
640 | + |
641 | + cache = apt_cache() |
642 | _pkgs = [] |
643 | for package in packages: |
644 | try: |
645 | @@ -73,6 +122,15 @@ |
646 | return _pkgs |
647 | |
648 | |
649 | +def apt_cache(in_memory=True): |
650 | + """Build and return an apt cache""" |
651 | + apt_pkg.init() |
652 | + if in_memory: |
653 | + apt_pkg.config.set("Dir::Cache::pkgcache", "") |
654 | + apt_pkg.config.set("Dir::Cache::srcpkgcache", "") |
655 | + return apt_pkg.Cache() |
656 | + |
657 | + |
658 | def apt_install(packages, options=None, fatal=False): |
659 | """Install one or more packages""" |
660 | if options is None: |
661 | @@ -87,14 +145,7 @@ |
662 | cmd.extend(packages) |
663 | log("Installing {} with options: {}".format(packages, |
664 | options)) |
665 | - env = os.environ.copy() |
666 | - if 'DEBIAN_FRONTEND' not in env: |
667 | - env['DEBIAN_FRONTEND'] = 'noninteractive' |
668 | - |
669 | - if fatal: |
670 | - subprocess.check_call(cmd, env=env) |
671 | - else: |
672 | - subprocess.call(cmd, env=env) |
673 | + _run_apt_command(cmd, fatal) |
674 | |
675 | |
676 | def apt_upgrade(options=None, fatal=False, dist=False): |
677 | @@ -109,24 +160,13 @@ |
678 | else: |
679 | cmd.append('upgrade') |
680 | log("Upgrading with options: {}".format(options)) |
681 | - |
682 | - env = os.environ.copy() |
683 | - if 'DEBIAN_FRONTEND' not in env: |
684 | - env['DEBIAN_FRONTEND'] = 'noninteractive' |
685 | - |
686 | - if fatal: |
687 | - subprocess.check_call(cmd, env=env) |
688 | - else: |
689 | - subprocess.call(cmd, env=env) |
690 | + _run_apt_command(cmd, fatal) |
691 | |
692 | |
693 | def apt_update(fatal=False): |
694 | """Update local apt cache""" |
695 | cmd = ['apt-get', 'update'] |
696 | - if fatal: |
697 | - subprocess.check_call(cmd) |
698 | - else: |
699 | - subprocess.call(cmd) |
700 | + _run_apt_command(cmd, fatal) |
701 | |
702 | |
703 | def apt_purge(packages, fatal=False): |
704 | @@ -137,10 +177,7 @@ |
705 | else: |
706 | cmd.extend(packages) |
707 | log("Purging {}".format(packages)) |
708 | - if fatal: |
709 | - subprocess.check_call(cmd) |
710 | - else: |
711 | - subprocess.call(cmd) |
712 | + _run_apt_command(cmd, fatal) |
713 | |
714 | |
715 | def apt_hold(packages, fatal=False): |
716 | @@ -151,6 +188,7 @@ |
717 | else: |
718 | cmd.extend(packages) |
719 | log("Holding {}".format(packages)) |
720 | + |
721 | if fatal: |
722 | subprocess.check_call(cmd) |
723 | else: |
724 | @@ -184,14 +222,10 @@ |
725 | apt.write(PROPOSED_POCKET.format(release)) |
726 | if key: |
727 | subprocess.check_call(['apt-key', 'adv', '--keyserver', |
728 | - 'keyserver.ubuntu.com', '--recv', |
729 | + 'hkp://keyserver.ubuntu.com:80', '--recv', |
730 | key]) |
731 | |
732 | |
733 | -class SourceConfigError(Exception): |
734 | - pass |
735 | - |
736 | - |
737 | def configure_sources(update=False, |
738 | sources_var='install_sources', |
739 | keys_var='install_keys'): |
740 | @@ -224,17 +258,6 @@ |
741 | if update: |
742 | apt_update(fatal=True) |
743 | |
744 | -# The order of this list is very important. Handlers should be listed in from |
745 | -# least- to most-specific URL matching. |
746 | -FETCH_HANDLERS = ( |
747 | - 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler', |
748 | - 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler', |
749 | -) |
750 | - |
751 | - |
752 | -class UnhandledSource(Exception): |
753 | - pass |
754 | - |
755 | |
756 | def install_remote(source): |
757 | """ |
758 | @@ -265,30 +288,6 @@ |
759 | return install_remote(source) |
760 | |
761 | |
762 | -class BaseFetchHandler(object): |
763 | - |
764 | - """Base class for FetchHandler implementations in fetch plugins""" |
765 | - |
766 | - def can_handle(self, source): |
767 | - """Returns True if the source can be handled. Otherwise returns |
768 | - a string explaining why it cannot""" |
769 | - return "Wrong source type" |
770 | - |
771 | - def install(self, source): |
772 | - """Try to download and unpack the source. Return the path to the |
773 | - unpacked files or raise UnhandledSource.""" |
774 | - raise UnhandledSource("Wrong source type {}".format(source)) |
775 | - |
776 | - def parse_url(self, url): |
777 | - return urlparse(url) |
778 | - |
779 | - def base_url(self, url): |
780 | - """Return url without querystring or fragment""" |
781 | - parts = list(self.parse_url(url)) |
782 | - parts[4:] = ['' for i in parts[4:]] |
783 | - return urlunparse(parts) |
784 | - |
785 | - |
786 | def plugins(fetch_handlers=None): |
787 | if not fetch_handlers: |
788 | fetch_handlers = FETCH_HANDLERS |
789 | @@ -306,3 +305,40 @@ |
790 | log("FetchHandler {} not found, skipping plugin".format( |
791 | handler_name)) |
792 | return plugin_list |
793 | + |
794 | + |
795 | +def _run_apt_command(cmd, fatal=False): |
796 | + """ |
797 | + Run an APT command, checking output and retrying if the fatal flag is set |
798 | + to True. |
799 | + |
800 | + :param: cmd: str: The apt command to run. |
801 | + :param: fatal: bool: Whether the command's output should be checked and |
802 | + retried. |
803 | + """ |
804 | + env = os.environ.copy() |
805 | + |
806 | + if 'DEBIAN_FRONTEND' not in env: |
807 | + env['DEBIAN_FRONTEND'] = 'noninteractive' |
808 | + |
809 | + if fatal: |
810 | + retry_count = 0 |
811 | + result = None |
812 | + |
813 | + # If the command is considered "fatal", we need to retry if the apt |
814 | + # lock was not acquired. |
815 | + |
816 | + while result is None or result == APT_NO_LOCK: |
817 | + try: |
818 | + result = subprocess.check_call(cmd, env=env) |
819 | + except subprocess.CalledProcessError, e: |
820 | + retry_count = retry_count + 1 |
821 | + if retry_count > APT_NO_LOCK_RETRY_COUNT: |
822 | + raise |
823 | + result = e.returncode |
824 | + log("Couldn't acquire DPKG lock. Will retry in {} seconds." |
825 | + "".format(APT_NO_LOCK_RETRY_DELAY)) |
826 | + time.sleep(APT_NO_LOCK_RETRY_DELAY) |
827 | + |
828 | + else: |
829 | + subprocess.call(cmd, env=env) |
830 | |
831 | === modified file 'hooks/charmhelpers/fetch/bzrurl.py' |
832 | --- hooks/charmhelpers/fetch/bzrurl.py 2013-12-17 15:33:05 +0000 |
833 | +++ hooks/charmhelpers/fetch/bzrurl.py 2014-08-27 07:08:22 +0000 |
834 | @@ -39,7 +39,8 @@ |
835 | def install(self, source): |
836 | url_parts = self.parse_url(source) |
837 | branch_name = url_parts.path.strip("/").split("/")[-1] |
838 | - dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name) |
839 | + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", |
840 | + branch_name) |
841 | if not os.path.exists(dest_dir): |
842 | mkdir(dest_dir, perms=0755) |
843 | try: |
Looks good! +1