Merge lp:~tribaal/charms/trusty/mysql/charm-helpers-sync into lp:charms/trusty/mysql

Proposed by Chris Glass
Status: Merged
Approved by: David Britton
Approved revision: 127
Merge reported by: David Britton
Merged at revision: not available
Proposed branch: lp:~tribaal/charms/trusty/mysql/charm-helpers-sync
Merge into: lp:charms/trusty/mysql
Diff against target: 1685 lines (+1204/-116)
12 files modified
.bzrignore (+1/-0)
Makefile (+7/-2)
hooks/charmhelpers/core/fstab.py (+116/-0)
hooks/charmhelpers/core/hookenv.py (+132/-7)
hooks/charmhelpers/core/host.py (+100/-12)
hooks/charmhelpers/core/services/__init__.py (+2/-0)
hooks/charmhelpers/core/services/base.py (+313/-0)
hooks/charmhelpers/core/services/helpers.py (+239/-0)
hooks/charmhelpers/core/templating.py (+51/-0)
hooks/charmhelpers/fetch/__init__.py (+192/-90)
hooks/charmhelpers/fetch/archiveurl.py (+49/-4)
hooks/charmhelpers/fetch/bzrurl.py (+2/-1)
To merge this branch: bzr merge lp:~tribaal/charms/trusty/mysql/charm-helpers-sync
Reviewer Review Type Date Requested Status
David Britton Approve
Review Queue (community) automated testing Needs Fixing
Review via email: mp+236067@code.launchpad.net

Description of the change

This branch updates charm-helpers so that the charm take advantage of the new in-memory apt-cache building (fixes race conditions).

To post a comment you must log in.
Revision history for this message
Review Queue (review-queue) wrote :

This items has failed automated testing! Results available here http://reports.vapour.ws/charm-tests/charm-bundle-test-1109-results

review: Needs Fixing (automated testing)
Revision history for this message
David Britton (dpb) wrote :

Look great Chris, Thanks! -- I checked the automated test results, the error was from hpcloud and was spurious.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== added file '.bzrignore'
2--- .bzrignore 1970-01-01 00:00:00 +0000
3+++ .bzrignore 2014-09-26 07:21:45 +0000
4@@ -0,0 +1,1 @@
5+bin/
6
7=== modified file 'Makefile'
8--- Makefile 2014-03-04 17:28:10 +0000
9+++ Makefile 2014-09-26 07:21:45 +0000
10@@ -11,5 +11,10 @@
11 # @echo Starting tests...
12 # @$(PYTHON) /usr/bin/nosetests --nologcapture unit_tests
13
14-sync:
15- @charm-helper-sync -c charm-helpers.yaml
16+bin/charm_helpers_sync.py:
17+ @mkdir -p bin
18+ @bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \
19+ > bin/charm_helpers_sync.py
20+
21+sync: bin/charm_helpers_sync.py
22+ $(PYTHON) bin/charm_helpers_sync.py -c charm-helpers.yaml
23
24=== added file 'hooks/charmhelpers/core/fstab.py'
25--- hooks/charmhelpers/core/fstab.py 1970-01-01 00:00:00 +0000
26+++ hooks/charmhelpers/core/fstab.py 2014-09-26 07:21:45 +0000
27@@ -0,0 +1,116 @@
28+#!/usr/bin/env python
29+# -*- coding: utf-8 -*-
30+
31+__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
32+
33+import os
34+
35+
36+class Fstab(file):
37+ """This class extends file in order to implement a file reader/writer
38+ for file `/etc/fstab`
39+ """
40+
41+ class Entry(object):
42+ """Entry class represents a non-comment line on the `/etc/fstab` file
43+ """
44+ def __init__(self, device, mountpoint, filesystem,
45+ options, d=0, p=0):
46+ self.device = device
47+ self.mountpoint = mountpoint
48+ self.filesystem = filesystem
49+
50+ if not options:
51+ options = "defaults"
52+
53+ self.options = options
54+ self.d = d
55+ self.p = p
56+
57+ def __eq__(self, o):
58+ return str(self) == str(o)
59+
60+ def __str__(self):
61+ return "{} {} {} {} {} {}".format(self.device,
62+ self.mountpoint,
63+ self.filesystem,
64+ self.options,
65+ self.d,
66+ self.p)
67+
68+ DEFAULT_PATH = os.path.join(os.path.sep, 'etc', 'fstab')
69+
70+ def __init__(self, path=None):
71+ if path:
72+ self._path = path
73+ else:
74+ self._path = self.DEFAULT_PATH
75+ file.__init__(self, self._path, 'r+')
76+
77+ def _hydrate_entry(self, line):
78+ # NOTE: use split with no arguments to split on any
79+ # whitespace including tabs
80+ return Fstab.Entry(*filter(
81+ lambda x: x not in ('', None),
82+ line.strip("\n").split()))
83+
84+ @property
85+ def entries(self):
86+ self.seek(0)
87+ for line in self.readlines():
88+ try:
89+ if not line.startswith("#"):
90+ yield self._hydrate_entry(line)
91+ except ValueError:
92+ pass
93+
94+ def get_entry_by_attr(self, attr, value):
95+ for entry in self.entries:
96+ e_attr = getattr(entry, attr)
97+ if e_attr == value:
98+ return entry
99+ return None
100+
101+ def add_entry(self, entry):
102+ if self.get_entry_by_attr('device', entry.device):
103+ return False
104+
105+ self.write(str(entry) + '\n')
106+ self.truncate()
107+ return entry
108+
109+ def remove_entry(self, entry):
110+ self.seek(0)
111+
112+ lines = self.readlines()
113+
114+ found = False
115+ for index, line in enumerate(lines):
116+ if not line.startswith("#"):
117+ if self._hydrate_entry(line) == entry:
118+ found = True
119+ break
120+
121+ if not found:
122+ return False
123+
124+ lines.remove(line)
125+
126+ self.seek(0)
127+ self.write(''.join(lines))
128+ self.truncate()
129+ return True
130+
131+ @classmethod
132+ def remove_by_mountpoint(cls, mountpoint, path=None):
133+ fstab = cls(path=path)
134+ entry = fstab.get_entry_by_attr('mountpoint', mountpoint)
135+ if entry:
136+ return fstab.remove_entry(entry)
137+ return False
138+
139+ @classmethod
140+ def add(cls, device, mountpoint, filesystem, options=None, path=None):
141+ return cls(path=path).add_entry(Fstab.Entry(device,
142+ mountpoint, filesystem,
143+ options=options))
144
145=== modified file 'hooks/charmhelpers/core/hookenv.py'
146--- hooks/charmhelpers/core/hookenv.py 2014-02-19 14:49:31 +0000
147+++ hooks/charmhelpers/core/hookenv.py 2014-09-26 07:21:45 +0000
148@@ -25,7 +25,7 @@
149 def cached(func):
150 """Cache return values for multiple executions of func + args
151
152- For example:
153+ For example::
154
155 @cached
156 def unit_get(attribute):
157@@ -155,6 +155,121 @@
158 return os.path.basename(sys.argv[0])
159
160
161+class Config(dict):
162+ """A dictionary representation of the charm's config.yaml, with some
163+ extra features:
164+
165+ - See which values in the dictionary have changed since the previous hook.
166+ - For values that have changed, see what the previous value was.
167+ - Store arbitrary data for use in a later hook.
168+
169+ NOTE: Do not instantiate this object directly - instead call
170+ ``hookenv.config()``, which will return an instance of :class:`Config`.
171+
172+ Example usage::
173+
174+ >>> # inside a hook
175+ >>> from charmhelpers.core import hookenv
176+ >>> config = hookenv.config()
177+ >>> config['foo']
178+ 'bar'
179+ >>> # store a new key/value for later use
180+ >>> config['mykey'] = 'myval'
181+
182+
183+ >>> # user runs `juju set mycharm foo=baz`
184+ >>> # now we're inside subsequent config-changed hook
185+ >>> config = hookenv.config()
186+ >>> config['foo']
187+ 'baz'
188+ >>> # test to see if this val has changed since last hook
189+ >>> config.changed('foo')
190+ True
191+ >>> # what was the previous value?
192+ >>> config.previous('foo')
193+ 'bar'
194+ >>> # keys/values that we add are preserved across hooks
195+ >>> config['mykey']
196+ 'myval'
197+
198+ """
199+ CONFIG_FILE_NAME = '.juju-persistent-config'
200+
201+ def __init__(self, *args, **kw):
202+ super(Config, self).__init__(*args, **kw)
203+ self.implicit_save = True
204+ self._prev_dict = None
205+ self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
206+ if os.path.exists(self.path):
207+ self.load_previous()
208+
209+ def __getitem__(self, key):
210+ """For regular dict lookups, check the current juju config first,
211+ then the previous (saved) copy. This ensures that user-saved values
212+ will be returned by a dict lookup.
213+
214+ """
215+ try:
216+ return dict.__getitem__(self, key)
217+ except KeyError:
218+ return (self._prev_dict or {})[key]
219+
220+ def load_previous(self, path=None):
221+ """Load previous copy of config from disk.
222+
223+ In normal usage you don't need to call this method directly - it
224+ is called automatically at object initialization.
225+
226+ :param path:
227+
228+ File path from which to load the previous config. If `None`,
229+ config is loaded from the default location. If `path` is
230+ specified, subsequent `save()` calls will write to the same
231+ path.
232+
233+ """
234+ self.path = path or self.path
235+ with open(self.path) as f:
236+ self._prev_dict = json.load(f)
237+
238+ def changed(self, key):
239+ """Return True if the current value for this key is different from
240+ the previous value.
241+
242+ """
243+ if self._prev_dict is None:
244+ return True
245+ return self.previous(key) != self.get(key)
246+
247+ def previous(self, key):
248+ """Return previous value for this key, or None if there
249+ is no previous value.
250+
251+ """
252+ if self._prev_dict:
253+ return self._prev_dict.get(key)
254+ return None
255+
256+ def save(self):
257+ """Save this config to disk.
258+
259+ If the charm is using the :mod:`Services Framework <services.base>`
260+ or :meth:'@hook <Hooks.hook>' decorator, this
261+ is called automatically at the end of successful hook execution.
262+ Otherwise, it should be called directly by user code.
263+
264+ To disable automatic saves, set ``implicit_save=False`` on this
265+ instance.
266+
267+ """
268+ if self._prev_dict:
269+ for k, v in self._prev_dict.iteritems():
270+ if k not in self:
271+ self[k] = v
272+ with open(self.path, 'w') as f:
273+ json.dump(self, f)
274+
275+
276 @cached
277 def config(scope=None):
278 """Juju charm configuration"""
279@@ -163,7 +278,10 @@
280 config_cmd_line.append(scope)
281 config_cmd_line.append('--format=json')
282 try:
283- return json.loads(subprocess.check_output(config_cmd_line))
284+ config_data = json.loads(subprocess.check_output(config_cmd_line))
285+ if scope is not None:
286+ return config_data
287+ return Config(config_data)
288 except ValueError:
289 return None
290
291@@ -188,8 +306,9 @@
292 raise
293
294
295-def relation_set(relation_id=None, relation_settings={}, **kwargs):
296+def relation_set(relation_id=None, relation_settings=None, **kwargs):
297 """Set relation information for the current unit"""
298+ relation_settings = relation_settings if relation_settings else {}
299 relation_cmd_line = ['relation-set']
300 if relation_id is not None:
301 relation_cmd_line.extend(('-r', relation_id))
302@@ -348,27 +467,29 @@
303 class Hooks(object):
304 """A convenient handler for hook functions.
305
306- Example:
307+ Example::
308+
309 hooks = Hooks()
310
311 # register a hook, taking its name from the function name
312 @hooks.hook()
313 def install():
314- ...
315+ pass # your code here
316
317 # register a hook, providing a custom hook name
318 @hooks.hook("config-changed")
319 def config_changed():
320- ...
321+ pass # your code here
322
323 if __name__ == "__main__":
324 # execute a hook based on the name the program is called by
325 hooks.execute(sys.argv)
326 """
327
328- def __init__(self):
329+ def __init__(self, config_save=True):
330 super(Hooks, self).__init__()
331 self._hooks = {}
332+ self._config_save = config_save
333
334 def register(self, name, function):
335 """Register a hook"""
336@@ -379,6 +500,10 @@
337 hook_name = os.path.basename(args[0])
338 if hook_name in self._hooks:
339 self._hooks[hook_name]()
340+ if self._config_save:
341+ cfg = config()
342+ if cfg.implicit_save:
343+ cfg.save()
344 else:
345 raise UnregisteredHookError(hook_name)
346
347
348=== modified file 'hooks/charmhelpers/core/host.py'
349--- hooks/charmhelpers/core/host.py 2014-02-19 14:49:31 +0000
350+++ hooks/charmhelpers/core/host.py 2014-09-26 07:21:45 +0000
351@@ -12,10 +12,13 @@
352 import string
353 import subprocess
354 import hashlib
355+import shutil
356+from contextlib import contextmanager
357
358 from collections import OrderedDict
359
360 from hookenv import log
361+from fstab import Fstab
362
363
364 def service_start(service_name):
365@@ -34,7 +37,8 @@
366
367
368 def service_reload(service_name, restart_on_failure=False):
369- """Reload a system service, optionally falling back to restart if reload fails"""
370+ """Reload a system service, optionally falling back to restart if
371+ reload fails"""
372 service_result = service('reload', service_name)
373 if not service_result and restart_on_failure:
374 service_result = service('restart', service_name)
375@@ -50,7 +54,7 @@
376 def service_running(service):
377 """Determine whether a system service is running"""
378 try:
379- output = subprocess.check_output(['service', service, 'status'])
380+ output = subprocess.check_output(['service', service, 'status'], stderr=subprocess.STDOUT)
381 except subprocess.CalledProcessError:
382 return False
383 else:
384@@ -60,6 +64,16 @@
385 return False
386
387
388+def service_available(service_name):
389+ """Determine whether a system service is available"""
390+ try:
391+ subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT)
392+ except subprocess.CalledProcessError as e:
393+ return 'unrecognized service' not in e.output
394+ else:
395+ return True
396+
397+
398 def adduser(username, password=None, shell='/bin/bash', system_user=False):
399 """Add a user to the system"""
400 try:
401@@ -143,7 +157,19 @@
402 target.write(content)
403
404
405-def mount(device, mountpoint, options=None, persist=False):
406+def fstab_remove(mp):
407+ """Remove the given mountpoint entry from /etc/fstab
408+ """
409+ return Fstab.remove_by_mountpoint(mp)
410+
411+
412+def fstab_add(dev, mp, fs, options=None):
413+ """Adds the given device entry to the /etc/fstab file
414+ """
415+ return Fstab.add(dev, mp, fs, options=options)
416+
417+
418+def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"):
419 """Mount a filesystem at a particular mountpoint"""
420 cmd_args = ['mount']
421 if options is not None:
422@@ -154,9 +180,9 @@
423 except subprocess.CalledProcessError, e:
424 log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output))
425 return False
426+
427 if persist:
428- # TODO: update fstab
429- pass
430+ return fstab_add(device, mountpoint, filesystem, options=options)
431 return True
432
433
434@@ -168,9 +194,9 @@
435 except subprocess.CalledProcessError, e:
436 log('Error unmounting {}\n{}'.format(mountpoint, e.output))
437 return False
438+
439 if persist:
440- # TODO: update fstab
441- pass
442+ return fstab_remove(mountpoint)
443 return True
444
445
446@@ -183,10 +209,15 @@
447 return system_mounts
448
449
450-def file_hash(path):
451- """Generate a md5 hash of the contents of 'path' or None if not found """
452+def file_hash(path, hash_type='md5'):
453+ """
454+ Generate a hash checksum of the contents of 'path' or None if not found.
455+
456+ :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`,
457+ such as md5, sha1, sha256, sha512, etc.
458+ """
459 if os.path.exists(path):
460- h = hashlib.md5()
461+ h = getattr(hashlib, hash_type)()
462 with open(path, 'r') as source:
463 h.update(source.read()) # IGNORE:E1101 - it does have update
464 return h.hexdigest()
465@@ -194,16 +225,36 @@
466 return None
467
468
469+def check_hash(path, checksum, hash_type='md5'):
470+ """
471+ Validate a file using a cryptographic checksum.
472+
473+ :param str checksum: Value of the checksum used to validate the file.
474+ :param str hash_type: Hash algorithm used to generate `checksum`.
475+ Can be any hash alrgorithm supported by :mod:`hashlib`,
476+ such as md5, sha1, sha256, sha512, etc.
477+ :raises ChecksumError: If the file fails the checksum
478+
479+ """
480+ actual_checksum = file_hash(path, hash_type)
481+ if checksum != actual_checksum:
482+ raise ChecksumError("'%s' != '%s'" % (checksum, actual_checksum))
483+
484+
485+class ChecksumError(ValueError):
486+ pass
487+
488+
489 def restart_on_change(restart_map, stopstart=False):
490 """Restart services based on configuration files changing
491
492- This function is used a decorator, for example
493+ This function is used a decorator, for example::
494
495 @restart_on_change({
496 '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
497 })
498 def ceph_client_changed():
499- ...
500+ pass # your code here
501
502 In this example, the cinder-api and cinder-volume services
503 would be restarted if /etc/ceph/ceph.conf is changed by the
504@@ -295,3 +346,40 @@
505 if 'link/ether' in words:
506 hwaddr = words[words.index('link/ether') + 1]
507 return hwaddr
508+
509+
510+def cmp_pkgrevno(package, revno, pkgcache=None):
511+ '''Compare supplied revno with the revno of the installed package
512+
513+ * 1 => Installed revno is greater than supplied arg
514+ * 0 => Installed revno is the same as supplied arg
515+ * -1 => Installed revno is less than supplied arg
516+
517+ '''
518+ import apt_pkg
519+ from charmhelpers.fetch import apt_cache
520+ if not pkgcache:
521+ pkgcache = apt_cache()
522+ pkg = pkgcache[package]
523+ return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
524+
525+
526+@contextmanager
527+def chdir(d):
528+ cur = os.getcwd()
529+ try:
530+ yield os.chdir(d)
531+ finally:
532+ os.chdir(cur)
533+
534+
535+def chownr(path, owner, group):
536+ uid = pwd.getpwnam(owner).pw_uid
537+ gid = grp.getgrnam(group).gr_gid
538+
539+ for root, dirs, files in os.walk(path):
540+ for name in dirs + files:
541+ full = os.path.join(root, name)
542+ broken_symlink = os.path.lexists(full) and not os.path.exists(full)
543+ if not broken_symlink:
544+ os.chown(full, uid, gid)
545
546=== added directory 'hooks/charmhelpers/core/services'
547=== added file 'hooks/charmhelpers/core/services/__init__.py'
548--- hooks/charmhelpers/core/services/__init__.py 1970-01-01 00:00:00 +0000
549+++ hooks/charmhelpers/core/services/__init__.py 2014-09-26 07:21:45 +0000
550@@ -0,0 +1,2 @@
551+from .base import *
552+from .helpers import *
553
554=== added file 'hooks/charmhelpers/core/services/base.py'
555--- hooks/charmhelpers/core/services/base.py 1970-01-01 00:00:00 +0000
556+++ hooks/charmhelpers/core/services/base.py 2014-09-26 07:21:45 +0000
557@@ -0,0 +1,313 @@
558+import os
559+import re
560+import json
561+from collections import Iterable
562+
563+from charmhelpers.core import host
564+from charmhelpers.core import hookenv
565+
566+
567+__all__ = ['ServiceManager', 'ManagerCallback',
568+ 'PortManagerCallback', 'open_ports', 'close_ports', 'manage_ports',
569+ 'service_restart', 'service_stop']
570+
571+
572+class ServiceManager(object):
573+ def __init__(self, services=None):
574+ """
575+ Register a list of services, given their definitions.
576+
577+ Service definitions are dicts in the following formats (all keys except
578+ 'service' are optional)::
579+
580+ {
581+ "service": <service name>,
582+ "required_data": <list of required data contexts>,
583+ "provided_data": <list of provided data contexts>,
584+ "data_ready": <one or more callbacks>,
585+ "data_lost": <one or more callbacks>,
586+ "start": <one or more callbacks>,
587+ "stop": <one or more callbacks>,
588+ "ports": <list of ports to manage>,
589+ }
590+
591+ The 'required_data' list should contain dicts of required data (or
592+ dependency managers that act like dicts and know how to collect the data).
593+ Only when all items in the 'required_data' list are populated are the list
594+ of 'data_ready' and 'start' callbacks executed. See `is_ready()` for more
595+ information.
596+
597+ The 'provided_data' list should contain relation data providers, most likely
598+ a subclass of :class:`charmhelpers.core.services.helpers.RelationContext`,
599+ that will indicate a set of data to set on a given relation.
600+
601+ The 'data_ready' value should be either a single callback, or a list of
602+ callbacks, to be called when all items in 'required_data' pass `is_ready()`.
603+ Each callback will be called with the service name as the only parameter.
604+ After all of the 'data_ready' callbacks are called, the 'start' callbacks
605+ are fired.
606+
607+ The 'data_lost' value should be either a single callback, or a list of
608+ callbacks, to be called when a 'required_data' item no longer passes
609+ `is_ready()`. Each callback will be called with the service name as the
610+ only parameter. After all of the 'data_lost' callbacks are called,
611+ the 'stop' callbacks are fired.
612+
613+ The 'start' value should be either a single callback, or a list of
614+ callbacks, to be called when starting the service, after the 'data_ready'
615+ callbacks are complete. Each callback will be called with the service
616+ name as the only parameter. This defaults to
617+ `[host.service_start, services.open_ports]`.
618+
619+ The 'stop' value should be either a single callback, or a list of
620+ callbacks, to be called when stopping the service. If the service is
621+ being stopped because it no longer has all of its 'required_data', this
622+ will be called after all of the 'data_lost' callbacks are complete.
623+ Each callback will be called with the service name as the only parameter.
624+ This defaults to `[services.close_ports, host.service_stop]`.
625+
626+ The 'ports' value should be a list of ports to manage. The default
627+ 'start' handler will open the ports after the service is started,
628+ and the default 'stop' handler will close the ports prior to stopping
629+ the service.
630+
631+
632+ Examples:
633+
634+ The following registers an Upstart service called bingod that depends on
635+ a mongodb relation and which runs a custom `db_migrate` function prior to
636+ restarting the service, and a Runit service called spadesd::
637+
638+ manager = services.ServiceManager([
639+ {
640+ 'service': 'bingod',
641+ 'ports': [80, 443],
642+ 'required_data': [MongoRelation(), config(), {'my': 'data'}],
643+ 'data_ready': [
644+ services.template(source='bingod.conf'),
645+ services.template(source='bingod.ini',
646+ target='/etc/bingod.ini',
647+ owner='bingo', perms=0400),
648+ ],
649+ },
650+ {
651+ 'service': 'spadesd',
652+ 'data_ready': services.template(source='spadesd_run.j2',
653+ target='/etc/sv/spadesd/run',
654+ perms=0555),
655+ 'start': runit_start,
656+ 'stop': runit_stop,
657+ },
658+ ])
659+ manager.manage()
660+ """
661+ self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json')
662+ self._ready = None
663+ self.services = {}
664+ for service in services or []:
665+ service_name = service['service']
666+ self.services[service_name] = service
667+
668+ def manage(self):
669+ """
670+ Handle the current hook by doing The Right Thing with the registered services.
671+ """
672+ hook_name = hookenv.hook_name()
673+ if hook_name == 'stop':
674+ self.stop_services()
675+ else:
676+ self.provide_data()
677+ self.reconfigure_services()
678+ cfg = hookenv.config()
679+ if cfg.implicit_save:
680+ cfg.save()
681+
682+ def provide_data(self):
683+ """
684+ Set the relation data for each provider in the ``provided_data`` list.
685+
686+ A provider must have a `name` attribute, which indicates which relation
687+ to set data on, and a `provide_data()` method, which returns a dict of
688+ data to set.
689+ """
690+ hook_name = hookenv.hook_name()
691+ for service in self.services.values():
692+ for provider in service.get('provided_data', []):
693+ if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name):
694+ data = provider.provide_data()
695+ _ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data
696+ if _ready:
697+ hookenv.relation_set(None, data)
698+
699+ def reconfigure_services(self, *service_names):
700+ """
701+ Update all files for one or more registered services, and,
702+ if ready, optionally restart them.
703+
704+ If no service names are given, reconfigures all registered services.
705+ """
706+ for service_name in service_names or self.services.keys():
707+ if self.is_ready(service_name):
708+ self.fire_event('data_ready', service_name)
709+ self.fire_event('start', service_name, default=[
710+ service_restart,
711+ manage_ports])
712+ self.save_ready(service_name)
713+ else:
714+ if self.was_ready(service_name):
715+ self.fire_event('data_lost', service_name)
716+ self.fire_event('stop', service_name, default=[
717+ manage_ports,
718+ service_stop])
719+ self.save_lost(service_name)
720+
721+ def stop_services(self, *service_names):
722+ """
723+ Stop one or more registered services, by name.
724+
725+ If no service names are given, stops all registered services.
726+ """
727+ for service_name in service_names or self.services.keys():
728+ self.fire_event('stop', service_name, default=[
729+ manage_ports,
730+ service_stop])
731+
732+ def get_service(self, service_name):
733+ """
734+ Given the name of a registered service, return its service definition.
735+ """
736+ service = self.services.get(service_name)
737+ if not service:
738+ raise KeyError('Service not registered: %s' % service_name)
739+ return service
740+
741+ def fire_event(self, event_name, service_name, default=None):
742+ """
743+ Fire a data_ready, data_lost, start, or stop event on a given service.
744+ """
745+ service = self.get_service(service_name)
746+ callbacks = service.get(event_name, default)
747+ if not callbacks:
748+ return
749+ if not isinstance(callbacks, Iterable):
750+ callbacks = [callbacks]
751+ for callback in callbacks:
752+ if isinstance(callback, ManagerCallback):
753+ callback(self, service_name, event_name)
754+ else:
755+ callback(service_name)
756+
757+ def is_ready(self, service_name):
758+ """
759+ Determine if a registered service is ready, by checking its 'required_data'.
760+
761+ A 'required_data' item can be any mapping type, and is considered ready
762+ if `bool(item)` evaluates as True.
763+ """
764+ service = self.get_service(service_name)
765+ reqs = service.get('required_data', [])
766+ return all(bool(req) for req in reqs)
767+
768+ def _load_ready_file(self):
769+ if self._ready is not None:
770+ return
771+ if os.path.exists(self._ready_file):
772+ with open(self._ready_file) as fp:
773+ self._ready = set(json.load(fp))
774+ else:
775+ self._ready = set()
776+
777+ def _save_ready_file(self):
778+ if self._ready is None:
779+ return
780+ with open(self._ready_file, 'w') as fp:
781+ json.dump(list(self._ready), fp)
782+
783+ def save_ready(self, service_name):
784+ """
785+ Save an indicator that the given service is now data_ready.
786+ """
787+ self._load_ready_file()
788+ self._ready.add(service_name)
789+ self._save_ready_file()
790+
791+ def save_lost(self, service_name):
792+ """
793+ Save an indicator that the given service is no longer data_ready.
794+ """
795+ self._load_ready_file()
796+ self._ready.discard(service_name)
797+ self._save_ready_file()
798+
799+ def was_ready(self, service_name):
800+ """
801+ Determine if the given service was previously data_ready.
802+ """
803+ self._load_ready_file()
804+ return service_name in self._ready
805+
806+
807+class ManagerCallback(object):
808+ """
809+ Special case of a callback that takes the `ServiceManager` instance
810+ in addition to the service name.
811+
812+ Subclasses should implement `__call__` which should accept three parameters:
813+
814+ * `manager` The `ServiceManager` instance
815+ * `service_name` The name of the service it's being triggered for
816+ * `event_name` The name of the event that this callback is handling
817+ """
818+ def __call__(self, manager, service_name, event_name):
819+ raise NotImplementedError()
820+
821+
822+class PortManagerCallback(ManagerCallback):
823+ """
824+ Callback class that will open or close ports, for use as either
825+ a start or stop action.
826+ """
827+ def __call__(self, manager, service_name, event_name):
828+ service = manager.get_service(service_name)
829+ new_ports = service.get('ports', [])
830+ port_file = os.path.join(hookenv.charm_dir(), '.{}.ports'.format(service_name))
831+ if os.path.exists(port_file):
832+ with open(port_file) as fp:
833+ old_ports = fp.read().split(',')
834+ for old_port in old_ports:
835+ if bool(old_port):
836+ old_port = int(old_port)
837+ if old_port not in new_ports:
838+ hookenv.close_port(old_port)
839+ with open(port_file, 'w') as fp:
840+ fp.write(','.join(str(port) for port in new_ports))
841+ for port in new_ports:
842+ if event_name == 'start':
843+ hookenv.open_port(port)
844+ elif event_name == 'stop':
845+ hookenv.close_port(port)
846+
847+
848+def service_stop(service_name):
849+ """
850+ Wrapper around host.service_stop to prevent spurious "unknown service"
851+ messages in the logs.
852+ """
853+ if host.service_running(service_name):
854+ host.service_stop(service_name)
855+
856+
857+def service_restart(service_name):
858+ """
859+ Wrapper around host.service_restart to prevent spurious "unknown service"
860+ messages in the logs.
861+ """
862+ if host.service_available(service_name):
863+ if host.service_running(service_name):
864+ host.service_restart(service_name)
865+ else:
866+ host.service_start(service_name)
867+
868+
869+# Convenience aliases
870+open_ports = close_ports = manage_ports = PortManagerCallback()
871
872=== added file 'hooks/charmhelpers/core/services/helpers.py'
873--- hooks/charmhelpers/core/services/helpers.py 1970-01-01 00:00:00 +0000
874+++ hooks/charmhelpers/core/services/helpers.py 2014-09-26 07:21:45 +0000
875@@ -0,0 +1,239 @@
876+import os
877+import yaml
878+from charmhelpers.core import hookenv
879+from charmhelpers.core import templating
880+
881+from charmhelpers.core.services.base import ManagerCallback
882+
883+
884+__all__ = ['RelationContext', 'TemplateCallback',
885+ 'render_template', 'template']
886+
887+
888+class RelationContext(dict):
889+ """
890+ Base class for a context generator that gets relation data from juju.
891+
892+ Subclasses must provide the attributes `name`, which is the name of the
893+ interface of interest, `interface`, which is the type of the interface of
894+ interest, and `required_keys`, which is the set of keys required for the
895+ relation to be considered complete. The data for all interfaces matching
896+ the `name` attribute that are complete will used to populate the dictionary
897+ values (see `get_data`, below).
898+
899+ The generated context will be namespaced under the relation :attr:`name`,
900+ to prevent potential naming conflicts.
901+
902+ :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
903+ :param list additional_required_keys: Extend the list of :attr:`required_keys`
904+ """
905+ name = None
906+ interface = None
907+ required_keys = []
908+
909+ def __init__(self, name=None, additional_required_keys=None):
910+ if name is not None:
911+ self.name = name
912+ if additional_required_keys is not None:
913+ self.required_keys.extend(additional_required_keys)
914+ self.get_data()
915+
916+ def __bool__(self):
917+ """
918+ Returns True if all of the required_keys are available.
919+ """
920+ return self.is_ready()
921+
922+ __nonzero__ = __bool__
923+
924+ def __repr__(self):
925+ return super(RelationContext, self).__repr__()
926+
927+ def is_ready(self):
928+ """
929+ Returns True if all of the `required_keys` are available from any units.
930+ """
931+ ready = len(self.get(self.name, [])) > 0
932+ if not ready:
933+ hookenv.log('Incomplete relation: {}'.format(self.__class__.__name__), hookenv.DEBUG)
934+ return ready
935+
936+ def _is_ready(self, unit_data):
937+ """
938+ Helper method that tests a set of relation data and returns True if
939+ all of the `required_keys` are present.
940+ """
941+ return set(unit_data.keys()).issuperset(set(self.required_keys))
942+
943+ def get_data(self):
944+ """
945+ Retrieve the relation data for each unit involved in a relation and,
946+ if complete, store it in a list under `self[self.name]`. This
947+ is automatically called when the RelationContext is instantiated.
948+
949+ The units are sorted lexographically first by the service ID, then by
950+ the unit ID. Thus, if an interface has two other services, 'db:1'
951+ and 'db:2', with 'db:1' having two units, 'wordpress/0' and 'wordpress/1',
952+ and 'db:2' having one unit, 'mediawiki/0', all of which have a complete
953+ set of data, the relation data for the units will be stored in the
954+ order: 'wordpress/0', 'wordpress/1', 'mediawiki/0'.
955+
956+ If you only care about a single unit on the relation, you can just
957+ access it as `{{ interface[0]['key'] }}`. However, if you can at all
958+ support multiple units on a relation, you should iterate over the list,
959+ like::
960+
961+ {% for unit in interface -%}
962+ {{ unit['key'] }}{% if not loop.last %},{% endif %}
963+ {%- endfor %}
964+
965+ Note that since all sets of relation data from all related services and
966+ units are in a single list, if you need to know which service or unit a
967+ set of data came from, you'll need to extend this class to preserve
968+ that information.
969+ """
970+ if not hookenv.relation_ids(self.name):
971+ return
972+
973+ ns = self.setdefault(self.name, [])
974+ for rid in sorted(hookenv.relation_ids(self.name)):
975+ for unit in sorted(hookenv.related_units(rid)):
976+ reldata = hookenv.relation_get(rid=rid, unit=unit)
977+ if self._is_ready(reldata):
978+ ns.append(reldata)
979+
980+ def provide_data(self):
981+ """
982+ Return data to be relation_set for this interface.
983+ """
984+ return {}
985+
986+
987+class MysqlRelation(RelationContext):
988+ """
989+ Relation context for the `mysql` interface.
990+
991+ :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
992+ :param list additional_required_keys: Extend the list of :attr:`required_keys`
993+ """
994+ name = 'db'
995+ interface = 'mysql'
996+ required_keys = ['host', 'user', 'password', 'database']
997+
998+
999+class HttpRelation(RelationContext):
1000+ """
1001+ Relation context for the `http` interface.
1002+
1003+ :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
1004+ :param list additional_required_keys: Extend the list of :attr:`required_keys`
1005+ """
1006+ name = 'website'
1007+ interface = 'http'
1008+ required_keys = ['host', 'port']
1009+
1010+ def provide_data(self):
1011+ return {
1012+ 'host': hookenv.unit_get('private-address'),
1013+ 'port': 80,
1014+ }
1015+
1016+
1017+class RequiredConfig(dict):
1018+ """
1019+ Data context that loads config options with one or more mandatory options.
1020+
1021+ Once the required options have been changed from their default values, all
1022+ config options will be available, namespaced under `config` to prevent
1023+ potential naming conflicts (for example, between a config option and a
1024+ relation property).
1025+
1026+ :param list *args: List of options that must be changed from their default values.
1027+ """
1028+
1029+ def __init__(self, *args):
1030+ self.required_options = args
1031+ self['config'] = hookenv.config()
1032+ with open(os.path.join(hookenv.charm_dir(), 'config.yaml')) as fp:
1033+ self.config = yaml.load(fp).get('options', {})
1034+
1035+ def __bool__(self):
1036+ for option in self.required_options:
1037+ if option not in self['config']:
1038+ return False
1039+ current_value = self['config'][option]
1040+ default_value = self.config[option].get('default')
1041+ if current_value == default_value:
1042+ return False
1043+ if current_value in (None, '') and default_value in (None, ''):
1044+ return False
1045+ return True
1046+
1047+ def __nonzero__(self):
1048+ return self.__bool__()
1049+
1050+
1051+class StoredContext(dict):
1052+ """
1053+ A data context that always returns the data that it was first created with.
1054+
1055+ This is useful to do a one-time generation of things like passwords, that
1056+ will thereafter use the same value that was originally generated, instead
1057+ of generating a new value each time it is run.
1058+ """
1059+ def __init__(self, file_name, config_data):
1060+ """
1061+ If the file exists, populate `self` with the data from the file.
1062+ Otherwise, populate with the given data and persist it to the file.
1063+ """
1064+ if os.path.exists(file_name):
1065+ self.update(self.read_context(file_name))
1066+ else:
1067+ self.store_context(file_name, config_data)
1068+ self.update(config_data)
1069+
1070+ def store_context(self, file_name, config_data):
1071+ if not os.path.isabs(file_name):
1072+ file_name = os.path.join(hookenv.charm_dir(), file_name)
1073+ with open(file_name, 'w') as file_stream:
1074+ os.fchmod(file_stream.fileno(), 0600)
1075+ yaml.dump(config_data, file_stream)
1076+
1077+ def read_context(self, file_name):
1078+ if not os.path.isabs(file_name):
1079+ file_name = os.path.join(hookenv.charm_dir(), file_name)
1080+ with open(file_name, 'r') as file_stream:
1081+ data = yaml.load(file_stream)
1082+ if not data:
1083+ raise OSError("%s is empty" % file_name)
1084+ return data
1085+
1086+
1087+class TemplateCallback(ManagerCallback):
1088+ """
1089+ Callback class that will render a Jinja2 template, for use as a ready action.
1090+
1091+ :param str source: The template source file, relative to `$CHARM_DIR/templates`
1092+ :param str target: The target to write the rendered template to
1093+ :param str owner: The owner of the rendered file
1094+ :param str group: The group of the rendered file
1095+ :param int perms: The permissions of the rendered file
1096+ """
1097+ def __init__(self, source, target, owner='root', group='root', perms=0444):
1098+ self.source = source
1099+ self.target = target
1100+ self.owner = owner
1101+ self.group = group
1102+ self.perms = perms
1103+
1104+ def __call__(self, manager, service_name, event_name):
1105+ service = manager.get_service(service_name)
1106+ context = {}
1107+ for ctx in service.get('required_data', []):
1108+ context.update(ctx)
1109+ templating.render(self.source, self.target, context,
1110+ self.owner, self.group, self.perms)
1111+
1112+
1113+# Convenience aliases for templates
1114+render_template = template = TemplateCallback
1115
1116=== added file 'hooks/charmhelpers/core/templating.py'
1117--- hooks/charmhelpers/core/templating.py 1970-01-01 00:00:00 +0000
1118+++ hooks/charmhelpers/core/templating.py 2014-09-26 07:21:45 +0000
1119@@ -0,0 +1,51 @@
1120+import os
1121+
1122+from charmhelpers.core import host
1123+from charmhelpers.core import hookenv
1124+
1125+
1126+def render(source, target, context, owner='root', group='root', perms=0444, templates_dir=None):
1127+ """
1128+ Render a template.
1129+
1130+ The `source` path, if not absolute, is relative to the `templates_dir`.
1131+
1132+ The `target` path should be absolute.
1133+
1134+ The context should be a dict containing the values to be replaced in the
1135+ template.
1136+
1137+ The `owner`, `group`, and `perms` options will be passed to `write_file`.
1138+
1139+ If omitted, `templates_dir` defaults to the `templates` folder in the charm.
1140+
1141+ Note: Using this requires python-jinja2; if it is not installed, calling
1142+ this will attempt to use charmhelpers.fetch.apt_install to install it.
1143+ """
1144+ try:
1145+ from jinja2 import FileSystemLoader, Environment, exceptions
1146+ except ImportError:
1147+ try:
1148+ from charmhelpers.fetch import apt_install
1149+ except ImportError:
1150+ hookenv.log('Could not import jinja2, and could not import '
1151+ 'charmhelpers.fetch to install it',
1152+ level=hookenv.ERROR)
1153+ raise
1154+ apt_install('python-jinja2', fatal=True)
1155+ from jinja2 import FileSystemLoader, Environment, exceptions
1156+
1157+ if templates_dir is None:
1158+ templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
1159+ loader = Environment(loader=FileSystemLoader(templates_dir))
1160+ try:
1161+ source = source
1162+ template = loader.get_template(source)
1163+ except exceptions.TemplateNotFound as e:
1164+ hookenv.log('Could not load template %s from %s.' %
1165+ (source, templates_dir),
1166+ level=hookenv.ERROR)
1167+ raise e
1168+ content = template.render(context)
1169+ host.mkdir(os.path.dirname(target))
1170+ host.write_file(target, content, owner, group, perms)
1171
1172=== modified file 'hooks/charmhelpers/fetch/__init__.py'
1173--- hooks/charmhelpers/fetch/__init__.py 2014-05-08 10:22:43 +0000
1174+++ hooks/charmhelpers/fetch/__init__.py 2014-09-26 07:21:45 +0000
1175@@ -1,4 +1,6 @@
1176 import importlib
1177+from tempfile import NamedTemporaryFile
1178+import time
1179 from yaml import safe_load
1180 from charmhelpers.core.host import (
1181 lsb_release
1182@@ -12,9 +14,9 @@
1183 config,
1184 log,
1185 )
1186-import apt_pkg
1187 import os
1188
1189+
1190 CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
1191 deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
1192 """
1193@@ -54,13 +56,68 @@
1194 'icehouse/proposed': 'precise-proposed/icehouse',
1195 'precise-icehouse/proposed': 'precise-proposed/icehouse',
1196 'precise-proposed/icehouse': 'precise-proposed/icehouse',
1197+ # Juno
1198+ 'juno': 'trusty-updates/juno',
1199+ 'trusty-juno': 'trusty-updates/juno',
1200+ 'trusty-juno/updates': 'trusty-updates/juno',
1201+ 'trusty-updates/juno': 'trusty-updates/juno',
1202+ 'juno/proposed': 'trusty-proposed/juno',
1203+ 'juno/proposed': 'trusty-proposed/juno',
1204+ 'trusty-juno/proposed': 'trusty-proposed/juno',
1205+ 'trusty-proposed/juno': 'trusty-proposed/juno',
1206 }
1207
1208+# The order of this list is very important. Handlers should be listed in from
1209+# least- to most-specific URL matching.
1210+FETCH_HANDLERS = (
1211+ 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
1212+ 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
1213+)
1214+
1215+APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
1216+APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
1217+APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
1218+
1219+
1220+class SourceConfigError(Exception):
1221+ pass
1222+
1223+
1224+class UnhandledSource(Exception):
1225+ pass
1226+
1227+
1228+class AptLockError(Exception):
1229+ pass
1230+
1231+
1232+class BaseFetchHandler(object):
1233+
1234+ """Base class for FetchHandler implementations in fetch plugins"""
1235+
1236+ def can_handle(self, source):
1237+ """Returns True if the source can be handled. Otherwise returns
1238+ a string explaining why it cannot"""
1239+ return "Wrong source type"
1240+
1241+ def install(self, source):
1242+ """Try to download and unpack the source. Return the path to the
1243+ unpacked files or raise UnhandledSource."""
1244+ raise UnhandledSource("Wrong source type {}".format(source))
1245+
1246+ def parse_url(self, url):
1247+ return urlparse(url)
1248+
1249+ def base_url(self, url):
1250+ """Return url without querystring or fragment"""
1251+ parts = list(self.parse_url(url))
1252+ parts[4:] = ['' for i in parts[4:]]
1253+ return urlunparse(parts)
1254+
1255
1256 def filter_installed_packages(packages):
1257 """Returns a list of packages that require installation"""
1258- apt_pkg.init()
1259- cache = apt_pkg.Cache()
1260+ cache = apt_cache()
1261 _pkgs = []
1262 for package in packages:
1263 try:
1264@@ -73,6 +130,16 @@
1265 return _pkgs
1266
1267
1268+def apt_cache(in_memory=True):
1269+ """Build and return an apt cache"""
1270+ import apt_pkg
1271+ apt_pkg.init()
1272+ if in_memory:
1273+ apt_pkg.config.set("Dir::Cache::pkgcache", "")
1274+ apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
1275+ return apt_pkg.Cache()
1276+
1277+
1278 def apt_install(packages, options=None, fatal=False):
1279 """Install one or more packages"""
1280 if options is None:
1281@@ -87,14 +154,7 @@
1282 cmd.extend(packages)
1283 log("Installing {} with options: {}".format(packages,
1284 options))
1285- env = os.environ.copy()
1286- if 'DEBIAN_FRONTEND' not in env:
1287- env['DEBIAN_FRONTEND'] = 'noninteractive'
1288-
1289- if fatal:
1290- subprocess.check_call(cmd, env=env)
1291- else:
1292- subprocess.call(cmd, env=env)
1293+ _run_apt_command(cmd, fatal)
1294
1295
1296 def apt_upgrade(options=None, fatal=False, dist=False):
1297@@ -109,24 +169,13 @@
1298 else:
1299 cmd.append('upgrade')
1300 log("Upgrading with options: {}".format(options))
1301-
1302- env = os.environ.copy()
1303- if 'DEBIAN_FRONTEND' not in env:
1304- env['DEBIAN_FRONTEND'] = 'noninteractive'
1305-
1306- if fatal:
1307- subprocess.check_call(cmd, env=env)
1308- else:
1309- subprocess.call(cmd, env=env)
1310+ _run_apt_command(cmd, fatal)
1311
1312
1313 def apt_update(fatal=False):
1314 """Update local apt cache"""
1315 cmd = ['apt-get', 'update']
1316- if fatal:
1317- subprocess.check_call(cmd)
1318- else:
1319- subprocess.call(cmd)
1320+ _run_apt_command(cmd, fatal)
1321
1322
1323 def apt_purge(packages, fatal=False):
1324@@ -137,10 +186,7 @@
1325 else:
1326 cmd.extend(packages)
1327 log("Purging {}".format(packages))
1328- if fatal:
1329- subprocess.check_call(cmd)
1330- else:
1331- subprocess.call(cmd)
1332+ _run_apt_command(cmd, fatal)
1333
1334
1335 def apt_hold(packages, fatal=False):
1336@@ -151,6 +197,7 @@
1337 else:
1338 cmd.extend(packages)
1339 log("Holding {}".format(packages))
1340+
1341 if fatal:
1342 subprocess.check_call(cmd)
1343 else:
1344@@ -158,6 +205,28 @@
1345
1346
1347 def add_source(source, key=None):
1348+ """Add a package source to this system.
1349+
1350+ @param source: a URL or sources.list entry, as supported by
1351+ add-apt-repository(1). Examples::
1352+
1353+ ppa:charmers/example
1354+ deb https://stub:key@private.example.com/ubuntu trusty main
1355+
1356+ In addition:
1357+ 'proposed:' may be used to enable the standard 'proposed'
1358+ pocket for the release.
1359+ 'cloud:' may be used to activate official cloud archive pockets,
1360+ such as 'cloud:icehouse'
1361+
1362+ @param key: A key to be added to the system's APT keyring and used
1363+ to verify the signatures on packages. Ideally, this should be an
1364+ ASCII format GPG public key including the block headers. A GPG key
1365+ id may also be used, but be aware that only insecure protocols are
1366+ available to retrieve the actual public key from a public keyserver
1367+ placing your Juju environment at risk. ppa and cloud archive keys
1368+ are securely added automtically, so sould not be provided.
1369+ """
1370 if source is None:
1371 log('Source is not present. Skipping')
1372 return
1373@@ -182,76 +251,96 @@
1374 release = lsb_release()['DISTRIB_CODENAME']
1375 with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
1376 apt.write(PROPOSED_POCKET.format(release))
1377+ else:
1378+ raise SourceConfigError("Unknown source: {!r}".format(source))
1379+
1380 if key:
1381- subprocess.check_call(['apt-key', 'adv', '--keyserver',
1382- 'hkp://keyserver.ubuntu.com:80', '--recv',
1383- key])
1384-
1385-
1386-class SourceConfigError(Exception):
1387- pass
1388+ if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
1389+ with NamedTemporaryFile() as key_file:
1390+ key_file.write(key)
1391+ key_file.flush()
1392+ key_file.seek(0)
1393+ subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
1394+ else:
1395+ # Note that hkp: is in no way a secure protocol. Using a
1396+ # GPG key id is pointless from a security POV unless you
1397+ # absolutely trust your network and DNS.
1398+ subprocess.check_call(['apt-key', 'adv', '--keyserver',
1399+ 'hkp://keyserver.ubuntu.com:80', '--recv',
1400+ key])
1401
1402
1403 def configure_sources(update=False,
1404 sources_var='install_sources',
1405 keys_var='install_keys'):
1406 """
1407- Configure multiple sources from charm configuration
1408+ Configure multiple sources from charm configuration.
1409+
1410+ The lists are encoded as yaml fragments in the configuration.
1411+ The frament needs to be included as a string. Sources and their
1412+ corresponding keys are of the types supported by add_source().
1413
1414 Example config:
1415- install_sources:
1416+ install_sources: |
1417 - "ppa:foo"
1418 - "http://example.com/repo precise main"
1419- install_keys:
1420+ install_keys: |
1421 - null
1422 - "a1b2c3d4"
1423
1424 Note that 'null' (a.k.a. None) should not be quoted.
1425 """
1426- sources = safe_load(config(sources_var))
1427- keys = config(keys_var)
1428- if keys is not None:
1429- keys = safe_load(keys)
1430- if isinstance(sources, basestring) and (
1431- keys is None or isinstance(keys, basestring)):
1432- add_source(sources, keys)
1433+ sources = safe_load((config(sources_var) or '').strip()) or []
1434+ keys = safe_load((config(keys_var) or '').strip()) or None
1435+
1436+ if isinstance(sources, basestring):
1437+ sources = [sources]
1438+
1439+ if keys is None:
1440+ for source in sources:
1441+ add_source(source, None)
1442 else:
1443- if not len(sources) == len(keys):
1444- msg = 'Install sources and keys lists are different lengths'
1445- raise SourceConfigError(msg)
1446- for src_num in range(len(sources)):
1447- add_source(sources[src_num], keys[src_num])
1448+ if isinstance(keys, basestring):
1449+ keys = [keys]
1450+
1451+ if len(sources) != len(keys):
1452+ raise SourceConfigError(
1453+ 'Install sources and keys lists are different lengths')
1454+ for source, key in zip(sources, keys):
1455+ add_source(source, key)
1456 if update:
1457 apt_update(fatal=True)
1458
1459-# The order of this list is very important. Handlers should be listed in from
1460-# least- to most-specific URL matching.
1461-FETCH_HANDLERS = (
1462- 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
1463- 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
1464-)
1465-
1466-
1467-class UnhandledSource(Exception):
1468- pass
1469-
1470-
1471-def install_remote(source):
1472+
1473+def install_remote(source, *args, **kwargs):
1474 """
1475 Install a file tree from a remote source
1476
1477 The specified source should be a url of the form:
1478 scheme://[host]/path[#[option=value][&...]]
1479
1480- Schemes supported are based on this modules submodules
1481- Options supported are submodule-specific"""
1482+ Schemes supported are based on this modules submodules.
1483+ Options supported are submodule-specific.
1484+ Additional arguments are passed through to the submodule.
1485+
1486+ For example::
1487+
1488+ dest = install_remote('http://example.com/archive.tgz',
1489+ checksum='deadbeef',
1490+ hash_type='sha1')
1491+
1492+ This will download `archive.tgz`, validate it using SHA1 and, if
1493+ the file is ok, extract it and return the directory in which it
1494+ was extracted. If the checksum fails, it will raise
1495+ :class:`charmhelpers.core.host.ChecksumError`.
1496+ """
1497 # We ONLY check for True here because can_handle may return a string
1498 # explaining why it can't handle a given source.
1499 handlers = [h for h in plugins() if h.can_handle(source) is True]
1500 installed_to = None
1501 for handler in handlers:
1502 try:
1503- installed_to = handler.install(source)
1504+ installed_to = handler.install(source, *args, **kwargs)
1505 except UnhandledSource:
1506 pass
1507 if not installed_to:
1508@@ -265,30 +354,6 @@
1509 return install_remote(source)
1510
1511
1512-class BaseFetchHandler(object):
1513-
1514- """Base class for FetchHandler implementations in fetch plugins"""
1515-
1516- def can_handle(self, source):
1517- """Returns True if the source can be handled. Otherwise returns
1518- a string explaining why it cannot"""
1519- return "Wrong source type"
1520-
1521- def install(self, source):
1522- """Try to download and unpack the source. Return the path to the
1523- unpacked files or raise UnhandledSource."""
1524- raise UnhandledSource("Wrong source type {}".format(source))
1525-
1526- def parse_url(self, url):
1527- return urlparse(url)
1528-
1529- def base_url(self, url):
1530- """Return url without querystring or fragment"""
1531- parts = list(self.parse_url(url))
1532- parts[4:] = ['' for i in parts[4:]]
1533- return urlunparse(parts)
1534-
1535-
1536 def plugins(fetch_handlers=None):
1537 if not fetch_handlers:
1538 fetch_handlers = FETCH_HANDLERS
1539@@ -306,3 +371,40 @@
1540 log("FetchHandler {} not found, skipping plugin".format(
1541 handler_name))
1542 return plugin_list
1543+
1544+
1545+def _run_apt_command(cmd, fatal=False):
1546+ """
1547+ Run an APT command, checking output and retrying if the fatal flag is set
1548+ to True.
1549+
1550+ :param: cmd: str: The apt command to run.
1551+ :param: fatal: bool: Whether the command's output should be checked and
1552+ retried.
1553+ """
1554+ env = os.environ.copy()
1555+
1556+ if 'DEBIAN_FRONTEND' not in env:
1557+ env['DEBIAN_FRONTEND'] = 'noninteractive'
1558+
1559+ if fatal:
1560+ retry_count = 0
1561+ result = None
1562+
1563+ # If the command is considered "fatal", we need to retry if the apt
1564+ # lock was not acquired.
1565+
1566+ while result is None or result == APT_NO_LOCK:
1567+ try:
1568+ result = subprocess.check_call(cmd, env=env)
1569+ except subprocess.CalledProcessError, e:
1570+ retry_count = retry_count + 1
1571+ if retry_count > APT_NO_LOCK_RETRY_COUNT:
1572+ raise
1573+ result = e.returncode
1574+ log("Couldn't acquire DPKG lock. Will retry in {} seconds."
1575+ "".format(APT_NO_LOCK_RETRY_DELAY))
1576+ time.sleep(APT_NO_LOCK_RETRY_DELAY)
1577+
1578+ else:
1579+ subprocess.call(cmd, env=env)
1580
1581=== modified file 'hooks/charmhelpers/fetch/archiveurl.py'
1582--- hooks/charmhelpers/fetch/archiveurl.py 2014-04-17 10:53:00 +0000
1583+++ hooks/charmhelpers/fetch/archiveurl.py 2014-09-26 07:21:45 +0000
1584@@ -1,6 +1,8 @@
1585 import os
1586 import urllib2
1587+from urllib import urlretrieve
1588 import urlparse
1589+import hashlib
1590
1591 from charmhelpers.fetch import (
1592 BaseFetchHandler,
1593@@ -10,11 +12,19 @@
1594 get_archive_handler,
1595 extract,
1596 )
1597-from charmhelpers.core.host import mkdir
1598+from charmhelpers.core.host import mkdir, check_hash
1599
1600
1601 class ArchiveUrlFetchHandler(BaseFetchHandler):
1602- """Handler for archives via generic URLs"""
1603+ """
1604+ Handler to download archive files from arbitrary URLs.
1605+
1606+ Can fetch from http, https, ftp, and file URLs.
1607+
1608+ Can install either tarballs (.tar, .tgz, .tbz2, etc) or zip files.
1609+
1610+ Installs the contents of the archive in $CHARM_DIR/fetched/.
1611+ """
1612 def can_handle(self, source):
1613 url_parts = self.parse_url(source)
1614 if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
1615@@ -24,6 +34,12 @@
1616 return False
1617
1618 def download(self, source, dest):
1619+ """
1620+ Download an archive file.
1621+
1622+ :param str source: URL pointing to an archive file.
1623+ :param str dest: Local path location to download archive file to.
1624+ """
1625 # propogate all exceptions
1626 # URLError, OSError, etc
1627 proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
1628@@ -48,7 +64,30 @@
1629 os.unlink(dest)
1630 raise e
1631
1632- def install(self, source):
1633+ # Mandatory file validation via Sha1 or MD5 hashing.
1634+ def download_and_validate(self, url, hashsum, validate="sha1"):
1635+ tempfile, headers = urlretrieve(url)
1636+ check_hash(tempfile, hashsum, validate)
1637+ return tempfile
1638+
1639+ def install(self, source, dest=None, checksum=None, hash_type='sha1'):
1640+ """
1641+ Download and install an archive file, with optional checksum validation.
1642+
1643+ The checksum can also be given on the `source` URL's fragment.
1644+ For example::
1645+
1646+ handler.install('http://example.com/file.tgz#sha1=deadbeef')
1647+
1648+ :param str source: URL pointing to an archive file.
1649+ :param str dest: Local destination path to install to. If not given,
1650+ installs to `$CHARM_DIR/archives/archive_file_name`.
1651+ :param str checksum: If given, validate the archive file after download.
1652+ :param str hash_type: Algorithm used to generate `checksum`.
1653+ Can be any hash alrgorithm supported by :mod:`hashlib`,
1654+ such as md5, sha1, sha256, sha512, etc.
1655+
1656+ """
1657 url_parts = self.parse_url(source)
1658 dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
1659 if not os.path.exists(dest_dir):
1660@@ -60,4 +99,10 @@
1661 raise UnhandledSource(e.reason)
1662 except OSError as e:
1663 raise UnhandledSource(e.strerror)
1664- return extract(dld_file)
1665+ options = urlparse.parse_qs(url_parts.fragment)
1666+ for key, value in options.items():
1667+ if key in hashlib.algorithms:
1668+ check_hash(dld_file, value, key)
1669+ if checksum:
1670+ check_hash(dld_file, checksum, hash_type)
1671+ return extract(dld_file, dest)
1672
1673=== modified file 'hooks/charmhelpers/fetch/bzrurl.py'
1674--- hooks/charmhelpers/fetch/bzrurl.py 2014-02-19 14:49:31 +0000
1675+++ hooks/charmhelpers/fetch/bzrurl.py 2014-09-26 07:21:45 +0000
1676@@ -39,7 +39,8 @@
1677 def install(self, source):
1678 url_parts = self.parse_url(source)
1679 branch_name = url_parts.path.strip("/").split("/")[-1]
1680- dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name)
1681+ dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
1682+ branch_name)
1683 if not os.path.exists(dest_dir):
1684 mkdir(dest_dir, perms=0755)
1685 try:

Subscribers

People subscribed via source and target branches

to all changes: