Merge lp:~blr/charms/precise/squid-forwardproxy/fix-unit-state into lp:~canonical-launchpad-branches/charms/precise/squid-forwardproxy/trunk

Proposed by Kit Randel on 2015-12-15
Status: Merged
Merged at revision: 35
Proposed branch: lp:~blr/charms/precise/squid-forwardproxy/fix-unit-state
Merge into: lp:~canonical-launchpad-branches/charms/precise/squid-forwardproxy/trunk
Diff against target: 3974 lines (+3760/-43)
20 files modified
Makefile (+13/-0)
charm-helpers.yaml (+4/-0)
hooks/charmhelpers/__init__.py (+38/-0)
hooks/charmhelpers/core/__init__.py (+15/-0)
hooks/charmhelpers/core/decorators.py (+57/-0)
hooks/charmhelpers/core/files.py (+45/-0)
hooks/charmhelpers/core/fstab.py (+134/-0)
hooks/charmhelpers/core/hookenv.py (+978/-0)
hooks/charmhelpers/core/host.py (+641/-0)
hooks/charmhelpers/core/hugepage.py (+71/-0)
hooks/charmhelpers/core/kernel.py (+68/-0)
hooks/charmhelpers/core/services/__init__.py (+18/-0)
hooks/charmhelpers/core/services/base.py (+353/-0)
hooks/charmhelpers/core/services/helpers.py (+292/-0)
hooks/charmhelpers/core/strutils.py (+72/-0)
hooks/charmhelpers/core/sysctl.py (+56/-0)
hooks/charmhelpers/core/templating.py (+81/-0)
hooks/charmhelpers/core/unitdata.py (+521/-0)
hooks/hooks.py (+50/-43)
scripts/charm_helpers_sync.py (+253/-0)
To merge this branch: bzr merge lp:~blr/charms/precise/squid-forwardproxy/fix-unit-state
Reviewer Review Type Date Requested Status
William Grant code 2015-12-15 Approve on 2015-12-17
Review via email: mp+280650@code.launchpad.net

Commit message

Manage local unit state with charmhelpers unitdata.kv.

Description of the change

This branch correctly manages the persistence of hook states 'start' and 'relation-auth-helper-joined' using the charmhelpers kv store, instead of the broken and nonsensical mess I merged earlier. Mea culpa.

To post a comment you must log in.
William Grant (wgrant) :
review: Needs Fixing (code)
63. By Kit Randel on 2015-12-16

Unset hook_start unitdata on stop hook.

64. By Kit Randel on 2015-12-16

Reloading squid3 is redundant given fallthrough restart case.

65. By Kit Randel on 2015-12-17

Remove unnecessary unitdata keys 'state_delayed_start' and 'hook_auth_helper_joined'.

66. By Kit Randel on 2015-12-17

* Remove potential vector for symlink attack.
* Update charmhelpers.

67. By Kit Randel on 2015-12-17

Factor out test for delayed service start.

68. By Kit Randel on 2015-12-17

Move invalid squid configuration CRITICAL log to service_squid3('check').

William Grant (wgrant) :
review: Approve (code)
69. By Kit Randel on 2015-12-17

Unset hook_start unitdata key, regardless of service status.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== added file 'Makefile'
--- Makefile 1970-01-01 00:00:00 +0000
+++ Makefile 2015-12-17 05:44:32 +0000
@@ -0,0 +1,13 @@
1# -*- mode: makefile -*-
2
3PYTHON := /usr/bin/env python
4
5all: sync
6
7sync:
8 @bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py > scripts/charm_helpers_sync.py
9 @echo "Syncing charmhelpers..."
10 @mkdir -p hooks/charmhelpers
11 @$(PYTHON) scripts/charm_helpers_sync.py -c charm-helpers.yaml
12
13
014
=== added file 'charm-helpers.yaml'
--- charm-helpers.yaml 1970-01-01 00:00:00 +0000
+++ charm-helpers.yaml 2015-12-17 05:44:32 +0000
@@ -0,0 +1,4 @@
1destination: hooks/charmhelpers
2branch: lp:charm-helpers
3include:
4 - core
0\ No newline at end of file5\ No newline at end of file
16
=== added directory 'hooks/charmhelpers'
=== added file 'hooks/charmhelpers/__init__.py'
--- hooks/charmhelpers/__init__.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/__init__.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,38 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
17# Bootstrap charm-helpers, installing its dependencies if necessary using
18# only standard libraries.
19import subprocess
20import sys
21
22try:
23 import six # flake8: noqa
24except ImportError:
25 if sys.version_info.major == 2:
26 subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
27 else:
28 subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
29 import six # flake8: noqa
30
31try:
32 import yaml # flake8: noqa
33except ImportError:
34 if sys.version_info.major == 2:
35 subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
36 else:
37 subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
38 import yaml # flake8: noqa
039
=== added directory 'hooks/charmhelpers/core'
=== added file 'hooks/charmhelpers/core/__init__.py'
--- hooks/charmhelpers/core/__init__.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/__init__.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,15 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
016
=== added file 'hooks/charmhelpers/core/decorators.py'
--- hooks/charmhelpers/core/decorators.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/decorators.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,57 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
17#
18# Copyright 2014 Canonical Ltd.
19#
20# Authors:
21# Edward Hope-Morley <opentastic@gmail.com>
22#
23
24import time
25
26from charmhelpers.core.hookenv import (
27 log,
28 INFO,
29)
30
31
32def retry_on_exception(num_retries, base_delay=0, exc_type=Exception):
33 """If the decorated function raises exception exc_type, allow num_retries
34 retry attempts before raise the exception.
35 """
36 def _retry_on_exception_inner_1(f):
37 def _retry_on_exception_inner_2(*args, **kwargs):
38 retries = num_retries
39 multiplier = 1
40 while True:
41 try:
42 return f(*args, **kwargs)
43 except exc_type:
44 if not retries:
45 raise
46
47 delay = base_delay * multiplier
48 multiplier += 1
49 log("Retrying '%s' %d more times (delay=%s)" %
50 (f.__name__, retries, delay), level=INFO)
51 retries -= 1
52 if delay:
53 time.sleep(delay)
54
55 return _retry_on_exception_inner_2
56
57 return _retry_on_exception_inner_1
058
=== added file 'hooks/charmhelpers/core/files.py'
--- hooks/charmhelpers/core/files.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/files.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,45 @@
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4# Copyright 2014-2015 Canonical Limited.
5#
6# This file is part of charm-helpers.
7#
8# charm-helpers is free software: you can redistribute it and/or modify
9# it under the terms of the GNU Lesser General Public License version 3 as
10# published by the Free Software Foundation.
11#
12# charm-helpers is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU Lesser General Public License for more details.
16#
17# You should have received a copy of the GNU Lesser General Public License
18# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
19
20__author__ = 'Jorge Niedbalski <niedbalski@ubuntu.com>'
21
22import os
23import subprocess
24
25
26def sed(filename, before, after, flags='g'):
27 """
28 Search and replaces the given pattern on filename.
29
30 :param filename: relative or absolute file path.
31 :param before: expression to be replaced (see 'man sed')
32 :param after: expression to replace with (see 'man sed')
33 :param flags: sed-compatible regex flags in example, to make
34 the search and replace case insensitive, specify ``flags="i"``.
35 The ``g`` flag is always specified regardless, so you do not
36 need to remember to include it when overriding this parameter.
37 :returns: If the sed command exit code was zero then return,
38 otherwise raise CalledProcessError.
39 """
40 expression = r's/{0}/{1}/{2}'.format(before,
41 after, flags)
42
43 return subprocess.check_call(["sed", "-i", "-r", "-e",
44 expression,
45 os.path.expanduser(filename)])
046
=== added file 'hooks/charmhelpers/core/fstab.py'
--- hooks/charmhelpers/core/fstab.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/fstab.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,134 @@
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4# Copyright 2014-2015 Canonical Limited.
5#
6# This file is part of charm-helpers.
7#
8# charm-helpers is free software: you can redistribute it and/or modify
9# it under the terms of the GNU Lesser General Public License version 3 as
10# published by the Free Software Foundation.
11#
12# charm-helpers is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU Lesser General Public License for more details.
16#
17# You should have received a copy of the GNU Lesser General Public License
18# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
19
20import io
21import os
22
23__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
24
25
26class Fstab(io.FileIO):
27 """This class extends file in order to implement a file reader/writer
28 for file `/etc/fstab`
29 """
30
31 class Entry(object):
32 """Entry class represents a non-comment line on the `/etc/fstab` file
33 """
34 def __init__(self, device, mountpoint, filesystem,
35 options, d=0, p=0):
36 self.device = device
37 self.mountpoint = mountpoint
38 self.filesystem = filesystem
39
40 if not options:
41 options = "defaults"
42
43 self.options = options
44 self.d = int(d)
45 self.p = int(p)
46
47 def __eq__(self, o):
48 return str(self) == str(o)
49
50 def __str__(self):
51 return "{} {} {} {} {} {}".format(self.device,
52 self.mountpoint,
53 self.filesystem,
54 self.options,
55 self.d,
56 self.p)
57
58 DEFAULT_PATH = os.path.join(os.path.sep, 'etc', 'fstab')
59
60 def __init__(self, path=None):
61 if path:
62 self._path = path
63 else:
64 self._path = self.DEFAULT_PATH
65 super(Fstab, self).__init__(self._path, 'rb+')
66
67 def _hydrate_entry(self, line):
68 # NOTE: use split with no arguments to split on any
69 # whitespace including tabs
70 return Fstab.Entry(*filter(
71 lambda x: x not in ('', None),
72 line.strip("\n").split()))
73
74 @property
75 def entries(self):
76 self.seek(0)
77 for line in self.readlines():
78 line = line.decode('us-ascii')
79 try:
80 if line.strip() and not line.strip().startswith("#"):
81 yield self._hydrate_entry(line)
82 except ValueError:
83 pass
84
85 def get_entry_by_attr(self, attr, value):
86 for entry in self.entries:
87 e_attr = getattr(entry, attr)
88 if e_attr == value:
89 return entry
90 return None
91
92 def add_entry(self, entry):
93 if self.get_entry_by_attr('device', entry.device):
94 return False
95
96 self.write((str(entry) + '\n').encode('us-ascii'))
97 self.truncate()
98 return entry
99
100 def remove_entry(self, entry):
101 self.seek(0)
102
103 lines = [l.decode('us-ascii') for l in self.readlines()]
104
105 found = False
106 for index, line in enumerate(lines):
107 if line.strip() and not line.strip().startswith("#"):
108 if self._hydrate_entry(line) == entry:
109 found = True
110 break
111
112 if not found:
113 return False
114
115 lines.remove(line)
116
117 self.seek(0)
118 self.write(''.join(lines).encode('us-ascii'))
119 self.truncate()
120 return True
121
122 @classmethod
123 def remove_by_mountpoint(cls, mountpoint, path=None):
124 fstab = cls(path=path)
125 entry = fstab.get_entry_by_attr('mountpoint', mountpoint)
126 if entry:
127 return fstab.remove_entry(entry)
128 return False
129
130 @classmethod
131 def add(cls, device, mountpoint, filesystem, options=None, path=None):
132 return cls(path=path).add_entry(Fstab.Entry(device,
133 mountpoint, filesystem,
134 options=options))
0135
=== added file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/hookenv.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,978 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
17"Interactions with the Juju environment"
18# Copyright 2013 Canonical Ltd.
19#
20# Authors:
21# Charm Helpers Developers <juju@lists.ubuntu.com>
22
23from __future__ import print_function
24import copy
25from distutils.version import LooseVersion
26from functools import wraps
27import glob
28import os
29import json
30import yaml
31import subprocess
32import sys
33import errno
34import tempfile
35from subprocess import CalledProcessError
36
37import six
38if not six.PY3:
39 from UserDict import UserDict
40else:
41 from collections import UserDict
42
43CRITICAL = "CRITICAL"
44ERROR = "ERROR"
45WARNING = "WARNING"
46INFO = "INFO"
47DEBUG = "DEBUG"
48MARKER = object()
49
50cache = {}
51
52
53def cached(func):
54 """Cache return values for multiple executions of func + args
55
56 For example::
57
58 @cached
59 def unit_get(attribute):
60 pass
61
62 unit_get('test')
63
64 will cache the result of unit_get + 'test' for future calls.
65 """
66 @wraps(func)
67 def wrapper(*args, **kwargs):
68 global cache
69 key = str((func, args, kwargs))
70 try:
71 return cache[key]
72 except KeyError:
73 pass # Drop out of the exception handler scope.
74 res = func(*args, **kwargs)
75 cache[key] = res
76 return res
77 wrapper._wrapped = func
78 return wrapper
79
80
81def flush(key):
82 """Flushes any entries from function cache where the
83 key is found in the function+args """
84 flush_list = []
85 for item in cache:
86 if key in item:
87 flush_list.append(item)
88 for item in flush_list:
89 del cache[item]
90
91
92def log(message, level=None):
93 """Write a message to the juju log"""
94 command = ['juju-log']
95 if level:
96 command += ['-l', level]
97 if not isinstance(message, six.string_types):
98 message = repr(message)
99 command += [message]
100 # Missing juju-log should not cause failures in unit tests
101 # Send log output to stderr
102 try:
103 subprocess.call(command)
104 except OSError as e:
105 if e.errno == errno.ENOENT:
106 if level:
107 message = "{}: {}".format(level, message)
108 message = "juju-log: {}".format(message)
109 print(message, file=sys.stderr)
110 else:
111 raise
112
113
114class Serializable(UserDict):
115 """Wrapper, an object that can be serialized to yaml or json"""
116
117 def __init__(self, obj):
118 # wrap the object
119 UserDict.__init__(self)
120 self.data = obj
121
122 def __getattr__(self, attr):
123 # See if this object has attribute.
124 if attr in ("json", "yaml", "data"):
125 return self.__dict__[attr]
126 # Check for attribute in wrapped object.
127 got = getattr(self.data, attr, MARKER)
128 if got is not MARKER:
129 return got
130 # Proxy to the wrapped object via dict interface.
131 try:
132 return self.data[attr]
133 except KeyError:
134 raise AttributeError(attr)
135
136 def __getstate__(self):
137 # Pickle as a standard dictionary.
138 return self.data
139
140 def __setstate__(self, state):
141 # Unpickle into our wrapper.
142 self.data = state
143
144 def json(self):
145 """Serialize the object to json"""
146 return json.dumps(self.data)
147
148 def yaml(self):
149 """Serialize the object to yaml"""
150 return yaml.dump(self.data)
151
152
153def execution_environment():
154 """A convenient bundling of the current execution context"""
155 context = {}
156 context['conf'] = config()
157 if relation_id():
158 context['reltype'] = relation_type()
159 context['relid'] = relation_id()
160 context['rel'] = relation_get()
161 context['unit'] = local_unit()
162 context['rels'] = relations()
163 context['env'] = os.environ
164 return context
165
166
167def in_relation_hook():
168 """Determine whether we're running in a relation hook"""
169 return 'JUJU_RELATION' in os.environ
170
171
172def relation_type():
173 """The scope for the current relation hook"""
174 return os.environ.get('JUJU_RELATION', None)
175
176
177@cached
178def relation_id(relation_name=None, service_or_unit=None):
179 """The relation ID for the current or a specified relation"""
180 if not relation_name and not service_or_unit:
181 return os.environ.get('JUJU_RELATION_ID', None)
182 elif relation_name and service_or_unit:
183 service_name = service_or_unit.split('/')[0]
184 for relid in relation_ids(relation_name):
185 remote_service = remote_service_name(relid)
186 if remote_service == service_name:
187 return relid
188 else:
189 raise ValueError('Must specify neither or both of relation_name and service_or_unit')
190
191
192def local_unit():
193 """Local unit ID"""
194 return os.environ['JUJU_UNIT_NAME']
195
196
197def remote_unit():
198 """The remote unit for the current relation hook"""
199 return os.environ.get('JUJU_REMOTE_UNIT', None)
200
201
202def service_name():
203 """The name service group this unit belongs to"""
204 return local_unit().split('/')[0]
205
206
207@cached
208def remote_service_name(relid=None):
209 """The remote service name for a given relation-id (or the current relation)"""
210 if relid is None:
211 unit = remote_unit()
212 else:
213 units = related_units(relid)
214 unit = units[0] if units else None
215 return unit.split('/')[0] if unit else None
216
217
218def hook_name():
219 """The name of the currently executing hook"""
220 return os.environ.get('JUJU_HOOK_NAME', os.path.basename(sys.argv[0]))
221
222
223class Config(dict):
224 """A dictionary representation of the charm's config.yaml, with some
225 extra features:
226
227 - See which values in the dictionary have changed since the previous hook.
228 - For values that have changed, see what the previous value was.
229 - Store arbitrary data for use in a later hook.
230
231 NOTE: Do not instantiate this object directly - instead call
232 ``hookenv.config()``, which will return an instance of :class:`Config`.
233
234 Example usage::
235
236 >>> # inside a hook
237 >>> from charmhelpers.core import hookenv
238 >>> config = hookenv.config()
239 >>> config['foo']
240 'bar'
241 >>> # store a new key/value for later use
242 >>> config['mykey'] = 'myval'
243
244
245 >>> # user runs `juju set mycharm foo=baz`
246 >>> # now we're inside subsequent config-changed hook
247 >>> config = hookenv.config()
248 >>> config['foo']
249 'baz'
250 >>> # test to see if this val has changed since last hook
251 >>> config.changed('foo')
252 True
253 >>> # what was the previous value?
254 >>> config.previous('foo')
255 'bar'
256 >>> # keys/values that we add are preserved across hooks
257 >>> config['mykey']
258 'myval'
259
260 """
261 CONFIG_FILE_NAME = '.juju-persistent-config'
262
263 def __init__(self, *args, **kw):
264 super(Config, self).__init__(*args, **kw)
265 self.implicit_save = True
266 self._prev_dict = None
267 self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
268 if os.path.exists(self.path):
269 self.load_previous()
270 atexit(self._implicit_save)
271
272 def load_previous(self, path=None):
273 """Load previous copy of config from disk.
274
275 In normal usage you don't need to call this method directly - it
276 is called automatically at object initialization.
277
278 :param path:
279
280 File path from which to load the previous config. If `None`,
281 config is loaded from the default location. If `path` is
282 specified, subsequent `save()` calls will write to the same
283 path.
284
285 """
286 self.path = path or self.path
287 with open(self.path) as f:
288 self._prev_dict = json.load(f)
289 for k, v in copy.deepcopy(self._prev_dict).items():
290 if k not in self:
291 self[k] = v
292
293 def changed(self, key):
294 """Return True if the current value for this key is different from
295 the previous value.
296
297 """
298 if self._prev_dict is None:
299 return True
300 return self.previous(key) != self.get(key)
301
302 def previous(self, key):
303 """Return previous value for this key, or None if there
304 is no previous value.
305
306 """
307 if self._prev_dict:
308 return self._prev_dict.get(key)
309 return None
310
311 def save(self):
312 """Save this config to disk.
313
314 If the charm is using the :mod:`Services Framework <services.base>`
315 or :meth:'@hook <Hooks.hook>' decorator, this
316 is called automatically at the end of successful hook execution.
317 Otherwise, it should be called directly by user code.
318
319 To disable automatic saves, set ``implicit_save=False`` on this
320 instance.
321
322 """
323 with open(self.path, 'w') as f:
324 json.dump(self, f)
325
326 def _implicit_save(self):
327 if self.implicit_save:
328 self.save()
329
330
331@cached
332def config(scope=None):
333 """Juju charm configuration"""
334 config_cmd_line = ['config-get']
335 if scope is not None:
336 config_cmd_line.append(scope)
337 config_cmd_line.append('--format=json')
338 try:
339 config_data = json.loads(
340 subprocess.check_output(config_cmd_line).decode('UTF-8'))
341 if scope is not None:
342 return config_data
343 return Config(config_data)
344 except ValueError:
345 return None
346
347
348@cached
349def relation_get(attribute=None, unit=None, rid=None):
350 """Get relation information"""
351 _args = ['relation-get', '--format=json']
352 if rid:
353 _args.append('-r')
354 _args.append(rid)
355 _args.append(attribute or '-')
356 if unit:
357 _args.append(unit)
358 try:
359 return json.loads(subprocess.check_output(_args).decode('UTF-8'))
360 except ValueError:
361 return None
362 except CalledProcessError as e:
363 if e.returncode == 2:
364 return None
365 raise
366
367
368def relation_set(relation_id=None, relation_settings=None, **kwargs):
369 """Set relation information for the current unit"""
370 relation_settings = relation_settings if relation_settings else {}
371 relation_cmd_line = ['relation-set']
372 accepts_file = "--file" in subprocess.check_output(
373 relation_cmd_line + ["--help"], universal_newlines=True)
374 if relation_id is not None:
375 relation_cmd_line.extend(('-r', relation_id))
376 settings = relation_settings.copy()
377 settings.update(kwargs)
378 for key, value in settings.items():
379 # Force value to be a string: it always should, but some call
380 # sites pass in things like dicts or numbers.
381 if value is not None:
382 settings[key] = "{}".format(value)
383 if accepts_file:
384 # --file was introduced in Juju 1.23.2. Use it by default if
385 # available, since otherwise we'll break if the relation data is
386 # too big. Ideally we should tell relation-set to read the data from
387 # stdin, but that feature is broken in 1.23.2: Bug #1454678.
388 with tempfile.NamedTemporaryFile(delete=False) as settings_file:
389 settings_file.write(yaml.safe_dump(settings).encode("utf-8"))
390 subprocess.check_call(
391 relation_cmd_line + ["--file", settings_file.name])
392 os.remove(settings_file.name)
393 else:
394 for key, value in settings.items():
395 if value is None:
396 relation_cmd_line.append('{}='.format(key))
397 else:
398 relation_cmd_line.append('{}={}'.format(key, value))
399 subprocess.check_call(relation_cmd_line)
400 # Flush cache of any relation-gets for local unit
401 flush(local_unit())
402
403
404def relation_clear(r_id=None):
405 ''' Clears any relation data already set on relation r_id '''
406 settings = relation_get(rid=r_id,
407 unit=local_unit())
408 for setting in settings:
409 if setting not in ['public-address', 'private-address']:
410 settings[setting] = None
411 relation_set(relation_id=r_id,
412 **settings)
413
414
415@cached
416def relation_ids(reltype=None):
417 """A list of relation_ids"""
418 reltype = reltype or relation_type()
419 relid_cmd_line = ['relation-ids', '--format=json']
420 if reltype is not None:
421 relid_cmd_line.append(reltype)
422 return json.loads(
423 subprocess.check_output(relid_cmd_line).decode('UTF-8')) or []
424 return []
425
426
427@cached
428def related_units(relid=None):
429 """A list of related units"""
430 relid = relid or relation_id()
431 units_cmd_line = ['relation-list', '--format=json']
432 if relid is not None:
433 units_cmd_line.extend(('-r', relid))
434 return json.loads(
435 subprocess.check_output(units_cmd_line).decode('UTF-8')) or []
436
437
438@cached
439def relation_for_unit(unit=None, rid=None):
440 """Get the json represenation of a unit's relation"""
441 unit = unit or remote_unit()
442 relation = relation_get(unit=unit, rid=rid)
443 for key in relation:
444 if key.endswith('-list'):
445 relation[key] = relation[key].split()
446 relation['__unit__'] = unit
447 return relation
448
449
450@cached
451def relations_for_id(relid=None):
452 """Get relations of a specific relation ID"""
453 relation_data = []
454 relid = relid or relation_ids()
455 for unit in related_units(relid):
456 unit_data = relation_for_unit(unit, relid)
457 unit_data['__relid__'] = relid
458 relation_data.append(unit_data)
459 return relation_data
460
461
462@cached
463def relations_of_type(reltype=None):
464 """Get relations of a specific type"""
465 relation_data = []
466 reltype = reltype or relation_type()
467 for relid in relation_ids(reltype):
468 for relation in relations_for_id(relid):
469 relation['__relid__'] = relid
470 relation_data.append(relation)
471 return relation_data
472
473
474@cached
475def metadata():
476 """Get the current charm metadata.yaml contents as a python object"""
477 with open(os.path.join(charm_dir(), 'metadata.yaml')) as md:
478 return yaml.safe_load(md)
479
480
481@cached
482def relation_types():
483 """Get a list of relation types supported by this charm"""
484 rel_types = []
485 md = metadata()
486 for key in ('provides', 'requires', 'peers'):
487 section = md.get(key)
488 if section:
489 rel_types.extend(section.keys())
490 return rel_types
491
492
493@cached
494def peer_relation_id():
495 '''Get the peers relation id if a peers relation has been joined, else None.'''
496 md = metadata()
497 section = md.get('peers')
498 if section:
499 for key in section:
500 relids = relation_ids(key)
501 if relids:
502 return relids[0]
503 return None
504
505
506@cached
507def relation_to_interface(relation_name):
508 """
509 Given the name of a relation, return the interface that relation uses.
510
511 :returns: The interface name, or ``None``.
512 """
513 return relation_to_role_and_interface(relation_name)[1]
514
515
516@cached
517def relation_to_role_and_interface(relation_name):
518 """
519 Given the name of a relation, return the role and the name of the interface
520 that relation uses (where role is one of ``provides``, ``requires``, or ``peers``).
521
522 :returns: A tuple containing ``(role, interface)``, or ``(None, None)``.
523 """
524 _metadata = metadata()
525 for role in ('provides', 'requires', 'peers'):
526 interface = _metadata.get(role, {}).get(relation_name, {}).get('interface')
527 if interface:
528 return role, interface
529 return None, None
530
531
532@cached
533def role_and_interface_to_relations(role, interface_name):
534 """
535 Given a role and interface name, return a list of relation names for the
536 current charm that use that interface under that role (where role is one
537 of ``provides``, ``requires``, or ``peers``).
538
539 :returns: A list of relation names.
540 """
541 _metadata = metadata()
542 results = []
543 for relation_name, relation in _metadata.get(role, {}).items():
544 if relation['interface'] == interface_name:
545 results.append(relation_name)
546 return results
547
548
549@cached
550def interface_to_relations(interface_name):
551 """
552 Given an interface, return a list of relation names for the current
553 charm that use that interface.
554
555 :returns: A list of relation names.
556 """
557 results = []
558 for role in ('provides', 'requires', 'peers'):
559 results.extend(role_and_interface_to_relations(role, interface_name))
560 return results
561
562
563@cached
564def charm_name():
565 """Get the name of the current charm as is specified on metadata.yaml"""
566 return metadata().get('name')
567
568
569@cached
570def relations():
571 """Get a nested dictionary of relation data for all related units"""
572 rels = {}
573 for reltype in relation_types():
574 relids = {}
575 for relid in relation_ids(reltype):
576 units = {local_unit(): relation_get(unit=local_unit(), rid=relid)}
577 for unit in related_units(relid):
578 reldata = relation_get(unit=unit, rid=relid)
579 units[unit] = reldata
580 relids[relid] = units
581 rels[reltype] = relids
582 return rels
583
584
585@cached
586def is_relation_made(relation, keys='private-address'):
587 '''
588 Determine whether a relation is established by checking for
589 presence of key(s). If a list of keys is provided, they
590 must all be present for the relation to be identified as made
591 '''
592 if isinstance(keys, str):
593 keys = [keys]
594 for r_id in relation_ids(relation):
595 for unit in related_units(r_id):
596 context = {}
597 for k in keys:
598 context[k] = relation_get(k, rid=r_id,
599 unit=unit)
600 if None not in context.values():
601 return True
602 return False
603
604
605def open_port(port, protocol="TCP"):
606 """Open a service network port"""
607 _args = ['open-port']
608 _args.append('{}/{}'.format(port, protocol))
609 subprocess.check_call(_args)
610
611
612def close_port(port, protocol="TCP"):
613 """Close a service network port"""
614 _args = ['close-port']
615 _args.append('{}/{}'.format(port, protocol))
616 subprocess.check_call(_args)
617
618
619@cached
620def unit_get(attribute):
621 """Get the unit ID for the remote unit"""
622 _args = ['unit-get', '--format=json', attribute]
623 try:
624 return json.loads(subprocess.check_output(_args).decode('UTF-8'))
625 except ValueError:
626 return None
627
628
629def unit_public_ip():
630 """Get this unit's public IP address"""
631 return unit_get('public-address')
632
633
634def unit_private_ip():
635 """Get this unit's private IP address"""
636 return unit_get('private-address')
637
638
639@cached
640def storage_get(attribute=None, storage_id=None):
641 """Get storage attributes"""
642 _args = ['storage-get', '--format=json']
643 if storage_id:
644 _args.extend(('-s', storage_id))
645 if attribute:
646 _args.append(attribute)
647 try:
648 return json.loads(subprocess.check_output(_args).decode('UTF-8'))
649 except ValueError:
650 return None
651
652
653@cached
654def storage_list(storage_name=None):
655 """List the storage IDs for the unit"""
656 _args = ['storage-list', '--format=json']
657 if storage_name:
658 _args.append(storage_name)
659 try:
660 return json.loads(subprocess.check_output(_args).decode('UTF-8'))
661 except ValueError:
662 return None
663 except OSError as e:
664 import errno
665 if e.errno == errno.ENOENT:
666 # storage-list does not exist
667 return []
668 raise
669
670
671class UnregisteredHookError(Exception):
672 """Raised when an undefined hook is called"""
673 pass
674
675
676class Hooks(object):
677 """A convenient handler for hook functions.
678
679 Example::
680
681 hooks = Hooks()
682
683 # register a hook, taking its name from the function name
684 @hooks.hook()
685 def install():
686 pass # your code here
687
688 # register a hook, providing a custom hook name
689 @hooks.hook("config-changed")
690 def config_changed():
691 pass # your code here
692
693 if __name__ == "__main__":
694 # execute a hook based on the name the program is called by
695 hooks.execute(sys.argv)
696 """
697
698 def __init__(self, config_save=None):
699 super(Hooks, self).__init__()
700 self._hooks = {}
701
702 # For unknown reasons, we allow the Hooks constructor to override
703 # config().implicit_save.
704 if config_save is not None:
705 config().implicit_save = config_save
706
707 def register(self, name, function):
708 """Register a hook"""
709 self._hooks[name] = function
710
711 def execute(self, args):
712 """Execute a registered hook based on args[0]"""
713 _run_atstart()
714 hook_name = os.path.basename(args[0])
715 if hook_name in self._hooks:
716 try:
717 self._hooks[hook_name]()
718 except SystemExit as x:
719 if x.code is None or x.code == 0:
720 _run_atexit()
721 raise
722 _run_atexit()
723 else:
724 raise UnregisteredHookError(hook_name)
725
726 def hook(self, *hook_names):
727 """Decorator, registering them as hooks"""
728 def wrapper(decorated):
729 for hook_name in hook_names:
730 self.register(hook_name, decorated)
731 else:
732 self.register(decorated.__name__, decorated)
733 if '_' in decorated.__name__:
734 self.register(
735 decorated.__name__.replace('_', '-'), decorated)
736 return decorated
737 return wrapper
738
739
740def charm_dir():
741 """Return the root directory of the current charm"""
742 return os.environ.get('CHARM_DIR')
743
744
745@cached
746def action_get(key=None):
747 """Gets the value of an action parameter, or all key/value param pairs"""
748 cmd = ['action-get']
749 if key is not None:
750 cmd.append(key)
751 cmd.append('--format=json')
752 action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8'))
753 return action_data
754
755
756def action_set(values):
757 """Sets the values to be returned after the action finishes"""
758 cmd = ['action-set']
759 for k, v in list(values.items()):
760 cmd.append('{}={}'.format(k, v))
761 subprocess.check_call(cmd)
762
763
764def action_fail(message):
765 """Sets the action status to failed and sets the error message.
766
767 The results set by action_set are preserved."""
768 subprocess.check_call(['action-fail', message])
769
770
771def action_name():
772 """Get the name of the currently executing action."""
773 return os.environ.get('JUJU_ACTION_NAME')
774
775
776def action_uuid():
777 """Get the UUID of the currently executing action."""
778 return os.environ.get('JUJU_ACTION_UUID')
779
780
781def action_tag():
782 """Get the tag for the currently executing action."""
783 return os.environ.get('JUJU_ACTION_TAG')
784
785
786def status_set(workload_state, message):
787 """Set the workload state with a message
788
789 Use status-set to set the workload state with a message which is visible
790 to the user via juju status. If the status-set command is not found then
791 assume this is juju < 1.23 and juju-log the message unstead.
792
793 workload_state -- valid juju workload state.
794 message -- status update message
795 """
796 valid_states = ['maintenance', 'blocked', 'waiting', 'active']
797 if workload_state not in valid_states:
798 raise ValueError(
799 '{!r} is not a valid workload state'.format(workload_state)
800 )
801 cmd = ['status-set', workload_state, message]
802 try:
803 ret = subprocess.call(cmd)
804 if ret == 0:
805 return
806 except OSError as e:
807 if e.errno != errno.ENOENT:
808 raise
809 log_message = 'status-set failed: {} {}'.format(workload_state,
810 message)
811 log(log_message, level='INFO')
812
813
814def status_get():
815 """Retrieve the previously set juju workload state and message
816
817 If the status-get command is not found then assume this is juju < 1.23 and
818 return 'unknown', ""
819
820 """
821 cmd = ['status-get', "--format=json", "--include-data"]
822 try:
823 raw_status = subprocess.check_output(cmd)
824 except OSError as e:
825 if e.errno == errno.ENOENT:
826 return ('unknown', "")
827 else:
828 raise
829 else:
830 status = json.loads(raw_status.decode("UTF-8"))
831 return (status["status"], status["message"])
832
833
834def translate_exc(from_exc, to_exc):
835 def inner_translate_exc1(f):
836 @wraps(f)
837 def inner_translate_exc2(*args, **kwargs):
838 try:
839 return f(*args, **kwargs)
840 except from_exc:
841 raise to_exc
842
843 return inner_translate_exc2
844
845 return inner_translate_exc1
846
847
848@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
849def is_leader():
850 """Does the current unit hold the juju leadership
851
852 Uses juju to determine whether the current unit is the leader of its peers
853 """
854 cmd = ['is-leader', '--format=json']
855 return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
856
857
858@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
859def leader_get(attribute=None):
860 """Juju leader get value(s)"""
861 cmd = ['leader-get', '--format=json'] + [attribute or '-']
862 return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
863
864
865@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
866def leader_set(settings=None, **kwargs):
867 """Juju leader set value(s)"""
868 # Don't log secrets.
869 # log("Juju leader-set '%s'" % (settings), level=DEBUG)
870 cmd = ['leader-set']
871 settings = settings or {}
872 settings.update(kwargs)
873 for k, v in settings.items():
874 if v is None:
875 cmd.append('{}='.format(k))
876 else:
877 cmd.append('{}={}'.format(k, v))
878 subprocess.check_call(cmd)
879
880
881@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
882def payload_register(ptype, klass, pid):
883 """ is used while a hook is running to let Juju know that a
884 payload has been started."""
885 cmd = ['payload-register']
886 for x in [ptype, klass, pid]:
887 cmd.append(x)
888 subprocess.check_call(cmd)
889
890
891@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
892def payload_unregister(klass, pid):
893 """ is used while a hook is running to let Juju know
894 that a payload has been manually stopped. The <class> and <id> provided
895 must match a payload that has been previously registered with juju using
896 payload-register."""
897 cmd = ['payload-unregister']
898 for x in [klass, pid]:
899 cmd.append(x)
900 subprocess.check_call(cmd)
901
902
903@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
904def payload_status_set(klass, pid, status):
905 """is used to update the current status of a registered payload.
906 The <class> and <id> provided must match a payload that has been previously
907 registered with juju using payload-register. The <status> must be one of the
908 follow: starting, started, stopping, stopped"""
909 cmd = ['payload-status-set']
910 for x in [klass, pid, status]:
911 cmd.append(x)
912 subprocess.check_call(cmd)
913
914
915@cached
916def juju_version():
917 """Full version string (eg. '1.23.3.1-trusty-amd64')"""
918 # Per https://bugs.launchpad.net/juju-core/+bug/1455368/comments/1
919 jujud = glob.glob('/var/lib/juju/tools/machine-*/jujud')[0]
920 return subprocess.check_output([jujud, 'version'],
921 universal_newlines=True).strip()
922
923
924@cached
925def has_juju_version(minimum_version):
926 """Return True if the Juju version is at least the provided version"""
927 return LooseVersion(juju_version()) >= LooseVersion(minimum_version)
928
929
930_atexit = []
931_atstart = []
932
933
934def atstart(callback, *args, **kwargs):
935 '''Schedule a callback to run before the main hook.
936
937 Callbacks are run in the order they were added.
938
939 This is useful for modules and classes to perform initialization
940 and inject behavior. In particular:
941
942 - Run common code before all of your hooks, such as logging
943 the hook name or interesting relation data.
944 - Defer object or module initialization that requires a hook
945 context until we know there actually is a hook context,
946 making testing easier.
947 - Rather than requiring charm authors to include boilerplate to
948 invoke your helper's behavior, have it run automatically if
949 your object is instantiated or module imported.
950
951 This is not at all useful after your hook framework as been launched.
952 '''
953 global _atstart
954 _atstart.append((callback, args, kwargs))
955
956
957def atexit(callback, *args, **kwargs):
958 '''Schedule a callback to run on successful hook completion.
959
960 Callbacks are run in the reverse order that they were added.'''
961 _atexit.append((callback, args, kwargs))
962
963
964def _run_atstart():
965 '''Hook frameworks must invoke this before running the main hook body.'''
966 global _atstart
967 for callback, args, kwargs in _atstart:
968 callback(*args, **kwargs)
969 del _atstart[:]
970
971
972def _run_atexit():
973 '''Hook frameworks must invoke this after the main hook body has
974 successfully completed. Do not invoke it if the hook fails.'''
975 global _atexit
976 for callback, args, kwargs in reversed(_atexit):
977 callback(*args, **kwargs)
978 del _atexit[:]
0979
=== added file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/host.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,641 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
17"""Tools for working with the host system"""
18# Copyright 2012 Canonical Ltd.
19#
20# Authors:
21# Nick Moffitt <nick.moffitt@canonical.com>
22# Matthew Wedgwood <matthew.wedgwood@canonical.com>
23
24import os
25import re
26import pwd
27import glob
28import grp
29import random
30import string
31import subprocess
32import hashlib
33from contextlib import contextmanager
34from collections import OrderedDict
35
36import six
37
38from .hookenv import log
39from .fstab import Fstab
40
41
42def service_start(service_name):
43 """Start a system service"""
44 return service('start', service_name)
45
46
47def service_stop(service_name):
48 """Stop a system service"""
49 return service('stop', service_name)
50
51
52def service_restart(service_name):
53 """Restart a system service"""
54 return service('restart', service_name)
55
56
57def service_reload(service_name, restart_on_failure=False):
58 """Reload a system service, optionally falling back to restart if
59 reload fails"""
60 service_result = service('reload', service_name)
61 if not service_result and restart_on_failure:
62 service_result = service('restart', service_name)
63 return service_result
64
65
66def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"):
67 """Pause a system service.
68
69 Stop it, and prevent it from starting again at boot."""
70 stopped = True
71 if service_running(service_name):
72 stopped = service_stop(service_name)
73 upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
74 sysv_file = os.path.join(initd_dir, service_name)
75 if os.path.exists(upstart_file):
76 override_path = os.path.join(
77 init_dir, '{}.override'.format(service_name))
78 with open(override_path, 'w') as fh:
79 fh.write("manual\n")
80 elif os.path.exists(sysv_file):
81 subprocess.check_call(["update-rc.d", service_name, "disable"])
82 else:
83 # XXX: Support SystemD too
84 raise ValueError(
85 "Unable to detect {0} as either Upstart {1} or SysV {2}".format(
86 service_name, upstart_file, sysv_file))
87 return stopped
88
89
90def service_resume(service_name, init_dir="/etc/init",
91 initd_dir="/etc/init.d"):
92 """Resume a system service.
93
94 Reenable starting again at boot. Start the service"""
95 upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
96 sysv_file = os.path.join(initd_dir, service_name)
97 if os.path.exists(upstart_file):
98 override_path = os.path.join(
99 init_dir, '{}.override'.format(service_name))
100 if os.path.exists(override_path):
101 os.unlink(override_path)
102 elif os.path.exists(sysv_file):
103 subprocess.check_call(["update-rc.d", service_name, "enable"])
104 else:
105 # XXX: Support SystemD too
106 raise ValueError(
107 "Unable to detect {0} as either Upstart {1} or SysV {2}".format(
108 service_name, upstart_file, sysv_file))
109
110 started = service_running(service_name)
111 if not started:
112 started = service_start(service_name)
113 return started
114
115
116def service(action, service_name):
117 """Control a system service"""
118 cmd = ['service', service_name, action]
119 return subprocess.call(cmd) == 0
120
121
122def service_running(service):
123 """Determine whether a system service is running"""
124 try:
125 output = subprocess.check_output(
126 ['service', service, 'status'],
127 stderr=subprocess.STDOUT).decode('UTF-8')
128 except subprocess.CalledProcessError:
129 return False
130 else:
131 if ("start/running" in output or "is running" in output):
132 return True
133 else:
134 return False
135
136
137def service_available(service_name):
138 """Determine whether a system service is available"""
139 try:
140 subprocess.check_output(
141 ['service', service_name, 'status'],
142 stderr=subprocess.STDOUT).decode('UTF-8')
143 except subprocess.CalledProcessError as e:
144 return b'unrecognized service' not in e.output
145 else:
146 return True
147
148
149def adduser(username, password=None, shell='/bin/bash', system_user=False,
150 primary_group=None, secondary_groups=None):
151 """
152 Add a user to the system.
153
154 Will log but otherwise succeed if the user already exists.
155
156 :param str username: Username to create
157 :param str password: Password for user; if ``None``, create a system user
158 :param str shell: The default shell for the user
159 :param bool system_user: Whether to create a login or system user
160 :param str primary_group: Primary group for user; defaults to their username
161 :param list secondary_groups: Optional list of additional groups
162
163 :returns: The password database entry struct, as returned by `pwd.getpwnam`
164 """
165 try:
166 user_info = pwd.getpwnam(username)
167 log('user {0} already exists!'.format(username))
168 except KeyError:
169 log('creating user {0}'.format(username))
170 cmd = ['useradd']
171 if system_user or password is None:
172 cmd.append('--system')
173 else:
174 cmd.extend([
175 '--create-home',
176 '--shell', shell,
177 '--password', password,
178 ])
179 if not primary_group:
180 try:
181 grp.getgrnam(username)
182 primary_group = username # avoid "group exists" error
183 except KeyError:
184 pass
185 if primary_group:
186 cmd.extend(['-g', primary_group])
187 if secondary_groups:
188 cmd.extend(['-G', ','.join(secondary_groups)])
189 cmd.append(username)
190 subprocess.check_call(cmd)
191 user_info = pwd.getpwnam(username)
192 return user_info
193
194
195def user_exists(username):
196 """Check if a user exists"""
197 try:
198 pwd.getpwnam(username)
199 user_exists = True
200 except KeyError:
201 user_exists = False
202 return user_exists
203
204
205def add_group(group_name, system_group=False):
206 """Add a group to the system"""
207 try:
208 group_info = grp.getgrnam(group_name)
209 log('group {0} already exists!'.format(group_name))
210 except KeyError:
211 log('creating group {0}'.format(group_name))
212 cmd = ['addgroup']
213 if system_group:
214 cmd.append('--system')
215 else:
216 cmd.extend([
217 '--group',
218 ])
219 cmd.append(group_name)
220 subprocess.check_call(cmd)
221 group_info = grp.getgrnam(group_name)
222 return group_info
223
224
225def add_user_to_group(username, group):
226 """Add a user to a group"""
227 cmd = ['gpasswd', '-a', username, group]
228 log("Adding user {} to group {}".format(username, group))
229 subprocess.check_call(cmd)
230
231
232def rsync(from_path, to_path, flags='-r', options=None):
233 """Replicate the contents of a path"""
234 options = options or ['--delete', '--executability']
235 cmd = ['/usr/bin/rsync', flags]
236 cmd.extend(options)
237 cmd.append(from_path)
238 cmd.append(to_path)
239 log(" ".join(cmd))
240 return subprocess.check_output(cmd).decode('UTF-8').strip()
241
242
243def symlink(source, destination):
244 """Create a symbolic link"""
245 log("Symlinking {} as {}".format(source, destination))
246 cmd = [
247 'ln',
248 '-sf',
249 source,
250 destination,
251 ]
252 subprocess.check_call(cmd)
253
254
255def mkdir(path, owner='root', group='root', perms=0o555, force=False):
256 """Create a directory"""
257 log("Making dir {} {}:{} {:o}".format(path, owner, group,
258 perms))
259 uid = pwd.getpwnam(owner).pw_uid
260 gid = grp.getgrnam(group).gr_gid
261 realpath = os.path.abspath(path)
262 path_exists = os.path.exists(realpath)
263 if path_exists and force:
264 if not os.path.isdir(realpath):
265 log("Removing non-directory file {} prior to mkdir()".format(path))
266 os.unlink(realpath)
267 os.makedirs(realpath, perms)
268 elif not path_exists:
269 os.makedirs(realpath, perms)
270 os.chown(realpath, uid, gid)
271 os.chmod(realpath, perms)
272
273
274def write_file(path, content, owner='root', group='root', perms=0o444):
275 """Create or overwrite a file with the contents of a byte string."""
276 log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
277 uid = pwd.getpwnam(owner).pw_uid
278 gid = grp.getgrnam(group).gr_gid
279 with open(path, 'wb') as target:
280 os.fchown(target.fileno(), uid, gid)
281 os.fchmod(target.fileno(), perms)
282 target.write(content)
283
284
285def fstab_remove(mp):
286 """Remove the given mountpoint entry from /etc/fstab
287 """
288 return Fstab.remove_by_mountpoint(mp)
289
290
291def fstab_add(dev, mp, fs, options=None):
292 """Adds the given device entry to the /etc/fstab file
293 """
294 return Fstab.add(dev, mp, fs, options=options)
295
296
297def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"):
298 """Mount a filesystem at a particular mountpoint"""
299 cmd_args = ['mount']
300 if options is not None:
301 cmd_args.extend(['-o', options])
302 cmd_args.extend([device, mountpoint])
303 try:
304 subprocess.check_output(cmd_args)
305 except subprocess.CalledProcessError as e:
306 log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output))
307 return False
308
309 if persist:
310 return fstab_add(device, mountpoint, filesystem, options=options)
311 return True
312
313
314def umount(mountpoint, persist=False):
315 """Unmount a filesystem"""
316 cmd_args = ['umount', mountpoint]
317 try:
318 subprocess.check_output(cmd_args)
319 except subprocess.CalledProcessError as e:
320 log('Error unmounting {}\n{}'.format(mountpoint, e.output))
321 return False
322
323 if persist:
324 return fstab_remove(mountpoint)
325 return True
326
327
328def mounts():
329 """Get a list of all mounted volumes as [[mountpoint,device],[...]]"""
330 with open('/proc/mounts') as f:
331 # [['/mount/point','/dev/path'],[...]]
332 system_mounts = [m[1::-1] for m in [l.strip().split()
333 for l in f.readlines()]]
334 return system_mounts
335
336
337def fstab_mount(mountpoint):
338 """Mount filesystem using fstab"""
339 cmd_args = ['mount', mountpoint]
340 try:
341 subprocess.check_output(cmd_args)
342 except subprocess.CalledProcessError as e:
343 log('Error unmounting {}\n{}'.format(mountpoint, e.output))
344 return False
345 return True
346
347
348def file_hash(path, hash_type='md5'):
349 """
350 Generate a hash checksum of the contents of 'path' or None if not found.
351
352 :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`,
353 such as md5, sha1, sha256, sha512, etc.
354 """
355 if os.path.exists(path):
356 h = getattr(hashlib, hash_type)()
357 with open(path, 'rb') as source:
358 h.update(source.read())
359 return h.hexdigest()
360 else:
361 return None
362
363
364def path_hash(path):
365 """
366 Generate a hash checksum of all files matching 'path'. Standard wildcards
367 like '*' and '?' are supported, see documentation for the 'glob' module for
368 more information.
369
370 :return: dict: A { filename: hash } dictionary for all matched files.
371 Empty if none found.
372 """
373 return {
374 filename: file_hash(filename)
375 for filename in glob.iglob(path)
376 }
377
378
379def check_hash(path, checksum, hash_type='md5'):
380 """
381 Validate a file using a cryptographic checksum.
382
383 :param str checksum: Value of the checksum used to validate the file.
384 :param str hash_type: Hash algorithm used to generate `checksum`.
385 Can be any hash alrgorithm supported by :mod:`hashlib`,
386 such as md5, sha1, sha256, sha512, etc.
387 :raises ChecksumError: If the file fails the checksum
388
389 """
390 actual_checksum = file_hash(path, hash_type)
391 if checksum != actual_checksum:
392 raise ChecksumError("'%s' != '%s'" % (checksum, actual_checksum))
393
394
395class ChecksumError(ValueError):
396 pass
397
398
399def restart_on_change(restart_map, stopstart=False):
400 """Restart services based on configuration files changing
401
402 This function is used a decorator, for example::
403
404 @restart_on_change({
405 '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
406 '/etc/apache/sites-enabled/*': [ 'apache2' ]
407 })
408 def config_changed():
409 pass # your code here
410
411 In this example, the cinder-api and cinder-volume services
412 would be restarted if /etc/ceph/ceph.conf is changed by the
413 ceph_client_changed function. The apache2 service would be
414 restarted if any file matching the pattern got changed, created
415 or removed. Standard wildcards are supported, see documentation
416 for the 'glob' module for more information.
417 """
418 def wrap(f):
419 def wrapped_f(*args, **kwargs):
420 checksums = {path: path_hash(path) for path in restart_map}
421 f(*args, **kwargs)
422 restarts = []
423 for path in restart_map:
424 if path_hash(path) != checksums[path]:
425 restarts += restart_map[path]
426 services_list = list(OrderedDict.fromkeys(restarts))
427 if not stopstart:
428 for service_name in services_list:
429 service('restart', service_name)
430 else:
431 for action in ['stop', 'start']:
432 for service_name in services_list:
433 service(action, service_name)
434 return wrapped_f
435 return wrap
436
437
438def lsb_release():
439 """Return /etc/lsb-release in a dict"""
440 d = {}
441 with open('/etc/lsb-release', 'r') as lsb:
442 for l in lsb:
443 k, v = l.split('=')
444 d[k.strip()] = v.strip()
445 return d
446
447
448def pwgen(length=None):
449 """Generate a random pasword."""
450 if length is None:
451 # A random length is ok to use a weak PRNG
452 length = random.choice(range(35, 45))
453 alphanumeric_chars = [
454 l for l in (string.ascii_letters + string.digits)
455 if l not in 'l0QD1vAEIOUaeiou']
456 # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the
457 # actual password
458 random_generator = random.SystemRandom()
459 random_chars = [
460 random_generator.choice(alphanumeric_chars) for _ in range(length)]
461 return(''.join(random_chars))
462
463
464def is_phy_iface(interface):
465 """Returns True if interface is not virtual, otherwise False."""
466 if interface:
467 sys_net = '/sys/class/net'
468 if os.path.isdir(sys_net):
469 for iface in glob.glob(os.path.join(sys_net, '*')):
470 if '/virtual/' in os.path.realpath(iface):
471 continue
472
473 if interface == os.path.basename(iface):
474 return True
475
476 return False
477
478
479def get_bond_master(interface):
480 """Returns bond master if interface is bond slave otherwise None.
481
482 NOTE: the provided interface is expected to be physical
483 """
484 if interface:
485 iface_path = '/sys/class/net/%s' % (interface)
486 if os.path.exists(iface_path):
487 if '/virtual/' in os.path.realpath(iface_path):
488 return None
489
490 master = os.path.join(iface_path, 'master')
491 if os.path.exists(master):
492 master = os.path.realpath(master)
493 # make sure it is a bond master
494 if os.path.exists(os.path.join(master, 'bonding')):
495 return os.path.basename(master)
496
497 return None
498
499
500def list_nics(nic_type=None):
501 '''Return a list of nics of given type(s)'''
502 if isinstance(nic_type, six.string_types):
503 int_types = [nic_type]
504 else:
505 int_types = nic_type
506
507 interfaces = []
508 if nic_type:
509 for int_type in int_types:
510 cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
511 ip_output = subprocess.check_output(cmd).decode('UTF-8')
512 ip_output = ip_output.split('\n')
513 ip_output = (line for line in ip_output if line)
514 for line in ip_output:
515 if line.split()[1].startswith(int_type):
516 matched = re.search('.*: (' + int_type +
517 r'[0-9]+\.[0-9]+)@.*', line)
518 if matched:
519 iface = matched.groups()[0]
520 else:
521 iface = line.split()[1].replace(":", "")
522
523 if iface not in interfaces:
524 interfaces.append(iface)
525 else:
526 cmd = ['ip', 'a']
527 ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
528 ip_output = (line.strip() for line in ip_output if line)
529
530 key = re.compile('^[0-9]+:\s+(.+):')
531 for line in ip_output:
532 matched = re.search(key, line)
533 if matched:
534 iface = matched.group(1)
535 iface = iface.partition("@")[0]
536 if iface not in interfaces:
537 interfaces.append(iface)
538
539 return interfaces
540
541
542def set_nic_mtu(nic, mtu):
543 '''Set MTU on a network interface'''
544 cmd = ['ip', 'link', 'set', nic, 'mtu', mtu]
545 subprocess.check_call(cmd)
546
547
548def get_nic_mtu(nic):
549 cmd = ['ip', 'addr', 'show', nic]
550 ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
551 mtu = ""
552 for line in ip_output:
553 words = line.split()
554 if 'mtu' in words:
555 mtu = words[words.index("mtu") + 1]
556 return mtu
557
558
559def get_nic_hwaddr(nic):
560 cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
561 ip_output = subprocess.check_output(cmd).decode('UTF-8')
562 hwaddr = ""
563 words = ip_output.split()
564 if 'link/ether' in words:
565 hwaddr = words[words.index('link/ether') + 1]
566 return hwaddr
567
568
569def cmp_pkgrevno(package, revno, pkgcache=None):
570 '''Compare supplied revno with the revno of the installed package
571
572 * 1 => Installed revno is greater than supplied arg
573 * 0 => Installed revno is the same as supplied arg
574 * -1 => Installed revno is less than supplied arg
575
576 This function imports apt_cache function from charmhelpers.fetch if
577 the pkgcache argument is None. Be sure to add charmhelpers.fetch if
578 you call this function, or pass an apt_pkg.Cache() instance.
579 '''
580 import apt_pkg
581 if not pkgcache:
582 from charmhelpers.fetch import apt_cache
583 pkgcache = apt_cache()
584 pkg = pkgcache[package]
585 return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
586
587
588@contextmanager
589def chdir(d):
590 cur = os.getcwd()
591 try:
592 yield os.chdir(d)
593 finally:
594 os.chdir(cur)
595
596
597def chownr(path, owner, group, follow_links=True, chowntopdir=False):
598 """
599 Recursively change user and group ownership of files and directories
600 in given path. Doesn't chown path itself by default, only its children.
601
602 :param bool follow_links: Also Chown links if True
603 :param bool chowntopdir: Also chown path itself if True
604 """
605 uid = pwd.getpwnam(owner).pw_uid
606 gid = grp.getgrnam(group).gr_gid
607 if follow_links:
608 chown = os.chown
609 else:
610 chown = os.lchown
611
612 if chowntopdir:
613 broken_symlink = os.path.lexists(path) and not os.path.exists(path)
614 if not broken_symlink:
615 chown(path, uid, gid)
616 for root, dirs, files in os.walk(path):
617 for name in dirs + files:
618 full = os.path.join(root, name)
619 broken_symlink = os.path.lexists(full) and not os.path.exists(full)
620 if not broken_symlink:
621 chown(full, uid, gid)
622
623
624def lchownr(path, owner, group):
625 chownr(path, owner, group, follow_links=False)
626
627
628def get_total_ram():
629 '''The total amount of system RAM in bytes.
630
631 This is what is reported by the OS, and may be overcommitted when
632 there are multiple containers hosted on the same machine.
633 '''
634 with open('/proc/meminfo', 'r') as f:
635 for line in f.readlines():
636 if line:
637 key, value, unit = line.split()
638 if key == 'MemTotal:':
639 assert unit == 'kB', 'Unknown unit'
640 return int(value) * 1024 # Classic, not KiB.
641 raise NotImplementedError()
0642
=== added file 'hooks/charmhelpers/core/hugepage.py'
--- hooks/charmhelpers/core/hugepage.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/hugepage.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,71 @@
1# -*- coding: utf-8 -*-
2
3# Copyright 2014-2015 Canonical Limited.
4#
5# This file is part of charm-helpers.
6#
7# charm-helpers is free software: you can redistribute it and/or modify
8# it under the terms of the GNU Lesser General Public License version 3 as
9# published by the Free Software Foundation.
10#
11# charm-helpers is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU Lesser General Public License for more details.
15#
16# You should have received a copy of the GNU Lesser General Public License
17# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
18
19import yaml
20from charmhelpers.core import fstab
21from charmhelpers.core import sysctl
22from charmhelpers.core.host import (
23 add_group,
24 add_user_to_group,
25 fstab_mount,
26 mkdir,
27)
28from charmhelpers.core.strutils import bytes_from_string
29from subprocess import check_output
30
31
32def hugepage_support(user, group='hugetlb', nr_hugepages=256,
33 max_map_count=65536, mnt_point='/run/hugepages/kvm',
34 pagesize='2MB', mount=True, set_shmmax=False):
35 """Enable hugepages on system.
36
37 Args:
38 user (str) -- Username to allow access to hugepages to
39 group (str) -- Group name to own hugepages
40 nr_hugepages (int) -- Number of pages to reserve
41 max_map_count (int) -- Number of Virtual Memory Areas a process can own
42 mnt_point (str) -- Directory to mount hugepages on
43 pagesize (str) -- Size of hugepages
44 mount (bool) -- Whether to Mount hugepages
45 """
46 group_info = add_group(group)
47 gid = group_info.gr_gid
48 add_user_to_group(user, group)
49 if max_map_count < 2 * nr_hugepages:
50 max_map_count = 2 * nr_hugepages
51 sysctl_settings = {
52 'vm.nr_hugepages': nr_hugepages,
53 'vm.max_map_count': max_map_count,
54 'vm.hugetlb_shm_group': gid,
55 }
56 if set_shmmax:
57 shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax']))
58 shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages
59 if shmmax_minsize > shmmax_current:
60 sysctl_settings['kernel.shmmax'] = shmmax_minsize
61 sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf')
62 mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False)
63 lfstab = fstab.Fstab()
64 fstab_entry = lfstab.get_entry_by_attr('mountpoint', mnt_point)
65 if fstab_entry:
66 lfstab.remove_entry(fstab_entry)
67 entry = lfstab.Entry('nodev', mnt_point, 'hugetlbfs',
68 'mode=1770,gid={},pagesize={}'.format(gid, pagesize), 0, 0)
69 lfstab.add_entry(entry)
70 if mount:
71 fstab_mount(mnt_point)
072
=== added file 'hooks/charmhelpers/core/kernel.py'
--- hooks/charmhelpers/core/kernel.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/kernel.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,68 @@
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4# Copyright 2014-2015 Canonical Limited.
5#
6# This file is part of charm-helpers.
7#
8# charm-helpers is free software: you can redistribute it and/or modify
9# it under the terms of the GNU Lesser General Public License version 3 as
10# published by the Free Software Foundation.
11#
12# charm-helpers is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU Lesser General Public License for more details.
16#
17# You should have received a copy of the GNU Lesser General Public License
18# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
19
20__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
21
22from charmhelpers.core.hookenv import (
23 log,
24 INFO
25)
26
27from subprocess import check_call, check_output
28import re
29
30
31def modprobe(module, persist=True):
32 """Load a kernel module and configure for auto-load on reboot."""
33 cmd = ['modprobe', module]
34
35 log('Loading kernel module %s' % module, level=INFO)
36
37 check_call(cmd)
38 if persist:
39 with open('/etc/modules', 'r+') as modules:
40 if module not in modules.read():
41 modules.write(module)
42
43
44def rmmod(module, force=False):
45 """Remove a module from the linux kernel"""
46 cmd = ['rmmod']
47 if force:
48 cmd.append('-f')
49 cmd.append(module)
50 log('Removing kernel module %s' % module, level=INFO)
51 return check_call(cmd)
52
53
54def lsmod():
55 """Shows what kernel modules are currently loaded"""
56 return check_output(['lsmod'],
57 universal_newlines=True)
58
59
60def is_module_loaded(module):
61 """Checks if a kernel module is already loaded"""
62 matches = re.findall('^%s[ ]+' % module, lsmod(), re.M)
63 return len(matches) > 0
64
65
66def update_initramfs(version='all'):
67 """Updates an initramfs image"""
68 return check_call(["update-initramfs", "-k", version, "-u"])
069
=== added directory 'hooks/charmhelpers/core/services'
=== added file 'hooks/charmhelpers/core/services/__init__.py'
--- hooks/charmhelpers/core/services/__init__.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/services/__init__.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,18 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
17from .base import * # NOQA
18from .helpers import * # NOQA
019
=== added file 'hooks/charmhelpers/core/services/base.py'
--- hooks/charmhelpers/core/services/base.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/services/base.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,353 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
17import os
18import json
19from inspect import getargspec
20from collections import Iterable, OrderedDict
21
22from charmhelpers.core import host
23from charmhelpers.core import hookenv
24
25
26__all__ = ['ServiceManager', 'ManagerCallback',
27 'PortManagerCallback', 'open_ports', 'close_ports', 'manage_ports',
28 'service_restart', 'service_stop']
29
30
31class ServiceManager(object):
32 def __init__(self, services=None):
33 """
34 Register a list of services, given their definitions.
35
36 Service definitions are dicts in the following formats (all keys except
37 'service' are optional)::
38
39 {
40 "service": <service name>,
41 "required_data": <list of required data contexts>,
42 "provided_data": <list of provided data contexts>,
43 "data_ready": <one or more callbacks>,
44 "data_lost": <one or more callbacks>,
45 "start": <one or more callbacks>,
46 "stop": <one or more callbacks>,
47 "ports": <list of ports to manage>,
48 }
49
50 The 'required_data' list should contain dicts of required data (or
51 dependency managers that act like dicts and know how to collect the data).
52 Only when all items in the 'required_data' list are populated are the list
53 of 'data_ready' and 'start' callbacks executed. See `is_ready()` for more
54 information.
55
56 The 'provided_data' list should contain relation data providers, most likely
57 a subclass of :class:`charmhelpers.core.services.helpers.RelationContext`,
58 that will indicate a set of data to set on a given relation.
59
60 The 'data_ready' value should be either a single callback, or a list of
61 callbacks, to be called when all items in 'required_data' pass `is_ready()`.
62 Each callback will be called with the service name as the only parameter.
63 After all of the 'data_ready' callbacks are called, the 'start' callbacks
64 are fired.
65
66 The 'data_lost' value should be either a single callback, or a list of
67 callbacks, to be called when a 'required_data' item no longer passes
68 `is_ready()`. Each callback will be called with the service name as the
69 only parameter. After all of the 'data_lost' callbacks are called,
70 the 'stop' callbacks are fired.
71
72 The 'start' value should be either a single callback, or a list of
73 callbacks, to be called when starting the service, after the 'data_ready'
74 callbacks are complete. Each callback will be called with the service
75 name as the only parameter. This defaults to
76 `[host.service_start, services.open_ports]`.
77
78 The 'stop' value should be either a single callback, or a list of
79 callbacks, to be called when stopping the service. If the service is
80 being stopped because it no longer has all of its 'required_data', this
81 will be called after all of the 'data_lost' callbacks are complete.
82 Each callback will be called with the service name as the only parameter.
83 This defaults to `[services.close_ports, host.service_stop]`.
84
85 The 'ports' value should be a list of ports to manage. The default
86 'start' handler will open the ports after the service is started,
87 and the default 'stop' handler will close the ports prior to stopping
88 the service.
89
90
91 Examples:
92
93 The following registers an Upstart service called bingod that depends on
94 a mongodb relation and which runs a custom `db_migrate` function prior to
95 restarting the service, and a Runit service called spadesd::
96
97 manager = services.ServiceManager([
98 {
99 'service': 'bingod',
100 'ports': [80, 443],
101 'required_data': [MongoRelation(), config(), {'my': 'data'}],
102 'data_ready': [
103 services.template(source='bingod.conf'),
104 services.template(source='bingod.ini',
105 target='/etc/bingod.ini',
106 owner='bingo', perms=0400),
107 ],
108 },
109 {
110 'service': 'spadesd',
111 'data_ready': services.template(source='spadesd_run.j2',
112 target='/etc/sv/spadesd/run',
113 perms=0555),
114 'start': runit_start,
115 'stop': runit_stop,
116 },
117 ])
118 manager.manage()
119 """
120 self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json')
121 self._ready = None
122 self.services = OrderedDict()
123 for service in services or []:
124 service_name = service['service']
125 self.services[service_name] = service
126
127 def manage(self):
128 """
129 Handle the current hook by doing The Right Thing with the registered services.
130 """
131 hookenv._run_atstart()
132 try:
133 hook_name = hookenv.hook_name()
134 if hook_name == 'stop':
135 self.stop_services()
136 else:
137 self.reconfigure_services()
138 self.provide_data()
139 except SystemExit as x:
140 if x.code is None or x.code == 0:
141 hookenv._run_atexit()
142 hookenv._run_atexit()
143
144 def provide_data(self):
145 """
146 Set the relation data for each provider in the ``provided_data`` list.
147
148 A provider must have a `name` attribute, which indicates which relation
149 to set data on, and a `provide_data()` method, which returns a dict of
150 data to set.
151
152 The `provide_data()` method can optionally accept two parameters:
153
154 * ``remote_service`` The name of the remote service that the data will
155 be provided to. The `provide_data()` method will be called once
156 for each connected service (not unit). This allows the method to
157 tailor its data to the given service.
158 * ``service_ready`` Whether or not the service definition had all of
159 its requirements met, and thus the ``data_ready`` callbacks run.
160
161 Note that the ``provided_data`` methods are now called **after** the
162 ``data_ready`` callbacks are run. This gives the ``data_ready`` callbacks
163 a chance to generate any data necessary for the providing to the remote
164 services.
165 """
166 for service_name, service in self.services.items():
167 service_ready = self.is_ready(service_name)
168 for provider in service.get('provided_data', []):
169 for relid in hookenv.relation_ids(provider.name):
170 units = hookenv.related_units(relid)
171 if not units:
172 continue
173 remote_service = units[0].split('/')[0]
174 argspec = getargspec(provider.provide_data)
175 if len(argspec.args) > 1:
176 data = provider.provide_data(remote_service, service_ready)
177 else:
178 data = provider.provide_data()
179 if data:
180 hookenv.relation_set(relid, data)
181
182 def reconfigure_services(self, *service_names):
183 """
184 Update all files for one or more registered services, and,
185 if ready, optionally restart them.
186
187 If no service names are given, reconfigures all registered services.
188 """
189 for service_name in service_names or self.services.keys():
190 if self.is_ready(service_name):
191 self.fire_event('data_ready', service_name)
192 self.fire_event('start', service_name, default=[
193 service_restart,
194 manage_ports])
195 self.save_ready(service_name)
196 else:
197 if self.was_ready(service_name):
198 self.fire_event('data_lost', service_name)
199 self.fire_event('stop', service_name, default=[
200 manage_ports,
201 service_stop])
202 self.save_lost(service_name)
203
204 def stop_services(self, *service_names):
205 """
206 Stop one or more registered services, by name.
207
208 If no service names are given, stops all registered services.
209 """
210 for service_name in service_names or self.services.keys():
211 self.fire_event('stop', service_name, default=[
212 manage_ports,
213 service_stop])
214
215 def get_service(self, service_name):
216 """
217 Given the name of a registered service, return its service definition.
218 """
219 service = self.services.get(service_name)
220 if not service:
221 raise KeyError('Service not registered: %s' % service_name)
222 return service
223
224 def fire_event(self, event_name, service_name, default=None):
225 """
226 Fire a data_ready, data_lost, start, or stop event on a given service.
227 """
228 service = self.get_service(service_name)
229 callbacks = service.get(event_name, default)
230 if not callbacks:
231 return
232 if not isinstance(callbacks, Iterable):
233 callbacks = [callbacks]
234 for callback in callbacks:
235 if isinstance(callback, ManagerCallback):
236 callback(self, service_name, event_name)
237 else:
238 callback(service_name)
239
240 def is_ready(self, service_name):
241 """
242 Determine if a registered service is ready, by checking its 'required_data'.
243
244 A 'required_data' item can be any mapping type, and is considered ready
245 if `bool(item)` evaluates as True.
246 """
247 service = self.get_service(service_name)
248 reqs = service.get('required_data', [])
249 return all(bool(req) for req in reqs)
250
251 def _load_ready_file(self):
252 if self._ready is not None:
253 return
254 if os.path.exists(self._ready_file):
255 with open(self._ready_file) as fp:
256 self._ready = set(json.load(fp))
257 else:
258 self._ready = set()
259
260 def _save_ready_file(self):
261 if self._ready is None:
262 return
263 with open(self._ready_file, 'w') as fp:
264 json.dump(list(self._ready), fp)
265
266 def save_ready(self, service_name):
267 """
268 Save an indicator that the given service is now data_ready.
269 """
270 self._load_ready_file()
271 self._ready.add(service_name)
272 self._save_ready_file()
273
274 def save_lost(self, service_name):
275 """
276 Save an indicator that the given service is no longer data_ready.
277 """
278 self._load_ready_file()
279 self._ready.discard(service_name)
280 self._save_ready_file()
281
282 def was_ready(self, service_name):
283 """
284 Determine if the given service was previously data_ready.
285 """
286 self._load_ready_file()
287 return service_name in self._ready
288
289
290class ManagerCallback(object):
291 """
292 Special case of a callback that takes the `ServiceManager` instance
293 in addition to the service name.
294
295 Subclasses should implement `__call__` which should accept three parameters:
296
297 * `manager` The `ServiceManager` instance
298 * `service_name` The name of the service it's being triggered for
299 * `event_name` The name of the event that this callback is handling
300 """
301 def __call__(self, manager, service_name, event_name):
302 raise NotImplementedError()
303
304
305class PortManagerCallback(ManagerCallback):
306 """
307 Callback class that will open or close ports, for use as either
308 a start or stop action.
309 """
310 def __call__(self, manager, service_name, event_name):
311 service = manager.get_service(service_name)
312 new_ports = service.get('ports', [])
313 port_file = os.path.join(hookenv.charm_dir(), '.{}.ports'.format(service_name))
314 if os.path.exists(port_file):
315 with open(port_file) as fp:
316 old_ports = fp.read().split(',')
317 for old_port in old_ports:
318 if bool(old_port):
319 old_port = int(old_port)
320 if old_port not in new_ports:
321 hookenv.close_port(old_port)
322 with open(port_file, 'w') as fp:
323 fp.write(','.join(str(port) for port in new_ports))
324 for port in new_ports:
325 if event_name == 'start':
326 hookenv.open_port(port)
327 elif event_name == 'stop':
328 hookenv.close_port(port)
329
330
331def service_stop(service_name):
332 """
333 Wrapper around host.service_stop to prevent spurious "unknown service"
334 messages in the logs.
335 """
336 if host.service_running(service_name):
337 host.service_stop(service_name)
338
339
340def service_restart(service_name):
341 """
342 Wrapper around host.service_restart to prevent spurious "unknown service"
343 messages in the logs.
344 """
345 if host.service_available(service_name):
346 if host.service_running(service_name):
347 host.service_restart(service_name)
348 else:
349 host.service_start(service_name)
350
351
352# Convenience aliases
353open_ports = close_ports = manage_ports = PortManagerCallback()
0354
=== added file 'hooks/charmhelpers/core/services/helpers.py'
--- hooks/charmhelpers/core/services/helpers.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/services/helpers.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,292 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
17import os
18import yaml
19
20from charmhelpers.core import hookenv
21from charmhelpers.core import host
22from charmhelpers.core import templating
23
24from charmhelpers.core.services.base import ManagerCallback
25
26
27__all__ = ['RelationContext', 'TemplateCallback',
28 'render_template', 'template']
29
30
31class RelationContext(dict):
32 """
33 Base class for a context generator that gets relation data from juju.
34
35 Subclasses must provide the attributes `name`, which is the name of the
36 interface of interest, `interface`, which is the type of the interface of
37 interest, and `required_keys`, which is the set of keys required for the
38 relation to be considered complete. The data for all interfaces matching
39 the `name` attribute that are complete will used to populate the dictionary
40 values (see `get_data`, below).
41
42 The generated context will be namespaced under the relation :attr:`name`,
43 to prevent potential naming conflicts.
44
45 :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
46 :param list additional_required_keys: Extend the list of :attr:`required_keys`
47 """
48 name = None
49 interface = None
50
51 def __init__(self, name=None, additional_required_keys=None):
52 if not hasattr(self, 'required_keys'):
53 self.required_keys = []
54
55 if name is not None:
56 self.name = name
57 if additional_required_keys:
58 self.required_keys.extend(additional_required_keys)
59 self.get_data()
60
61 def __bool__(self):
62 """
63 Returns True if all of the required_keys are available.
64 """
65 return self.is_ready()
66
67 __nonzero__ = __bool__
68
69 def __repr__(self):
70 return super(RelationContext, self).__repr__()
71
72 def is_ready(self):
73 """
74 Returns True if all of the `required_keys` are available from any units.
75 """
76 ready = len(self.get(self.name, [])) > 0
77 if not ready:
78 hookenv.log('Incomplete relation: {}'.format(self.__class__.__name__), hookenv.DEBUG)
79 return ready
80
81 def _is_ready(self, unit_data):
82 """
83 Helper method that tests a set of relation data and returns True if
84 all of the `required_keys` are present.
85 """
86 return set(unit_data.keys()).issuperset(set(self.required_keys))
87
88 def get_data(self):
89 """
90 Retrieve the relation data for each unit involved in a relation and,
91 if complete, store it in a list under `self[self.name]`. This
92 is automatically called when the RelationContext is instantiated.
93
94 The units are sorted lexographically first by the service ID, then by
95 the unit ID. Thus, if an interface has two other services, 'db:1'
96 and 'db:2', with 'db:1' having two units, 'wordpress/0' and 'wordpress/1',
97 and 'db:2' having one unit, 'mediawiki/0', all of which have a complete
98 set of data, the relation data for the units will be stored in the
99 order: 'wordpress/0', 'wordpress/1', 'mediawiki/0'.
100
101 If you only care about a single unit on the relation, you can just
102 access it as `{{ interface[0]['key'] }}`. However, if you can at all
103 support multiple units on a relation, you should iterate over the list,
104 like::
105
106 {% for unit in interface -%}
107 {{ unit['key'] }}{% if not loop.last %},{% endif %}
108 {%- endfor %}
109
110 Note that since all sets of relation data from all related services and
111 units are in a single list, if you need to know which service or unit a
112 set of data came from, you'll need to extend this class to preserve
113 that information.
114 """
115 if not hookenv.relation_ids(self.name):
116 return
117
118 ns = self.setdefault(self.name, [])
119 for rid in sorted(hookenv.relation_ids(self.name)):
120 for unit in sorted(hookenv.related_units(rid)):
121 reldata = hookenv.relation_get(rid=rid, unit=unit)
122 if self._is_ready(reldata):
123 ns.append(reldata)
124
125 def provide_data(self):
126 """
127 Return data to be relation_set for this interface.
128 """
129 return {}
130
131
132class MysqlRelation(RelationContext):
133 """
134 Relation context for the `mysql` interface.
135
136 :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
137 :param list additional_required_keys: Extend the list of :attr:`required_keys`
138 """
139 name = 'db'
140 interface = 'mysql'
141
142 def __init__(self, *args, **kwargs):
143 self.required_keys = ['host', 'user', 'password', 'database']
144 RelationContext.__init__(self, *args, **kwargs)
145
146
147class HttpRelation(RelationContext):
148 """
149 Relation context for the `http` interface.
150
151 :param str name: Override the relation :attr:`name`, since it can vary from charm to charm
152 :param list additional_required_keys: Extend the list of :attr:`required_keys`
153 """
154 name = 'website'
155 interface = 'http'
156
157 def __init__(self, *args, **kwargs):
158 self.required_keys = ['host', 'port']
159 RelationContext.__init__(self, *args, **kwargs)
160
161 def provide_data(self):
162 return {
163 'host': hookenv.unit_get('private-address'),
164 'port': 80,
165 }
166
167
168class RequiredConfig(dict):
169 """
170 Data context that loads config options with one or more mandatory options.
171
172 Once the required options have been changed from their default values, all
173 config options will be available, namespaced under `config` to prevent
174 potential naming conflicts (for example, between a config option and a
175 relation property).
176
177 :param list *args: List of options that must be changed from their default values.
178 """
179
180 def __init__(self, *args):
181 self.required_options = args
182 self['config'] = hookenv.config()
183 with open(os.path.join(hookenv.charm_dir(), 'config.yaml')) as fp:
184 self.config = yaml.load(fp).get('options', {})
185
186 def __bool__(self):
187 for option in self.required_options:
188 if option not in self['config']:
189 return False
190 current_value = self['config'][option]
191 default_value = self.config[option].get('default')
192 if current_value == default_value:
193 return False
194 if current_value in (None, '') and default_value in (None, ''):
195 return False
196 return True
197
198 def __nonzero__(self):
199 return self.__bool__()
200
201
202class StoredContext(dict):
203 """
204 A data context that always returns the data that it was first created with.
205
206 This is useful to do a one-time generation of things like passwords, that
207 will thereafter use the same value that was originally generated, instead
208 of generating a new value each time it is run.
209 """
210 def __init__(self, file_name, config_data):
211 """
212 If the file exists, populate `self` with the data from the file.
213 Otherwise, populate with the given data and persist it to the file.
214 """
215 if os.path.exists(file_name):
216 self.update(self.read_context(file_name))
217 else:
218 self.store_context(file_name, config_data)
219 self.update(config_data)
220
221 def store_context(self, file_name, config_data):
222 if not os.path.isabs(file_name):
223 file_name = os.path.join(hookenv.charm_dir(), file_name)
224 with open(file_name, 'w') as file_stream:
225 os.fchmod(file_stream.fileno(), 0o600)
226 yaml.dump(config_data, file_stream)
227
228 def read_context(self, file_name):
229 if not os.path.isabs(file_name):
230 file_name = os.path.join(hookenv.charm_dir(), file_name)
231 with open(file_name, 'r') as file_stream:
232 data = yaml.load(file_stream)
233 if not data:
234 raise OSError("%s is empty" % file_name)
235 return data
236
237
238class TemplateCallback(ManagerCallback):
239 """
240 Callback class that will render a Jinja2 template, for use as a ready
241 action.
242
243 :param str source: The template source file, relative to
244 `$CHARM_DIR/templates`
245
246 :param str target: The target to write the rendered template to (or None)
247 :param str owner: The owner of the rendered file
248 :param str group: The group of the rendered file
249 :param int perms: The permissions of the rendered file
250 :param partial on_change_action: functools partial to be executed when
251 rendered file changes
252 :param jinja2 loader template_loader: A jinja2 template loader
253
254 :return str: The rendered template
255 """
256 def __init__(self, source, target,
257 owner='root', group='root', perms=0o444,
258 on_change_action=None, template_loader=None):
259 self.source = source
260 self.target = target
261 self.owner = owner
262 self.group = group
263 self.perms = perms
264 self.on_change_action = on_change_action
265 self.template_loader = template_loader
266
267 def __call__(self, manager, service_name, event_name):
268 pre_checksum = ''
269 if self.on_change_action and os.path.isfile(self.target):
270 pre_checksum = host.file_hash(self.target)
271 service = manager.get_service(service_name)
272 context = {'ctx': {}}
273 for ctx in service.get('required_data', []):
274 context.update(ctx)
275 context['ctx'].update(ctx)
276
277 result = templating.render(self.source, self.target, context,
278 self.owner, self.group, self.perms,
279 template_loader=self.template_loader)
280 if self.on_change_action:
281 if pre_checksum == host.file_hash(self.target):
282 hookenv.log(
283 'No change detected: {}'.format(self.target),
284 hookenv.DEBUG)
285 else:
286 self.on_change_action()
287
288 return result
289
290
291# Convenience aliases for templates
292render_template = template = TemplateCallback
0293
=== added file 'hooks/charmhelpers/core/strutils.py'
--- hooks/charmhelpers/core/strutils.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/strutils.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,72 @@
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4# Copyright 2014-2015 Canonical Limited.
5#
6# This file is part of charm-helpers.
7#
8# charm-helpers is free software: you can redistribute it and/or modify
9# it under the terms of the GNU Lesser General Public License version 3 as
10# published by the Free Software Foundation.
11#
12# charm-helpers is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU Lesser General Public License for more details.
16#
17# You should have received a copy of the GNU Lesser General Public License
18# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
19
20import six
21import re
22
23
24def bool_from_string(value):
25 """Interpret string value as boolean.
26
27 Returns True if value translates to True otherwise False.
28 """
29 if isinstance(value, six.string_types):
30 value = six.text_type(value)
31 else:
32 msg = "Unable to interpret non-string value '%s' as boolean" % (value)
33 raise ValueError(msg)
34
35 value = value.strip().lower()
36
37 if value in ['y', 'yes', 'true', 't', 'on']:
38 return True
39 elif value in ['n', 'no', 'false', 'f', 'off']:
40 return False
41
42 msg = "Unable to interpret string value '%s' as boolean" % (value)
43 raise ValueError(msg)
44
45
46def bytes_from_string(value):
47 """Interpret human readable string value as bytes.
48
49 Returns int
50 """
51 BYTE_POWER = {
52 'K': 1,
53 'KB': 1,
54 'M': 2,
55 'MB': 2,
56 'G': 3,
57 'GB': 3,
58 'T': 4,
59 'TB': 4,
60 'P': 5,
61 'PB': 5,
62 }
63 if isinstance(value, six.string_types):
64 value = six.text_type(value)
65 else:
66 msg = "Unable to interpret non-string value '%s' as boolean" % (value)
67 raise ValueError(msg)
68 matches = re.match("([0-9]+)([a-zA-Z]+)", value)
69 if not matches:
70 msg = "Unable to interpret string value '%s' as bytes" % (value)
71 raise ValueError(msg)
72 return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
073
=== added file 'hooks/charmhelpers/core/sysctl.py'
--- hooks/charmhelpers/core/sysctl.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/sysctl.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,56 @@
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4# Copyright 2014-2015 Canonical Limited.
5#
6# This file is part of charm-helpers.
7#
8# charm-helpers is free software: you can redistribute it and/or modify
9# it under the terms of the GNU Lesser General Public License version 3 as
10# published by the Free Software Foundation.
11#
12# charm-helpers is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU Lesser General Public License for more details.
16#
17# You should have received a copy of the GNU Lesser General Public License
18# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
19
20import yaml
21
22from subprocess import check_call
23
24from charmhelpers.core.hookenv import (
25 log,
26 DEBUG,
27 ERROR,
28)
29
30__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
31
32
33def create(sysctl_dict, sysctl_file):
34 """Creates a sysctl.conf file from a YAML associative array
35
36 :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }"
37 :type sysctl_dict: str
38 :param sysctl_file: path to the sysctl file to be saved
39 :type sysctl_file: str or unicode
40 :returns: None
41 """
42 try:
43 sysctl_dict_parsed = yaml.safe_load(sysctl_dict)
44 except yaml.YAMLError:
45 log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict),
46 level=ERROR)
47 return
48
49 with open(sysctl_file, "w") as fd:
50 for key, value in sysctl_dict_parsed.items():
51 fd.write("{}={}\n".format(key, value))
52
53 log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed),
54 level=DEBUG)
55
56 check_call(["sysctl", "-p", sysctl_file])
057
=== added file 'hooks/charmhelpers/core/templating.py'
--- hooks/charmhelpers/core/templating.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/templating.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,81 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
17import os
18
19from charmhelpers.core import host
20from charmhelpers.core import hookenv
21
22
23def render(source, target, context, owner='root', group='root',
24 perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None):
25 """
26 Render a template.
27
28 The `source` path, if not absolute, is relative to the `templates_dir`.
29
30 The `target` path should be absolute. It can also be `None`, in which
31 case no file will be written.
32
33 The context should be a dict containing the values to be replaced in the
34 template.
35
36 The `owner`, `group`, and `perms` options will be passed to `write_file`.
37
38 If omitted, `templates_dir` defaults to the `templates` folder in the charm.
39
40 The rendered template will be written to the file as well as being returned
41 as a string.
42
43 Note: Using this requires python-jinja2; if it is not installed, calling
44 this will attempt to use charmhelpers.fetch.apt_install to install it.
45 """
46 try:
47 from jinja2 import FileSystemLoader, Environment, exceptions
48 except ImportError:
49 try:
50 from charmhelpers.fetch import apt_install
51 except ImportError:
52 hookenv.log('Could not import jinja2, and could not import '
53 'charmhelpers.fetch to install it',
54 level=hookenv.ERROR)
55 raise
56 apt_install('python-jinja2', fatal=True)
57 from jinja2 import FileSystemLoader, Environment, exceptions
58
59 if template_loader:
60 template_env = Environment(loader=template_loader)
61 else:
62 if templates_dir is None:
63 templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
64 template_env = Environment(loader=FileSystemLoader(templates_dir))
65 try:
66 source = source
67 template = template_env.get_template(source)
68 except exceptions.TemplateNotFound as e:
69 hookenv.log('Could not load template %s from %s.' %
70 (source, templates_dir),
71 level=hookenv.ERROR)
72 raise e
73 content = template.render(context)
74 if target is not None:
75 target_dir = os.path.dirname(target)
76 if not os.path.exists(target_dir):
77 # This is a terrible default directory permission, as the file
78 # or its siblings will often contain secrets.
79 host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
80 host.write_file(target, content.encode(encoding), owner, group, perms)
81 return content
082
=== added file 'hooks/charmhelpers/core/unitdata.py'
--- hooks/charmhelpers/core/unitdata.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/unitdata.py 2015-12-17 05:44:32 +0000
@@ -0,0 +1,521 @@
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3#
4# Copyright 2014-2015 Canonical Limited.
5#
6# This file is part of charm-helpers.
7#
8# charm-helpers is free software: you can redistribute it and/or modify
9# it under the terms of the GNU Lesser General Public License version 3 as
10# published by the Free Software Foundation.
11#
12# charm-helpers is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU Lesser General Public License for more details.
16#
17# You should have received a copy of the GNU Lesser General Public License
18# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
19#
20#
21# Authors:
22# Kapil Thangavelu <kapil.foss@gmail.com>
23#
24"""
25Intro
26-----
27
28A simple way to store state in units. This provides a key value
29storage with support for versioned, transactional operation,
30and can calculate deltas from previous values to simplify unit logic
31when processing changes.
32
33
34Hook Integration
35----------------
36
37There are several extant frameworks for hook execution, including
38
39 - charmhelpers.core.hookenv.Hooks
40 - charmhelpers.core.services.ServiceManager
41
42The storage classes are framework agnostic, one simple integration is
43via the HookData contextmanager. It will record the current hook
44execution environment (including relation data, config data, etc.),
45setup a transaction and allow easy access to the changes from
46previously seen values. One consequence of the integration is the
47reservation of particular keys ('rels', 'unit', 'env', 'config',
48'charm_revisions') for their respective values.
49
50Here's a fully worked integration example using hookenv.Hooks::
51
52 from charmhelper.core import hookenv, unitdata
53
54 hook_data = unitdata.HookData()
55 db = unitdata.kv()
56 hooks = hookenv.Hooks()
57
58 @hooks.hook
59 def config_changed():
60 # Print all changes to configuration from previously seen
61 # values.
62 for changed, (prev, cur) in hook_data.conf.items():
63 print('config changed', changed,
64 'previous value', prev,
65 'current value', cur)
66
67 # Get some unit specific bookeeping
68 if not db.get('pkg_key'):
69 key = urllib.urlopen('https://example.com/pkg_key').read()
70 db.set('pkg_key', key)
71
72 # Directly access all charm config as a mapping.
73 conf = db.getrange('config', True)
74
75 # Directly access all relation data as a mapping
76 rels = db.getrange('rels', True)
77
78 if __name__ == '__main__':
79 with hook_data():
80 hook.execute()
81
82
83A more basic integration is via the hook_scope context manager which simply
84manages transaction scope (and records hook name, and timestamp)::
85
86 >>> from unitdata import kv
87 >>> db = kv()
88 >>> with db.hook_scope('install'):
89 ... # do work, in transactional scope.
90 ... db.set('x', 1)
91 >>> db.get('x')
92 1
93
94
95Usage
96-----
97
98Values are automatically json de/serialized to preserve basic typing
99and complex data struct capabilities (dicts, lists, ints, booleans, etc).
100
101Individual values can be manipulated via get/set::
102
103 >>> kv.set('y', True)
104 >>> kv.get('y')
105 True
106
107 # We can set complex values (dicts, lists) as a single key.
108 >>> kv.set('config', {'a': 1, 'b': True'})
109
110 # Also supports returning dictionaries as a record which
111 # provides attribute access.
112 >>> config = kv.get('config', record=True)
113 >>> config.b
114 True
115
116
117Groups of keys can be manipulated with update/getrange::
118
119 >>> kv.update({'z': 1, 'y': 2}, prefix="gui.")
120 >>> kv.getrange('gui.', strip=True)
121 {'z': 1, 'y': 2}
122
123When updating values, its very helpful to understand which values
124have actually changed and how have they changed. The storage
125provides a delta method to provide for this::
126
127 >>> data = {'debug': True, 'option': 2}
128 >>> delta = kv.delta(data, 'config.')
129 >>> delta.debug.previous
130 None
131 >>> delta.debug.current
132 True
133 >>> delta
134 {'debug': (None, True), 'option': (None, 2)}
135
136Note the delta method does not persist the actual change, it needs to
137be explicitly saved via 'update' method::
138
139 >>> kv.update(data, 'config.')
140
141Values modified in the context of a hook scope retain historical values
142associated to the hookname.
143
144 >>> with db.hook_scope('config-changed'):
145 ... db.set('x', 42)
146 >>> db.gethistory('x')
147 [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'),
148 (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')]
149
150"""
151
152import collections
153import contextlib
154import datetime
155import itertools
156import json
157import os
158import pprint
159import sqlite3
160import sys
161
162__author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>'
163
164
165class Storage(object):
166 """Simple key value database for local unit state within charms.
167
168 Modifications are not persisted unless :meth:`flush` is called.
169
170 To support dicts, lists, integer, floats, and booleans values
171 are automatically json encoded/decoded.
172 """
173 def __init__(self, path=None):
174 self.db_path = path
175 if path is None:
176 if 'UNIT_STATE_DB' in os.environ:
177 self.db_path = os.environ['UNIT_STATE_DB']
178 else:
179 self.db_path = os.path.join(
180 os.environ.get('CHARM_DIR', ''), '.unit-state.db')
181 self.conn = sqlite3.connect('%s' % self.db_path)
182 self.cursor = self.conn.cursor()
183 self.revision = None
184 self._closed = False
185 self._init()
186
187 def close(self):
188 if self._closed:
189 return
190 self.flush(False)
191 self.cursor.close()
192 self.conn.close()
193 self._closed = True
194
195 def get(self, key, default=None, record=False):
196 self.cursor.execute('select data from kv where key=?', [key])
197 result = self.cursor.fetchone()
198 if not result:
199 return default
200 if record:
201 return Record(json.loads(result[0]))
202 return json.loads(result[0])
203
204 def getrange(self, key_prefix, strip=False):
205 """
206 Get a range of keys starting with a common prefix as a mapping of
207 keys to values.
208
209 :param str key_prefix: Common prefix among all keys
210 :param bool strip: Optionally strip the common prefix from the key
211 names in the returned dict
212 :return dict: A (possibly empty) dict of key-value mappings
213 """
214 self.cursor.execute("select key, data from kv where key like ?",
215 ['%s%%' % key_prefix])
216 result = self.cursor.fetchall()
217
218 if not result:
219 return {}
220 if not strip:
221 key_prefix = ''
222 return dict([
223 (k[len(key_prefix):], json.loads(v)) for k, v in result])
224
225 def update(self, mapping, prefix=""):
226 """
227 Set the values of multiple keys at once.
228
229 :param dict mapping: Mapping of keys to values
230 :param str prefix: Optional prefix to apply to all keys in `mapping`
231 before setting
232 """
233 for k, v in mapping.items():
234 self.set("%s%s" % (prefix, k), v)
235
236 def unset(self, key):
237 """
238 Remove a key from the database entirely.
239 """
240 self.cursor.execute('delete from kv where key=?', [key])
241 if self.revision and self.cursor.rowcount:
242 self.cursor.execute(
243 'insert into kv_revisions values (?, ?, ?)',
244 [key, self.revision, json.dumps('DELETED')])
245
246 def unsetrange(self, keys=None, prefix=""):
247 """
248 Remove a range of keys starting with a common prefix, from the database
249 entirely.
250
251 :param list keys: List of keys to remove.
252 :param str prefix: Optional prefix to apply to all keys in ``keys``
253 before removing.
254 """
255 if keys is not None:
256 keys = ['%s%s' % (prefix, key) for key in keys]
257 self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys)
258 if self.revision and self.cursor.rowcount:
259 self.cursor.execute(
260 'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)),
261 list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys)))
262 else:
263 self.cursor.execute('delete from kv where key like ?',
264 ['%s%%' % prefix])
265 if self.revision and self.cursor.rowcount:
266 self.cursor.execute(
267 'insert into kv_revisions values (?, ?, ?)',
268 ['%s%%' % prefix, self.revision, json.dumps('DELETED')])
269
270 def set(self, key, value):
271 """
272 Set a value in the database.
273
274 :param str key: Key to set the value for
275 :param value: Any JSON-serializable value to be set
276 """
277 serialized = json.dumps(value)
278
279 self.cursor.execute('select data from kv where key=?', [key])
280 exists = self.cursor.fetchone()
281
282 # Skip mutations to the same value
283 if exists:
284 if exists[0] == serialized:
285 return value
286
287 if not exists:
288 self.cursor.execute(
289 'insert into kv (key, data) values (?, ?)',
290 (key, serialized))
291 else:
292 self.cursor.execute('''
293 update kv
294 set data = ?
295 where key = ?''', [serialized, key])
296
297 # Save
298 if not self.revision:
299 return value
300
301 self.cursor.execute(
302 'select 1 from kv_revisions where key=? and revision=?',
303 [key, self.revision])
304 exists = self.cursor.fetchone()
305
306 if not exists:
307 self.cursor.execute(
308 '''insert into kv_revisions (
309 revision, key, data) values (?, ?, ?)''',
310 (self.revision, key, serialized))
311 else:
312 self.cursor.execute(
313 '''
314 update kv_revisions
315 set data = ?
316 where key = ?
317 and revision = ?''',
318 [serialized, key, self.revision])
319
320 return value
321
322 def delta(self, mapping, prefix):
323 """
324 return a delta containing values that have changed.
325 """
326 previous = self.getrange(prefix, strip=True)
327 if not previous:
328 pk = set()
329 else:
330 pk = set(previous.keys())
331 ck = set(mapping.keys())
332 delta = DeltaSet()
333
334 # added
335 for k in ck.difference(pk):
336 delta[k] = Delta(None, mapping[k])
337
338 # removed
339 for k in pk.difference(ck):
340 delta[k] = Delta(previous[k], None)
341
342 # changed
343 for k in pk.intersection(ck):
344 c = mapping[k]
345 p = previous[k]
346 if c != p:
347 delta[k] = Delta(p, c)
348
349 return delta
350
351 @contextlib.contextmanager
352 def hook_scope(self, name=""):
353 """Scope all future interactions to the current hook execution
354 revision."""
355 assert not self.revision
356 self.cursor.execute(
357 'insert into hooks (hook, date) values (?, ?)',
358 (name or sys.argv[0],
359 datetime.datetime.utcnow().isoformat()))
360 self.revision = self.cursor.lastrowid
361 try:
362 yield self.revision
363 self.revision = None
364 except:
365 self.flush(False)
366 self.revision = None
367 raise
368 else:
369 self.flush()
370
371 def flush(self, save=True):
372 if save:
373 self.conn.commit()
374 elif self._closed:
375 return
376 else:
377 self.conn.rollback()
378
379 def _init(self):
380 self.cursor.execute('''
381 create table if not exists kv (
382 key text,
383 data text,
384 primary key (key)
385 )''')
386 self.cursor.execute('''
387 create table if not exists kv_revisions (
388 key text,
389 revision integer,
390 data text,
391 primary key (key, revision)
392 )''')
393 self.cursor.execute('''
394 create table if not exists hooks (
395 version integer primary key autoincrement,
396 hook text,
397 date text
398 )''')
399 self.conn.commit()
400
401 def gethistory(self, key, deserialize=False):
402 self.cursor.execute(
403 '''
404 select kv.revision, kv.key, kv.data, h.hook, h.date
405 from kv_revisions kv,
406 hooks h
407 where kv.key=?
408 and kv.revision = h.version
409 ''', [key])
410 if deserialize is False:
411 return self.cursor.fetchall()
412 return map(_parse_history, self.cursor.fetchall())
413
414 def debug(self, fh=sys.stderr):
415 self.cursor.execute('select * from kv')
416 pprint.pprint(self.cursor.fetchall(), stream=fh)
417 self.cursor.execute('select * from kv_revisions')
418 pprint.pprint(self.cursor.fetchall(), stream=fh)
419
420
421def _parse_history(d):
422 return (d[0], d[1], json.loads(d[2]), d[3],
423 datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f"))
424
425
426class HookData(object):
427 """Simple integration for existing hook exec frameworks.
428
429 Records all unit information, and stores deltas for processing
430 by the hook.
431
432 Sample::
433
434 from charmhelper.core import hookenv, unitdata
435
436 changes = unitdata.HookData()
437 db = unitdata.kv()
438 hooks = hookenv.Hooks()
439
440 @hooks.hook
441 def config_changed():
442 # View all changes to configuration
443 for changed, (prev, cur) in changes.conf.items():
444 print('config changed', changed,
445 'previous value', prev,
446 'current value', cur)
447
448 # Get some unit specific bookeeping
449 if not db.get('pkg_key'):
450 key = urllib.urlopen('https://example.com/pkg_key').read()
451 db.set('pkg_key', key)
452
453 if __name__ == '__main__':
454 with changes():
455 hook.execute()
456
457 """
458 def __init__(self):
459 self.kv = kv()
460 self.conf = None
461 self.rels = None
462
463 @contextlib.contextmanager
464 def __call__(self):
465 from charmhelpers.core import hookenv
466 hook_name = hookenv.hook_name()
467
468 with self.kv.hook_scope(hook_name):
469 self._record_charm_version(hookenv.charm_dir())
470 delta_config, delta_relation = self._record_hook(hookenv)
471 yield self.kv, delta_config, delta_relation
472
473 def _record_charm_version(self, charm_dir):
474 # Record revisions.. charm revisions are meaningless
475 # to charm authors as they don't control the revision.
476 # so logic dependnent on revision is not particularly
477 # useful, however it is useful for debugging analysis.
478 charm_rev = open(
479 os.path.join(charm_dir, 'revision')).read().strip()
480 charm_rev = charm_rev or '0'
481 revs = self.kv.get('charm_revisions', [])
482 if charm_rev not in revs:
483 revs.append(charm_rev.strip() or '0')
484 self.kv.set('charm_revisions', revs)
485
486 def _record_hook(self, hookenv):
487 data = hookenv.execution_environment()
488 self.conf = conf_delta = self.kv.delta(data['conf'], 'config')
489 self.rels = rels_delta = self.kv.delta(data['rels'], 'rels')
490 self.kv.set('env', dict(data['env']))
491 self.kv.set('unit', data['unit'])
492 self.kv.set('relid', data.get('relid'))
493 return conf_delta, rels_delta
494
495
496class Record(dict):
497
498 __slots__ = ()
499
500 def __getattr__(self, k):
501 if k in self:
502 return self[k]
503 raise AttributeError(k)
504
505
506class DeltaSet(Record):
507
508 __slots__ = ()
509
510
511Delta = collections.namedtuple('Delta', ['previous', 'current'])
512
513
514_KV = None
515
516
517def kv():
518 global _KV
519 if _KV is None:
520 _KV = Storage()
521 return _KV
0522
=== modified file 'hooks/hooks.py'
--- hooks/hooks.py 2015-12-06 22:48:27 +0000
+++ hooks/hooks.py 2015-12-17 05:44:32 +0000
@@ -15,6 +15,9 @@
15import glob15import glob
16import utils16import utils
1717
18from charmhelpers.core import hookenv
19from charmhelpers.core import unitdata
20
18###############################################################################21###############################################################################
19# Global variables22# Global variables
20###############################################################################23###############################################################################
@@ -22,9 +25,8 @@
22default_squid3_config = "%s/squid.conf" % default_squid3_config_dir25default_squid3_config = "%s/squid.conf" % default_squid3_config_dir
23default_squid3_config_cache_dir = "/var/run/squid3"26default_squid3_config_cache_dir = "/var/run/squid3"
24hook_name = os.path.basename(sys.argv[0])27hook_name = os.path.basename(sys.argv[0])
25HOOK_START = False28db = unitdata.kv()
26HOOK_AUTH_HELPER_JOINED = False29db_changes = unitdata.HookData()
27STATE_DELAYED_START = False
28###############################################################################30###############################################################################
29# Supporting functions31# Supporting functions
30###############################################################################32###############################################################################
@@ -321,24 +323,26 @@
321 retVal = subprocess.call(323 retVal = subprocess.call(
322 ['/usr/sbin/squid3', '-f', squid3_config, '-k', 'parse'])324 ['/usr/sbin/squid3', '-f', squid3_config, '-k', 'parse'])
323 if retVal == 1:325 if retVal == 1:
324 return(False)326 utils.juju_log('CRITICAL', 'Invalid squid configuration.')
327 return False
325 elif retVal == 0:328 elif retVal == 0:
326 return(True)329 return True
327 else:330 else:
328 return(False)331 utils.juju_log('CRITICAL', 'Invalid squid configuration.')
332 return False
329 elif action == 'status':333 elif action == 'status':
330 status = subprocess.check_output(['status', 'squid3'])334 status = subprocess.check_output(['status', 'squid3'])
331 if re.search('running', status) is not None:335 if re.search('running', status) is not None:
332 return(True)336 return True
333 else:337 else:
334 return(False)338 return False
335 elif action in ('start', 'stop', 'reload', 'restart'):339 elif action in ('start', 'stop', 'reload', 'restart'):
336 utils.juju_log('INFO', 'Requesting %s of squid3 service.' % action)340 utils.juju_log('INFO', 'Requesting %s of squid3 service.' % action)
337 retVal = subprocess.call([action, 'squid3'])341 retVal = subprocess.call([action, 'squid3'])
338 if retVal == 0:342 if retVal == 0:
339 return(True)343 return True
340 else:344 else:
341 return(False)345 return False
342346
343347
344def update_nrpe_checks():348def update_nrpe_checks():
@@ -400,6 +404,22 @@
400 return (utils.install_unattended('squid3', 'python-jinja2'))404 return (utils.install_unattended('squid3', 'python-jinja2'))
401405
402406
407def service_can_start():
408 # If wait_for_auth_helper is set, wait until squid has started and
409 # the squid-auth-helper relation has been joined.
410 config_data = config_get()
411 if not config_data['wait_for_auth_helper']:
412 return True
413 if hookenv.is_relation_made('squid-auth-helper') and db.get('hook_start'):
414 utils.juju_log('INFO',
415 'Squid auth helper available, squid may start...')
416 return True
417 else:
418 utils.juju_log('INFO', 'Squid not ready, waiting for auth helper...')
419 service_squid3('stop')
420 return False
421
422
403def config_changed():423def config_changed():
404 current_service_ports = get_service_ports()424 current_service_ports = get_service_ports()
405 construct_squid3_config()425 construct_squid3_config()
@@ -408,11 +428,7 @@
408 updated_service_ports = get_service_ports()428 updated_service_ports = get_service_ports()
409 update_service_ports(current_service_ports, updated_service_ports)429 update_service_ports(current_service_ports, updated_service_ports)
410430
411 config_data = config_get()431 if not service_can_start():
412 if config_data['wait_for_auth_helper'] and not STATE_DELAYED_START:
413 # unable to parse squid3 configuration without auth helper in
414 # place.
415 utils.juju_log('INFO', 'Squid not started, waiting for auth helper...')
416 return432 return
417433
418 if service_squid3('check'):434 if service_squid3('check'):
@@ -424,36 +440,26 @@
424 sys.exit(1)440 sys.exit(1)
425441
426442
427def start_hook(start=None, auth_helper=None):443def start_hook(start=None):
428 global HOOK_START
429 global HOOK_AUTH_HELPER_JOINED
430
431 if start:444 if start:
432 HOOK_START = True445 db.set('hook_start', True)
433 if auth_helper:446
434 HOOK_AUTH_HELPER_JOINED = True447 if service_can_start():
435448 if not service_squid3('check'):
436 config_data = config_get()449 sys.exit(1)
437 if config_data['wait_for_auth_helper']:450 else:
438 if HOOK_AUTH_HELPER_JOINED and HOOK_START:451 return
439 utils.juju_log('INFO', 'Squid auth helper available, starting...')452
440 if service_squid3('check'):453 if service_squid3('status'):
441 STATE_DELAYED_START = True454 return(service_squid3('restart'))
442 else:455 else:
443 sys.exit(1)456 return(service_squid3('start'))
444 else:
445 utils.juju_log('INFO', 'Waiting for auth helper...')
446 return
447
448 if service_squid3("status"):
449 return(service_squid3("restart"))
450 else:
451 return(service_squid3("start"))
452457
453458
454def stop_hook():459def stop_hook():
455 if service_squid3("status"):460 db.unset('hook_start')
456 return(service_squid3("stop"))461 if service_squid3('status'):
462 return(service_squid3('stop'))
457463
458464
459def proxy_interface(hook_name=None):465def proxy_interface(hook_name=None):
@@ -478,7 +484,7 @@
478 elif hook_name == "stop":484 elif hook_name == "stop":
479 stop_hook()485 stop_hook()
480 elif hook_name == "squid-auth-helper-relation-joined":486 elif hook_name == "squid-auth-helper-relation-joined":
481 start_hook(auth_helper=True)487 start_hook()
482 elif hook_name == "cached-website-relation-joined":488 elif hook_name == "cached-website-relation-joined":
483 proxy_interface("joined")489 proxy_interface("joined")
484 elif hook_name == "cached-website-relation-changed":490 elif hook_name == "cached-website-relation-changed":
@@ -498,4 +5