Merge lp:~brad-marshall/charms/trusty/ceilometer-agent/nagios-fix-servicegroups into lp:~openstack-charmers-archive/charms/trusty/ceilometer-agent/next
- Trusty Tahr (14.04)
- nagios-fix-servicegroups
- Merge into next
Proposed by
Brad Marshall
Status: | Merged |
---|---|
Merged at revision: | 48 |
Proposed branch: | lp:~brad-marshall/charms/trusty/ceilometer-agent/nagios-fix-servicegroups |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/ceilometer-agent/next |
Diff against target: |
993 lines (+670/-42) 16 files modified
config.yaml (+6/-0) hooks/charmhelpers/contrib/charmsupport/nrpe.py (+41/-7) hooks/charmhelpers/contrib/hahelpers/cluster.py (+5/-1) hooks/charmhelpers/contrib/openstack/amulet/deployment.py (+5/-2) hooks/charmhelpers/contrib/openstack/files/__init__.py (+18/-0) hooks/charmhelpers/contrib/openstack/ip.py (+37/-0) hooks/charmhelpers/contrib/openstack/utils.py (+1/-0) hooks/charmhelpers/contrib/python/packages.py (+2/-2) hooks/charmhelpers/core/fstab.py (+4/-4) hooks/charmhelpers/core/host.py (+5/-5) hooks/charmhelpers/core/strutils.py (+42/-0) hooks/charmhelpers/core/sysctl.py (+13/-7) hooks/charmhelpers/core/templating.py (+3/-3) hooks/charmhelpers/core/unitdata.py (+477/-0) hooks/charmhelpers/fetch/archiveurl.py (+10/-10) hooks/charmhelpers/fetch/giturl.py (+1/-1) |
To merge this branch: | bzr merge lp:~brad-marshall/charms/trusty/ceilometer-agent/nagios-fix-servicegroups |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Liam Young (community) | Approve | ||
Review via email: mp+250712@code.launchpad.net |
Commit message
Description of the change
Synced charmhelpers, added nagios_servicegroup config option
To post a comment you must log in.
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #2016 ceilometer-
UNIT OK: passed
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #2173 ceilometer-
AMULET FAIL: amulet-test missing
AMULET Results (max last 2 lines):
INFO:root:Search string not found in makefile target commands.
ERROR:root:No make target was executed.
Full amulet test output: http://
Build: http://
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'config.yaml' |
2 | --- config.yaml 2014-11-18 05:51:05 +0000 |
3 | +++ config.yaml 2015-02-24 06:28:18 +0000 |
4 | @@ -24,3 +24,9 @@ |
5 | juju-myservice-0 |
6 | If you're running multiple environments with the same services in them |
7 | this allows you to differentiate between them. |
8 | + nagios_servicegroups: |
9 | + default: "" |
10 | + type: string |
11 | + description: | |
12 | + A comma-separated list of nagios servicegroups. |
13 | + If left empty, the nagios_context will be used as the servicegroup |
14 | |
15 | === modified file 'hooks/charmhelpers/contrib/charmsupport/nrpe.py' |
16 | --- hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-01-26 07:50:21 +0000 |
17 | +++ hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-02-24 06:28:18 +0000 |
18 | @@ -24,6 +24,8 @@ |
19 | import pwd |
20 | import grp |
21 | import os |
22 | +import glob |
23 | +import shutil |
24 | import re |
25 | import shlex |
26 | import yaml |
27 | @@ -161,7 +163,7 @@ |
28 | log('Check command not found: {}'.format(parts[0])) |
29 | return '' |
30 | |
31 | - def write(self, nagios_context, hostname, nagios_servicegroups=None): |
32 | + def write(self, nagios_context, hostname, nagios_servicegroups): |
33 | nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( |
34 | self.command) |
35 | with open(nrpe_check_file, 'w') as nrpe_check_config: |
36 | @@ -177,14 +179,11 @@ |
37 | nagios_servicegroups) |
38 | |
39 | def write_service_config(self, nagios_context, hostname, |
40 | - nagios_servicegroups=None): |
41 | + nagios_servicegroups): |
42 | for f in os.listdir(NRPE.nagios_exportdir): |
43 | if re.search('.*{}.cfg'.format(self.command), f): |
44 | os.remove(os.path.join(NRPE.nagios_exportdir, f)) |
45 | |
46 | - if not nagios_servicegroups: |
47 | - nagios_servicegroups = nagios_context |
48 | - |
49 | templ_vars = { |
50 | 'nagios_hostname': hostname, |
51 | 'nagios_servicegroup': nagios_servicegroups, |
52 | @@ -211,10 +210,10 @@ |
53 | super(NRPE, self).__init__() |
54 | self.config = config() |
55 | self.nagios_context = self.config['nagios_context'] |
56 | - if 'nagios_servicegroups' in self.config: |
57 | + if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: |
58 | self.nagios_servicegroups = self.config['nagios_servicegroups'] |
59 | else: |
60 | - self.nagios_servicegroups = 'juju' |
61 | + self.nagios_servicegroups = self.nagios_context |
62 | self.unit_name = local_unit().replace('/', '-') |
63 | if hostname: |
64 | self.hostname = hostname |
65 | @@ -322,3 +321,38 @@ |
66 | check_cmd='check_status_file.py -f ' |
67 | '/var/lib/nagios/service-check-%s.txt' % svc, |
68 | ) |
69 | + |
70 | + |
71 | +def copy_nrpe_checks(): |
72 | + """ |
73 | + Copy the nrpe checks into place |
74 | + |
75 | + """ |
76 | + NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' |
77 | + nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks', |
78 | + 'charmhelpers', 'contrib', 'openstack', |
79 | + 'files') |
80 | + |
81 | + if not os.path.exists(NAGIOS_PLUGINS): |
82 | + os.makedirs(NAGIOS_PLUGINS) |
83 | + for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): |
84 | + if os.path.isfile(fname): |
85 | + shutil.copy2(fname, |
86 | + os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) |
87 | + |
88 | + |
89 | +def add_haproxy_checks(nrpe, unit_name): |
90 | + """ |
91 | + Add checks for each service in list |
92 | + |
93 | + :param NRPE nrpe: NRPE object to add check to |
94 | + :param str unit_name: Unit name to use in check description |
95 | + """ |
96 | + nrpe.add_check( |
97 | + shortname='haproxy_servers', |
98 | + description='Check HAProxy {%s}' % unit_name, |
99 | + check_cmd='check_haproxy.sh') |
100 | + nrpe.add_check( |
101 | + shortname='haproxy_queue', |
102 | + description='Check HAProxy queue depth {%s}' % unit_name, |
103 | + check_cmd='check_haproxy_queue_depth.sh') |
104 | |
105 | === modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py' |
106 | --- hooks/charmhelpers/contrib/hahelpers/cluster.py 2015-01-26 10:55:38 +0000 |
107 | +++ hooks/charmhelpers/contrib/hahelpers/cluster.py 2015-02-24 06:28:18 +0000 |
108 | @@ -48,6 +48,9 @@ |
109 | from charmhelpers.core.decorators import ( |
110 | retry_on_exception, |
111 | ) |
112 | +from charmhelpers.core.strutils import ( |
113 | + bool_from_string, |
114 | +) |
115 | |
116 | |
117 | class HAIncompleteConfig(Exception): |
118 | @@ -164,7 +167,8 @@ |
119 | . |
120 | returns: boolean |
121 | ''' |
122 | - if config_get('use-https') == "yes": |
123 | + use_https = config_get('use-https') |
124 | + if use_https and bool_from_string(use_https): |
125 | return True |
126 | if config_get('ssl_cert') and config_get('ssl_key'): |
127 | return True |
128 | |
129 | === modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py' |
130 | --- hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2015-01-26 10:55:38 +0000 |
131 | +++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2015-02-24 06:28:18 +0000 |
132 | @@ -71,16 +71,19 @@ |
133 | services.append(this_service) |
134 | use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', |
135 | 'ceph-osd', 'ceph-radosgw'] |
136 | + # Openstack subordinate charms do not expose an origin option as that |
137 | + # is controlled by the principle |
138 | + ignore = ['neutron-openvswitch'] |
139 | |
140 | if self.openstack: |
141 | for svc in services: |
142 | - if svc['name'] not in use_source: |
143 | + if svc['name'] not in use_source + ignore: |
144 | config = {'openstack-origin': self.openstack} |
145 | self.d.configure(svc['name'], config) |
146 | |
147 | if self.source: |
148 | for svc in services: |
149 | - if svc['name'] in use_source: |
150 | + if svc['name'] in use_source and svc['name'] not in ignore: |
151 | config = {'source': self.source} |
152 | self.d.configure(svc['name'], config) |
153 | |
154 | |
155 | === added directory 'hooks/charmhelpers/contrib/openstack/files' |
156 | === added file 'hooks/charmhelpers/contrib/openstack/files/__init__.py' |
157 | --- hooks/charmhelpers/contrib/openstack/files/__init__.py 1970-01-01 00:00:00 +0000 |
158 | +++ hooks/charmhelpers/contrib/openstack/files/__init__.py 2015-02-24 06:28:18 +0000 |
159 | @@ -0,0 +1,18 @@ |
160 | +# Copyright 2014-2015 Canonical Limited. |
161 | +# |
162 | +# This file is part of charm-helpers. |
163 | +# |
164 | +# charm-helpers is free software: you can redistribute it and/or modify |
165 | +# it under the terms of the GNU Lesser General Public License version 3 as |
166 | +# published by the Free Software Foundation. |
167 | +# |
168 | +# charm-helpers is distributed in the hope that it will be useful, |
169 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
170 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
171 | +# GNU Lesser General Public License for more details. |
172 | +# |
173 | +# You should have received a copy of the GNU Lesser General Public License |
174 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
175 | + |
176 | +# dummy __init__.py to fool syncer into thinking this is a syncable python |
177 | +# module |
178 | |
179 | === modified file 'hooks/charmhelpers/contrib/openstack/ip.py' |
180 | --- hooks/charmhelpers/contrib/openstack/ip.py 2015-01-26 10:55:38 +0000 |
181 | +++ hooks/charmhelpers/contrib/openstack/ip.py 2015-02-24 06:28:18 +0000 |
182 | @@ -26,6 +26,8 @@ |
183 | ) |
184 | from charmhelpers.contrib.hahelpers.cluster import is_clustered |
185 | |
186 | +from functools import partial |
187 | + |
188 | PUBLIC = 'public' |
189 | INTERNAL = 'int' |
190 | ADMIN = 'admin' |
191 | @@ -107,3 +109,38 @@ |
192 | "clustered=%s)" % (net_type, clustered)) |
193 | |
194 | return resolved_address |
195 | + |
196 | + |
197 | +def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC, |
198 | + override=None): |
199 | + """Returns the correct endpoint URL to advertise to Keystone. |
200 | + |
201 | + This method provides the correct endpoint URL which should be advertised to |
202 | + the keystone charm for endpoint creation. This method allows for the url to |
203 | + be overridden to force a keystone endpoint to have specific URL for any of |
204 | + the defined scopes (admin, internal, public). |
205 | + |
206 | + :param configs: OSTemplateRenderer config templating object to inspect |
207 | + for a complete https context. |
208 | + :param url_template: str format string for creating the url template. Only |
209 | + two values will be passed - the scheme+hostname |
210 | + returned by the canonical_url and the port. |
211 | + :param endpoint_type: str endpoint type to resolve. |
212 | + :param override: str the name of the config option which overrides the |
213 | + endpoint URL defined by the charm itself. None will |
214 | + disable any overrides (default). |
215 | + """ |
216 | + if override: |
217 | + # Return any user-defined overrides for the keystone endpoint URL. |
218 | + user_value = config(override) |
219 | + if user_value: |
220 | + return user_value.strip() |
221 | + |
222 | + return url_template % (canonical_url(configs, endpoint_type), port) |
223 | + |
224 | + |
225 | +public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC) |
226 | + |
227 | +internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL) |
228 | + |
229 | +admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN) |
230 | |
231 | === modified file 'hooks/charmhelpers/contrib/openstack/utils.py' |
232 | --- hooks/charmhelpers/contrib/openstack/utils.py 2015-01-26 10:55:38 +0000 |
233 | +++ hooks/charmhelpers/contrib/openstack/utils.py 2015-02-24 06:28:18 +0000 |
234 | @@ -103,6 +103,7 @@ |
235 | ('2.1.0', 'juno'), |
236 | ('2.2.0', 'juno'), |
237 | ('2.2.1', 'kilo'), |
238 | + ('2.2.2', 'kilo'), |
239 | ]) |
240 | |
241 | DEFAULT_LOOPBACK_SIZE = '5G' |
242 | |
243 | === modified file 'hooks/charmhelpers/contrib/python/packages.py' |
244 | --- hooks/charmhelpers/contrib/python/packages.py 2015-01-26 10:55:38 +0000 |
245 | +++ hooks/charmhelpers/contrib/python/packages.py 2015-02-24 06:28:18 +0000 |
246 | @@ -17,8 +17,6 @@ |
247 | # You should have received a copy of the GNU Lesser General Public License |
248 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
249 | |
250 | -__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>" |
251 | - |
252 | from charmhelpers.fetch import apt_install, apt_update |
253 | from charmhelpers.core.hookenv import log |
254 | |
255 | @@ -29,6 +27,8 @@ |
256 | apt_install('python-pip') |
257 | from pip import main as pip_execute |
258 | |
259 | +__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>" |
260 | + |
261 | |
262 | def parse_options(given, available): |
263 | """Given a set of options, check if available""" |
264 | |
265 | === modified file 'hooks/charmhelpers/core/fstab.py' |
266 | --- hooks/charmhelpers/core/fstab.py 2015-01-26 10:55:38 +0000 |
267 | +++ hooks/charmhelpers/core/fstab.py 2015-02-24 06:28:18 +0000 |
268 | @@ -17,11 +17,11 @@ |
269 | # You should have received a copy of the GNU Lesser General Public License |
270 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
271 | |
272 | -__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
273 | - |
274 | import io |
275 | import os |
276 | |
277 | +__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
278 | + |
279 | |
280 | class Fstab(io.FileIO): |
281 | """This class extends file in order to implement a file reader/writer |
282 | @@ -77,7 +77,7 @@ |
283 | for line in self.readlines(): |
284 | line = line.decode('us-ascii') |
285 | try: |
286 | - if line.strip() and not line.startswith("#"): |
287 | + if line.strip() and not line.strip().startswith("#"): |
288 | yield self._hydrate_entry(line) |
289 | except ValueError: |
290 | pass |
291 | @@ -104,7 +104,7 @@ |
292 | |
293 | found = False |
294 | for index, line in enumerate(lines): |
295 | - if not line.startswith("#"): |
296 | + if line.strip() and not line.strip().startswith("#"): |
297 | if self._hydrate_entry(line) == entry: |
298 | found = True |
299 | break |
300 | |
301 | === modified file 'hooks/charmhelpers/core/host.py' |
302 | --- hooks/charmhelpers/core/host.py 2015-01-26 10:55:38 +0000 |
303 | +++ hooks/charmhelpers/core/host.py 2015-02-24 06:28:18 +0000 |
304 | @@ -191,11 +191,11 @@ |
305 | |
306 | |
307 | def write_file(path, content, owner='root', group='root', perms=0o444): |
308 | - """Create or overwrite a file with the contents of a string""" |
309 | + """Create or overwrite a file with the contents of a byte string.""" |
310 | log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) |
311 | uid = pwd.getpwnam(owner).pw_uid |
312 | gid = grp.getgrnam(group).gr_gid |
313 | - with open(path, 'w') as target: |
314 | + with open(path, 'wb') as target: |
315 | os.fchown(target.fileno(), uid, gid) |
316 | os.fchmod(target.fileno(), perms) |
317 | target.write(content) |
318 | @@ -305,11 +305,11 @@ |
319 | ceph_client_changed function. |
320 | """ |
321 | def wrap(f): |
322 | - def wrapped_f(*args): |
323 | + def wrapped_f(*args, **kwargs): |
324 | checksums = {} |
325 | for path in restart_map: |
326 | checksums[path] = file_hash(path) |
327 | - f(*args) |
328 | + f(*args, **kwargs) |
329 | restarts = [] |
330 | for path in restart_map: |
331 | if checksums[path] != file_hash(path): |
332 | @@ -361,7 +361,7 @@ |
333 | ip_output = (line for line in ip_output if line) |
334 | for line in ip_output: |
335 | if line.split()[1].startswith(int_type): |
336 | - matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line) |
337 | + matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line) |
338 | if matched: |
339 | interface = matched.groups()[0] |
340 | else: |
341 | |
342 | === added file 'hooks/charmhelpers/core/strutils.py' |
343 | --- hooks/charmhelpers/core/strutils.py 1970-01-01 00:00:00 +0000 |
344 | +++ hooks/charmhelpers/core/strutils.py 2015-02-24 06:28:18 +0000 |
345 | @@ -0,0 +1,42 @@ |
346 | +#!/usr/bin/env python |
347 | +# -*- coding: utf-8 -*- |
348 | + |
349 | +# Copyright 2014-2015 Canonical Limited. |
350 | +# |
351 | +# This file is part of charm-helpers. |
352 | +# |
353 | +# charm-helpers is free software: you can redistribute it and/or modify |
354 | +# it under the terms of the GNU Lesser General Public License version 3 as |
355 | +# published by the Free Software Foundation. |
356 | +# |
357 | +# charm-helpers is distributed in the hope that it will be useful, |
358 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
359 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
360 | +# GNU Lesser General Public License for more details. |
361 | +# |
362 | +# You should have received a copy of the GNU Lesser General Public License |
363 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
364 | + |
365 | +import six |
366 | + |
367 | + |
368 | +def bool_from_string(value): |
369 | + """Interpret string value as boolean. |
370 | + |
371 | + Returns True if value translates to True otherwise False. |
372 | + """ |
373 | + if isinstance(value, six.string_types): |
374 | + value = six.text_type(value) |
375 | + else: |
376 | + msg = "Unable to interpret non-string value '%s' as boolean" % (value) |
377 | + raise ValueError(msg) |
378 | + |
379 | + value = value.strip().lower() |
380 | + |
381 | + if value in ['y', 'yes', 'true', 't']: |
382 | + return True |
383 | + elif value in ['n', 'no', 'false', 'f']: |
384 | + return False |
385 | + |
386 | + msg = "Unable to interpret string value '%s' as boolean" % (value) |
387 | + raise ValueError(msg) |
388 | |
389 | === modified file 'hooks/charmhelpers/core/sysctl.py' |
390 | --- hooks/charmhelpers/core/sysctl.py 2015-01-26 10:55:38 +0000 |
391 | +++ hooks/charmhelpers/core/sysctl.py 2015-02-24 06:28:18 +0000 |
392 | @@ -17,8 +17,6 @@ |
393 | # You should have received a copy of the GNU Lesser General Public License |
394 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
395 | |
396 | -__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
397 | - |
398 | import yaml |
399 | |
400 | from subprocess import check_call |
401 | @@ -26,25 +24,33 @@ |
402 | from charmhelpers.core.hookenv import ( |
403 | log, |
404 | DEBUG, |
405 | + ERROR, |
406 | ) |
407 | |
408 | +__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
409 | + |
410 | |
411 | def create(sysctl_dict, sysctl_file): |
412 | """Creates a sysctl.conf file from a YAML associative array |
413 | |
414 | - :param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 } |
415 | - :type sysctl_dict: dict |
416 | + :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" |
417 | + :type sysctl_dict: str |
418 | :param sysctl_file: path to the sysctl file to be saved |
419 | :type sysctl_file: str or unicode |
420 | :returns: None |
421 | """ |
422 | - sysctl_dict = yaml.load(sysctl_dict) |
423 | + try: |
424 | + sysctl_dict_parsed = yaml.safe_load(sysctl_dict) |
425 | + except yaml.YAMLError: |
426 | + log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), |
427 | + level=ERROR) |
428 | + return |
429 | |
430 | with open(sysctl_file, "w") as fd: |
431 | - for key, value in sysctl_dict.items(): |
432 | + for key, value in sysctl_dict_parsed.items(): |
433 | fd.write("{}={}\n".format(key, value)) |
434 | |
435 | - log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict), |
436 | + log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed), |
437 | level=DEBUG) |
438 | |
439 | check_call(["sysctl", "-p", sysctl_file]) |
440 | |
441 | === modified file 'hooks/charmhelpers/core/templating.py' |
442 | --- hooks/charmhelpers/core/templating.py 2015-01-26 10:55:38 +0000 |
443 | +++ hooks/charmhelpers/core/templating.py 2015-02-24 06:28:18 +0000 |
444 | @@ -21,7 +21,7 @@ |
445 | |
446 | |
447 | def render(source, target, context, owner='root', group='root', |
448 | - perms=0o444, templates_dir=None): |
449 | + perms=0o444, templates_dir=None, encoding='UTF-8'): |
450 | """ |
451 | Render a template. |
452 | |
453 | @@ -64,5 +64,5 @@ |
454 | level=hookenv.ERROR) |
455 | raise e |
456 | content = template.render(context) |
457 | - host.mkdir(os.path.dirname(target), owner, group) |
458 | - host.write_file(target, content, owner, group, perms) |
459 | + host.mkdir(os.path.dirname(target), owner, group, perms=0o755) |
460 | + host.write_file(target, content.encode(encoding), owner, group, perms) |
461 | |
462 | === added file 'hooks/charmhelpers/core/unitdata.py' |
463 | --- hooks/charmhelpers/core/unitdata.py 1970-01-01 00:00:00 +0000 |
464 | +++ hooks/charmhelpers/core/unitdata.py 2015-02-24 06:28:18 +0000 |
465 | @@ -0,0 +1,477 @@ |
466 | +#!/usr/bin/env python |
467 | +# -*- coding: utf-8 -*- |
468 | +# |
469 | +# Copyright 2014-2015 Canonical Limited. |
470 | +# |
471 | +# This file is part of charm-helpers. |
472 | +# |
473 | +# charm-helpers is free software: you can redistribute it and/or modify |
474 | +# it under the terms of the GNU Lesser General Public License version 3 as |
475 | +# published by the Free Software Foundation. |
476 | +# |
477 | +# charm-helpers is distributed in the hope that it will be useful, |
478 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
479 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
480 | +# GNU Lesser General Public License for more details. |
481 | +# |
482 | +# You should have received a copy of the GNU Lesser General Public License |
483 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
484 | +# |
485 | +# |
486 | +# Authors: |
487 | +# Kapil Thangavelu <kapil.foss@gmail.com> |
488 | +# |
489 | +""" |
490 | +Intro |
491 | +----- |
492 | + |
493 | +A simple way to store state in units. This provides a key value |
494 | +storage with support for versioned, transactional operation, |
495 | +and can calculate deltas from previous values to simplify unit logic |
496 | +when processing changes. |
497 | + |
498 | + |
499 | +Hook Integration |
500 | +---------------- |
501 | + |
502 | +There are several extant frameworks for hook execution, including |
503 | + |
504 | + - charmhelpers.core.hookenv.Hooks |
505 | + - charmhelpers.core.services.ServiceManager |
506 | + |
507 | +The storage classes are framework agnostic, one simple integration is |
508 | +via the HookData contextmanager. It will record the current hook |
509 | +execution environment (including relation data, config data, etc.), |
510 | +setup a transaction and allow easy access to the changes from |
511 | +previously seen values. One consequence of the integration is the |
512 | +reservation of particular keys ('rels', 'unit', 'env', 'config', |
513 | +'charm_revisions') for their respective values. |
514 | + |
515 | +Here's a fully worked integration example using hookenv.Hooks:: |
516 | + |
517 | + from charmhelper.core import hookenv, unitdata |
518 | + |
519 | + hook_data = unitdata.HookData() |
520 | + db = unitdata.kv() |
521 | + hooks = hookenv.Hooks() |
522 | + |
523 | + @hooks.hook |
524 | + def config_changed(): |
525 | + # Print all changes to configuration from previously seen |
526 | + # values. |
527 | + for changed, (prev, cur) in hook_data.conf.items(): |
528 | + print('config changed', changed, |
529 | + 'previous value', prev, |
530 | + 'current value', cur) |
531 | + |
532 | + # Get some unit specific bookeeping |
533 | + if not db.get('pkg_key'): |
534 | + key = urllib.urlopen('https://example.com/pkg_key').read() |
535 | + db.set('pkg_key', key) |
536 | + |
537 | + # Directly access all charm config as a mapping. |
538 | + conf = db.getrange('config', True) |
539 | + |
540 | + # Directly access all relation data as a mapping |
541 | + rels = db.getrange('rels', True) |
542 | + |
543 | + if __name__ == '__main__': |
544 | + with hook_data(): |
545 | + hook.execute() |
546 | + |
547 | + |
548 | +A more basic integration is via the hook_scope context manager which simply |
549 | +manages transaction scope (and records hook name, and timestamp):: |
550 | + |
551 | + >>> from unitdata import kv |
552 | + >>> db = kv() |
553 | + >>> with db.hook_scope('install'): |
554 | + ... # do work, in transactional scope. |
555 | + ... db.set('x', 1) |
556 | + >>> db.get('x') |
557 | + 1 |
558 | + |
559 | + |
560 | +Usage |
561 | +----- |
562 | + |
563 | +Values are automatically json de/serialized to preserve basic typing |
564 | +and complex data struct capabilities (dicts, lists, ints, booleans, etc). |
565 | + |
566 | +Individual values can be manipulated via get/set:: |
567 | + |
568 | + >>> kv.set('y', True) |
569 | + >>> kv.get('y') |
570 | + True |
571 | + |
572 | + # We can set complex values (dicts, lists) as a single key. |
573 | + >>> kv.set('config', {'a': 1, 'b': True'}) |
574 | + |
575 | + # Also supports returning dictionaries as a record which |
576 | + # provides attribute access. |
577 | + >>> config = kv.get('config', record=True) |
578 | + >>> config.b |
579 | + True |
580 | + |
581 | + |
582 | +Groups of keys can be manipulated with update/getrange:: |
583 | + |
584 | + >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") |
585 | + >>> kv.getrange('gui.', strip=True) |
586 | + {'z': 1, 'y': 2} |
587 | + |
588 | +When updating values, its very helpful to understand which values |
589 | +have actually changed and how have they changed. The storage |
590 | +provides a delta method to provide for this:: |
591 | + |
592 | + >>> data = {'debug': True, 'option': 2} |
593 | + >>> delta = kv.delta(data, 'config.') |
594 | + >>> delta.debug.previous |
595 | + None |
596 | + >>> delta.debug.current |
597 | + True |
598 | + >>> delta |
599 | + {'debug': (None, True), 'option': (None, 2)} |
600 | + |
601 | +Note the delta method does not persist the actual change, it needs to |
602 | +be explicitly saved via 'update' method:: |
603 | + |
604 | + >>> kv.update(data, 'config.') |
605 | + |
606 | +Values modified in the context of a hook scope retain historical values |
607 | +associated to the hookname. |
608 | + |
609 | + >>> with db.hook_scope('config-changed'): |
610 | + ... db.set('x', 42) |
611 | + >>> db.gethistory('x') |
612 | + [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), |
613 | + (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] |
614 | + |
615 | +""" |
616 | + |
617 | +import collections |
618 | +import contextlib |
619 | +import datetime |
620 | +import json |
621 | +import os |
622 | +import pprint |
623 | +import sqlite3 |
624 | +import sys |
625 | + |
626 | +__author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>' |
627 | + |
628 | + |
629 | +class Storage(object): |
630 | + """Simple key value database for local unit state within charms. |
631 | + |
632 | + Modifications are automatically committed at hook exit. That's |
633 | + currently regardless of exit code. |
634 | + |
635 | + To support dicts, lists, integer, floats, and booleans values |
636 | + are automatically json encoded/decoded. |
637 | + """ |
638 | + def __init__(self, path=None): |
639 | + self.db_path = path |
640 | + if path is None: |
641 | + self.db_path = os.path.join( |
642 | + os.environ.get('CHARM_DIR', ''), '.unit-state.db') |
643 | + self.conn = sqlite3.connect('%s' % self.db_path) |
644 | + self.cursor = self.conn.cursor() |
645 | + self.revision = None |
646 | + self._closed = False |
647 | + self._init() |
648 | + |
649 | + def close(self): |
650 | + if self._closed: |
651 | + return |
652 | + self.flush(False) |
653 | + self.cursor.close() |
654 | + self.conn.close() |
655 | + self._closed = True |
656 | + |
657 | + def _scoped_query(self, stmt, params=None): |
658 | + if params is None: |
659 | + params = [] |
660 | + return stmt, params |
661 | + |
662 | + def get(self, key, default=None, record=False): |
663 | + self.cursor.execute( |
664 | + *self._scoped_query( |
665 | + 'select data from kv where key=?', [key])) |
666 | + result = self.cursor.fetchone() |
667 | + if not result: |
668 | + return default |
669 | + if record: |
670 | + return Record(json.loads(result[0])) |
671 | + return json.loads(result[0]) |
672 | + |
673 | + def getrange(self, key_prefix, strip=False): |
674 | + stmt = "select key, data from kv where key like '%s%%'" % key_prefix |
675 | + self.cursor.execute(*self._scoped_query(stmt)) |
676 | + result = self.cursor.fetchall() |
677 | + |
678 | + if not result: |
679 | + return None |
680 | + if not strip: |
681 | + key_prefix = '' |
682 | + return dict([ |
683 | + (k[len(key_prefix):], json.loads(v)) for k, v in result]) |
684 | + |
685 | + def update(self, mapping, prefix=""): |
686 | + for k, v in mapping.items(): |
687 | + self.set("%s%s" % (prefix, k), v) |
688 | + |
689 | + def unset(self, key): |
690 | + self.cursor.execute('delete from kv where key=?', [key]) |
691 | + if self.revision and self.cursor.rowcount: |
692 | + self.cursor.execute( |
693 | + 'insert into kv_revisions values (?, ?, ?)', |
694 | + [key, self.revision, json.dumps('DELETED')]) |
695 | + |
696 | + def set(self, key, value): |
697 | + serialized = json.dumps(value) |
698 | + |
699 | + self.cursor.execute( |
700 | + 'select data from kv where key=?', [key]) |
701 | + exists = self.cursor.fetchone() |
702 | + |
703 | + # Skip mutations to the same value |
704 | + if exists: |
705 | + if exists[0] == serialized: |
706 | + return value |
707 | + |
708 | + if not exists: |
709 | + self.cursor.execute( |
710 | + 'insert into kv (key, data) values (?, ?)', |
711 | + (key, serialized)) |
712 | + else: |
713 | + self.cursor.execute(''' |
714 | + update kv |
715 | + set data = ? |
716 | + where key = ?''', [serialized, key]) |
717 | + |
718 | + # Save |
719 | + if not self.revision: |
720 | + return value |
721 | + |
722 | + self.cursor.execute( |
723 | + 'select 1 from kv_revisions where key=? and revision=?', |
724 | + [key, self.revision]) |
725 | + exists = self.cursor.fetchone() |
726 | + |
727 | + if not exists: |
728 | + self.cursor.execute( |
729 | + '''insert into kv_revisions ( |
730 | + revision, key, data) values (?, ?, ?)''', |
731 | + (self.revision, key, serialized)) |
732 | + else: |
733 | + self.cursor.execute( |
734 | + ''' |
735 | + update kv_revisions |
736 | + set data = ? |
737 | + where key = ? |
738 | + and revision = ?''', |
739 | + [serialized, key, self.revision]) |
740 | + |
741 | + return value |
742 | + |
743 | + def delta(self, mapping, prefix): |
744 | + """ |
745 | + return a delta containing values that have changed. |
746 | + """ |
747 | + previous = self.getrange(prefix, strip=True) |
748 | + if not previous: |
749 | + pk = set() |
750 | + else: |
751 | + pk = set(previous.keys()) |
752 | + ck = set(mapping.keys()) |
753 | + delta = DeltaSet() |
754 | + |
755 | + # added |
756 | + for k in ck.difference(pk): |
757 | + delta[k] = Delta(None, mapping[k]) |
758 | + |
759 | + # removed |
760 | + for k in pk.difference(ck): |
761 | + delta[k] = Delta(previous[k], None) |
762 | + |
763 | + # changed |
764 | + for k in pk.intersection(ck): |
765 | + c = mapping[k] |
766 | + p = previous[k] |
767 | + if c != p: |
768 | + delta[k] = Delta(p, c) |
769 | + |
770 | + return delta |
771 | + |
772 | + @contextlib.contextmanager |
773 | + def hook_scope(self, name=""): |
774 | + """Scope all future interactions to the current hook execution |
775 | + revision.""" |
776 | + assert not self.revision |
777 | + self.cursor.execute( |
778 | + 'insert into hooks (hook, date) values (?, ?)', |
779 | + (name or sys.argv[0], |
780 | + datetime.datetime.utcnow().isoformat())) |
781 | + self.revision = self.cursor.lastrowid |
782 | + try: |
783 | + yield self.revision |
784 | + self.revision = None |
785 | + except: |
786 | + self.flush(False) |
787 | + self.revision = None |
788 | + raise |
789 | + else: |
790 | + self.flush() |
791 | + |
792 | + def flush(self, save=True): |
793 | + if save: |
794 | + self.conn.commit() |
795 | + elif self._closed: |
796 | + return |
797 | + else: |
798 | + self.conn.rollback() |
799 | + |
800 | + def _init(self): |
801 | + self.cursor.execute(''' |
802 | + create table if not exists kv ( |
803 | + key text, |
804 | + data text, |
805 | + primary key (key) |
806 | + )''') |
807 | + self.cursor.execute(''' |
808 | + create table if not exists kv_revisions ( |
809 | + key text, |
810 | + revision integer, |
811 | + data text, |
812 | + primary key (key, revision) |
813 | + )''') |
814 | + self.cursor.execute(''' |
815 | + create table if not exists hooks ( |
816 | + version integer primary key autoincrement, |
817 | + hook text, |
818 | + date text |
819 | + )''') |
820 | + self.conn.commit() |
821 | + |
822 | + def gethistory(self, key, deserialize=False): |
823 | + self.cursor.execute( |
824 | + ''' |
825 | + select kv.revision, kv.key, kv.data, h.hook, h.date |
826 | + from kv_revisions kv, |
827 | + hooks h |
828 | + where kv.key=? |
829 | + and kv.revision = h.version |
830 | + ''', [key]) |
831 | + if deserialize is False: |
832 | + return self.cursor.fetchall() |
833 | + return map(_parse_history, self.cursor.fetchall()) |
834 | + |
835 | + def debug(self, fh=sys.stderr): |
836 | + self.cursor.execute('select * from kv') |
837 | + pprint.pprint(self.cursor.fetchall(), stream=fh) |
838 | + self.cursor.execute('select * from kv_revisions') |
839 | + pprint.pprint(self.cursor.fetchall(), stream=fh) |
840 | + |
841 | + |
842 | +def _parse_history(d): |
843 | + return (d[0], d[1], json.loads(d[2]), d[3], |
844 | + datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) |
845 | + |
846 | + |
847 | +class HookData(object): |
848 | + """Simple integration for existing hook exec frameworks. |
849 | + |
850 | + Records all unit information, and stores deltas for processing |
851 | + by the hook. |
852 | + |
853 | + Sample:: |
854 | + |
855 | + from charmhelper.core import hookenv, unitdata |
856 | + |
857 | + changes = unitdata.HookData() |
858 | + db = unitdata.kv() |
859 | + hooks = hookenv.Hooks() |
860 | + |
861 | + @hooks.hook |
862 | + def config_changed(): |
863 | + # View all changes to configuration |
864 | + for changed, (prev, cur) in changes.conf.items(): |
865 | + print('config changed', changed, |
866 | + 'previous value', prev, |
867 | + 'current value', cur) |
868 | + |
869 | + # Get some unit specific bookeeping |
870 | + if not db.get('pkg_key'): |
871 | + key = urllib.urlopen('https://example.com/pkg_key').read() |
872 | + db.set('pkg_key', key) |
873 | + |
874 | + if __name__ == '__main__': |
875 | + with changes(): |
876 | + hook.execute() |
877 | + |
878 | + """ |
879 | + def __init__(self): |
880 | + self.kv = kv() |
881 | + self.conf = None |
882 | + self.rels = None |
883 | + |
884 | + @contextlib.contextmanager |
885 | + def __call__(self): |
886 | + from charmhelpers.core import hookenv |
887 | + hook_name = hookenv.hook_name() |
888 | + |
889 | + with self.kv.hook_scope(hook_name): |
890 | + self._record_charm_version(hookenv.charm_dir()) |
891 | + delta_config, delta_relation = self._record_hook(hookenv) |
892 | + yield self.kv, delta_config, delta_relation |
893 | + |
894 | + def _record_charm_version(self, charm_dir): |
895 | + # Record revisions.. charm revisions are meaningless |
896 | + # to charm authors as they don't control the revision. |
897 | + # so logic dependnent on revision is not particularly |
898 | + # useful, however it is useful for debugging analysis. |
899 | + charm_rev = open( |
900 | + os.path.join(charm_dir, 'revision')).read().strip() |
901 | + charm_rev = charm_rev or '0' |
902 | + revs = self.kv.get('charm_revisions', []) |
903 | + if charm_rev not in revs: |
904 | + revs.append(charm_rev.strip() or '0') |
905 | + self.kv.set('charm_revisions', revs) |
906 | + |
907 | + def _record_hook(self, hookenv): |
908 | + data = hookenv.execution_environment() |
909 | + self.conf = conf_delta = self.kv.delta(data['conf'], 'config') |
910 | + self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') |
911 | + self.kv.set('env', data['env']) |
912 | + self.kv.set('unit', data['unit']) |
913 | + self.kv.set('relid', data.get('relid')) |
914 | + return conf_delta, rels_delta |
915 | + |
916 | + |
917 | +class Record(dict): |
918 | + |
919 | + __slots__ = () |
920 | + |
921 | + def __getattr__(self, k): |
922 | + if k in self: |
923 | + return self[k] |
924 | + raise AttributeError(k) |
925 | + |
926 | + |
927 | +class DeltaSet(Record): |
928 | + |
929 | + __slots__ = () |
930 | + |
931 | + |
932 | +Delta = collections.namedtuple('Delta', ['previous', 'current']) |
933 | + |
934 | + |
935 | +_KV = None |
936 | + |
937 | + |
938 | +def kv(): |
939 | + global _KV |
940 | + if _KV is None: |
941 | + _KV = Storage() |
942 | + return _KV |
943 | |
944 | === modified file 'hooks/charmhelpers/fetch/archiveurl.py' |
945 | --- hooks/charmhelpers/fetch/archiveurl.py 2015-01-26 10:55:38 +0000 |
946 | +++ hooks/charmhelpers/fetch/archiveurl.py 2015-02-24 06:28:18 +0000 |
947 | @@ -18,6 +18,16 @@ |
948 | import hashlib |
949 | import re |
950 | |
951 | +from charmhelpers.fetch import ( |
952 | + BaseFetchHandler, |
953 | + UnhandledSource |
954 | +) |
955 | +from charmhelpers.payload.archive import ( |
956 | + get_archive_handler, |
957 | + extract, |
958 | +) |
959 | +from charmhelpers.core.host import mkdir, check_hash |
960 | + |
961 | import six |
962 | if six.PY3: |
963 | from urllib.request import ( |
964 | @@ -35,16 +45,6 @@ |
965 | ) |
966 | from urlparse import urlparse, urlunparse, parse_qs |
967 | |
968 | -from charmhelpers.fetch import ( |
969 | - BaseFetchHandler, |
970 | - UnhandledSource |
971 | -) |
972 | -from charmhelpers.payload.archive import ( |
973 | - get_archive_handler, |
974 | - extract, |
975 | -) |
976 | -from charmhelpers.core.host import mkdir, check_hash |
977 | - |
978 | |
979 | def splituser(host): |
980 | '''urllib.splituser(), but six's support of this seems broken''' |
981 | |
982 | === modified file 'hooks/charmhelpers/fetch/giturl.py' |
983 | --- hooks/charmhelpers/fetch/giturl.py 2015-01-26 10:55:38 +0000 |
984 | +++ hooks/charmhelpers/fetch/giturl.py 2015-02-24 06:28:18 +0000 |
985 | @@ -32,7 +32,7 @@ |
986 | apt_install("python-git") |
987 | from git import Repo |
988 | |
989 | -from git.exc import GitCommandError |
990 | +from git.exc import GitCommandError # noqa E402 |
991 | |
992 | |
993 | class GitUrlFetchHandler(BaseFetchHandler): |
charm_lint_check #2227 ceilometer- agent-next for brad-marshall mp250712
LINT OK: passed
Build: http:// 10.245. 162.77: 8080/job/ charm_lint_ check/2227/