Merge lp:~brad-marshall/charms/trusty/ceph-osd/nagios-fix-servicegroups into lp:~openstack-charmers-archive/charms/trusty/ceph-osd/next
- Trusty Tahr (14.04)
- nagios-fix-servicegroups
- Merge into next
Proposed by
Brad Marshall
Status: | Merged |
---|---|
Merged at revision: | 39 |
Proposed branch: | lp:~brad-marshall/charms/trusty/ceph-osd/nagios-fix-servicegroups |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/ceph-osd/next |
Diff against target: |
1021 lines (+731/-42) 13 files modified
config.yaml (+7/-0) hooks/charmhelpers/contrib/charmsupport/nrpe.py (+41/-7) hooks/charmhelpers/core/fstab.py (+4/-4) hooks/charmhelpers/core/host.py (+5/-5) hooks/charmhelpers/core/strutils.py (+42/-0) hooks/charmhelpers/core/sysctl.py (+13/-7) hooks/charmhelpers/core/templating.py (+3/-3) hooks/charmhelpers/core/unitdata.py (+477/-0) hooks/charmhelpers/fetch/archiveurl.py (+10/-10) hooks/charmhelpers/fetch/giturl.py (+1/-1) tests/basic_deployment.py (+1/-1) tests/charmhelpers/contrib/amulet/utils.py (+122/-2) tests/charmhelpers/contrib/openstack/amulet/deployment.py (+5/-2) |
To merge this branch: | bzr merge lp:~brad-marshall/charms/trusty/ceph-osd/nagios-fix-servicegroups |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Liam Young (community) | Approve | ||
Review via email: mp+250710@code.launchpad.net |
Commit message
Description of the change
Synced charmhelpers, added nagios_servicegroup config option, and added haproxy nrpe checks.
To post a comment you must log in.
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #2225 ceph-osd-next for brad-marshall mp250710
LINT OK: passed
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #2171 ceph-osd-next for brad-marshall mp250710
AMULET FAIL: amulet-test failed
AMULET Results (max last 2 lines):
ERROR subprocess encountered error code 1
make: *** [test] Error 1
Full amulet test output: http://
Build: http://
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'config.yaml' |
2 | --- config.yaml 2015-02-19 10:17:21 +0000 |
3 | +++ config.yaml 2015-02-24 06:25:16 +0000 |
4 | @@ -134,6 +134,7 @@ |
5 | nagios_context: |
6 | type: string |
7 | default: "juju" |
8 | + type: string |
9 | description: | |
10 | Used by the nrpe-external-master subordinate charm. |
11 | A string that will be prepended to instance name to set the host name |
12 | @@ -141,3 +142,9 @@ |
13 | juju-myservice-0 |
14 | If you're running multiple environments with the same services in them |
15 | this allows you to differentiate between them. |
16 | + nagios_servicegroups: |
17 | + default: "" |
18 | + type: string |
19 | + description: | |
20 | + A comma-separated list of nagios servicegroups. |
21 | + If left empty, the nagios_context will be used as the servicegroup |
22 | |
23 | === modified file 'hooks/charmhelpers/contrib/charmsupport/nrpe.py' |
24 | --- hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-01-26 11:51:28 +0000 |
25 | +++ hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-02-24 06:25:16 +0000 |
26 | @@ -24,6 +24,8 @@ |
27 | import pwd |
28 | import grp |
29 | import os |
30 | +import glob |
31 | +import shutil |
32 | import re |
33 | import shlex |
34 | import yaml |
35 | @@ -161,7 +163,7 @@ |
36 | log('Check command not found: {}'.format(parts[0])) |
37 | return '' |
38 | |
39 | - def write(self, nagios_context, hostname, nagios_servicegroups=None): |
40 | + def write(self, nagios_context, hostname, nagios_servicegroups): |
41 | nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( |
42 | self.command) |
43 | with open(nrpe_check_file, 'w') as nrpe_check_config: |
44 | @@ -177,14 +179,11 @@ |
45 | nagios_servicegroups) |
46 | |
47 | def write_service_config(self, nagios_context, hostname, |
48 | - nagios_servicegroups=None): |
49 | + nagios_servicegroups): |
50 | for f in os.listdir(NRPE.nagios_exportdir): |
51 | if re.search('.*{}.cfg'.format(self.command), f): |
52 | os.remove(os.path.join(NRPE.nagios_exportdir, f)) |
53 | |
54 | - if not nagios_servicegroups: |
55 | - nagios_servicegroups = nagios_context |
56 | - |
57 | templ_vars = { |
58 | 'nagios_hostname': hostname, |
59 | 'nagios_servicegroup': nagios_servicegroups, |
60 | @@ -211,10 +210,10 @@ |
61 | super(NRPE, self).__init__() |
62 | self.config = config() |
63 | self.nagios_context = self.config['nagios_context'] |
64 | - if 'nagios_servicegroups' in self.config: |
65 | + if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: |
66 | self.nagios_servicegroups = self.config['nagios_servicegroups'] |
67 | else: |
68 | - self.nagios_servicegroups = 'juju' |
69 | + self.nagios_servicegroups = self.nagios_context |
70 | self.unit_name = local_unit().replace('/', '-') |
71 | if hostname: |
72 | self.hostname = hostname |
73 | @@ -322,3 +321,38 @@ |
74 | check_cmd='check_status_file.py -f ' |
75 | '/var/lib/nagios/service-check-%s.txt' % svc, |
76 | ) |
77 | + |
78 | + |
79 | +def copy_nrpe_checks(): |
80 | + """ |
81 | + Copy the nrpe checks into place |
82 | + |
83 | + """ |
84 | + NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' |
85 | + nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks', |
86 | + 'charmhelpers', 'contrib', 'openstack', |
87 | + 'files') |
88 | + |
89 | + if not os.path.exists(NAGIOS_PLUGINS): |
90 | + os.makedirs(NAGIOS_PLUGINS) |
91 | + for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): |
92 | + if os.path.isfile(fname): |
93 | + shutil.copy2(fname, |
94 | + os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) |
95 | + |
96 | + |
97 | +def add_haproxy_checks(nrpe, unit_name): |
98 | + """ |
99 | + Add checks for each service in list |
100 | + |
101 | + :param NRPE nrpe: NRPE object to add check to |
102 | + :param str unit_name: Unit name to use in check description |
103 | + """ |
104 | + nrpe.add_check( |
105 | + shortname='haproxy_servers', |
106 | + description='Check HAProxy {%s}' % unit_name, |
107 | + check_cmd='check_haproxy.sh') |
108 | + nrpe.add_check( |
109 | + shortname='haproxy_queue', |
110 | + description='Check HAProxy queue depth {%s}' % unit_name, |
111 | + check_cmd='check_haproxy_queue_depth.sh') |
112 | |
113 | === modified file 'hooks/charmhelpers/core/fstab.py' |
114 | --- hooks/charmhelpers/core/fstab.py 2015-01-26 11:51:28 +0000 |
115 | +++ hooks/charmhelpers/core/fstab.py 2015-02-24 06:25:16 +0000 |
116 | @@ -17,11 +17,11 @@ |
117 | # You should have received a copy of the GNU Lesser General Public License |
118 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
119 | |
120 | -__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
121 | - |
122 | import io |
123 | import os |
124 | |
125 | +__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
126 | + |
127 | |
128 | class Fstab(io.FileIO): |
129 | """This class extends file in order to implement a file reader/writer |
130 | @@ -77,7 +77,7 @@ |
131 | for line in self.readlines(): |
132 | line = line.decode('us-ascii') |
133 | try: |
134 | - if line.strip() and not line.startswith("#"): |
135 | + if line.strip() and not line.strip().startswith("#"): |
136 | yield self._hydrate_entry(line) |
137 | except ValueError: |
138 | pass |
139 | @@ -104,7 +104,7 @@ |
140 | |
141 | found = False |
142 | for index, line in enumerate(lines): |
143 | - if not line.startswith("#"): |
144 | + if line.strip() and not line.strip().startswith("#"): |
145 | if self._hydrate_entry(line) == entry: |
146 | found = True |
147 | break |
148 | |
149 | === modified file 'hooks/charmhelpers/core/host.py' |
150 | --- hooks/charmhelpers/core/host.py 2015-01-26 11:51:28 +0000 |
151 | +++ hooks/charmhelpers/core/host.py 2015-02-24 06:25:16 +0000 |
152 | @@ -191,11 +191,11 @@ |
153 | |
154 | |
155 | def write_file(path, content, owner='root', group='root', perms=0o444): |
156 | - """Create or overwrite a file with the contents of a string""" |
157 | + """Create or overwrite a file with the contents of a byte string.""" |
158 | log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) |
159 | uid = pwd.getpwnam(owner).pw_uid |
160 | gid = grp.getgrnam(group).gr_gid |
161 | - with open(path, 'w') as target: |
162 | + with open(path, 'wb') as target: |
163 | os.fchown(target.fileno(), uid, gid) |
164 | os.fchmod(target.fileno(), perms) |
165 | target.write(content) |
166 | @@ -305,11 +305,11 @@ |
167 | ceph_client_changed function. |
168 | """ |
169 | def wrap(f): |
170 | - def wrapped_f(*args): |
171 | + def wrapped_f(*args, **kwargs): |
172 | checksums = {} |
173 | for path in restart_map: |
174 | checksums[path] = file_hash(path) |
175 | - f(*args) |
176 | + f(*args, **kwargs) |
177 | restarts = [] |
178 | for path in restart_map: |
179 | if checksums[path] != file_hash(path): |
180 | @@ -361,7 +361,7 @@ |
181 | ip_output = (line for line in ip_output if line) |
182 | for line in ip_output: |
183 | if line.split()[1].startswith(int_type): |
184 | - matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line) |
185 | + matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line) |
186 | if matched: |
187 | interface = matched.groups()[0] |
188 | else: |
189 | |
190 | === added file 'hooks/charmhelpers/core/strutils.py' |
191 | --- hooks/charmhelpers/core/strutils.py 1970-01-01 00:00:00 +0000 |
192 | +++ hooks/charmhelpers/core/strutils.py 2015-02-24 06:25:16 +0000 |
193 | @@ -0,0 +1,42 @@ |
194 | +#!/usr/bin/env python |
195 | +# -*- coding: utf-8 -*- |
196 | + |
197 | +# Copyright 2014-2015 Canonical Limited. |
198 | +# |
199 | +# This file is part of charm-helpers. |
200 | +# |
201 | +# charm-helpers is free software: you can redistribute it and/or modify |
202 | +# it under the terms of the GNU Lesser General Public License version 3 as |
203 | +# published by the Free Software Foundation. |
204 | +# |
205 | +# charm-helpers is distributed in the hope that it will be useful, |
206 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
207 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
208 | +# GNU Lesser General Public License for more details. |
209 | +# |
210 | +# You should have received a copy of the GNU Lesser General Public License |
211 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
212 | + |
213 | +import six |
214 | + |
215 | + |
216 | +def bool_from_string(value): |
217 | + """Interpret string value as boolean. |
218 | + |
219 | + Returns True if value translates to True otherwise False. |
220 | + """ |
221 | + if isinstance(value, six.string_types): |
222 | + value = six.text_type(value) |
223 | + else: |
224 | + msg = "Unable to interpret non-string value '%s' as boolean" % (value) |
225 | + raise ValueError(msg) |
226 | + |
227 | + value = value.strip().lower() |
228 | + |
229 | + if value in ['y', 'yes', 'true', 't']: |
230 | + return True |
231 | + elif value in ['n', 'no', 'false', 'f']: |
232 | + return False |
233 | + |
234 | + msg = "Unable to interpret string value '%s' as boolean" % (value) |
235 | + raise ValueError(msg) |
236 | |
237 | === modified file 'hooks/charmhelpers/core/sysctl.py' |
238 | --- hooks/charmhelpers/core/sysctl.py 2015-01-26 11:51:28 +0000 |
239 | +++ hooks/charmhelpers/core/sysctl.py 2015-02-24 06:25:16 +0000 |
240 | @@ -17,8 +17,6 @@ |
241 | # You should have received a copy of the GNU Lesser General Public License |
242 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
243 | |
244 | -__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
245 | - |
246 | import yaml |
247 | |
248 | from subprocess import check_call |
249 | @@ -26,25 +24,33 @@ |
250 | from charmhelpers.core.hookenv import ( |
251 | log, |
252 | DEBUG, |
253 | + ERROR, |
254 | ) |
255 | |
256 | +__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
257 | + |
258 | |
259 | def create(sysctl_dict, sysctl_file): |
260 | """Creates a sysctl.conf file from a YAML associative array |
261 | |
262 | - :param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 } |
263 | - :type sysctl_dict: dict |
264 | + :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" |
265 | + :type sysctl_dict: str |
266 | :param sysctl_file: path to the sysctl file to be saved |
267 | :type sysctl_file: str or unicode |
268 | :returns: None |
269 | """ |
270 | - sysctl_dict = yaml.load(sysctl_dict) |
271 | + try: |
272 | + sysctl_dict_parsed = yaml.safe_load(sysctl_dict) |
273 | + except yaml.YAMLError: |
274 | + log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), |
275 | + level=ERROR) |
276 | + return |
277 | |
278 | with open(sysctl_file, "w") as fd: |
279 | - for key, value in sysctl_dict.items(): |
280 | + for key, value in sysctl_dict_parsed.items(): |
281 | fd.write("{}={}\n".format(key, value)) |
282 | |
283 | - log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict), |
284 | + log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed), |
285 | level=DEBUG) |
286 | |
287 | check_call(["sysctl", "-p", sysctl_file]) |
288 | |
289 | === modified file 'hooks/charmhelpers/core/templating.py' |
290 | --- hooks/charmhelpers/core/templating.py 2015-01-26 11:51:28 +0000 |
291 | +++ hooks/charmhelpers/core/templating.py 2015-02-24 06:25:16 +0000 |
292 | @@ -21,7 +21,7 @@ |
293 | |
294 | |
295 | def render(source, target, context, owner='root', group='root', |
296 | - perms=0o444, templates_dir=None): |
297 | + perms=0o444, templates_dir=None, encoding='UTF-8'): |
298 | """ |
299 | Render a template. |
300 | |
301 | @@ -64,5 +64,5 @@ |
302 | level=hookenv.ERROR) |
303 | raise e |
304 | content = template.render(context) |
305 | - host.mkdir(os.path.dirname(target), owner, group) |
306 | - host.write_file(target, content, owner, group, perms) |
307 | + host.mkdir(os.path.dirname(target), owner, group, perms=0o755) |
308 | + host.write_file(target, content.encode(encoding), owner, group, perms) |
309 | |
310 | === added file 'hooks/charmhelpers/core/unitdata.py' |
311 | --- hooks/charmhelpers/core/unitdata.py 1970-01-01 00:00:00 +0000 |
312 | +++ hooks/charmhelpers/core/unitdata.py 2015-02-24 06:25:16 +0000 |
313 | @@ -0,0 +1,477 @@ |
314 | +#!/usr/bin/env python |
315 | +# -*- coding: utf-8 -*- |
316 | +# |
317 | +# Copyright 2014-2015 Canonical Limited. |
318 | +# |
319 | +# This file is part of charm-helpers. |
320 | +# |
321 | +# charm-helpers is free software: you can redistribute it and/or modify |
322 | +# it under the terms of the GNU Lesser General Public License version 3 as |
323 | +# published by the Free Software Foundation. |
324 | +# |
325 | +# charm-helpers is distributed in the hope that it will be useful, |
326 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
327 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
328 | +# GNU Lesser General Public License for more details. |
329 | +# |
330 | +# You should have received a copy of the GNU Lesser General Public License |
331 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
332 | +# |
333 | +# |
334 | +# Authors: |
335 | +# Kapil Thangavelu <kapil.foss@gmail.com> |
336 | +# |
337 | +""" |
338 | +Intro |
339 | +----- |
340 | + |
341 | +A simple way to store state in units. This provides a key value |
342 | +storage with support for versioned, transactional operation, |
343 | +and can calculate deltas from previous values to simplify unit logic |
344 | +when processing changes. |
345 | + |
346 | + |
347 | +Hook Integration |
348 | +---------------- |
349 | + |
350 | +There are several extant frameworks for hook execution, including |
351 | + |
352 | + - charmhelpers.core.hookenv.Hooks |
353 | + - charmhelpers.core.services.ServiceManager |
354 | + |
355 | +The storage classes are framework agnostic, one simple integration is |
356 | +via the HookData contextmanager. It will record the current hook |
357 | +execution environment (including relation data, config data, etc.), |
358 | +setup a transaction and allow easy access to the changes from |
359 | +previously seen values. One consequence of the integration is the |
360 | +reservation of particular keys ('rels', 'unit', 'env', 'config', |
361 | +'charm_revisions') for their respective values. |
362 | + |
363 | +Here's a fully worked integration example using hookenv.Hooks:: |
364 | + |
365 | + from charmhelper.core import hookenv, unitdata |
366 | + |
367 | + hook_data = unitdata.HookData() |
368 | + db = unitdata.kv() |
369 | + hooks = hookenv.Hooks() |
370 | + |
371 | + @hooks.hook |
372 | + def config_changed(): |
373 | + # Print all changes to configuration from previously seen |
374 | + # values. |
375 | + for changed, (prev, cur) in hook_data.conf.items(): |
376 | + print('config changed', changed, |
377 | + 'previous value', prev, |
378 | + 'current value', cur) |
379 | + |
380 | + # Get some unit specific bookeeping |
381 | + if not db.get('pkg_key'): |
382 | + key = urllib.urlopen('https://example.com/pkg_key').read() |
383 | + db.set('pkg_key', key) |
384 | + |
385 | + # Directly access all charm config as a mapping. |
386 | + conf = db.getrange('config', True) |
387 | + |
388 | + # Directly access all relation data as a mapping |
389 | + rels = db.getrange('rels', True) |
390 | + |
391 | + if __name__ == '__main__': |
392 | + with hook_data(): |
393 | + hook.execute() |
394 | + |
395 | + |
396 | +A more basic integration is via the hook_scope context manager which simply |
397 | +manages transaction scope (and records hook name, and timestamp):: |
398 | + |
399 | + >>> from unitdata import kv |
400 | + >>> db = kv() |
401 | + >>> with db.hook_scope('install'): |
402 | + ... # do work, in transactional scope. |
403 | + ... db.set('x', 1) |
404 | + >>> db.get('x') |
405 | + 1 |
406 | + |
407 | + |
408 | +Usage |
409 | +----- |
410 | + |
411 | +Values are automatically json de/serialized to preserve basic typing |
412 | +and complex data struct capabilities (dicts, lists, ints, booleans, etc). |
413 | + |
414 | +Individual values can be manipulated via get/set:: |
415 | + |
416 | + >>> kv.set('y', True) |
417 | + >>> kv.get('y') |
418 | + True |
419 | + |
420 | + # We can set complex values (dicts, lists) as a single key. |
421 | + >>> kv.set('config', {'a': 1, 'b': True'}) |
422 | + |
423 | + # Also supports returning dictionaries as a record which |
424 | + # provides attribute access. |
425 | + >>> config = kv.get('config', record=True) |
426 | + >>> config.b |
427 | + True |
428 | + |
429 | + |
430 | +Groups of keys can be manipulated with update/getrange:: |
431 | + |
432 | + >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") |
433 | + >>> kv.getrange('gui.', strip=True) |
434 | + {'z': 1, 'y': 2} |
435 | + |
436 | +When updating values, its very helpful to understand which values |
437 | +have actually changed and how have they changed. The storage |
438 | +provides a delta method to provide for this:: |
439 | + |
440 | + >>> data = {'debug': True, 'option': 2} |
441 | + >>> delta = kv.delta(data, 'config.') |
442 | + >>> delta.debug.previous |
443 | + None |
444 | + >>> delta.debug.current |
445 | + True |
446 | + >>> delta |
447 | + {'debug': (None, True), 'option': (None, 2)} |
448 | + |
449 | +Note the delta method does not persist the actual change, it needs to |
450 | +be explicitly saved via 'update' method:: |
451 | + |
452 | + >>> kv.update(data, 'config.') |
453 | + |
454 | +Values modified in the context of a hook scope retain historical values |
455 | +associated to the hookname. |
456 | + |
457 | + >>> with db.hook_scope('config-changed'): |
458 | + ... db.set('x', 42) |
459 | + >>> db.gethistory('x') |
460 | + [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), |
461 | + (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] |
462 | + |
463 | +""" |
464 | + |
465 | +import collections |
466 | +import contextlib |
467 | +import datetime |
468 | +import json |
469 | +import os |
470 | +import pprint |
471 | +import sqlite3 |
472 | +import sys |
473 | + |
474 | +__author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>' |
475 | + |
476 | + |
477 | +class Storage(object): |
478 | + """Simple key value database for local unit state within charms. |
479 | + |
480 | + Modifications are automatically committed at hook exit. That's |
481 | + currently regardless of exit code. |
482 | + |
483 | + To support dicts, lists, integer, floats, and booleans values |
484 | + are automatically json encoded/decoded. |
485 | + """ |
486 | + def __init__(self, path=None): |
487 | + self.db_path = path |
488 | + if path is None: |
489 | + self.db_path = os.path.join( |
490 | + os.environ.get('CHARM_DIR', ''), '.unit-state.db') |
491 | + self.conn = sqlite3.connect('%s' % self.db_path) |
492 | + self.cursor = self.conn.cursor() |
493 | + self.revision = None |
494 | + self._closed = False |
495 | + self._init() |
496 | + |
497 | + def close(self): |
498 | + if self._closed: |
499 | + return |
500 | + self.flush(False) |
501 | + self.cursor.close() |
502 | + self.conn.close() |
503 | + self._closed = True |
504 | + |
505 | + def _scoped_query(self, stmt, params=None): |
506 | + if params is None: |
507 | + params = [] |
508 | + return stmt, params |
509 | + |
510 | + def get(self, key, default=None, record=False): |
511 | + self.cursor.execute( |
512 | + *self._scoped_query( |
513 | + 'select data from kv where key=?', [key])) |
514 | + result = self.cursor.fetchone() |
515 | + if not result: |
516 | + return default |
517 | + if record: |
518 | + return Record(json.loads(result[0])) |
519 | + return json.loads(result[0]) |
520 | + |
521 | + def getrange(self, key_prefix, strip=False): |
522 | + stmt = "select key, data from kv where key like '%s%%'" % key_prefix |
523 | + self.cursor.execute(*self._scoped_query(stmt)) |
524 | + result = self.cursor.fetchall() |
525 | + |
526 | + if not result: |
527 | + return None |
528 | + if not strip: |
529 | + key_prefix = '' |
530 | + return dict([ |
531 | + (k[len(key_prefix):], json.loads(v)) for k, v in result]) |
532 | + |
533 | + def update(self, mapping, prefix=""): |
534 | + for k, v in mapping.items(): |
535 | + self.set("%s%s" % (prefix, k), v) |
536 | + |
537 | + def unset(self, key): |
538 | + self.cursor.execute('delete from kv where key=?', [key]) |
539 | + if self.revision and self.cursor.rowcount: |
540 | + self.cursor.execute( |
541 | + 'insert into kv_revisions values (?, ?, ?)', |
542 | + [key, self.revision, json.dumps('DELETED')]) |
543 | + |
544 | + def set(self, key, value): |
545 | + serialized = json.dumps(value) |
546 | + |
547 | + self.cursor.execute( |
548 | + 'select data from kv where key=?', [key]) |
549 | + exists = self.cursor.fetchone() |
550 | + |
551 | + # Skip mutations to the same value |
552 | + if exists: |
553 | + if exists[0] == serialized: |
554 | + return value |
555 | + |
556 | + if not exists: |
557 | + self.cursor.execute( |
558 | + 'insert into kv (key, data) values (?, ?)', |
559 | + (key, serialized)) |
560 | + else: |
561 | + self.cursor.execute(''' |
562 | + update kv |
563 | + set data = ? |
564 | + where key = ?''', [serialized, key]) |
565 | + |
566 | + # Save |
567 | + if not self.revision: |
568 | + return value |
569 | + |
570 | + self.cursor.execute( |
571 | + 'select 1 from kv_revisions where key=? and revision=?', |
572 | + [key, self.revision]) |
573 | + exists = self.cursor.fetchone() |
574 | + |
575 | + if not exists: |
576 | + self.cursor.execute( |
577 | + '''insert into kv_revisions ( |
578 | + revision, key, data) values (?, ?, ?)''', |
579 | + (self.revision, key, serialized)) |
580 | + else: |
581 | + self.cursor.execute( |
582 | + ''' |
583 | + update kv_revisions |
584 | + set data = ? |
585 | + where key = ? |
586 | + and revision = ?''', |
587 | + [serialized, key, self.revision]) |
588 | + |
589 | + return value |
590 | + |
591 | + def delta(self, mapping, prefix): |
592 | + """ |
593 | + return a delta containing values that have changed. |
594 | + """ |
595 | + previous = self.getrange(prefix, strip=True) |
596 | + if not previous: |
597 | + pk = set() |
598 | + else: |
599 | + pk = set(previous.keys()) |
600 | + ck = set(mapping.keys()) |
601 | + delta = DeltaSet() |
602 | + |
603 | + # added |
604 | + for k in ck.difference(pk): |
605 | + delta[k] = Delta(None, mapping[k]) |
606 | + |
607 | + # removed |
608 | + for k in pk.difference(ck): |
609 | + delta[k] = Delta(previous[k], None) |
610 | + |
611 | + # changed |
612 | + for k in pk.intersection(ck): |
613 | + c = mapping[k] |
614 | + p = previous[k] |
615 | + if c != p: |
616 | + delta[k] = Delta(p, c) |
617 | + |
618 | + return delta |
619 | + |
620 | + @contextlib.contextmanager |
621 | + def hook_scope(self, name=""): |
622 | + """Scope all future interactions to the current hook execution |
623 | + revision.""" |
624 | + assert not self.revision |
625 | + self.cursor.execute( |
626 | + 'insert into hooks (hook, date) values (?, ?)', |
627 | + (name or sys.argv[0], |
628 | + datetime.datetime.utcnow().isoformat())) |
629 | + self.revision = self.cursor.lastrowid |
630 | + try: |
631 | + yield self.revision |
632 | + self.revision = None |
633 | + except: |
634 | + self.flush(False) |
635 | + self.revision = None |
636 | + raise |
637 | + else: |
638 | + self.flush() |
639 | + |
640 | + def flush(self, save=True): |
641 | + if save: |
642 | + self.conn.commit() |
643 | + elif self._closed: |
644 | + return |
645 | + else: |
646 | + self.conn.rollback() |
647 | + |
648 | + def _init(self): |
649 | + self.cursor.execute(''' |
650 | + create table if not exists kv ( |
651 | + key text, |
652 | + data text, |
653 | + primary key (key) |
654 | + )''') |
655 | + self.cursor.execute(''' |
656 | + create table if not exists kv_revisions ( |
657 | + key text, |
658 | + revision integer, |
659 | + data text, |
660 | + primary key (key, revision) |
661 | + )''') |
662 | + self.cursor.execute(''' |
663 | + create table if not exists hooks ( |
664 | + version integer primary key autoincrement, |
665 | + hook text, |
666 | + date text |
667 | + )''') |
668 | + self.conn.commit() |
669 | + |
670 | + def gethistory(self, key, deserialize=False): |
671 | + self.cursor.execute( |
672 | + ''' |
673 | + select kv.revision, kv.key, kv.data, h.hook, h.date |
674 | + from kv_revisions kv, |
675 | + hooks h |
676 | + where kv.key=? |
677 | + and kv.revision = h.version |
678 | + ''', [key]) |
679 | + if deserialize is False: |
680 | + return self.cursor.fetchall() |
681 | + return map(_parse_history, self.cursor.fetchall()) |
682 | + |
683 | + def debug(self, fh=sys.stderr): |
684 | + self.cursor.execute('select * from kv') |
685 | + pprint.pprint(self.cursor.fetchall(), stream=fh) |
686 | + self.cursor.execute('select * from kv_revisions') |
687 | + pprint.pprint(self.cursor.fetchall(), stream=fh) |
688 | + |
689 | + |
690 | +def _parse_history(d): |
691 | + return (d[0], d[1], json.loads(d[2]), d[3], |
692 | + datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) |
693 | + |
694 | + |
695 | +class HookData(object): |
696 | + """Simple integration for existing hook exec frameworks. |
697 | + |
698 | + Records all unit information, and stores deltas for processing |
699 | + by the hook. |
700 | + |
701 | + Sample:: |
702 | + |
703 | + from charmhelper.core import hookenv, unitdata |
704 | + |
705 | + changes = unitdata.HookData() |
706 | + db = unitdata.kv() |
707 | + hooks = hookenv.Hooks() |
708 | + |
709 | + @hooks.hook |
710 | + def config_changed(): |
711 | + # View all changes to configuration |
712 | + for changed, (prev, cur) in changes.conf.items(): |
713 | + print('config changed', changed, |
714 | + 'previous value', prev, |
715 | + 'current value', cur) |
716 | + |
717 | + # Get some unit specific bookeeping |
718 | + if not db.get('pkg_key'): |
719 | + key = urllib.urlopen('https://example.com/pkg_key').read() |
720 | + db.set('pkg_key', key) |
721 | + |
722 | + if __name__ == '__main__': |
723 | + with changes(): |
724 | + hook.execute() |
725 | + |
726 | + """ |
727 | + def __init__(self): |
728 | + self.kv = kv() |
729 | + self.conf = None |
730 | + self.rels = None |
731 | + |
732 | + @contextlib.contextmanager |
733 | + def __call__(self): |
734 | + from charmhelpers.core import hookenv |
735 | + hook_name = hookenv.hook_name() |
736 | + |
737 | + with self.kv.hook_scope(hook_name): |
738 | + self._record_charm_version(hookenv.charm_dir()) |
739 | + delta_config, delta_relation = self._record_hook(hookenv) |
740 | + yield self.kv, delta_config, delta_relation |
741 | + |
742 | + def _record_charm_version(self, charm_dir): |
743 | + # Record revisions.. charm revisions are meaningless |
744 | + # to charm authors as they don't control the revision. |
745 | + # so logic dependnent on revision is not particularly |
746 | + # useful, however it is useful for debugging analysis. |
747 | + charm_rev = open( |
748 | + os.path.join(charm_dir, 'revision')).read().strip() |
749 | + charm_rev = charm_rev or '0' |
750 | + revs = self.kv.get('charm_revisions', []) |
751 | + if charm_rev not in revs: |
752 | + revs.append(charm_rev.strip() or '0') |
753 | + self.kv.set('charm_revisions', revs) |
754 | + |
755 | + def _record_hook(self, hookenv): |
756 | + data = hookenv.execution_environment() |
757 | + self.conf = conf_delta = self.kv.delta(data['conf'], 'config') |
758 | + self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') |
759 | + self.kv.set('env', data['env']) |
760 | + self.kv.set('unit', data['unit']) |
761 | + self.kv.set('relid', data.get('relid')) |
762 | + return conf_delta, rels_delta |
763 | + |
764 | + |
765 | +class Record(dict): |
766 | + |
767 | + __slots__ = () |
768 | + |
769 | + def __getattr__(self, k): |
770 | + if k in self: |
771 | + return self[k] |
772 | + raise AttributeError(k) |
773 | + |
774 | + |
775 | +class DeltaSet(Record): |
776 | + |
777 | + __slots__ = () |
778 | + |
779 | + |
780 | +Delta = collections.namedtuple('Delta', ['previous', 'current']) |
781 | + |
782 | + |
783 | +_KV = None |
784 | + |
785 | + |
786 | +def kv(): |
787 | + global _KV |
788 | + if _KV is None: |
789 | + _KV = Storage() |
790 | + return _KV |
791 | |
792 | === modified file 'hooks/charmhelpers/fetch/archiveurl.py' |
793 | --- hooks/charmhelpers/fetch/archiveurl.py 2015-01-26 11:51:28 +0000 |
794 | +++ hooks/charmhelpers/fetch/archiveurl.py 2015-02-24 06:25:16 +0000 |
795 | @@ -18,6 +18,16 @@ |
796 | import hashlib |
797 | import re |
798 | |
799 | +from charmhelpers.fetch import ( |
800 | + BaseFetchHandler, |
801 | + UnhandledSource |
802 | +) |
803 | +from charmhelpers.payload.archive import ( |
804 | + get_archive_handler, |
805 | + extract, |
806 | +) |
807 | +from charmhelpers.core.host import mkdir, check_hash |
808 | + |
809 | import six |
810 | if six.PY3: |
811 | from urllib.request import ( |
812 | @@ -35,16 +45,6 @@ |
813 | ) |
814 | from urlparse import urlparse, urlunparse, parse_qs |
815 | |
816 | -from charmhelpers.fetch import ( |
817 | - BaseFetchHandler, |
818 | - UnhandledSource |
819 | -) |
820 | -from charmhelpers.payload.archive import ( |
821 | - get_archive_handler, |
822 | - extract, |
823 | -) |
824 | -from charmhelpers.core.host import mkdir, check_hash |
825 | - |
826 | |
827 | def splituser(host): |
828 | '''urllib.splituser(), but six's support of this seems broken''' |
829 | |
830 | === modified file 'hooks/charmhelpers/fetch/giturl.py' |
831 | --- hooks/charmhelpers/fetch/giturl.py 2015-01-26 11:51:28 +0000 |
832 | +++ hooks/charmhelpers/fetch/giturl.py 2015-02-24 06:25:16 +0000 |
833 | @@ -32,7 +32,7 @@ |
834 | apt_install("python-git") |
835 | from git import Repo |
836 | |
837 | -from git.exc import GitCommandError |
838 | +from git.exc import GitCommandError # noqa E402 |
839 | |
840 | |
841 | class GitUrlFetchHandler(BaseFetchHandler): |
842 | |
843 | === modified file 'tests/basic_deployment.py' |
844 | --- tests/basic_deployment.py 2014-09-27 18:17:20 +0000 |
845 | +++ tests/basic_deployment.py 2015-02-24 06:25:16 +0000 |
846 | @@ -18,7 +18,7 @@ |
847 | """Amulet tests on a basic ceph-osd deployment.""" |
848 | |
849 | def __init__(self, series=None, openstack=None, source=None, |
850 | - stable=False): |
851 | + stable=True): |
852 | """Deploy the entire test environment.""" |
853 | super(CephOsdBasicDeployment, self).__init__(series, openstack, |
854 | source, stable) |
855 | |
856 | === modified file 'tests/charmhelpers/contrib/amulet/utils.py' |
857 | --- tests/charmhelpers/contrib/amulet/utils.py 2015-01-26 11:51:28 +0000 |
858 | +++ tests/charmhelpers/contrib/amulet/utils.py 2015-02-24 06:25:16 +0000 |
859 | @@ -169,8 +169,13 @@ |
860 | cmd = 'pgrep -o -f {}'.format(service) |
861 | else: |
862 | cmd = 'pgrep -o {}'.format(service) |
863 | - proc_dir = '/proc/{}'.format(sentry_unit.run(cmd)[0].strip()) |
864 | - return self._get_dir_mtime(sentry_unit, proc_dir) |
865 | + cmd = cmd + ' | grep -v pgrep || exit 0' |
866 | + cmd_out = sentry_unit.run(cmd) |
867 | + self.log.debug('CMDout: ' + str(cmd_out)) |
868 | + if cmd_out[0]: |
869 | + self.log.debug('Pid for %s %s' % (service, str(cmd_out[0]))) |
870 | + proc_dir = '/proc/{}'.format(cmd_out[0].strip()) |
871 | + return self._get_dir_mtime(sentry_unit, proc_dir) |
872 | |
873 | def service_restarted(self, sentry_unit, service, filename, |
874 | pgrep_full=False, sleep_time=20): |
875 | @@ -187,6 +192,121 @@ |
876 | else: |
877 | return False |
878 | |
879 | + def service_restarted_since(self, sentry_unit, mtime, service, |
880 | + pgrep_full=False, sleep_time=20, |
881 | + retry_count=2): |
882 | + """Check if service was been started after a given time. |
883 | + |
884 | + Args: |
885 | + sentry_unit (sentry): The sentry unit to check for the service on |
886 | + mtime (float): The epoch time to check against |
887 | + service (string): service name to look for in process table |
888 | + pgrep_full (boolean): Use full command line search mode with pgrep |
889 | + sleep_time (int): Seconds to sleep before looking for process |
890 | + retry_count (int): If service is not found, how many times to retry |
891 | + |
892 | + Returns: |
893 | + bool: True if service found and its start time it newer than mtime, |
894 | + False if service is older than mtime or if service was |
895 | + not found. |
896 | + """ |
897 | + self.log.debug('Checking %s restarted since %s' % (service, mtime)) |
898 | + time.sleep(sleep_time) |
899 | + proc_start_time = self._get_proc_start_time(sentry_unit, service, |
900 | + pgrep_full) |
901 | + while retry_count > 0 and not proc_start_time: |
902 | + self.log.debug('No pid file found for service %s, will retry %i ' |
903 | + 'more times' % (service, retry_count)) |
904 | + time.sleep(30) |
905 | + proc_start_time = self._get_proc_start_time(sentry_unit, service, |
906 | + pgrep_full) |
907 | + retry_count = retry_count - 1 |
908 | + |
909 | + if not proc_start_time: |
910 | + self.log.warn('No proc start time found, assuming service did ' |
911 | + 'not start') |
912 | + return False |
913 | + if proc_start_time >= mtime: |
914 | + self.log.debug('proc start time is newer than provided mtime' |
915 | + '(%s >= %s)' % (proc_start_time, mtime)) |
916 | + return True |
917 | + else: |
918 | + self.log.warn('proc start time (%s) is older than provided mtime ' |
919 | + '(%s), service did not restart' % (proc_start_time, |
920 | + mtime)) |
921 | + return False |
922 | + |
923 | + def config_updated_since(self, sentry_unit, filename, mtime, |
924 | + sleep_time=20): |
925 | + """Check if file was modified after a given time. |
926 | + |
927 | + Args: |
928 | + sentry_unit (sentry): The sentry unit to check the file mtime on |
929 | + filename (string): The file to check mtime of |
930 | + mtime (float): The epoch time to check against |
931 | + sleep_time (int): Seconds to sleep before looking for process |
932 | + |
933 | + Returns: |
934 | + bool: True if file was modified more recently than mtime, False if |
935 | + file was modified before mtime, |
936 | + """ |
937 | + self.log.debug('Checking %s updated since %s' % (filename, mtime)) |
938 | + time.sleep(sleep_time) |
939 | + file_mtime = self._get_file_mtime(sentry_unit, filename) |
940 | + if file_mtime >= mtime: |
941 | + self.log.debug('File mtime is newer than provided mtime ' |
942 | + '(%s >= %s)' % (file_mtime, mtime)) |
943 | + return True |
944 | + else: |
945 | + self.log.warn('File mtime %s is older than provided mtime %s' |
946 | + % (file_mtime, mtime)) |
947 | + return False |
948 | + |
949 | + def validate_service_config_changed(self, sentry_unit, mtime, service, |
950 | + filename, pgrep_full=False, |
951 | + sleep_time=20, retry_count=2): |
952 | + """Check service and file were updated after mtime |
953 | + |
954 | + Args: |
955 | + sentry_unit (sentry): The sentry unit to check for the service on |
956 | + mtime (float): The epoch time to check against |
957 | + service (string): service name to look for in process table |
958 | + filename (string): The file to check mtime of |
959 | + pgrep_full (boolean): Use full command line search mode with pgrep |
960 | + sleep_time (int): Seconds to sleep before looking for process |
961 | + retry_count (int): If service is not found, how many times to retry |
962 | + |
963 | + Typical Usage: |
964 | + u = OpenStackAmuletUtils(ERROR) |
965 | + ... |
966 | + mtime = u.get_sentry_time(self.cinder_sentry) |
967 | + self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'}) |
968 | + if not u.validate_service_config_changed(self.cinder_sentry, |
969 | + mtime, |
970 | + 'cinder-api', |
971 | + '/etc/cinder/cinder.conf') |
972 | + amulet.raise_status(amulet.FAIL, msg='update failed') |
973 | + Returns: |
974 | + bool: True if both service and file where updated/restarted after |
975 | + mtime, False if service is older than mtime or if service was |
976 | + not found or if filename was modified before mtime. |
977 | + """ |
978 | + self.log.debug('Checking %s restarted since %s' % (service, mtime)) |
979 | + time.sleep(sleep_time) |
980 | + service_restart = self.service_restarted_since(sentry_unit, mtime, |
981 | + service, |
982 | + pgrep_full=pgrep_full, |
983 | + sleep_time=0, |
984 | + retry_count=retry_count) |
985 | + config_update = self.config_updated_since(sentry_unit, filename, mtime, |
986 | + sleep_time=0) |
987 | + return service_restart and config_update |
988 | + |
989 | + def get_sentry_time(self, sentry_unit): |
990 | + """Return current epoch time on a sentry""" |
991 | + cmd = "date +'%s'" |
992 | + return float(sentry_unit.run(cmd)[0]) |
993 | + |
994 | def relation_error(self, name, data): |
995 | return 'unexpected relation data in {} - {}'.format(name, data) |
996 | |
997 | |
998 | === modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py' |
999 | --- tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-01-26 11:51:28 +0000 |
1000 | +++ tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-02-24 06:25:16 +0000 |
1001 | @@ -71,16 +71,19 @@ |
1002 | services.append(this_service) |
1003 | use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', |
1004 | 'ceph-osd', 'ceph-radosgw'] |
1005 | + # Openstack subordinate charms do not expose an origin option as that |
1006 | + # is controlled by the principle |
1007 | + ignore = ['neutron-openvswitch'] |
1008 | |
1009 | if self.openstack: |
1010 | for svc in services: |
1011 | - if svc['name'] not in use_source: |
1012 | + if svc['name'] not in use_source + ignore: |
1013 | config = {'openstack-origin': self.openstack} |
1014 | self.d.configure(svc['name'], config) |
1015 | |
1016 | if self.source: |
1017 | for svc in services: |
1018 | - if svc['name'] in use_source: |
1019 | + if svc['name'] in use_source and svc['name'] not in ignore: |
1020 | config = {'source': self.source} |
1021 | self.d.configure(svc['name'], config) |
1022 |
charm_unit_test #2014 ceph-osd-next for brad-marshall mp250710
UNIT OK: passed
Build: http:// 10.245. 162.77: 8080/job/ charm_unit_ test/2014/