Merge lp:~brad-marshall/charms/trusty/swift-storage/fix-nagios into lp:~openstack-charmers-archive/charms/trusty/swift-storage/next
- Trusty Tahr (14.04)
- fix-nagios
- Merge into next
Proposed by
Brad Marshall
Status: | Merged |
---|---|
Merged at revision: | 58 |
Proposed branch: | lp:~brad-marshall/charms/trusty/swift-storage/fix-nagios |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/swift-storage/next |
Diff against target: |
1245 lines (+860/-47) 21 files modified
config.yaml (+6/-0) hooks/charmhelpers/contrib/charmsupport/nrpe.py (+41/-7) hooks/charmhelpers/contrib/hahelpers/cluster.py (+5/-1) hooks/charmhelpers/contrib/openstack/amulet/deployment.py (+5/-2) hooks/charmhelpers/contrib/openstack/files/__init__.py (+18/-0) hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh (+32/-0) hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh (+30/-0) hooks/charmhelpers/contrib/openstack/ip.py (+37/-0) hooks/charmhelpers/contrib/openstack/utils.py (+1/-0) hooks/charmhelpers/contrib/python/packages.py (+2/-2) hooks/charmhelpers/core/fstab.py (+4/-4) hooks/charmhelpers/core/host.py (+5/-5) hooks/charmhelpers/core/strutils.py (+42/-0) hooks/charmhelpers/core/sysctl.py (+13/-7) hooks/charmhelpers/core/templating.py (+3/-3) hooks/charmhelpers/core/unitdata.py (+477/-0) hooks/charmhelpers/fetch/archiveurl.py (+10/-10) hooks/charmhelpers/fetch/giturl.py (+1/-1) tests/basic_deployment.py (+1/-1) tests/charmhelpers/contrib/amulet/utils.py (+122/-2) tests/charmhelpers/contrib/openstack/amulet/deployment.py (+5/-2) |
To merge this branch: | bzr merge lp:~brad-marshall/charms/trusty/swift-storage/fix-nagios |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
OpenStack Charmers | Pending | ||
Review via email: mp+250699@code.launchpad.net |
Commit message
Description of the change
Synced charmhelpers, added nagios_servicegroup config option, and added haproxy nrpe checks.
To post a comment you must log in.
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #2004 swift-storage-next for brad-marshall mp250699
UNIT OK: passed
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #2161 swift-storage-next for brad-marshall mp250699
AMULET FAIL: amulet-test failed
AMULET Results (max last 2 lines):
ERROR subprocess encountered error code 1
make: *** [test] Error 1
Full amulet test output: http://
Build: http://
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'config.yaml' |
2 | --- config.yaml 2014-10-30 05:52:15 +0000 |
3 | +++ config.yaml 2015-02-24 06:05:56 +0000 |
4 | @@ -109,3 +109,9 @@ |
5 | juju-myservice-0 |
6 | If you're running multiple environments with the same services in them |
7 | this allows you to differentiate between them. |
8 | + nagios_servicegroups: |
9 | + default: "" |
10 | + type: string |
11 | + description: | |
12 | + A comma-separated list of nagios servicegroups. |
13 | + If left empty, the nagios_context will be used as the servicegroup |
14 | |
15 | === modified file 'hooks/charmhelpers/contrib/charmsupport/nrpe.py' |
16 | --- hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-01-26 09:47:56 +0000 |
17 | +++ hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-02-24 06:05:56 +0000 |
18 | @@ -24,6 +24,8 @@ |
19 | import pwd |
20 | import grp |
21 | import os |
22 | +import glob |
23 | +import shutil |
24 | import re |
25 | import shlex |
26 | import yaml |
27 | @@ -161,7 +163,7 @@ |
28 | log('Check command not found: {}'.format(parts[0])) |
29 | return '' |
30 | |
31 | - def write(self, nagios_context, hostname, nagios_servicegroups=None): |
32 | + def write(self, nagios_context, hostname, nagios_servicegroups): |
33 | nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( |
34 | self.command) |
35 | with open(nrpe_check_file, 'w') as nrpe_check_config: |
36 | @@ -177,14 +179,11 @@ |
37 | nagios_servicegroups) |
38 | |
39 | def write_service_config(self, nagios_context, hostname, |
40 | - nagios_servicegroups=None): |
41 | + nagios_servicegroups): |
42 | for f in os.listdir(NRPE.nagios_exportdir): |
43 | if re.search('.*{}.cfg'.format(self.command), f): |
44 | os.remove(os.path.join(NRPE.nagios_exportdir, f)) |
45 | |
46 | - if not nagios_servicegroups: |
47 | - nagios_servicegroups = nagios_context |
48 | - |
49 | templ_vars = { |
50 | 'nagios_hostname': hostname, |
51 | 'nagios_servicegroup': nagios_servicegroups, |
52 | @@ -211,10 +210,10 @@ |
53 | super(NRPE, self).__init__() |
54 | self.config = config() |
55 | self.nagios_context = self.config['nagios_context'] |
56 | - if 'nagios_servicegroups' in self.config: |
57 | + if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: |
58 | self.nagios_servicegroups = self.config['nagios_servicegroups'] |
59 | else: |
60 | - self.nagios_servicegroups = 'juju' |
61 | + self.nagios_servicegroups = self.nagios_context |
62 | self.unit_name = local_unit().replace('/', '-') |
63 | if hostname: |
64 | self.hostname = hostname |
65 | @@ -322,3 +321,38 @@ |
66 | check_cmd='check_status_file.py -f ' |
67 | '/var/lib/nagios/service-check-%s.txt' % svc, |
68 | ) |
69 | + |
70 | + |
71 | +def copy_nrpe_checks(): |
72 | + """ |
73 | + Copy the nrpe checks into place |
74 | + |
75 | + """ |
76 | + NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' |
77 | + nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks', |
78 | + 'charmhelpers', 'contrib', 'openstack', |
79 | + 'files') |
80 | + |
81 | + if not os.path.exists(NAGIOS_PLUGINS): |
82 | + os.makedirs(NAGIOS_PLUGINS) |
83 | + for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): |
84 | + if os.path.isfile(fname): |
85 | + shutil.copy2(fname, |
86 | + os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) |
87 | + |
88 | + |
89 | +def add_haproxy_checks(nrpe, unit_name): |
90 | + """ |
91 | + Add checks for each service in list |
92 | + |
93 | + :param NRPE nrpe: NRPE object to add check to |
94 | + :param str unit_name: Unit name to use in check description |
95 | + """ |
96 | + nrpe.add_check( |
97 | + shortname='haproxy_servers', |
98 | + description='Check HAProxy {%s}' % unit_name, |
99 | + check_cmd='check_haproxy.sh') |
100 | + nrpe.add_check( |
101 | + shortname='haproxy_queue', |
102 | + description='Check HAProxy queue depth {%s}' % unit_name, |
103 | + check_cmd='check_haproxy_queue_depth.sh') |
104 | |
105 | === modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py' |
106 | --- hooks/charmhelpers/contrib/hahelpers/cluster.py 2015-01-26 09:47:56 +0000 |
107 | +++ hooks/charmhelpers/contrib/hahelpers/cluster.py 2015-02-24 06:05:56 +0000 |
108 | @@ -48,6 +48,9 @@ |
109 | from charmhelpers.core.decorators import ( |
110 | retry_on_exception, |
111 | ) |
112 | +from charmhelpers.core.strutils import ( |
113 | + bool_from_string, |
114 | +) |
115 | |
116 | |
117 | class HAIncompleteConfig(Exception): |
118 | @@ -164,7 +167,8 @@ |
119 | . |
120 | returns: boolean |
121 | ''' |
122 | - if config_get('use-https') == "yes": |
123 | + use_https = config_get('use-https') |
124 | + if use_https and bool_from_string(use_https): |
125 | return True |
126 | if config_get('ssl_cert') and config_get('ssl_key'): |
127 | return True |
128 | |
129 | === modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py' |
130 | --- hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2015-01-26 09:47:56 +0000 |
131 | +++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2015-02-24 06:05:56 +0000 |
132 | @@ -71,16 +71,19 @@ |
133 | services.append(this_service) |
134 | use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', |
135 | 'ceph-osd', 'ceph-radosgw'] |
136 | + # Openstack subordinate charms do not expose an origin option as that |
137 | + # is controlled by the principle |
138 | + ignore = ['neutron-openvswitch'] |
139 | |
140 | if self.openstack: |
141 | for svc in services: |
142 | - if svc['name'] not in use_source: |
143 | + if svc['name'] not in use_source + ignore: |
144 | config = {'openstack-origin': self.openstack} |
145 | self.d.configure(svc['name'], config) |
146 | |
147 | if self.source: |
148 | for svc in services: |
149 | - if svc['name'] in use_source: |
150 | + if svc['name'] in use_source and svc['name'] not in ignore: |
151 | config = {'source': self.source} |
152 | self.d.configure(svc['name'], config) |
153 | |
154 | |
155 | === added directory 'hooks/charmhelpers/contrib/openstack/files' |
156 | === added file 'hooks/charmhelpers/contrib/openstack/files/__init__.py' |
157 | --- hooks/charmhelpers/contrib/openstack/files/__init__.py 1970-01-01 00:00:00 +0000 |
158 | +++ hooks/charmhelpers/contrib/openstack/files/__init__.py 2015-02-24 06:05:56 +0000 |
159 | @@ -0,0 +1,18 @@ |
160 | +# Copyright 2014-2015 Canonical Limited. |
161 | +# |
162 | +# This file is part of charm-helpers. |
163 | +# |
164 | +# charm-helpers is free software: you can redistribute it and/or modify |
165 | +# it under the terms of the GNU Lesser General Public License version 3 as |
166 | +# published by the Free Software Foundation. |
167 | +# |
168 | +# charm-helpers is distributed in the hope that it will be useful, |
169 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
170 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
171 | +# GNU Lesser General Public License for more details. |
172 | +# |
173 | +# You should have received a copy of the GNU Lesser General Public License |
174 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
175 | + |
176 | +# dummy __init__.py to fool syncer into thinking this is a syncable python |
177 | +# module |
178 | |
179 | === added file 'hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh' |
180 | --- hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh 1970-01-01 00:00:00 +0000 |
181 | +++ hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh 2015-02-24 06:05:56 +0000 |
182 | @@ -0,0 +1,32 @@ |
183 | +#!/bin/bash |
184 | +#-------------------------------------------- |
185 | +# This file is managed by Juju |
186 | +#-------------------------------------------- |
187 | +# |
188 | +# Copyright 2009,2012 Canonical Ltd. |
189 | +# Author: Tom Haddon |
190 | + |
191 | +CRITICAL=0 |
192 | +NOTACTIVE='' |
193 | +LOGFILE=/var/log/nagios/check_haproxy.log |
194 | +AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}') |
195 | + |
196 | +for appserver in $(grep ' server' /etc/haproxy/haproxy.cfg | awk '{print $2'}); |
197 | +do |
198 | + output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 --regex="class=\"(active|backup)(2|3).*${appserver}" -e ' 200 OK') |
199 | + if [ $? != 0 ]; then |
200 | + date >> $LOGFILE |
201 | + echo $output >> $LOGFILE |
202 | + /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -v | grep $appserver >> $LOGFILE 2>&1 |
203 | + CRITICAL=1 |
204 | + NOTACTIVE="${NOTACTIVE} $appserver" |
205 | + fi |
206 | +done |
207 | + |
208 | +if [ $CRITICAL = 1 ]; then |
209 | + echo "CRITICAL:${NOTACTIVE}" |
210 | + exit 2 |
211 | +fi |
212 | + |
213 | +echo "OK: All haproxy instances looking good" |
214 | +exit 0 |
215 | |
216 | === added file 'hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh' |
217 | --- hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh 1970-01-01 00:00:00 +0000 |
218 | +++ hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh 2015-02-24 06:05:56 +0000 |
219 | @@ -0,0 +1,30 @@ |
220 | +#!/bin/bash |
221 | +#-------------------------------------------- |
222 | +# This file is managed by Juju |
223 | +#-------------------------------------------- |
224 | +# |
225 | +# Copyright 2009,2012 Canonical Ltd. |
226 | +# Author: Tom Haddon |
227 | + |
228 | +# These should be config options at some stage |
229 | +CURRQthrsh=0 |
230 | +MAXQthrsh=100 |
231 | + |
232 | +AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}') |
233 | + |
234 | +HAPROXYSTATS=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v) |
235 | + |
236 | +for BACKEND in $(echo $HAPROXYSTATS| xargs -n1 | grep BACKEND | awk -F , '{print $1}') |
237 | +do |
238 | + CURRQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 3) |
239 | + MAXQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 4) |
240 | + |
241 | + if [[ $CURRQ -gt $CURRQthrsh || $MAXQ -gt $MAXQthrsh ]] ; then |
242 | + echo "CRITICAL: queue depth for $BACKEND - CURRENT:$CURRQ MAX:$MAXQ" |
243 | + exit 2 |
244 | + fi |
245 | +done |
246 | + |
247 | +echo "OK: All haproxy queue depths looking good" |
248 | +exit 0 |
249 | + |
250 | |
251 | === modified file 'hooks/charmhelpers/contrib/openstack/ip.py' |
252 | --- hooks/charmhelpers/contrib/openstack/ip.py 2015-01-26 09:47:56 +0000 |
253 | +++ hooks/charmhelpers/contrib/openstack/ip.py 2015-02-24 06:05:56 +0000 |
254 | @@ -26,6 +26,8 @@ |
255 | ) |
256 | from charmhelpers.contrib.hahelpers.cluster import is_clustered |
257 | |
258 | +from functools import partial |
259 | + |
260 | PUBLIC = 'public' |
261 | INTERNAL = 'int' |
262 | ADMIN = 'admin' |
263 | @@ -107,3 +109,38 @@ |
264 | "clustered=%s)" % (net_type, clustered)) |
265 | |
266 | return resolved_address |
267 | + |
268 | + |
269 | +def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC, |
270 | + override=None): |
271 | + """Returns the correct endpoint URL to advertise to Keystone. |
272 | + |
273 | + This method provides the correct endpoint URL which should be advertised to |
274 | + the keystone charm for endpoint creation. This method allows for the url to |
275 | + be overridden to force a keystone endpoint to have specific URL for any of |
276 | + the defined scopes (admin, internal, public). |
277 | + |
278 | + :param configs: OSTemplateRenderer config templating object to inspect |
279 | + for a complete https context. |
280 | + :param url_template: str format string for creating the url template. Only |
281 | + two values will be passed - the scheme+hostname |
282 | + returned by the canonical_url and the port. |
283 | + :param endpoint_type: str endpoint type to resolve. |
284 | + :param override: str the name of the config option which overrides the |
285 | + endpoint URL defined by the charm itself. None will |
286 | + disable any overrides (default). |
287 | + """ |
288 | + if override: |
289 | + # Return any user-defined overrides for the keystone endpoint URL. |
290 | + user_value = config(override) |
291 | + if user_value: |
292 | + return user_value.strip() |
293 | + |
294 | + return url_template % (canonical_url(configs, endpoint_type), port) |
295 | + |
296 | + |
297 | +public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC) |
298 | + |
299 | +internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL) |
300 | + |
301 | +admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN) |
302 | |
303 | === modified file 'hooks/charmhelpers/contrib/openstack/utils.py' |
304 | --- hooks/charmhelpers/contrib/openstack/utils.py 2015-01-26 09:47:56 +0000 |
305 | +++ hooks/charmhelpers/contrib/openstack/utils.py 2015-02-24 06:05:56 +0000 |
306 | @@ -103,6 +103,7 @@ |
307 | ('2.1.0', 'juno'), |
308 | ('2.2.0', 'juno'), |
309 | ('2.2.1', 'kilo'), |
310 | + ('2.2.2', 'kilo'), |
311 | ]) |
312 | |
313 | DEFAULT_LOOPBACK_SIZE = '5G' |
314 | |
315 | === modified file 'hooks/charmhelpers/contrib/python/packages.py' |
316 | --- hooks/charmhelpers/contrib/python/packages.py 2015-01-26 09:47:56 +0000 |
317 | +++ hooks/charmhelpers/contrib/python/packages.py 2015-02-24 06:05:56 +0000 |
318 | @@ -17,8 +17,6 @@ |
319 | # You should have received a copy of the GNU Lesser General Public License |
320 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
321 | |
322 | -__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>" |
323 | - |
324 | from charmhelpers.fetch import apt_install, apt_update |
325 | from charmhelpers.core.hookenv import log |
326 | |
327 | @@ -29,6 +27,8 @@ |
328 | apt_install('python-pip') |
329 | from pip import main as pip_execute |
330 | |
331 | +__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>" |
332 | + |
333 | |
334 | def parse_options(given, available): |
335 | """Given a set of options, check if available""" |
336 | |
337 | === modified file 'hooks/charmhelpers/core/fstab.py' |
338 | --- hooks/charmhelpers/core/fstab.py 2015-01-26 09:47:56 +0000 |
339 | +++ hooks/charmhelpers/core/fstab.py 2015-02-24 06:05:56 +0000 |
340 | @@ -17,11 +17,11 @@ |
341 | # You should have received a copy of the GNU Lesser General Public License |
342 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
343 | |
344 | -__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
345 | - |
346 | import io |
347 | import os |
348 | |
349 | +__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
350 | + |
351 | |
352 | class Fstab(io.FileIO): |
353 | """This class extends file in order to implement a file reader/writer |
354 | @@ -77,7 +77,7 @@ |
355 | for line in self.readlines(): |
356 | line = line.decode('us-ascii') |
357 | try: |
358 | - if line.strip() and not line.startswith("#"): |
359 | + if line.strip() and not line.strip().startswith("#"): |
360 | yield self._hydrate_entry(line) |
361 | except ValueError: |
362 | pass |
363 | @@ -104,7 +104,7 @@ |
364 | |
365 | found = False |
366 | for index, line in enumerate(lines): |
367 | - if not line.startswith("#"): |
368 | + if line.strip() and not line.strip().startswith("#"): |
369 | if self._hydrate_entry(line) == entry: |
370 | found = True |
371 | break |
372 | |
373 | === modified file 'hooks/charmhelpers/core/host.py' |
374 | --- hooks/charmhelpers/core/host.py 2015-01-26 09:47:56 +0000 |
375 | +++ hooks/charmhelpers/core/host.py 2015-02-24 06:05:56 +0000 |
376 | @@ -191,11 +191,11 @@ |
377 | |
378 | |
379 | def write_file(path, content, owner='root', group='root', perms=0o444): |
380 | - """Create or overwrite a file with the contents of a string""" |
381 | + """Create or overwrite a file with the contents of a byte string.""" |
382 | log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) |
383 | uid = pwd.getpwnam(owner).pw_uid |
384 | gid = grp.getgrnam(group).gr_gid |
385 | - with open(path, 'w') as target: |
386 | + with open(path, 'wb') as target: |
387 | os.fchown(target.fileno(), uid, gid) |
388 | os.fchmod(target.fileno(), perms) |
389 | target.write(content) |
390 | @@ -305,11 +305,11 @@ |
391 | ceph_client_changed function. |
392 | """ |
393 | def wrap(f): |
394 | - def wrapped_f(*args): |
395 | + def wrapped_f(*args, **kwargs): |
396 | checksums = {} |
397 | for path in restart_map: |
398 | checksums[path] = file_hash(path) |
399 | - f(*args) |
400 | + f(*args, **kwargs) |
401 | restarts = [] |
402 | for path in restart_map: |
403 | if checksums[path] != file_hash(path): |
404 | @@ -361,7 +361,7 @@ |
405 | ip_output = (line for line in ip_output if line) |
406 | for line in ip_output: |
407 | if line.split()[1].startswith(int_type): |
408 | - matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line) |
409 | + matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line) |
410 | if matched: |
411 | interface = matched.groups()[0] |
412 | else: |
413 | |
414 | === added file 'hooks/charmhelpers/core/strutils.py' |
415 | --- hooks/charmhelpers/core/strutils.py 1970-01-01 00:00:00 +0000 |
416 | +++ hooks/charmhelpers/core/strutils.py 2015-02-24 06:05:56 +0000 |
417 | @@ -0,0 +1,42 @@ |
418 | +#!/usr/bin/env python |
419 | +# -*- coding: utf-8 -*- |
420 | + |
421 | +# Copyright 2014-2015 Canonical Limited. |
422 | +# |
423 | +# This file is part of charm-helpers. |
424 | +# |
425 | +# charm-helpers is free software: you can redistribute it and/or modify |
426 | +# it under the terms of the GNU Lesser General Public License version 3 as |
427 | +# published by the Free Software Foundation. |
428 | +# |
429 | +# charm-helpers is distributed in the hope that it will be useful, |
430 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
431 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
432 | +# GNU Lesser General Public License for more details. |
433 | +# |
434 | +# You should have received a copy of the GNU Lesser General Public License |
435 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
436 | + |
437 | +import six |
438 | + |
439 | + |
440 | +def bool_from_string(value): |
441 | + """Interpret string value as boolean. |
442 | + |
443 | + Returns True if value translates to True otherwise False. |
444 | + """ |
445 | + if isinstance(value, six.string_types): |
446 | + value = six.text_type(value) |
447 | + else: |
448 | + msg = "Unable to interpret non-string value '%s' as boolean" % (value) |
449 | + raise ValueError(msg) |
450 | + |
451 | + value = value.strip().lower() |
452 | + |
453 | + if value in ['y', 'yes', 'true', 't']: |
454 | + return True |
455 | + elif value in ['n', 'no', 'false', 'f']: |
456 | + return False |
457 | + |
458 | + msg = "Unable to interpret string value '%s' as boolean" % (value) |
459 | + raise ValueError(msg) |
460 | |
461 | === modified file 'hooks/charmhelpers/core/sysctl.py' |
462 | --- hooks/charmhelpers/core/sysctl.py 2015-01-26 09:47:56 +0000 |
463 | +++ hooks/charmhelpers/core/sysctl.py 2015-02-24 06:05:56 +0000 |
464 | @@ -17,8 +17,6 @@ |
465 | # You should have received a copy of the GNU Lesser General Public License |
466 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
467 | |
468 | -__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
469 | - |
470 | import yaml |
471 | |
472 | from subprocess import check_call |
473 | @@ -26,25 +24,33 @@ |
474 | from charmhelpers.core.hookenv import ( |
475 | log, |
476 | DEBUG, |
477 | + ERROR, |
478 | ) |
479 | |
480 | +__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
481 | + |
482 | |
483 | def create(sysctl_dict, sysctl_file): |
484 | """Creates a sysctl.conf file from a YAML associative array |
485 | |
486 | - :param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 } |
487 | - :type sysctl_dict: dict |
488 | + :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" |
489 | + :type sysctl_dict: str |
490 | :param sysctl_file: path to the sysctl file to be saved |
491 | :type sysctl_file: str or unicode |
492 | :returns: None |
493 | """ |
494 | - sysctl_dict = yaml.load(sysctl_dict) |
495 | + try: |
496 | + sysctl_dict_parsed = yaml.safe_load(sysctl_dict) |
497 | + except yaml.YAMLError: |
498 | + log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), |
499 | + level=ERROR) |
500 | + return |
501 | |
502 | with open(sysctl_file, "w") as fd: |
503 | - for key, value in sysctl_dict.items(): |
504 | + for key, value in sysctl_dict_parsed.items(): |
505 | fd.write("{}={}\n".format(key, value)) |
506 | |
507 | - log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict), |
508 | + log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed), |
509 | level=DEBUG) |
510 | |
511 | check_call(["sysctl", "-p", sysctl_file]) |
512 | |
513 | === modified file 'hooks/charmhelpers/core/templating.py' |
514 | --- hooks/charmhelpers/core/templating.py 2015-01-26 09:47:56 +0000 |
515 | +++ hooks/charmhelpers/core/templating.py 2015-02-24 06:05:56 +0000 |
516 | @@ -21,7 +21,7 @@ |
517 | |
518 | |
519 | def render(source, target, context, owner='root', group='root', |
520 | - perms=0o444, templates_dir=None): |
521 | + perms=0o444, templates_dir=None, encoding='UTF-8'): |
522 | """ |
523 | Render a template. |
524 | |
525 | @@ -64,5 +64,5 @@ |
526 | level=hookenv.ERROR) |
527 | raise e |
528 | content = template.render(context) |
529 | - host.mkdir(os.path.dirname(target), owner, group) |
530 | - host.write_file(target, content, owner, group, perms) |
531 | + host.mkdir(os.path.dirname(target), owner, group, perms=0o755) |
532 | + host.write_file(target, content.encode(encoding), owner, group, perms) |
533 | |
534 | === added file 'hooks/charmhelpers/core/unitdata.py' |
535 | --- hooks/charmhelpers/core/unitdata.py 1970-01-01 00:00:00 +0000 |
536 | +++ hooks/charmhelpers/core/unitdata.py 2015-02-24 06:05:56 +0000 |
537 | @@ -0,0 +1,477 @@ |
538 | +#!/usr/bin/env python |
539 | +# -*- coding: utf-8 -*- |
540 | +# |
541 | +# Copyright 2014-2015 Canonical Limited. |
542 | +# |
543 | +# This file is part of charm-helpers. |
544 | +# |
545 | +# charm-helpers is free software: you can redistribute it and/or modify |
546 | +# it under the terms of the GNU Lesser General Public License version 3 as |
547 | +# published by the Free Software Foundation. |
548 | +# |
549 | +# charm-helpers is distributed in the hope that it will be useful, |
550 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
551 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
552 | +# GNU Lesser General Public License for more details. |
553 | +# |
554 | +# You should have received a copy of the GNU Lesser General Public License |
555 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
556 | +# |
557 | +# |
558 | +# Authors: |
559 | +# Kapil Thangavelu <kapil.foss@gmail.com> |
560 | +# |
561 | +""" |
562 | +Intro |
563 | +----- |
564 | + |
565 | +A simple way to store state in units. This provides a key value |
566 | +storage with support for versioned, transactional operation, |
567 | +and can calculate deltas from previous values to simplify unit logic |
568 | +when processing changes. |
569 | + |
570 | + |
571 | +Hook Integration |
572 | +---------------- |
573 | + |
574 | +There are several extant frameworks for hook execution, including |
575 | + |
576 | + - charmhelpers.core.hookenv.Hooks |
577 | + - charmhelpers.core.services.ServiceManager |
578 | + |
579 | +The storage classes are framework agnostic, one simple integration is |
580 | +via the HookData contextmanager. It will record the current hook |
581 | +execution environment (including relation data, config data, etc.), |
582 | +setup a transaction and allow easy access to the changes from |
583 | +previously seen values. One consequence of the integration is the |
584 | +reservation of particular keys ('rels', 'unit', 'env', 'config', |
585 | +'charm_revisions') for their respective values. |
586 | + |
587 | +Here's a fully worked integration example using hookenv.Hooks:: |
588 | + |
589 | + from charmhelper.core import hookenv, unitdata |
590 | + |
591 | + hook_data = unitdata.HookData() |
592 | + db = unitdata.kv() |
593 | + hooks = hookenv.Hooks() |
594 | + |
595 | + @hooks.hook |
596 | + def config_changed(): |
597 | + # Print all changes to configuration from previously seen |
598 | + # values. |
599 | + for changed, (prev, cur) in hook_data.conf.items(): |
600 | + print('config changed', changed, |
601 | + 'previous value', prev, |
602 | + 'current value', cur) |
603 | + |
604 | + # Get some unit specific bookeeping |
605 | + if not db.get('pkg_key'): |
606 | + key = urllib.urlopen('https://example.com/pkg_key').read() |
607 | + db.set('pkg_key', key) |
608 | + |
609 | + # Directly access all charm config as a mapping. |
610 | + conf = db.getrange('config', True) |
611 | + |
612 | + # Directly access all relation data as a mapping |
613 | + rels = db.getrange('rels', True) |
614 | + |
615 | + if __name__ == '__main__': |
616 | + with hook_data(): |
617 | + hook.execute() |
618 | + |
619 | + |
620 | +A more basic integration is via the hook_scope context manager which simply |
621 | +manages transaction scope (and records hook name, and timestamp):: |
622 | + |
623 | + >>> from unitdata import kv |
624 | + >>> db = kv() |
625 | + >>> with db.hook_scope('install'): |
626 | + ... # do work, in transactional scope. |
627 | + ... db.set('x', 1) |
628 | + >>> db.get('x') |
629 | + 1 |
630 | + |
631 | + |
632 | +Usage |
633 | +----- |
634 | + |
635 | +Values are automatically json de/serialized to preserve basic typing |
636 | +and complex data struct capabilities (dicts, lists, ints, booleans, etc). |
637 | + |
638 | +Individual values can be manipulated via get/set:: |
639 | + |
640 | + >>> kv.set('y', True) |
641 | + >>> kv.get('y') |
642 | + True |
643 | + |
644 | + # We can set complex values (dicts, lists) as a single key. |
645 | + >>> kv.set('config', {'a': 1, 'b': True'}) |
646 | + |
647 | + # Also supports returning dictionaries as a record which |
648 | + # provides attribute access. |
649 | + >>> config = kv.get('config', record=True) |
650 | + >>> config.b |
651 | + True |
652 | + |
653 | + |
654 | +Groups of keys can be manipulated with update/getrange:: |
655 | + |
656 | + >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") |
657 | + >>> kv.getrange('gui.', strip=True) |
658 | + {'z': 1, 'y': 2} |
659 | + |
660 | +When updating values, its very helpful to understand which values |
661 | +have actually changed and how have they changed. The storage |
662 | +provides a delta method to provide for this:: |
663 | + |
664 | + >>> data = {'debug': True, 'option': 2} |
665 | + >>> delta = kv.delta(data, 'config.') |
666 | + >>> delta.debug.previous |
667 | + None |
668 | + >>> delta.debug.current |
669 | + True |
670 | + >>> delta |
671 | + {'debug': (None, True), 'option': (None, 2)} |
672 | + |
673 | +Note the delta method does not persist the actual change, it needs to |
674 | +be explicitly saved via 'update' method:: |
675 | + |
676 | + >>> kv.update(data, 'config.') |
677 | + |
678 | +Values modified in the context of a hook scope retain historical values |
679 | +associated to the hookname. |
680 | + |
681 | + >>> with db.hook_scope('config-changed'): |
682 | + ... db.set('x', 42) |
683 | + >>> db.gethistory('x') |
684 | + [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), |
685 | + (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] |
686 | + |
687 | +""" |
688 | + |
689 | +import collections |
690 | +import contextlib |
691 | +import datetime |
692 | +import json |
693 | +import os |
694 | +import pprint |
695 | +import sqlite3 |
696 | +import sys |
697 | + |
698 | +__author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>' |
699 | + |
700 | + |
701 | +class Storage(object): |
702 | + """Simple key value database for local unit state within charms. |
703 | + |
704 | + Modifications are automatically committed at hook exit. That's |
705 | + currently regardless of exit code. |
706 | + |
707 | + To support dicts, lists, integer, floats, and booleans values |
708 | + are automatically json encoded/decoded. |
709 | + """ |
710 | + def __init__(self, path=None): |
711 | + self.db_path = path |
712 | + if path is None: |
713 | + self.db_path = os.path.join( |
714 | + os.environ.get('CHARM_DIR', ''), '.unit-state.db') |
715 | + self.conn = sqlite3.connect('%s' % self.db_path) |
716 | + self.cursor = self.conn.cursor() |
717 | + self.revision = None |
718 | + self._closed = False |
719 | + self._init() |
720 | + |
721 | + def close(self): |
722 | + if self._closed: |
723 | + return |
724 | + self.flush(False) |
725 | + self.cursor.close() |
726 | + self.conn.close() |
727 | + self._closed = True |
728 | + |
729 | + def _scoped_query(self, stmt, params=None): |
730 | + if params is None: |
731 | + params = [] |
732 | + return stmt, params |
733 | + |
734 | + def get(self, key, default=None, record=False): |
735 | + self.cursor.execute( |
736 | + *self._scoped_query( |
737 | + 'select data from kv where key=?', [key])) |
738 | + result = self.cursor.fetchone() |
739 | + if not result: |
740 | + return default |
741 | + if record: |
742 | + return Record(json.loads(result[0])) |
743 | + return json.loads(result[0]) |
744 | + |
745 | + def getrange(self, key_prefix, strip=False): |
746 | + stmt = "select key, data from kv where key like '%s%%'" % key_prefix |
747 | + self.cursor.execute(*self._scoped_query(stmt)) |
748 | + result = self.cursor.fetchall() |
749 | + |
750 | + if not result: |
751 | + return None |
752 | + if not strip: |
753 | + key_prefix = '' |
754 | + return dict([ |
755 | + (k[len(key_prefix):], json.loads(v)) for k, v in result]) |
756 | + |
757 | + def update(self, mapping, prefix=""): |
758 | + for k, v in mapping.items(): |
759 | + self.set("%s%s" % (prefix, k), v) |
760 | + |
761 | + def unset(self, key): |
762 | + self.cursor.execute('delete from kv where key=?', [key]) |
763 | + if self.revision and self.cursor.rowcount: |
764 | + self.cursor.execute( |
765 | + 'insert into kv_revisions values (?, ?, ?)', |
766 | + [key, self.revision, json.dumps('DELETED')]) |
767 | + |
768 | + def set(self, key, value): |
769 | + serialized = json.dumps(value) |
770 | + |
771 | + self.cursor.execute( |
772 | + 'select data from kv where key=?', [key]) |
773 | + exists = self.cursor.fetchone() |
774 | + |
775 | + # Skip mutations to the same value |
776 | + if exists: |
777 | + if exists[0] == serialized: |
778 | + return value |
779 | + |
780 | + if not exists: |
781 | + self.cursor.execute( |
782 | + 'insert into kv (key, data) values (?, ?)', |
783 | + (key, serialized)) |
784 | + else: |
785 | + self.cursor.execute(''' |
786 | + update kv |
787 | + set data = ? |
788 | + where key = ?''', [serialized, key]) |
789 | + |
790 | + # Save |
791 | + if not self.revision: |
792 | + return value |
793 | + |
794 | + self.cursor.execute( |
795 | + 'select 1 from kv_revisions where key=? and revision=?', |
796 | + [key, self.revision]) |
797 | + exists = self.cursor.fetchone() |
798 | + |
799 | + if not exists: |
800 | + self.cursor.execute( |
801 | + '''insert into kv_revisions ( |
802 | + revision, key, data) values (?, ?, ?)''', |
803 | + (self.revision, key, serialized)) |
804 | + else: |
805 | + self.cursor.execute( |
806 | + ''' |
807 | + update kv_revisions |
808 | + set data = ? |
809 | + where key = ? |
810 | + and revision = ?''', |
811 | + [serialized, key, self.revision]) |
812 | + |
813 | + return value |
814 | + |
815 | + def delta(self, mapping, prefix): |
816 | + """ |
817 | + return a delta containing values that have changed. |
818 | + """ |
819 | + previous = self.getrange(prefix, strip=True) |
820 | + if not previous: |
821 | + pk = set() |
822 | + else: |
823 | + pk = set(previous.keys()) |
824 | + ck = set(mapping.keys()) |
825 | + delta = DeltaSet() |
826 | + |
827 | + # added |
828 | + for k in ck.difference(pk): |
829 | + delta[k] = Delta(None, mapping[k]) |
830 | + |
831 | + # removed |
832 | + for k in pk.difference(ck): |
833 | + delta[k] = Delta(previous[k], None) |
834 | + |
835 | + # changed |
836 | + for k in pk.intersection(ck): |
837 | + c = mapping[k] |
838 | + p = previous[k] |
839 | + if c != p: |
840 | + delta[k] = Delta(p, c) |
841 | + |
842 | + return delta |
843 | + |
844 | + @contextlib.contextmanager |
845 | + def hook_scope(self, name=""): |
846 | + """Scope all future interactions to the current hook execution |
847 | + revision.""" |
848 | + assert not self.revision |
849 | + self.cursor.execute( |
850 | + 'insert into hooks (hook, date) values (?, ?)', |
851 | + (name or sys.argv[0], |
852 | + datetime.datetime.utcnow().isoformat())) |
853 | + self.revision = self.cursor.lastrowid |
854 | + try: |
855 | + yield self.revision |
856 | + self.revision = None |
857 | + except: |
858 | + self.flush(False) |
859 | + self.revision = None |
860 | + raise |
861 | + else: |
862 | + self.flush() |
863 | + |
864 | + def flush(self, save=True): |
865 | + if save: |
866 | + self.conn.commit() |
867 | + elif self._closed: |
868 | + return |
869 | + else: |
870 | + self.conn.rollback() |
871 | + |
872 | + def _init(self): |
873 | + self.cursor.execute(''' |
874 | + create table if not exists kv ( |
875 | + key text, |
876 | + data text, |
877 | + primary key (key) |
878 | + )''') |
879 | + self.cursor.execute(''' |
880 | + create table if not exists kv_revisions ( |
881 | + key text, |
882 | + revision integer, |
883 | + data text, |
884 | + primary key (key, revision) |
885 | + )''') |
886 | + self.cursor.execute(''' |
887 | + create table if not exists hooks ( |
888 | + version integer primary key autoincrement, |
889 | + hook text, |
890 | + date text |
891 | + )''') |
892 | + self.conn.commit() |
893 | + |
894 | + def gethistory(self, key, deserialize=False): |
895 | + self.cursor.execute( |
896 | + ''' |
897 | + select kv.revision, kv.key, kv.data, h.hook, h.date |
898 | + from kv_revisions kv, |
899 | + hooks h |
900 | + where kv.key=? |
901 | + and kv.revision = h.version |
902 | + ''', [key]) |
903 | + if deserialize is False: |
904 | + return self.cursor.fetchall() |
905 | + return map(_parse_history, self.cursor.fetchall()) |
906 | + |
907 | + def debug(self, fh=sys.stderr): |
908 | + self.cursor.execute('select * from kv') |
909 | + pprint.pprint(self.cursor.fetchall(), stream=fh) |
910 | + self.cursor.execute('select * from kv_revisions') |
911 | + pprint.pprint(self.cursor.fetchall(), stream=fh) |
912 | + |
913 | + |
914 | +def _parse_history(d): |
915 | + return (d[0], d[1], json.loads(d[2]), d[3], |
916 | + datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) |
917 | + |
918 | + |
919 | +class HookData(object): |
920 | + """Simple integration for existing hook exec frameworks. |
921 | + |
922 | + Records all unit information, and stores deltas for processing |
923 | + by the hook. |
924 | + |
925 | + Sample:: |
926 | + |
927 | + from charmhelper.core import hookenv, unitdata |
928 | + |
929 | + changes = unitdata.HookData() |
930 | + db = unitdata.kv() |
931 | + hooks = hookenv.Hooks() |
932 | + |
933 | + @hooks.hook |
934 | + def config_changed(): |
935 | + # View all changes to configuration |
936 | + for changed, (prev, cur) in changes.conf.items(): |
937 | + print('config changed', changed, |
938 | + 'previous value', prev, |
939 | + 'current value', cur) |
940 | + |
941 | + # Get some unit specific bookeeping |
942 | + if not db.get('pkg_key'): |
943 | + key = urllib.urlopen('https://example.com/pkg_key').read() |
944 | + db.set('pkg_key', key) |
945 | + |
946 | + if __name__ == '__main__': |
947 | + with changes(): |
948 | + hook.execute() |
949 | + |
950 | + """ |
951 | + def __init__(self): |
952 | + self.kv = kv() |
953 | + self.conf = None |
954 | + self.rels = None |
955 | + |
956 | + @contextlib.contextmanager |
957 | + def __call__(self): |
958 | + from charmhelpers.core import hookenv |
959 | + hook_name = hookenv.hook_name() |
960 | + |
961 | + with self.kv.hook_scope(hook_name): |
962 | + self._record_charm_version(hookenv.charm_dir()) |
963 | + delta_config, delta_relation = self._record_hook(hookenv) |
964 | + yield self.kv, delta_config, delta_relation |
965 | + |
966 | + def _record_charm_version(self, charm_dir): |
967 | + # Record revisions.. charm revisions are meaningless |
968 | + # to charm authors as they don't control the revision. |
969 | + # so logic dependnent on revision is not particularly |
970 | + # useful, however it is useful for debugging analysis. |
971 | + charm_rev = open( |
972 | + os.path.join(charm_dir, 'revision')).read().strip() |
973 | + charm_rev = charm_rev or '0' |
974 | + revs = self.kv.get('charm_revisions', []) |
975 | + if charm_rev not in revs: |
976 | + revs.append(charm_rev.strip() or '0') |
977 | + self.kv.set('charm_revisions', revs) |
978 | + |
979 | + def _record_hook(self, hookenv): |
980 | + data = hookenv.execution_environment() |
981 | + self.conf = conf_delta = self.kv.delta(data['conf'], 'config') |
982 | + self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') |
983 | + self.kv.set('env', data['env']) |
984 | + self.kv.set('unit', data['unit']) |
985 | + self.kv.set('relid', data.get('relid')) |
986 | + return conf_delta, rels_delta |
987 | + |
988 | + |
989 | +class Record(dict): |
990 | + |
991 | + __slots__ = () |
992 | + |
993 | + def __getattr__(self, k): |
994 | + if k in self: |
995 | + return self[k] |
996 | + raise AttributeError(k) |
997 | + |
998 | + |
999 | +class DeltaSet(Record): |
1000 | + |
1001 | + __slots__ = () |
1002 | + |
1003 | + |
1004 | +Delta = collections.namedtuple('Delta', ['previous', 'current']) |
1005 | + |
1006 | + |
1007 | +_KV = None |
1008 | + |
1009 | + |
1010 | +def kv(): |
1011 | + global _KV |
1012 | + if _KV is None: |
1013 | + _KV = Storage() |
1014 | + return _KV |
1015 | |
1016 | === modified file 'hooks/charmhelpers/fetch/archiveurl.py' |
1017 | --- hooks/charmhelpers/fetch/archiveurl.py 2015-01-26 09:47:56 +0000 |
1018 | +++ hooks/charmhelpers/fetch/archiveurl.py 2015-02-24 06:05:56 +0000 |
1019 | @@ -18,6 +18,16 @@ |
1020 | import hashlib |
1021 | import re |
1022 | |
1023 | +from charmhelpers.fetch import ( |
1024 | + BaseFetchHandler, |
1025 | + UnhandledSource |
1026 | +) |
1027 | +from charmhelpers.payload.archive import ( |
1028 | + get_archive_handler, |
1029 | + extract, |
1030 | +) |
1031 | +from charmhelpers.core.host import mkdir, check_hash |
1032 | + |
1033 | import six |
1034 | if six.PY3: |
1035 | from urllib.request import ( |
1036 | @@ -35,16 +45,6 @@ |
1037 | ) |
1038 | from urlparse import urlparse, urlunparse, parse_qs |
1039 | |
1040 | -from charmhelpers.fetch import ( |
1041 | - BaseFetchHandler, |
1042 | - UnhandledSource |
1043 | -) |
1044 | -from charmhelpers.payload.archive import ( |
1045 | - get_archive_handler, |
1046 | - extract, |
1047 | -) |
1048 | -from charmhelpers.core.host import mkdir, check_hash |
1049 | - |
1050 | |
1051 | def splituser(host): |
1052 | '''urllib.splituser(), but six's support of this seems broken''' |
1053 | |
1054 | === modified file 'hooks/charmhelpers/fetch/giturl.py' |
1055 | --- hooks/charmhelpers/fetch/giturl.py 2015-01-26 09:47:56 +0000 |
1056 | +++ hooks/charmhelpers/fetch/giturl.py 2015-02-24 06:05:56 +0000 |
1057 | @@ -32,7 +32,7 @@ |
1058 | apt_install("python-git") |
1059 | from git import Repo |
1060 | |
1061 | -from git.exc import GitCommandError |
1062 | +from git.exc import GitCommandError # noqa E402 |
1063 | |
1064 | |
1065 | class GitUrlFetchHandler(BaseFetchHandler): |
1066 | |
1067 | === modified file 'tests/basic_deployment.py' |
1068 | --- tests/basic_deployment.py 2014-10-09 20:25:51 +0000 |
1069 | +++ tests/basic_deployment.py 2015-02-24 06:05:56 +0000 |
1070 | @@ -20,7 +20,7 @@ |
1071 | class SwiftStorageBasicDeployment(OpenStackAmuletDeployment): |
1072 | """Amulet tests on a basic swift-storage deployment.""" |
1073 | |
1074 | - def __init__(self, series, openstack=None, source=None, stable=False): |
1075 | + def __init__(self, series, openstack=None, source=None, stable=True): |
1076 | """Deploy the entire test environment.""" |
1077 | super(SwiftStorageBasicDeployment, self).__init__(series, openstack, |
1078 | source, stable) |
1079 | |
1080 | === modified file 'tests/charmhelpers/contrib/amulet/utils.py' |
1081 | --- tests/charmhelpers/contrib/amulet/utils.py 2015-01-26 09:47:56 +0000 |
1082 | +++ tests/charmhelpers/contrib/amulet/utils.py 2015-02-24 06:05:56 +0000 |
1083 | @@ -169,8 +169,13 @@ |
1084 | cmd = 'pgrep -o -f {}'.format(service) |
1085 | else: |
1086 | cmd = 'pgrep -o {}'.format(service) |
1087 | - proc_dir = '/proc/{}'.format(sentry_unit.run(cmd)[0].strip()) |
1088 | - return self._get_dir_mtime(sentry_unit, proc_dir) |
1089 | + cmd = cmd + ' | grep -v pgrep || exit 0' |
1090 | + cmd_out = sentry_unit.run(cmd) |
1091 | + self.log.debug('CMDout: ' + str(cmd_out)) |
1092 | + if cmd_out[0]: |
1093 | + self.log.debug('Pid for %s %s' % (service, str(cmd_out[0]))) |
1094 | + proc_dir = '/proc/{}'.format(cmd_out[0].strip()) |
1095 | + return self._get_dir_mtime(sentry_unit, proc_dir) |
1096 | |
1097 | def service_restarted(self, sentry_unit, service, filename, |
1098 | pgrep_full=False, sleep_time=20): |
1099 | @@ -187,6 +192,121 @@ |
1100 | else: |
1101 | return False |
1102 | |
1103 | + def service_restarted_since(self, sentry_unit, mtime, service, |
1104 | + pgrep_full=False, sleep_time=20, |
1105 | + retry_count=2): |
1106 | + """Check if service was been started after a given time. |
1107 | + |
1108 | + Args: |
1109 | + sentry_unit (sentry): The sentry unit to check for the service on |
1110 | + mtime (float): The epoch time to check against |
1111 | + service (string): service name to look for in process table |
1112 | + pgrep_full (boolean): Use full command line search mode with pgrep |
1113 | + sleep_time (int): Seconds to sleep before looking for process |
1114 | + retry_count (int): If service is not found, how many times to retry |
1115 | + |
1116 | + Returns: |
1117 | + bool: True if service found and its start time it newer than mtime, |
1118 | + False if service is older than mtime or if service was |
1119 | + not found. |
1120 | + """ |
1121 | + self.log.debug('Checking %s restarted since %s' % (service, mtime)) |
1122 | + time.sleep(sleep_time) |
1123 | + proc_start_time = self._get_proc_start_time(sentry_unit, service, |
1124 | + pgrep_full) |
1125 | + while retry_count > 0 and not proc_start_time: |
1126 | + self.log.debug('No pid file found for service %s, will retry %i ' |
1127 | + 'more times' % (service, retry_count)) |
1128 | + time.sleep(30) |
1129 | + proc_start_time = self._get_proc_start_time(sentry_unit, service, |
1130 | + pgrep_full) |
1131 | + retry_count = retry_count - 1 |
1132 | + |
1133 | + if not proc_start_time: |
1134 | + self.log.warn('No proc start time found, assuming service did ' |
1135 | + 'not start') |
1136 | + return False |
1137 | + if proc_start_time >= mtime: |
1138 | + self.log.debug('proc start time is newer than provided mtime' |
1139 | + '(%s >= %s)' % (proc_start_time, mtime)) |
1140 | + return True |
1141 | + else: |
1142 | + self.log.warn('proc start time (%s) is older than provided mtime ' |
1143 | + '(%s), service did not restart' % (proc_start_time, |
1144 | + mtime)) |
1145 | + return False |
1146 | + |
1147 | + def config_updated_since(self, sentry_unit, filename, mtime, |
1148 | + sleep_time=20): |
1149 | + """Check if file was modified after a given time. |
1150 | + |
1151 | + Args: |
1152 | + sentry_unit (sentry): The sentry unit to check the file mtime on |
1153 | + filename (string): The file to check mtime of |
1154 | + mtime (float): The epoch time to check against |
1155 | + sleep_time (int): Seconds to sleep before looking for process |
1156 | + |
1157 | + Returns: |
1158 | + bool: True if file was modified more recently than mtime, False if |
1159 | + file was modified before mtime, |
1160 | + """ |
1161 | + self.log.debug('Checking %s updated since %s' % (filename, mtime)) |
1162 | + time.sleep(sleep_time) |
1163 | + file_mtime = self._get_file_mtime(sentry_unit, filename) |
1164 | + if file_mtime >= mtime: |
1165 | + self.log.debug('File mtime is newer than provided mtime ' |
1166 | + '(%s >= %s)' % (file_mtime, mtime)) |
1167 | + return True |
1168 | + else: |
1169 | + self.log.warn('File mtime %s is older than provided mtime %s' |
1170 | + % (file_mtime, mtime)) |
1171 | + return False |
1172 | + |
1173 | + def validate_service_config_changed(self, sentry_unit, mtime, service, |
1174 | + filename, pgrep_full=False, |
1175 | + sleep_time=20, retry_count=2): |
1176 | + """Check service and file were updated after mtime |
1177 | + |
1178 | + Args: |
1179 | + sentry_unit (sentry): The sentry unit to check for the service on |
1180 | + mtime (float): The epoch time to check against |
1181 | + service (string): service name to look for in process table |
1182 | + filename (string): The file to check mtime of |
1183 | + pgrep_full (boolean): Use full command line search mode with pgrep |
1184 | + sleep_time (int): Seconds to sleep before looking for process |
1185 | + retry_count (int): If service is not found, how many times to retry |
1186 | + |
1187 | + Typical Usage: |
1188 | + u = OpenStackAmuletUtils(ERROR) |
1189 | + ... |
1190 | + mtime = u.get_sentry_time(self.cinder_sentry) |
1191 | + self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'}) |
1192 | + if not u.validate_service_config_changed(self.cinder_sentry, |
1193 | + mtime, |
1194 | + 'cinder-api', |
1195 | + '/etc/cinder/cinder.conf') |
1196 | + amulet.raise_status(amulet.FAIL, msg='update failed') |
1197 | + Returns: |
1198 | + bool: True if both service and file where updated/restarted after |
1199 | + mtime, False if service is older than mtime or if service was |
1200 | + not found or if filename was modified before mtime. |
1201 | + """ |
1202 | + self.log.debug('Checking %s restarted since %s' % (service, mtime)) |
1203 | + time.sleep(sleep_time) |
1204 | + service_restart = self.service_restarted_since(sentry_unit, mtime, |
1205 | + service, |
1206 | + pgrep_full=pgrep_full, |
1207 | + sleep_time=0, |
1208 | + retry_count=retry_count) |
1209 | + config_update = self.config_updated_since(sentry_unit, filename, mtime, |
1210 | + sleep_time=0) |
1211 | + return service_restart and config_update |
1212 | + |
1213 | + def get_sentry_time(self, sentry_unit): |
1214 | + """Return current epoch time on a sentry""" |
1215 | + cmd = "date +'%s'" |
1216 | + return float(sentry_unit.run(cmd)[0]) |
1217 | + |
1218 | def relation_error(self, name, data): |
1219 | return 'unexpected relation data in {} - {}'.format(name, data) |
1220 | |
1221 | |
1222 | === modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py' |
1223 | --- tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-01-26 09:47:56 +0000 |
1224 | +++ tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-02-24 06:05:56 +0000 |
1225 | @@ -71,16 +71,19 @@ |
1226 | services.append(this_service) |
1227 | use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', |
1228 | 'ceph-osd', 'ceph-radosgw'] |
1229 | + # Openstack subordinate charms do not expose an origin option as that |
1230 | + # is controlled by the principle |
1231 | + ignore = ['neutron-openvswitch'] |
1232 | |
1233 | if self.openstack: |
1234 | for svc in services: |
1235 | - if svc['name'] not in use_source: |
1236 | + if svc['name'] not in use_source + ignore: |
1237 | config = {'openstack-origin': self.openstack} |
1238 | self.d.configure(svc['name'], config) |
1239 | |
1240 | if self.source: |
1241 | for svc in services: |
1242 | - if svc['name'] in use_source: |
1243 | + if svc['name'] in use_source and svc['name'] not in ignore: |
1244 | config = {'source': self.source} |
1245 | self.d.configure(svc['name'], config) |
1246 |
charm_lint_check #2215 swift-storage-next for brad-marshall mp250699
LINT OK: passed
Build: http:// 10.245. 162.77: 8080/job/ charm_lint_ check/2215/