Merge lp:~brad-marshall/charms/trusty/neutron-api/add-haproxy-nrpe-fix-servicegroups into lp:~openstack-charmers-archive/charms/trusty/neutron-api/next
- Trusty Tahr (14.04)
- add-haproxy-nrpe-fix-servicegroups
- Merge into next
Proposed by
Brad Marshall
Status: | Merged |
---|---|
Merged at revision: | 81 |
Proposed branch: | lp:~brad-marshall/charms/trusty/neutron-api/add-haproxy-nrpe-fix-servicegroups |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/neutron-api/next |
Diff against target: |
1455 lines (+264/-499) 16 files modified
Makefile (+1/-1) config.yaml (+6/-0) hooks/charmhelpers/contrib/charmsupport/nrpe.py (+41/-7) hooks/charmhelpers/contrib/hahelpers/cluster.py (+5/-1) hooks/charmhelpers/contrib/openstack/amulet/deployment.py (+5/-2) hooks/charmhelpers/contrib/openstack/context.py (+28/-9) hooks/charmhelpers/contrib/openstack/files/__init__.py (+18/-0) hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh (+32/-0) hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh (+30/-0) hooks/charmhelpers/contrib/openstack/ip.py (+37/-0) hooks/charmhelpers/contrib/openstack/templates/zeromq (+14/-0) hooks/charmhelpers/contrib/openstack/utils.py (+1/-0) hooks/charmhelpers/core/fstab.py (+2/-2) hooks/charmhelpers/core/strutils.py (+42/-0) hooks/charmhelpers/core/unitdata.py (+0/-477) hooks/neutron_api_hooks.py (+2/-0) |
To merge this branch: | bzr merge lp:~brad-marshall/charms/trusty/neutron-api/add-haproxy-nrpe-fix-servicegroups |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Liam Young (community) | Approve | ||
Review via email: mp+250705@code.launchpad.net |
Commit message
Description of the change
Synced charmhelpers, added nagios_servicegroup config option, and added haproxy nrpe checks.
To post a comment you must log in.
- 80. By Edward Hope-Morley
-
[trivial] charmhelpers sync
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #2009 neutron-api-next for brad-marshall mp250705
UNIT OK: passed
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #2166 neutron-api-next for brad-marshall mp250705
AMULET FAIL: amulet-test missing
AMULET Results (max last 2 lines):
INFO:root:Search string not found in makefile target commands.
ERROR:root:No make target was executed.
Full amulet test output: http://
Build: http://
- 81. By Brad Marshall
-
[bradm] Fixed merge conflicts
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'Makefile' |
2 | --- Makefile 2015-02-17 07:10:15 +0000 |
3 | +++ Makefile 2015-02-26 04:23:37 +0000 |
4 | @@ -15,7 +15,7 @@ |
5 | > bin/charm_helpers_sync.py |
6 | |
7 | sync: bin/charm_helpers_sync.py |
8 | -# @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-hooks.yaml |
9 | + @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-hooks.yaml |
10 | @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-tests.yaml |
11 | |
12 | test: |
13 | |
14 | === modified file 'config.yaml' |
15 | --- config.yaml 2015-01-13 14:43:04 +0000 |
16 | +++ config.yaml 2015-02-26 04:23:37 +0000 |
17 | @@ -213,3 +213,9 @@ |
18 | juju-myservice-0 |
19 | If you're running multiple environments with the same services in them |
20 | this allows you to differentiate between them. |
21 | + nagios_servicegroups: |
22 | + default: "" |
23 | + type: string |
24 | + description: | |
25 | + A comma-separated list of nagios servicegroups. |
26 | + If left empty, the nagios_context will be used as the servicegroup |
27 | |
28 | === modified file 'hooks/charmhelpers/contrib/charmsupport/nrpe.py' |
29 | --- hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-01-26 09:44:26 +0000 |
30 | +++ hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-02-26 04:23:37 +0000 |
31 | @@ -24,6 +24,8 @@ |
32 | import pwd |
33 | import grp |
34 | import os |
35 | +import glob |
36 | +import shutil |
37 | import re |
38 | import shlex |
39 | import yaml |
40 | @@ -161,7 +163,7 @@ |
41 | log('Check command not found: {}'.format(parts[0])) |
42 | return '' |
43 | |
44 | - def write(self, nagios_context, hostname, nagios_servicegroups=None): |
45 | + def write(self, nagios_context, hostname, nagios_servicegroups): |
46 | nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( |
47 | self.command) |
48 | with open(nrpe_check_file, 'w') as nrpe_check_config: |
49 | @@ -177,14 +179,11 @@ |
50 | nagios_servicegroups) |
51 | |
52 | def write_service_config(self, nagios_context, hostname, |
53 | - nagios_servicegroups=None): |
54 | + nagios_servicegroups): |
55 | for f in os.listdir(NRPE.nagios_exportdir): |
56 | if re.search('.*{}.cfg'.format(self.command), f): |
57 | os.remove(os.path.join(NRPE.nagios_exportdir, f)) |
58 | |
59 | - if not nagios_servicegroups: |
60 | - nagios_servicegroups = nagios_context |
61 | - |
62 | templ_vars = { |
63 | 'nagios_hostname': hostname, |
64 | 'nagios_servicegroup': nagios_servicegroups, |
65 | @@ -211,10 +210,10 @@ |
66 | super(NRPE, self).__init__() |
67 | self.config = config() |
68 | self.nagios_context = self.config['nagios_context'] |
69 | - if 'nagios_servicegroups' in self.config: |
70 | + if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: |
71 | self.nagios_servicegroups = self.config['nagios_servicegroups'] |
72 | else: |
73 | - self.nagios_servicegroups = 'juju' |
74 | + self.nagios_servicegroups = self.nagios_context |
75 | self.unit_name = local_unit().replace('/', '-') |
76 | if hostname: |
77 | self.hostname = hostname |
78 | @@ -322,3 +321,38 @@ |
79 | check_cmd='check_status_file.py -f ' |
80 | '/var/lib/nagios/service-check-%s.txt' % svc, |
81 | ) |
82 | + |
83 | + |
84 | +def copy_nrpe_checks(): |
85 | + """ |
86 | + Copy the nrpe checks into place |
87 | + |
88 | + """ |
89 | + NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' |
90 | + nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks', |
91 | + 'charmhelpers', 'contrib', 'openstack', |
92 | + 'files') |
93 | + |
94 | + if not os.path.exists(NAGIOS_PLUGINS): |
95 | + os.makedirs(NAGIOS_PLUGINS) |
96 | + for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): |
97 | + if os.path.isfile(fname): |
98 | + shutil.copy2(fname, |
99 | + os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) |
100 | + |
101 | + |
102 | +def add_haproxy_checks(nrpe, unit_name): |
103 | + """ |
104 | + Add checks for each service in list |
105 | + |
106 | + :param NRPE nrpe: NRPE object to add check to |
107 | + :param str unit_name: Unit name to use in check description |
108 | + """ |
109 | + nrpe.add_check( |
110 | + shortname='haproxy_servers', |
111 | + description='Check HAProxy {%s}' % unit_name, |
112 | + check_cmd='check_haproxy.sh') |
113 | + nrpe.add_check( |
114 | + shortname='haproxy_queue', |
115 | + description='Check HAProxy queue depth {%s}' % unit_name, |
116 | + check_cmd='check_haproxy_queue_depth.sh') |
117 | |
118 | === modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py' |
119 | --- hooks/charmhelpers/contrib/hahelpers/cluster.py 2015-01-26 09:44:26 +0000 |
120 | +++ hooks/charmhelpers/contrib/hahelpers/cluster.py 2015-02-26 04:23:37 +0000 |
121 | @@ -48,6 +48,9 @@ |
122 | from charmhelpers.core.decorators import ( |
123 | retry_on_exception, |
124 | ) |
125 | +from charmhelpers.core.strutils import ( |
126 | + bool_from_string, |
127 | +) |
128 | |
129 | |
130 | class HAIncompleteConfig(Exception): |
131 | @@ -164,7 +167,8 @@ |
132 | . |
133 | returns: boolean |
134 | ''' |
135 | - if config_get('use-https') == "yes": |
136 | + use_https = config_get('use-https') |
137 | + if use_https and bool_from_string(use_https): |
138 | return True |
139 | if config_get('ssl_cert') and config_get('ssl_key'): |
140 | return True |
141 | |
142 | === modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py' |
143 | --- hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2015-01-26 09:44:26 +0000 |
144 | +++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2015-02-26 04:23:37 +0000 |
145 | @@ -71,16 +71,19 @@ |
146 | services.append(this_service) |
147 | use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', |
148 | 'ceph-osd', 'ceph-radosgw'] |
149 | + # Openstack subordinate charms do not expose an origin option as that |
150 | + # is controlled by the principle |
151 | + ignore = ['neutron-openvswitch'] |
152 | |
153 | if self.openstack: |
154 | for svc in services: |
155 | - if svc['name'] not in use_source: |
156 | + if svc['name'] not in use_source + ignore: |
157 | config = {'openstack-origin': self.openstack} |
158 | self.d.configure(svc['name'], config) |
159 | |
160 | if self.source: |
161 | for svc in services: |
162 | - if svc['name'] in use_source: |
163 | + if svc['name'] in use_source and svc['name'] not in ignore: |
164 | config = {'source': self.source} |
165 | self.d.configure(svc['name'], config) |
166 | |
167 | |
168 | === modified file 'hooks/charmhelpers/contrib/openstack/context.py' |
169 | --- hooks/charmhelpers/contrib/openstack/context.py 2015-01-26 09:44:26 +0000 |
170 | +++ hooks/charmhelpers/contrib/openstack/context.py 2015-02-26 04:23:37 +0000 |
171 | @@ -279,9 +279,25 @@ |
172 | class IdentityServiceContext(OSContextGenerator): |
173 | interfaces = ['identity-service'] |
174 | |
175 | + def __init__(self, service=None, service_user=None): |
176 | + self.service = service |
177 | + self.service_user = service_user |
178 | + |
179 | def __call__(self): |
180 | log('Generating template context for identity-service', level=DEBUG) |
181 | ctxt = {} |
182 | + |
183 | + if self.service and self.service_user: |
184 | + # This is required for pki token signing if we don't want /tmp to |
185 | + # be used. |
186 | + cachedir = '/var/cache/%s' % (self.service) |
187 | + if not os.path.isdir(cachedir): |
188 | + log("Creating service cache dir %s" % (cachedir), level=DEBUG) |
189 | + mkdir(path=cachedir, owner=self.service_user, |
190 | + group=self.service_user, perms=0o700) |
191 | + |
192 | + ctxt['signing_dir'] = cachedir |
193 | + |
194 | for rid in relation_ids('identity-service'): |
195 | for unit in related_units(rid): |
196 | rdata = relation_get(rid=rid, unit=unit) |
197 | @@ -291,15 +307,16 @@ |
198 | auth_host = format_ipv6_addr(auth_host) or auth_host |
199 | svc_protocol = rdata.get('service_protocol') or 'http' |
200 | auth_protocol = rdata.get('auth_protocol') or 'http' |
201 | - ctxt = {'service_port': rdata.get('service_port'), |
202 | - 'service_host': serv_host, |
203 | - 'auth_host': auth_host, |
204 | - 'auth_port': rdata.get('auth_port'), |
205 | - 'admin_tenant_name': rdata.get('service_tenant'), |
206 | - 'admin_user': rdata.get('service_username'), |
207 | - 'admin_password': rdata.get('service_password'), |
208 | - 'service_protocol': svc_protocol, |
209 | - 'auth_protocol': auth_protocol} |
210 | + ctxt.update({'service_port': rdata.get('service_port'), |
211 | + 'service_host': serv_host, |
212 | + 'auth_host': auth_host, |
213 | + 'auth_port': rdata.get('auth_port'), |
214 | + 'admin_tenant_name': rdata.get('service_tenant'), |
215 | + 'admin_user': rdata.get('service_username'), |
216 | + 'admin_password': rdata.get('service_password'), |
217 | + 'service_protocol': svc_protocol, |
218 | + 'auth_protocol': auth_protocol}) |
219 | + |
220 | if context_complete(ctxt): |
221 | # NOTE(jamespage) this is required for >= icehouse |
222 | # so a missing value just indicates keystone needs |
223 | @@ -1021,6 +1038,8 @@ |
224 | for unit in related_units(rid): |
225 | ctxt['zmq_nonce'] = relation_get('nonce', unit, rid) |
226 | ctxt['zmq_host'] = relation_get('host', unit, rid) |
227 | + ctxt['zmq_redis_address'] = relation_get( |
228 | + 'zmq_redis_address', unit, rid) |
229 | |
230 | return ctxt |
231 | |
232 | |
233 | === added directory 'hooks/charmhelpers/contrib/openstack/files' |
234 | === added file 'hooks/charmhelpers/contrib/openstack/files/__init__.py' |
235 | --- hooks/charmhelpers/contrib/openstack/files/__init__.py 1970-01-01 00:00:00 +0000 |
236 | +++ hooks/charmhelpers/contrib/openstack/files/__init__.py 2015-02-26 04:23:37 +0000 |
237 | @@ -0,0 +1,18 @@ |
238 | +# Copyright 2014-2015 Canonical Limited. |
239 | +# |
240 | +# This file is part of charm-helpers. |
241 | +# |
242 | +# charm-helpers is free software: you can redistribute it and/or modify |
243 | +# it under the terms of the GNU Lesser General Public License version 3 as |
244 | +# published by the Free Software Foundation. |
245 | +# |
246 | +# charm-helpers is distributed in the hope that it will be useful, |
247 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
248 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
249 | +# GNU Lesser General Public License for more details. |
250 | +# |
251 | +# You should have received a copy of the GNU Lesser General Public License |
252 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
253 | + |
254 | +# dummy __init__.py to fool syncer into thinking this is a syncable python |
255 | +# module |
256 | |
257 | === added file 'hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh' |
258 | --- hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh 1970-01-01 00:00:00 +0000 |
259 | +++ hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh 2015-02-26 04:23:37 +0000 |
260 | @@ -0,0 +1,32 @@ |
261 | +#!/bin/bash |
262 | +#-------------------------------------------- |
263 | +# This file is managed by Juju |
264 | +#-------------------------------------------- |
265 | +# |
266 | +# Copyright 2009,2012 Canonical Ltd. |
267 | +# Author: Tom Haddon |
268 | + |
269 | +CRITICAL=0 |
270 | +NOTACTIVE='' |
271 | +LOGFILE=/var/log/nagios/check_haproxy.log |
272 | +AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}') |
273 | + |
274 | +for appserver in $(grep ' server' /etc/haproxy/haproxy.cfg | awk '{print $2'}); |
275 | +do |
276 | + output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 --regex="class=\"(active|backup)(2|3).*${appserver}" -e ' 200 OK') |
277 | + if [ $? != 0 ]; then |
278 | + date >> $LOGFILE |
279 | + echo $output >> $LOGFILE |
280 | + /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -v | grep $appserver >> $LOGFILE 2>&1 |
281 | + CRITICAL=1 |
282 | + NOTACTIVE="${NOTACTIVE} $appserver" |
283 | + fi |
284 | +done |
285 | + |
286 | +if [ $CRITICAL = 1 ]; then |
287 | + echo "CRITICAL:${NOTACTIVE}" |
288 | + exit 2 |
289 | +fi |
290 | + |
291 | +echo "OK: All haproxy instances looking good" |
292 | +exit 0 |
293 | |
294 | === added file 'hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh' |
295 | --- hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh 1970-01-01 00:00:00 +0000 |
296 | +++ hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh 2015-02-26 04:23:37 +0000 |
297 | @@ -0,0 +1,30 @@ |
298 | +#!/bin/bash |
299 | +#-------------------------------------------- |
300 | +# This file is managed by Juju |
301 | +#-------------------------------------------- |
302 | +# |
303 | +# Copyright 2009,2012 Canonical Ltd. |
304 | +# Author: Tom Haddon |
305 | + |
306 | +# These should be config options at some stage |
307 | +CURRQthrsh=0 |
308 | +MAXQthrsh=100 |
309 | + |
310 | +AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}') |
311 | + |
312 | +HAPROXYSTATS=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v) |
313 | + |
314 | +for BACKEND in $(echo $HAPROXYSTATS| xargs -n1 | grep BACKEND | awk -F , '{print $1}') |
315 | +do |
316 | + CURRQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 3) |
317 | + MAXQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 4) |
318 | + |
319 | + if [[ $CURRQ -gt $CURRQthrsh || $MAXQ -gt $MAXQthrsh ]] ; then |
320 | + echo "CRITICAL: queue depth for $BACKEND - CURRENT:$CURRQ MAX:$MAXQ" |
321 | + exit 2 |
322 | + fi |
323 | +done |
324 | + |
325 | +echo "OK: All haproxy queue depths looking good" |
326 | +exit 0 |
327 | + |
328 | |
329 | === modified file 'hooks/charmhelpers/contrib/openstack/ip.py' |
330 | --- hooks/charmhelpers/contrib/openstack/ip.py 2015-01-26 09:44:26 +0000 |
331 | +++ hooks/charmhelpers/contrib/openstack/ip.py 2015-02-26 04:23:37 +0000 |
332 | @@ -26,6 +26,8 @@ |
333 | ) |
334 | from charmhelpers.contrib.hahelpers.cluster import is_clustered |
335 | |
336 | +from functools import partial |
337 | + |
338 | PUBLIC = 'public' |
339 | INTERNAL = 'int' |
340 | ADMIN = 'admin' |
341 | @@ -107,3 +109,38 @@ |
342 | "clustered=%s)" % (net_type, clustered)) |
343 | |
344 | return resolved_address |
345 | + |
346 | + |
347 | +def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC, |
348 | + override=None): |
349 | + """Returns the correct endpoint URL to advertise to Keystone. |
350 | + |
351 | + This method provides the correct endpoint URL which should be advertised to |
352 | + the keystone charm for endpoint creation. This method allows for the url to |
353 | + be overridden to force a keystone endpoint to have specific URL for any of |
354 | + the defined scopes (admin, internal, public). |
355 | + |
356 | + :param configs: OSTemplateRenderer config templating object to inspect |
357 | + for a complete https context. |
358 | + :param url_template: str format string for creating the url template. Only |
359 | + two values will be passed - the scheme+hostname |
360 | + returned by the canonical_url and the port. |
361 | + :param endpoint_type: str endpoint type to resolve. |
362 | + :param override: str the name of the config option which overrides the |
363 | + endpoint URL defined by the charm itself. None will |
364 | + disable any overrides (default). |
365 | + """ |
366 | + if override: |
367 | + # Return any user-defined overrides for the keystone endpoint URL. |
368 | + user_value = config(override) |
369 | + if user_value: |
370 | + return user_value.strip() |
371 | + |
372 | + return url_template % (canonical_url(configs, endpoint_type), port) |
373 | + |
374 | + |
375 | +public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC) |
376 | + |
377 | +internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL) |
378 | + |
379 | +admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN) |
380 | |
381 | === added file 'hooks/charmhelpers/contrib/openstack/templates/zeromq' |
382 | --- hooks/charmhelpers/contrib/openstack/templates/zeromq 1970-01-01 00:00:00 +0000 |
383 | +++ hooks/charmhelpers/contrib/openstack/templates/zeromq 2015-02-26 04:23:37 +0000 |
384 | @@ -0,0 +1,14 @@ |
385 | +{% if zmq_host -%} |
386 | +# ZeroMQ configuration (restart-nonce: {{ zmq_nonce }}) |
387 | +rpc_backend = zmq |
388 | +rpc_zmq_host = {{ zmq_host }} |
389 | +{% if zmq_redis_address -%} |
390 | +rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_redis.MatchMakerRedis |
391 | +matchmaker_heartbeat_freq = 15 |
392 | +matchmaker_heartbeat_ttl = 30 |
393 | +[matchmaker_redis] |
394 | +host = {{ zmq_redis_address }} |
395 | +{% else -%} |
396 | +rpc_zmq_matchmaker = oslo.messaging._drivers.matchmaker_ring.MatchMakerRing |
397 | +{% endif -%} |
398 | +{% endif -%} |
399 | |
400 | === modified file 'hooks/charmhelpers/contrib/openstack/utils.py' |
401 | --- hooks/charmhelpers/contrib/openstack/utils.py 2015-01-26 09:44:26 +0000 |
402 | +++ hooks/charmhelpers/contrib/openstack/utils.py 2015-02-26 04:23:37 +0000 |
403 | @@ -103,6 +103,7 @@ |
404 | ('2.1.0', 'juno'), |
405 | ('2.2.0', 'juno'), |
406 | ('2.2.1', 'kilo'), |
407 | + ('2.2.2', 'kilo'), |
408 | ]) |
409 | |
410 | DEFAULT_LOOPBACK_SIZE = '5G' |
411 | |
412 | === modified file 'hooks/charmhelpers/core/fstab.py' |
413 | --- hooks/charmhelpers/core/fstab.py 2015-02-17 07:10:15 +0000 |
414 | +++ hooks/charmhelpers/core/fstab.py 2015-02-26 04:23:37 +0000 |
415 | @@ -77,7 +77,7 @@ |
416 | for line in self.readlines(): |
417 | line = line.decode('us-ascii') |
418 | try: |
419 | - if line.strip() and not line.startswith("#"): |
420 | + if line.strip() and not line.strip().startswith("#"): |
421 | yield self._hydrate_entry(line) |
422 | except ValueError: |
423 | pass |
424 | @@ -104,7 +104,7 @@ |
425 | |
426 | found = False |
427 | for index, line in enumerate(lines): |
428 | - if not line.startswith("#"): |
429 | + if line.strip() and not line.strip().startswith("#"): |
430 | if self._hydrate_entry(line) == entry: |
431 | found = True |
432 | break |
433 | |
434 | === added file 'hooks/charmhelpers/core/strutils.py' |
435 | --- hooks/charmhelpers/core/strutils.py 1970-01-01 00:00:00 +0000 |
436 | +++ hooks/charmhelpers/core/strutils.py 2015-02-26 04:23:37 +0000 |
437 | @@ -0,0 +1,42 @@ |
438 | +#!/usr/bin/env python |
439 | +# -*- coding: utf-8 -*- |
440 | + |
441 | +# Copyright 2014-2015 Canonical Limited. |
442 | +# |
443 | +# This file is part of charm-helpers. |
444 | +# |
445 | +# charm-helpers is free software: you can redistribute it and/or modify |
446 | +# it under the terms of the GNU Lesser General Public License version 3 as |
447 | +# published by the Free Software Foundation. |
448 | +# |
449 | +# charm-helpers is distributed in the hope that it will be useful, |
450 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
451 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
452 | +# GNU Lesser General Public License for more details. |
453 | +# |
454 | +# You should have received a copy of the GNU Lesser General Public License |
455 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
456 | + |
457 | +import six |
458 | + |
459 | + |
460 | +def bool_from_string(value): |
461 | + """Interpret string value as boolean. |
462 | + |
463 | + Returns True if value translates to True otherwise False. |
464 | + """ |
465 | + if isinstance(value, six.string_types): |
466 | + value = six.text_type(value) |
467 | + else: |
468 | + msg = "Unable to interpret non-string value '%s' as boolean" % (value) |
469 | + raise ValueError(msg) |
470 | + |
471 | + value = value.strip().lower() |
472 | + |
473 | + if value in ['y', 'yes', 'true', 't']: |
474 | + return True |
475 | + elif value in ['n', 'no', 'false', 'f']: |
476 | + return False |
477 | + |
478 | + msg = "Unable to interpret string value '%s' as boolean" % (value) |
479 | + raise ValueError(msg) |
480 | |
481 | === added file 'hooks/charmhelpers/core/unitdata.py' |
482 | --- hooks/charmhelpers/core/unitdata.py 1970-01-01 00:00:00 +0000 |
483 | +++ hooks/charmhelpers/core/unitdata.py 2015-02-26 04:23:37 +0000 |
484 | @@ -0,0 +1,477 @@ |
485 | +#!/usr/bin/env python |
486 | +# -*- coding: utf-8 -*- |
487 | +# |
488 | +# Copyright 2014-2015 Canonical Limited. |
489 | +# |
490 | +# This file is part of charm-helpers. |
491 | +# |
492 | +# charm-helpers is free software: you can redistribute it and/or modify |
493 | +# it under the terms of the GNU Lesser General Public License version 3 as |
494 | +# published by the Free Software Foundation. |
495 | +# |
496 | +# charm-helpers is distributed in the hope that it will be useful, |
497 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
498 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
499 | +# GNU Lesser General Public License for more details. |
500 | +# |
501 | +# You should have received a copy of the GNU Lesser General Public License |
502 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
503 | +# |
504 | +# |
505 | +# Authors: |
506 | +# Kapil Thangavelu <kapil.foss@gmail.com> |
507 | +# |
508 | +""" |
509 | +Intro |
510 | +----- |
511 | + |
512 | +A simple way to store state in units. This provides a key value |
513 | +storage with support for versioned, transactional operation, |
514 | +and can calculate deltas from previous values to simplify unit logic |
515 | +when processing changes. |
516 | + |
517 | + |
518 | +Hook Integration |
519 | +---------------- |
520 | + |
521 | +There are several extant frameworks for hook execution, including |
522 | + |
523 | + - charmhelpers.core.hookenv.Hooks |
524 | + - charmhelpers.core.services.ServiceManager |
525 | + |
526 | +The storage classes are framework agnostic, one simple integration is |
527 | +via the HookData contextmanager. It will record the current hook |
528 | +execution environment (including relation data, config data, etc.), |
529 | +setup a transaction and allow easy access to the changes from |
530 | +previously seen values. One consequence of the integration is the |
531 | +reservation of particular keys ('rels', 'unit', 'env', 'config', |
532 | +'charm_revisions') for their respective values. |
533 | + |
534 | +Here's a fully worked integration example using hookenv.Hooks:: |
535 | + |
536 | + from charmhelper.core import hookenv, unitdata |
537 | + |
538 | + hook_data = unitdata.HookData() |
539 | + db = unitdata.kv() |
540 | + hooks = hookenv.Hooks() |
541 | + |
542 | + @hooks.hook |
543 | + def config_changed(): |
544 | + # Print all changes to configuration from previously seen |
545 | + # values. |
546 | + for changed, (prev, cur) in hook_data.conf.items(): |
547 | + print('config changed', changed, |
548 | + 'previous value', prev, |
549 | + 'current value', cur) |
550 | + |
551 | + # Get some unit specific bookeeping |
552 | + if not db.get('pkg_key'): |
553 | + key = urllib.urlopen('https://example.com/pkg_key').read() |
554 | + db.set('pkg_key', key) |
555 | + |
556 | + # Directly access all charm config as a mapping. |
557 | + conf = db.getrange('config', True) |
558 | + |
559 | + # Directly access all relation data as a mapping |
560 | + rels = db.getrange('rels', True) |
561 | + |
562 | + if __name__ == '__main__': |
563 | + with hook_data(): |
564 | + hook.execute() |
565 | + |
566 | + |
567 | +A more basic integration is via the hook_scope context manager which simply |
568 | +manages transaction scope (and records hook name, and timestamp):: |
569 | + |
570 | + >>> from unitdata import kv |
571 | + >>> db = kv() |
572 | + >>> with db.hook_scope('install'): |
573 | + ... # do work, in transactional scope. |
574 | + ... db.set('x', 1) |
575 | + >>> db.get('x') |
576 | + 1 |
577 | + |
578 | + |
579 | +Usage |
580 | +----- |
581 | + |
582 | +Values are automatically json de/serialized to preserve basic typing |
583 | +and complex data struct capabilities (dicts, lists, ints, booleans, etc). |
584 | + |
585 | +Individual values can be manipulated via get/set:: |
586 | + |
587 | + >>> kv.set('y', True) |
588 | + >>> kv.get('y') |
589 | + True |
590 | + |
591 | + # We can set complex values (dicts, lists) as a single key. |
592 | + >>> kv.set('config', {'a': 1, 'b': True'}) |
593 | + |
594 | + # Also supports returning dictionaries as a record which |
595 | + # provides attribute access. |
596 | + >>> config = kv.get('config', record=True) |
597 | + >>> config.b |
598 | + True |
599 | + |
600 | + |
601 | +Groups of keys can be manipulated with update/getrange:: |
602 | + |
603 | + >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") |
604 | + >>> kv.getrange('gui.', strip=True) |
605 | + {'z': 1, 'y': 2} |
606 | + |
607 | +When updating values, its very helpful to understand which values |
608 | +have actually changed and how have they changed. The storage |
609 | +provides a delta method to provide for this:: |
610 | + |
611 | + >>> data = {'debug': True, 'option': 2} |
612 | + >>> delta = kv.delta(data, 'config.') |
613 | + >>> delta.debug.previous |
614 | + None |
615 | + >>> delta.debug.current |
616 | + True |
617 | + >>> delta |
618 | + {'debug': (None, True), 'option': (None, 2)} |
619 | + |
620 | +Note the delta method does not persist the actual change, it needs to |
621 | +be explicitly saved via 'update' method:: |
622 | + |
623 | + >>> kv.update(data, 'config.') |
624 | + |
625 | +Values modified in the context of a hook scope retain historical values |
626 | +associated to the hookname. |
627 | + |
628 | + >>> with db.hook_scope('config-changed'): |
629 | + ... db.set('x', 42) |
630 | + >>> db.gethistory('x') |
631 | + [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), |
632 | + (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] |
633 | + |
634 | +""" |
635 | + |
636 | +import collections |
637 | +import contextlib |
638 | +import datetime |
639 | +import json |
640 | +import os |
641 | +import pprint |
642 | +import sqlite3 |
643 | +import sys |
644 | + |
645 | +__author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>' |
646 | + |
647 | + |
648 | +class Storage(object): |
649 | + """Simple key value database for local unit state within charms. |
650 | + |
651 | + Modifications are automatically committed at hook exit. That's |
652 | + currently regardless of exit code. |
653 | + |
654 | + To support dicts, lists, integer, floats, and booleans values |
655 | + are automatically json encoded/decoded. |
656 | + """ |
657 | + def __init__(self, path=None): |
658 | + self.db_path = path |
659 | + if path is None: |
660 | + self.db_path = os.path.join( |
661 | + os.environ.get('CHARM_DIR', ''), '.unit-state.db') |
662 | + self.conn = sqlite3.connect('%s' % self.db_path) |
663 | + self.cursor = self.conn.cursor() |
664 | + self.revision = None |
665 | + self._closed = False |
666 | + self._init() |
667 | + |
668 | + def close(self): |
669 | + if self._closed: |
670 | + return |
671 | + self.flush(False) |
672 | + self.cursor.close() |
673 | + self.conn.close() |
674 | + self._closed = True |
675 | + |
676 | + def _scoped_query(self, stmt, params=None): |
677 | + if params is None: |
678 | + params = [] |
679 | + return stmt, params |
680 | + |
681 | + def get(self, key, default=None, record=False): |
682 | + self.cursor.execute( |
683 | + *self._scoped_query( |
684 | + 'select data from kv where key=?', [key])) |
685 | + result = self.cursor.fetchone() |
686 | + if not result: |
687 | + return default |
688 | + if record: |
689 | + return Record(json.loads(result[0])) |
690 | + return json.loads(result[0]) |
691 | + |
692 | + def getrange(self, key_prefix, strip=False): |
693 | + stmt = "select key, data from kv where key like '%s%%'" % key_prefix |
694 | + self.cursor.execute(*self._scoped_query(stmt)) |
695 | + result = self.cursor.fetchall() |
696 | + |
697 | + if not result: |
698 | + return None |
699 | + if not strip: |
700 | + key_prefix = '' |
701 | + return dict([ |
702 | + (k[len(key_prefix):], json.loads(v)) for k, v in result]) |
703 | + |
704 | + def update(self, mapping, prefix=""): |
705 | + for k, v in mapping.items(): |
706 | + self.set("%s%s" % (prefix, k), v) |
707 | + |
708 | + def unset(self, key): |
709 | + self.cursor.execute('delete from kv where key=?', [key]) |
710 | + if self.revision and self.cursor.rowcount: |
711 | + self.cursor.execute( |
712 | + 'insert into kv_revisions values (?, ?, ?)', |
713 | + [key, self.revision, json.dumps('DELETED')]) |
714 | + |
715 | + def set(self, key, value): |
716 | + serialized = json.dumps(value) |
717 | + |
718 | + self.cursor.execute( |
719 | + 'select data from kv where key=?', [key]) |
720 | + exists = self.cursor.fetchone() |
721 | + |
722 | + # Skip mutations to the same value |
723 | + if exists: |
724 | + if exists[0] == serialized: |
725 | + return value |
726 | + |
727 | + if not exists: |
728 | + self.cursor.execute( |
729 | + 'insert into kv (key, data) values (?, ?)', |
730 | + (key, serialized)) |
731 | + else: |
732 | + self.cursor.execute(''' |
733 | + update kv |
734 | + set data = ? |
735 | + where key = ?''', [serialized, key]) |
736 | + |
737 | + # Save |
738 | + if not self.revision: |
739 | + return value |
740 | + |
741 | + self.cursor.execute( |
742 | + 'select 1 from kv_revisions where key=? and revision=?', |
743 | + [key, self.revision]) |
744 | + exists = self.cursor.fetchone() |
745 | + |
746 | + if not exists: |
747 | + self.cursor.execute( |
748 | + '''insert into kv_revisions ( |
749 | + revision, key, data) values (?, ?, ?)''', |
750 | + (self.revision, key, serialized)) |
751 | + else: |
752 | + self.cursor.execute( |
753 | + ''' |
754 | + update kv_revisions |
755 | + set data = ? |
756 | + where key = ? |
757 | + and revision = ?''', |
758 | + [serialized, key, self.revision]) |
759 | + |
760 | + return value |
761 | + |
762 | + def delta(self, mapping, prefix): |
763 | + """ |
764 | + return a delta containing values that have changed. |
765 | + """ |
766 | + previous = self.getrange(prefix, strip=True) |
767 | + if not previous: |
768 | + pk = set() |
769 | + else: |
770 | + pk = set(previous.keys()) |
771 | + ck = set(mapping.keys()) |
772 | + delta = DeltaSet() |
773 | + |
774 | + # added |
775 | + for k in ck.difference(pk): |
776 | + delta[k] = Delta(None, mapping[k]) |
777 | + |
778 | + # removed |
779 | + for k in pk.difference(ck): |
780 | + delta[k] = Delta(previous[k], None) |
781 | + |
782 | + # changed |
783 | + for k in pk.intersection(ck): |
784 | + c = mapping[k] |
785 | + p = previous[k] |
786 | + if c != p: |
787 | + delta[k] = Delta(p, c) |
788 | + |
789 | + return delta |
790 | + |
791 | + @contextlib.contextmanager |
792 | + def hook_scope(self, name=""): |
793 | + """Scope all future interactions to the current hook execution |
794 | + revision.""" |
795 | + assert not self.revision |
796 | + self.cursor.execute( |
797 | + 'insert into hooks (hook, date) values (?, ?)', |
798 | + (name or sys.argv[0], |
799 | + datetime.datetime.utcnow().isoformat())) |
800 | + self.revision = self.cursor.lastrowid |
801 | + try: |
802 | + yield self.revision |
803 | + self.revision = None |
804 | + except: |
805 | + self.flush(False) |
806 | + self.revision = None |
807 | + raise |
808 | + else: |
809 | + self.flush() |
810 | + |
811 | + def flush(self, save=True): |
812 | + if save: |
813 | + self.conn.commit() |
814 | + elif self._closed: |
815 | + return |
816 | + else: |
817 | + self.conn.rollback() |
818 | + |
819 | + def _init(self): |
820 | + self.cursor.execute(''' |
821 | + create table if not exists kv ( |
822 | + key text, |
823 | + data text, |
824 | + primary key (key) |
825 | + )''') |
826 | + self.cursor.execute(''' |
827 | + create table if not exists kv_revisions ( |
828 | + key text, |
829 | + revision integer, |
830 | + data text, |
831 | + primary key (key, revision) |
832 | + )''') |
833 | + self.cursor.execute(''' |
834 | + create table if not exists hooks ( |
835 | + version integer primary key autoincrement, |
836 | + hook text, |
837 | + date text |
838 | + )''') |
839 | + self.conn.commit() |
840 | + |
841 | + def gethistory(self, key, deserialize=False): |
842 | + self.cursor.execute( |
843 | + ''' |
844 | + select kv.revision, kv.key, kv.data, h.hook, h.date |
845 | + from kv_revisions kv, |
846 | + hooks h |
847 | + where kv.key=? |
848 | + and kv.revision = h.version |
849 | + ''', [key]) |
850 | + if deserialize is False: |
851 | + return self.cursor.fetchall() |
852 | + return map(_parse_history, self.cursor.fetchall()) |
853 | + |
854 | + def debug(self, fh=sys.stderr): |
855 | + self.cursor.execute('select * from kv') |
856 | + pprint.pprint(self.cursor.fetchall(), stream=fh) |
857 | + self.cursor.execute('select * from kv_revisions') |
858 | + pprint.pprint(self.cursor.fetchall(), stream=fh) |
859 | + |
860 | + |
861 | +def _parse_history(d): |
862 | + return (d[0], d[1], json.loads(d[2]), d[3], |
863 | + datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) |
864 | + |
865 | + |
866 | +class HookData(object): |
867 | + """Simple integration for existing hook exec frameworks. |
868 | + |
869 | + Records all unit information, and stores deltas for processing |
870 | + by the hook. |
871 | + |
872 | + Sample:: |
873 | + |
874 | + from charmhelper.core import hookenv, unitdata |
875 | + |
876 | + changes = unitdata.HookData() |
877 | + db = unitdata.kv() |
878 | + hooks = hookenv.Hooks() |
879 | + |
880 | + @hooks.hook |
881 | + def config_changed(): |
882 | + # View all changes to configuration |
883 | + for changed, (prev, cur) in changes.conf.items(): |
884 | + print('config changed', changed, |
885 | + 'previous value', prev, |
886 | + 'current value', cur) |
887 | + |
888 | + # Get some unit specific bookeeping |
889 | + if not db.get('pkg_key'): |
890 | + key = urllib.urlopen('https://example.com/pkg_key').read() |
891 | + db.set('pkg_key', key) |
892 | + |
893 | + if __name__ == '__main__': |
894 | + with changes(): |
895 | + hook.execute() |
896 | + |
897 | + """ |
898 | + def __init__(self): |
899 | + self.kv = kv() |
900 | + self.conf = None |
901 | + self.rels = None |
902 | + |
903 | + @contextlib.contextmanager |
904 | + def __call__(self): |
905 | + from charmhelpers.core import hookenv |
906 | + hook_name = hookenv.hook_name() |
907 | + |
908 | + with self.kv.hook_scope(hook_name): |
909 | + self._record_charm_version(hookenv.charm_dir()) |
910 | + delta_config, delta_relation = self._record_hook(hookenv) |
911 | + yield self.kv, delta_config, delta_relation |
912 | + |
913 | + def _record_charm_version(self, charm_dir): |
914 | + # Record revisions.. charm revisions are meaningless |
915 | + # to charm authors as they don't control the revision. |
916 | + # so logic dependnent on revision is not particularly |
917 | + # useful, however it is useful for debugging analysis. |
918 | + charm_rev = open( |
919 | + os.path.join(charm_dir, 'revision')).read().strip() |
920 | + charm_rev = charm_rev or '0' |
921 | + revs = self.kv.get('charm_revisions', []) |
922 | + if charm_rev not in revs: |
923 | + revs.append(charm_rev.strip() or '0') |
924 | + self.kv.set('charm_revisions', revs) |
925 | + |
926 | + def _record_hook(self, hookenv): |
927 | + data = hookenv.execution_environment() |
928 | + self.conf = conf_delta = self.kv.delta(data['conf'], 'config') |
929 | + self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') |
930 | + self.kv.set('env', data['env']) |
931 | + self.kv.set('unit', data['unit']) |
932 | + self.kv.set('relid', data.get('relid')) |
933 | + return conf_delta, rels_delta |
934 | + |
935 | + |
936 | +class Record(dict): |
937 | + |
938 | + __slots__ = () |
939 | + |
940 | + def __getattr__(self, k): |
941 | + if k in self: |
942 | + return self[k] |
943 | + raise AttributeError(k) |
944 | + |
945 | + |
946 | +class DeltaSet(Record): |
947 | + |
948 | + __slots__ = () |
949 | + |
950 | + |
951 | +Delta = collections.namedtuple('Delta', ['previous', 'current']) |
952 | + |
953 | + |
954 | +_KV = None |
955 | + |
956 | + |
957 | +def kv(): |
958 | + global _KV |
959 | + if _KV is None: |
960 | + _KV = Storage() |
961 | + return _KV |
962 | |
963 | === removed file 'hooks/charmhelpers/core/unitdata.py' |
964 | --- hooks/charmhelpers/core/unitdata.py 2015-02-17 07:10:15 +0000 |
965 | +++ hooks/charmhelpers/core/unitdata.py 1970-01-01 00:00:00 +0000 |
966 | @@ -1,477 +0,0 @@ |
967 | -#!/usr/bin/env python |
968 | -# -*- coding: utf-8 -*- |
969 | -# |
970 | -# Copyright 2014-2015 Canonical Limited. |
971 | -# |
972 | -# This file is part of charm-helpers. |
973 | -# |
974 | -# charm-helpers is free software: you can redistribute it and/or modify |
975 | -# it under the terms of the GNU Lesser General Public License version 3 as |
976 | -# published by the Free Software Foundation. |
977 | -# |
978 | -# charm-helpers is distributed in the hope that it will be useful, |
979 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
980 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
981 | -# GNU Lesser General Public License for more details. |
982 | -# |
983 | -# You should have received a copy of the GNU Lesser General Public License |
984 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
985 | -# |
986 | -# |
987 | -# Authors: |
988 | -# Kapil Thangavelu <kapil.foss@gmail.com> |
989 | -# |
990 | -""" |
991 | -Intro |
992 | ------ |
993 | - |
994 | -A simple way to store state in units. This provides a key value |
995 | -storage with support for versioned, transactional operation, |
996 | -and can calculate deltas from previous values to simplify unit logic |
997 | -when processing changes. |
998 | - |
999 | - |
1000 | -Hook Integration |
1001 | ----------------- |
1002 | - |
1003 | -There are several extant frameworks for hook execution, including |
1004 | - |
1005 | - - charmhelpers.core.hookenv.Hooks |
1006 | - - charmhelpers.core.services.ServiceManager |
1007 | - |
1008 | -The storage classes are framework agnostic, one simple integration is |
1009 | -via the HookData contextmanager. It will record the current hook |
1010 | -execution environment (including relation data, config data, etc.), |
1011 | -setup a transaction and allow easy access to the changes from |
1012 | -previously seen values. One consequence of the integration is the |
1013 | -reservation of particular keys ('rels', 'unit', 'env', 'config', |
1014 | -'charm_revisions') for their respective values. |
1015 | - |
1016 | -Here's a fully worked integration example using hookenv.Hooks:: |
1017 | - |
1018 | - from charmhelper.core import hookenv, unitdata |
1019 | - |
1020 | - hook_data = unitdata.HookData() |
1021 | - db = unitdata.kv() |
1022 | - hooks = hookenv.Hooks() |
1023 | - |
1024 | - @hooks.hook |
1025 | - def config_changed(): |
1026 | - # Print all changes to configuration from previously seen |
1027 | - # values. |
1028 | - for changed, (prev, cur) in hook_data.conf.items(): |
1029 | - print('config changed', changed, |
1030 | - 'previous value', prev, |
1031 | - 'current value', cur) |
1032 | - |
1033 | - # Get some unit specific bookeeping |
1034 | - if not db.get('pkg_key'): |
1035 | - key = urllib.urlopen('https://example.com/pkg_key').read() |
1036 | - db.set('pkg_key', key) |
1037 | - |
1038 | - # Directly access all charm config as a mapping. |
1039 | - conf = db.getrange('config', True) |
1040 | - |
1041 | - # Directly access all relation data as a mapping |
1042 | - rels = db.getrange('rels', True) |
1043 | - |
1044 | - if __name__ == '__main__': |
1045 | - with hook_data(): |
1046 | - hook.execute() |
1047 | - |
1048 | - |
1049 | -A more basic integration is via the hook_scope context manager which simply |
1050 | -manages transaction scope (and records hook name, and timestamp):: |
1051 | - |
1052 | - >>> from unitdata import kv |
1053 | - >>> db = kv() |
1054 | - >>> with db.hook_scope('install'): |
1055 | - ... # do work, in transactional scope. |
1056 | - ... db.set('x', 1) |
1057 | - >>> db.get('x') |
1058 | - 1 |
1059 | - |
1060 | - |
1061 | -Usage |
1062 | ------ |
1063 | - |
1064 | -Values are automatically json de/serialized to preserve basic typing |
1065 | -and complex data struct capabilities (dicts, lists, ints, booleans, etc). |
1066 | - |
1067 | -Individual values can be manipulated via get/set:: |
1068 | - |
1069 | - >>> kv.set('y', True) |
1070 | - >>> kv.get('y') |
1071 | - True |
1072 | - |
1073 | - # We can set complex values (dicts, lists) as a single key. |
1074 | - >>> kv.set('config', {'a': 1, 'b': True'}) |
1075 | - |
1076 | - # Also supports returning dictionaries as a record which |
1077 | - # provides attribute access. |
1078 | - >>> config = kv.get('config', record=True) |
1079 | - >>> config.b |
1080 | - True |
1081 | - |
1082 | - |
1083 | -Groups of keys can be manipulated with update/getrange:: |
1084 | - |
1085 | - >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") |
1086 | - >>> kv.getrange('gui.', strip=True) |
1087 | - {'z': 1, 'y': 2} |
1088 | - |
1089 | -When updating values, its very helpful to understand which values |
1090 | -have actually changed and how have they changed. The storage |
1091 | -provides a delta method to provide for this:: |
1092 | - |
1093 | - >>> data = {'debug': True, 'option': 2} |
1094 | - >>> delta = kv.delta(data, 'config.') |
1095 | - >>> delta.debug.previous |
1096 | - None |
1097 | - >>> delta.debug.current |
1098 | - True |
1099 | - >>> delta |
1100 | - {'debug': (None, True), 'option': (None, 2)} |
1101 | - |
1102 | -Note the delta method does not persist the actual change, it needs to |
1103 | -be explicitly saved via 'update' method:: |
1104 | - |
1105 | - >>> kv.update(data, 'config.') |
1106 | - |
1107 | -Values modified in the context of a hook scope retain historical values |
1108 | -associated to the hookname. |
1109 | - |
1110 | - >>> with db.hook_scope('config-changed'): |
1111 | - ... db.set('x', 42) |
1112 | - >>> db.gethistory('x') |
1113 | - [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), |
1114 | - (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] |
1115 | - |
1116 | -""" |
1117 | - |
1118 | -import collections |
1119 | -import contextlib |
1120 | -import datetime |
1121 | -import json |
1122 | -import os |
1123 | -import pprint |
1124 | -import sqlite3 |
1125 | -import sys |
1126 | - |
1127 | -__author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>' |
1128 | - |
1129 | - |
1130 | -class Storage(object): |
1131 | - """Simple key value database for local unit state within charms. |
1132 | - |
1133 | - Modifications are automatically committed at hook exit. That's |
1134 | - currently regardless of exit code. |
1135 | - |
1136 | - To support dicts, lists, integer, floats, and booleans values |
1137 | - are automatically json encoded/decoded. |
1138 | - """ |
1139 | - def __init__(self, path=None): |
1140 | - self.db_path = path |
1141 | - if path is None: |
1142 | - self.db_path = os.path.join( |
1143 | - os.environ.get('CHARM_DIR', ''), '.unit-state.db') |
1144 | - self.conn = sqlite3.connect('%s' % self.db_path) |
1145 | - self.cursor = self.conn.cursor() |
1146 | - self.revision = None |
1147 | - self._closed = False |
1148 | - self._init() |
1149 | - |
1150 | - def close(self): |
1151 | - if self._closed: |
1152 | - return |
1153 | - self.flush(False) |
1154 | - self.cursor.close() |
1155 | - self.conn.close() |
1156 | - self._closed = True |
1157 | - |
1158 | - def _scoped_query(self, stmt, params=None): |
1159 | - if params is None: |
1160 | - params = [] |
1161 | - return stmt, params |
1162 | - |
1163 | - def get(self, key, default=None, record=False): |
1164 | - self.cursor.execute( |
1165 | - *self._scoped_query( |
1166 | - 'select data from kv where key=?', [key])) |
1167 | - result = self.cursor.fetchone() |
1168 | - if not result: |
1169 | - return default |
1170 | - if record: |
1171 | - return Record(json.loads(result[0])) |
1172 | - return json.loads(result[0]) |
1173 | - |
1174 | - def getrange(self, key_prefix, strip=False): |
1175 | - stmt = "select key, data from kv where key like '%s%%'" % key_prefix |
1176 | - self.cursor.execute(*self._scoped_query(stmt)) |
1177 | - result = self.cursor.fetchall() |
1178 | - |
1179 | - if not result: |
1180 | - return None |
1181 | - if not strip: |
1182 | - key_prefix = '' |
1183 | - return dict([ |
1184 | - (k[len(key_prefix):], json.loads(v)) for k, v in result]) |
1185 | - |
1186 | - def update(self, mapping, prefix=""): |
1187 | - for k, v in mapping.items(): |
1188 | - self.set("%s%s" % (prefix, k), v) |
1189 | - |
1190 | - def unset(self, key): |
1191 | - self.cursor.execute('delete from kv where key=?', [key]) |
1192 | - if self.revision and self.cursor.rowcount: |
1193 | - self.cursor.execute( |
1194 | - 'insert into kv_revisions values (?, ?, ?)', |
1195 | - [key, self.revision, json.dumps('DELETED')]) |
1196 | - |
1197 | - def set(self, key, value): |
1198 | - serialized = json.dumps(value) |
1199 | - |
1200 | - self.cursor.execute( |
1201 | - 'select data from kv where key=?', [key]) |
1202 | - exists = self.cursor.fetchone() |
1203 | - |
1204 | - # Skip mutations to the same value |
1205 | - if exists: |
1206 | - if exists[0] == serialized: |
1207 | - return value |
1208 | - |
1209 | - if not exists: |
1210 | - self.cursor.execute( |
1211 | - 'insert into kv (key, data) values (?, ?)', |
1212 | - (key, serialized)) |
1213 | - else: |
1214 | - self.cursor.execute(''' |
1215 | - update kv |
1216 | - set data = ? |
1217 | - where key = ?''', [serialized, key]) |
1218 | - |
1219 | - # Save |
1220 | - if not self.revision: |
1221 | - return value |
1222 | - |
1223 | - self.cursor.execute( |
1224 | - 'select 1 from kv_revisions where key=? and revision=?', |
1225 | - [key, self.revision]) |
1226 | - exists = self.cursor.fetchone() |
1227 | - |
1228 | - if not exists: |
1229 | - self.cursor.execute( |
1230 | - '''insert into kv_revisions ( |
1231 | - revision, key, data) values (?, ?, ?)''', |
1232 | - (self.revision, key, serialized)) |
1233 | - else: |
1234 | - self.cursor.execute( |
1235 | - ''' |
1236 | - update kv_revisions |
1237 | - set data = ? |
1238 | - where key = ? |
1239 | - and revision = ?''', |
1240 | - [serialized, key, self.revision]) |
1241 | - |
1242 | - return value |
1243 | - |
1244 | - def delta(self, mapping, prefix): |
1245 | - """ |
1246 | - return a delta containing values that have changed. |
1247 | - """ |
1248 | - previous = self.getrange(prefix, strip=True) |
1249 | - if not previous: |
1250 | - pk = set() |
1251 | - else: |
1252 | - pk = set(previous.keys()) |
1253 | - ck = set(mapping.keys()) |
1254 | - delta = DeltaSet() |
1255 | - |
1256 | - # added |
1257 | - for k in ck.difference(pk): |
1258 | - delta[k] = Delta(None, mapping[k]) |
1259 | - |
1260 | - # removed |
1261 | - for k in pk.difference(ck): |
1262 | - delta[k] = Delta(previous[k], None) |
1263 | - |
1264 | - # changed |
1265 | - for k in pk.intersection(ck): |
1266 | - c = mapping[k] |
1267 | - p = previous[k] |
1268 | - if c != p: |
1269 | - delta[k] = Delta(p, c) |
1270 | - |
1271 | - return delta |
1272 | - |
1273 | - @contextlib.contextmanager |
1274 | - def hook_scope(self, name=""): |
1275 | - """Scope all future interactions to the current hook execution |
1276 | - revision.""" |
1277 | - assert not self.revision |
1278 | - self.cursor.execute( |
1279 | - 'insert into hooks (hook, date) values (?, ?)', |
1280 | - (name or sys.argv[0], |
1281 | - datetime.datetime.utcnow().isoformat())) |
1282 | - self.revision = self.cursor.lastrowid |
1283 | - try: |
1284 | - yield self.revision |
1285 | - self.revision = None |
1286 | - except: |
1287 | - self.flush(False) |
1288 | - self.revision = None |
1289 | - raise |
1290 | - else: |
1291 | - self.flush() |
1292 | - |
1293 | - def flush(self, save=True): |
1294 | - if save: |
1295 | - self.conn.commit() |
1296 | - elif self._closed: |
1297 | - return |
1298 | - else: |
1299 | - self.conn.rollback() |
1300 | - |
1301 | - def _init(self): |
1302 | - self.cursor.execute(''' |
1303 | - create table if not exists kv ( |
1304 | - key text, |
1305 | - data text, |
1306 | - primary key (key) |
1307 | - )''') |
1308 | - self.cursor.execute(''' |
1309 | - create table if not exists kv_revisions ( |
1310 | - key text, |
1311 | - revision integer, |
1312 | - data text, |
1313 | - primary key (key, revision) |
1314 | - )''') |
1315 | - self.cursor.execute(''' |
1316 | - create table if not exists hooks ( |
1317 | - version integer primary key autoincrement, |
1318 | - hook text, |
1319 | - date text |
1320 | - )''') |
1321 | - self.conn.commit() |
1322 | - |
1323 | - def gethistory(self, key, deserialize=False): |
1324 | - self.cursor.execute( |
1325 | - ''' |
1326 | - select kv.revision, kv.key, kv.data, h.hook, h.date |
1327 | - from kv_revisions kv, |
1328 | - hooks h |
1329 | - where kv.key=? |
1330 | - and kv.revision = h.version |
1331 | - ''', [key]) |
1332 | - if deserialize is False: |
1333 | - return self.cursor.fetchall() |
1334 | - return map(_parse_history, self.cursor.fetchall()) |
1335 | - |
1336 | - def debug(self, fh=sys.stderr): |
1337 | - self.cursor.execute('select * from kv') |
1338 | - pprint.pprint(self.cursor.fetchall(), stream=fh) |
1339 | - self.cursor.execute('select * from kv_revisions') |
1340 | - pprint.pprint(self.cursor.fetchall(), stream=fh) |
1341 | - |
1342 | - |
1343 | -def _parse_history(d): |
1344 | - return (d[0], d[1], json.loads(d[2]), d[3], |
1345 | - datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) |
1346 | - |
1347 | - |
1348 | -class HookData(object): |
1349 | - """Simple integration for existing hook exec frameworks. |
1350 | - |
1351 | - Records all unit information, and stores deltas for processing |
1352 | - by the hook. |
1353 | - |
1354 | - Sample:: |
1355 | - |
1356 | - from charmhelper.core import hookenv, unitdata |
1357 | - |
1358 | - changes = unitdata.HookData() |
1359 | - db = unitdata.kv() |
1360 | - hooks = hookenv.Hooks() |
1361 | - |
1362 | - @hooks.hook |
1363 | - def config_changed(): |
1364 | - # View all changes to configuration |
1365 | - for changed, (prev, cur) in changes.conf.items(): |
1366 | - print('config changed', changed, |
1367 | - 'previous value', prev, |
1368 | - 'current value', cur) |
1369 | - |
1370 | - # Get some unit specific bookeeping |
1371 | - if not db.get('pkg_key'): |
1372 | - key = urllib.urlopen('https://example.com/pkg_key').read() |
1373 | - db.set('pkg_key', key) |
1374 | - |
1375 | - if __name__ == '__main__': |
1376 | - with changes(): |
1377 | - hook.execute() |
1378 | - |
1379 | - """ |
1380 | - def __init__(self): |
1381 | - self.kv = kv() |
1382 | - self.conf = None |
1383 | - self.rels = None |
1384 | - |
1385 | - @contextlib.contextmanager |
1386 | - def __call__(self): |
1387 | - from charmhelpers.core import hookenv |
1388 | - hook_name = hookenv.hook_name() |
1389 | - |
1390 | - with self.kv.hook_scope(hook_name): |
1391 | - self._record_charm_version(hookenv.charm_dir()) |
1392 | - delta_config, delta_relation = self._record_hook(hookenv) |
1393 | - yield self.kv, delta_config, delta_relation |
1394 | - |
1395 | - def _record_charm_version(self, charm_dir): |
1396 | - # Record revisions.. charm revisions are meaningless |
1397 | - # to charm authors as they don't control the revision. |
1398 | - # so logic dependnent on revision is not particularly |
1399 | - # useful, however it is useful for debugging analysis. |
1400 | - charm_rev = open( |
1401 | - os.path.join(charm_dir, 'revision')).read().strip() |
1402 | - charm_rev = charm_rev or '0' |
1403 | - revs = self.kv.get('charm_revisions', []) |
1404 | - if charm_rev not in revs: |
1405 | - revs.append(charm_rev.strip() or '0') |
1406 | - self.kv.set('charm_revisions', revs) |
1407 | - |
1408 | - def _record_hook(self, hookenv): |
1409 | - data = hookenv.execution_environment() |
1410 | - self.conf = conf_delta = self.kv.delta(data['conf'], 'config') |
1411 | - self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') |
1412 | - self.kv.set('env', data['env']) |
1413 | - self.kv.set('unit', data['unit']) |
1414 | - self.kv.set('relid', data.get('relid')) |
1415 | - return conf_delta, rels_delta |
1416 | - |
1417 | - |
1418 | -class Record(dict): |
1419 | - |
1420 | - __slots__ = () |
1421 | - |
1422 | - def __getattr__(self, k): |
1423 | - if k in self: |
1424 | - return self[k] |
1425 | - raise AttributeError(k) |
1426 | - |
1427 | - |
1428 | -class DeltaSet(Record): |
1429 | - |
1430 | - __slots__ = () |
1431 | - |
1432 | - |
1433 | -Delta = collections.namedtuple('Delta', ['previous', 'current']) |
1434 | - |
1435 | - |
1436 | -_KV = None |
1437 | - |
1438 | - |
1439 | -def kv(): |
1440 | - global _KV |
1441 | - if _KV is None: |
1442 | - _KV = Storage() |
1443 | - return _KV |
1444 | |
1445 | === modified file 'hooks/neutron_api_hooks.py' |
1446 | --- hooks/neutron_api_hooks.py 2015-02-13 15:02:52 +0000 |
1447 | +++ hooks/neutron_api_hooks.py 2015-02-26 04:23:37 +0000 |
1448 | @@ -382,7 +382,9 @@ |
1449 | hostname = nrpe.get_nagios_hostname() |
1450 | current_unit = nrpe.get_nagios_unit_name() |
1451 | nrpe_setup = nrpe.NRPE(hostname=hostname) |
1452 | + nrpe.copy_nrpe_checks() |
1453 | nrpe.add_init_service_checks(nrpe_setup, services(), current_unit) |
1454 | + nrpe.add_haproxy_checks(nrpe_setup, current_unit) |
1455 | nrpe_setup.write() |
1456 | |
1457 |
charm_lint_check #2220 neutron-api-next for brad-marshall mp250705
LINT OK: passed
Build: http:// 10.245. 162.77: 8080/job/ charm_lint_ check/2220/