Merge lp:~hopem/charms/trusty/nova-compute/charm-helpers-sync into lp:~openstack-charmers-archive/charms/trusty/nova-compute/next
- Trusty Tahr (14.04)
- charm-helpers-sync
- Merge into next
Proposed by
Edward Hope-Morley
Status: | Merged |
---|---|
Merged at revision: | 102 |
Proposed branch: | lp:~hopem/charms/trusty/nova-compute/charm-helpers-sync |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/nova-compute/next |
Diff against target: |
716 lines (+559/-28) 5 files modified
hooks/charmhelpers/contrib/network/ufw.py (+63/-15) hooks/charmhelpers/core/host.py (+5/-5) hooks/charmhelpers/core/sysctl.py (+11/-5) hooks/charmhelpers/core/templating.py (+3/-3) hooks/charmhelpers/core/unitdata.py (+477/-0) |
To merge this branch: | bzr merge lp:~hopem/charms/trusty/nova-compute/charm-helpers-sync |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Liam Young (community) | Approve | ||
Review via email: mp+249317@code.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #1717 nova-compute-next for hopem mp249317
UNIT OK: passed
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #1864 nova-compute-next for hopem mp249317
AMULET OK: passed
Build: http://
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'hooks/charmhelpers/contrib/network/ufw.py' | |||
2 | --- hooks/charmhelpers/contrib/network/ufw.py 2015-01-26 09:46:57 +0000 | |||
3 | +++ hooks/charmhelpers/contrib/network/ufw.py 2015-02-11 12:43:24 +0000 | |||
4 | @@ -46,6 +46,10 @@ | |||
5 | 46 | from charmhelpers.core import hookenv | 46 | from charmhelpers.core import hookenv |
6 | 47 | 47 | ||
7 | 48 | 48 | ||
8 | 49 | class UFWError(Exception): | ||
9 | 50 | pass | ||
10 | 51 | |||
11 | 52 | |||
12 | 49 | def is_enabled(): | 53 | def is_enabled(): |
13 | 50 | """ | 54 | """ |
14 | 51 | Check if `ufw` is enabled | 55 | Check if `ufw` is enabled |
15 | @@ -53,6 +57,7 @@ | |||
16 | 53 | :returns: True if ufw is enabled | 57 | :returns: True if ufw is enabled |
17 | 54 | """ | 58 | """ |
18 | 55 | output = subprocess.check_output(['ufw', 'status'], | 59 | output = subprocess.check_output(['ufw', 'status'], |
19 | 60 | universal_newlines=True, | ||
20 | 56 | env={'LANG': 'en_US', | 61 | env={'LANG': 'en_US', |
21 | 57 | 'PATH': os.environ['PATH']}) | 62 | 'PATH': os.environ['PATH']}) |
22 | 58 | 63 | ||
23 | @@ -61,6 +66,53 @@ | |||
24 | 61 | return len(m) >= 1 | 66 | return len(m) >= 1 |
25 | 62 | 67 | ||
26 | 63 | 68 | ||
27 | 69 | def is_ipv6_ok(): | ||
28 | 70 | """ | ||
29 | 71 | Check if IPv6 support is present and ip6tables functional | ||
30 | 72 | |||
31 | 73 | :returns: True if IPv6 is working, False otherwise | ||
32 | 74 | """ | ||
33 | 75 | |||
34 | 76 | # do we have IPv6 in the machine? | ||
35 | 77 | if os.path.isdir('/proc/sys/net/ipv6'): | ||
36 | 78 | # is ip6tables kernel module loaded? | ||
37 | 79 | lsmod = subprocess.check_output(['lsmod'], universal_newlines=True) | ||
38 | 80 | matches = re.findall('^ip6_tables[ ]+', lsmod, re.M) | ||
39 | 81 | if len(matches) == 0: | ||
40 | 82 | # ip6tables support isn't complete, let's try to load it | ||
41 | 83 | try: | ||
42 | 84 | subprocess.check_output(['modprobe', 'ip6_tables'], | ||
43 | 85 | universal_newlines=True) | ||
44 | 86 | # great, we could load the module | ||
45 | 87 | return True | ||
46 | 88 | except subprocess.CalledProcessError as ex: | ||
47 | 89 | hookenv.log("Couldn't load ip6_tables module: %s" % ex.output, | ||
48 | 90 | level="WARN") | ||
49 | 91 | # we are in a world where ip6tables isn't working | ||
50 | 92 | # so we inform that the machine doesn't have IPv6 | ||
51 | 93 | return False | ||
52 | 94 | else: | ||
53 | 95 | # the module is present :) | ||
54 | 96 | return True | ||
55 | 97 | |||
56 | 98 | else: | ||
57 | 99 | # the system doesn't have IPv6 | ||
58 | 100 | return False | ||
59 | 101 | |||
60 | 102 | |||
61 | 103 | def disable_ipv6(): | ||
62 | 104 | """ | ||
63 | 105 | Disable ufw IPv6 support in /etc/default/ufw | ||
64 | 106 | """ | ||
65 | 107 | exit_code = subprocess.call(['sed', '-i', 's/IPV6=.*/IPV6=no/g', | ||
66 | 108 | '/etc/default/ufw']) | ||
67 | 109 | if exit_code == 0: | ||
68 | 110 | hookenv.log('IPv6 support in ufw disabled', level='INFO') | ||
69 | 111 | else: | ||
70 | 112 | hookenv.log("Couldn't disable IPv6 support in ufw", level="ERROR") | ||
71 | 113 | raise UFWError("Couldn't disable IPv6 support in ufw") | ||
72 | 114 | |||
73 | 115 | |||
74 | 64 | def enable(): | 116 | def enable(): |
75 | 65 | """ | 117 | """ |
76 | 66 | Enable ufw | 118 | Enable ufw |
77 | @@ -70,18 +122,11 @@ | |||
78 | 70 | if is_enabled(): | 122 | if is_enabled(): |
79 | 71 | return True | 123 | return True |
80 | 72 | 124 | ||
91 | 73 | if not os.path.isdir('/proc/sys/net/ipv6'): | 125 | if not is_ipv6_ok(): |
92 | 74 | # disable IPv6 support in ufw | 126 | disable_ipv6() |
83 | 75 | hookenv.log("This machine doesn't have IPv6 enabled", level="INFO") | ||
84 | 76 | exit_code = subprocess.call(['sed', '-i', 's/IPV6=yes/IPV6=no/g', | ||
85 | 77 | '/etc/default/ufw']) | ||
86 | 78 | if exit_code == 0: | ||
87 | 79 | hookenv.log('IPv6 support in ufw disabled', level='INFO') | ||
88 | 80 | else: | ||
89 | 81 | hookenv.log("Couldn't disable IPv6 support in ufw", level="ERROR") | ||
90 | 82 | raise Exception("Couldn't disable IPv6 support in ufw") | ||
93 | 83 | 127 | ||
94 | 84 | output = subprocess.check_output(['ufw', 'enable'], | 128 | output = subprocess.check_output(['ufw', 'enable'], |
95 | 129 | universal_newlines=True, | ||
96 | 85 | env={'LANG': 'en_US', | 130 | env={'LANG': 'en_US', |
97 | 86 | 'PATH': os.environ['PATH']}) | 131 | 'PATH': os.environ['PATH']}) |
98 | 87 | 132 | ||
99 | @@ -107,6 +152,7 @@ | |||
100 | 107 | return True | 152 | return True |
101 | 108 | 153 | ||
102 | 109 | output = subprocess.check_output(['ufw', 'disable'], | 154 | output = subprocess.check_output(['ufw', 'disable'], |
103 | 155 | universal_newlines=True, | ||
104 | 110 | env={'LANG': 'en_US', | 156 | env={'LANG': 'en_US', |
105 | 111 | 'PATH': os.environ['PATH']}) | 157 | 'PATH': os.environ['PATH']}) |
106 | 112 | 158 | ||
107 | @@ -151,7 +197,7 @@ | |||
108 | 151 | cmd += ['to', dst] | 197 | cmd += ['to', dst] |
109 | 152 | 198 | ||
110 | 153 | if port is not None: | 199 | if port is not None: |
112 | 154 | cmd += ['port', port] | 200 | cmd += ['port', str(port)] |
113 | 155 | 201 | ||
114 | 156 | if proto is not None: | 202 | if proto is not None: |
115 | 157 | cmd += ['proto', proto] | 203 | cmd += ['proto', proto] |
116 | @@ -208,9 +254,11 @@ | |||
117 | 208 | :param action: `open` or `close` | 254 | :param action: `open` or `close` |
118 | 209 | """ | 255 | """ |
119 | 210 | if action == 'open': | 256 | if action == 'open': |
121 | 211 | subprocess.check_output(['ufw', 'allow', name]) | 257 | subprocess.check_output(['ufw', 'allow', str(name)], |
122 | 258 | universal_newlines=True) | ||
123 | 212 | elif action == 'close': | 259 | elif action == 'close': |
125 | 213 | subprocess.check_output(['ufw', 'delete', 'allow', name]) | 260 | subprocess.check_output(['ufw', 'delete', 'allow', str(name)], |
126 | 261 | universal_newlines=True) | ||
127 | 214 | else: | 262 | else: |
130 | 215 | raise Exception(("'{}' not supported, use 'allow' " | 263 | raise UFWError(("'{}' not supported, use 'allow' " |
131 | 216 | "or 'delete'").format(action)) | 264 | "or 'delete'").format(action)) |
132 | 217 | 265 | ||
133 | === modified file 'hooks/charmhelpers/core/host.py' | |||
134 | --- hooks/charmhelpers/core/host.py 2015-01-26 09:46:57 +0000 | |||
135 | +++ hooks/charmhelpers/core/host.py 2015-02-11 12:43:24 +0000 | |||
136 | @@ -191,11 +191,11 @@ | |||
137 | 191 | 191 | ||
138 | 192 | 192 | ||
139 | 193 | def write_file(path, content, owner='root', group='root', perms=0o444): | 193 | def write_file(path, content, owner='root', group='root', perms=0o444): |
141 | 194 | """Create or overwrite a file with the contents of a string""" | 194 | """Create or overwrite a file with the contents of a byte string.""" |
142 | 195 | log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) | 195 | log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) |
143 | 196 | uid = pwd.getpwnam(owner).pw_uid | 196 | uid = pwd.getpwnam(owner).pw_uid |
144 | 197 | gid = grp.getgrnam(group).gr_gid | 197 | gid = grp.getgrnam(group).gr_gid |
146 | 198 | with open(path, 'w') as target: | 198 | with open(path, 'wb') as target: |
147 | 199 | os.fchown(target.fileno(), uid, gid) | 199 | os.fchown(target.fileno(), uid, gid) |
148 | 200 | os.fchmod(target.fileno(), perms) | 200 | os.fchmod(target.fileno(), perms) |
149 | 201 | target.write(content) | 201 | target.write(content) |
150 | @@ -305,11 +305,11 @@ | |||
151 | 305 | ceph_client_changed function. | 305 | ceph_client_changed function. |
152 | 306 | """ | 306 | """ |
153 | 307 | def wrap(f): | 307 | def wrap(f): |
155 | 308 | def wrapped_f(*args): | 308 | def wrapped_f(*args, **kwargs): |
156 | 309 | checksums = {} | 309 | checksums = {} |
157 | 310 | for path in restart_map: | 310 | for path in restart_map: |
158 | 311 | checksums[path] = file_hash(path) | 311 | checksums[path] = file_hash(path) |
160 | 312 | f(*args) | 312 | f(*args, **kwargs) |
161 | 313 | restarts = [] | 313 | restarts = [] |
162 | 314 | for path in restart_map: | 314 | for path in restart_map: |
163 | 315 | if checksums[path] != file_hash(path): | 315 | if checksums[path] != file_hash(path): |
164 | @@ -361,7 +361,7 @@ | |||
165 | 361 | ip_output = (line for line in ip_output if line) | 361 | ip_output = (line for line in ip_output if line) |
166 | 362 | for line in ip_output: | 362 | for line in ip_output: |
167 | 363 | if line.split()[1].startswith(int_type): | 363 | if line.split()[1].startswith(int_type): |
169 | 364 | matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line) | 364 | matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line) |
170 | 365 | if matched: | 365 | if matched: |
171 | 366 | interface = matched.groups()[0] | 366 | interface = matched.groups()[0] |
172 | 367 | else: | 367 | else: |
173 | 368 | 368 | ||
174 | === modified file 'hooks/charmhelpers/core/sysctl.py' | |||
175 | --- hooks/charmhelpers/core/sysctl.py 2015-01-26 09:46:57 +0000 | |||
176 | +++ hooks/charmhelpers/core/sysctl.py 2015-02-11 12:43:24 +0000 | |||
177 | @@ -26,25 +26,31 @@ | |||
178 | 26 | from charmhelpers.core.hookenv import ( | 26 | from charmhelpers.core.hookenv import ( |
179 | 27 | log, | 27 | log, |
180 | 28 | DEBUG, | 28 | DEBUG, |
181 | 29 | ERROR, | ||
182 | 29 | ) | 30 | ) |
183 | 30 | 31 | ||
184 | 31 | 32 | ||
185 | 32 | def create(sysctl_dict, sysctl_file): | 33 | def create(sysctl_dict, sysctl_file): |
186 | 33 | """Creates a sysctl.conf file from a YAML associative array | 34 | """Creates a sysctl.conf file from a YAML associative array |
187 | 34 | 35 | ||
190 | 35 | :param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 } | 36 | :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" |
191 | 36 | :type sysctl_dict: dict | 37 | :type sysctl_dict: str |
192 | 37 | :param sysctl_file: path to the sysctl file to be saved | 38 | :param sysctl_file: path to the sysctl file to be saved |
193 | 38 | :type sysctl_file: str or unicode | 39 | :type sysctl_file: str or unicode |
194 | 39 | :returns: None | 40 | :returns: None |
195 | 40 | """ | 41 | """ |
197 | 41 | sysctl_dict = yaml.load(sysctl_dict) | 42 | try: |
198 | 43 | sysctl_dict_parsed = yaml.safe_load(sysctl_dict) | ||
199 | 44 | except yaml.YAMLError: | ||
200 | 45 | log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), | ||
201 | 46 | level=ERROR) | ||
202 | 47 | return | ||
203 | 42 | 48 | ||
204 | 43 | with open(sysctl_file, "w") as fd: | 49 | with open(sysctl_file, "w") as fd: |
206 | 44 | for key, value in sysctl_dict.items(): | 50 | for key, value in sysctl_dict_parsed.items(): |
207 | 45 | fd.write("{}={}\n".format(key, value)) | 51 | fd.write("{}={}\n".format(key, value)) |
208 | 46 | 52 | ||
210 | 47 | log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict), | 53 | log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed), |
211 | 48 | level=DEBUG) | 54 | level=DEBUG) |
212 | 49 | 55 | ||
213 | 50 | check_call(["sysctl", "-p", sysctl_file]) | 56 | check_call(["sysctl", "-p", sysctl_file]) |
214 | 51 | 57 | ||
215 | === modified file 'hooks/charmhelpers/core/templating.py' | |||
216 | --- hooks/charmhelpers/core/templating.py 2015-01-26 09:46:57 +0000 | |||
217 | +++ hooks/charmhelpers/core/templating.py 2015-02-11 12:43:24 +0000 | |||
218 | @@ -21,7 +21,7 @@ | |||
219 | 21 | 21 | ||
220 | 22 | 22 | ||
221 | 23 | def render(source, target, context, owner='root', group='root', | 23 | def render(source, target, context, owner='root', group='root', |
223 | 24 | perms=0o444, templates_dir=None): | 24 | perms=0o444, templates_dir=None, encoding='UTF-8'): |
224 | 25 | """ | 25 | """ |
225 | 26 | Render a template. | 26 | Render a template. |
226 | 27 | 27 | ||
227 | @@ -64,5 +64,5 @@ | |||
228 | 64 | level=hookenv.ERROR) | 64 | level=hookenv.ERROR) |
229 | 65 | raise e | 65 | raise e |
230 | 66 | content = template.render(context) | 66 | content = template.render(context) |
233 | 67 | host.mkdir(os.path.dirname(target), owner, group) | 67 | host.mkdir(os.path.dirname(target), owner, group, perms=0o755) |
234 | 68 | host.write_file(target, content, owner, group, perms) | 68 | host.write_file(target, content.encode(encoding), owner, group, perms) |
235 | 69 | 69 | ||
236 | === added file 'hooks/charmhelpers/core/unitdata.py' | |||
237 | --- hooks/charmhelpers/core/unitdata.py 1970-01-01 00:00:00 +0000 | |||
238 | +++ hooks/charmhelpers/core/unitdata.py 2015-02-11 12:43:24 +0000 | |||
239 | @@ -0,0 +1,477 @@ | |||
240 | 1 | #!/usr/bin/env python | ||
241 | 2 | # -*- coding: utf-8 -*- | ||
242 | 3 | # | ||
243 | 4 | # Copyright 2014-2015 Canonical Limited. | ||
244 | 5 | # | ||
245 | 6 | # This file is part of charm-helpers. | ||
246 | 7 | # | ||
247 | 8 | # charm-helpers is free software: you can redistribute it and/or modify | ||
248 | 9 | # it under the terms of the GNU Lesser General Public License version 3 as | ||
249 | 10 | # published by the Free Software Foundation. | ||
250 | 11 | # | ||
251 | 12 | # charm-helpers is distributed in the hope that it will be useful, | ||
252 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
253 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
254 | 15 | # GNU Lesser General Public License for more details. | ||
255 | 16 | # | ||
256 | 17 | # You should have received a copy of the GNU Lesser General Public License | ||
257 | 18 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. | ||
258 | 19 | # | ||
259 | 20 | # | ||
260 | 21 | # Authors: | ||
261 | 22 | # Kapil Thangavelu <kapil.foss@gmail.com> | ||
262 | 23 | # | ||
263 | 24 | """ | ||
264 | 25 | Intro | ||
265 | 26 | ----- | ||
266 | 27 | |||
267 | 28 | A simple way to store state in units. This provides a key value | ||
268 | 29 | storage with support for versioned, transactional operation, | ||
269 | 30 | and can calculate deltas from previous values to simplify unit logic | ||
270 | 31 | when processing changes. | ||
271 | 32 | |||
272 | 33 | |||
273 | 34 | Hook Integration | ||
274 | 35 | ---------------- | ||
275 | 36 | |||
276 | 37 | There are several extant frameworks for hook execution, including | ||
277 | 38 | |||
278 | 39 | - charmhelpers.core.hookenv.Hooks | ||
279 | 40 | - charmhelpers.core.services.ServiceManager | ||
280 | 41 | |||
281 | 42 | The storage classes are framework agnostic, one simple integration is | ||
282 | 43 | via the HookData contextmanager. It will record the current hook | ||
283 | 44 | execution environment (including relation data, config data, etc.), | ||
284 | 45 | setup a transaction and allow easy access to the changes from | ||
285 | 46 | previously seen values. One consequence of the integration is the | ||
286 | 47 | reservation of particular keys ('rels', 'unit', 'env', 'config', | ||
287 | 48 | 'charm_revisions') for their respective values. | ||
288 | 49 | |||
289 | 50 | Here's a fully worked integration example using hookenv.Hooks:: | ||
290 | 51 | |||
291 | 52 | from charmhelper.core import hookenv, unitdata | ||
292 | 53 | |||
293 | 54 | hook_data = unitdata.HookData() | ||
294 | 55 | db = unitdata.kv() | ||
295 | 56 | hooks = hookenv.Hooks() | ||
296 | 57 | |||
297 | 58 | @hooks.hook | ||
298 | 59 | def config_changed(): | ||
299 | 60 | # Print all changes to configuration from previously seen | ||
300 | 61 | # values. | ||
301 | 62 | for changed, (prev, cur) in hook_data.conf.items(): | ||
302 | 63 | print('config changed', changed, | ||
303 | 64 | 'previous value', prev, | ||
304 | 65 | 'current value', cur) | ||
305 | 66 | |||
306 | 67 | # Get some unit specific bookeeping | ||
307 | 68 | if not db.get('pkg_key'): | ||
308 | 69 | key = urllib.urlopen('https://example.com/pkg_key').read() | ||
309 | 70 | db.set('pkg_key', key) | ||
310 | 71 | |||
311 | 72 | # Directly access all charm config as a mapping. | ||
312 | 73 | conf = db.getrange('config', True) | ||
313 | 74 | |||
314 | 75 | # Directly access all relation data as a mapping | ||
315 | 76 | rels = db.getrange('rels', True) | ||
316 | 77 | |||
317 | 78 | if __name__ == '__main__': | ||
318 | 79 | with hook_data(): | ||
319 | 80 | hook.execute() | ||
320 | 81 | |||
321 | 82 | |||
322 | 83 | A more basic integration is via the hook_scope context manager which simply | ||
323 | 84 | manages transaction scope (and records hook name, and timestamp):: | ||
324 | 85 | |||
325 | 86 | >>> from unitdata import kv | ||
326 | 87 | >>> db = kv() | ||
327 | 88 | >>> with db.hook_scope('install'): | ||
328 | 89 | ... # do work, in transactional scope. | ||
329 | 90 | ... db.set('x', 1) | ||
330 | 91 | >>> db.get('x') | ||
331 | 92 | 1 | ||
332 | 93 | |||
333 | 94 | |||
334 | 95 | Usage | ||
335 | 96 | ----- | ||
336 | 97 | |||
337 | 98 | Values are automatically json de/serialized to preserve basic typing | ||
338 | 99 | and complex data struct capabilities (dicts, lists, ints, booleans, etc). | ||
339 | 100 | |||
340 | 101 | Individual values can be manipulated via get/set:: | ||
341 | 102 | |||
342 | 103 | >>> kv.set('y', True) | ||
343 | 104 | >>> kv.get('y') | ||
344 | 105 | True | ||
345 | 106 | |||
346 | 107 | # We can set complex values (dicts, lists) as a single key. | ||
347 | 108 | >>> kv.set('config', {'a': 1, 'b': True'}) | ||
348 | 109 | |||
349 | 110 | # Also supports returning dictionaries as a record which | ||
350 | 111 | # provides attribute access. | ||
351 | 112 | >>> config = kv.get('config', record=True) | ||
352 | 113 | >>> config.b | ||
353 | 114 | True | ||
354 | 115 | |||
355 | 116 | |||
356 | 117 | Groups of keys can be manipulated with update/getrange:: | ||
357 | 118 | |||
358 | 119 | >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") | ||
359 | 120 | >>> kv.getrange('gui.', strip=True) | ||
360 | 121 | {'z': 1, 'y': 2} | ||
361 | 122 | |||
362 | 123 | When updating values, its very helpful to understand which values | ||
363 | 124 | have actually changed and how have they changed. The storage | ||
364 | 125 | provides a delta method to provide for this:: | ||
365 | 126 | |||
366 | 127 | >>> data = {'debug': True, 'option': 2} | ||
367 | 128 | >>> delta = kv.delta(data, 'config.') | ||
368 | 129 | >>> delta.debug.previous | ||
369 | 130 | None | ||
370 | 131 | >>> delta.debug.current | ||
371 | 132 | True | ||
372 | 133 | >>> delta | ||
373 | 134 | {'debug': (None, True), 'option': (None, 2)} | ||
374 | 135 | |||
375 | 136 | Note the delta method does not persist the actual change, it needs to | ||
376 | 137 | be explicitly saved via 'update' method:: | ||
377 | 138 | |||
378 | 139 | >>> kv.update(data, 'config.') | ||
379 | 140 | |||
380 | 141 | Values modified in the context of a hook scope retain historical values | ||
381 | 142 | associated to the hookname. | ||
382 | 143 | |||
383 | 144 | >>> with db.hook_scope('config-changed'): | ||
384 | 145 | ... db.set('x', 42) | ||
385 | 146 | >>> db.gethistory('x') | ||
386 | 147 | [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), | ||
387 | 148 | (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] | ||
388 | 149 | |||
389 | 150 | """ | ||
390 | 151 | |||
391 | 152 | import collections | ||
392 | 153 | import contextlib | ||
393 | 154 | import datetime | ||
394 | 155 | import json | ||
395 | 156 | import os | ||
396 | 157 | import pprint | ||
397 | 158 | import sqlite3 | ||
398 | 159 | import sys | ||
399 | 160 | |||
400 | 161 | __author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>' | ||
401 | 162 | |||
402 | 163 | |||
403 | 164 | class Storage(object): | ||
404 | 165 | """Simple key value database for local unit state within charms. | ||
405 | 166 | |||
406 | 167 | Modifications are automatically committed at hook exit. That's | ||
407 | 168 | currently regardless of exit code. | ||
408 | 169 | |||
409 | 170 | To support dicts, lists, integer, floats, and booleans values | ||
410 | 171 | are automatically json encoded/decoded. | ||
411 | 172 | """ | ||
412 | 173 | def __init__(self, path=None): | ||
413 | 174 | self.db_path = path | ||
414 | 175 | if path is None: | ||
415 | 176 | self.db_path = os.path.join( | ||
416 | 177 | os.environ.get('CHARM_DIR', ''), '.unit-state.db') | ||
417 | 178 | self.conn = sqlite3.connect('%s' % self.db_path) | ||
418 | 179 | self.cursor = self.conn.cursor() | ||
419 | 180 | self.revision = None | ||
420 | 181 | self._closed = False | ||
421 | 182 | self._init() | ||
422 | 183 | |||
423 | 184 | def close(self): | ||
424 | 185 | if self._closed: | ||
425 | 186 | return | ||
426 | 187 | self.flush(False) | ||
427 | 188 | self.cursor.close() | ||
428 | 189 | self.conn.close() | ||
429 | 190 | self._closed = True | ||
430 | 191 | |||
431 | 192 | def _scoped_query(self, stmt, params=None): | ||
432 | 193 | if params is None: | ||
433 | 194 | params = [] | ||
434 | 195 | return stmt, params | ||
435 | 196 | |||
436 | 197 | def get(self, key, default=None, record=False): | ||
437 | 198 | self.cursor.execute( | ||
438 | 199 | *self._scoped_query( | ||
439 | 200 | 'select data from kv where key=?', [key])) | ||
440 | 201 | result = self.cursor.fetchone() | ||
441 | 202 | if not result: | ||
442 | 203 | return default | ||
443 | 204 | if record: | ||
444 | 205 | return Record(json.loads(result[0])) | ||
445 | 206 | return json.loads(result[0]) | ||
446 | 207 | |||
447 | 208 | def getrange(self, key_prefix, strip=False): | ||
448 | 209 | stmt = "select key, data from kv where key like '%s%%'" % key_prefix | ||
449 | 210 | self.cursor.execute(*self._scoped_query(stmt)) | ||
450 | 211 | result = self.cursor.fetchall() | ||
451 | 212 | |||
452 | 213 | if not result: | ||
453 | 214 | return None | ||
454 | 215 | if not strip: | ||
455 | 216 | key_prefix = '' | ||
456 | 217 | return dict([ | ||
457 | 218 | (k[len(key_prefix):], json.loads(v)) for k, v in result]) | ||
458 | 219 | |||
459 | 220 | def update(self, mapping, prefix=""): | ||
460 | 221 | for k, v in mapping.items(): | ||
461 | 222 | self.set("%s%s" % (prefix, k), v) | ||
462 | 223 | |||
463 | 224 | def unset(self, key): | ||
464 | 225 | self.cursor.execute('delete from kv where key=?', [key]) | ||
465 | 226 | if self.revision and self.cursor.rowcount: | ||
466 | 227 | self.cursor.execute( | ||
467 | 228 | 'insert into kv_revisions values (?, ?, ?)', | ||
468 | 229 | [key, self.revision, json.dumps('DELETED')]) | ||
469 | 230 | |||
470 | 231 | def set(self, key, value): | ||
471 | 232 | serialized = json.dumps(value) | ||
472 | 233 | |||
473 | 234 | self.cursor.execute( | ||
474 | 235 | 'select data from kv where key=?', [key]) | ||
475 | 236 | exists = self.cursor.fetchone() | ||
476 | 237 | |||
477 | 238 | # Skip mutations to the same value | ||
478 | 239 | if exists: | ||
479 | 240 | if exists[0] == serialized: | ||
480 | 241 | return value | ||
481 | 242 | |||
482 | 243 | if not exists: | ||
483 | 244 | self.cursor.execute( | ||
484 | 245 | 'insert into kv (key, data) values (?, ?)', | ||
485 | 246 | (key, serialized)) | ||
486 | 247 | else: | ||
487 | 248 | self.cursor.execute(''' | ||
488 | 249 | update kv | ||
489 | 250 | set data = ? | ||
490 | 251 | where key = ?''', [serialized, key]) | ||
491 | 252 | |||
492 | 253 | # Save | ||
493 | 254 | if not self.revision: | ||
494 | 255 | return value | ||
495 | 256 | |||
496 | 257 | self.cursor.execute( | ||
497 | 258 | 'select 1 from kv_revisions where key=? and revision=?', | ||
498 | 259 | [key, self.revision]) | ||
499 | 260 | exists = self.cursor.fetchone() | ||
500 | 261 | |||
501 | 262 | if not exists: | ||
502 | 263 | self.cursor.execute( | ||
503 | 264 | '''insert into kv_revisions ( | ||
504 | 265 | revision, key, data) values (?, ?, ?)''', | ||
505 | 266 | (self.revision, key, serialized)) | ||
506 | 267 | else: | ||
507 | 268 | self.cursor.execute( | ||
508 | 269 | ''' | ||
509 | 270 | update kv_revisions | ||
510 | 271 | set data = ? | ||
511 | 272 | where key = ? | ||
512 | 273 | and revision = ?''', | ||
513 | 274 | [serialized, key, self.revision]) | ||
514 | 275 | |||
515 | 276 | return value | ||
516 | 277 | |||
517 | 278 | def delta(self, mapping, prefix): | ||
518 | 279 | """ | ||
519 | 280 | return a delta containing values that have changed. | ||
520 | 281 | """ | ||
521 | 282 | previous = self.getrange(prefix, strip=True) | ||
522 | 283 | if not previous: | ||
523 | 284 | pk = set() | ||
524 | 285 | else: | ||
525 | 286 | pk = set(previous.keys()) | ||
526 | 287 | ck = set(mapping.keys()) | ||
527 | 288 | delta = DeltaSet() | ||
528 | 289 | |||
529 | 290 | # added | ||
530 | 291 | for k in ck.difference(pk): | ||
531 | 292 | delta[k] = Delta(None, mapping[k]) | ||
532 | 293 | |||
533 | 294 | # removed | ||
534 | 295 | for k in pk.difference(ck): | ||
535 | 296 | delta[k] = Delta(previous[k], None) | ||
536 | 297 | |||
537 | 298 | # changed | ||
538 | 299 | for k in pk.intersection(ck): | ||
539 | 300 | c = mapping[k] | ||
540 | 301 | p = previous[k] | ||
541 | 302 | if c != p: | ||
542 | 303 | delta[k] = Delta(p, c) | ||
543 | 304 | |||
544 | 305 | return delta | ||
545 | 306 | |||
546 | 307 | @contextlib.contextmanager | ||
547 | 308 | def hook_scope(self, name=""): | ||
548 | 309 | """Scope all future interactions to the current hook execution | ||
549 | 310 | revision.""" | ||
550 | 311 | assert not self.revision | ||
551 | 312 | self.cursor.execute( | ||
552 | 313 | 'insert into hooks (hook, date) values (?, ?)', | ||
553 | 314 | (name or sys.argv[0], | ||
554 | 315 | datetime.datetime.utcnow().isoformat())) | ||
555 | 316 | self.revision = self.cursor.lastrowid | ||
556 | 317 | try: | ||
557 | 318 | yield self.revision | ||
558 | 319 | self.revision = None | ||
559 | 320 | except: | ||
560 | 321 | self.flush(False) | ||
561 | 322 | self.revision = None | ||
562 | 323 | raise | ||
563 | 324 | else: | ||
564 | 325 | self.flush() | ||
565 | 326 | |||
566 | 327 | def flush(self, save=True): | ||
567 | 328 | if save: | ||
568 | 329 | self.conn.commit() | ||
569 | 330 | elif self._closed: | ||
570 | 331 | return | ||
571 | 332 | else: | ||
572 | 333 | self.conn.rollback() | ||
573 | 334 | |||
574 | 335 | def _init(self): | ||
575 | 336 | self.cursor.execute(''' | ||
576 | 337 | create table if not exists kv ( | ||
577 | 338 | key text, | ||
578 | 339 | data text, | ||
579 | 340 | primary key (key) | ||
580 | 341 | )''') | ||
581 | 342 | self.cursor.execute(''' | ||
582 | 343 | create table if not exists kv_revisions ( | ||
583 | 344 | key text, | ||
584 | 345 | revision integer, | ||
585 | 346 | data text, | ||
586 | 347 | primary key (key, revision) | ||
587 | 348 | )''') | ||
588 | 349 | self.cursor.execute(''' | ||
589 | 350 | create table if not exists hooks ( | ||
590 | 351 | version integer primary key autoincrement, | ||
591 | 352 | hook text, | ||
592 | 353 | date text | ||
593 | 354 | )''') | ||
594 | 355 | self.conn.commit() | ||
595 | 356 | |||
596 | 357 | def gethistory(self, key, deserialize=False): | ||
597 | 358 | self.cursor.execute( | ||
598 | 359 | ''' | ||
599 | 360 | select kv.revision, kv.key, kv.data, h.hook, h.date | ||
600 | 361 | from kv_revisions kv, | ||
601 | 362 | hooks h | ||
602 | 363 | where kv.key=? | ||
603 | 364 | and kv.revision = h.version | ||
604 | 365 | ''', [key]) | ||
605 | 366 | if deserialize is False: | ||
606 | 367 | return self.cursor.fetchall() | ||
607 | 368 | return map(_parse_history, self.cursor.fetchall()) | ||
608 | 369 | |||
609 | 370 | def debug(self, fh=sys.stderr): | ||
610 | 371 | self.cursor.execute('select * from kv') | ||
611 | 372 | pprint.pprint(self.cursor.fetchall(), stream=fh) | ||
612 | 373 | self.cursor.execute('select * from kv_revisions') | ||
613 | 374 | pprint.pprint(self.cursor.fetchall(), stream=fh) | ||
614 | 375 | |||
615 | 376 | |||
616 | 377 | def _parse_history(d): | ||
617 | 378 | return (d[0], d[1], json.loads(d[2]), d[3], | ||
618 | 379 | datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) | ||
619 | 380 | |||
620 | 381 | |||
621 | 382 | class HookData(object): | ||
622 | 383 | """Simple integration for existing hook exec frameworks. | ||
623 | 384 | |||
624 | 385 | Records all unit information, and stores deltas for processing | ||
625 | 386 | by the hook. | ||
626 | 387 | |||
627 | 388 | Sample:: | ||
628 | 389 | |||
629 | 390 | from charmhelper.core import hookenv, unitdata | ||
630 | 391 | |||
631 | 392 | changes = unitdata.HookData() | ||
632 | 393 | db = unitdata.kv() | ||
633 | 394 | hooks = hookenv.Hooks() | ||
634 | 395 | |||
635 | 396 | @hooks.hook | ||
636 | 397 | def config_changed(): | ||
637 | 398 | # View all changes to configuration | ||
638 | 399 | for changed, (prev, cur) in changes.conf.items(): | ||
639 | 400 | print('config changed', changed, | ||
640 | 401 | 'previous value', prev, | ||
641 | 402 | 'current value', cur) | ||
642 | 403 | |||
643 | 404 | # Get some unit specific bookeeping | ||
644 | 405 | if not db.get('pkg_key'): | ||
645 | 406 | key = urllib.urlopen('https://example.com/pkg_key').read() | ||
646 | 407 | db.set('pkg_key', key) | ||
647 | 408 | |||
648 | 409 | if __name__ == '__main__': | ||
649 | 410 | with changes(): | ||
650 | 411 | hook.execute() | ||
651 | 412 | |||
652 | 413 | """ | ||
653 | 414 | def __init__(self): | ||
654 | 415 | self.kv = kv() | ||
655 | 416 | self.conf = None | ||
656 | 417 | self.rels = None | ||
657 | 418 | |||
658 | 419 | @contextlib.contextmanager | ||
659 | 420 | def __call__(self): | ||
660 | 421 | from charmhelpers.core import hookenv | ||
661 | 422 | hook_name = hookenv.hook_name() | ||
662 | 423 | |||
663 | 424 | with self.kv.hook_scope(hook_name): | ||
664 | 425 | self._record_charm_version(hookenv.charm_dir()) | ||
665 | 426 | delta_config, delta_relation = self._record_hook(hookenv) | ||
666 | 427 | yield self.kv, delta_config, delta_relation | ||
667 | 428 | |||
668 | 429 | def _record_charm_version(self, charm_dir): | ||
669 | 430 | # Record revisions.. charm revisions are meaningless | ||
670 | 431 | # to charm authors as they don't control the revision. | ||
671 | 432 | # so logic dependnent on revision is not particularly | ||
672 | 433 | # useful, however it is useful for debugging analysis. | ||
673 | 434 | charm_rev = open( | ||
674 | 435 | os.path.join(charm_dir, 'revision')).read().strip() | ||
675 | 436 | charm_rev = charm_rev or '0' | ||
676 | 437 | revs = self.kv.get('charm_revisions', []) | ||
677 | 438 | if not charm_rev in revs: | ||
678 | 439 | revs.append(charm_rev.strip() or '0') | ||
679 | 440 | self.kv.set('charm_revisions', revs) | ||
680 | 441 | |||
681 | 442 | def _record_hook(self, hookenv): | ||
682 | 443 | data = hookenv.execution_environment() | ||
683 | 444 | self.conf = conf_delta = self.kv.delta(data['conf'], 'config') | ||
684 | 445 | self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') | ||
685 | 446 | self.kv.set('env', data['env']) | ||
686 | 447 | self.kv.set('unit', data['unit']) | ||
687 | 448 | self.kv.set('relid', data.get('relid')) | ||
688 | 449 | return conf_delta, rels_delta | ||
689 | 450 | |||
690 | 451 | |||
691 | 452 | class Record(dict): | ||
692 | 453 | |||
693 | 454 | __slots__ = () | ||
694 | 455 | |||
695 | 456 | def __getattr__(self, k): | ||
696 | 457 | if k in self: | ||
697 | 458 | return self[k] | ||
698 | 459 | raise AttributeError(k) | ||
699 | 460 | |||
700 | 461 | |||
701 | 462 | class DeltaSet(Record): | ||
702 | 463 | |||
703 | 464 | __slots__ = () | ||
704 | 465 | |||
705 | 466 | |||
706 | 467 | Delta = collections.namedtuple('Delta', ['previous', 'current']) | ||
707 | 468 | |||
708 | 469 | |||
709 | 470 | _KV = None | ||
710 | 471 | |||
711 | 472 | |||
712 | 473 | def kv(): | ||
713 | 474 | global _KV | ||
714 | 475 | if _KV is None: | ||
715 | 476 | _KV = Storage() | ||
716 | 477 | return _KV |
charm_lint_check #1886 nova-compute-next for hopem mp249317
LINT OK: passed
Build: http:// 10.245. 162.77: 8080/job/ charm_lint_ check/1886/