Merge lp:~hopem/charms/trusty/percona-cluster/fix-mysql-helper-passwd-backcompat into lp:~openstack-charmers-archive/charms/trusty/percona-cluster/trunk
- Trusty Tahr (14.04)
- fix-mysql-helper-passwd-backcompat
- Merge into trunk
Proposed by
Edward Hope-Morley
Status: | Merged | ||||
---|---|---|---|---|---|
Merged at revision: | 46 | ||||
Proposed branch: | lp:~hopem/charms/trusty/percona-cluster/fix-mysql-helper-passwd-backcompat | ||||
Merge into: | lp:~openstack-charmers-archive/charms/trusty/percona-cluster/trunk | ||||
Diff against target: |
793 lines (+584/-31) 9 files modified
hooks/charmhelpers/contrib/database/mysql.py (+41/-11) hooks/charmhelpers/contrib/hahelpers/cluster.py (+5/-1) hooks/charmhelpers/core/fstab.py (+4/-4) hooks/charmhelpers/core/host.py (+2/-2) hooks/charmhelpers/core/strutils.py (+42/-0) hooks/charmhelpers/core/sysctl.py (+2/-2) hooks/charmhelpers/core/unitdata.py (+477/-0) hooks/charmhelpers/fetch/archiveurl.py (+10/-10) hooks/charmhelpers/fetch/giturl.py (+1/-1) |
||||
To merge this branch: | bzr merge lp:~hopem/charms/trusty/percona-cluster/fix-mysql-helper-passwd-backcompat | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Liam Young (community) | Approve | ||
Jorge Niedbalski (community) | Approve | ||
Review via email: mp+250994@code.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'hooks/charmhelpers/contrib/database/mysql.py' |
2 | --- hooks/charmhelpers/contrib/database/mysql.py 2015-02-05 10:18:02 +0000 |
3 | +++ hooks/charmhelpers/contrib/database/mysql.py 2015-02-25 20:50:49 +0000 |
4 | @@ -43,13 +43,20 @@ |
5 | |
6 | class MySQLHelper(object): |
7 | |
8 | - def __init__(self, rpasswdf_template, upasswdf_template, host='localhost'): |
9 | + def __init__(self, rpasswdf_template, upasswdf_template, host='localhost', |
10 | + migrate_passwd_to_peer_relation=True, |
11 | + delete_ondisk_passwd_file=True): |
12 | self.host = host |
13 | # Password file path templates |
14 | self.root_passwd_file_template = rpasswdf_template |
15 | self.user_passwd_file_template = upasswdf_template |
16 | |
17 | + self.migrate_passwd_to_peer_relation = migrate_passwd_to_peer_relation |
18 | + # If we migrate we have the option to delete local copy of root passwd |
19 | + self.delete_ondisk_passwd_file = delete_ondisk_passwd_file |
20 | + |
21 | def connect(self, user='root', password=None): |
22 | + log("Opening db connection for %s@%s" % (user, self.host), level=DEBUG) |
23 | self.connection = MySQLdb.connect(user=user, host=self.host, |
24 | passwd=password) |
25 | |
26 | @@ -126,18 +133,23 @@ |
27 | finally: |
28 | cursor.close() |
29 | |
30 | - def migrate_passwords_to_peer_relation(self): |
31 | + def migrate_passwords_to_peer_relation(self, excludes=None): |
32 | """Migrate any passwords storage on disk to cluster peer relation.""" |
33 | dirname = os.path.dirname(self.root_passwd_file_template) |
34 | path = os.path.join(dirname, '*.passwd') |
35 | for f in glob.glob(path): |
36 | + if excludes and f in excludes: |
37 | + log("Excluding %s from peer migration" % (f), level=DEBUG) |
38 | + continue |
39 | + |
40 | _key = os.path.basename(f) |
41 | with open(f, 'r') as passwd: |
42 | _value = passwd.read().strip() |
43 | |
44 | try: |
45 | peer_store(_key, _value) |
46 | - os.unlink(f) |
47 | + if self.delete_ondisk_passwd_file: |
48 | + os.unlink(f) |
49 | except ValueError: |
50 | # NOTE cluster relation not yet ready - skip for now |
51 | pass |
52 | @@ -153,13 +165,20 @@ |
53 | |
54 | _password = None |
55 | if os.path.exists(passwd_file): |
56 | + log("Using existing password file '%s'" % passwd_file, level=DEBUG) |
57 | with open(passwd_file, 'r') as passwd: |
58 | _password = passwd.read().strip() |
59 | else: |
60 | - mkdir(os.path.dirname(passwd_file), owner='root', group='root', |
61 | - perms=0o770) |
62 | - # Force permissions - for some reason the chmod in makedirs fails |
63 | - os.chmod(os.path.dirname(passwd_file), 0o770) |
64 | + log("Generating new password file '%s'" % passwd_file, level=DEBUG) |
65 | + if not os.path.isdir(os.path.dirname(passwd_file)): |
66 | + # NOTE: need to ensure this is not mysql root dir (which needs |
67 | + # to be mysql readable) |
68 | + mkdir(os.path.dirname(passwd_file), owner='root', group='root', |
69 | + perms=0o770) |
70 | + # Force permissions - for some reason the chmod in makedirs |
71 | + # fails |
72 | + os.chmod(os.path.dirname(passwd_file), 0o770) |
73 | + |
74 | _password = password or pwgen(length=32) |
75 | write_file(passwd_file, _password, owner='root', group='root', |
76 | perms=0o660) |
77 | @@ -169,7 +188,9 @@ |
78 | def get_mysql_password(self, username=None, password=None): |
79 | """Retrieve, generate or store a mysql password for the provided |
80 | username using peer relation cluster.""" |
81 | - self.migrate_passwords_to_peer_relation() |
82 | + excludes = [] |
83 | + |
84 | + # First check peer relation |
85 | if username: |
86 | _key = 'mysql-{}.passwd'.format(username) |
87 | else: |
88 | @@ -177,13 +198,22 @@ |
89 | |
90 | try: |
91 | _password = peer_retrieve(_key) |
92 | - if _password is None: |
93 | - _password = password or pwgen(length=32) |
94 | - peer_store(_key, _password) |
95 | + # If root password available don't update peer relation from local |
96 | + if _password and not username: |
97 | + excludes.append(self.root_passwd_file_template) |
98 | + |
99 | except ValueError: |
100 | # cluster relation is not yet started; use on-disk |
101 | + _password = None |
102 | + |
103 | + # If none available, generate new one |
104 | + if not _password: |
105 | _password = self.get_mysql_password_on_disk(username, password) |
106 | |
107 | + # Put on wire if required |
108 | + if self.migrate_passwd_to_peer_relation: |
109 | + self.migrate_passwords_to_peer_relation(excludes=excludes) |
110 | + |
111 | return _password |
112 | |
113 | def get_mysql_root_password(self, password=None): |
114 | |
115 | === modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py' |
116 | --- hooks/charmhelpers/contrib/hahelpers/cluster.py 2015-02-04 18:56:00 +0000 |
117 | +++ hooks/charmhelpers/contrib/hahelpers/cluster.py 2015-02-25 20:50:49 +0000 |
118 | @@ -48,6 +48,9 @@ |
119 | from charmhelpers.core.decorators import ( |
120 | retry_on_exception, |
121 | ) |
122 | +from charmhelpers.core.strutils import ( |
123 | + bool_from_string, |
124 | +) |
125 | |
126 | |
127 | class HAIncompleteConfig(Exception): |
128 | @@ -164,7 +167,8 @@ |
129 | . |
130 | returns: boolean |
131 | ''' |
132 | - if config_get('use-https') == "yes": |
133 | + use_https = config_get('use-https') |
134 | + if use_https and bool_from_string(use_https): |
135 | return True |
136 | if config_get('ssl_cert') and config_get('ssl_key'): |
137 | return True |
138 | |
139 | === modified file 'hooks/charmhelpers/core/fstab.py' |
140 | --- hooks/charmhelpers/core/fstab.py 2015-02-04 18:56:00 +0000 |
141 | +++ hooks/charmhelpers/core/fstab.py 2015-02-25 20:50:49 +0000 |
142 | @@ -17,11 +17,11 @@ |
143 | # You should have received a copy of the GNU Lesser General Public License |
144 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
145 | |
146 | -__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
147 | - |
148 | import io |
149 | import os |
150 | |
151 | +__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
152 | + |
153 | |
154 | class Fstab(io.FileIO): |
155 | """This class extends file in order to implement a file reader/writer |
156 | @@ -77,7 +77,7 @@ |
157 | for line in self.readlines(): |
158 | line = line.decode('us-ascii') |
159 | try: |
160 | - if line.strip() and not line.startswith("#"): |
161 | + if line.strip() and not line.strip().startswith("#"): |
162 | yield self._hydrate_entry(line) |
163 | except ValueError: |
164 | pass |
165 | @@ -104,7 +104,7 @@ |
166 | |
167 | found = False |
168 | for index, line in enumerate(lines): |
169 | - if not line.startswith("#"): |
170 | + if line.strip() and not line.strip().startswith("#"): |
171 | if self._hydrate_entry(line) == entry: |
172 | found = True |
173 | break |
174 | |
175 | === modified file 'hooks/charmhelpers/core/host.py' |
176 | --- hooks/charmhelpers/core/host.py 2015-02-04 18:56:00 +0000 |
177 | +++ hooks/charmhelpers/core/host.py 2015-02-25 20:50:49 +0000 |
178 | @@ -305,11 +305,11 @@ |
179 | ceph_client_changed function. |
180 | """ |
181 | def wrap(f): |
182 | - def wrapped_f(*args): |
183 | + def wrapped_f(*args, **kwargs): |
184 | checksums = {} |
185 | for path in restart_map: |
186 | checksums[path] = file_hash(path) |
187 | - f(*args) |
188 | + f(*args, **kwargs) |
189 | restarts = [] |
190 | for path in restart_map: |
191 | if checksums[path] != file_hash(path): |
192 | |
193 | === added file 'hooks/charmhelpers/core/strutils.py' |
194 | --- hooks/charmhelpers/core/strutils.py 1970-01-01 00:00:00 +0000 |
195 | +++ hooks/charmhelpers/core/strutils.py 2015-02-25 20:50:49 +0000 |
196 | @@ -0,0 +1,42 @@ |
197 | +#!/usr/bin/env python |
198 | +# -*- coding: utf-8 -*- |
199 | + |
200 | +# Copyright 2014-2015 Canonical Limited. |
201 | +# |
202 | +# This file is part of charm-helpers. |
203 | +# |
204 | +# charm-helpers is free software: you can redistribute it and/or modify |
205 | +# it under the terms of the GNU Lesser General Public License version 3 as |
206 | +# published by the Free Software Foundation. |
207 | +# |
208 | +# charm-helpers is distributed in the hope that it will be useful, |
209 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
210 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
211 | +# GNU Lesser General Public License for more details. |
212 | +# |
213 | +# You should have received a copy of the GNU Lesser General Public License |
214 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
215 | + |
216 | +import six |
217 | + |
218 | + |
219 | +def bool_from_string(value): |
220 | + """Interpret string value as boolean. |
221 | + |
222 | + Returns True if value translates to True otherwise False. |
223 | + """ |
224 | + if isinstance(value, six.string_types): |
225 | + value = six.text_type(value) |
226 | + else: |
227 | + msg = "Unable to interpret non-string value '%s' as boolean" % (value) |
228 | + raise ValueError(msg) |
229 | + |
230 | + value = value.strip().lower() |
231 | + |
232 | + if value in ['y', 'yes', 'true', 't']: |
233 | + return True |
234 | + elif value in ['n', 'no', 'false', 'f']: |
235 | + return False |
236 | + |
237 | + msg = "Unable to interpret string value '%s' as boolean" % (value) |
238 | + raise ValueError(msg) |
239 | |
240 | === modified file 'hooks/charmhelpers/core/sysctl.py' |
241 | --- hooks/charmhelpers/core/sysctl.py 2015-02-04 18:56:00 +0000 |
242 | +++ hooks/charmhelpers/core/sysctl.py 2015-02-25 20:50:49 +0000 |
243 | @@ -17,8 +17,6 @@ |
244 | # You should have received a copy of the GNU Lesser General Public License |
245 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
246 | |
247 | -__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
248 | - |
249 | import yaml |
250 | |
251 | from subprocess import check_call |
252 | @@ -29,6 +27,8 @@ |
253 | ERROR, |
254 | ) |
255 | |
256 | +__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
257 | + |
258 | |
259 | def create(sysctl_dict, sysctl_file): |
260 | """Creates a sysctl.conf file from a YAML associative array |
261 | |
262 | === added file 'hooks/charmhelpers/core/unitdata.py' |
263 | --- hooks/charmhelpers/core/unitdata.py 1970-01-01 00:00:00 +0000 |
264 | +++ hooks/charmhelpers/core/unitdata.py 2015-02-25 20:50:49 +0000 |
265 | @@ -0,0 +1,477 @@ |
266 | +#!/usr/bin/env python |
267 | +# -*- coding: utf-8 -*- |
268 | +# |
269 | +# Copyright 2014-2015 Canonical Limited. |
270 | +# |
271 | +# This file is part of charm-helpers. |
272 | +# |
273 | +# charm-helpers is free software: you can redistribute it and/or modify |
274 | +# it under the terms of the GNU Lesser General Public License version 3 as |
275 | +# published by the Free Software Foundation. |
276 | +# |
277 | +# charm-helpers is distributed in the hope that it will be useful, |
278 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
279 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
280 | +# GNU Lesser General Public License for more details. |
281 | +# |
282 | +# You should have received a copy of the GNU Lesser General Public License |
283 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
284 | +# |
285 | +# |
286 | +# Authors: |
287 | +# Kapil Thangavelu <kapil.foss@gmail.com> |
288 | +# |
289 | +""" |
290 | +Intro |
291 | +----- |
292 | + |
293 | +A simple way to store state in units. This provides a key value |
294 | +storage with support for versioned, transactional operation, |
295 | +and can calculate deltas from previous values to simplify unit logic |
296 | +when processing changes. |
297 | + |
298 | + |
299 | +Hook Integration |
300 | +---------------- |
301 | + |
302 | +There are several extant frameworks for hook execution, including |
303 | + |
304 | + - charmhelpers.core.hookenv.Hooks |
305 | + - charmhelpers.core.services.ServiceManager |
306 | + |
307 | +The storage classes are framework agnostic, one simple integration is |
308 | +via the HookData contextmanager. It will record the current hook |
309 | +execution environment (including relation data, config data, etc.), |
310 | +setup a transaction and allow easy access to the changes from |
311 | +previously seen values. One consequence of the integration is the |
312 | +reservation of particular keys ('rels', 'unit', 'env', 'config', |
313 | +'charm_revisions') for their respective values. |
314 | + |
315 | +Here's a fully worked integration example using hookenv.Hooks:: |
316 | + |
317 | + from charmhelper.core import hookenv, unitdata |
318 | + |
319 | + hook_data = unitdata.HookData() |
320 | + db = unitdata.kv() |
321 | + hooks = hookenv.Hooks() |
322 | + |
323 | + @hooks.hook |
324 | + def config_changed(): |
325 | + # Print all changes to configuration from previously seen |
326 | + # values. |
327 | + for changed, (prev, cur) in hook_data.conf.items(): |
328 | + print('config changed', changed, |
329 | + 'previous value', prev, |
330 | + 'current value', cur) |
331 | + |
332 | + # Get some unit specific bookeeping |
333 | + if not db.get('pkg_key'): |
334 | + key = urllib.urlopen('https://example.com/pkg_key').read() |
335 | + db.set('pkg_key', key) |
336 | + |
337 | + # Directly access all charm config as a mapping. |
338 | + conf = db.getrange('config', True) |
339 | + |
340 | + # Directly access all relation data as a mapping |
341 | + rels = db.getrange('rels', True) |
342 | + |
343 | + if __name__ == '__main__': |
344 | + with hook_data(): |
345 | + hook.execute() |
346 | + |
347 | + |
348 | +A more basic integration is via the hook_scope context manager which simply |
349 | +manages transaction scope (and records hook name, and timestamp):: |
350 | + |
351 | + >>> from unitdata import kv |
352 | + >>> db = kv() |
353 | + >>> with db.hook_scope('install'): |
354 | + ... # do work, in transactional scope. |
355 | + ... db.set('x', 1) |
356 | + >>> db.get('x') |
357 | + 1 |
358 | + |
359 | + |
360 | +Usage |
361 | +----- |
362 | + |
363 | +Values are automatically json de/serialized to preserve basic typing |
364 | +and complex data struct capabilities (dicts, lists, ints, booleans, etc). |
365 | + |
366 | +Individual values can be manipulated via get/set:: |
367 | + |
368 | + >>> kv.set('y', True) |
369 | + >>> kv.get('y') |
370 | + True |
371 | + |
372 | + # We can set complex values (dicts, lists) as a single key. |
373 | + >>> kv.set('config', {'a': 1, 'b': True'}) |
374 | + |
375 | + # Also supports returning dictionaries as a record which |
376 | + # provides attribute access. |
377 | + >>> config = kv.get('config', record=True) |
378 | + >>> config.b |
379 | + True |
380 | + |
381 | + |
382 | +Groups of keys can be manipulated with update/getrange:: |
383 | + |
384 | + >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") |
385 | + >>> kv.getrange('gui.', strip=True) |
386 | + {'z': 1, 'y': 2} |
387 | + |
388 | +When updating values, its very helpful to understand which values |
389 | +have actually changed and how have they changed. The storage |
390 | +provides a delta method to provide for this:: |
391 | + |
392 | + >>> data = {'debug': True, 'option': 2} |
393 | + >>> delta = kv.delta(data, 'config.') |
394 | + >>> delta.debug.previous |
395 | + None |
396 | + >>> delta.debug.current |
397 | + True |
398 | + >>> delta |
399 | + {'debug': (None, True), 'option': (None, 2)} |
400 | + |
401 | +Note the delta method does not persist the actual change, it needs to |
402 | +be explicitly saved via 'update' method:: |
403 | + |
404 | + >>> kv.update(data, 'config.') |
405 | + |
406 | +Values modified in the context of a hook scope retain historical values |
407 | +associated to the hookname. |
408 | + |
409 | + >>> with db.hook_scope('config-changed'): |
410 | + ... db.set('x', 42) |
411 | + >>> db.gethistory('x') |
412 | + [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), |
413 | + (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] |
414 | + |
415 | +""" |
416 | + |
417 | +import collections |
418 | +import contextlib |
419 | +import datetime |
420 | +import json |
421 | +import os |
422 | +import pprint |
423 | +import sqlite3 |
424 | +import sys |
425 | + |
426 | +__author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>' |
427 | + |
428 | + |
429 | +class Storage(object): |
430 | + """Simple key value database for local unit state within charms. |
431 | + |
432 | + Modifications are automatically committed at hook exit. That's |
433 | + currently regardless of exit code. |
434 | + |
435 | + To support dicts, lists, integer, floats, and booleans values |
436 | + are automatically json encoded/decoded. |
437 | + """ |
438 | + def __init__(self, path=None): |
439 | + self.db_path = path |
440 | + if path is None: |
441 | + self.db_path = os.path.join( |
442 | + os.environ.get('CHARM_DIR', ''), '.unit-state.db') |
443 | + self.conn = sqlite3.connect('%s' % self.db_path) |
444 | + self.cursor = self.conn.cursor() |
445 | + self.revision = None |
446 | + self._closed = False |
447 | + self._init() |
448 | + |
449 | + def close(self): |
450 | + if self._closed: |
451 | + return |
452 | + self.flush(False) |
453 | + self.cursor.close() |
454 | + self.conn.close() |
455 | + self._closed = True |
456 | + |
457 | + def _scoped_query(self, stmt, params=None): |
458 | + if params is None: |
459 | + params = [] |
460 | + return stmt, params |
461 | + |
462 | + def get(self, key, default=None, record=False): |
463 | + self.cursor.execute( |
464 | + *self._scoped_query( |
465 | + 'select data from kv where key=?', [key])) |
466 | + result = self.cursor.fetchone() |
467 | + if not result: |
468 | + return default |
469 | + if record: |
470 | + return Record(json.loads(result[0])) |
471 | + return json.loads(result[0]) |
472 | + |
473 | + def getrange(self, key_prefix, strip=False): |
474 | + stmt = "select key, data from kv where key like '%s%%'" % key_prefix |
475 | + self.cursor.execute(*self._scoped_query(stmt)) |
476 | + result = self.cursor.fetchall() |
477 | + |
478 | + if not result: |
479 | + return None |
480 | + if not strip: |
481 | + key_prefix = '' |
482 | + return dict([ |
483 | + (k[len(key_prefix):], json.loads(v)) for k, v in result]) |
484 | + |
485 | + def update(self, mapping, prefix=""): |
486 | + for k, v in mapping.items(): |
487 | + self.set("%s%s" % (prefix, k), v) |
488 | + |
489 | + def unset(self, key): |
490 | + self.cursor.execute('delete from kv where key=?', [key]) |
491 | + if self.revision and self.cursor.rowcount: |
492 | + self.cursor.execute( |
493 | + 'insert into kv_revisions values (?, ?, ?)', |
494 | + [key, self.revision, json.dumps('DELETED')]) |
495 | + |
496 | + def set(self, key, value): |
497 | + serialized = json.dumps(value) |
498 | + |
499 | + self.cursor.execute( |
500 | + 'select data from kv where key=?', [key]) |
501 | + exists = self.cursor.fetchone() |
502 | + |
503 | + # Skip mutations to the same value |
504 | + if exists: |
505 | + if exists[0] == serialized: |
506 | + return value |
507 | + |
508 | + if not exists: |
509 | + self.cursor.execute( |
510 | + 'insert into kv (key, data) values (?, ?)', |
511 | + (key, serialized)) |
512 | + else: |
513 | + self.cursor.execute(''' |
514 | + update kv |
515 | + set data = ? |
516 | + where key = ?''', [serialized, key]) |
517 | + |
518 | + # Save |
519 | + if not self.revision: |
520 | + return value |
521 | + |
522 | + self.cursor.execute( |
523 | + 'select 1 from kv_revisions where key=? and revision=?', |
524 | + [key, self.revision]) |
525 | + exists = self.cursor.fetchone() |
526 | + |
527 | + if not exists: |
528 | + self.cursor.execute( |
529 | + '''insert into kv_revisions ( |
530 | + revision, key, data) values (?, ?, ?)''', |
531 | + (self.revision, key, serialized)) |
532 | + else: |
533 | + self.cursor.execute( |
534 | + ''' |
535 | + update kv_revisions |
536 | + set data = ? |
537 | + where key = ? |
538 | + and revision = ?''', |
539 | + [serialized, key, self.revision]) |
540 | + |
541 | + return value |
542 | + |
543 | + def delta(self, mapping, prefix): |
544 | + """ |
545 | + return a delta containing values that have changed. |
546 | + """ |
547 | + previous = self.getrange(prefix, strip=True) |
548 | + if not previous: |
549 | + pk = set() |
550 | + else: |
551 | + pk = set(previous.keys()) |
552 | + ck = set(mapping.keys()) |
553 | + delta = DeltaSet() |
554 | + |
555 | + # added |
556 | + for k in ck.difference(pk): |
557 | + delta[k] = Delta(None, mapping[k]) |
558 | + |
559 | + # removed |
560 | + for k in pk.difference(ck): |
561 | + delta[k] = Delta(previous[k], None) |
562 | + |
563 | + # changed |
564 | + for k in pk.intersection(ck): |
565 | + c = mapping[k] |
566 | + p = previous[k] |
567 | + if c != p: |
568 | + delta[k] = Delta(p, c) |
569 | + |
570 | + return delta |
571 | + |
572 | + @contextlib.contextmanager |
573 | + def hook_scope(self, name=""): |
574 | + """Scope all future interactions to the current hook execution |
575 | + revision.""" |
576 | + assert not self.revision |
577 | + self.cursor.execute( |
578 | + 'insert into hooks (hook, date) values (?, ?)', |
579 | + (name or sys.argv[0], |
580 | + datetime.datetime.utcnow().isoformat())) |
581 | + self.revision = self.cursor.lastrowid |
582 | + try: |
583 | + yield self.revision |
584 | + self.revision = None |
585 | + except: |
586 | + self.flush(False) |
587 | + self.revision = None |
588 | + raise |
589 | + else: |
590 | + self.flush() |
591 | + |
592 | + def flush(self, save=True): |
593 | + if save: |
594 | + self.conn.commit() |
595 | + elif self._closed: |
596 | + return |
597 | + else: |
598 | + self.conn.rollback() |
599 | + |
600 | + def _init(self): |
601 | + self.cursor.execute(''' |
602 | + create table if not exists kv ( |
603 | + key text, |
604 | + data text, |
605 | + primary key (key) |
606 | + )''') |
607 | + self.cursor.execute(''' |
608 | + create table if not exists kv_revisions ( |
609 | + key text, |
610 | + revision integer, |
611 | + data text, |
612 | + primary key (key, revision) |
613 | + )''') |
614 | + self.cursor.execute(''' |
615 | + create table if not exists hooks ( |
616 | + version integer primary key autoincrement, |
617 | + hook text, |
618 | + date text |
619 | + )''') |
620 | + self.conn.commit() |
621 | + |
622 | + def gethistory(self, key, deserialize=False): |
623 | + self.cursor.execute( |
624 | + ''' |
625 | + select kv.revision, kv.key, kv.data, h.hook, h.date |
626 | + from kv_revisions kv, |
627 | + hooks h |
628 | + where kv.key=? |
629 | + and kv.revision = h.version |
630 | + ''', [key]) |
631 | + if deserialize is False: |
632 | + return self.cursor.fetchall() |
633 | + return map(_parse_history, self.cursor.fetchall()) |
634 | + |
635 | + def debug(self, fh=sys.stderr): |
636 | + self.cursor.execute('select * from kv') |
637 | + pprint.pprint(self.cursor.fetchall(), stream=fh) |
638 | + self.cursor.execute('select * from kv_revisions') |
639 | + pprint.pprint(self.cursor.fetchall(), stream=fh) |
640 | + |
641 | + |
642 | +def _parse_history(d): |
643 | + return (d[0], d[1], json.loads(d[2]), d[3], |
644 | + datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) |
645 | + |
646 | + |
647 | +class HookData(object): |
648 | + """Simple integration for existing hook exec frameworks. |
649 | + |
650 | + Records all unit information, and stores deltas for processing |
651 | + by the hook. |
652 | + |
653 | + Sample:: |
654 | + |
655 | + from charmhelper.core import hookenv, unitdata |
656 | + |
657 | + changes = unitdata.HookData() |
658 | + db = unitdata.kv() |
659 | + hooks = hookenv.Hooks() |
660 | + |
661 | + @hooks.hook |
662 | + def config_changed(): |
663 | + # View all changes to configuration |
664 | + for changed, (prev, cur) in changes.conf.items(): |
665 | + print('config changed', changed, |
666 | + 'previous value', prev, |
667 | + 'current value', cur) |
668 | + |
669 | + # Get some unit specific bookeeping |
670 | + if not db.get('pkg_key'): |
671 | + key = urllib.urlopen('https://example.com/pkg_key').read() |
672 | + db.set('pkg_key', key) |
673 | + |
674 | + if __name__ == '__main__': |
675 | + with changes(): |
676 | + hook.execute() |
677 | + |
678 | + """ |
679 | + def __init__(self): |
680 | + self.kv = kv() |
681 | + self.conf = None |
682 | + self.rels = None |
683 | + |
684 | + @contextlib.contextmanager |
685 | + def __call__(self): |
686 | + from charmhelpers.core import hookenv |
687 | + hook_name = hookenv.hook_name() |
688 | + |
689 | + with self.kv.hook_scope(hook_name): |
690 | + self._record_charm_version(hookenv.charm_dir()) |
691 | + delta_config, delta_relation = self._record_hook(hookenv) |
692 | + yield self.kv, delta_config, delta_relation |
693 | + |
694 | + def _record_charm_version(self, charm_dir): |
695 | + # Record revisions.. charm revisions are meaningless |
696 | + # to charm authors as they don't control the revision. |
697 | + # so logic dependnent on revision is not particularly |
698 | + # useful, however it is useful for debugging analysis. |
699 | + charm_rev = open( |
700 | + os.path.join(charm_dir, 'revision')).read().strip() |
701 | + charm_rev = charm_rev or '0' |
702 | + revs = self.kv.get('charm_revisions', []) |
703 | + if charm_rev not in revs: |
704 | + revs.append(charm_rev.strip() or '0') |
705 | + self.kv.set('charm_revisions', revs) |
706 | + |
707 | + def _record_hook(self, hookenv): |
708 | + data = hookenv.execution_environment() |
709 | + self.conf = conf_delta = self.kv.delta(data['conf'], 'config') |
710 | + self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') |
711 | + self.kv.set('env', data['env']) |
712 | + self.kv.set('unit', data['unit']) |
713 | + self.kv.set('relid', data.get('relid')) |
714 | + return conf_delta, rels_delta |
715 | + |
716 | + |
717 | +class Record(dict): |
718 | + |
719 | + __slots__ = () |
720 | + |
721 | + def __getattr__(self, k): |
722 | + if k in self: |
723 | + return self[k] |
724 | + raise AttributeError(k) |
725 | + |
726 | + |
727 | +class DeltaSet(Record): |
728 | + |
729 | + __slots__ = () |
730 | + |
731 | + |
732 | +Delta = collections.namedtuple('Delta', ['previous', 'current']) |
733 | + |
734 | + |
735 | +_KV = None |
736 | + |
737 | + |
738 | +def kv(): |
739 | + global _KV |
740 | + if _KV is None: |
741 | + _KV = Storage() |
742 | + return _KV |
743 | |
744 | === modified file 'hooks/charmhelpers/fetch/archiveurl.py' |
745 | --- hooks/charmhelpers/fetch/archiveurl.py 2015-02-04 18:56:00 +0000 |
746 | +++ hooks/charmhelpers/fetch/archiveurl.py 2015-02-25 20:50:49 +0000 |
747 | @@ -18,6 +18,16 @@ |
748 | import hashlib |
749 | import re |
750 | |
751 | +from charmhelpers.fetch import ( |
752 | + BaseFetchHandler, |
753 | + UnhandledSource |
754 | +) |
755 | +from charmhelpers.payload.archive import ( |
756 | + get_archive_handler, |
757 | + extract, |
758 | +) |
759 | +from charmhelpers.core.host import mkdir, check_hash |
760 | + |
761 | import six |
762 | if six.PY3: |
763 | from urllib.request import ( |
764 | @@ -35,16 +45,6 @@ |
765 | ) |
766 | from urlparse import urlparse, urlunparse, parse_qs |
767 | |
768 | -from charmhelpers.fetch import ( |
769 | - BaseFetchHandler, |
770 | - UnhandledSource |
771 | -) |
772 | -from charmhelpers.payload.archive import ( |
773 | - get_archive_handler, |
774 | - extract, |
775 | -) |
776 | -from charmhelpers.core.host import mkdir, check_hash |
777 | - |
778 | |
779 | def splituser(host): |
780 | '''urllib.splituser(), but six's support of this seems broken''' |
781 | |
782 | === modified file 'hooks/charmhelpers/fetch/giturl.py' |
783 | --- hooks/charmhelpers/fetch/giturl.py 2015-02-04 18:56:00 +0000 |
784 | +++ hooks/charmhelpers/fetch/giturl.py 2015-02-25 20:50:49 +0000 |
785 | @@ -32,7 +32,7 @@ |
786 | apt_install("python-git") |
787 | from git import Repo |
788 | |
789 | -from git.exc import GitCommandError |
790 | +from git.exc import GitCommandError # noqa E402 |
791 | |
792 | |
793 | class GitUrlFetchHandler(BaseFetchHandler): |
Ran tests / local deployment in HA, all works fine.
LGTM +1