Merge lp:~corey.bryant/charms/trusty/percona-cluster/render into lp:~openstack-charmers-archive/charms/trusty/percona-cluster/trunk
- Trusty Tahr (14.04)
- render
- Merge into trunk
Proposed by
Corey Bryant
Status: | Merged |
---|---|
Merged at revision: | 42 |
Proposed branch: | lp:~corey.bryant/charms/trusty/percona-cluster/render |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/percona-cluster/trunk |
Diff against target: |
995 lines (+246/-128) 15 files modified
hooks/charmhelpers/__init__.py (+22/-0) hooks/charmhelpers/contrib/hahelpers/cluster.py (+16/-7) hooks/charmhelpers/contrib/network/ip.py (+2/-2) hooks/charmhelpers/contrib/peerstorage/__init__.py (+4/-3) hooks/charmhelpers/core/fstab.py (+10/-8) hooks/charmhelpers/core/hookenv.py (+36/-16) hooks/charmhelpers/core/host.py (+52/-24) hooks/charmhelpers/core/services/helpers.py (+9/-5) hooks/charmhelpers/core/templating.py (+3/-2) hooks/charmhelpers/fetch/__init__.py (+13/-11) hooks/charmhelpers/fetch/archiveurl.py (+53/-16) hooks/charmhelpers/fetch/bzrurl.py (+5/-1) hooks/charmhelpers/fetch/giturl.py (+12/-5) hooks/percona_hooks.py (+2/-4) hooks/percona_utils.py (+7/-24) |
To merge this branch: | bzr merge lp:~corey.bryant/charms/trusty/percona-cluster/render |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Liam Young (community) | Approve | ||
OpenStack Charmers | Pending | ||
Review via email: mp+245040@code.launchpad.net |
Commit message
Description of the change
This is just simplifying the code and using render() from charm-helpers instead of the roll-your-own render_template().
To post a comment you must log in.
Revision history for this message
Corey Bryant (corey.bryant) wrote : | # |
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'hooks/charmhelpers/__init__.py' |
2 | --- hooks/charmhelpers/__init__.py 2013-09-03 16:52:02 +0000 |
3 | +++ hooks/charmhelpers/__init__.py 2014-12-17 21:09:26 +0000 |
4 | @@ -0,0 +1,22 @@ |
5 | +# Bootstrap charm-helpers, installing its dependencies if necessary using |
6 | +# only standard libraries. |
7 | +import subprocess |
8 | +import sys |
9 | + |
10 | +try: |
11 | + import six # flake8: noqa |
12 | +except ImportError: |
13 | + if sys.version_info.major == 2: |
14 | + subprocess.check_call(['apt-get', 'install', '-y', 'python-six']) |
15 | + else: |
16 | + subprocess.check_call(['apt-get', 'install', '-y', 'python3-six']) |
17 | + import six # flake8: noqa |
18 | + |
19 | +try: |
20 | + import yaml # flake8: noqa |
21 | +except ImportError: |
22 | + if sys.version_info.major == 2: |
23 | + subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml']) |
24 | + else: |
25 | + subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) |
26 | + import yaml # flake8: noqa |
27 | |
28 | === modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py' |
29 | --- hooks/charmhelpers/contrib/hahelpers/cluster.py 2014-10-22 10:34:07 +0000 |
30 | +++ hooks/charmhelpers/contrib/hahelpers/cluster.py 2014-12-17 21:09:26 +0000 |
31 | @@ -13,9 +13,10 @@ |
32 | |
33 | import subprocess |
34 | import os |
35 | - |
36 | from socket import gethostname as get_unit_hostname |
37 | |
38 | +import six |
39 | + |
40 | from charmhelpers.core.hookenv import ( |
41 | log, |
42 | relation_ids, |
43 | @@ -77,7 +78,7 @@ |
44 | "show", resource |
45 | ] |
46 | try: |
47 | - status = subprocess.check_output(cmd) |
48 | + status = subprocess.check_output(cmd).decode('UTF-8') |
49 | except subprocess.CalledProcessError: |
50 | return False |
51 | else: |
52 | @@ -150,34 +151,42 @@ |
53 | return False |
54 | |
55 | |
56 | -def determine_api_port(public_port): |
57 | +def determine_api_port(public_port, singlenode_mode=False): |
58 | ''' |
59 | Determine correct API server listening port based on |
60 | existence of HTTPS reverse proxy and/or haproxy. |
61 | |
62 | public_port: int: standard public port for given service |
63 | |
64 | + singlenode_mode: boolean: Shuffle ports when only a single unit is present |
65 | + |
66 | returns: int: the correct listening port for the API service |
67 | ''' |
68 | i = 0 |
69 | - if len(peer_units()) > 0 or is_clustered(): |
70 | + if singlenode_mode: |
71 | + i += 1 |
72 | + elif len(peer_units()) > 0 or is_clustered(): |
73 | i += 1 |
74 | if https(): |
75 | i += 1 |
76 | return public_port - (i * 10) |
77 | |
78 | |
79 | -def determine_apache_port(public_port): |
80 | +def determine_apache_port(public_port, singlenode_mode=False): |
81 | ''' |
82 | Description: Determine correct apache listening port based on public IP + |
83 | state of the cluster. |
84 | |
85 | public_port: int: standard public port for given service |
86 | |
87 | + singlenode_mode: boolean: Shuffle ports when only a single unit is present |
88 | + |
89 | returns: int: the correct listening port for the HAProxy service |
90 | ''' |
91 | i = 0 |
92 | - if len(peer_units()) > 0 or is_clustered(): |
93 | + if singlenode_mode: |
94 | + i += 1 |
95 | + elif len(peer_units()) > 0 or is_clustered(): |
96 | i += 1 |
97 | return public_port - (i * 10) |
98 | |
99 | @@ -197,7 +206,7 @@ |
100 | for setting in settings: |
101 | conf[setting] = config_get(setting) |
102 | missing = [] |
103 | - [missing.append(s) for s, v in conf.iteritems() if v is None] |
104 | + [missing.append(s) for s, v in six.iteritems(conf) if v is None] |
105 | if missing: |
106 | log('Insufficient config data to configure hacluster.', level=ERROR) |
107 | raise HAIncompleteConfig |
108 | |
109 | === modified file 'hooks/charmhelpers/contrib/network/ip.py' |
110 | --- hooks/charmhelpers/contrib/network/ip.py 2014-11-21 19:25:26 +0000 |
111 | +++ hooks/charmhelpers/contrib/network/ip.py 2014-12-17 21:09:26 +0000 |
112 | @@ -228,7 +228,7 @@ |
113 | raise Exception("Interface '%s' doesn't have any %s addresses." % |
114 | (iface, inet_type)) |
115 | |
116 | - return addresses |
117 | + return sorted(addresses) |
118 | |
119 | |
120 | get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET') |
121 | @@ -302,7 +302,7 @@ |
122 | if global_addrs: |
123 | # Make sure any found global addresses are not temporary |
124 | cmd = ['ip', 'addr', 'show', iface] |
125 | - out = subprocess.check_output(cmd) |
126 | + out = subprocess.check_output(cmd).decode('UTF-8') |
127 | if dynamic_only: |
128 | key = re.compile("inet6 (.+)/[0-9]+ scope global dynamic.*") |
129 | else: |
130 | |
131 | === modified file 'hooks/charmhelpers/contrib/peerstorage/__init__.py' |
132 | --- hooks/charmhelpers/contrib/peerstorage/__init__.py 2014-10-22 10:34:07 +0000 |
133 | +++ hooks/charmhelpers/contrib/peerstorage/__init__.py 2014-12-17 21:09:26 +0000 |
134 | @@ -1,3 +1,4 @@ |
135 | +import six |
136 | from charmhelpers.core.hookenv import relation_id as current_relation_id |
137 | from charmhelpers.core.hookenv import ( |
138 | is_relation_made, |
139 | @@ -93,7 +94,7 @@ |
140 | if ex in echo_data: |
141 | echo_data.pop(ex) |
142 | else: |
143 | - for attribute, value in rdata.iteritems(): |
144 | + for attribute, value in six.iteritems(rdata): |
145 | for include in includes: |
146 | if include in attribute: |
147 | echo_data[attribute] = value |
148 | @@ -119,8 +120,8 @@ |
149 | relation_settings=relation_settings, |
150 | **kwargs) |
151 | if is_relation_made(peer_relation_name): |
152 | - for key, value in dict(kwargs.items() + |
153 | - relation_settings.items()).iteritems(): |
154 | + for key, value in six.iteritems(dict(list(kwargs.items()) + |
155 | + list(relation_settings.items()))): |
156 | key_prefix = relation_id or current_relation_id() |
157 | peer_store(key_prefix + delimiter + key, |
158 | value, |
159 | |
160 | === modified file 'hooks/charmhelpers/core/fstab.py' |
161 | --- hooks/charmhelpers/core/fstab.py 2014-06-23 10:01:12 +0000 |
162 | +++ hooks/charmhelpers/core/fstab.py 2014-12-17 21:09:26 +0000 |
163 | @@ -3,10 +3,11 @@ |
164 | |
165 | __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' |
166 | |
167 | +import io |
168 | import os |
169 | |
170 | |
171 | -class Fstab(file): |
172 | +class Fstab(io.FileIO): |
173 | """This class extends file in order to implement a file reader/writer |
174 | for file `/etc/fstab` |
175 | """ |
176 | @@ -24,8 +25,8 @@ |
177 | options = "defaults" |
178 | |
179 | self.options = options |
180 | - self.d = d |
181 | - self.p = p |
182 | + self.d = int(d) |
183 | + self.p = int(p) |
184 | |
185 | def __eq__(self, o): |
186 | return str(self) == str(o) |
187 | @@ -45,7 +46,7 @@ |
188 | self._path = path |
189 | else: |
190 | self._path = self.DEFAULT_PATH |
191 | - file.__init__(self, self._path, 'r+') |
192 | + super(Fstab, self).__init__(self._path, 'rb+') |
193 | |
194 | def _hydrate_entry(self, line): |
195 | # NOTE: use split with no arguments to split on any |
196 | @@ -58,8 +59,9 @@ |
197 | def entries(self): |
198 | self.seek(0) |
199 | for line in self.readlines(): |
200 | + line = line.decode('us-ascii') |
201 | try: |
202 | - if not line.startswith("#"): |
203 | + if line.strip() and not line.startswith("#"): |
204 | yield self._hydrate_entry(line) |
205 | except ValueError: |
206 | pass |
207 | @@ -75,14 +77,14 @@ |
208 | if self.get_entry_by_attr('device', entry.device): |
209 | return False |
210 | |
211 | - self.write(str(entry) + '\n') |
212 | + self.write((str(entry) + '\n').encode('us-ascii')) |
213 | self.truncate() |
214 | return entry |
215 | |
216 | def remove_entry(self, entry): |
217 | self.seek(0) |
218 | |
219 | - lines = self.readlines() |
220 | + lines = [l.decode('us-ascii') for l in self.readlines()] |
221 | |
222 | found = False |
223 | for index, line in enumerate(lines): |
224 | @@ -97,7 +99,7 @@ |
225 | lines.remove(line) |
226 | |
227 | self.seek(0) |
228 | - self.write(''.join(lines)) |
229 | + self.write(''.join(lines).encode('us-ascii')) |
230 | self.truncate() |
231 | return True |
232 | |
233 | |
234 | === modified file 'hooks/charmhelpers/core/hookenv.py' |
235 | --- hooks/charmhelpers/core/hookenv.py 2014-11-21 19:25:26 +0000 |
236 | +++ hooks/charmhelpers/core/hookenv.py 2014-12-17 21:09:26 +0000 |
237 | @@ -9,9 +9,14 @@ |
238 | import yaml |
239 | import subprocess |
240 | import sys |
241 | -import UserDict |
242 | from subprocess import CalledProcessError |
243 | |
244 | +import six |
245 | +if not six.PY3: |
246 | + from UserDict import UserDict |
247 | +else: |
248 | + from collections import UserDict |
249 | + |
250 | CRITICAL = "CRITICAL" |
251 | ERROR = "ERROR" |
252 | WARNING = "WARNING" |
253 | @@ -63,16 +68,18 @@ |
254 | command = ['juju-log'] |
255 | if level: |
256 | command += ['-l', level] |
257 | + if not isinstance(message, six.string_types): |
258 | + message = repr(message) |
259 | command += [message] |
260 | subprocess.call(command) |
261 | |
262 | |
263 | -class Serializable(UserDict.IterableUserDict): |
264 | +class Serializable(UserDict): |
265 | """Wrapper, an object that can be serialized to yaml or json""" |
266 | |
267 | def __init__(self, obj): |
268 | # wrap the object |
269 | - UserDict.IterableUserDict.__init__(self) |
270 | + UserDict.__init__(self) |
271 | self.data = obj |
272 | |
273 | def __getattr__(self, attr): |
274 | @@ -218,7 +225,7 @@ |
275 | prev_keys = [] |
276 | if self._prev_dict is not None: |
277 | prev_keys = self._prev_dict.keys() |
278 | - return list(set(prev_keys + dict.keys(self))) |
279 | + return list(set(prev_keys + list(dict.keys(self)))) |
280 | |
281 | def load_previous(self, path=None): |
282 | """Load previous copy of config from disk. |
283 | @@ -269,7 +276,7 @@ |
284 | |
285 | """ |
286 | if self._prev_dict: |
287 | - for k, v in self._prev_dict.iteritems(): |
288 | + for k, v in six.iteritems(self._prev_dict): |
289 | if k not in self: |
290 | self[k] = v |
291 | with open(self.path, 'w') as f: |
292 | @@ -284,7 +291,8 @@ |
293 | config_cmd_line.append(scope) |
294 | config_cmd_line.append('--format=json') |
295 | try: |
296 | - config_data = json.loads(subprocess.check_output(config_cmd_line)) |
297 | + config_data = json.loads( |
298 | + subprocess.check_output(config_cmd_line).decode('UTF-8')) |
299 | if scope is not None: |
300 | return config_data |
301 | return Config(config_data) |
302 | @@ -303,10 +311,10 @@ |
303 | if unit: |
304 | _args.append(unit) |
305 | try: |
306 | - return json.loads(subprocess.check_output(_args)) |
307 | + return json.loads(subprocess.check_output(_args).decode('UTF-8')) |
308 | except ValueError: |
309 | return None |
310 | - except CalledProcessError, e: |
311 | + except CalledProcessError as e: |
312 | if e.returncode == 2: |
313 | return None |
314 | raise |
315 | @@ -318,7 +326,7 @@ |
316 | relation_cmd_line = ['relation-set'] |
317 | if relation_id is not None: |
318 | relation_cmd_line.extend(('-r', relation_id)) |
319 | - for k, v in (relation_settings.items() + kwargs.items()): |
320 | + for k, v in (list(relation_settings.items()) + list(kwargs.items())): |
321 | if v is None: |
322 | relation_cmd_line.append('{}='.format(k)) |
323 | else: |
324 | @@ -335,7 +343,8 @@ |
325 | relid_cmd_line = ['relation-ids', '--format=json'] |
326 | if reltype is not None: |
327 | relid_cmd_line.append(reltype) |
328 | - return json.loads(subprocess.check_output(relid_cmd_line)) or [] |
329 | + return json.loads( |
330 | + subprocess.check_output(relid_cmd_line).decode('UTF-8')) or [] |
331 | return [] |
332 | |
333 | |
334 | @@ -346,7 +355,8 @@ |
335 | units_cmd_line = ['relation-list', '--format=json'] |
336 | if relid is not None: |
337 | units_cmd_line.extend(('-r', relid)) |
338 | - return json.loads(subprocess.check_output(units_cmd_line)) or [] |
339 | + return json.loads( |
340 | + subprocess.check_output(units_cmd_line).decode('UTF-8')) or [] |
341 | |
342 | |
343 | @cached |
344 | @@ -386,21 +396,31 @@ |
345 | |
346 | |
347 | @cached |
348 | +def metadata(): |
349 | + """Get the current charm metadata.yaml contents as a python object""" |
350 | + with open(os.path.join(charm_dir(), 'metadata.yaml')) as md: |
351 | + return yaml.safe_load(md) |
352 | + |
353 | + |
354 | +@cached |
355 | def relation_types(): |
356 | """Get a list of relation types supported by this charm""" |
357 | - charmdir = os.environ.get('CHARM_DIR', '') |
358 | - mdf = open(os.path.join(charmdir, 'metadata.yaml')) |
359 | - md = yaml.safe_load(mdf) |
360 | rel_types = [] |
361 | + md = metadata() |
362 | for key in ('provides', 'requires', 'peers'): |
363 | section = md.get(key) |
364 | if section: |
365 | rel_types.extend(section.keys()) |
366 | - mdf.close() |
367 | return rel_types |
368 | |
369 | |
370 | @cached |
371 | +def charm_name(): |
372 | + """Get the name of the current charm as is specified on metadata.yaml""" |
373 | + return metadata().get('name') |
374 | + |
375 | + |
376 | +@cached |
377 | def relations(): |
378 | """Get a nested dictionary of relation data for all related units""" |
379 | rels = {} |
380 | @@ -455,7 +475,7 @@ |
381 | """Get the unit ID for the remote unit""" |
382 | _args = ['unit-get', '--format=json', attribute] |
383 | try: |
384 | - return json.loads(subprocess.check_output(_args)) |
385 | + return json.loads(subprocess.check_output(_args).decode('UTF-8')) |
386 | except ValueError: |
387 | return None |
388 | |
389 | |
390 | === modified file 'hooks/charmhelpers/core/host.py' |
391 | --- hooks/charmhelpers/core/host.py 2014-11-21 19:25:26 +0000 |
392 | +++ hooks/charmhelpers/core/host.py 2014-12-17 21:09:26 +0000 |
393 | @@ -14,11 +14,12 @@ |
394 | import subprocess |
395 | import hashlib |
396 | from contextlib import contextmanager |
397 | - |
398 | from collections import OrderedDict |
399 | |
400 | -from hookenv import log |
401 | -from fstab import Fstab |
402 | +import six |
403 | + |
404 | +from .hookenv import log |
405 | +from .fstab import Fstab |
406 | |
407 | |
408 | def service_start(service_name): |
409 | @@ -54,7 +55,9 @@ |
410 | def service_running(service): |
411 | """Determine whether a system service is running""" |
412 | try: |
413 | - output = subprocess.check_output(['service', service, 'status'], stderr=subprocess.STDOUT) |
414 | + output = subprocess.check_output( |
415 | + ['service', service, 'status'], |
416 | + stderr=subprocess.STDOUT).decode('UTF-8') |
417 | except subprocess.CalledProcessError: |
418 | return False |
419 | else: |
420 | @@ -67,7 +70,9 @@ |
421 | def service_available(service_name): |
422 | """Determine whether a system service is available""" |
423 | try: |
424 | - subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT) |
425 | + subprocess.check_output( |
426 | + ['service', service_name, 'status'], |
427 | + stderr=subprocess.STDOUT).decode('UTF-8') |
428 | except subprocess.CalledProcessError as e: |
429 | return 'unrecognized service' not in e.output |
430 | else: |
431 | @@ -96,6 +101,26 @@ |
432 | return user_info |
433 | |
434 | |
435 | +def add_group(group_name, system_group=False): |
436 | + """Add a group to the system""" |
437 | + try: |
438 | + group_info = grp.getgrnam(group_name) |
439 | + log('group {0} already exists!'.format(group_name)) |
440 | + except KeyError: |
441 | + log('creating group {0}'.format(group_name)) |
442 | + cmd = ['addgroup'] |
443 | + if system_group: |
444 | + cmd.append('--system') |
445 | + else: |
446 | + cmd.extend([ |
447 | + '--group', |
448 | + ]) |
449 | + cmd.append(group_name) |
450 | + subprocess.check_call(cmd) |
451 | + group_info = grp.getgrnam(group_name) |
452 | + return group_info |
453 | + |
454 | + |
455 | def add_user_to_group(username, group): |
456 | """Add a user to a group""" |
457 | cmd = [ |
458 | @@ -115,7 +140,7 @@ |
459 | cmd.append(from_path) |
460 | cmd.append(to_path) |
461 | log(" ".join(cmd)) |
462 | - return subprocess.check_output(cmd).strip() |
463 | + return subprocess.check_output(cmd).decode('UTF-8').strip() |
464 | |
465 | |
466 | def symlink(source, destination): |
467 | @@ -130,23 +155,26 @@ |
468 | subprocess.check_call(cmd) |
469 | |
470 | |
471 | -def mkdir(path, owner='root', group='root', perms=0555, force=False): |
472 | +def mkdir(path, owner='root', group='root', perms=0o555, force=False): |
473 | """Create a directory""" |
474 | log("Making dir {} {}:{} {:o}".format(path, owner, group, |
475 | perms)) |
476 | uid = pwd.getpwnam(owner).pw_uid |
477 | gid = grp.getgrnam(group).gr_gid |
478 | realpath = os.path.abspath(path) |
479 | - if os.path.exists(realpath): |
480 | - if force and not os.path.isdir(realpath): |
481 | + path_exists = os.path.exists(realpath) |
482 | + if path_exists and force: |
483 | + if not os.path.isdir(realpath): |
484 | log("Removing non-directory file {} prior to mkdir()".format(path)) |
485 | os.unlink(realpath) |
486 | - else: |
487 | + os.makedirs(realpath, perms) |
488 | + os.chown(realpath, uid, gid) |
489 | + elif not path_exists: |
490 | os.makedirs(realpath, perms) |
491 | - os.chown(realpath, uid, gid) |
492 | - |
493 | - |
494 | -def write_file(path, content, owner='root', group='root', perms=0444): |
495 | + os.chown(realpath, uid, gid) |
496 | + |
497 | + |
498 | +def write_file(path, content, owner='root', group='root', perms=0o444): |
499 | """Create or overwrite a file with the contents of a string""" |
500 | log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) |
501 | uid = pwd.getpwnam(owner).pw_uid |
502 | @@ -177,7 +205,7 @@ |
503 | cmd_args.extend([device, mountpoint]) |
504 | try: |
505 | subprocess.check_output(cmd_args) |
506 | - except subprocess.CalledProcessError, e: |
507 | + except subprocess.CalledProcessError as e: |
508 | log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output)) |
509 | return False |
510 | |
511 | @@ -191,7 +219,7 @@ |
512 | cmd_args = ['umount', mountpoint] |
513 | try: |
514 | subprocess.check_output(cmd_args) |
515 | - except subprocess.CalledProcessError, e: |
516 | + except subprocess.CalledProcessError as e: |
517 | log('Error unmounting {}\n{}'.format(mountpoint, e.output)) |
518 | return False |
519 | |
520 | @@ -218,8 +246,8 @@ |
521 | """ |
522 | if os.path.exists(path): |
523 | h = getattr(hashlib, hash_type)() |
524 | - with open(path, 'r') as source: |
525 | - h.update(source.read()) # IGNORE:E1101 - it does have update |
526 | + with open(path, 'rb') as source: |
527 | + h.update(source.read()) |
528 | return h.hexdigest() |
529 | else: |
530 | return None |
531 | @@ -297,7 +325,7 @@ |
532 | if length is None: |
533 | length = random.choice(range(35, 45)) |
534 | alphanumeric_chars = [ |
535 | - l for l in (string.letters + string.digits) |
536 | + l for l in (string.ascii_letters + string.digits) |
537 | if l not in 'l0QD1vAEIOUaeiou'] |
538 | random_chars = [ |
539 | random.choice(alphanumeric_chars) for _ in range(length)] |
540 | @@ -306,14 +334,14 @@ |
541 | |
542 | def list_nics(nic_type): |
543 | '''Return a list of nics of given type(s)''' |
544 | - if isinstance(nic_type, basestring): |
545 | + if isinstance(nic_type, six.string_types): |
546 | int_types = [nic_type] |
547 | else: |
548 | int_types = nic_type |
549 | interfaces = [] |
550 | for int_type in int_types: |
551 | cmd = ['ip', 'addr', 'show', 'label', int_type + '*'] |
552 | - ip_output = subprocess.check_output(cmd).split('\n') |
553 | + ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') |
554 | ip_output = (line for line in ip_output if line) |
555 | for line in ip_output: |
556 | if line.split()[1].startswith(int_type): |
557 | @@ -335,7 +363,7 @@ |
558 | |
559 | def get_nic_mtu(nic): |
560 | cmd = ['ip', 'addr', 'show', nic] |
561 | - ip_output = subprocess.check_output(cmd).split('\n') |
562 | + ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') |
563 | mtu = "" |
564 | for line in ip_output: |
565 | words = line.split() |
566 | @@ -346,7 +374,7 @@ |
567 | |
568 | def get_nic_hwaddr(nic): |
569 | cmd = ['ip', '-o', '-0', 'addr', 'show', nic] |
570 | - ip_output = subprocess.check_output(cmd) |
571 | + ip_output = subprocess.check_output(cmd).decode('UTF-8') |
572 | hwaddr = "" |
573 | words = ip_output.split() |
574 | if 'link/ether' in words: |
575 | @@ -363,8 +391,8 @@ |
576 | |
577 | ''' |
578 | import apt_pkg |
579 | - from charmhelpers.fetch import apt_cache |
580 | if not pkgcache: |
581 | + from charmhelpers.fetch import apt_cache |
582 | pkgcache = apt_cache() |
583 | pkg = pkgcache[package] |
584 | return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) |
585 | |
586 | === modified file 'hooks/charmhelpers/core/services/helpers.py' |
587 | --- hooks/charmhelpers/core/services/helpers.py 2014-10-22 10:34:07 +0000 |
588 | +++ hooks/charmhelpers/core/services/helpers.py 2014-12-17 21:09:26 +0000 |
589 | @@ -196,7 +196,7 @@ |
590 | if not os.path.isabs(file_name): |
591 | file_name = os.path.join(hookenv.charm_dir(), file_name) |
592 | with open(file_name, 'w') as file_stream: |
593 | - os.fchmod(file_stream.fileno(), 0600) |
594 | + os.fchmod(file_stream.fileno(), 0o600) |
595 | yaml.dump(config_data, file_stream) |
596 | |
597 | def read_context(self, file_name): |
598 | @@ -211,15 +211,19 @@ |
599 | |
600 | class TemplateCallback(ManagerCallback): |
601 | """ |
602 | - Callback class that will render a Jinja2 template, for use as a ready action. |
603 | - |
604 | - :param str source: The template source file, relative to `$CHARM_DIR/templates` |
605 | + Callback class that will render a Jinja2 template, for use as a ready |
606 | + action. |
607 | + |
608 | + :param str source: The template source file, relative to |
609 | + `$CHARM_DIR/templates` |
610 | + |
611 | :param str target: The target to write the rendered template to |
612 | :param str owner: The owner of the rendered file |
613 | :param str group: The group of the rendered file |
614 | :param int perms: The permissions of the rendered file |
615 | """ |
616 | - def __init__(self, source, target, owner='root', group='root', perms=0444): |
617 | + def __init__(self, source, target, |
618 | + owner='root', group='root', perms=0o444): |
619 | self.source = source |
620 | self.target = target |
621 | self.owner = owner |
622 | |
623 | === modified file 'hooks/charmhelpers/core/templating.py' |
624 | --- hooks/charmhelpers/core/templating.py 2014-08-18 12:39:26 +0000 |
625 | +++ hooks/charmhelpers/core/templating.py 2014-12-17 21:09:26 +0000 |
626 | @@ -4,7 +4,8 @@ |
627 | from charmhelpers.core import hookenv |
628 | |
629 | |
630 | -def render(source, target, context, owner='root', group='root', perms=0444, templates_dir=None): |
631 | +def render(source, target, context, owner='root', group='root', |
632 | + perms=0o444, templates_dir=None): |
633 | """ |
634 | Render a template. |
635 | |
636 | @@ -47,5 +48,5 @@ |
637 | level=hookenv.ERROR) |
638 | raise e |
639 | content = template.render(context) |
640 | - host.mkdir(os.path.dirname(target)) |
641 | + host.mkdir(os.path.dirname(target), owner, group) |
642 | host.write_file(target, content, owner, group, perms) |
643 | |
644 | === modified file 'hooks/charmhelpers/fetch/__init__.py' |
645 | --- hooks/charmhelpers/fetch/__init__.py 2014-11-21 19:25:26 +0000 |
646 | +++ hooks/charmhelpers/fetch/__init__.py 2014-12-17 21:09:26 +0000 |
647 | @@ -5,10 +5,6 @@ |
648 | from charmhelpers.core.host import ( |
649 | lsb_release |
650 | ) |
651 | -from urlparse import ( |
652 | - urlparse, |
653 | - urlunparse, |
654 | -) |
655 | import subprocess |
656 | from charmhelpers.core.hookenv import ( |
657 | config, |
658 | @@ -16,6 +12,12 @@ |
659 | ) |
660 | import os |
661 | |
662 | +import six |
663 | +if six.PY3: |
664 | + from urllib.parse import urlparse, urlunparse |
665 | +else: |
666 | + from urlparse import urlparse, urlunparse |
667 | + |
668 | |
669 | CLOUD_ARCHIVE = """# Ubuntu Cloud Archive |
670 | deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main |
671 | @@ -149,7 +151,7 @@ |
672 | cmd = ['apt-get', '--assume-yes'] |
673 | cmd.extend(options) |
674 | cmd.append('install') |
675 | - if isinstance(packages, basestring): |
676 | + if isinstance(packages, six.string_types): |
677 | cmd.append(packages) |
678 | else: |
679 | cmd.extend(packages) |
680 | @@ -182,7 +184,7 @@ |
681 | def apt_purge(packages, fatal=False): |
682 | """Purge one or more packages""" |
683 | cmd = ['apt-get', '--assume-yes', 'purge'] |
684 | - if isinstance(packages, basestring): |
685 | + if isinstance(packages, six.string_types): |
686 | cmd.append(packages) |
687 | else: |
688 | cmd.extend(packages) |
689 | @@ -193,7 +195,7 @@ |
690 | def apt_hold(packages, fatal=False): |
691 | """Hold one or more packages""" |
692 | cmd = ['apt-mark', 'hold'] |
693 | - if isinstance(packages, basestring): |
694 | + if isinstance(packages, six.string_types): |
695 | cmd.append(packages) |
696 | else: |
697 | cmd.extend(packages) |
698 | @@ -260,7 +262,7 @@ |
699 | |
700 | if key: |
701 | if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key: |
702 | - with NamedTemporaryFile() as key_file: |
703 | + with NamedTemporaryFile('w+') as key_file: |
704 | key_file.write(key) |
705 | key_file.flush() |
706 | key_file.seek(0) |
707 | @@ -297,14 +299,14 @@ |
708 | sources = safe_load((config(sources_var) or '').strip()) or [] |
709 | keys = safe_load((config(keys_var) or '').strip()) or None |
710 | |
711 | - if isinstance(sources, basestring): |
712 | + if isinstance(sources, six.string_types): |
713 | sources = [sources] |
714 | |
715 | if keys is None: |
716 | for source in sources: |
717 | add_source(source, None) |
718 | else: |
719 | - if isinstance(keys, basestring): |
720 | + if isinstance(keys, six.string_types): |
721 | keys = [keys] |
722 | |
723 | if len(sources) != len(keys): |
724 | @@ -401,7 +403,7 @@ |
725 | while result is None or result == APT_NO_LOCK: |
726 | try: |
727 | result = subprocess.check_call(cmd, env=env) |
728 | - except subprocess.CalledProcessError, e: |
729 | + except subprocess.CalledProcessError as e: |
730 | retry_count = retry_count + 1 |
731 | if retry_count > APT_NO_LOCK_RETRY_COUNT: |
732 | raise |
733 | |
734 | === modified file 'hooks/charmhelpers/fetch/archiveurl.py' |
735 | --- hooks/charmhelpers/fetch/archiveurl.py 2014-10-22 10:34:07 +0000 |
736 | +++ hooks/charmhelpers/fetch/archiveurl.py 2014-12-17 21:09:26 +0000 |
737 | @@ -1,8 +1,23 @@ |
738 | import os |
739 | -import urllib2 |
740 | -from urllib import urlretrieve |
741 | -import urlparse |
742 | import hashlib |
743 | +import re |
744 | + |
745 | +import six |
746 | +if six.PY3: |
747 | + from urllib.request import ( |
748 | + build_opener, install_opener, urlopen, urlretrieve, |
749 | + HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, |
750 | + ) |
751 | + from urllib.parse import urlparse, urlunparse, parse_qs |
752 | + from urllib.error import URLError |
753 | +else: |
754 | + from urllib import urlretrieve |
755 | + from urllib2 import ( |
756 | + build_opener, install_opener, urlopen, |
757 | + HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, |
758 | + URLError |
759 | + ) |
760 | + from urlparse import urlparse, urlunparse, parse_qs |
761 | |
762 | from charmhelpers.fetch import ( |
763 | BaseFetchHandler, |
764 | @@ -15,6 +30,24 @@ |
765 | from charmhelpers.core.host import mkdir, check_hash |
766 | |
767 | |
768 | +def splituser(host): |
769 | + '''urllib.splituser(), but six's support of this seems broken''' |
770 | + _userprog = re.compile('^(.*)@(.*)$') |
771 | + match = _userprog.match(host) |
772 | + if match: |
773 | + return match.group(1, 2) |
774 | + return None, host |
775 | + |
776 | + |
777 | +def splitpasswd(user): |
778 | + '''urllib.splitpasswd(), but six's support of this is missing''' |
779 | + _passwdprog = re.compile('^([^:]*):(.*)$', re.S) |
780 | + match = _passwdprog.match(user) |
781 | + if match: |
782 | + return match.group(1, 2) |
783 | + return user, None |
784 | + |
785 | + |
786 | class ArchiveUrlFetchHandler(BaseFetchHandler): |
787 | """ |
788 | Handler to download archive files from arbitrary URLs. |
789 | @@ -42,20 +75,20 @@ |
790 | """ |
791 | # propogate all exceptions |
792 | # URLError, OSError, etc |
793 | - proto, netloc, path, params, query, fragment = urlparse.urlparse(source) |
794 | + proto, netloc, path, params, query, fragment = urlparse(source) |
795 | if proto in ('http', 'https'): |
796 | - auth, barehost = urllib2.splituser(netloc) |
797 | + auth, barehost = splituser(netloc) |
798 | if auth is not None: |
799 | - source = urlparse.urlunparse((proto, barehost, path, params, query, fragment)) |
800 | - username, password = urllib2.splitpasswd(auth) |
801 | - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() |
802 | + source = urlunparse((proto, barehost, path, params, query, fragment)) |
803 | + username, password = splitpasswd(auth) |
804 | + passman = HTTPPasswordMgrWithDefaultRealm() |
805 | # Realm is set to None in add_password to force the username and password |
806 | # to be used whatever the realm |
807 | passman.add_password(None, source, username, password) |
808 | - authhandler = urllib2.HTTPBasicAuthHandler(passman) |
809 | - opener = urllib2.build_opener(authhandler) |
810 | - urllib2.install_opener(opener) |
811 | - response = urllib2.urlopen(source) |
812 | + authhandler = HTTPBasicAuthHandler(passman) |
813 | + opener = build_opener(authhandler) |
814 | + install_opener(opener) |
815 | + response = urlopen(source) |
816 | try: |
817 | with open(dest, 'w') as dest_file: |
818 | dest_file.write(response.read()) |
819 | @@ -91,17 +124,21 @@ |
820 | url_parts = self.parse_url(source) |
821 | dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched') |
822 | if not os.path.exists(dest_dir): |
823 | - mkdir(dest_dir, perms=0755) |
824 | + mkdir(dest_dir, perms=0o755) |
825 | dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path)) |
826 | try: |
827 | self.download(source, dld_file) |
828 | - except urllib2.URLError as e: |
829 | + except URLError as e: |
830 | raise UnhandledSource(e.reason) |
831 | except OSError as e: |
832 | raise UnhandledSource(e.strerror) |
833 | - options = urlparse.parse_qs(url_parts.fragment) |
834 | + options = parse_qs(url_parts.fragment) |
835 | for key, value in options.items(): |
836 | - if key in hashlib.algorithms: |
837 | + if not six.PY3: |
838 | + algorithms = hashlib.algorithms |
839 | + else: |
840 | + algorithms = hashlib.algorithms_available |
841 | + if key in algorithms: |
842 | check_hash(dld_file, value, key) |
843 | if checksum: |
844 | check_hash(dld_file, checksum, hash_type) |
845 | |
846 | === modified file 'hooks/charmhelpers/fetch/bzrurl.py' |
847 | --- hooks/charmhelpers/fetch/bzrurl.py 2014-06-23 09:47:35 +0000 |
848 | +++ hooks/charmhelpers/fetch/bzrurl.py 2014-12-17 21:09:26 +0000 |
849 | @@ -5,6 +5,10 @@ |
850 | ) |
851 | from charmhelpers.core.host import mkdir |
852 | |
853 | +import six |
854 | +if six.PY3: |
855 | + raise ImportError('bzrlib does not support Python3') |
856 | + |
857 | try: |
858 | from bzrlib.branch import Branch |
859 | except ImportError: |
860 | @@ -42,7 +46,7 @@ |
861 | dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", |
862 | branch_name) |
863 | if not os.path.exists(dest_dir): |
864 | - mkdir(dest_dir, perms=0755) |
865 | + mkdir(dest_dir, perms=0o755) |
866 | try: |
867 | self.branch(source, dest_dir) |
868 | except OSError as e: |
869 | |
870 | === modified file 'hooks/charmhelpers/fetch/giturl.py' |
871 | --- hooks/charmhelpers/fetch/giturl.py 2014-11-21 19:25:26 +0000 |
872 | +++ hooks/charmhelpers/fetch/giturl.py 2014-12-17 21:09:26 +0000 |
873 | @@ -5,6 +5,10 @@ |
874 | ) |
875 | from charmhelpers.core.host import mkdir |
876 | |
877 | +import six |
878 | +if six.PY3: |
879 | + raise ImportError('GitPython does not support Python 3') |
880 | + |
881 | try: |
882 | from git import Repo |
883 | except ImportError: |
884 | @@ -17,7 +21,7 @@ |
885 | """Handler for git branches via generic and github URLs""" |
886 | def can_handle(self, source): |
887 | url_parts = self.parse_url(source) |
888 | - #TODO (mattyw) no support for ssh git@ yet |
889 | + # TODO (mattyw) no support for ssh git@ yet |
890 | if url_parts.scheme not in ('http', 'https', 'git'): |
891 | return False |
892 | else: |
893 | @@ -30,13 +34,16 @@ |
894 | repo = Repo.clone_from(source, dest) |
895 | repo.git.checkout(branch) |
896 | |
897 | - def install(self, source, branch="master"): |
898 | + def install(self, source, branch="master", dest=None): |
899 | url_parts = self.parse_url(source) |
900 | branch_name = url_parts.path.strip("/").split("/")[-1] |
901 | - dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", |
902 | - branch_name) |
903 | + if dest: |
904 | + dest_dir = os.path.join(dest, branch_name) |
905 | + else: |
906 | + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", |
907 | + branch_name) |
908 | if not os.path.exists(dest_dir): |
909 | - mkdir(dest_dir, perms=0755) |
910 | + mkdir(dest_dir, perms=0o755) |
911 | try: |
912 | self.clone(source, dest_dir, branch) |
913 | except OSError as e: |
914 | |
915 | === modified file 'hooks/percona_hooks.py' |
916 | --- hooks/percona_hooks.py 2014-11-25 14:59:34 +0000 |
917 | +++ hooks/percona_hooks.py 2014-12-17 21:09:26 +0000 |
918 | @@ -26,6 +26,7 @@ |
919 | write_file, |
920 | lsb_release, |
921 | ) |
922 | +from charmhelpers.core.templating import render |
923 | from charmhelpers.fetch import ( |
924 | apt_update, |
925 | apt_install, |
926 | @@ -40,7 +41,6 @@ |
927 | PACKAGES, |
928 | MY_CNF, |
929 | setup_percona_repo, |
930 | - render_template, |
931 | get_host_ip, |
932 | get_cluster_hosts, |
933 | configure_sstuser, |
934 | @@ -123,9 +123,7 @@ |
935 | context['ipv6'] = False |
936 | |
937 | context.update(parse_config()) |
938 | - write_file(path=MY_CNF, |
939 | - content=render_template(os.path.basename(MY_CNF), context), |
940 | - perms=0o444) |
941 | + render(os.path.basename(MY_CNF), MY_CNF, context, perms=0o444) |
942 | |
943 | |
944 | @hooks.hook('upgrade-charm') |
945 | |
946 | === modified file 'hooks/percona_utils.py' |
947 | --- hooks/percona_utils.py 2014-11-21 19:25:26 +0000 |
948 | +++ hooks/percona_utils.py 2014-12-17 21:09:26 +0000 |
949 | @@ -28,20 +28,6 @@ |
950 | from mysql import get_mysql_root_password, MySQLHelper |
951 | |
952 | |
953 | -try: |
954 | - import jinja2 |
955 | -except ImportError: |
956 | - apt_install(filter_installed_packages(['python-jinja2']), |
957 | - fatal=True) |
958 | - import jinja2 |
959 | - |
960 | -try: |
961 | - import dns.resolver |
962 | -except ImportError: |
963 | - apt_install(filter_installed_packages(['python-dnspython']), |
964 | - fatal=True) |
965 | - import dns.resolver |
966 | - |
967 | PACKAGES = [ |
968 | 'percona-xtradb-cluster-server-5.5', |
969 | 'percona-xtradb-cluster-client-5.5', |
970 | @@ -72,18 +58,15 @@ |
971 | sources.write(REPO.format(release=lsb_release()['DISTRIB_CODENAME'])) |
972 | subprocess.check_call(['apt-key', 'add', KEY]) |
973 | |
974 | -TEMPLATES_DIR = 'templates' |
975 | -FILES_DIR = 'files' |
976 | - |
977 | - |
978 | -def render_template(template_name, context, template_dir=TEMPLATES_DIR): |
979 | - templates = jinja2.Environment( |
980 | - loader=jinja2.FileSystemLoader(template_dir)) |
981 | - template = templates.get_template(template_name) |
982 | - return template.render(context) |
983 | - |
984 | |
985 | def get_host_ip(hostname=None): |
986 | + try: |
987 | + import dns.resolver |
988 | + except ImportError: |
989 | + apt_install(filter_installed_packages(['python-dnspython']), |
990 | + fatal=True) |
991 | + import dns.resolver |
992 | + |
993 | if config('prefer-ipv6'): |
994 | # Ensure we have a valid ipv6 address configured |
995 | get_ipv6_addr(exc_list=[config('vip')], fatal=True)[0] |
This merge also fixes a pre-existing unit test failure.