Merge lp:~gnuoy/charms/trusty/cinder/next-haproxy-always into lp:~openstack-charmers-archive/charms/trusty/cinder/next

Proposed by Liam Young on 2014-11-26
Status: Merged
Merged at revision: 60
Proposed branch: lp:~gnuoy/charms/trusty/cinder/next-haproxy-always
Merge into: lp:~openstack-charmers-archive/charms/trusty/cinder/next
Diff against target: 2996 lines (+769/-498)
29 files modified
hooks/charmhelpers/contrib/hahelpers/cluster.py (+16/-7)
hooks/charmhelpers/contrib/network/ip.py (+52/-50)
hooks/charmhelpers/contrib/openstack/amulet/deployment.py (+2/-1)
hooks/charmhelpers/contrib/openstack/amulet/utils.py (+3/-1)
hooks/charmhelpers/contrib/openstack/context.py (+319/-226)
hooks/charmhelpers/contrib/openstack/neutron.py (+20/-4)
hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg (+2/-2)
hooks/charmhelpers/contrib/openstack/templating.py (+5/-5)
hooks/charmhelpers/contrib/openstack/utils.py (+32/-7)
hooks/charmhelpers/contrib/storage/linux/ceph.py (+89/-102)
hooks/charmhelpers/contrib/storage/linux/loopback.py (+4/-4)
hooks/charmhelpers/contrib/storage/linux/lvm.py (+1/-0)
hooks/charmhelpers/contrib/storage/linux/utils.py (+3/-2)
hooks/charmhelpers/core/fstab.py (+10/-8)
hooks/charmhelpers/core/hookenv.py (+25/-11)
hooks/charmhelpers/core/host.py (+30/-19)
hooks/charmhelpers/core/services/__init__.py (+2/-2)
hooks/charmhelpers/core/services/helpers.py (+9/-5)
hooks/charmhelpers/core/templating.py (+2/-1)
hooks/charmhelpers/fetch/__init__.py (+18/-12)
hooks/charmhelpers/fetch/archiveurl.py (+53/-16)
hooks/charmhelpers/fetch/bzrurl.py (+5/-1)
hooks/charmhelpers/fetch/giturl.py (+48/-0)
hooks/cinder_contexts.py (+4/-2)
hooks/cinder_utils.py (+1/-1)
tests/charmhelpers/contrib/amulet/deployment.py (+3/-3)
tests/charmhelpers/contrib/amulet/utils.py (+6/-4)
tests/charmhelpers/contrib/openstack/amulet/deployment.py (+2/-1)
tests/charmhelpers/contrib/openstack/amulet/utils.py (+3/-1)
To merge this branch: bzr merge lp:~gnuoy/charms/trusty/cinder/next-haproxy-always
Reviewer Review Type Date Requested Status
OpenStack Charmers 2014-11-26 Pending
Review via email: mp+242902@code.launchpad.net
To post a comment you must log in.

UOSCI bot says:
charm_lint_check #1233 cinder-next for gnuoy mp242902
    LINT OK: passed

LINT Results (max last 5 lines):
  I: config.yaml: option ssl_ca has no default value
  I: config.yaml: option config-flags has no default value
  I: config.yaml: option ssl_cert has no default value
  I: config.yaml: option os-internal-network has no default value
  I: config.yaml: option os-public-network has no default value

Full lint test output: http://paste.ubuntu.com/9252849/
Build: http://10.98.191.181:8080/job/charm_lint_check/1233/

UOSCI bot says:
charm_unit_test #1067 cinder-next for gnuoy mp242902
    UNIT OK: passed

UNIT Results (max last 5 lines):
  hooks/cinder_hooks 194 6 97% 99-100, 115, 122, 288-289
  hooks/cinder_utils 173 14 92% 191, 410-415, 452-464
  TOTAL 419 26 94%
  Ran 86 tests in 5.000s
  OK

Full unit test output: http://paste.ubuntu.com/9252857/
Build: http://10.98.191.181:8080/job/charm_unit_test/1067/

UOSCI bot says:
charm_amulet_test #536 cinder-next for gnuoy mp242902
    AMULET FAIL: amulet-test failed

AMULET Results (max last 5 lines):
  juju-test.conductor DEBUG : Calling "juju destroy-environment -y osci-sv11"
  WARNING cannot delete security group "juju-osci-sv11-0". Used by another environment?
  juju-test INFO : Results: 1 passed, 2 failed, 0 errored
  ERROR subprocess encountered error code 2
  make: *** [test] Error 2

Full amulet test output: http://paste.ubuntu.com/9253275/
Build: http://10.98.191.181:8080/job/charm_amulet_test/536/

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
2--- hooks/charmhelpers/contrib/hahelpers/cluster.py 2014-10-01 22:07:44 +0000
3+++ hooks/charmhelpers/contrib/hahelpers/cluster.py 2014-11-26 10:42:22 +0000
4@@ -13,9 +13,10 @@
5
6 import subprocess
7 import os
8-
9 from socket import gethostname as get_unit_hostname
10
11+import six
12+
13 from charmhelpers.core.hookenv import (
14 log,
15 relation_ids,
16@@ -77,7 +78,7 @@
17 "show", resource
18 ]
19 try:
20- status = subprocess.check_output(cmd)
21+ status = subprocess.check_output(cmd).decode('UTF-8')
22 except subprocess.CalledProcessError:
23 return False
24 else:
25@@ -150,34 +151,42 @@
26 return False
27
28
29-def determine_api_port(public_port):
30+def determine_api_port(public_port, singlenode_mode=False):
31 '''
32 Determine correct API server listening port based on
33 existence of HTTPS reverse proxy and/or haproxy.
34
35 public_port: int: standard public port for given service
36
37+ singlenode_mode: boolean: Shuffle ports when only a single unit is present
38+
39 returns: int: the correct listening port for the API service
40 '''
41 i = 0
42- if len(peer_units()) > 0 or is_clustered():
43+ if singlenode_mode:
44+ i += 1
45+ elif len(peer_units()) > 0 or is_clustered():
46 i += 1
47 if https():
48 i += 1
49 return public_port - (i * 10)
50
51
52-def determine_apache_port(public_port):
53+def determine_apache_port(public_port, singlenode_mode=False):
54 '''
55 Description: Determine correct apache listening port based on public IP +
56 state of the cluster.
57
58 public_port: int: standard public port for given service
59
60+ singlenode_mode: boolean: Shuffle ports when only a single unit is present
61+
62 returns: int: the correct listening port for the HAProxy service
63 '''
64 i = 0
65- if len(peer_units()) > 0 or is_clustered():
66+ if singlenode_mode:
67+ i += 1
68+ elif len(peer_units()) > 0 or is_clustered():
69 i += 1
70 return public_port - (i * 10)
71
72@@ -197,7 +206,7 @@
73 for setting in settings:
74 conf[setting] = config_get(setting)
75 missing = []
76- [missing.append(s) for s, v in conf.iteritems() if v is None]
77+ [missing.append(s) for s, v in six.iteritems(conf) if v is None]
78 if missing:
79 log('Insufficient config data to configure hacluster.', level=ERROR)
80 raise HAIncompleteConfig
81
82=== modified file 'hooks/charmhelpers/contrib/network/ip.py'
83--- hooks/charmhelpers/contrib/network/ip.py 2014-10-09 10:20:37 +0000
84+++ hooks/charmhelpers/contrib/network/ip.py 2014-11-26 10:42:22 +0000
85@@ -1,15 +1,12 @@
86 import glob
87 import re
88 import subprocess
89-import sys
90
91 from functools import partial
92
93 from charmhelpers.core.hookenv import unit_get
94 from charmhelpers.fetch import apt_install
95 from charmhelpers.core.hookenv import (
96- WARNING,
97- ERROR,
98 log
99 )
100
101@@ -34,31 +31,28 @@
102 network)
103
104
105+def no_ip_found_error_out(network):
106+ errmsg = ("No IP address found in network: %s" % network)
107+ raise ValueError(errmsg)
108+
109+
110 def get_address_in_network(network, fallback=None, fatal=False):
111- """
112- Get an IPv4 or IPv6 address within the network from the host.
113+ """Get an IPv4 or IPv6 address within the network from the host.
114
115 :param network (str): CIDR presentation format. For example,
116 '192.168.1.0/24'.
117 :param fallback (str): If no address is found, return fallback.
118 :param fatal (boolean): If no address is found, fallback is not
119 set and fatal is True then exit(1).
120-
121 """
122-
123- def not_found_error_out():
124- log("No IP address found in network: %s" % network,
125- level=ERROR)
126- sys.exit(1)
127-
128 if network is None:
129 if fallback is not None:
130 return fallback
131+
132+ if fatal:
133+ no_ip_found_error_out(network)
134 else:
135- if fatal:
136- not_found_error_out()
137- else:
138- return None
139+ return None
140
141 _validate_cidr(network)
142 network = netaddr.IPNetwork(network)
143@@ -70,6 +64,7 @@
144 cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
145 if cidr in network:
146 return str(cidr.ip)
147+
148 if network.version == 6 and netifaces.AF_INET6 in addresses:
149 for addr in addresses[netifaces.AF_INET6]:
150 if not addr['addr'].startswith('fe80'):
151@@ -82,20 +77,20 @@
152 return fallback
153
154 if fatal:
155- not_found_error_out()
156+ no_ip_found_error_out(network)
157
158 return None
159
160
161 def is_ipv6(address):
162- '''Determine whether provided address is IPv6 or not'''
163+ """Determine whether provided address is IPv6 or not."""
164 try:
165 address = netaddr.IPAddress(address)
166 except netaddr.AddrFormatError:
167 # probably a hostname - so not an address at all!
168 return False
169- else:
170- return address.version == 6
171+
172+ return address.version == 6
173
174
175 def is_address_in_network(network, address):
176@@ -113,11 +108,13 @@
177 except (netaddr.core.AddrFormatError, ValueError):
178 raise ValueError("Network (%s) is not in CIDR presentation format" %
179 network)
180+
181 try:
182 address = netaddr.IPAddress(address)
183 except (netaddr.core.AddrFormatError, ValueError):
184 raise ValueError("Address (%s) is not in correct presentation format" %
185 address)
186+
187 if address in network:
188 return True
189 else:
190@@ -147,6 +144,7 @@
191 return iface
192 else:
193 return addresses[netifaces.AF_INET][0][key]
194+
195 if address.version == 6 and netifaces.AF_INET6 in addresses:
196 for addr in addresses[netifaces.AF_INET6]:
197 if not addr['addr'].startswith('fe80'):
198@@ -160,41 +158,42 @@
199 return str(cidr).split('/')[1]
200 else:
201 return addr[key]
202+
203 return None
204
205
206 get_iface_for_address = partial(_get_for_address, key='iface')
207
208+
209 get_netmask_for_address = partial(_get_for_address, key='netmask')
210
211
212 def format_ipv6_addr(address):
213- """
214- IPv6 needs to be wrapped with [] in url link to parse correctly.
215+ """If address is IPv6, wrap it in '[]' otherwise return None.
216+
217+ This is required by most configuration files when specifying IPv6
218+ addresses.
219 """
220 if is_ipv6(address):
221- address = "[%s]" % address
222- else:
223- log("Not a valid ipv6 address: %s" % address, level=WARNING)
224- address = None
225+ return "[%s]" % address
226
227- return address
228+ return None
229
230
231 def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,
232 fatal=True, exc_list=None):
233- """
234- Return the assigned IP address for a given interface, if any, or [].
235- """
236+ """Return the assigned IP address for a given interface, if any."""
237 # Extract nic if passed /dev/ethX
238 if '/' in iface:
239 iface = iface.split('/')[-1]
240+
241 if not exc_list:
242 exc_list = []
243+
244 try:
245 inet_num = getattr(netifaces, inet_type)
246 except AttributeError:
247- raise Exception('Unknown inet type ' + str(inet_type))
248+ raise Exception("Unknown inet type '%s'" % str(inet_type))
249
250 interfaces = netifaces.interfaces()
251 if inc_aliases:
252@@ -202,15 +201,18 @@
253 for _iface in interfaces:
254 if iface == _iface or _iface.split(':')[0] == iface:
255 ifaces.append(_iface)
256+
257 if fatal and not ifaces:
258 raise Exception("Invalid interface '%s'" % iface)
259+
260 ifaces.sort()
261 else:
262 if iface not in interfaces:
263 if fatal:
264- raise Exception("%s not found " % (iface))
265+ raise Exception("Interface '%s' not found " % (iface))
266 else:
267 return []
268+
269 else:
270 ifaces = [iface]
271
272@@ -221,10 +223,13 @@
273 for entry in net_info[inet_num]:
274 if 'addr' in entry and entry['addr'] not in exc_list:
275 addresses.append(entry['addr'])
276+
277 if fatal and not addresses:
278 raise Exception("Interface '%s' doesn't have any %s addresses." %
279 (iface, inet_type))
280- return addresses
281+
282+ return sorted(addresses)
283+
284
285 get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET')
286
287@@ -241,6 +246,7 @@
288 raw = re.match(ll_key, _addr)
289 if raw:
290 _addr = raw.group(1)
291+
292 if _addr == addr:
293 log("Address '%s' is configured on iface '%s'" %
294 (addr, iface))
295@@ -251,8 +257,9 @@
296
297
298 def sniff_iface(f):
299- """If no iface provided, inject net iface inferred from unit private
300- address.
301+ """Ensure decorated function is called with a value for iface.
302+
303+ If no iface provided, inject net iface inferred from unit private address.
304 """
305 def iface_sniffer(*args, **kwargs):
306 if not kwargs.get('iface', None):
307@@ -295,7 +302,7 @@
308 if global_addrs:
309 # Make sure any found global addresses are not temporary
310 cmd = ['ip', 'addr', 'show', iface]
311- out = subprocess.check_output(cmd)
312+ out = subprocess.check_output(cmd).decode('UTF-8')
313 if dynamic_only:
314 key = re.compile("inet6 (.+)/[0-9]+ scope global dynamic.*")
315 else:
316@@ -317,33 +324,28 @@
317 return addrs
318
319 if fatal:
320- raise Exception("Interface '%s' doesn't have a scope global "
321+ raise Exception("Interface '%s' does not have a scope global "
322 "non-temporary ipv6 address." % iface)
323
324 return []
325
326
327 def get_bridges(vnic_dir='/sys/devices/virtual/net'):
328- """
329- Return a list of bridges on the system or []
330- """
331- b_rgex = vnic_dir + '/*/bridge'
332- return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_rgex)]
333+ """Return a list of bridges on the system."""
334+ b_regex = "%s/*/bridge" % vnic_dir
335+ return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_regex)]
336
337
338 def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'):
339- """
340- Return a list of nics comprising a given bridge on the system or []
341- """
342- brif_rgex = "%s/%s/brif/*" % (vnic_dir, bridge)
343- return [x.split('/')[-1] for x in glob.glob(brif_rgex)]
344+ """Return a list of nics comprising a given bridge on the system."""
345+ brif_regex = "%s/%s/brif/*" % (vnic_dir, bridge)
346+ return [x.split('/')[-1] for x in glob.glob(brif_regex)]
347
348
349 def is_bridge_member(nic):
350- """
351- Check if a given nic is a member of a bridge
352- """
353+ """Check if a given nic is a member of a bridge."""
354 for bridge in get_bridges():
355 if nic in get_bridge_nics(bridge):
356 return True
357+
358 return False
359
360=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
361--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2014-10-01 22:07:44 +0000
362+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2014-11-26 10:42:22 +0000
363@@ -1,3 +1,4 @@
364+import six
365 from charmhelpers.contrib.amulet.deployment import (
366 AmuletDeployment
367 )
368@@ -69,7 +70,7 @@
369
370 def _configure_services(self, configs):
371 """Configure all of the services."""
372- for service, config in configs.iteritems():
373+ for service, config in six.iteritems(configs):
374 self.d.configure(service, config)
375
376 def _get_openstack_release(self):
377
378=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
379--- hooks/charmhelpers/contrib/openstack/amulet/utils.py 2014-10-01 22:07:44 +0000
380+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py 2014-11-26 10:42:22 +0000
381@@ -7,6 +7,8 @@
382 import keystoneclient.v2_0 as keystone_client
383 import novaclient.v1_1.client as nova_client
384
385+import six
386+
387 from charmhelpers.contrib.amulet.utils import (
388 AmuletUtils
389 )
390@@ -60,7 +62,7 @@
391 expected service catalog endpoints.
392 """
393 self.log.debug('actual: {}'.format(repr(actual)))
394- for k, v in expected.iteritems():
395+ for k, v in six.iteritems(expected):
396 if k in actual:
397 ret = self._validate_dict_data(expected[k][0], actual[k][0])
398 if ret:
399
400=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
401--- hooks/charmhelpers/contrib/openstack/context.py 2014-10-07 12:27:23 +0000
402+++ hooks/charmhelpers/contrib/openstack/context.py 2014-11-26 10:42:22 +0000
403@@ -1,20 +1,18 @@
404 import json
405 import os
406 import time
407-
408 from base64 import b64decode
409+from subprocess import check_call
410
411-from subprocess import (
412- check_call
413-)
414+import six
415
416 from charmhelpers.fetch import (
417 apt_install,
418 filter_installed_packages,
419 )
420-
421 from charmhelpers.core.hookenv import (
422 config,
423+ is_relation_made,
424 local_unit,
425 log,
426 relation_get,
427@@ -23,43 +21,40 @@
428 relation_set,
429 unit_get,
430 unit_private_ip,
431+ DEBUG,
432+ INFO,
433+ WARNING,
434 ERROR,
435- INFO
436 )
437-
438 from charmhelpers.core.host import (
439 mkdir,
440- write_file
441+ write_file,
442 )
443-
444 from charmhelpers.contrib.hahelpers.cluster import (
445 determine_apache_port,
446 determine_api_port,
447 https,
448- is_clustered
449+ is_clustered,
450 )
451-
452 from charmhelpers.contrib.hahelpers.apache import (
453 get_cert,
454 get_ca_cert,
455 install_ca_cert,
456 )
457-
458 from charmhelpers.contrib.openstack.neutron import (
459 neutron_plugin_attribute,
460 )
461-
462 from charmhelpers.contrib.network.ip import (
463 get_address_in_network,
464 get_ipv6_addr,
465 get_netmask_for_address,
466 format_ipv6_addr,
467- is_address_in_network
468+ is_address_in_network,
469 )
470-
471 from charmhelpers.contrib.openstack.utils import get_host_ip
472
473 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
474+ADDRESS_TYPES = ['admin', 'internal', 'public']
475
476
477 class OSContextError(Exception):
478@@ -67,7 +62,7 @@
479
480
481 def ensure_packages(packages):
482- '''Install but do not upgrade required plugin packages'''
483+ """Install but do not upgrade required plugin packages."""
484 required = filter_installed_packages(packages)
485 if required:
486 apt_install(required, fatal=True)
487@@ -75,20 +70,27 @@
488
489 def context_complete(ctxt):
490 _missing = []
491- for k, v in ctxt.iteritems():
492+ for k, v in six.iteritems(ctxt):
493 if v is None or v == '':
494 _missing.append(k)
495+
496 if _missing:
497- log('Missing required data: %s' % ' '.join(_missing), level='INFO')
498+ log('Missing required data: %s' % ' '.join(_missing), level=INFO)
499 return False
500+
501 return True
502
503
504 def config_flags_parser(config_flags):
505+ """Parses config flags string into dict.
506+
507+ The provided config_flags string may be a list of comma-separated values
508+ which themselves may be comma-separated list of values.
509+ """
510 if config_flags.find('==') >= 0:
511- log("config_flags is not in expected format (key=value)",
512- level=ERROR)
513+ log("config_flags is not in expected format (key=value)", level=ERROR)
514 raise OSContextError
515+
516 # strip the following from each value.
517 post_strippers = ' ,'
518 # we strip any leading/trailing '=' or ' ' from the string then
519@@ -96,7 +98,7 @@
520 split = config_flags.strip(' =').split('=')
521 limit = len(split)
522 flags = {}
523- for i in xrange(0, limit - 1):
524+ for i in range(0, limit - 1):
525 current = split[i]
526 next = split[i + 1]
527 vindex = next.rfind(',')
528@@ -111,17 +113,18 @@
529 # if this not the first entry, expect an embedded key.
530 index = current.rfind(',')
531 if index < 0:
532- log("invalid config value(s) at index %s" % (i),
533- level=ERROR)
534+ log("Invalid config value(s) at index %s" % (i), level=ERROR)
535 raise OSContextError
536 key = current[index + 1:]
537
538 # Add to collection.
539 flags[key.strip(post_strippers)] = value.rstrip(post_strippers)
540+
541 return flags
542
543
544 class OSContextGenerator(object):
545+ """Base class for all context generators."""
546 interfaces = []
547
548 def __call__(self):
549@@ -133,11 +136,11 @@
550
551 def __init__(self,
552 database=None, user=None, relation_prefix=None, ssl_dir=None):
553- '''
554- Allows inspecting relation for settings prefixed with relation_prefix.
555- This is useful for parsing access for multiple databases returned via
556- the shared-db interface (eg, nova_password, quantum_password)
557- '''
558+ """Allows inspecting relation for settings prefixed with
559+ relation_prefix. This is useful for parsing access for multiple
560+ databases returned via the shared-db interface (eg, nova_password,
561+ quantum_password)
562+ """
563 self.relation_prefix = relation_prefix
564 self.database = database
565 self.user = user
566@@ -147,9 +150,8 @@
567 self.database = self.database or config('database')
568 self.user = self.user or config('database-user')
569 if None in [self.database, self.user]:
570- log('Could not generate shared_db context. '
571- 'Missing required charm config options. '
572- '(database name and user)')
573+ log("Could not generate shared_db context. Missing required charm "
574+ "config options. (database name and user)", level=ERROR)
575 raise OSContextError
576
577 ctxt = {}
578@@ -202,23 +204,24 @@
579 def __call__(self):
580 self.database = self.database or config('database')
581 if self.database is None:
582- log('Could not generate postgresql_db context. '
583- 'Missing required charm config options. '
584- '(database name)')
585+ log('Could not generate postgresql_db context. Missing required '
586+ 'charm config options. (database name)', level=ERROR)
587 raise OSContextError
588+
589 ctxt = {}
590-
591 for rid in relation_ids(self.interfaces[0]):
592 for unit in related_units(rid):
593- ctxt = {
594- 'database_host': relation_get('host', rid=rid, unit=unit),
595- 'database': self.database,
596- 'database_user': relation_get('user', rid=rid, unit=unit),
597- 'database_password': relation_get('password', rid=rid, unit=unit),
598- 'database_type': 'postgresql',
599- }
600+ rel_host = relation_get('host', rid=rid, unit=unit)
601+ rel_user = relation_get('user', rid=rid, unit=unit)
602+ rel_passwd = relation_get('password', rid=rid, unit=unit)
603+ ctxt = {'database_host': rel_host,
604+ 'database': self.database,
605+ 'database_user': rel_user,
606+ 'database_password': rel_passwd,
607+ 'database_type': 'postgresql'}
608 if context_complete(ctxt):
609 return ctxt
610+
611 return {}
612
613
614@@ -227,23 +230,29 @@
615 ca_path = os.path.join(ssl_dir, 'db-client.ca')
616 with open(ca_path, 'w') as fh:
617 fh.write(b64decode(rdata['ssl_ca']))
618+
619 ctxt['database_ssl_ca'] = ca_path
620 elif 'ssl_ca' in rdata:
621- log("Charm not setup for ssl support but ssl ca found")
622+ log("Charm not setup for ssl support but ssl ca found", level=INFO)
623 return ctxt
624+
625 if 'ssl_cert' in rdata:
626 cert_path = os.path.join(
627 ssl_dir, 'db-client.cert')
628 if not os.path.exists(cert_path):
629- log("Waiting 1m for ssl client cert validity")
630+ log("Waiting 1m for ssl client cert validity", level=INFO)
631 time.sleep(60)
632+
633 with open(cert_path, 'w') as fh:
634 fh.write(b64decode(rdata['ssl_cert']))
635+
636 ctxt['database_ssl_cert'] = cert_path
637 key_path = os.path.join(ssl_dir, 'db-client.key')
638 with open(key_path, 'w') as fh:
639 fh.write(b64decode(rdata['ssl_key']))
640+
641 ctxt['database_ssl_key'] = key_path
642+
643 return ctxt
644
645
646@@ -251,9 +260,8 @@
647 interfaces = ['identity-service']
648
649 def __call__(self):
650- log('Generating template context for identity-service')
651+ log('Generating template context for identity-service', level=DEBUG)
652 ctxt = {}
653-
654 for rid in relation_ids('identity-service'):
655 for unit in related_units(rid):
656 rdata = relation_get(rid=rid, unit=unit)
657@@ -261,26 +269,24 @@
658 serv_host = format_ipv6_addr(serv_host) or serv_host
659 auth_host = rdata.get('auth_host')
660 auth_host = format_ipv6_addr(auth_host) or auth_host
661-
662- ctxt = {
663- 'service_port': rdata.get('service_port'),
664- 'service_host': serv_host,
665- 'auth_host': auth_host,
666- 'auth_port': rdata.get('auth_port'),
667- 'admin_tenant_name': rdata.get('service_tenant'),
668- 'admin_user': rdata.get('service_username'),
669- 'admin_password': rdata.get('service_password'),
670- 'service_protocol':
671- rdata.get('service_protocol') or 'http',
672- 'auth_protocol':
673- rdata.get('auth_protocol') or 'http',
674- }
675+ svc_protocol = rdata.get('service_protocol') or 'http'
676+ auth_protocol = rdata.get('auth_protocol') or 'http'
677+ ctxt = {'service_port': rdata.get('service_port'),
678+ 'service_host': serv_host,
679+ 'auth_host': auth_host,
680+ 'auth_port': rdata.get('auth_port'),
681+ 'admin_tenant_name': rdata.get('service_tenant'),
682+ 'admin_user': rdata.get('service_username'),
683+ 'admin_password': rdata.get('service_password'),
684+ 'service_protocol': svc_protocol,
685+ 'auth_protocol': auth_protocol}
686 if context_complete(ctxt):
687 # NOTE(jamespage) this is required for >= icehouse
688 # so a missing value just indicates keystone needs
689 # upgrading
690 ctxt['admin_tenant_id'] = rdata.get('service_tenant_id')
691 return ctxt
692+
693 return {}
694
695
696@@ -293,21 +299,23 @@
697 self.interfaces = [rel_name]
698
699 def __call__(self):
700- log('Generating template context for amqp')
701+ log('Generating template context for amqp', level=DEBUG)
702 conf = config()
703- user_setting = 'rabbit-user'
704- vhost_setting = 'rabbit-vhost'
705 if self.relation_prefix:
706- user_setting = self.relation_prefix + '-rabbit-user'
707- vhost_setting = self.relation_prefix + '-rabbit-vhost'
708+ user_setting = '%s-rabbit-user' % (self.relation_prefix)
709+ vhost_setting = '%s-rabbit-vhost' % (self.relation_prefix)
710+ else:
711+ user_setting = 'rabbit-user'
712+ vhost_setting = 'rabbit-vhost'
713
714 try:
715 username = conf[user_setting]
716 vhost = conf[vhost_setting]
717 except KeyError as e:
718- log('Could not generate shared_db context. '
719- 'Missing required charm config options: %s.' % e)
720+ log('Could not generate shared_db context. Missing required charm '
721+ 'config options: %s.' % e, level=ERROR)
722 raise OSContextError
723+
724 ctxt = {}
725 for rid in relation_ids(self.rel_name):
726 ha_vip_only = False
727@@ -321,6 +329,7 @@
728 host = relation_get('private-address', rid=rid, unit=unit)
729 host = format_ipv6_addr(host) or host
730 ctxt['rabbitmq_host'] = host
731+
732 ctxt.update({
733 'rabbitmq_user': username,
734 'rabbitmq_password': relation_get('password', rid=rid,
735@@ -331,6 +340,7 @@
736 ssl_port = relation_get('ssl_port', rid=rid, unit=unit)
737 if ssl_port:
738 ctxt['rabbit_ssl_port'] = ssl_port
739+
740 ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit)
741 if ssl_ca:
742 ctxt['rabbit_ssl_ca'] = ssl_ca
743@@ -344,41 +354,45 @@
744 if context_complete(ctxt):
745 if 'rabbit_ssl_ca' in ctxt:
746 if not self.ssl_dir:
747- log(("Charm not setup for ssl support "
748- "but ssl ca found"))
749+ log("Charm not setup for ssl support but ssl ca "
750+ "found", level=INFO)
751 break
752+
753 ca_path = os.path.join(
754 self.ssl_dir, 'rabbit-client-ca.pem')
755 with open(ca_path, 'w') as fh:
756 fh.write(b64decode(ctxt['rabbit_ssl_ca']))
757 ctxt['rabbit_ssl_ca'] = ca_path
758+
759 # Sufficient information found = break out!
760 break
761+
762 # Used for active/active rabbitmq >= grizzly
763- if ('clustered' not in ctxt or ha_vip_only) \
764- and len(related_units(rid)) > 1:
765+ if (('clustered' not in ctxt or ha_vip_only) and
766+ len(related_units(rid)) > 1):
767 rabbitmq_hosts = []
768 for unit in related_units(rid):
769 host = relation_get('private-address', rid=rid, unit=unit)
770 host = format_ipv6_addr(host) or host
771 rabbitmq_hosts.append(host)
772- ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts)
773+
774+ ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts))
775+
776 if not context_complete(ctxt):
777 return {}
778- else:
779- return ctxt
780+
781+ return ctxt
782
783
784 class CephContext(OSContextGenerator):
785+ """Generates context for /etc/ceph/ceph.conf templates."""
786 interfaces = ['ceph']
787
788 def __call__(self):
789- '''This generates context for /etc/ceph/ceph.conf templates'''
790 if not relation_ids('ceph'):
791 return {}
792
793- log('Generating template context for ceph')
794-
795+ log('Generating template context for ceph', level=DEBUG)
796 mon_hosts = []
797 auth = None
798 key = None
799@@ -387,18 +401,18 @@
800 for unit in related_units(rid):
801 auth = relation_get('auth', rid=rid, unit=unit)
802 key = relation_get('key', rid=rid, unit=unit)
803- ceph_addr = \
804- relation_get('ceph-public-address', rid=rid, unit=unit) or \
805- relation_get('private-address', rid=rid, unit=unit)
806+ ceph_pub_addr = relation_get('ceph-public-address', rid=rid,
807+ unit=unit)
808+ unit_priv_addr = relation_get('private-address', rid=rid,
809+ unit=unit)
810+ ceph_addr = ceph_pub_addr or unit_priv_addr
811 ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
812 mon_hosts.append(ceph_addr)
813
814- ctxt = {
815- 'mon_hosts': ' '.join(mon_hosts),
816- 'auth': auth,
817- 'key': key,
818- 'use_syslog': use_syslog
819- }
820+ ctxt = {'mon_hosts': ' '.join(sorted(mon_hosts)),
821+ 'auth': auth,
822+ 'key': key,
823+ 'use_syslog': use_syslog}
824
825 if not os.path.isdir('/etc/ceph'):
826 os.mkdir('/etc/ceph')
827@@ -407,79 +421,68 @@
828 return {}
829
830 ensure_packages(['ceph-common'])
831-
832 return ctxt
833
834
835-ADDRESS_TYPES = ['admin', 'internal', 'public']
836-
837-
838 class HAProxyContext(OSContextGenerator):
839+ """Provides half a context for the haproxy template, which describes
840+ all peers to be included in the cluster. Each charm needs to include
841+ its own context generator that describes the port mapping.
842+ """
843 interfaces = ['cluster']
844
845+ def __init__(self, singlenode_mode=False):
846+ self.singlenode_mode = singlenode_mode
847+
848 def __call__(self):
849- '''
850- Builds half a context for the haproxy template, which describes
851- all peers to be included in the cluster. Each charm needs to include
852- its own context generator that describes the port mapping.
853- '''
854- if not relation_ids('cluster'):
855+ if not relation_ids('cluster') and not self.singlenode_mode:
856 return {}
857
858- l_unit = local_unit().replace('/', '-')
859-
860 if config('prefer-ipv6'):
861 addr = get_ipv6_addr(exc_list=[config('vip')])[0]
862 else:
863 addr = get_host_ip(unit_get('private-address'))
864
865+ l_unit = local_unit().replace('/', '-')
866 cluster_hosts = {}
867
868 # NOTE(jamespage): build out map of configured network endpoints
869 # and associated backends
870 for addr_type in ADDRESS_TYPES:
871- laddr = get_address_in_network(
872- config('os-{}-network'.format(addr_type)))
873+ cfg_opt = 'os-{}-network'.format(addr_type)
874+ laddr = get_address_in_network(config(cfg_opt))
875 if laddr:
876- cluster_hosts[laddr] = {}
877- cluster_hosts[laddr]['network'] = "{}/{}".format(
878- laddr,
879- get_netmask_for_address(laddr)
880- )
881- cluster_hosts[laddr]['backends'] = {}
882- cluster_hosts[laddr]['backends'][l_unit] = laddr
883+ netmask = get_netmask_for_address(laddr)
884+ cluster_hosts[laddr] = {'network': "{}/{}".format(laddr,
885+ netmask),
886+ 'backends': {l_unit: laddr}}
887 for rid in relation_ids('cluster'):
888 for unit in related_units(rid):
889- _unit = unit.replace('/', '-')
890 _laddr = relation_get('{}-address'.format(addr_type),
891 rid=rid, unit=unit)
892 if _laddr:
893+ _unit = unit.replace('/', '-')
894 cluster_hosts[laddr]['backends'][_unit] = _laddr
895
896 # NOTE(jamespage) no split configurations found, just use
897 # private addresses
898 if not cluster_hosts:
899- cluster_hosts[addr] = {}
900- cluster_hosts[addr]['network'] = "{}/{}".format(
901- addr,
902- get_netmask_for_address(addr)
903- )
904- cluster_hosts[addr]['backends'] = {}
905- cluster_hosts[addr]['backends'][l_unit] = addr
906+ netmask = get_netmask_for_address(addr)
907+ cluster_hosts[addr] = {'network': "{}/{}".format(addr, netmask),
908+ 'backends': {l_unit: addr}}
909 for rid in relation_ids('cluster'):
910 for unit in related_units(rid):
911- _unit = unit.replace('/', '-')
912 _laddr = relation_get('private-address',
913 rid=rid, unit=unit)
914 if _laddr:
915+ _unit = unit.replace('/', '-')
916 cluster_hosts[addr]['backends'][_unit] = _laddr
917
918- ctxt = {
919- 'frontends': cluster_hosts,
920- }
921+ ctxt = {'frontends': cluster_hosts}
922
923 if config('haproxy-server-timeout'):
924 ctxt['haproxy_server_timeout'] = config('haproxy-server-timeout')
925+
926 if config('haproxy-client-timeout'):
927 ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout')
928
929@@ -493,13 +496,18 @@
930 ctxt['stat_port'] = ':8888'
931
932 for frontend in cluster_hosts:
933- if len(cluster_hosts[frontend]['backends']) > 1:
934+ if (len(cluster_hosts[frontend]['backends']) > 1 or
935+ self.singlenode_mode):
936 # Enable haproxy when we have enough peers.
937- log('Ensuring haproxy enabled in /etc/default/haproxy.')
938+ log('Ensuring haproxy enabled in /etc/default/haproxy.',
939+ level=DEBUG)
940 with open('/etc/default/haproxy', 'w') as out:
941 out.write('ENABLED=1\n')
942+
943 return ctxt
944- log('HAProxy context is incomplete, this unit has no peers.')
945+
946+ log('HAProxy context is incomplete, this unit has no peers.',
947+ level=INFO)
948 return {}
949
950
951@@ -507,29 +515,28 @@
952 interfaces = ['image-service']
953
954 def __call__(self):
955- '''
956- Obtains the glance API server from the image-service relation. Useful
957- in nova and cinder (currently).
958- '''
959- log('Generating template context for image-service.')
960+ """Obtains the glance API server from the image-service relation.
961+ Useful in nova and cinder (currently).
962+ """
963+ log('Generating template context for image-service.', level=DEBUG)
964 rids = relation_ids('image-service')
965 if not rids:
966 return {}
967+
968 for rid in rids:
969 for unit in related_units(rid):
970 api_server = relation_get('glance-api-server',
971 rid=rid, unit=unit)
972 if api_server:
973 return {'glance_api_servers': api_server}
974- log('ImageService context is incomplete. '
975- 'Missing required relation data.')
976+
977+ log("ImageService context is incomplete. Missing required relation "
978+ "data.", level=INFO)
979 return {}
980
981
982 class ApacheSSLContext(OSContextGenerator):
983-
984- """
985- Generates a context for an apache vhost configuration that configures
986+ """Generates a context for an apache vhost configuration that configures
987 HTTPS reverse proxying for one or many endpoints. Generated context
988 looks something like::
989
990@@ -563,6 +570,7 @@
991 else:
992 cert_filename = 'cert'
993 key_filename = 'key'
994+
995 write_file(path=os.path.join(ssl_dir, cert_filename),
996 content=b64decode(cert))
997 write_file(path=os.path.join(ssl_dir, key_filename),
998@@ -574,7 +582,8 @@
999 install_ca_cert(b64decode(ca_cert))
1000
1001 def canonical_names(self):
1002- '''Figure out which canonical names clients will access this service'''
1003+ """Figure out which canonical names clients will access this service.
1004+ """
1005 cns = []
1006 for r_id in relation_ids('identity-service'):
1007 for unit in related_units(r_id):
1008@@ -582,55 +591,80 @@
1009 for k in rdata:
1010 if k.startswith('ssl_key_'):
1011 cns.append(k.lstrip('ssl_key_'))
1012- return list(set(cns))
1013+
1014+ return sorted(list(set(cns)))
1015+
1016+ def get_network_addresses(self):
1017+ """For each network configured, return corresponding address and vip
1018+ (if available).
1019+
1020+ Returns a list of tuples of the form:
1021+
1022+ [(address_in_net_a, vip_in_net_a),
1023+ (address_in_net_b, vip_in_net_b),
1024+ ...]
1025+
1026+ or, if no vip(s) available:
1027+
1028+ [(address_in_net_a, address_in_net_a),
1029+ (address_in_net_b, address_in_net_b),
1030+ ...]
1031+ """
1032+ addresses = []
1033+ if config('vip'):
1034+ vips = config('vip').split()
1035+ else:
1036+ vips = []
1037+
1038+ for net_type in ['os-internal-network', 'os-admin-network',
1039+ 'os-public-network']:
1040+ addr = get_address_in_network(config(net_type),
1041+ unit_get('private-address'))
1042+ if len(vips) > 1 and is_clustered():
1043+ if not config(net_type):
1044+ log("Multiple networks configured but net_type "
1045+ "is None (%s)." % net_type, level=WARNING)
1046+ continue
1047+
1048+ for vip in vips:
1049+ if is_address_in_network(config(net_type), vip):
1050+ addresses.append((addr, vip))
1051+ break
1052+
1053+ elif is_clustered() and config('vip'):
1054+ addresses.append((addr, config('vip')))
1055+ else:
1056+ addresses.append((addr, addr))
1057+
1058+ return sorted(addresses)
1059
1060 def __call__(self):
1061- if isinstance(self.external_ports, basestring):
1062+ if isinstance(self.external_ports, six.string_types):
1063 self.external_ports = [self.external_ports]
1064- if (not self.external_ports or not https()):
1065+
1066+ if not self.external_ports or not https():
1067 return {}
1068
1069 self.configure_ca()
1070 self.enable_modules()
1071
1072- ctxt = {
1073- 'namespace': self.service_namespace,
1074- 'endpoints': [],
1075- 'ext_ports': []
1076- }
1077+ ctxt = {'namespace': self.service_namespace,
1078+ 'endpoints': [],
1079+ 'ext_ports': []}
1080
1081 for cn in self.canonical_names():
1082 self.configure_cert(cn)
1083
1084- addresses = []
1085- vips = []
1086- if config('vip'):
1087- vips = config('vip').split()
1088-
1089- for network_type in ['os-internal-network',
1090- 'os-admin-network',
1091- 'os-public-network']:
1092- address = get_address_in_network(config(network_type),
1093- unit_get('private-address'))
1094- if len(vips) > 0 and is_clustered():
1095- for vip in vips:
1096- if is_address_in_network(config(network_type),
1097- vip):
1098- addresses.append((address, vip))
1099- break
1100- elif is_clustered():
1101- addresses.append((address, config('vip')))
1102- else:
1103- addresses.append((address, address))
1104-
1105- for address, endpoint in set(addresses):
1106+ addresses = self.get_network_addresses()
1107+ for address, endpoint in sorted(set(addresses)):
1108 for api_port in self.external_ports:
1109 ext_port = determine_apache_port(api_port)
1110 int_port = determine_api_port(api_port)
1111 portmap = (address, endpoint, int(ext_port), int(int_port))
1112 ctxt['endpoints'].append(portmap)
1113 ctxt['ext_ports'].append(int(ext_port))
1114- ctxt['ext_ports'] = list(set(ctxt['ext_ports']))
1115+
1116+ ctxt['ext_ports'] = sorted(list(set(ctxt['ext_ports'])))
1117 return ctxt
1118
1119
1120@@ -647,21 +681,23 @@
1121
1122 @property
1123 def packages(self):
1124- return neutron_plugin_attribute(
1125- self.plugin, 'packages', self.network_manager)
1126+ return neutron_plugin_attribute(self.plugin, 'packages',
1127+ self.network_manager)
1128
1129 @property
1130 def neutron_security_groups(self):
1131 return None
1132
1133 def _ensure_packages(self):
1134- [ensure_packages(pkgs) for pkgs in self.packages]
1135+ for pkgs in self.packages:
1136+ ensure_packages(pkgs)
1137
1138 def _save_flag_file(self):
1139 if self.network_manager == 'quantum':
1140 _file = '/etc/nova/quantum_plugin.conf'
1141 else:
1142 _file = '/etc/nova/neutron_plugin.conf'
1143+
1144 with open(_file, 'wb') as out:
1145 out.write(self.plugin + '\n')
1146
1147@@ -670,13 +706,11 @@
1148 self.network_manager)
1149 config = neutron_plugin_attribute(self.plugin, 'config',
1150 self.network_manager)
1151- ovs_ctxt = {
1152- 'core_plugin': driver,
1153- 'neutron_plugin': 'ovs',
1154- 'neutron_security_groups': self.neutron_security_groups,
1155- 'local_ip': unit_private_ip(),
1156- 'config': config
1157- }
1158+ ovs_ctxt = {'core_plugin': driver,
1159+ 'neutron_plugin': 'ovs',
1160+ 'neutron_security_groups': self.neutron_security_groups,
1161+ 'local_ip': unit_private_ip(),
1162+ 'config': config}
1163
1164 return ovs_ctxt
1165
1166@@ -685,13 +719,11 @@
1167 self.network_manager)
1168 config = neutron_plugin_attribute(self.plugin, 'config',
1169 self.network_manager)
1170- nvp_ctxt = {
1171- 'core_plugin': driver,
1172- 'neutron_plugin': 'nvp',
1173- 'neutron_security_groups': self.neutron_security_groups,
1174- 'local_ip': unit_private_ip(),
1175- 'config': config
1176- }
1177+ nvp_ctxt = {'core_plugin': driver,
1178+ 'neutron_plugin': 'nvp',
1179+ 'neutron_security_groups': self.neutron_security_groups,
1180+ 'local_ip': unit_private_ip(),
1181+ 'config': config}
1182
1183 return nvp_ctxt
1184
1185@@ -700,35 +732,50 @@
1186 self.network_manager)
1187 n1kv_config = neutron_plugin_attribute(self.plugin, 'config',
1188 self.network_manager)
1189- n1kv_ctxt = {
1190- 'core_plugin': driver,
1191- 'neutron_plugin': 'n1kv',
1192- 'neutron_security_groups': self.neutron_security_groups,
1193- 'local_ip': unit_private_ip(),
1194- 'config': n1kv_config,
1195- 'vsm_ip': config('n1kv-vsm-ip'),
1196- 'vsm_username': config('n1kv-vsm-username'),
1197- 'vsm_password': config('n1kv-vsm-password'),
1198- 'restrict_policy_profiles': config(
1199- 'n1kv_restrict_policy_profiles'),
1200- }
1201+ n1kv_user_config_flags = config('n1kv-config-flags')
1202+ restrict_policy_profiles = config('n1kv-restrict-policy-profiles')
1203+ n1kv_ctxt = {'core_plugin': driver,
1204+ 'neutron_plugin': 'n1kv',
1205+ 'neutron_security_groups': self.neutron_security_groups,
1206+ 'local_ip': unit_private_ip(),
1207+ 'config': n1kv_config,
1208+ 'vsm_ip': config('n1kv-vsm-ip'),
1209+ 'vsm_username': config('n1kv-vsm-username'),
1210+ 'vsm_password': config('n1kv-vsm-password'),
1211+ 'restrict_policy_profiles': restrict_policy_profiles}
1212+
1213+ if n1kv_user_config_flags:
1214+ flags = config_flags_parser(n1kv_user_config_flags)
1215+ n1kv_ctxt['user_config_flags'] = flags
1216
1217 return n1kv_ctxt
1218
1219+ def calico_ctxt(self):
1220+ driver = neutron_plugin_attribute(self.plugin, 'driver',
1221+ self.network_manager)
1222+ config = neutron_plugin_attribute(self.plugin, 'config',
1223+ self.network_manager)
1224+ calico_ctxt = {'core_plugin': driver,
1225+ 'neutron_plugin': 'Calico',
1226+ 'neutron_security_groups': self.neutron_security_groups,
1227+ 'local_ip': unit_private_ip(),
1228+ 'config': config}
1229+
1230+ return calico_ctxt
1231+
1232 def neutron_ctxt(self):
1233 if https():
1234 proto = 'https'
1235 else:
1236 proto = 'http'
1237+
1238 if is_clustered():
1239 host = config('vip')
1240 else:
1241 host = unit_get('private-address')
1242- url = '%s://%s:%s' % (proto, host, '9696')
1243- ctxt = {
1244- 'network_manager': self.network_manager,
1245- 'neutron_url': url,
1246- }
1247+
1248+ ctxt = {'network_manager': self.network_manager,
1249+ 'neutron_url': '%s://%s:%s' % (proto, host, '9696')}
1250 return ctxt
1251
1252 def __call__(self):
1253@@ -748,6 +795,8 @@
1254 ctxt.update(self.nvp_ctxt())
1255 elif self.plugin == 'n1kv':
1256 ctxt.update(self.n1kv_ctxt())
1257+ elif self.plugin == 'Calico':
1258+ ctxt.update(self.calico_ctxt())
1259
1260 alchemy_flags = config('neutron-alchemy-flags')
1261 if alchemy_flags:
1262@@ -759,23 +808,40 @@
1263
1264
1265 class OSConfigFlagContext(OSContextGenerator):
1266-
1267- """
1268- Responsible for adding user-defined config-flags in charm config to a
1269- template context.
1270+ """Provides support for user-defined config flags.
1271+
1272+ Users can define a comma-seperated list of key=value pairs
1273+ in the charm configuration and apply them at any point in
1274+ any file by using a template flag.
1275+
1276+ Sometimes users might want config flags inserted within a
1277+ specific section so this class allows users to specify the
1278+ template flag name, allowing for multiple template flags
1279+ (sections) within the same context.
1280
1281 NOTE: the value of config-flags may be a comma-separated list of
1282 key=value pairs and some Openstack config files support
1283 comma-separated lists as values.
1284 """
1285
1286+ def __init__(self, charm_flag='config-flags',
1287+ template_flag='user_config_flags'):
1288+ """
1289+ :param charm_flag: config flags in charm configuration.
1290+ :param template_flag: insert point for user-defined flags in template
1291+ file.
1292+ """
1293+ super(OSConfigFlagContext, self).__init__()
1294+ self._charm_flag = charm_flag
1295+ self._template_flag = template_flag
1296+
1297 def __call__(self):
1298- config_flags = config('config-flags')
1299+ config_flags = config(self._charm_flag)
1300 if not config_flags:
1301 return {}
1302
1303- flags = config_flags_parser(config_flags)
1304- return {'user_config_flags': flags}
1305+ return {self._template_flag:
1306+ config_flags_parser(config_flags)}
1307
1308
1309 class SubordinateConfigContext(OSContextGenerator):
1310@@ -819,7 +885,6 @@
1311 },
1312 }
1313 }
1314-
1315 """
1316
1317 def __init__(self, service, config_file, interface):
1318@@ -849,26 +914,28 @@
1319
1320 if self.service not in sub_config:
1321 log('Found subordinate_config on %s but it contained'
1322- 'nothing for %s service' % (rid, self.service))
1323+ 'nothing for %s service' % (rid, self.service),
1324+ level=INFO)
1325 continue
1326
1327 sub_config = sub_config[self.service]
1328 if self.config_file not in sub_config:
1329 log('Found subordinate_config on %s but it contained'
1330- 'nothing for %s' % (rid, self.config_file))
1331+ 'nothing for %s' % (rid, self.config_file),
1332+ level=INFO)
1333 continue
1334
1335 sub_config = sub_config[self.config_file]
1336- for k, v in sub_config.iteritems():
1337+ for k, v in six.iteritems(sub_config):
1338 if k == 'sections':
1339- for section, config_dict in v.iteritems():
1340- log("adding section '%s'" % (section))
1341+ for section, config_dict in six.iteritems(v):
1342+ log("adding section '%s'" % (section),
1343+ level=DEBUG)
1344 ctxt[k][section] = config_dict
1345 else:
1346 ctxt[k] = v
1347
1348- log("%d section(s) found" % (len(ctxt['sections'])), level=INFO)
1349-
1350+ log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
1351 return ctxt
1352
1353
1354@@ -880,15 +947,14 @@
1355 False if config('debug') is None else config('debug')
1356 ctxt['verbose'] = \
1357 False if config('verbose') is None else config('verbose')
1358+
1359 return ctxt
1360
1361
1362 class SyslogContext(OSContextGenerator):
1363
1364 def __call__(self):
1365- ctxt = {
1366- 'use_syslog': config('use-syslog')
1367- }
1368+ ctxt = {'use_syslog': config('use-syslog')}
1369 return ctxt
1370
1371
1372@@ -896,13 +962,9 @@
1373
1374 def __call__(self):
1375 if config('prefer-ipv6'):
1376- return {
1377- 'bind_host': '::'
1378- }
1379+ return {'bind_host': '::'}
1380 else:
1381- return {
1382- 'bind_host': '0.0.0.0'
1383- }
1384+ return {'bind_host': '0.0.0.0'}
1385
1386
1387 class WorkerConfigContext(OSContextGenerator):
1388@@ -914,11 +976,42 @@
1389 except ImportError:
1390 apt_install('python-psutil', fatal=True)
1391 from psutil import NUM_CPUS
1392+
1393 return NUM_CPUS
1394
1395 def __call__(self):
1396- multiplier = config('worker-multiplier') or 1
1397- ctxt = {
1398- "workers": self.num_cpus * multiplier
1399- }
1400+ multiplier = config('worker-multiplier') or 0
1401+ ctxt = {"workers": self.num_cpus * multiplier}
1402+ return ctxt
1403+
1404+
1405+class ZeroMQContext(OSContextGenerator):
1406+ interfaces = ['zeromq-configuration']
1407+
1408+ def __call__(self):
1409+ ctxt = {}
1410+ if is_relation_made('zeromq-configuration', 'host'):
1411+ for rid in relation_ids('zeromq-configuration'):
1412+ for unit in related_units(rid):
1413+ ctxt['zmq_nonce'] = relation_get('nonce', unit, rid)
1414+ ctxt['zmq_host'] = relation_get('host', unit, rid)
1415+
1416+ return ctxt
1417+
1418+
1419+class NotificationDriverContext(OSContextGenerator):
1420+
1421+ def __init__(self, zmq_relation='zeromq-configuration',
1422+ amqp_relation='amqp'):
1423+ """
1424+ :param zmq_relation: Name of Zeromq relation to check
1425+ """
1426+ self.zmq_relation = zmq_relation
1427+ self.amqp_relation = amqp_relation
1428+
1429+ def __call__(self):
1430+ ctxt = {'notifications': 'False'}
1431+ if is_relation_made(self.amqp_relation):
1432+ ctxt['notifications'] = "True"
1433+
1434 return ctxt
1435
1436=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
1437--- hooks/charmhelpers/contrib/openstack/neutron.py 2014-07-25 08:11:52 +0000
1438+++ hooks/charmhelpers/contrib/openstack/neutron.py 2014-11-26 10:42:22 +0000
1439@@ -14,7 +14,7 @@
1440 def headers_package():
1441 """Ensures correct linux-headers for running kernel are installed,
1442 for building DKMS package"""
1443- kver = check_output(['uname', '-r']).strip()
1444+ kver = check_output(['uname', '-r']).decode('UTF-8').strip()
1445 return 'linux-headers-%s' % kver
1446
1447 QUANTUM_CONF_DIR = '/etc/quantum'
1448@@ -22,7 +22,7 @@
1449
1450 def kernel_version():
1451 """ Retrieve the current major kernel version as a tuple e.g. (3, 13) """
1452- kver = check_output(['uname', '-r']).strip()
1453+ kver = check_output(['uname', '-r']).decode('UTF-8').strip()
1454 kver = kver.split('.')
1455 return (int(kver[0]), int(kver[1]))
1456
1457@@ -138,10 +138,25 @@
1458 relation_prefix='neutron',
1459 ssl_dir=NEUTRON_CONF_DIR)],
1460 'services': [],
1461- 'packages': [['neutron-plugin-cisco']],
1462+ 'packages': [[headers_package()] + determine_dkms_package(),
1463+ ['neutron-plugin-cisco']],
1464 'server_packages': ['neutron-server',
1465 'neutron-plugin-cisco'],
1466 'server_services': ['neutron-server']
1467+ },
1468+ 'Calico': {
1469+ 'config': '/etc/neutron/plugins/ml2/ml2_conf.ini',
1470+ 'driver': 'neutron.plugins.ml2.plugin.Ml2Plugin',
1471+ 'contexts': [
1472+ context.SharedDBContext(user=config('neutron-database-user'),
1473+ database=config('neutron-database'),
1474+ relation_prefix='neutron',
1475+ ssl_dir=NEUTRON_CONF_DIR)],
1476+ 'services': ['calico-compute', 'bird', 'neutron-dhcp-agent'],
1477+ 'packages': [[headers_package()] + determine_dkms_package(),
1478+ ['calico-compute', 'bird', 'neutron-dhcp-agent']],
1479+ 'server_packages': ['neutron-server', 'calico-control'],
1480+ 'server_services': ['neutron-server']
1481 }
1482 }
1483 if release >= 'icehouse':
1484@@ -162,7 +177,8 @@
1485 elif manager == 'neutron':
1486 plugins = neutron_plugins()
1487 else:
1488- log('Error: Network manager does not support plugins.')
1489+ log("Network manager '%s' does not support plugins." % (manager),
1490+ level=ERROR)
1491 raise Exception
1492
1493 try:
1494
1495=== modified file 'hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg'
1496--- hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg 2014-10-01 22:07:44 +0000
1497+++ hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg 2014-11-26 10:42:22 +0000
1498@@ -35,7 +35,7 @@
1499 stats auth admin:password
1500
1501 {% if frontends -%}
1502-{% for service, ports in service_ports.iteritems() -%}
1503+{% for service, ports in service_ports.items() -%}
1504 frontend tcp-in_{{ service }}
1505 bind *:{{ ports[0] }}
1506 bind :::{{ ports[0] }}
1507@@ -46,7 +46,7 @@
1508 {% for frontend in frontends -%}
1509 backend {{ service }}_{{ frontend }}
1510 balance leastconn
1511- {% for unit, address in frontends[frontend]['backends'].iteritems() -%}
1512+ {% for unit, address in frontends[frontend]['backends'].items() -%}
1513 server {{ unit }} {{ address }}:{{ ports[1] }} check
1514 {% endfor %}
1515 {% endfor -%}
1516
1517=== modified file 'hooks/charmhelpers/contrib/openstack/templating.py'
1518--- hooks/charmhelpers/contrib/openstack/templating.py 2014-07-25 08:11:52 +0000
1519+++ hooks/charmhelpers/contrib/openstack/templating.py 2014-11-26 10:42:22 +0000
1520@@ -1,13 +1,13 @@
1521 import os
1522
1523+import six
1524+
1525 from charmhelpers.fetch import apt_install
1526-
1527 from charmhelpers.core.hookenv import (
1528 log,
1529 ERROR,
1530 INFO
1531 )
1532-
1533 from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES
1534
1535 try:
1536@@ -43,7 +43,7 @@
1537 order by OpenStack release.
1538 """
1539 tmpl_dirs = [(rel, os.path.join(templates_dir, rel))
1540- for rel in OPENSTACK_CODENAMES.itervalues()]
1541+ for rel in six.itervalues(OPENSTACK_CODENAMES)]
1542
1543 if not os.path.isdir(templates_dir):
1544 log('Templates directory not found @ %s.' % templates_dir,
1545@@ -258,7 +258,7 @@
1546 """
1547 Write out all registered config files.
1548 """
1549- [self.write(k) for k in self.templates.iterkeys()]
1550+ [self.write(k) for k in six.iterkeys(self.templates)]
1551
1552 def set_release(self, openstack_release):
1553 """
1554@@ -275,5 +275,5 @@
1555 '''
1556 interfaces = []
1557 [interfaces.extend(i.complete_contexts())
1558- for i in self.templates.itervalues()]
1559+ for i in six.itervalues(self.templates)]
1560 return interfaces
1561
1562=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
1563--- hooks/charmhelpers/contrib/openstack/utils.py 2014-10-06 21:49:00 +0000
1564+++ hooks/charmhelpers/contrib/openstack/utils.py 2014-11-26 10:42:22 +0000
1565@@ -2,6 +2,7 @@
1566
1567 # Common python helper functions used for OpenStack charms.
1568 from collections import OrderedDict
1569+from functools import wraps
1570
1571 import subprocess
1572 import json
1573@@ -9,6 +10,8 @@
1574 import socket
1575 import sys
1576
1577+import six
1578+
1579 from charmhelpers.core.hookenv import (
1580 config,
1581 log as juju_log,
1582@@ -112,7 +115,7 @@
1583
1584 # Best guess match based on deb string provided
1585 if src.startswith('deb') or src.startswith('ppa'):
1586- for k, v in OPENSTACK_CODENAMES.iteritems():
1587+ for k, v in six.iteritems(OPENSTACK_CODENAMES):
1588 if v in src:
1589 return v
1590
1591@@ -133,7 +136,7 @@
1592
1593 def get_os_version_codename(codename):
1594 '''Determine OpenStack version number from codename.'''
1595- for k, v in OPENSTACK_CODENAMES.iteritems():
1596+ for k, v in six.iteritems(OPENSTACK_CODENAMES):
1597 if v == codename:
1598 return k
1599 e = 'Could not derive OpenStack version for '\
1600@@ -193,7 +196,7 @@
1601 else:
1602 vers_map = OPENSTACK_CODENAMES
1603
1604- for version, cname in vers_map.iteritems():
1605+ for version, cname in six.iteritems(vers_map):
1606 if cname == codename:
1607 return version
1608 # e = "Could not determine OpenStack version for package: %s" % pkg
1609@@ -317,7 +320,7 @@
1610 rc_script.write(
1611 "#!/bin/bash\n")
1612 [rc_script.write('export %s=%s\n' % (u, p))
1613- for u, p in env_vars.iteritems() if u != "script_path"]
1614+ for u, p in six.iteritems(env_vars) if u != "script_path"]
1615
1616
1617 def openstack_upgrade_available(package):
1618@@ -417,7 +420,7 @@
1619
1620 if isinstance(address, dns.name.Name):
1621 rtype = 'PTR'
1622- elif isinstance(address, basestring):
1623+ elif isinstance(address, six.string_types):
1624 rtype = 'A'
1625 else:
1626 return None
1627@@ -468,6 +471,14 @@
1628 return result.split('.')[0]
1629
1630
1631+def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'):
1632+ mm_map = {}
1633+ if os.path.isfile(mm_file):
1634+ with open(mm_file, 'r') as f:
1635+ mm_map = json.load(f)
1636+ return mm_map
1637+
1638+
1639 def sync_db_with_multi_ipv6_addresses(database, database_user,
1640 relation_prefix=None):
1641 hosts = get_ipv6_addr(dynamic_only=False)
1642@@ -477,10 +488,24 @@
1643 'hostname': json.dumps(hosts)}
1644
1645 if relation_prefix:
1646- keys = kwargs.keys()
1647- for key in keys:
1648+ for key in list(kwargs.keys()):
1649 kwargs["%s_%s" % (relation_prefix, key)] = kwargs[key]
1650 del kwargs[key]
1651
1652 for rid in relation_ids('shared-db'):
1653 relation_set(relation_id=rid, **kwargs)
1654+
1655+
1656+def os_requires_version(ostack_release, pkg):
1657+ """
1658+ Decorator for hook to specify minimum supported release
1659+ """
1660+ def wrap(f):
1661+ @wraps(f)
1662+ def wrapped_f(*args):
1663+ if os_release(pkg) < ostack_release:
1664+ raise Exception("This hook is not supported on releases"
1665+ " before %s" % ostack_release)
1666+ f(*args)
1667+ return wrapped_f
1668+ return wrap
1669
1670=== modified file 'hooks/charmhelpers/contrib/storage/linux/ceph.py'
1671--- hooks/charmhelpers/contrib/storage/linux/ceph.py 2014-07-25 08:11:52 +0000
1672+++ hooks/charmhelpers/contrib/storage/linux/ceph.py 2014-11-26 10:42:22 +0000
1673@@ -16,19 +16,18 @@
1674 from subprocess import (
1675 check_call,
1676 check_output,
1677- CalledProcessError
1678+ CalledProcessError,
1679 )
1680-
1681 from charmhelpers.core.hookenv import (
1682 relation_get,
1683 relation_ids,
1684 related_units,
1685 log,
1686+ DEBUG,
1687 INFO,
1688 WARNING,
1689- ERROR
1690+ ERROR,
1691 )
1692-
1693 from charmhelpers.core.host import (
1694 mount,
1695 mounts,
1696@@ -37,7 +36,6 @@
1697 service_running,
1698 umount,
1699 )
1700-
1701 from charmhelpers.fetch import (
1702 apt_install,
1703 )
1704@@ -56,99 +54,85 @@
1705
1706
1707 def install():
1708- ''' Basic Ceph client installation '''
1709+ """Basic Ceph client installation."""
1710 ceph_dir = "/etc/ceph"
1711 if not os.path.exists(ceph_dir):
1712 os.mkdir(ceph_dir)
1713+
1714 apt_install('ceph-common', fatal=True)
1715
1716
1717 def rbd_exists(service, pool, rbd_img):
1718- ''' Check to see if a RADOS block device exists '''
1719+ """Check to see if a RADOS block device exists."""
1720 try:
1721- out = check_output(['rbd', 'list', '--id', service,
1722- '--pool', pool])
1723+ out = check_output(['rbd', 'list', '--id',
1724+ service, '--pool', pool]).decode('UTF-8')
1725 except CalledProcessError:
1726 return False
1727- else:
1728- return rbd_img in out
1729+
1730+ return rbd_img in out
1731
1732
1733 def create_rbd_image(service, pool, image, sizemb):
1734- ''' Create a new RADOS block device '''
1735- cmd = [
1736- 'rbd',
1737- 'create',
1738- image,
1739- '--size',
1740- str(sizemb),
1741- '--id',
1742- service,
1743- '--pool',
1744- pool
1745- ]
1746+ """Create a new RADOS block device."""
1747+ cmd = ['rbd', 'create', image, '--size', str(sizemb), '--id', service,
1748+ '--pool', pool]
1749 check_call(cmd)
1750
1751
1752 def pool_exists(service, name):
1753- ''' Check to see if a RADOS pool already exists '''
1754+ """Check to see if a RADOS pool already exists."""
1755 try:
1756- out = check_output(['rados', '--id', service, 'lspools'])
1757+ out = check_output(['rados', '--id', service,
1758+ 'lspools']).decode('UTF-8')
1759 except CalledProcessError:
1760 return False
1761- else:
1762- return name in out
1763+
1764+ return name in out
1765
1766
1767 def get_osds(service):
1768- '''
1769- Return a list of all Ceph Object Storage Daemons
1770- currently in the cluster
1771- '''
1772+ """Return a list of all Ceph Object Storage Daemons currently in the
1773+ cluster.
1774+ """
1775 version = ceph_version()
1776 if version and version >= '0.56':
1777 return json.loads(check_output(['ceph', '--id', service,
1778- 'osd', 'ls', '--format=json']))
1779- else:
1780- return None
1781-
1782-
1783-def create_pool(service, name, replicas=2):
1784- ''' Create a new RADOS pool '''
1785+ 'osd', 'ls',
1786+ '--format=json']).decode('UTF-8'))
1787+
1788+ return None
1789+
1790+
1791+def create_pool(service, name, replicas=3):
1792+ """Create a new RADOS pool."""
1793 if pool_exists(service, name):
1794 log("Ceph pool {} already exists, skipping creation".format(name),
1795 level=WARNING)
1796 return
1797+
1798 # Calculate the number of placement groups based
1799 # on upstream recommended best practices.
1800 osds = get_osds(service)
1801 if osds:
1802- pgnum = (len(osds) * 100 / replicas)
1803+ pgnum = (len(osds) * 100 // replicas)
1804 else:
1805 # NOTE(james-page): Default to 200 for older ceph versions
1806 # which don't support OSD query from cli
1807 pgnum = 200
1808- cmd = [
1809- 'ceph', '--id', service,
1810- 'osd', 'pool', 'create',
1811- name, str(pgnum)
1812- ]
1813+
1814+ cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pgnum)]
1815 check_call(cmd)
1816- cmd = [
1817- 'ceph', '--id', service,
1818- 'osd', 'pool', 'set', name,
1819- 'size', str(replicas)
1820- ]
1821+
1822+ cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', name, 'size',
1823+ str(replicas)]
1824 check_call(cmd)
1825
1826
1827 def delete_pool(service, name):
1828- ''' Delete a RADOS pool from ceph '''
1829- cmd = [
1830- 'ceph', '--id', service,
1831- 'osd', 'pool', 'delete',
1832- name, '--yes-i-really-really-mean-it'
1833- ]
1834+ """Delete a RADOS pool from ceph."""
1835+ cmd = ['ceph', '--id', service, 'osd', 'pool', 'delete', name,
1836+ '--yes-i-really-really-mean-it']
1837 check_call(cmd)
1838
1839
1840@@ -161,44 +145,43 @@
1841
1842
1843 def create_keyring(service, key):
1844- ''' Create a new Ceph keyring containing key'''
1845+ """Create a new Ceph keyring containing key."""
1846 keyring = _keyring_path(service)
1847 if os.path.exists(keyring):
1848- log('ceph: Keyring exists at %s.' % keyring, level=WARNING)
1849+ log('Ceph keyring exists at %s.' % keyring, level=WARNING)
1850 return
1851- cmd = [
1852- 'ceph-authtool',
1853- keyring,
1854- '--create-keyring',
1855- '--name=client.{}'.format(service),
1856- '--add-key={}'.format(key)
1857- ]
1858+
1859+ cmd = ['ceph-authtool', keyring, '--create-keyring',
1860+ '--name=client.{}'.format(service), '--add-key={}'.format(key)]
1861 check_call(cmd)
1862- log('ceph: Created new ring at %s.' % keyring, level=INFO)
1863+ log('Created new ceph keyring at %s.' % keyring, level=DEBUG)
1864
1865
1866 def create_key_file(service, key):
1867- ''' Create a file containing key '''
1868+ """Create a file containing key."""
1869 keyfile = _keyfile_path(service)
1870 if os.path.exists(keyfile):
1871- log('ceph: Keyfile exists at %s.' % keyfile, level=WARNING)
1872+ log('Keyfile exists at %s.' % keyfile, level=WARNING)
1873 return
1874+
1875 with open(keyfile, 'w') as fd:
1876 fd.write(key)
1877- log('ceph: Created new keyfile at %s.' % keyfile, level=INFO)
1878+
1879+ log('Created new keyfile at %s.' % keyfile, level=INFO)
1880
1881
1882 def get_ceph_nodes():
1883- ''' Query named relation 'ceph' to detemine current nodes '''
1884+ """Query named relation 'ceph' to determine current nodes."""
1885 hosts = []
1886 for r_id in relation_ids('ceph'):
1887 for unit in related_units(r_id):
1888 hosts.append(relation_get('private-address', unit=unit, rid=r_id))
1889+
1890 return hosts
1891
1892
1893 def configure(service, key, auth, use_syslog):
1894- ''' Perform basic configuration of Ceph '''
1895+ """Perform basic configuration of Ceph."""
1896 create_keyring(service, key)
1897 create_key_file(service, key)
1898 hosts = get_ceph_nodes()
1899@@ -211,17 +194,17 @@
1900
1901
1902 def image_mapped(name):
1903- ''' Determine whether a RADOS block device is mapped locally '''
1904+ """Determine whether a RADOS block device is mapped locally."""
1905 try:
1906- out = check_output(['rbd', 'showmapped'])
1907+ out = check_output(['rbd', 'showmapped']).decode('UTF-8')
1908 except CalledProcessError:
1909 return False
1910- else:
1911- return name in out
1912+
1913+ return name in out
1914
1915
1916 def map_block_storage(service, pool, image):
1917- ''' Map a RADOS block device for local use '''
1918+ """Map a RADOS block device for local use."""
1919 cmd = [
1920 'rbd',
1921 'map',
1922@@ -235,31 +218,32 @@
1923
1924
1925 def filesystem_mounted(fs):
1926- ''' Determine whether a filesytems is already mounted '''
1927+ """Determine whether a filesytems is already mounted."""
1928 return fs in [f for f, m in mounts()]
1929
1930
1931 def make_filesystem(blk_device, fstype='ext4', timeout=10):
1932- ''' Make a new filesystem on the specified block device '''
1933+ """Make a new filesystem on the specified block device."""
1934 count = 0
1935 e_noent = os.errno.ENOENT
1936 while not os.path.exists(blk_device):
1937 if count >= timeout:
1938- log('ceph: gave up waiting on block device %s' % blk_device,
1939+ log('Gave up waiting on block device %s' % blk_device,
1940 level=ERROR)
1941 raise IOError(e_noent, os.strerror(e_noent), blk_device)
1942- log('ceph: waiting for block device %s to appear' % blk_device,
1943- level=INFO)
1944+
1945+ log('Waiting for block device %s to appear' % blk_device,
1946+ level=DEBUG)
1947 count += 1
1948 time.sleep(1)
1949 else:
1950- log('ceph: Formatting block device %s as filesystem %s.' %
1951+ log('Formatting block device %s as filesystem %s.' %
1952 (blk_device, fstype), level=INFO)
1953 check_call(['mkfs', '-t', fstype, blk_device])
1954
1955
1956 def place_data_on_block_device(blk_device, data_src_dst):
1957- ''' Migrate data in data_src_dst to blk_device and then remount '''
1958+ """Migrate data in data_src_dst to blk_device and then remount."""
1959 # mount block device into /mnt
1960 mount(blk_device, '/mnt')
1961 # copy data to /mnt
1962@@ -279,8 +263,8 @@
1963
1964 # TODO: re-use
1965 def modprobe(module):
1966- ''' Load a kernel module and configure for auto-load on reboot '''
1967- log('ceph: Loading kernel module', level=INFO)
1968+ """Load a kernel module and configure for auto-load on reboot."""
1969+ log('Loading kernel module', level=INFO)
1970 cmd = ['modprobe', module]
1971 check_call(cmd)
1972 with open('/etc/modules', 'r+') as modules:
1973@@ -289,7 +273,7 @@
1974
1975
1976 def copy_files(src, dst, symlinks=False, ignore=None):
1977- ''' Copy files from src to dst '''
1978+ """Copy files from src to dst."""
1979 for item in os.listdir(src):
1980 s = os.path.join(src, item)
1981 d = os.path.join(dst, item)
1982@@ -300,9 +284,9 @@
1983
1984
1985 def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
1986- blk_device, fstype, system_services=[]):
1987- """
1988- NOTE: This function must only be called from a single service unit for
1989+ blk_device, fstype, system_services=[],
1990+ replicas=3):
1991+ """NOTE: This function must only be called from a single service unit for
1992 the same rbd_img otherwise data loss will occur.
1993
1994 Ensures given pool and RBD image exists, is mapped to a block device,
1995@@ -316,15 +300,16 @@
1996 """
1997 # Ensure pool, RBD image, RBD mappings are in place.
1998 if not pool_exists(service, pool):
1999- log('ceph: Creating new pool {}.'.format(pool))
2000- create_pool(service, pool)
2001+ log('Creating new pool {}.'.format(pool), level=INFO)
2002+ create_pool(service, pool, replicas=replicas)
2003
2004 if not rbd_exists(service, pool, rbd_img):
2005- log('ceph: Creating RBD image ({}).'.format(rbd_img))
2006+ log('Creating RBD image ({}).'.format(rbd_img), level=INFO)
2007 create_rbd_image(service, pool, rbd_img, sizemb)
2008
2009 if not image_mapped(rbd_img):
2010- log('ceph: Mapping RBD Image {} as a Block Device.'.format(rbd_img))
2011+ log('Mapping RBD Image {} as a Block Device.'.format(rbd_img),
2012+ level=INFO)
2013 map_block_storage(service, pool, rbd_img)
2014
2015 # make file system
2016@@ -339,45 +324,47 @@
2017
2018 for svc in system_services:
2019 if service_running(svc):
2020- log('ceph: Stopping services {} prior to migrating data.'
2021- .format(svc))
2022+ log('Stopping services {} prior to migrating data.'
2023+ .format(svc), level=DEBUG)
2024 service_stop(svc)
2025
2026 place_data_on_block_device(blk_device, mount_point)
2027
2028 for svc in system_services:
2029- log('ceph: Starting service {} after migrating data.'
2030- .format(svc))
2031+ log('Starting service {} after migrating data.'
2032+ .format(svc), level=DEBUG)
2033 service_start(svc)
2034
2035
2036 def ensure_ceph_keyring(service, user=None, group=None):
2037- '''
2038- Ensures a ceph keyring is created for a named service
2039- and optionally ensures user and group ownership.
2040+ """Ensures a ceph keyring is created for a named service and optionally
2041+ ensures user and group ownership.
2042
2043 Returns False if no ceph key is available in relation state.
2044- '''
2045+ """
2046 key = None
2047 for rid in relation_ids('ceph'):
2048 for unit in related_units(rid):
2049 key = relation_get('key', rid=rid, unit=unit)
2050 if key:
2051 break
2052+
2053 if not key:
2054 return False
2055+
2056 create_keyring(service=service, key=key)
2057 keyring = _keyring_path(service)
2058 if user and group:
2059 check_call(['chown', '%s.%s' % (user, group), keyring])
2060+
2061 return True
2062
2063
2064 def ceph_version():
2065- ''' Retrieve the local version of ceph '''
2066+ """Retrieve the local version of ceph."""
2067 if os.path.exists('/usr/bin/ceph'):
2068 cmd = ['ceph', '-v']
2069- output = check_output(cmd)
2070+ output = check_output(cmd).decode('US-ASCII')
2071 output = output.split()
2072 if len(output) > 3:
2073 return output[2]
2074
2075=== modified file 'hooks/charmhelpers/contrib/storage/linux/loopback.py'
2076--- hooks/charmhelpers/contrib/storage/linux/loopback.py 2013-10-17 21:48:08 +0000
2077+++ hooks/charmhelpers/contrib/storage/linux/loopback.py 2014-11-26 10:42:22 +0000
2078@@ -1,12 +1,12 @@
2079-
2080 import os
2081 import re
2082-
2083 from subprocess import (
2084 check_call,
2085 check_output,
2086 )
2087
2088+import six
2089+
2090
2091 ##################################################
2092 # loopback device helpers.
2093@@ -37,7 +37,7 @@
2094 '''
2095 file_path = os.path.abspath(file_path)
2096 check_call(['losetup', '--find', file_path])
2097- for d, f in loopback_devices().iteritems():
2098+ for d, f in six.iteritems(loopback_devices()):
2099 if f == file_path:
2100 return d
2101
2102@@ -51,7 +51,7 @@
2103
2104 :returns: str: Full path to the ensured loopback device (eg, /dev/loop0)
2105 '''
2106- for d, f in loopback_devices().iteritems():
2107+ for d, f in six.iteritems(loopback_devices()):
2108 if f == path:
2109 return d
2110
2111
2112=== modified file 'hooks/charmhelpers/contrib/storage/linux/lvm.py'
2113--- hooks/charmhelpers/contrib/storage/linux/lvm.py 2014-05-19 11:43:29 +0000
2114+++ hooks/charmhelpers/contrib/storage/linux/lvm.py 2014-11-26 10:42:22 +0000
2115@@ -61,6 +61,7 @@
2116 vg = None
2117 pvd = check_output(['pvdisplay', block_device]).splitlines()
2118 for l in pvd:
2119+ l = l.decode('UTF-8')
2120 if l.strip().startswith('VG Name'):
2121 vg = ' '.join(l.strip().split()[2:])
2122 return vg
2123
2124=== modified file 'hooks/charmhelpers/contrib/storage/linux/utils.py'
2125--- hooks/charmhelpers/contrib/storage/linux/utils.py 2014-09-18 12:47:52 +0000
2126+++ hooks/charmhelpers/contrib/storage/linux/utils.py 2014-11-26 10:42:22 +0000
2127@@ -30,7 +30,8 @@
2128 # sometimes sgdisk exits non-zero; this is OK, dd will clean up
2129 call(['sgdisk', '--zap-all', '--mbrtogpt',
2130 '--clear', block_device])
2131- dev_end = check_output(['blockdev', '--getsz', block_device])
2132+ dev_end = check_output(['blockdev', '--getsz',
2133+ block_device]).decode('UTF-8')
2134 gpt_end = int(dev_end.split()[0]) - 100
2135 check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
2136 'bs=1M', 'count=1'])
2137@@ -47,7 +48,7 @@
2138 it doesn't.
2139 '''
2140 is_partition = bool(re.search(r".*[0-9]+\b", device))
2141- out = check_output(['mount'])
2142+ out = check_output(['mount']).decode('UTF-8')
2143 if is_partition:
2144 return bool(re.search(device + r"\b", out))
2145 return bool(re.search(device + r"[0-9]+\b", out))
2146
2147=== modified file 'hooks/charmhelpers/core/fstab.py'
2148--- hooks/charmhelpers/core/fstab.py 2014-07-03 12:44:32 +0000
2149+++ hooks/charmhelpers/core/fstab.py 2014-11-26 10:42:22 +0000
2150@@ -3,10 +3,11 @@
2151
2152 __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
2153
2154+import io
2155 import os
2156
2157
2158-class Fstab(file):
2159+class Fstab(io.FileIO):
2160 """This class extends file in order to implement a file reader/writer
2161 for file `/etc/fstab`
2162 """
2163@@ -24,8 +25,8 @@
2164 options = "defaults"
2165
2166 self.options = options
2167- self.d = d
2168- self.p = p
2169+ self.d = int(d)
2170+ self.p = int(p)
2171
2172 def __eq__(self, o):
2173 return str(self) == str(o)
2174@@ -45,7 +46,7 @@
2175 self._path = path
2176 else:
2177 self._path = self.DEFAULT_PATH
2178- file.__init__(self, self._path, 'r+')
2179+ super(Fstab, self).__init__(self._path, 'rb+')
2180
2181 def _hydrate_entry(self, line):
2182 # NOTE: use split with no arguments to split on any
2183@@ -58,8 +59,9 @@
2184 def entries(self):
2185 self.seek(0)
2186 for line in self.readlines():
2187+ line = line.decode('us-ascii')
2188 try:
2189- if not line.startswith("#"):
2190+ if line.strip() and not line.startswith("#"):
2191 yield self._hydrate_entry(line)
2192 except ValueError:
2193 pass
2194@@ -75,14 +77,14 @@
2195 if self.get_entry_by_attr('device', entry.device):
2196 return False
2197
2198- self.write(str(entry) + '\n')
2199+ self.write((str(entry) + '\n').encode('us-ascii'))
2200 self.truncate()
2201 return entry
2202
2203 def remove_entry(self, entry):
2204 self.seek(0)
2205
2206- lines = self.readlines()
2207+ lines = [l.decode('us-ascii') for l in self.readlines()]
2208
2209 found = False
2210 for index, line in enumerate(lines):
2211@@ -97,7 +99,7 @@
2212 lines.remove(line)
2213
2214 self.seek(0)
2215- self.write(''.join(lines))
2216+ self.write(''.join(lines).encode('us-ascii'))
2217 self.truncate()
2218 return True
2219
2220
2221=== modified file 'hooks/charmhelpers/core/hookenv.py'
2222--- hooks/charmhelpers/core/hookenv.py 2014-10-01 22:07:44 +0000
2223+++ hooks/charmhelpers/core/hookenv.py 2014-11-26 10:42:22 +0000
2224@@ -9,9 +9,14 @@
2225 import yaml
2226 import subprocess
2227 import sys
2228-import UserDict
2229 from subprocess import CalledProcessError
2230
2231+import six
2232+if not six.PY3:
2233+ from UserDict import UserDict
2234+else:
2235+ from collections import UserDict
2236+
2237 CRITICAL = "CRITICAL"
2238 ERROR = "ERROR"
2239 WARNING = "WARNING"
2240@@ -67,12 +72,12 @@
2241 subprocess.call(command)
2242
2243
2244-class Serializable(UserDict.IterableUserDict):
2245+class Serializable(UserDict):
2246 """Wrapper, an object that can be serialized to yaml or json"""
2247
2248 def __init__(self, obj):
2249 # wrap the object
2250- UserDict.IterableUserDict.__init__(self)
2251+ UserDict.__init__(self)
2252 self.data = obj
2253
2254 def __getattr__(self, attr):
2255@@ -214,6 +219,12 @@
2256 except KeyError:
2257 return (self._prev_dict or {})[key]
2258
2259+ def keys(self):
2260+ prev_keys = []
2261+ if self._prev_dict is not None:
2262+ prev_keys = self._prev_dict.keys()
2263+ return list(set(prev_keys + list(dict.keys(self))))
2264+
2265 def load_previous(self, path=None):
2266 """Load previous copy of config from disk.
2267
2268@@ -263,7 +274,7 @@
2269
2270 """
2271 if self._prev_dict:
2272- for k, v in self._prev_dict.iteritems():
2273+ for k, v in six.iteritems(self._prev_dict):
2274 if k not in self:
2275 self[k] = v
2276 with open(self.path, 'w') as f:
2277@@ -278,7 +289,8 @@
2278 config_cmd_line.append(scope)
2279 config_cmd_line.append('--format=json')
2280 try:
2281- config_data = json.loads(subprocess.check_output(config_cmd_line))
2282+ config_data = json.loads(
2283+ subprocess.check_output(config_cmd_line).decode('UTF-8'))
2284 if scope is not None:
2285 return config_data
2286 return Config(config_data)
2287@@ -297,10 +309,10 @@
2288 if unit:
2289 _args.append(unit)
2290 try:
2291- return json.loads(subprocess.check_output(_args))
2292+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
2293 except ValueError:
2294 return None
2295- except CalledProcessError, e:
2296+ except CalledProcessError as e:
2297 if e.returncode == 2:
2298 return None
2299 raise
2300@@ -312,7 +324,7 @@
2301 relation_cmd_line = ['relation-set']
2302 if relation_id is not None:
2303 relation_cmd_line.extend(('-r', relation_id))
2304- for k, v in (relation_settings.items() + kwargs.items()):
2305+ for k, v in (list(relation_settings.items()) + list(kwargs.items())):
2306 if v is None:
2307 relation_cmd_line.append('{}='.format(k))
2308 else:
2309@@ -329,7 +341,8 @@
2310 relid_cmd_line = ['relation-ids', '--format=json']
2311 if reltype is not None:
2312 relid_cmd_line.append(reltype)
2313- return json.loads(subprocess.check_output(relid_cmd_line)) or []
2314+ return json.loads(
2315+ subprocess.check_output(relid_cmd_line).decode('UTF-8')) or []
2316 return []
2317
2318
2319@@ -340,7 +353,8 @@
2320 units_cmd_line = ['relation-list', '--format=json']
2321 if relid is not None:
2322 units_cmd_line.extend(('-r', relid))
2323- return json.loads(subprocess.check_output(units_cmd_line)) or []
2324+ return json.loads(
2325+ subprocess.check_output(units_cmd_line).decode('UTF-8')) or []
2326
2327
2328 @cached
2329@@ -449,7 +463,7 @@
2330 """Get the unit ID for the remote unit"""
2331 _args = ['unit-get', '--format=json', attribute]
2332 try:
2333- return json.loads(subprocess.check_output(_args))
2334+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
2335 except ValueError:
2336 return None
2337
2338
2339=== modified file 'hooks/charmhelpers/core/host.py'
2340--- hooks/charmhelpers/core/host.py 2014-10-01 22:07:44 +0000
2341+++ hooks/charmhelpers/core/host.py 2014-11-26 10:42:22 +0000
2342@@ -6,19 +6,20 @@
2343 # Matthew Wedgwood <matthew.wedgwood@canonical.com>
2344
2345 import os
2346+import re
2347 import pwd
2348 import grp
2349 import random
2350 import string
2351 import subprocess
2352 import hashlib
2353-import shutil
2354 from contextlib import contextmanager
2355-
2356 from collections import OrderedDict
2357
2358-from hookenv import log
2359-from fstab import Fstab
2360+import six
2361+
2362+from .hookenv import log
2363+from .fstab import Fstab
2364
2365
2366 def service_start(service_name):
2367@@ -54,7 +55,9 @@
2368 def service_running(service):
2369 """Determine whether a system service is running"""
2370 try:
2371- output = subprocess.check_output(['service', service, 'status'], stderr=subprocess.STDOUT)
2372+ output = subprocess.check_output(
2373+ ['service', service, 'status'],
2374+ stderr=subprocess.STDOUT).decode('UTF-8')
2375 except subprocess.CalledProcessError:
2376 return False
2377 else:
2378@@ -67,7 +70,9 @@
2379 def service_available(service_name):
2380 """Determine whether a system service is available"""
2381 try:
2382- subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT)
2383+ subprocess.check_output(
2384+ ['service', service_name, 'status'],
2385+ stderr=subprocess.STDOUT).decode('UTF-8')
2386 except subprocess.CalledProcessError as e:
2387 return 'unrecognized service' not in e.output
2388 else:
2389@@ -115,7 +120,7 @@
2390 cmd.append(from_path)
2391 cmd.append(to_path)
2392 log(" ".join(cmd))
2393- return subprocess.check_output(cmd).strip()
2394+ return subprocess.check_output(cmd).decode('UTF-8').strip()
2395
2396
2397 def symlink(source, destination):
2398@@ -130,7 +135,7 @@
2399 subprocess.check_call(cmd)
2400
2401
2402-def mkdir(path, owner='root', group='root', perms=0555, force=False):
2403+def mkdir(path, owner='root', group='root', perms=0o555, force=False):
2404 """Create a directory"""
2405 log("Making dir {} {}:{} {:o}".format(path, owner, group,
2406 perms))
2407@@ -146,7 +151,7 @@
2408 os.chown(realpath, uid, gid)
2409
2410
2411-def write_file(path, content, owner='root', group='root', perms=0444):
2412+def write_file(path, content, owner='root', group='root', perms=0o444):
2413 """Create or overwrite a file with the contents of a string"""
2414 log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
2415 uid = pwd.getpwnam(owner).pw_uid
2416@@ -177,7 +182,7 @@
2417 cmd_args.extend([device, mountpoint])
2418 try:
2419 subprocess.check_output(cmd_args)
2420- except subprocess.CalledProcessError, e:
2421+ except subprocess.CalledProcessError as e:
2422 log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output))
2423 return False
2424
2425@@ -191,7 +196,7 @@
2426 cmd_args = ['umount', mountpoint]
2427 try:
2428 subprocess.check_output(cmd_args)
2429- except subprocess.CalledProcessError, e:
2430+ except subprocess.CalledProcessError as e:
2431 log('Error unmounting {}\n{}'.format(mountpoint, e.output))
2432 return False
2433
2434@@ -218,8 +223,8 @@
2435 """
2436 if os.path.exists(path):
2437 h = getattr(hashlib, hash_type)()
2438- with open(path, 'r') as source:
2439- h.update(source.read()) # IGNORE:E1101 - it does have update
2440+ with open(path, 'rb') as source:
2441+ h.update(source.read())
2442 return h.hexdigest()
2443 else:
2444 return None
2445@@ -297,7 +302,7 @@
2446 if length is None:
2447 length = random.choice(range(35, 45))
2448 alphanumeric_chars = [
2449- l for l in (string.letters + string.digits)
2450+ l for l in (string.ascii_letters + string.digits)
2451 if l not in 'l0QD1vAEIOUaeiou']
2452 random_chars = [
2453 random.choice(alphanumeric_chars) for _ in range(length)]
2454@@ -306,18 +311,24 @@
2455
2456 def list_nics(nic_type):
2457 '''Return a list of nics of given type(s)'''
2458- if isinstance(nic_type, basestring):
2459+ if isinstance(nic_type, six.string_types):
2460 int_types = [nic_type]
2461 else:
2462 int_types = nic_type
2463 interfaces = []
2464 for int_type in int_types:
2465 cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
2466- ip_output = subprocess.check_output(cmd).split('\n')
2467+ ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
2468 ip_output = (line for line in ip_output if line)
2469 for line in ip_output:
2470 if line.split()[1].startswith(int_type):
2471- interfaces.append(line.split()[1].replace(":", ""))
2472+ matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line)
2473+ if matched:
2474+ interface = matched.groups()[0]
2475+ else:
2476+ interface = line.split()[1].replace(":", "")
2477+ interfaces.append(interface)
2478+
2479 return interfaces
2480
2481
2482@@ -329,7 +340,7 @@
2483
2484 def get_nic_mtu(nic):
2485 cmd = ['ip', 'addr', 'show', nic]
2486- ip_output = subprocess.check_output(cmd).split('\n')
2487+ ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
2488 mtu = ""
2489 for line in ip_output:
2490 words = line.split()
2491@@ -340,7 +351,7 @@
2492
2493 def get_nic_hwaddr(nic):
2494 cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
2495- ip_output = subprocess.check_output(cmd)
2496+ ip_output = subprocess.check_output(cmd).decode('UTF-8')
2497 hwaddr = ""
2498 words = ip_output.split()
2499 if 'link/ether' in words:
2500
2501=== modified file 'hooks/charmhelpers/core/services/__init__.py'
2502--- hooks/charmhelpers/core/services/__init__.py 2014-08-13 13:10:26 +0000
2503+++ hooks/charmhelpers/core/services/__init__.py 2014-11-26 10:42:22 +0000
2504@@ -1,2 +1,2 @@
2505-from .base import *
2506-from .helpers import *
2507+from .base import * # NOQA
2508+from .helpers import * # NOQA
2509
2510=== modified file 'hooks/charmhelpers/core/services/helpers.py'
2511--- hooks/charmhelpers/core/services/helpers.py 2014-10-01 22:07:44 +0000
2512+++ hooks/charmhelpers/core/services/helpers.py 2014-11-26 10:42:22 +0000
2513@@ -196,7 +196,7 @@
2514 if not os.path.isabs(file_name):
2515 file_name = os.path.join(hookenv.charm_dir(), file_name)
2516 with open(file_name, 'w') as file_stream:
2517- os.fchmod(file_stream.fileno(), 0600)
2518+ os.fchmod(file_stream.fileno(), 0o600)
2519 yaml.dump(config_data, file_stream)
2520
2521 def read_context(self, file_name):
2522@@ -211,15 +211,19 @@
2523
2524 class TemplateCallback(ManagerCallback):
2525 """
2526- Callback class that will render a Jinja2 template, for use as a ready action.
2527-
2528- :param str source: The template source file, relative to `$CHARM_DIR/templates`
2529+ Callback class that will render a Jinja2 template, for use as a ready
2530+ action.
2531+
2532+ :param str source: The template source file, relative to
2533+ `$CHARM_DIR/templates`
2534+
2535 :param str target: The target to write the rendered template to
2536 :param str owner: The owner of the rendered file
2537 :param str group: The group of the rendered file
2538 :param int perms: The permissions of the rendered file
2539 """
2540- def __init__(self, source, target, owner='root', group='root', perms=0444):
2541+ def __init__(self, source, target,
2542+ owner='root', group='root', perms=0o444):
2543 self.source = source
2544 self.target = target
2545 self.owner = owner
2546
2547=== modified file 'hooks/charmhelpers/core/templating.py'
2548--- hooks/charmhelpers/core/templating.py 2014-08-13 13:10:26 +0000
2549+++ hooks/charmhelpers/core/templating.py 2014-11-26 10:42:22 +0000
2550@@ -4,7 +4,8 @@
2551 from charmhelpers.core import hookenv
2552
2553
2554-def render(source, target, context, owner='root', group='root', perms=0444, templates_dir=None):
2555+def render(source, target, context, owner='root', group='root',
2556+ perms=0o444, templates_dir=None):
2557 """
2558 Render a template.
2559
2560
2561=== modified file 'hooks/charmhelpers/fetch/__init__.py'
2562--- hooks/charmhelpers/fetch/__init__.py 2014-10-01 22:07:44 +0000
2563+++ hooks/charmhelpers/fetch/__init__.py 2014-11-26 10:42:22 +0000
2564@@ -5,10 +5,6 @@
2565 from charmhelpers.core.host import (
2566 lsb_release
2567 )
2568-from urlparse import (
2569- urlparse,
2570- urlunparse,
2571-)
2572 import subprocess
2573 from charmhelpers.core.hookenv import (
2574 config,
2575@@ -16,6 +12,12 @@
2576 )
2577 import os
2578
2579+import six
2580+if six.PY3:
2581+ from urllib.parse import urlparse, urlunparse
2582+else:
2583+ from urlparse import urlparse, urlunparse
2584+
2585
2586 CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
2587 deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
2588@@ -72,6 +74,7 @@
2589 FETCH_HANDLERS = (
2590 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
2591 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
2592+ 'charmhelpers.fetch.giturl.GitUrlFetchHandler',
2593 )
2594
2595 APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
2596@@ -148,7 +151,7 @@
2597 cmd = ['apt-get', '--assume-yes']
2598 cmd.extend(options)
2599 cmd.append('install')
2600- if isinstance(packages, basestring):
2601+ if isinstance(packages, six.string_types):
2602 cmd.append(packages)
2603 else:
2604 cmd.extend(packages)
2605@@ -181,7 +184,7 @@
2606 def apt_purge(packages, fatal=False):
2607 """Purge one or more packages"""
2608 cmd = ['apt-get', '--assume-yes', 'purge']
2609- if isinstance(packages, basestring):
2610+ if isinstance(packages, six.string_types):
2611 cmd.append(packages)
2612 else:
2613 cmd.extend(packages)
2614@@ -192,7 +195,7 @@
2615 def apt_hold(packages, fatal=False):
2616 """Hold one or more packages"""
2617 cmd = ['apt-mark', 'hold']
2618- if isinstance(packages, basestring):
2619+ if isinstance(packages, six.string_types):
2620 cmd.append(packages)
2621 else:
2622 cmd.extend(packages)
2623@@ -218,6 +221,7 @@
2624 pocket for the release.
2625 'cloud:' may be used to activate official cloud archive pockets,
2626 such as 'cloud:icehouse'
2627+ 'distro' may be used as a noop
2628
2629 @param key: A key to be added to the system's APT keyring and used
2630 to verify the signatures on packages. Ideally, this should be an
2631@@ -251,12 +255,14 @@
2632 release = lsb_release()['DISTRIB_CODENAME']
2633 with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
2634 apt.write(PROPOSED_POCKET.format(release))
2635+ elif source == 'distro':
2636+ pass
2637 else:
2638- raise SourceConfigError("Unknown source: {!r}".format(source))
2639+ log("Unknown source: {!r}".format(source))
2640
2641 if key:
2642 if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
2643- with NamedTemporaryFile() as key_file:
2644+ with NamedTemporaryFile('w+') as key_file:
2645 key_file.write(key)
2646 key_file.flush()
2647 key_file.seek(0)
2648@@ -293,14 +299,14 @@
2649 sources = safe_load((config(sources_var) or '').strip()) or []
2650 keys = safe_load((config(keys_var) or '').strip()) or None
2651
2652- if isinstance(sources, basestring):
2653+ if isinstance(sources, six.string_types):
2654 sources = [sources]
2655
2656 if keys is None:
2657 for source in sources:
2658 add_source(source, None)
2659 else:
2660- if isinstance(keys, basestring):
2661+ if isinstance(keys, six.string_types):
2662 keys = [keys]
2663
2664 if len(sources) != len(keys):
2665@@ -397,7 +403,7 @@
2666 while result is None or result == APT_NO_LOCK:
2667 try:
2668 result = subprocess.check_call(cmd, env=env)
2669- except subprocess.CalledProcessError, e:
2670+ except subprocess.CalledProcessError as e:
2671 retry_count = retry_count + 1
2672 if retry_count > APT_NO_LOCK_RETRY_COUNT:
2673 raise
2674
2675=== modified file 'hooks/charmhelpers/fetch/archiveurl.py'
2676--- hooks/charmhelpers/fetch/archiveurl.py 2014-10-01 22:07:44 +0000
2677+++ hooks/charmhelpers/fetch/archiveurl.py 2014-11-26 10:42:22 +0000
2678@@ -1,8 +1,23 @@
2679 import os
2680-import urllib2
2681-from urllib import urlretrieve
2682-import urlparse
2683 import hashlib
2684+import re
2685+
2686+import six
2687+if six.PY3:
2688+ from urllib.request import (
2689+ build_opener, install_opener, urlopen, urlretrieve,
2690+ HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
2691+ )
2692+ from urllib.parse import urlparse, urlunparse, parse_qs
2693+ from urllib.error import URLError
2694+else:
2695+ from urllib import urlretrieve
2696+ from urllib2 import (
2697+ build_opener, install_opener, urlopen,
2698+ HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
2699+ URLError
2700+ )
2701+ from urlparse import urlparse, urlunparse, parse_qs
2702
2703 from charmhelpers.fetch import (
2704 BaseFetchHandler,
2705@@ -15,6 +30,24 @@
2706 from charmhelpers.core.host import mkdir, check_hash
2707
2708
2709+def splituser(host):
2710+ '''urllib.splituser(), but six's support of this seems broken'''
2711+ _userprog = re.compile('^(.*)@(.*)$')
2712+ match = _userprog.match(host)
2713+ if match:
2714+ return match.group(1, 2)
2715+ return None, host
2716+
2717+
2718+def splitpasswd(user):
2719+ '''urllib.splitpasswd(), but six's support of this is missing'''
2720+ _passwdprog = re.compile('^([^:]*):(.*)$', re.S)
2721+ match = _passwdprog.match(user)
2722+ if match:
2723+ return match.group(1, 2)
2724+ return user, None
2725+
2726+
2727 class ArchiveUrlFetchHandler(BaseFetchHandler):
2728 """
2729 Handler to download archive files from arbitrary URLs.
2730@@ -42,20 +75,20 @@
2731 """
2732 # propogate all exceptions
2733 # URLError, OSError, etc
2734- proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
2735+ proto, netloc, path, params, query, fragment = urlparse(source)
2736 if proto in ('http', 'https'):
2737- auth, barehost = urllib2.splituser(netloc)
2738+ auth, barehost = splituser(netloc)
2739 if auth is not None:
2740- source = urlparse.urlunparse((proto, barehost, path, params, query, fragment))
2741- username, password = urllib2.splitpasswd(auth)
2742- passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
2743+ source = urlunparse((proto, barehost, path, params, query, fragment))
2744+ username, password = splitpasswd(auth)
2745+ passman = HTTPPasswordMgrWithDefaultRealm()
2746 # Realm is set to None in add_password to force the username and password
2747 # to be used whatever the realm
2748 passman.add_password(None, source, username, password)
2749- authhandler = urllib2.HTTPBasicAuthHandler(passman)
2750- opener = urllib2.build_opener(authhandler)
2751- urllib2.install_opener(opener)
2752- response = urllib2.urlopen(source)
2753+ authhandler = HTTPBasicAuthHandler(passman)
2754+ opener = build_opener(authhandler)
2755+ install_opener(opener)
2756+ response = urlopen(source)
2757 try:
2758 with open(dest, 'w') as dest_file:
2759 dest_file.write(response.read())
2760@@ -91,17 +124,21 @@
2761 url_parts = self.parse_url(source)
2762 dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
2763 if not os.path.exists(dest_dir):
2764- mkdir(dest_dir, perms=0755)
2765+ mkdir(dest_dir, perms=0o755)
2766 dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
2767 try:
2768 self.download(source, dld_file)
2769- except urllib2.URLError as e:
2770+ except URLError as e:
2771 raise UnhandledSource(e.reason)
2772 except OSError as e:
2773 raise UnhandledSource(e.strerror)
2774- options = urlparse.parse_qs(url_parts.fragment)
2775+ options = parse_qs(url_parts.fragment)
2776 for key, value in options.items():
2777- if key in hashlib.algorithms:
2778+ if not six.PY3:
2779+ algorithms = hashlib.algorithms
2780+ else:
2781+ algorithms = hashlib.algorithms_available
2782+ if key in algorithms:
2783 check_hash(dld_file, value, key)
2784 if checksum:
2785 check_hash(dld_file, checksum, hash_type)
2786
2787=== modified file 'hooks/charmhelpers/fetch/bzrurl.py'
2788--- hooks/charmhelpers/fetch/bzrurl.py 2014-07-25 08:11:52 +0000
2789+++ hooks/charmhelpers/fetch/bzrurl.py 2014-11-26 10:42:22 +0000
2790@@ -5,6 +5,10 @@
2791 )
2792 from charmhelpers.core.host import mkdir
2793
2794+import six
2795+if six.PY3:
2796+ raise ImportError('bzrlib does not support Python3')
2797+
2798 try:
2799 from bzrlib.branch import Branch
2800 except ImportError:
2801@@ -42,7 +46,7 @@
2802 dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
2803 branch_name)
2804 if not os.path.exists(dest_dir):
2805- mkdir(dest_dir, perms=0755)
2806+ mkdir(dest_dir, perms=0o755)
2807 try:
2808 self.branch(source, dest_dir)
2809 except OSError as e:
2810
2811=== added file 'hooks/charmhelpers/fetch/giturl.py'
2812--- hooks/charmhelpers/fetch/giturl.py 1970-01-01 00:00:00 +0000
2813+++ hooks/charmhelpers/fetch/giturl.py 2014-11-26 10:42:22 +0000
2814@@ -0,0 +1,48 @@
2815+import os
2816+from charmhelpers.fetch import (
2817+ BaseFetchHandler,
2818+ UnhandledSource
2819+)
2820+from charmhelpers.core.host import mkdir
2821+
2822+import six
2823+if six.PY3:
2824+ raise ImportError('GitPython does not support Python 3')
2825+
2826+try:
2827+ from git import Repo
2828+except ImportError:
2829+ from charmhelpers.fetch import apt_install
2830+ apt_install("python-git")
2831+ from git import Repo
2832+
2833+
2834+class GitUrlFetchHandler(BaseFetchHandler):
2835+ """Handler for git branches via generic and github URLs"""
2836+ def can_handle(self, source):
2837+ url_parts = self.parse_url(source)
2838+ # TODO (mattyw) no support for ssh git@ yet
2839+ if url_parts.scheme not in ('http', 'https', 'git'):
2840+ return False
2841+ else:
2842+ return True
2843+
2844+ def clone(self, source, dest, branch):
2845+ if not self.can_handle(source):
2846+ raise UnhandledSource("Cannot handle {}".format(source))
2847+
2848+ repo = Repo.clone_from(source, dest)
2849+ repo.git.checkout(branch)
2850+
2851+ def install(self, source, branch="master"):
2852+ url_parts = self.parse_url(source)
2853+ branch_name = url_parts.path.strip("/").split("/")[-1]
2854+ dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
2855+ branch_name)
2856+ if not os.path.exists(dest_dir):
2857+ mkdir(dest_dir, perms=0o755)
2858+ try:
2859+ self.clone(source, dest_dir, branch)
2860+ except OSError as e:
2861+ raise UnhandledSource(e.strerror)
2862+ return dest_dir
2863
2864=== modified file 'hooks/cinder_contexts.py'
2865--- hooks/cinder_contexts.py 2014-09-24 15:56:28 +0000
2866+++ hooks/cinder_contexts.py 2014-11-26 10:42:22 +0000
2867@@ -64,8 +64,10 @@
2868 Also used to extend cinder.conf context with correct api_listening_port
2869 '''
2870 haproxy_port = config('api-listening-port')
2871- api_port = determine_api_port(config('api-listening-port'))
2872- apache_port = determine_apache_port(config('api-listening-port'))
2873+ api_port = determine_api_port(config('api-listening-port'),
2874+ singlenode_mode=True)
2875+ apache_port = determine_apache_port(config('api-listening-port'),
2876+ singlenode_mode=True)
2877
2878 ctxt = {
2879 'service_ports': {'cinder_api': [haproxy_port, apache_port]},
2880
2881=== modified file 'hooks/cinder_utils.py'
2882--- hooks/cinder_utils.py 2014-10-14 15:59:34 +0000
2883+++ hooks/cinder_utils.py 2014-11-26 10:42:22 +0000
2884@@ -145,7 +145,7 @@
2885 'services': ['cinder-volume']
2886 }),
2887 (HAPROXY_CONF, {
2888- 'hook_contexts': [context.HAProxyContext(),
2889+ 'hook_contexts': [context.HAProxyContext(singlenode_mode=True),
2890 cinder_contexts.HAProxyContext()],
2891 'services': ['haproxy'],
2892 }),
2893
2894=== modified file 'tests/charmhelpers/contrib/amulet/deployment.py'
2895--- tests/charmhelpers/contrib/amulet/deployment.py 2014-10-06 21:50:53 +0000
2896+++ tests/charmhelpers/contrib/amulet/deployment.py 2014-11-26 10:42:22 +0000
2897@@ -1,6 +1,6 @@
2898 import amulet
2899-
2900 import os
2901+import six
2902
2903
2904 class AmuletDeployment(object):
2905@@ -52,12 +52,12 @@
2906
2907 def _add_relations(self, relations):
2908 """Add all of the relations for the services."""
2909- for k, v in relations.iteritems():
2910+ for k, v in six.iteritems(relations):
2911 self.d.relate(k, v)
2912
2913 def _configure_services(self, configs):
2914 """Configure all of the services."""
2915- for service, config in configs.iteritems():
2916+ for service, config in six.iteritems(configs):
2917 self.d.configure(service, config)
2918
2919 def _deploy(self):
2920
2921=== modified file 'tests/charmhelpers/contrib/amulet/utils.py'
2922--- tests/charmhelpers/contrib/amulet/utils.py 2014-10-06 21:50:53 +0000
2923+++ tests/charmhelpers/contrib/amulet/utils.py 2014-11-26 10:42:22 +0000
2924@@ -5,6 +5,8 @@
2925 import sys
2926 import time
2927
2928+import six
2929+
2930
2931 class AmuletUtils(object):
2932 """Amulet utilities.
2933@@ -58,7 +60,7 @@
2934 Verify the specified services are running on the corresponding
2935 service units.
2936 """
2937- for k, v in commands.iteritems():
2938+ for k, v in six.iteritems(commands):
2939 for cmd in v:
2940 output, code = k.run(cmd)
2941 if code != 0:
2942@@ -100,11 +102,11 @@
2943 longs, or can be a function that evaluate a variable and returns a
2944 bool.
2945 """
2946- for k, v in expected.iteritems():
2947+ for k, v in six.iteritems(expected):
2948 if k in actual:
2949- if (isinstance(v, basestring) or
2950+ if (isinstance(v, six.string_types) or
2951 isinstance(v, bool) or
2952- isinstance(v, (int, long))):
2953+ isinstance(v, six.integer_types)):
2954 if v != actual[k]:
2955 return "{}:{}".format(k, actual[k])
2956 elif not v(actual[k]):
2957
2958=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
2959--- tests/charmhelpers/contrib/openstack/amulet/deployment.py 2014-10-06 21:50:53 +0000
2960+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py 2014-11-26 10:42:22 +0000
2961@@ -1,3 +1,4 @@
2962+import six
2963 from charmhelpers.contrib.amulet.deployment import (
2964 AmuletDeployment
2965 )
2966@@ -69,7 +70,7 @@
2967
2968 def _configure_services(self, configs):
2969 """Configure all of the services."""
2970- for service, config in configs.iteritems():
2971+ for service, config in six.iteritems(configs):
2972 self.d.configure(service, config)
2973
2974 def _get_openstack_release(self):
2975
2976=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
2977--- tests/charmhelpers/contrib/openstack/amulet/utils.py 2014-10-06 21:50:53 +0000
2978+++ tests/charmhelpers/contrib/openstack/amulet/utils.py 2014-11-26 10:42:22 +0000
2979@@ -7,6 +7,8 @@
2980 import keystoneclient.v2_0 as keystone_client
2981 import novaclient.v1_1.client as nova_client
2982
2983+import six
2984+
2985 from charmhelpers.contrib.amulet.utils import (
2986 AmuletUtils
2987 )
2988@@ -60,7 +62,7 @@
2989 expected service catalog endpoints.
2990 """
2991 self.log.debug('actual: {}'.format(repr(actual)))
2992- for k, v in expected.iteritems():
2993+ for k, v in six.iteritems(expected):
2994 if k in actual:
2995 ret = self._validate_dict_data(expected[k][0], actual[k][0])
2996 if ret:

Subscribers

People subscribed via source and target branches