Merge lp:~james-page/charms/trusty/nova-compute-vmware/resync into lp:~openstack-charmers/charms/trusty/nova-compute-vmware/next

Proposed by James Page
Status: Merged
Merged at revision: 111
Proposed branch: lp:~james-page/charms/trusty/nova-compute-vmware/resync
Merge into: lp:~openstack-charmers/charms/trusty/nova-compute-vmware/next
Diff against target: 7005 lines (+4730/-512)
58 files modified
hooks/charmhelpers/__init__.py (+16/-0)
hooks/charmhelpers/contrib/__init__.py (+15/-0)
hooks/charmhelpers/contrib/hahelpers/__init__.py (+15/-0)
hooks/charmhelpers/contrib/hahelpers/apache.py (+16/-0)
hooks/charmhelpers/contrib/hahelpers/cluster.py (+73/-5)
hooks/charmhelpers/contrib/network/__init__.py (+15/-0)
hooks/charmhelpers/contrib/network/ip.py (+104/-1)
hooks/charmhelpers/contrib/network/ovs/__init__.py (+16/-0)
hooks/charmhelpers/contrib/network/ufw.py (+140/-22)
hooks/charmhelpers/contrib/openstack/__init__.py (+15/-0)
hooks/charmhelpers/contrib/openstack/alternatives.py (+16/-0)
hooks/charmhelpers/contrib/openstack/amulet/__init__.py (+15/-0)
hooks/charmhelpers/contrib/openstack/amulet/deployment.py (+118/-13)
hooks/charmhelpers/contrib/openstack/amulet/utils.py (+736/-51)
hooks/charmhelpers/contrib/openstack/context.py (+437/-59)
hooks/charmhelpers/contrib/openstack/files/__init__.py (+18/-0)
hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh (+32/-0)
hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh (+30/-0)
hooks/charmhelpers/contrib/openstack/ip.py (+65/-7)
hooks/charmhelpers/contrib/openstack/neutron.py (+136/-3)
hooks/charmhelpers/contrib/openstack/templates/__init__.py (+16/-0)
hooks/charmhelpers/contrib/openstack/templates/ceph.conf (+6/-6)
hooks/charmhelpers/contrib/openstack/templates/git.upstart (+17/-0)
hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken (+9/-0)
hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo (+22/-0)
hooks/charmhelpers/contrib/openstack/templates/section-zeromq (+14/-0)
hooks/charmhelpers/contrib/openstack/templating.py (+46/-3)
hooks/charmhelpers/contrib/openstack/utils.py (+467/-169)
hooks/charmhelpers/contrib/python/__init__.py (+15/-0)
hooks/charmhelpers/contrib/python/packages.py (+50/-6)
hooks/charmhelpers/contrib/storage/__init__.py (+15/-0)
hooks/charmhelpers/contrib/storage/linux/__init__.py (+15/-0)
hooks/charmhelpers/contrib/storage/linux/ceph.py (+248/-19)
hooks/charmhelpers/contrib/storage/linux/loopback.py (+16/-0)
hooks/charmhelpers/contrib/storage/linux/lvm.py (+16/-0)
hooks/charmhelpers/contrib/storage/linux/utils.py (+20/-3)
hooks/charmhelpers/core/__init__.py (+15/-0)
hooks/charmhelpers/core/decorators.py (+16/-0)
hooks/charmhelpers/core/files.py (+45/-0)
hooks/charmhelpers/core/fstab.py (+19/-3)
hooks/charmhelpers/core/hookenv.py (+389/-43)
hooks/charmhelpers/core/host.py (+199/-32)
hooks/charmhelpers/core/hugepage.py (+62/-0)
hooks/charmhelpers/core/kernel.py (+68/-0)
hooks/charmhelpers/core/services/__init__.py (+16/-0)
hooks/charmhelpers/core/services/base.py (+59/-19)
hooks/charmhelpers/core/services/helpers.py (+46/-6)
hooks/charmhelpers/core/strutils.py (+42/-0)
hooks/charmhelpers/core/sysctl.py (+28/-6)
hooks/charmhelpers/core/templating.py (+19/-3)
hooks/charmhelpers/core/unitdata.py (+521/-0)
hooks/charmhelpers/fetch/__init__.py (+48/-15)
hooks/charmhelpers/fetch/archiveurl.py (+33/-11)
hooks/charmhelpers/fetch/bzrurl.py (+25/-1)
hooks/charmhelpers/fetch/giturl.py (+27/-5)
hooks/charmhelpers/payload/__init__.py (+16/-0)
hooks/charmhelpers/payload/execd.py (+16/-0)
unit_tests/test_nova_vmware_contexts.py (+1/-1)
To merge this branch: bzr merge lp:~james-page/charms/trusty/nova-compute-vmware/resync
Reviewer Review Type Date Requested Status
Billy Olsen Approve
OpenStack Charmers Pending
Review via email: mp+271791@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Billy Olsen (billy-olsen) wrote :

Looks to be a simple c-h sync. Unit tests pass, looks fairly straightforward. I'll let OSCI vote before merging, but +1 from me.

review: Approve
Revision history for this message
Billy Olsen (billy-olsen) wrote :

Realized that OSCI vote won't touch this other than unit-tests + lint, which I've already run. Lint fails with warnings, but not due to this change. Will merge.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'hooks/charmhelpers/__init__.py'
--- hooks/charmhelpers/__init__.py 2014-12-11 17:48:55 +0000
+++ hooks/charmhelpers/__init__.py 2015-09-21 10:47:21 +0000
@@ -1,3 +1,19 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1# Bootstrap charm-helpers, installing its dependencies if necessary using17# Bootstrap charm-helpers, installing its dependencies if necessary using
2# only standard libraries.18# only standard libraries.
3import subprocess19import subprocess
420
=== modified file 'hooks/charmhelpers/contrib/__init__.py'
--- hooks/charmhelpers/contrib/__init__.py 2013-07-19 02:37:30 +0000
+++ hooks/charmhelpers/contrib/__init__.py 2015-09-21 10:47:21 +0000
@@ -0,0 +1,15 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
016
=== modified file 'hooks/charmhelpers/contrib/hahelpers/__init__.py'
--- hooks/charmhelpers/contrib/hahelpers/__init__.py 2013-07-19 02:37:30 +0000
+++ hooks/charmhelpers/contrib/hahelpers/__init__.py 2015-09-21 10:47:21 +0000
@@ -0,0 +1,15 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
016
=== modified file 'hooks/charmhelpers/contrib/hahelpers/apache.py'
--- hooks/charmhelpers/contrib/hahelpers/apache.py 2014-10-23 17:30:13 +0000
+++ hooks/charmhelpers/contrib/hahelpers/apache.py 2015-09-21 10:47:21 +0000
@@ -1,3 +1,19 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1#17#
2# Copyright 2012 Canonical Ltd.18# Copyright 2012 Canonical Ltd.
3#19#
420
=== modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
--- hooks/charmhelpers/contrib/hahelpers/cluster.py 2015-01-14 15:30:19 +0000
+++ hooks/charmhelpers/contrib/hahelpers/cluster.py 2015-09-21 10:47:21 +0000
@@ -1,3 +1,19 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1#17#
2# Copyright 2012 Canonical Ltd.18# Copyright 2012 Canonical Ltd.
3#19#
@@ -28,10 +44,16 @@
28 ERROR,44 ERROR,
29 WARNING,45 WARNING,
30 unit_get,46 unit_get,
47 is_leader as juju_is_leader
31)48)
32from charmhelpers.core.decorators import (49from charmhelpers.core.decorators import (
33 retry_on_exception,50 retry_on_exception,
34)51)
52from charmhelpers.core.strutils import (
53 bool_from_string,
54)
55
56DC_RESOURCE_NAME = 'DC'
3557
3658
37class HAIncompleteConfig(Exception):59class HAIncompleteConfig(Exception):
@@ -42,17 +64,30 @@
42 pass64 pass
4365
4466
67class CRMDCNotFound(Exception):
68 pass
69
70
45def is_elected_leader(resource):71def is_elected_leader(resource):
46 """72 """
47 Returns True if the charm executing this is the elected cluster leader.73 Returns True if the charm executing this is the elected cluster leader.
4874
49 It relies on two mechanisms to determine leadership:75 It relies on two mechanisms to determine leadership:
50 1. If the charm is part of a corosync cluster, call corosync to76 1. If juju is sufficiently new and leadership election is supported,
77 the is_leader command will be used.
78 2. If the charm is part of a corosync cluster, call corosync to
51 determine leadership.79 determine leadership.
52 2. If the charm is not part of a corosync cluster, the leader is80 3. If the charm is not part of a corosync cluster, the leader is
53 determined as being "the alive unit with the lowest unit numer". In81 determined as being "the alive unit with the lowest unit numer". In
54 other words, the oldest surviving unit.82 other words, the oldest surviving unit.
55 """83 """
84 try:
85 return juju_is_leader()
86 except NotImplementedError:
87 log('Juju leadership election feature not enabled'
88 ', using fallback support',
89 level=WARNING)
90
56 if is_clustered():91 if is_clustered():
57 if not is_crm_leader(resource):92 if not is_crm_leader(resource):
58 log('Deferring action to CRM leader.', level=INFO)93 log('Deferring action to CRM leader.', level=INFO)
@@ -76,7 +111,33 @@
76 return False111 return False
77112
78113
79@retry_on_exception(5, base_delay=2, exc_type=CRMResourceNotFound)114def is_crm_dc():
115 """
116 Determine leadership by querying the pacemaker Designated Controller
117 """
118 cmd = ['crm', 'status']
119 try:
120 status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
121 if not isinstance(status, six.text_type):
122 status = six.text_type(status, "utf-8")
123 except subprocess.CalledProcessError as ex:
124 raise CRMDCNotFound(str(ex))
125
126 current_dc = ''
127 for line in status.split('\n'):
128 if line.startswith('Current DC'):
129 # Current DC: juju-lytrusty-machine-2 (168108163) - partition with quorum
130 current_dc = line.split(':')[1].split()[0]
131 if current_dc == get_unit_hostname():
132 return True
133 elif current_dc == 'NONE':
134 raise CRMDCNotFound('Current DC: NONE')
135
136 return False
137
138
139@retry_on_exception(5, base_delay=2,
140 exc_type=(CRMResourceNotFound, CRMDCNotFound))
80def is_crm_leader(resource, retry=False):141def is_crm_leader(resource, retry=False):
81 """142 """
82 Returns True if the charm calling this is the elected corosync leader,143 Returns True if the charm calling this is the elected corosync leader,
@@ -85,6 +146,8 @@
85 We allow this operation to be retried to avoid the possibility of getting a146 We allow this operation to be retried to avoid the possibility of getting a
86 false negative. See LP #1396246 for more info.147 false negative. See LP #1396246 for more info.
87 """148 """
149 if resource == DC_RESOURCE_NAME:
150 return is_crm_dc()
88 cmd = ['crm', 'resource', 'show', resource]151 cmd = ['crm', 'resource', 'show', resource]
89 try:152 try:
90 status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)153 status = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
@@ -148,7 +211,8 @@
148 .211 .
149 returns: boolean212 returns: boolean
150 '''213 '''
151 if config_get('use-https') == "yes":214 use_https = config_get('use-https')
215 if use_https and bool_from_string(use_https):
152 return True216 return True
153 if config_get('ssl_cert') and config_get('ssl_key'):217 if config_get('ssl_cert') and config_get('ssl_key'):
154 return True218 return True
@@ -205,19 +269,23 @@
205 return public_port - (i * 10)269 return public_port - (i * 10)
206270
207271
208def get_hacluster_config():272def get_hacluster_config(exclude_keys=None):
209 '''273 '''
210 Obtains all relevant configuration from charm configuration required274 Obtains all relevant configuration from charm configuration required
211 for initiating a relation to hacluster:275 for initiating a relation to hacluster:
212276
213 ha-bindiface, ha-mcastport, vip277 ha-bindiface, ha-mcastport, vip
214278
279 param: exclude_keys: list of setting key(s) to be excluded.
215 returns: dict: A dict containing settings keyed by setting name.280 returns: dict: A dict containing settings keyed by setting name.
216 raises: HAIncompleteConfig if settings are missing.281 raises: HAIncompleteConfig if settings are missing.
217 '''282 '''
218 settings = ['ha-bindiface', 'ha-mcastport', 'vip']283 settings = ['ha-bindiface', 'ha-mcastport', 'vip']
219 conf = {}284 conf = {}
220 for setting in settings:285 for setting in settings:
286 if exclude_keys and setting in exclude_keys:
287 continue
288
221 conf[setting] = config_get(setting)289 conf[setting] = config_get(setting)
222 missing = []290 missing = []
223 [missing.append(s) for s, v in six.iteritems(conf) if v is None]291 [missing.append(s) for s, v in six.iteritems(conf) if v is None]
224292
=== modified file 'hooks/charmhelpers/contrib/network/__init__.py'
--- hooks/charmhelpers/contrib/network/__init__.py 2014-10-23 17:30:13 +0000
+++ hooks/charmhelpers/contrib/network/__init__.py 2015-09-21 10:47:21 +0000
@@ -0,0 +1,15 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
016
=== modified file 'hooks/charmhelpers/contrib/network/ip.py'
--- hooks/charmhelpers/contrib/network/ip.py 2014-12-10 20:28:57 +0000
+++ hooks/charmhelpers/contrib/network/ip.py 2015-09-21 10:47:21 +0000
@@ -1,13 +1,32 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1import glob17import glob
2import re18import re
3import subprocess19import subprocess
20import six
21import socket
422
5from functools import partial23from functools import partial
624
7from charmhelpers.core.hookenv import unit_get25from charmhelpers.core.hookenv import unit_get
8from charmhelpers.fetch import apt_install26from charmhelpers.fetch import apt_install
9from charmhelpers.core.hookenv import (27from charmhelpers.core.hookenv import (
10 log28 log,
29 WARNING,
11)30)
1231
13try:32try:
@@ -349,3 +368,87 @@
349 return True368 return True
350369
351 return False370 return False
371
372
373def is_ip(address):
374 """
375 Returns True if address is a valid IP address.
376 """
377 try:
378 # Test to see if already an IPv4 address
379 socket.inet_aton(address)
380 return True
381 except socket.error:
382 return False
383
384
385def ns_query(address):
386 try:
387 import dns.resolver
388 except ImportError:
389 apt_install('python-dnspython')
390 import dns.resolver
391
392 if isinstance(address, dns.name.Name):
393 rtype = 'PTR'
394 elif isinstance(address, six.string_types):
395 rtype = 'A'
396 else:
397 return None
398
399 answers = dns.resolver.query(address, rtype)
400 if answers:
401 return str(answers[0])
402 return None
403
404
405def get_host_ip(hostname, fallback=None):
406 """
407 Resolves the IP for a given hostname, or returns
408 the input if it is already an IP.
409 """
410 if is_ip(hostname):
411 return hostname
412
413 ip_addr = ns_query(hostname)
414 if not ip_addr:
415 try:
416 ip_addr = socket.gethostbyname(hostname)
417 except:
418 log("Failed to resolve hostname '%s'" % (hostname),
419 level=WARNING)
420 return fallback
421 return ip_addr
422
423
424def get_hostname(address, fqdn=True):
425 """
426 Resolves hostname for given IP, or returns the input
427 if it is already a hostname.
428 """
429 if is_ip(address):
430 try:
431 import dns.reversename
432 except ImportError:
433 apt_install("python-dnspython")
434 import dns.reversename
435
436 rev = dns.reversename.from_address(address)
437 result = ns_query(rev)
438
439 if not result:
440 try:
441 result = socket.gethostbyaddr(address)[0]
442 except:
443 return None
444 else:
445 result = address
446
447 if fqdn:
448 # strip trailing .
449 if result.endswith('.'):
450 return result[:-1]
451 else:
452 return result
453 else:
454 return result.split('.')[0]
352455
=== modified file 'hooks/charmhelpers/contrib/network/ovs/__init__.py'
--- hooks/charmhelpers/contrib/network/ovs/__init__.py 2014-10-23 17:30:13 +0000
+++ hooks/charmhelpers/contrib/network/ovs/__init__.py 2015-09-21 10:47:21 +0000
@@ -1,3 +1,19 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1''' Helpers for interacting with OpenvSwitch '''17''' Helpers for interacting with OpenvSwitch '''
2import subprocess18import subprocess
3import os19import os
420
=== modified file 'hooks/charmhelpers/contrib/network/ufw.py'
--- hooks/charmhelpers/contrib/network/ufw.py 2015-01-14 15:30:19 +0000
+++ hooks/charmhelpers/contrib/network/ufw.py 2015-09-21 10:47:21 +0000
@@ -1,3 +1,19 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1"""17"""
2This module contains helpers to add and remove ufw rules.18This module contains helpers to add and remove ufw rules.
319
@@ -21,13 +37,22 @@
21 >>> ufw.enable()37 >>> ufw.enable()
22 >>> ufw.service('4949', 'close') # munin38 >>> ufw.service('4949', 'close') # munin
23"""39"""
24
25__author__ = "Felipe Reyes <felipe.reyes@canonical.com>"
26
27import re40import re
28import os41import os
29import subprocess42import subprocess
43
30from charmhelpers.core import hookenv44from charmhelpers.core import hookenv
45from charmhelpers.core.kernel import modprobe, is_module_loaded
46
47__author__ = "Felipe Reyes <felipe.reyes@canonical.com>"
48
49
50class UFWError(Exception):
51 pass
52
53
54class UFWIPv6Error(UFWError):
55 pass
3156
3257
33def is_enabled():58def is_enabled():
@@ -37,6 +62,7 @@
37 :returns: True if ufw is enabled62 :returns: True if ufw is enabled
38 """63 """
39 output = subprocess.check_output(['ufw', 'status'],64 output = subprocess.check_output(['ufw', 'status'],
65 universal_newlines=True,
40 env={'LANG': 'en_US',66 env={'LANG': 'en_US',
41 'PATH': os.environ['PATH']})67 'PATH': os.environ['PATH']})
4268
@@ -45,27 +71,73 @@
45 return len(m) >= 171 return len(m) >= 1
4672
4773
48def enable():74def is_ipv6_ok(soft_fail=False):
75 """
76 Check if IPv6 support is present and ip6tables functional
77
78 :param soft_fail: If set to True and IPv6 support is broken, then reports
79 that the host doesn't have IPv6 support, otherwise a
80 UFWIPv6Error exception is raised.
81 :returns: True if IPv6 is working, False otherwise
82 """
83
84 # do we have IPv6 in the machine?
85 if os.path.isdir('/proc/sys/net/ipv6'):
86 # is ip6tables kernel module loaded?
87 if not is_module_loaded('ip6_tables'):
88 # ip6tables support isn't complete, let's try to load it
89 try:
90 modprobe('ip6_tables')
91 # great, we can load the module
92 return True
93 except subprocess.CalledProcessError as ex:
94 hookenv.log("Couldn't load ip6_tables module: %s" % ex.output,
95 level="WARN")
96 # we are in a world where ip6tables isn't working
97 if soft_fail:
98 # so we inform that the machine doesn't have IPv6
99 return False
100 else:
101 raise UFWIPv6Error("IPv6 firewall support broken")
102 else:
103 # the module is present :)
104 return True
105
106 else:
107 # the system doesn't have IPv6
108 return False
109
110
111def disable_ipv6():
112 """
113 Disable ufw IPv6 support in /etc/default/ufw
114 """
115 exit_code = subprocess.call(['sed', '-i', 's/IPV6=.*/IPV6=no/g',
116 '/etc/default/ufw'])
117 if exit_code == 0:
118 hookenv.log('IPv6 support in ufw disabled', level='INFO')
119 else:
120 hookenv.log("Couldn't disable IPv6 support in ufw", level="ERROR")
121 raise UFWError("Couldn't disable IPv6 support in ufw")
122
123
124def enable(soft_fail=False):
49 """125 """
50 Enable ufw126 Enable ufw
51127
128 :param soft_fail: If set to True silently disables IPv6 support in ufw,
129 otherwise a UFWIPv6Error exception is raised when IP6
130 support is broken.
52 :returns: True if ufw is successfully enabled131 :returns: True if ufw is successfully enabled
53 """132 """
54 if is_enabled():133 if is_enabled():
55 return True134 return True
56135
57 if not os.path.isdir('/proc/sys/net/ipv6'):136 if not is_ipv6_ok(soft_fail):
58 # disable IPv6 support in ufw137 disable_ipv6()
59 hookenv.log("This machine doesn't have IPv6 enabled", level="INFO")
60 exit_code = subprocess.call(['sed', '-i', 's/IPV6=yes/IPV6=no/g',
61 '/etc/default/ufw'])
62 if exit_code == 0:
63 hookenv.log('IPv6 support in ufw disabled', level='INFO')
64 else:
65 hookenv.log("Couldn't disable IPv6 support in ufw", level="ERROR")
66 raise Exception("Couldn't disable IPv6 support in ufw")
67138
68 output = subprocess.check_output(['ufw', 'enable'],139 output = subprocess.check_output(['ufw', 'enable'],
140 universal_newlines=True,
69 env={'LANG': 'en_US',141 env={'LANG': 'en_US',
70 'PATH': os.environ['PATH']})142 'PATH': os.environ['PATH']})
71143
@@ -91,6 +163,7 @@
91 return True163 return True
92164
93 output = subprocess.check_output(['ufw', 'disable'],165 output = subprocess.check_output(['ufw', 'disable'],
166 universal_newlines=True,
94 env={'LANG': 'en_US',167 env={'LANG': 'en_US',
95 'PATH': os.environ['PATH']})168 'PATH': os.environ['PATH']})
96169
@@ -106,7 +179,43 @@
106 return True179 return True
107180
108181
109def modify_access(src, dst='any', port=None, proto=None, action='allow'):182def default_policy(policy='deny', direction='incoming'):
183 """
184 Changes the default policy for traffic `direction`
185
186 :param policy: allow, deny or reject
187 :param direction: traffic direction, possible values: incoming, outgoing,
188 routed
189 """
190 if policy not in ['allow', 'deny', 'reject']:
191 raise UFWError(('Unknown policy %s, valid values: '
192 'allow, deny, reject') % policy)
193
194 if direction not in ['incoming', 'outgoing', 'routed']:
195 raise UFWError(('Unknown direction %s, valid values: '
196 'incoming, outgoing, routed') % direction)
197
198 output = subprocess.check_output(['ufw', 'default', policy, direction],
199 universal_newlines=True,
200 env={'LANG': 'en_US',
201 'PATH': os.environ['PATH']})
202 hookenv.log(output, level='DEBUG')
203
204 m = re.findall("^Default %s policy changed to '%s'\n" % (direction,
205 policy),
206 output, re.M)
207 if len(m) == 0:
208 hookenv.log("ufw couldn't change the default policy to %s for %s"
209 % (policy, direction), level='WARN')
210 return False
211 else:
212 hookenv.log("ufw default policy for %s changed to %s"
213 % (direction, policy), level='INFO')
214 return True
215
216
217def modify_access(src, dst='any', port=None, proto=None, action='allow',
218 index=None):
110 """219 """
111 Grant access to an address or subnet220 Grant access to an address or subnet
112221
@@ -118,6 +227,8 @@
118 :param port: destiny port227 :param port: destiny port
119 :param proto: protocol (tcp or udp)228 :param proto: protocol (tcp or udp)
120 :param action: `allow` or `delete`229 :param action: `allow` or `delete`
230 :param index: if different from None the rule is inserted at the given
231 `index`.
121 """232 """
122 if not is_enabled():233 if not is_enabled():
123 hookenv.log('ufw is disabled, skipping modify_access()', level='WARN')234 hookenv.log('ufw is disabled, skipping modify_access()', level='WARN')
@@ -125,6 +236,8 @@
125236
126 if action == 'delete':237 if action == 'delete':
127 cmd = ['ufw', 'delete', 'allow']238 cmd = ['ufw', 'delete', 'allow']
239 elif index is not None:
240 cmd = ['ufw', 'insert', str(index), action]
128 else:241 else:
129 cmd = ['ufw', action]242 cmd = ['ufw', action]
130243
@@ -135,7 +248,7 @@
135 cmd += ['to', dst]248 cmd += ['to', dst]
136249
137 if port is not None:250 if port is not None:
138 cmd += ['port', port]251 cmd += ['port', str(port)]
139252
140 if proto is not None:253 if proto is not None:
141 cmd += ['proto', proto]254 cmd += ['proto', proto]
@@ -153,7 +266,7 @@
153 level='ERROR')266 level='ERROR')
154267
155268
156def grant_access(src, dst='any', port=None, proto=None):269def grant_access(src, dst='any', port=None, proto=None, index=None):
157 """270 """
158 Grant access to an address or subnet271 Grant access to an address or subnet
159272
@@ -164,8 +277,11 @@
164 field has to be set.277 field has to be set.
165 :param port: destiny port278 :param port: destiny port
166 :param proto: protocol (tcp or udp)279 :param proto: protocol (tcp or udp)
280 :param index: if different from None the rule is inserted at the given
281 `index`.
167 """282 """
168 return modify_access(src, dst=dst, port=port, proto=proto, action='allow')283 return modify_access(src, dst=dst, port=port, proto=proto, action='allow',
284 index=index)
169285
170286
171def revoke_access(src, dst='any', port=None, proto=None):287def revoke_access(src, dst='any', port=None, proto=None):
@@ -192,9 +308,11 @@
192 :param action: `open` or `close`308 :param action: `open` or `close`
193 """309 """
194 if action == 'open':310 if action == 'open':
195 subprocess.check_output(['ufw', 'allow', name])311 subprocess.check_output(['ufw', 'allow', str(name)],
312 universal_newlines=True)
196 elif action == 'close':313 elif action == 'close':
197 subprocess.check_output(['ufw', 'delete', 'allow', name])314 subprocess.check_output(['ufw', 'delete', 'allow', str(name)],
315 universal_newlines=True)
198 else:316 else:
199 raise Exception(("'{}' not supported, use 'allow' "317 raise UFWError(("'{}' not supported, use 'allow' "
200 "or 'delete'").format(action))318 "or 'delete'").format(action))
201319
=== modified file 'hooks/charmhelpers/contrib/openstack/__init__.py'
--- hooks/charmhelpers/contrib/openstack/__init__.py 2013-07-19 02:37:30 +0000
+++ hooks/charmhelpers/contrib/openstack/__init__.py 2015-09-21 10:47:21 +0000
@@ -0,0 +1,15 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
016
=== modified file 'hooks/charmhelpers/contrib/openstack/alternatives.py'
--- hooks/charmhelpers/contrib/openstack/alternatives.py 2013-10-10 11:09:48 +0000
+++ hooks/charmhelpers/contrib/openstack/alternatives.py 2015-09-21 10:47:21 +0000
@@ -1,3 +1,19 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1''' Helper for managing alternatives for file conflict resolution '''17''' Helper for managing alternatives for file conflict resolution '''
218
3import subprocess19import subprocess
420
=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/__init__.py'
--- hooks/charmhelpers/contrib/openstack/amulet/__init__.py 2014-10-23 17:30:13 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/__init__.py 2015-09-21 10:47:21 +0000
@@ -0,0 +1,15 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
016
=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2014-12-10 20:28:57 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2015-09-21 10:47:21 +0000
@@ -1,4 +1,21 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1import six17import six
18from collections import OrderedDict
2from charmhelpers.contrib.amulet.deployment import (19from charmhelpers.contrib.amulet.deployment import (
3 AmuletDeployment20 AmuletDeployment
4)21)
@@ -27,21 +44,40 @@
27 Determine if the local branch being tested is derived from its44 Determine if the local branch being tested is derived from its
28 stable or next (dev) branch, and based on this, use the corresonding45 stable or next (dev) branch, and based on this, use the corresonding
29 stable or next branches for the other_services."""46 stable or next branches for the other_services."""
30 base_charms = ['mysql', 'mongodb', 'rabbitmq-server']47
3148 # Charms outside the lp:~openstack-charmers namespace
32 if self.stable:49 base_charms = ['mysql', 'mongodb', 'nrpe']
33 for svc in other_services:50
34 temp = 'lp:charms/{}'51 # Force these charms to current series even when using an older series.
35 svc['location'] = temp.format(svc['name'])52 # ie. Use trusty/nrpe even when series is precise, as the P charm
53 # does not possess the necessary external master config and hooks.
54 force_series_current = ['nrpe']
55
56 if self.series in ['precise', 'trusty']:
57 base_series = self.series
36 else:58 else:
37 for svc in other_services:59 base_series = self.current_next
60
61 for svc in other_services:
62 if svc['name'] in force_series_current:
63 base_series = self.current_next
64 # If a location has been explicitly set, use it
65 if svc.get('location'):
66 continue
67 if self.stable:
68 temp = 'lp:charms/{}/{}'
69 svc['location'] = temp.format(base_series,
70 svc['name'])
71 else:
38 if svc['name'] in base_charms:72 if svc['name'] in base_charms:
39 temp = 'lp:charms/{}'73 temp = 'lp:charms/{}/{}'
40 svc['location'] = temp.format(svc['name'])74 svc['location'] = temp.format(base_series,
75 svc['name'])
41 else:76 else:
42 temp = 'lp:~openstack-charmers/charms/{}/{}/next'77 temp = 'lp:~openstack-charmers/charms/{}/{}/next'
43 svc['location'] = temp.format(self.current_next,78 svc['location'] = temp.format(self.current_next,
44 svc['name'])79 svc['name'])
80
45 return other_services81 return other_services
4682
47 def _add_services(self, this_service, other_services):83 def _add_services(self, this_service, other_services):
@@ -53,18 +89,23 @@
5389
54 services = other_services90 services = other_services
55 services.append(this_service)91 services.append(this_service)
92
93 # Charms which should use the source config option
56 use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',94 use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
57 'ceph-osd', 'ceph-radosgw']95 'ceph-osd', 'ceph-radosgw']
5896
97 # Charms which can not use openstack-origin, ie. many subordinates
98 no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
99
59 if self.openstack:100 if self.openstack:
60 for svc in services:101 for svc in services:
61 if svc['name'] not in use_source:102 if svc['name'] not in use_source + no_origin:
62 config = {'openstack-origin': self.openstack}103 config = {'openstack-origin': self.openstack}
63 self.d.configure(svc['name'], config)104 self.d.configure(svc['name'], config)
64105
65 if self.source:106 if self.source:
66 for svc in services:107 for svc in services:
67 if svc['name'] in use_source:108 if svc['name'] in use_source and svc['name'] not in no_origin:
68 config = {'source': self.source}109 config = {'source': self.source}
69 self.d.configure(svc['name'], config)110 self.d.configure(svc['name'], config)
70111
@@ -79,14 +120,78 @@
79 Return an integer representing the enum value of the openstack120 Return an integer representing the enum value of the openstack
80 release.121 release.
81 """122 """
123 # Must be ordered by OpenStack release (not by Ubuntu release):
82 (self.precise_essex, self.precise_folsom, self.precise_grizzly,124 (self.precise_essex, self.precise_folsom, self.precise_grizzly,
83 self.precise_havana, self.precise_icehouse,125 self.precise_havana, self.precise_icehouse,
84 self.trusty_icehouse) = range(6)126 self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
127 self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
128 self.wily_liberty) = range(12)
129
85 releases = {130 releases = {
86 ('precise', None): self.precise_essex,131 ('precise', None): self.precise_essex,
87 ('precise', 'cloud:precise-folsom'): self.precise_folsom,132 ('precise', 'cloud:precise-folsom'): self.precise_folsom,
88 ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,133 ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
89 ('precise', 'cloud:precise-havana'): self.precise_havana,134 ('precise', 'cloud:precise-havana'): self.precise_havana,
90 ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,135 ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
91 ('trusty', None): self.trusty_icehouse}136 ('trusty', None): self.trusty_icehouse,
137 ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
138 ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
139 ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
140 ('utopic', None): self.utopic_juno,
141 ('vivid', None): self.vivid_kilo,
142 ('wily', None): self.wily_liberty}
92 return releases[(self.series, self.openstack)]143 return releases[(self.series, self.openstack)]
144
145 def _get_openstack_release_string(self):
146 """Get openstack release string.
147
148 Return a string representing the openstack release.
149 """
150 releases = OrderedDict([
151 ('precise', 'essex'),
152 ('quantal', 'folsom'),
153 ('raring', 'grizzly'),
154 ('saucy', 'havana'),
155 ('trusty', 'icehouse'),
156 ('utopic', 'juno'),
157 ('vivid', 'kilo'),
158 ('wily', 'liberty'),
159 ])
160 if self.openstack:
161 os_origin = self.openstack.split(':')[1]
162 return os_origin.split('%s-' % self.series)[1].split('/')[0]
163 else:
164 return releases[self.series]
165
166 def get_ceph_expected_pools(self, radosgw=False):
167 """Return a list of expected ceph pools in a ceph + cinder + glance
168 test scenario, based on OpenStack release and whether ceph radosgw
169 is flagged as present or not."""
170
171 if self._get_openstack_release() >= self.trusty_kilo:
172 # Kilo or later
173 pools = [
174 'rbd',
175 'cinder',
176 'glance'
177 ]
178 else:
179 # Juno or earlier
180 pools = [
181 'data',
182 'metadata',
183 'rbd',
184 'cinder',
185 'glance'
186 ]
187
188 if radosgw:
189 pools.extend([
190 '.rgw.root',
191 '.rgw.control',
192 '.rgw',
193 '.rgw.gc',
194 '.users.uid'
195 ])
196
197 return pools
93198
=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
--- hooks/charmhelpers/contrib/openstack/amulet/utils.py 2014-12-10 20:28:57 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py 2015-09-21 10:47:21 +0000
@@ -1,13 +1,34 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
17import amulet
18import json
1import logging19import logging
2import os20import os
21import six
3import time22import time
4import urllib23import urllib
524
25import cinderclient.v1.client as cinder_client
6import glanceclient.v1.client as glance_client26import glanceclient.v1.client as glance_client
27import heatclient.v1.client as heat_client
7import keystoneclient.v2_0 as keystone_client28import keystoneclient.v2_0 as keystone_client
8import novaclient.v1_1.client as nova_client29import novaclient.v1_1.client as nova_client
930import pika
10import six31import swiftclient
1132
12from charmhelpers.contrib.amulet.utils import (33from charmhelpers.contrib.amulet.utils import (
13 AmuletUtils34 AmuletUtils
@@ -21,7 +42,7 @@
21 """OpenStack amulet utilities.42 """OpenStack amulet utilities.
2243
23 This class inherits from AmuletUtils and has additional support44 This class inherits from AmuletUtils and has additional support
24 that is specifically for use by OpenStack charms.45 that is specifically for use by OpenStack charm tests.
25 """46 """
2647
27 def __init__(self, log_level=ERROR):48 def __init__(self, log_level=ERROR):
@@ -35,6 +56,8 @@
35 Validate actual endpoint data vs expected endpoint data. The ports56 Validate actual endpoint data vs expected endpoint data. The ports
36 are used to find the matching endpoint.57 are used to find the matching endpoint.
37 """58 """
59 self.log.debug('Validating endpoint data...')
60 self.log.debug('actual: {}'.format(repr(endpoints)))
38 found = False61 found = False
39 for ep in endpoints:62 for ep in endpoints:
40 self.log.debug('endpoint: {}'.format(repr(ep)))63 self.log.debug('endpoint: {}'.format(repr(ep)))
@@ -61,6 +84,7 @@
61 Validate a list of actual service catalog endpoints vs a list of84 Validate a list of actual service catalog endpoints vs a list of
62 expected service catalog endpoints.85 expected service catalog endpoints.
63 """86 """
87 self.log.debug('Validating service catalog endpoint data...')
64 self.log.debug('actual: {}'.format(repr(actual)))88 self.log.debug('actual: {}'.format(repr(actual)))
65 for k, v in six.iteritems(expected):89 for k, v in six.iteritems(expected):
66 if k in actual:90 if k in actual:
@@ -77,6 +101,7 @@
77 Validate a list of actual tenant data vs list of expected tenant101 Validate a list of actual tenant data vs list of expected tenant
78 data.102 data.
79 """103 """
104 self.log.debug('Validating tenant data...')
80 self.log.debug('actual: {}'.format(repr(actual)))105 self.log.debug('actual: {}'.format(repr(actual)))
81 for e in expected:106 for e in expected:
82 found = False107 found = False
@@ -98,6 +123,7 @@
98 Validate a list of actual role data vs a list of expected role123 Validate a list of actual role data vs a list of expected role
99 data.124 data.
100 """125 """
126 self.log.debug('Validating role data...')
101 self.log.debug('actual: {}'.format(repr(actual)))127 self.log.debug('actual: {}'.format(repr(actual)))
102 for e in expected:128 for e in expected:
103 found = False129 found = False
@@ -118,6 +144,7 @@
118 Validate a list of actual user data vs a list of expected user144 Validate a list of actual user data vs a list of expected user
119 data.145 data.
120 """146 """
147 self.log.debug('Validating user data...')
121 self.log.debug('actual: {}'.format(repr(actual)))148 self.log.debug('actual: {}'.format(repr(actual)))
122 for e in expected:149 for e in expected:
123 found = False150 found = False
@@ -139,17 +166,30 @@
139166
140 Validate a list of actual flavors vs a list of expected flavors.167 Validate a list of actual flavors vs a list of expected flavors.
141 """168 """
169 self.log.debug('Validating flavor data...')
142 self.log.debug('actual: {}'.format(repr(actual)))170 self.log.debug('actual: {}'.format(repr(actual)))
143 act = [a.name for a in actual]171 act = [a.name for a in actual]
144 return self._validate_list_data(expected, act)172 return self._validate_list_data(expected, act)
145173
146 def tenant_exists(self, keystone, tenant):174 def tenant_exists(self, keystone, tenant):
147 """Return True if tenant exists."""175 """Return True if tenant exists."""
176 self.log.debug('Checking if tenant exists ({})...'.format(tenant))
148 return tenant in [t.name for t in keystone.tenants.list()]177 return tenant in [t.name for t in keystone.tenants.list()]
149178
179 def authenticate_cinder_admin(self, keystone_sentry, username,
180 password, tenant):
181 """Authenticates admin user with cinder."""
182 # NOTE(beisner): cinder python client doesn't accept tokens.
183 service_ip = \
184 keystone_sentry.relation('shared-db',
185 'mysql:shared-db')['private-address']
186 ept = "http://{}:5000/v2.0".format(service_ip.strip().decode('utf-8'))
187 return cinder_client.Client(username, password, tenant, ept)
188
150 def authenticate_keystone_admin(self, keystone_sentry, user, password,189 def authenticate_keystone_admin(self, keystone_sentry, user, password,
151 tenant):190 tenant):
152 """Authenticates admin user with the keystone admin endpoint."""191 """Authenticates admin user with the keystone admin endpoint."""
192 self.log.debug('Authenticating keystone admin...')
153 unit = keystone_sentry193 unit = keystone_sentry
154 service_ip = unit.relation('shared-db',194 service_ip = unit.relation('shared-db',
155 'mysql:shared-db')['private-address']195 'mysql:shared-db')['private-address']
@@ -159,6 +199,7 @@
159199
160 def authenticate_keystone_user(self, keystone, user, password, tenant):200 def authenticate_keystone_user(self, keystone, user, password, tenant):
161 """Authenticates a regular user with the keystone public endpoint."""201 """Authenticates a regular user with the keystone public endpoint."""
202 self.log.debug('Authenticating keystone user ({})...'.format(user))
162 ep = keystone.service_catalog.url_for(service_type='identity',203 ep = keystone.service_catalog.url_for(service_type='identity',
163 endpoint_type='publicURL')204 endpoint_type='publicURL')
164 return keystone_client.Client(username=user, password=password,205 return keystone_client.Client(username=user, password=password,
@@ -166,19 +207,49 @@
166207
167 def authenticate_glance_admin(self, keystone):208 def authenticate_glance_admin(self, keystone):
168 """Authenticates admin user with glance."""209 """Authenticates admin user with glance."""
210 self.log.debug('Authenticating glance admin...')
169 ep = keystone.service_catalog.url_for(service_type='image',211 ep = keystone.service_catalog.url_for(service_type='image',
170 endpoint_type='adminURL')212 endpoint_type='adminURL')
171 return glance_client.Client(ep, token=keystone.auth_token)213 return glance_client.Client(ep, token=keystone.auth_token)
172214
215 def authenticate_heat_admin(self, keystone):
216 """Authenticates the admin user with heat."""
217 self.log.debug('Authenticating heat admin...')
218 ep = keystone.service_catalog.url_for(service_type='orchestration',
219 endpoint_type='publicURL')
220 return heat_client.Client(endpoint=ep, token=keystone.auth_token)
221
173 def authenticate_nova_user(self, keystone, user, password, tenant):222 def authenticate_nova_user(self, keystone, user, password, tenant):
174 """Authenticates a regular user with nova-api."""223 """Authenticates a regular user with nova-api."""
224 self.log.debug('Authenticating nova user ({})...'.format(user))
175 ep = keystone.service_catalog.url_for(service_type='identity',225 ep = keystone.service_catalog.url_for(service_type='identity',
176 endpoint_type='publicURL')226 endpoint_type='publicURL')
177 return nova_client.Client(username=user, api_key=password,227 return nova_client.Client(username=user, api_key=password,
178 project_id=tenant, auth_url=ep)228 project_id=tenant, auth_url=ep)
179229
230 def authenticate_swift_user(self, keystone, user, password, tenant):
231 """Authenticates a regular user with swift api."""
232 self.log.debug('Authenticating swift user ({})...'.format(user))
233 ep = keystone.service_catalog.url_for(service_type='identity',
234 endpoint_type='publicURL')
235 return swiftclient.Connection(authurl=ep,
236 user=user,
237 key=password,
238 tenant_name=tenant,
239 auth_version='2.0')
240
180 def create_cirros_image(self, glance, image_name):241 def create_cirros_image(self, glance, image_name):
181 """Download the latest cirros image and upload it to glance."""242 """Download the latest cirros image and upload it to glance,
243 validate and return a resource pointer.
244
245 :param glance: pointer to authenticated glance connection
246 :param image_name: display name for new image
247 :returns: glance image pointer
248 """
249 self.log.debug('Creating glance cirros image '
250 '({})...'.format(image_name))
251
252 # Download cirros image
182 http_proxy = os.getenv('AMULET_HTTP_PROXY')253 http_proxy = os.getenv('AMULET_HTTP_PROXY')
183 self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))254 self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
184 if http_proxy:255 if http_proxy:
@@ -187,57 +258,67 @@
187 else:258 else:
188 opener = urllib.FancyURLopener()259 opener = urllib.FancyURLopener()
189260
190 f = opener.open("http://download.cirros-cloud.net/version/released")261 f = opener.open('http://download.cirros-cloud.net/version/released')
191 version = f.read().strip()262 version = f.read().strip()
192 cirros_img = "cirros-{}-x86_64-disk.img".format(version)263 cirros_img = 'cirros-{}-x86_64-disk.img'.format(version)
193 local_path = os.path.join('tests', cirros_img)264 local_path = os.path.join('tests', cirros_img)
194265
195 if not os.path.exists(local_path):266 if not os.path.exists(local_path):
196 cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net",267 cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net',
197 version, cirros_img)268 version, cirros_img)
198 opener.retrieve(cirros_url, local_path)269 opener.retrieve(cirros_url, local_path)
199 f.close()270 f.close()
200271
272 # Create glance image
201 with open(local_path) as f:273 with open(local_path) as f:
202 image = glance.images.create(name=image_name, is_public=True,274 image = glance.images.create(name=image_name, is_public=True,
203 disk_format='qcow2',275 disk_format='qcow2',
204 container_format='bare', data=f)276 container_format='bare', data=f)
205 count = 1277
206 status = image.status278 # Wait for image to reach active status
207 while status != 'active' and count < 10:279 img_id = image.id
208 time.sleep(3)280 ret = self.resource_reaches_status(glance.images, img_id,
209 image = glance.images.get(image.id)281 expected_stat='active',
210 status = image.status282 msg='Image status wait')
211 self.log.debug('image status: {}'.format(status))283 if not ret:
212 count += 1284 msg = 'Glance image failed to reach expected state.'
213285 amulet.raise_status(amulet.FAIL, msg=msg)
214 if status != 'active':286
215 self.log.error('image creation timed out')287 # Re-validate new image
216 return None288 self.log.debug('Validating image attributes...')
289 val_img_name = glance.images.get(img_id).name
290 val_img_stat = glance.images.get(img_id).status
291 val_img_pub = glance.images.get(img_id).is_public
292 val_img_cfmt = glance.images.get(img_id).container_format
293 val_img_dfmt = glance.images.get(img_id).disk_format
294 msg_attr = ('Image attributes - name:{} public:{} id:{} stat:{} '
295 'container fmt:{} disk fmt:{}'.format(
296 val_img_name, val_img_pub, img_id,
297 val_img_stat, val_img_cfmt, val_img_dfmt))
298
299 if val_img_name == image_name and val_img_stat == 'active' \
300 and val_img_pub is True and val_img_cfmt == 'bare' \
301 and val_img_dfmt == 'qcow2':
302 self.log.debug(msg_attr)
303 else:
304 msg = ('Volume validation failed, {}'.format(msg_attr))
305 amulet.raise_status(amulet.FAIL, msg=msg)
217306
218 return image307 return image
219308
220 def delete_image(self, glance, image):309 def delete_image(self, glance, image):
221 """Delete the specified image."""310 """Delete the specified image."""
222 num_before = len(list(glance.images.list()))311
223 glance.images.delete(image)312 # /!\ DEPRECATION WARNING
224313 self.log.warn('/!\\ DEPRECATION WARNING: use '
225 count = 1314 'delete_resource instead of delete_image.')
226 num_after = len(list(glance.images.list()))315 self.log.debug('Deleting glance image ({})...'.format(image))
227 while num_after != (num_before - 1) and count < 10:316 return self.delete_resource(glance.images, image, msg='glance image')
228 time.sleep(3)
229 num_after = len(list(glance.images.list()))
230 self.log.debug('number of images: {}'.format(num_after))
231 count += 1
232
233 if num_after != (num_before - 1):
234 self.log.error('image deletion timed out')
235 return False
236
237 return True
238317
239 def create_instance(self, nova, image_name, instance_name, flavor):318 def create_instance(self, nova, image_name, instance_name, flavor):
240 """Create the specified instance."""319 """Create the specified instance."""
320 self.log.debug('Creating instance '
321 '({}|{}|{})'.format(instance_name, image_name, flavor))
241 image = nova.images.find(name=image_name)322 image = nova.images.find(name=image_name)
242 flavor = nova.flavors.find(name=flavor)323 flavor = nova.flavors.find(name=flavor)
243 instance = nova.servers.create(name=instance_name, image=image,324 instance = nova.servers.create(name=instance_name, image=image,
@@ -260,19 +341,623 @@
260341
261 def delete_instance(self, nova, instance):342 def delete_instance(self, nova, instance):
262 """Delete the specified instance."""343 """Delete the specified instance."""
263 num_before = len(list(nova.servers.list()))344
264 nova.servers.delete(instance)345 # /!\ DEPRECATION WARNING
265346 self.log.warn('/!\\ DEPRECATION WARNING: use '
266 count = 1347 'delete_resource instead of delete_instance.')
267 num_after = len(list(nova.servers.list()))348 self.log.debug('Deleting instance ({})...'.format(instance))
268 while num_after != (num_before - 1) and count < 10:349 return self.delete_resource(nova.servers, instance,
269 time.sleep(3)350 msg='nova instance')
270 num_after = len(list(nova.servers.list()))351
271 self.log.debug('number of instances: {}'.format(num_after))352 def create_or_get_keypair(self, nova, keypair_name="testkey"):
272 count += 1353 """Create a new keypair, or return pointer if it already exists."""
273354 try:
274 if num_after != (num_before - 1):355 _keypair = nova.keypairs.get(keypair_name)
275 self.log.error('instance deletion timed out')356 self.log.debug('Keypair ({}) already exists, '
276 return False357 'using it.'.format(keypair_name))
277358 return _keypair
278 return True359 except:
360 self.log.debug('Keypair ({}) does not exist, '
361 'creating it.'.format(keypair_name))
362
363 _keypair = nova.keypairs.create(name=keypair_name)
364 return _keypair
365
366 def create_cinder_volume(self, cinder, vol_name="demo-vol", vol_size=1,
367 img_id=None, src_vol_id=None, snap_id=None):
368 """Create cinder volume, optionally from a glance image, OR
369 optionally as a clone of an existing volume, OR optionally
370 from a snapshot. Wait for the new volume status to reach
371 the expected status, validate and return a resource pointer.
372
373 :param vol_name: cinder volume display name
374 :param vol_size: size in gigabytes
375 :param img_id: optional glance image id
376 :param src_vol_id: optional source volume id to clone
377 :param snap_id: optional snapshot id to use
378 :returns: cinder volume pointer
379 """
380 # Handle parameter input and avoid impossible combinations
381 if img_id and not src_vol_id and not snap_id:
382 # Create volume from image
383 self.log.debug('Creating cinder volume from glance image...')
384 bootable = 'true'
385 elif src_vol_id and not img_id and not snap_id:
386 # Clone an existing volume
387 self.log.debug('Cloning cinder volume...')
388 bootable = cinder.volumes.get(src_vol_id).bootable
389 elif snap_id and not src_vol_id and not img_id:
390 # Create volume from snapshot
391 self.log.debug('Creating cinder volume from snapshot...')
392 snap = cinder.volume_snapshots.find(id=snap_id)
393 vol_size = snap.size
394 snap_vol_id = cinder.volume_snapshots.get(snap_id).volume_id
395 bootable = cinder.volumes.get(snap_vol_id).bootable
396 elif not img_id and not src_vol_id and not snap_id:
397 # Create volume
398 self.log.debug('Creating cinder volume...')
399 bootable = 'false'
400 else:
401 # Impossible combination of parameters
402 msg = ('Invalid method use - name:{} size:{} img_id:{} '
403 'src_vol_id:{} snap_id:{}'.format(vol_name, vol_size,
404 img_id, src_vol_id,
405 snap_id))
406 amulet.raise_status(amulet.FAIL, msg=msg)
407
408 # Create new volume
409 try:
410 vol_new = cinder.volumes.create(display_name=vol_name,
411 imageRef=img_id,
412 size=vol_size,
413 source_volid=src_vol_id,
414 snapshot_id=snap_id)
415 vol_id = vol_new.id
416 except Exception as e:
417 msg = 'Failed to create volume: {}'.format(e)
418 amulet.raise_status(amulet.FAIL, msg=msg)
419
420 # Wait for volume to reach available status
421 ret = self.resource_reaches_status(cinder.volumes, vol_id,
422 expected_stat="available",
423 msg="Volume status wait")
424 if not ret:
425 msg = 'Cinder volume failed to reach expected state.'
426 amulet.raise_status(amulet.FAIL, msg=msg)
427
428 # Re-validate new volume
429 self.log.debug('Validating volume attributes...')
430 val_vol_name = cinder.volumes.get(vol_id).display_name
431 val_vol_boot = cinder.volumes.get(vol_id).bootable
432 val_vol_stat = cinder.volumes.get(vol_id).status
433 val_vol_size = cinder.volumes.get(vol_id).size
434 msg_attr = ('Volume attributes - name:{} id:{} stat:{} boot:'
435 '{} size:{}'.format(val_vol_name, vol_id,
436 val_vol_stat, val_vol_boot,
437 val_vol_size))
438
439 if val_vol_boot == bootable and val_vol_stat == 'available' \
440 and val_vol_name == vol_name and val_vol_size == vol_size:
441 self.log.debug(msg_attr)
442 else:
443 msg = ('Volume validation failed, {}'.format(msg_attr))
444 amulet.raise_status(amulet.FAIL, msg=msg)
445
446 return vol_new
447
448 def delete_resource(self, resource, resource_id,
449 msg="resource", max_wait=120):
450 """Delete one openstack resource, such as one instance, keypair,
451 image, volume, stack, etc., and confirm deletion within max wait time.
452
453 :param resource: pointer to os resource type, ex:glance_client.images
454 :param resource_id: unique name or id for the openstack resource
455 :param msg: text to identify purpose in logging
456 :param max_wait: maximum wait time in seconds
457 :returns: True if successful, otherwise False
458 """
459 self.log.debug('Deleting OpenStack resource '
460 '{} ({})'.format(resource_id, msg))
461 num_before = len(list(resource.list()))
462 resource.delete(resource_id)
463
464 tries = 0
465 num_after = len(list(resource.list()))
466 while num_after != (num_before - 1) and tries < (max_wait / 4):
467 self.log.debug('{} delete check: '
468 '{} [{}:{}] {}'.format(msg, tries,
469 num_before,
470 num_after,
471 resource_id))
472 time.sleep(4)
473 num_after = len(list(resource.list()))
474 tries += 1
475
476 self.log.debug('{}: expected, actual count = {}, '
477 '{}'.format(msg, num_before - 1, num_after))
478
479 if num_after == (num_before - 1):
480 return True
481 else:
482 self.log.error('{} delete timed out'.format(msg))
483 return False
484
485 def resource_reaches_status(self, resource, resource_id,
486 expected_stat='available',
487 msg='resource', max_wait=120):
488 """Wait for an openstack resources status to reach an
489 expected status within a specified time. Useful to confirm that
490 nova instances, cinder vols, snapshots, glance images, heat stacks
491 and other resources eventually reach the expected status.
492
493 :param resource: pointer to os resource type, ex: heat_client.stacks
494 :param resource_id: unique id for the openstack resource
495 :param expected_stat: status to expect resource to reach
496 :param msg: text to identify purpose in logging
497 :param max_wait: maximum wait time in seconds
498 :returns: True if successful, False if status is not reached
499 """
500
501 tries = 0
502 resource_stat = resource.get(resource_id).status
503 while resource_stat != expected_stat and tries < (max_wait / 4):
504 self.log.debug('{} status check: '
505 '{} [{}:{}] {}'.format(msg, tries,
506 resource_stat,
507 expected_stat,
508 resource_id))
509 time.sleep(4)
510 resource_stat = resource.get(resource_id).status
511 tries += 1
512
513 self.log.debug('{}: expected, actual status = {}, '
514 '{}'.format(msg, resource_stat, expected_stat))
515
516 if resource_stat == expected_stat:
517 return True
518 else:
519 self.log.debug('{} never reached expected status: '
520 '{}'.format(resource_id, expected_stat))
521 return False
522
523 def get_ceph_osd_id_cmd(self, index):
524 """Produce a shell command that will return a ceph-osd id."""
525 return ("`initctl list | grep 'ceph-osd ' | "
526 "awk 'NR=={} {{ print $2 }}' | "
527 "grep -o '[0-9]*'`".format(index + 1))
528
529 def get_ceph_pools(self, sentry_unit):
530 """Return a dict of ceph pools from a single ceph unit, with
531 pool name as keys, pool id as vals."""
532 pools = {}
533 cmd = 'sudo ceph osd lspools'
534 output, code = sentry_unit.run(cmd)
535 if code != 0:
536 msg = ('{} `{}` returned {} '
537 '{}'.format(sentry_unit.info['unit_name'],
538 cmd, code, output))
539 amulet.raise_status(amulet.FAIL, msg=msg)
540
541 # Example output: 0 data,1 metadata,2 rbd,3 cinder,4 glance,
542 for pool in str(output).split(','):
543 pool_id_name = pool.split(' ')
544 if len(pool_id_name) == 2:
545 pool_id = pool_id_name[0]
546 pool_name = pool_id_name[1]
547 pools[pool_name] = int(pool_id)
548
549 self.log.debug('Pools on {}: {}'.format(sentry_unit.info['unit_name'],
550 pools))
551 return pools
552
553 def get_ceph_df(self, sentry_unit):
554 """Return dict of ceph df json output, including ceph pool state.
555
556 :param sentry_unit: Pointer to amulet sentry instance (juju unit)
557 :returns: Dict of ceph df output
558 """
559 cmd = 'sudo ceph df --format=json'
560 output, code = sentry_unit.run(cmd)
561 if code != 0:
562 msg = ('{} `{}` returned {} '
563 '{}'.format(sentry_unit.info['unit_name'],
564 cmd, code, output))
565 amulet.raise_status(amulet.FAIL, msg=msg)
566 return json.loads(output)
567
568 def get_ceph_pool_sample(self, sentry_unit, pool_id=0):
569 """Take a sample of attributes of a ceph pool, returning ceph
570 pool name, object count and disk space used for the specified
571 pool ID number.
572
573 :param sentry_unit: Pointer to amulet sentry instance (juju unit)
574 :param pool_id: Ceph pool ID
575 :returns: List of pool name, object count, kb disk space used
576 """
577 df = self.get_ceph_df(sentry_unit)
578 pool_name = df['pools'][pool_id]['name']
579 obj_count = df['pools'][pool_id]['stats']['objects']
580 kb_used = df['pools'][pool_id]['stats']['kb_used']
581 self.log.debug('Ceph {} pool (ID {}): {} objects, '
582 '{} kb used'.format(pool_name, pool_id,
583 obj_count, kb_used))
584 return pool_name, obj_count, kb_used
585
586 def validate_ceph_pool_samples(self, samples, sample_type="resource pool"):
587 """Validate ceph pool samples taken over time, such as pool
588 object counts or pool kb used, before adding, after adding, and
589 after deleting items which affect those pool attributes. The
590 2nd element is expected to be greater than the 1st; 3rd is expected
591 to be less than the 2nd.
592
593 :param samples: List containing 3 data samples
594 :param sample_type: String for logging and usage context
595 :returns: None if successful, Failure message otherwise
596 """
597 original, created, deleted = range(3)
598 if samples[created] <= samples[original] or \
599 samples[deleted] >= samples[created]:
600 return ('Ceph {} samples ({}) '
601 'unexpected.'.format(sample_type, samples))
602 else:
603 self.log.debug('Ceph {} samples (OK): '
604 '{}'.format(sample_type, samples))
605 return None
606
607# rabbitmq/amqp specific helpers:
608 def add_rmq_test_user(self, sentry_units,
609 username="testuser1", password="changeme"):
610 """Add a test user via the first rmq juju unit, check connection as
611 the new user against all sentry units.
612
613 :param sentry_units: list of sentry unit pointers
614 :param username: amqp user name, default to testuser1
615 :param password: amqp user password
616 :returns: None if successful. Raise on error.
617 """
618 self.log.debug('Adding rmq user ({})...'.format(username))
619
620 # Check that user does not already exist
621 cmd_user_list = 'rabbitmqctl list_users'
622 output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
623 if username in output:
624 self.log.warning('User ({}) already exists, returning '
625 'gracefully.'.format(username))
626 return
627
628 perms = '".*" ".*" ".*"'
629 cmds = ['rabbitmqctl add_user {} {}'.format(username, password),
630 'rabbitmqctl set_permissions {} {}'.format(username, perms)]
631
632 # Add user via first unit
633 for cmd in cmds:
634 output, _ = self.run_cmd_unit(sentry_units[0], cmd)
635
636 # Check connection against the other sentry_units
637 self.log.debug('Checking user connect against units...')
638 for sentry_unit in sentry_units:
639 connection = self.connect_amqp_by_unit(sentry_unit, ssl=False,
640 username=username,
641 password=password)
642 connection.close()
643
644 def delete_rmq_test_user(self, sentry_units, username="testuser1"):
645 """Delete a rabbitmq user via the first rmq juju unit.
646
647 :param sentry_units: list of sentry unit pointers
648 :param username: amqp user name, default to testuser1
649 :param password: amqp user password
650 :returns: None if successful or no such user.
651 """
652 self.log.debug('Deleting rmq user ({})...'.format(username))
653
654 # Check that the user exists
655 cmd_user_list = 'rabbitmqctl list_users'
656 output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
657
658 if username not in output:
659 self.log.warning('User ({}) does not exist, returning '
660 'gracefully.'.format(username))
661 return
662
663 # Delete the user
664 cmd_user_del = 'rabbitmqctl delete_user {}'.format(username)
665 output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del)
666
667 def get_rmq_cluster_status(self, sentry_unit):
668 """Execute rabbitmq cluster status command on a unit and return
669 the full output.
670
671 :param unit: sentry unit
672 :returns: String containing console output of cluster status command
673 """
674 cmd = 'rabbitmqctl cluster_status'
675 output, _ = self.run_cmd_unit(sentry_unit, cmd)
676 self.log.debug('{} cluster_status:\n{}'.format(
677 sentry_unit.info['unit_name'], output))
678 return str(output)
679
680 def get_rmq_cluster_running_nodes(self, sentry_unit):
681 """Parse rabbitmqctl cluster_status output string, return list of
682 running rabbitmq cluster nodes.
683
684 :param unit: sentry unit
685 :returns: List containing node names of running nodes
686 """
687 # NOTE(beisner): rabbitmqctl cluster_status output is not
688 # json-parsable, do string chop foo, then json.loads that.
689 str_stat = self.get_rmq_cluster_status(sentry_unit)
690 if 'running_nodes' in str_stat:
691 pos_start = str_stat.find("{running_nodes,") + 15
692 pos_end = str_stat.find("]},", pos_start) + 1
693 str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"')
694 run_nodes = json.loads(str_run_nodes)
695 return run_nodes
696 else:
697 return []
698
699 def validate_rmq_cluster_running_nodes(self, sentry_units):
700 """Check that all rmq unit hostnames are represented in the
701 cluster_status output of all units.
702
703 :param host_names: dict of juju unit names to host names
704 :param units: list of sentry unit pointers (all rmq units)
705 :returns: None if successful, otherwise return error message
706 """
707 host_names = self.get_unit_hostnames(sentry_units)
708 errors = []
709
710 # Query every unit for cluster_status running nodes
711 for query_unit in sentry_units:
712 query_unit_name = query_unit.info['unit_name']
713 running_nodes = self.get_rmq_cluster_running_nodes(query_unit)
714
715 # Confirm that every unit is represented in the queried unit's
716 # cluster_status running nodes output.
717 for validate_unit in sentry_units:
718 val_host_name = host_names[validate_unit.info['unit_name']]
719 val_node_name = 'rabbit@{}'.format(val_host_name)
720
721 if val_node_name not in running_nodes:
722 errors.append('Cluster member check failed on {}: {} not '
723 'in {}\n'.format(query_unit_name,
724 val_node_name,
725 running_nodes))
726 if errors:
727 return ''.join(errors)
728
729 def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None):
730 """Check a single juju rmq unit for ssl and port in the config file."""
731 host = sentry_unit.info['public-address']
732 unit_name = sentry_unit.info['unit_name']
733
734 conf_file = '/etc/rabbitmq/rabbitmq.config'
735 conf_contents = str(self.file_contents_safe(sentry_unit,
736 conf_file, max_wait=16))
737 # Checks
738 conf_ssl = 'ssl' in conf_contents
739 conf_port = str(port) in conf_contents
740
741 # Port explicitly checked in config
742 if port and conf_port and conf_ssl:
743 self.log.debug('SSL is enabled @{}:{} '
744 '({})'.format(host, port, unit_name))
745 return True
746 elif port and not conf_port and conf_ssl:
747 self.log.debug('SSL is enabled @{} but not on port {} '
748 '({})'.format(host, port, unit_name))
749 return False
750 # Port not checked (useful when checking that ssl is disabled)
751 elif not port and conf_ssl:
752 self.log.debug('SSL is enabled @{}:{} '
753 '({})'.format(host, port, unit_name))
754 return True
755 elif not port and not conf_ssl:
756 self.log.debug('SSL not enabled @{}:{} '
757 '({})'.format(host, port, unit_name))
758 return False
759 else:
760 msg = ('Unknown condition when checking SSL status @{}:{} '
761 '({})'.format(host, port, unit_name))
762 amulet.raise_status(amulet.FAIL, msg)
763
764 def validate_rmq_ssl_enabled_units(self, sentry_units, port=None):
765 """Check that ssl is enabled on rmq juju sentry units.
766
767 :param sentry_units: list of all rmq sentry units
768 :param port: optional ssl port override to validate
769 :returns: None if successful, otherwise return error message
770 """
771 for sentry_unit in sentry_units:
772 if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port):
773 return ('Unexpected condition: ssl is disabled on unit '
774 '({})'.format(sentry_unit.info['unit_name']))
775 return None
776
777 def validate_rmq_ssl_disabled_units(self, sentry_units):
778 """Check that ssl is enabled on listed rmq juju sentry units.
779
780 :param sentry_units: list of all rmq sentry units
781 :returns: True if successful. Raise on error.
782 """
783 for sentry_unit in sentry_units:
784 if self.rmq_ssl_is_enabled_on_unit(sentry_unit):
785 return ('Unexpected condition: ssl is enabled on unit '
786 '({})'.format(sentry_unit.info['unit_name']))
787 return None
788
789 def configure_rmq_ssl_on(self, sentry_units, deployment,
790 port=None, max_wait=60):
791 """Turn ssl charm config option on, with optional non-default
792 ssl port specification. Confirm that it is enabled on every
793 unit.
794
795 :param sentry_units: list of sentry units
796 :param deployment: amulet deployment object pointer
797 :param port: amqp port, use defaults if None
798 :param max_wait: maximum time to wait in seconds to confirm
799 :returns: None if successful. Raise on error.
800 """
801 self.log.debug('Setting ssl charm config option: on')
802
803 # Enable RMQ SSL
804 config = {'ssl': 'on'}
805 if port:
806 config['ssl_port'] = port
807
808 deployment.configure('rabbitmq-server', config)
809
810 # Confirm
811 tries = 0
812 ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
813 while ret and tries < (max_wait / 4):
814 time.sleep(4)
815 self.log.debug('Attempt {}: {}'.format(tries, ret))
816 ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
817 tries += 1
818
819 if ret:
820 amulet.raise_status(amulet.FAIL, ret)
821
822 def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60):
823 """Turn ssl charm config option off, confirm that it is disabled
824 on every unit.
825
826 :param sentry_units: list of sentry units
827 :param deployment: amulet deployment object pointer
828 :param max_wait: maximum time to wait in seconds to confirm
829 :returns: None if successful. Raise on error.
830 """
831 self.log.debug('Setting ssl charm config option: off')
832
833 # Disable RMQ SSL
834 config = {'ssl': 'off'}
835 deployment.configure('rabbitmq-server', config)
836
837 # Confirm
838 tries = 0
839 ret = self.validate_rmq_ssl_disabled_units(sentry_units)
840 while ret and tries < (max_wait / 4):
841 time.sleep(4)
842 self.log.debug('Attempt {}: {}'.format(tries, ret))
843 ret = self.validate_rmq_ssl_disabled_units(sentry_units)
844 tries += 1
845
846 if ret:
847 amulet.raise_status(amulet.FAIL, ret)
848
849 def connect_amqp_by_unit(self, sentry_unit, ssl=False,
850 port=None, fatal=True,
851 username="testuser1", password="changeme"):
852 """Establish and return a pika amqp connection to the rabbitmq service
853 running on a rmq juju unit.
854
855 :param sentry_unit: sentry unit pointer
856 :param ssl: boolean, default to False
857 :param port: amqp port, use defaults if None
858 :param fatal: boolean, default to True (raises on connect error)
859 :param username: amqp user name, default to testuser1
860 :param password: amqp user password
861 :returns: pika amqp connection pointer or None if failed and non-fatal
862 """
863 host = sentry_unit.info['public-address']
864 unit_name = sentry_unit.info['unit_name']
865
866 # Default port logic if port is not specified
867 if ssl and not port:
868 port = 5671
869 elif not ssl and not port:
870 port = 5672
871
872 self.log.debug('Connecting to amqp on {}:{} ({}) as '
873 '{}...'.format(host, port, unit_name, username))
874
875 try:
876 credentials = pika.PlainCredentials(username, password)
877 parameters = pika.ConnectionParameters(host=host, port=port,
878 credentials=credentials,
879 ssl=ssl,
880 connection_attempts=3,
881 retry_delay=5,
882 socket_timeout=1)
883 connection = pika.BlockingConnection(parameters)
884 assert connection.server_properties['product'] == 'RabbitMQ'
885 self.log.debug('Connect OK')
886 return connection
887 except Exception as e:
888 msg = ('amqp connection failed to {}:{} as '
889 '{} ({})'.format(host, port, username, str(e)))
890 if fatal:
891 amulet.raise_status(amulet.FAIL, msg)
892 else:
893 self.log.warn(msg)
894 return None
895
896 def publish_amqp_message_by_unit(self, sentry_unit, message,
897 queue="test", ssl=False,
898 username="testuser1",
899 password="changeme",
900 port=None):
901 """Publish an amqp message to a rmq juju unit.
902
903 :param sentry_unit: sentry unit pointer
904 :param message: amqp message string
905 :param queue: message queue, default to test
906 :param username: amqp user name, default to testuser1
907 :param password: amqp user password
908 :param ssl: boolean, default to False
909 :param port: amqp port, use defaults if None
910 :returns: None. Raises exception if publish failed.
911 """
912 self.log.debug('Publishing message to {} queue:\n{}'.format(queue,
913 message))
914 connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
915 port=port,
916 username=username,
917 password=password)
918
919 # NOTE(beisner): extra debug here re: pika hang potential:
920 # https://github.com/pika/pika/issues/297
921 # https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw
922 self.log.debug('Defining channel...')
923 channel = connection.channel()
924 self.log.debug('Declaring queue...')
925 channel.queue_declare(queue=queue, auto_delete=False, durable=True)
926 self.log.debug('Publishing message...')
927 channel.basic_publish(exchange='', routing_key=queue, body=message)
928 self.log.debug('Closing channel...')
929 channel.close()
930 self.log.debug('Closing connection...')
931 connection.close()
932
933 def get_amqp_message_by_unit(self, sentry_unit, queue="test",
934 username="testuser1",
935 password="changeme",
936 ssl=False, port=None):
937 """Get an amqp message from a rmq juju unit.
938
939 :param sentry_unit: sentry unit pointer
940 :param queue: message queue, default to test
941 :param username: amqp user name, default to testuser1
942 :param password: amqp user password
943 :param ssl: boolean, default to False
944 :param port: amqp port, use defaults if None
945 :returns: amqp message body as string. Raise if get fails.
946 """
947 connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
948 port=port,
949 username=username,
950 password=password)
951 channel = connection.channel()
952 method_frame, _, body = channel.basic_get(queue)
953
954 if method_frame:
955 self.log.debug('Retreived message from {} queue:\n{}'.format(queue,
956 body))
957 channel.basic_ack(method_frame.delivery_tag)
958 channel.close()
959 connection.close()
960 return body
961 else:
962 msg = 'No message retrieved.'
963 amulet.raise_status(amulet.FAIL, msg)
279964
=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py 2015-01-14 15:30:19 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py 2015-09-21 10:47:21 +0000
@@ -1,10 +1,28 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1import json17import json
2import os18import os
19import re
3import time20import time
4from base64 import b64decode21from base64 import b64decode
5from subprocess import check_call22from subprocess import check_call
623
7import six24import six
25import yaml
826
9from charmhelpers.fetch import (27from charmhelpers.fetch import (
10 apt_install,28 apt_install,
@@ -29,8 +47,13 @@
29)47)
3048
31from charmhelpers.core.sysctl import create as sysctl_create49from charmhelpers.core.sysctl import create as sysctl_create
50from charmhelpers.core.strutils import bool_from_string
3251
33from charmhelpers.core.host import (52from charmhelpers.core.host import (
53 get_bond_master,
54 is_phy_iface,
55 list_nics,
56 get_nic_hwaddr,
34 mkdir,57 mkdir,
35 write_file,58 write_file,
36)59)
@@ -47,16 +70,22 @@
47)70)
48from charmhelpers.contrib.openstack.neutron import (71from charmhelpers.contrib.openstack.neutron import (
49 neutron_plugin_attribute,72 neutron_plugin_attribute,
73 parse_data_port_mappings,
74)
75from charmhelpers.contrib.openstack.ip import (
76 resolve_address,
77 INTERNAL,
50)78)
51from charmhelpers.contrib.network.ip import (79from charmhelpers.contrib.network.ip import (
52 get_address_in_network,80 get_address_in_network,
81 get_ipv4_addr,
53 get_ipv6_addr,82 get_ipv6_addr,
54 get_netmask_for_address,83 get_netmask_for_address,
55 format_ipv6_addr,84 format_ipv6_addr,
56 is_address_in_network,85 is_address_in_network,
86 is_bridge_member,
57)87)
58from charmhelpers.contrib.openstack.utils import get_host_ip88from charmhelpers.contrib.openstack.utils import get_host_ip
59
60CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'89CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
61ADDRESS_TYPES = ['admin', 'internal', 'public']90ADDRESS_TYPES = ['admin', 'internal', 'public']
6291
@@ -88,9 +117,44 @@
88def config_flags_parser(config_flags):117def config_flags_parser(config_flags):
89 """Parses config flags string into dict.118 """Parses config flags string into dict.
90119
120 This parsing method supports a few different formats for the config
121 flag values to be parsed:
122
123 1. A string in the simple format of key=value pairs, with the possibility
124 of specifying multiple key value pairs within the same string. For
125 example, a string in the format of 'key1=value1, key2=value2' will
126 return a dict of:
127
128 {'key1': 'value1',
129 'key2': 'value2'}.
130
131 2. A string in the above format, but supporting a comma-delimited list
132 of values for the same key. For example, a string in the format of
133 'key1=value1, key2=value3,value4,value5' will return a dict of:
134
135 {'key1', 'value1',
136 'key2', 'value2,value3,value4'}
137
138 3. A string containing a colon character (:) prior to an equal
139 character (=) will be treated as yaml and parsed as such. This can be
140 used to specify more complex key value pairs. For example,
141 a string in the format of 'key1: subkey1=value1, subkey2=value2' will
142 return a dict of:
143
144 {'key1', 'subkey1=value1, subkey2=value2'}
145
91 The provided config_flags string may be a list of comma-separated values146 The provided config_flags string may be a list of comma-separated values
92 which themselves may be comma-separated list of values.147 which themselves may be comma-separated list of values.
93 """148 """
149 # If we find a colon before an equals sign then treat it as yaml.
150 # Note: limit it to finding the colon first since this indicates assignment
151 # for inline yaml.
152 colon = config_flags.find(':')
153 equals = config_flags.find('=')
154 if colon > 0:
155 if colon < equals or equals < 0:
156 return yaml.safe_load(config_flags)
157
94 if config_flags.find('==') >= 0:158 if config_flags.find('==') >= 0:
95 log("config_flags is not in expected format (key=value)", level=ERROR)159 log("config_flags is not in expected format (key=value)", level=ERROR)
96 raise OSContextError160 raise OSContextError
@@ -130,10 +194,50 @@
130class OSContextGenerator(object):194class OSContextGenerator(object):
131 """Base class for all context generators."""195 """Base class for all context generators."""
132 interfaces = []196 interfaces = []
197 related = False
198 complete = False
199 missing_data = []
133200
134 def __call__(self):201 def __call__(self):
135 raise NotImplementedError202 raise NotImplementedError
136203
204 def context_complete(self, ctxt):
205 """Check for missing data for the required context data.
206 Set self.missing_data if it exists and return False.
207 Set self.complete if no missing data and return True.
208 """
209 # Fresh start
210 self.complete = False
211 self.missing_data = []
212 for k, v in six.iteritems(ctxt):
213 if v is None or v == '':
214 if k not in self.missing_data:
215 self.missing_data.append(k)
216
217 if self.missing_data:
218 self.complete = False
219 log('Missing required data: %s' % ' '.join(self.missing_data), level=INFO)
220 else:
221 self.complete = True
222 return self.complete
223
224 def get_related(self):
225 """Check if any of the context interfaces have relation ids.
226 Set self.related and return True if one of the interfaces
227 has relation ids.
228 """
229 # Fresh start
230 self.related = False
231 try:
232 for interface in self.interfaces:
233 if relation_ids(interface):
234 self.related = True
235 return self.related
236 except AttributeError as e:
237 log("{} {}"
238 "".format(self, e), 'INFO')
239 return self.related
240
137241
138class SharedDBContext(OSContextGenerator):242class SharedDBContext(OSContextGenerator):
139 interfaces = ['shared-db']243 interfaces = ['shared-db']
@@ -149,6 +253,7 @@
149 self.database = database253 self.database = database
150 self.user = user254 self.user = user
151 self.ssl_dir = ssl_dir255 self.ssl_dir = ssl_dir
256 self.rel_name = self.interfaces[0]
152257
153 def __call__(self):258 def __call__(self):
154 self.database = self.database or config('database')259 self.database = self.database or config('database')
@@ -175,13 +280,14 @@
175 unit=local_unit())280 unit=local_unit())
176 if set_hostname != access_hostname:281 if set_hostname != access_hostname:
177 relation_set(relation_settings={hostname_key: access_hostname})282 relation_set(relation_settings={hostname_key: access_hostname})
178 return ctxt # Defer any further hook execution for now....283 return None # Defer any further hook execution for now....
179284
180 password_setting = 'password'285 password_setting = 'password'
181 if self.relation_prefix:286 if self.relation_prefix:
182 password_setting = self.relation_prefix + '_password'287 password_setting = self.relation_prefix + '_password'
183288
184 for rid in relation_ids('shared-db'):289 for rid in relation_ids(self.interfaces[0]):
290 self.related = True
185 for unit in related_units(rid):291 for unit in related_units(rid):
186 rdata = relation_get(rid=rid, unit=unit)292 rdata = relation_get(rid=rid, unit=unit)
187 host = rdata.get('db_host')293 host = rdata.get('db_host')
@@ -193,7 +299,7 @@
193 'database_password': rdata.get(password_setting),299 'database_password': rdata.get(password_setting),
194 'database_type': 'mysql'300 'database_type': 'mysql'
195 }301 }
196 if context_complete(ctxt):302 if self.context_complete(ctxt):
197 db_ssl(rdata, ctxt, self.ssl_dir)303 db_ssl(rdata, ctxt, self.ssl_dir)
198 return ctxt304 return ctxt
199 return {}305 return {}
@@ -214,6 +320,7 @@
214320
215 ctxt = {}321 ctxt = {}
216 for rid in relation_ids(self.interfaces[0]):322 for rid in relation_ids(self.interfaces[0]):
323 self.related = True
217 for unit in related_units(rid):324 for unit in related_units(rid):
218 rel_host = relation_get('host', rid=rid, unit=unit)325 rel_host = relation_get('host', rid=rid, unit=unit)
219 rel_user = relation_get('user', rid=rid, unit=unit)326 rel_user = relation_get('user', rid=rid, unit=unit)
@@ -223,7 +330,7 @@
223 'database_user': rel_user,330 'database_user': rel_user,
224 'database_password': rel_passwd,331 'database_password': rel_passwd,
225 'database_type': 'postgresql'}332 'database_type': 'postgresql'}
226 if context_complete(ctxt):333 if self.context_complete(ctxt):
227 return ctxt334 return ctxt
228335
229 return {}336 return {}
@@ -261,12 +368,30 @@
261368
262369
263class IdentityServiceContext(OSContextGenerator):370class IdentityServiceContext(OSContextGenerator):
264 interfaces = ['identity-service']371
372 def __init__(self, service=None, service_user=None, rel_name='identity-service'):
373 self.service = service
374 self.service_user = service_user
375 self.rel_name = rel_name
376 self.interfaces = [self.rel_name]
265377
266 def __call__(self):378 def __call__(self):
267 log('Generating template context for identity-service', level=DEBUG)379 log('Generating template context for ' + self.rel_name, level=DEBUG)
268 ctxt = {}380 ctxt = {}
269 for rid in relation_ids('identity-service'):381
382 if self.service and self.service_user:
383 # This is required for pki token signing if we don't want /tmp to
384 # be used.
385 cachedir = '/var/cache/%s' % (self.service)
386 if not os.path.isdir(cachedir):
387 log("Creating service cache dir %s" % (cachedir), level=DEBUG)
388 mkdir(path=cachedir, owner=self.service_user,
389 group=self.service_user, perms=0o700)
390
391 ctxt['signing_dir'] = cachedir
392
393 for rid in relation_ids(self.rel_name):
394 self.related = True
270 for unit in related_units(rid):395 for unit in related_units(rid):
271 rdata = relation_get(rid=rid, unit=unit)396 rdata = relation_get(rid=rid, unit=unit)
272 serv_host = rdata.get('service_host')397 serv_host = rdata.get('service_host')
@@ -275,16 +400,17 @@
275 auth_host = format_ipv6_addr(auth_host) or auth_host400 auth_host = format_ipv6_addr(auth_host) or auth_host
276 svc_protocol = rdata.get('service_protocol') or 'http'401 svc_protocol = rdata.get('service_protocol') or 'http'
277 auth_protocol = rdata.get('auth_protocol') or 'http'402 auth_protocol = rdata.get('auth_protocol') or 'http'
278 ctxt = {'service_port': rdata.get('service_port'),403 ctxt.update({'service_port': rdata.get('service_port'),
279 'service_host': serv_host,404 'service_host': serv_host,
280 'auth_host': auth_host,405 'auth_host': auth_host,
281 'auth_port': rdata.get('auth_port'),406 'auth_port': rdata.get('auth_port'),
282 'admin_tenant_name': rdata.get('service_tenant'),407 'admin_tenant_name': rdata.get('service_tenant'),
283 'admin_user': rdata.get('service_username'),408 'admin_user': rdata.get('service_username'),
284 'admin_password': rdata.get('service_password'),409 'admin_password': rdata.get('service_password'),
285 'service_protocol': svc_protocol,410 'service_protocol': svc_protocol,
286 'auth_protocol': auth_protocol}411 'auth_protocol': auth_protocol})
287 if context_complete(ctxt):412
413 if self.context_complete(ctxt):
288 # NOTE(jamespage) this is required for >= icehouse414 # NOTE(jamespage) this is required for >= icehouse
289 # so a missing value just indicates keystone needs415 # so a missing value just indicates keystone needs
290 # upgrading416 # upgrading
@@ -323,6 +449,7 @@
323 ctxt = {}449 ctxt = {}
324 for rid in relation_ids(self.rel_name):450 for rid in relation_ids(self.rel_name):
325 ha_vip_only = False451 ha_vip_only = False
452 self.related = True
326 for unit in related_units(rid):453 for unit in related_units(rid):
327 if relation_get('clustered', rid=rid, unit=unit):454 if relation_get('clustered', rid=rid, unit=unit):
328 ctxt['clustered'] = True455 ctxt['clustered'] = True
@@ -355,7 +482,7 @@
355 ha_vip_only = relation_get('ha-vip-only',482 ha_vip_only = relation_get('ha-vip-only',
356 rid=rid, unit=unit) is not None483 rid=rid, unit=unit) is not None
357484
358 if context_complete(ctxt):485 if self.context_complete(ctxt):
359 if 'rabbit_ssl_ca' in ctxt:486 if 'rabbit_ssl_ca' in ctxt:
360 if not self.ssl_dir:487 if not self.ssl_dir:
361 log("Charm not setup for ssl support but ssl ca "488 log("Charm not setup for ssl support but ssl ca "
@@ -382,7 +509,12 @@
382509
383 ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts))510 ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts))
384511
385 if not context_complete(ctxt):512 oslo_messaging_flags = conf.get('oslo-messaging-flags', None)
513 if oslo_messaging_flags:
514 ctxt['oslo_messaging_flags'] = config_flags_parser(
515 oslo_messaging_flags)
516
517 if not self.complete:
386 return {}518 return {}
387519
388 return ctxt520 return ctxt
@@ -398,13 +530,15 @@
398530
399 log('Generating template context for ceph', level=DEBUG)531 log('Generating template context for ceph', level=DEBUG)
400 mon_hosts = []532 mon_hosts = []
401 auth = None533 ctxt = {
402 key = None534 'use_syslog': str(config('use-syslog')).lower()
403 use_syslog = str(config('use-syslog')).lower()535 }
404 for rid in relation_ids('ceph'):536 for rid in relation_ids('ceph'):
405 for unit in related_units(rid):537 for unit in related_units(rid):
406 auth = relation_get('auth', rid=rid, unit=unit)538 if not ctxt.get('auth'):
407 key = relation_get('key', rid=rid, unit=unit)539 ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
540 if not ctxt.get('key'):
541 ctxt['key'] = relation_get('key', rid=rid, unit=unit)
408 ceph_pub_addr = relation_get('ceph-public-address', rid=rid,542 ceph_pub_addr = relation_get('ceph-public-address', rid=rid,
409 unit=unit)543 unit=unit)
410 unit_priv_addr = relation_get('private-address', rid=rid,544 unit_priv_addr = relation_get('private-address', rid=rid,
@@ -413,15 +547,12 @@
413 ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr547 ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
414 mon_hosts.append(ceph_addr)548 mon_hosts.append(ceph_addr)
415549
416 ctxt = {'mon_hosts': ' '.join(sorted(mon_hosts)),550 ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts))
417 'auth': auth,
418 'key': key,
419 'use_syslog': use_syslog}
420551
421 if not os.path.isdir('/etc/ceph'):552 if not os.path.isdir('/etc/ceph'):
422 os.mkdir('/etc/ceph')553 os.mkdir('/etc/ceph')
423554
424 if not context_complete(ctxt):555 if not self.context_complete(ctxt):
425 return {}556 return {}
426557
427 ensure_packages(['ceph-common'])558 ensure_packages(['ceph-common'])
@@ -661,7 +792,14 @@
661 'endpoints': [],792 'endpoints': [],
662 'ext_ports': []}793 'ext_ports': []}
663794
664 for cn in self.canonical_names():795 cns = self.canonical_names()
796 if cns:
797 for cn in cns:
798 self.configure_cert(cn)
799 else:
800 # Expect cert/key provided in config (currently assumed that ca
801 # uses ip for cn)
802 cn = resolve_address(endpoint_type=INTERNAL)
665 self.configure_cert(cn)803 self.configure_cert(cn)
666804
667 addresses = self.get_network_addresses()805 addresses = self.get_network_addresses()
@@ -724,6 +862,19 @@
724862
725 return ovs_ctxt863 return ovs_ctxt
726864
865 def nuage_ctxt(self):
866 driver = neutron_plugin_attribute(self.plugin, 'driver',
867 self.network_manager)
868 config = neutron_plugin_attribute(self.plugin, 'config',
869 self.network_manager)
870 nuage_ctxt = {'core_plugin': driver,
871 'neutron_plugin': 'vsp',
872 'neutron_security_groups': self.neutron_security_groups,
873 'local_ip': unit_private_ip(),
874 'config': config}
875
876 return nuage_ctxt
877
727 def nvp_ctxt(self):878 def nvp_ctxt(self):
728 driver = neutron_plugin_attribute(self.plugin, 'driver',879 driver = neutron_plugin_attribute(self.plugin, 'driver',
729 self.network_manager)880 self.network_manager)
@@ -788,9 +939,19 @@
788 'neutron_url': '%s://%s:%s' % (proto, host, '9696')}939 'neutron_url': '%s://%s:%s' % (proto, host, '9696')}
789 return ctxt940 return ctxt
790941
942 def pg_ctxt(self):
943 driver = neutron_plugin_attribute(self.plugin, 'driver',
944 self.network_manager)
945 config = neutron_plugin_attribute(self.plugin, 'config',
946 self.network_manager)
947 ovs_ctxt = {'core_plugin': driver,
948 'neutron_plugin': 'plumgrid',
949 'neutron_security_groups': self.neutron_security_groups,
950 'local_ip': unit_private_ip(),
951 'config': config}
952 return ovs_ctxt
953
791 def __call__(self):954 def __call__(self):
792 self._ensure_packages()
793
794 if self.network_manager not in ['quantum', 'neutron']:955 if self.network_manager not in ['quantum', 'neutron']:
795 return {}956 return {}
796957
@@ -807,6 +968,10 @@
807 ctxt.update(self.n1kv_ctxt())968 ctxt.update(self.n1kv_ctxt())
808 elif self.plugin == 'Calico':969 elif self.plugin == 'Calico':
809 ctxt.update(self.calico_ctxt())970 ctxt.update(self.calico_ctxt())
971 elif self.plugin == 'vsp':
972 ctxt.update(self.nuage_ctxt())
973 elif self.plugin == 'plumgrid':
974 ctxt.update(self.pg_ctxt())
810975
811 alchemy_flags = config('neutron-alchemy-flags')976 alchemy_flags = config('neutron-alchemy-flags')
812 if alchemy_flags:977 if alchemy_flags:
@@ -817,6 +982,59 @@
817 return ctxt982 return ctxt
818983
819984
985class NeutronPortContext(OSContextGenerator):
986
987 def resolve_ports(self, ports):
988 """Resolve NICs not yet bound to bridge(s)
989
990 If hwaddress provided then returns resolved hwaddress otherwise NIC.
991 """
992 if not ports:
993 return None
994
995 hwaddr_to_nic = {}
996 hwaddr_to_ip = {}
997 for nic in list_nics():
998 # Ignore virtual interfaces (bond masters will be identified from
999 # their slaves)
1000 if not is_phy_iface(nic):
1001 continue
1002
1003 _nic = get_bond_master(nic)
1004 if _nic:
1005 log("Replacing iface '%s' with bond master '%s'" % (nic, _nic),
1006 level=DEBUG)
1007 nic = _nic
1008
1009 hwaddr = get_nic_hwaddr(nic)
1010 hwaddr_to_nic[hwaddr] = nic
1011 addresses = get_ipv4_addr(nic, fatal=False)
1012 addresses += get_ipv6_addr(iface=nic, fatal=False)
1013 hwaddr_to_ip[hwaddr] = addresses
1014
1015 resolved = []
1016 mac_regex = re.compile(r'([0-9A-F]{2}[:-]){5}([0-9A-F]{2})', re.I)
1017 for entry in ports:
1018 if re.match(mac_regex, entry):
1019 # NIC is in known NICs and does NOT hace an IP address
1020 if entry in hwaddr_to_nic and not hwaddr_to_ip[entry]:
1021 # If the nic is part of a bridge then don't use it
1022 if is_bridge_member(hwaddr_to_nic[entry]):
1023 continue
1024
1025 # Entry is a MAC address for a valid interface that doesn't
1026 # have an IP address assigned yet.
1027 resolved.append(hwaddr_to_nic[entry])
1028 else:
1029 # If the passed entry is not a MAC address, assume it's a valid
1030 # interface, and that the user put it there on purpose (we can
1031 # trust it to be the real external network).
1032 resolved.append(entry)
1033
1034 # Ensure no duplicates
1035 return list(set(resolved))
1036
1037
820class OSConfigFlagContext(OSContextGenerator):1038class OSConfigFlagContext(OSContextGenerator):
821 """Provides support for user-defined config flags.1039 """Provides support for user-defined config flags.
8221040
@@ -904,13 +1122,22 @@
904 :param config_file : Service's config file to query sections1122 :param config_file : Service's config file to query sections
905 :param interface : Subordinate interface to inspect1123 :param interface : Subordinate interface to inspect
906 """1124 """
907 self.service = service
908 self.config_file = config_file1125 self.config_file = config_file
909 self.interface = interface1126 if isinstance(service, list):
1127 self.services = service
1128 else:
1129 self.services = [service]
1130 if isinstance(interface, list):
1131 self.interfaces = interface
1132 else:
1133 self.interfaces = [interface]
9101134
911 def __call__(self):1135 def __call__(self):
912 ctxt = {'sections': {}}1136 ctxt = {'sections': {}}
913 for rid in relation_ids(self.interface):1137 rids = []
1138 for interface in self.interfaces:
1139 rids.extend(relation_ids(interface))
1140 for rid in rids:
914 for unit in related_units(rid):1141 for unit in related_units(rid):
915 sub_config = relation_get('subordinate_configuration',1142 sub_config = relation_get('subordinate_configuration',
916 rid=rid, unit=unit)1143 rid=rid, unit=unit)
@@ -922,29 +1149,32 @@
922 'setting from %s' % rid, level=ERROR)1149 'setting from %s' % rid, level=ERROR)
923 continue1150 continue
9241151
925 if self.service not in sub_config:1152 for service in self.services:
926 log('Found subordinate_config on %s but it contained'1153 if service not in sub_config:
927 'nothing for %s service' % (rid, self.service),1154 log('Found subordinate_config on %s but it contained'
928 level=INFO)1155 'nothing for %s service' % (rid, service),
929 continue1156 level=INFO)
9301157 continue
931 sub_config = sub_config[self.service]1158
932 if self.config_file not in sub_config:1159 sub_config = sub_config[service]
933 log('Found subordinate_config on %s but it contained'1160 if self.config_file not in sub_config:
934 'nothing for %s' % (rid, self.config_file),1161 log('Found subordinate_config on %s but it contained'
935 level=INFO)1162 'nothing for %s' % (rid, self.config_file),
936 continue1163 level=INFO)
9371164 continue
938 sub_config = sub_config[self.config_file]1165
939 for k, v in six.iteritems(sub_config):1166 sub_config = sub_config[self.config_file]
940 if k == 'sections':1167 for k, v in six.iteritems(sub_config):
941 for section, config_dict in six.iteritems(v):1168 if k == 'sections':
942 log("adding section '%s'" % (section),1169 for section, config_list in six.iteritems(v):
943 level=DEBUG)1170 log("adding section '%s'" % (section),
944 ctxt[k][section] = config_dict1171 level=DEBUG)
945 else:1172 if ctxt[k].get(section):
946 ctxt[k] = v1173 ctxt[k][section].extend(config_list)
9471174 else:
1175 ctxt[k][section] = config_list
1176 else:
1177 ctxt[k] = v
948 log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)1178 log("%d section(s) found" % (len(ctxt['sections'])), level=DEBUG)
949 return ctxt1179 return ctxt
9501180
@@ -1005,6 +1235,8 @@
1005 for unit in related_units(rid):1235 for unit in related_units(rid):
1006 ctxt['zmq_nonce'] = relation_get('nonce', unit, rid)1236 ctxt['zmq_nonce'] = relation_get('nonce', unit, rid)
1007 ctxt['zmq_host'] = relation_get('host', unit, rid)1237 ctxt['zmq_host'] = relation_get('host', unit, rid)
1238 ctxt['zmq_redis_address'] = relation_get(
1239 'zmq_redis_address', unit, rid)
10081240
1009 return ctxt1241 return ctxt
10101242
@@ -1036,3 +1268,149 @@
1036 sysctl_create(sysctl_dict,1268 sysctl_create(sysctl_dict,
1037 '/etc/sysctl.d/50-{0}.conf'.format(charm_name()))1269 '/etc/sysctl.d/50-{0}.conf'.format(charm_name()))
1038 return {'sysctl': sysctl_dict}1270 return {'sysctl': sysctl_dict}
1271
1272
1273class NeutronAPIContext(OSContextGenerator):
1274 '''
1275 Inspects current neutron-plugin-api relation for neutron settings. Return
1276 defaults if it is not present.
1277 '''
1278 interfaces = ['neutron-plugin-api']
1279
1280 def __call__(self):
1281 self.neutron_defaults = {
1282 'l2_population': {
1283 'rel_key': 'l2-population',
1284 'default': False,
1285 },
1286 'overlay_network_type': {
1287 'rel_key': 'overlay-network-type',
1288 'default': 'gre',
1289 },
1290 'neutron_security_groups': {
1291 'rel_key': 'neutron-security-groups',
1292 'default': False,
1293 },
1294 'network_device_mtu': {
1295 'rel_key': 'network-device-mtu',
1296 'default': None,
1297 },
1298 'enable_dvr': {
1299 'rel_key': 'enable-dvr',
1300 'default': False,
1301 },
1302 'enable_l3ha': {
1303 'rel_key': 'enable-l3ha',
1304 'default': False,
1305 },
1306 }
1307 ctxt = self.get_neutron_options({})
1308 for rid in relation_ids('neutron-plugin-api'):
1309 for unit in related_units(rid):
1310 rdata = relation_get(rid=rid, unit=unit)
1311 if 'l2-population' in rdata:
1312 ctxt.update(self.get_neutron_options(rdata))
1313
1314 return ctxt
1315
1316 def get_neutron_options(self, rdata):
1317 settings = {}
1318 for nkey in self.neutron_defaults.keys():
1319 defv = self.neutron_defaults[nkey]['default']
1320 rkey = self.neutron_defaults[nkey]['rel_key']
1321 if rkey in rdata.keys():
1322 if type(defv) is bool:
1323 settings[nkey] = bool_from_string(rdata[rkey])
1324 else:
1325 settings[nkey] = rdata[rkey]
1326 else:
1327 settings[nkey] = defv
1328 return settings
1329
1330
1331class ExternalPortContext(NeutronPortContext):
1332
1333 def __call__(self):
1334 ctxt = {}
1335 ports = config('ext-port')
1336 if ports:
1337 ports = [p.strip() for p in ports.split()]
1338 ports = self.resolve_ports(ports)
1339 if ports:
1340 ctxt = {"ext_port": ports[0]}
1341 napi_settings = NeutronAPIContext()()
1342 mtu = napi_settings.get('network_device_mtu')
1343 if mtu:
1344 ctxt['ext_port_mtu'] = mtu
1345
1346 return ctxt
1347
1348
1349class DataPortContext(NeutronPortContext):
1350
1351 def __call__(self):
1352 ports = config('data-port')
1353 if ports:
1354 # Map of {port/mac:bridge}
1355 portmap = parse_data_port_mappings(ports)
1356 ports = portmap.keys()
1357 # Resolve provided ports or mac addresses and filter out those
1358 # already attached to a bridge.
1359 resolved = self.resolve_ports(ports)
1360 # FIXME: is this necessary?
1361 normalized = {get_nic_hwaddr(port): port for port in resolved
1362 if port not in ports}
1363 normalized.update({port: port for port in resolved
1364 if port in ports})
1365 if resolved:
1366 return {bridge: normalized[port] for port, bridge in
1367 six.iteritems(portmap) if port in normalized.keys()}
1368
1369 return None
1370
1371
1372class PhyNICMTUContext(DataPortContext):
1373
1374 def __call__(self):
1375 ctxt = {}
1376 mappings = super(PhyNICMTUContext, self).__call__()
1377 if mappings and mappings.values():
1378 ports = mappings.values()
1379 napi_settings = NeutronAPIContext()()
1380 mtu = napi_settings.get('network_device_mtu')
1381 if mtu:
1382 ctxt["devs"] = '\\n'.join(ports)
1383 ctxt['mtu'] = mtu
1384
1385 return ctxt
1386
1387
1388class NetworkServiceContext(OSContextGenerator):
1389
1390 def __init__(self, rel_name='quantum-network-service'):
1391 self.rel_name = rel_name
1392 self.interfaces = [rel_name]
1393
1394 def __call__(self):
1395 for rid in relation_ids(self.rel_name):
1396 for unit in related_units(rid):
1397 rdata = relation_get(rid=rid, unit=unit)
1398 ctxt = {
1399 'keystone_host': rdata.get('keystone_host'),
1400 'service_port': rdata.get('service_port'),
1401 'auth_port': rdata.get('auth_port'),
1402 'service_tenant': rdata.get('service_tenant'),
1403 'service_username': rdata.get('service_username'),
1404 'service_password': rdata.get('service_password'),
1405 'quantum_host': rdata.get('quantum_host'),
1406 'quantum_port': rdata.get('quantum_port'),
1407 'quantum_url': rdata.get('quantum_url'),
1408 'region': rdata.get('region'),
1409 'service_protocol':
1410 rdata.get('service_protocol') or 'http',
1411 'auth_protocol':
1412 rdata.get('auth_protocol') or 'http',
1413 }
1414 if self.context_complete(ctxt):
1415 return ctxt
1416 return {}
10391417
=== added directory 'hooks/charmhelpers/contrib/openstack/files'
=== added file 'hooks/charmhelpers/contrib/openstack/files/__init__.py'
--- hooks/charmhelpers/contrib/openstack/files/__init__.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/files/__init__.py 2015-09-21 10:47:21 +0000
@@ -0,0 +1,18 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
17# dummy __init__.py to fool syncer into thinking this is a syncable python
18# module
019
=== added file 'hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh'
--- hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/files/check_haproxy.sh 2015-09-21 10:47:21 +0000
@@ -0,0 +1,32 @@
1#!/bin/bash
2#--------------------------------------------
3# This file is managed by Juju
4#--------------------------------------------
5#
6# Copyright 2009,2012 Canonical Ltd.
7# Author: Tom Haddon
8
9CRITICAL=0
10NOTACTIVE=''
11LOGFILE=/var/log/nagios/check_haproxy.log
12AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
13
14for appserver in $(grep ' server' /etc/haproxy/haproxy.cfg | awk '{print $2'});
15do
16 output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 --regex="class=\"(active|backup)(2|3).*${appserver}" -e ' 200 OK')
17 if [ $? != 0 ]; then
18 date >> $LOGFILE
19 echo $output >> $LOGFILE
20 /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -v | grep $appserver >> $LOGFILE 2>&1
21 CRITICAL=1
22 NOTACTIVE="${NOTACTIVE} $appserver"
23 fi
24done
25
26if [ $CRITICAL = 1 ]; then
27 echo "CRITICAL:${NOTACTIVE}"
28 exit 2
29fi
30
31echo "OK: All haproxy instances looking good"
32exit 0
033
=== added file 'hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh'
--- hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/files/check_haproxy_queue_depth.sh 2015-09-21 10:47:21 +0000
@@ -0,0 +1,30 @@
1#!/bin/bash
2#--------------------------------------------
3# This file is managed by Juju
4#--------------------------------------------
5#
6# Copyright 2009,2012 Canonical Ltd.
7# Author: Tom Haddon
8
9# These should be config options at some stage
10CURRQthrsh=0
11MAXQthrsh=100
12
13AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
14
15HAPROXYSTATS=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v)
16
17for BACKEND in $(echo $HAPROXYSTATS| xargs -n1 | grep BACKEND | awk -F , '{print $1}')
18do
19 CURRQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 3)
20 MAXQ=$(echo "$HAPROXYSTATS" | grep $BACKEND | grep BACKEND | cut -d , -f 4)
21
22 if [[ $CURRQ -gt $CURRQthrsh || $MAXQ -gt $MAXQthrsh ]] ; then
23 echo "CRITICAL: queue depth for $BACKEND - CURRENT:$CURRQ MAX:$MAXQ"
24 exit 2
25 fi
26done
27
28echo "OK: All haproxy queue depths looking good"
29exit 0
30
031
=== modified file 'hooks/charmhelpers/contrib/openstack/ip.py'
--- hooks/charmhelpers/contrib/openstack/ip.py 2014-12-10 20:28:57 +0000
+++ hooks/charmhelpers/contrib/openstack/ip.py 2015-09-21 10:47:21 +0000
@@ -1,6 +1,23 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1from charmhelpers.core.hookenv import (17from charmhelpers.core.hookenv import (
2 config,18 config,
3 unit_get,19 unit_get,
20 service_name,
4)21)
5from charmhelpers.contrib.network.ip import (22from charmhelpers.contrib.network.ip import (
6 get_address_in_network,23 get_address_in_network,
@@ -17,15 +34,18 @@
17ADDRESS_MAP = {34ADDRESS_MAP = {
18 PUBLIC: {35 PUBLIC: {
19 'config': 'os-public-network',36 'config': 'os-public-network',
20 'fallback': 'public-address'37 'fallback': 'public-address',
38 'override': 'os-public-hostname',
21 },39 },
22 INTERNAL: {40 INTERNAL: {
23 'config': 'os-internal-network',41 'config': 'os-internal-network',
24 'fallback': 'private-address'42 'fallback': 'private-address',
43 'override': 'os-internal-hostname',
25 },44 },
26 ADMIN: {45 ADMIN: {
27 'config': 'os-admin-network',46 'config': 'os-admin-network',
28 'fallback': 'private-address'47 'fallback': 'private-address',
48 'override': 'os-admin-hostname',
29 }49 }
30}50}
3151
@@ -39,15 +59,50 @@
39 :param endpoint_type: str endpoint type to resolve.59 :param endpoint_type: str endpoint type to resolve.
40 :param returns: str base URL for services on the current service unit.60 :param returns: str base URL for services on the current service unit.
41 """61 """
42 scheme = 'http'62 scheme = _get_scheme(configs)
43 if 'https' in configs.complete_contexts():63
44 scheme = 'https'
45 address = resolve_address(endpoint_type)64 address = resolve_address(endpoint_type)
46 if is_ipv6(address):65 if is_ipv6(address):
47 address = "[{}]".format(address)66 address = "[{}]".format(address)
67
48 return '%s://%s' % (scheme, address)68 return '%s://%s' % (scheme, address)
4969
5070
71def _get_scheme(configs):
72 """Returns the scheme to use for the url (either http or https)
73 depending upon whether https is in the configs value.
74
75 :param configs: OSTemplateRenderer config templating object to inspect
76 for a complete https context.
77 :returns: either 'http' or 'https' depending on whether https is
78 configured within the configs context.
79 """
80 scheme = 'http'
81 if configs and 'https' in configs.complete_contexts():
82 scheme = 'https'
83 return scheme
84
85
86def _get_address_override(endpoint_type=PUBLIC):
87 """Returns any address overrides that the user has defined based on the
88 endpoint type.
89
90 Note: this function allows for the service name to be inserted into the
91 address if the user specifies {service_name}.somehost.org.
92
93 :param endpoint_type: the type of endpoint to retrieve the override
94 value for.
95 :returns: any endpoint address or hostname that the user has overridden
96 or None if an override is not present.
97 """
98 override_key = ADDRESS_MAP[endpoint_type]['override']
99 addr_override = config(override_key)
100 if not addr_override:
101 return None
102 else:
103 return addr_override.format(service_name=service_name())
104
105
51def resolve_address(endpoint_type=PUBLIC):106def resolve_address(endpoint_type=PUBLIC):
52 """Return unit address depending on net config.107 """Return unit address depending on net config.
53108
@@ -59,7 +114,10 @@
59114
60 :param endpoint_type: Network endpoing type115 :param endpoint_type: Network endpoing type
61 """116 """
62 resolved_address = None117 resolved_address = _get_address_override(endpoint_type)
118 if resolved_address:
119 return resolved_address
120
63 vips = config('vip')121 vips = config('vip')
64 if vips:122 if vips:
65 vips = vips.split()123 vips = vips.split()
66124
=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
--- hooks/charmhelpers/contrib/openstack/neutron.py 2015-01-14 15:30:19 +0000
+++ hooks/charmhelpers/contrib/openstack/neutron.py 2015-09-21 10:47:21 +0000
@@ -1,5 +1,22 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1# Various utilies for dealing with Neutron and the renaming from Quantum.17# Various utilies for dealing with Neutron and the renaming from Quantum.
218
19import six
3from subprocess import check_output20from subprocess import check_output
421
5from charmhelpers.core.hookenv import (22from charmhelpers.core.hookenv import (
@@ -155,13 +172,42 @@
155 'services': ['calico-felix',172 'services': ['calico-felix',
156 'bird',173 'bird',
157 'neutron-dhcp-agent',174 'neutron-dhcp-agent',
158 'nova-api-metadata'],175 'nova-api-metadata',
176 'etcd'],
159 'packages': [[headers_package()] + determine_dkms_package(),177 'packages': [[headers_package()] + determine_dkms_package(),
160 ['calico-compute',178 ['calico-compute',
161 'bird',179 'bird',
162 'neutron-dhcp-agent',180 'neutron-dhcp-agent',
163 'nova-api-metadata']],181 'nova-api-metadata',
164 'server_packages': ['neutron-server', 'calico-control'],182 'etcd']],
183 'server_packages': ['neutron-server', 'calico-control', 'etcd'],
184 'server_services': ['neutron-server', 'etcd']
185 },
186 'vsp': {
187 'config': '/etc/neutron/plugins/nuage/nuage_plugin.ini',
188 'driver': 'neutron.plugins.nuage.plugin.NuagePlugin',
189 'contexts': [
190 context.SharedDBContext(user=config('neutron-database-user'),
191 database=config('neutron-database'),
192 relation_prefix='neutron',
193 ssl_dir=NEUTRON_CONF_DIR)],
194 'services': [],
195 'packages': [],
196 'server_packages': ['neutron-server', 'neutron-plugin-nuage'],
197 'server_services': ['neutron-server']
198 },
199 'plumgrid': {
200 'config': '/etc/neutron/plugins/plumgrid/plumgrid.ini',
201 'driver': 'neutron.plugins.plumgrid.plumgrid_plugin.plumgrid_plugin.NeutronPluginPLUMgridV2',
202 'contexts': [
203 context.SharedDBContext(user=config('database-user'),
204 database=config('database'),
205 ssl_dir=NEUTRON_CONF_DIR)],
206 'services': [],
207 'packages': [['plumgrid-lxc'],
208 ['iovisor-dkms']],
209 'server_packages': ['neutron-server',
210 'neutron-plugin-plumgrid'],
165 'server_services': ['neutron-server']211 'server_services': ['neutron-server']
166 }212 }
167 }213 }
@@ -221,3 +267,90 @@
221 else:267 else:
222 # ensure accurate naming for all releases post-H268 # ensure accurate naming for all releases post-H
223 return 'neutron'269 return 'neutron'
270
271
272def parse_mappings(mappings, key_rvalue=False):
273 """By default mappings are lvalue keyed.
274
275 If key_rvalue is True, the mapping will be reversed to allow multiple
276 configs for the same lvalue.
277 """
278 parsed = {}
279 if mappings:
280 mappings = mappings.split()
281 for m in mappings:
282 p = m.partition(':')
283
284 if key_rvalue:
285 key_index = 2
286 val_index = 0
287 # if there is no rvalue skip to next
288 if not p[1]:
289 continue
290 else:
291 key_index = 0
292 val_index = 2
293
294 key = p[key_index].strip()
295 parsed[key] = p[val_index].strip()
296
297 return parsed
298
299
300def parse_bridge_mappings(mappings):
301 """Parse bridge mappings.
302
303 Mappings must be a space-delimited list of provider:bridge mappings.
304
305 Returns dict of the form {provider:bridge}.
306 """
307 return parse_mappings(mappings)
308
309
310def parse_data_port_mappings(mappings, default_bridge='br-data'):
311 """Parse data port mappings.
312
313 Mappings must be a space-delimited list of port:bridge mappings.
314
315 Returns dict of the form {port:bridge} where port may be an mac address or
316 interface name.
317 """
318
319 # NOTE(dosaboy): we use rvalue for key to allow multiple values to be
320 # proposed for <port> since it may be a mac address which will differ
321 # across units this allowing first-known-good to be chosen.
322 _mappings = parse_mappings(mappings, key_rvalue=True)
323 if not _mappings or list(_mappings.values()) == ['']:
324 if not mappings:
325 return {}
326
327 # For backwards-compatibility we need to support port-only provided in
328 # config.
329 _mappings = {mappings.split()[0]: default_bridge}
330
331 ports = _mappings.keys()
332 if len(set(ports)) != len(ports):
333 raise Exception("It is not allowed to have the same port configured "
334 "on more than one bridge")
335
336 return _mappings
337
338
339def parse_vlan_range_mappings(mappings):
340 """Parse vlan range mappings.
341
342 Mappings must be a space-delimited list of provider:start:end mappings.
343
344 The start:end range is optional and may be omitted.
345
346 Returns dict of the form {provider: (start, end)}.
347 """
348 _mappings = parse_mappings(mappings)
349 if not _mappings:
350 return {}
351
352 mappings = {}
353 for p, r in six.iteritems(_mappings):
354 mappings[p] = tuple(r.split(':'))
355
356 return mappings
224357
=== modified file 'hooks/charmhelpers/contrib/openstack/templates/__init__.py'
--- hooks/charmhelpers/contrib/openstack/templates/__init__.py 2013-07-19 02:37:30 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/__init__.py 2015-09-21 10:47:21 +0000
@@ -1,2 +1,18 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1# dummy __init__.py to fool syncer into thinking this is a syncable python17# dummy __init__.py to fool syncer into thinking this is a syncable python
2# module18# module
319
=== modified file 'hooks/charmhelpers/contrib/openstack/templates/ceph.conf'
--- hooks/charmhelpers/contrib/openstack/templates/ceph.conf 2014-04-04 16:45:38 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/ceph.conf 2015-09-21 10:47:21 +0000
@@ -5,11 +5,11 @@
5###############################################################################5###############################################################################
6[global]6[global]
7{% if auth -%}7{% if auth -%}
8 auth_supported = {{ auth }}8auth_supported = {{ auth }}
9 keyring = /etc/ceph/$cluster.$name.keyring9keyring = /etc/ceph/$cluster.$name.keyring
10 mon host = {{ mon_hosts }}10mon host = {{ mon_hosts }}
11{% endif -%}11{% endif -%}
12 log to syslog = {{ use_syslog }}12log to syslog = {{ use_syslog }}
13 err to syslog = {{ use_syslog }}13err to syslog = {{ use_syslog }}
14 clog to syslog = {{ use_syslog }}14clog to syslog = {{ use_syslog }}
1515
1616
=== added file 'hooks/charmhelpers/contrib/openstack/templates/git.upstart'
--- hooks/charmhelpers/contrib/openstack/templates/git.upstart 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/git.upstart 2015-09-21 10:47:21 +0000
@@ -0,0 +1,17 @@
1description "{{ service_description }}"
2author "Juju {{ service_name }} Charm <juju@localhost>"
3
4start on runlevel [2345]
5stop on runlevel [!2345]
6
7respawn
8
9exec start-stop-daemon --start --chuid {{ user_name }} \
10 --chdir {{ start_dir }} --name {{ process_name }} \
11 --exec {{ executable_name }} -- \
12 {% for config_file in config_files -%}
13 --config-file={{ config_file }} \
14 {% endfor -%}
15 {% if log_file -%}
16 --log-file={{ log_file }}
17 {% endif -%}
018
=== added file 'hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken'
--- hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken 2015-09-21 10:47:21 +0000
@@ -0,0 +1,9 @@
1{% if auth_host -%}
2[keystone_authtoken]
3identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }}
4auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }}
5admin_tenant_name = {{ admin_tenant_name }}
6admin_user = {{ admin_user }}
7admin_password = {{ admin_password }}
8signing_dir = {{ signing_dir }}
9{% endif -%}
010
=== added file 'hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo'
--- hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/section-rabbitmq-oslo 2015-09-21 10:47:21 +0000
@@ -0,0 +1,22 @@
1{% if rabbitmq_host or rabbitmq_hosts -%}
2[oslo_messaging_rabbit]
3rabbit_userid = {{ rabbitmq_user }}
4rabbit_virtual_host = {{ rabbitmq_virtual_host }}
5rabbit_password = {{ rabbitmq_password }}
6{% if rabbitmq_hosts -%}
7rabbit_hosts = {{ rabbitmq_hosts }}
8{% if rabbitmq_ha_queues -%}
9rabbit_ha_queues = True
10rabbit_durable_queues = False
11{% endif -%}
12{% else -%}
13rabbit_host = {{ rabbitmq_host }}
14{% endif -%}
15{% if rabbit_ssl_port -%}
16rabbit_use_ssl = True
17rabbit_port = {{ rabbit_ssl_port }}
18{% if rabbit_ssl_ca -%}
19kombu_ssl_ca_certs = {{ rabbit_ssl_ca }}
20{% endif -%}
21{% endif -%}
22{% endif -%}
023
=== added file 'hooks/charmhelpers/contrib/openstack/templates/section-zeromq'
--- hooks/charmhelpers/contrib/openstack/templates/section-zeromq 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/section-zeromq 2015-09-21 10:47:21 +0000
@@ -0,0 +1,14 @@
1{% if zmq_host -%}
2# ZeroMQ configuration (restart-nonce: {{ zmq_nonce }})
3rpc_backend = zmq
4rpc_zmq_host = {{ zmq_host }}
5{% if zmq_redis_address -%}
6rpc_zmq_matchmaker = redis
7matchmaker_heartbeat_freq = 15
8matchmaker_heartbeat_ttl = 30
9[matchmaker_redis]
10host = {{ zmq_redis_address }}
11{% else -%}
12rpc_zmq_matchmaker = ring
13{% endif -%}
14{% endif -%}
015
=== modified file 'hooks/charmhelpers/contrib/openstack/templating.py'
--- hooks/charmhelpers/contrib/openstack/templating.py 2014-12-10 20:28:57 +0000
+++ hooks/charmhelpers/contrib/openstack/templating.py 2015-09-21 10:47:21 +0000
@@ -1,3 +1,19 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1import os17import os
218
3import six19import six
@@ -13,8 +29,8 @@
13try:29try:
14 from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions30 from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
15except ImportError:31except ImportError:
16 # python-jinja2 may not be installed yet, or we're running unittests.32 apt_install('python-jinja2', fatal=True)
17 FileSystemLoader = ChoiceLoader = Environment = exceptions = None33 from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
1834
1935
20class OSConfigException(Exception):36class OSConfigException(Exception):
@@ -96,7 +112,7 @@
96112
97 def complete_contexts(self):113 def complete_contexts(self):
98 '''114 '''
99 Return a list of interfaces that have atisfied contexts.115 Return a list of interfaces that have satisfied contexts.
100 '''116 '''
101 if self._complete_contexts:117 if self._complete_contexts:
102 return self._complete_contexts118 return self._complete_contexts
@@ -277,3 +293,30 @@
277 [interfaces.extend(i.complete_contexts())293 [interfaces.extend(i.complete_contexts())
278 for i in six.itervalues(self.templates)]294 for i in six.itervalues(self.templates)]
279 return interfaces295 return interfaces
296
297 def get_incomplete_context_data(self, interfaces):
298 '''
299 Return dictionary of relation status of interfaces and any missing
300 required context data. Example:
301 {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True},
302 'zeromq-configuration': {'related': False}}
303 '''
304 incomplete_context_data = {}
305
306 for i in six.itervalues(self.templates):
307 for context in i.contexts:
308 for interface in interfaces:
309 related = False
310 if interface in context.interfaces:
311 related = context.get_related()
312 missing_data = context.missing_data
313 if missing_data:
314 incomplete_context_data[interface] = {'missing_data': missing_data}
315 if related:
316 if incomplete_context_data.get(interface):
317 incomplete_context_data[interface].update({'related': True})
318 else:
319 incomplete_context_data[interface] = {'related': True}
320 else:
321 incomplete_context_data[interface] = {'related': False}
322 return incomplete_context_data
280323
=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py 2015-01-14 15:30:19 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py 2015-09-21 10:47:21 +0000
@@ -1,4 +1,18 @@
1#!/usr/bin/python1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
216
3# Common python helper functions used for OpenStack charms.17# Common python helper functions used for OpenStack charms.
4from collections import OrderedDict18from collections import OrderedDict
@@ -7,19 +21,27 @@
7import subprocess21import subprocess
8import json22import json
9import os23import os
10import socket
11import sys24import sys
25import re
1226
13import six27import six
14import yaml28import yaml
1529
30from charmhelpers.contrib.network import ip
31
32from charmhelpers.core import (
33 unitdata,
34)
35
16from charmhelpers.core.hookenv import (36from charmhelpers.core.hookenv import (
17 config,37 config,
18 log as juju_log,38 log as juju_log,
19 charm_dir,39 charm_dir,
20 INFO,40 INFO,
21 relation_ids,41 relation_ids,
22 relation_set42 relation_set,
43 status_set,
44 hook_name
23)45)
2446
25from charmhelpers.contrib.storage.linux.lvm import (47from charmhelpers.contrib.storage.linux.lvm import (
@@ -32,9 +54,13 @@
32 get_ipv6_addr54 get_ipv6_addr
33)55)
3456
57from charmhelpers.contrib.python.packages import (
58 pip_create_virtualenv,
59 pip_install,
60)
61
35from charmhelpers.core.host import lsb_release, mounts, umount62from charmhelpers.core.host import lsb_release, mounts, umount
36from charmhelpers.fetch import apt_install, apt_cache, install_remote63from charmhelpers.fetch import apt_install, apt_cache, install_remote
37from charmhelpers.contrib.python.packages import pip_install
38from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk64from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
39from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device65from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
4066
@@ -44,7 +70,6 @@
44DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed '70DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed '
45 'restricted main multiverse universe')71 'restricted main multiverse universe')
4672
47
48UBUNTU_OPENSTACK_RELEASE = OrderedDict([73UBUNTU_OPENSTACK_RELEASE = OrderedDict([
49 ('oneiric', 'diablo'),74 ('oneiric', 'diablo'),
50 ('precise', 'essex'),75 ('precise', 'essex'),
@@ -54,6 +79,7 @@
54 ('trusty', 'icehouse'),79 ('trusty', 'icehouse'),
55 ('utopic', 'juno'),80 ('utopic', 'juno'),
56 ('vivid', 'kilo'),81 ('vivid', 'kilo'),
82 ('wily', 'liberty'),
57])83])
5884
5985
@@ -66,6 +92,7 @@
66 ('2014.1', 'icehouse'),92 ('2014.1', 'icehouse'),
67 ('2014.2', 'juno'),93 ('2014.2', 'juno'),
68 ('2015.1', 'kilo'),94 ('2015.1', 'kilo'),
95 ('2015.2', 'liberty'),
69])96])
7097
71# The ugly duckling98# The ugly duckling
@@ -87,8 +114,42 @@
87 ('2.1.0', 'juno'),114 ('2.1.0', 'juno'),
88 ('2.2.0', 'juno'),115 ('2.2.0', 'juno'),
89 ('2.2.1', 'kilo'),116 ('2.2.1', 'kilo'),
117 ('2.2.2', 'kilo'),
118 ('2.3.0', 'liberty'),
119 ('2.4.0', 'liberty'),
90])120])
91121
122# >= Liberty version->codename mapping
123PACKAGE_CODENAMES = {
124 'nova-common': OrderedDict([
125 ('12.0.0', 'liberty'),
126 ]),
127 'neutron-common': OrderedDict([
128 ('7.0.0', 'liberty'),
129 ]),
130 'cinder-common': OrderedDict([
131 ('7.0.0', 'liberty'),
132 ]),
133 'keystone': OrderedDict([
134 ('8.0.0', 'liberty'),
135 ]),
136 'horizon-common': OrderedDict([
137 ('8.0.0', 'liberty'),
138 ]),
139 'ceilometer-common': OrderedDict([
140 ('5.0.0', 'liberty'),
141 ]),
142 'heat-common': OrderedDict([
143 ('5.0.0', 'liberty'),
144 ]),
145 'glance-common': OrderedDict([
146 ('11.0.0', 'liberty'),
147 ]),
148 'openstack-dashboard': OrderedDict([
149 ('8.0.0', 'liberty'),
150 ]),
151}
152
92DEFAULT_LOOPBACK_SIZE = '5G'153DEFAULT_LOOPBACK_SIZE = '5G'
93154
94155
@@ -138,9 +199,9 @@
138 error_out(e)199 error_out(e)
139200
140201
141def get_os_version_codename(codename):202def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES):
142 '''Determine OpenStack version number from codename.'''203 '''Determine OpenStack version number from codename.'''
143 for k, v in six.iteritems(OPENSTACK_CODENAMES):204 for k, v in six.iteritems(version_map):
144 if v == codename:205 if v == codename:
145 return k206 return k
146 e = 'Could not derive OpenStack version for '\207 e = 'Could not derive OpenStack version for '\
@@ -172,20 +233,31 @@
172 error_out(e)233 error_out(e)
173234
174 vers = apt.upstream_version(pkg.current_ver.ver_str)235 vers = apt.upstream_version(pkg.current_ver.ver_str)
236 match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
237 if match:
238 vers = match.group(0)
175239
176 try:240 # >= Liberty independent project versions
177 if 'swift' in pkg.name:241 if (package in PACKAGE_CODENAMES and
178 swift_vers = vers[:5]242 vers in PACKAGE_CODENAMES[package]):
179 if swift_vers not in SWIFT_CODENAMES:243 return PACKAGE_CODENAMES[package][vers]
180 # Deal with 1.10.0 upward244 else:
181 swift_vers = vers[:6]245 # < Liberty co-ordinated project versions
182 return SWIFT_CODENAMES[swift_vers]246 try:
183 else:247 if 'swift' in pkg.name:
184 vers = vers[:6]248 swift_vers = vers[:5]
185 return OPENSTACK_CODENAMES[vers]249 if swift_vers not in SWIFT_CODENAMES:
186 except KeyError:250 # Deal with 1.10.0 upward
187 e = 'Could not determine OpenStack codename for version %s' % vers251 swift_vers = vers[:6]
188 error_out(e)252 return SWIFT_CODENAMES[swift_vers]
253 else:
254 vers = vers[:6]
255 return OPENSTACK_CODENAMES[vers]
256 except KeyError:
257 if not fatal:
258 return None
259 e = 'Could not determine OpenStack codename for version %s' % vers
260 error_out(e)
189261
190262
191def get_os_version_package(pkg, fatal=True):263def get_os_version_package(pkg, fatal=True):
@@ -295,6 +367,9 @@
295 'kilo': 'trusty-updates/kilo',367 'kilo': 'trusty-updates/kilo',
296 'kilo/updates': 'trusty-updates/kilo',368 'kilo/updates': 'trusty-updates/kilo',
297 'kilo/proposed': 'trusty-proposed/kilo',369 'kilo/proposed': 'trusty-proposed/kilo',
370 'liberty': 'trusty-updates/liberty',
371 'liberty/updates': 'trusty-updates/liberty',
372 'liberty/proposed': 'trusty-proposed/liberty',
298 }373 }
299374
300 try:375 try:
@@ -312,6 +387,21 @@
312 error_out("Invalid openstack-release specified: %s" % rel)387 error_out("Invalid openstack-release specified: %s" % rel)
313388
314389
390def config_value_changed(option):
391 """
392 Determine if config value changed since last call to this function.
393 """
394 hook_data = unitdata.HookData()
395 with hook_data():
396 db = unitdata.kv()
397 current = config(option)
398 saved = db.get(option)
399 db.set(option, current)
400 if saved is None:
401 return False
402 return current != saved
403
404
315def save_script_rc(script_path="scripts/scriptrc", **env_vars):405def save_script_rc(script_path="scripts/scriptrc", **env_vars):
316 """406 """
317 Write an rc file in the charm-delivered directory containing407 Write an rc file in the charm-delivered directory containing
@@ -345,7 +435,11 @@
345 import apt_pkg as apt435 import apt_pkg as apt
346 src = config('openstack-origin')436 src = config('openstack-origin')
347 cur_vers = get_os_version_package(package)437 cur_vers = get_os_version_package(package)
348 available_vers = get_os_version_install_source(src)438 if "swift" in package:
439 codename = get_os_codename_install_source(src)
440 available_vers = get_os_version_codename(codename, SWIFT_CODENAMES)
441 else:
442 available_vers = get_os_version_install_source(src)
349 apt.init()443 apt.init()
350 return apt.version_compare(available_vers, cur_vers) == 1444 return apt.version_compare(available_vers, cur_vers) == 1
351445
@@ -404,77 +498,10 @@
404 else:498 else:
405 zap_disk(block_device)499 zap_disk(block_device)
406500
407501is_ip = ip.is_ip
408def is_ip(address):502ns_query = ip.ns_query
409 """503get_host_ip = ip.get_host_ip
410 Returns True if address is a valid IP address.504get_hostname = ip.get_hostname
411 """
412 try:
413 # Test to see if already an IPv4 address
414 socket.inet_aton(address)
415 return True
416 except socket.error:
417 return False
418
419
420def ns_query(address):
421 try:
422 import dns.resolver
423 except ImportError:
424 apt_install('python-dnspython')
425 import dns.resolver
426
427 if isinstance(address, dns.name.Name):
428 rtype = 'PTR'
429 elif isinstance(address, six.string_types):
430 rtype = 'A'
431 else:
432 return None
433
434 answers = dns.resolver.query(address, rtype)
435 if answers:
436 return str(answers[0])
437 return None
438
439
440def get_host_ip(hostname):
441 """
442 Resolves the IP for a given hostname, or returns
443 the input if it is already an IP.
444 """
445 if is_ip(hostname):
446 return hostname
447
448 return ns_query(hostname)
449
450
451def get_hostname(address, fqdn=True):
452 """
453 Resolves hostname for given IP, or returns the input
454 if it is already a hostname.
455 """
456 if is_ip(address):
457 try:
458 import dns.reversename
459 except ImportError:
460 apt_install('python-dnspython')
461 import dns.reversename
462
463 rev = dns.reversename.from_address(address)
464 result = ns_query(rev)
465 if not result:
466 return None
467 else:
468 result = address
469
470 if fqdn:
471 # strip trailing .
472 if result.endswith('.'):
473 return result[:-1]
474 else:
475 return result
476 else:
477 return result.split('.')[0]
478505
479506
480def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'):507def get_matchmaker_map(mm_file='/etc/oslo/matchmaker_ring.json'):
@@ -518,108 +545,379 @@
518545
519546
520def git_install_requested():547def git_install_requested():
521 """Returns true if openstack-origin-git is specified."""548 """
522 return config('openstack-origin-git') != "None"549 Returns true if openstack-origin-git is specified.
550 """
551 return config('openstack-origin-git') is not None
523552
524553
525requirements_dir = None554requirements_dir = None
526555
527556
528def git_clone_and_install(file_name, core_project):557def _git_yaml_load(projects_yaml):
529 """Clone/install all OpenStack repos specified in yaml config file."""558 """
530 global requirements_dir559 Load the specified yaml into a dictionary.
531560 """
532 if file_name == "None":561 if not projects_yaml:
533 return562 return None
534563
535 yaml_file = os.path.join(charm_dir(), file_name)564 return yaml.load(projects_yaml)
536565
537 # clone/install the requirements project first566
538 installed = _git_clone_and_install_subset(yaml_file,567def git_clone_and_install(projects_yaml, core_project, depth=1):
539 whitelist=['requirements'])568 """
540 if 'requirements' not in installed:569 Clone/install all specified OpenStack repositories.
541 error_out('requirements git repository must be specified')570
542571 The expected format of projects_yaml is:
543 # clone/install all other projects except requirements and the core project572
544 blacklist = ['requirements', core_project]573 repositories:
545 _git_clone_and_install_subset(yaml_file, blacklist=blacklist,574 - {name: keystone,
546 update_requirements=True)575 repository: 'git://git.openstack.org/openstack/keystone.git',
547576 branch: 'stable/icehouse'}
548 # clone/install the core project577 - {name: requirements,
549 whitelist = [core_project]578 repository: 'git://git.openstack.org/openstack/requirements.git',
550 installed = _git_clone_and_install_subset(yaml_file, whitelist=whitelist,579 branch: 'stable/icehouse'}
551 update_requirements=True)580
552 if core_project not in installed:581 directory: /mnt/openstack-git
553 error_out('{} git repository must be specified'.format(core_project))582 http_proxy: squid-proxy-url
554583 https_proxy: squid-proxy-url
555584
556def _git_clone_and_install_subset(yaml_file, whitelist=[], blacklist=[],585 The directory, http_proxy, and https_proxy keys are optional.
557 update_requirements=False):586
558 """Clone/install subset of OpenStack repos specified in yaml config file."""587 """
559 global requirements_dir588 global requirements_dir
560 installed = []589 parent_dir = '/mnt/openstack-git'
561590 http_proxy = None
562 with open(yaml_file, 'r') as fd:591
563 projects = yaml.load(fd)592 projects = _git_yaml_load(projects_yaml)
564 for proj, val in projects.items():593 _git_validate_projects_yaml(projects, core_project)
565 # The project subset is chosen based on the following 3 rules:594
566 # 1) If project is in blacklist, we don't clone/install it, period.595 old_environ = dict(os.environ)
567 # 2) If whitelist is empty, we clone/install everything else.596
568 # 3) If whitelist is not empty, we clone/install everything in the597 if 'http_proxy' in projects.keys():
569 # whitelist.598 http_proxy = projects['http_proxy']
570 if proj in blacklist:599 os.environ['http_proxy'] = projects['http_proxy']
571 continue600 if 'https_proxy' in projects.keys():
572 if whitelist and proj not in whitelist:601 os.environ['https_proxy'] = projects['https_proxy']
573 continue602
574 repo = val['repository']603 if 'directory' in projects.keys():
575 branch = val['branch']604 parent_dir = projects['directory']
576 repo_dir = _git_clone_and_install_single(repo, branch,605
577 update_requirements)606 pip_create_virtualenv(os.path.join(parent_dir, 'venv'))
578 if proj == 'requirements':607
579 requirements_dir = repo_dir608 # Upgrade setuptools and pip from default virtualenv versions. The default
580 installed.append(proj)609 # versions in trusty break master OpenStack branch deployments.
581 return installed610 for p in ['pip', 'setuptools']:
582611 pip_install(p, upgrade=True, proxy=http_proxy,
583612 venv=os.path.join(parent_dir, 'venv'))
584def _git_clone_and_install_single(repo, branch, update_requirements=False):613
585 """Clone and install a single git repository."""614 for p in projects['repositories']:
586 dest_parent_dir = "/mnt/openstack-git/"615 repo = p['repository']
587 dest_dir = os.path.join(dest_parent_dir, os.path.basename(repo))616 branch = p['branch']
588617 if p['name'] == 'requirements':
589 if not os.path.exists(dest_parent_dir):618 repo_dir = _git_clone_and_install_single(repo, branch, depth,
590 juju_log('Host dir not mounted at {}. '619 parent_dir, http_proxy,
591 'Creating directory there instead.'.format(dest_parent_dir))620 update_requirements=False)
592 os.mkdir(dest_parent_dir)621 requirements_dir = repo_dir
622 else:
623 repo_dir = _git_clone_and_install_single(repo, branch, depth,
624 parent_dir, http_proxy,
625 update_requirements=True)
626
627 os.environ = old_environ
628
629
630def _git_validate_projects_yaml(projects, core_project):
631 """
632 Validate the projects yaml.
633 """
634 _git_ensure_key_exists('repositories', projects)
635
636 for project in projects['repositories']:
637 _git_ensure_key_exists('name', project.keys())
638 _git_ensure_key_exists('repository', project.keys())
639 _git_ensure_key_exists('branch', project.keys())
640
641 if projects['repositories'][0]['name'] != 'requirements':
642 error_out('{} git repo must be specified first'.format('requirements'))
643
644 if projects['repositories'][-1]['name'] != core_project:
645 error_out('{} git repo must be specified last'.format(core_project))
646
647
648def _git_ensure_key_exists(key, keys):
649 """
650 Ensure that key exists in keys.
651 """
652 if key not in keys:
653 error_out('openstack-origin-git key \'{}\' is missing'.format(key))
654
655
656def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
657 update_requirements):
658 """
659 Clone and install a single git repository.
660 """
661 dest_dir = os.path.join(parent_dir, os.path.basename(repo))
662
663 if not os.path.exists(parent_dir):
664 juju_log('Directory already exists at {}. '
665 'No need to create directory.'.format(parent_dir))
666 os.mkdir(parent_dir)
593667
594 if not os.path.exists(dest_dir):668 if not os.path.exists(dest_dir):
595 juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))669 juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
596 repo_dir = install_remote(repo, dest=dest_parent_dir, branch=branch)670 repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
671 depth=depth)
597 else:672 else:
598 repo_dir = dest_dir673 repo_dir = dest_dir
599674
675 venv = os.path.join(parent_dir, 'venv')
676
600 if update_requirements:677 if update_requirements:
601 if not requirements_dir:678 if not requirements_dir:
602 error_out('requirements repo must be cloned before '679 error_out('requirements repo must be cloned before '
603 'updating from global requirements.')680 'updating from global requirements.')
604 _git_update_requirements(repo_dir, requirements_dir)681 _git_update_requirements(venv, repo_dir, requirements_dir)
605682
606 juju_log('Installing git repo from dir: {}'.format(repo_dir))683 juju_log('Installing git repo from dir: {}'.format(repo_dir))
607 pip_install(repo_dir)684 if http_proxy:
685 pip_install(repo_dir, proxy=http_proxy, venv=venv)
686 else:
687 pip_install(repo_dir, venv=venv)
608688
609 return repo_dir689 return repo_dir
610690
611691
612def _git_update_requirements(package_dir, reqs_dir):692def _git_update_requirements(venv, package_dir, reqs_dir):
613 """Update from global requirements.693 """
694 Update from global requirements.
614695
615 Update an OpenStack git directory's requirements.txt and696 Update an OpenStack git directory's requirements.txt and
616 test-requirements.txt from global-requirements.txt."""697 test-requirements.txt from global-requirements.txt.
698 """
617 orig_dir = os.getcwd()699 orig_dir = os.getcwd()
618 os.chdir(reqs_dir)700 os.chdir(reqs_dir)
619 cmd = "python update.py {}".format(package_dir)701 python = os.path.join(venv, 'bin/python')
702 cmd = [python, 'update.py', package_dir]
620 try:703 try:
621 subprocess.check_call(cmd.split(' '))704 subprocess.check_call(cmd)
622 except subprocess.CalledProcessError:705 except subprocess.CalledProcessError:
623 package = os.path.basename(package_dir)706 package = os.path.basename(package_dir)
624 error_out("Error updating {} from global-requirements.txt".format(package))707 error_out("Error updating {} from "
708 "global-requirements.txt".format(package))
625 os.chdir(orig_dir)709 os.chdir(orig_dir)
710
711
712def git_pip_venv_dir(projects_yaml):
713 """
714 Return the pip virtualenv path.
715 """
716 parent_dir = '/mnt/openstack-git'
717
718 projects = _git_yaml_load(projects_yaml)
719
720 if 'directory' in projects.keys():
721 parent_dir = projects['directory']
722
723 return os.path.join(parent_dir, 'venv')
724
725
726def git_src_dir(projects_yaml, project):
727 """
728 Return the directory where the specified project's source is located.
729 """
730 parent_dir = '/mnt/openstack-git'
731
732 projects = _git_yaml_load(projects_yaml)
733
734 if 'directory' in projects.keys():
735 parent_dir = projects['directory']
736
737 for p in projects['repositories']:
738 if p['name'] == project:
739 return os.path.join(parent_dir, os.path.basename(p['repository']))
740
741 return None
742
743
744def git_yaml_value(projects_yaml, key):
745 """
746 Return the value in projects_yaml for the specified key.
747 """
748 projects = _git_yaml_load(projects_yaml)
749
750 if key in projects.keys():
751 return projects[key]
752
753 return None
754
755
756def os_workload_status(configs, required_interfaces, charm_func=None):
757 """
758 Decorator to set workload status based on complete contexts
759 """
760 def wrap(f):
761 @wraps(f)
762 def wrapped_f(*args, **kwargs):
763 # Run the original function first
764 f(*args, **kwargs)
765 # Set workload status now that contexts have been
766 # acted on
767 set_os_workload_status(configs, required_interfaces, charm_func)
768 return wrapped_f
769 return wrap
770
771
772def set_os_workload_status(configs, required_interfaces, charm_func=None):
773 """
774 Set workload status based on complete contexts.
775 status-set missing or incomplete contexts
776 and juju-log details of missing required data.
777 charm_func is a charm specific function to run checking
778 for charm specific requirements such as a VIP setting.
779 """
780 incomplete_rel_data = incomplete_relation_data(configs, required_interfaces)
781 state = 'active'
782 missing_relations = []
783 incomplete_relations = []
784 message = None
785 charm_state = None
786 charm_message = None
787
788 for generic_interface in incomplete_rel_data.keys():
789 related_interface = None
790 missing_data = {}
791 # Related or not?
792 for interface in incomplete_rel_data[generic_interface]:
793 if incomplete_rel_data[generic_interface][interface].get('related'):
794 related_interface = interface
795 missing_data = incomplete_rel_data[generic_interface][interface].get('missing_data')
796 # No relation ID for the generic_interface
797 if not related_interface:
798 juju_log("{} relation is missing and must be related for "
799 "functionality. ".format(generic_interface), 'WARN')
800 state = 'blocked'
801 if generic_interface not in missing_relations:
802 missing_relations.append(generic_interface)
803 else:
804 # Relation ID exists but no related unit
805 if not missing_data:
806 # Edge case relation ID exists but departing
807 if ('departed' in hook_name() or 'broken' in hook_name()) \
808 and related_interface in hook_name():
809 state = 'blocked'
810 if generic_interface not in missing_relations:
811 missing_relations.append(generic_interface)
812 juju_log("{} relation's interface, {}, "
813 "relationship is departed or broken "
814 "and is required for functionality."
815 "".format(generic_interface, related_interface), "WARN")
816 # Normal case relation ID exists but no related unit
817 # (joining)
818 else:
819 juju_log("{} relations's interface, {}, is related but has "
820 "no units in the relation."
821 "".format(generic_interface, related_interface), "INFO")
822 # Related unit exists and data missing on the relation
823 else:
824 juju_log("{} relation's interface, {}, is related awaiting "
825 "the following data from the relationship: {}. "
826 "".format(generic_interface, related_interface,
827 ", ".join(missing_data)), "INFO")
828 if state != 'blocked':
829 state = 'waiting'
830 if generic_interface not in incomplete_relations \
831 and generic_interface not in missing_relations:
832 incomplete_relations.append(generic_interface)
833
834 if missing_relations:
835 message = "Missing relations: {}".format(", ".join(missing_relations))
836 if incomplete_relations:
837 message += "; incomplete relations: {}" \
838 "".format(", ".join(incomplete_relations))
839 state = 'blocked'
840 elif incomplete_relations:
841 message = "Incomplete relations: {}" \
842 "".format(", ".join(incomplete_relations))
843 state = 'waiting'
844
845 # Run charm specific checks
846 if charm_func:
847 charm_state, charm_message = charm_func(configs)
848 if charm_state != 'active' and charm_state != 'unknown':
849 state = workload_state_compare(state, charm_state)
850 if message:
851 message = "{} {}".format(message, charm_message)
852 else:
853 message = charm_message
854
855 # Set to active if all requirements have been met
856 if state == 'active':
857 message = "Unit is ready"
858 juju_log(message, "INFO")
859
860 status_set(state, message)
861
862
863def workload_state_compare(current_workload_state, workload_state):
864 """ Return highest priority of two states"""
865 hierarchy = {'unknown': -1,
866 'active': 0,
867 'maintenance': 1,
868 'waiting': 2,
869 'blocked': 3,
870 }
871
872 if hierarchy.get(workload_state) is None:
873 workload_state = 'unknown'
874 if hierarchy.get(current_workload_state) is None:
875 current_workload_state = 'unknown'
876
877 # Set workload_state based on hierarchy of statuses
878 if hierarchy.get(current_workload_state) > hierarchy.get(workload_state):
879 return current_workload_state
880 else:
881 return workload_state
882
883
884def incomplete_relation_data(configs, required_interfaces):
885 """
886 Check complete contexts against required_interfaces
887 Return dictionary of incomplete relation data.
888
889 configs is an OSConfigRenderer object with configs registered
890
891 required_interfaces is a dictionary of required general interfaces
892 with dictionary values of possible specific interfaces.
893 Example:
894 required_interfaces = {'database': ['shared-db', 'pgsql-db']}
895
896 The interface is said to be satisfied if anyone of the interfaces in the
897 list has a complete context.
898
899 Return dictionary of incomplete or missing required contexts with relation
900 status of interfaces and any missing data points. Example:
901 {'message':
902 {'amqp': {'missing_data': ['rabbitmq_password'], 'related': True},
903 'zeromq-configuration': {'related': False}},
904 'identity':
905 {'identity-service': {'related': False}},
906 'database':
907 {'pgsql-db': {'related': False},
908 'shared-db': {'related': True}}}
909 """
910 complete_ctxts = configs.complete_contexts()
911 incomplete_relations = []
912 for svc_type in required_interfaces.keys():
913 # Avoid duplicates
914 found_ctxt = False
915 for interface in required_interfaces[svc_type]:
916 if interface in complete_ctxts:
917 found_ctxt = True
918 if not found_ctxt:
919 incomplete_relations.append(svc_type)
920 incomplete_context_data = {}
921 for i in incomplete_relations:
922 incomplete_context_data[i] = configs.get_incomplete_context_data(required_interfaces[i])
923 return incomplete_context_data
626924
=== modified file 'hooks/charmhelpers/contrib/python/__init__.py'
--- hooks/charmhelpers/contrib/python/__init__.py 2014-12-10 20:28:57 +0000
+++ hooks/charmhelpers/contrib/python/__init__.py 2015-09-21 10:47:21 +0000
@@ -0,0 +1,15 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
016
=== modified file 'hooks/charmhelpers/contrib/python/packages.py'
--- hooks/charmhelpers/contrib/python/packages.py 2014-12-10 20:28:57 +0000
+++ hooks/charmhelpers/contrib/python/packages.py 2015-09-21 10:47:21 +0000
@@ -1,10 +1,27 @@
1#!/usr/bin/env python1#!/usr/bin/env python
2# coding: utf-82# coding: utf-8
33
4__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"4# Copyright 2014-2015 Canonical Limited.
5#
6# This file is part of charm-helpers.
7#
8# charm-helpers is free software: you can redistribute it and/or modify
9# it under the terms of the GNU Lesser General Public License version 3 as
10# published by the Free Software Foundation.
11#
12# charm-helpers is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU Lesser General Public License for more details.
16#
17# You should have received a copy of the GNU Lesser General Public License
18# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
19
20import os
21import subprocess
522
6from charmhelpers.fetch import apt_install, apt_update23from charmhelpers.fetch import apt_install, apt_update
7from charmhelpers.core.hookenv import log24from charmhelpers.core.hookenv import charm_dir, log
825
9try:26try:
10 from pip import main as pip_execute27 from pip import main as pip_execute
@@ -13,10 +30,14 @@
13 apt_install('python-pip')30 apt_install('python-pip')
14 from pip import main as pip_execute31 from pip import main as pip_execute
1532
33__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
34
1635
17def parse_options(given, available):36def parse_options(given, available):
18 """Given a set of options, check if available"""37 """Given a set of options, check if available"""
19 for key, value in sorted(given.items()):38 for key, value in sorted(given.items()):
39 if not value:
40 continue
20 if key in available:41 if key in available:
21 yield "--{0}={1}".format(key, value)42 yield "--{0}={1}".format(key, value)
2243
@@ -35,14 +56,21 @@
35 pip_execute(command)56 pip_execute(command)
3657
3758
38def pip_install(package, fatal=False, **options):59def pip_install(package, fatal=False, upgrade=False, venv=None, **options):
39 """Install a python package"""60 """Install a python package"""
40 command = ["install"]61 if venv:
62 venv_python = os.path.join(venv, 'bin/pip')
63 command = [venv_python, "install"]
64 else:
65 command = ["install"]
4166
42 available_options = ('proxy', 'src', 'log', "index-url", )67 available_options = ('proxy', 'src', 'log', 'index-url', )
43 for option in parse_options(options, available_options):68 for option in parse_options(options, available_options):
44 command.append(option)69 command.append(option)
4570
71 if upgrade:
72 command.append('--upgrade')
73
46 if isinstance(package, list):74 if isinstance(package, list):
47 command.extend(package)75 command.extend(package)
48 else:76 else:
@@ -50,7 +78,10 @@
5078
51 log("Installing {} package with options: {}".format(package,79 log("Installing {} package with options: {}".format(package,
52 command))80 command))
53 pip_execute(command)81 if venv:
82 subprocess.check_call(command)
83 else:
84 pip_execute(command)
5485
5586
56def pip_uninstall(package, **options):87def pip_uninstall(package, **options):
@@ -75,3 +106,16 @@
75 """Returns the list of current python installed packages106 """Returns the list of current python installed packages
76 """107 """
77 return pip_execute(["list"])108 return pip_execute(["list"])
109
110
111def pip_create_virtualenv(path=None):
112 """Create an isolated Python environment."""
113 apt_install('python-virtualenv')
114
115 if path:
116 venv_path = path
117 else:
118 venv_path = os.path.join(charm_dir(), 'venv')
119
120 if not os.path.exists(venv_path):
121 subprocess.check_call(['virtualenv', venv_path])
78122
=== modified file 'hooks/charmhelpers/contrib/storage/__init__.py'
--- hooks/charmhelpers/contrib/storage/__init__.py 2013-07-19 02:37:30 +0000
+++ hooks/charmhelpers/contrib/storage/__init__.py 2015-09-21 10:47:21 +0000
@@ -0,0 +1,15 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
016
=== modified file 'hooks/charmhelpers/contrib/storage/linux/__init__.py'
--- hooks/charmhelpers/contrib/storage/linux/__init__.py 2013-07-19 02:37:30 +0000
+++ hooks/charmhelpers/contrib/storage/linux/__init__.py 2015-09-21 10:47:21 +0000
@@ -0,0 +1,15 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
016
=== modified file 'hooks/charmhelpers/contrib/storage/linux/ceph.py'
--- hooks/charmhelpers/contrib/storage/linux/ceph.py 2015-01-14 15:30:19 +0000
+++ hooks/charmhelpers/contrib/storage/linux/ceph.py 2015-09-21 10:47:21 +0000
@@ -1,3 +1,19 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1#17#
2# Copyright 2012 Canonical Ltd.18# Copyright 2012 Canonical Ltd.
3#19#
@@ -12,6 +28,7 @@
12import shutil28import shutil
13import json29import json
14import time30import time
31import uuid
1532
16from subprocess import (33from subprocess import (
17 check_call,34 check_call,
@@ -19,8 +36,10 @@
19 CalledProcessError,36 CalledProcessError,
20)37)
21from charmhelpers.core.hookenv import (38from charmhelpers.core.hookenv import (
39 local_unit,
22 relation_get,40 relation_get,
23 relation_ids,41 relation_ids,
42 relation_set,
24 related_units,43 related_units,
25 log,44 log,
26 DEBUG,45 DEBUG,
@@ -40,16 +59,18 @@
40 apt_install,59 apt_install,
41)60)
4261
62from charmhelpers.core.kernel import modprobe
63
43KEYRING = '/etc/ceph/ceph.client.{}.keyring'64KEYRING = '/etc/ceph/ceph.client.{}.keyring'
44KEYFILE = '/etc/ceph/ceph.client.{}.key'65KEYFILE = '/etc/ceph/ceph.client.{}.key'
4566
46CEPH_CONF = """[global]67CEPH_CONF = """[global]
47 auth supported = {auth}68auth supported = {auth}
48 keyring = {keyring}69keyring = {keyring}
49 mon host = {mon_hosts}70mon host = {mon_hosts}
50 log to syslog = {use_syslog}71log to syslog = {use_syslog}
51 err to syslog = {use_syslog}72err to syslog = {use_syslog}
52 clog to syslog = {use_syslog}73clog to syslog = {use_syslog}
53"""74"""
5475
5576
@@ -272,17 +293,6 @@
272 os.chown(data_src_dst, uid, gid)293 os.chown(data_src_dst, uid, gid)
273294
274295
275# TODO: re-use
276def modprobe(module):
277 """Load a kernel module and configure for auto-load on reboot."""
278 log('Loading kernel module', level=INFO)
279 cmd = ['modprobe', module]
280 check_call(cmd)
281 with open('/etc/modules', 'r+') as modules:
282 if module not in modules.read():
283 modules.write(module)
284
285
286def copy_files(src, dst, symlinks=False, ignore=None):296def copy_files(src, dst, symlinks=False, ignore=None):
287 """Copy files from src to dst."""297 """Copy files from src to dst."""
288 for item in os.listdir(src):298 for item in os.listdir(src):
@@ -395,17 +405,52 @@
395405
396 The API is versioned and defaults to version 1.406 The API is versioned and defaults to version 1.
397 """407 """
398 def __init__(self, api_version=1):408 def __init__(self, api_version=1, request_id=None):
399 self.api_version = api_version409 self.api_version = api_version
410 if request_id:
411 self.request_id = request_id
412 else:
413 self.request_id = str(uuid.uuid1())
400 self.ops = []414 self.ops = []
401415
402 def add_op_create_pool(self, name, replica_count=3):416 def add_op_create_pool(self, name, replica_count=3):
403 self.ops.append({'op': 'create-pool', 'name': name,417 self.ops.append({'op': 'create-pool', 'name': name,
404 'replicas': replica_count})418 'replicas': replica_count})
405419
420 def set_ops(self, ops):
421 """Set request ops to provided value.
422
423 Useful for injecting ops that come from a previous request
424 to allow comparisons to ensure validity.
425 """
426 self.ops = ops
427
406 @property428 @property
407 def request(self):429 def request(self):
408 return json.dumps({'api-version': self.api_version, 'ops': self.ops})430 return json.dumps({'api-version': self.api_version, 'ops': self.ops,
431 'request-id': self.request_id})
432
433 def _ops_equal(self, other):
434 if len(self.ops) == len(other.ops):
435 for req_no in range(0, len(self.ops)):
436 for key in ['replicas', 'name', 'op']:
437 if self.ops[req_no][key] != other.ops[req_no][key]:
438 return False
439 else:
440 return False
441 return True
442
443 def __eq__(self, other):
444 if not isinstance(other, self.__class__):
445 return False
446 if self.api_version == other.api_version and \
447 self._ops_equal(other):
448 return True
449 else:
450 return False
451
452 def __ne__(self, other):
453 return not self.__eq__(other)
409454
410455
411class CephBrokerRsp(object):456class CephBrokerRsp(object):
@@ -415,14 +460,198 @@
415460
416 The API is versioned and defaults to version 1.461 The API is versioned and defaults to version 1.
417 """462 """
463
418 def __init__(self, encoded_rsp):464 def __init__(self, encoded_rsp):
419 self.api_version = None465 self.api_version = None
420 self.rsp = json.loads(encoded_rsp)466 self.rsp = json.loads(encoded_rsp)
421467
422 @property468 @property
469 def request_id(self):
470 return self.rsp.get('request-id')
471
472 @property
423 def exit_code(self):473 def exit_code(self):
424 return self.rsp.get('exit-code')474 return self.rsp.get('exit-code')
425475
426 @property476 @property
427 def exit_msg(self):477 def exit_msg(self):
428 return self.rsp.get('stderr')478 return self.rsp.get('stderr')
479
480
481# Ceph Broker Conversation:
482# If a charm needs an action to be taken by ceph it can create a CephBrokerRq
483# and send that request to ceph via the ceph relation. The CephBrokerRq has a
484# unique id so that the client can identity which CephBrokerRsp is associated
485# with the request. Ceph will also respond to each client unit individually
486# creating a response key per client unit eg glance/0 will get a CephBrokerRsp
487# via key broker-rsp-glance-0
488#
489# To use this the charm can just do something like:
490#
491# from charmhelpers.contrib.storage.linux.ceph import (
492# send_request_if_needed,
493# is_request_complete,
494# CephBrokerRq,
495# )
496#
497# @hooks.hook('ceph-relation-changed')
498# def ceph_changed():
499# rq = CephBrokerRq()
500# rq.add_op_create_pool(name='poolname', replica_count=3)
501#
502# if is_request_complete(rq):
503# <Request complete actions>
504# else:
505# send_request_if_needed(get_ceph_request())
506#
507# CephBrokerRq and CephBrokerRsp are serialized into JSON. Below is an example
508# of glance having sent a request to ceph which ceph has successfully processed
509# 'ceph:8': {
510# 'ceph/0': {
511# 'auth': 'cephx',
512# 'broker-rsp-glance-0': '{"request-id": "0bc7dc54", "exit-code": 0}',
513# 'broker_rsp': '{"request-id": "0da543b8", "exit-code": 0}',
514# 'ceph-public-address': '10.5.44.103',
515# 'key': 'AQCLDttVuHXINhAAvI144CB09dYchhHyTUY9BQ==',
516# 'private-address': '10.5.44.103',
517# },
518# 'glance/0': {
519# 'broker_req': ('{"api-version": 1, "request-id": "0bc7dc54", '
520# '"ops": [{"replicas": 3, "name": "glance", '
521# '"op": "create-pool"}]}'),
522# 'private-address': '10.5.44.109',
523# },
524# }
525
526def get_previous_request(rid):
527 """Return the last ceph broker request sent on a given relation
528
529 @param rid: Relation id to query for request
530 """
531 request = None
532 broker_req = relation_get(attribute='broker_req', rid=rid,
533 unit=local_unit())
534 if broker_req:
535 request_data = json.loads(broker_req)
536 request = CephBrokerRq(api_version=request_data['api-version'],
537 request_id=request_data['request-id'])
538 request.set_ops(request_data['ops'])
539
540 return request
541
542
543def get_request_states(request):
544 """Return a dict of requests per relation id with their corresponding
545 completion state.
546
547 This allows a charm, which has a request for ceph, to see whether there is
548 an equivalent request already being processed and if so what state that
549 request is in.
550
551 @param request: A CephBrokerRq object
552 """
553 complete = []
554 requests = {}
555 for rid in relation_ids('ceph'):
556 complete = False
557 previous_request = get_previous_request(rid)
558 if request == previous_request:
559 sent = True
560 complete = is_request_complete_for_rid(previous_request, rid)
561 else:
562 sent = False
563 complete = False
564
565 requests[rid] = {
566 'sent': sent,
567 'complete': complete,
568 }
569
570 return requests
571
572
573def is_request_sent(request):
574 """Check to see if a functionally equivalent request has already been sent
575
576 Returns True if a similair request has been sent
577
578 @param request: A CephBrokerRq object
579 """
580 states = get_request_states(request)
581 for rid in states.keys():
582 if not states[rid]['sent']:
583 return False
584
585 return True
586
587
588def is_request_complete(request):
589 """Check to see if a functionally equivalent request has already been
590 completed
591
592 Returns True if a similair request has been completed
593
594 @param request: A CephBrokerRq object
595 """
596 states = get_request_states(request)
597 for rid in states.keys():
598 if not states[rid]['complete']:
599 return False
600
601 return True
602
603
604def is_request_complete_for_rid(request, rid):
605 """Check if a given request has been completed on the given relation
606
607 @param request: A CephBrokerRq object
608 @param rid: Relation ID
609 """
610 broker_key = get_broker_rsp_key()
611 for unit in related_units(rid):
612 rdata = relation_get(rid=rid, unit=unit)
613 if rdata.get(broker_key):
614 rsp = CephBrokerRsp(rdata.get(broker_key))
615 if rsp.request_id == request.request_id:
616 if not rsp.exit_code:
617 return True
618 else:
619 # The remote unit sent no reply targeted at this unit so either the
620 # remote ceph cluster does not support unit targeted replies or it
621 # has not processed our request yet.
622 if rdata.get('broker_rsp'):
623 request_data = json.loads(rdata['broker_rsp'])
624 if request_data.get('request-id'):
625 log('Ignoring legacy broker_rsp without unit key as remote '
626 'service supports unit specific replies', level=DEBUG)
627 else:
628 log('Using legacy broker_rsp as remote service does not '
629 'supports unit specific replies', level=DEBUG)
630 rsp = CephBrokerRsp(rdata['broker_rsp'])
631 if not rsp.exit_code:
632 return True
633
634 return False
635
636
637def get_broker_rsp_key():
638 """Return broker response key for this unit
639
640 This is the key that ceph is going to use to pass request status
641 information back to this unit
642 """
643 return 'broker-rsp-' + local_unit().replace('/', '-')
644
645
646def send_request_if_needed(request):
647 """Send broker request if an equivalent request has not already been sent
648
649 @param request: A CephBrokerRq object
650 """
651 if is_request_sent(request):
652 log('Request already sent but not complete, not sending new request',
653 level=DEBUG)
654 else:
655 for rid in relation_ids('ceph'):
656 log('Sending request {}'.format(request.request_id), level=DEBUG)
657 relation_set(relation_id=rid, broker_req=request.request)
429658
=== modified file 'hooks/charmhelpers/contrib/storage/linux/loopback.py'
--- hooks/charmhelpers/contrib/storage/linux/loopback.py 2014-12-10 20:28:57 +0000
+++ hooks/charmhelpers/contrib/storage/linux/loopback.py 2015-09-21 10:47:21 +0000
@@ -1,3 +1,19 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1import os17import os
2import re18import re
3from subprocess import (19from subprocess import (
420
=== modified file 'hooks/charmhelpers/contrib/storage/linux/lvm.py'
--- hooks/charmhelpers/contrib/storage/linux/lvm.py 2014-12-10 20:28:57 +0000
+++ hooks/charmhelpers/contrib/storage/linux/lvm.py 2015-09-21 10:47:21 +0000
@@ -1,3 +1,19 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1from subprocess import (17from subprocess import (
2 CalledProcessError,18 CalledProcessError,
3 check_call,19 check_call,
420
=== modified file 'hooks/charmhelpers/contrib/storage/linux/utils.py'
--- hooks/charmhelpers/contrib/storage/linux/utils.py 2014-12-10 20:28:57 +0000
+++ hooks/charmhelpers/contrib/storage/linux/utils.py 2015-09-21 10:47:21 +0000
@@ -1,3 +1,19 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1import os17import os
2import re18import re
3from stat import S_ISBLK19from stat import S_ISBLK
@@ -27,9 +43,10 @@
2743
28 :param block_device: str: Full path of block device to clean.44 :param block_device: str: Full path of block device to clean.
29 '''45 '''
46 # https://github.com/ceph/ceph/commit/fdd7f8d83afa25c4e09aaedd90ab93f3b64a677b
30 # sometimes sgdisk exits non-zero; this is OK, dd will clean up47 # sometimes sgdisk exits non-zero; this is OK, dd will clean up
31 call(['sgdisk', '--zap-all', '--mbrtogpt',48 call(['sgdisk', '--zap-all', '--', block_device])
32 '--clear', block_device])49 call(['sgdisk', '--clear', '--mbrtogpt', '--', block_device])
33 dev_end = check_output(['blockdev', '--getsz',50 dev_end = check_output(['blockdev', '--getsz',
34 block_device]).decode('UTF-8')51 block_device]).decode('UTF-8')
35 gpt_end = int(dev_end.split()[0]) - 10052 gpt_end = int(dev_end.split()[0]) - 100
@@ -51,4 +68,4 @@
51 out = check_output(['mount']).decode('UTF-8')68 out = check_output(['mount']).decode('UTF-8')
52 if is_partition:69 if is_partition:
53 return bool(re.search(device + r"\b", out))70 return bool(re.search(device + r"\b", out))
54 return bool(re.search(device + r"[0-9]+\b", out))71 return bool(re.search(device + r"[0-9]*\b", out))
5572
=== modified file 'hooks/charmhelpers/core/__init__.py'
--- hooks/charmhelpers/core/__init__.py 2013-07-19 02:37:30 +0000
+++ hooks/charmhelpers/core/__init__.py 2015-09-21 10:47:21 +0000
@@ -0,0 +1,15 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
016
=== modified file 'hooks/charmhelpers/core/decorators.py'
--- hooks/charmhelpers/core/decorators.py 2015-01-14 15:30:27 +0000
+++ hooks/charmhelpers/core/decorators.py 2015-09-21 10:47:21 +0000
@@ -1,3 +1,19 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1#17#
2# Copyright 2014 Canonical Ltd.18# Copyright 2014 Canonical Ltd.
3#19#
420
=== added file 'hooks/charmhelpers/core/files.py'
--- hooks/charmhelpers/core/files.py 1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/files.py 2015-09-21 10:47:21 +0000
@@ -0,0 +1,45 @@
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4# Copyright 2014-2015 Canonical Limited.
5#
6# This file is part of charm-helpers.
7#
8# charm-helpers is free software: you can redistribute it and/or modify
9# it under the terms of the GNU Lesser General Public License version 3 as
10# published by the Free Software Foundation.
11#
12# charm-helpers is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU Lesser General Public License for more details.
16#
17# You should have received a copy of the GNU Lesser General Public License
18# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
19
20__author__ = 'Jorge Niedbalski <niedbalski@ubuntu.com>'
21
22import os
23import subprocess
24
25
26def sed(filename, before, after, flags='g'):
27 """
28 Search and replaces the given pattern on filename.
29
30 :param filename: relative or absolute file path.
31 :param before: expression to be replaced (see 'man sed')
32 :param after: expression to replace with (see 'man sed')
33 :param flags: sed-compatible regex flags in example, to make
34 the search and replace case insensitive, specify ``flags="i"``.
35 The ``g`` flag is always specified regardless, so you do not
36 need to remember to include it when overriding this parameter.
37 :returns: If the sed command exit code was zero then return,
38 otherwise raise CalledProcessError.
39 """
40 expression = r's/{0}/{1}/{2}'.format(before,
41 after, flags)
42
43 return subprocess.check_call(["sed", "-i", "-r", "-e",
44 expression,
45 os.path.expanduser(filename)])
046
=== modified file 'hooks/charmhelpers/core/fstab.py'
--- hooks/charmhelpers/core/fstab.py 2014-12-10 20:28:57 +0000
+++ hooks/charmhelpers/core/fstab.py 2015-09-21 10:47:21 +0000
@@ -1,11 +1,27 @@
1#!/usr/bin/env python1#!/usr/bin/env python
2# -*- coding: utf-8 -*-2# -*- coding: utf-8 -*-
33
4__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'4# Copyright 2014-2015 Canonical Limited.
5#
6# This file is part of charm-helpers.
7#
8# charm-helpers is free software: you can redistribute it and/or modify
9# it under the terms of the GNU Lesser General Public License version 3 as
10# published by the Free Software Foundation.
11#
12# charm-helpers is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU Lesser General Public License for more details.
16#
17# You should have received a copy of the GNU Lesser General Public License
18# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
519
6import io20import io
7import os21import os
822
23__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
24
925
10class Fstab(io.FileIO):26class Fstab(io.FileIO):
11 """This class extends file in order to implement a file reader/writer27 """This class extends file in order to implement a file reader/writer
@@ -61,7 +77,7 @@
61 for line in self.readlines():77 for line in self.readlines():
62 line = line.decode('us-ascii')78 line = line.decode('us-ascii')
63 try:79 try:
64 if line.strip() and not line.startswith("#"):80 if line.strip() and not line.strip().startswith("#"):
65 yield self._hydrate_entry(line)81 yield self._hydrate_entry(line)
66 except ValueError:82 except ValueError:
67 pass83 pass
@@ -88,7 +104,7 @@
88104
89 found = False105 found = False
90 for index, line in enumerate(lines):106 for index, line in enumerate(lines):
91 if not line.startswith("#"):107 if line.strip() and not line.strip().startswith("#"):
92 if self._hydrate_entry(line) == entry:108 if self._hydrate_entry(line) == entry:
93 found = True109 found = True
94 break110 break
95111
=== modified file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py 2014-12-11 13:41:43 +0000
+++ hooks/charmhelpers/core/hookenv.py 2015-09-21 10:47:21 +0000
@@ -1,14 +1,37 @@
1# Copyright 2014-2015 Canonical Limited.
2#
3# This file is part of charm-helpers.
4#
5# charm-helpers is free software: you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License version 3 as
7# published by the Free Software Foundation.
8#
9# charm-helpers is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
16
1"Interactions with the Juju environment"17"Interactions with the Juju environment"
2# Copyright 2013 Canonical Ltd.18# Copyright 2013 Canonical Ltd.
3#19#
4# Authors:20# Authors:
5# Charm Helpers Developers <juju@lists.ubuntu.com>21# Charm Helpers Developers <juju@lists.ubuntu.com>
622
23from __future__ import print_function
24import copy
25from distutils.version import LooseVersion
26from functools import wraps
27import glob
7import os28import os
8import json29import json
9import yaml30import yaml
10import subprocess31import subprocess
11import sys32import sys
33import errno
34import tempfile
12from subprocess import CalledProcessError35from subprocess import CalledProcessError
1336
14import six37import six
@@ -40,15 +63,18 @@
4063
41 will cache the result of unit_get + 'test' for future calls.64 will cache the result of unit_get + 'test' for future calls.
42 """65 """
66 @wraps(func)
43 def wrapper(*args, **kwargs):67 def wrapper(*args, **kwargs):
44 global cache68 global cache
45 key = str((func, args, kwargs))69 key = str((func, args, kwargs))
46 try:70 try:
47 return cache[key]71 return cache[key]
48 except KeyError:72 except KeyError:
49 res = func(*args, **kwargs)73 pass # Drop out of the exception handler scope.
50 cache[key] = res74 res = func(*args, **kwargs)
51 return res75 cache[key] = res
76 return res
77 wrapper._wrapped = func
52 return wrapper78 return wrapper
5379
5480
@@ -71,7 +97,18 @@
71 if not isinstance(message, six.string_types):97 if not isinstance(message, six.string_types):
72 message = repr(message)98 message = repr(message)
73 command += [message]99 command += [message]
74 subprocess.call(command)100 # Missing juju-log should not cause failures in unit tests
101 # Send log output to stderr
102 try:
103 subprocess.call(command)
104 except OSError as e:
105 if e.errno == errno.ENOENT:
106 if level:
107 message = "{}: {}".format(level, message)
108 message = "juju-log: {}".format(message)
109 print(message, file=sys.stderr)
110 else:
111 raise
75112
76113
77class Serializable(UserDict):114class Serializable(UserDict):
@@ -137,9 +174,19 @@
137 return os.environ.get('JUJU_RELATION', None)174 return os.environ.get('JUJU_RELATION', None)
138175
139176
140def relation_id():177@cached
141 """The relation ID for the current relation hook"""178def relation_id(relation_name=None, service_or_unit=None):
142 return os.environ.get('JUJU_RELATION_ID', None)179 """The relation ID for the current or a specified relation"""
180 if not relation_name and not service_or_unit:
181 return os.environ.get('JUJU_RELATION_ID', None)
182 elif relation_name and service_or_unit:
183 service_name = service_or_unit.split('/')[0]
184 for relid in relation_ids(relation_name):
185 remote_service = remote_service_name(relid)
186 if remote_service == service_name:
187 return relid
188 else:
189 raise ValueError('Must specify neither or both of relation_name and service_or_unit')
143190
144191
145def local_unit():192def local_unit():
@@ -149,7 +196,7 @@
149196
150def remote_unit():197def remote_unit():
151 """The remote unit for the current relation hook"""198 """The remote unit for the current relation hook"""
152 return os.environ['JUJU_REMOTE_UNIT']199 return os.environ.get('JUJU_REMOTE_UNIT', None)
153200
154201
155def service_name():202def service_name():
@@ -157,9 +204,20 @@
157 return local_unit().split('/')[0]204 return local_unit().split('/')[0]
158205
159206
207@cached
208def remote_service_name(relid=None):
209 """The remote service name for a given relation-id (or the current relation)"""
210 if relid is None:
211 unit = remote_unit()
212 else:
213 units = related_units(relid)
214 unit = units[0] if units else None
215 return unit.split('/')[0] if unit else None
216
217
160def hook_name():218def hook_name():
161 """The name of the currently executing hook"""219 """The name of the currently executing hook"""
162 return os.path.basename(sys.argv[0])220 return os.environ.get('JUJU_HOOK_NAME', os.path.basename(sys.argv[0]))
163221
164222
165class Config(dict):223class Config(dict):
@@ -209,23 +267,7 @@
209 self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)267 self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
210 if os.path.exists(self.path):268 if os.path.exists(self.path):
211 self.load_previous()269 self.load_previous()
212270 atexit(self._implicit_save)
213 def __getitem__(self, key):
214 """For regular dict lookups, check the current juju config first,
215 then the previous (saved) copy. This ensures that user-saved values
216 will be returned by a dict lookup.
217
218 """
219 try:
220 return dict.__getitem__(self, key)
221 except KeyError:
222 return (self._prev_dict or {})[key]
223
224 def keys(self):
225 prev_keys = []
226 if self._prev_dict is not None:
227 prev_keys = self._prev_dict.keys()
228 return list(set(prev_keys + list(dict.keys(self))))
229271
230 def load_previous(self, path=None):272 def load_previous(self, path=None):
231 """Load previous copy of config from disk.273 """Load previous copy of config from disk.
@@ -244,6 +286,9 @@
244 self.path = path or self.path286 self.path = path or self.path
245 with open(self.path) as f:287 with open(self.path) as f:
246 self._prev_dict = json.load(f)288 self._prev_dict = json.load(f)
289 for k, v in copy.deepcopy(self._prev_dict).items():
290 if k not in self:
291 self[k] = v
247292
248 def changed(self, key):293 def changed(self, key):
249 """Return True if the current value for this key is different from294 """Return True if the current value for this key is different from
@@ -275,13 +320,13 @@
275 instance.320 instance.
276321
277 """322 """
278 if self._prev_dict:
279 for k, v in six.iteritems(self._prev_dict):
280 if k not in self:
281 self[k] = v
282 with open(self.path, 'w') as f:323 with open(self.path, 'w') as f:
283 json.dump(self, f)324 json.dump(self, f)
284325
326 def _implicit_save(self):
327 if self.implicit_save:
328 self.save()
329
285330
286@cached331@cached
287def config(scope=None):332def config(scope=None):
@@ -324,18 +369,49 @@
324 """Set relation information for the current unit"""369 """Set relation information for the current unit"""
325 relation_settings = relation_settings if relation_settings else {}370 relation_settings = relation_settings if relation_settings else {}
326 relation_cmd_line = ['relation-set']371 relation_cmd_line = ['relation-set']
372 accepts_file = "--file" in subprocess.check_output(
373 relation_cmd_line + ["--help"], universal_newlines=True)
327 if relation_id is not None:374 if relation_id is not None:
328 relation_cmd_line.extend(('-r', relation_id))375 relation_cmd_line.extend(('-r', relation_id))
329 for k, v in (list(relation_settings.items()) + list(kwargs.items())):376 settings = relation_settings.copy()
330 if v is None:377 settings.update(kwargs)
331 relation_cmd_line.append('{}='.format(k))378 for key, value in settings.items():
332 else:379 # Force value to be a string: it always should, but some call
333 relation_cmd_line.append('{}={}'.format(k, v))380 # sites pass in things like dicts or numbers.
334 subprocess.check_call(relation_cmd_line)381 if value is not None:
382 settings[key] = "{}".format(value)
383 if accepts_file:
384 # --file was introduced in Juju 1.23.2. Use it by default if
385 # available, since otherwise we'll break if the relation data is
386 # too big. Ideally we should tell relation-set to read the data from
387 # stdin, but that feature is broken in 1.23.2: Bug #1454678.
388 with tempfile.NamedTemporaryFile(delete=False) as settings_file:
389 settings_file.write(yaml.safe_dump(settings).encode("utf-8"))
390 subprocess.check_call(
391 relation_cmd_line + ["--file", settings_file.name])
392 os.remove(settings_file.name)
393 else:
394 for key, value in settings.items():
395 if value is None:
396 relation_cmd_line.append('{}='.format(key))
397 else:
398 relation_cmd_line.append('{}={}'.format(key, value))
399 subprocess.check_call(relation_cmd_line)
335 # Flush cache of any relation-gets for local unit400 # Flush cache of any relation-gets for local unit
336 flush(local_unit())401 flush(local_unit())
337402
338403
404def relation_clear(r_id=None):
405 ''' Clears any relation data already set on relation r_id '''
406 settings = relation_get(rid=r_id,
407 unit=local_unit())
408 for setting in settings:
409 if setting not in ['public-address', 'private-address']:
410 settings[setting] = None
411 relation_set(relation_id=r_id,
412 **settings)
413
414
339@cached415@cached
340def relation_ids(reltype=None):416def relation_ids(reltype=None):
341 """A list of relation_ids"""417 """A list of relation_ids"""
@@ -415,6 +491,63 @@
415491
416492
417@cached493@cached
494def relation_to_interface(relation_name):
495 """
496 Given the name of a relation, return the interface that relation uses.
497
498 :returns: The interface name, or ``None``.
499 """
500 return relation_to_role_and_interface(relation_name)[1]
501
502
503@cached
504def relation_to_role_and_interface(relation_name):
505 """
506 Given the name of a relation, return the role and the name of the interface
507 that relation uses (where role is one of ``provides``, ``requires``, or ``peer``).
508
509 :returns: A tuple containing ``(role, interface)``, or ``(None, None)``.
510 """
511 _metadata = metadata()
512 for role in ('provides', 'requires', 'peer'):
513 interface = _metadata.get(role, {}).get(relation_name, {}).get('interface')
514 if interface:
515 return role, interface
516 return None, None
517
518
519@cached
520def role_and_interface_to_relations(role, interface_name):
521 """
522 Given a role and interface name, return a list of relation names for the
523 current charm that use that interface under that role (where role is one
524 of ``provides``, ``requires``, or ``peer``).
525
526 :returns: A list of relation names.
527 """
528 _metadata = metadata()
529 results = []
530 for relation_name, relation in _metadata.get(role, {}).items():
531 if relation['interface'] == interface_name:
532 results.append(relation_name)
533 return results
534
535
536@cached
537def interface_to_relations(interface_name):
538 """
539 Given an interface, return a list of relation names for the current
540 charm that use that interface.
541
542 :returns: A list of relation names.
543 """
544 results = []
545 for role in ('provides', 'requires', 'peer'):
546 results.extend(role_and_interface_to_relations(role, interface_name))
547 return results
548
549
550@cached
418def charm_name():551def charm_name():
419 """Get the name of the current charm as is specified on metadata.yaml"""552 """Get the name of the current charm as is specified on metadata.yaml"""
420 return metadata().get('name')553 return metadata().get('name')
@@ -480,6 +613,11 @@
480 return None613 return None
481614
482615
616def unit_public_ip():
617 """Get this unit's public IP address"""
618 return unit_get('public-address')
619
620
483def unit_private_ip():621def unit_private_ip():
484 """Get this unit's private IP address"""622 """Get this unit's private IP address"""
485 return unit_get('private-address')623 return unit_get('private-address')
@@ -512,10 +650,14 @@
512 hooks.execute(sys.argv)650 hooks.execute(sys.argv)
513 """651 """
514652
515 def __init__(self, config_save=True):653 def __init__(self, config_save=None):
516 super(Hooks, self).__init__()654 super(Hooks, self).__init__()
517 self._hooks = {}655 self._hooks = {}
518 self._config_save = config_save656
657 # For unknown reasons, we allow the Hooks constructor to override
658 # config().implicit_save.
659 if config_save is not None:
660 config().implicit_save = config_save
519661
520 def register(self, name, function):662 def register(self, name, function):
521 """Register a hook"""663 """Register a hook"""
@@ -523,13 +665,16 @@
523665
524 def execute(self, args):666 def execute(self, args):
525 """Execute a registered hook based on args[0]"""667 """Execute a registered hook based on args[0]"""
668 _run_atstart()
526 hook_name = os.path.basename(args[0])669 hook_name = os.path.basename(args[0])
527 if hook_name in self._hooks:670 if hook_name in self._hooks:
528 self._hooks[hook_name]()
The diff has been truncated for viewing.

Subscribers

People subscribed via source and target branches

to all changes: