Merge lp:~chad.smith/charms/trusty/glance-simplestreams-sync/sync-charmhelpers-for-add-apt-repo-retry into lp:~openstack-charmers/charms/trusty/glance-simplestreams-sync/next

Proposed by Chad Smith
Status: Needs review
Proposed branch: lp:~chad.smith/charms/trusty/glance-simplestreams-sync/sync-charmhelpers-for-add-apt-repo-retry
Merge into: lp:~openstack-charmers/charms/trusty/glance-simplestreams-sync/next
Diff against target: 10125 lines (+5920/-1725)
74 files modified
charm-helpers-sync.yaml (+1/-0)
charmhelpers/__init__.py (+11/-13)
charmhelpers/contrib/__init__.py (+11/-13)
charmhelpers/contrib/charmsupport/__init__.py (+11/-13)
charmhelpers/contrib/charmsupport/nrpe.py (+103/-34)
charmhelpers/contrib/charmsupport/volumes.py (+11/-13)
charmhelpers/contrib/hahelpers/__init__.py (+11/-13)
charmhelpers/contrib/hahelpers/apache.py (+30/-17)
charmhelpers/contrib/hahelpers/cluster.py (+70/-23)
charmhelpers/contrib/network/__init__.py (+11/-13)
charmhelpers/contrib/network/ip.py (+99/-42)
charmhelpers/contrib/openstack/__init__.py (+11/-13)
charmhelpers/contrib/openstack/alternatives.py (+11/-13)
charmhelpers/contrib/openstack/amulet/__init__.py (+11/-13)
charmhelpers/contrib/openstack/amulet/deployment.py (+200/-69)
charmhelpers/contrib/openstack/amulet/utils.py (+354/-38)
charmhelpers/contrib/openstack/context.py (+313/-127)
charmhelpers/contrib/openstack/exceptions.py (+21/-0)
charmhelpers/contrib/openstack/files/__init__.py (+11/-13)
charmhelpers/contrib/openstack/files/check_haproxy.sh (+7/-5)
charmhelpers/contrib/openstack/ha/__init__.py (+13/-0)
charmhelpers/contrib/openstack/ha/utils.py (+139/-0)
charmhelpers/contrib/openstack/ip.py (+60/-25)
charmhelpers/contrib/openstack/keystone.py (+178/-0)
charmhelpers/contrib/openstack/neutron.py (+57/-23)
charmhelpers/contrib/openstack/templates/__init__.py (+11/-13)
charmhelpers/contrib/openstack/templates/haproxy.cfg (+19/-11)
charmhelpers/contrib/openstack/templates/memcached.conf (+53/-0)
charmhelpers/contrib/openstack/templates/openstack_https_frontend (+5/-0)
charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf (+5/-0)
charmhelpers/contrib/openstack/templates/section-keystone-authtoken (+8/-5)
charmhelpers/contrib/openstack/templates/section-keystone-authtoken-legacy (+10/-0)
charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka (+20/-0)
charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf (+100/-0)
charmhelpers/contrib/openstack/templating.py (+19/-15)
charmhelpers/contrib/openstack/utils.py (+1163/-151)
charmhelpers/contrib/python/__init__.py (+11/-13)
charmhelpers/contrib/python/packages.py (+59/-26)
charmhelpers/contrib/storage/__init__.py (+11/-13)
charmhelpers/contrib/storage/linux/__init__.py (+11/-13)
charmhelpers/contrib/storage/linux/ceph.py (+766/-72)
charmhelpers/contrib/storage/linux/loopback.py (+21/-13)
charmhelpers/contrib/storage/linux/lvm.py (+11/-13)
charmhelpers/contrib/storage/linux/utils.py (+16/-18)
charmhelpers/core/__init__.py (+11/-13)
charmhelpers/core/decorators.py (+11/-13)
charmhelpers/core/files.py (+11/-13)
charmhelpers/core/fstab.py (+11/-13)
charmhelpers/core/hookenv.py (+157/-19)
charmhelpers/core/host.py (+482/-150)
charmhelpers/core/host_factory/centos.py (+56/-0)
charmhelpers/core/host_factory/ubuntu.py (+56/-0)
charmhelpers/core/hugepage.py (+13/-13)
charmhelpers/core/kernel.py (+34/-30)
charmhelpers/core/kernel_factory/centos.py (+17/-0)
charmhelpers/core/kernel_factory/ubuntu.py (+13/-0)
charmhelpers/core/services/__init__.py (+11/-13)
charmhelpers/core/services/base.py (+11/-13)
charmhelpers/core/services/helpers.py (+25/-18)
charmhelpers/core/strutils.py (+11/-13)
charmhelpers/core/sysctl.py (+11/-13)
charmhelpers/core/templating.py (+40/-24)
charmhelpers/core/unitdata.py (+11/-14)
charmhelpers/fetch/__init__.py (+43/-302)
charmhelpers/fetch/archiveurl.py (+12/-14)
charmhelpers/fetch/bzrurl.py (+48/-50)
charmhelpers/fetch/centos.py (+171/-0)
charmhelpers/fetch/giturl.py (+33/-37)
charmhelpers/fetch/snap.py (+122/-0)
charmhelpers/fetch/ubuntu.py (+364/-0)
charmhelpers/osplatform.py (+25/-0)
charmhelpers/payload/__init__.py (+11/-13)
charmhelpers/payload/archive.py (+11/-13)
charmhelpers/payload/execd.py (+14/-15)
To merge this branch: bzr merge lp:~chad.smith/charms/trusty/glance-simplestreams-sync/sync-charmhelpers-for-add-apt-repo-retry
Reviewer Review Type Date Requested Status
OpenStack Charmers Pending
Review via email: mp+318976@code.launchpad.net

Description of the change

Make sync of charmhelpers updates to get latest fetch.add_source retries logic.

The only official change to this branch is in charm-helpers-sync because host now depends on osplatform
--- charm-helpers-sync.yaml revid:<email address hidden>
+++ charm-helpers-sync.yaml revid:<email address hidden>
@@ -3,6 +3,7 @@
 include:
     - core
     - fetch
+ - osplatform
     - payload
     - contrib.openstack|inc=*
     - contrib.charmsupport

To post a comment you must log in.

Unmerged revisions

60. By Chad Smith on 2017-03-04

make sync pulling in updated charmhelpers

59. By Chad Smith on 2017-03-04

Update charmhelpers dependencies prior to make sync since charmhelpers.host now imports get_platform from charmhelpers.osplatform, we need to import osplatform too

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'charm-helpers-sync.yaml'
2--- charm-helpers-sync.yaml 2015-09-28 20:36:18 +0000
3+++ charm-helpers-sync.yaml 2017-03-04 02:21:16 +0000
4@@ -3,6 +3,7 @@
5 include:
6 - core
7 - fetch
8+ - osplatform
9 - payload
10 - contrib.openstack|inc=*
11 - contrib.charmsupport
12
13=== modified file 'charmhelpers/__init__.py'
14--- charmhelpers/__init__.py 2015-09-28 20:09:02 +0000
15+++ charmhelpers/__init__.py 2017-03-04 02:21:16 +0000
16@@ -1,18 +1,16 @@
17 # Copyright 2014-2015 Canonical Limited.
18 #
19-# This file is part of charm-helpers.
20-#
21-# charm-helpers is free software: you can redistribute it and/or modify
22-# it under the terms of the GNU Lesser General Public License version 3 as
23-# published by the Free Software Foundation.
24-#
25-# charm-helpers is distributed in the hope that it will be useful,
26-# but WITHOUT ANY WARRANTY; without even the implied warranty of
27-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28-# GNU Lesser General Public License for more details.
29-#
30-# You should have received a copy of the GNU Lesser General Public License
31-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
32+# Licensed under the Apache License, Version 2.0 (the "License");
33+# you may not use this file except in compliance with the License.
34+# You may obtain a copy of the License at
35+#
36+# http://www.apache.org/licenses/LICENSE-2.0
37+#
38+# Unless required by applicable law or agreed to in writing, software
39+# distributed under the License is distributed on an "AS IS" BASIS,
40+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
41+# See the License for the specific language governing permissions and
42+# limitations under the License.
43
44 # Bootstrap charm-helpers, installing its dependencies if necessary using
45 # only standard libraries.
46
47=== modified file 'charmhelpers/contrib/__init__.py'
48--- charmhelpers/contrib/__init__.py 2015-09-28 20:09:02 +0000
49+++ charmhelpers/contrib/__init__.py 2017-03-04 02:21:16 +0000
50@@ -1,15 +1,13 @@
51 # Copyright 2014-2015 Canonical Limited.
52 #
53-# This file is part of charm-helpers.
54-#
55-# charm-helpers is free software: you can redistribute it and/or modify
56-# it under the terms of the GNU Lesser General Public License version 3 as
57-# published by the Free Software Foundation.
58-#
59-# charm-helpers is distributed in the hope that it will be useful,
60-# but WITHOUT ANY WARRANTY; without even the implied warranty of
61-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
62-# GNU Lesser General Public License for more details.
63-#
64-# You should have received a copy of the GNU Lesser General Public License
65-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
66+# Licensed under the Apache License, Version 2.0 (the "License");
67+# you may not use this file except in compliance with the License.
68+# You may obtain a copy of the License at
69+#
70+# http://www.apache.org/licenses/LICENSE-2.0
71+#
72+# Unless required by applicable law or agreed to in writing, software
73+# distributed under the License is distributed on an "AS IS" BASIS,
74+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
75+# See the License for the specific language governing permissions and
76+# limitations under the License.
77
78=== modified file 'charmhelpers/contrib/charmsupport/__init__.py'
79--- charmhelpers/contrib/charmsupport/__init__.py 2015-09-28 20:09:02 +0000
80+++ charmhelpers/contrib/charmsupport/__init__.py 2017-03-04 02:21:16 +0000
81@@ -1,15 +1,13 @@
82 # Copyright 2014-2015 Canonical Limited.
83 #
84-# This file is part of charm-helpers.
85-#
86-# charm-helpers is free software: you can redistribute it and/or modify
87-# it under the terms of the GNU Lesser General Public License version 3 as
88-# published by the Free Software Foundation.
89-#
90-# charm-helpers is distributed in the hope that it will be useful,
91-# but WITHOUT ANY WARRANTY; without even the implied warranty of
92-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
93-# GNU Lesser General Public License for more details.
94-#
95-# You should have received a copy of the GNU Lesser General Public License
96-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
97+# Licensed under the Apache License, Version 2.0 (the "License");
98+# you may not use this file except in compliance with the License.
99+# You may obtain a copy of the License at
100+#
101+# http://www.apache.org/licenses/LICENSE-2.0
102+#
103+# Unless required by applicable law or agreed to in writing, software
104+# distributed under the License is distributed on an "AS IS" BASIS,
105+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
106+# See the License for the specific language governing permissions and
107+# limitations under the License.
108
109=== modified file 'charmhelpers/contrib/charmsupport/nrpe.py'
110--- charmhelpers/contrib/charmsupport/nrpe.py 2015-09-28 20:09:02 +0000
111+++ charmhelpers/contrib/charmsupport/nrpe.py 2017-03-04 02:21:16 +0000
112@@ -1,18 +1,16 @@
113 # Copyright 2014-2015 Canonical Limited.
114 #
115-# This file is part of charm-helpers.
116-#
117-# charm-helpers is free software: you can redistribute it and/or modify
118-# it under the terms of the GNU Lesser General Public License version 3 as
119-# published by the Free Software Foundation.
120-#
121-# charm-helpers is distributed in the hope that it will be useful,
122-# but WITHOUT ANY WARRANTY; without even the implied warranty of
123-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
124-# GNU Lesser General Public License for more details.
125-#
126-# You should have received a copy of the GNU Lesser General Public License
127-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
128+# Licensed under the Apache License, Version 2.0 (the "License");
129+# you may not use this file except in compliance with the License.
130+# You may obtain a copy of the License at
131+#
132+# http://www.apache.org/licenses/LICENSE-2.0
133+#
134+# Unless required by applicable law or agreed to in writing, software
135+# distributed under the License is distributed on an "AS IS" BASIS,
136+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
137+# See the License for the specific language governing permissions and
138+# limitations under the License.
139
140 """Compatibility with the nrpe-external-master charm"""
141 # Copyright 2012 Canonical Ltd.
142@@ -40,6 +38,7 @@
143 )
144
145 from charmhelpers.core.host import service
146+from charmhelpers.core import host
147
148 # This module adds compatibility with the nrpe-external-master and plain nrpe
149 # subordinate charms. To use it in your charm:
150@@ -110,6 +109,13 @@
151 # def local_monitors_relation_changed():
152 # update_nrpe_config()
153 #
154+# 4.a If your charm is a subordinate charm set primary=False
155+#
156+# from charmsupport.nrpe import NRPE
157+# (...)
158+# def update_nrpe_config():
159+# nrpe_compat = NRPE(primary=False)
160+#
161 # 5. ln -s hooks.py nrpe-external-master-relation-changed
162 # ln -s hooks.py local-monitors-relation-changed
163
164@@ -148,6 +154,13 @@
165 self.description = description
166 self.check_cmd = self._locate_cmd(check_cmd)
167
168+ def _get_check_filename(self):
169+ return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command))
170+
171+ def _get_service_filename(self, hostname):
172+ return os.path.join(NRPE.nagios_exportdir,
173+ 'service__{}_{}.cfg'.format(hostname, self.command))
174+
175 def _locate_cmd(self, check_cmd):
176 search_path = (
177 '/usr/lib/nagios/plugins',
178@@ -163,9 +176,21 @@
179 log('Check command not found: {}'.format(parts[0]))
180 return ''
181
182+ def _remove_service_files(self):
183+ if not os.path.exists(NRPE.nagios_exportdir):
184+ return
185+ for f in os.listdir(NRPE.nagios_exportdir):
186+ if f.endswith('_{}.cfg'.format(self.command)):
187+ os.remove(os.path.join(NRPE.nagios_exportdir, f))
188+
189+ def remove(self, hostname):
190+ nrpe_check_file = self._get_check_filename()
191+ if os.path.exists(nrpe_check_file):
192+ os.remove(nrpe_check_file)
193+ self._remove_service_files()
194+
195 def write(self, nagios_context, hostname, nagios_servicegroups):
196- nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format(
197- self.command)
198+ nrpe_check_file = self._get_check_filename()
199 with open(nrpe_check_file, 'w') as nrpe_check_config:
200 nrpe_check_config.write("# check {}\n".format(self.shortname))
201 nrpe_check_config.write("command[{}]={}\n".format(
202@@ -180,9 +205,7 @@
203
204 def write_service_config(self, nagios_context, hostname,
205 nagios_servicegroups):
206- for f in os.listdir(NRPE.nagios_exportdir):
207- if re.search('.*{}.cfg'.format(self.command), f):
208- os.remove(os.path.join(NRPE.nagios_exportdir, f))
209+ self._remove_service_files()
210
211 templ_vars = {
212 'nagios_hostname': hostname,
213@@ -192,8 +215,7 @@
214 'command': self.command,
215 }
216 nrpe_service_text = Check.service_template.format(**templ_vars)
217- nrpe_service_file = '{}/service__{}_{}.cfg'.format(
218- NRPE.nagios_exportdir, hostname, self.command)
219+ nrpe_service_file = self._get_service_filename(hostname)
220 with open(nrpe_service_file, 'w') as nrpe_service_config:
221 nrpe_service_config.write(str(nrpe_service_text))
222
223@@ -206,9 +228,10 @@
224 nagios_exportdir = '/var/lib/nagios/export'
225 nrpe_confdir = '/etc/nagios/nrpe.d'
226
227- def __init__(self, hostname=None):
228+ def __init__(self, hostname=None, primary=True):
229 super(NRPE, self).__init__()
230 self.config = config()
231+ self.primary = primary
232 self.nagios_context = self.config['nagios_context']
233 if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']:
234 self.nagios_servicegroups = self.config['nagios_servicegroups']
235@@ -218,12 +241,38 @@
236 if hostname:
237 self.hostname = hostname
238 else:
239- self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
240+ nagios_hostname = get_nagios_hostname()
241+ if nagios_hostname:
242+ self.hostname = nagios_hostname
243+ else:
244+ self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
245 self.checks = []
246+ # Iff in an nrpe-external-master relation hook, set primary status
247+ relation = relation_ids('nrpe-external-master')
248+ if relation:
249+ log("Setting charm primary status {}".format(primary))
250+ for rid in relation_ids('nrpe-external-master'):
251+ relation_set(relation_id=rid, relation_settings={'primary': self.primary})
252
253 def add_check(self, *args, **kwargs):
254 self.checks.append(Check(*args, **kwargs))
255
256+ def remove_check(self, *args, **kwargs):
257+ if kwargs.get('shortname') is None:
258+ raise ValueError('shortname of check must be specified')
259+
260+ # Use sensible defaults if they're not specified - these are not
261+ # actually used during removal, but they're required for constructing
262+ # the Check object; check_disk is chosen because it's part of the
263+ # nagios-plugins-basic package.
264+ if kwargs.get('check_cmd') is None:
265+ kwargs['check_cmd'] = 'check_disk'
266+ if kwargs.get('description') is None:
267+ kwargs['description'] = ''
268+
269+ check = Check(*args, **kwargs)
270+ check.remove(self.hostname)
271+
272 def write(self):
273 try:
274 nagios_uid = pwd.getpwnam('nagios').pw_uid
275@@ -260,7 +309,7 @@
276 :param str relation_name: Name of relation nrpe sub joined to
277 """
278 for rel in relations_of_type(relation_name):
279- if 'nagios_hostname' in rel:
280+ if 'nagios_host_context' in rel:
281 return rel['nagios_host_context']
282
283
284@@ -289,18 +338,30 @@
285 return unit
286
287
288-def add_init_service_checks(nrpe, services, unit_name):
289+def add_init_service_checks(nrpe, services, unit_name, immediate_check=True):
290 """
291 Add checks for each service in list
292
293 :param NRPE nrpe: NRPE object to add check to
294 :param list services: List of services to check
295 :param str unit_name: Unit name to use in check description
296+ :param bool immediate_check: For sysv init, run the service check immediately
297 """
298 for svc in services:
299+ # Don't add a check for these services from neutron-gateway
300+ if svc in ['ext-port', 'os-charm-phy-nic-mtu']:
301+ next
302+
303 upstart_init = '/etc/init/%s.conf' % svc
304 sysv_init = '/etc/init.d/%s' % svc
305- if os.path.exists(upstart_init):
306+
307+ if host.init_is_systemd():
308+ nrpe.add_check(
309+ shortname=svc,
310+ description='process check {%s}' % unit_name,
311+ check_cmd='check_systemd.py %s' % svc
312+ )
313+ elif os.path.exists(upstart_init):
314 nrpe.add_check(
315 shortname=svc,
316 description='process check {%s}' % unit_name,
317@@ -308,21 +369,29 @@
318 )
319 elif os.path.exists(sysv_init):
320 cronpath = '/etc/cron.d/nagios-service-check-%s' % svc
321- cron_file = ('*/5 * * * * root '
322- '/usr/local/lib/nagios/plugins/check_exit_status.pl '
323- '-s /etc/init.d/%s status > '
324- '/var/lib/nagios/service-check-%s.txt\n' % (svc,
325- svc)
326- )
327+ checkpath = '/var/lib/nagios/service-check-%s.txt' % svc
328+ croncmd = (
329+ '/usr/local/lib/nagios/plugins/check_exit_status.pl '
330+ '-s /etc/init.d/%s status' % svc
331+ )
332+ cron_file = '*/5 * * * * root %s > %s\n' % (croncmd, checkpath)
333 f = open(cronpath, 'w')
334 f.write(cron_file)
335 f.close()
336 nrpe.add_check(
337 shortname=svc,
338- description='process check {%s}' % unit_name,
339- check_cmd='check_status_file.py -f '
340- '/var/lib/nagios/service-check-%s.txt' % svc,
341+ description='service check {%s}' % unit_name,
342+ check_cmd='check_status_file.py -f %s' % checkpath,
343 )
344+ if immediate_check:
345+ f = open(checkpath, 'w')
346+ subprocess.call(
347+ croncmd.split(),
348+ stdout=f,
349+ stderr=subprocess.STDOUT
350+ )
351+ f.close()
352+ os.chmod(checkpath, 0o644)
353
354
355 def copy_nrpe_checks():
356
357=== modified file 'charmhelpers/contrib/charmsupport/volumes.py'
358--- charmhelpers/contrib/charmsupport/volumes.py 2015-09-28 20:09:02 +0000
359+++ charmhelpers/contrib/charmsupport/volumes.py 2017-03-04 02:21:16 +0000
360@@ -1,18 +1,16 @@
361 # Copyright 2014-2015 Canonical Limited.
362 #
363-# This file is part of charm-helpers.
364-#
365-# charm-helpers is free software: you can redistribute it and/or modify
366-# it under the terms of the GNU Lesser General Public License version 3 as
367-# published by the Free Software Foundation.
368-#
369-# charm-helpers is distributed in the hope that it will be useful,
370-# but WITHOUT ANY WARRANTY; without even the implied warranty of
371-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
372-# GNU Lesser General Public License for more details.
373-#
374-# You should have received a copy of the GNU Lesser General Public License
375-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
376+# Licensed under the Apache License, Version 2.0 (the "License");
377+# you may not use this file except in compliance with the License.
378+# You may obtain a copy of the License at
379+#
380+# http://www.apache.org/licenses/LICENSE-2.0
381+#
382+# Unless required by applicable law or agreed to in writing, software
383+# distributed under the License is distributed on an "AS IS" BASIS,
384+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
385+# See the License for the specific language governing permissions and
386+# limitations under the License.
387
388 '''
389 Functions for managing volumes in juju units. One volume is supported per unit.
390
391=== modified file 'charmhelpers/contrib/hahelpers/__init__.py'
392--- charmhelpers/contrib/hahelpers/__init__.py 2015-09-28 20:09:02 +0000
393+++ charmhelpers/contrib/hahelpers/__init__.py 2017-03-04 02:21:16 +0000
394@@ -1,15 +1,13 @@
395 # Copyright 2014-2015 Canonical Limited.
396 #
397-# This file is part of charm-helpers.
398-#
399-# charm-helpers is free software: you can redistribute it and/or modify
400-# it under the terms of the GNU Lesser General Public License version 3 as
401-# published by the Free Software Foundation.
402-#
403-# charm-helpers is distributed in the hope that it will be useful,
404-# but WITHOUT ANY WARRANTY; without even the implied warranty of
405-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
406-# GNU Lesser General Public License for more details.
407-#
408-# You should have received a copy of the GNU Lesser General Public License
409-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
410+# Licensed under the Apache License, Version 2.0 (the "License");
411+# you may not use this file except in compliance with the License.
412+# You may obtain a copy of the License at
413+#
414+# http://www.apache.org/licenses/LICENSE-2.0
415+#
416+# Unless required by applicable law or agreed to in writing, software
417+# distributed under the License is distributed on an "AS IS" BASIS,
418+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
419+# See the License for the specific language governing permissions and
420+# limitations under the License.
421
422=== modified file 'charmhelpers/contrib/hahelpers/apache.py'
423--- charmhelpers/contrib/hahelpers/apache.py 2015-09-28 20:09:02 +0000
424+++ charmhelpers/contrib/hahelpers/apache.py 2017-03-04 02:21:16 +0000
425@@ -1,18 +1,16 @@
426 # Copyright 2014-2015 Canonical Limited.
427 #
428-# This file is part of charm-helpers.
429-#
430-# charm-helpers is free software: you can redistribute it and/or modify
431-# it under the terms of the GNU Lesser General Public License version 3 as
432-# published by the Free Software Foundation.
433-#
434-# charm-helpers is distributed in the hope that it will be useful,
435-# but WITHOUT ANY WARRANTY; without even the implied warranty of
436-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
437-# GNU Lesser General Public License for more details.
438-#
439-# You should have received a copy of the GNU Lesser General Public License
440-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
441+# Licensed under the Apache License, Version 2.0 (the "License");
442+# you may not use this file except in compliance with the License.
443+# You may obtain a copy of the License at
444+#
445+# http://www.apache.org/licenses/LICENSE-2.0
446+#
447+# Unless required by applicable law or agreed to in writing, software
448+# distributed under the License is distributed on an "AS IS" BASIS,
449+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
450+# See the License for the specific language governing permissions and
451+# limitations under the License.
452
453 #
454 # Copyright 2012 Canonical Ltd.
455@@ -24,6 +22,7 @@
456 # Adam Gandelman <adamg@ubuntu.com>
457 #
458
459+import os
460 import subprocess
461
462 from charmhelpers.core.hookenv import (
463@@ -74,9 +73,23 @@
464 return ca_cert
465
466
467+def retrieve_ca_cert(cert_file):
468+ cert = None
469+ if os.path.isfile(cert_file):
470+ with open(cert_file, 'r') as crt:
471+ cert = crt.read()
472+ return cert
473+
474+
475 def install_ca_cert(ca_cert):
476 if ca_cert:
477- with open('/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt',
478- 'w') as crt:
479- crt.write(ca_cert)
480- subprocess.check_call(['update-ca-certificates', '--fresh'])
481+ cert_file = ('/usr/local/share/ca-certificates/'
482+ 'keystone_juju_ca_cert.crt')
483+ old_cert = retrieve_ca_cert(cert_file)
484+ if old_cert and old_cert == ca_cert:
485+ log("CA cert is the same as installed version", level=INFO)
486+ else:
487+ log("Installing new CA cert", level=INFO)
488+ with open(cert_file, 'w') as crt:
489+ crt.write(ca_cert)
490+ subprocess.check_call(['update-ca-certificates', '--fresh'])
491
492=== modified file 'charmhelpers/contrib/hahelpers/cluster.py'
493--- charmhelpers/contrib/hahelpers/cluster.py 2015-09-28 20:09:02 +0000
494+++ charmhelpers/contrib/hahelpers/cluster.py 2017-03-04 02:21:16 +0000
495@@ -1,18 +1,16 @@
496 # Copyright 2014-2015 Canonical Limited.
497 #
498-# This file is part of charm-helpers.
499-#
500-# charm-helpers is free software: you can redistribute it and/or modify
501-# it under the terms of the GNU Lesser General Public License version 3 as
502-# published by the Free Software Foundation.
503-#
504-# charm-helpers is distributed in the hope that it will be useful,
505-# but WITHOUT ANY WARRANTY; without even the implied warranty of
506-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
507-# GNU Lesser General Public License for more details.
508-#
509-# You should have received a copy of the GNU Lesser General Public License
510-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
511+# Licensed under the Apache License, Version 2.0 (the "License");
512+# you may not use this file except in compliance with the License.
513+# You may obtain a copy of the License at
514+#
515+# http://www.apache.org/licenses/LICENSE-2.0
516+#
517+# Unless required by applicable law or agreed to in writing, software
518+# distributed under the License is distributed on an "AS IS" BASIS,
519+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
520+# See the License for the specific language governing permissions and
521+# limitations under the License.
522
523 #
524 # Copyright 2012 Canonical Ltd.
525@@ -41,10 +39,11 @@
526 relation_get,
527 config as config_get,
528 INFO,
529- ERROR,
530+ DEBUG,
531 WARNING,
532 unit_get,
533- is_leader as juju_is_leader
534+ is_leader as juju_is_leader,
535+ status_set,
536 )
537 from charmhelpers.core.decorators import (
538 retry_on_exception,
539@@ -60,6 +59,10 @@
540 pass
541
542
543+class HAIncorrectConfig(Exception):
544+ pass
545+
546+
547 class CRMResourceNotFound(Exception):
548 pass
549
550@@ -274,27 +277,71 @@
551 Obtains all relevant configuration from charm configuration required
552 for initiating a relation to hacluster:
553
554- ha-bindiface, ha-mcastport, vip
555+ ha-bindiface, ha-mcastport, vip, os-internal-hostname,
556+ os-admin-hostname, os-public-hostname, os-access-hostname
557
558 param: exclude_keys: list of setting key(s) to be excluded.
559 returns: dict: A dict containing settings keyed by setting name.
560- raises: HAIncompleteConfig if settings are missing.
561+ raises: HAIncompleteConfig if settings are missing or incorrect.
562 '''
563- settings = ['ha-bindiface', 'ha-mcastport', 'vip']
564+ settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'os-internal-hostname',
565+ 'os-admin-hostname', 'os-public-hostname', 'os-access-hostname']
566 conf = {}
567 for setting in settings:
568 if exclude_keys and setting in exclude_keys:
569 continue
570
571 conf[setting] = config_get(setting)
572- missing = []
573- [missing.append(s) for s, v in six.iteritems(conf) if v is None]
574- if missing:
575- log('Insufficient config data to configure hacluster.', level=ERROR)
576- raise HAIncompleteConfig
577+
578+ if not valid_hacluster_config():
579+ raise HAIncorrectConfig('Insufficient or incorrect config data to '
580+ 'configure hacluster.')
581 return conf
582
583
584+def valid_hacluster_config():
585+ '''
586+ Check that either vip or dns-ha is set. If dns-ha then one of os-*-hostname
587+ must be set.
588+
589+ Note: ha-bindiface and ha-macastport both have defaults and will always
590+ be set. We only care that either vip or dns-ha is set.
591+
592+ :returns: boolean: valid config returns true.
593+ raises: HAIncompatibileConfig if settings conflict.
594+ raises: HAIncompleteConfig if settings are missing.
595+ '''
596+ vip = config_get('vip')
597+ dns = config_get('dns-ha')
598+ if not(bool(vip) ^ bool(dns)):
599+ msg = ('HA: Either vip or dns-ha must be set but not both in order to '
600+ 'use high availability')
601+ status_set('blocked', msg)
602+ raise HAIncorrectConfig(msg)
603+
604+ # If dns-ha then one of os-*-hostname must be set
605+ if dns:
606+ dns_settings = ['os-internal-hostname', 'os-admin-hostname',
607+ 'os-public-hostname', 'os-access-hostname']
608+ # At this point it is unknown if one or all of the possible
609+ # network spaces are in HA. Validate at least one is set which is
610+ # the minimum required.
611+ for setting in dns_settings:
612+ if config_get(setting):
613+ log('DNS HA: At least one hostname is set {}: {}'
614+ ''.format(setting, config_get(setting)),
615+ level=DEBUG)
616+ return True
617+
618+ msg = ('DNS HA: At least one os-*-hostname(s) must be set to use '
619+ 'DNS HA')
620+ status_set('blocked', msg)
621+ raise HAIncompleteConfig(msg)
622+
623+ log('VIP HA: VIP is set {}'.format(vip), level=DEBUG)
624+ return True
625+
626+
627 def canonical_url(configs, vip_setting='vip'):
628 '''
629 Returns the correct HTTP URL to this host given the state of HTTPS
630
631=== modified file 'charmhelpers/contrib/network/__init__.py'
632--- charmhelpers/contrib/network/__init__.py 2015-09-28 20:09:02 +0000
633+++ charmhelpers/contrib/network/__init__.py 2017-03-04 02:21:16 +0000
634@@ -1,15 +1,13 @@
635 # Copyright 2014-2015 Canonical Limited.
636 #
637-# This file is part of charm-helpers.
638-#
639-# charm-helpers is free software: you can redistribute it and/or modify
640-# it under the terms of the GNU Lesser General Public License version 3 as
641-# published by the Free Software Foundation.
642-#
643-# charm-helpers is distributed in the hope that it will be useful,
644-# but WITHOUT ANY WARRANTY; without even the implied warranty of
645-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
646-# GNU Lesser General Public License for more details.
647-#
648-# You should have received a copy of the GNU Lesser General Public License
649-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
650+# Licensed under the Apache License, Version 2.0 (the "License");
651+# you may not use this file except in compliance with the License.
652+# You may obtain a copy of the License at
653+#
654+# http://www.apache.org/licenses/LICENSE-2.0
655+#
656+# Unless required by applicable law or agreed to in writing, software
657+# distributed under the License is distributed on an "AS IS" BASIS,
658+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
659+# See the License for the specific language governing permissions and
660+# limitations under the License.
661
662=== modified file 'charmhelpers/contrib/network/ip.py'
663--- charmhelpers/contrib/network/ip.py 2015-09-28 20:09:02 +0000
664+++ charmhelpers/contrib/network/ip.py 2017-03-04 02:21:16 +0000
665@@ -1,18 +1,16 @@
666 # Copyright 2014-2015 Canonical Limited.
667 #
668-# This file is part of charm-helpers.
669-#
670-# charm-helpers is free software: you can redistribute it and/or modify
671-# it under the terms of the GNU Lesser General Public License version 3 as
672-# published by the Free Software Foundation.
673-#
674-# charm-helpers is distributed in the hope that it will be useful,
675-# but WITHOUT ANY WARRANTY; without even the implied warranty of
676-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
677-# GNU Lesser General Public License for more details.
678-#
679-# You should have received a copy of the GNU Lesser General Public License
680-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
681+# Licensed under the Apache License, Version 2.0 (the "License");
682+# you may not use this file except in compliance with the License.
683+# You may obtain a copy of the License at
684+#
685+# http://www.apache.org/licenses/LICENSE-2.0
686+#
687+# Unless required by applicable law or agreed to in writing, software
688+# distributed under the License is distributed on an "AS IS" BASIS,
689+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
690+# See the License for the specific language governing permissions and
691+# limitations under the License.
692
693 import glob
694 import re
695@@ -33,14 +31,20 @@
696 import netifaces
697 except ImportError:
698 apt_update(fatal=True)
699- apt_install('python-netifaces', fatal=True)
700+ if six.PY2:
701+ apt_install('python-netifaces', fatal=True)
702+ else:
703+ apt_install('python3-netifaces', fatal=True)
704 import netifaces
705
706 try:
707 import netaddr
708 except ImportError:
709 apt_update(fatal=True)
710- apt_install('python-netaddr', fatal=True)
711+ if six.PY2:
712+ apt_install('python-netaddr', fatal=True)
713+ else:
714+ apt_install('python3-netaddr', fatal=True)
715 import netaddr
716
717
718@@ -53,7 +57,7 @@
719
720
721 def no_ip_found_error_out(network):
722- errmsg = ("No IP address found in network: %s" % network)
723+ errmsg = ("No IP address found in network(s): %s" % network)
724 raise ValueError(errmsg)
725
726
727@@ -61,7 +65,7 @@
728 """Get an IPv4 or IPv6 address within the network from the host.
729
730 :param network (str): CIDR presentation format. For example,
731- '192.168.1.0/24'.
732+ '192.168.1.0/24'. Supports multiple networks as a space-delimited list.
733 :param fallback (str): If no address is found, return fallback.
734 :param fatal (boolean): If no address is found, fallback is not
735 set and fatal is True then exit(1).
736@@ -75,24 +79,26 @@
737 else:
738 return None
739
740- _validate_cidr(network)
741- network = netaddr.IPNetwork(network)
742- for iface in netifaces.interfaces():
743- addresses = netifaces.ifaddresses(iface)
744- if network.version == 4 and netifaces.AF_INET in addresses:
745- addr = addresses[netifaces.AF_INET][0]['addr']
746- netmask = addresses[netifaces.AF_INET][0]['netmask']
747- cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
748- if cidr in network:
749- return str(cidr.ip)
750+ networks = network.split() or [network]
751+ for network in networks:
752+ _validate_cidr(network)
753+ network = netaddr.IPNetwork(network)
754+ for iface in netifaces.interfaces():
755+ addresses = netifaces.ifaddresses(iface)
756+ if network.version == 4 and netifaces.AF_INET in addresses:
757+ addr = addresses[netifaces.AF_INET][0]['addr']
758+ netmask = addresses[netifaces.AF_INET][0]['netmask']
759+ cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
760+ if cidr in network:
761+ return str(cidr.ip)
762
763- if network.version == 6 and netifaces.AF_INET6 in addresses:
764- for addr in addresses[netifaces.AF_INET6]:
765- if not addr['addr'].startswith('fe80'):
766- cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
767- addr['netmask']))
768- if cidr in network:
769- return str(cidr.ip)
770+ if network.version == 6 and netifaces.AF_INET6 in addresses:
771+ for addr in addresses[netifaces.AF_INET6]:
772+ if not addr['addr'].startswith('fe80'):
773+ cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
774+ addr['netmask']))
775+ if cidr in network:
776+ return str(cidr.ip)
777
778 if fallback is not None:
779 return fallback
780@@ -189,6 +195,15 @@
781 get_netmask_for_address = partial(_get_for_address, key='netmask')
782
783
784+def resolve_network_cidr(ip_address):
785+ '''
786+ Resolves the full address cidr of an ip_address based on
787+ configured network interfaces
788+ '''
789+ netmask = get_netmask_for_address(ip_address)
790+ return str(netaddr.IPNetwork("%s/%s" % (ip_address, netmask)).cidr)
791+
792+
793 def format_ipv6_addr(address):
794 """If address is IPv6, wrap it in '[]' otherwise return None.
795
796@@ -203,7 +218,16 @@
797
798 def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,
799 fatal=True, exc_list=None):
800- """Return the assigned IP address for a given interface, if any."""
801+ """Return the assigned IP address for a given interface, if any.
802+
803+ :param iface: network interface on which address(es) are expected to
804+ be found.
805+ :param inet_type: inet address family
806+ :param inc_aliases: include alias interfaces in search
807+ :param fatal: if True, raise exception if address not found
808+ :param exc_list: list of addresses to ignore
809+ :return: list of ip addresses
810+ """
811 # Extract nic if passed /dev/ethX
812 if '/' in iface:
813 iface = iface.split('/')[-1]
814@@ -304,6 +328,14 @@
815 We currently only support scope global IPv6 addresses i.e. non-temporary
816 addresses. If no global IPv6 address is found, return the first one found
817 in the ipv6 address list.
818+
819+ :param iface: network interface on which ipv6 address(es) are expected to
820+ be found.
821+ :param inc_aliases: include alias interfaces in search
822+ :param fatal: if True, raise exception if address not found
823+ :param exc_list: list of addresses to ignore
824+ :param dynamic_only: only recognise dynamic addresses
825+ :return: list of ipv6 addresses
826 """
827 addresses = get_iface_addr(iface=iface, inet_type='AF_INET6',
828 inc_aliases=inc_aliases, fatal=fatal,
829@@ -325,7 +357,7 @@
830 cmd = ['ip', 'addr', 'show', iface]
831 out = subprocess.check_output(cmd).decode('UTF-8')
832 if dynamic_only:
833- key = re.compile("inet6 (.+)/[0-9]+ scope global dynamic.*")
834+ key = re.compile("inet6 (.+)/[0-9]+ scope global.* dynamic.*")
835 else:
836 key = re.compile("inet6 (.+)/[0-9]+ scope global.*")
837
838@@ -377,10 +409,10 @@
839 Returns True if address is a valid IP address.
840 """
841 try:
842- # Test to see if already an IPv4 address
843- socket.inet_aton(address)
844+ # Test to see if already an IPv4/IPv6 address
845+ address = netaddr.IPAddress(address)
846 return True
847- except socket.error:
848+ except (netaddr.AddrFormatError, ValueError):
849 return False
850
851
852@@ -388,7 +420,10 @@
853 try:
854 import dns.resolver
855 except ImportError:
856- apt_install('python-dnspython')
857+ if six.PY2:
858+ apt_install('python-dnspython', fatal=True)
859+ else:
860+ apt_install('python3-dnspython', fatal=True)
861 import dns.resolver
862
863 if isinstance(address, dns.name.Name):
864@@ -398,7 +433,11 @@
865 else:
866 return None
867
868- answers = dns.resolver.query(address, rtype)
869+ try:
870+ answers = dns.resolver.query(address, rtype)
871+ except dns.resolver.NXDOMAIN:
872+ return None
873+
874 if answers:
875 return str(answers[0])
876 return None
877@@ -432,7 +471,10 @@
878 try:
879 import dns.reversename
880 except ImportError:
881- apt_install("python-dnspython")
882+ if six.PY2:
883+ apt_install("python-dnspython", fatal=True)
884+ else:
885+ apt_install("python3-dnspython", fatal=True)
886 import dns.reversename
887
888 rev = dns.reversename.from_address(address)
889@@ -454,3 +496,18 @@
890 return result
891 else:
892 return result.split('.')[0]
893+
894+
895+def port_has_listener(address, port):
896+ """
897+ Returns True if the address:port is open and being listened to,
898+ else False.
899+
900+ @param address: an IP address or hostname
901+ @param port: integer port
902+
903+ Note calls 'zc' via a subprocess shell
904+ """
905+ cmd = ['nc', '-z', address, str(port)]
906+ result = subprocess.call(cmd)
907+ return not(bool(result))
908
909=== modified file 'charmhelpers/contrib/openstack/__init__.py'
910--- charmhelpers/contrib/openstack/__init__.py 2015-09-28 20:09:02 +0000
911+++ charmhelpers/contrib/openstack/__init__.py 2017-03-04 02:21:16 +0000
912@@ -1,15 +1,13 @@
913 # Copyright 2014-2015 Canonical Limited.
914 #
915-# This file is part of charm-helpers.
916-#
917-# charm-helpers is free software: you can redistribute it and/or modify
918-# it under the terms of the GNU Lesser General Public License version 3 as
919-# published by the Free Software Foundation.
920-#
921-# charm-helpers is distributed in the hope that it will be useful,
922-# but WITHOUT ANY WARRANTY; without even the implied warranty of
923-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
924-# GNU Lesser General Public License for more details.
925-#
926-# You should have received a copy of the GNU Lesser General Public License
927-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
928+# Licensed under the Apache License, Version 2.0 (the "License");
929+# you may not use this file except in compliance with the License.
930+# You may obtain a copy of the License at
931+#
932+# http://www.apache.org/licenses/LICENSE-2.0
933+#
934+# Unless required by applicable law or agreed to in writing, software
935+# distributed under the License is distributed on an "AS IS" BASIS,
936+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
937+# See the License for the specific language governing permissions and
938+# limitations under the License.
939
940=== modified file 'charmhelpers/contrib/openstack/alternatives.py'
941--- charmhelpers/contrib/openstack/alternatives.py 2015-09-28 20:09:02 +0000
942+++ charmhelpers/contrib/openstack/alternatives.py 2017-03-04 02:21:16 +0000
943@@ -1,18 +1,16 @@
944 # Copyright 2014-2015 Canonical Limited.
945 #
946-# This file is part of charm-helpers.
947-#
948-# charm-helpers is free software: you can redistribute it and/or modify
949-# it under the terms of the GNU Lesser General Public License version 3 as
950-# published by the Free Software Foundation.
951-#
952-# charm-helpers is distributed in the hope that it will be useful,
953-# but WITHOUT ANY WARRANTY; without even the implied warranty of
954-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
955-# GNU Lesser General Public License for more details.
956-#
957-# You should have received a copy of the GNU Lesser General Public License
958-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
959+# Licensed under the Apache License, Version 2.0 (the "License");
960+# you may not use this file except in compliance with the License.
961+# You may obtain a copy of the License at
962+#
963+# http://www.apache.org/licenses/LICENSE-2.0
964+#
965+# Unless required by applicable law or agreed to in writing, software
966+# distributed under the License is distributed on an "AS IS" BASIS,
967+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
968+# See the License for the specific language governing permissions and
969+# limitations under the License.
970
971 ''' Helper for managing alternatives for file conflict resolution '''
972
973
974=== modified file 'charmhelpers/contrib/openstack/amulet/__init__.py'
975--- charmhelpers/contrib/openstack/amulet/__init__.py 2015-09-28 20:09:02 +0000
976+++ charmhelpers/contrib/openstack/amulet/__init__.py 2017-03-04 02:21:16 +0000
977@@ -1,15 +1,13 @@
978 # Copyright 2014-2015 Canonical Limited.
979 #
980-# This file is part of charm-helpers.
981-#
982-# charm-helpers is free software: you can redistribute it and/or modify
983-# it under the terms of the GNU Lesser General Public License version 3 as
984-# published by the Free Software Foundation.
985-#
986-# charm-helpers is distributed in the hope that it will be useful,
987-# but WITHOUT ANY WARRANTY; without even the implied warranty of
988-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
989-# GNU Lesser General Public License for more details.
990-#
991-# You should have received a copy of the GNU Lesser General Public License
992-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
993+# Licensed under the Apache License, Version 2.0 (the "License");
994+# you may not use this file except in compliance with the License.
995+# You may obtain a copy of the License at
996+#
997+# http://www.apache.org/licenses/LICENSE-2.0
998+#
999+# Unless required by applicable law or agreed to in writing, software
1000+# distributed under the License is distributed on an "AS IS" BASIS,
1001+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1002+# See the License for the specific language governing permissions and
1003+# limitations under the License.
1004
1005=== modified file 'charmhelpers/contrib/openstack/amulet/deployment.py'
1006--- charmhelpers/contrib/openstack/amulet/deployment.py 2015-09-28 20:09:02 +0000
1007+++ charmhelpers/contrib/openstack/amulet/deployment.py 2017-03-04 02:21:16 +0000
1008@@ -1,25 +1,29 @@
1009 # Copyright 2014-2015 Canonical Limited.
1010 #
1011-# This file is part of charm-helpers.
1012-#
1013-# charm-helpers is free software: you can redistribute it and/or modify
1014-# it under the terms of the GNU Lesser General Public License version 3 as
1015-# published by the Free Software Foundation.
1016-#
1017-# charm-helpers is distributed in the hope that it will be useful,
1018-# but WITHOUT ANY WARRANTY; without even the implied warranty of
1019-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1020-# GNU Lesser General Public License for more details.
1021-#
1022-# You should have received a copy of the GNU Lesser General Public License
1023-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
1024+# Licensed under the Apache License, Version 2.0 (the "License");
1025+# you may not use this file except in compliance with the License.
1026+# You may obtain a copy of the License at
1027+#
1028+# http://www.apache.org/licenses/LICENSE-2.0
1029+#
1030+# Unless required by applicable law or agreed to in writing, software
1031+# distributed under the License is distributed on an "AS IS" BASIS,
1032+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1033+# See the License for the specific language governing permissions and
1034+# limitations under the License.
1035
1036+import logging
1037+import re
1038+import sys
1039 import six
1040 from collections import OrderedDict
1041 from charmhelpers.contrib.amulet.deployment import (
1042 AmuletDeployment
1043 )
1044
1045+DEBUG = logging.DEBUG
1046+ERROR = logging.ERROR
1047+
1048
1049 class OpenStackAmuletDeployment(AmuletDeployment):
1050 """OpenStack amulet deployment.
1051@@ -28,15 +32,31 @@
1052 that is specifically for use by OpenStack charms.
1053 """
1054
1055- def __init__(self, series=None, openstack=None, source=None, stable=True):
1056+ def __init__(self, series=None, openstack=None, source=None,
1057+ stable=True, log_level=DEBUG):
1058 """Initialize the deployment environment."""
1059 super(OpenStackAmuletDeployment, self).__init__(series)
1060+ self.log = self.get_logger(level=log_level)
1061+ self.log.info('OpenStackAmuletDeployment: init')
1062 self.openstack = openstack
1063 self.source = source
1064 self.stable = stable
1065- # Note(coreycb): this needs to be changed when new next branches come
1066- # out.
1067- self.current_next = "trusty"
1068+
1069+ def get_logger(self, name="deployment-logger", level=logging.DEBUG):
1070+ """Get a logger object that will log to stdout."""
1071+ log = logging
1072+ logger = log.getLogger(name)
1073+ fmt = log.Formatter("%(asctime)s %(funcName)s "
1074+ "%(levelname)s: %(message)s")
1075+
1076+ handler = log.StreamHandler(stream=sys.stdout)
1077+ handler.setLevel(level)
1078+ handler.setFormatter(fmt)
1079+
1080+ logger.addHandler(handler)
1081+ logger.setLevel(level)
1082+
1083+ return logger
1084
1085 def _determine_branch_locations(self, other_services):
1086 """Determine the branch locations for the other services.
1087@@ -45,43 +65,82 @@
1088 stable or next (dev) branch, and based on this, use the corresonding
1089 stable or next branches for the other_services."""
1090
1091- # Charms outside the lp:~openstack-charmers namespace
1092- base_charms = ['mysql', 'mongodb', 'nrpe']
1093-
1094- # Force these charms to current series even when using an older series.
1095- # ie. Use trusty/nrpe even when series is precise, as the P charm
1096- # does not possess the necessary external master config and hooks.
1097- force_series_current = ['nrpe']
1098-
1099- if self.series in ['precise', 'trusty']:
1100- base_series = self.series
1101- else:
1102- base_series = self.current_next
1103+ self.log.info('OpenStackAmuletDeployment: determine branch locations')
1104+
1105+ # Charms outside the ~openstack-charmers
1106+ base_charms = {
1107+ 'mysql': ['trusty'],
1108+ 'mongodb': ['trusty'],
1109+ 'nrpe': ['trusty', 'xenial'],
1110+ }
1111
1112 for svc in other_services:
1113- if svc['name'] in force_series_current:
1114- base_series = self.current_next
1115 # If a location has been explicitly set, use it
1116 if svc.get('location'):
1117 continue
1118- if self.stable:
1119- temp = 'lp:charms/{}/{}'
1120- svc['location'] = temp.format(base_series,
1121- svc['name'])
1122+ if svc['name'] in base_charms:
1123+ # NOTE: not all charms have support for all series we
1124+ # want/need to test against, so fix to most recent
1125+ # that each base charm supports
1126+ target_series = self.series
1127+ if self.series not in base_charms[svc['name']]:
1128+ target_series = base_charms[svc['name']][-1]
1129+ svc['location'] = 'cs:{}/{}'.format(target_series,
1130+ svc['name'])
1131+ elif self.stable:
1132+ svc['location'] = 'cs:{}/{}'.format(self.series,
1133+ svc['name'])
1134 else:
1135- if svc['name'] in base_charms:
1136- temp = 'lp:charms/{}/{}'
1137- svc['location'] = temp.format(base_series,
1138- svc['name'])
1139- else:
1140- temp = 'lp:~openstack-charmers/charms/{}/{}/next'
1141- svc['location'] = temp.format(self.current_next,
1142- svc['name'])
1143+ svc['location'] = 'cs:~openstack-charmers-next/{}/{}'.format(
1144+ self.series,
1145+ svc['name']
1146+ )
1147
1148 return other_services
1149
1150- def _add_services(self, this_service, other_services):
1151- """Add services to the deployment and set openstack-origin/source."""
1152+ def _add_services(self, this_service, other_services, use_source=None,
1153+ no_origin=None):
1154+ """Add services to the deployment and optionally set
1155+ openstack-origin/source.
1156+
1157+ :param this_service dict: Service dictionary describing the service
1158+ whose amulet tests are being run
1159+ :param other_services dict: List of service dictionaries describing
1160+ the services needed to support the target
1161+ service
1162+ :param use_source list: List of services which use the 'source' config
1163+ option rather than 'openstack-origin'
1164+ :param no_origin list: List of services which do not support setting
1165+ the Cloud Archive.
1166+ Service Dict:
1167+ {
1168+ 'name': str charm-name,
1169+ 'units': int number of units,
1170+ 'constraints': dict of juju constraints,
1171+ 'location': str location of charm,
1172+ }
1173+ eg
1174+ this_service = {
1175+ 'name': 'openvswitch-odl',
1176+ 'constraints': {'mem': '8G'},
1177+ }
1178+ other_services = [
1179+ {
1180+ 'name': 'nova-compute',
1181+ 'units': 2,
1182+ 'constraints': {'mem': '4G'},
1183+ 'location': cs:~bob/xenial/nova-compute
1184+ },
1185+ {
1186+ 'name': 'mysql',
1187+ 'constraints': {'mem': '2G'},
1188+ },
1189+ {'neutron-api-odl'}]
1190+ use_source = ['mysql']
1191+ no_origin = ['neutron-api-odl']
1192+ """
1193+ self.log.info('OpenStackAmuletDeployment: adding services')
1194+
1195 other_services = self._determine_branch_locations(other_services)
1196
1197 super(OpenStackAmuletDeployment, self)._add_services(this_service,
1198@@ -90,12 +149,22 @@
1199 services = other_services
1200 services.append(this_service)
1201
1202+ use_source = use_source or []
1203+ no_origin = no_origin or []
1204+
1205 # Charms which should use the source config option
1206- use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
1207- 'ceph-osd', 'ceph-radosgw']
1208+ use_source = list(set(
1209+ use_source + ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
1210+ 'ceph-osd', 'ceph-radosgw', 'ceph-mon',
1211+ 'ceph-proxy', 'percona-cluster', 'lxd']))
1212
1213 # Charms which can not use openstack-origin, ie. many subordinates
1214- no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
1215+ no_origin = list(set(
1216+ no_origin + ['cinder-ceph', 'hacluster', 'neutron-openvswitch',
1217+ 'nrpe', 'openvswitch-odl', 'neutron-api-odl',
1218+ 'odl-controller', 'cinder-backup', 'nexentaedge-data',
1219+ 'nexentaedge-iscsi-gw', 'nexentaedge-swift-gw',
1220+ 'cinder-nexentaedge', 'nexentaedge-mgmt']))
1221
1222 if self.openstack:
1223 for svc in services:
1224@@ -111,9 +180,79 @@
1225
1226 def _configure_services(self, configs):
1227 """Configure all of the services."""
1228+ self.log.info('OpenStackAmuletDeployment: configure services')
1229 for service, config in six.iteritems(configs):
1230 self.d.configure(service, config)
1231
1232+ def _auto_wait_for_status(self, message=None, exclude_services=None,
1233+ include_only=None, timeout=1800):
1234+ """Wait for all units to have a specific extended status, except
1235+ for any defined as excluded. Unless specified via message, any
1236+ status containing any case of 'ready' will be considered a match.
1237+
1238+ Examples of message usage:
1239+
1240+ Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
1241+ message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
1242+
1243+ Wait for all units to reach this status (exact match):
1244+ message = re.compile('^Unit is ready and clustered$')
1245+
1246+ Wait for all units to reach any one of these (exact match):
1247+ message = re.compile('Unit is ready|OK|Ready')
1248+
1249+ Wait for at least one unit to reach this status (exact match):
1250+ message = {'ready'}
1251+
1252+ See Amulet's sentry.wait_for_messages() for message usage detail.
1253+ https://github.com/juju/amulet/blob/master/amulet/sentry.py
1254+
1255+ :param message: Expected status match
1256+ :param exclude_services: List of juju service names to ignore,
1257+ not to be used in conjuction with include_only.
1258+ :param include_only: List of juju service names to exclusively check,
1259+ not to be used in conjuction with exclude_services.
1260+ :param timeout: Maximum time in seconds to wait for status match
1261+ :returns: None. Raises if timeout is hit.
1262+ """
1263+ self.log.info('Waiting for extended status on units...')
1264+
1265+ all_services = self.d.services.keys()
1266+
1267+ if exclude_services and include_only:
1268+ raise ValueError('exclude_services can not be used '
1269+ 'with include_only')
1270+
1271+ if message:
1272+ if isinstance(message, re._pattern_type):
1273+ match = message.pattern
1274+ else:
1275+ match = message
1276+
1277+ self.log.debug('Custom extended status wait match: '
1278+ '{}'.format(match))
1279+ else:
1280+ self.log.debug('Default extended status wait match: contains '
1281+ 'READY (case-insensitive)')
1282+ message = re.compile('.*ready.*', re.IGNORECASE)
1283+
1284+ if exclude_services:
1285+ self.log.debug('Excluding services from extended status match: '
1286+ '{}'.format(exclude_services))
1287+ else:
1288+ exclude_services = []
1289+
1290+ if include_only:
1291+ services = include_only
1292+ else:
1293+ services = list(set(all_services) - set(exclude_services))
1294+
1295+ self.log.debug('Waiting up to {}s for extended status on services: '
1296+ '{}'.format(timeout, services))
1297+ service_messages = {service: message for service in services}
1298+ self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
1299+ self.log.info('OK')
1300+
1301 def _get_openstack_release(self):
1302 """Get openstack release.
1303
1304@@ -121,25 +260,21 @@
1305 release.
1306 """
1307 # Must be ordered by OpenStack release (not by Ubuntu release):
1308- (self.precise_essex, self.precise_folsom, self.precise_grizzly,
1309- self.precise_havana, self.precise_icehouse,
1310- self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
1311- self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
1312- self.wily_liberty) = range(12)
1313+ (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
1314+ self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
1315+ self.yakkety_newton, self.xenial_ocata, self.zesty_ocata) = range(9)
1316
1317 releases = {
1318- ('precise', None): self.precise_essex,
1319- ('precise', 'cloud:precise-folsom'): self.precise_folsom,
1320- ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
1321- ('precise', 'cloud:precise-havana'): self.precise_havana,
1322- ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
1323 ('trusty', None): self.trusty_icehouse,
1324- ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
1325 ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
1326 ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
1327- ('utopic', None): self.utopic_juno,
1328- ('vivid', None): self.vivid_kilo,
1329- ('wily', None): self.wily_liberty}
1330+ ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka,
1331+ ('xenial', None): self.xenial_mitaka,
1332+ ('xenial', 'cloud:xenial-newton'): self.xenial_newton,
1333+ ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
1334+ ('yakkety', None): self.yakkety_newton,
1335+ ('zesty', None): self.zesty_ocata,
1336+ }
1337 return releases[(self.series, self.openstack)]
1338
1339 def _get_openstack_release_string(self):
1340@@ -148,14 +283,10 @@
1341 Return a string representing the openstack release.
1342 """
1343 releases = OrderedDict([
1344- ('precise', 'essex'),
1345- ('quantal', 'folsom'),
1346- ('raring', 'grizzly'),
1347- ('saucy', 'havana'),
1348 ('trusty', 'icehouse'),
1349- ('utopic', 'juno'),
1350- ('vivid', 'kilo'),
1351- ('wily', 'liberty'),
1352+ ('xenial', 'mitaka'),
1353+ ('yakkety', 'newton'),
1354+ ('zesty', 'ocata'),
1355 ])
1356 if self.openstack:
1357 os_origin = self.openstack.split(':')[1]
1358
1359=== modified file 'charmhelpers/contrib/openstack/amulet/utils.py'
1360--- charmhelpers/contrib/openstack/amulet/utils.py 2015-09-28 20:38:07 +0000
1361+++ charmhelpers/contrib/openstack/amulet/utils.py 2017-03-04 02:21:16 +0000
1362@@ -1,42 +1,51 @@
1363 # Copyright 2014-2015 Canonical Limited.
1364 #
1365-# This file is part of charm-helpers.
1366-#
1367-# charm-helpers is free software: you can redistribute it and/or modify
1368-# it under the terms of the GNU Lesser General Public License version 3 as
1369-# published by the Free Software Foundation.
1370-#
1371-# charm-helpers is distributed in the hope that it will be useful,
1372-# but WITHOUT ANY WARRANTY; without even the implied warranty of
1373-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1374-# GNU Lesser General Public License for more details.
1375-#
1376-# You should have received a copy of the GNU Lesser General Public License
1377-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
1378+# Licensed under the Apache License, Version 2.0 (the "License");
1379+# you may not use this file except in compliance with the License.
1380+# You may obtain a copy of the License at
1381+#
1382+# http://www.apache.org/licenses/LICENSE-2.0
1383+#
1384+# Unless required by applicable law or agreed to in writing, software
1385+# distributed under the License is distributed on an "AS IS" BASIS,
1386+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1387+# See the License for the specific language governing permissions and
1388+# limitations under the License.
1389
1390 import amulet
1391 import json
1392 import logging
1393 import os
1394+import re
1395 import six
1396 import time
1397 import urllib
1398+import urlparse
1399
1400 import cinderclient.v1.client as cinder_client
1401 import glanceclient.v1.client as glance_client
1402 import heatclient.v1.client as heat_client
1403 import keystoneclient.v2_0 as keystone_client
1404-import novaclient.v1_1.client as nova_client
1405+from keystoneclient.auth.identity import v3 as keystone_id_v3
1406+from keystoneclient import session as keystone_session
1407+from keystoneclient.v3 import client as keystone_client_v3
1408+from novaclient import exceptions
1409+
1410+import novaclient.client as nova_client
1411+import novaclient
1412 import pika
1413 import swiftclient
1414
1415 from charmhelpers.contrib.amulet.utils import (
1416 AmuletUtils
1417 )
1418+from charmhelpers.core.decorators import retry_on_exception
1419
1420 DEBUG = logging.DEBUG
1421 ERROR = logging.ERROR
1422
1423+NOVA_CLIENT_VERSION = "2"
1424+
1425
1426 class OpenStackAmuletUtils(AmuletUtils):
1427 """OpenStack amulet utilities.
1428@@ -78,6 +87,56 @@
1429 if not found:
1430 return 'endpoint not found'
1431
1432+ def validate_v3_endpoint_data(self, endpoints, admin_port, internal_port,
1433+ public_port, expected):
1434+ """Validate keystone v3 endpoint data.
1435+
1436+ Validate the v3 endpoint data which has changed from v2. The
1437+ ports are used to find the matching endpoint.
1438+
1439+ The new v3 endpoint data looks like:
1440+
1441+ [<Endpoint enabled=True,
1442+ id=0432655fc2f74d1e9fa17bdaa6f6e60b,
1443+ interface=admin,
1444+ links={u'self': u'<RESTful URL of this endpoint>'},
1445+ region=RegionOne,
1446+ region_id=RegionOne,
1447+ service_id=17f842a0dc084b928e476fafe67e4095,
1448+ url=http://10.5.6.5:9312>,
1449+ <Endpoint enabled=True,
1450+ id=6536cb6cb92f4f41bf22b079935c7707,
1451+ interface=admin,
1452+ links={u'self': u'<RESTful url of this endpoint>'},
1453+ region=RegionOne,
1454+ region_id=RegionOne,
1455+ service_id=72fc8736fb41435e8b3584205bb2cfa3,
1456+ url=http://10.5.6.6:35357/v3>,
1457+ ... ]
1458+ """
1459+ self.log.debug('Validating v3 endpoint data...')
1460+ self.log.debug('actual: {}'.format(repr(endpoints)))
1461+ found = []
1462+ for ep in endpoints:
1463+ self.log.debug('endpoint: {}'.format(repr(ep)))
1464+ if ((admin_port in ep.url and ep.interface == 'admin') or
1465+ (internal_port in ep.url and ep.interface == 'internal') or
1466+ (public_port in ep.url and ep.interface == 'public')):
1467+ found.append(ep.interface)
1468+ # note we ignore the links member.
1469+ actual = {'id': ep.id,
1470+ 'region': ep.region,
1471+ 'region_id': ep.region_id,
1472+ 'interface': self.not_null,
1473+ 'url': ep.url,
1474+ 'service_id': ep.service_id, }
1475+ ret = self._validate_dict_data(expected, actual)
1476+ if ret:
1477+ return 'unexpected endpoint data - {}'.format(ret)
1478+
1479+ if len(found) != 3:
1480+ return 'Unexpected number of endpoints found'
1481+
1482 def validate_svc_catalog_endpoint_data(self, expected, actual):
1483 """Validate service catalog endpoint data.
1484
1485@@ -95,6 +154,72 @@
1486 return "endpoint {} does not exist".format(k)
1487 return ret
1488
1489+ def validate_v3_svc_catalog_endpoint_data(self, expected, actual):
1490+ """Validate the keystone v3 catalog endpoint data.
1491+
1492+ Validate a list of dictinaries that make up the keystone v3 service
1493+ catalogue.
1494+
1495+ It is in the form of:
1496+
1497+
1498+ {u'identity': [{u'id': u'48346b01c6804b298cdd7349aadb732e',
1499+ u'interface': u'admin',
1500+ u'region': u'RegionOne',
1501+ u'region_id': u'RegionOne',
1502+ u'url': u'http://10.5.5.224:35357/v3'},
1503+ {u'id': u'8414f7352a4b47a69fddd9dbd2aef5cf',
1504+ u'interface': u'public',
1505+ u'region': u'RegionOne',
1506+ u'region_id': u'RegionOne',
1507+ u'url': u'http://10.5.5.224:5000/v3'},
1508+ {u'id': u'd5ca31440cc24ee1bf625e2996fb6a5b',
1509+ u'interface': u'internal',
1510+ u'region': u'RegionOne',
1511+ u'region_id': u'RegionOne',
1512+ u'url': u'http://10.5.5.224:5000/v3'}],
1513+ u'key-manager': [{u'id': u'68ebc17df0b045fcb8a8a433ebea9e62',
1514+ u'interface': u'public',
1515+ u'region': u'RegionOne',
1516+ u'region_id': u'RegionOne',
1517+ u'url': u'http://10.5.5.223:9311'},
1518+ {u'id': u'9cdfe2a893c34afd8f504eb218cd2f9d',
1519+ u'interface': u'internal',
1520+ u'region': u'RegionOne',
1521+ u'region_id': u'RegionOne',
1522+ u'url': u'http://10.5.5.223:9311'},
1523+ {u'id': u'f629388955bc407f8b11d8b7ca168086',
1524+ u'interface': u'admin',
1525+ u'region': u'RegionOne',
1526+ u'region_id': u'RegionOne',
1527+ u'url': u'http://10.5.5.223:9312'}]}
1528+
1529+ Note, that an added complication is that the order of admin, public,
1530+ internal against 'interface' in each region.
1531+
1532+ Thus, the function sorts the expected and actual lists using the
1533+ interface key as a sort key, prior to the comparison.
1534+ """
1535+ self.log.debug('Validating v3 service catalog endpoint data...')
1536+ self.log.debug('actual: {}'.format(repr(actual)))
1537+ for k, v in six.iteritems(expected):
1538+ if k in actual:
1539+ l_expected = sorted(v, key=lambda x: x['interface'])
1540+ l_actual = sorted(actual[k], key=lambda x: x['interface'])
1541+ if len(l_actual) != len(l_expected):
1542+ return ("endpoint {} has differing number of interfaces "
1543+ " - expected({}), actual({})"
1544+ .format(k, len(l_expected), len(l_actual)))
1545+ for i_expected, i_actual in zip(l_expected, l_actual):
1546+ self.log.debug("checking interface {}"
1547+ .format(i_expected['interface']))
1548+ ret = self._validate_dict_data(i_expected, i_actual)
1549+ if ret:
1550+ return self.endpoint_error(k, ret)
1551+ else:
1552+ return "endpoint {} does not exist".format(k)
1553+ return ret
1554+
1555 def validate_tenant_data(self, expected, actual):
1556 """Validate tenant data.
1557
1558@@ -138,7 +263,7 @@
1559 return "role {} does not exist".format(e['name'])
1560 return ret
1561
1562- def validate_user_data(self, expected, actual):
1563+ def validate_user_data(self, expected, actual, api_version=None):
1564 """Validate user data.
1565
1566 Validate a list of actual user data vs a list of expected user
1567@@ -149,10 +274,15 @@
1568 for e in expected:
1569 found = False
1570 for act in actual:
1571- a = {'enabled': act.enabled, 'name': act.name,
1572- 'email': act.email, 'tenantId': act.tenantId,
1573- 'id': act.id}
1574- if e['name'] == a['name']:
1575+ if e['name'] == act.name:
1576+ a = {'enabled': act.enabled, 'name': act.name,
1577+ 'email': act.email, 'id': act.id}
1578+ if api_version == 3:
1579+ a['default_project_id'] = getattr(act,
1580+ 'default_project_id',
1581+ 'none')
1582+ else:
1583+ a['tenantId'] = act.tenantId
1584 found = True
1585 ret = self._validate_dict_data(e, a)
1586 if ret:
1587@@ -176,34 +306,115 @@
1588 self.log.debug('Checking if tenant exists ({})...'.format(tenant))
1589 return tenant in [t.name for t in keystone.tenants.list()]
1590
1591+ @retry_on_exception(5, base_delay=10)
1592+ def keystone_wait_for_propagation(self, sentry_relation_pairs,
1593+ api_version):
1594+ """Iterate over list of sentry and relation tuples and verify that
1595+ api_version has the expected value.
1596+
1597+ :param sentry_relation_pairs: list of sentry, relation name tuples used
1598+ for monitoring propagation of relation
1599+ data
1600+ :param api_version: api_version to expect in relation data
1601+ :returns: None if successful. Raise on error.
1602+ """
1603+ for (sentry, relation_name) in sentry_relation_pairs:
1604+ rel = sentry.relation('identity-service',
1605+ relation_name)
1606+ self.log.debug('keystone relation data: {}'.format(rel))
1607+ if rel['api_version'] != str(api_version):
1608+ raise Exception("api_version not propagated through relation"
1609+ " data yet ('{}' != '{}')."
1610+ "".format(rel['api_version'], api_version))
1611+
1612+ def keystone_configure_api_version(self, sentry_relation_pairs, deployment,
1613+ api_version):
1614+ """Configure preferred-api-version of keystone in deployment and
1615+ monitor provided list of relation objects for propagation
1616+ before returning to caller.
1617+
1618+ :param sentry_relation_pairs: list of sentry, relation tuples used for
1619+ monitoring propagation of relation data
1620+ :param deployment: deployment to configure
1621+ :param api_version: value preferred-api-version will be set to
1622+ :returns: None if successful. Raise on error.
1623+ """
1624+ self.log.debug("Setting keystone preferred-api-version: '{}'"
1625+ "".format(api_version))
1626+
1627+ config = {'preferred-api-version': api_version}
1628+ deployment.d.configure('keystone', config)
1629+ self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
1630+
1631 def authenticate_cinder_admin(self, keystone_sentry, username,
1632 password, tenant):
1633 """Authenticates admin user with cinder."""
1634 # NOTE(beisner): cinder python client doesn't accept tokens.
1635- service_ip = \
1636- keystone_sentry.relation('shared-db',
1637- 'mysql:shared-db')['private-address']
1638- ept = "http://{}:5000/v2.0".format(service_ip.strip().decode('utf-8'))
1639+ keystone_ip = keystone_sentry.info['public-address']
1640+ ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8'))
1641 return cinder_client.Client(username, password, tenant, ept)
1642
1643+ def authenticate_keystone(self, keystone_ip, username, password,
1644+ api_version=False, admin_port=False,
1645+ user_domain_name=None, domain_name=None,
1646+ project_domain_name=None, project_name=None):
1647+ """Authenticate with Keystone"""
1648+ self.log.debug('Authenticating with keystone...')
1649+ port = 5000
1650+ if admin_port:
1651+ port = 35357
1652+ base_ep = "http://{}:{}".format(keystone_ip.strip().decode('utf-8'),
1653+ port)
1654+ if not api_version or api_version == 2:
1655+ ep = base_ep + "/v2.0"
1656+ return keystone_client.Client(username=username, password=password,
1657+ tenant_name=project_name,
1658+ auth_url=ep)
1659+ else:
1660+ ep = base_ep + "/v3"
1661+ auth = keystone_id_v3.Password(
1662+ user_domain_name=user_domain_name,
1663+ username=username,
1664+ password=password,
1665+ domain_name=domain_name,
1666+ project_domain_name=project_domain_name,
1667+ project_name=project_name,
1668+ auth_url=ep
1669+ )
1670+ return keystone_client_v3.Client(
1671+ session=keystone_session.Session(auth=auth)
1672+ )
1673+
1674 def authenticate_keystone_admin(self, keystone_sentry, user, password,
1675- tenant):
1676+ tenant=None, api_version=None,
1677+ keystone_ip=None):
1678 """Authenticates admin user with the keystone admin endpoint."""
1679 self.log.debug('Authenticating keystone admin...')
1680- unit = keystone_sentry
1681- service_ip = unit.relation('shared-db',
1682- 'mysql:shared-db')['private-address']
1683- ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
1684- return keystone_client.Client(username=user, password=password,
1685- tenant_name=tenant, auth_url=ep)
1686+ if not keystone_ip:
1687+ keystone_ip = keystone_sentry.info['public-address']
1688+
1689+ user_domain_name = None
1690+ domain_name = None
1691+ if api_version == 3:
1692+ user_domain_name = 'admin_domain'
1693+ domain_name = user_domain_name
1694+
1695+ return self.authenticate_keystone(keystone_ip, user, password,
1696+ project_name=tenant,
1697+ api_version=api_version,
1698+ user_domain_name=user_domain_name,
1699+ domain_name=domain_name,
1700+ admin_port=True)
1701
1702 def authenticate_keystone_user(self, keystone, user, password, tenant):
1703 """Authenticates a regular user with the keystone public endpoint."""
1704 self.log.debug('Authenticating keystone user ({})...'.format(user))
1705 ep = keystone.service_catalog.url_for(service_type='identity',
1706 endpoint_type='publicURL')
1707- return keystone_client.Client(username=user, password=password,
1708- tenant_name=tenant, auth_url=ep)
1709+ keystone_ip = urlparse.urlparse(ep).hostname
1710+
1711+ return self.authenticate_keystone(keystone_ip, user, password,
1712+ project_name=tenant)
1713
1714 def authenticate_glance_admin(self, keystone):
1715 """Authenticates admin user with glance."""
1716@@ -224,8 +435,14 @@
1717 self.log.debug('Authenticating nova user ({})...'.format(user))
1718 ep = keystone.service_catalog.url_for(service_type='identity',
1719 endpoint_type='publicURL')
1720- return nova_client.Client(username=user, api_key=password,
1721- project_id=tenant, auth_url=ep)
1722+ if novaclient.__version__[0] >= "7":
1723+ return nova_client.Client(NOVA_CLIENT_VERSION,
1724+ username=user, password=password,
1725+ project_name=tenant, auth_url=ep)
1726+ else:
1727+ return nova_client.Client(NOVA_CLIENT_VERSION,
1728+ username=user, api_key=password,
1729+ project_id=tenant, auth_url=ep)
1730
1731 def authenticate_swift_user(self, keystone, user, password, tenant):
1732 """Authenticates a regular user with swift api."""
1733@@ -238,6 +455,16 @@
1734 tenant_name=tenant,
1735 auth_version='2.0')
1736
1737+ def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto",
1738+ ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True):
1739+ """Create the specified flavor."""
1740+ try:
1741+ nova.flavors.find(name=name)
1742+ except (exceptions.NotFound, exceptions.NoUniqueMatch):
1743+ self.log.debug('Creating flavor ({})'.format(name))
1744+ nova.flavors.create(name, ram, vcpus, disk, flavorid,
1745+ ephemeral, swap, rxtx_factor, is_public)
1746+
1747 def create_cirros_image(self, glance, image_name):
1748 """Download the latest cirros image and upload it to glance,
1749 validate and return a resource pointer.
1750@@ -604,7 +831,22 @@
1751 '{}'.format(sample_type, samples))
1752 return None
1753
1754-# rabbitmq/amqp specific helpers:
1755+ # rabbitmq/amqp specific helpers:
1756+
1757+ def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
1758+ """Wait for rmq units extended status to show cluster readiness,
1759+ after an optional initial sleep period. Initial sleep is likely
1760+ necessary to be effective following a config change, as status
1761+ message may not instantly update to non-ready."""
1762+
1763+ if init_sleep:
1764+ time.sleep(init_sleep)
1765+
1766+ message = re.compile('^Unit is ready and clustered$')
1767+ deployment._auto_wait_for_status(message=message,
1768+ timeout=timeout,
1769+ include_only=['rabbitmq-server'])
1770+
1771 def add_rmq_test_user(self, sentry_units,
1772 username="testuser1", password="changeme"):
1773 """Add a test user via the first rmq juju unit, check connection as
1774@@ -805,7 +1047,10 @@
1775 if port:
1776 config['ssl_port'] = port
1777
1778- deployment.configure('rabbitmq-server', config)
1779+ deployment.d.configure('rabbitmq-server', config)
1780+
1781+ # Wait for unit status
1782+ self.rmq_wait_for_cluster(deployment)
1783
1784 # Confirm
1785 tries = 0
1786@@ -832,7 +1077,10 @@
1787
1788 # Disable RMQ SSL
1789 config = {'ssl': 'off'}
1790- deployment.configure('rabbitmq-server', config)
1791+ deployment.d.configure('rabbitmq-server', config)
1792+
1793+ # Wait for unit status
1794+ self.rmq_wait_for_cluster(deployment)
1795
1796 # Confirm
1797 tries = 0
1798@@ -881,7 +1129,8 @@
1799 retry_delay=5,
1800 socket_timeout=1)
1801 connection = pika.BlockingConnection(parameters)
1802- assert connection.server_properties['product'] == 'RabbitMQ'
1803+ assert connection.is_open is True
1804+ assert connection.is_closing is False
1805 self.log.debug('Connect OK')
1806 return connection
1807 except Exception as e:
1808@@ -961,3 +1210,70 @@
1809 else:
1810 msg = 'No message retrieved.'
1811 amulet.raise_status(amulet.FAIL, msg)
1812+
1813+ def validate_memcache(self, sentry_unit, conf, os_release,
1814+ earliest_release=5, section='keystone_authtoken',
1815+ check_kvs=None):
1816+ """Check Memcache is running and is configured to be used
1817+
1818+ Example call from Amulet test:
1819+
1820+ def test_110_memcache(self):
1821+ u.validate_memcache(self.neutron_api_sentry,
1822+ '/etc/neutron/neutron.conf',
1823+ self._get_openstack_release())
1824+
1825+ :param sentry_unit: sentry unit
1826+ :param conf: OpenStack config file to check memcache settings
1827+ :param os_release: Current OpenStack release int code
1828+ :param earliest_release: Earliest Openstack release to check int code
1829+ :param section: OpenStack config file section to check
1830+ :param check_kvs: Dict of settings to check in config file
1831+ :returns: None
1832+ """
1833+ if os_release < earliest_release:
1834+ self.log.debug('Skipping memcache checks for deployment. {} <'
1835+ 'mitaka'.format(os_release))
1836+ return
1837+ _kvs = check_kvs or {'memcached_servers': 'inet6:[::1]:11211'}
1838+ self.log.debug('Checking memcached is running')
1839+ ret = self.validate_services_by_name({sentry_unit: ['memcached']})
1840+ if ret:
1841+ amulet.raise_status(amulet.FAIL, msg='Memcache running check'
1842+ 'failed {}'.format(ret))
1843+ else:
1844+ self.log.debug('OK')
1845+ self.log.debug('Checking memcache url is configured in {}'.format(
1846+ conf))
1847+ if self.validate_config_data(sentry_unit, conf, section, _kvs):
1848+ message = "Memcache config error in: {}".format(conf)
1849+ amulet.raise_status(amulet.FAIL, msg=message)
1850+ else:
1851+ self.log.debug('OK')
1852+ self.log.debug('Checking memcache configuration in '
1853+ '/etc/memcached.conf')
1854+ contents = self.file_contents_safe(sentry_unit, '/etc/memcached.conf',
1855+ fatal=True)
1856+ ubuntu_release, _ = self.run_cmd_unit(sentry_unit, 'lsb_release -cs')
1857+ if ubuntu_release <= 'trusty':
1858+ memcache_listen_addr = 'ip6-localhost'
1859+ else:
1860+ memcache_listen_addr = '::1'
1861+ expected = {
1862+ '-p': '11211',
1863+ '-l': memcache_listen_addr}
1864+ found = []
1865+ for key, value in expected.items():
1866+ for line in contents.split('\n'):
1867+ if line.startswith(key):
1868+ self.log.debug('Checking {} is set to {}'.format(
1869+ key,
1870+ value))
1871+ assert value == line.split()[-1]
1872+ self.log.debug(line.split()[-1])
1873+ found.append(key)
1874+ if sorted(found) == sorted(expected.keys()):
1875+ self.log.debug('OK')
1876+ else:
1877+ message = "Memcache config error in: /etc/memcached.conf"
1878+ amulet.raise_status(amulet.FAIL, msg=message)
1879
1880=== modified file 'charmhelpers/contrib/openstack/context.py'
1881--- charmhelpers/contrib/openstack/context.py 2015-09-28 20:09:02 +0000
1882+++ charmhelpers/contrib/openstack/context.py 2017-03-04 02:21:16 +0000
1883@@ -1,29 +1,27 @@
1884 # Copyright 2014-2015 Canonical Limited.
1885 #
1886-# This file is part of charm-helpers.
1887-#
1888-# charm-helpers is free software: you can redistribute it and/or modify
1889-# it under the terms of the GNU Lesser General Public License version 3 as
1890-# published by the Free Software Foundation.
1891-#
1892-# charm-helpers is distributed in the hope that it will be useful,
1893-# but WITHOUT ANY WARRANTY; without even the implied warranty of
1894-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1895-# GNU Lesser General Public License for more details.
1896-#
1897-# You should have received a copy of the GNU Lesser General Public License
1898-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
1899+# Licensed under the Apache License, Version 2.0 (the "License");
1900+# you may not use this file except in compliance with the License.
1901+# You may obtain a copy of the License at
1902+#
1903+# http://www.apache.org/licenses/LICENSE-2.0
1904+#
1905+# Unless required by applicable law or agreed to in writing, software
1906+# distributed under the License is distributed on an "AS IS" BASIS,
1907+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1908+# See the License for the specific language governing permissions and
1909+# limitations under the License.
1910
1911 import glob
1912 import json
1913+import math
1914 import os
1915 import re
1916 import time
1917 from base64 import b64decode
1918-from subprocess import check_call
1919+from subprocess import check_call, CalledProcessError
1920
1921 import six
1922-import yaml
1923
1924 from charmhelpers.fetch import (
1925 apt_install,
1926@@ -45,10 +43,12 @@
1927 INFO,
1928 WARNING,
1929 ERROR,
1930+ status_set,
1931 )
1932
1933 from charmhelpers.core.sysctl import create as sysctl_create
1934 from charmhelpers.core.strutils import bool_from_string
1935+from charmhelpers.contrib.openstack.exceptions import OSContextError
1936
1937 from charmhelpers.core.host import (
1938 get_bond_master,
1939@@ -57,6 +57,8 @@
1940 get_nic_hwaddr,
1941 mkdir,
1942 write_file,
1943+ pwgen,
1944+ lsb_release,
1945 )
1946 from charmhelpers.contrib.hahelpers.cluster import (
1947 determine_apache_port,
1948@@ -86,15 +88,28 @@
1949 is_address_in_network,
1950 is_bridge_member,
1951 )
1952-from charmhelpers.contrib.openstack.utils import get_host_ip
1953+from charmhelpers.contrib.openstack.utils import (
1954+ config_flags_parser,
1955+ get_host_ip,
1956+ git_determine_usr_bin,
1957+ git_determine_python_path,
1958+ enable_memcache,
1959+)
1960+from charmhelpers.core.unitdata import kv
1961+
1962+try:
1963+ import psutil
1964+except ImportError:
1965+ if six.PY2:
1966+ apt_install('python-psutil', fatal=True)
1967+ else:
1968+ apt_install('python3-psutil', fatal=True)
1969+ import psutil
1970+
1971 CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
1972 ADDRESS_TYPES = ['admin', 'internal', 'public']
1973
1974
1975-class OSContextError(Exception):
1976- pass
1977-
1978-
1979 def ensure_packages(packages):
1980 """Install but do not upgrade required plugin packages."""
1981 required = filter_installed_packages(packages)
1982@@ -115,83 +130,6 @@
1983 return True
1984
1985
1986-def config_flags_parser(config_flags):
1987- """Parses config flags string into dict.
1988-
1989- This parsing method supports a few different formats for the config
1990- flag values to be parsed:
1991-
1992- 1. A string in the simple format of key=value pairs, with the possibility
1993- of specifying multiple key value pairs within the same string. For
1994- example, a string in the format of 'key1=value1, key2=value2' will
1995- return a dict of:
1996-
1997- {'key1': 'value1',
1998- 'key2': 'value2'}.
1999-
2000- 2. A string in the above format, but supporting a comma-delimited list
2001- of values for the same key. For example, a string in the format of
2002- 'key1=value1, key2=value3,value4,value5' will return a dict of:
2003-
2004- {'key1', 'value1',
2005- 'key2', 'value2,value3,value4'}
2006-
2007- 3. A string containing a colon character (:) prior to an equal
2008- character (=) will be treated as yaml and parsed as such. This can be
2009- used to specify more complex key value pairs. For example,
2010- a string in the format of 'key1: subkey1=value1, subkey2=value2' will
2011- return a dict of:
2012-
2013- {'key1', 'subkey1=value1, subkey2=value2'}
2014-
2015- The provided config_flags string may be a list of comma-separated values
2016- which themselves may be comma-separated list of values.
2017- """
2018- # If we find a colon before an equals sign then treat it as yaml.
2019- # Note: limit it to finding the colon first since this indicates assignment
2020- # for inline yaml.
2021- colon = config_flags.find(':')
2022- equals = config_flags.find('=')
2023- if colon > 0:
2024- if colon < equals or equals < 0:
2025- return yaml.safe_load(config_flags)
2026-
2027- if config_flags.find('==') >= 0:
2028- log("config_flags is not in expected format (key=value)", level=ERROR)
2029- raise OSContextError
2030-
2031- # strip the following from each value.
2032- post_strippers = ' ,'
2033- # we strip any leading/trailing '=' or ' ' from the string then
2034- # split on '='.
2035- split = config_flags.strip(' =').split('=')
2036- limit = len(split)
2037- flags = {}
2038- for i in range(0, limit - 1):
2039- current = split[i]
2040- next = split[i + 1]
2041- vindex = next.rfind(',')
2042- if (i == limit - 2) or (vindex < 0):
2043- value = next
2044- else:
2045- value = next[:vindex]
2046-
2047- if i == 0:
2048- key = current
2049- else:
2050- # if this not the first entry, expect an embedded key.
2051- index = current.rfind(',')
2052- if index < 0:
2053- log("Invalid config value(s) at index %s" % (i), level=ERROR)
2054- raise OSContextError
2055- key = current[index + 1:]
2056-
2057- # Add to collection.
2058- flags[key.strip(post_strippers)] = value.rstrip(post_strippers)
2059-
2060- return flags
2061-
2062-
2063 class OSContextGenerator(object):
2064 """Base class for all context generators."""
2065 interfaces = []
2066@@ -401,6 +339,7 @@
2067 auth_host = format_ipv6_addr(auth_host) or auth_host
2068 svc_protocol = rdata.get('service_protocol') or 'http'
2069 auth_protocol = rdata.get('auth_protocol') or 'http'
2070+ api_version = rdata.get('api_version') or '2.0'
2071 ctxt.update({'service_port': rdata.get('service_port'),
2072 'service_host': serv_host,
2073 'auth_host': auth_host,
2074@@ -409,7 +348,12 @@
2075 'admin_user': rdata.get('service_username'),
2076 'admin_password': rdata.get('service_password'),
2077 'service_protocol': svc_protocol,
2078- 'auth_protocol': auth_protocol})
2079+ 'auth_protocol': auth_protocol,
2080+ 'api_version': api_version})
2081+
2082+ if float(api_version) > 2:
2083+ ctxt.update({'admin_domain_name':
2084+ rdata.get('service_domain')})
2085
2086 if self.context_complete(ctxt):
2087 # NOTE(jamespage) this is required for >= icehouse
2088@@ -451,16 +395,20 @@
2089 for rid in relation_ids(self.rel_name):
2090 ha_vip_only = False
2091 self.related = True
2092+ transport_hosts = None
2093+ rabbitmq_port = '5672'
2094 for unit in related_units(rid):
2095 if relation_get('clustered', rid=rid, unit=unit):
2096 ctxt['clustered'] = True
2097 vip = relation_get('vip', rid=rid, unit=unit)
2098 vip = format_ipv6_addr(vip) or vip
2099 ctxt['rabbitmq_host'] = vip
2100+ transport_hosts = [vip]
2101 else:
2102 host = relation_get('private-address', rid=rid, unit=unit)
2103 host = format_ipv6_addr(host) or host
2104 ctxt['rabbitmq_host'] = host
2105+ transport_hosts = [host]
2106
2107 ctxt.update({
2108 'rabbitmq_user': username,
2109@@ -472,6 +420,7 @@
2110 ssl_port = relation_get('ssl_port', rid=rid, unit=unit)
2111 if ssl_port:
2112 ctxt['rabbit_ssl_port'] = ssl_port
2113+ rabbitmq_port = ssl_port
2114
2115 ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit)
2116 if ssl_ca:
2117@@ -509,6 +458,20 @@
2118 rabbitmq_hosts.append(host)
2119
2120 ctxt['rabbitmq_hosts'] = ','.join(sorted(rabbitmq_hosts))
2121+ transport_hosts = rabbitmq_hosts
2122+
2123+ if transport_hosts:
2124+ transport_url_hosts = ''
2125+ for host in transport_hosts:
2126+ if transport_url_hosts:
2127+ format_string = ",{}:{}@{}:{}"
2128+ else:
2129+ format_string = "{}:{}@{}:{}"
2130+ transport_url_hosts += format_string.format(
2131+ ctxt['rabbitmq_user'], ctxt['rabbitmq_password'],
2132+ host, rabbitmq_port)
2133+ ctxt['transport_url'] = "rabbit://{}/{}".format(
2134+ transport_url_hosts, vhost)
2135
2136 oslo_messaging_flags = conf.get('oslo-messaging-flags', None)
2137 if oslo_messaging_flags:
2138@@ -540,13 +503,16 @@
2139 ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
2140 if not ctxt.get('key'):
2141 ctxt['key'] = relation_get('key', rid=rid, unit=unit)
2142- ceph_pub_addr = relation_get('ceph-public-address', rid=rid,
2143+
2144+ ceph_addrs = relation_get('ceph-public-address', rid=rid,
2145+ unit=unit)
2146+ if ceph_addrs:
2147+ for addr in ceph_addrs.split(' '):
2148+ mon_hosts.append(format_ipv6_addr(addr) or addr)
2149+ else:
2150+ priv_addr = relation_get('private-address', rid=rid,
2151 unit=unit)
2152- unit_priv_addr = relation_get('private-address', rid=rid,
2153- unit=unit)
2154- ceph_addr = ceph_pub_addr or unit_priv_addr
2155- ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
2156- mon_hosts.append(ceph_addr)
2157+ mon_hosts.append(format_ipv6_addr(priv_addr) or priv_addr)
2158
2159 ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts))
2160
2161@@ -626,15 +592,28 @@
2162 if config('haproxy-client-timeout'):
2163 ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout')
2164
2165+ if config('haproxy-queue-timeout'):
2166+ ctxt['haproxy_queue_timeout'] = config('haproxy-queue-timeout')
2167+
2168+ if config('haproxy-connect-timeout'):
2169+ ctxt['haproxy_connect_timeout'] = config('haproxy-connect-timeout')
2170+
2171 if config('prefer-ipv6'):
2172 ctxt['ipv6'] = True
2173 ctxt['local_host'] = 'ip6-localhost'
2174 ctxt['haproxy_host'] = '::'
2175- ctxt['stat_port'] = ':::8888'
2176 else:
2177 ctxt['local_host'] = '127.0.0.1'
2178 ctxt['haproxy_host'] = '0.0.0.0'
2179- ctxt['stat_port'] = ':8888'
2180+
2181+ ctxt['stat_port'] = '8888'
2182+
2183+ db = kv()
2184+ ctxt['stat_password'] = db.get('stat-password')
2185+ if not ctxt['stat_password']:
2186+ ctxt['stat_password'] = db.set('stat-password',
2187+ pwgen(32))
2188+ db.flush()
2189
2190 for frontend in cluster_hosts:
2191 if (len(cluster_hosts[frontend]['backends']) > 1 or
2192@@ -698,7 +677,7 @@
2193 service_namespace = None
2194
2195 def enable_modules(self):
2196- cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http']
2197+ cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http', 'headers']
2198 check_call(cmd)
2199
2200 def configure_cert(self, cn=None):
2201@@ -952,6 +931,19 @@
2202 'config': config}
2203 return ovs_ctxt
2204
2205+ def midonet_ctxt(self):
2206+ driver = neutron_plugin_attribute(self.plugin, 'driver',
2207+ self.network_manager)
2208+ midonet_config = neutron_plugin_attribute(self.plugin, 'config',
2209+ self.network_manager)
2210+ mido_ctxt = {'core_plugin': driver,
2211+ 'neutron_plugin': 'midonet',
2212+ 'neutron_security_groups': self.neutron_security_groups,
2213+ 'local_ip': unit_private_ip(),
2214+ 'config': midonet_config}
2215+
2216+ return mido_ctxt
2217+
2218 def __call__(self):
2219 if self.network_manager not in ['quantum', 'neutron']:
2220 return {}
2221@@ -973,6 +965,8 @@
2222 ctxt.update(self.nuage_ctxt())
2223 elif self.plugin == 'plumgrid':
2224 ctxt.update(self.pg_ctxt())
2225+ elif self.plugin == 'midonet':
2226+ ctxt.update(self.midonet_ctxt())
2227
2228 alchemy_flags = config('neutron-alchemy-flags')
2229 if alchemy_flags:
2230@@ -1073,6 +1067,20 @@
2231 config_flags_parser(config_flags)}
2232
2233
2234+class LibvirtConfigFlagsContext(OSContextGenerator):
2235+ """
2236+ This context provides support for extending
2237+ the libvirt section through user-defined flags.
2238+ """
2239+ def __call__(self):
2240+ ctxt = {}
2241+ libvirt_flags = config('libvirt-flags')
2242+ if libvirt_flags:
2243+ ctxt['libvirt_flags'] = config_flags_parser(
2244+ libvirt_flags)
2245+ return ctxt
2246+
2247+
2248 class SubordinateConfigContext(OSContextGenerator):
2249
2250 """
2251@@ -1105,7 +1113,7 @@
2252
2253 ctxt = {
2254 ... other context ...
2255- 'subordinate_config': {
2256+ 'subordinate_configuration': {
2257 'DEFAULT': {
2258 'key1': 'value1',
2259 },
2260@@ -1146,22 +1154,23 @@
2261 try:
2262 sub_config = json.loads(sub_config)
2263 except:
2264- log('Could not parse JSON from subordinate_config '
2265- 'setting from %s' % rid, level=ERROR)
2266+ log('Could not parse JSON from '
2267+ 'subordinate_configuration setting from %s'
2268+ % rid, level=ERROR)
2269 continue
2270
2271 for service in self.services:
2272 if service not in sub_config:
2273- log('Found subordinate_config on %s but it contained'
2274- 'nothing for %s service' % (rid, service),
2275- level=INFO)
2276+ log('Found subordinate_configuration on %s but it '
2277+ 'contained nothing for %s service'
2278+ % (rid, service), level=INFO)
2279 continue
2280
2281 sub_config = sub_config[service]
2282 if self.config_file not in sub_config:
2283- log('Found subordinate_config on %s but it contained'
2284- 'nothing for %s' % (rid, self.config_file),
2285- level=INFO)
2286+ log('Found subordinate_configuration on %s but it '
2287+ 'contained nothing for %s'
2288+ % (rid, self.config_file), level=INFO)
2289 continue
2290
2291 sub_config = sub_config[self.config_file]
2292@@ -1212,17 +1221,55 @@
2293
2294 @property
2295 def num_cpus(self):
2296- try:
2297- from psutil import NUM_CPUS
2298- except ImportError:
2299- apt_install('python-psutil', fatal=True)
2300- from psutil import NUM_CPUS
2301-
2302- return NUM_CPUS
2303+ # NOTE: use cpu_count if present (16.04 support)
2304+ if hasattr(psutil, 'cpu_count'):
2305+ return psutil.cpu_count()
2306+ else:
2307+ return psutil.NUM_CPUS
2308
2309 def __call__(self):
2310 multiplier = config('worker-multiplier') or 0
2311- ctxt = {"workers": self.num_cpus * multiplier}
2312+ count = int(self.num_cpus * multiplier)
2313+ if multiplier > 0 and count == 0:
2314+ count = 1
2315+ ctxt = {"workers": count}
2316+ return ctxt
2317+
2318+
2319+class WSGIWorkerConfigContext(WorkerConfigContext):
2320+
2321+ def __init__(self, name=None, script=None, admin_script=None,
2322+ public_script=None, process_weight=1.00,
2323+ admin_process_weight=0.75, public_process_weight=0.25):
2324+ self.service_name = name
2325+ self.user = name
2326+ self.group = name
2327+ self.script = script
2328+ self.admin_script = admin_script
2329+ self.public_script = public_script
2330+ self.process_weight = process_weight
2331+ self.admin_process_weight = admin_process_weight
2332+ self.public_process_weight = public_process_weight
2333+
2334+ def __call__(self):
2335+ multiplier = config('worker-multiplier') or 1
2336+ total_processes = self.num_cpus * multiplier
2337+ ctxt = {
2338+ "service_name": self.service_name,
2339+ "user": self.user,
2340+ "group": self.group,
2341+ "script": self.script,
2342+ "admin_script": self.admin_script,
2343+ "public_script": self.public_script,
2344+ "processes": int(math.ceil(self.process_weight * total_processes)),
2345+ "admin_processes": int(math.ceil(self.admin_process_weight *
2346+ total_processes)),
2347+ "public_processes": int(math.ceil(self.public_process_weight *
2348+ total_processes)),
2349+ "threads": 1,
2350+ "usr_bin": git_determine_usr_bin(),
2351+ "python_path": git_determine_python_path(),
2352+ }
2353 return ctxt
2354
2355
2356@@ -1364,7 +1411,7 @@
2357 normalized.update({port: port for port in resolved
2358 if port in ports})
2359 if resolved:
2360- return {bridge: normalized[port] for port, bridge in
2361+ return {normalized[port]: bridge for port, bridge in
2362 six.iteritems(portmap) if port in normalized.keys()}
2363
2364 return None
2365@@ -1375,8 +1422,8 @@
2366 def __call__(self):
2367 ctxt = {}
2368 mappings = super(PhyNICMTUContext, self).__call__()
2369- if mappings and mappings.values():
2370- ports = mappings.values()
2371+ if mappings and mappings.keys():
2372+ ports = sorted(mappings.keys())
2373 napi_settings = NeutronAPIContext()()
2374 mtu = napi_settings.get('network_device_mtu')
2375 all_ports = set()
2376@@ -1421,7 +1468,146 @@
2377 rdata.get('service_protocol') or 'http',
2378 'auth_protocol':
2379 rdata.get('auth_protocol') or 'http',
2380+ 'api_version':
2381+ rdata.get('api_version') or '2.0',
2382 }
2383 if self.context_complete(ctxt):
2384 return ctxt
2385 return {}
2386+
2387+
2388+class InternalEndpointContext(OSContextGenerator):
2389+ """Internal endpoint context.
2390+
2391+ This context provides the endpoint type used for communication between
2392+ services e.g. between Nova and Cinder internally. Openstack uses Public
2393+ endpoints by default so this allows admins to optionally use internal
2394+ endpoints.
2395+ """
2396+ def __call__(self):
2397+ return {'use_internal_endpoints': config('use-internal-endpoints')}
2398+
2399+
2400+class AppArmorContext(OSContextGenerator):
2401+ """Base class for apparmor contexts."""
2402+
2403+ def __init__(self, profile_name=None):
2404+ self._ctxt = None
2405+ self.aa_profile = profile_name
2406+ self.aa_utils_packages = ['apparmor-utils']
2407+
2408+ @property
2409+ def ctxt(self):
2410+ if self._ctxt is not None:
2411+ return self._ctxt
2412+ self._ctxt = self._determine_ctxt()
2413+ return self._ctxt
2414+
2415+ def _determine_ctxt(self):
2416+ """
2417+ Validate aa-profile-mode settings is disable, enforce, or complain.
2418+
2419+ :return ctxt: Dictionary of the apparmor profile or None
2420+ """
2421+ if config('aa-profile-mode') in ['disable', 'enforce', 'complain']:
2422+ ctxt = {'aa_profile_mode': config('aa-profile-mode'),
2423+ 'ubuntu_release': lsb_release()['DISTRIB_RELEASE']}
2424+ if self.aa_profile:
2425+ ctxt['aa_profile'] = self.aa_profile
2426+ else:
2427+ ctxt = None
2428+ return ctxt
2429+
2430+ def __call__(self):
2431+ return self.ctxt
2432+
2433+ def install_aa_utils(self):
2434+ """
2435+ Install packages required for apparmor configuration.
2436+ """
2437+ log("Installing apparmor utils.")
2438+ ensure_packages(self.aa_utils_packages)
2439+
2440+ def manually_disable_aa_profile(self):
2441+ """
2442+ Manually disable an apparmor profile.
2443+
2444+ If aa-profile-mode is set to disabled (default) this is required as the
2445+ template has been written but apparmor is yet unaware of the profile
2446+ and aa-disable aa-profile fails. Without this the profile would kick
2447+ into enforce mode on the next service restart.
2448+
2449+ """
2450+ profile_path = '/etc/apparmor.d'
2451+ disable_path = '/etc/apparmor.d/disable'
2452+ if not os.path.lexists(os.path.join(disable_path, self.aa_profile)):
2453+ os.symlink(os.path.join(profile_path, self.aa_profile),
2454+ os.path.join(disable_path, self.aa_profile))
2455+
2456+ def setup_aa_profile(self):
2457+ """
2458+ Setup an apparmor profile.
2459+ The ctxt dictionary will contain the apparmor profile mode and
2460+ the apparmor profile name.
2461+ Makes calls out to aa-disable, aa-complain, or aa-enforce to setup
2462+ the apparmor profile.
2463+ """
2464+ self()
2465+ if not self.ctxt:
2466+ log("Not enabling apparmor Profile")
2467+ return
2468+ self.install_aa_utils()
2469+ cmd = ['aa-{}'.format(self.ctxt['aa_profile_mode'])]
2470+ cmd.append(self.ctxt['aa_profile'])
2471+ log("Setting up the apparmor profile for {} in {} mode."
2472+ "".format(self.ctxt['aa_profile'], self.ctxt['aa_profile_mode']))
2473+ try:
2474+ check_call(cmd)
2475+ except CalledProcessError as e:
2476+ # If aa-profile-mode is set to disabled (default) manual
2477+ # disabling is required as the template has been written but
2478+ # apparmor is yet unaware of the profile and aa-disable aa-profile
2479+ # fails. If aa-disable learns to read profile files first this can
2480+ # be removed.
2481+ if self.ctxt['aa_profile_mode'] == 'disable':
2482+ log("Manually disabling the apparmor profile for {}."
2483+ "".format(self.ctxt['aa_profile']))
2484+ self.manually_disable_aa_profile()
2485+ return
2486+ status_set('blocked', "Apparmor profile {} failed to be set to {}."
2487+ "".format(self.ctxt['aa_profile'],
2488+ self.ctxt['aa_profile_mode']))
2489+ raise e
2490+
2491+
2492+class MemcacheContext(OSContextGenerator):
2493+ """Memcache context
2494+
2495+ This context provides options for configuring a local memcache client and
2496+ server
2497+ """
2498+
2499+ def __init__(self, package=None):
2500+ """
2501+ @param package: Package to examine to extrapolate OpenStack release.
2502+ Used when charms have no openstack-origin config
2503+ option (ie subordinates)
2504+ """
2505+ self.package = package
2506+
2507+ def __call__(self):
2508+ ctxt = {}
2509+ ctxt['use_memcache'] = enable_memcache(package=self.package)
2510+ if ctxt['use_memcache']:
2511+ # Trusty version of memcached does not support ::1 as a listen
2512+ # address so use host file entry instead
2513+ if lsb_release()['DISTRIB_CODENAME'].lower() > 'trusty':
2514+ ctxt['memcache_server'] = '::1'
2515+ else:
2516+ ctxt['memcache_server'] = 'ip6-localhost'
2517+ ctxt['memcache_server_formatted'] = '[::1]'
2518+ ctxt['memcache_port'] = '11211'
2519+ ctxt['memcache_url'] = 'inet6:{}:{}'.format(
2520+ ctxt['memcache_server_formatted'],
2521+ ctxt['memcache_port'])
2522+ return ctxt
2523
2524=== added file 'charmhelpers/contrib/openstack/exceptions.py'
2525--- charmhelpers/contrib/openstack/exceptions.py 1970-01-01 00:00:00 +0000
2526+++ charmhelpers/contrib/openstack/exceptions.py 2017-03-04 02:21:16 +0000
2527@@ -0,0 +1,21 @@
2528+# Copyright 2016 Canonical Ltd
2529+#
2530+# Licensed under the Apache License, Version 2.0 (the "License");
2531+# you may not use this file except in compliance with the License.
2532+# You may obtain a copy of the License at
2533+#
2534+# http://www.apache.org/licenses/LICENSE-2.0
2535+#
2536+# Unless required by applicable law or agreed to in writing, software
2537+# distributed under the License is distributed on an "AS IS" BASIS,
2538+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2539+# See the License for the specific language governing permissions and
2540+# limitations under the License.
2541+
2542+
2543+class OSContextError(Exception):
2544+ """Raised when an error occurs during context generation.
2545+
2546+ This exception is principally used in contrib.openstack.context
2547+ """
2548+ pass
2549
2550=== modified file 'charmhelpers/contrib/openstack/files/__init__.py'
2551--- charmhelpers/contrib/openstack/files/__init__.py 2015-09-28 20:09:02 +0000
2552+++ charmhelpers/contrib/openstack/files/__init__.py 2017-03-04 02:21:16 +0000
2553@@ -1,18 +1,16 @@
2554 # Copyright 2014-2015 Canonical Limited.
2555 #
2556-# This file is part of charm-helpers.
2557-#
2558-# charm-helpers is free software: you can redistribute it and/or modify
2559-# it under the terms of the GNU Lesser General Public License version 3 as
2560-# published by the Free Software Foundation.
2561-#
2562-# charm-helpers is distributed in the hope that it will be useful,
2563-# but WITHOUT ANY WARRANTY; without even the implied warranty of
2564-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2565-# GNU Lesser General Public License for more details.
2566-#
2567-# You should have received a copy of the GNU Lesser General Public License
2568-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2569+# Licensed under the Apache License, Version 2.0 (the "License");
2570+# you may not use this file except in compliance with the License.
2571+# You may obtain a copy of the License at
2572+#
2573+# http://www.apache.org/licenses/LICENSE-2.0
2574+#
2575+# Unless required by applicable law or agreed to in writing, software
2576+# distributed under the License is distributed on an "AS IS" BASIS,
2577+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2578+# See the License for the specific language governing permissions and
2579+# limitations under the License.
2580
2581 # dummy __init__.py to fool syncer into thinking this is a syncable python
2582 # module
2583
2584=== modified file 'charmhelpers/contrib/openstack/files/check_haproxy.sh'
2585--- charmhelpers/contrib/openstack/files/check_haproxy.sh 2015-09-28 20:09:02 +0000
2586+++ charmhelpers/contrib/openstack/files/check_haproxy.sh 2017-03-04 02:21:16 +0000
2587@@ -9,15 +9,17 @@
2588 CRITICAL=0
2589 NOTACTIVE=''
2590 LOGFILE=/var/log/nagios/check_haproxy.log
2591-AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}')
2592+AUTH=$(grep -r "stats auth" /etc/haproxy | awk 'NR=1{print $4}')
2593
2594-for appserver in $(grep ' server' /etc/haproxy/haproxy.cfg | awk '{print $2'});
2595+typeset -i N_INSTANCES=0
2596+for appserver in $(awk '/^\s+server/{print $2}' /etc/haproxy/haproxy.cfg)
2597 do
2598- output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 --regex="class=\"(active|backup)(2|3).*${appserver}" -e ' 200 OK')
2599+ N_INSTANCES=N_INSTANCES+1
2600+ output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' --regex=",${appserver},.*,UP.*" -e ' 200 OK')
2601 if [ $? != 0 ]; then
2602 date >> $LOGFILE
2603 echo $output >> $LOGFILE
2604- /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -v | grep $appserver >> $LOGFILE 2>&1
2605+ /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v | grep ",${appserver}," >> $LOGFILE 2>&1
2606 CRITICAL=1
2607 NOTACTIVE="${NOTACTIVE} $appserver"
2608 fi
2609@@ -28,5 +30,5 @@
2610 exit 2
2611 fi
2612
2613-echo "OK: All haproxy instances looking good"
2614+echo "OK: All haproxy instances ($N_INSTANCES) looking good"
2615 exit 0
2616
2617=== added directory 'charmhelpers/contrib/openstack/ha'
2618=== added file 'charmhelpers/contrib/openstack/ha/__init__.py'
2619--- charmhelpers/contrib/openstack/ha/__init__.py 1970-01-01 00:00:00 +0000
2620+++ charmhelpers/contrib/openstack/ha/__init__.py 2017-03-04 02:21:16 +0000
2621@@ -0,0 +1,13 @@
2622+# Copyright 2016 Canonical Ltd
2623+#
2624+# Licensed under the Apache License, Version 2.0 (the "License");
2625+# you may not use this file except in compliance with the License.
2626+# You may obtain a copy of the License at
2627+#
2628+# http://www.apache.org/licenses/LICENSE-2.0
2629+#
2630+# Unless required by applicable law or agreed to in writing, software
2631+# distributed under the License is distributed on an "AS IS" BASIS,
2632+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2633+# See the License for the specific language governing permissions and
2634+# limitations under the License.
2635
2636=== added file 'charmhelpers/contrib/openstack/ha/utils.py'
2637--- charmhelpers/contrib/openstack/ha/utils.py 1970-01-01 00:00:00 +0000
2638+++ charmhelpers/contrib/openstack/ha/utils.py 2017-03-04 02:21:16 +0000
2639@@ -0,0 +1,139 @@
2640+# Copyright 2014-2016 Canonical Limited.
2641+#
2642+# Licensed under the Apache License, Version 2.0 (the "License");
2643+# you may not use this file except in compliance with the License.
2644+# You may obtain a copy of the License at
2645+#
2646+# http://www.apache.org/licenses/LICENSE-2.0
2647+#
2648+# Unless required by applicable law or agreed to in writing, software
2649+# distributed under the License is distributed on an "AS IS" BASIS,
2650+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2651+# See the License for the specific language governing permissions and
2652+# limitations under the License.
2653+
2654+#
2655+# Copyright 2016 Canonical Ltd.
2656+#
2657+# Authors:
2658+# Openstack Charmers <
2659+#
2660+
2661+"""
2662+Helpers for high availability.
2663+"""
2664+
2665+import re
2666+
2667+from charmhelpers.core.hookenv import (
2668+ log,
2669+ relation_set,
2670+ charm_name,
2671+ config,
2672+ status_set,
2673+ DEBUG,
2674+)
2675+
2676+from charmhelpers.core.host import (
2677+ lsb_release
2678+)
2679+
2680+from charmhelpers.contrib.openstack.ip import (
2681+ resolve_address,
2682+)
2683+
2684+
2685+class DNSHAException(Exception):
2686+ """Raised when an error occurs setting up DNS HA
2687+ """
2688+
2689+ pass
2690+
2691+
2692+def update_dns_ha_resource_params(resources, resource_params,
2693+ relation_id=None,
2694+ crm_ocf='ocf:maas:dns'):
2695+ """ Check for os-*-hostname settings and update resource dictionaries for
2696+ the HA relation.
2697+
2698+ @param resources: Pointer to dictionary of resources.
2699+ Usually instantiated in ha_joined().
2700+ @param resource_params: Pointer to dictionary of resource parameters.
2701+ Usually instantiated in ha_joined()
2702+ @param relation_id: Relation ID of the ha relation
2703+ @param crm_ocf: Corosync Open Cluster Framework resource agent to use for
2704+ DNS HA
2705+ """
2706+
2707+ # Validate the charm environment for DNS HA
2708+ assert_charm_supports_dns_ha()
2709+
2710+ settings = ['os-admin-hostname', 'os-internal-hostname',
2711+ 'os-public-hostname', 'os-access-hostname']
2712+
2713+ # Check which DNS settings are set and update dictionaries
2714+ hostname_group = []
2715+ for setting in settings:
2716+ hostname = config(setting)
2717+ if hostname is None:
2718+ log('DNS HA: Hostname setting {} is None. Ignoring.'
2719+ ''.format(setting),
2720+ DEBUG)
2721+ continue
2722+ m = re.search('os-(.+?)-hostname', setting)
2723+ if m:
2724+ networkspace = m.group(1)
2725+ else:
2726+ msg = ('Unexpected DNS hostname setting: {}. '
2727+ 'Cannot determine network space name'
2728+ ''.format(setting))
2729+ status_set('blocked', msg)
2730+ raise DNSHAException(msg)
2731+
2732+ hostname_key = 'res_{}_{}_hostname'.format(charm_name(), networkspace)
2733+ if hostname_key in hostname_group:
2734+ log('DNS HA: Resource {}: {} already exists in '
2735+ 'hostname group - skipping'.format(hostname_key, hostname),
2736+ DEBUG)
2737+ continue
2738+
2739+ hostname_group.append(hostname_key)
2740+ resources[hostname_key] = crm_ocf
2741+ resource_params[hostname_key] = (
2742+ 'params fqdn="{}" ip_address="{}" '
2743+ ''.format(hostname, resolve_address(endpoint_type=networkspace,
2744+ override=False)))
2745+
2746+ if len(hostname_group) >= 1:
2747+ log('DNS HA: Hostname group is set with {} as members. '
2748+ 'Informing the ha relation'.format(' '.join(hostname_group)),
2749+ DEBUG)
2750+ relation_set(relation_id=relation_id, groups={
2751+ 'grp_{}_hostnames'.format(charm_name()): ' '.join(hostname_group)})
2752+ else:
2753+ msg = 'DNS HA: Hostname group has no members.'
2754+ status_set('blocked', msg)
2755+ raise DNSHAException(msg)
2756+
2757+
2758+def assert_charm_supports_dns_ha():
2759+ """Validate prerequisites for DNS HA
2760+ The MAAS client is only available on Xenial or greater
2761+ """
2762+ if lsb_release().get('DISTRIB_RELEASE') < '16.04':
2763+ msg = ('DNS HA is only supported on 16.04 and greater '
2764+ 'versions of Ubuntu.')
2765+ status_set('blocked', msg)
2766+ raise DNSHAException(msg)
2767+ return True
2768+
2769+
2770+def expect_ha():
2771+ """ Determine if the unit expects to be in HA
2772+
2773+ Check for VIP or dns-ha settings which indicate the unit should expect to
2774+ be related to hacluster.
2775+
2776+ @returns boolean
2777+ """
2778+ return config('vip') or config('dns-ha')
2779
2780=== modified file 'charmhelpers/contrib/openstack/ip.py'
2781--- charmhelpers/contrib/openstack/ip.py 2015-09-28 20:09:02 +0000
2782+++ charmhelpers/contrib/openstack/ip.py 2017-03-04 02:21:16 +0000
2783@@ -1,52 +1,62 @@
2784 # Copyright 2014-2015 Canonical Limited.
2785 #
2786-# This file is part of charm-helpers.
2787-#
2788-# charm-helpers is free software: you can redistribute it and/or modify
2789-# it under the terms of the GNU Lesser General Public License version 3 as
2790-# published by the Free Software Foundation.
2791-#
2792-# charm-helpers is distributed in the hope that it will be useful,
2793-# but WITHOUT ANY WARRANTY; without even the implied warranty of
2794-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2795-# GNU Lesser General Public License for more details.
2796-#
2797-# You should have received a copy of the GNU Lesser General Public License
2798-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2799+# Licensed under the Apache License, Version 2.0 (the "License");
2800+# you may not use this file except in compliance with the License.
2801+# You may obtain a copy of the License at
2802+#
2803+# http://www.apache.org/licenses/LICENSE-2.0
2804+#
2805+# Unless required by applicable law or agreed to in writing, software
2806+# distributed under the License is distributed on an "AS IS" BASIS,
2807+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2808+# See the License for the specific language governing permissions and
2809+# limitations under the License.
2810
2811 from charmhelpers.core.hookenv import (
2812 config,
2813 unit_get,
2814 service_name,
2815+ network_get_primary_address,
2816 )
2817 from charmhelpers.contrib.network.ip import (
2818 get_address_in_network,
2819 is_address_in_network,
2820 is_ipv6,
2821 get_ipv6_addr,
2822+ resolve_network_cidr,
2823 )
2824 from charmhelpers.contrib.hahelpers.cluster import is_clustered
2825
2826 PUBLIC = 'public'
2827 INTERNAL = 'int'
2828 ADMIN = 'admin'
2829+ACCESS = 'access'
2830
2831 ADDRESS_MAP = {
2832 PUBLIC: {
2833+ 'binding': 'public',
2834 'config': 'os-public-network',
2835 'fallback': 'public-address',
2836 'override': 'os-public-hostname',
2837 },
2838 INTERNAL: {
2839+ 'binding': 'internal',
2840 'config': 'os-internal-network',
2841 'fallback': 'private-address',
2842 'override': 'os-internal-hostname',
2843 },
2844 ADMIN: {
2845+ 'binding': 'admin',
2846 'config': 'os-admin-network',
2847 'fallback': 'private-address',
2848 'override': 'os-admin-hostname',
2849- }
2850+ },
2851+ ACCESS: {
2852+ 'binding': 'access',
2853+ 'config': 'access-network',
2854+ 'fallback': 'private-address',
2855+ 'override': 'os-access-hostname',
2856+ },
2857 }
2858
2859
2860@@ -103,20 +113,23 @@
2861 return addr_override.format(service_name=service_name())
2862
2863
2864-def resolve_address(endpoint_type=PUBLIC):
2865+def resolve_address(endpoint_type=PUBLIC, override=True):
2866 """Return unit address depending on net config.
2867
2868 If unit is clustered with vip(s) and has net splits defined, return vip on
2869 correct network. If clustered with no nets defined, return primary vip.
2870
2871 If not clustered, return unit address ensuring address is on configured net
2872- split if one is configured.
2873+ split if one is configured, or a Juju 2.0 extra-binding has been used.
2874
2875 :param endpoint_type: Network endpoing type
2876+ :param override: Accept hostname overrides or not
2877 """
2878- resolved_address = _get_address_override(endpoint_type)
2879- if resolved_address:
2880- return resolved_address
2881+ resolved_address = None
2882+ if override:
2883+ resolved_address = _get_address_override(endpoint_type)
2884+ if resolved_address:
2885+ return resolved_address
2886
2887 vips = config('vip')
2888 if vips:
2889@@ -125,23 +138,45 @@
2890 net_type = ADDRESS_MAP[endpoint_type]['config']
2891 net_addr = config(net_type)
2892 net_fallback = ADDRESS_MAP[endpoint_type]['fallback']
2893+ binding = ADDRESS_MAP[endpoint_type]['binding']
2894 clustered = is_clustered()
2895- if clustered:
2896- if not net_addr:
2897- # If no net-splits defined, we expect a single vip
2898- resolved_address = vips[0]
2899- else:
2900+
2901+ if clustered and vips:
2902+ if net_addr:
2903 for vip in vips:
2904 if is_address_in_network(net_addr, vip):
2905 resolved_address = vip
2906 break
2907+ else:
2908+ # NOTE: endeavour to check vips against network space
2909+ # bindings
2910+ try:
2911+ bound_cidr = resolve_network_cidr(
2912+ network_get_primary_address(binding)
2913+ )
2914+ for vip in vips:
2915+ if is_address_in_network(bound_cidr, vip):
2916+ resolved_address = vip
2917+ break
2918+ except NotImplementedError:
2919+ # If no net-splits configured and no support for extra
2920+ # bindings/network spaces so we expect a single vip
2921+ resolved_address = vips[0]
2922 else:
2923 if config('prefer-ipv6'):
2924 fallback_addr = get_ipv6_addr(exc_list=vips)[0]
2925 else:
2926 fallback_addr = unit_get(net_fallback)
2927
2928- resolved_address = get_address_in_network(net_addr, fallback_addr)
2929+ if net_addr:
2930+ resolved_address = get_address_in_network(net_addr, fallback_addr)
2931+ else:
2932+ # NOTE: only try to use extra bindings if legacy network
2933+ # configuration is not in use
2934+ try:
2935+ resolved_address = network_get_primary_address(binding)
2936+ except NotImplementedError:
2937+ resolved_address = fallback_addr
2938
2939 if resolved_address is None:
2940 raise ValueError("Unable to resolve a suitable IP address based on "
2941
2942=== added file 'charmhelpers/contrib/openstack/keystone.py'
2943--- charmhelpers/contrib/openstack/keystone.py 1970-01-01 00:00:00 +0000
2944+++ charmhelpers/contrib/openstack/keystone.py 2017-03-04 02:21:16 +0000
2945@@ -0,0 +1,178 @@
2946+#!/usr/bin/python
2947+#
2948+# Copyright 2017 Canonical Ltd
2949+#
2950+# Licensed under the Apache License, Version 2.0 (the "License");
2951+# you may not use this file except in compliance with the License.
2952+# You may obtain a copy of the License at
2953+#
2954+# http://www.apache.org/licenses/LICENSE-2.0
2955+#
2956+# Unless required by applicable law or agreed to in writing, software
2957+# distributed under the License is distributed on an "AS IS" BASIS,
2958+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2959+# See the License for the specific language governing permissions and
2960+# limitations under the License.
2961+
2962+import six
2963+from charmhelpers.fetch import apt_install
2964+from charmhelpers.contrib.openstack.context import IdentityServiceContext
2965+from charmhelpers.core.hookenv import (
2966+ log,
2967+ ERROR,
2968+)
2969+
2970+
2971+def get_api_suffix(api_version):
2972+ """Return the formatted api suffix for the given version
2973+ @param api_version: version of the keystone endpoint
2974+ @returns the api suffix formatted according to the given api
2975+ version
2976+ """
2977+ return 'v2.0' if api_version in (2, "2.0") else 'v3'
2978+
2979+
2980+def format_endpoint(schema, addr, port, api_version):
2981+ """Return a formatted keystone endpoint
2982+ @param schema: http or https
2983+ @param addr: ipv4/ipv6 host of the keystone service
2984+ @param port: port of the keystone service
2985+ @param api_version: 2 or 3
2986+ @returns a fully formatted keystone endpoint
2987+ """
2988+ return '{}://{}:{}/{}/'.format(schema, addr, port,
2989+ get_api_suffix(api_version))
2990+
2991+
2992+def get_keystone_manager(endpoint, api_version, **kwargs):
2993+ """Return a keystonemanager for the correct API version
2994+
2995+ @param endpoint: the keystone endpoint to point client at
2996+ @param api_version: version of the keystone api the client should use
2997+ @param kwargs: token or username/tenant/password information
2998+ @returns keystonemanager class used for interrogating keystone
2999+ """
3000+ if api_version == 2:
3001+ return KeystoneManager2(endpoint, **kwargs)
3002+ if api_version == 3:
3003+ return KeystoneManager3(endpoint, **kwargs)
3004+ raise ValueError('No manager found for api version {}'.format(api_version))
3005+
3006+
3007+def get_keystone_manager_from_identity_service_context():
3008+ """Return a keystonmanager generated from a
3009+ instance of charmhelpers.contrib.openstack.context.IdentityServiceContext
3010+ @returns keystonamenager instance
3011+ """
3012+ context = IdentityServiceContext()()
3013+ if not context:
3014+ msg = "Identity service context cannot be generated"
3015+ log(msg, level=ERROR)
3016+ raise ValueError(msg)
3017+
3018+ endpoint = format_endpoint(context['service_protocol'],
3019+ context['service_host'],
3020+ context['service_port'],
3021+ context['api_version'])
3022+
3023+ if context['api_version'] in (2, "2.0"):
3024+ api_version = 2
3025+ else:
3026+ api_version = 3
3027+
3028+ return get_keystone_manager(endpoint, api_version,
3029+ username=context['admin_user'],
3030+ password=context['admin_password'],
3031+ tenant_name=context['admin_tenant_name'])
3032+
3033+
3034+class KeystoneManager(object):
3035+
3036+ def resolve_service_id(self, service_name=None, service_type=None):
3037+ """Find the service_id of a given service"""
3038+ services = [s._info for s in self.api.services.list()]
3039+
3040+ service_name = service_name.lower()
3041+ for s in services:
3042+ name = s['name'].lower()
3043+ if service_type and service_name:
3044+ if (service_name == name and service_type == s['type']):
3045+ return s['id']
3046+ elif service_name and service_name == name:
3047+ return s['id']
3048+ elif service_type and service_type == s['type']:
3049+ return s['id']
3050+ return None
3051+
3052+ def service_exists(self, service_name=None, service_type=None):
3053+ """Determine if the given service exists on the service list"""
3054+ return self.resolve_service_id(service_name, service_type) is not None
3055+
3056+
3057+class KeystoneManager2(KeystoneManager):
3058+
3059+ def __init__(self, endpoint, **kwargs):
3060+ try:
3061+ from keystoneclient.v2_0 import client
3062+ from keystoneclient.auth.identity import v2
3063+ from keystoneclient import session
3064+ except ImportError:
3065+ if six.PY2:
3066+ apt_install(["python-keystoneclient"], fatal=True)
3067+ else:
3068+ apt_install(["python3-keystoneclient"], fatal=True)
3069+
3070+ from keystoneclient.v2_0 import client
3071+ from keystoneclient.auth.identity import v2
3072+ from keystoneclient import session
3073+
3074+ self.api_version = 2
3075+
3076+ token = kwargs.get("token", None)
3077+ if token:
3078+ api = client.Client(endpoint=endpoint, token=token)
3079+ else:
3080+ auth = v2.Password(username=kwargs.get("username"),
3081+ password=kwargs.get("password"),
3082+ tenant_name=kwargs.get("tenant_name"),
3083+ auth_url=endpoint)
3084+ sess = session.Session(auth=auth)
3085+ api = client.Client(session=sess)
3086+
3087+ self.api = api
3088+
3089+
3090+class KeystoneManager3(KeystoneManager):
3091+
3092+ def __init__(self, endpoint, **kwargs):
3093+ try:
3094+ from keystoneclient.v3 import client
3095+ from keystoneclient.auth import token_endpoint
3096+ from keystoneclient import session
3097+ from keystoneclient.auth.identity import v3
3098+ except ImportError:
3099+ if six.PY2:
3100+ apt_install(["python-keystoneclient"], fatal=True)
3101+ else:
3102+ apt_install(["python3-keystoneclient"], fatal=True)
3103+
3104+ from keystoneclient.v3 import client
3105+ from keystoneclient.auth import token_endpoint
3106+ from keystoneclient import session
3107+ from keystoneclient.auth.identity import v3
3108+
3109+ self.api_version = 3
3110+
3111+ token = kwargs.get("token", None)
3112+ if token:
3113+ auth = token_endpoint.Token(endpoint=endpoint,
3114+ token=token)
3115+ sess = session.Session(auth=auth)
3116+ else:
3117+ auth = v3.Password(auth_url=endpoint,
3118+ user_id=kwargs.get("username"),
3119+ password=kwargs.get("password"),
3120+ project_id=kwargs.get("tenant_name"))
3121+ sess = session.Session(auth=auth)
3122+
3123+ self.api = client.Client(session=sess)
3124
3125=== modified file 'charmhelpers/contrib/openstack/neutron.py'
3126--- charmhelpers/contrib/openstack/neutron.py 2015-09-28 20:09:02 +0000
3127+++ charmhelpers/contrib/openstack/neutron.py 2017-03-04 02:21:16 +0000
3128@@ -1,18 +1,16 @@
3129 # Copyright 2014-2015 Canonical Limited.
3130 #
3131-# This file is part of charm-helpers.
3132-#
3133-# charm-helpers is free software: you can redistribute it and/or modify
3134-# it under the terms of the GNU Lesser General Public License version 3 as
3135-# published by the Free Software Foundation.
3136-#
3137-# charm-helpers is distributed in the hope that it will be useful,
3138-# but WITHOUT ANY WARRANTY; without even the implied warranty of
3139-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3140-# GNU Lesser General Public License for more details.
3141-#
3142-# You should have received a copy of the GNU Lesser General Public License
3143-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3144+# Licensed under the Apache License, Version 2.0 (the "License");
3145+# you may not use this file except in compliance with the License.
3146+# You may obtain a copy of the License at
3147+#
3148+# http://www.apache.org/licenses/LICENSE-2.0
3149+#
3150+# Unless required by applicable law or agreed to in writing, software
3151+# distributed under the License is distributed on an "AS IS" BASIS,
3152+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3153+# See the License for the specific language governing permissions and
3154+# limitations under the License.
3155
3156 # Various utilies for dealing with Neutron and the renaming from Quantum.
3157
3158@@ -34,6 +32,7 @@
3159 kver = check_output(['uname', '-r']).decode('UTF-8').strip()
3160 return 'linux-headers-%s' % kver
3161
3162+
3163 QUANTUM_CONF_DIR = '/etc/quantum'
3164
3165
3166@@ -50,7 +49,7 @@
3167 if kernel_version() >= (3, 13):
3168 return []
3169 else:
3170- return ['openvswitch-datapath-dkms']
3171+ return [headers_package(), 'openvswitch-datapath-dkms']
3172
3173
3174 # legacy
3175@@ -70,7 +69,7 @@
3176 relation_prefix='neutron',
3177 ssl_dir=QUANTUM_CONF_DIR)],
3178 'services': ['quantum-plugin-openvswitch-agent'],
3179- 'packages': [[headers_package()] + determine_dkms_package(),
3180+ 'packages': [determine_dkms_package(),
3181 ['quantum-plugin-openvswitch-agent']],
3182 'server_packages': ['quantum-server',
3183 'quantum-plugin-openvswitch'],
3184@@ -93,6 +92,7 @@
3185 }
3186 }
3187
3188+
3189 NEUTRON_CONF_DIR = '/etc/neutron'
3190
3191
3192@@ -111,7 +111,7 @@
3193 relation_prefix='neutron',
3194 ssl_dir=NEUTRON_CONF_DIR)],
3195 'services': ['neutron-plugin-openvswitch-agent'],
3196- 'packages': [[headers_package()] + determine_dkms_package(),
3197+ 'packages': [determine_dkms_package(),
3198 ['neutron-plugin-openvswitch-agent']],
3199 'server_packages': ['neutron-server',
3200 'neutron-plugin-openvswitch'],
3201@@ -155,7 +155,7 @@
3202 relation_prefix='neutron',
3203 ssl_dir=NEUTRON_CONF_DIR)],
3204 'services': [],
3205- 'packages': [[headers_package()] + determine_dkms_package(),
3206+ 'packages': [determine_dkms_package(),
3207 ['neutron-plugin-cisco']],
3208 'server_packages': ['neutron-server',
3209 'neutron-plugin-cisco'],
3210@@ -174,7 +174,7 @@
3211 'neutron-dhcp-agent',
3212 'nova-api-metadata',
3213 'etcd'],
3214- 'packages': [[headers_package()] + determine_dkms_package(),
3215+ 'packages': [determine_dkms_package(),
3216 ['calico-compute',
3217 'bird',
3218 'neutron-dhcp-agent',
3219@@ -204,11 +204,25 @@
3220 database=config('database'),
3221 ssl_dir=NEUTRON_CONF_DIR)],
3222 'services': [],
3223- 'packages': [['plumgrid-lxc'],
3224- ['iovisor-dkms']],
3225+ 'packages': ['plumgrid-lxc',
3226+ 'iovisor-dkms'],
3227 'server_packages': ['neutron-server',
3228 'neutron-plugin-plumgrid'],
3229 'server_services': ['neutron-server']
3230+ },
3231+ 'midonet': {
3232+ 'config': '/etc/neutron/plugins/midonet/midonet.ini',
3233+ 'driver': 'midonet.neutron.plugin.MidonetPluginV2',
3234+ 'contexts': [
3235+ context.SharedDBContext(user=config('neutron-database-user'),
3236+ database=config('neutron-database'),
3237+ relation_prefix='neutron',
3238+ ssl_dir=NEUTRON_CONF_DIR)],
3239+ 'services': [],
3240+ 'packages': [determine_dkms_package()],
3241+ 'server_packages': ['neutron-server',
3242+ 'python-neutron-plugin-midonet'],
3243+ 'server_services': ['neutron-server']
3244 }
3245 }
3246 if release >= 'icehouse':
3247@@ -219,6 +233,26 @@
3248 'neutron-plugin-ml2']
3249 # NOTE: patch in vmware renames nvp->nsx for icehouse onwards
3250 plugins['nvp'] = plugins['nsx']
3251+ if release >= 'kilo':
3252+ plugins['midonet']['driver'] = (
3253+ 'neutron.plugins.midonet.plugin.MidonetPluginV2')
3254+ if release >= 'liberty':
3255+ plugins['midonet']['driver'] = (
3256+ 'midonet.neutron.plugin_v1.MidonetPluginV2')
3257+ plugins['midonet']['server_packages'].remove(
3258+ 'python-neutron-plugin-midonet')
3259+ plugins['midonet']['server_packages'].append(
3260+ 'python-networking-midonet')
3261+ plugins['plumgrid']['driver'] = (
3262+ 'networking_plumgrid.neutron.plugins.plugin.NeutronPluginPLUMgridV2')
3263+ plugins['plumgrid']['server_packages'].remove(
3264+ 'neutron-plugin-plumgrid')
3265+ if release >= 'mitaka':
3266+ plugins['nsx']['server_packages'].remove('neutron-plugin-vmware')
3267+ plugins['nsx']['server_packages'].append('python-vmware-nsx')
3268+ plugins['nsx']['config'] = '/etc/neutron/nsx.ini'
3269+ plugins['vsp']['driver'] = (
3270+ 'nuage_neutron.plugins.nuage.plugin.NuagePlugin')
3271 return plugins
3272
3273
3274@@ -310,10 +344,10 @@
3275 def parse_data_port_mappings(mappings, default_bridge='br-data'):
3276 """Parse data port mappings.
3277
3278- Mappings must be a space-delimited list of port:bridge mappings.
3279+ Mappings must be a space-delimited list of bridge:port.
3280
3281- Returns dict of the form {port:bridge} where port may be an mac address or
3282- interface name.
3283+ Returns dict of the form {port:bridge} where ports may be mac addresses or
3284+ interface names.
3285 """
3286
3287 # NOTE(dosaboy): we use rvalue for key to allow multiple values to be
3288
3289=== modified file 'charmhelpers/contrib/openstack/templates/__init__.py'
3290--- charmhelpers/contrib/openstack/templates/__init__.py 2015-09-28 20:09:02 +0000
3291+++ charmhelpers/contrib/openstack/templates/__init__.py 2017-03-04 02:21:16 +0000
3292@@ -1,18 +1,16 @@
3293 # Copyright 2014-2015 Canonical Limited.
3294 #
3295-# This file is part of charm-helpers.
3296-#
3297-# charm-helpers is free software: you can redistribute it and/or modify
3298-# it under the terms of the GNU Lesser General Public License version 3 as
3299-# published by the Free Software Foundation.
3300-#
3301-# charm-helpers is distributed in the hope that it will be useful,
3302-# but WITHOUT ANY WARRANTY; without even the implied warranty of
3303-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3304-# GNU Lesser General Public License for more details.
3305-#
3306-# You should have received a copy of the GNU Lesser General Public License
3307-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3308+# Licensed under the Apache License, Version 2.0 (the "License");
3309+# you may not use this file except in compliance with the License.
3310+# You may obtain a copy of the License at
3311+#
3312+# http://www.apache.org/licenses/LICENSE-2.0
3313+#
3314+# Unless required by applicable law or agreed to in writing, software
3315+# distributed under the License is distributed on an "AS IS" BASIS,
3316+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3317+# See the License for the specific language governing permissions and
3318+# limitations under the License.
3319
3320 # dummy __init__.py to fool syncer into thinking this is a syncable python
3321 # module
3322
3323=== modified file 'charmhelpers/contrib/openstack/templates/haproxy.cfg'
3324--- charmhelpers/contrib/openstack/templates/haproxy.cfg 2015-09-28 20:09:02 +0000
3325+++ charmhelpers/contrib/openstack/templates/haproxy.cfg 2017-03-04 02:21:16 +0000
3326@@ -12,27 +12,35 @@
3327 option tcplog
3328 option dontlognull
3329 retries 3
3330- timeout queue 1000
3331- timeout connect 1000
3332-{% if haproxy_client_timeout -%}
3333+{%- if haproxy_queue_timeout %}
3334+ timeout queue {{ haproxy_queue_timeout }}
3335+{%- else %}
3336+ timeout queue 5000
3337+{%- endif %}
3338+{%- if haproxy_connect_timeout %}
3339+ timeout connect {{ haproxy_connect_timeout }}
3340+{%- else %}
3341+ timeout connect 5000
3342+{%- endif %}
3343+{%- if haproxy_client_timeout %}
3344 timeout client {{ haproxy_client_timeout }}
3345-{% else -%}
3346+{%- else %}
3347 timeout client 30000
3348-{% endif -%}
3349-
3350-{% if haproxy_server_timeout -%}
3351+{%- endif %}
3352+{%- if haproxy_server_timeout %}
3353 timeout server {{ haproxy_server_timeout }}
3354-{% else -%}
3355+{%- else %}
3356 timeout server 30000
3357-{% endif -%}
3358+{%- endif %}
3359
3360-listen stats {{ stat_port }}
3361+listen stats
3362+ bind {{ local_host }}:{{ stat_port }}
3363 mode http
3364 stats enable
3365 stats hide-version
3366 stats realm Haproxy\ Statistics
3367 stats uri /
3368- stats auth admin:password
3369+ stats auth admin:{{ stat_password }}
3370
3371 {% if frontends -%}
3372 {% for service, ports in service_ports.items() -%}
3373
3374=== added file 'charmhelpers/contrib/openstack/templates/memcached.conf'
3375--- charmhelpers/contrib/openstack/templates/memcached.conf 1970-01-01 00:00:00 +0000
3376+++ charmhelpers/contrib/openstack/templates/memcached.conf 2017-03-04 02:21:16 +0000
3377@@ -0,0 +1,53 @@
3378+###############################################################################
3379+# [ WARNING ]
3380+# memcached configuration file maintained by Juju
3381+# local changes may be overwritten.
3382+###############################################################################
3383+
3384+# memcached default config file
3385+# 2003 - Jay Bonci <jaybonci@debian.org>
3386+# This configuration file is read by the start-memcached script provided as
3387+# part of the Debian GNU/Linux distribution.
3388+
3389+# Run memcached as a daemon. This command is implied, and is not needed for the
3390+# daemon to run. See the README.Debian that comes with this package for more
3391+# information.
3392+-d
3393+
3394+# Log memcached's output to /var/log/memcached
3395+logfile /var/log/memcached.log
3396+
3397+# Be verbose
3398+# -v
3399+
3400+# Be even more verbose (print client commands as well)
3401+# -vv
3402+
3403+# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default
3404+# Note that the daemon will grow to this size, but does not start out holding this much
3405+# memory
3406+-m 64
3407+
3408+# Default connection port is 11211
3409+-p {{ memcache_port }}
3410+
3411+# Run the daemon as root. The start-memcached will default to running as root if no
3412+# -u command is present in this config file
3413+-u memcache
3414+
3415+# Specify which IP address to listen on. The default is to listen on all IP addresses
3416+# This parameter is one of the only security measures that memcached has, so make sure
3417+# it's listening on a firewalled interface.
3418+-l {{ memcache_server }}
3419+
3420+# Limit the number of simultaneous incoming connections. The daemon default is 1024
3421+# -c 1024
3422+
3423+# Lock down all paged memory. Consult with the README and homepage before you do this
3424+# -k
3425+
3426+# Return error when memory is exhausted (rather than removing items)
3427+# -M
3428+
3429+# Maximize core file limit
3430+# -r
3431
3432=== modified file 'charmhelpers/contrib/openstack/templates/openstack_https_frontend'
3433--- charmhelpers/contrib/openstack/templates/openstack_https_frontend 2015-09-28 20:09:02 +0000
3434+++ charmhelpers/contrib/openstack/templates/openstack_https_frontend 2017-03-04 02:21:16 +0000
3435@@ -6,11 +6,16 @@
3436 <VirtualHost {{ address }}:{{ ext }}>
3437 ServerName {{ endpoint }}
3438 SSLEngine on
3439+ SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2
3440+ SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM
3441 SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
3442+ # See LP 1484489 - this is to support <= 2.4.7 and >= 2.4.8
3443+ SSLCertificateChainFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
3444 SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }}
3445 ProxyPass / http://localhost:{{ int }}/
3446 ProxyPassReverse / http://localhost:{{ int }}/
3447 ProxyPreserveHost on
3448+ RequestHeader set X-Forwarded-Proto "https"
3449 </VirtualHost>
3450 {% endfor -%}
3451 <Proxy *>
3452
3453=== modified file 'charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf'
3454--- charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf 2015-09-28 20:09:02 +0000
3455+++ charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf 2017-03-04 02:21:16 +0000
3456@@ -6,11 +6,16 @@
3457 <VirtualHost {{ address }}:{{ ext }}>
3458 ServerName {{ endpoint }}
3459 SSLEngine on
3460+ SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2
3461+ SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM
3462 SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
3463+ # See LP 1484489 - this is to support <= 2.4.7 and >= 2.4.8
3464+ SSLCertificateChainFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
3465 SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }}
3466 ProxyPass / http://localhost:{{ int }}/
3467 ProxyPassReverse / http://localhost:{{ int }}/
3468 ProxyPreserveHost on
3469+ RequestHeader set X-Forwarded-Proto "https"
3470 </VirtualHost>
3471 {% endfor -%}
3472 <Proxy *>
3473
3474=== modified file 'charmhelpers/contrib/openstack/templates/section-keystone-authtoken'
3475--- charmhelpers/contrib/openstack/templates/section-keystone-authtoken 2015-09-28 20:09:02 +0000
3476+++ charmhelpers/contrib/openstack/templates/section-keystone-authtoken 2017-03-04 02:21:16 +0000
3477@@ -1,9 +1,12 @@
3478 {% if auth_host -%}
3479 [keystone_authtoken]
3480-identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }}
3481-auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }}
3482-admin_tenant_name = {{ admin_tenant_name }}
3483-admin_user = {{ admin_user }}
3484-admin_password = {{ admin_password }}
3485+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
3486+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
3487+auth_plugin = password
3488+project_domain_id = default
3489+user_domain_id = default
3490+project_name = {{ admin_tenant_name }}
3491+username = {{ admin_user }}
3492+password = {{ admin_password }}
3493 signing_dir = {{ signing_dir }}
3494 {% endif -%}
3495
3496=== added file 'charmhelpers/contrib/openstack/templates/section-keystone-authtoken-legacy'
3497--- charmhelpers/contrib/openstack/templates/section-keystone-authtoken-legacy 1970-01-01 00:00:00 +0000
3498+++ charmhelpers/contrib/openstack/templates/section-keystone-authtoken-legacy 2017-03-04 02:21:16 +0000
3499@@ -0,0 +1,10 @@
3500+{% if auth_host -%}
3501+[keystone_authtoken]
3502+# Juno specific config (Bug #1557223)
3503+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }}
3504+identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
3505+admin_tenant_name = {{ admin_tenant_name }}
3506+admin_user = {{ admin_user }}
3507+admin_password = {{ admin_password }}
3508+signing_dir = {{ signing_dir }}
3509+{% endif -%}
3510
3511=== added file 'charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka'
3512--- charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka 1970-01-01 00:00:00 +0000
3513+++ charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka 2017-03-04 02:21:16 +0000
3514@@ -0,0 +1,20 @@
3515+{% if auth_host -%}
3516+[keystone_authtoken]
3517+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
3518+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
3519+auth_type = password
3520+{% if api_version == "3" -%}
3521+project_domain_name = {{ admin_domain_name }}
3522+user_domain_name = {{ admin_domain_name }}
3523+{% else -%}
3524+project_domain_name = default
3525+user_domain_name = default
3526+{% endif -%}
3527+project_name = {{ admin_tenant_name }}
3528+username = {{ admin_user }}
3529+password = {{ admin_password }}
3530+signing_dir = {{ signing_dir }}
3531+{% if use_memcache == true %}
3532+memcached_servers = {{ memcache_url }}
3533+{% endif -%}
3534+{% endif -%}
3535
3536=== added file 'charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf'
3537--- charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf 1970-01-01 00:00:00 +0000
3538+++ charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf 2017-03-04 02:21:16 +0000
3539@@ -0,0 +1,100 @@
3540+# Configuration file maintained by Juju. Local changes may be overwritten.
3541+
3542+{% if port -%}
3543+Listen {{ port }}
3544+{% endif -%}
3545+
3546+{% if admin_port -%}
3547+Listen {{ admin_port }}
3548+{% endif -%}
3549+
3550+{% if public_port -%}
3551+Listen {{ public_port }}
3552+{% endif -%}
3553+
3554+{% if port -%}
3555+<VirtualHost *:{{ port }}>
3556+ WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
3557+{% if python_path -%}
3558+ python-path={{ python_path }} \
3559+{% endif -%}
3560+ display-name=%{GROUP}
3561+ WSGIProcessGroup {{ service_name }}
3562+ WSGIScriptAlias / {{ script }}
3563+ WSGIApplicationGroup %{GLOBAL}
3564+ WSGIPassAuthorization On
3565+ <IfVersion >= 2.4>
3566+ ErrorLogFormat "%{cu}t %M"
3567+ </IfVersion>
3568+ ErrorLog /var/log/apache2/{{ service_name }}_error.log
3569+ CustomLog /var/log/apache2/{{ service_name }}_access.log combined
3570+
3571+ <Directory {{ usr_bin }}>
3572+ <IfVersion >= 2.4>
3573+ Require all granted
3574+ </IfVersion>
3575+ <IfVersion < 2.4>
3576+ Order allow,deny
3577+ Allow from all
3578+ </IfVersion>
3579+ </Directory>
3580+</VirtualHost>
3581+{% endif -%}
3582+
3583+{% if admin_port -%}
3584+<VirtualHost *:{{ admin_port }}>
3585+ WSGIDaemonProcess {{ service_name }}-admin processes={{ admin_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
3586+{% if python_path -%}
3587+ python-path={{ python_path }} \
3588+{% endif -%}
3589+ display-name=%{GROUP}
3590+ WSGIProcessGroup {{ service_name }}-admin
3591+ WSGIScriptAlias / {{ admin_script }}
3592+ WSGIApplicationGroup %{GLOBAL}
3593+ WSGIPassAuthorization On
3594+ <IfVersion >= 2.4>
3595+ ErrorLogFormat "%{cu}t %M"
3596+ </IfVersion>
3597+ ErrorLog /var/log/apache2/{{ service_name }}_error.log
3598+ CustomLog /var/log/apache2/{{ service_name }}_access.log combined
3599+
3600+ <Directory {{ usr_bin }}>
3601+ <IfVersion >= 2.4>
3602+ Require all granted
3603+ </IfVersion>
3604+ <IfVersion < 2.4>
3605+ Order allow,deny
3606+ Allow from all
3607+ </IfVersion>
3608+ </Directory>
3609+</VirtualHost>
3610+{% endif -%}
3611+
3612+{% if public_port -%}
3613+<VirtualHost *:{{ public_port }}>
3614+ WSGIDaemonProcess {{ service_name }}-public processes={{ public_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
3615+{% if python_path -%}
3616+ python-path={{ python_path }} \
3617+{% endif -%}
3618+ display-name=%{GROUP}
3619+ WSGIProcessGroup {{ service_name }}-public
3620+ WSGIScriptAlias / {{ public_script }}
3621+ WSGIApplicationGroup %{GLOBAL}
3622+ WSGIPassAuthorization On
3623+ <IfVersion >= 2.4>
3624+ ErrorLogFormat "%{cu}t %M"
3625+ </IfVersion>
3626+ ErrorLog /var/log/apache2/{{ service_name }}_error.log
3627+ CustomLog /var/log/apache2/{{ service_name }}_access.log combined
3628+
3629+ <Directory {{ usr_bin }}>
3630+ <IfVersion >= 2.4>
3631+ Require all granted
3632+ </IfVersion>
3633+ <IfVersion < 2.4>
3634+ Order allow,deny
3635+ Allow from all
3636+ </IfVersion>
3637+ </Directory>
3638+</VirtualHost>
3639+{% endif -%}
3640
3641=== modified file 'charmhelpers/contrib/openstack/templating.py'
3642--- charmhelpers/contrib/openstack/templating.py 2015-09-28 20:09:02 +0000
3643+++ charmhelpers/contrib/openstack/templating.py 2017-03-04 02:21:16 +0000
3644@@ -1,18 +1,16 @@
3645 # Copyright 2014-2015 Canonical Limited.
3646 #
3647-# This file is part of charm-helpers.
3648-#
3649-# charm-helpers is free software: you can redistribute it and/or modify
3650-# it under the terms of the GNU Lesser General Public License version 3 as
3651-# published by the Free Software Foundation.
3652-#
3653-# charm-helpers is distributed in the hope that it will be useful,
3654-# but WITHOUT ANY WARRANTY; without even the implied warranty of
3655-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3656-# GNU Lesser General Public License for more details.
3657-#
3658-# You should have received a copy of the GNU Lesser General Public License
3659-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3660+# Licensed under the Apache License, Version 2.0 (the "License");
3661+# you may not use this file except in compliance with the License.
3662+# You may obtain a copy of the License at
3663+#
3664+# http://www.apache.org/licenses/LICENSE-2.0
3665+#
3666+# Unless required by applicable law or agreed to in writing, software
3667+# distributed under the License is distributed on an "AS IS" BASIS,
3668+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3669+# See the License for the specific language governing permissions and
3670+# limitations under the License.
3671
3672 import os
3673
3674@@ -30,7 +28,10 @@
3675 from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
3676 except ImportError:
3677 apt_update(fatal=True)
3678- apt_install('python-jinja2', fatal=True)
3679+ if six.PY2:
3680+ apt_install('python-jinja2', fatal=True)
3681+ else:
3682+ apt_install('python3-jinja2', fatal=True)
3683 from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
3684
3685
3686@@ -209,7 +210,10 @@
3687 # if this code is running, the object is created pre-install hook.
3688 # jinja2 shouldn't get touched until the module is reloaded on next
3689 # hook execution, with proper jinja2 bits successfully imported.
3690- apt_install('python-jinja2')
3691+ if six.PY2:
3692+ apt_install('python-jinja2')
3693+ else:
3694+ apt_install('python3-jinja2')
3695
3696 def register(self, config_file, contexts):
3697 """
3698
3699=== modified file 'charmhelpers/contrib/openstack/utils.py'
3700--- charmhelpers/contrib/openstack/utils.py 2015-09-28 20:09:02 +0000
3701+++ charmhelpers/contrib/openstack/utils.py 2017-03-04 02:21:16 +0000
3702@@ -1,18 +1,16 @@
3703 # Copyright 2014-2015 Canonical Limited.
3704 #
3705-# This file is part of charm-helpers.
3706-#
3707-# charm-helpers is free software: you can redistribute it and/or modify
3708-# it under the terms of the GNU Lesser General Public License version 3 as
3709-# published by the Free Software Foundation.
3710-#
3711-# charm-helpers is distributed in the hope that it will be useful,
3712-# but WITHOUT ANY WARRANTY; without even the implied warranty of
3713-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3714-# GNU Lesser General Public License for more details.
3715-#
3716-# You should have received a copy of the GNU Lesser General Public License
3717-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3718+# Licensed under the Apache License, Version 2.0 (the "License");
3719+# you may not use this file except in compliance with the License.
3720+# You may obtain a copy of the License at
3721+#
3722+# http://www.apache.org/licenses/LICENSE-2.0
3723+#
3724+# Unless required by applicable law or agreed to in writing, software
3725+# distributed under the License is distributed on an "AS IS" BASIS,
3726+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3727+# See the License for the specific language governing permissions and
3728+# limitations under the License.
3729
3730 # Common python helper functions used for OpenStack charms.
3731 from collections import OrderedDict
3732@@ -23,9 +21,14 @@
3733 import os
3734 import sys
3735 import re
3736+import itertools
3737+import functools
3738+import shutil
3739
3740 import six
3741+import tempfile
3742 import traceback
3743+import uuid
3744 import yaml
3745
3746 from charmhelpers.contrib.network import ip
3747@@ -40,11 +43,16 @@
3748 config,
3749 log as juju_log,
3750 charm_dir,
3751+ DEBUG,
3752 INFO,
3753+ ERROR,
3754+ related_units,
3755 relation_ids,
3756 relation_set,
3757+ service_name,
3758 status_set,
3759- hook_name
3760+ hook_name,
3761+ application_version_set,
3762 )
3763
3764 from charmhelpers.contrib.storage.linux.lvm import (
3765@@ -56,6 +64,7 @@
3766 from charmhelpers.contrib.network.ip import (
3767 get_ipv6_addr,
3768 is_ipv6,
3769+ port_has_listener,
3770 )
3771
3772 from charmhelpers.contrib.python.packages import (
3773@@ -63,10 +72,24 @@
3774 pip_install,
3775 )
3776
3777-from charmhelpers.core.host import lsb_release, mounts, umount
3778-from charmhelpers.fetch import apt_install, apt_cache, install_remote
3779+from charmhelpers.core.host import (
3780+ lsb_release,
3781+ mounts,
3782+ umount,
3783+ service_running,
3784+ service_pause,
3785+ service_resume,
3786+ restart_on_change_helper,
3787+)
3788+from charmhelpers.fetch import (
3789+ apt_install,
3790+ apt_cache,
3791+ install_remote,
3792+ get_upstream_version
3793+)
3794 from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
3795 from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
3796+from charmhelpers.contrib.openstack.exceptions import OSContextError
3797
3798 CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
3799 CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
3800@@ -84,6 +107,9 @@
3801 ('utopic', 'juno'),
3802 ('vivid', 'kilo'),
3803 ('wily', 'liberty'),
3804+ ('xenial', 'mitaka'),
3805+ ('yakkety', 'newton'),
3806+ ('zesty', 'ocata'),
3807 ])
3808
3809
3810@@ -97,63 +123,118 @@
3811 ('2014.2', 'juno'),
3812 ('2015.1', 'kilo'),
3813 ('2015.2', 'liberty'),
3814+ ('2016.1', 'mitaka'),
3815+ ('2016.2', 'newton'),
3816+ ('2017.1', 'ocata'),
3817 ])
3818
3819-# The ugly duckling
3820+# The ugly duckling - must list releases oldest to newest
3821 SWIFT_CODENAMES = OrderedDict([
3822- ('1.4.3', 'diablo'),
3823- ('1.4.8', 'essex'),
3824- ('1.7.4', 'folsom'),
3825- ('1.8.0', 'grizzly'),
3826- ('1.7.7', 'grizzly'),
3827- ('1.7.6', 'grizzly'),
3828- ('1.10.0', 'havana'),
3829- ('1.9.1', 'havana'),
3830- ('1.9.0', 'havana'),
3831- ('1.13.1', 'icehouse'),
3832- ('1.13.0', 'icehouse'),
3833- ('1.12.0', 'icehouse'),
3834- ('1.11.0', 'icehouse'),
3835- ('2.0.0', 'juno'),
3836- ('2.1.0', 'juno'),
3837- ('2.2.0', 'juno'),
3838- ('2.2.1', 'kilo'),
3839- ('2.2.2', 'kilo'),
3840- ('2.3.0', 'liberty'),
3841- ('2.4.0', 'liberty'),
3842+ ('diablo',
3843+ ['1.4.3']),
3844+ ('essex',
3845+ ['1.4.8']),
3846+ ('folsom',
3847+ ['1.7.4']),
3848+ ('grizzly',
3849+ ['1.7.6', '1.7.7', '1.8.0']),
3850+ ('havana',
3851+ ['1.9.0', '1.9.1', '1.10.0']),
3852+ ('icehouse',
3853+ ['1.11.0', '1.12.0', '1.13.0', '1.13.1']),
3854+ ('juno',
3855+ ['2.0.0', '2.1.0', '2.2.0']),
3856+ ('kilo',
3857+ ['2.2.1', '2.2.2']),
3858+ ('liberty',
3859+ ['2.3.0', '2.4.0', '2.5.0']),
3860+ ('mitaka',
3861+ ['2.5.0', '2.6.0', '2.7.0']),
3862+ ('newton',
3863+ ['2.8.0', '2.9.0', '2.10.0']),
3864+ ('ocata',
3865+ ['2.11.0', '2.12.0', '2.13.0']),
3866 ])
3867
3868 # >= Liberty version->codename mapping
3869 PACKAGE_CODENAMES = {
3870 'nova-common': OrderedDict([
3871- ('12.0.0', 'liberty'),
3872+ ('12', 'liberty'),
3873+ ('13', 'mitaka'),
3874+ ('14', 'newton'),
3875+ ('15', 'ocata'),
3876 ]),
3877 'neutron-common': OrderedDict([
3878- ('7.0.0', 'liberty'),
3879+ ('7', 'liberty'),
3880+ ('8', 'mitaka'),
3881+ ('9', 'newton'),
3882+ ('10', 'ocata'),
3883 ]),
3884 'cinder-common': OrderedDict([
3885- ('7.0.0', 'liberty'),
3886+ ('7', 'liberty'),
3887+ ('8', 'mitaka'),
3888+ ('9', 'newton'),
3889+ ('10', 'ocata'),
3890 ]),
3891 'keystone': OrderedDict([
3892- ('8.0.0', 'liberty'),
3893+ ('8', 'liberty'),
3894+ ('9', 'mitaka'),
3895+ ('10', 'newton'),
3896+ ('11', 'ocata'),
3897 ]),
3898 'horizon-common': OrderedDict([
3899- ('8.0.0', 'liberty'),
3900+ ('8', 'liberty'),
3901+ ('9', 'mitaka'),
3902+ ('10', 'newton'),
3903+ ('11', 'ocata'),
3904 ]),
3905 'ceilometer-common': OrderedDict([
3906- ('5.0.0', 'liberty'),
3907+ ('5', 'liberty'),
3908+ ('6', 'mitaka'),
3909+ ('7', 'newton'),
3910+ ('8', 'ocata'),
3911 ]),
3912 'heat-common': OrderedDict([
3913- ('5.0.0', 'liberty'),
3914+ ('5', 'liberty'),
3915+ ('6', 'mitaka'),
3916+ ('7', 'newton'),
3917+ ('8', 'ocata'),
3918 ]),
3919 'glance-common': OrderedDict([
3920- ('11.0.0', 'liberty'),
3921+ ('11', 'liberty'),
3922+ ('12', 'mitaka'),
3923+ ('13', 'newton'),
3924+ ('14', 'ocata'),
3925 ]),
3926 'openstack-dashboard': OrderedDict([
3927- ('8.0.0', 'liberty'),
3928+ ('8', 'liberty'),
3929+ ('9', 'mitaka'),
3930+ ('10', 'newton'),
3931+ ('11', 'ocata'),
3932 ]),
3933 }
3934
3935+GIT_DEFAULT_REPOS = {
3936+ 'requirements': 'git://github.com/openstack/requirements',
3937+ 'cinder': 'git://github.com/openstack/cinder',
3938+ 'glance': 'git://github.com/openstack/glance',
3939+ 'horizon': 'git://github.com/openstack/horizon',
3940+ 'keystone': 'git://github.com/openstack/keystone',
3941+ 'networking-hyperv': 'git://github.com/openstack/networking-hyperv',
3942+ 'neutron': 'git://github.com/openstack/neutron',
3943+ 'neutron-fwaas': 'git://github.com/openstack/neutron-fwaas',
3944+ 'neutron-lbaas': 'git://github.com/openstack/neutron-lbaas',
3945+ 'neutron-vpnaas': 'git://github.com/openstack/neutron-vpnaas',
3946+ 'nova': 'git://github.com/openstack/nova',
3947+}
3948+
3949+GIT_DEFAULT_BRANCHES = {
3950+ 'liberty': 'stable/liberty',
3951+ 'mitaka': 'stable/mitaka',
3952+ 'newton': 'stable/newton',
3953+ 'master': 'master',
3954+}
3955+
3956 DEFAULT_LOOPBACK_SIZE = '5G'
3957
3958
3959@@ -213,6 +294,44 @@
3960 error_out(e)
3961
3962
3963+def get_os_version_codename_swift(codename):
3964+ '''Determine OpenStack version number of swift from codename.'''
3965+ for k, v in six.iteritems(SWIFT_CODENAMES):
3966+ if k == codename:
3967+ return v[-1]
3968+ e = 'Could not derive swift version for '\
3969+ 'codename: %s' % codename
3970+ error_out(e)
3971+
3972+
3973+def get_swift_codename(version):
3974+ '''Determine OpenStack codename that corresponds to swift version.'''
3975+ codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v]
3976+
3977+ if len(codenames) > 1:
3978+ # If more than one release codename contains this version we determine
3979+ # the actual codename based on the highest available install source.
3980+ for codename in reversed(codenames):
3981+ releases = UBUNTU_OPENSTACK_RELEASE
3982+ release = [k for k, v in six.iteritems(releases) if codename in v]
3983+ ret = subprocess.check_output(['apt-cache', 'policy', 'swift'])
3984+ if codename in ret or release[0] in ret:
3985+ return codename
3986+ elif len(codenames) == 1:
3987+ return codenames[0]
3988+
3989+ # NOTE: fallback - attempt to match with just major.minor version
3990+ match = re.match('^(\d+)\.(\d+)', version)
3991+ if match:
3992+ major_minor_version = match.group(0)
3993+ for codename, versions in six.iteritems(SWIFT_CODENAMES):
3994+ for release_version in versions:
3995+ if release_version.startswith(major_minor_version):
3996+ return codename
3997+
3998+ return None
3999+
4000+
4001 def get_os_codename_package(package, fatal=True):
4002 '''Derive OpenStack release codename from an installed package.'''
4003 import apt_pkg as apt
4004@@ -237,25 +356,30 @@
4005 error_out(e)
4006
4007 vers = apt.upstream_version(pkg.current_ver.ver_str)
4008- match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
4009+ if 'swift' in pkg.name:
4010+ # Fully x.y.z match for swift versions
4011+ match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
4012+ else:
4013+ # x.y match only for 20XX.X
4014+ # and ignore patch level for other packages
4015+ match = re.match('^(\d+)\.(\d+)', vers)
4016+
4017 if match:
4018 vers = match.group(0)
4019
4020+ # Generate a major version number for newer semantic
4021+ # versions of openstack projects
4022+ major_vers = vers.split('.')[0]
4023 # >= Liberty independent project versions
4024 if (package in PACKAGE_CODENAMES and
4025- vers in PACKAGE_CODENAMES[package]):
4026- return PACKAGE_CODENAMES[package][vers]
4027+ major_vers in PACKAGE_CODENAMES[package]):
4028+ return PACKAGE_CODENAMES[package][major_vers]
4029 else:
4030 # < Liberty co-ordinated project versions
4031 try:
4032 if 'swift' in pkg.name:
4033- swift_vers = vers[:5]
4034- if swift_vers not in SWIFT_CODENAMES:
4035- # Deal with 1.10.0 upward
4036- swift_vers = vers[:6]
4037- return SWIFT_CODENAMES[swift_vers]
4038+ return get_swift_codename(vers)
4039 else:
4040- vers = vers[:6]
4041 return OPENSTACK_CODENAMES[vers]
4042 except KeyError:
4043 if not fatal:
4044@@ -273,12 +397,14 @@
4045
4046 if 'swift' in pkg:
4047 vers_map = SWIFT_CODENAMES
4048+ for cname, version in six.iteritems(vers_map):
4049+ if cname == codename:
4050+ return version[-1]
4051 else:
4052 vers_map = OPENSTACK_CODENAMES
4053-
4054- for version, cname in six.iteritems(vers_map):
4055- if cname == codename:
4056- return version
4057+ for version, cname in six.iteritems(vers_map):
4058+ if cname == codename:
4059+ return version
4060 # e = "Could not determine OpenStack version for package: %s" % pkg
4061 # error_out(e)
4062
4063@@ -286,29 +412,72 @@
4064 os_rel = None
4065
4066
4067-def os_release(package, base='essex'):
4068+def reset_os_release():
4069+ '''Unset the cached os_release version'''
4070+ global os_rel
4071+ os_rel = None
4072+
4073+
4074+def os_release(package, base='essex', reset_cache=False):
4075 '''
4076 Returns OpenStack release codename from a cached global.
4077+
4078+ If reset_cache then unset the cached os_release version and return the
4079+ freshly determined version.
4080+
4081 If the codename can not be determined from either an installed package or
4082 the installation source, the earliest release supported by the charm should
4083 be returned.
4084 '''
4085 global os_rel
4086+ if reset_cache:
4087+ reset_os_release()
4088 if os_rel:
4089 return os_rel
4090- os_rel = (get_os_codename_package(package, fatal=False) or
4091+ os_rel = (git_os_codename_install_source(config('openstack-origin-git')) or
4092+ get_os_codename_package(package, fatal=False) or
4093 get_os_codename_install_source(config('openstack-origin')) or
4094 base)
4095 return os_rel
4096
4097
4098 def import_key(keyid):
4099- cmd = "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 " \
4100- "--recv-keys %s" % keyid
4101- try:
4102- subprocess.check_call(cmd.split(' '))
4103- except subprocess.CalledProcessError:
4104- error_out("Error importing repo key %s" % keyid)
4105+ key = keyid.strip()
4106+ if (key.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----') and
4107+ key.endswith('-----END PGP PUBLIC KEY BLOCK-----')):
4108+ juju_log("PGP key found (looks like ASCII Armor format)", level=DEBUG)
4109+ juju_log("Importing ASCII Armor PGP key", level=DEBUG)
4110+ with tempfile.NamedTemporaryFile() as keyfile:
4111+ with open(keyfile.name, 'w') as fd:
4112+ fd.write(key)
4113+ fd.write("\n")
4114+
4115+ cmd = ['apt-key', 'add', keyfile.name]
4116+ try:
4117+ subprocess.check_call(cmd)
4118+ except subprocess.CalledProcessError:
4119+ error_out("Error importing PGP key '%s'" % key)
4120+ else:
4121+ juju_log("PGP key found (looks like Radix64 format)", level=DEBUG)
4122+ juju_log("Importing PGP key from keyserver", level=DEBUG)
4123+ cmd = ['apt-key', 'adv', '--keyserver',
4124+ 'hkp://keyserver.ubuntu.com:80', '--recv-keys', key]
4125+ try:
4126+ subprocess.check_call(cmd)
4127+ except subprocess.CalledProcessError:
4128+ error_out("Error importing PGP key '%s'" % key)
4129+
4130+
4131+def get_source_and_pgp_key(input):
4132+ """Look for a pgp key ID or ascii-armor key in the given input."""
4133+ index = input.strip()
4134+ index = input.rfind('|')
4135+ if index < 0:
4136+ return input, None
4137+
4138+ key = input[index + 1:].strip('|')
4139+ source = input[:index]
4140+ return source, key
4141
4142
4143 def configure_installation_source(rel):
4144@@ -320,16 +489,16 @@
4145 with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f:
4146 f.write(DISTRO_PROPOSED % ubuntu_rel)
4147 elif rel[:4] == "ppa:":
4148- src = rel
4149+ src, key = get_source_and_pgp_key(rel)
4150+ if key:
4151+ import_key(key)
4152+
4153 subprocess.check_call(["add-apt-repository", "-y", src])
4154 elif rel[:3] == "deb":
4155- l = len(rel.split('|'))
4156- if l == 2:
4157- src, key = rel.split('|')
4158- juju_log("Importing PPA key from keyserver for %s" % src)
4159+ src, key = get_source_and_pgp_key(rel)
4160+ if key:
4161 import_key(key)
4162- elif l == 1:
4163- src = rel
4164+
4165 with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f:
4166 f.write(src)
4167 elif rel[:6] == 'cloud:':
4168@@ -374,6 +543,15 @@
4169 'liberty': 'trusty-updates/liberty',
4170 'liberty/updates': 'trusty-updates/liberty',
4171 'liberty/proposed': 'trusty-proposed/liberty',
4172+ 'mitaka': 'trusty-updates/mitaka',
4173+ 'mitaka/updates': 'trusty-updates/mitaka',
4174+ 'mitaka/proposed': 'trusty-proposed/mitaka',
4175+ 'newton': 'xenial-updates/newton',
4176+ 'newton/updates': 'xenial-updates/newton',
4177+ 'newton/proposed': 'xenial-proposed/newton',
4178+ 'ocata': 'xenial-updates/ocata',
4179+ 'ocata/updates': 'xenial-updates/ocata',
4180+ 'ocata/proposed': 'xenial-proposed/ocata',
4181 }
4182
4183 try:
4184@@ -441,11 +619,16 @@
4185 cur_vers = get_os_version_package(package)
4186 if "swift" in package:
4187 codename = get_os_codename_install_source(src)
4188- available_vers = get_os_version_codename(codename, SWIFT_CODENAMES)
4189+ avail_vers = get_os_version_codename_swift(codename)
4190 else:
4191- available_vers = get_os_version_install_source(src)
4192+ avail_vers = get_os_version_install_source(src)
4193 apt.init()
4194- return apt.version_compare(available_vers, cur_vers) == 1
4195+ if "swift" in package:
4196+ major_cur_vers = cur_vers.split('.', 1)[0]
4197+ major_avail_vers = avail_vers.split('.', 1)[0]
4198+ major_diff = apt.version_compare(major_avail_vers, major_cur_vers)
4199+ return avail_vers > cur_vers and (major_diff == 1 or major_diff == 0)
4200+ return apt.version_compare(avail_vers, cur_vers) == 1
4201
4202
4203 def ensure_block_device(block_device):
4204@@ -502,6 +685,7 @@
4205 else:
4206 zap_disk(block_device)
4207
4208+
4209 is_ip = ip.is_ip
4210 ns_query = ip.ns_query
4211 get_host_ip = ip.get_host_ip
4212@@ -561,7 +745,86 @@
4213 return config('openstack-origin-git') is not None
4214
4215
4216-requirements_dir = None
4217+def git_os_codename_install_source(projects_yaml):
4218+ """
4219+ Returns OpenStack codename of release being installed from source.
4220+ """
4221+ if git_install_requested():
4222+ projects = _git_yaml_load(projects_yaml)
4223+
4224+ if projects in GIT_DEFAULT_BRANCHES.keys():
4225+ if projects == 'master':
4226+ return 'ocata'
4227+ return projects
4228+
4229+ if 'release' in projects:
4230+ if projects['release'] == 'master':
4231+ return 'ocata'
4232+ return projects['release']
4233+
4234+ return None
4235+
4236+
4237+def git_default_repos(projects_yaml):
4238+ """
4239+ Returns default repos if a default openstack-origin-git value is specified.
4240+ """
4241+ service = service_name()
4242+ core_project = service
4243+
4244+ for default, branch in GIT_DEFAULT_BRANCHES.iteritems():
4245+ if projects_yaml == default:
4246+
4247+ # add the requirements repo first
4248+ repo = {
4249+ 'name': 'requirements',
4250+ 'repository': GIT_DEFAULT_REPOS['requirements'],
4251+ 'branch': branch,
4252+ }
4253+ repos = [repo]
4254+
4255+ # neutron-* and nova-* charms require some additional repos
4256+ if service in ['neutron-api', 'neutron-gateway',
4257+ 'neutron-openvswitch']:
4258+ core_project = 'neutron'
4259+ if service == 'neutron-api':
4260+ repo = {
4261+ 'name': 'networking-hyperv',
4262+ 'repository': GIT_DEFAULT_REPOS['networking-hyperv'],
4263+ 'branch': branch,
4264+ }
4265+ repos.append(repo)
4266+ for project in ['neutron-fwaas', 'neutron-lbaas',
4267+ 'neutron-vpnaas', 'nova']:
4268+ repo = {
4269+ 'name': project,
4270+ 'repository': GIT_DEFAULT_REPOS[project],
4271+ 'branch': branch,
4272+ }
4273+ repos.append(repo)
4274+
4275+ elif service in ['nova-cloud-controller', 'nova-compute']:
4276+ core_project = 'nova'
4277+ repo = {
4278+ 'name': 'neutron',
4279+ 'repository': GIT_DEFAULT_REPOS['neutron'],
4280+ 'branch': branch,
4281+ }
4282+ repos.append(repo)
4283+ elif service == 'openstack-dashboard':
4284+ core_project = 'horizon'
4285+
4286+ # finally add the current service's core project repo
4287+ repo = {
4288+ 'name': core_project,
4289+ 'repository': GIT_DEFAULT_REPOS[core_project],
4290+ 'branch': branch,
4291+ }
4292+ repos.append(repo)
4293+
4294+ return yaml.dump(dict(repositories=repos, release=default))
4295+
4296+ return projects_yaml
4297
4298
4299 def _git_yaml_load(projects_yaml):
4300@@ -574,7 +837,10 @@
4301 return yaml.load(projects_yaml)
4302
4303
4304-def git_clone_and_install(projects_yaml, core_project, depth=1):
4305+requirements_dir = None
4306+
4307+
4308+def git_clone_and_install(projects_yaml, core_project):
4309 """
4310 Clone/install all specified OpenStack repositories.
4311
4312@@ -621,18 +887,31 @@
4313 pip_install(p, upgrade=True, proxy=http_proxy,
4314 venv=os.path.join(parent_dir, 'venv'))
4315
4316+ constraints = None
4317 for p in projects['repositories']:
4318 repo = p['repository']
4319 branch = p['branch']
4320+ depth = '1'
4321+ if 'depth' in p.keys():
4322+ depth = p['depth']
4323 if p['name'] == 'requirements':
4324 repo_dir = _git_clone_and_install_single(repo, branch, depth,
4325 parent_dir, http_proxy,
4326 update_requirements=False)
4327 requirements_dir = repo_dir
4328+ constraints = os.path.join(repo_dir, "upper-constraints.txt")
4329+ # upper-constraints didn't exist until after icehouse
4330+ if not os.path.isfile(constraints):
4331+ constraints = None
4332+ # use constraints unless project yaml sets use_constraints to false
4333+ if 'use_constraints' in projects.keys():
4334+ if not projects['use_constraints']:
4335+ constraints = None
4336 else:
4337 repo_dir = _git_clone_and_install_single(repo, branch, depth,
4338 parent_dir, http_proxy,
4339- update_requirements=True)
4340+ update_requirements=True,
4341+ constraints=constraints)
4342
4343 os.environ = old_environ
4344
4345@@ -654,6 +933,8 @@
4346 if projects['repositories'][-1]['name'] != core_project:
4347 error_out('{} git repo must be specified last'.format(core_project))
4348
4349+ _git_ensure_key_exists('release', projects)
4350+
4351
4352 def _git_ensure_key_exists(key, keys):
4353 """
4354@@ -664,23 +945,18 @@
4355
4356
4357 def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy,
4358- update_requirements):
4359+ update_requirements, constraints=None):
4360 """
4361 Clone and install a single git repository.
4362 """
4363- dest_dir = os.path.join(parent_dir, os.path.basename(repo))
4364-
4365 if not os.path.exists(parent_dir):
4366 juju_log('Directory already exists at {}. '
4367 'No need to create directory.'.format(parent_dir))
4368 os.mkdir(parent_dir)
4369
4370- if not os.path.exists(dest_dir):
4371- juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
4372- repo_dir = install_remote(repo, dest=parent_dir, branch=branch,
4373- depth=depth)
4374- else:
4375- repo_dir = dest_dir
4376+ juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch))
4377+ repo_dir = install_remote(
4378+ repo, dest=parent_dir, branch=branch, depth=depth)
4379
4380 venv = os.path.join(parent_dir, 'venv')
4381
4382@@ -692,9 +968,10 @@
4383
4384 juju_log('Installing git repo from dir: {}'.format(repo_dir))
4385 if http_proxy:
4386- pip_install(repo_dir, proxy=http_proxy, venv=venv)
4387+ pip_install(repo_dir, proxy=http_proxy, venv=venv,
4388+ constraints=constraints)
4389 else:
4390- pip_install(repo_dir, venv=venv)
4391+ pip_install(repo_dir, venv=venv, constraints=constraints)
4392
4393 return repo_dir
4394
4395@@ -763,6 +1040,114 @@
4396 return None
4397
4398
4399+def git_generate_systemd_init_files(templates_dir):
4400+ """
4401+ Generate systemd init files.
4402+
4403+ Generates and installs systemd init units and script files based on the
4404+ *.init.in files contained in the templates_dir directory.
4405+
4406+ This code is based on the openstack-pkg-tools package and its init
4407+ script generation, which is used by the OpenStack packages.
4408+ """
4409+ for f in os.listdir(templates_dir):
4410+ # Create the init script and systemd unit file from the template
4411+ if f.endswith(".init.in"):
4412+ init_in_file = f
4413+ init_file = f[:-8]
4414+ service_file = "{}.service".format(init_file)
4415+
4416+ init_in_source = os.path.join(templates_dir, init_in_file)
4417+ init_source = os.path.join(templates_dir, init_file)
4418+ service_source = os.path.join(templates_dir, service_file)
4419+
4420+ init_dest = os.path.join('/etc/init.d', init_file)
4421+ service_dest = os.path.join('/lib/systemd/system', service_file)
4422+
4423+ shutil.copyfile(init_in_source, init_source)
4424+ with open(init_source, 'a') as outfile:
4425+ template = '/usr/share/openstack-pkg-tools/init-script-template'
4426+ with open(template) as infile:
4427+ outfile.write('\n\n{}'.format(infile.read()))
4428+
4429+ cmd = ['pkgos-gen-systemd-unit', init_in_source]
4430+ subprocess.check_call(cmd)
4431+
4432+ if os.path.exists(init_dest):
4433+ os.remove(init_dest)
4434+ if os.path.exists(service_dest):
4435+ os.remove(service_dest)
4436+ shutil.copyfile(init_source, init_dest)
4437+ shutil.copyfile(service_source, service_dest)
4438+ os.chmod(init_dest, 0o755)
4439+
4440+ for f in os.listdir(templates_dir):
4441+ # If there's a service.in file, use it instead of the generated one
4442+ if f.endswith(".service.in"):
4443+ service_in_file = f
4444+ service_file = f[:-3]
4445+
4446+ service_in_source = os.path.join(templates_dir, service_in_file)
4447+ service_source = os.path.join(templates_dir, service_file)
4448+ service_dest = os.path.join('/lib/systemd/system', service_file)
4449+
4450+ shutil.copyfile(service_in_source, service_source)
4451+
4452+ if os.path.exists(service_dest):
4453+ os.remove(service_dest)
4454+ shutil.copyfile(service_source, service_dest)
4455+
4456+ for f in os.listdir(templates_dir):
4457+ # Generate the systemd unit if there's no existing .service.in
4458+ if f.endswith(".init.in"):
4459+ init_in_file = f
4460+ init_file = f[:-8]
4461+ service_in_file = "{}.service.in".format(init_file)
4462+ service_file = "{}.service".format(init_file)
4463+
4464+ init_in_source = os.path.join(templates_dir, init_in_file)
4465+ service_in_source = os.path.join(templates_dir, service_in_file)
4466+ service_source = os.path.join(templates_dir, service_file)
4467+ service_dest = os.path.join('/lib/systemd/system', service_file)
4468+
4469+ if not os.path.exists(service_in_source):
4470+ cmd = ['pkgos-gen-systemd-unit', init_in_source]
4471+ subprocess.check_call(cmd)
4472+
4473+ if os.path.exists(service_dest):
4474+ os.remove(service_dest)
4475+ shutil.copyfile(service_source, service_dest)
4476+
4477+
4478+def git_determine_usr_bin():
4479+ """Return the /usr/bin path for Apache2 config.
4480+
4481+ The /usr/bin path will be located in the virtualenv if the charm
4482+ is configured to deploy from source.
4483+ """
4484+ if git_install_requested():
4485+ projects_yaml = config('openstack-origin-git')
4486+ projects_yaml = git_default_repos(projects_yaml)
4487+ return os.path.join(git_pip_venv_dir(projects_yaml), 'bin')
4488+ else:
4489+ return '/usr/bin'
4490+
4491+
4492+def git_determine_python_path():
4493+ """Return the python-path for Apache2 config.
4494+
4495+ Returns 'None' unless the charm is configured to deploy from source,
4496+ in which case the path of the virtualenv's site-packages is returned.
4497+ """
4498+ if git_install_requested():
4499+ projects_yaml = config('openstack-origin-git')
4500+ projects_yaml = git_default_repos(projects_yaml)
4501+ return os.path.join(git_pip_venv_dir(projects_yaml),
4502+ 'lib/python2.7/site-packages')
4503+ else:
4504+ return None
4505+
4506+
4507 def os_workload_status(configs, required_interfaces, charm_func=None):
4508 """
4509 Decorator to set workload status based on complete contexts
4510@@ -779,56 +1164,155 @@
4511 return wrap
4512
4513
4514-def set_os_workload_status(configs, required_interfaces, charm_func=None):
4515- """
4516- Set workload status based on complete contexts.
4517- status-set missing or incomplete contexts
4518- and juju-log details of missing required data.
4519- charm_func is a charm specific function to run checking
4520- for charm specific requirements such as a VIP setting.
4521- """
4522- incomplete_rel_data = incomplete_relation_data(configs, required_interfaces)
4523- state = 'active'
4524- missing_relations = []
4525- incomplete_relations = []
4526+def set_os_workload_status(configs, required_interfaces, charm_func=None,
4527+ services=None, ports=None):
4528+ """Set the state of the workload status for the charm.
4529+
4530+ This calls _determine_os_workload_status() to get the new state, message
4531+ and sets the status using status_set()
4532+
4533+ @param configs: a templating.OSConfigRenderer() object
4534+ @param required_interfaces: {generic: [specific, specific2, ...]}
4535+ @param charm_func: a callable function that returns state, message. The
4536+ signature is charm_func(configs) -> (state, message)
4537+ @param services: list of strings OR dictionary specifying services/ports
4538+ @param ports: OPTIONAL list of port numbers.
4539+ @returns state, message: the new workload status, user message
4540+ """
4541+ state, message = _determine_os_workload_status(
4542+ configs, required_interfaces, charm_func, services, ports)
4543+ status_set(state, message)
4544+
4545+
4546+def _determine_os_workload_status(
4547+ configs, required_interfaces, charm_func=None,
4548+ services=None, ports=None):
4549+ """Determine the state of the workload status for the charm.
4550+
4551+ This function returns the new workload status for the charm based
4552+ on the state of the interfaces, the paused state and whether the
4553+ services are actually running and any specified ports are open.
4554+
4555+ This checks:
4556+
4557+ 1. if the unit should be paused, that it is actually paused. If so the
4558+ state is 'maintenance' + message, else 'broken'.
4559+ 2. that the interfaces/relations are complete. If they are not then
4560+ it sets the state to either 'broken' or 'waiting' and an appropriate
4561+ message.
4562+ 3. If all the relation data is set, then it checks that the actual
4563+ services really are running. If not it sets the state to 'broken'.
4564+
4565+ If everything is okay then the state returns 'active'.
4566+
4567+ @param configs: a templating.OSConfigRenderer() object
4568+ @param required_interfaces: {generic: [specific, specific2, ...]}
4569+ @param charm_func: a callable function that returns state, message. The
4570+ signature is charm_func(configs) -> (state, message)
4571+ @param services: list of strings OR dictionary specifying services/ports
4572+ @param ports: OPTIONAL list of port numbers.
4573+ @returns state, message: the new workload status, user message
4574+ """
4575+ state, message = _ows_check_if_paused(services, ports)
4576+
4577+ if state is None:
4578+ state, message = _ows_check_generic_interfaces(
4579+ configs, required_interfaces)
4580+
4581+ if state != 'maintenance' and charm_func:
4582+ # _ows_check_charm_func() may modify the state, message
4583+ state, message = _ows_check_charm_func(
4584+ state, message, lambda: charm_func(configs))
4585+
4586+ if state is None:
4587+ state, message = _ows_check_services_running(services, ports)
4588+
4589+ if state is None:
4590+ state = 'active'
4591+ message = "Unit is ready"
4592+ juju_log(message, 'INFO')
4593+
4594+ return state, message
4595+
4596+
4597+def _ows_check_if_paused(services=None, ports=None):
4598+ """Check if the unit is supposed to be paused, and if so check that the
4599+ services/ports (if passed) are actually stopped/not being listened to.
4600+
4601+ if the unit isn't supposed to be paused, just return None, None
4602+
4603+ @param services: OPTIONAL services spec or list of service names.
4604+ @param ports: OPTIONAL list of port numbers.
4605+ @returns state, message or None, None
4606+ """
4607+ if is_unit_paused_set():
4608+ state, message = check_actually_paused(services=services,
4609+ ports=ports)
4610+ if state is None:
4611+ # we're paused okay, so set maintenance and return
4612+ state = "maintenance"
4613+ message = "Paused. Use 'resume' action to resume normal service."
4614+ return state, message
4615+ return None, None
4616+
4617+
4618+def _ows_check_generic_interfaces(configs, required_interfaces):
4619+ """Check the complete contexts to determine the workload status.
4620+
4621+ - Checks for missing or incomplete contexts
4622+ - juju log details of missing required data.
4623+ - determines the correct workload status
4624+ - creates an appropriate message for status_set(...)
4625+
4626+ if there are no problems then the function returns None, None
4627+
4628+ @param configs: a templating.OSConfigRenderer() object
4629+ @params required_interfaces: {generic_interface: [specific_interface], }
4630+ @returns state, message or None, None
4631+ """
4632+ incomplete_rel_data = incomplete_relation_data(configs,
4633+ required_interfaces)
4634+ state = None
4635 message = None
4636- charm_state = None
4637- charm_message = None
4638+ missing_relations = set()
4639+ incomplete_relations = set()
4640
4641- for generic_interface in incomplete_rel_data.keys():
4642+ for generic_interface, relations_states in incomplete_rel_data.items():
4643 related_interface = None
4644 missing_data = {}
4645 # Related or not?
4646- for interface in incomplete_rel_data[generic_interface]:
4647- if incomplete_rel_data[generic_interface][interface].get('related'):
4648+ for interface, relation_state in relations_states.items():
4649+ if relation_state.get('related'):
4650 related_interface = interface
4651- missing_data = incomplete_rel_data[generic_interface][interface].get('missing_data')
4652- # No relation ID for the generic_interface
4653+ missing_data = relation_state.get('missing_data')
4654+ break
4655+ # No relation ID for the generic_interface?
4656 if not related_interface:
4657 juju_log("{} relation is missing and must be related for "
4658 "functionality. ".format(generic_interface), 'WARN')
4659 state = 'blocked'
4660- if generic_interface not in missing_relations:
4661- missing_relations.append(generic_interface)
4662+ missing_relations.add(generic_interface)
4663 else:
4664- # Relation ID exists but no related unit
4665+ # Relation ID eists but no related unit
4666 if not missing_data:
4667- # Edge case relation ID exists but departing
4668- if ('departed' in hook_name() or 'broken' in hook_name()) \
4669- and related_interface in hook_name():
4670+ # Edge case - relation ID exists but departings
4671+ _hook_name = hook_name()
4672+ if (('departed' in _hook_name or 'broken' in _hook_name) and
4673+ related_interface in _hook_name):
4674 state = 'blocked'
4675- if generic_interface not in missing_relations:
4676- missing_relations.append(generic_interface)
4677+ missing_relations.add(generic_interface)
4678 juju_log("{} relation's interface, {}, "
4679 "relationship is departed or broken "
4680 "and is required for functionality."
4681- "".format(generic_interface, related_interface), "WARN")
4682+ "".format(generic_interface, related_interface),
4683+ "WARN")
4684 # Normal case relation ID exists but no related unit
4685 # (joining)
4686 else:
4687- juju_log("{} relations's interface, {}, is related but has "
4688- "no units in the relation."
4689- "".format(generic_interface, related_interface), "INFO")
4690+ juju_log("{} relations's interface, {}, is related but has"
4691+ " no units in the relation."
4692+ "".format(generic_interface, related_interface),
4693+ "INFO")
4694 # Related unit exists and data missing on the relation
4695 else:
4696 juju_log("{} relation's interface, {}, is related awaiting "
4697@@ -837,9 +1321,8 @@
4698 ", ".join(missing_data)), "INFO")
4699 if state != 'blocked':
4700 state = 'waiting'
4701- if generic_interface not in incomplete_relations \
4702- and generic_interface not in missing_relations:
4703- incomplete_relations.append(generic_interface)
4704+ if generic_interface not in missing_relations:
4705+ incomplete_relations.add(generic_interface)
4706
4707 if missing_relations:
4708 message = "Missing relations: {}".format(", ".join(missing_relations))
4709@@ -852,22 +1335,175 @@
4710 "".format(", ".join(incomplete_relations))
4711 state = 'waiting'
4712
4713- # Run charm specific checks
4714- if charm_func:
4715- charm_state, charm_message = charm_func(configs)
4716+ return state, message
4717+
4718+
4719+def _ows_check_charm_func(state, message, charm_func_with_configs):
4720+ """Run a custom check function for the charm to see if it wants to
4721+ change the state. This is only run if not in 'maintenance' and
4722+ tests to see if the new state is more important that the previous
4723+ one determined by the interfaces/relations check.
4724+
4725+ @param state: the previously determined state so far.
4726+ @param message: the user orientated message so far.
4727+ @param charm_func: a callable function that returns state, message
4728+ @returns state, message strings.
4729+ """
4730+ if charm_func_with_configs:
4731+ charm_state, charm_message = charm_func_with_configs()
4732 if charm_state != 'active' and charm_state != 'unknown':
4733 state = workload_state_compare(state, charm_state)
4734 if message:
4735- message = "{} {}".format(message, charm_message)
4736+ charm_message = charm_message.replace("Incomplete relations: ",
4737+ "")
4738+ message = "{}, {}".format(message, charm_message)
4739 else:
4740 message = charm_message
4741-
4742- # Set to active if all requirements have been met
4743- if state == 'active':
4744- message = "Unit is ready"
4745- juju_log(message, "INFO")
4746-
4747- status_set(state, message)
4748+ return state, message
4749+
4750+
4751+def _ows_check_services_running(services, ports):
4752+ """Check that the services that should be running are actually running
4753+ and that any ports specified are being listened to.
4754+
4755+ @param services: list of strings OR dictionary specifying services/ports
4756+ @param ports: list of ports
4757+ @returns state, message: strings or None, None
4758+ """
4759+ messages = []
4760+ state = None
4761+ if services is not None:
4762+ services = _extract_services_list_helper(services)
4763+ services_running, running = _check_running_services(services)
4764+ if not all(running):
4765+ messages.append(
4766+ "Services not running that should be: {}"
4767+ .format(", ".join(_filter_tuples(services_running, False))))
4768+ state = 'blocked'
4769+ # also verify that the ports that should be open are open
4770+ # NB, that ServiceManager objects only OPTIONALLY have ports
4771+ map_not_open, ports_open = (
4772+ _check_listening_on_services_ports(services))
4773+ if not all(ports_open):
4774+ # find which service has missing ports. They are in service
4775+ # order which makes it a bit easier.
4776+ message_parts = {service: ", ".join([str(v) for v in open_ports])
4777+ for service, open_ports in map_not_open.items()}
4778+ message = ", ".join(
4779+ ["{}: [{}]".format(s, sp) for s, sp in message_parts.items()])
4780+ messages.append(
4781+ "Services with ports not open that should be: {}"
4782+ .format(message))
4783+ state = 'blocked'
4784+
4785+ if ports is not None:
4786+ # and we can also check ports which we don't know the service for
4787+ ports_open, ports_open_bools = _check_listening_on_ports_list(ports)
4788+ if not all(ports_open_bools):
4789+ messages.append(
4790+ "Ports which should be open, but are not: {}"
4791+ .format(", ".join([str(p) for p, v in ports_open
4792+ if not v])))
4793+ state = 'blocked'
4794+
4795+ if state is not None:
4796+ message = "; ".join(messages)
4797+ return state, message
4798+
4799+ return None, None
4800+
4801+
4802+def _extract_services_list_helper(services):
4803+ """Extract a OrderedDict of {service: [ports]} of the supplied services
4804+ for use by the other functions.
4805+
4806+ The services object can either be:
4807+ - None : no services were passed (an empty dict is returned)
4808+ - a list of strings
4809+ - A dictionary (optionally OrderedDict) {service_name: {'service': ..}}
4810+ - An array of [{'service': service_name, ...}, ...]
4811+
4812+ @param services: see above
4813+ @returns OrderedDict(service: [ports], ...)
4814+ """
4815+ if services is None:
4816+ return {}
4817+ if isinstance(services, dict):
4818+ services = services.values()
4819+ # either extract the list of services from the dictionary, or if
4820+ # it is a simple string, use that. i.e. works with mixed lists.
4821+ _s = OrderedDict()
4822+ for s in services:
4823+ if isinstance(s, dict) and 'service' in s:
4824+ _s[s['service']] = s.get('ports', [])
4825+ if isinstance(s, str):
4826+ _s[s] = []
4827+ return _s
4828+
4829+
4830+def _check_running_services(services):
4831+ """Check that the services dict provided is actually running and provide
4832+ a list of (service, boolean) tuples for each service.
4833+
4834+ Returns both a zipped list of (service, boolean) and a list of booleans
4835+ in the same order as the services.
4836+
4837+ @param services: OrderedDict of strings: [ports], one for each service to
4838+ check.
4839+ @returns [(service, boolean), ...], : results for checks
4840+ [boolean] : just the result of the service checks
4841+ """
4842+ services_running = [service_running(s) for s in services]
4843+ return list(zip(services, services_running)), services_running
4844+
4845+
4846+def _check_listening_on_services_ports(services, test=False):
4847+ """Check that the unit is actually listening (has the port open) on the
4848+ ports that the service specifies are open. If test is True then the
4849+ function returns the services with ports that are open rather than
4850+ closed.
4851+
4852+ Returns an OrderedDict of service: ports and a list of booleans
4853+
4854+ @param services: OrderedDict(service: [port, ...], ...)
4855+ @param test: default=False, if False, test for closed, otherwise open.
4856+ @returns OrderedDict(service: [port-not-open, ...]...), [boolean]
4857+ """
4858+ test = not(not(test)) # ensure test is True or False
4859+ all_ports = list(itertools.chain(*services.values()))
4860+ ports_states = [port_has_listener('0.0.0.0', p) for p in all_ports]
4861+ map_ports = OrderedDict()
4862+ matched_ports = [p for p, opened in zip(all_ports, ports_states)
4863+ if opened == test] # essentially opened xor test
4864+ for service, ports in services.items():
4865+ set_ports = set(ports).intersection(matched_ports)
4866+ if set_ports:
4867+ map_ports[service] = set_ports
4868+ return map_ports, ports_states
4869+
4870+
4871+def _check_listening_on_ports_list(ports):
4872+ """Check that the ports list given are being listened to
4873+
4874+ Returns a list of ports being listened to and a list of the
4875+ booleans.
4876+
4877+ @param ports: LIST or port numbers.
4878+ @returns [(port_num, boolean), ...], [boolean]
4879+ """
4880+ ports_open = [port_has_listener('0.0.0.0', p) for p in ports]
4881+ return zip(ports, ports_open), ports_open
4882+
4883+
4884+def _filter_tuples(services_states, state):
4885+ """Return a simple list from a list of tuples according to the condition
4886+
4887+ @param services_states: LIST of (string, boolean): service and running
4888+ state.
4889+ @param state: Boolean to match the tuple against.
4890+ @returns [LIST of strings] that matched the tuple RHS.
4891+ """
4892+ return [s for s, b in services_states if b == state]
4893
4894
4895 def workload_state_compare(current_workload_state, workload_state):
4896@@ -892,8 +1528,7 @@
4897
4898
4899 def incomplete_relation_data(configs, required_interfaces):
4900- """
4901- Check complete contexts against required_interfaces
4902+ """Check complete contexts against required_interfaces
4903 Return dictionary of incomplete relation data.
4904
4905 configs is an OSConfigRenderer object with configs registered
4906@@ -918,19 +1553,13 @@
4907 'shared-db': {'related': True}}}
4908 """
4909 complete_ctxts = configs.complete_contexts()
4910- incomplete_relations = []
4911- for svc_type in required_interfaces.keys():
4912- # Avoid duplicates
4913- found_ctxt = False
4914- for interface in required_interfaces[svc_type]:
4915- if interface in complete_ctxts:
4916- found_ctxt = True
4917- if not found_ctxt:
4918- incomplete_relations.append(svc_type)
4919- incomplete_context_data = {}
4920- for i in incomplete_relations:
4921- incomplete_context_data[i] = configs.get_incomplete_context_data(required_interfaces[i])
4922- return incomplete_context_data
4923+ incomplete_relations = [
4924+ svc_type
4925+ for svc_type, interfaces in required_interfaces.items()
4926+ if not set(interfaces).intersection(complete_ctxts)]
4927+ return {
4928+ i: configs.get_incomplete_context_data(required_interfaces[i])
4929+ for i in incomplete_relations}
4930
4931
4932 def do_action_openstack_upgrade(package, upgrade_callback, configs):
4933@@ -975,3 +1604,386 @@
4934 action_set({'outcome': 'no upgrade available.'})
4935
4936 return ret
4937+
4938+
4939+def remote_restart(rel_name, remote_service=None):
4940+ trigger = {
4941+ 'restart-trigger': str(uuid.uuid4()),
4942+ }
4943+ if remote_service:
4944+ trigger['remote-service'] = remote_service
4945+ for rid in relation_ids(rel_name):
4946+ # This subordinate can be related to two seperate services using
4947+ # different subordinate relations so only issue the restart if
4948+ # the principle is conencted down the relation we think it is
4949+ if related_units(relid=rid):
4950+ relation_set(relation_id=rid,
4951+ relation_settings=trigger,
4952+ )
4953+
4954+
4955+def check_actually_paused(services=None, ports=None):
4956+ """Check that services listed in the services object and and ports
4957+ are actually closed (not listened to), to verify that the unit is
4958+ properly paused.
4959+
4960+ @param services: See _extract_services_list_helper
4961+ @returns status, : string for status (None if okay)
4962+ message : string for problem for status_set
4963+ """
4964+ state = None
4965+ message = None
4966+ messages = []
4967+ if services is not None:
4968+ services = _extract_services_list_helper(services)
4969+ services_running, services_states = _check_running_services(services)
4970+ if any(services_states):
4971+ # there shouldn't be any running so this is a problem
4972+ messages.append("these services running: {}"
4973+ .format(", ".join(
4974+ _filter_tuples(services_running, True))))
4975+ state = "blocked"
4976+ ports_open, ports_open_bools = (
4977+ _check_listening_on_services_ports(services, True))
4978+ if any(ports_open_bools):
4979+ message_parts = {service: ", ".join([str(v) for v in open_ports])
4980+ for service, open_ports in ports_open.items()}
4981+ message = ", ".join(
4982+ ["{}: [{}]".format(s, sp) for s, sp in message_parts.items()])
4983+ messages.append(
4984+ "these service:ports are open: {}".format(message))
4985+ state = 'blocked'
4986+ if ports is not None:
4987+ ports_open, bools = _check_listening_on_ports_list(ports)
4988+ if any(bools):
4989+ messages.append(
4990+ "these ports which should be closed, but are open: {}"
4991+ .format(", ".join([str(p) for p, v in ports_open if v])))
4992+ state = 'blocked'
4993+ if messages:
4994+ message = ("Services should be paused but {}"
4995+ .format(", ".join(messages)))
4996+ return state, message
4997+
4998+
4999+def set_unit_paused():
5000+ """Set the unit to a paused state in the local kv() store.
The diff has been truncated for viewing.

Subscribers

People subscribed via source and target branches

to all changes: