Merge lp:~gnuoy/charms/trusty/glance-simplestreams-sync/ksv3 into lp:charms/trusty/glance-simplestreams-sync
- Trusty Tahr (14.04)
- ksv3
- Merge into trunk
Proposed by
Liam Young
Status: | Work in progress |
---|---|
Proposed branch: | lp:~gnuoy/charms/trusty/glance-simplestreams-sync/ksv3 |
Merge into: | lp:charms/trusty/glance-simplestreams-sync |
Diff against target: |
8587 lines (+4648/-1649) 72 files modified
charm-helpers-sync.yaml (+1/-0) charmhelpers/__init__.py (+11/-13) charmhelpers/contrib/__init__.py (+11/-13) charmhelpers/contrib/charmsupport/__init__.py (+11/-13) charmhelpers/contrib/charmsupport/nrpe.py (+84/-24) charmhelpers/contrib/charmsupport/volumes.py (+11/-13) charmhelpers/contrib/hahelpers/__init__.py (+11/-13) charmhelpers/contrib/hahelpers/apache.py (+30/-17) charmhelpers/contrib/hahelpers/cluster.py (+70/-23) charmhelpers/contrib/network/__init__.py (+11/-13) charmhelpers/contrib/network/ip.py (+80/-39) charmhelpers/contrib/openstack/__init__.py (+11/-13) charmhelpers/contrib/openstack/alternatives.py (+11/-13) charmhelpers/contrib/openstack/amulet/__init__.py (+11/-13) charmhelpers/contrib/openstack/amulet/deployment.py (+145/-47) charmhelpers/contrib/openstack/amulet/utils.py (+194/-30) charmhelpers/contrib/openstack/context.py (+201/-120) charmhelpers/contrib/openstack/exceptions.py (+21/-0) charmhelpers/contrib/openstack/files/__init__.py (+11/-13) charmhelpers/contrib/openstack/files/check_haproxy.sh (+7/-5) charmhelpers/contrib/openstack/ha/__init__.py (+13/-0) charmhelpers/contrib/openstack/ha/utils.py (+128/-0) charmhelpers/contrib/openstack/ip.py (+52/-24) charmhelpers/contrib/openstack/neutron.py (+53/-23) charmhelpers/contrib/openstack/templates/__init__.py (+11/-13) charmhelpers/contrib/openstack/templates/haproxy.cfg (+19/-11) charmhelpers/contrib/openstack/templates/openstack_https_frontend (+2/-0) charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf (+2/-0) charmhelpers/contrib/openstack/templates/section-keystone-authtoken (+8/-5) charmhelpers/contrib/openstack/templates/section-keystone-authtoken-legacy (+10/-0) charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka (+12/-0) charmhelpers/contrib/openstack/templating.py (+11/-13) charmhelpers/contrib/openstack/utils.py (+1062/-148) charmhelpers/contrib/python/__init__.py (+11/-13) charmhelpers/contrib/python/packages.py (+51/-25) charmhelpers/contrib/storage/__init__.py (+11/-13) charmhelpers/contrib/storage/linux/__init__.py (+11/-13) charmhelpers/contrib/storage/linux/ceph.py (+754/-72) charmhelpers/contrib/storage/linux/loopback.py (+21/-13) charmhelpers/contrib/storage/linux/lvm.py (+11/-13) charmhelpers/contrib/storage/linux/utils.py (+16/-18) charmhelpers/core/__init__.py (+11/-13) charmhelpers/core/decorators.py (+11/-13) charmhelpers/core/files.py (+11/-13) charmhelpers/core/fstab.py (+11/-13) charmhelpers/core/hookenv.py (+110/-19) charmhelpers/core/host.py (+270/-122) charmhelpers/core/host_factory/centos.py (+56/-0) charmhelpers/core/host_factory/ubuntu.py (+56/-0) charmhelpers/core/hugepage.py (+13/-13) charmhelpers/core/kernel.py (+34/-30) charmhelpers/core/kernel_factory/centos.py (+17/-0) charmhelpers/core/kernel_factory/ubuntu.py (+13/-0) charmhelpers/core/services/__init__.py (+11/-13) charmhelpers/core/services/base.py (+11/-13) charmhelpers/core/services/helpers.py (+25/-18) charmhelpers/core/strutils.py (+11/-13) charmhelpers/core/sysctl.py (+11/-13) charmhelpers/core/templating.py (+40/-24) charmhelpers/core/unitdata.py (+11/-14) charmhelpers/fetch/__init__.py (+42/-302) charmhelpers/fetch/archiveurl.py (+12/-14) charmhelpers/fetch/bzrurl.py (+48/-50) charmhelpers/fetch/centos.py (+171/-0) charmhelpers/fetch/giturl.py (+33/-37) charmhelpers/fetch/ubuntu.py (+313/-0) charmhelpers/osplatform.py (+19/-0) charmhelpers/payload/__init__.py (+11/-13) charmhelpers/payload/archive.py (+11/-13) charmhelpers/payload/execd.py (+11/-13) scripts/glance-simplestreams-sync.py (+28/-9) templates/identity.yaml (+4/-2) |
To merge this branch: | bzr merge lp:~gnuoy/charms/trusty/glance-simplestreams-sync/ksv3 |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
charmers | Pending | ||
Review via email: mp+306066@code.launchpad.net |
Commit message
Description of the change
Depends on https:/
To post a comment you must log in.
Unmerged revisions
- 75. By Liam Young
-
Added support for Keystone API v3
This change generates a keystone client with the correct version for the
Keystone API URL that is passed in. - 74. By Liam Young
-
Added new compulsary osplatform module
- 73. By Liam Young
-
Charmhelpers sync
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'charm-helpers-sync.yaml' |
2 | --- charm-helpers-sync.yaml 2015-09-28 20:36:18 +0000 |
3 | +++ charm-helpers-sync.yaml 2016-09-19 09:44:09 +0000 |
4 | @@ -3,6 +3,7 @@ |
5 | include: |
6 | - core |
7 | - fetch |
8 | + - osplatform |
9 | - payload |
10 | - contrib.openstack|inc=* |
11 | - contrib.charmsupport |
12 | |
13 | === modified file 'charmhelpers/__init__.py' |
14 | --- charmhelpers/__init__.py 2015-09-28 20:09:02 +0000 |
15 | +++ charmhelpers/__init__.py 2016-09-19 09:44:09 +0000 |
16 | @@ -1,18 +1,16 @@ |
17 | # Copyright 2014-2015 Canonical Limited. |
18 | # |
19 | -# This file is part of charm-helpers. |
20 | -# |
21 | -# charm-helpers is free software: you can redistribute it and/or modify |
22 | -# it under the terms of the GNU Lesser General Public License version 3 as |
23 | -# published by the Free Software Foundation. |
24 | -# |
25 | -# charm-helpers is distributed in the hope that it will be useful, |
26 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
27 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
28 | -# GNU Lesser General Public License for more details. |
29 | -# |
30 | -# You should have received a copy of the GNU Lesser General Public License |
31 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
32 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
33 | +# you may not use this file except in compliance with the License. |
34 | +# You may obtain a copy of the License at |
35 | +# |
36 | +# http://www.apache.org/licenses/LICENSE-2.0 |
37 | +# |
38 | +# Unless required by applicable law or agreed to in writing, software |
39 | +# distributed under the License is distributed on an "AS IS" BASIS, |
40 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
41 | +# See the License for the specific language governing permissions and |
42 | +# limitations under the License. |
43 | |
44 | # Bootstrap charm-helpers, installing its dependencies if necessary using |
45 | # only standard libraries. |
46 | |
47 | === modified file 'charmhelpers/contrib/__init__.py' |
48 | --- charmhelpers/contrib/__init__.py 2015-09-28 20:09:02 +0000 |
49 | +++ charmhelpers/contrib/__init__.py 2016-09-19 09:44:09 +0000 |
50 | @@ -1,15 +1,13 @@ |
51 | # Copyright 2014-2015 Canonical Limited. |
52 | # |
53 | -# This file is part of charm-helpers. |
54 | -# |
55 | -# charm-helpers is free software: you can redistribute it and/or modify |
56 | -# it under the terms of the GNU Lesser General Public License version 3 as |
57 | -# published by the Free Software Foundation. |
58 | -# |
59 | -# charm-helpers is distributed in the hope that it will be useful, |
60 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
61 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
62 | -# GNU Lesser General Public License for more details. |
63 | -# |
64 | -# You should have received a copy of the GNU Lesser General Public License |
65 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
66 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
67 | +# you may not use this file except in compliance with the License. |
68 | +# You may obtain a copy of the License at |
69 | +# |
70 | +# http://www.apache.org/licenses/LICENSE-2.0 |
71 | +# |
72 | +# Unless required by applicable law or agreed to in writing, software |
73 | +# distributed under the License is distributed on an "AS IS" BASIS, |
74 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
75 | +# See the License for the specific language governing permissions and |
76 | +# limitations under the License. |
77 | |
78 | === modified file 'charmhelpers/contrib/charmsupport/__init__.py' |
79 | --- charmhelpers/contrib/charmsupport/__init__.py 2015-09-28 20:09:02 +0000 |
80 | +++ charmhelpers/contrib/charmsupport/__init__.py 2016-09-19 09:44:09 +0000 |
81 | @@ -1,15 +1,13 @@ |
82 | # Copyright 2014-2015 Canonical Limited. |
83 | # |
84 | -# This file is part of charm-helpers. |
85 | -# |
86 | -# charm-helpers is free software: you can redistribute it and/or modify |
87 | -# it under the terms of the GNU Lesser General Public License version 3 as |
88 | -# published by the Free Software Foundation. |
89 | -# |
90 | -# charm-helpers is distributed in the hope that it will be useful, |
91 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
92 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
93 | -# GNU Lesser General Public License for more details. |
94 | -# |
95 | -# You should have received a copy of the GNU Lesser General Public License |
96 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
97 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
98 | +# you may not use this file except in compliance with the License. |
99 | +# You may obtain a copy of the License at |
100 | +# |
101 | +# http://www.apache.org/licenses/LICENSE-2.0 |
102 | +# |
103 | +# Unless required by applicable law or agreed to in writing, software |
104 | +# distributed under the License is distributed on an "AS IS" BASIS, |
105 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
106 | +# See the License for the specific language governing permissions and |
107 | +# limitations under the License. |
108 | |
109 | === modified file 'charmhelpers/contrib/charmsupport/nrpe.py' |
110 | --- charmhelpers/contrib/charmsupport/nrpe.py 2015-09-28 20:09:02 +0000 |
111 | +++ charmhelpers/contrib/charmsupport/nrpe.py 2016-09-19 09:44:09 +0000 |
112 | @@ -1,18 +1,16 @@ |
113 | # Copyright 2014-2015 Canonical Limited. |
114 | # |
115 | -# This file is part of charm-helpers. |
116 | -# |
117 | -# charm-helpers is free software: you can redistribute it and/or modify |
118 | -# it under the terms of the GNU Lesser General Public License version 3 as |
119 | -# published by the Free Software Foundation. |
120 | -# |
121 | -# charm-helpers is distributed in the hope that it will be useful, |
122 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
123 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
124 | -# GNU Lesser General Public License for more details. |
125 | -# |
126 | -# You should have received a copy of the GNU Lesser General Public License |
127 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
128 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
129 | +# you may not use this file except in compliance with the License. |
130 | +# You may obtain a copy of the License at |
131 | +# |
132 | +# http://www.apache.org/licenses/LICENSE-2.0 |
133 | +# |
134 | +# Unless required by applicable law or agreed to in writing, software |
135 | +# distributed under the License is distributed on an "AS IS" BASIS, |
136 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
137 | +# See the License for the specific language governing permissions and |
138 | +# limitations under the License. |
139 | |
140 | """Compatibility with the nrpe-external-master charm""" |
141 | # Copyright 2012 Canonical Ltd. |
142 | @@ -40,6 +38,7 @@ |
143 | ) |
144 | |
145 | from charmhelpers.core.host import service |
146 | +from charmhelpers.core import host |
147 | |
148 | # This module adds compatibility with the nrpe-external-master and plain nrpe |
149 | # subordinate charms. To use it in your charm: |
150 | @@ -110,6 +109,13 @@ |
151 | # def local_monitors_relation_changed(): |
152 | # update_nrpe_config() |
153 | # |
154 | +# 4.a If your charm is a subordinate charm set primary=False |
155 | +# |
156 | +# from charmsupport.nrpe import NRPE |
157 | +# (...) |
158 | +# def update_nrpe_config(): |
159 | +# nrpe_compat = NRPE(primary=False) |
160 | +# |
161 | # 5. ln -s hooks.py nrpe-external-master-relation-changed |
162 | # ln -s hooks.py local-monitors-relation-changed |
163 | |
164 | @@ -148,6 +154,13 @@ |
165 | self.description = description |
166 | self.check_cmd = self._locate_cmd(check_cmd) |
167 | |
168 | + def _get_check_filename(self): |
169 | + return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command)) |
170 | + |
171 | + def _get_service_filename(self, hostname): |
172 | + return os.path.join(NRPE.nagios_exportdir, |
173 | + 'service__{}_{}.cfg'.format(hostname, self.command)) |
174 | + |
175 | def _locate_cmd(self, check_cmd): |
176 | search_path = ( |
177 | '/usr/lib/nagios/plugins', |
178 | @@ -163,9 +176,21 @@ |
179 | log('Check command not found: {}'.format(parts[0])) |
180 | return '' |
181 | |
182 | + def _remove_service_files(self): |
183 | + if not os.path.exists(NRPE.nagios_exportdir): |
184 | + return |
185 | + for f in os.listdir(NRPE.nagios_exportdir): |
186 | + if f.endswith('_{}.cfg'.format(self.command)): |
187 | + os.remove(os.path.join(NRPE.nagios_exportdir, f)) |
188 | + |
189 | + def remove(self, hostname): |
190 | + nrpe_check_file = self._get_check_filename() |
191 | + if os.path.exists(nrpe_check_file): |
192 | + os.remove(nrpe_check_file) |
193 | + self._remove_service_files() |
194 | + |
195 | def write(self, nagios_context, hostname, nagios_servicegroups): |
196 | - nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( |
197 | - self.command) |
198 | + nrpe_check_file = self._get_check_filename() |
199 | with open(nrpe_check_file, 'w') as nrpe_check_config: |
200 | nrpe_check_config.write("# check {}\n".format(self.shortname)) |
201 | nrpe_check_config.write("command[{}]={}\n".format( |
202 | @@ -180,9 +205,7 @@ |
203 | |
204 | def write_service_config(self, nagios_context, hostname, |
205 | nagios_servicegroups): |
206 | - for f in os.listdir(NRPE.nagios_exportdir): |
207 | - if re.search('.*{}.cfg'.format(self.command), f): |
208 | - os.remove(os.path.join(NRPE.nagios_exportdir, f)) |
209 | + self._remove_service_files() |
210 | |
211 | templ_vars = { |
212 | 'nagios_hostname': hostname, |
213 | @@ -192,8 +215,7 @@ |
214 | 'command': self.command, |
215 | } |
216 | nrpe_service_text = Check.service_template.format(**templ_vars) |
217 | - nrpe_service_file = '{}/service__{}_{}.cfg'.format( |
218 | - NRPE.nagios_exportdir, hostname, self.command) |
219 | + nrpe_service_file = self._get_service_filename(hostname) |
220 | with open(nrpe_service_file, 'w') as nrpe_service_config: |
221 | nrpe_service_config.write(str(nrpe_service_text)) |
222 | |
223 | @@ -206,9 +228,10 @@ |
224 | nagios_exportdir = '/var/lib/nagios/export' |
225 | nrpe_confdir = '/etc/nagios/nrpe.d' |
226 | |
227 | - def __init__(self, hostname=None): |
228 | + def __init__(self, hostname=None, primary=True): |
229 | super(NRPE, self).__init__() |
230 | self.config = config() |
231 | + self.primary = primary |
232 | self.nagios_context = self.config['nagios_context'] |
233 | if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: |
234 | self.nagios_servicegroups = self.config['nagios_servicegroups'] |
235 | @@ -218,12 +241,38 @@ |
236 | if hostname: |
237 | self.hostname = hostname |
238 | else: |
239 | - self.hostname = "{}-{}".format(self.nagios_context, self.unit_name) |
240 | + nagios_hostname = get_nagios_hostname() |
241 | + if nagios_hostname: |
242 | + self.hostname = nagios_hostname |
243 | + else: |
244 | + self.hostname = "{}-{}".format(self.nagios_context, self.unit_name) |
245 | self.checks = [] |
246 | + # Iff in an nrpe-external-master relation hook, set primary status |
247 | + relation = relation_ids('nrpe-external-master') |
248 | + if relation: |
249 | + log("Setting charm primary status {}".format(primary)) |
250 | + for rid in relation_ids('nrpe-external-master'): |
251 | + relation_set(relation_id=rid, relation_settings={'primary': self.primary}) |
252 | |
253 | def add_check(self, *args, **kwargs): |
254 | self.checks.append(Check(*args, **kwargs)) |
255 | |
256 | + def remove_check(self, *args, **kwargs): |
257 | + if kwargs.get('shortname') is None: |
258 | + raise ValueError('shortname of check must be specified') |
259 | + |
260 | + # Use sensible defaults if they're not specified - these are not |
261 | + # actually used during removal, but they're required for constructing |
262 | + # the Check object; check_disk is chosen because it's part of the |
263 | + # nagios-plugins-basic package. |
264 | + if kwargs.get('check_cmd') is None: |
265 | + kwargs['check_cmd'] = 'check_disk' |
266 | + if kwargs.get('description') is None: |
267 | + kwargs['description'] = '' |
268 | + |
269 | + check = Check(*args, **kwargs) |
270 | + check.remove(self.hostname) |
271 | + |
272 | def write(self): |
273 | try: |
274 | nagios_uid = pwd.getpwnam('nagios').pw_uid |
275 | @@ -260,7 +309,7 @@ |
276 | :param str relation_name: Name of relation nrpe sub joined to |
277 | """ |
278 | for rel in relations_of_type(relation_name): |
279 | - if 'nagios_hostname' in rel: |
280 | + if 'nagios_host_context' in rel: |
281 | return rel['nagios_host_context'] |
282 | |
283 | |
284 | @@ -298,9 +347,20 @@ |
285 | :param str unit_name: Unit name to use in check description |
286 | """ |
287 | for svc in services: |
288 | + # Don't add a check for these services from neutron-gateway |
289 | + if svc in ['ext-port', 'os-charm-phy-nic-mtu']: |
290 | + next |
291 | + |
292 | upstart_init = '/etc/init/%s.conf' % svc |
293 | sysv_init = '/etc/init.d/%s' % svc |
294 | - if os.path.exists(upstart_init): |
295 | + |
296 | + if host.init_is_systemd(): |
297 | + nrpe.add_check( |
298 | + shortname=svc, |
299 | + description='process check {%s}' % unit_name, |
300 | + check_cmd='check_systemd.py %s' % svc |
301 | + ) |
302 | + elif os.path.exists(upstart_init): |
303 | nrpe.add_check( |
304 | shortname=svc, |
305 | description='process check {%s}' % unit_name, |
306 | |
307 | === modified file 'charmhelpers/contrib/charmsupport/volumes.py' |
308 | --- charmhelpers/contrib/charmsupport/volumes.py 2015-09-28 20:09:02 +0000 |
309 | +++ charmhelpers/contrib/charmsupport/volumes.py 2016-09-19 09:44:09 +0000 |
310 | @@ -1,18 +1,16 @@ |
311 | # Copyright 2014-2015 Canonical Limited. |
312 | # |
313 | -# This file is part of charm-helpers. |
314 | -# |
315 | -# charm-helpers is free software: you can redistribute it and/or modify |
316 | -# it under the terms of the GNU Lesser General Public License version 3 as |
317 | -# published by the Free Software Foundation. |
318 | -# |
319 | -# charm-helpers is distributed in the hope that it will be useful, |
320 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
321 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
322 | -# GNU Lesser General Public License for more details. |
323 | -# |
324 | -# You should have received a copy of the GNU Lesser General Public License |
325 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
326 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
327 | +# you may not use this file except in compliance with the License. |
328 | +# You may obtain a copy of the License at |
329 | +# |
330 | +# http://www.apache.org/licenses/LICENSE-2.0 |
331 | +# |
332 | +# Unless required by applicable law or agreed to in writing, software |
333 | +# distributed under the License is distributed on an "AS IS" BASIS, |
334 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
335 | +# See the License for the specific language governing permissions and |
336 | +# limitations under the License. |
337 | |
338 | ''' |
339 | Functions for managing volumes in juju units. One volume is supported per unit. |
340 | |
341 | === modified file 'charmhelpers/contrib/hahelpers/__init__.py' |
342 | --- charmhelpers/contrib/hahelpers/__init__.py 2015-09-28 20:09:02 +0000 |
343 | +++ charmhelpers/contrib/hahelpers/__init__.py 2016-09-19 09:44:09 +0000 |
344 | @@ -1,15 +1,13 @@ |
345 | # Copyright 2014-2015 Canonical Limited. |
346 | # |
347 | -# This file is part of charm-helpers. |
348 | -# |
349 | -# charm-helpers is free software: you can redistribute it and/or modify |
350 | -# it under the terms of the GNU Lesser General Public License version 3 as |
351 | -# published by the Free Software Foundation. |
352 | -# |
353 | -# charm-helpers is distributed in the hope that it will be useful, |
354 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
355 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
356 | -# GNU Lesser General Public License for more details. |
357 | -# |
358 | -# You should have received a copy of the GNU Lesser General Public License |
359 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
360 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
361 | +# you may not use this file except in compliance with the License. |
362 | +# You may obtain a copy of the License at |
363 | +# |
364 | +# http://www.apache.org/licenses/LICENSE-2.0 |
365 | +# |
366 | +# Unless required by applicable law or agreed to in writing, software |
367 | +# distributed under the License is distributed on an "AS IS" BASIS, |
368 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
369 | +# See the License for the specific language governing permissions and |
370 | +# limitations under the License. |
371 | |
372 | === modified file 'charmhelpers/contrib/hahelpers/apache.py' |
373 | --- charmhelpers/contrib/hahelpers/apache.py 2015-09-28 20:09:02 +0000 |
374 | +++ charmhelpers/contrib/hahelpers/apache.py 2016-09-19 09:44:09 +0000 |
375 | @@ -1,18 +1,16 @@ |
376 | # Copyright 2014-2015 Canonical Limited. |
377 | # |
378 | -# This file is part of charm-helpers. |
379 | -# |
380 | -# charm-helpers is free software: you can redistribute it and/or modify |
381 | -# it under the terms of the GNU Lesser General Public License version 3 as |
382 | -# published by the Free Software Foundation. |
383 | -# |
384 | -# charm-helpers is distributed in the hope that it will be useful, |
385 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
386 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
387 | -# GNU Lesser General Public License for more details. |
388 | -# |
389 | -# You should have received a copy of the GNU Lesser General Public License |
390 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
391 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
392 | +# you may not use this file except in compliance with the License. |
393 | +# You may obtain a copy of the License at |
394 | +# |
395 | +# http://www.apache.org/licenses/LICENSE-2.0 |
396 | +# |
397 | +# Unless required by applicable law or agreed to in writing, software |
398 | +# distributed under the License is distributed on an "AS IS" BASIS, |
399 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
400 | +# See the License for the specific language governing permissions and |
401 | +# limitations under the License. |
402 | |
403 | # |
404 | # Copyright 2012 Canonical Ltd. |
405 | @@ -24,6 +22,7 @@ |
406 | # Adam Gandelman <adamg@ubuntu.com> |
407 | # |
408 | |
409 | +import os |
410 | import subprocess |
411 | |
412 | from charmhelpers.core.hookenv import ( |
413 | @@ -74,9 +73,23 @@ |
414 | return ca_cert |
415 | |
416 | |
417 | +def retrieve_ca_cert(cert_file): |
418 | + cert = None |
419 | + if os.path.isfile(cert_file): |
420 | + with open(cert_file, 'r') as crt: |
421 | + cert = crt.read() |
422 | + return cert |
423 | + |
424 | + |
425 | def install_ca_cert(ca_cert): |
426 | if ca_cert: |
427 | - with open('/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt', |
428 | - 'w') as crt: |
429 | - crt.write(ca_cert) |
430 | - subprocess.check_call(['update-ca-certificates', '--fresh']) |
431 | + cert_file = ('/usr/local/share/ca-certificates/' |
432 | + 'keystone_juju_ca_cert.crt') |
433 | + old_cert = retrieve_ca_cert(cert_file) |
434 | + if old_cert and old_cert == ca_cert: |
435 | + log("CA cert is the same as installed version", level=INFO) |
436 | + else: |
437 | + log("Installing new CA cert", level=INFO) |
438 | + with open(cert_file, 'w') as crt: |
439 | + crt.write(ca_cert) |
440 | + subprocess.check_call(['update-ca-certificates', '--fresh']) |
441 | |
442 | === modified file 'charmhelpers/contrib/hahelpers/cluster.py' |
443 | --- charmhelpers/contrib/hahelpers/cluster.py 2015-09-28 20:09:02 +0000 |
444 | +++ charmhelpers/contrib/hahelpers/cluster.py 2016-09-19 09:44:09 +0000 |
445 | @@ -1,18 +1,16 @@ |
446 | # Copyright 2014-2015 Canonical Limited. |
447 | # |
448 | -# This file is part of charm-helpers. |
449 | -# |
450 | -# charm-helpers is free software: you can redistribute it and/or modify |
451 | -# it under the terms of the GNU Lesser General Public License version 3 as |
452 | -# published by the Free Software Foundation. |
453 | -# |
454 | -# charm-helpers is distributed in the hope that it will be useful, |
455 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
456 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
457 | -# GNU Lesser General Public License for more details. |
458 | -# |
459 | -# You should have received a copy of the GNU Lesser General Public License |
460 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
461 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
462 | +# you may not use this file except in compliance with the License. |
463 | +# You may obtain a copy of the License at |
464 | +# |
465 | +# http://www.apache.org/licenses/LICENSE-2.0 |
466 | +# |
467 | +# Unless required by applicable law or agreed to in writing, software |
468 | +# distributed under the License is distributed on an "AS IS" BASIS, |
469 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
470 | +# See the License for the specific language governing permissions and |
471 | +# limitations under the License. |
472 | |
473 | # |
474 | # Copyright 2012 Canonical Ltd. |
475 | @@ -41,10 +39,11 @@ |
476 | relation_get, |
477 | config as config_get, |
478 | INFO, |
479 | - ERROR, |
480 | + DEBUG, |
481 | WARNING, |
482 | unit_get, |
483 | - is_leader as juju_is_leader |
484 | + is_leader as juju_is_leader, |
485 | + status_set, |
486 | ) |
487 | from charmhelpers.core.decorators import ( |
488 | retry_on_exception, |
489 | @@ -60,6 +59,10 @@ |
490 | pass |
491 | |
492 | |
493 | +class HAIncorrectConfig(Exception): |
494 | + pass |
495 | + |
496 | + |
497 | class CRMResourceNotFound(Exception): |
498 | pass |
499 | |
500 | @@ -274,27 +277,71 @@ |
501 | Obtains all relevant configuration from charm configuration required |
502 | for initiating a relation to hacluster: |
503 | |
504 | - ha-bindiface, ha-mcastport, vip |
505 | + ha-bindiface, ha-mcastport, vip, os-internal-hostname, |
506 | + os-admin-hostname, os-public-hostname, os-access-hostname |
507 | |
508 | param: exclude_keys: list of setting key(s) to be excluded. |
509 | returns: dict: A dict containing settings keyed by setting name. |
510 | - raises: HAIncompleteConfig if settings are missing. |
511 | + raises: HAIncompleteConfig if settings are missing or incorrect. |
512 | ''' |
513 | - settings = ['ha-bindiface', 'ha-mcastport', 'vip'] |
514 | + settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'os-internal-hostname', |
515 | + 'os-admin-hostname', 'os-public-hostname', 'os-access-hostname'] |
516 | conf = {} |
517 | for setting in settings: |
518 | if exclude_keys and setting in exclude_keys: |
519 | continue |
520 | |
521 | conf[setting] = config_get(setting) |
522 | - missing = [] |
523 | - [missing.append(s) for s, v in six.iteritems(conf) if v is None] |
524 | - if missing: |
525 | - log('Insufficient config data to configure hacluster.', level=ERROR) |
526 | - raise HAIncompleteConfig |
527 | + |
528 | + if not valid_hacluster_config(): |
529 | + raise HAIncorrectConfig('Insufficient or incorrect config data to ' |
530 | + 'configure hacluster.') |
531 | return conf |
532 | |
533 | |
534 | +def valid_hacluster_config(): |
535 | + ''' |
536 | + Check that either vip or dns-ha is set. If dns-ha then one of os-*-hostname |
537 | + must be set. |
538 | + |
539 | + Note: ha-bindiface and ha-macastport both have defaults and will always |
540 | + be set. We only care that either vip or dns-ha is set. |
541 | + |
542 | + :returns: boolean: valid config returns true. |
543 | + raises: HAIncompatibileConfig if settings conflict. |
544 | + raises: HAIncompleteConfig if settings are missing. |
545 | + ''' |
546 | + vip = config_get('vip') |
547 | + dns = config_get('dns-ha') |
548 | + if not(bool(vip) ^ bool(dns)): |
549 | + msg = ('HA: Either vip or dns-ha must be set but not both in order to ' |
550 | + 'use high availability') |
551 | + status_set('blocked', msg) |
552 | + raise HAIncorrectConfig(msg) |
553 | + |
554 | + # If dns-ha then one of os-*-hostname must be set |
555 | + if dns: |
556 | + dns_settings = ['os-internal-hostname', 'os-admin-hostname', |
557 | + 'os-public-hostname', 'os-access-hostname'] |
558 | + # At this point it is unknown if one or all of the possible |
559 | + # network spaces are in HA. Validate at least one is set which is |
560 | + # the minimum required. |
561 | + for setting in dns_settings: |
562 | + if config_get(setting): |
563 | + log('DNS HA: At least one hostname is set {}: {}' |
564 | + ''.format(setting, config_get(setting)), |
565 | + level=DEBUG) |
566 | + return True |
567 | + |
568 | + msg = ('DNS HA: At least one os-*-hostname(s) must be set to use ' |
569 | + 'DNS HA') |
570 | + status_set('blocked', msg) |
571 | + raise HAIncompleteConfig(msg) |
572 | + |
573 | + log('VIP HA: VIP is set {}'.format(vip), level=DEBUG) |
574 | + return True |
575 | + |
576 | + |
577 | def canonical_url(configs, vip_setting='vip'): |
578 | ''' |
579 | Returns the correct HTTP URL to this host given the state of HTTPS |
580 | |
581 | === modified file 'charmhelpers/contrib/network/__init__.py' |
582 | --- charmhelpers/contrib/network/__init__.py 2015-09-28 20:09:02 +0000 |
583 | +++ charmhelpers/contrib/network/__init__.py 2016-09-19 09:44:09 +0000 |
584 | @@ -1,15 +1,13 @@ |
585 | # Copyright 2014-2015 Canonical Limited. |
586 | # |
587 | -# This file is part of charm-helpers. |
588 | -# |
589 | -# charm-helpers is free software: you can redistribute it and/or modify |
590 | -# it under the terms of the GNU Lesser General Public License version 3 as |
591 | -# published by the Free Software Foundation. |
592 | -# |
593 | -# charm-helpers is distributed in the hope that it will be useful, |
594 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
595 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
596 | -# GNU Lesser General Public License for more details. |
597 | -# |
598 | -# You should have received a copy of the GNU Lesser General Public License |
599 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
600 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
601 | +# you may not use this file except in compliance with the License. |
602 | +# You may obtain a copy of the License at |
603 | +# |
604 | +# http://www.apache.org/licenses/LICENSE-2.0 |
605 | +# |
606 | +# Unless required by applicable law or agreed to in writing, software |
607 | +# distributed under the License is distributed on an "AS IS" BASIS, |
608 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
609 | +# See the License for the specific language governing permissions and |
610 | +# limitations under the License. |
611 | |
612 | === modified file 'charmhelpers/contrib/network/ip.py' |
613 | --- charmhelpers/contrib/network/ip.py 2015-09-28 20:09:02 +0000 |
614 | +++ charmhelpers/contrib/network/ip.py 2016-09-19 09:44:09 +0000 |
615 | @@ -1,18 +1,16 @@ |
616 | # Copyright 2014-2015 Canonical Limited. |
617 | # |
618 | -# This file is part of charm-helpers. |
619 | -# |
620 | -# charm-helpers is free software: you can redistribute it and/or modify |
621 | -# it under the terms of the GNU Lesser General Public License version 3 as |
622 | -# published by the Free Software Foundation. |
623 | -# |
624 | -# charm-helpers is distributed in the hope that it will be useful, |
625 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
626 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
627 | -# GNU Lesser General Public License for more details. |
628 | -# |
629 | -# You should have received a copy of the GNU Lesser General Public License |
630 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
631 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
632 | +# you may not use this file except in compliance with the License. |
633 | +# You may obtain a copy of the License at |
634 | +# |
635 | +# http://www.apache.org/licenses/LICENSE-2.0 |
636 | +# |
637 | +# Unless required by applicable law or agreed to in writing, software |
638 | +# distributed under the License is distributed on an "AS IS" BASIS, |
639 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
640 | +# See the License for the specific language governing permissions and |
641 | +# limitations under the License. |
642 | |
643 | import glob |
644 | import re |
645 | @@ -53,7 +51,7 @@ |
646 | |
647 | |
648 | def no_ip_found_error_out(network): |
649 | - errmsg = ("No IP address found in network: %s" % network) |
650 | + errmsg = ("No IP address found in network(s): %s" % network) |
651 | raise ValueError(errmsg) |
652 | |
653 | |
654 | @@ -61,7 +59,7 @@ |
655 | """Get an IPv4 or IPv6 address within the network from the host. |
656 | |
657 | :param network (str): CIDR presentation format. For example, |
658 | - '192.168.1.0/24'. |
659 | + '192.168.1.0/24'. Supports multiple networks as a space-delimited list. |
660 | :param fallback (str): If no address is found, return fallback. |
661 | :param fatal (boolean): If no address is found, fallback is not |
662 | set and fatal is True then exit(1). |
663 | @@ -75,24 +73,26 @@ |
664 | else: |
665 | return None |
666 | |
667 | - _validate_cidr(network) |
668 | - network = netaddr.IPNetwork(network) |
669 | - for iface in netifaces.interfaces(): |
670 | - addresses = netifaces.ifaddresses(iface) |
671 | - if network.version == 4 and netifaces.AF_INET in addresses: |
672 | - addr = addresses[netifaces.AF_INET][0]['addr'] |
673 | - netmask = addresses[netifaces.AF_INET][0]['netmask'] |
674 | - cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) |
675 | - if cidr in network: |
676 | - return str(cidr.ip) |
677 | + networks = network.split() or [network] |
678 | + for network in networks: |
679 | + _validate_cidr(network) |
680 | + network = netaddr.IPNetwork(network) |
681 | + for iface in netifaces.interfaces(): |
682 | + addresses = netifaces.ifaddresses(iface) |
683 | + if network.version == 4 and netifaces.AF_INET in addresses: |
684 | + addr = addresses[netifaces.AF_INET][0]['addr'] |
685 | + netmask = addresses[netifaces.AF_INET][0]['netmask'] |
686 | + cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) |
687 | + if cidr in network: |
688 | + return str(cidr.ip) |
689 | |
690 | - if network.version == 6 and netifaces.AF_INET6 in addresses: |
691 | - for addr in addresses[netifaces.AF_INET6]: |
692 | - if not addr['addr'].startswith('fe80'): |
693 | - cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], |
694 | - addr['netmask'])) |
695 | - if cidr in network: |
696 | - return str(cidr.ip) |
697 | + if network.version == 6 and netifaces.AF_INET6 in addresses: |
698 | + for addr in addresses[netifaces.AF_INET6]: |
699 | + if not addr['addr'].startswith('fe80'): |
700 | + cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], |
701 | + addr['netmask'])) |
702 | + if cidr in network: |
703 | + return str(cidr.ip) |
704 | |
705 | if fallback is not None: |
706 | return fallback |
707 | @@ -189,6 +189,15 @@ |
708 | get_netmask_for_address = partial(_get_for_address, key='netmask') |
709 | |
710 | |
711 | +def resolve_network_cidr(ip_address): |
712 | + ''' |
713 | + Resolves the full address cidr of an ip_address based on |
714 | + configured network interfaces |
715 | + ''' |
716 | + netmask = get_netmask_for_address(ip_address) |
717 | + return str(netaddr.IPNetwork("%s/%s" % (ip_address, netmask)).cidr) |
718 | + |
719 | + |
720 | def format_ipv6_addr(address): |
721 | """If address is IPv6, wrap it in '[]' otherwise return None. |
722 | |
723 | @@ -203,7 +212,16 @@ |
724 | |
725 | def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, |
726 | fatal=True, exc_list=None): |
727 | - """Return the assigned IP address for a given interface, if any.""" |
728 | + """Return the assigned IP address for a given interface, if any. |
729 | + |
730 | + :param iface: network interface on which address(es) are expected to |
731 | + be found. |
732 | + :param inet_type: inet address family |
733 | + :param inc_aliases: include alias interfaces in search |
734 | + :param fatal: if True, raise exception if address not found |
735 | + :param exc_list: list of addresses to ignore |
736 | + :return: list of ip addresses |
737 | + """ |
738 | # Extract nic if passed /dev/ethX |
739 | if '/' in iface: |
740 | iface = iface.split('/')[-1] |
741 | @@ -304,6 +322,14 @@ |
742 | We currently only support scope global IPv6 addresses i.e. non-temporary |
743 | addresses. If no global IPv6 address is found, return the first one found |
744 | in the ipv6 address list. |
745 | + |
746 | + :param iface: network interface on which ipv6 address(es) are expected to |
747 | + be found. |
748 | + :param inc_aliases: include alias interfaces in search |
749 | + :param fatal: if True, raise exception if address not found |
750 | + :param exc_list: list of addresses to ignore |
751 | + :param dynamic_only: only recognise dynamic addresses |
752 | + :return: list of ipv6 addresses |
753 | """ |
754 | addresses = get_iface_addr(iface=iface, inet_type='AF_INET6', |
755 | inc_aliases=inc_aliases, fatal=fatal, |
756 | @@ -325,7 +351,7 @@ |
757 | cmd = ['ip', 'addr', 'show', iface] |
758 | out = subprocess.check_output(cmd).decode('UTF-8') |
759 | if dynamic_only: |
760 | - key = re.compile("inet6 (.+)/[0-9]+ scope global dynamic.*") |
761 | + key = re.compile("inet6 (.+)/[0-9]+ scope global.* dynamic.*") |
762 | else: |
763 | key = re.compile("inet6 (.+)/[0-9]+ scope global.*") |
764 | |
765 | @@ -377,10 +403,10 @@ |
766 | Returns True if address is a valid IP address. |
767 | """ |
768 | try: |
769 | - # Test to see if already an IPv4 address |
770 | - socket.inet_aton(address) |
771 | + # Test to see if already an IPv4/IPv6 address |
772 | + address = netaddr.IPAddress(address) |
773 | return True |
774 | - except socket.error: |
775 | + except netaddr.AddrFormatError: |
776 | return False |
777 | |
778 | |
779 | @@ -388,7 +414,7 @@ |
780 | try: |
781 | import dns.resolver |
782 | except ImportError: |
783 | - apt_install('python-dnspython') |
784 | + apt_install('python-dnspython', fatal=True) |
785 | import dns.resolver |
786 | |
787 | if isinstance(address, dns.name.Name): |
788 | @@ -432,7 +458,7 @@ |
789 | try: |
790 | import dns.reversename |
791 | except ImportError: |
792 | - apt_install("python-dnspython") |
793 | + apt_install("python-dnspython", fatal=True) |
794 | import dns.reversename |
795 | |
796 | rev = dns.reversename.from_address(address) |
797 | @@ -454,3 +480,18 @@ |
798 | return result |
799 | else: |
800 | return result.split('.')[0] |
801 | + |
802 | + |
803 | +def port_has_listener(address, port): |
804 | + """ |
805 | + Returns True if the address:port is open and being listened to, |
806 | + else False. |
807 | + |
808 | + @param address: an IP address or hostname |
809 | + @param port: integer port |
810 | + |
811 | + Note calls 'zc' via a subprocess shell |
812 | + """ |
813 | + cmd = ['nc', '-z', address, str(port)] |
814 | + result = subprocess.call(cmd) |
815 | + return not(bool(result)) |
816 | |
817 | === modified file 'charmhelpers/contrib/openstack/__init__.py' |
818 | --- charmhelpers/contrib/openstack/__init__.py 2015-09-28 20:09:02 +0000 |
819 | +++ charmhelpers/contrib/openstack/__init__.py 2016-09-19 09:44:09 +0000 |
820 | @@ -1,15 +1,13 @@ |
821 | # Copyright 2014-2015 Canonical Limited. |
822 | # |
823 | -# This file is part of charm-helpers. |
824 | -# |
825 | -# charm-helpers is free software: you can redistribute it and/or modify |
826 | -# it under the terms of the GNU Lesser General Public License version 3 as |
827 | -# published by the Free Software Foundation. |
828 | -# |
829 | -# charm-helpers is distributed in the hope that it will be useful, |
830 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
831 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
832 | -# GNU Lesser General Public License for more details. |
833 | -# |
834 | -# You should have received a copy of the GNU Lesser General Public License |
835 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
836 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
837 | +# you may not use this file except in compliance with the License. |
838 | +# You may obtain a copy of the License at |
839 | +# |
840 | +# http://www.apache.org/licenses/LICENSE-2.0 |
841 | +# |
842 | +# Unless required by applicable law or agreed to in writing, software |
843 | +# distributed under the License is distributed on an "AS IS" BASIS, |
844 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
845 | +# See the License for the specific language governing permissions and |
846 | +# limitations under the License. |
847 | |
848 | === modified file 'charmhelpers/contrib/openstack/alternatives.py' |
849 | --- charmhelpers/contrib/openstack/alternatives.py 2015-09-28 20:09:02 +0000 |
850 | +++ charmhelpers/contrib/openstack/alternatives.py 2016-09-19 09:44:09 +0000 |
851 | @@ -1,18 +1,16 @@ |
852 | # Copyright 2014-2015 Canonical Limited. |
853 | # |
854 | -# This file is part of charm-helpers. |
855 | -# |
856 | -# charm-helpers is free software: you can redistribute it and/or modify |
857 | -# it under the terms of the GNU Lesser General Public License version 3 as |
858 | -# published by the Free Software Foundation. |
859 | -# |
860 | -# charm-helpers is distributed in the hope that it will be useful, |
861 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
862 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
863 | -# GNU Lesser General Public License for more details. |
864 | -# |
865 | -# You should have received a copy of the GNU Lesser General Public License |
866 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
867 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
868 | +# you may not use this file except in compliance with the License. |
869 | +# You may obtain a copy of the License at |
870 | +# |
871 | +# http://www.apache.org/licenses/LICENSE-2.0 |
872 | +# |
873 | +# Unless required by applicable law or agreed to in writing, software |
874 | +# distributed under the License is distributed on an "AS IS" BASIS, |
875 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
876 | +# See the License for the specific language governing permissions and |
877 | +# limitations under the License. |
878 | |
879 | ''' Helper for managing alternatives for file conflict resolution ''' |
880 | |
881 | |
882 | === modified file 'charmhelpers/contrib/openstack/amulet/__init__.py' |
883 | --- charmhelpers/contrib/openstack/amulet/__init__.py 2015-09-28 20:09:02 +0000 |
884 | +++ charmhelpers/contrib/openstack/amulet/__init__.py 2016-09-19 09:44:09 +0000 |
885 | @@ -1,15 +1,13 @@ |
886 | # Copyright 2014-2015 Canonical Limited. |
887 | # |
888 | -# This file is part of charm-helpers. |
889 | -# |
890 | -# charm-helpers is free software: you can redistribute it and/or modify |
891 | -# it under the terms of the GNU Lesser General Public License version 3 as |
892 | -# published by the Free Software Foundation. |
893 | -# |
894 | -# charm-helpers is distributed in the hope that it will be useful, |
895 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
896 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
897 | -# GNU Lesser General Public License for more details. |
898 | -# |
899 | -# You should have received a copy of the GNU Lesser General Public License |
900 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
901 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
902 | +# you may not use this file except in compliance with the License. |
903 | +# You may obtain a copy of the License at |
904 | +# |
905 | +# http://www.apache.org/licenses/LICENSE-2.0 |
906 | +# |
907 | +# Unless required by applicable law or agreed to in writing, software |
908 | +# distributed under the License is distributed on an "AS IS" BASIS, |
909 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
910 | +# See the License for the specific language governing permissions and |
911 | +# limitations under the License. |
912 | |
913 | === modified file 'charmhelpers/contrib/openstack/amulet/deployment.py' |
914 | --- charmhelpers/contrib/openstack/amulet/deployment.py 2015-09-28 20:09:02 +0000 |
915 | +++ charmhelpers/contrib/openstack/amulet/deployment.py 2016-09-19 09:44:09 +0000 |
916 | @@ -1,25 +1,29 @@ |
917 | # Copyright 2014-2015 Canonical Limited. |
918 | # |
919 | -# This file is part of charm-helpers. |
920 | -# |
921 | -# charm-helpers is free software: you can redistribute it and/or modify |
922 | -# it under the terms of the GNU Lesser General Public License version 3 as |
923 | -# published by the Free Software Foundation. |
924 | -# |
925 | -# charm-helpers is distributed in the hope that it will be useful, |
926 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
927 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
928 | -# GNU Lesser General Public License for more details. |
929 | -# |
930 | -# You should have received a copy of the GNU Lesser General Public License |
931 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
932 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
933 | +# you may not use this file except in compliance with the License. |
934 | +# You may obtain a copy of the License at |
935 | +# |
936 | +# http://www.apache.org/licenses/LICENSE-2.0 |
937 | +# |
938 | +# Unless required by applicable law or agreed to in writing, software |
939 | +# distributed under the License is distributed on an "AS IS" BASIS, |
940 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
941 | +# See the License for the specific language governing permissions and |
942 | +# limitations under the License. |
943 | |
944 | +import logging |
945 | +import re |
946 | +import sys |
947 | import six |
948 | from collections import OrderedDict |
949 | from charmhelpers.contrib.amulet.deployment import ( |
950 | AmuletDeployment |
951 | ) |
952 | |
953 | +DEBUG = logging.DEBUG |
954 | +ERROR = logging.ERROR |
955 | + |
956 | |
957 | class OpenStackAmuletDeployment(AmuletDeployment): |
958 | """OpenStack amulet deployment. |
959 | @@ -28,15 +32,31 @@ |
960 | that is specifically for use by OpenStack charms. |
961 | """ |
962 | |
963 | - def __init__(self, series=None, openstack=None, source=None, stable=True): |
964 | + def __init__(self, series=None, openstack=None, source=None, |
965 | + stable=True, log_level=DEBUG): |
966 | """Initialize the deployment environment.""" |
967 | super(OpenStackAmuletDeployment, self).__init__(series) |
968 | + self.log = self.get_logger(level=log_level) |
969 | + self.log.info('OpenStackAmuletDeployment: init') |
970 | self.openstack = openstack |
971 | self.source = source |
972 | self.stable = stable |
973 | - # Note(coreycb): this needs to be changed when new next branches come |
974 | - # out. |
975 | - self.current_next = "trusty" |
976 | + |
977 | + def get_logger(self, name="deployment-logger", level=logging.DEBUG): |
978 | + """Get a logger object that will log to stdout.""" |
979 | + log = logging |
980 | + logger = log.getLogger(name) |
981 | + fmt = log.Formatter("%(asctime)s %(funcName)s " |
982 | + "%(levelname)s: %(message)s") |
983 | + |
984 | + handler = log.StreamHandler(stream=sys.stdout) |
985 | + handler.setLevel(level) |
986 | + handler.setFormatter(fmt) |
987 | + |
988 | + logger.addHandler(handler) |
989 | + logger.setLevel(level) |
990 | + |
991 | + return logger |
992 | |
993 | def _determine_branch_locations(self, other_services): |
994 | """Determine the branch locations for the other services. |
995 | @@ -45,43 +65,43 @@ |
996 | stable or next (dev) branch, and based on this, use the corresonding |
997 | stable or next branches for the other_services.""" |
998 | |
999 | - # Charms outside the lp:~openstack-charmers namespace |
1000 | - base_charms = ['mysql', 'mongodb', 'nrpe'] |
1001 | - |
1002 | - # Force these charms to current series even when using an older series. |
1003 | - # ie. Use trusty/nrpe even when series is precise, as the P charm |
1004 | - # does not possess the necessary external master config and hooks. |
1005 | - force_series_current = ['nrpe'] |
1006 | - |
1007 | - if self.series in ['precise', 'trusty']: |
1008 | - base_series = self.series |
1009 | - else: |
1010 | - base_series = self.current_next |
1011 | + self.log.info('OpenStackAmuletDeployment: determine branch locations') |
1012 | + |
1013 | + # Charms outside the ~openstack-charmers |
1014 | + base_charms = { |
1015 | + 'mysql': ['precise', 'trusty'], |
1016 | + 'mongodb': ['precise', 'trusty'], |
1017 | + 'nrpe': ['precise', 'trusty', 'wily', 'xenial'], |
1018 | + } |
1019 | |
1020 | for svc in other_services: |
1021 | - if svc['name'] in force_series_current: |
1022 | - base_series = self.current_next |
1023 | # If a location has been explicitly set, use it |
1024 | if svc.get('location'): |
1025 | continue |
1026 | - if self.stable: |
1027 | - temp = 'lp:charms/{}/{}' |
1028 | - svc['location'] = temp.format(base_series, |
1029 | - svc['name']) |
1030 | + if svc['name'] in base_charms: |
1031 | + # NOTE: not all charms have support for all series we |
1032 | + # want/need to test against, so fix to most recent |
1033 | + # that each base charm supports |
1034 | + target_series = self.series |
1035 | + if self.series not in base_charms[svc['name']]: |
1036 | + target_series = base_charms[svc['name']][-1] |
1037 | + svc['location'] = 'cs:{}/{}'.format(target_series, |
1038 | + svc['name']) |
1039 | + elif self.stable: |
1040 | + svc['location'] = 'cs:{}/{}'.format(self.series, |
1041 | + svc['name']) |
1042 | else: |
1043 | - if svc['name'] in base_charms: |
1044 | - temp = 'lp:charms/{}/{}' |
1045 | - svc['location'] = temp.format(base_series, |
1046 | - svc['name']) |
1047 | - else: |
1048 | - temp = 'lp:~openstack-charmers/charms/{}/{}/next' |
1049 | - svc['location'] = temp.format(self.current_next, |
1050 | - svc['name']) |
1051 | + svc['location'] = 'cs:~openstack-charmers-next/{}/{}'.format( |
1052 | + self.series, |
1053 | + svc['name'] |
1054 | + ) |
1055 | |
1056 | return other_services |
1057 | |
1058 | def _add_services(self, this_service, other_services): |
1059 | """Add services to the deployment and set openstack-origin/source.""" |
1060 | + self.log.info('OpenStackAmuletDeployment: adding services') |
1061 | + |
1062 | other_services = self._determine_branch_locations(other_services) |
1063 | |
1064 | super(OpenStackAmuletDeployment, self)._add_services(this_service, |
1065 | @@ -92,10 +112,14 @@ |
1066 | |
1067 | # Charms which should use the source config option |
1068 | use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', |
1069 | - 'ceph-osd', 'ceph-radosgw'] |
1070 | + 'ceph-osd', 'ceph-radosgw', 'ceph-mon', 'ceph-proxy'] |
1071 | |
1072 | # Charms which can not use openstack-origin, ie. many subordinates |
1073 | - no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe'] |
1074 | + no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe', |
1075 | + 'openvswitch-odl', 'neutron-api-odl', 'odl-controller', |
1076 | + 'cinder-backup', 'nexentaedge-data', |
1077 | + 'nexentaedge-iscsi-gw', 'nexentaedge-swift-gw', |
1078 | + 'cinder-nexentaedge', 'nexentaedge-mgmt'] |
1079 | |
1080 | if self.openstack: |
1081 | for svc in services: |
1082 | @@ -111,9 +135,79 @@ |
1083 | |
1084 | def _configure_services(self, configs): |
1085 | """Configure all of the services.""" |
1086 | + self.log.info('OpenStackAmuletDeployment: configure services') |
1087 | for service, config in six.iteritems(configs): |
1088 | self.d.configure(service, config) |
1089 | |
1090 | + def _auto_wait_for_status(self, message=None, exclude_services=None, |
1091 | + include_only=None, timeout=1800): |
1092 | + """Wait for all units to have a specific extended status, except |
1093 | + for any defined as excluded. Unless specified via message, any |
1094 | + status containing any case of 'ready' will be considered a match. |
1095 | + |
1096 | + Examples of message usage: |
1097 | + |
1098 | + Wait for all unit status to CONTAIN any case of 'ready' or 'ok': |
1099 | + message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE) |
1100 | + |
1101 | + Wait for all units to reach this status (exact match): |
1102 | + message = re.compile('^Unit is ready and clustered$') |
1103 | + |
1104 | + Wait for all units to reach any one of these (exact match): |
1105 | + message = re.compile('Unit is ready|OK|Ready') |
1106 | + |
1107 | + Wait for at least one unit to reach this status (exact match): |
1108 | + message = {'ready'} |
1109 | + |
1110 | + See Amulet's sentry.wait_for_messages() for message usage detail. |
1111 | + https://github.com/juju/amulet/blob/master/amulet/sentry.py |
1112 | + |
1113 | + :param message: Expected status match |
1114 | + :param exclude_services: List of juju service names to ignore, |
1115 | + not to be used in conjuction with include_only. |
1116 | + :param include_only: List of juju service names to exclusively check, |
1117 | + not to be used in conjuction with exclude_services. |
1118 | + :param timeout: Maximum time in seconds to wait for status match |
1119 | + :returns: None. Raises if timeout is hit. |
1120 | + """ |
1121 | + self.log.info('Waiting for extended status on units...') |
1122 | + |
1123 | + all_services = self.d.services.keys() |
1124 | + |
1125 | + if exclude_services and include_only: |
1126 | + raise ValueError('exclude_services can not be used ' |
1127 | + 'with include_only') |
1128 | + |
1129 | + if message: |
1130 | + if isinstance(message, re._pattern_type): |
1131 | + match = message.pattern |
1132 | + else: |
1133 | + match = message |
1134 | + |
1135 | + self.log.debug('Custom extended status wait match: ' |
1136 | + '{}'.format(match)) |
1137 | + else: |
1138 | + self.log.debug('Default extended status wait match: contains ' |
1139 | + 'READY (case-insensitive)') |
1140 | + message = re.compile('.*ready.*', re.IGNORECASE) |
1141 | + |
1142 | + if exclude_services: |
1143 | + self.log.debug('Excluding services from extended status match: ' |
1144 | + '{}'.format(exclude_services)) |
1145 | + else: |
1146 | + exclude_services = [] |
1147 | + |
1148 | + if include_only: |
1149 | + services = include_only |
1150 | + else: |
1151 | + services = list(set(all_services) - set(exclude_services)) |
1152 | + |
1153 | + self.log.debug('Waiting up to {}s for extended status on services: ' |
1154 | + '{}'.format(timeout, services)) |
1155 | + service_messages = {service: message for service in services} |
1156 | + self.d.sentry.wait_for_messages(service_messages, timeout=timeout) |
1157 | + self.log.info('OK') |
1158 | + |
1159 | def _get_openstack_release(self): |
1160 | """Get openstack release. |
1161 | |
1162 | @@ -125,7 +219,8 @@ |
1163 | self.precise_havana, self.precise_icehouse, |
1164 | self.trusty_icehouse, self.trusty_juno, self.utopic_juno, |
1165 | self.trusty_kilo, self.vivid_kilo, self.trusty_liberty, |
1166 | - self.wily_liberty) = range(12) |
1167 | + self.wily_liberty, self.trusty_mitaka, |
1168 | + self.xenial_mitaka) = range(14) |
1169 | |
1170 | releases = { |
1171 | ('precise', None): self.precise_essex, |
1172 | @@ -137,9 +232,11 @@ |
1173 | ('trusty', 'cloud:trusty-juno'): self.trusty_juno, |
1174 | ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo, |
1175 | ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty, |
1176 | + ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka, |
1177 | ('utopic', None): self.utopic_juno, |
1178 | ('vivid', None): self.vivid_kilo, |
1179 | - ('wily', None): self.wily_liberty} |
1180 | + ('wily', None): self.wily_liberty, |
1181 | + ('xenial', None): self.xenial_mitaka} |
1182 | return releases[(self.series, self.openstack)] |
1183 | |
1184 | def _get_openstack_release_string(self): |
1185 | @@ -156,6 +253,7 @@ |
1186 | ('utopic', 'juno'), |
1187 | ('vivid', 'kilo'), |
1188 | ('wily', 'liberty'), |
1189 | + ('xenial', 'mitaka'), |
1190 | ]) |
1191 | if self.openstack: |
1192 | os_origin = self.openstack.split(':')[1] |
1193 | |
1194 | === modified file 'charmhelpers/contrib/openstack/amulet/utils.py' |
1195 | --- charmhelpers/contrib/openstack/amulet/utils.py 2015-09-28 20:38:07 +0000 |
1196 | +++ charmhelpers/contrib/openstack/amulet/utils.py 2016-09-19 09:44:09 +0000 |
1197 | @@ -1,23 +1,22 @@ |
1198 | # Copyright 2014-2015 Canonical Limited. |
1199 | # |
1200 | -# This file is part of charm-helpers. |
1201 | -# |
1202 | -# charm-helpers is free software: you can redistribute it and/or modify |
1203 | -# it under the terms of the GNU Lesser General Public License version 3 as |
1204 | -# published by the Free Software Foundation. |
1205 | -# |
1206 | -# charm-helpers is distributed in the hope that it will be useful, |
1207 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
1208 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
1209 | -# GNU Lesser General Public License for more details. |
1210 | -# |
1211 | -# You should have received a copy of the GNU Lesser General Public License |
1212 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
1213 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
1214 | +# you may not use this file except in compliance with the License. |
1215 | +# You may obtain a copy of the License at |
1216 | +# |
1217 | +# http://www.apache.org/licenses/LICENSE-2.0 |
1218 | +# |
1219 | +# Unless required by applicable law or agreed to in writing, software |
1220 | +# distributed under the License is distributed on an "AS IS" BASIS, |
1221 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
1222 | +# See the License for the specific language governing permissions and |
1223 | +# limitations under the License. |
1224 | |
1225 | import amulet |
1226 | import json |
1227 | import logging |
1228 | import os |
1229 | +import re |
1230 | import six |
1231 | import time |
1232 | import urllib |
1233 | @@ -26,7 +25,11 @@ |
1234 | import glanceclient.v1.client as glance_client |
1235 | import heatclient.v1.client as heat_client |
1236 | import keystoneclient.v2_0 as keystone_client |
1237 | -import novaclient.v1_1.client as nova_client |
1238 | +from keystoneclient.auth.identity import v3 as keystone_id_v3 |
1239 | +from keystoneclient import session as keystone_session |
1240 | +from keystoneclient.v3 import client as keystone_client_v3 |
1241 | + |
1242 | +import novaclient.client as nova_client |
1243 | import pika |
1244 | import swiftclient |
1245 | |
1246 | @@ -37,6 +40,8 @@ |
1247 | DEBUG = logging.DEBUG |
1248 | ERROR = logging.ERROR |
1249 | |
1250 | +NOVA_CLIENT_VERSION = "2" |
1251 | + |
1252 | |
1253 | class OpenStackAmuletUtils(AmuletUtils): |
1254 | """OpenStack amulet utilities. |
1255 | @@ -78,6 +83,56 @@ |
1256 | if not found: |
1257 | return 'endpoint not found' |
1258 | |
1259 | + def validate_v3_endpoint_data(self, endpoints, admin_port, internal_port, |
1260 | + public_port, expected): |
1261 | + """Validate keystone v3 endpoint data. |
1262 | + |
1263 | + Validate the v3 endpoint data which has changed from v2. The |
1264 | + ports are used to find the matching endpoint. |
1265 | + |
1266 | + The new v3 endpoint data looks like: |
1267 | + |
1268 | + [<Endpoint enabled=True, |
1269 | + id=0432655fc2f74d1e9fa17bdaa6f6e60b, |
1270 | + interface=admin, |
1271 | + links={u'self': u'<RESTful URL of this endpoint>'}, |
1272 | + region=RegionOne, |
1273 | + region_id=RegionOne, |
1274 | + service_id=17f842a0dc084b928e476fafe67e4095, |
1275 | + url=http://10.5.6.5:9312>, |
1276 | + <Endpoint enabled=True, |
1277 | + id=6536cb6cb92f4f41bf22b079935c7707, |
1278 | + interface=admin, |
1279 | + links={u'self': u'<RESTful url of this endpoint>'}, |
1280 | + region=RegionOne, |
1281 | + region_id=RegionOne, |
1282 | + service_id=72fc8736fb41435e8b3584205bb2cfa3, |
1283 | + url=http://10.5.6.6:35357/v3>, |
1284 | + ... ] |
1285 | + """ |
1286 | + self.log.debug('Validating v3 endpoint data...') |
1287 | + self.log.debug('actual: {}'.format(repr(endpoints))) |
1288 | + found = [] |
1289 | + for ep in endpoints: |
1290 | + self.log.debug('endpoint: {}'.format(repr(ep))) |
1291 | + if ((admin_port in ep.url and ep.interface == 'admin') or |
1292 | + (internal_port in ep.url and ep.interface == 'internal') or |
1293 | + (public_port in ep.url and ep.interface == 'public')): |
1294 | + found.append(ep.interface) |
1295 | + # note we ignore the links member. |
1296 | + actual = {'id': ep.id, |
1297 | + 'region': ep.region, |
1298 | + 'region_id': ep.region_id, |
1299 | + 'interface': self.not_null, |
1300 | + 'url': ep.url, |
1301 | + 'service_id': ep.service_id, } |
1302 | + ret = self._validate_dict_data(expected, actual) |
1303 | + if ret: |
1304 | + return 'unexpected endpoint data - {}'.format(ret) |
1305 | + |
1306 | + if len(found) != 3: |
1307 | + return 'Unexpected number of endpoints found' |
1308 | + |
1309 | def validate_svc_catalog_endpoint_data(self, expected, actual): |
1310 | """Validate service catalog endpoint data. |
1311 | |
1312 | @@ -95,6 +150,72 @@ |
1313 | return "endpoint {} does not exist".format(k) |
1314 | return ret |
1315 | |
1316 | + def validate_v3_svc_catalog_endpoint_data(self, expected, actual): |
1317 | + """Validate the keystone v3 catalog endpoint data. |
1318 | + |
1319 | + Validate a list of dictinaries that make up the keystone v3 service |
1320 | + catalogue. |
1321 | + |
1322 | + It is in the form of: |
1323 | + |
1324 | + |
1325 | + {u'identity': [{u'id': u'48346b01c6804b298cdd7349aadb732e', |
1326 | + u'interface': u'admin', |
1327 | + u'region': u'RegionOne', |
1328 | + u'region_id': u'RegionOne', |
1329 | + u'url': u'http://10.5.5.224:35357/v3'}, |
1330 | + {u'id': u'8414f7352a4b47a69fddd9dbd2aef5cf', |
1331 | + u'interface': u'public', |
1332 | + u'region': u'RegionOne', |
1333 | + u'region_id': u'RegionOne', |
1334 | + u'url': u'http://10.5.5.224:5000/v3'}, |
1335 | + {u'id': u'd5ca31440cc24ee1bf625e2996fb6a5b', |
1336 | + u'interface': u'internal', |
1337 | + u'region': u'RegionOne', |
1338 | + u'region_id': u'RegionOne', |
1339 | + u'url': u'http://10.5.5.224:5000/v3'}], |
1340 | + u'key-manager': [{u'id': u'68ebc17df0b045fcb8a8a433ebea9e62', |
1341 | + u'interface': u'public', |
1342 | + u'region': u'RegionOne', |
1343 | + u'region_id': u'RegionOne', |
1344 | + u'url': u'http://10.5.5.223:9311'}, |
1345 | + {u'id': u'9cdfe2a893c34afd8f504eb218cd2f9d', |
1346 | + u'interface': u'internal', |
1347 | + u'region': u'RegionOne', |
1348 | + u'region_id': u'RegionOne', |
1349 | + u'url': u'http://10.5.5.223:9311'}, |
1350 | + {u'id': u'f629388955bc407f8b11d8b7ca168086', |
1351 | + u'interface': u'admin', |
1352 | + u'region': u'RegionOne', |
1353 | + u'region_id': u'RegionOne', |
1354 | + u'url': u'http://10.5.5.223:9312'}]} |
1355 | + |
1356 | + Note, that an added complication is that the order of admin, public, |
1357 | + internal against 'interface' in each region. |
1358 | + |
1359 | + Thus, the function sorts the expected and actual lists using the |
1360 | + interface key as a sort key, prior to the comparison. |
1361 | + """ |
1362 | + self.log.debug('Validating v3 service catalog endpoint data...') |
1363 | + self.log.debug('actual: {}'.format(repr(actual))) |
1364 | + for k, v in six.iteritems(expected): |
1365 | + if k in actual: |
1366 | + l_expected = sorted(v, key=lambda x: x['interface']) |
1367 | + l_actual = sorted(actual[k], key=lambda x: x['interface']) |
1368 | + if len(l_actual) != len(l_expected): |
1369 | + return ("endpoint {} has differing number of interfaces " |
1370 | + " - expected({}), actual({})" |
1371 | + .format(k, len(l_expected), len(l_actual))) |
1372 | + for i_expected, i_actual in zip(l_expected, l_actual): |
1373 | + self.log.debug("checking interface {}" |
1374 | + .format(i_expected['interface'])) |
1375 | + ret = self._validate_dict_data(i_expected, i_actual) |
1376 | + if ret: |
1377 | + return self.endpoint_error(k, ret) |
1378 | + else: |
1379 | + return "endpoint {} does not exist".format(k) |
1380 | + return ret |
1381 | + |
1382 | def validate_tenant_data(self, expected, actual): |
1383 | """Validate tenant data. |
1384 | |
1385 | @@ -138,7 +259,7 @@ |
1386 | return "role {} does not exist".format(e['name']) |
1387 | return ret |
1388 | |
1389 | - def validate_user_data(self, expected, actual): |
1390 | + def validate_user_data(self, expected, actual, api_version=None): |
1391 | """Validate user data. |
1392 | |
1393 | Validate a list of actual user data vs a list of expected user |
1394 | @@ -149,10 +270,15 @@ |
1395 | for e in expected: |
1396 | found = False |
1397 | for act in actual: |
1398 | - a = {'enabled': act.enabled, 'name': act.name, |
1399 | - 'email': act.email, 'tenantId': act.tenantId, |
1400 | - 'id': act.id} |
1401 | - if e['name'] == a['name']: |
1402 | + if e['name'] == act.name: |
1403 | + a = {'enabled': act.enabled, 'name': act.name, |
1404 | + 'email': act.email, 'id': act.id} |
1405 | + if api_version == 3: |
1406 | + a['default_project_id'] = getattr(act, |
1407 | + 'default_project_id', |
1408 | + 'none') |
1409 | + else: |
1410 | + a['tenantId'] = act.tenantId |
1411 | found = True |
1412 | ret = self._validate_dict_data(e, a) |
1413 | if ret: |
1414 | @@ -187,15 +313,30 @@ |
1415 | return cinder_client.Client(username, password, tenant, ept) |
1416 | |
1417 | def authenticate_keystone_admin(self, keystone_sentry, user, password, |
1418 | - tenant): |
1419 | + tenant=None, api_version=None, |
1420 | + keystone_ip=None): |
1421 | """Authenticates admin user with the keystone admin endpoint.""" |
1422 | self.log.debug('Authenticating keystone admin...') |
1423 | unit = keystone_sentry |
1424 | - service_ip = unit.relation('shared-db', |
1425 | - 'mysql:shared-db')['private-address'] |
1426 | - ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8')) |
1427 | - return keystone_client.Client(username=user, password=password, |
1428 | - tenant_name=tenant, auth_url=ep) |
1429 | + if not keystone_ip: |
1430 | + keystone_ip = unit.relation('shared-db', |
1431 | + 'mysql:shared-db')['private-address'] |
1432 | + base_ep = "http://{}:35357".format(keystone_ip.strip().decode('utf-8')) |
1433 | + if not api_version or api_version == 2: |
1434 | + ep = base_ep + "/v2.0" |
1435 | + return keystone_client.Client(username=user, password=password, |
1436 | + tenant_name=tenant, auth_url=ep) |
1437 | + else: |
1438 | + ep = base_ep + "/v3" |
1439 | + auth = keystone_id_v3.Password( |
1440 | + user_domain_name='admin_domain', |
1441 | + username=user, |
1442 | + password=password, |
1443 | + domain_name='admin_domain', |
1444 | + auth_url=ep, |
1445 | + ) |
1446 | + sess = keystone_session.Session(auth=auth) |
1447 | + return keystone_client_v3.Client(session=sess) |
1448 | |
1449 | def authenticate_keystone_user(self, keystone, user, password, tenant): |
1450 | """Authenticates a regular user with the keystone public endpoint.""" |
1451 | @@ -224,7 +365,8 @@ |
1452 | self.log.debug('Authenticating nova user ({})...'.format(user)) |
1453 | ep = keystone.service_catalog.url_for(service_type='identity', |
1454 | endpoint_type='publicURL') |
1455 | - return nova_client.Client(username=user, api_key=password, |
1456 | + return nova_client.Client(NOVA_CLIENT_VERSION, |
1457 | + username=user, api_key=password, |
1458 | project_id=tenant, auth_url=ep) |
1459 | |
1460 | def authenticate_swift_user(self, keystone, user, password, tenant): |
1461 | @@ -604,7 +746,22 @@ |
1462 | '{}'.format(sample_type, samples)) |
1463 | return None |
1464 | |
1465 | -# rabbitmq/amqp specific helpers: |
1466 | + # rabbitmq/amqp specific helpers: |
1467 | + |
1468 | + def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200): |
1469 | + """Wait for rmq units extended status to show cluster readiness, |
1470 | + after an optional initial sleep period. Initial sleep is likely |
1471 | + necessary to be effective following a config change, as status |
1472 | + message may not instantly update to non-ready.""" |
1473 | + |
1474 | + if init_sleep: |
1475 | + time.sleep(init_sleep) |
1476 | + |
1477 | + message = re.compile('^Unit is ready and clustered$') |
1478 | + deployment._auto_wait_for_status(message=message, |
1479 | + timeout=timeout, |
1480 | + include_only=['rabbitmq-server']) |
1481 | + |
1482 | def add_rmq_test_user(self, sentry_units, |
1483 | username="testuser1", password="changeme"): |
1484 | """Add a test user via the first rmq juju unit, check connection as |
1485 | @@ -805,7 +962,10 @@ |
1486 | if port: |
1487 | config['ssl_port'] = port |
1488 | |
1489 | - deployment.configure('rabbitmq-server', config) |
1490 | + deployment.d.configure('rabbitmq-server', config) |
1491 | + |
1492 | + # Wait for unit status |
1493 | + self.rmq_wait_for_cluster(deployment) |
1494 | |
1495 | # Confirm |
1496 | tries = 0 |
1497 | @@ -832,7 +992,10 @@ |
1498 | |
1499 | # Disable RMQ SSL |
1500 | config = {'ssl': 'off'} |
1501 | - deployment.configure('rabbitmq-server', config) |
1502 | + deployment.d.configure('rabbitmq-server', config) |
1503 | + |
1504 | + # Wait for unit status |
1505 | + self.rmq_wait_for_cluster(deployment) |
1506 | |
1507 | # Confirm |
1508 | tries = 0 |
1509 | @@ -881,7 +1044,8 @@ |
1510 | retry_delay=5, |
1511 | socket_timeout=1) |
1512 | connection = pika.BlockingConnection(parameters) |
1513 | - assert connection.server_properties['product'] == 'RabbitMQ' |
1514 | + assert connection.is_open is True |
1515 | + assert connection.is_closing is False |
1516 | self.log.debug('Connect OK') |
1517 | return connection |
1518 | except Exception as e: |
1519 | |
1520 | === modified file 'charmhelpers/contrib/openstack/context.py' |
1521 | --- charmhelpers/contrib/openstack/context.py 2015-09-28 20:09:02 +0000 |
1522 | +++ charmhelpers/contrib/openstack/context.py 2016-09-19 09:44:09 +0000 |
1523 | @@ -1,18 +1,16 @@ |
1524 | # Copyright 2014-2015 Canonical Limited. |
1525 | # |
1526 | -# This file is part of charm-helpers. |
1527 | -# |
1528 | -# charm-helpers is free software: you can redistribute it and/or modify |
1529 | -# it under the terms of the GNU Lesser General Public License version 3 as |
1530 | -# published by the Free Software Foundation. |
1531 | -# |
1532 | -# charm-helpers is distributed in the hope that it will be useful, |
1533 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
1534 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
1535 | -# GNU Lesser General Public License for more details. |
1536 | -# |
1537 | -# You should have received a copy of the GNU Lesser General Public License |
1538 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
1539 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
1540 | +# you may not use this file except in compliance with the License. |
1541 | +# You may obtain a copy of the License at |
1542 | +# |
1543 | +# http://www.apache.org/licenses/LICENSE-2.0 |
1544 | +# |
1545 | +# Unless required by applicable law or agreed to in writing, software |
1546 | +# distributed under the License is distributed on an "AS IS" BASIS, |
1547 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
1548 | +# See the License for the specific language governing permissions and |
1549 | +# limitations under the License. |
1550 | |
1551 | import glob |
1552 | import json |
1553 | @@ -20,10 +18,9 @@ |
1554 | import re |
1555 | import time |
1556 | from base64 import b64decode |
1557 | -from subprocess import check_call |
1558 | +from subprocess import check_call, CalledProcessError |
1559 | |
1560 | import six |
1561 | -import yaml |
1562 | |
1563 | from charmhelpers.fetch import ( |
1564 | apt_install, |
1565 | @@ -45,10 +42,12 @@ |
1566 | INFO, |
1567 | WARNING, |
1568 | ERROR, |
1569 | + status_set, |
1570 | ) |
1571 | |
1572 | from charmhelpers.core.sysctl import create as sysctl_create |
1573 | from charmhelpers.core.strutils import bool_from_string |
1574 | +from charmhelpers.contrib.openstack.exceptions import OSContextError |
1575 | |
1576 | from charmhelpers.core.host import ( |
1577 | get_bond_master, |
1578 | @@ -57,6 +56,8 @@ |
1579 | get_nic_hwaddr, |
1580 | mkdir, |
1581 | write_file, |
1582 | + pwgen, |
1583 | + lsb_release, |
1584 | ) |
1585 | from charmhelpers.contrib.hahelpers.cluster import ( |
1586 | determine_apache_port, |
1587 | @@ -86,15 +87,22 @@ |
1588 | is_address_in_network, |
1589 | is_bridge_member, |
1590 | ) |
1591 | -from charmhelpers.contrib.openstack.utils import get_host_ip |
1592 | +from charmhelpers.contrib.openstack.utils import ( |
1593 | + config_flags_parser, |
1594 | + get_host_ip, |
1595 | +) |
1596 | +from charmhelpers.core.unitdata import kv |
1597 | + |
1598 | +try: |
1599 | + import psutil |
1600 | +except ImportError: |
1601 | + apt_install('python-psutil', fatal=True) |
1602 | + import psutil |
1603 | + |
1604 | CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' |
1605 | ADDRESS_TYPES = ['admin', 'internal', 'public'] |
1606 | |
1607 | |
1608 | -class OSContextError(Exception): |
1609 | - pass |
1610 | - |
1611 | - |
1612 | def ensure_packages(packages): |
1613 | """Install but do not upgrade required plugin packages.""" |
1614 | required = filter_installed_packages(packages) |
1615 | @@ -115,83 +123,6 @@ |
1616 | return True |
1617 | |
1618 | |
1619 | -def config_flags_parser(config_flags): |
1620 | - """Parses config flags string into dict. |
1621 | - |
1622 | - This parsing method supports a few different formats for the config |
1623 | - flag values to be parsed: |
1624 | - |
1625 | - 1. A string in the simple format of key=value pairs, with the possibility |
1626 | - of specifying multiple key value pairs within the same string. For |
1627 | - example, a string in the format of 'key1=value1, key2=value2' will |
1628 | - return a dict of: |
1629 | - |
1630 | - {'key1': 'value1', |
1631 | - 'key2': 'value2'}. |
1632 | - |
1633 | - 2. A string in the above format, but supporting a comma-delimited list |
1634 | - of values for the same key. For example, a string in the format of |
1635 | - 'key1=value1, key2=value3,value4,value5' will return a dict of: |
1636 | - |
1637 | - {'key1', 'value1', |
1638 | - 'key2', 'value2,value3,value4'} |
1639 | - |
1640 | - 3. A string containing a colon character (:) prior to an equal |
1641 | - character (=) will be treated as yaml and parsed as such. This can be |
1642 | - used to specify more complex key value pairs. For example, |
1643 | - a string in the format of 'key1: subkey1=value1, subkey2=value2' will |
1644 | - return a dict of: |
1645 | - |
1646 | - {'key1', 'subkey1=value1, subkey2=value2'} |
1647 | - |
1648 | - The provided config_flags string may be a list of comma-separated values |
1649 | - which themselves may be comma-separated list of values. |
1650 | - """ |
1651 | - # If we find a colon before an equals sign then treat it as yaml. |
1652 | - # Note: limit it to finding the colon first since this indicates assignment |
1653 | - # for inline yaml. |
1654 | - colon = config_flags.find(':') |
1655 | - equals = config_flags.find('=') |
1656 | - if colon > 0: |
1657 | - if colon < equals or equals < 0: |
1658 | - return yaml.safe_load(config_flags) |
1659 | - |
1660 | - if config_flags.find('==') >= 0: |
1661 | - log("config_flags is not in expected format (key=value)", level=ERROR) |
1662 | - raise OSContextError |
1663 | - |
1664 | - # strip the following from each value. |
1665 | - post_strippers = ' ,' |
1666 | - # we strip any leading/trailing '=' or ' ' from the string then |
1667 | - # split on '='. |
1668 | - split = config_flags.strip(' =').split('=') |
1669 | - limit = len(split) |
1670 | - flags = {} |
1671 | - for i in range(0, limit - 1): |
1672 | - current = split[i] |
1673 | - next = split[i + 1] |
1674 | - vindex = next.rfind(',') |
1675 | - if (i == limit - 2) or (vindex < 0): |
1676 | - value = next |
1677 | - else: |
1678 | - value = next[:vindex] |
1679 | - |
1680 | - if i == 0: |
1681 | - key = current |
1682 | - else: |
1683 | - # if this not the first entry, expect an embedded key. |
1684 | - index = current.rfind(',') |
1685 | - if index < 0: |
1686 | - log("Invalid config value(s) at index %s" % (i), level=ERROR) |
1687 | - raise OSContextError |
1688 | - key = current[index + 1:] |
1689 | - |
1690 | - # Add to collection. |
1691 | - flags[key.strip(post_strippers)] = value.rstrip(post_strippers) |
1692 | - |
1693 | - return flags |
1694 | - |
1695 | - |
1696 | class OSContextGenerator(object): |
1697 | """Base class for all context generators.""" |
1698 | interfaces = [] |
1699 | @@ -401,6 +332,7 @@ |
1700 | auth_host = format_ipv6_addr(auth_host) or auth_host |
1701 | svc_protocol = rdata.get('service_protocol') or 'http' |
1702 | auth_protocol = rdata.get('auth_protocol') or 'http' |
1703 | + api_version = rdata.get('api_version') or '2.0' |
1704 | ctxt.update({'service_port': rdata.get('service_port'), |
1705 | 'service_host': serv_host, |
1706 | 'auth_host': auth_host, |
1707 | @@ -409,7 +341,8 @@ |
1708 | 'admin_user': rdata.get('service_username'), |
1709 | 'admin_password': rdata.get('service_password'), |
1710 | 'service_protocol': svc_protocol, |
1711 | - 'auth_protocol': auth_protocol}) |
1712 | + 'auth_protocol': auth_protocol, |
1713 | + 'api_version': api_version}) |
1714 | |
1715 | if self.context_complete(ctxt): |
1716 | # NOTE(jamespage) this is required for >= icehouse |
1717 | @@ -626,15 +559,28 @@ |
1718 | if config('haproxy-client-timeout'): |
1719 | ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout') |
1720 | |
1721 | + if config('haproxy-queue-timeout'): |
1722 | + ctxt['haproxy_queue_timeout'] = config('haproxy-queue-timeout') |
1723 | + |
1724 | + if config('haproxy-connect-timeout'): |
1725 | + ctxt['haproxy_connect_timeout'] = config('haproxy-connect-timeout') |
1726 | + |
1727 | if config('prefer-ipv6'): |
1728 | ctxt['ipv6'] = True |
1729 | ctxt['local_host'] = 'ip6-localhost' |
1730 | ctxt['haproxy_host'] = '::' |
1731 | - ctxt['stat_port'] = ':::8888' |
1732 | else: |
1733 | ctxt['local_host'] = '127.0.0.1' |
1734 | ctxt['haproxy_host'] = '0.0.0.0' |
1735 | - ctxt['stat_port'] = ':8888' |
1736 | + |
1737 | + ctxt['stat_port'] = '8888' |
1738 | + |
1739 | + db = kv() |
1740 | + ctxt['stat_password'] = db.get('stat-password') |
1741 | + if not ctxt['stat_password']: |
1742 | + ctxt['stat_password'] = db.set('stat-password', |
1743 | + pwgen(32)) |
1744 | + db.flush() |
1745 | |
1746 | for frontend in cluster_hosts: |
1747 | if (len(cluster_hosts[frontend]['backends']) > 1 or |
1748 | @@ -952,6 +898,19 @@ |
1749 | 'config': config} |
1750 | return ovs_ctxt |
1751 | |
1752 | + def midonet_ctxt(self): |
1753 | + driver = neutron_plugin_attribute(self.plugin, 'driver', |
1754 | + self.network_manager) |
1755 | + midonet_config = neutron_plugin_attribute(self.plugin, 'config', |
1756 | + self.network_manager) |
1757 | + mido_ctxt = {'core_plugin': driver, |
1758 | + 'neutron_plugin': 'midonet', |
1759 | + 'neutron_security_groups': self.neutron_security_groups, |
1760 | + 'local_ip': unit_private_ip(), |
1761 | + 'config': midonet_config} |
1762 | + |
1763 | + return mido_ctxt |
1764 | + |
1765 | def __call__(self): |
1766 | if self.network_manager not in ['quantum', 'neutron']: |
1767 | return {} |
1768 | @@ -973,6 +932,8 @@ |
1769 | ctxt.update(self.nuage_ctxt()) |
1770 | elif self.plugin == 'plumgrid': |
1771 | ctxt.update(self.pg_ctxt()) |
1772 | + elif self.plugin == 'midonet': |
1773 | + ctxt.update(self.midonet_ctxt()) |
1774 | |
1775 | alchemy_flags = config('neutron-alchemy-flags') |
1776 | if alchemy_flags: |
1777 | @@ -1073,6 +1034,20 @@ |
1778 | config_flags_parser(config_flags)} |
1779 | |
1780 | |
1781 | +class LibvirtConfigFlagsContext(OSContextGenerator): |
1782 | + """ |
1783 | + This context provides support for extending |
1784 | + the libvirt section through user-defined flags. |
1785 | + """ |
1786 | + def __call__(self): |
1787 | + ctxt = {} |
1788 | + libvirt_flags = config('libvirt-flags') |
1789 | + if libvirt_flags: |
1790 | + ctxt['libvirt_flags'] = config_flags_parser( |
1791 | + libvirt_flags) |
1792 | + return ctxt |
1793 | + |
1794 | + |
1795 | class SubordinateConfigContext(OSContextGenerator): |
1796 | |
1797 | """ |
1798 | @@ -1105,7 +1080,7 @@ |
1799 | |
1800 | ctxt = { |
1801 | ... other context ... |
1802 | - 'subordinate_config': { |
1803 | + 'subordinate_configuration': { |
1804 | 'DEFAULT': { |
1805 | 'key1': 'value1', |
1806 | }, |
1807 | @@ -1146,22 +1121,23 @@ |
1808 | try: |
1809 | sub_config = json.loads(sub_config) |
1810 | except: |
1811 | - log('Could not parse JSON from subordinate_config ' |
1812 | - 'setting from %s' % rid, level=ERROR) |
1813 | + log('Could not parse JSON from ' |
1814 | + 'subordinate_configuration setting from %s' |
1815 | + % rid, level=ERROR) |
1816 | continue |
1817 | |
1818 | for service in self.services: |
1819 | if service not in sub_config: |
1820 | - log('Found subordinate_config on %s but it contained' |
1821 | - 'nothing for %s service' % (rid, service), |
1822 | - level=INFO) |
1823 | + log('Found subordinate_configuration on %s but it ' |
1824 | + 'contained nothing for %s service' |
1825 | + % (rid, service), level=INFO) |
1826 | continue |
1827 | |
1828 | sub_config = sub_config[service] |
1829 | if self.config_file not in sub_config: |
1830 | - log('Found subordinate_config on %s but it contained' |
1831 | - 'nothing for %s' % (rid, self.config_file), |
1832 | - level=INFO) |
1833 | + log('Found subordinate_configuration on %s but it ' |
1834 | + 'contained nothing for %s' |
1835 | + % (rid, self.config_file), level=INFO) |
1836 | continue |
1837 | |
1838 | sub_config = sub_config[self.config_file] |
1839 | @@ -1212,17 +1188,18 @@ |
1840 | |
1841 | @property |
1842 | def num_cpus(self): |
1843 | - try: |
1844 | - from psutil import NUM_CPUS |
1845 | - except ImportError: |
1846 | - apt_install('python-psutil', fatal=True) |
1847 | - from psutil import NUM_CPUS |
1848 | - |
1849 | - return NUM_CPUS |
1850 | + # NOTE: use cpu_count if present (16.04 support) |
1851 | + if hasattr(psutil, 'cpu_count'): |
1852 | + return psutil.cpu_count() |
1853 | + else: |
1854 | + return psutil.NUM_CPUS |
1855 | |
1856 | def __call__(self): |
1857 | multiplier = config('worker-multiplier') or 0 |
1858 | - ctxt = {"workers": self.num_cpus * multiplier} |
1859 | + count = int(self.num_cpus * multiplier) |
1860 | + if multiplier > 0 and count == 0: |
1861 | + count = 1 |
1862 | + ctxt = {"workers": count} |
1863 | return ctxt |
1864 | |
1865 | |
1866 | @@ -1364,7 +1341,7 @@ |
1867 | normalized.update({port: port for port in resolved |
1868 | if port in ports}) |
1869 | if resolved: |
1870 | - return {bridge: normalized[port] for port, bridge in |
1871 | + return {normalized[port]: bridge for port, bridge in |
1872 | six.iteritems(portmap) if port in normalized.keys()} |
1873 | |
1874 | return None |
1875 | @@ -1375,8 +1352,8 @@ |
1876 | def __call__(self): |
1877 | ctxt = {} |
1878 | mappings = super(PhyNICMTUContext, self).__call__() |
1879 | - if mappings and mappings.values(): |
1880 | - ports = mappings.values() |
1881 | + if mappings and mappings.keys(): |
1882 | + ports = sorted(mappings.keys()) |
1883 | napi_settings = NeutronAPIContext()() |
1884 | mtu = napi_settings.get('network_device_mtu') |
1885 | all_ports = set() |
1886 | @@ -1421,7 +1398,111 @@ |
1887 | rdata.get('service_protocol') or 'http', |
1888 | 'auth_protocol': |
1889 | rdata.get('auth_protocol') or 'http', |
1890 | + 'api_version': |
1891 | + rdata.get('api_version') or '2.0', |
1892 | } |
1893 | if self.context_complete(ctxt): |
1894 | return ctxt |
1895 | return {} |
1896 | + |
1897 | + |
1898 | +class InternalEndpointContext(OSContextGenerator): |
1899 | + """Internal endpoint context. |
1900 | + |
1901 | + This context provides the endpoint type used for communication between |
1902 | + services e.g. between Nova and Cinder internally. Openstack uses Public |
1903 | + endpoints by default so this allows admins to optionally use internal |
1904 | + endpoints. |
1905 | + """ |
1906 | + def __call__(self): |
1907 | + return {'use_internal_endpoints': config('use-internal-endpoints')} |
1908 | + |
1909 | + |
1910 | +class AppArmorContext(OSContextGenerator): |
1911 | + """Base class for apparmor contexts.""" |
1912 | + |
1913 | + def __init__(self): |
1914 | + self._ctxt = None |
1915 | + self.aa_profile = None |
1916 | + self.aa_utils_packages = ['apparmor-utils'] |
1917 | + |
1918 | + @property |
1919 | + def ctxt(self): |
1920 | + if self._ctxt is not None: |
1921 | + return self._ctxt |
1922 | + self._ctxt = self._determine_ctxt() |
1923 | + return self._ctxt |
1924 | + |
1925 | + def _determine_ctxt(self): |
1926 | + """ |
1927 | + Validate aa-profile-mode settings is disable, enforce, or complain. |
1928 | + |
1929 | + :return ctxt: Dictionary of the apparmor profile or None |
1930 | + """ |
1931 | + if config('aa-profile-mode') in ['disable', 'enforce', 'complain']: |
1932 | + ctxt = {'aa_profile_mode': config('aa-profile-mode'), |
1933 | + 'ubuntu_release': lsb_release()['DISTRIB_RELEASE']} |
1934 | + else: |
1935 | + ctxt = None |
1936 | + return ctxt |
1937 | + |
1938 | + def __call__(self): |
1939 | + return self.ctxt |
1940 | + |
1941 | + def install_aa_utils(self): |
1942 | + """ |
1943 | + Install packages required for apparmor configuration. |
1944 | + """ |
1945 | + log("Installing apparmor utils.") |
1946 | + ensure_packages(self.aa_utils_packages) |
1947 | + |
1948 | + def manually_disable_aa_profile(self): |
1949 | + """ |
1950 | + Manually disable an apparmor profile. |
1951 | + |
1952 | + If aa-profile-mode is set to disabled (default) this is required as the |
1953 | + template has been written but apparmor is yet unaware of the profile |
1954 | + and aa-disable aa-profile fails. Without this the profile would kick |
1955 | + into enforce mode on the next service restart. |
1956 | + |
1957 | + """ |
1958 | + profile_path = '/etc/apparmor.d' |
1959 | + disable_path = '/etc/apparmor.d/disable' |
1960 | + if not os.path.lexists(os.path.join(disable_path, self.aa_profile)): |
1961 | + os.symlink(os.path.join(profile_path, self.aa_profile), |
1962 | + os.path.join(disable_path, self.aa_profile)) |
1963 | + |
1964 | + def setup_aa_profile(self): |
1965 | + """ |
1966 | + Setup an apparmor profile. |
1967 | + The ctxt dictionary will contain the apparmor profile mode and |
1968 | + the apparmor profile name. |
1969 | + Makes calls out to aa-disable, aa-complain, or aa-enforce to setup |
1970 | + the apparmor profile. |
1971 | + """ |
1972 | + self() |
1973 | + if not self.ctxt: |
1974 | + log("Not enabling apparmor Profile") |
1975 | + return |
1976 | + self.install_aa_utils() |
1977 | + cmd = ['aa-{}'.format(self.ctxt['aa_profile_mode'])] |
1978 | + cmd.append(self.ctxt['aa_profile']) |
1979 | + log("Setting up the apparmor profile for {} in {} mode." |
1980 | + "".format(self.ctxt['aa_profile'], self.ctxt['aa_profile_mode'])) |
1981 | + try: |
1982 | + check_call(cmd) |
1983 | + except CalledProcessError as e: |
1984 | + # If aa-profile-mode is set to disabled (default) manual |
1985 | + # disabling is required as the template has been written but |
1986 | + # apparmor is yet unaware of the profile and aa-disable aa-profile |
1987 | + # fails. If aa-disable learns to read profile files first this can |
1988 | + # be removed. |
1989 | + if self.ctxt['aa_profile_mode'] == 'disable': |
1990 | + log("Manually disabling the apparmor profile for {}." |
1991 | + "".format(self.ctxt['aa_profile'])) |
1992 | + self.manually_disable_aa_profile() |
1993 | + return |
1994 | + status_set('blocked', "Apparmor profile {} failed to be set to {}." |
1995 | + "".format(self.ctxt['aa_profile'], |
1996 | + self.ctxt['aa_profile_mode'])) |
1997 | + raise e |
1998 | |
1999 | === added file 'charmhelpers/contrib/openstack/exceptions.py' |
2000 | --- charmhelpers/contrib/openstack/exceptions.py 1970-01-01 00:00:00 +0000 |
2001 | +++ charmhelpers/contrib/openstack/exceptions.py 2016-09-19 09:44:09 +0000 |
2002 | @@ -0,0 +1,21 @@ |
2003 | +# Copyright 2016 Canonical Ltd |
2004 | +# |
2005 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
2006 | +# you may not use this file except in compliance with the License. |
2007 | +# You may obtain a copy of the License at |
2008 | +# |
2009 | +# http://www.apache.org/licenses/LICENSE-2.0 |
2010 | +# |
2011 | +# Unless required by applicable law or agreed to in writing, software |
2012 | +# distributed under the License is distributed on an "AS IS" BASIS, |
2013 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
2014 | +# See the License for the specific language governing permissions and |
2015 | +# limitations under the License. |
2016 | + |
2017 | + |
2018 | +class OSContextError(Exception): |
2019 | + """Raised when an error occurs during context generation. |
2020 | + |
2021 | + This exception is principally used in contrib.openstack.context |
2022 | + """ |
2023 | + pass |
2024 | |
2025 | === modified file 'charmhelpers/contrib/openstack/files/__init__.py' |
2026 | --- charmhelpers/contrib/openstack/files/__init__.py 2015-09-28 20:09:02 +0000 |
2027 | +++ charmhelpers/contrib/openstack/files/__init__.py 2016-09-19 09:44:09 +0000 |
2028 | @@ -1,18 +1,16 @@ |
2029 | # Copyright 2014-2015 Canonical Limited. |
2030 | # |
2031 | -# This file is part of charm-helpers. |
2032 | -# |
2033 | -# charm-helpers is free software: you can redistribute it and/or modify |
2034 | -# it under the terms of the GNU Lesser General Public License version 3 as |
2035 | -# published by the Free Software Foundation. |
2036 | -# |
2037 | -# charm-helpers is distributed in the hope that it will be useful, |
2038 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
2039 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
2040 | -# GNU Lesser General Public License for more details. |
2041 | -# |
2042 | -# You should have received a copy of the GNU Lesser General Public License |
2043 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
2044 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
2045 | +# you may not use this file except in compliance with the License. |
2046 | +# You may obtain a copy of the License at |
2047 | +# |
2048 | +# http://www.apache.org/licenses/LICENSE-2.0 |
2049 | +# |
2050 | +# Unless required by applicable law or agreed to in writing, software |
2051 | +# distributed under the License is distributed on an "AS IS" BASIS, |
2052 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
2053 | +# See the License for the specific language governing permissions and |
2054 | +# limitations under the License. |
2055 | |
2056 | # dummy __init__.py to fool syncer into thinking this is a syncable python |
2057 | # module |
2058 | |
2059 | === modified file 'charmhelpers/contrib/openstack/files/check_haproxy.sh' |
2060 | --- charmhelpers/contrib/openstack/files/check_haproxy.sh 2015-09-28 20:09:02 +0000 |
2061 | +++ charmhelpers/contrib/openstack/files/check_haproxy.sh 2016-09-19 09:44:09 +0000 |
2062 | @@ -9,15 +9,17 @@ |
2063 | CRITICAL=0 |
2064 | NOTACTIVE='' |
2065 | LOGFILE=/var/log/nagios/check_haproxy.log |
2066 | -AUTH=$(grep -r "stats auth" /etc/haproxy | head -1 | awk '{print $4}') |
2067 | +AUTH=$(grep -r "stats auth" /etc/haproxy | awk 'NR=1{print $4}') |
2068 | |
2069 | -for appserver in $(grep ' server' /etc/haproxy/haproxy.cfg | awk '{print $2'}); |
2070 | +typeset -i N_INSTANCES=0 |
2071 | +for appserver in $(awk '/^\s+server/{print $2}' /etc/haproxy/haproxy.cfg) |
2072 | do |
2073 | - output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 --regex="class=\"(active|backup)(2|3).*${appserver}" -e ' 200 OK') |
2074 | + N_INSTANCES=N_INSTANCES+1 |
2075 | + output=$(/usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' --regex=",${appserver},.*,UP.*" -e ' 200 OK') |
2076 | if [ $? != 0 ]; then |
2077 | date >> $LOGFILE |
2078 | echo $output >> $LOGFILE |
2079 | - /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -v | grep $appserver >> $LOGFILE 2>&1 |
2080 | + /usr/lib/nagios/plugins/check_http -a ${AUTH} -I 127.0.0.1 -p 8888 -u '/;csv' -v | grep ",${appserver}," >> $LOGFILE 2>&1 |
2081 | CRITICAL=1 |
2082 | NOTACTIVE="${NOTACTIVE} $appserver" |
2083 | fi |
2084 | @@ -28,5 +30,5 @@ |
2085 | exit 2 |
2086 | fi |
2087 | |
2088 | -echo "OK: All haproxy instances looking good" |
2089 | +echo "OK: All haproxy instances ($N_INSTANCES) looking good" |
2090 | exit 0 |
2091 | |
2092 | === added directory 'charmhelpers/contrib/openstack/ha' |
2093 | === added file 'charmhelpers/contrib/openstack/ha/__init__.py' |
2094 | --- charmhelpers/contrib/openstack/ha/__init__.py 1970-01-01 00:00:00 +0000 |
2095 | +++ charmhelpers/contrib/openstack/ha/__init__.py 2016-09-19 09:44:09 +0000 |
2096 | @@ -0,0 +1,13 @@ |
2097 | +# Copyright 2016 Canonical Ltd |
2098 | +# |
2099 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
2100 | +# you may not use this file except in compliance with the License. |
2101 | +# You may obtain a copy of the License at |
2102 | +# |
2103 | +# http://www.apache.org/licenses/LICENSE-2.0 |
2104 | +# |
2105 | +# Unless required by applicable law or agreed to in writing, software |
2106 | +# distributed under the License is distributed on an "AS IS" BASIS, |
2107 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
2108 | +# See the License for the specific language governing permissions and |
2109 | +# limitations under the License. |
2110 | |
2111 | === added file 'charmhelpers/contrib/openstack/ha/utils.py' |
2112 | --- charmhelpers/contrib/openstack/ha/utils.py 1970-01-01 00:00:00 +0000 |
2113 | +++ charmhelpers/contrib/openstack/ha/utils.py 2016-09-19 09:44:09 +0000 |
2114 | @@ -0,0 +1,128 @@ |
2115 | +# Copyright 2014-2016 Canonical Limited. |
2116 | +# |
2117 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
2118 | +# you may not use this file except in compliance with the License. |
2119 | +# You may obtain a copy of the License at |
2120 | +# |
2121 | +# http://www.apache.org/licenses/LICENSE-2.0 |
2122 | +# |
2123 | +# Unless required by applicable law or agreed to in writing, software |
2124 | +# distributed under the License is distributed on an "AS IS" BASIS, |
2125 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
2126 | +# See the License for the specific language governing permissions and |
2127 | +# limitations under the License. |
2128 | + |
2129 | +# |
2130 | +# Copyright 2016 Canonical Ltd. |
2131 | +# |
2132 | +# Authors: |
2133 | +# Openstack Charmers < |
2134 | +# |
2135 | + |
2136 | +""" |
2137 | +Helpers for high availability. |
2138 | +""" |
2139 | + |
2140 | +import re |
2141 | + |
2142 | +from charmhelpers.core.hookenv import ( |
2143 | + log, |
2144 | + relation_set, |
2145 | + charm_name, |
2146 | + config, |
2147 | + status_set, |
2148 | + DEBUG, |
2149 | +) |
2150 | + |
2151 | +from charmhelpers.core.host import ( |
2152 | + lsb_release |
2153 | +) |
2154 | + |
2155 | +from charmhelpers.contrib.openstack.ip import ( |
2156 | + resolve_address, |
2157 | +) |
2158 | + |
2159 | + |
2160 | +class DNSHAException(Exception): |
2161 | + """Raised when an error occurs setting up DNS HA |
2162 | + """ |
2163 | + |
2164 | + pass |
2165 | + |
2166 | + |
2167 | +def update_dns_ha_resource_params(resources, resource_params, |
2168 | + relation_id=None, |
2169 | + crm_ocf='ocf:maas:dns'): |
2170 | + """ Check for os-*-hostname settings and update resource dictionaries for |
2171 | + the HA relation. |
2172 | + |
2173 | + @param resources: Pointer to dictionary of resources. |
2174 | + Usually instantiated in ha_joined(). |
2175 | + @param resource_params: Pointer to dictionary of resource parameters. |
2176 | + Usually instantiated in ha_joined() |
2177 | + @param relation_id: Relation ID of the ha relation |
2178 | + @param crm_ocf: Corosync Open Cluster Framework resource agent to use for |
2179 | + DNS HA |
2180 | + """ |
2181 | + |
2182 | + # Validate the charm environment for DNS HA |
2183 | + assert_charm_supports_dns_ha() |
2184 | + |
2185 | + settings = ['os-admin-hostname', 'os-internal-hostname', |
2186 | + 'os-public-hostname', 'os-access-hostname'] |
2187 | + |
2188 | + # Check which DNS settings are set and update dictionaries |
2189 | + hostname_group = [] |
2190 | + for setting in settings: |
2191 | + hostname = config(setting) |
2192 | + if hostname is None: |
2193 | + log('DNS HA: Hostname setting {} is None. Ignoring.' |
2194 | + ''.format(setting), |
2195 | + DEBUG) |
2196 | + continue |
2197 | + m = re.search('os-(.+?)-hostname', setting) |
2198 | + if m: |
2199 | + networkspace = m.group(1) |
2200 | + else: |
2201 | + msg = ('Unexpected DNS hostname setting: {}. ' |
2202 | + 'Cannot determine network space name' |
2203 | + ''.format(setting)) |
2204 | + status_set('blocked', msg) |
2205 | + raise DNSHAException(msg) |
2206 | + |
2207 | + hostname_key = 'res_{}_{}_hostname'.format(charm_name(), networkspace) |
2208 | + if hostname_key in hostname_group: |
2209 | + log('DNS HA: Resource {}: {} already exists in ' |
2210 | + 'hostname group - skipping'.format(hostname_key, hostname), |
2211 | + DEBUG) |
2212 | + continue |
2213 | + |
2214 | + hostname_group.append(hostname_key) |
2215 | + resources[hostname_key] = crm_ocf |
2216 | + resource_params[hostname_key] = ( |
2217 | + 'params fqdn="{}" ip_address="{}" ' |
2218 | + ''.format(hostname, resolve_address(endpoint_type=networkspace, |
2219 | + override=False))) |
2220 | + |
2221 | + if len(hostname_group) >= 1: |
2222 | + log('DNS HA: Hostname group is set with {} as members. ' |
2223 | + 'Informing the ha relation'.format(' '.join(hostname_group)), |
2224 | + DEBUG) |
2225 | + relation_set(relation_id=relation_id, groups={ |
2226 | + 'grp_{}_hostnames'.format(charm_name()): ' '.join(hostname_group)}) |
2227 | + else: |
2228 | + msg = 'DNS HA: Hostname group has no members.' |
2229 | + status_set('blocked', msg) |
2230 | + raise DNSHAException(msg) |
2231 | + |
2232 | + |
2233 | +def assert_charm_supports_dns_ha(): |
2234 | + """Validate prerequisites for DNS HA |
2235 | + The MAAS client is only available on Xenial or greater |
2236 | + """ |
2237 | + if lsb_release().get('DISTRIB_RELEASE') < '16.04': |
2238 | + msg = ('DNS HA is only supported on 16.04 and greater ' |
2239 | + 'versions of Ubuntu.') |
2240 | + status_set('blocked', msg) |
2241 | + raise DNSHAException(msg) |
2242 | + return True |
2243 | |
2244 | === modified file 'charmhelpers/contrib/openstack/ip.py' |
2245 | --- charmhelpers/contrib/openstack/ip.py 2015-09-28 20:09:02 +0000 |
2246 | +++ charmhelpers/contrib/openstack/ip.py 2016-09-19 09:44:09 +0000 |
2247 | @@ -1,29 +1,29 @@ |
2248 | # Copyright 2014-2015 Canonical Limited. |
2249 | # |
2250 | -# This file is part of charm-helpers. |
2251 | -# |
2252 | -# charm-helpers is free software: you can redistribute it and/or modify |
2253 | -# it under the terms of the GNU Lesser General Public License version 3 as |
2254 | -# published by the Free Software Foundation. |
2255 | -# |
2256 | -# charm-helpers is distributed in the hope that it will be useful, |
2257 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
2258 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
2259 | -# GNU Lesser General Public License for more details. |
2260 | -# |
2261 | -# You should have received a copy of the GNU Lesser General Public License |
2262 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
2263 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
2264 | +# you may not use this file except in compliance with the License. |
2265 | +# You may obtain a copy of the License at |
2266 | +# |
2267 | +# http://www.apache.org/licenses/LICENSE-2.0 |
2268 | +# |
2269 | +# Unless required by applicable law or agreed to in writing, software |
2270 | +# distributed under the License is distributed on an "AS IS" BASIS, |
2271 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
2272 | +# See the License for the specific language governing permissions and |
2273 | +# limitations under the License. |
2274 | |
2275 | from charmhelpers.core.hookenv import ( |
2276 | config, |
2277 | unit_get, |
2278 | service_name, |
2279 | + network_get_primary_address, |
2280 | ) |
2281 | from charmhelpers.contrib.network.ip import ( |
2282 | get_address_in_network, |
2283 | is_address_in_network, |
2284 | is_ipv6, |
2285 | get_ipv6_addr, |
2286 | + resolve_network_cidr, |
2287 | ) |
2288 | from charmhelpers.contrib.hahelpers.cluster import is_clustered |
2289 | |
2290 | @@ -33,16 +33,19 @@ |
2291 | |
2292 | ADDRESS_MAP = { |
2293 | PUBLIC: { |
2294 | + 'binding': 'public', |
2295 | 'config': 'os-public-network', |
2296 | 'fallback': 'public-address', |
2297 | 'override': 'os-public-hostname', |
2298 | }, |
2299 | INTERNAL: { |
2300 | + 'binding': 'internal', |
2301 | 'config': 'os-internal-network', |
2302 | 'fallback': 'private-address', |
2303 | 'override': 'os-internal-hostname', |
2304 | }, |
2305 | ADMIN: { |
2306 | + 'binding': 'admin', |
2307 | 'config': 'os-admin-network', |
2308 | 'fallback': 'private-address', |
2309 | 'override': 'os-admin-hostname', |
2310 | @@ -103,20 +106,23 @@ |
2311 | return addr_override.format(service_name=service_name()) |
2312 | |
2313 | |
2314 | -def resolve_address(endpoint_type=PUBLIC): |
2315 | +def resolve_address(endpoint_type=PUBLIC, override=True): |
2316 | """Return unit address depending on net config. |
2317 | |
2318 | If unit is clustered with vip(s) and has net splits defined, return vip on |
2319 | correct network. If clustered with no nets defined, return primary vip. |
2320 | |
2321 | If not clustered, return unit address ensuring address is on configured net |
2322 | - split if one is configured. |
2323 | + split if one is configured, or a Juju 2.0 extra-binding has been used. |
2324 | |
2325 | :param endpoint_type: Network endpoing type |
2326 | + :param override: Accept hostname overrides or not |
2327 | """ |
2328 | - resolved_address = _get_address_override(endpoint_type) |
2329 | - if resolved_address: |
2330 | - return resolved_address |
2331 | + resolved_address = None |
2332 | + if override: |
2333 | + resolved_address = _get_address_override(endpoint_type) |
2334 | + if resolved_address: |
2335 | + return resolved_address |
2336 | |
2337 | vips = config('vip') |
2338 | if vips: |
2339 | @@ -125,23 +131,45 @@ |
2340 | net_type = ADDRESS_MAP[endpoint_type]['config'] |
2341 | net_addr = config(net_type) |
2342 | net_fallback = ADDRESS_MAP[endpoint_type]['fallback'] |
2343 | + binding = ADDRESS_MAP[endpoint_type]['binding'] |
2344 | clustered = is_clustered() |
2345 | - if clustered: |
2346 | - if not net_addr: |
2347 | - # If no net-splits defined, we expect a single vip |
2348 | - resolved_address = vips[0] |
2349 | - else: |
2350 | + |
2351 | + if clustered and vips: |
2352 | + if net_addr: |
2353 | for vip in vips: |
2354 | if is_address_in_network(net_addr, vip): |
2355 | resolved_address = vip |
2356 | break |
2357 | + else: |
2358 | + # NOTE: endeavour to check vips against network space |
2359 | + # bindings |
2360 | + try: |
2361 | + bound_cidr = resolve_network_cidr( |
2362 | + network_get_primary_address(binding) |
2363 | + ) |
2364 | + for vip in vips: |
2365 | + if is_address_in_network(bound_cidr, vip): |
2366 | + resolved_address = vip |
2367 | + break |
2368 | + except NotImplementedError: |
2369 | + # If no net-splits configured and no support for extra |
2370 | + # bindings/network spaces so we expect a single vip |
2371 | + resolved_address = vips[0] |
2372 | else: |
2373 | if config('prefer-ipv6'): |
2374 | fallback_addr = get_ipv6_addr(exc_list=vips)[0] |
2375 | else: |
2376 | fallback_addr = unit_get(net_fallback) |
2377 | |
2378 | - resolved_address = get_address_in_network(net_addr, fallback_addr) |
2379 | + if net_addr: |
2380 | + resolved_address = get_address_in_network(net_addr, fallback_addr) |
2381 | + else: |
2382 | + # NOTE: only try to use extra bindings if legacy network |
2383 | + # configuration is not in use |
2384 | + try: |
2385 | + resolved_address = network_get_primary_address(binding) |
2386 | + except NotImplementedError: |
2387 | + resolved_address = fallback_addr |
2388 | |
2389 | if resolved_address is None: |
2390 | raise ValueError("Unable to resolve a suitable IP address based on " |
2391 | |
2392 | === modified file 'charmhelpers/contrib/openstack/neutron.py' |
2393 | --- charmhelpers/contrib/openstack/neutron.py 2015-09-28 20:09:02 +0000 |
2394 | +++ charmhelpers/contrib/openstack/neutron.py 2016-09-19 09:44:09 +0000 |
2395 | @@ -1,18 +1,16 @@ |
2396 | # Copyright 2014-2015 Canonical Limited. |
2397 | # |
2398 | -# This file is part of charm-helpers. |
2399 | -# |
2400 | -# charm-helpers is free software: you can redistribute it and/or modify |
2401 | -# it under the terms of the GNU Lesser General Public License version 3 as |
2402 | -# published by the Free Software Foundation. |
2403 | -# |
2404 | -# charm-helpers is distributed in the hope that it will be useful, |
2405 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
2406 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
2407 | -# GNU Lesser General Public License for more details. |
2408 | -# |
2409 | -# You should have received a copy of the GNU Lesser General Public License |
2410 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
2411 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
2412 | +# you may not use this file except in compliance with the License. |
2413 | +# You may obtain a copy of the License at |
2414 | +# |
2415 | +# http://www.apache.org/licenses/LICENSE-2.0 |
2416 | +# |
2417 | +# Unless required by applicable law or agreed to in writing, software |
2418 | +# distributed under the License is distributed on an "AS IS" BASIS, |
2419 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
2420 | +# See the License for the specific language governing permissions and |
2421 | +# limitations under the License. |
2422 | |
2423 | # Various utilies for dealing with Neutron and the renaming from Quantum. |
2424 | |
2425 | @@ -50,7 +48,7 @@ |
2426 | if kernel_version() >= (3, 13): |
2427 | return [] |
2428 | else: |
2429 | - return ['openvswitch-datapath-dkms'] |
2430 | + return [headers_package(), 'openvswitch-datapath-dkms'] |
2431 | |
2432 | |
2433 | # legacy |
2434 | @@ -70,7 +68,7 @@ |
2435 | relation_prefix='neutron', |
2436 | ssl_dir=QUANTUM_CONF_DIR)], |
2437 | 'services': ['quantum-plugin-openvswitch-agent'], |
2438 | - 'packages': [[headers_package()] + determine_dkms_package(), |
2439 | + 'packages': [determine_dkms_package(), |
2440 | ['quantum-plugin-openvswitch-agent']], |
2441 | 'server_packages': ['quantum-server', |
2442 | 'quantum-plugin-openvswitch'], |
2443 | @@ -111,7 +109,7 @@ |
2444 | relation_prefix='neutron', |
2445 | ssl_dir=NEUTRON_CONF_DIR)], |
2446 | 'services': ['neutron-plugin-openvswitch-agent'], |
2447 | - 'packages': [[headers_package()] + determine_dkms_package(), |
2448 | + 'packages': [determine_dkms_package(), |
2449 | ['neutron-plugin-openvswitch-agent']], |
2450 | 'server_packages': ['neutron-server', |
2451 | 'neutron-plugin-openvswitch'], |
2452 | @@ -155,7 +153,7 @@ |
2453 | relation_prefix='neutron', |
2454 | ssl_dir=NEUTRON_CONF_DIR)], |
2455 | 'services': [], |
2456 | - 'packages': [[headers_package()] + determine_dkms_package(), |
2457 | + 'packages': [determine_dkms_package(), |
2458 | ['neutron-plugin-cisco']], |
2459 | 'server_packages': ['neutron-server', |
2460 | 'neutron-plugin-cisco'], |
2461 | @@ -174,7 +172,7 @@ |
2462 | 'neutron-dhcp-agent', |
2463 | 'nova-api-metadata', |
2464 | 'etcd'], |
2465 | - 'packages': [[headers_package()] + determine_dkms_package(), |
2466 | + 'packages': [determine_dkms_package(), |
2467 | ['calico-compute', |
2468 | 'bird', |
2469 | 'neutron-dhcp-agent', |
2470 | @@ -204,11 +202,25 @@ |
2471 | database=config('database'), |
2472 | ssl_dir=NEUTRON_CONF_DIR)], |
2473 | 'services': [], |
2474 | - 'packages': [['plumgrid-lxc'], |
2475 | - ['iovisor-dkms']], |
2476 | + 'packages': ['plumgrid-lxc', |
2477 | + 'iovisor-dkms'], |
2478 | 'server_packages': ['neutron-server', |
2479 | 'neutron-plugin-plumgrid'], |
2480 | 'server_services': ['neutron-server'] |
2481 | + }, |
2482 | + 'midonet': { |
2483 | + 'config': '/etc/neutron/plugins/midonet/midonet.ini', |
2484 | + 'driver': 'midonet.neutron.plugin.MidonetPluginV2', |
2485 | + 'contexts': [ |
2486 | + context.SharedDBContext(user=config('neutron-database-user'), |
2487 | + database=config('neutron-database'), |
2488 | + relation_prefix='neutron', |
2489 | + ssl_dir=NEUTRON_CONF_DIR)], |
2490 | + 'services': [], |
2491 | + 'packages': [determine_dkms_package()], |
2492 | + 'server_packages': ['neutron-server', |
2493 | + 'python-neutron-plugin-midonet'], |
2494 | + 'server_services': ['neutron-server'] |
2495 | } |
2496 | } |
2497 | if release >= 'icehouse': |
2498 | @@ -219,6 +231,24 @@ |
2499 | 'neutron-plugin-ml2'] |
2500 | # NOTE: patch in vmware renames nvp->nsx for icehouse onwards |
2501 | plugins['nvp'] = plugins['nsx'] |
2502 | + if release >= 'kilo': |
2503 | + plugins['midonet']['driver'] = ( |
2504 | + 'neutron.plugins.midonet.plugin.MidonetPluginV2') |
2505 | + if release >= 'liberty': |
2506 | + plugins['midonet']['driver'] = ( |
2507 | + 'midonet.neutron.plugin_v1.MidonetPluginV2') |
2508 | + plugins['midonet']['server_packages'].remove( |
2509 | + 'python-neutron-plugin-midonet') |
2510 | + plugins['midonet']['server_packages'].append( |
2511 | + 'python-networking-midonet') |
2512 | + plugins['plumgrid']['driver'] = ( |
2513 | + 'networking_plumgrid.neutron.plugins.plugin.NeutronPluginPLUMgridV2') |
2514 | + plugins['plumgrid']['server_packages'].remove( |
2515 | + 'neutron-plugin-plumgrid') |
2516 | + if release >= 'mitaka': |
2517 | + plugins['nsx']['server_packages'].remove('neutron-plugin-vmware') |
2518 | + plugins['nsx']['server_packages'].append('python-vmware-nsx') |
2519 | + plugins['nsx']['config'] = '/etc/neutron/nsx.ini' |
2520 | return plugins |
2521 | |
2522 | |
2523 | @@ -310,10 +340,10 @@ |
2524 | def parse_data_port_mappings(mappings, default_bridge='br-data'): |
2525 | """Parse data port mappings. |
2526 | |
2527 | - Mappings must be a space-delimited list of port:bridge mappings. |
2528 | + Mappings must be a space-delimited list of bridge:port. |
2529 | |
2530 | - Returns dict of the form {port:bridge} where port may be an mac address or |
2531 | - interface name. |
2532 | + Returns dict of the form {port:bridge} where ports may be mac addresses or |
2533 | + interface names. |
2534 | """ |
2535 | |
2536 | # NOTE(dosaboy): we use rvalue for key to allow multiple values to be |
2537 | |
2538 | === modified file 'charmhelpers/contrib/openstack/templates/__init__.py' |
2539 | --- charmhelpers/contrib/openstack/templates/__init__.py 2015-09-28 20:09:02 +0000 |
2540 | +++ charmhelpers/contrib/openstack/templates/__init__.py 2016-09-19 09:44:09 +0000 |
2541 | @@ -1,18 +1,16 @@ |
2542 | # Copyright 2014-2015 Canonical Limited. |
2543 | # |
2544 | -# This file is part of charm-helpers. |
2545 | -# |
2546 | -# charm-helpers is free software: you can redistribute it and/or modify |
2547 | -# it under the terms of the GNU Lesser General Public License version 3 as |
2548 | -# published by the Free Software Foundation. |
2549 | -# |
2550 | -# charm-helpers is distributed in the hope that it will be useful, |
2551 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
2552 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
2553 | -# GNU Lesser General Public License for more details. |
2554 | -# |
2555 | -# You should have received a copy of the GNU Lesser General Public License |
2556 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
2557 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
2558 | +# you may not use this file except in compliance with the License. |
2559 | +# You may obtain a copy of the License at |
2560 | +# |
2561 | +# http://www.apache.org/licenses/LICENSE-2.0 |
2562 | +# |
2563 | +# Unless required by applicable law or agreed to in writing, software |
2564 | +# distributed under the License is distributed on an "AS IS" BASIS, |
2565 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
2566 | +# See the License for the specific language governing permissions and |
2567 | +# limitations under the License. |
2568 | |
2569 | # dummy __init__.py to fool syncer into thinking this is a syncable python |
2570 | # module |
2571 | |
2572 | === modified file 'charmhelpers/contrib/openstack/templates/haproxy.cfg' |
2573 | --- charmhelpers/contrib/openstack/templates/haproxy.cfg 2015-09-28 20:09:02 +0000 |
2574 | +++ charmhelpers/contrib/openstack/templates/haproxy.cfg 2016-09-19 09:44:09 +0000 |
2575 | @@ -12,27 +12,35 @@ |
2576 | option tcplog |
2577 | option dontlognull |
2578 | retries 3 |
2579 | - timeout queue 1000 |
2580 | - timeout connect 1000 |
2581 | -{% if haproxy_client_timeout -%} |
2582 | +{%- if haproxy_queue_timeout %} |
2583 | + timeout queue {{ haproxy_queue_timeout }} |
2584 | +{%- else %} |
2585 | + timeout queue 5000 |
2586 | +{%- endif %} |
2587 | +{%- if haproxy_connect_timeout %} |
2588 | + timeout connect {{ haproxy_connect_timeout }} |
2589 | +{%- else %} |
2590 | + timeout connect 5000 |
2591 | +{%- endif %} |
2592 | +{%- if haproxy_client_timeout %} |
2593 | timeout client {{ haproxy_client_timeout }} |
2594 | -{% else -%} |
2595 | +{%- else %} |
2596 | timeout client 30000 |
2597 | -{% endif -%} |
2598 | - |
2599 | -{% if haproxy_server_timeout -%} |
2600 | +{%- endif %} |
2601 | +{%- if haproxy_server_timeout %} |
2602 | timeout server {{ haproxy_server_timeout }} |
2603 | -{% else -%} |
2604 | +{%- else %} |
2605 | timeout server 30000 |
2606 | -{% endif -%} |
2607 | +{%- endif %} |
2608 | |
2609 | -listen stats {{ stat_port }} |
2610 | +listen stats |
2611 | + bind {{ local_host }}:{{ stat_port }} |
2612 | mode http |
2613 | stats enable |
2614 | stats hide-version |
2615 | stats realm Haproxy\ Statistics |
2616 | stats uri / |
2617 | - stats auth admin:password |
2618 | + stats auth admin:{{ stat_password }} |
2619 | |
2620 | {% if frontends -%} |
2621 | {% for service, ports in service_ports.items() -%} |
2622 | |
2623 | === modified file 'charmhelpers/contrib/openstack/templates/openstack_https_frontend' |
2624 | --- charmhelpers/contrib/openstack/templates/openstack_https_frontend 2015-09-28 20:09:02 +0000 |
2625 | +++ charmhelpers/contrib/openstack/templates/openstack_https_frontend 2016-09-19 09:44:09 +0000 |
2626 | @@ -6,6 +6,8 @@ |
2627 | <VirtualHost {{ address }}:{{ ext }}> |
2628 | ServerName {{ endpoint }} |
2629 | SSLEngine on |
2630 | + SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2 |
2631 | + SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM |
2632 | SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }} |
2633 | SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }} |
2634 | ProxyPass / http://localhost:{{ int }}/ |
2635 | |
2636 | === modified file 'charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf' |
2637 | --- charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf 2015-09-28 20:09:02 +0000 |
2638 | +++ charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf 2016-09-19 09:44:09 +0000 |
2639 | @@ -6,6 +6,8 @@ |
2640 | <VirtualHost {{ address }}:{{ ext }}> |
2641 | ServerName {{ endpoint }} |
2642 | SSLEngine on |
2643 | + SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2 |
2644 | + SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM |
2645 | SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }} |
2646 | SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }} |
2647 | ProxyPass / http://localhost:{{ int }}/ |
2648 | |
2649 | === modified file 'charmhelpers/contrib/openstack/templates/section-keystone-authtoken' |
2650 | --- charmhelpers/contrib/openstack/templates/section-keystone-authtoken 2015-09-28 20:09:02 +0000 |
2651 | +++ charmhelpers/contrib/openstack/templates/section-keystone-authtoken 2016-09-19 09:44:09 +0000 |
2652 | @@ -1,9 +1,12 @@ |
2653 | {% if auth_host -%} |
2654 | [keystone_authtoken] |
2655 | -identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/{{ auth_admin_prefix }} |
2656 | -auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }} |
2657 | -admin_tenant_name = {{ admin_tenant_name }} |
2658 | -admin_user = {{ admin_user }} |
2659 | -admin_password = {{ admin_password }} |
2660 | +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }} |
2661 | +auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }} |
2662 | +auth_plugin = password |
2663 | +project_domain_id = default |
2664 | +user_domain_id = default |
2665 | +project_name = {{ admin_tenant_name }} |
2666 | +username = {{ admin_user }} |
2667 | +password = {{ admin_password }} |
2668 | signing_dir = {{ signing_dir }} |
2669 | {% endif -%} |
2670 | |
2671 | === added file 'charmhelpers/contrib/openstack/templates/section-keystone-authtoken-legacy' |
2672 | --- charmhelpers/contrib/openstack/templates/section-keystone-authtoken-legacy 1970-01-01 00:00:00 +0000 |
2673 | +++ charmhelpers/contrib/openstack/templates/section-keystone-authtoken-legacy 2016-09-19 09:44:09 +0000 |
2674 | @@ -0,0 +1,10 @@ |
2675 | +{% if auth_host -%} |
2676 | +[keystone_authtoken] |
2677 | +# Juno specific config (Bug #1557223) |
2678 | +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/{{ service_admin_prefix }} |
2679 | +identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }} |
2680 | +admin_tenant_name = {{ admin_tenant_name }} |
2681 | +admin_user = {{ admin_user }} |
2682 | +admin_password = {{ admin_password }} |
2683 | +signing_dir = {{ signing_dir }} |
2684 | +{% endif -%} |
2685 | |
2686 | === added file 'charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka' |
2687 | --- charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka 1970-01-01 00:00:00 +0000 |
2688 | +++ charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka 2016-09-19 09:44:09 +0000 |
2689 | @@ -0,0 +1,12 @@ |
2690 | +{% if auth_host -%} |
2691 | +[keystone_authtoken] |
2692 | +auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }} |
2693 | +auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }} |
2694 | +auth_type = password |
2695 | +project_domain_name = default |
2696 | +user_domain_name = default |
2697 | +project_name = {{ admin_tenant_name }} |
2698 | +username = {{ admin_user }} |
2699 | +password = {{ admin_password }} |
2700 | +signing_dir = {{ signing_dir }} |
2701 | +{% endif -%} |
2702 | |
2703 | === modified file 'charmhelpers/contrib/openstack/templating.py' |
2704 | --- charmhelpers/contrib/openstack/templating.py 2015-09-28 20:09:02 +0000 |
2705 | +++ charmhelpers/contrib/openstack/templating.py 2016-09-19 09:44:09 +0000 |
2706 | @@ -1,18 +1,16 @@ |
2707 | # Copyright 2014-2015 Canonical Limited. |
2708 | # |
2709 | -# This file is part of charm-helpers. |
2710 | -# |
2711 | -# charm-helpers is free software: you can redistribute it and/or modify |
2712 | -# it under the terms of the GNU Lesser General Public License version 3 as |
2713 | -# published by the Free Software Foundation. |
2714 | -# |
2715 | -# charm-helpers is distributed in the hope that it will be useful, |
2716 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
2717 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
2718 | -# GNU Lesser General Public License for more details. |
2719 | -# |
2720 | -# You should have received a copy of the GNU Lesser General Public License |
2721 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
2722 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
2723 | +# you may not use this file except in compliance with the License. |
2724 | +# You may obtain a copy of the License at |
2725 | +# |
2726 | +# http://www.apache.org/licenses/LICENSE-2.0 |
2727 | +# |
2728 | +# Unless required by applicable law or agreed to in writing, software |
2729 | +# distributed under the License is distributed on an "AS IS" BASIS, |
2730 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
2731 | +# See the License for the specific language governing permissions and |
2732 | +# limitations under the License. |
2733 | |
2734 | import os |
2735 | |
2736 | |
2737 | === modified file 'charmhelpers/contrib/openstack/utils.py' |
2738 | --- charmhelpers/contrib/openstack/utils.py 2015-09-28 20:09:02 +0000 |
2739 | +++ charmhelpers/contrib/openstack/utils.py 2016-09-19 09:44:09 +0000 |
2740 | @@ -1,18 +1,16 @@ |
2741 | # Copyright 2014-2015 Canonical Limited. |
2742 | # |
2743 | -# This file is part of charm-helpers. |
2744 | -# |
2745 | -# charm-helpers is free software: you can redistribute it and/or modify |
2746 | -# it under the terms of the GNU Lesser General Public License version 3 as |
2747 | -# published by the Free Software Foundation. |
2748 | -# |
2749 | -# charm-helpers is distributed in the hope that it will be useful, |
2750 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
2751 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
2752 | -# GNU Lesser General Public License for more details. |
2753 | -# |
2754 | -# You should have received a copy of the GNU Lesser General Public License |
2755 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
2756 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
2757 | +# you may not use this file except in compliance with the License. |
2758 | +# You may obtain a copy of the License at |
2759 | +# |
2760 | +# http://www.apache.org/licenses/LICENSE-2.0 |
2761 | +# |
2762 | +# Unless required by applicable law or agreed to in writing, software |
2763 | +# distributed under the License is distributed on an "AS IS" BASIS, |
2764 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
2765 | +# See the License for the specific language governing permissions and |
2766 | +# limitations under the License. |
2767 | |
2768 | # Common python helper functions used for OpenStack charms. |
2769 | from collections import OrderedDict |
2770 | @@ -23,9 +21,14 @@ |
2771 | import os |
2772 | import sys |
2773 | import re |
2774 | +import itertools |
2775 | +import functools |
2776 | +import shutil |
2777 | |
2778 | import six |
2779 | +import tempfile |
2780 | import traceback |
2781 | +import uuid |
2782 | import yaml |
2783 | |
2784 | from charmhelpers.contrib.network import ip |
2785 | @@ -40,9 +43,13 @@ |
2786 | config, |
2787 | log as juju_log, |
2788 | charm_dir, |
2789 | + DEBUG, |
2790 | INFO, |
2791 | + ERROR, |
2792 | + related_units, |
2793 | relation_ids, |
2794 | relation_set, |
2795 | + service_name, |
2796 | status_set, |
2797 | hook_name |
2798 | ) |
2799 | @@ -56,6 +63,7 @@ |
2800 | from charmhelpers.contrib.network.ip import ( |
2801 | get_ipv6_addr, |
2802 | is_ipv6, |
2803 | + port_has_listener, |
2804 | ) |
2805 | |
2806 | from charmhelpers.contrib.python.packages import ( |
2807 | @@ -63,10 +71,19 @@ |
2808 | pip_install, |
2809 | ) |
2810 | |
2811 | -from charmhelpers.core.host import lsb_release, mounts, umount |
2812 | +from charmhelpers.core.host import ( |
2813 | + lsb_release, |
2814 | + mounts, |
2815 | + umount, |
2816 | + service_running, |
2817 | + service_pause, |
2818 | + service_resume, |
2819 | + restart_on_change_helper, |
2820 | +) |
2821 | from charmhelpers.fetch import apt_install, apt_cache, install_remote |
2822 | from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk |
2823 | from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device |
2824 | +from charmhelpers.contrib.openstack.exceptions import OSContextError |
2825 | |
2826 | CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu" |
2827 | CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' |
2828 | @@ -84,6 +101,9 @@ |
2829 | ('utopic', 'juno'), |
2830 | ('vivid', 'kilo'), |
2831 | ('wily', 'liberty'), |
2832 | + ('xenial', 'mitaka'), |
2833 | + ('yakkety', 'newton'), |
2834 | + ('zebra', 'ocata'), # TODO: upload with real Z name |
2835 | ]) |
2836 | |
2837 | |
2838 | @@ -97,63 +117,115 @@ |
2839 | ('2014.2', 'juno'), |
2840 | ('2015.1', 'kilo'), |
2841 | ('2015.2', 'liberty'), |
2842 | + ('2016.1', 'mitaka'), |
2843 | + ('2016.2', 'newton'), |
2844 | + ('2017.1', 'ocata'), |
2845 | ]) |
2846 | |
2847 | -# The ugly duckling |
2848 | +# The ugly duckling - must list releases oldest to newest |
2849 | SWIFT_CODENAMES = OrderedDict([ |
2850 | - ('1.4.3', 'diablo'), |
2851 | - ('1.4.8', 'essex'), |
2852 | - ('1.7.4', 'folsom'), |
2853 | - ('1.8.0', 'grizzly'), |
2854 | - ('1.7.7', 'grizzly'), |
2855 | - ('1.7.6', 'grizzly'), |
2856 | - ('1.10.0', 'havana'), |
2857 | - ('1.9.1', 'havana'), |
2858 | - ('1.9.0', 'havana'), |
2859 | - ('1.13.1', 'icehouse'), |
2860 | - ('1.13.0', 'icehouse'), |
2861 | - ('1.12.0', 'icehouse'), |
2862 | - ('1.11.0', 'icehouse'), |
2863 | - ('2.0.0', 'juno'), |
2864 | - ('2.1.0', 'juno'), |
2865 | - ('2.2.0', 'juno'), |
2866 | - ('2.2.1', 'kilo'), |
2867 | - ('2.2.2', 'kilo'), |
2868 | - ('2.3.0', 'liberty'), |
2869 | - ('2.4.0', 'liberty'), |
2870 | + ('diablo', |
2871 | + ['1.4.3']), |
2872 | + ('essex', |
2873 | + ['1.4.8']), |
2874 | + ('folsom', |
2875 | + ['1.7.4']), |
2876 | + ('grizzly', |
2877 | + ['1.7.6', '1.7.7', '1.8.0']), |
2878 | + ('havana', |
2879 | + ['1.9.0', '1.9.1', '1.10.0']), |
2880 | + ('icehouse', |
2881 | + ['1.11.0', '1.12.0', '1.13.0', '1.13.1']), |
2882 | + ('juno', |
2883 | + ['2.0.0', '2.1.0', '2.2.0']), |
2884 | + ('kilo', |
2885 | + ['2.2.1', '2.2.2']), |
2886 | + ('liberty', |
2887 | + ['2.3.0', '2.4.0', '2.5.0']), |
2888 | + ('mitaka', |
2889 | + ['2.5.0', '2.6.0', '2.7.0']), |
2890 | + ('newton', |
2891 | + ['2.8.0', '2.9.0']), |
2892 | ]) |
2893 | |
2894 | # >= Liberty version->codename mapping |
2895 | PACKAGE_CODENAMES = { |
2896 | 'nova-common': OrderedDict([ |
2897 | - ('12.0.0', 'liberty'), |
2898 | + ('12', 'liberty'), |
2899 | + ('13', 'mitaka'), |
2900 | + ('14', 'newton'), |
2901 | + ('15', 'ocata'), |
2902 | ]), |
2903 | 'neutron-common': OrderedDict([ |
2904 | - ('7.0.0', 'liberty'), |
2905 | + ('7', 'liberty'), |
2906 | + ('8', 'mitaka'), |
2907 | + ('9', 'newton'), |
2908 | + ('10', 'ocata'), |
2909 | ]), |
2910 | 'cinder-common': OrderedDict([ |
2911 | - ('7.0.0', 'liberty'), |
2912 | + ('7', 'liberty'), |
2913 | + ('8', 'mitaka'), |
2914 | + ('9', 'newton'), |
2915 | + ('10', 'ocata'), |
2916 | ]), |
2917 | 'keystone': OrderedDict([ |
2918 | - ('8.0.0', 'liberty'), |
2919 | + ('8', 'liberty'), |
2920 | + ('9', 'mitaka'), |
2921 | + ('10', 'newton'), |
2922 | + ('11', 'ocata'), |
2923 | ]), |
2924 | 'horizon-common': OrderedDict([ |
2925 | - ('8.0.0', 'liberty'), |
2926 | + ('8', 'liberty'), |
2927 | + ('9', 'mitaka'), |
2928 | + ('10', 'newton'), |
2929 | + ('11', 'ocata'), |
2930 | ]), |
2931 | 'ceilometer-common': OrderedDict([ |
2932 | - ('5.0.0', 'liberty'), |
2933 | + ('5', 'liberty'), |
2934 | + ('6', 'mitaka'), |
2935 | + ('7', 'newton'), |
2936 | + ('8', 'ocata'), |
2937 | ]), |
2938 | 'heat-common': OrderedDict([ |
2939 | - ('5.0.0', 'liberty'), |
2940 | + ('5', 'liberty'), |
2941 | + ('6', 'mitaka'), |
2942 | + ('7', 'newton'), |
2943 | + ('8', 'ocata'), |
2944 | ]), |
2945 | 'glance-common': OrderedDict([ |
2946 | - ('11.0.0', 'liberty'), |
2947 | + ('11', 'liberty'), |
2948 | + ('12', 'mitaka'), |
2949 | + ('13', 'newton'), |
2950 | + ('14', 'ocata'), |
2951 | ]), |
2952 | 'openstack-dashboard': OrderedDict([ |
2953 | - ('8.0.0', 'liberty'), |
2954 | + ('8', 'liberty'), |
2955 | + ('9', 'mitaka'), |
2956 | + ('10', 'newton'), |
2957 | + ('11', 'ocata'), |
2958 | ]), |
2959 | } |
2960 | |
2961 | +GIT_DEFAULT_REPOS = { |
2962 | + 'requirements': 'git://github.com/openstack/requirements', |
2963 | + 'cinder': 'git://github.com/openstack/cinder', |
2964 | + 'glance': 'git://github.com/openstack/glance', |
2965 | + 'horizon': 'git://github.com/openstack/horizon', |
2966 | + 'keystone': 'git://github.com/openstack/keystone', |
2967 | + 'networking-hyperv': 'git://github.com/openstack/networking-hyperv', |
2968 | + 'neutron': 'git://github.com/openstack/neutron', |
2969 | + 'neutron-fwaas': 'git://github.com/openstack/neutron-fwaas', |
2970 | + 'neutron-lbaas': 'git://github.com/openstack/neutron-lbaas', |
2971 | + 'neutron-vpnaas': 'git://github.com/openstack/neutron-vpnaas', |
2972 | + 'nova': 'git://github.com/openstack/nova', |
2973 | +} |
2974 | + |
2975 | +GIT_DEFAULT_BRANCHES = { |
2976 | + 'liberty': 'stable/liberty', |
2977 | + 'mitaka': 'stable/mitaka', |
2978 | + 'master': 'master', |
2979 | +} |
2980 | + |
2981 | DEFAULT_LOOPBACK_SIZE = '5G' |
2982 | |
2983 | |
2984 | @@ -213,6 +285,44 @@ |
2985 | error_out(e) |
2986 | |
2987 | |
2988 | +def get_os_version_codename_swift(codename): |
2989 | + '''Determine OpenStack version number of swift from codename.''' |
2990 | + for k, v in six.iteritems(SWIFT_CODENAMES): |
2991 | + if k == codename: |
2992 | + return v[-1] |
2993 | + e = 'Could not derive swift version for '\ |
2994 | + 'codename: %s' % codename |
2995 | + error_out(e) |
2996 | + |
2997 | + |
2998 | +def get_swift_codename(version): |
2999 | + '''Determine OpenStack codename that corresponds to swift version.''' |
3000 | + codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v] |
3001 | + |
3002 | + if len(codenames) > 1: |
3003 | + # If more than one release codename contains this version we determine |
3004 | + # the actual codename based on the highest available install source. |
3005 | + for codename in reversed(codenames): |
3006 | + releases = UBUNTU_OPENSTACK_RELEASE |
3007 | + release = [k for k, v in six.iteritems(releases) if codename in v] |
3008 | + ret = subprocess.check_output(['apt-cache', 'policy', 'swift']) |
3009 | + if codename in ret or release[0] in ret: |
3010 | + return codename |
3011 | + elif len(codenames) == 1: |
3012 | + return codenames[0] |
3013 | + |
3014 | + # NOTE: fallback - attempt to match with just major.minor version |
3015 | + match = re.match('^(\d+)\.(\d+)', version) |
3016 | + if match: |
3017 | + major_minor_version = match.group(0) |
3018 | + for codename, versions in six.iteritems(SWIFT_CODENAMES): |
3019 | + for release_version in versions: |
3020 | + if release_version.startswith(major_minor_version): |
3021 | + return codename |
3022 | + |
3023 | + return None |
3024 | + |
3025 | + |
3026 | def get_os_codename_package(package, fatal=True): |
3027 | '''Derive OpenStack release codename from an installed package.''' |
3028 | import apt_pkg as apt |
3029 | @@ -237,25 +347,30 @@ |
3030 | error_out(e) |
3031 | |
3032 | vers = apt.upstream_version(pkg.current_ver.ver_str) |
3033 | - match = re.match('^(\d+)\.(\d+)\.(\d+)', vers) |
3034 | + if 'swift' in pkg.name: |
3035 | + # Fully x.y.z match for swift versions |
3036 | + match = re.match('^(\d+)\.(\d+)\.(\d+)', vers) |
3037 | + else: |
3038 | + # x.y match only for 20XX.X |
3039 | + # and ignore patch level for other packages |
3040 | + match = re.match('^(\d+)\.(\d+)', vers) |
3041 | + |
3042 | if match: |
3043 | vers = match.group(0) |
3044 | |
3045 | + # Generate a major version number for newer semantic |
3046 | + # versions of openstack projects |
3047 | + major_vers = vers.split('.')[0] |
3048 | # >= Liberty independent project versions |
3049 | if (package in PACKAGE_CODENAMES and |
3050 | - vers in PACKAGE_CODENAMES[package]): |
3051 | - return PACKAGE_CODENAMES[package][vers] |
3052 | + major_vers in PACKAGE_CODENAMES[package]): |
3053 | + return PACKAGE_CODENAMES[package][major_vers] |
3054 | else: |
3055 | # < Liberty co-ordinated project versions |
3056 | try: |
3057 | if 'swift' in pkg.name: |
3058 | - swift_vers = vers[:5] |
3059 | - if swift_vers not in SWIFT_CODENAMES: |
3060 | - # Deal with 1.10.0 upward |
3061 | - swift_vers = vers[:6] |
3062 | - return SWIFT_CODENAMES[swift_vers] |
3063 | + return get_swift_codename(vers) |
3064 | else: |
3065 | - vers = vers[:6] |
3066 | return OPENSTACK_CODENAMES[vers] |
3067 | except KeyError: |
3068 | if not fatal: |
3069 | @@ -273,12 +388,14 @@ |
3070 | |
3071 | if 'swift' in pkg: |
3072 | vers_map = SWIFT_CODENAMES |
3073 | + for cname, version in six.iteritems(vers_map): |
3074 | + if cname == codename: |
3075 | + return version[-1] |
3076 | else: |
3077 | vers_map = OPENSTACK_CODENAMES |
3078 | - |
3079 | - for version, cname in six.iteritems(vers_map): |
3080 | - if cname == codename: |
3081 | - return version |
3082 | + for version, cname in six.iteritems(vers_map): |
3083 | + if cname == codename: |
3084 | + return version |
3085 | # e = "Could not determine OpenStack version for package: %s" % pkg |
3086 | # error_out(e) |
3087 | |
3088 | @@ -296,19 +413,50 @@ |
3089 | global os_rel |
3090 | if os_rel: |
3091 | return os_rel |
3092 | - os_rel = (get_os_codename_package(package, fatal=False) or |
3093 | + os_rel = (git_os_codename_install_source(config('openstack-origin-git')) or |
3094 | + get_os_codename_package(package, fatal=False) or |
3095 | get_os_codename_install_source(config('openstack-origin')) or |
3096 | base) |
3097 | return os_rel |
3098 | |
3099 | |
3100 | def import_key(keyid): |
3101 | - cmd = "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 " \ |
3102 | - "--recv-keys %s" % keyid |
3103 | - try: |
3104 | - subprocess.check_call(cmd.split(' ')) |
3105 | - except subprocess.CalledProcessError: |
3106 | - error_out("Error importing repo key %s" % keyid) |
3107 | + key = keyid.strip() |
3108 | + if (key.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----') and |
3109 | + key.endswith('-----END PGP PUBLIC KEY BLOCK-----')): |
3110 | + juju_log("PGP key found (looks like ASCII Armor format)", level=DEBUG) |
3111 | + juju_log("Importing ASCII Armor PGP key", level=DEBUG) |
3112 | + with tempfile.NamedTemporaryFile() as keyfile: |
3113 | + with open(keyfile.name, 'w') as fd: |
3114 | + fd.write(key) |
3115 | + fd.write("\n") |
3116 | + |
3117 | + cmd = ['apt-key', 'add', keyfile.name] |
3118 | + try: |
3119 | + subprocess.check_call(cmd) |
3120 | + except subprocess.CalledProcessError: |
3121 | + error_out("Error importing PGP key '%s'" % key) |
3122 | + else: |
3123 | + juju_log("PGP key found (looks like Radix64 format)", level=DEBUG) |
3124 | + juju_log("Importing PGP key from keyserver", level=DEBUG) |
3125 | + cmd = ['apt-key', 'adv', '--keyserver', |
3126 | + 'hkp://keyserver.ubuntu.com:80', '--recv-keys', key] |
3127 | + try: |
3128 | + subprocess.check_call(cmd) |
3129 | + except subprocess.CalledProcessError: |
3130 | + error_out("Error importing PGP key '%s'" % key) |
3131 | + |
3132 | + |
3133 | +def get_source_and_pgp_key(input): |
3134 | + """Look for a pgp key ID or ascii-armor key in the given input.""" |
3135 | + index = input.strip() |
3136 | + index = input.rfind('|') |
3137 | + if index < 0: |
3138 | + return input, None |
3139 | + |
3140 | + key = input[index + 1:].strip('|') |
3141 | + source = input[:index] |
3142 | + return source, key |
3143 | |
3144 | |
3145 | def configure_installation_source(rel): |
3146 | @@ -320,16 +468,16 @@ |
3147 | with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f: |
3148 | f.write(DISTRO_PROPOSED % ubuntu_rel) |
3149 | elif rel[:4] == "ppa:": |
3150 | - src = rel |
3151 | + src, key = get_source_and_pgp_key(rel) |
3152 | + if key: |
3153 | + import_key(key) |
3154 | + |
3155 | subprocess.check_call(["add-apt-repository", "-y", src]) |
3156 | elif rel[:3] == "deb": |
3157 | - l = len(rel.split('|')) |
3158 | - if l == 2: |
3159 | - src, key = rel.split('|') |
3160 | - juju_log("Importing PPA key from keyserver for %s" % src) |
3161 | + src, key = get_source_and_pgp_key(rel) |
3162 | + if key: |
3163 | import_key(key) |
3164 | - elif l == 1: |
3165 | - src = rel |
3166 | + |
3167 | with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f: |
3168 | f.write(src) |
3169 | elif rel[:6] == 'cloud:': |
3170 | @@ -374,6 +522,12 @@ |
3171 | 'liberty': 'trusty-updates/liberty', |
3172 | 'liberty/updates': 'trusty-updates/liberty', |
3173 | 'liberty/proposed': 'trusty-proposed/liberty', |
3174 | + 'mitaka': 'trusty-updates/mitaka', |
3175 | + 'mitaka/updates': 'trusty-updates/mitaka', |
3176 | + 'mitaka/proposed': 'trusty-proposed/mitaka', |
3177 | + 'newton': 'xenial-updates/newton', |
3178 | + 'newton/updates': 'xenial-updates/newton', |
3179 | + 'newton/proposed': 'xenial-proposed/newton', |
3180 | } |
3181 | |
3182 | try: |
3183 | @@ -441,11 +595,16 @@ |
3184 | cur_vers = get_os_version_package(package) |
3185 | if "swift" in package: |
3186 | codename = get_os_codename_install_source(src) |
3187 | - available_vers = get_os_version_codename(codename, SWIFT_CODENAMES) |
3188 | + avail_vers = get_os_version_codename_swift(codename) |
3189 | else: |
3190 | - available_vers = get_os_version_install_source(src) |
3191 | + avail_vers = get_os_version_install_source(src) |
3192 | apt.init() |
3193 | - return apt.version_compare(available_vers, cur_vers) == 1 |
3194 | + if "swift" in package: |
3195 | + major_cur_vers = cur_vers.split('.', 1)[0] |
3196 | + major_avail_vers = avail_vers.split('.', 1)[0] |
3197 | + major_diff = apt.version_compare(major_avail_vers, major_cur_vers) |
3198 | + return avail_vers > cur_vers and (major_diff == 1 or major_diff == 0) |
3199 | + return apt.version_compare(avail_vers, cur_vers) == 1 |
3200 | |
3201 | |
3202 | def ensure_block_device(block_device): |
3203 | @@ -561,7 +720,86 @@ |
3204 | return config('openstack-origin-git') is not None |
3205 | |
3206 | |
3207 | -requirements_dir = None |
3208 | +def git_os_codename_install_source(projects_yaml): |
3209 | + """ |
3210 | + Returns OpenStack codename of release being installed from source. |
3211 | + """ |
3212 | + if git_install_requested(): |
3213 | + projects = _git_yaml_load(projects_yaml) |
3214 | + |
3215 | + if projects in GIT_DEFAULT_BRANCHES.keys(): |
3216 | + if projects == 'master': |
3217 | + return 'newton' |
3218 | + return projects |
3219 | + |
3220 | + if 'release' in projects: |
3221 | + if projects['release'] == 'master': |
3222 | + return 'newton' |
3223 | + return projects['release'] |
3224 | + |
3225 | + return None |
3226 | + |
3227 | + |
3228 | +def git_default_repos(projects_yaml): |
3229 | + """ |
3230 | + Returns default repos if a default openstack-origin-git value is specified. |
3231 | + """ |
3232 | + service = service_name() |
3233 | + core_project = service |
3234 | + |
3235 | + for default, branch in GIT_DEFAULT_BRANCHES.iteritems(): |
3236 | + if projects_yaml == default: |
3237 | + |
3238 | + # add the requirements repo first |
3239 | + repo = { |
3240 | + 'name': 'requirements', |
3241 | + 'repository': GIT_DEFAULT_REPOS['requirements'], |
3242 | + 'branch': branch, |
3243 | + } |
3244 | + repos = [repo] |
3245 | + |
3246 | + # neutron-* and nova-* charms require some additional repos |
3247 | + if service in ['neutron-api', 'neutron-gateway', |
3248 | + 'neutron-openvswitch']: |
3249 | + core_project = 'neutron' |
3250 | + if service == 'neutron-api': |
3251 | + repo = { |
3252 | + 'name': 'networking-hyperv', |
3253 | + 'repository': GIT_DEFAULT_REPOS['networking-hyperv'], |
3254 | + 'branch': branch, |
3255 | + } |
3256 | + repos.append(repo) |
3257 | + for project in ['neutron-fwaas', 'neutron-lbaas', |
3258 | + 'neutron-vpnaas', 'nova']: |
3259 | + repo = { |
3260 | + 'name': project, |
3261 | + 'repository': GIT_DEFAULT_REPOS[project], |
3262 | + 'branch': branch, |
3263 | + } |
3264 | + repos.append(repo) |
3265 | + |
3266 | + elif service in ['nova-cloud-controller', 'nova-compute']: |
3267 | + core_project = 'nova' |
3268 | + repo = { |
3269 | + 'name': 'neutron', |
3270 | + 'repository': GIT_DEFAULT_REPOS['neutron'], |
3271 | + 'branch': branch, |
3272 | + } |
3273 | + repos.append(repo) |
3274 | + elif service == 'openstack-dashboard': |
3275 | + core_project = 'horizon' |
3276 | + |
3277 | + # finally add the current service's core project repo |
3278 | + repo = { |
3279 | + 'name': core_project, |
3280 | + 'repository': GIT_DEFAULT_REPOS[core_project], |
3281 | + 'branch': branch, |
3282 | + } |
3283 | + repos.append(repo) |
3284 | + |
3285 | + return yaml.dump(dict(repositories=repos, release=default)) |
3286 | + |
3287 | + return projects_yaml |
3288 | |
3289 | |
3290 | def _git_yaml_load(projects_yaml): |
3291 | @@ -574,7 +812,10 @@ |
3292 | return yaml.load(projects_yaml) |
3293 | |
3294 | |
3295 | -def git_clone_and_install(projects_yaml, core_project, depth=1): |
3296 | +requirements_dir = None |
3297 | + |
3298 | + |
3299 | +def git_clone_and_install(projects_yaml, core_project): |
3300 | """ |
3301 | Clone/install all specified OpenStack repositories. |
3302 | |
3303 | @@ -621,18 +862,31 @@ |
3304 | pip_install(p, upgrade=True, proxy=http_proxy, |
3305 | venv=os.path.join(parent_dir, 'venv')) |
3306 | |
3307 | + constraints = None |
3308 | for p in projects['repositories']: |
3309 | repo = p['repository'] |
3310 | branch = p['branch'] |
3311 | + depth = '1' |
3312 | + if 'depth' in p.keys(): |
3313 | + depth = p['depth'] |
3314 | if p['name'] == 'requirements': |
3315 | repo_dir = _git_clone_and_install_single(repo, branch, depth, |
3316 | parent_dir, http_proxy, |
3317 | update_requirements=False) |
3318 | requirements_dir = repo_dir |
3319 | + constraints = os.path.join(repo_dir, "upper-constraints.txt") |
3320 | + # upper-constraints didn't exist until after icehouse |
3321 | + if not os.path.isfile(constraints): |
3322 | + constraints = None |
3323 | + # use constraints unless project yaml sets use_constraints to false |
3324 | + if 'use_constraints' in projects.keys(): |
3325 | + if not projects['use_constraints']: |
3326 | + constraints = None |
3327 | else: |
3328 | repo_dir = _git_clone_and_install_single(repo, branch, depth, |
3329 | parent_dir, http_proxy, |
3330 | - update_requirements=True) |
3331 | + update_requirements=True, |
3332 | + constraints=constraints) |
3333 | |
3334 | os.environ = old_environ |
3335 | |
3336 | @@ -654,6 +908,8 @@ |
3337 | if projects['repositories'][-1]['name'] != core_project: |
3338 | error_out('{} git repo must be specified last'.format(core_project)) |
3339 | |
3340 | + _git_ensure_key_exists('release', projects) |
3341 | + |
3342 | |
3343 | def _git_ensure_key_exists(key, keys): |
3344 | """ |
3345 | @@ -664,23 +920,18 @@ |
3346 | |
3347 | |
3348 | def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, |
3349 | - update_requirements): |
3350 | + update_requirements, constraints=None): |
3351 | """ |
3352 | Clone and install a single git repository. |
3353 | """ |
3354 | - dest_dir = os.path.join(parent_dir, os.path.basename(repo)) |
3355 | - |
3356 | if not os.path.exists(parent_dir): |
3357 | juju_log('Directory already exists at {}. ' |
3358 | 'No need to create directory.'.format(parent_dir)) |
3359 | os.mkdir(parent_dir) |
3360 | |
3361 | - if not os.path.exists(dest_dir): |
3362 | - juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) |
3363 | - repo_dir = install_remote(repo, dest=parent_dir, branch=branch, |
3364 | - depth=depth) |
3365 | - else: |
3366 | - repo_dir = dest_dir |
3367 | + juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) |
3368 | + repo_dir = install_remote( |
3369 | + repo, dest=parent_dir, branch=branch, depth=depth) |
3370 | |
3371 | venv = os.path.join(parent_dir, 'venv') |
3372 | |
3373 | @@ -692,9 +943,10 @@ |
3374 | |
3375 | juju_log('Installing git repo from dir: {}'.format(repo_dir)) |
3376 | if http_proxy: |
3377 | - pip_install(repo_dir, proxy=http_proxy, venv=venv) |
3378 | + pip_install(repo_dir, proxy=http_proxy, venv=venv, |
3379 | + constraints=constraints) |
3380 | else: |
3381 | - pip_install(repo_dir, venv=venv) |
3382 | + pip_install(repo_dir, venv=venv, constraints=constraints) |
3383 | |
3384 | return repo_dir |
3385 | |
3386 | @@ -763,6 +1015,85 @@ |
3387 | return None |
3388 | |
3389 | |
3390 | +def git_generate_systemd_init_files(templates_dir): |
3391 | + """ |
3392 | + Generate systemd init files. |
3393 | + |
3394 | + Generates and installs systemd init units and script files based on the |
3395 | + *.init.in files contained in the templates_dir directory. |
3396 | + |
3397 | + This code is based on the openstack-pkg-tools package and its init |
3398 | + script generation, which is used by the OpenStack packages. |
3399 | + """ |
3400 | + for f in os.listdir(templates_dir): |
3401 | + # Create the init script and systemd unit file from the template |
3402 | + if f.endswith(".init.in"): |
3403 | + init_in_file = f |
3404 | + init_file = f[:-8] |
3405 | + service_file = "{}.service".format(init_file) |
3406 | + |
3407 | + init_in_source = os.path.join(templates_dir, init_in_file) |
3408 | + init_source = os.path.join(templates_dir, init_file) |
3409 | + service_source = os.path.join(templates_dir, service_file) |
3410 | + |
3411 | + init_dest = os.path.join('/etc/init.d', init_file) |
3412 | + service_dest = os.path.join('/lib/systemd/system', service_file) |
3413 | + |
3414 | + shutil.copyfile(init_in_source, init_source) |
3415 | + with open(init_source, 'a') as outfile: |
3416 | + template = '/usr/share/openstack-pkg-tools/init-script-template' |
3417 | + with open(template) as infile: |
3418 | + outfile.write('\n\n{}'.format(infile.read())) |
3419 | + |
3420 | + cmd = ['pkgos-gen-systemd-unit', init_in_source] |
3421 | + subprocess.check_call(cmd) |
3422 | + |
3423 | + if os.path.exists(init_dest): |
3424 | + os.remove(init_dest) |
3425 | + if os.path.exists(service_dest): |
3426 | + os.remove(service_dest) |
3427 | + shutil.copyfile(init_source, init_dest) |
3428 | + shutil.copyfile(service_source, service_dest) |
3429 | + os.chmod(init_dest, 0o755) |
3430 | + |
3431 | + for f in os.listdir(templates_dir): |
3432 | + # If there's a service.in file, use it instead of the generated one |
3433 | + if f.endswith(".service.in"): |
3434 | + service_in_file = f |
3435 | + service_file = f[:-3] |
3436 | + |
3437 | + service_in_source = os.path.join(templates_dir, service_in_file) |
3438 | + service_source = os.path.join(templates_dir, service_file) |
3439 | + service_dest = os.path.join('/lib/systemd/system', service_file) |
3440 | + |
3441 | + shutil.copyfile(service_in_source, service_source) |
3442 | + |
3443 | + if os.path.exists(service_dest): |
3444 | + os.remove(service_dest) |
3445 | + shutil.copyfile(service_source, service_dest) |
3446 | + |
3447 | + for f in os.listdir(templates_dir): |
3448 | + # Generate the systemd unit if there's no existing .service.in |
3449 | + if f.endswith(".init.in"): |
3450 | + init_in_file = f |
3451 | + init_file = f[:-8] |
3452 | + service_in_file = "{}.service.in".format(init_file) |
3453 | + service_file = "{}.service".format(init_file) |
3454 | + |
3455 | + init_in_source = os.path.join(templates_dir, init_in_file) |
3456 | + service_in_source = os.path.join(templates_dir, service_in_file) |
3457 | + service_source = os.path.join(templates_dir, service_file) |
3458 | + service_dest = os.path.join('/lib/systemd/system', service_file) |
3459 | + |
3460 | + if not os.path.exists(service_in_source): |
3461 | + cmd = ['pkgos-gen-systemd-unit', init_in_source] |
3462 | + subprocess.check_call(cmd) |
3463 | + |
3464 | + if os.path.exists(service_dest): |
3465 | + os.remove(service_dest) |
3466 | + shutil.copyfile(service_source, service_dest) |
3467 | + |
3468 | + |
3469 | def os_workload_status(configs, required_interfaces, charm_func=None): |
3470 | """ |
3471 | Decorator to set workload status based on complete contexts |
3472 | @@ -779,56 +1110,155 @@ |
3473 | return wrap |
3474 | |
3475 | |
3476 | -def set_os_workload_status(configs, required_interfaces, charm_func=None): |
3477 | - """ |
3478 | - Set workload status based on complete contexts. |
3479 | - status-set missing or incomplete contexts |
3480 | - and juju-log details of missing required data. |
3481 | - charm_func is a charm specific function to run checking |
3482 | - for charm specific requirements such as a VIP setting. |
3483 | - """ |
3484 | - incomplete_rel_data = incomplete_relation_data(configs, required_interfaces) |
3485 | - state = 'active' |
3486 | - missing_relations = [] |
3487 | - incomplete_relations = [] |
3488 | +def set_os_workload_status(configs, required_interfaces, charm_func=None, |
3489 | + services=None, ports=None): |
3490 | + """Set the state of the workload status for the charm. |
3491 | + |
3492 | + This calls _determine_os_workload_status() to get the new state, message |
3493 | + and sets the status using status_set() |
3494 | + |
3495 | + @param configs: a templating.OSConfigRenderer() object |
3496 | + @param required_interfaces: {generic: [specific, specific2, ...]} |
3497 | + @param charm_func: a callable function that returns state, message. The |
3498 | + signature is charm_func(configs) -> (state, message) |
3499 | + @param services: list of strings OR dictionary specifying services/ports |
3500 | + @param ports: OPTIONAL list of port numbers. |
3501 | + @returns state, message: the new workload status, user message |
3502 | + """ |
3503 | + state, message = _determine_os_workload_status( |
3504 | + configs, required_interfaces, charm_func, services, ports) |
3505 | + status_set(state, message) |
3506 | + |
3507 | + |
3508 | +def _determine_os_workload_status( |
3509 | + configs, required_interfaces, charm_func=None, |
3510 | + services=None, ports=None): |
3511 | + """Determine the state of the workload status for the charm. |
3512 | + |
3513 | + This function returns the new workload status for the charm based |
3514 | + on the state of the interfaces, the paused state and whether the |
3515 | + services are actually running and any specified ports are open. |
3516 | + |
3517 | + This checks: |
3518 | + |
3519 | + 1. if the unit should be paused, that it is actually paused. If so the |
3520 | + state is 'maintenance' + message, else 'broken'. |
3521 | + 2. that the interfaces/relations are complete. If they are not then |
3522 | + it sets the state to either 'broken' or 'waiting' and an appropriate |
3523 | + message. |
3524 | + 3. If all the relation data is set, then it checks that the actual |
3525 | + services really are running. If not it sets the state to 'broken'. |
3526 | + |
3527 | + If everything is okay then the state returns 'active'. |
3528 | + |
3529 | + @param configs: a templating.OSConfigRenderer() object |
3530 | + @param required_interfaces: {generic: [specific, specific2, ...]} |
3531 | + @param charm_func: a callable function that returns state, message. The |
3532 | + signature is charm_func(configs) -> (state, message) |
3533 | + @param services: list of strings OR dictionary specifying services/ports |
3534 | + @param ports: OPTIONAL list of port numbers. |
3535 | + @returns state, message: the new workload status, user message |
3536 | + """ |
3537 | + state, message = _ows_check_if_paused(services, ports) |
3538 | + |
3539 | + if state is None: |
3540 | + state, message = _ows_check_generic_interfaces( |
3541 | + configs, required_interfaces) |
3542 | + |
3543 | + if state != 'maintenance' and charm_func: |
3544 | + # _ows_check_charm_func() may modify the state, message |
3545 | + state, message = _ows_check_charm_func( |
3546 | + state, message, lambda: charm_func(configs)) |
3547 | + |
3548 | + if state is None: |
3549 | + state, message = _ows_check_services_running(services, ports) |
3550 | + |
3551 | + if state is None: |
3552 | + state = 'active' |
3553 | + message = "Unit is ready" |
3554 | + juju_log(message, 'INFO') |
3555 | + |
3556 | + return state, message |
3557 | + |
3558 | + |
3559 | +def _ows_check_if_paused(services=None, ports=None): |
3560 | + """Check if the unit is supposed to be paused, and if so check that the |
3561 | + services/ports (if passed) are actually stopped/not being listened to. |
3562 | + |
3563 | + if the unit isn't supposed to be paused, just return None, None |
3564 | + |
3565 | + @param services: OPTIONAL services spec or list of service names. |
3566 | + @param ports: OPTIONAL list of port numbers. |
3567 | + @returns state, message or None, None |
3568 | + """ |
3569 | + if is_unit_paused_set(): |
3570 | + state, message = check_actually_paused(services=services, |
3571 | + ports=ports) |
3572 | + if state is None: |
3573 | + # we're paused okay, so set maintenance and return |
3574 | + state = "maintenance" |
3575 | + message = "Paused. Use 'resume' action to resume normal service." |
3576 | + return state, message |
3577 | + return None, None |
3578 | + |
3579 | + |
3580 | +def _ows_check_generic_interfaces(configs, required_interfaces): |
3581 | + """Check the complete contexts to determine the workload status. |
3582 | + |
3583 | + - Checks for missing or incomplete contexts |
3584 | + - juju log details of missing required data. |
3585 | + - determines the correct workload status |
3586 | + - creates an appropriate message for status_set(...) |
3587 | + |
3588 | + if there are no problems then the function returns None, None |
3589 | + |
3590 | + @param configs: a templating.OSConfigRenderer() object |
3591 | + @params required_interfaces: {generic_interface: [specific_interface], } |
3592 | + @returns state, message or None, None |
3593 | + """ |
3594 | + incomplete_rel_data = incomplete_relation_data(configs, |
3595 | + required_interfaces) |
3596 | + state = None |
3597 | message = None |
3598 | - charm_state = None |
3599 | - charm_message = None |
3600 | + missing_relations = set() |
3601 | + incomplete_relations = set() |
3602 | |
3603 | - for generic_interface in incomplete_rel_data.keys(): |
3604 | + for generic_interface, relations_states in incomplete_rel_data.items(): |
3605 | related_interface = None |
3606 | missing_data = {} |
3607 | # Related or not? |
3608 | - for interface in incomplete_rel_data[generic_interface]: |
3609 | - if incomplete_rel_data[generic_interface][interface].get('related'): |
3610 | + for interface, relation_state in relations_states.items(): |
3611 | + if relation_state.get('related'): |
3612 | related_interface = interface |
3613 | - missing_data = incomplete_rel_data[generic_interface][interface].get('missing_data') |
3614 | - # No relation ID for the generic_interface |
3615 | + missing_data = relation_state.get('missing_data') |
3616 | + break |
3617 | + # No relation ID for the generic_interface? |
3618 | if not related_interface: |
3619 | juju_log("{} relation is missing and must be related for " |
3620 | "functionality. ".format(generic_interface), 'WARN') |
3621 | state = 'blocked' |
3622 | - if generic_interface not in missing_relations: |
3623 | - missing_relations.append(generic_interface) |
3624 | + missing_relations.add(generic_interface) |
3625 | else: |
3626 | - # Relation ID exists but no related unit |
3627 | + # Relation ID eists but no related unit |
3628 | if not missing_data: |
3629 | - # Edge case relation ID exists but departing |
3630 | - if ('departed' in hook_name() or 'broken' in hook_name()) \ |
3631 | - and related_interface in hook_name(): |
3632 | + # Edge case - relation ID exists but departings |
3633 | + _hook_name = hook_name() |
3634 | + if (('departed' in _hook_name or 'broken' in _hook_name) and |
3635 | + related_interface in _hook_name): |
3636 | state = 'blocked' |
3637 | - if generic_interface not in missing_relations: |
3638 | - missing_relations.append(generic_interface) |
3639 | + missing_relations.add(generic_interface) |
3640 | juju_log("{} relation's interface, {}, " |
3641 | "relationship is departed or broken " |
3642 | "and is required for functionality." |
3643 | - "".format(generic_interface, related_interface), "WARN") |
3644 | + "".format(generic_interface, related_interface), |
3645 | + "WARN") |
3646 | # Normal case relation ID exists but no related unit |
3647 | # (joining) |
3648 | else: |
3649 | - juju_log("{} relations's interface, {}, is related but has " |
3650 | - "no units in the relation." |
3651 | - "".format(generic_interface, related_interface), "INFO") |
3652 | + juju_log("{} relations's interface, {}, is related but has" |
3653 | + " no units in the relation." |
3654 | + "".format(generic_interface, related_interface), |
3655 | + "INFO") |
3656 | # Related unit exists and data missing on the relation |
3657 | else: |
3658 | juju_log("{} relation's interface, {}, is related awaiting " |
3659 | @@ -837,9 +1267,8 @@ |
3660 | ", ".join(missing_data)), "INFO") |
3661 | if state != 'blocked': |
3662 | state = 'waiting' |
3663 | - if generic_interface not in incomplete_relations \ |
3664 | - and generic_interface not in missing_relations: |
3665 | - incomplete_relations.append(generic_interface) |
3666 | + if generic_interface not in missing_relations: |
3667 | + incomplete_relations.add(generic_interface) |
3668 | |
3669 | if missing_relations: |
3670 | message = "Missing relations: {}".format(", ".join(missing_relations)) |
3671 | @@ -852,22 +1281,175 @@ |
3672 | "".format(", ".join(incomplete_relations)) |
3673 | state = 'waiting' |
3674 | |
3675 | - # Run charm specific checks |
3676 | - if charm_func: |
3677 | - charm_state, charm_message = charm_func(configs) |
3678 | + return state, message |
3679 | + |
3680 | + |
3681 | +def _ows_check_charm_func(state, message, charm_func_with_configs): |
3682 | + """Run a custom check function for the charm to see if it wants to |
3683 | + change the state. This is only run if not in 'maintenance' and |
3684 | + tests to see if the new state is more important that the previous |
3685 | + one determined by the interfaces/relations check. |
3686 | + |
3687 | + @param state: the previously determined state so far. |
3688 | + @param message: the user orientated message so far. |
3689 | + @param charm_func: a callable function that returns state, message |
3690 | + @returns state, message strings. |
3691 | + """ |
3692 | + if charm_func_with_configs: |
3693 | + charm_state, charm_message = charm_func_with_configs() |
3694 | if charm_state != 'active' and charm_state != 'unknown': |
3695 | state = workload_state_compare(state, charm_state) |
3696 | if message: |
3697 | - message = "{} {}".format(message, charm_message) |
3698 | + charm_message = charm_message.replace("Incomplete relations: ", |
3699 | + "") |
3700 | + message = "{}, {}".format(message, charm_message) |
3701 | else: |
3702 | message = charm_message |
3703 | - |
3704 | - # Set to active if all requirements have been met |
3705 | - if state == 'active': |
3706 | - message = "Unit is ready" |
3707 | - juju_log(message, "INFO") |
3708 | - |
3709 | - status_set(state, message) |
3710 | + return state, message |
3711 | + |
3712 | + |
3713 | +def _ows_check_services_running(services, ports): |
3714 | + """Check that the services that should be running are actually running |
3715 | + and that any ports specified are being listened to. |
3716 | + |
3717 | + @param services: list of strings OR dictionary specifying services/ports |
3718 | + @param ports: list of ports |
3719 | + @returns state, message: strings or None, None |
3720 | + """ |
3721 | + messages = [] |
3722 | + state = None |
3723 | + if services is not None: |
3724 | + services = _extract_services_list_helper(services) |
3725 | + services_running, running = _check_running_services(services) |
3726 | + if not all(running): |
3727 | + messages.append( |
3728 | + "Services not running that should be: {}" |
3729 | + .format(", ".join(_filter_tuples(services_running, False)))) |
3730 | + state = 'blocked' |
3731 | + # also verify that the ports that should be open are open |
3732 | + # NB, that ServiceManager objects only OPTIONALLY have ports |
3733 | + map_not_open, ports_open = ( |
3734 | + _check_listening_on_services_ports(services)) |
3735 | + if not all(ports_open): |
3736 | + # find which service has missing ports. They are in service |
3737 | + # order which makes it a bit easier. |
3738 | + message_parts = {service: ", ".join([str(v) for v in open_ports]) |
3739 | + for service, open_ports in map_not_open.items()} |
3740 | + message = ", ".join( |
3741 | + ["{}: [{}]".format(s, sp) for s, sp in message_parts.items()]) |
3742 | + messages.append( |
3743 | + "Services with ports not open that should be: {}" |
3744 | + .format(message)) |
3745 | + state = 'blocked' |
3746 | + |
3747 | + if ports is not None: |
3748 | + # and we can also check ports which we don't know the service for |
3749 | + ports_open, ports_open_bools = _check_listening_on_ports_list(ports) |
3750 | + if not all(ports_open_bools): |
3751 | + messages.append( |
3752 | + "Ports which should be open, but are not: {}" |
3753 | + .format(", ".join([str(p) for p, v in ports_open |
3754 | + if not v]))) |
3755 | + state = 'blocked' |
3756 | + |
3757 | + if state is not None: |
3758 | + message = "; ".join(messages) |
3759 | + return state, message |
3760 | + |
3761 | + return None, None |
3762 | + |
3763 | + |
3764 | +def _extract_services_list_helper(services): |
3765 | + """Extract a OrderedDict of {service: [ports]} of the supplied services |
3766 | + for use by the other functions. |
3767 | + |
3768 | + The services object can either be: |
3769 | + - None : no services were passed (an empty dict is returned) |
3770 | + - a list of strings |
3771 | + - A dictionary (optionally OrderedDict) {service_name: {'service': ..}} |
3772 | + - An array of [{'service': service_name, ...}, ...] |
3773 | + |
3774 | + @param services: see above |
3775 | + @returns OrderedDict(service: [ports], ...) |
3776 | + """ |
3777 | + if services is None: |
3778 | + return {} |
3779 | + if isinstance(services, dict): |
3780 | + services = services.values() |
3781 | + # either extract the list of services from the dictionary, or if |
3782 | + # it is a simple string, use that. i.e. works with mixed lists. |
3783 | + _s = OrderedDict() |
3784 | + for s in services: |
3785 | + if isinstance(s, dict) and 'service' in s: |
3786 | + _s[s['service']] = s.get('ports', []) |
3787 | + if isinstance(s, str): |
3788 | + _s[s] = [] |
3789 | + return _s |
3790 | + |
3791 | + |
3792 | +def _check_running_services(services): |
3793 | + """Check that the services dict provided is actually running and provide |
3794 | + a list of (service, boolean) tuples for each service. |
3795 | + |
3796 | + Returns both a zipped list of (service, boolean) and a list of booleans |
3797 | + in the same order as the services. |
3798 | + |
3799 | + @param services: OrderedDict of strings: [ports], one for each service to |
3800 | + check. |
3801 | + @returns [(service, boolean), ...], : results for checks |
3802 | + [boolean] : just the result of the service checks |
3803 | + """ |
3804 | + services_running = [service_running(s) for s in services] |
3805 | + return list(zip(services, services_running)), services_running |
3806 | + |
3807 | + |
3808 | +def _check_listening_on_services_ports(services, test=False): |
3809 | + """Check that the unit is actually listening (has the port open) on the |
3810 | + ports that the service specifies are open. If test is True then the |
3811 | + function returns the services with ports that are open rather than |
3812 | + closed. |
3813 | + |
3814 | + Returns an OrderedDict of service: ports and a list of booleans |
3815 | + |
3816 | + @param services: OrderedDict(service: [port, ...], ...) |
3817 | + @param test: default=False, if False, test for closed, otherwise open. |
3818 | + @returns OrderedDict(service: [port-not-open, ...]...), [boolean] |
3819 | + """ |
3820 | + test = not(not(test)) # ensure test is True or False |
3821 | + all_ports = list(itertools.chain(*services.values())) |
3822 | + ports_states = [port_has_listener('0.0.0.0', p) for p in all_ports] |
3823 | + map_ports = OrderedDict() |
3824 | + matched_ports = [p for p, opened in zip(all_ports, ports_states) |
3825 | + if opened == test] # essentially opened xor test |
3826 | + for service, ports in services.items(): |
3827 | + set_ports = set(ports).intersection(matched_ports) |
3828 | + if set_ports: |
3829 | + map_ports[service] = set_ports |
3830 | + return map_ports, ports_states |
3831 | + |
3832 | + |
3833 | +def _check_listening_on_ports_list(ports): |
3834 | + """Check that the ports list given are being listened to |
3835 | + |
3836 | + Returns a list of ports being listened to and a list of the |
3837 | + booleans. |
3838 | + |
3839 | + @param ports: LIST or port numbers. |
3840 | + @returns [(port_num, boolean), ...], [boolean] |
3841 | + """ |
3842 | + ports_open = [port_has_listener('0.0.0.0', p) for p in ports] |
3843 | + return zip(ports, ports_open), ports_open |
3844 | + |
3845 | + |
3846 | +def _filter_tuples(services_states, state): |
3847 | + """Return a simple list from a list of tuples according to the condition |
3848 | + |
3849 | + @param services_states: LIST of (string, boolean): service and running |
3850 | + state. |
3851 | + @param state: Boolean to match the tuple against. |
3852 | + @returns [LIST of strings] that matched the tuple RHS. |
3853 | + """ |
3854 | + return [s for s, b in services_states if b == state] |
3855 | |
3856 | |
3857 | def workload_state_compare(current_workload_state, workload_state): |
3858 | @@ -892,8 +1474,7 @@ |
3859 | |
3860 | |
3861 | def incomplete_relation_data(configs, required_interfaces): |
3862 | - """ |
3863 | - Check complete contexts against required_interfaces |
3864 | + """Check complete contexts against required_interfaces |
3865 | Return dictionary of incomplete relation data. |
3866 | |
3867 | configs is an OSConfigRenderer object with configs registered |
3868 | @@ -918,19 +1499,13 @@ |
3869 | 'shared-db': {'related': True}}} |
3870 | """ |
3871 | complete_ctxts = configs.complete_contexts() |
3872 | - incomplete_relations = [] |
3873 | - for svc_type in required_interfaces.keys(): |
3874 | - # Avoid duplicates |
3875 | - found_ctxt = False |
3876 | - for interface in required_interfaces[svc_type]: |
3877 | - if interface in complete_ctxts: |
3878 | - found_ctxt = True |
3879 | - if not found_ctxt: |
3880 | - incomplete_relations.append(svc_type) |
3881 | - incomplete_context_data = {} |
3882 | - for i in incomplete_relations: |
3883 | - incomplete_context_data[i] = configs.get_incomplete_context_data(required_interfaces[i]) |
3884 | - return incomplete_context_data |
3885 | + incomplete_relations = [ |
3886 | + svc_type |
3887 | + for svc_type, interfaces in required_interfaces.items() |
3888 | + if not set(interfaces).intersection(complete_ctxts)] |
3889 | + return { |
3890 | + i: configs.get_incomplete_context_data(required_interfaces[i]) |
3891 | + for i in incomplete_relations} |
3892 | |
3893 | |
3894 | def do_action_openstack_upgrade(package, upgrade_callback, configs): |
3895 | @@ -975,3 +1550,342 @@ |
3896 | action_set({'outcome': 'no upgrade available.'}) |
3897 | |
3898 | return ret |
3899 | + |
3900 | + |
3901 | +def remote_restart(rel_name, remote_service=None): |
3902 | + trigger = { |
3903 | + 'restart-trigger': str(uuid.uuid4()), |
3904 | + } |
3905 | + if remote_service: |
3906 | + trigger['remote-service'] = remote_service |
3907 | + for rid in relation_ids(rel_name): |
3908 | + # This subordinate can be related to two seperate services using |
3909 | + # different subordinate relations so only issue the restart if |
3910 | + # the principle is conencted down the relation we think it is |
3911 | + if related_units(relid=rid): |
3912 | + relation_set(relation_id=rid, |
3913 | + relation_settings=trigger, |
3914 | + ) |
3915 | + |
3916 | + |
3917 | +def check_actually_paused(services=None, ports=None): |
3918 | + """Check that services listed in the services object and and ports |
3919 | + are actually closed (not listened to), to verify that the unit is |
3920 | + properly paused. |
3921 | + |
3922 | + @param services: See _extract_services_list_helper |
3923 | + @returns status, : string for status (None if okay) |
3924 | + message : string for problem for status_set |
3925 | + """ |
3926 | + state = None |
3927 | + message = None |
3928 | + messages = [] |
3929 | + if services is not None: |
3930 | + services = _extract_services_list_helper(services) |
3931 | + services_running, services_states = _check_running_services(services) |
3932 | + if any(services_states): |
3933 | + # there shouldn't be any running so this is a problem |
3934 | + messages.append("these services running: {}" |
3935 | + .format(", ".join( |
3936 | + _filter_tuples(services_running, True)))) |
3937 | + state = "blocked" |
3938 | + ports_open, ports_open_bools = ( |
3939 | + _check_listening_on_services_ports(services, True)) |
3940 | + if any(ports_open_bools): |
3941 | + message_parts = {service: ", ".join([str(v) for v in open_ports]) |
3942 | + for service, open_ports in ports_open.items()} |
3943 | + message = ", ".join( |
3944 | + ["{}: [{}]".format(s, sp) for s, sp in message_parts.items()]) |
3945 | + messages.append( |
3946 | + "these service:ports are open: {}".format(message)) |
3947 | + state = 'blocked' |
3948 | + if ports is not None: |
3949 | + ports_open, bools = _check_listening_on_ports_list(ports) |
3950 | + if any(bools): |
3951 | + messages.append( |
3952 | + "these ports which should be closed, but are open: {}" |
3953 | + .format(", ".join([str(p) for p, v in ports_open if v]))) |
3954 | + state = 'blocked' |
3955 | + if messages: |
3956 | + message = ("Services should be paused but {}" |
3957 | + .format(", ".join(messages))) |
3958 | + return state, message |
3959 | + |
3960 | + |
3961 | +def set_unit_paused(): |
3962 | + """Set the unit to a paused state in the local kv() store. |
3963 | + This does NOT actually pause the unit |
3964 | + """ |
3965 | + with unitdata.HookData()() as t: |
3966 | + kv = t[0] |
3967 | + kv.set('unit-paused', True) |
3968 | + |
3969 | + |
3970 | +def clear_unit_paused(): |
3971 | + """Clear the unit from a paused state in the local kv() store |
3972 | + This does NOT actually restart any services - it only clears the |
3973 | + local state. |
3974 | + """ |
3975 | + with unitdata.HookData()() as t: |
3976 | + kv = t[0] |
3977 | + kv.set('unit-paused', False) |
3978 | + |
3979 | + |
3980 | +def is_unit_paused_set(): |
3981 | + """Return the state of the kv().get('unit-paused'). |
3982 | + This does NOT verify that the unit really is paused. |
3983 | + |
3984 | + To help with units that don't have HookData() (testing) |
3985 | + if it excepts, return False |
3986 | + """ |
3987 | + try: |
3988 | + with unitdata.HookData()() as t: |
3989 | + kv = t[0] |
3990 | + # transform something truth-y into a Boolean. |
3991 | + return not(not(kv.get('unit-paused'))) |
3992 | + except: |
3993 | + return False |
3994 | + |
3995 | + |
3996 | +def pause_unit(assess_status_func, services=None, ports=None, |
3997 | + charm_func=None): |
3998 | + """Pause a unit by stopping the services and setting 'unit-paused' |
3999 | + in the local kv() store. |
4000 | + |
4001 | + Also checks that the services have stopped and ports are no longer |
4002 | + being listened to. |
4003 | + |
4004 | + An optional charm_func() can be called that can either raise an |
4005 | + Exception or return non None, None to indicate that the unit |
4006 | + didn't pause cleanly. |
4007 | + |
4008 | + The signature for charm_func is: |
4009 | + charm_func() -> message: string |
4010 | + |
4011 | + charm_func() is executed after any services are stopped, if supplied. |
4012 | + |
4013 | + The services object can either be: |
4014 | + - None : no services were passed (an empty dict is returned) |
4015 | + - a list of strings |
4016 | + - A dictionary (optionally OrderedDict) {service_name: {'service': ..}} |
4017 | + - An array of [{'service': service_name, ...}, ...] |
4018 | + |
4019 | + @param assess_status_func: (f() -> message: string | None) or None |
4020 | + @param services: OPTIONAL see above |
4021 | + @param ports: OPTIONAL list of port |
4022 | + @param charm_func: function to run for custom charm pausing. |
4023 | + @returns None |
4024 | + @raises Exception(message) on an error for action_fail(). |
4025 | + """ |
4026 | + services = _extract_services_list_helper(services) |
4027 | + messages = [] |
4028 | + if services: |
4029 | + for service in services.keys(): |
4030 | + stopped = service_pause(service) |
4031 | + if not stopped: |
4032 | + messages.append("{} didn't stop cleanly.".format(service)) |
4033 | + if charm_func: |
4034 | + try: |
4035 | + message = charm_func() |
4036 | + if message: |
4037 | + messages.append(message) |
4038 | + except Exception as e: |
4039 | + message.append(str(e)) |
4040 | + set_unit_paused() |
4041 | + if assess_status_func: |
4042 | + message = assess_status_func() |
4043 | + if message: |
4044 | + messages.append(message) |
4045 | + if messages: |
4046 | + raise Exception("Couldn't pause: {}".format("; ".join(messages))) |
4047 | + |
4048 | + |
4049 | +def resume_unit(assess_status_func, services=None, ports=None, |
4050 | + charm_func=None): |
4051 | + """Resume a unit by starting the services and clearning 'unit-paused' |
4052 | + in the local kv() store. |
4053 | + |
4054 | + Also checks that the services have started and ports are being listened to. |
4055 | + |
4056 | + An optional charm_func() can be called that can either raise an |
4057 | + Exception or return non None to indicate that the unit |
4058 | + didn't resume cleanly. |
4059 | + |
4060 | + The signature for charm_func is: |
4061 | + charm_func() -> message: string |
4062 | + |
4063 | + charm_func() is executed after any services are started, if supplied. |
4064 | + |
4065 | + The services object can either be: |
4066 | + - None : no services were passed (an empty dict is returned) |
4067 | + - a list of strings |
4068 | + - A dictionary (optionally OrderedDict) {service_name: {'service': ..}} |
4069 | + - An array of [{'service': service_name, ...}, ...] |
4070 | + |
4071 | + @param assess_status_func: (f() -> message: string | None) or None |
4072 | + @param services: OPTIONAL see above |
4073 | + @param ports: OPTIONAL list of port |
4074 | + @param charm_func: function to run for custom charm resuming. |
4075 | + @returns None |
4076 | + @raises Exception(message) on an error for action_fail(). |
4077 | + """ |
4078 | + services = _extract_services_list_helper(services) |
4079 | + messages = [] |
4080 | + if services: |
4081 | + for service in services.keys(): |
4082 | + started = service_resume(service) |
4083 | + if not started: |
4084 | + messages.append("{} didn't start cleanly.".format(service)) |
4085 | + if charm_func: |
4086 | + try: |
4087 | + message = charm_func() |
4088 | + if message: |
4089 | + messages.append(message) |
4090 | + except Exception as e: |
4091 | + message.append(str(e)) |
4092 | + clear_unit_paused() |
4093 | + if assess_status_func: |
4094 | + message = assess_status_func() |
4095 | + if message: |
4096 | + messages.append(message) |
4097 | + if messages: |
4098 | + raise Exception("Couldn't resume: {}".format("; ".join(messages))) |
4099 | + |
4100 | + |
4101 | +def make_assess_status_func(*args, **kwargs): |
4102 | + """Creates an assess_status_func() suitable for handing to pause_unit() |
4103 | + and resume_unit(). |
4104 | + |
4105 | + This uses the _determine_os_workload_status(...) function to determine |
4106 | + what the workload_status should be for the unit. If the unit is |
4107 | + not in maintenance or active states, then the message is returned to |
4108 | + the caller. This is so an action that doesn't result in either a |
4109 | + complete pause or complete resume can signal failure with an action_fail() |
4110 | + """ |
4111 | + def _assess_status_func(): |
4112 | + state, message = _determine_os_workload_status(*args, **kwargs) |
4113 | + status_set(state, message) |
4114 | + if state not in ['maintenance', 'active']: |
4115 | + return message |
4116 | + return None |
4117 | + |
4118 | + return _assess_status_func |
4119 | + |
4120 | + |
4121 | +def pausable_restart_on_change(restart_map, stopstart=False, |
4122 | + restart_functions=None): |
4123 | + """A restart_on_change decorator that checks to see if the unit is |
4124 | + paused. If it is paused then the decorated function doesn't fire. |
4125 | + |
4126 | + This is provided as a helper, as the @restart_on_change(...) decorator |
4127 | + is in core.host, yet the openstack specific helpers are in this file |
4128 | + (contrib.openstack.utils). Thus, this needs to be an optional feature |
4129 | + for openstack charms (or charms that wish to use the openstack |
4130 | + pause/resume type features). |
4131 | + |
4132 | + It is used as follows: |
4133 | + |
4134 | + from contrib.openstack.utils import ( |
4135 | + pausable_restart_on_change as restart_on_change) |
4136 | + |
4137 | + @restart_on_change(restart_map, stopstart=<boolean>) |
4138 | + def some_hook(...): |
4139 | + pass |
4140 | + |
4141 | + see core.utils.restart_on_change() for more details. |
4142 | + |
4143 | + @param f: the function to decorate |
4144 | + @param restart_map: the restart map {conf_file: [services]} |
4145 | + @param stopstart: DEFAULT false; whether to stop, start or just restart |
4146 | + @returns decorator to use a restart_on_change with pausability |
4147 | + """ |
4148 | + def wrap(f): |
4149 | + @functools.wraps(f) |
4150 | + def wrapped_f(*args, **kwargs): |
4151 | + if is_unit_paused_set(): |
4152 | + return f(*args, **kwargs) |
4153 | + # otherwise, normal restart_on_change functionality |
4154 | + return restart_on_change_helper( |
4155 | + (lambda: f(*args, **kwargs)), restart_map, stopstart, |
4156 | + restart_functions) |
4157 | + return wrapped_f |
4158 | + return wrap |
4159 | + |
4160 | + |
4161 | +def config_flags_parser(config_flags): |
4162 | + """Parses config flags string into dict. |
4163 | + |
4164 | + This parsing method supports a few different formats for the config |
4165 | + flag values to be parsed: |
4166 | + |
4167 | + 1. A string in the simple format of key=value pairs, with the possibility |
4168 | + of specifying multiple key value pairs within the same string. For |
4169 | + example, a string in the format of 'key1=value1, key2=value2' will |
4170 | + return a dict of: |
4171 | + |
4172 | + {'key1': 'value1', |
4173 | + 'key2': 'value2'}. |
4174 | + |
4175 | + 2. A string in the above format, but supporting a comma-delimited list |
4176 | + of values for the same key. For example, a string in the format of |
4177 | + 'key1=value1, key2=value3,value4,value5' will return a dict of: |
4178 | + |
4179 | + {'key1', 'value1', |
4180 | + 'key2', 'value2,value3,value4'} |
4181 | + |
4182 | + 3. A string containing a colon character (:) prior to an equal |
4183 | + character (=) will be treated as yaml and parsed as such. This can be |
4184 | + used to specify more complex key value pairs. For example, |
4185 | + a string in the format of 'key1: subkey1=value1, subkey2=value2' will |
4186 | + return a dict of: |
4187 | + |
4188 | + {'key1', 'subkey1=value1, subkey2=value2'} |
4189 | + |
4190 | + The provided config_flags string may be a list of comma-separated values |
4191 | + which themselves may be comma-separated list of values. |
4192 | + """ |
4193 | + # If we find a colon before an equals sign then treat it as yaml. |
4194 | + # Note: limit it to finding the colon first since this indicates assignment |
4195 | + # for inline yaml. |
4196 | + colon = config_flags.find(':') |
4197 | + equals = config_flags.find('=') |
4198 | + if colon > 0: |
4199 | + if colon < equals or equals < 0: |
4200 | + return yaml.safe_load(config_flags) |
4201 | + |
4202 | + if config_flags.find('==') >= 0: |
4203 | + juju_log("config_flags is not in expected format (key=value)", |
4204 | + level=ERROR) |
4205 | + raise OSContextError |
4206 | + |
4207 | + # strip the following from each value. |
4208 | + post_strippers = ' ,' |
4209 | + # we strip any leading/trailing '=' or ' ' from the string then |
4210 | + # split on '='. |
4211 | + split = config_flags.strip(' =').split('=') |
4212 | + limit = len(split) |
4213 | + flags = {} |
4214 | + for i in range(0, limit - 1): |
4215 | + current = split[i] |
4216 | + next = split[i + 1] |
4217 | + vindex = next.rfind(',') |
4218 | + if (i == limit - 2) or (vindex < 0): |
4219 | + value = next |
4220 | + else: |
4221 | + value = next[:vindex] |
4222 | + |
4223 | + if i == 0: |
4224 | + key = current |
4225 | + else: |
4226 | + # if this not the first entry, expect an embedded key. |
4227 | + index = current.rfind(',') |
4228 | + if index < 0: |
4229 | + juju_log("Invalid config value(s) at index %s" % (i), |
4230 | + level=ERROR) |
4231 | + raise OSContextError |
4232 | + key = current[index + 1:] |
4233 | + |
4234 | + # Add to collection. |
4235 | + flags[key.strip(post_strippers)] = value.rstrip(post_strippers) |
4236 | + |
4237 | + return flags |
4238 | |
4239 | === modified file 'charmhelpers/contrib/python/__init__.py' |
4240 | --- charmhelpers/contrib/python/__init__.py 2015-09-28 20:09:02 +0000 |
4241 | +++ charmhelpers/contrib/python/__init__.py 2016-09-19 09:44:09 +0000 |
4242 | @@ -1,15 +1,13 @@ |
4243 | # Copyright 2014-2015 Canonical Limited. |
4244 | # |
4245 | -# This file is part of charm-helpers. |
4246 | -# |
4247 | -# charm-helpers is free software: you can redistribute it and/or modify |
4248 | -# it under the terms of the GNU Lesser General Public License version 3 as |
4249 | -# published by the Free Software Foundation. |
4250 | -# |
4251 | -# charm-helpers is distributed in the hope that it will be useful, |
4252 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
4253 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
4254 | -# GNU Lesser General Public License for more details. |
4255 | -# |
4256 | -# You should have received a copy of the GNU Lesser General Public License |
4257 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
4258 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
4259 | +# you may not use this file except in compliance with the License. |
4260 | +# You may obtain a copy of the License at |
4261 | +# |
4262 | +# http://www.apache.org/licenses/LICENSE-2.0 |
4263 | +# |
4264 | +# Unless required by applicable law or agreed to in writing, software |
4265 | +# distributed under the License is distributed on an "AS IS" BASIS, |
4266 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
4267 | +# See the License for the specific language governing permissions and |
4268 | +# limitations under the License. |
4269 | |
4270 | === modified file 'charmhelpers/contrib/python/packages.py' |
4271 | --- charmhelpers/contrib/python/packages.py 2015-09-28 20:09:02 +0000 |
4272 | +++ charmhelpers/contrib/python/packages.py 2016-09-19 09:44:09 +0000 |
4273 | @@ -3,36 +3,49 @@ |
4274 | |
4275 | # Copyright 2014-2015 Canonical Limited. |
4276 | # |
4277 | -# This file is part of charm-helpers. |
4278 | -# |
4279 | -# charm-helpers is free software: you can redistribute it and/or modify |
4280 | -# it under the terms of the GNU Lesser General Public License version 3 as |
4281 | -# published by the Free Software Foundation. |
4282 | -# |
4283 | -# charm-helpers is distributed in the hope that it will be useful, |
4284 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
4285 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
4286 | -# GNU Lesser General Public License for more details. |
4287 | -# |
4288 | -# You should have received a copy of the GNU Lesser General Public License |
4289 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
4290 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
4291 | +# you may not use this file except in compliance with the License. |
4292 | +# You may obtain a copy of the License at |
4293 | +# |
4294 | +# http://www.apache.org/licenses/LICENSE-2.0 |
4295 | +# |
4296 | +# Unless required by applicable law or agreed to in writing, software |
4297 | +# distributed under the License is distributed on an "AS IS" BASIS, |
4298 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
4299 | +# See the License for the specific language governing permissions and |
4300 | +# limitations under the License. |
4301 | |
4302 | import os |
4303 | import subprocess |
4304 | +import sys |
4305 | |
4306 | from charmhelpers.fetch import apt_install, apt_update |
4307 | from charmhelpers.core.hookenv import charm_dir, log |
4308 | |
4309 | -try: |
4310 | - from pip import main as pip_execute |
4311 | -except ImportError: |
4312 | - apt_update() |
4313 | - apt_install('python-pip') |
4314 | - from pip import main as pip_execute |
4315 | - |
4316 | __author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>" |
4317 | |
4318 | |
4319 | +def pip_execute(*args, **kwargs): |
4320 | + """Overriden pip_execute() to stop sys.path being changed. |
4321 | + |
4322 | + The act of importing main from the pip module seems to cause add wheels |
4323 | + from the /usr/share/python-wheels which are installed by various tools. |
4324 | + This function ensures that sys.path remains the same after the call is |
4325 | + executed. |
4326 | + """ |
4327 | + try: |
4328 | + _path = sys.path |
4329 | + try: |
4330 | + from pip import main as _pip_execute |
4331 | + except ImportError: |
4332 | + apt_update() |
4333 | + apt_install('python-pip') |
4334 | + from pip import main as _pip_execute |
4335 | + _pip_execute(*args, **kwargs) |
4336 | + finally: |
4337 | + sys.path = _path |
4338 | + |
4339 | + |
4340 | def parse_options(given, available): |
4341 | """Given a set of options, check if available""" |
4342 | for key, value in sorted(given.items()): |
4343 | @@ -42,8 +55,12 @@ |
4344 | yield "--{0}={1}".format(key, value) |
4345 | |
4346 | |
4347 | -def pip_install_requirements(requirements, **options): |
4348 | - """Install a requirements file """ |
4349 | +def pip_install_requirements(requirements, constraints=None, **options): |
4350 | + """Install a requirements file. |
4351 | + |
4352 | + :param constraints: Path to pip constraints file. |
4353 | + http://pip.readthedocs.org/en/stable/user_guide/#constraints-files |
4354 | + """ |
4355 | command = ["install"] |
4356 | |
4357 | available_options = ('proxy', 'src', 'log', ) |
4358 | @@ -51,12 +68,18 @@ |
4359 | command.append(option) |
4360 | |
4361 | command.append("-r {0}".format(requirements)) |
4362 | - log("Installing from file: {} with options: {}".format(requirements, |
4363 | - command)) |
4364 | + if constraints: |
4365 | + command.append("-c {0}".format(constraints)) |
4366 | + log("Installing from file: {} with constraints {} " |
4367 | + "and options: {}".format(requirements, constraints, command)) |
4368 | + else: |
4369 | + log("Installing from file: {} with options: {}".format(requirements, |
4370 | + command)) |
4371 | pip_execute(command) |
4372 | |
4373 | |
4374 | -def pip_install(package, fatal=False, upgrade=False, venv=None, **options): |
4375 | +def pip_install(package, fatal=False, upgrade=False, venv=None, |
4376 | + constraints=None, **options): |
4377 | """Install a python package""" |
4378 | if venv: |
4379 | venv_python = os.path.join(venv, 'bin/pip') |
4380 | @@ -71,6 +94,9 @@ |
4381 | if upgrade: |
4382 | command.append('--upgrade') |
4383 | |
4384 | + if constraints: |
4385 | + command.extend(['-c', constraints]) |
4386 | + |
4387 | if isinstance(package, list): |
4388 | command.extend(package) |
4389 | else: |
4390 | |
4391 | === modified file 'charmhelpers/contrib/storage/__init__.py' |
4392 | --- charmhelpers/contrib/storage/__init__.py 2015-09-28 20:09:02 +0000 |
4393 | +++ charmhelpers/contrib/storage/__init__.py 2016-09-19 09:44:09 +0000 |
4394 | @@ -1,15 +1,13 @@ |
4395 | # Copyright 2014-2015 Canonical Limited. |
4396 | # |
4397 | -# This file is part of charm-helpers. |
4398 | -# |
4399 | -# charm-helpers is free software: you can redistribute it and/or modify |
4400 | -# it under the terms of the GNU Lesser General Public License version 3 as |
4401 | -# published by the Free Software Foundation. |
4402 | -# |
4403 | -# charm-helpers is distributed in the hope that it will be useful, |
4404 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
4405 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
4406 | -# GNU Lesser General Public License for more details. |
4407 | -# |
4408 | -# You should have received a copy of the GNU Lesser General Public License |
4409 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
4410 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
4411 | +# you may not use this file except in compliance with the License. |
4412 | +# You may obtain a copy of the License at |
4413 | +# |
4414 | +# http://www.apache.org/licenses/LICENSE-2.0 |
4415 | +# |
4416 | +# Unless required by applicable law or agreed to in writing, software |
4417 | +# distributed under the License is distributed on an "AS IS" BASIS, |
4418 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
4419 | +# See the License for the specific language governing permissions and |
4420 | +# limitations under the License. |
4421 | |
4422 | === modified file 'charmhelpers/contrib/storage/linux/__init__.py' |
4423 | --- charmhelpers/contrib/storage/linux/__init__.py 2015-09-28 20:09:02 +0000 |
4424 | +++ charmhelpers/contrib/storage/linux/__init__.py 2016-09-19 09:44:09 +0000 |
4425 | @@ -1,15 +1,13 @@ |
4426 | # Copyright 2014-2015 Canonical Limited. |
4427 | # |
4428 | -# This file is part of charm-helpers. |
4429 | -# |
4430 | -# charm-helpers is free software: you can redistribute it and/or modify |
4431 | -# it under the terms of the GNU Lesser General Public License version 3 as |
4432 | -# published by the Free Software Foundation. |
4433 | -# |
4434 | -# charm-helpers is distributed in the hope that it will be useful, |
4435 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
4436 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
4437 | -# GNU Lesser General Public License for more details. |
4438 | -# |
4439 | -# You should have received a copy of the GNU Lesser General Public License |
4440 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
4441 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
4442 | +# you may not use this file except in compliance with the License. |
4443 | +# You may obtain a copy of the License at |
4444 | +# |
4445 | +# http://www.apache.org/licenses/LICENSE-2.0 |
4446 | +# |
4447 | +# Unless required by applicable law or agreed to in writing, software |
4448 | +# distributed under the License is distributed on an "AS IS" BASIS, |
4449 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
4450 | +# See the License for the specific language governing permissions and |
4451 | +# limitations under the License. |
4452 | |
4453 | === modified file 'charmhelpers/contrib/storage/linux/ceph.py' |
4454 | --- charmhelpers/contrib/storage/linux/ceph.py 2015-09-28 20:09:02 +0000 |
4455 | +++ charmhelpers/contrib/storage/linux/ceph.py 2016-09-19 09:44:09 +0000 |
4456 | @@ -1,18 +1,16 @@ |
4457 | # Copyright 2014-2015 Canonical Limited. |
4458 | # |
4459 | -# This file is part of charm-helpers. |
4460 | -# |
4461 | -# charm-helpers is free software: you can redistribute it and/or modify |
4462 | -# it under the terms of the GNU Lesser General Public License version 3 as |
4463 | -# published by the Free Software Foundation. |
4464 | -# |
4465 | -# charm-helpers is distributed in the hope that it will be useful, |
4466 | -# but WITHOUT ANY WARRANTY; without even the implied warranty of |
4467 | -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
4468 | -# GNU Lesser General Public License for more details. |
4469 | -# |
4470 | -# You should have received a copy of the GNU Lesser General Public License |
4471 | -# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
4472 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
4473 | +# you may not use this file except in compliance with the License. |
4474 | +# You may obtain a copy of the License at |
4475 | +# |
4476 | +# http://www.apache.org/licenses/LICENSE-2.0 |
4477 | +# |
4478 | +# Unless required by applicable law or agreed to in writing, software |
4479 | +# distributed under the License is distributed on an "AS IS" BASIS, |
4480 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
4481 | +# See the License for the specific language governing permissions and |
4482 | +# limitations under the License. |
4483 | |
4484 | # |
4485 | # Copyright 2012 Canonical Ltd. |
4486 | @@ -24,6 +22,11 @@ |
4487 | # Adam Gandelman <adamg@ubuntu.com> |
4488 | # |
4489 | |
4490 | +import errno |
4491 | +import hashlib |
4492 | +import math |
4493 | +import six |
4494 | + |
4495 | import os |
4496 | import shutil |
4497 | import json |
4498 | @@ -36,6 +39,7 @@ |
4499 | CalledProcessError, |
4500 | ) |
4501 | from charmhelpers.core.hookenv import ( |
4502 | + config, |
4503 | local_unit, |
4504 | relation_get, |
4505 | relation_ids, |
4506 | @@ -60,6 +64,7 @@ |
4507 | ) |
4508 | |
4509 | from charmhelpers.core.kernel import modprobe |
4510 | +from charmhelpers.contrib.openstack.utils import config_flags_parser |
4511 | |
4512 | KEYRING = '/etc/ceph/ceph.client.{}.keyring' |
4513 | KEYFILE = '/etc/ceph/ceph.client.{}.key' |
4514 | @@ -73,6 +78,646 @@ |
4515 | clog to syslog = {use_syslog} |
4516 | """ |
4517 | |
4518 | +# The number of placement groups per OSD to target for placement group |
4519 | +# calculations. This number is chosen as 100 due to the ceph PG Calc |
4520 | +# documentation recommending to choose 100 for clusters which are not |
4521 | +# expected to increase in the foreseeable future. Since the majority of the |
4522 | +# calculations are done on deployment, target the case of non-expanding |
4523 | +# clusters as the default. |
4524 | +DEFAULT_PGS_PER_OSD_TARGET = 100 |
4525 | +DEFAULT_POOL_WEIGHT = 10.0 |
4526 | +LEGACY_PG_COUNT = 200 |
4527 | +DEFAULT_MINIMUM_PGS = 2 |
4528 | + |
4529 | + |
4530 | +def validator(value, valid_type, valid_range=None): |
4531 | + """ |
4532 | + Used to validate these: http://docs.ceph.com/docs/master/rados/operations/pools/#set-pool-values |
4533 | + Example input: |
4534 | + validator(value=1, |
4535 | + valid_type=int, |
4536 | + valid_range=[0, 2]) |
4537 | + This says I'm testing value=1. It must be an int inclusive in [0,2] |
4538 | + |
4539 | + :param value: The value to validate |
4540 | + :param valid_type: The type that value should be. |
4541 | + :param valid_range: A range of values that value can assume. |
4542 | + :return: |
4543 | + """ |
4544 | + assert isinstance(value, valid_type), "{} is not a {}".format( |
4545 | + value, |
4546 | + valid_type) |
4547 | + if valid_range is not None: |
4548 | + assert isinstance(valid_range, list), \ |
4549 | + "valid_range must be a list, was given {}".format(valid_range) |
4550 | + # If we're dealing with strings |
4551 | + if valid_type is six.string_types: |
4552 | + assert value in valid_range, \ |
4553 | + "{} is not in the list {}".format(value, valid_range) |
4554 | + # Integer, float should have a min and max |
4555 | + else: |
4556 | + if len(valid_range) != 2: |
4557 | + raise ValueError( |
4558 | + "Invalid valid_range list of {} for {}. " |
4559 | + "List must be [min,max]".format(valid_range, value)) |
4560 | + assert value >= valid_range[0], \ |
4561 | + "{} is less than minimum allowed value of {}".format( |
4562 | + value, valid_range[0]) |
4563 | + assert value <= valid_range[1], \ |
4564 | + "{} is greater than maximum allowed value of {}".format( |
4565 | + value, valid_range[1]) |
4566 | + |
4567 | + |
4568 | +class PoolCreationError(Exception): |
4569 | + """ |
4570 | + A custom error to inform the caller that a pool creation failed. Provides an error message |
4571 | + """ |
4572 | + |
4573 | + def __init__(self, message): |
4574 | + super(PoolCreationError, self).__init__(message) |
4575 | + |
4576 | + |
4577 | +class Pool(object): |
4578 | + """ |
4579 | + An object oriented approach to Ceph pool creation. This base class is inherited by ReplicatedPool and ErasurePool. |
4580 | + Do not call create() on this base class as it will not do anything. Instantiate a child class and call create(). |
4581 | + """ |
4582 | + |
4583 | + def __init__(self, service, name): |
4584 | + self.service = service |
4585 | + self.name = name |
4586 | + |
4587 | + # Create the pool if it doesn't exist already |
4588 | + # To be implemented by subclasses |
4589 | + def create(self): |
4590 | + pass |
4591 | + |
4592 | + def add_cache_tier(self, cache_pool, mode): |
4593 | + """ |
4594 | + Adds a new cache tier to an existing pool. |
4595 | + :param cache_pool: six.string_types. The cache tier pool name to add. |
4596 | + :param mode: six.string_types. The caching mode to use for this pool. valid range = ["readonly", "writeback"] |
4597 | + :return: None |
4598 | + """ |
4599 | + # Check the input types and values |
4600 | + validator(value=cache_pool, valid_type=six.string_types) |
4601 | + validator(value=mode, valid_type=six.string_types, valid_range=["readonly", "writeback"]) |
4602 | + |
4603 | + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'add', self.name, cache_pool]) |
4604 | + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, mode]) |
4605 | + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'set-overlay', self.name, cache_pool]) |
4606 | + check_call(['ceph', '--id', self.service, 'osd', 'pool', 'set', cache_pool, 'hit_set_type', 'bloom']) |
4607 | + |
4608 | + def remove_cache_tier(self, cache_pool): |
4609 | + """ |
4610 | + Removes a cache tier from Ceph. Flushes all dirty objects from writeback pools and waits for that to complete. |
4611 | + :param cache_pool: six.string_types. The cache tier pool name to remove. |
4612 | + :return: None |
4613 | + """ |
4614 | + # read-only is easy, writeback is much harder |
4615 | + mode = get_cache_mode(self.service, cache_pool) |
4616 | + version = ceph_version() |
4617 | + if mode == 'readonly': |
4618 | + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', cache_pool, 'none']) |
4619 | + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool]) |
4620 | + |
4621 | + elif mode == 'writeback': |
4622 | + pool_forward_cmd = ['ceph', '--id', self.service, 'osd', 'tier', |
4623 | + 'cache-mode', cache_pool, 'forward'] |
4624 | + if version >= '10.1': |
4625 | + # Jewel added a mandatory flag |
4626 | + pool_forward_cmd.append('--yes-i-really-mean-it') |
4627 | + |
4628 | + check_call(pool_forward_cmd) |
4629 | + # Flush the cache and wait for it to return |
4630 | + check_call(['rados', '--id', self.service, '-p', cache_pool, 'cache-flush-evict-all']) |
4631 | + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove-overlay', self.name]) |
4632 | + check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', self.name, cache_pool]) |
4633 | + |
4634 | + def get_pgs(self, pool_size, percent_data=DEFAULT_POOL_WEIGHT): |
4635 | + """Return the number of placement groups to use when creating the pool. |
4636 | + |
4637 | + Returns the number of placement groups which should be specified when |
4638 | + creating the pool. This is based upon the calculation guidelines |
4639 | + provided by the Ceph Placement Group Calculator (located online at |
4640 | + http://ceph.com/pgcalc/). |
4641 | + |
4642 | + The number of placement groups are calculated using the following: |
4643 | + |
4644 | + (Target PGs per OSD) * (OSD #) * (%Data) |
4645 | + ---------------------------------------- |
4646 | + (Pool size) |
4647 | + |
4648 | + Per the upstream guidelines, the OSD # should really be considered |
4649 | + based on the number of OSDs which are eligible to be selected by the |
4650 | + pool. Since the pool creation doesn't specify any of CRUSH set rules, |
4651 | + the default rule will be dependent upon the type of pool being |
4652 | + created (replicated or erasure). |
4653 | + |
4654 | + This code makes no attempt to determine the number of OSDs which can be |
4655 | + selected for the specific rule, rather it is left to the user to tune |
4656 | + in the form of 'expected-osd-count' config option. |
4657 | + |
4658 | + :param pool_size: int. pool_size is either the number of replicas for |
4659 | + replicated pools or the K+M sum for erasure coded pools |
4660 | + :param percent_data: float. the percentage of data that is expected to |
4661 | + be contained in the pool for the specific OSD set. Default value |
4662 | + is to assume 10% of the data is for this pool, which is a |
4663 | + relatively low % of the data but allows for the pg_num to be |
4664 | + increased. NOTE: the default is primarily to handle the scenario |
4665 | + where related charms requiring pools has not been upgraded to |
4666 | + include an update to indicate their relative usage of the pools. |
4667 | + :return: int. The number of pgs to use. |
4668 | + """ |
4669 | + |
4670 | + # Note: This calculation follows the approach that is provided |
4671 | + # by the Ceph PG Calculator located at http://ceph.com/pgcalc/. |
4672 | + validator(value=pool_size, valid_type=int) |
4673 | + |
4674 | + # Ensure that percent data is set to something - even with a default |
4675 | + # it can be set to None, which would wreak havoc below. |
4676 | + if percent_data is None: |
4677 | + percent_data = DEFAULT_POOL_WEIGHT |
4678 | + |
4679 | + # If the expected-osd-count is specified, then use the max between |
4680 | + # the expected-osd-count and the actual osd_count |
4681 | + osd_list = get_osds(self.service) |
4682 | + expected = config('expected-osd-count') or 0 |
4683 | + |
4684 | + if osd_list: |
4685 | + osd_count = max(expected, len(osd_list)) |
4686 | + |
4687 | + # Log a message to provide some insight if the calculations claim |
4688 | + # to be off because someone is setting the expected count and |
4689 | + # there are more OSDs in reality. Try to make a proper guess |
4690 | + # based upon the cluster itself. |
4691 | + if expected and osd_count != expected: |
4692 | + log("Found more OSDs than provided expected count. " |
4693 | + "Using the actual count instead", INFO) |
4694 | + elif expected: |
4695 | + # Use the expected-osd-count in older ceph versions to allow for |
4696 | + # a more accurate pg calculations |
4697 | + osd_count = expected |
4698 | + else: |
4699 | + # NOTE(james-page): Default to 200 for older ceph versions |
4700 | + # which don't support OSD query from cli |
4701 | + return LEGACY_PG_COUNT |
4702 | + |
4703 | + percent_data /= 100.0 |
4704 | + target_pgs_per_osd = config('pgs-per-osd') or DEFAULT_PGS_PER_OSD_TARGET |
4705 | + num_pg = (target_pgs_per_osd * osd_count * percent_data) // pool_size |
4706 | + |
4707 | + # NOTE: ensure a sane minimum number of PGS otherwise we don't get any |
4708 | + # reasonable data distribution in minimal OSD configurations |
4709 | + if num_pg < DEFAULT_MINIMUM_PGS: |
4710 | + num_pg = DEFAULT_MINIMUM_PGS |
4711 | + |
4712 | + # The CRUSH algorithm has a slight optimization for placement groups |
4713 | + # with powers of 2 so find the nearest power of 2. If the nearest |
4714 | + # power of 2 is more than 25% below the original value, the next |
4715 | + # highest value is used. To do this, find the nearest power of 2 such |
4716 | + # that 2^n <= num_pg, check to see if its within the 25% tolerance. |
4717 | + exponent = math.floor(math.log(num_pg, 2)) |
4718 | + nearest = 2 ** exponent |
4719 | + if (num_pg - nearest) > (num_pg * 0.25): |
4720 | + # Choose the next highest power of 2 since the nearest is more |
4721 | + # than 25% below the original value. |
4722 | + return int(nearest * 2) |
4723 | + else: |
4724 | + return int(nearest) |
4725 | + |
4726 | + |
4727 | +class ReplicatedPool(Pool): |
4728 | + def __init__(self, service, name, pg_num=None, replicas=2, |
4729 | + percent_data=10.0): |
4730 | + super(ReplicatedPool, self).__init__(service=service, name=name) |
4731 | + self.replicas = replicas |
4732 | + if pg_num: |
4733 | + # Since the number of placement groups were specified, ensure |
4734 | + # that there aren't too many created. |
4735 | + max_pgs = self.get_pgs(self.replicas, 100.0) |
4736 | + self.pg_num = min(pg_num, max_pgs) |
4737 | + else: |
4738 | + self.pg_num = self.get_pgs(self.replicas, percent_data) |
4739 | + |
4740 | + def create(self): |
4741 | + if not pool_exists(self.service, self.name): |
4742 | + # Create it |
4743 | + cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', |
4744 | + self.name, str(self.pg_num)] |
4745 | + try: |
4746 | + check_call(cmd) |
4747 | + # Set the pool replica size |
4748 | + update_pool(client=self.service, |
4749 | + pool=self.name, |
4750 | + settings={'size': str(self.replicas)}) |
4751 | + except CalledProcessError: |
4752 | + raise |
4753 | + |
4754 | + |
4755 | +# Default jerasure erasure coded pool |
4756 | +class ErasurePool(Pool): |
4757 | + def __init__(self, service, name, erasure_code_profile="default", |
4758 | + percent_data=10.0): |
4759 | + super(ErasurePool, self).__init__(service=service, name=name) |
4760 | + self.erasure_code_profile = erasure_code_profile |
4761 | + self.percent_data = percent_data |
4762 | + |
4763 | + def create(self): |
4764 | + if not pool_exists(self.service, self.name): |
4765 | + # Try to find the erasure profile information in order to properly |
4766 | + # size the number of placement groups. The size of an erasure |
4767 | + # coded placement group is calculated as k+m. |
4768 | + erasure_profile = get_erasure_profile(self.service, |
4769 | + self.erasure_code_profile) |
4770 | + |
4771 | + # Check for errors |
4772 | + if erasure_profile is None: |
4773 | + msg = ("Failed to discover erasure profile named " |
4774 | + "{}".format(self.erasure_code_profile)) |
4775 | + log(msg, level=ERROR) |
4776 | + raise PoolCreationError(msg) |
4777 | + if 'k' not in erasure_profile or 'm' not in erasure_profile: |
4778 | + # Error |
4779 | + msg = ("Unable to find k (data chunks) or m (coding chunks) " |
4780 | + "in erasure profile {}".format(erasure_profile)) |
4781 | + log(msg, level=ERROR) |
4782 | + raise PoolCreationError(msg) |
4783 | + |
4784 | + k = int(erasure_profile['k']) |
4785 | + m = int(erasure_profile['m']) |
4786 | + pgs = self.get_pgs(k + m, self.percent_data) |
4787 | + # Create it |
4788 | + cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', |
4789 | + self.name, str(pgs), str(pgs), |
4790 | + 'erasure', self.erasure_code_profile] |
4791 | + try: |
4792 | + check_call(cmd) |
4793 | + except CalledProcessError: |
4794 | + raise |
4795 | + |
4796 | + """Get an existing erasure code profile if it already exists. |
4797 | + Returns json formatted output""" |
4798 | + |
4799 | + |
4800 | +def get_mon_map(service): |
4801 | + """ |
4802 | + Returns the current monitor map. |
4803 | + :param service: six.string_types. The Ceph user name to run the command under |
4804 | + :return: json string. :raise: ValueError if the monmap fails to parse. |
4805 | + Also raises CalledProcessError if our ceph command fails |
4806 | + """ |
4807 | + try: |
4808 | + mon_status = check_output( |
4809 | + ['ceph', '--id', service, |
4810 | + 'mon_status', '--format=json']) |
4811 | + try: |
4812 | + return json.loads(mon_status) |
4813 | + except ValueError as v: |
4814 | + log("Unable to parse mon_status json: {}. Error: {}".format( |
4815 | + mon_status, v.message)) |
4816 | + raise |
4817 | + except CalledProcessError as e: |
4818 | + log("mon_status command failed with message: {}".format( |
4819 | + e.message)) |
4820 | + raise |
4821 | + |
4822 | + |
4823 | +def hash_monitor_names(service): |
4824 | + """ |
4825 | + Uses the get_mon_map() function to get information about the monitor |
4826 | + cluster. |
4827 | + Hash the name of each monitor. Return a sorted list of monitor hashes |
4828 | + in an ascending order. |
4829 | + :param service: six.string_types. The Ceph user name to run the command under |
4830 | + :rtype : dict. json dict of monitor name, ip address and rank |
4831 | + example: { |
4832 | + 'name': 'ip-172-31-13-165', |
4833 | + 'rank': 0, |
4834 | + 'addr': '172.31.13.165:6789/0'} |
4835 | + """ |
4836 | + try: |
4837 | + hash_list = [] |
4838 | + monitor_list = get_mon_map(service=service) |
4839 | + if monitor_list['monmap']['mons']: |
4840 | + for mon in monitor_list['monmap']['mons']: |
4841 | + hash_list.append( |
4842 | + hashlib.sha224(mon['name'].encode('utf-8')).hexdigest()) |
4843 | + return sorted(hash_list) |
4844 | + else: |
4845 | + return None |
4846 | + except (ValueError, CalledProcessError): |
4847 | + raise |
4848 | + |
4849 | + |
4850 | +def monitor_key_delete(service, key): |
4851 | + """ |
4852 | + Delete a key and value pair from the monitor cluster |
4853 | + :param service: six.string_types. The Ceph user name to run the command under |
4854 | + Deletes a key value pair on the monitor cluster. |
4855 | + :param key: six.string_types. The key to delete. |
4856 | + """ |
4857 | + try: |
4858 | + check_output( |
4859 | + ['ceph', '--id', service, |
4860 | + 'config-key', 'del', str(key)]) |
4861 | + except CalledProcessError as e: |
4862 | + log("Monitor config-key put failed with message: {}".format( |
4863 | + e.output)) |
4864 | + raise |
4865 | + |
4866 | + |
4867 | +def monitor_key_set(service, key, value): |
4868 | + """ |
4869 | + Sets a key value pair on the monitor cluster. |
4870 | + :param service: six.string_types. The Ceph user name to run the command under |
4871 | + :param key: six.string_types. The key to set. |
4872 | + :param value: The value to set. This will be converted to a string |
4873 | + before setting |
4874 | + """ |
4875 | + try: |
4876 | + check_output( |
4877 | + ['ceph', '--id', service, |
4878 | + 'config-key', 'put', str(key), str(value)]) |
4879 | + except CalledProcessError as e: |
4880 | + log("Monitor config-key put failed with message: {}".format( |
4881 | + e.output)) |
4882 | + raise |
4883 | + |
4884 | + |
4885 | +def monitor_key_get(service, key): |
4886 | + """ |
4887 | + Gets the value of an existing key in the monitor cluster. |
4888 | + :param service: six.string_types. The Ceph user name to run the command under |
4889 | + :param key: six.string_types. The key to search for. |
4890 | + :return: Returns the value of that key or None if not found. |
4891 | + """ |
4892 | + try: |
4893 | + output = check_output( |
4894 | + ['ceph', '--id', service, |
4895 | + 'config-key', 'get', str(key)]) |
4896 | + return output |
4897 | + except CalledProcessError as e: |
4898 | + log("Monitor config-key get failed with message: {}".format( |
4899 | + e.output)) |
4900 | + return None |
4901 | + |
4902 | + |
4903 | +def monitor_key_exists(service, key): |
4904 | + """ |
4905 | + Searches for the existence of a key in the monitor cluster. |
4906 | + :param service: six.string_types. The Ceph user name to run the command under |
4907 | + :param key: six.string_types. The key to search for |
4908 | + :return: Returns True if the key exists, False if not and raises an |
4909 | + exception if an unknown error occurs. :raise: CalledProcessError if |
4910 | + an unknown error occurs |
4911 | + """ |
4912 | + try: |
4913 | + check_call( |
4914 | + ['ceph', '--id', service, |
4915 | + 'config-key', 'exists', str(key)]) |
4916 | + # I can return true here regardless because Ceph returns |
4917 | + # ENOENT if the key wasn't found |
4918 | + return True |
4919 | + except CalledProcessError as e: |
4920 | + if e.returncode == errno.ENOENT: |
4921 | + return False |
4922 | + else: |
4923 | + log("Unknown error from ceph config-get exists: {} {}".format( |
4924 | + e.returncode, e.output)) |
4925 | + raise |
4926 | + |
4927 | + |
4928 | +def get_erasure_profile(service, name): |
4929 | + """ |
4930 | + :param service: six.string_types. The Ceph user name to run the command under |
4931 | + :param name: |
4932 | + :return: |
4933 | + """ |
4934 | + try: |
4935 | + out = check_output(['ceph', '--id', service, |
4936 | + 'osd', 'erasure-code-profile', 'get', |
4937 | + name, '--format=json']) |
4938 | + return json.loads(out) |
4939 | + except (CalledProcessError, OSError, ValueError): |
4940 | + return None |
4941 | + |
4942 | + |
4943 | +def pool_set(service, pool_name, key, value): |
4944 | + """ |
4945 | + Sets a value for a RADOS pool in ceph. |
4946 | + :param service: six.string_types. The Ceph user name to run the command under |
4947 | + :param pool_name: six.string_types |
4948 | + :param key: six.string_types |
4949 | + :param value: |
4950 | + :return: None. Can raise CalledProcessError |
4951 | + """ |
4952 | + cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', pool_name, key, value] |
4953 | + try: |
4954 | + check_call(cmd) |
4955 | + except CalledProcessError: |
4956 | + raise |
4957 | + |
4958 | + |
4959 | +def snapshot_pool(service, pool_name, snapshot_name): |
4960 | + """ |
4961 | + Snapshots a RADOS pool in ceph. |
4962 | + :param service: six.string_types. The Ceph user name to run the command under |
4963 | + :param pool_name: six.string_types |
4964 | + :param snapshot_name: six.string_types |
4965 | + :return: None. Can raise CalledProcessError |
4966 | + """ |
4967 | + cmd = ['ceph', '--id', service, 'osd', 'pool', 'mksnap', pool_name, snapshot_name] |
4968 | + try: |
4969 | + check_call(cmd) |
4970 | + except CalledProcessError: |
4971 | + raise |
4972 | + |
4973 | + |
4974 | +def remove_pool_snapshot(service, pool_name, snapshot_name): |
4975 | + """ |
4976 | + Remove a snapshot from a RADOS pool in ceph. |
4977 | + :param service: six.string_types. The Ceph user name to run the command under |
4978 | + :param pool_name: six.string_types |
4979 | + :param snapshot_name: six.string_types |
4980 | + :return: None. Can raise CalledProcessError |
4981 | + """ |
4982 | + cmd = ['ceph', '--id', service, 'osd', 'pool', 'rmsnap', pool_name, snapshot_name] |
4983 | + try: |
4984 | + check_call(cmd) |
4985 | + except CalledProcessError: |
4986 | + raise |
4987 | + |
4988 | + |
4989 | +# max_bytes should be an int or long |
4990 | +def set_pool_quota(service, pool_name, max_bytes): |
4991 | + """ |
4992 | + :param service: six.string_types. The Ceph user name to run the command under |
4993 | + :param pool_name: six.string_types |
4994 | + :param max_bytes: int or long |
4995 | + :return: None. Can raise CalledProcessError |
4996 | + """ |
4997 | + # Set a byte quota on a RADOS pool in ceph. |
4998 | + cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, |
4999 | + 'max_bytes', str(max_bytes)] |
5000 | + try: |
The diff has been truncated for viewing.