Merge lp:~hopem/charms/trusty/ceph/lp1517846 into lp:~openstack-charmers-archive/charms/trusty/ceph/next
- Trusty Tahr (14.04)
- lp1517846
- Merge into next
Proposed by
Edward Hope-Morley
Status: | Merged |
---|---|
Merged at revision: | 123 |
Proposed branch: | lp:~hopem/charms/trusty/ceph/lp1517846 |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/ceph/next |
Diff against target: |
1254 lines (+541/-123) 16 files modified
hooks/charmhelpers/cli/__init__.py (+3/-3) hooks/charmhelpers/contrib/charmsupport/nrpe.py (+44/-8) hooks/charmhelpers/contrib/network/ip.py (+5/-3) hooks/charmhelpers/contrib/storage/linux/ceph.py (+53/-46) hooks/charmhelpers/core/hookenv.py (+46/-0) hooks/charmhelpers/core/host.py (+66/-19) hooks/charmhelpers/core/hugepage.py (+10/-1) hooks/charmhelpers/core/kernel.py (+68/-0) hooks/charmhelpers/core/services/helpers.py (+5/-2) hooks/charmhelpers/core/strutils.py (+30/-0) hooks/charmhelpers/core/templating.py (+13/-6) hooks/charmhelpers/fetch/__init__.py (+1/-1) tests/charmhelpers/contrib/amulet/deployment.py (+4/-2) tests/charmhelpers/contrib/amulet/utils.py (+56/-16) tests/charmhelpers/contrib/openstack/amulet/deployment.py (+111/-12) tests/charmhelpers/contrib/openstack/amulet/utils.py (+26/-4) |
To merge this branch: | bzr merge lp:~hopem/charms/trusty/ceph/lp1517846 |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Liam Young (community) | Approve | ||
Review via email: mp+278040@code.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #14071 ceph-next for hopem mp278040
LINT OK: passed
Build: http://
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #13116 ceph-next for hopem mp278040
UNIT OK: passed
- 124. By Edward Hope-Morley
-
more sync
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #13118 ceph-next for hopem mp278040
UNIT OK: passed
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #14074 ceph-next for hopem mp278040
LINT OK: passed
Build: http://
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #7944 ceph-next for hopem mp278040
AMULET OK: passed
Build: http://
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #7946 ceph-next for hopem mp278040
AMULET OK: passed
Build: http://
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'hooks/charmhelpers/cli/__init__.py' |
2 | --- hooks/charmhelpers/cli/__init__.py 2015-08-19 00:51:43 +0000 |
3 | +++ hooks/charmhelpers/cli/__init__.py 2015-11-19 19:15:27 +0000 |
4 | @@ -20,7 +20,7 @@ |
5 | |
6 | from six.moves import zip |
7 | |
8 | -from charmhelpers.core import unitdata |
9 | +import charmhelpers.core.unitdata |
10 | |
11 | |
12 | class OutputFormatter(object): |
13 | @@ -163,8 +163,8 @@ |
14 | if getattr(arguments.func, '_cli_no_output', False): |
15 | output = '' |
16 | self.formatter.format_output(output, arguments.format) |
17 | - if unitdata._KV: |
18 | - unitdata._KV.flush() |
19 | + if charmhelpers.core.unitdata._KV: |
20 | + charmhelpers.core.unitdata._KV.flush() |
21 | |
22 | |
23 | cmdline = CommandLine() |
24 | |
25 | === modified file 'hooks/charmhelpers/contrib/charmsupport/nrpe.py' |
26 | --- hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-04-19 09:01:44 +0000 |
27 | +++ hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-11-19 19:15:27 +0000 |
28 | @@ -148,6 +148,13 @@ |
29 | self.description = description |
30 | self.check_cmd = self._locate_cmd(check_cmd) |
31 | |
32 | + def _get_check_filename(self): |
33 | + return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command)) |
34 | + |
35 | + def _get_service_filename(self, hostname): |
36 | + return os.path.join(NRPE.nagios_exportdir, |
37 | + 'service__{}_{}.cfg'.format(hostname, self.command)) |
38 | + |
39 | def _locate_cmd(self, check_cmd): |
40 | search_path = ( |
41 | '/usr/lib/nagios/plugins', |
42 | @@ -163,9 +170,21 @@ |
43 | log('Check command not found: {}'.format(parts[0])) |
44 | return '' |
45 | |
46 | + def _remove_service_files(self): |
47 | + if not os.path.exists(NRPE.nagios_exportdir): |
48 | + return |
49 | + for f in os.listdir(NRPE.nagios_exportdir): |
50 | + if f.endswith('_{}.cfg'.format(self.command)): |
51 | + os.remove(os.path.join(NRPE.nagios_exportdir, f)) |
52 | + |
53 | + def remove(self, hostname): |
54 | + nrpe_check_file = self._get_check_filename() |
55 | + if os.path.exists(nrpe_check_file): |
56 | + os.remove(nrpe_check_file) |
57 | + self._remove_service_files() |
58 | + |
59 | def write(self, nagios_context, hostname, nagios_servicegroups): |
60 | - nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( |
61 | - self.command) |
62 | + nrpe_check_file = self._get_check_filename() |
63 | with open(nrpe_check_file, 'w') as nrpe_check_config: |
64 | nrpe_check_config.write("# check {}\n".format(self.shortname)) |
65 | nrpe_check_config.write("command[{}]={}\n".format( |
66 | @@ -180,9 +199,7 @@ |
67 | |
68 | def write_service_config(self, nagios_context, hostname, |
69 | nagios_servicegroups): |
70 | - for f in os.listdir(NRPE.nagios_exportdir): |
71 | - if re.search('.*{}.cfg'.format(self.command), f): |
72 | - os.remove(os.path.join(NRPE.nagios_exportdir, f)) |
73 | + self._remove_service_files() |
74 | |
75 | templ_vars = { |
76 | 'nagios_hostname': hostname, |
77 | @@ -192,8 +209,7 @@ |
78 | 'command': self.command, |
79 | } |
80 | nrpe_service_text = Check.service_template.format(**templ_vars) |
81 | - nrpe_service_file = '{}/service__{}_{}.cfg'.format( |
82 | - NRPE.nagios_exportdir, hostname, self.command) |
83 | + nrpe_service_file = self._get_service_filename(hostname) |
84 | with open(nrpe_service_file, 'w') as nrpe_service_config: |
85 | nrpe_service_config.write(str(nrpe_service_text)) |
86 | |
87 | @@ -218,12 +234,32 @@ |
88 | if hostname: |
89 | self.hostname = hostname |
90 | else: |
91 | - self.hostname = "{}-{}".format(self.nagios_context, self.unit_name) |
92 | + nagios_hostname = get_nagios_hostname() |
93 | + if nagios_hostname: |
94 | + self.hostname = nagios_hostname |
95 | + else: |
96 | + self.hostname = "{}-{}".format(self.nagios_context, self.unit_name) |
97 | self.checks = [] |
98 | |
99 | def add_check(self, *args, **kwargs): |
100 | self.checks.append(Check(*args, **kwargs)) |
101 | |
102 | + def remove_check(self, *args, **kwargs): |
103 | + if kwargs.get('shortname') is None: |
104 | + raise ValueError('shortname of check must be specified') |
105 | + |
106 | + # Use sensible defaults if they're not specified - these are not |
107 | + # actually used during removal, but they're required for constructing |
108 | + # the Check object; check_disk is chosen because it's part of the |
109 | + # nagios-plugins-basic package. |
110 | + if kwargs.get('check_cmd') is None: |
111 | + kwargs['check_cmd'] = 'check_disk' |
112 | + if kwargs.get('description') is None: |
113 | + kwargs['description'] = '' |
114 | + |
115 | + check = Check(*args, **kwargs) |
116 | + check.remove(self.hostname) |
117 | + |
118 | def write(self): |
119 | try: |
120 | nagios_uid = pwd.getpwnam('nagios').pw_uid |
121 | |
122 | === modified file 'hooks/charmhelpers/contrib/network/ip.py' |
123 | --- hooks/charmhelpers/contrib/network/ip.py 2015-09-03 09:42:00 +0000 |
124 | +++ hooks/charmhelpers/contrib/network/ip.py 2015-11-19 19:15:27 +0000 |
125 | @@ -23,7 +23,7 @@ |
126 | from functools import partial |
127 | |
128 | from charmhelpers.core.hookenv import unit_get |
129 | -from charmhelpers.fetch import apt_install |
130 | +from charmhelpers.fetch import apt_install, apt_update |
131 | from charmhelpers.core.hookenv import ( |
132 | log, |
133 | WARNING, |
134 | @@ -32,13 +32,15 @@ |
135 | try: |
136 | import netifaces |
137 | except ImportError: |
138 | - apt_install('python-netifaces') |
139 | + apt_update(fatal=True) |
140 | + apt_install('python-netifaces', fatal=True) |
141 | import netifaces |
142 | |
143 | try: |
144 | import netaddr |
145 | except ImportError: |
146 | - apt_install('python-netaddr') |
147 | + apt_update(fatal=True) |
148 | + apt_install('python-netaddr', fatal=True) |
149 | import netaddr |
150 | |
151 | |
152 | |
153 | === modified file 'hooks/charmhelpers/contrib/storage/linux/ceph.py' |
154 | --- hooks/charmhelpers/contrib/storage/linux/ceph.py 2015-09-10 09:29:50 +0000 |
155 | +++ hooks/charmhelpers/contrib/storage/linux/ceph.py 2015-11-19 19:15:27 +0000 |
156 | @@ -26,6 +26,7 @@ |
157 | |
158 | import os |
159 | import shutil |
160 | +import six |
161 | import json |
162 | import time |
163 | import uuid |
164 | @@ -59,6 +60,8 @@ |
165 | apt_install, |
166 | ) |
167 | |
168 | +from charmhelpers.core.kernel import modprobe |
169 | + |
170 | KEYRING = '/etc/ceph/ceph.client.{}.keyring' |
171 | KEYFILE = '/etc/ceph/ceph.client.{}.key' |
172 | |
173 | @@ -123,29 +126,37 @@ |
174 | return None |
175 | |
176 | |
177 | -def create_pool(service, name, replicas=3): |
178 | +def update_pool(client, pool, settings): |
179 | + cmd = ['ceph', '--id', client, 'osd', 'pool', 'set', pool] |
180 | + for k, v in six.iteritems(settings): |
181 | + cmd.append(k) |
182 | + cmd.append(v) |
183 | + |
184 | + check_call(cmd) |
185 | + |
186 | + |
187 | +def create_pool(service, name, replicas=3, pg_num=None): |
188 | """Create a new RADOS pool.""" |
189 | if pool_exists(service, name): |
190 | log("Ceph pool {} already exists, skipping creation".format(name), |
191 | level=WARNING) |
192 | return |
193 | |
194 | - # Calculate the number of placement groups based |
195 | - # on upstream recommended best practices. |
196 | - osds = get_osds(service) |
197 | - if osds: |
198 | - pgnum = (len(osds) * 100 // replicas) |
199 | - else: |
200 | - # NOTE(james-page): Default to 200 for older ceph versions |
201 | - # which don't support OSD query from cli |
202 | - pgnum = 200 |
203 | - |
204 | - cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pgnum)] |
205 | - check_call(cmd) |
206 | - |
207 | - cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', name, 'size', |
208 | - str(replicas)] |
209 | - check_call(cmd) |
210 | + if not pg_num: |
211 | + # Calculate the number of placement groups based |
212 | + # on upstream recommended best practices. |
213 | + osds = get_osds(service) |
214 | + if osds: |
215 | + pg_num = (len(osds) * 100 // replicas) |
216 | + else: |
217 | + # NOTE(james-page): Default to 200 for older ceph versions |
218 | + # which don't support OSD query from cli |
219 | + pg_num = 200 |
220 | + |
221 | + cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pg_num)] |
222 | + check_call(cmd) |
223 | + |
224 | + update_pool(service, name, settings={'size': str(replicas)}) |
225 | |
226 | |
227 | def delete_pool(service, name): |
228 | @@ -200,10 +211,10 @@ |
229 | log('Created new keyfile at %s.' % keyfile, level=INFO) |
230 | |
231 | |
232 | -def get_ceph_nodes(): |
233 | - """Query named relation 'ceph' to determine current nodes.""" |
234 | +def get_ceph_nodes(relation='ceph'): |
235 | + """Query named relation to determine current nodes.""" |
236 | hosts = [] |
237 | - for r_id in relation_ids('ceph'): |
238 | + for r_id in relation_ids(relation): |
239 | for unit in related_units(r_id): |
240 | hosts.append(relation_get('private-address', unit=unit, rid=r_id)) |
241 | |
242 | @@ -291,17 +302,6 @@ |
243 | os.chown(data_src_dst, uid, gid) |
244 | |
245 | |
246 | -# TODO: re-use |
247 | -def modprobe(module): |
248 | - """Load a kernel module and configure for auto-load on reboot.""" |
249 | - log('Loading kernel module', level=INFO) |
250 | - cmd = ['modprobe', module] |
251 | - check_call(cmd) |
252 | - with open('/etc/modules', 'r+') as modules: |
253 | - if module not in modules.read(): |
254 | - modules.write(module) |
255 | - |
256 | - |
257 | def copy_files(src, dst, symlinks=False, ignore=None): |
258 | """Copy files from src to dst.""" |
259 | for item in os.listdir(src): |
260 | @@ -366,14 +366,14 @@ |
261 | service_start(svc) |
262 | |
263 | |
264 | -def ensure_ceph_keyring(service, user=None, group=None): |
265 | +def ensure_ceph_keyring(service, user=None, group=None, relation='ceph'): |
266 | """Ensures a ceph keyring is created for a named service and optionally |
267 | ensures user and group ownership. |
268 | |
269 | Returns False if no ceph key is available in relation state. |
270 | """ |
271 | key = None |
272 | - for rid in relation_ids('ceph'): |
273 | + for rid in relation_ids(relation): |
274 | for unit in related_units(rid): |
275 | key = relation_get('key', rid=rid, unit=unit) |
276 | if key: |
277 | @@ -422,9 +422,16 @@ |
278 | self.request_id = str(uuid.uuid1()) |
279 | self.ops = [] |
280 | |
281 | - def add_op_create_pool(self, name, replica_count=3): |
282 | + def add_op_create_pool(self, name, replica_count=3, pg_num=None): |
283 | + """Adds an operation to create a pool. |
284 | + |
285 | + @param pg_num setting: optional setting. If not provided, this value |
286 | + will be calculated by the broker based on how many OSDs are in the |
287 | + cluster at the time of creation. Note that, if provided, this value |
288 | + will be capped at the current available maximum. |
289 | + """ |
290 | self.ops.append({'op': 'create-pool', 'name': name, |
291 | - 'replicas': replica_count}) |
292 | + 'replicas': replica_count, 'pg_num': pg_num}) |
293 | |
294 | def set_ops(self, ops): |
295 | """Set request ops to provided value. |
296 | @@ -442,8 +449,8 @@ |
297 | def _ops_equal(self, other): |
298 | if len(self.ops) == len(other.ops): |
299 | for req_no in range(0, len(self.ops)): |
300 | - for key in ['replicas', 'name', 'op']: |
301 | - if self.ops[req_no][key] != other.ops[req_no][key]: |
302 | + for key in ['replicas', 'name', 'op', 'pg_num']: |
303 | + if self.ops[req_no].get(key) != other.ops[req_no].get(key): |
304 | return False |
305 | else: |
306 | return False |
307 | @@ -549,7 +556,7 @@ |
308 | return request |
309 | |
310 | |
311 | -def get_request_states(request): |
312 | +def get_request_states(request, relation='ceph'): |
313 | """Return a dict of requests per relation id with their corresponding |
314 | completion state. |
315 | |
316 | @@ -561,7 +568,7 @@ |
317 | """ |
318 | complete = [] |
319 | requests = {} |
320 | - for rid in relation_ids('ceph'): |
321 | + for rid in relation_ids(relation): |
322 | complete = False |
323 | previous_request = get_previous_request(rid) |
324 | if request == previous_request: |
325 | @@ -579,14 +586,14 @@ |
326 | return requests |
327 | |
328 | |
329 | -def is_request_sent(request): |
330 | +def is_request_sent(request, relation='ceph'): |
331 | """Check to see if a functionally equivalent request has already been sent |
332 | |
333 | Returns True if a similair request has been sent |
334 | |
335 | @param request: A CephBrokerRq object |
336 | """ |
337 | - states = get_request_states(request) |
338 | + states = get_request_states(request, relation=relation) |
339 | for rid in states.keys(): |
340 | if not states[rid]['sent']: |
341 | return False |
342 | @@ -594,7 +601,7 @@ |
343 | return True |
344 | |
345 | |
346 | -def is_request_complete(request): |
347 | +def is_request_complete(request, relation='ceph'): |
348 | """Check to see if a functionally equivalent request has already been |
349 | completed |
350 | |
351 | @@ -602,7 +609,7 @@ |
352 | |
353 | @param request: A CephBrokerRq object |
354 | """ |
355 | - states = get_request_states(request) |
356 | + states = get_request_states(request, relation=relation) |
357 | for rid in states.keys(): |
358 | if not states[rid]['complete']: |
359 | return False |
360 | @@ -652,15 +659,15 @@ |
361 | return 'broker-rsp-' + local_unit().replace('/', '-') |
362 | |
363 | |
364 | -def send_request_if_needed(request): |
365 | +def send_request_if_needed(request, relation='ceph'): |
366 | """Send broker request if an equivalent request has not already been sent |
367 | |
368 | @param request: A CephBrokerRq object |
369 | """ |
370 | - if is_request_sent(request): |
371 | + if is_request_sent(request, relation=relation): |
372 | log('Request already sent but not complete, not sending new request', |
373 | level=DEBUG) |
374 | else: |
375 | - for rid in relation_ids('ceph'): |
376 | + for rid in relation_ids(relation): |
377 | log('Sending request {}'.format(request.request_id), level=DEBUG) |
378 | relation_set(relation_id=rid, broker_req=request.request) |
379 | |
380 | === modified file 'hooks/charmhelpers/core/hookenv.py' |
381 | --- hooks/charmhelpers/core/hookenv.py 2015-09-03 09:42:00 +0000 |
382 | +++ hooks/charmhelpers/core/hookenv.py 2015-11-19 19:15:27 +0000 |
383 | @@ -491,6 +491,19 @@ |
384 | |
385 | |
386 | @cached |
387 | +def peer_relation_id(): |
388 | + '''Get a peer relation id if a peer relation has been joined, else None.''' |
389 | + md = metadata() |
390 | + section = md.get('peers') |
391 | + if section: |
392 | + for key in section: |
393 | + relids = relation_ids(key) |
394 | + if relids: |
395 | + return relids[0] |
396 | + return None |
397 | + |
398 | + |
399 | +@cached |
400 | def relation_to_interface(relation_name): |
401 | """ |
402 | Given the name of a relation, return the interface that relation uses. |
403 | @@ -623,6 +636,38 @@ |
404 | return unit_get('private-address') |
405 | |
406 | |
407 | +@cached |
408 | +def storage_get(attribute="", storage_id=""): |
409 | + """Get storage attributes""" |
410 | + _args = ['storage-get', '--format=json'] |
411 | + if storage_id: |
412 | + _args.extend(('-s', storage_id)) |
413 | + if attribute: |
414 | + _args.append(attribute) |
415 | + try: |
416 | + return json.loads(subprocess.check_output(_args).decode('UTF-8')) |
417 | + except ValueError: |
418 | + return None |
419 | + |
420 | + |
421 | +@cached |
422 | +def storage_list(storage_name=""): |
423 | + """List the storage IDs for the unit""" |
424 | + _args = ['storage-list', '--format=json'] |
425 | + if storage_name: |
426 | + _args.append(storage_name) |
427 | + try: |
428 | + return json.loads(subprocess.check_output(_args).decode('UTF-8')) |
429 | + except ValueError: |
430 | + return None |
431 | + except OSError as e: |
432 | + import errno |
433 | + if e.errno == errno.ENOENT: |
434 | + # storage-list does not exist |
435 | + return [] |
436 | + raise |
437 | + |
438 | + |
439 | class UnregisteredHookError(Exception): |
440 | """Raised when an undefined hook is called""" |
441 | pass |
442 | @@ -788,6 +833,7 @@ |
443 | |
444 | def translate_exc(from_exc, to_exc): |
445 | def inner_translate_exc1(f): |
446 | + @wraps(f) |
447 | def inner_translate_exc2(*args, **kwargs): |
448 | try: |
449 | return f(*args, **kwargs) |
450 | |
451 | === modified file 'hooks/charmhelpers/core/host.py' |
452 | --- hooks/charmhelpers/core/host.py 2015-08-19 13:50:16 +0000 |
453 | +++ hooks/charmhelpers/core/host.py 2015-11-19 19:15:27 +0000 |
454 | @@ -63,33 +63,53 @@ |
455 | return service_result |
456 | |
457 | |
458 | -def service_pause(service_name, init_dir=None): |
459 | +def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"): |
460 | """Pause a system service. |
461 | |
462 | Stop it, and prevent it from starting again at boot.""" |
463 | - if init_dir is None: |
464 | - init_dir = "/etc/init" |
465 | - stopped = service_stop(service_name) |
466 | - # XXX: Support systemd too |
467 | - override_path = os.path.join( |
468 | - init_dir, '{}.override'.format(service_name)) |
469 | - with open(override_path, 'w') as fh: |
470 | - fh.write("manual\n") |
471 | + stopped = True |
472 | + if service_running(service_name): |
473 | + stopped = service_stop(service_name) |
474 | + upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) |
475 | + sysv_file = os.path.join(initd_dir, service_name) |
476 | + if os.path.exists(upstart_file): |
477 | + override_path = os.path.join( |
478 | + init_dir, '{}.override'.format(service_name)) |
479 | + with open(override_path, 'w') as fh: |
480 | + fh.write("manual\n") |
481 | + elif os.path.exists(sysv_file): |
482 | + subprocess.check_call(["update-rc.d", service_name, "disable"]) |
483 | + else: |
484 | + # XXX: Support SystemD too |
485 | + raise ValueError( |
486 | + "Unable to detect {0} as either Upstart {1} or SysV {2}".format( |
487 | + service_name, upstart_file, sysv_file)) |
488 | return stopped |
489 | |
490 | |
491 | -def service_resume(service_name, init_dir=None): |
492 | +def service_resume(service_name, init_dir="/etc/init", |
493 | + initd_dir="/etc/init.d"): |
494 | """Resume a system service. |
495 | |
496 | Reenable starting again at boot. Start the service""" |
497 | - # XXX: Support systemd too |
498 | - if init_dir is None: |
499 | - init_dir = "/etc/init" |
500 | - override_path = os.path.join( |
501 | - init_dir, '{}.override'.format(service_name)) |
502 | - if os.path.exists(override_path): |
503 | - os.unlink(override_path) |
504 | - started = service_start(service_name) |
505 | + upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) |
506 | + sysv_file = os.path.join(initd_dir, service_name) |
507 | + if os.path.exists(upstart_file): |
508 | + override_path = os.path.join( |
509 | + init_dir, '{}.override'.format(service_name)) |
510 | + if os.path.exists(override_path): |
511 | + os.unlink(override_path) |
512 | + elif os.path.exists(sysv_file): |
513 | + subprocess.check_call(["update-rc.d", service_name, "enable"]) |
514 | + else: |
515 | + # XXX: Support SystemD too |
516 | + raise ValueError( |
517 | + "Unable to detect {0} as either Upstart {1} or SysV {2}".format( |
518 | + service_name, upstart_file, sysv_file)) |
519 | + |
520 | + started = service_running(service_name) |
521 | + if not started: |
522 | + started = service_start(service_name) |
523 | return started |
524 | |
525 | |
526 | @@ -550,7 +570,14 @@ |
527 | os.chdir(cur) |
528 | |
529 | |
530 | -def chownr(path, owner, group, follow_links=True): |
531 | +def chownr(path, owner, group, follow_links=True, chowntopdir=False): |
532 | + """ |
533 | + Recursively change user and group ownership of files and directories |
534 | + in given path. Doesn't chown path itself by default, only its children. |
535 | + |
536 | + :param bool follow_links: Also Chown links if True |
537 | + :param bool chowntopdir: Also chown path itself if True |
538 | + """ |
539 | uid = pwd.getpwnam(owner).pw_uid |
540 | gid = grp.getgrnam(group).gr_gid |
541 | if follow_links: |
542 | @@ -558,6 +585,10 @@ |
543 | else: |
544 | chown = os.lchown |
545 | |
546 | + if chowntopdir: |
547 | + broken_symlink = os.path.lexists(path) and not os.path.exists(path) |
548 | + if not broken_symlink: |
549 | + chown(path, uid, gid) |
550 | for root, dirs, files in os.walk(path): |
551 | for name in dirs + files: |
552 | full = os.path.join(root, name) |
553 | @@ -568,3 +599,19 @@ |
554 | |
555 | def lchownr(path, owner, group): |
556 | chownr(path, owner, group, follow_links=False) |
557 | + |
558 | + |
559 | +def get_total_ram(): |
560 | + '''The total amount of system RAM in bytes. |
561 | + |
562 | + This is what is reported by the OS, and may be overcommitted when |
563 | + there are multiple containers hosted on the same machine. |
564 | + ''' |
565 | + with open('/proc/meminfo', 'r') as f: |
566 | + for line in f.readlines(): |
567 | + if line: |
568 | + key, value, unit = line.split() |
569 | + if key == 'MemTotal:': |
570 | + assert unit == 'kB', 'Unknown unit' |
571 | + return int(value) * 1024 # Classic, not KiB. |
572 | + raise NotImplementedError() |
573 | |
574 | === modified file 'hooks/charmhelpers/core/hugepage.py' |
575 | --- hooks/charmhelpers/core/hugepage.py 2015-08-19 13:50:16 +0000 |
576 | +++ hooks/charmhelpers/core/hugepage.py 2015-11-19 19:15:27 +0000 |
577 | @@ -25,11 +25,13 @@ |
578 | fstab_mount, |
579 | mkdir, |
580 | ) |
581 | +from charmhelpers.core.strutils import bytes_from_string |
582 | +from subprocess import check_output |
583 | |
584 | |
585 | def hugepage_support(user, group='hugetlb', nr_hugepages=256, |
586 | max_map_count=65536, mnt_point='/run/hugepages/kvm', |
587 | - pagesize='2MB', mount=True): |
588 | + pagesize='2MB', mount=True, set_shmmax=False): |
589 | """Enable hugepages on system. |
590 | |
591 | Args: |
592 | @@ -44,11 +46,18 @@ |
593 | group_info = add_group(group) |
594 | gid = group_info.gr_gid |
595 | add_user_to_group(user, group) |
596 | + if max_map_count < 2 * nr_hugepages: |
597 | + max_map_count = 2 * nr_hugepages |
598 | sysctl_settings = { |
599 | 'vm.nr_hugepages': nr_hugepages, |
600 | 'vm.max_map_count': max_map_count, |
601 | 'vm.hugetlb_shm_group': gid, |
602 | } |
603 | + if set_shmmax: |
604 | + shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax'])) |
605 | + shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages |
606 | + if shmmax_minsize > shmmax_current: |
607 | + sysctl_settings['kernel.shmmax'] = shmmax_minsize |
608 | sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf') |
609 | mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False) |
610 | lfstab = fstab.Fstab() |
611 | |
612 | === added file 'hooks/charmhelpers/core/kernel.py' |
613 | --- hooks/charmhelpers/core/kernel.py 1970-01-01 00:00:00 +0000 |
614 | +++ hooks/charmhelpers/core/kernel.py 2015-11-19 19:15:27 +0000 |
615 | @@ -0,0 +1,68 @@ |
616 | +#!/usr/bin/env python |
617 | +# -*- coding: utf-8 -*- |
618 | + |
619 | +# Copyright 2014-2015 Canonical Limited. |
620 | +# |
621 | +# This file is part of charm-helpers. |
622 | +# |
623 | +# charm-helpers is free software: you can redistribute it and/or modify |
624 | +# it under the terms of the GNU Lesser General Public License version 3 as |
625 | +# published by the Free Software Foundation. |
626 | +# |
627 | +# charm-helpers is distributed in the hope that it will be useful, |
628 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
629 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
630 | +# GNU Lesser General Public License for more details. |
631 | +# |
632 | +# You should have received a copy of the GNU Lesser General Public License |
633 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
634 | + |
635 | +__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>" |
636 | + |
637 | +from charmhelpers.core.hookenv import ( |
638 | + log, |
639 | + INFO |
640 | +) |
641 | + |
642 | +from subprocess import check_call, check_output |
643 | +import re |
644 | + |
645 | + |
646 | +def modprobe(module, persist=True): |
647 | + """Load a kernel module and configure for auto-load on reboot.""" |
648 | + cmd = ['modprobe', module] |
649 | + |
650 | + log('Loading kernel module %s' % module, level=INFO) |
651 | + |
652 | + check_call(cmd) |
653 | + if persist: |
654 | + with open('/etc/modules', 'r+') as modules: |
655 | + if module not in modules.read(): |
656 | + modules.write(module) |
657 | + |
658 | + |
659 | +def rmmod(module, force=False): |
660 | + """Remove a module from the linux kernel""" |
661 | + cmd = ['rmmod'] |
662 | + if force: |
663 | + cmd.append('-f') |
664 | + cmd.append(module) |
665 | + log('Removing kernel module %s' % module, level=INFO) |
666 | + return check_call(cmd) |
667 | + |
668 | + |
669 | +def lsmod(): |
670 | + """Shows what kernel modules are currently loaded""" |
671 | + return check_output(['lsmod'], |
672 | + universal_newlines=True) |
673 | + |
674 | + |
675 | +def is_module_loaded(module): |
676 | + """Checks if a kernel module is already loaded""" |
677 | + matches = re.findall('^%s[ ]+' % module, lsmod(), re.M) |
678 | + return len(matches) > 0 |
679 | + |
680 | + |
681 | +def update_initramfs(version='all'): |
682 | + """Updates an initramfs image""" |
683 | + return check_call(["update-initramfs", "-k", version, "-u"]) |
684 | |
685 | === modified file 'hooks/charmhelpers/core/services/helpers.py' |
686 | --- hooks/charmhelpers/core/services/helpers.py 2015-08-19 00:51:43 +0000 |
687 | +++ hooks/charmhelpers/core/services/helpers.py 2015-11-19 19:15:27 +0000 |
688 | @@ -249,16 +249,18 @@ |
689 | :param int perms: The permissions of the rendered file |
690 | :param partial on_change_action: functools partial to be executed when |
691 | rendered file changes |
692 | + :param jinja2 loader template_loader: A jinja2 template loader |
693 | """ |
694 | def __init__(self, source, target, |
695 | owner='root', group='root', perms=0o444, |
696 | - on_change_action=None): |
697 | + on_change_action=None, template_loader=None): |
698 | self.source = source |
699 | self.target = target |
700 | self.owner = owner |
701 | self.group = group |
702 | self.perms = perms |
703 | self.on_change_action = on_change_action |
704 | + self.template_loader = template_loader |
705 | |
706 | def __call__(self, manager, service_name, event_name): |
707 | pre_checksum = '' |
708 | @@ -269,7 +271,8 @@ |
709 | for ctx in service.get('required_data', []): |
710 | context.update(ctx) |
711 | templating.render(self.source, self.target, context, |
712 | - self.owner, self.group, self.perms) |
713 | + self.owner, self.group, self.perms, |
714 | + template_loader=self.template_loader) |
715 | if self.on_change_action: |
716 | if pre_checksum == host.file_hash(self.target): |
717 | hookenv.log( |
718 | |
719 | === modified file 'hooks/charmhelpers/core/strutils.py' |
720 | --- hooks/charmhelpers/core/strutils.py 2015-04-16 10:27:24 +0000 |
721 | +++ hooks/charmhelpers/core/strutils.py 2015-11-19 19:15:27 +0000 |
722 | @@ -18,6 +18,7 @@ |
723 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
724 | |
725 | import six |
726 | +import re |
727 | |
728 | |
729 | def bool_from_string(value): |
730 | @@ -40,3 +41,32 @@ |
731 | |
732 | msg = "Unable to interpret string value '%s' as boolean" % (value) |
733 | raise ValueError(msg) |
734 | + |
735 | + |
736 | +def bytes_from_string(value): |
737 | + """Interpret human readable string value as bytes. |
738 | + |
739 | + Returns int |
740 | + """ |
741 | + BYTE_POWER = { |
742 | + 'K': 1, |
743 | + 'KB': 1, |
744 | + 'M': 2, |
745 | + 'MB': 2, |
746 | + 'G': 3, |
747 | + 'GB': 3, |
748 | + 'T': 4, |
749 | + 'TB': 4, |
750 | + 'P': 5, |
751 | + 'PB': 5, |
752 | + } |
753 | + if isinstance(value, six.string_types): |
754 | + value = six.text_type(value) |
755 | + else: |
756 | + msg = "Unable to interpret non-string value '%s' as boolean" % (value) |
757 | + raise ValueError(msg) |
758 | + matches = re.match("([0-9]+)([a-zA-Z]+)", value) |
759 | + if not matches: |
760 | + msg = "Unable to interpret string value '%s' as bytes" % (value) |
761 | + raise ValueError(msg) |
762 | + return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) |
763 | |
764 | === modified file 'hooks/charmhelpers/core/templating.py' |
765 | --- hooks/charmhelpers/core/templating.py 2015-02-26 11:01:14 +0000 |
766 | +++ hooks/charmhelpers/core/templating.py 2015-11-19 19:15:27 +0000 |
767 | @@ -21,7 +21,7 @@ |
768 | |
769 | |
770 | def render(source, target, context, owner='root', group='root', |
771 | - perms=0o444, templates_dir=None, encoding='UTF-8'): |
772 | + perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None): |
773 | """ |
774 | Render a template. |
775 | |
776 | @@ -52,17 +52,24 @@ |
777 | apt_install('python-jinja2', fatal=True) |
778 | from jinja2 import FileSystemLoader, Environment, exceptions |
779 | |
780 | - if templates_dir is None: |
781 | - templates_dir = os.path.join(hookenv.charm_dir(), 'templates') |
782 | - loader = Environment(loader=FileSystemLoader(templates_dir)) |
783 | + if template_loader: |
784 | + template_env = Environment(loader=template_loader) |
785 | + else: |
786 | + if templates_dir is None: |
787 | + templates_dir = os.path.join(hookenv.charm_dir(), 'templates') |
788 | + template_env = Environment(loader=FileSystemLoader(templates_dir)) |
789 | try: |
790 | source = source |
791 | - template = loader.get_template(source) |
792 | + template = template_env.get_template(source) |
793 | except exceptions.TemplateNotFound as e: |
794 | hookenv.log('Could not load template %s from %s.' % |
795 | (source, templates_dir), |
796 | level=hookenv.ERROR) |
797 | raise e |
798 | content = template.render(context) |
799 | - host.mkdir(os.path.dirname(target), owner, group, perms=0o755) |
800 | + target_dir = os.path.dirname(target) |
801 | + if not os.path.exists(target_dir): |
802 | + # This is a terrible default directory permission, as the file |
803 | + # or its siblings will often contain secrets. |
804 | + host.mkdir(os.path.dirname(target), owner, group, perms=0o755) |
805 | host.write_file(target, content.encode(encoding), owner, group, perms) |
806 | |
807 | === modified file 'hooks/charmhelpers/fetch/__init__.py' |
808 | --- hooks/charmhelpers/fetch/__init__.py 2015-08-19 00:51:43 +0000 |
809 | +++ hooks/charmhelpers/fetch/__init__.py 2015-11-19 19:15:27 +0000 |
810 | @@ -225,12 +225,12 @@ |
811 | |
812 | def apt_mark(packages, mark, fatal=False): |
813 | """Flag one or more packages using apt-mark""" |
814 | + log("Marking {} as {}".format(packages, mark)) |
815 | cmd = ['apt-mark', mark] |
816 | if isinstance(packages, six.string_types): |
817 | cmd.append(packages) |
818 | else: |
819 | cmd.extend(packages) |
820 | - log("Holding {}".format(packages)) |
821 | |
822 | if fatal: |
823 | subprocess.check_call(cmd, universal_newlines=True) |
824 | |
825 | === modified file 'tests/charmhelpers/contrib/amulet/deployment.py' |
826 | --- tests/charmhelpers/contrib/amulet/deployment.py 2015-01-26 09:46:20 +0000 |
827 | +++ tests/charmhelpers/contrib/amulet/deployment.py 2015-11-19 19:15:27 +0000 |
828 | @@ -51,7 +51,8 @@ |
829 | if 'units' not in this_service: |
830 | this_service['units'] = 1 |
831 | |
832 | - self.d.add(this_service['name'], units=this_service['units']) |
833 | + self.d.add(this_service['name'], units=this_service['units'], |
834 | + constraints=this_service.get('constraints')) |
835 | |
836 | for svc in other_services: |
837 | if 'location' in svc: |
838 | @@ -64,7 +65,8 @@ |
839 | if 'units' not in svc: |
840 | svc['units'] = 1 |
841 | |
842 | - self.d.add(svc['name'], charm=branch_location, units=svc['units']) |
843 | + self.d.add(svc['name'], charm=branch_location, units=svc['units'], |
844 | + constraints=svc.get('constraints')) |
845 | |
846 | def _add_relations(self, relations): |
847 | """Add all of the relations for the services.""" |
848 | |
849 | === modified file 'tests/charmhelpers/contrib/amulet/utils.py' |
850 | --- tests/charmhelpers/contrib/amulet/utils.py 2015-09-10 09:29:50 +0000 |
851 | +++ tests/charmhelpers/contrib/amulet/utils.py 2015-11-19 19:15:27 +0000 |
852 | @@ -326,7 +326,7 @@ |
853 | |
854 | def service_restarted_since(self, sentry_unit, mtime, service, |
855 | pgrep_full=None, sleep_time=20, |
856 | - retry_count=2, retry_sleep_time=30): |
857 | + retry_count=30, retry_sleep_time=10): |
858 | """Check if service was been started after a given time. |
859 | |
860 | Args: |
861 | @@ -334,8 +334,9 @@ |
862 | mtime (float): The epoch time to check against |
863 | service (string): service name to look for in process table |
864 | pgrep_full: [Deprecated] Use full command line search mode with pgrep |
865 | - sleep_time (int): Seconds to sleep before looking for process |
866 | - retry_count (int): If service is not found, how many times to retry |
867 | + sleep_time (int): Initial sleep time (s) before looking for file |
868 | + retry_sleep_time (int): Time (s) to sleep between retries |
869 | + retry_count (int): If file is not found, how many times to retry |
870 | |
871 | Returns: |
872 | bool: True if service found and its start time it newer than mtime, |
873 | @@ -359,11 +360,12 @@ |
874 | pgrep_full) |
875 | self.log.debug('Attempt {} to get {} proc start time on {} ' |
876 | 'OK'.format(tries, service, unit_name)) |
877 | - except IOError: |
878 | + except IOError as e: |
879 | # NOTE(beisner) - race avoidance, proc may not exist yet. |
880 | # https://bugs.launchpad.net/charm-helpers/+bug/1474030 |
881 | self.log.debug('Attempt {} to get {} proc start time on {} ' |
882 | - 'failed'.format(tries, service, unit_name)) |
883 | + 'failed\n{}'.format(tries, service, |
884 | + unit_name, e)) |
885 | time.sleep(retry_sleep_time) |
886 | tries += 1 |
887 | |
888 | @@ -383,35 +385,62 @@ |
889 | return False |
890 | |
891 | def config_updated_since(self, sentry_unit, filename, mtime, |
892 | - sleep_time=20): |
893 | + sleep_time=20, retry_count=30, |
894 | + retry_sleep_time=10): |
895 | """Check if file was modified after a given time. |
896 | |
897 | Args: |
898 | sentry_unit (sentry): The sentry unit to check the file mtime on |
899 | filename (string): The file to check mtime of |
900 | mtime (float): The epoch time to check against |
901 | - sleep_time (int): Seconds to sleep before looking for process |
902 | + sleep_time (int): Initial sleep time (s) before looking for file |
903 | + retry_sleep_time (int): Time (s) to sleep between retries |
904 | + retry_count (int): If file is not found, how many times to retry |
905 | |
906 | Returns: |
907 | bool: True if file was modified more recently than mtime, False if |
908 | - file was modified before mtime, |
909 | + file was modified before mtime, or if file not found. |
910 | """ |
911 | - self.log.debug('Checking %s updated since %s' % (filename, mtime)) |
912 | + unit_name = sentry_unit.info['unit_name'] |
913 | + self.log.debug('Checking that %s updated since %s on ' |
914 | + '%s' % (filename, mtime, unit_name)) |
915 | time.sleep(sleep_time) |
916 | - file_mtime = self._get_file_mtime(sentry_unit, filename) |
917 | + file_mtime = None |
918 | + tries = 0 |
919 | + while tries <= retry_count and not file_mtime: |
920 | + try: |
921 | + file_mtime = self._get_file_mtime(sentry_unit, filename) |
922 | + self.log.debug('Attempt {} to get {} file mtime on {} ' |
923 | + 'OK'.format(tries, filename, unit_name)) |
924 | + except IOError as e: |
925 | + # NOTE(beisner) - race avoidance, file may not exist yet. |
926 | + # https://bugs.launchpad.net/charm-helpers/+bug/1474030 |
927 | + self.log.debug('Attempt {} to get {} file mtime on {} ' |
928 | + 'failed\n{}'.format(tries, filename, |
929 | + unit_name, e)) |
930 | + time.sleep(retry_sleep_time) |
931 | + tries += 1 |
932 | + |
933 | + if not file_mtime: |
934 | + self.log.warn('Could not determine file mtime, assuming ' |
935 | + 'file does not exist') |
936 | + return False |
937 | + |
938 | if file_mtime >= mtime: |
939 | self.log.debug('File mtime is newer than provided mtime ' |
940 | - '(%s >= %s)' % (file_mtime, mtime)) |
941 | + '(%s >= %s) on %s (OK)' % (file_mtime, |
942 | + mtime, unit_name)) |
943 | return True |
944 | else: |
945 | - self.log.warn('File mtime %s is older than provided mtime %s' |
946 | - % (file_mtime, mtime)) |
947 | + self.log.warn('File mtime is older than provided mtime' |
948 | + '(%s < on %s) on %s' % (file_mtime, |
949 | + mtime, unit_name)) |
950 | return False |
951 | |
952 | def validate_service_config_changed(self, sentry_unit, mtime, service, |
953 | filename, pgrep_full=None, |
954 | - sleep_time=20, retry_count=2, |
955 | - retry_sleep_time=30): |
956 | + sleep_time=20, retry_count=30, |
957 | + retry_sleep_time=10): |
958 | """Check service and file were updated after mtime |
959 | |
960 | Args: |
961 | @@ -456,7 +485,9 @@ |
962 | sentry_unit, |
963 | filename, |
964 | mtime, |
965 | - sleep_time=0) |
966 | + sleep_time=sleep_time, |
967 | + retry_count=retry_count, |
968 | + retry_sleep_time=retry_sleep_time) |
969 | |
970 | return service_restart and config_update |
971 | |
972 | @@ -776,3 +807,12 @@ |
973 | output = _check_output(command, universal_newlines=True) |
974 | data = json.loads(output) |
975 | return data.get(u"status") == "completed" |
976 | + |
977 | + def status_get(self, unit): |
978 | + """Return the current service status of this unit.""" |
979 | + raw_status, return_code = unit.run( |
980 | + "status-get --format=json --include-data") |
981 | + if return_code != 0: |
982 | + return ("unknown", "") |
983 | + status = json.loads(raw_status) |
984 | + return (status["status"], status["message"]) |
985 | |
986 | === modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py' |
987 | --- tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-09-10 09:29:50 +0000 |
988 | +++ tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-11-19 19:15:27 +0000 |
989 | @@ -14,12 +14,18 @@ |
990 | # You should have received a copy of the GNU Lesser General Public License |
991 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
992 | |
993 | +import logging |
994 | +import re |
995 | +import sys |
996 | import six |
997 | from collections import OrderedDict |
998 | from charmhelpers.contrib.amulet.deployment import ( |
999 | AmuletDeployment |
1000 | ) |
1001 | |
1002 | +DEBUG = logging.DEBUG |
1003 | +ERROR = logging.ERROR |
1004 | + |
1005 | |
1006 | class OpenStackAmuletDeployment(AmuletDeployment): |
1007 | """OpenStack amulet deployment. |
1008 | @@ -28,9 +34,12 @@ |
1009 | that is specifically for use by OpenStack charms. |
1010 | """ |
1011 | |
1012 | - def __init__(self, series=None, openstack=None, source=None, stable=True): |
1013 | + def __init__(self, series=None, openstack=None, source=None, |
1014 | + stable=True, log_level=DEBUG): |
1015 | """Initialize the deployment environment.""" |
1016 | super(OpenStackAmuletDeployment, self).__init__(series) |
1017 | + self.log = self.get_logger(level=log_level) |
1018 | + self.log.info('OpenStackAmuletDeployment: init') |
1019 | self.openstack = openstack |
1020 | self.source = source |
1021 | self.stable = stable |
1022 | @@ -38,6 +47,22 @@ |
1023 | # out. |
1024 | self.current_next = "trusty" |
1025 | |
1026 | + def get_logger(self, name="deployment-logger", level=logging.DEBUG): |
1027 | + """Get a logger object that will log to stdout.""" |
1028 | + log = logging |
1029 | + logger = log.getLogger(name) |
1030 | + fmt = log.Formatter("%(asctime)s %(funcName)s " |
1031 | + "%(levelname)s: %(message)s") |
1032 | + |
1033 | + handler = log.StreamHandler(stream=sys.stdout) |
1034 | + handler.setLevel(level) |
1035 | + handler.setFormatter(fmt) |
1036 | + |
1037 | + logger.addHandler(handler) |
1038 | + logger.setLevel(level) |
1039 | + |
1040 | + return logger |
1041 | + |
1042 | def _determine_branch_locations(self, other_services): |
1043 | """Determine the branch locations for the other services. |
1044 | |
1045 | @@ -45,6 +70,8 @@ |
1046 | stable or next (dev) branch, and based on this, use the corresonding |
1047 | stable or next branches for the other_services.""" |
1048 | |
1049 | + self.log.info('OpenStackAmuletDeployment: determine branch locations') |
1050 | + |
1051 | # Charms outside the lp:~openstack-charmers namespace |
1052 | base_charms = ['mysql', 'mongodb', 'nrpe'] |
1053 | |
1054 | @@ -58,19 +85,17 @@ |
1055 | else: |
1056 | base_series = self.current_next |
1057 | |
1058 | - if self.stable: |
1059 | - for svc in other_services: |
1060 | - if svc['name'] in force_series_current: |
1061 | - base_series = self.current_next |
1062 | - |
1063 | + for svc in other_services: |
1064 | + if svc['name'] in force_series_current: |
1065 | + base_series = self.current_next |
1066 | + # If a location has been explicitly set, use it |
1067 | + if svc.get('location'): |
1068 | + continue |
1069 | + if self.stable: |
1070 | temp = 'lp:charms/{}/{}' |
1071 | svc['location'] = temp.format(base_series, |
1072 | svc['name']) |
1073 | - else: |
1074 | - for svc in other_services: |
1075 | - if svc['name'] in force_series_current: |
1076 | - base_series = self.current_next |
1077 | - |
1078 | + else: |
1079 | if svc['name'] in base_charms: |
1080 | temp = 'lp:charms/{}/{}' |
1081 | svc['location'] = temp.format(base_series, |
1082 | @@ -79,10 +104,13 @@ |
1083 | temp = 'lp:~openstack-charmers/charms/{}/{}/next' |
1084 | svc['location'] = temp.format(self.current_next, |
1085 | svc['name']) |
1086 | + |
1087 | return other_services |
1088 | |
1089 | def _add_services(self, this_service, other_services): |
1090 | """Add services to the deployment and set openstack-origin/source.""" |
1091 | + self.log.info('OpenStackAmuletDeployment: adding services') |
1092 | + |
1093 | other_services = self._determine_branch_locations(other_services) |
1094 | |
1095 | super(OpenStackAmuletDeployment, self)._add_services(this_service, |
1096 | @@ -96,7 +124,8 @@ |
1097 | 'ceph-osd', 'ceph-radosgw'] |
1098 | |
1099 | # Charms which can not use openstack-origin, ie. many subordinates |
1100 | - no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe'] |
1101 | + no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe', |
1102 | + 'openvswitch-odl', 'neutron-api-odl', 'odl-controller'] |
1103 | |
1104 | if self.openstack: |
1105 | for svc in services: |
1106 | @@ -112,9 +141,79 @@ |
1107 | |
1108 | def _configure_services(self, configs): |
1109 | """Configure all of the services.""" |
1110 | + self.log.info('OpenStackAmuletDeployment: configure services') |
1111 | for service, config in six.iteritems(configs): |
1112 | self.d.configure(service, config) |
1113 | |
1114 | + def _auto_wait_for_status(self, message=None, exclude_services=None, |
1115 | + include_only=None, timeout=1800): |
1116 | + """Wait for all units to have a specific extended status, except |
1117 | + for any defined as excluded. Unless specified via message, any |
1118 | + status containing any case of 'ready' will be considered a match. |
1119 | + |
1120 | + Examples of message usage: |
1121 | + |
1122 | + Wait for all unit status to CONTAIN any case of 'ready' or 'ok': |
1123 | + message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE) |
1124 | + |
1125 | + Wait for all units to reach this status (exact match): |
1126 | + message = re.compile('^Unit is ready and clustered$') |
1127 | + |
1128 | + Wait for all units to reach any one of these (exact match): |
1129 | + message = re.compile('Unit is ready|OK|Ready') |
1130 | + |
1131 | + Wait for at least one unit to reach this status (exact match): |
1132 | + message = {'ready'} |
1133 | + |
1134 | + See Amulet's sentry.wait_for_messages() for message usage detail. |
1135 | + https://github.com/juju/amulet/blob/master/amulet/sentry.py |
1136 | + |
1137 | + :param message: Expected status match |
1138 | + :param exclude_services: List of juju service names to ignore, |
1139 | + not to be used in conjuction with include_only. |
1140 | + :param include_only: List of juju service names to exclusively check, |
1141 | + not to be used in conjuction with exclude_services. |
1142 | + :param timeout: Maximum time in seconds to wait for status match |
1143 | + :returns: None. Raises if timeout is hit. |
1144 | + """ |
1145 | + self.log.info('Waiting for extended status on units...') |
1146 | + |
1147 | + all_services = self.d.services.keys() |
1148 | + |
1149 | + if exclude_services and include_only: |
1150 | + raise ValueError('exclude_services can not be used ' |
1151 | + 'with include_only') |
1152 | + |
1153 | + if message: |
1154 | + if isinstance(message, re._pattern_type): |
1155 | + match = message.pattern |
1156 | + else: |
1157 | + match = message |
1158 | + |
1159 | + self.log.debug('Custom extended status wait match: ' |
1160 | + '{}'.format(match)) |
1161 | + else: |
1162 | + self.log.debug('Default extended status wait match: contains ' |
1163 | + 'READY (case-insensitive)') |
1164 | + message = re.compile('.*ready.*', re.IGNORECASE) |
1165 | + |
1166 | + if exclude_services: |
1167 | + self.log.debug('Excluding services from extended status match: ' |
1168 | + '{}'.format(exclude_services)) |
1169 | + else: |
1170 | + exclude_services = [] |
1171 | + |
1172 | + if include_only: |
1173 | + services = include_only |
1174 | + else: |
1175 | + services = list(set(all_services) - set(exclude_services)) |
1176 | + |
1177 | + self.log.debug('Waiting up to {}s for extended status on services: ' |
1178 | + '{}'.format(timeout, services)) |
1179 | + service_messages = {service: message for service in services} |
1180 | + self.d.sentry.wait_for_messages(service_messages, timeout=timeout) |
1181 | + self.log.info('OK') |
1182 | + |
1183 | def _get_openstack_release(self): |
1184 | """Get openstack release. |
1185 | |
1186 | |
1187 | === modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py' |
1188 | --- tests/charmhelpers/contrib/openstack/amulet/utils.py 2015-09-10 09:29:50 +0000 |
1189 | +++ tests/charmhelpers/contrib/openstack/amulet/utils.py 2015-11-19 19:15:27 +0000 |
1190 | @@ -18,6 +18,7 @@ |
1191 | import json |
1192 | import logging |
1193 | import os |
1194 | +import re |
1195 | import six |
1196 | import time |
1197 | import urllib |
1198 | @@ -604,7 +605,22 @@ |
1199 | '{}'.format(sample_type, samples)) |
1200 | return None |
1201 | |
1202 | -# rabbitmq/amqp specific helpers: |
1203 | + # rabbitmq/amqp specific helpers: |
1204 | + |
1205 | + def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200): |
1206 | + """Wait for rmq units extended status to show cluster readiness, |
1207 | + after an optional initial sleep period. Initial sleep is likely |
1208 | + necessary to be effective following a config change, as status |
1209 | + message may not instantly update to non-ready.""" |
1210 | + |
1211 | + if init_sleep: |
1212 | + time.sleep(init_sleep) |
1213 | + |
1214 | + message = re.compile('^Unit is ready and clustered$') |
1215 | + deployment._auto_wait_for_status(message=message, |
1216 | + timeout=timeout, |
1217 | + include_only=['rabbitmq-server']) |
1218 | + |
1219 | def add_rmq_test_user(self, sentry_units, |
1220 | username="testuser1", password="changeme"): |
1221 | """Add a test user via the first rmq juju unit, check connection as |
1222 | @@ -752,7 +768,7 @@ |
1223 | self.log.debug('SSL is enabled @{}:{} ' |
1224 | '({})'.format(host, port, unit_name)) |
1225 | return True |
1226 | - elif not port and not conf_ssl: |
1227 | + elif not conf_ssl: |
1228 | self.log.debug('SSL not enabled @{}:{} ' |
1229 | '({})'.format(host, port, unit_name)) |
1230 | return False |
1231 | @@ -805,7 +821,10 @@ |
1232 | if port: |
1233 | config['ssl_port'] = port |
1234 | |
1235 | - deployment.configure('rabbitmq-server', config) |
1236 | + deployment.d.configure('rabbitmq-server', config) |
1237 | + |
1238 | + # Wait for unit status |
1239 | + self.rmq_wait_for_cluster(deployment) |
1240 | |
1241 | # Confirm |
1242 | tries = 0 |
1243 | @@ -832,7 +851,10 @@ |
1244 | |
1245 | # Disable RMQ SSL |
1246 | config = {'ssl': 'off'} |
1247 | - deployment.configure('rabbitmq-server', config) |
1248 | + deployment.d.configure('rabbitmq-server', config) |
1249 | + |
1250 | + # Wait for unit status |
1251 | + self.rmq_wait_for_cluster(deployment) |
1252 | |
1253 | # Confirm |
1254 | tries = 0 |
+1 Assuming OSCI approves