Merge lp:~gnuoy/charms/trusty/cinder/1453940 into lp:~openstack-charmers-archive/charms/trusty/cinder/next

Proposed by Liam Young on 2015-08-27
Status: Merged
Merged at revision: 122
Proposed branch: lp:~gnuoy/charms/trusty/cinder/1453940
Merge into: lp:~openstack-charmers-archive/charms/trusty/cinder/next
Diff against target: 1274 lines (+867/-101)
7 files modified
hooks/charmhelpers/contrib/openstack/context.py (+8/-9)
hooks/charmhelpers/contrib/storage/linux/ceph.py (+224/-2)
hooks/cinder_hooks.py (+13/-19)
tests/charmhelpers/contrib/amulet/utils.py (+234/-52)
tests/charmhelpers/contrib/openstack/amulet/deployment.py (+20/-5)
tests/charmhelpers/contrib/openstack/amulet/utils.py (+359/-0)
unit_tests/test_cinder_hooks.py (+9/-14)
To merge this branch: bzr merge lp:~gnuoy/charms/trusty/cinder/1453940
Reviewer Review Type Date Requested Status
Edward Hope-Morley 2015-08-27 Approve on 2015-09-11
Review via email: mp+269378@code.launchpad.net
To post a comment you must log in.

charm_lint_check #9591 cinder-next for gnuoy mp269378
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/9591/

charm_unit_test #8829 cinder-next for gnuoy mp269378
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/8829/

charm_amulet_test #6314 cinder-next for gnuoy mp269378
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/6314/

123. By Liam Young on 2015-09-10

Charm helper sync

charm_lint_check #9704 cinder-next for gnuoy mp269378
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/9704/

charm_unit_test #8935 cinder-next for gnuoy mp269378
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/8935/

charm_amulet_test #6331 cinder-next for gnuoy mp269378
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/6331/

Edward Hope-Morley (hopem) wrote :

LGTM +1

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
2--- hooks/charmhelpers/contrib/openstack/context.py 2015-09-03 09:43:34 +0000
3+++ hooks/charmhelpers/contrib/openstack/context.py 2015-09-10 09:35:22 +0000
4@@ -485,13 +485,15 @@
5
6 log('Generating template context for ceph', level=DEBUG)
7 mon_hosts = []
8- auth = None
9- key = None
10- use_syslog = str(config('use-syslog')).lower()
11+ ctxt = {
12+ 'use_syslog': str(config('use-syslog')).lower()
13+ }
14 for rid in relation_ids('ceph'):
15 for unit in related_units(rid):
16- auth = relation_get('auth', rid=rid, unit=unit)
17- key = relation_get('key', rid=rid, unit=unit)
18+ if not ctxt.get('auth'):
19+ ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
20+ if not ctxt.get('key'):
21+ ctxt['key'] = relation_get('key', rid=rid, unit=unit)
22 ceph_pub_addr = relation_get('ceph-public-address', rid=rid,
23 unit=unit)
24 unit_priv_addr = relation_get('private-address', rid=rid,
25@@ -500,10 +502,7 @@
26 ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
27 mon_hosts.append(ceph_addr)
28
29- ctxt = {'mon_hosts': ' '.join(sorted(mon_hosts)),
30- 'auth': auth,
31- 'key': key,
32- 'use_syslog': use_syslog}
33+ ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts))
34
35 if not os.path.isdir('/etc/ceph'):
36 os.mkdir('/etc/ceph')
37
38=== modified file 'hooks/charmhelpers/contrib/storage/linux/ceph.py'
39--- hooks/charmhelpers/contrib/storage/linux/ceph.py 2015-07-16 20:17:07 +0000
40+++ hooks/charmhelpers/contrib/storage/linux/ceph.py 2015-09-10 09:35:22 +0000
41@@ -28,6 +28,7 @@
42 import shutil
43 import json
44 import time
45+import uuid
46
47 from subprocess import (
48 check_call,
49@@ -35,8 +36,10 @@
50 CalledProcessError,
51 )
52 from charmhelpers.core.hookenv import (
53+ local_unit,
54 relation_get,
55 relation_ids,
56+ relation_set,
57 related_units,
58 log,
59 DEBUG,
60@@ -411,17 +414,52 @@
61
62 The API is versioned and defaults to version 1.
63 """
64- def __init__(self, api_version=1):
65+ def __init__(self, api_version=1, request_id=None):
66 self.api_version = api_version
67+ if request_id:
68+ self.request_id = request_id
69+ else:
70+ self.request_id = str(uuid.uuid1())
71 self.ops = []
72
73 def add_op_create_pool(self, name, replica_count=3):
74 self.ops.append({'op': 'create-pool', 'name': name,
75 'replicas': replica_count})
76
77+ def set_ops(self, ops):
78+ """Set request ops to provided value.
79+
80+ Useful for injecting ops that come from a previous request
81+ to allow comparisons to ensure validity.
82+ """
83+ self.ops = ops
84+
85 @property
86 def request(self):
87- return json.dumps({'api-version': self.api_version, 'ops': self.ops})
88+ return json.dumps({'api-version': self.api_version, 'ops': self.ops,
89+ 'request-id': self.request_id})
90+
91+ def _ops_equal(self, other):
92+ if len(self.ops) == len(other.ops):
93+ for req_no in range(0, len(self.ops)):
94+ for key in ['replicas', 'name', 'op']:
95+ if self.ops[req_no][key] != other.ops[req_no][key]:
96+ return False
97+ else:
98+ return False
99+ return True
100+
101+ def __eq__(self, other):
102+ if not isinstance(other, self.__class__):
103+ return False
104+ if self.api_version == other.api_version and \
105+ self._ops_equal(other):
106+ return True
107+ else:
108+ return False
109+
110+ def __ne__(self, other):
111+ return not self.__eq__(other)
112
113
114 class CephBrokerRsp(object):
115@@ -431,14 +469,198 @@
116
117 The API is versioned and defaults to version 1.
118 """
119+
120 def __init__(self, encoded_rsp):
121 self.api_version = None
122 self.rsp = json.loads(encoded_rsp)
123
124 @property
125+ def request_id(self):
126+ return self.rsp.get('request-id')
127+
128+ @property
129 def exit_code(self):
130 return self.rsp.get('exit-code')
131
132 @property
133 def exit_msg(self):
134 return self.rsp.get('stderr')
135+
136+
137+# Ceph Broker Conversation:
138+# If a charm needs an action to be taken by ceph it can create a CephBrokerRq
139+# and send that request to ceph via the ceph relation. The CephBrokerRq has a
140+# unique id so that the client can identity which CephBrokerRsp is associated
141+# with the request. Ceph will also respond to each client unit individually
142+# creating a response key per client unit eg glance/0 will get a CephBrokerRsp
143+# via key broker-rsp-glance-0
144+#
145+# To use this the charm can just do something like:
146+#
147+# from charmhelpers.contrib.storage.linux.ceph import (
148+# send_request_if_needed,
149+# is_request_complete,
150+# CephBrokerRq,
151+# )
152+#
153+# @hooks.hook('ceph-relation-changed')
154+# def ceph_changed():
155+# rq = CephBrokerRq()
156+# rq.add_op_create_pool(name='poolname', replica_count=3)
157+#
158+# if is_request_complete(rq):
159+# <Request complete actions>
160+# else:
161+# send_request_if_needed(get_ceph_request())
162+#
163+# CephBrokerRq and CephBrokerRsp are serialized into JSON. Below is an example
164+# of glance having sent a request to ceph which ceph has successfully processed
165+# 'ceph:8': {
166+# 'ceph/0': {
167+# 'auth': 'cephx',
168+# 'broker-rsp-glance-0': '{"request-id": "0bc7dc54", "exit-code": 0}',
169+# 'broker_rsp': '{"request-id": "0da543b8", "exit-code": 0}',
170+# 'ceph-public-address': '10.5.44.103',
171+# 'key': 'AQCLDttVuHXINhAAvI144CB09dYchhHyTUY9BQ==',
172+# 'private-address': '10.5.44.103',
173+# },
174+# 'glance/0': {
175+# 'broker_req': ('{"api-version": 1, "request-id": "0bc7dc54", '
176+# '"ops": [{"replicas": 3, "name": "glance", '
177+# '"op": "create-pool"}]}'),
178+# 'private-address': '10.5.44.109',
179+# },
180+# }
181+
182+def get_previous_request(rid):
183+ """Return the last ceph broker request sent on a given relation
184+
185+ @param rid: Relation id to query for request
186+ """
187+ request = None
188+ broker_req = relation_get(attribute='broker_req', rid=rid,
189+ unit=local_unit())
190+ if broker_req:
191+ request_data = json.loads(broker_req)
192+ request = CephBrokerRq(api_version=request_data['api-version'],
193+ request_id=request_data['request-id'])
194+ request.set_ops(request_data['ops'])
195+
196+ return request
197+
198+
199+def get_request_states(request):
200+ """Return a dict of requests per relation id with their corresponding
201+ completion state.
202+
203+ This allows a charm, which has a request for ceph, to see whether there is
204+ an equivalent request already being processed and if so what state that
205+ request is in.
206+
207+ @param request: A CephBrokerRq object
208+ """
209+ complete = []
210+ requests = {}
211+ for rid in relation_ids('ceph'):
212+ complete = False
213+ previous_request = get_previous_request(rid)
214+ if request == previous_request:
215+ sent = True
216+ complete = is_request_complete_for_rid(previous_request, rid)
217+ else:
218+ sent = False
219+ complete = False
220+
221+ requests[rid] = {
222+ 'sent': sent,
223+ 'complete': complete,
224+ }
225+
226+ return requests
227+
228+
229+def is_request_sent(request):
230+ """Check to see if a functionally equivalent request has already been sent
231+
232+ Returns True if a similair request has been sent
233+
234+ @param request: A CephBrokerRq object
235+ """
236+ states = get_request_states(request)
237+ for rid in states.keys():
238+ if not states[rid]['sent']:
239+ return False
240+
241+ return True
242+
243+
244+def is_request_complete(request):
245+ """Check to see if a functionally equivalent request has already been
246+ completed
247+
248+ Returns True if a similair request has been completed
249+
250+ @param request: A CephBrokerRq object
251+ """
252+ states = get_request_states(request)
253+ for rid in states.keys():
254+ if not states[rid]['complete']:
255+ return False
256+
257+ return True
258+
259+
260+def is_request_complete_for_rid(request, rid):
261+ """Check if a given request has been completed on the given relation
262+
263+ @param request: A CephBrokerRq object
264+ @param rid: Relation ID
265+ """
266+ broker_key = get_broker_rsp_key()
267+ for unit in related_units(rid):
268+ rdata = relation_get(rid=rid, unit=unit)
269+ if rdata.get(broker_key):
270+ rsp = CephBrokerRsp(rdata.get(broker_key))
271+ if rsp.request_id == request.request_id:
272+ if not rsp.exit_code:
273+ return True
274+ else:
275+ # The remote unit sent no reply targeted at this unit so either the
276+ # remote ceph cluster does not support unit targeted replies or it
277+ # has not processed our request yet.
278+ if rdata.get('broker_rsp'):
279+ request_data = json.loads(rdata['broker_rsp'])
280+ if request_data.get('request-id'):
281+ log('Ignoring legacy broker_rsp without unit key as remote '
282+ 'service supports unit specific replies', level=DEBUG)
283+ else:
284+ log('Using legacy broker_rsp as remote service does not '
285+ 'supports unit specific replies', level=DEBUG)
286+ rsp = CephBrokerRsp(rdata['broker_rsp'])
287+ if not rsp.exit_code:
288+ return True
289+
290+ return False
291+
292+
293+def get_broker_rsp_key():
294+ """Return broker response key for this unit
295+
296+ This is the key that ceph is going to use to pass request status
297+ information back to this unit
298+ """
299+ return 'broker-rsp-' + local_unit().replace('/', '-')
300+
301+
302+def send_request_if_needed(request):
303+ """Send broker request if an equivalent request has not already been sent
304+
305+ @param request: A CephBrokerRq object
306+ """
307+ if is_request_sent(request):
308+ log('Request already sent but not complete, not sending new request',
309+ level=DEBUG)
310+ else:
311+ for rid in relation_ids('ceph'):
312+ log('Sending request {}'.format(request.request_id), level=DEBUG)
313+ relation_set(relation_id=rid, broker_req=request.request)
314
315=== modified file 'hooks/cinder_hooks.py'
316--- hooks/cinder_hooks.py 2015-09-01 15:20:27 +0000
317+++ hooks/cinder_hooks.py 2015-09-10 09:35:22 +0000
318@@ -42,7 +42,6 @@
319 unit_get,
320 log,
321 ERROR,
322- INFO,
323 )
324
325 from charmhelpers.fetch import (
326@@ -67,9 +66,10 @@
327 )
328
329 from charmhelpers.contrib.storage.linux.ceph import (
330+ send_request_if_needed,
331+ is_request_complete,
332 ensure_ceph_keyring,
333 CephBrokerRq,
334- CephBrokerRsp,
335 delete_keyring,
336 )
337
338@@ -320,6 +320,14 @@
339 apt_install('ceph-common', fatal=True)
340
341
342+def get_ceph_request():
343+ service = service_name()
344+ rq = CephBrokerRq()
345+ replicas = config('ceph-osd-replication-count')
346+ rq.add_op_create_pool(name=service, replica_count=replicas)
347+ return rq
348+
349+
350 @hooks.hook('ceph-relation-changed')
351 @restart_on_change(restart_map())
352 def ceph_changed(relation_id=None):
353@@ -333,17 +341,8 @@
354 juju_log('Could not create ceph keyring: peer not ready?')
355 return
356
357- settings = relation_get(rid=relation_id)
358- if settings and 'broker_rsp' in settings:
359- rsp = CephBrokerRsp(settings['broker_rsp'])
360- # Non-zero return code implies failure
361- if rsp.exit_code:
362- log("Ceph broker request failed (rc=%s, msg=%s)" %
363- (rsp.exit_code, rsp.exit_msg), level=ERROR)
364- return
365-
366- log("Ceph broker request succeeded (rc=%s, msg=%s)" %
367- (rsp.exit_code, rsp.exit_msg), level=INFO)
368+ if is_request_complete(get_ceph_request()):
369+ log('Request complete')
370 set_ceph_env_variables(service=service)
371 CONFIGS.write(CINDER_CONF)
372 CONFIGS.write(ceph_config_file())
373@@ -351,12 +350,7 @@
374 # guarantee that ceph resources are ready.
375 service_restart('cinder-volume')
376 else:
377- rq = CephBrokerRq()
378- replicas = config('ceph-osd-replication-count')
379- rq.add_op_create_pool(name=service, replica_count=replicas)
380- for rid in relation_ids('ceph'):
381- relation_set(relation_id=rid, broker_req=rq.request)
382- log("Request(s) sent to Ceph broker (rid=%s)" % (rid))
383+ send_request_if_needed(get_ceph_request())
384
385
386 @hooks.hook('ceph-relation-broken')
387
388=== modified file 'tests/charmhelpers/contrib/amulet/utils.py'
389--- tests/charmhelpers/contrib/amulet/utils.py 2015-08-18 17:34:33 +0000
390+++ tests/charmhelpers/contrib/amulet/utils.py 2015-09-10 09:35:22 +0000
391@@ -19,9 +19,11 @@
392 import logging
393 import os
394 import re
395+import socket
396 import subprocess
397 import sys
398 import time
399+import uuid
400
401 import amulet
402 import distro_info
403@@ -114,7 +116,7 @@
404 # /!\ DEPRECATION WARNING (beisner):
405 # New and existing tests should be rewritten to use
406 # validate_services_by_name() as it is aware of init systems.
407- self.log.warn('/!\\ DEPRECATION WARNING: use '
408+ self.log.warn('DEPRECATION WARNING: use '
409 'validate_services_by_name instead of validate_services '
410 'due to init system differences.')
411
412@@ -269,33 +271,52 @@
413 """Get last modification time of directory."""
414 return sentry_unit.directory_stat(directory)['mtime']
415
416- def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False):
417- """Get process' start time.
418-
419- Determine start time of the process based on the last modification
420- time of the /proc/pid directory. If pgrep_full is True, the process
421- name is matched against the full command line.
422- """
423- if pgrep_full:
424- cmd = 'pgrep -o -f {}'.format(service)
425- else:
426- cmd = 'pgrep -o {}'.format(service)
427- cmd = cmd + ' | grep -v pgrep || exit 0'
428- cmd_out = sentry_unit.run(cmd)
429- self.log.debug('CMDout: ' + str(cmd_out))
430- if cmd_out[0]:
431- self.log.debug('Pid for %s %s' % (service, str(cmd_out[0])))
432- proc_dir = '/proc/{}'.format(cmd_out[0].strip())
433- return self._get_dir_mtime(sentry_unit, proc_dir)
434+ def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None):
435+ """Get start time of a process based on the last modification time
436+ of the /proc/pid directory.
437+
438+ :sentry_unit: The sentry unit to check for the service on
439+ :service: service name to look for in process table
440+ :pgrep_full: [Deprecated] Use full command line search mode with pgrep
441+ :returns: epoch time of service process start
442+ :param commands: list of bash commands
443+ :param sentry_units: list of sentry unit pointers
444+ :returns: None if successful; Failure message otherwise
445+ """
446+ if pgrep_full is not None:
447+ # /!\ DEPRECATION WARNING (beisner):
448+ # No longer implemented, as pidof is now used instead of pgrep.
449+ # https://bugs.launchpad.net/charm-helpers/+bug/1474030
450+ self.log.warn('DEPRECATION WARNING: pgrep_full bool is no '
451+ 'longer implemented re: lp 1474030.')
452+
453+ pid_list = self.get_process_id_list(sentry_unit, service)
454+ pid = pid_list[0]
455+ proc_dir = '/proc/{}'.format(pid)
456+ self.log.debug('Pid for {} on {}: {}'.format(
457+ service, sentry_unit.info['unit_name'], pid))
458+
459+ return self._get_dir_mtime(sentry_unit, proc_dir)
460
461 def service_restarted(self, sentry_unit, service, filename,
462- pgrep_full=False, sleep_time=20):
463+ pgrep_full=None, sleep_time=20):
464 """Check if service was restarted.
465
466 Compare a service's start time vs a file's last modification time
467 (such as a config file for that service) to determine if the service
468 has been restarted.
469 """
470+ # /!\ DEPRECATION WARNING (beisner):
471+ # This method is prone to races in that no before-time is known.
472+ # Use validate_service_config_changed instead.
473+
474+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
475+ # used instead of pgrep. pgrep_full is still passed through to ensure
476+ # deprecation WARNS. lp1474030
477+ self.log.warn('DEPRECATION WARNING: use '
478+ 'validate_service_config_changed instead of '
479+ 'service_restarted due to known races.')
480+
481 time.sleep(sleep_time)
482 if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
483 self._get_file_mtime(sentry_unit, filename)):
484@@ -304,15 +325,15 @@
485 return False
486
487 def service_restarted_since(self, sentry_unit, mtime, service,
488- pgrep_full=False, sleep_time=20,
489- retry_count=2):
490+ pgrep_full=None, sleep_time=20,
491+ retry_count=2, retry_sleep_time=30):
492 """Check if service was been started after a given time.
493
494 Args:
495 sentry_unit (sentry): The sentry unit to check for the service on
496 mtime (float): The epoch time to check against
497 service (string): service name to look for in process table
498- pgrep_full (boolean): Use full command line search mode with pgrep
499+ pgrep_full: [Deprecated] Use full command line search mode with pgrep
500 sleep_time (int): Seconds to sleep before looking for process
501 retry_count (int): If service is not found, how many times to retry
502
503@@ -321,30 +342,44 @@
504 False if service is older than mtime or if service was
505 not found.
506 """
507- self.log.debug('Checking %s restarted since %s' % (service, mtime))
508+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
509+ # used instead of pgrep. pgrep_full is still passed through to ensure
510+ # deprecation WARNS. lp1474030
511+
512+ unit_name = sentry_unit.info['unit_name']
513+ self.log.debug('Checking that %s service restarted since %s on '
514+ '%s' % (service, mtime, unit_name))
515 time.sleep(sleep_time)
516- proc_start_time = self._get_proc_start_time(sentry_unit, service,
517- pgrep_full)
518- while retry_count > 0 and not proc_start_time:
519- self.log.debug('No pid file found for service %s, will retry %i '
520- 'more times' % (service, retry_count))
521- time.sleep(30)
522- proc_start_time = self._get_proc_start_time(sentry_unit, service,
523- pgrep_full)
524- retry_count = retry_count - 1
525+ proc_start_time = None
526+ tries = 0
527+ while tries <= retry_count and not proc_start_time:
528+ try:
529+ proc_start_time = self._get_proc_start_time(sentry_unit,
530+ service,
531+ pgrep_full)
532+ self.log.debug('Attempt {} to get {} proc start time on {} '
533+ 'OK'.format(tries, service, unit_name))
534+ except IOError:
535+ # NOTE(beisner) - race avoidance, proc may not exist yet.
536+ # https://bugs.launchpad.net/charm-helpers/+bug/1474030
537+ self.log.debug('Attempt {} to get {} proc start time on {} '
538+ 'failed'.format(tries, service, unit_name))
539+ time.sleep(retry_sleep_time)
540+ tries += 1
541
542 if not proc_start_time:
543 self.log.warn('No proc start time found, assuming service did '
544 'not start')
545 return False
546 if proc_start_time >= mtime:
547- self.log.debug('proc start time is newer than provided mtime'
548- '(%s >= %s)' % (proc_start_time, mtime))
549+ self.log.debug('Proc start time is newer than provided mtime'
550+ '(%s >= %s) on %s (OK)' % (proc_start_time,
551+ mtime, unit_name))
552 return True
553 else:
554- self.log.warn('proc start time (%s) is older than provided mtime '
555- '(%s), service did not restart' % (proc_start_time,
556- mtime))
557+ self.log.warn('Proc start time (%s) is older than provided mtime '
558+ '(%s) on %s, service did not '
559+ 'restart' % (proc_start_time, mtime, unit_name))
560 return False
561
562 def config_updated_since(self, sentry_unit, filename, mtime,
563@@ -374,8 +409,9 @@
564 return False
565
566 def validate_service_config_changed(self, sentry_unit, mtime, service,
567- filename, pgrep_full=False,
568- sleep_time=20, retry_count=2):
569+ filename, pgrep_full=None,
570+ sleep_time=20, retry_count=2,
571+ retry_sleep_time=30):
572 """Check service and file were updated after mtime
573
574 Args:
575@@ -383,9 +419,10 @@
576 mtime (float): The epoch time to check against
577 service (string): service name to look for in process table
578 filename (string): The file to check mtime of
579- pgrep_full (boolean): Use full command line search mode with pgrep
580- sleep_time (int): Seconds to sleep before looking for process
581+ pgrep_full: [Deprecated] Use full command line search mode with pgrep
582+ sleep_time (int): Initial sleep in seconds to pass to test helpers
583 retry_count (int): If service is not found, how many times to retry
584+ retry_sleep_time (int): Time in seconds to wait between retries
585
586 Typical Usage:
587 u = OpenStackAmuletUtils(ERROR)
588@@ -402,15 +439,25 @@
589 mtime, False if service is older than mtime or if service was
590 not found or if filename was modified before mtime.
591 """
592- self.log.debug('Checking %s restarted since %s' % (service, mtime))
593- time.sleep(sleep_time)
594- service_restart = self.service_restarted_since(sentry_unit, mtime,
595- service,
596- pgrep_full=pgrep_full,
597- sleep_time=0,
598- retry_count=retry_count)
599- config_update = self.config_updated_since(sentry_unit, filename, mtime,
600- sleep_time=0)
601+
602+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
603+ # used instead of pgrep. pgrep_full is still passed through to ensure
604+ # deprecation WARNS. lp1474030
605+
606+ service_restart = self.service_restarted_since(
607+ sentry_unit, mtime,
608+ service,
609+ pgrep_full=pgrep_full,
610+ sleep_time=sleep_time,
611+ retry_count=retry_count,
612+ retry_sleep_time=retry_sleep_time)
613+
614+ config_update = self.config_updated_since(
615+ sentry_unit,
616+ filename,
617+ mtime,
618+ sleep_time=0)
619+
620 return service_restart and config_update
621
622 def get_sentry_time(self, sentry_unit):
623@@ -428,7 +475,6 @@
624 """Return a list of all Ubuntu releases in order of release."""
625 _d = distro_info.UbuntuDistroInfo()
626 _release_list = _d.all
627- self.log.debug('Ubuntu release list: {}'.format(_release_list))
628 return _release_list
629
630 def file_to_url(self, file_rel_path):
631@@ -568,6 +614,142 @@
632
633 return None
634
635+ def validate_sectionless_conf(self, file_contents, expected):
636+ """A crude conf parser. Useful to inspect configuration files which
637+ do not have section headers (as would be necessary in order to use
638+ the configparser). Such as openstack-dashboard or rabbitmq confs."""
639+ for line in file_contents.split('\n'):
640+ if '=' in line:
641+ args = line.split('=')
642+ if len(args) <= 1:
643+ continue
644+ key = args[0].strip()
645+ value = args[1].strip()
646+ if key in expected.keys():
647+ if expected[key] != value:
648+ msg = ('Config mismatch. Expected, actual: {}, '
649+ '{}'.format(expected[key], value))
650+ amulet.raise_status(amulet.FAIL, msg=msg)
651+
652+ def get_unit_hostnames(self, units):
653+ """Return a dict of juju unit names to hostnames."""
654+ host_names = {}
655+ for unit in units:
656+ host_names[unit.info['unit_name']] = \
657+ str(unit.file_contents('/etc/hostname').strip())
658+ self.log.debug('Unit host names: {}'.format(host_names))
659+ return host_names
660+
661+ def run_cmd_unit(self, sentry_unit, cmd):
662+ """Run a command on a unit, return the output and exit code."""
663+ output, code = sentry_unit.run(cmd)
664+ if code == 0:
665+ self.log.debug('{} `{}` command returned {} '
666+ '(OK)'.format(sentry_unit.info['unit_name'],
667+ cmd, code))
668+ else:
669+ msg = ('{} `{}` command returned {} '
670+ '{}'.format(sentry_unit.info['unit_name'],
671+ cmd, code, output))
672+ amulet.raise_status(amulet.FAIL, msg=msg)
673+ return str(output), code
674+
675+ def file_exists_on_unit(self, sentry_unit, file_name):
676+ """Check if a file exists on a unit."""
677+ try:
678+ sentry_unit.file_stat(file_name)
679+ return True
680+ except IOError:
681+ return False
682+ except Exception as e:
683+ msg = 'Error checking file {}: {}'.format(file_name, e)
684+ amulet.raise_status(amulet.FAIL, msg=msg)
685+
686+ def file_contents_safe(self, sentry_unit, file_name,
687+ max_wait=60, fatal=False):
688+ """Get file contents from a sentry unit. Wrap amulet file_contents
689+ with retry logic to address races where a file checks as existing,
690+ but no longer exists by the time file_contents is called.
691+ Return None if file not found. Optionally raise if fatal is True."""
692+ unit_name = sentry_unit.info['unit_name']
693+ file_contents = False
694+ tries = 0
695+ while not file_contents and tries < (max_wait / 4):
696+ try:
697+ file_contents = sentry_unit.file_contents(file_name)
698+ except IOError:
699+ self.log.debug('Attempt {} to open file {} from {} '
700+ 'failed'.format(tries, file_name,
701+ unit_name))
702+ time.sleep(4)
703+ tries += 1
704+
705+ if file_contents:
706+ return file_contents
707+ elif not fatal:
708+ return None
709+ elif fatal:
710+ msg = 'Failed to get file contents from unit.'
711+ amulet.raise_status(amulet.FAIL, msg)
712+
713+ def port_knock_tcp(self, host="localhost", port=22, timeout=15):
714+ """Open a TCP socket to check for a listening sevice on a host.
715+
716+ :param host: host name or IP address, default to localhost
717+ :param port: TCP port number, default to 22
718+ :param timeout: Connect timeout, default to 15 seconds
719+ :returns: True if successful, False if connect failed
720+ """
721+
722+ # Resolve host name if possible
723+ try:
724+ connect_host = socket.gethostbyname(host)
725+ host_human = "{} ({})".format(connect_host, host)
726+ except socket.error as e:
727+ self.log.warn('Unable to resolve address: '
728+ '{} ({}) Trying anyway!'.format(host, e))
729+ connect_host = host
730+ host_human = connect_host
731+
732+ # Attempt socket connection
733+ try:
734+ knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
735+ knock.settimeout(timeout)
736+ knock.connect((connect_host, port))
737+ knock.close()
738+ self.log.debug('Socket connect OK for host '
739+ '{} on port {}.'.format(host_human, port))
740+ return True
741+ except socket.error as e:
742+ self.log.debug('Socket connect FAIL for'
743+ ' {} port {} ({})'.format(host_human, port, e))
744+ return False
745+
746+ def port_knock_units(self, sentry_units, port=22,
747+ timeout=15, expect_success=True):
748+ """Open a TCP socket to check for a listening sevice on each
749+ listed juju unit.
750+
751+ :param sentry_units: list of sentry unit pointers
752+ :param port: TCP port number, default to 22
753+ :param timeout: Connect timeout, default to 15 seconds
754+ :expect_success: True by default, set False to invert logic
755+ :returns: None if successful, Failure message otherwise
756+ """
757+ for unit in sentry_units:
758+ host = unit.info['public-address']
759+ connected = self.port_knock_tcp(host, port, timeout)
760+ if not connected and expect_success:
761+ return 'Socket connect failed.'
762+ elif connected and not expect_success:
763+ return 'Socket connected unexpectedly.'
764+
765+ def get_uuid_epoch_stamp(self):
766+ """Returns a stamp string based on uuid4 and epoch time. Useful in
767+ generating test messages which need to be unique-ish."""
768+ return '[{}-{}]'.format(uuid.uuid4(), time.time())
769+
770+# amulet juju action helpers:
771 def run_action(self, unit_sentry, action,
772 _check_output=subprocess.check_output):
773 """Run the named action on a given unit sentry.
774
775=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
776--- tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-08-18 17:34:33 +0000
777+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-09-10 09:35:22 +0000
778@@ -44,8 +44,15 @@
779 Determine if the local branch being tested is derived from its
780 stable or next (dev) branch, and based on this, use the corresonding
781 stable or next branches for the other_services."""
782+
783+ # Charms outside the lp:~openstack-charmers namespace
784 base_charms = ['mysql', 'mongodb', 'nrpe']
785
786+ # Force these charms to current series even when using an older series.
787+ # ie. Use trusty/nrpe even when series is precise, as the P charm
788+ # does not possess the necessary external master config and hooks.
789+ force_series_current = ['nrpe']
790+
791 if self.series in ['precise', 'trusty']:
792 base_series = self.series
793 else:
794@@ -53,11 +60,17 @@
795
796 if self.stable:
797 for svc in other_services:
798+ if svc['name'] in force_series_current:
799+ base_series = self.current_next
800+
801 temp = 'lp:charms/{}/{}'
802 svc['location'] = temp.format(base_series,
803 svc['name'])
804 else:
805 for svc in other_services:
806+ if svc['name'] in force_series_current:
807+ base_series = self.current_next
808+
809 if svc['name'] in base_charms:
810 temp = 'lp:charms/{}/{}'
811 svc['location'] = temp.format(base_series,
812@@ -77,21 +90,23 @@
813
814 services = other_services
815 services.append(this_service)
816+
817+ # Charms which should use the source config option
818 use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
819 'ceph-osd', 'ceph-radosgw']
820- # Most OpenStack subordinate charms do not expose an origin option
821- # as that is controlled by the principle.
822- ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
823+
824+ # Charms which can not use openstack-origin, ie. many subordinates
825+ no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe']
826
827 if self.openstack:
828 for svc in services:
829- if svc['name'] not in use_source + ignore:
830+ if svc['name'] not in use_source + no_origin:
831 config = {'openstack-origin': self.openstack}
832 self.d.configure(svc['name'], config)
833
834 if self.source:
835 for svc in services:
836- if svc['name'] in use_source and svc['name'] not in ignore:
837+ if svc['name'] in use_source and svc['name'] not in no_origin:
838 config = {'source': self.source}
839 self.d.configure(svc['name'], config)
840
841
842=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
843--- tests/charmhelpers/contrib/openstack/amulet/utils.py 2015-06-29 13:27:57 +0000
844+++ tests/charmhelpers/contrib/openstack/amulet/utils.py 2015-09-10 09:35:22 +0000
845@@ -27,6 +27,7 @@
846 import heatclient.v1.client as heat_client
847 import keystoneclient.v2_0 as keystone_client
848 import novaclient.v1_1.client as nova_client
849+import pika
850 import swiftclient
851
852 from charmhelpers.contrib.amulet.utils import (
853@@ -602,3 +603,361 @@
854 self.log.debug('Ceph {} samples (OK): '
855 '{}'.format(sample_type, samples))
856 return None
857+
858+# rabbitmq/amqp specific helpers:
859+ def add_rmq_test_user(self, sentry_units,
860+ username="testuser1", password="changeme"):
861+ """Add a test user via the first rmq juju unit, check connection as
862+ the new user against all sentry units.
863+
864+ :param sentry_units: list of sentry unit pointers
865+ :param username: amqp user name, default to testuser1
866+ :param password: amqp user password
867+ :returns: None if successful. Raise on error.
868+ """
869+ self.log.debug('Adding rmq user ({})...'.format(username))
870+
871+ # Check that user does not already exist
872+ cmd_user_list = 'rabbitmqctl list_users'
873+ output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
874+ if username in output:
875+ self.log.warning('User ({}) already exists, returning '
876+ 'gracefully.'.format(username))
877+ return
878+
879+ perms = '".*" ".*" ".*"'
880+ cmds = ['rabbitmqctl add_user {} {}'.format(username, password),
881+ 'rabbitmqctl set_permissions {} {}'.format(username, perms)]
882+
883+ # Add user via first unit
884+ for cmd in cmds:
885+ output, _ = self.run_cmd_unit(sentry_units[0], cmd)
886+
887+ # Check connection against the other sentry_units
888+ self.log.debug('Checking user connect against units...')
889+ for sentry_unit in sentry_units:
890+ connection = self.connect_amqp_by_unit(sentry_unit, ssl=False,
891+ username=username,
892+ password=password)
893+ connection.close()
894+
895+ def delete_rmq_test_user(self, sentry_units, username="testuser1"):
896+ """Delete a rabbitmq user via the first rmq juju unit.
897+
898+ :param sentry_units: list of sentry unit pointers
899+ :param username: amqp user name, default to testuser1
900+ :param password: amqp user password
901+ :returns: None if successful or no such user.
902+ """
903+ self.log.debug('Deleting rmq user ({})...'.format(username))
904+
905+ # Check that the user exists
906+ cmd_user_list = 'rabbitmqctl list_users'
907+ output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
908+
909+ if username not in output:
910+ self.log.warning('User ({}) does not exist, returning '
911+ 'gracefully.'.format(username))
912+ return
913+
914+ # Delete the user
915+ cmd_user_del = 'rabbitmqctl delete_user {}'.format(username)
916+ output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del)
917+
918+ def get_rmq_cluster_status(self, sentry_unit):
919+ """Execute rabbitmq cluster status command on a unit and return
920+ the full output.
921+
922+ :param unit: sentry unit
923+ :returns: String containing console output of cluster status command
924+ """
925+ cmd = 'rabbitmqctl cluster_status'
926+ output, _ = self.run_cmd_unit(sentry_unit, cmd)
927+ self.log.debug('{} cluster_status:\n{}'.format(
928+ sentry_unit.info['unit_name'], output))
929+ return str(output)
930+
931+ def get_rmq_cluster_running_nodes(self, sentry_unit):
932+ """Parse rabbitmqctl cluster_status output string, return list of
933+ running rabbitmq cluster nodes.
934+
935+ :param unit: sentry unit
936+ :returns: List containing node names of running nodes
937+ """
938+ # NOTE(beisner): rabbitmqctl cluster_status output is not
939+ # json-parsable, do string chop foo, then json.loads that.
940+ str_stat = self.get_rmq_cluster_status(sentry_unit)
941+ if 'running_nodes' in str_stat:
942+ pos_start = str_stat.find("{running_nodes,") + 15
943+ pos_end = str_stat.find("]},", pos_start) + 1
944+ str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"')
945+ run_nodes = json.loads(str_run_nodes)
946+ return run_nodes
947+ else:
948+ return []
949+
950+ def validate_rmq_cluster_running_nodes(self, sentry_units):
951+ """Check that all rmq unit hostnames are represented in the
952+ cluster_status output of all units.
953+
954+ :param host_names: dict of juju unit names to host names
955+ :param units: list of sentry unit pointers (all rmq units)
956+ :returns: None if successful, otherwise return error message
957+ """
958+ host_names = self.get_unit_hostnames(sentry_units)
959+ errors = []
960+
961+ # Query every unit for cluster_status running nodes
962+ for query_unit in sentry_units:
963+ query_unit_name = query_unit.info['unit_name']
964+ running_nodes = self.get_rmq_cluster_running_nodes(query_unit)
965+
966+ # Confirm that every unit is represented in the queried unit's
967+ # cluster_status running nodes output.
968+ for validate_unit in sentry_units:
969+ val_host_name = host_names[validate_unit.info['unit_name']]
970+ val_node_name = 'rabbit@{}'.format(val_host_name)
971+
972+ if val_node_name not in running_nodes:
973+ errors.append('Cluster member check failed on {}: {} not '
974+ 'in {}\n'.format(query_unit_name,
975+ val_node_name,
976+ running_nodes))
977+ if errors:
978+ return ''.join(errors)
979+
980+ def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None):
981+ """Check a single juju rmq unit for ssl and port in the config file."""
982+ host = sentry_unit.info['public-address']
983+ unit_name = sentry_unit.info['unit_name']
984+
985+ conf_file = '/etc/rabbitmq/rabbitmq.config'
986+ conf_contents = str(self.file_contents_safe(sentry_unit,
987+ conf_file, max_wait=16))
988+ # Checks
989+ conf_ssl = 'ssl' in conf_contents
990+ conf_port = str(port) in conf_contents
991+
992+ # Port explicitly checked in config
993+ if port and conf_port and conf_ssl:
994+ self.log.debug('SSL is enabled @{}:{} '
995+ '({})'.format(host, port, unit_name))
996+ return True
997+ elif port and not conf_port and conf_ssl:
998+ self.log.debug('SSL is enabled @{} but not on port {} '
999+ '({})'.format(host, port, unit_name))
1000+ return False
1001+ # Port not checked (useful when checking that ssl is disabled)
1002+ elif not port and conf_ssl:
1003+ self.log.debug('SSL is enabled @{}:{} '
1004+ '({})'.format(host, port, unit_name))
1005+ return True
1006+ elif not port and not conf_ssl:
1007+ self.log.debug('SSL not enabled @{}:{} '
1008+ '({})'.format(host, port, unit_name))
1009+ return False
1010+ else:
1011+ msg = ('Unknown condition when checking SSL status @{}:{} '
1012+ '({})'.format(host, port, unit_name))
1013+ amulet.raise_status(amulet.FAIL, msg)
1014+
1015+ def validate_rmq_ssl_enabled_units(self, sentry_units, port=None):
1016+ """Check that ssl is enabled on rmq juju sentry units.
1017+
1018+ :param sentry_units: list of all rmq sentry units
1019+ :param port: optional ssl port override to validate
1020+ :returns: None if successful, otherwise return error message
1021+ """
1022+ for sentry_unit in sentry_units:
1023+ if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port):
1024+ return ('Unexpected condition: ssl is disabled on unit '
1025+ '({})'.format(sentry_unit.info['unit_name']))
1026+ return None
1027+
1028+ def validate_rmq_ssl_disabled_units(self, sentry_units):
1029+ """Check that ssl is enabled on listed rmq juju sentry units.
1030+
1031+ :param sentry_units: list of all rmq sentry units
1032+ :returns: True if successful. Raise on error.
1033+ """
1034+ for sentry_unit in sentry_units:
1035+ if self.rmq_ssl_is_enabled_on_unit(sentry_unit):
1036+ return ('Unexpected condition: ssl is enabled on unit '
1037+ '({})'.format(sentry_unit.info['unit_name']))
1038+ return None
1039+
1040+ def configure_rmq_ssl_on(self, sentry_units, deployment,
1041+ port=None, max_wait=60):
1042+ """Turn ssl charm config option on, with optional non-default
1043+ ssl port specification. Confirm that it is enabled on every
1044+ unit.
1045+
1046+ :param sentry_units: list of sentry units
1047+ :param deployment: amulet deployment object pointer
1048+ :param port: amqp port, use defaults if None
1049+ :param max_wait: maximum time to wait in seconds to confirm
1050+ :returns: None if successful. Raise on error.
1051+ """
1052+ self.log.debug('Setting ssl charm config option: on')
1053+
1054+ # Enable RMQ SSL
1055+ config = {'ssl': 'on'}
1056+ if port:
1057+ config['ssl_port'] = port
1058+
1059+ deployment.configure('rabbitmq-server', config)
1060+
1061+ # Confirm
1062+ tries = 0
1063+ ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
1064+ while ret and tries < (max_wait / 4):
1065+ time.sleep(4)
1066+ self.log.debug('Attempt {}: {}'.format(tries, ret))
1067+ ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
1068+ tries += 1
1069+
1070+ if ret:
1071+ amulet.raise_status(amulet.FAIL, ret)
1072+
1073+ def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60):
1074+ """Turn ssl charm config option off, confirm that it is disabled
1075+ on every unit.
1076+
1077+ :param sentry_units: list of sentry units
1078+ :param deployment: amulet deployment object pointer
1079+ :param max_wait: maximum time to wait in seconds to confirm
1080+ :returns: None if successful. Raise on error.
1081+ """
1082+ self.log.debug('Setting ssl charm config option: off')
1083+
1084+ # Disable RMQ SSL
1085+ config = {'ssl': 'off'}
1086+ deployment.configure('rabbitmq-server', config)
1087+
1088+ # Confirm
1089+ tries = 0
1090+ ret = self.validate_rmq_ssl_disabled_units(sentry_units)
1091+ while ret and tries < (max_wait / 4):
1092+ time.sleep(4)
1093+ self.log.debug('Attempt {}: {}'.format(tries, ret))
1094+ ret = self.validate_rmq_ssl_disabled_units(sentry_units)
1095+ tries += 1
1096+
1097+ if ret:
1098+ amulet.raise_status(amulet.FAIL, ret)
1099+
1100+ def connect_amqp_by_unit(self, sentry_unit, ssl=False,
1101+ port=None, fatal=True,
1102+ username="testuser1", password="changeme"):
1103+ """Establish and return a pika amqp connection to the rabbitmq service
1104+ running on a rmq juju unit.
1105+
1106+ :param sentry_unit: sentry unit pointer
1107+ :param ssl: boolean, default to False
1108+ :param port: amqp port, use defaults if None
1109+ :param fatal: boolean, default to True (raises on connect error)
1110+ :param username: amqp user name, default to testuser1
1111+ :param password: amqp user password
1112+ :returns: pika amqp connection pointer or None if failed and non-fatal
1113+ """
1114+ host = sentry_unit.info['public-address']
1115+ unit_name = sentry_unit.info['unit_name']
1116+
1117+ # Default port logic if port is not specified
1118+ if ssl and not port:
1119+ port = 5671
1120+ elif not ssl and not port:
1121+ port = 5672
1122+
1123+ self.log.debug('Connecting to amqp on {}:{} ({}) as '
1124+ '{}...'.format(host, port, unit_name, username))
1125+
1126+ try:
1127+ credentials = pika.PlainCredentials(username, password)
1128+ parameters = pika.ConnectionParameters(host=host, port=port,
1129+ credentials=credentials,
1130+ ssl=ssl,
1131+ connection_attempts=3,
1132+ retry_delay=5,
1133+ socket_timeout=1)
1134+ connection = pika.BlockingConnection(parameters)
1135+ assert connection.server_properties['product'] == 'RabbitMQ'
1136+ self.log.debug('Connect OK')
1137+ return connection
1138+ except Exception as e:
1139+ msg = ('amqp connection failed to {}:{} as '
1140+ '{} ({})'.format(host, port, username, str(e)))
1141+ if fatal:
1142+ amulet.raise_status(amulet.FAIL, msg)
1143+ else:
1144+ self.log.warn(msg)
1145+ return None
1146+
1147+ def publish_amqp_message_by_unit(self, sentry_unit, message,
1148+ queue="test", ssl=False,
1149+ username="testuser1",
1150+ password="changeme",
1151+ port=None):
1152+ """Publish an amqp message to a rmq juju unit.
1153+
1154+ :param sentry_unit: sentry unit pointer
1155+ :param message: amqp message string
1156+ :param queue: message queue, default to test
1157+ :param username: amqp user name, default to testuser1
1158+ :param password: amqp user password
1159+ :param ssl: boolean, default to False
1160+ :param port: amqp port, use defaults if None
1161+ :returns: None. Raises exception if publish failed.
1162+ """
1163+ self.log.debug('Publishing message to {} queue:\n{}'.format(queue,
1164+ message))
1165+ connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
1166+ port=port,
1167+ username=username,
1168+ password=password)
1169+
1170+ # NOTE(beisner): extra debug here re: pika hang potential:
1171+ # https://github.com/pika/pika/issues/297
1172+ # https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw
1173+ self.log.debug('Defining channel...')
1174+ channel = connection.channel()
1175+ self.log.debug('Declaring queue...')
1176+ channel.queue_declare(queue=queue, auto_delete=False, durable=True)
1177+ self.log.debug('Publishing message...')
1178+ channel.basic_publish(exchange='', routing_key=queue, body=message)
1179+ self.log.debug('Closing channel...')
1180+ channel.close()
1181+ self.log.debug('Closing connection...')
1182+ connection.close()
1183+
1184+ def get_amqp_message_by_unit(self, sentry_unit, queue="test",
1185+ username="testuser1",
1186+ password="changeme",
1187+ ssl=False, port=None):
1188+ """Get an amqp message from a rmq juju unit.
1189+
1190+ :param sentry_unit: sentry unit pointer
1191+ :param queue: message queue, default to test
1192+ :param username: amqp user name, default to testuser1
1193+ :param password: amqp user password
1194+ :param ssl: boolean, default to False
1195+ :param port: amqp port, use defaults if None
1196+ :returns: amqp message body as string. Raise if get fails.
1197+ """
1198+ connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
1199+ port=port,
1200+ username=username,
1201+ password=password)
1202+ channel = connection.channel()
1203+ method_frame, _, body = channel.basic_get(queue)
1204+
1205+ if method_frame:
1206+ self.log.debug('Retreived message from {} queue:\n{}'.format(queue,
1207+ body))
1208+ channel.basic_ack(method_frame.delivery_tag)
1209+ channel.close()
1210+ connection.close()
1211+ return body
1212+ else:
1213+ msg = 'No message retrieved.'
1214+ amulet.raise_status(amulet.FAIL, msg)
1215
1216=== modified file 'unit_tests/test_cinder_hooks.py'
1217--- unit_tests/test_cinder_hooks.py 2015-09-01 16:22:10 +0000
1218+++ unit_tests/test_cinder_hooks.py 2015-09-10 09:35:22 +0000
1219@@ -31,6 +31,8 @@
1220
1221 TO_PATCH = [
1222 'check_call',
1223+ 'send_request_if_needed',
1224+ 'is_request_complete',
1225 # cinder_utils
1226 'configure_lvm_storage',
1227 'determine_packages',
1228@@ -572,24 +574,19 @@
1229 m = 'ceph relation incomplete. Peer not ready?'
1230 self.juju_log.assert_called_with(m)
1231
1232- @patch("cinder_hooks.relation_set")
1233- @patch("cinder_hooks.relation_get")
1234- def test_ceph_changed_broker_send_rq(self, mock_relation_get,
1235- mock_relation_set):
1236+ @patch.object(hooks, 'get_ceph_request')
1237+ def test_ceph_changed_broker_send_rq(self, mget_ceph_request):
1238+ mget_ceph_request.return_value = 'cephrq'
1239 self.CONFIGS.complete_contexts.return_value = ['ceph']
1240 self.service_name.return_value = 'cinder'
1241 self.ensure_ceph_keyring.return_value = True
1242+ self.is_request_complete.return_value = False
1243 self.ceph_config_file.return_value = '/var/lib/charm/cinder/ceph.conf'
1244- self.relation_ids.return_value = ['ceph:0']
1245 hooks.hooks.execute(['hooks/ceph-relation-changed'])
1246 self.ensure_ceph_keyring.assert_called_with(service='cinder',
1247 user='cinder',
1248 group='cinder')
1249- req = {'api-version': 1,
1250- 'ops': [{"op": "create-pool", "name": "cinder", "replicas": 3}]}
1251- broker_dict = json.dumps(req)
1252- mock_relation_set.assert_called_with(broker_req=broker_dict,
1253- relation_id='ceph:0')
1254+ self.send_request_if_needed.assert_called_with('cephrq')
1255 for c in [call('/var/lib/charm/cinder/ceph.conf'),
1256 call('/etc/cinder/cinder.conf')]:
1257 self.assertNotIn(c, self.CONFIGS.write.call_args_list)
1258@@ -616,14 +613,12 @@
1259 self.set_ceph_env_variables.assert_called_with(service='cinder')
1260 self.service_restart.assert_called_with('cinder-volume')
1261
1262- @patch("cinder_hooks.relation_get", autospec=True)
1263- def test_ceph_changed_broker_nonzero_rc(self, mock_relation_get):
1264+ def test_ceph_changed_broker_nonzero_rc(self):
1265 self.CONFIGS.complete_contexts.return_value = ['ceph']
1266 self.service_name.return_value = 'cinder'
1267 self.ensure_ceph_keyring.return_value = True
1268 self.ceph_config_file.return_value = '/var/lib/charm/cinder/ceph.conf'
1269- mock_relation_get.return_value = {'broker_rsp':
1270- json.dumps({'exit-code': 1})}
1271+ self.is_request_complete.return_value = False
1272 hooks.hooks.execute(['hooks/ceph-relation-changed'])
1273 self.ensure_ceph_keyring.assert_called_with(service='cinder',
1274 user='cinder',

Subscribers

People subscribed via source and target branches