Merge lp:~chris.macnaughton/charms/trusty/ceph-osd/trunk into lp:~openstack-charmers-archive/charms/trusty/ceph-osd/next
- Trusty Tahr (14.04)
- trunk
- Merge into next
Status: | Needs review |
---|---|
Proposed branch: | lp:~chris.macnaughton/charms/trusty/ceph-osd/trunk |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/ceph-osd/next |
Diff against target: |
1778 lines (+1103/-119) 18 files modified
config.yaml (+2/-1) hooks/ceph_hooks.py (+19/-3) hooks/charmhelpers/cli/__init__.py (+3/-3) hooks/charmhelpers/contrib/charmsupport/nrpe.py (+44/-8) hooks/charmhelpers/contrib/network/ip.py (+5/-3) hooks/charmhelpers/core/hookenv.py (+46/-0) hooks/charmhelpers/core/host.py (+60/-17) hooks/charmhelpers/core/hugepage.py (+10/-1) hooks/charmhelpers/core/kernel.py (+68/-0) hooks/charmhelpers/core/services/helpers.py (+5/-2) hooks/charmhelpers/core/strutils.py (+30/-0) hooks/charmhelpers/core/templating.py (+13/-6) hooks/charmhelpers/fetch/__init__.py (+1/-1) metadata.yaml (+5/-0) tests/charmhelpers/contrib/amulet/deployment.py (+4/-2) tests/charmhelpers/contrib/amulet/utils.py (+284/-62) tests/charmhelpers/contrib/openstack/amulet/deployment.py (+123/-10) tests/charmhelpers/contrib/openstack/amulet/utils.py (+381/-0) |
To merge this branch: | bzr merge lp:~chris.macnaughton/charms/trusty/ceph-osd/trunk |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
James Page | Needs Fixing | ||
Review via email: mp+277135@code.launchpad.net |
Commit message
Description of the change
Add storage hooks support
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #13425 ceph-osd-next for chris.macnaughton mp277135
LINT FAIL: lint-test failed
LINT FAIL: charm-proof failed
LINT Results (max last 2 lines):
make: *** [lint] Error 200
ERROR:root:Make target returned non-zero.
Full lint test output: http://
Build: http://
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #7833 ceph-osd-next for chris.macnaughton mp277135
AMULET FAIL: amulet-test failed
AMULET Results (max last 2 lines):
make: *** [functional_test] Error 124
ERROR:root:Make target returned non-zero.
Full amulet test output: http://
Build: http://
- 56. By Chris MacNaughton
-
add-storage-hooks
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #13062 ceph-osd-next for chris.macnaughton mp277135
UNIT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #14013 ceph-osd-next for chris.macnaughton mp277135
LINT FAIL: lint-test failed
LINT FAIL: charm-proof failed
LINT Results (max last 2 lines):
make: *** [lint] Error 200
ERROR:root:Make target returned non-zero.
Full lint test output: http://
Build: http://
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #7940 ceph-osd-next for chris.macnaughton mp277135
AMULET OK: passed
Build: http://
Ryan Beisner (1chb1n) wrote : | # |
FYI, the lint fail is https:/
James Page (james-page) wrote : | # |
Hi Chris
Please can you update this proposal to add support for the journal device as well - I've just landed that into the ceph charm.
Cheers
James
Chris MacNaughton (chris.macnaughton) wrote : | # |
James, referencing https:/
James Page (james-page) wrote : | # |
Chris - no not that one - specifically the changes that landed into ceph:
storage:
osd-devices:
type: block
multiple:
range: 0-
osd-journal:
type: block
multiple:
range: 0-1
Unmerged revisions
- 56. By Chris MacNaughton
-
add-storage-hooks
- 55. By Chris MacNaughton
-
add storage hooks
- 54. By Chris MacNaughton
-
update charmhelpers
Preview Diff
1 | === modified file 'config.yaml' | |||
2 | --- config.yaml 2015-07-10 14:12:01 +0000 | |||
3 | +++ config.yaml 2015-11-18 20:55:22 +0000 | |||
4 | @@ -6,7 +6,8 @@ | |||
5 | 6 | The devices to format and set up as osd volumes. | 6 | The devices to format and set up as osd volumes. |
6 | 7 | 7 | ||
7 | 8 | These devices are the range of devices that will be checked for and | 8 | These devices are the range of devices that will be checked for and |
9 | 9 | used across all service units. | 9 | used across all service units, in addition to any volumes attached |
10 | 10 | via the --storage flag during deployment | ||
11 | 10 | 11 | ||
12 | 11 | For ceph >= 0.56.6 these can also be directories instead of devices - the | 12 | For ceph >= 0.56.6 these can also be directories instead of devices - the |
13 | 12 | charm assumes anything not starting with /dev is a directory instead. | 13 | charm assumes anything not starting with /dev is a directory instead. |
14 | 13 | 14 | ||
15 | === modified file 'hooks/ceph_hooks.py' | |||
16 | --- hooks/ceph_hooks.py 2015-10-30 02:22:54 +0000 | |||
17 | +++ hooks/ceph_hooks.py 2015-11-18 20:55:22 +0000 | |||
18 | @@ -24,6 +24,8 @@ | |||
19 | 24 | UnregisteredHookError, | 24 | UnregisteredHookError, |
20 | 25 | service_name, | 25 | service_name, |
21 | 26 | status_set, | 26 | status_set, |
22 | 27 | storage_get, | ||
23 | 28 | storage_list | ||
24 | 27 | ) | 29 | ) |
25 | 28 | from charmhelpers.core.host import ( | 30 | from charmhelpers.core.host import ( |
26 | 29 | umount, | 31 | umount, |
27 | @@ -128,7 +130,14 @@ | |||
28 | 128 | ceph.zap_disk(osd_journal) | 130 | ceph.zap_disk(osd_journal) |
29 | 129 | with open(JOURNAL_ZAPPED, 'w') as zapped: | 131 | with open(JOURNAL_ZAPPED, 'w') as zapped: |
30 | 130 | zapped.write('DONE') | 132 | zapped.write('DONE') |
32 | 131 | 133 | storage_changed() | |
33 | 134 | |||
34 | 135 | |||
35 | 136 | @hooks.hook('osd-devices-storage-attached', | ||
36 | 137 | 'osd-fs-storage-attached', | ||
37 | 138 | 'osd-devices-storage-detaching', | ||
38 | 139 | 'osd-fs-storage-detaching') | ||
39 | 140 | def storage_changed(): | ||
40 | 132 | if ceph.is_bootstrapped(): | 141 | if ceph.is_bootstrapped(): |
41 | 133 | log('ceph bootstrapped, rescanning disks') | 142 | log('ceph bootstrapped, rescanning disks') |
42 | 134 | emit_cephconf() | 143 | emit_cephconf() |
43 | @@ -180,9 +189,16 @@ | |||
44 | 180 | 189 | ||
45 | 181 | def get_devices(): | 190 | def get_devices(): |
46 | 182 | if config('osd-devices'): | 191 | if config('osd-devices'): |
48 | 183 | return config('osd-devices').split(' ') | 192 | devices = config('osd-devices').split(' ') |
49 | 184 | else: | 193 | else: |
51 | 185 | return [] | 194 | devices = [] |
52 | 195 | # List storage instances for the 'osd-devices' | ||
53 | 196 | # storegdeclared for this charm too, and add | ||
54 | 197 | # their block device paths to the list. | ||
55 | 198 | storage_ids = storage_list('osd-devices') | ||
56 | 199 | storage_ids.extend(storage_list('osd-fs')) | ||
57 | 200 | devices.extend((storage_get('location', s) for s in storage_ids)) | ||
58 | 201 | return devices | ||
59 | 186 | 202 | ||
60 | 187 | 203 | ||
61 | 188 | @hooks.hook('mon-relation-changed', | 204 | @hooks.hook('mon-relation-changed', |
62 | 189 | 205 | ||
63 | === modified file 'hooks/charmhelpers/cli/__init__.py' | |||
64 | --- hooks/charmhelpers/cli/__init__.py 2015-08-19 13:50:42 +0000 | |||
65 | +++ hooks/charmhelpers/cli/__init__.py 2015-11-18 20:55:22 +0000 | |||
66 | @@ -20,7 +20,7 @@ | |||
67 | 20 | 20 | ||
68 | 21 | from six.moves import zip | 21 | from six.moves import zip |
69 | 22 | 22 | ||
71 | 23 | from charmhelpers.core import unitdata | 23 | import charmhelpers.core.unitdata |
72 | 24 | 24 | ||
73 | 25 | 25 | ||
74 | 26 | class OutputFormatter(object): | 26 | class OutputFormatter(object): |
75 | @@ -163,8 +163,8 @@ | |||
76 | 163 | if getattr(arguments.func, '_cli_no_output', False): | 163 | if getattr(arguments.func, '_cli_no_output', False): |
77 | 164 | output = '' | 164 | output = '' |
78 | 165 | self.formatter.format_output(output, arguments.format) | 165 | self.formatter.format_output(output, arguments.format) |
81 | 166 | if unitdata._KV: | 166 | if charmhelpers.core.unitdata._KV: |
82 | 167 | unitdata._KV.flush() | 167 | charmhelpers.core.unitdata._KV.flush() |
83 | 168 | 168 | ||
84 | 169 | 169 | ||
85 | 170 | cmdline = CommandLine() | 170 | cmdline = CommandLine() |
86 | 171 | 171 | ||
87 | === modified file 'hooks/charmhelpers/contrib/charmsupport/nrpe.py' | |||
88 | --- hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-06-17 16:59:15 +0000 | |||
89 | +++ hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-11-18 20:55:22 +0000 | |||
90 | @@ -148,6 +148,13 @@ | |||
91 | 148 | self.description = description | 148 | self.description = description |
92 | 149 | self.check_cmd = self._locate_cmd(check_cmd) | 149 | self.check_cmd = self._locate_cmd(check_cmd) |
93 | 150 | 150 | ||
94 | 151 | def _get_check_filename(self): | ||
95 | 152 | return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command)) | ||
96 | 153 | |||
97 | 154 | def _get_service_filename(self, hostname): | ||
98 | 155 | return os.path.join(NRPE.nagios_exportdir, | ||
99 | 156 | 'service__{}_{}.cfg'.format(hostname, self.command)) | ||
100 | 157 | |||
101 | 151 | def _locate_cmd(self, check_cmd): | 158 | def _locate_cmd(self, check_cmd): |
102 | 152 | search_path = ( | 159 | search_path = ( |
103 | 153 | '/usr/lib/nagios/plugins', | 160 | '/usr/lib/nagios/plugins', |
104 | @@ -163,9 +170,21 @@ | |||
105 | 163 | log('Check command not found: {}'.format(parts[0])) | 170 | log('Check command not found: {}'.format(parts[0])) |
106 | 164 | return '' | 171 | return '' |
107 | 165 | 172 | ||
108 | 173 | def _remove_service_files(self): | ||
109 | 174 | if not os.path.exists(NRPE.nagios_exportdir): | ||
110 | 175 | return | ||
111 | 176 | for f in os.listdir(NRPE.nagios_exportdir): | ||
112 | 177 | if f.endswith('_{}.cfg'.format(self.command)): | ||
113 | 178 | os.remove(os.path.join(NRPE.nagios_exportdir, f)) | ||
114 | 179 | |||
115 | 180 | def remove(self, hostname): | ||
116 | 181 | nrpe_check_file = self._get_check_filename() | ||
117 | 182 | if os.path.exists(nrpe_check_file): | ||
118 | 183 | os.remove(nrpe_check_file) | ||
119 | 184 | self._remove_service_files() | ||
120 | 185 | |||
121 | 166 | def write(self, nagios_context, hostname, nagios_servicegroups): | 186 | def write(self, nagios_context, hostname, nagios_servicegroups): |
124 | 167 | nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( | 187 | nrpe_check_file = self._get_check_filename() |
123 | 168 | self.command) | ||
125 | 169 | with open(nrpe_check_file, 'w') as nrpe_check_config: | 188 | with open(nrpe_check_file, 'w') as nrpe_check_config: |
126 | 170 | nrpe_check_config.write("# check {}\n".format(self.shortname)) | 189 | nrpe_check_config.write("# check {}\n".format(self.shortname)) |
127 | 171 | nrpe_check_config.write("command[{}]={}\n".format( | 190 | nrpe_check_config.write("command[{}]={}\n".format( |
128 | @@ -180,9 +199,7 @@ | |||
129 | 180 | 199 | ||
130 | 181 | def write_service_config(self, nagios_context, hostname, | 200 | def write_service_config(self, nagios_context, hostname, |
131 | 182 | nagios_servicegroups): | 201 | nagios_servicegroups): |
135 | 183 | for f in os.listdir(NRPE.nagios_exportdir): | 202 | self._remove_service_files() |
133 | 184 | if re.search('.*{}.cfg'.format(self.command), f): | ||
134 | 185 | os.remove(os.path.join(NRPE.nagios_exportdir, f)) | ||
136 | 186 | 203 | ||
137 | 187 | templ_vars = { | 204 | templ_vars = { |
138 | 188 | 'nagios_hostname': hostname, | 205 | 'nagios_hostname': hostname, |
139 | @@ -192,8 +209,7 @@ | |||
140 | 192 | 'command': self.command, | 209 | 'command': self.command, |
141 | 193 | } | 210 | } |
142 | 194 | nrpe_service_text = Check.service_template.format(**templ_vars) | 211 | nrpe_service_text = Check.service_template.format(**templ_vars) |
145 | 195 | nrpe_service_file = '{}/service__{}_{}.cfg'.format( | 212 | nrpe_service_file = self._get_service_filename(hostname) |
144 | 196 | NRPE.nagios_exportdir, hostname, self.command) | ||
146 | 197 | with open(nrpe_service_file, 'w') as nrpe_service_config: | 213 | with open(nrpe_service_file, 'w') as nrpe_service_config: |
147 | 198 | nrpe_service_config.write(str(nrpe_service_text)) | 214 | nrpe_service_config.write(str(nrpe_service_text)) |
148 | 199 | 215 | ||
149 | @@ -218,12 +234,32 @@ | |||
150 | 218 | if hostname: | 234 | if hostname: |
151 | 219 | self.hostname = hostname | 235 | self.hostname = hostname |
152 | 220 | else: | 236 | else: |
154 | 221 | self.hostname = "{}-{}".format(self.nagios_context, self.unit_name) | 237 | nagios_hostname = get_nagios_hostname() |
155 | 238 | if nagios_hostname: | ||
156 | 239 | self.hostname = nagios_hostname | ||
157 | 240 | else: | ||
158 | 241 | self.hostname = "{}-{}".format(self.nagios_context, self.unit_name) | ||
159 | 222 | self.checks = [] | 242 | self.checks = [] |
160 | 223 | 243 | ||
161 | 224 | def add_check(self, *args, **kwargs): | 244 | def add_check(self, *args, **kwargs): |
162 | 225 | self.checks.append(Check(*args, **kwargs)) | 245 | self.checks.append(Check(*args, **kwargs)) |
163 | 226 | 246 | ||
164 | 247 | def remove_check(self, *args, **kwargs): | ||
165 | 248 | if kwargs.get('shortname') is None: | ||
166 | 249 | raise ValueError('shortname of check must be specified') | ||
167 | 250 | |||
168 | 251 | # Use sensible defaults if they're not specified - these are not | ||
169 | 252 | # actually used during removal, but they're required for constructing | ||
170 | 253 | # the Check object; check_disk is chosen because it's part of the | ||
171 | 254 | # nagios-plugins-basic package. | ||
172 | 255 | if kwargs.get('check_cmd') is None: | ||
173 | 256 | kwargs['check_cmd'] = 'check_disk' | ||
174 | 257 | if kwargs.get('description') is None: | ||
175 | 258 | kwargs['description'] = '' | ||
176 | 259 | |||
177 | 260 | check = Check(*args, **kwargs) | ||
178 | 261 | check.remove(self.hostname) | ||
179 | 262 | |||
180 | 227 | def write(self): | 263 | def write(self): |
181 | 228 | try: | 264 | try: |
182 | 229 | nagios_uid = pwd.getpwnam('nagios').pw_uid | 265 | nagios_uid = pwd.getpwnam('nagios').pw_uid |
183 | 230 | 266 | ||
184 | === modified file 'hooks/charmhelpers/contrib/network/ip.py' | |||
185 | --- hooks/charmhelpers/contrib/network/ip.py 2015-09-03 09:42:18 +0000 | |||
186 | +++ hooks/charmhelpers/contrib/network/ip.py 2015-11-18 20:55:22 +0000 | |||
187 | @@ -23,7 +23,7 @@ | |||
188 | 23 | from functools import partial | 23 | from functools import partial |
189 | 24 | 24 | ||
190 | 25 | from charmhelpers.core.hookenv import unit_get | 25 | from charmhelpers.core.hookenv import unit_get |
192 | 26 | from charmhelpers.fetch import apt_install | 26 | from charmhelpers.fetch import apt_install, apt_update |
193 | 27 | from charmhelpers.core.hookenv import ( | 27 | from charmhelpers.core.hookenv import ( |
194 | 28 | log, | 28 | log, |
195 | 29 | WARNING, | 29 | WARNING, |
196 | @@ -32,13 +32,15 @@ | |||
197 | 32 | try: | 32 | try: |
198 | 33 | import netifaces | 33 | import netifaces |
199 | 34 | except ImportError: | 34 | except ImportError: |
201 | 35 | apt_install('python-netifaces') | 35 | apt_update(fatal=True) |
202 | 36 | apt_install('python-netifaces', fatal=True) | ||
203 | 36 | import netifaces | 37 | import netifaces |
204 | 37 | 38 | ||
205 | 38 | try: | 39 | try: |
206 | 39 | import netaddr | 40 | import netaddr |
207 | 40 | except ImportError: | 41 | except ImportError: |
209 | 41 | apt_install('python-netaddr') | 42 | apt_update(fatal=True) |
210 | 43 | apt_install('python-netaddr', fatal=True) | ||
211 | 42 | import netaddr | 44 | import netaddr |
212 | 43 | 45 | ||
213 | 44 | 46 | ||
214 | 45 | 47 | ||
215 | === modified file 'hooks/charmhelpers/core/hookenv.py' | |||
216 | --- hooks/charmhelpers/core/hookenv.py 2015-09-03 09:42:18 +0000 | |||
217 | +++ hooks/charmhelpers/core/hookenv.py 2015-11-18 20:55:22 +0000 | |||
218 | @@ -491,6 +491,19 @@ | |||
219 | 491 | 491 | ||
220 | 492 | 492 | ||
221 | 493 | @cached | 493 | @cached |
222 | 494 | def peer_relation_id(): | ||
223 | 495 | '''Get a peer relation id if a peer relation has been joined, else None.''' | ||
224 | 496 | md = metadata() | ||
225 | 497 | section = md.get('peers') | ||
226 | 498 | if section: | ||
227 | 499 | for key in section: | ||
228 | 500 | relids = relation_ids(key) | ||
229 | 501 | if relids: | ||
230 | 502 | return relids[0] | ||
231 | 503 | return None | ||
232 | 504 | |||
233 | 505 | |||
234 | 506 | @cached | ||
235 | 494 | def relation_to_interface(relation_name): | 507 | def relation_to_interface(relation_name): |
236 | 495 | """ | 508 | """ |
237 | 496 | Given the name of a relation, return the interface that relation uses. | 509 | Given the name of a relation, return the interface that relation uses. |
238 | @@ -623,6 +636,38 @@ | |||
239 | 623 | return unit_get('private-address') | 636 | return unit_get('private-address') |
240 | 624 | 637 | ||
241 | 625 | 638 | ||
242 | 639 | @cached | ||
243 | 640 | def storage_get(attribute="", storage_id=""): | ||
244 | 641 | """Get storage attributes""" | ||
245 | 642 | _args = ['storage-get', '--format=json'] | ||
246 | 643 | if storage_id: | ||
247 | 644 | _args.extend(('-s', storage_id)) | ||
248 | 645 | if attribute: | ||
249 | 646 | _args.append(attribute) | ||
250 | 647 | try: | ||
251 | 648 | return json.loads(subprocess.check_output(_args).decode('UTF-8')) | ||
252 | 649 | except ValueError: | ||
253 | 650 | return None | ||
254 | 651 | |||
255 | 652 | |||
256 | 653 | @cached | ||
257 | 654 | def storage_list(storage_name=""): | ||
258 | 655 | """List the storage IDs for the unit""" | ||
259 | 656 | _args = ['storage-list', '--format=json'] | ||
260 | 657 | if storage_name: | ||
261 | 658 | _args.append(storage_name) | ||
262 | 659 | try: | ||
263 | 660 | return json.loads(subprocess.check_output(_args).decode('UTF-8')) | ||
264 | 661 | except ValueError: | ||
265 | 662 | return None | ||
266 | 663 | except OSError as e: | ||
267 | 664 | import errno | ||
268 | 665 | if e.errno == errno.ENOENT: | ||
269 | 666 | # storage-list does not exist | ||
270 | 667 | return [] | ||
271 | 668 | raise | ||
272 | 669 | |||
273 | 670 | |||
274 | 626 | class UnregisteredHookError(Exception): | 671 | class UnregisteredHookError(Exception): |
275 | 627 | """Raised when an undefined hook is called""" | 672 | """Raised when an undefined hook is called""" |
276 | 628 | pass | 673 | pass |
277 | @@ -788,6 +833,7 @@ | |||
278 | 788 | 833 | ||
279 | 789 | def translate_exc(from_exc, to_exc): | 834 | def translate_exc(from_exc, to_exc): |
280 | 790 | def inner_translate_exc1(f): | 835 | def inner_translate_exc1(f): |
281 | 836 | @wraps(f) | ||
282 | 791 | def inner_translate_exc2(*args, **kwargs): | 837 | def inner_translate_exc2(*args, **kwargs): |
283 | 792 | try: | 838 | try: |
284 | 793 | return f(*args, **kwargs) | 839 | return f(*args, **kwargs) |
285 | 794 | 840 | ||
286 | === modified file 'hooks/charmhelpers/core/host.py' | |||
287 | --- hooks/charmhelpers/core/host.py 2015-08-19 13:50:42 +0000 | |||
288 | +++ hooks/charmhelpers/core/host.py 2015-11-18 20:55:22 +0000 | |||
289 | @@ -63,32 +63,48 @@ | |||
290 | 63 | return service_result | 63 | return service_result |
291 | 64 | 64 | ||
292 | 65 | 65 | ||
294 | 66 | def service_pause(service_name, init_dir=None): | 66 | def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"): |
295 | 67 | """Pause a system service. | 67 | """Pause a system service. |
296 | 68 | 68 | ||
297 | 69 | Stop it, and prevent it from starting again at boot.""" | 69 | Stop it, and prevent it from starting again at boot.""" |
298 | 70 | if init_dir is None: | ||
299 | 71 | init_dir = "/etc/init" | ||
300 | 72 | stopped = service_stop(service_name) | 70 | stopped = service_stop(service_name) |
306 | 73 | # XXX: Support systemd too | 71 | upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) |
307 | 74 | override_path = os.path.join( | 72 | sysv_file = os.path.join(initd_dir, service_name) |
308 | 75 | init_dir, '{}.override'.format(service_name)) | 73 | if os.path.exists(upstart_file): |
309 | 76 | with open(override_path, 'w') as fh: | 74 | override_path = os.path.join( |
310 | 77 | fh.write("manual\n") | 75 | init_dir, '{}.override'.format(service_name)) |
311 | 76 | with open(override_path, 'w') as fh: | ||
312 | 77 | fh.write("manual\n") | ||
313 | 78 | elif os.path.exists(sysv_file): | ||
314 | 79 | subprocess.check_call(["update-rc.d", service_name, "disable"]) | ||
315 | 80 | else: | ||
316 | 81 | # XXX: Support SystemD too | ||
317 | 82 | raise ValueError( | ||
318 | 83 | "Unable to detect {0} as either Upstart {1} or SysV {2}".format( | ||
319 | 84 | service_name, upstart_file, sysv_file)) | ||
320 | 78 | return stopped | 85 | return stopped |
321 | 79 | 86 | ||
322 | 80 | 87 | ||
324 | 81 | def service_resume(service_name, init_dir=None): | 88 | def service_resume(service_name, init_dir="/etc/init", |
325 | 89 | initd_dir="/etc/init.d"): | ||
326 | 82 | """Resume a system service. | 90 | """Resume a system service. |
327 | 83 | 91 | ||
328 | 84 | Reenable starting again at boot. Start the service""" | 92 | Reenable starting again at boot. Start the service""" |
336 | 85 | # XXX: Support systemd too | 93 | upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) |
337 | 86 | if init_dir is None: | 94 | sysv_file = os.path.join(initd_dir, service_name) |
338 | 87 | init_dir = "/etc/init" | 95 | if os.path.exists(upstart_file): |
339 | 88 | override_path = os.path.join( | 96 | override_path = os.path.join( |
340 | 89 | init_dir, '{}.override'.format(service_name)) | 97 | init_dir, '{}.override'.format(service_name)) |
341 | 90 | if os.path.exists(override_path): | 98 | if os.path.exists(override_path): |
342 | 91 | os.unlink(override_path) | 99 | os.unlink(override_path) |
343 | 100 | elif os.path.exists(sysv_file): | ||
344 | 101 | subprocess.check_call(["update-rc.d", service_name, "enable"]) | ||
345 | 102 | else: | ||
346 | 103 | # XXX: Support SystemD too | ||
347 | 104 | raise ValueError( | ||
348 | 105 | "Unable to detect {0} as either Upstart {1} or SysV {2}".format( | ||
349 | 106 | service_name, upstart_file, sysv_file)) | ||
350 | 107 | |||
351 | 92 | started = service_start(service_name) | 108 | started = service_start(service_name) |
352 | 93 | return started | 109 | return started |
353 | 94 | 110 | ||
354 | @@ -550,7 +566,14 @@ | |||
355 | 550 | os.chdir(cur) | 566 | os.chdir(cur) |
356 | 551 | 567 | ||
357 | 552 | 568 | ||
359 | 553 | def chownr(path, owner, group, follow_links=True): | 569 | def chownr(path, owner, group, follow_links=True, chowntopdir=False): |
360 | 570 | """ | ||
361 | 571 | Recursively change user and group ownership of files and directories | ||
362 | 572 | in given path. Doesn't chown path itself by default, only its children. | ||
363 | 573 | |||
364 | 574 | :param bool follow_links: Also Chown links if True | ||
365 | 575 | :param bool chowntopdir: Also chown path itself if True | ||
366 | 576 | """ | ||
367 | 554 | uid = pwd.getpwnam(owner).pw_uid | 577 | uid = pwd.getpwnam(owner).pw_uid |
368 | 555 | gid = grp.getgrnam(group).gr_gid | 578 | gid = grp.getgrnam(group).gr_gid |
369 | 556 | if follow_links: | 579 | if follow_links: |
370 | @@ -558,6 +581,10 @@ | |||
371 | 558 | else: | 581 | else: |
372 | 559 | chown = os.lchown | 582 | chown = os.lchown |
373 | 560 | 583 | ||
374 | 584 | if chowntopdir: | ||
375 | 585 | broken_symlink = os.path.lexists(path) and not os.path.exists(path) | ||
376 | 586 | if not broken_symlink: | ||
377 | 587 | chown(path, uid, gid) | ||
378 | 561 | for root, dirs, files in os.walk(path): | 588 | for root, dirs, files in os.walk(path): |
379 | 562 | for name in dirs + files: | 589 | for name in dirs + files: |
380 | 563 | full = os.path.join(root, name) | 590 | full = os.path.join(root, name) |
381 | @@ -568,3 +595,19 @@ | |||
382 | 568 | 595 | ||
383 | 569 | def lchownr(path, owner, group): | 596 | def lchownr(path, owner, group): |
384 | 570 | chownr(path, owner, group, follow_links=False) | 597 | chownr(path, owner, group, follow_links=False) |
385 | 598 | |||
386 | 599 | |||
387 | 600 | def get_total_ram(): | ||
388 | 601 | '''The total amount of system RAM in bytes. | ||
389 | 602 | |||
390 | 603 | This is what is reported by the OS, and may be overcommitted when | ||
391 | 604 | there are multiple containers hosted on the same machine. | ||
392 | 605 | ''' | ||
393 | 606 | with open('/proc/meminfo', 'r') as f: | ||
394 | 607 | for line in f.readlines(): | ||
395 | 608 | if line: | ||
396 | 609 | key, value, unit = line.split() | ||
397 | 610 | if key == 'MemTotal:': | ||
398 | 611 | assert unit == 'kB', 'Unknown unit' | ||
399 | 612 | return int(value) * 1024 # Classic, not KiB. | ||
400 | 613 | raise NotImplementedError() | ||
401 | 571 | 614 | ||
402 | === modified file 'hooks/charmhelpers/core/hugepage.py' | |||
403 | --- hooks/charmhelpers/core/hugepage.py 2015-08-19 13:50:42 +0000 | |||
404 | +++ hooks/charmhelpers/core/hugepage.py 2015-11-18 20:55:22 +0000 | |||
405 | @@ -25,11 +25,13 @@ | |||
406 | 25 | fstab_mount, | 25 | fstab_mount, |
407 | 26 | mkdir, | 26 | mkdir, |
408 | 27 | ) | 27 | ) |
409 | 28 | from charmhelpers.core.strutils import bytes_from_string | ||
410 | 29 | from subprocess import check_output | ||
411 | 28 | 30 | ||
412 | 29 | 31 | ||
413 | 30 | def hugepage_support(user, group='hugetlb', nr_hugepages=256, | 32 | def hugepage_support(user, group='hugetlb', nr_hugepages=256, |
414 | 31 | max_map_count=65536, mnt_point='/run/hugepages/kvm', | 33 | max_map_count=65536, mnt_point='/run/hugepages/kvm', |
416 | 32 | pagesize='2MB', mount=True): | 34 | pagesize='2MB', mount=True, set_shmmax=False): |
417 | 33 | """Enable hugepages on system. | 35 | """Enable hugepages on system. |
418 | 34 | 36 | ||
419 | 35 | Args: | 37 | Args: |
420 | @@ -44,11 +46,18 @@ | |||
421 | 44 | group_info = add_group(group) | 46 | group_info = add_group(group) |
422 | 45 | gid = group_info.gr_gid | 47 | gid = group_info.gr_gid |
423 | 46 | add_user_to_group(user, group) | 48 | add_user_to_group(user, group) |
424 | 49 | if max_map_count < 2 * nr_hugepages: | ||
425 | 50 | max_map_count = 2 * nr_hugepages | ||
426 | 47 | sysctl_settings = { | 51 | sysctl_settings = { |
427 | 48 | 'vm.nr_hugepages': nr_hugepages, | 52 | 'vm.nr_hugepages': nr_hugepages, |
428 | 49 | 'vm.max_map_count': max_map_count, | 53 | 'vm.max_map_count': max_map_count, |
429 | 50 | 'vm.hugetlb_shm_group': gid, | 54 | 'vm.hugetlb_shm_group': gid, |
430 | 51 | } | 55 | } |
431 | 56 | if set_shmmax: | ||
432 | 57 | shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax'])) | ||
433 | 58 | shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages | ||
434 | 59 | if shmmax_minsize > shmmax_current: | ||
435 | 60 | sysctl_settings['kernel.shmmax'] = shmmax_minsize | ||
436 | 52 | sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf') | 61 | sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf') |
437 | 53 | mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False) | 62 | mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False) |
438 | 54 | lfstab = fstab.Fstab() | 63 | lfstab = fstab.Fstab() |
439 | 55 | 64 | ||
440 | === added file 'hooks/charmhelpers/core/kernel.py' | |||
441 | --- hooks/charmhelpers/core/kernel.py 1970-01-01 00:00:00 +0000 | |||
442 | +++ hooks/charmhelpers/core/kernel.py 2015-11-18 20:55:22 +0000 | |||
443 | @@ -0,0 +1,68 @@ | |||
444 | 1 | #!/usr/bin/env python | ||
445 | 2 | # -*- coding: utf-8 -*- | ||
446 | 3 | |||
447 | 4 | # Copyright 2014-2015 Canonical Limited. | ||
448 | 5 | # | ||
449 | 6 | # This file is part of charm-helpers. | ||
450 | 7 | # | ||
451 | 8 | # charm-helpers is free software: you can redistribute it and/or modify | ||
452 | 9 | # it under the terms of the GNU Lesser General Public License version 3 as | ||
453 | 10 | # published by the Free Software Foundation. | ||
454 | 11 | # | ||
455 | 12 | # charm-helpers is distributed in the hope that it will be useful, | ||
456 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
457 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
458 | 15 | # GNU Lesser General Public License for more details. | ||
459 | 16 | # | ||
460 | 17 | # You should have received a copy of the GNU Lesser General Public License | ||
461 | 18 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. | ||
462 | 19 | |||
463 | 20 | __author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>" | ||
464 | 21 | |||
465 | 22 | from charmhelpers.core.hookenv import ( | ||
466 | 23 | log, | ||
467 | 24 | INFO | ||
468 | 25 | ) | ||
469 | 26 | |||
470 | 27 | from subprocess import check_call, check_output | ||
471 | 28 | import re | ||
472 | 29 | |||
473 | 30 | |||
474 | 31 | def modprobe(module, persist=True): | ||
475 | 32 | """Load a kernel module and configure for auto-load on reboot.""" | ||
476 | 33 | cmd = ['modprobe', module] | ||
477 | 34 | |||
478 | 35 | log('Loading kernel module %s' % module, level=INFO) | ||
479 | 36 | |||
480 | 37 | check_call(cmd) | ||
481 | 38 | if persist: | ||
482 | 39 | with open('/etc/modules', 'r+') as modules: | ||
483 | 40 | if module not in modules.read(): | ||
484 | 41 | modules.write(module) | ||
485 | 42 | |||
486 | 43 | |||
487 | 44 | def rmmod(module, force=False): | ||
488 | 45 | """Remove a module from the linux kernel""" | ||
489 | 46 | cmd = ['rmmod'] | ||
490 | 47 | if force: | ||
491 | 48 | cmd.append('-f') | ||
492 | 49 | cmd.append(module) | ||
493 | 50 | log('Removing kernel module %s' % module, level=INFO) | ||
494 | 51 | return check_call(cmd) | ||
495 | 52 | |||
496 | 53 | |||
497 | 54 | def lsmod(): | ||
498 | 55 | """Shows what kernel modules are currently loaded""" | ||
499 | 56 | return check_output(['lsmod'], | ||
500 | 57 | universal_newlines=True) | ||
501 | 58 | |||
502 | 59 | |||
503 | 60 | def is_module_loaded(module): | ||
504 | 61 | """Checks if a kernel module is already loaded""" | ||
505 | 62 | matches = re.findall('^%s[ ]+' % module, lsmod(), re.M) | ||
506 | 63 | return len(matches) > 0 | ||
507 | 64 | |||
508 | 65 | |||
509 | 66 | def update_initramfs(version='all'): | ||
510 | 67 | """Updates an initramfs image""" | ||
511 | 68 | return check_call(["update-initramfs", "-k", version, "-u"]) | ||
512 | 0 | 69 | ||
513 | === modified file 'hooks/charmhelpers/core/services/helpers.py' | |||
514 | --- hooks/charmhelpers/core/services/helpers.py 2015-08-19 13:50:42 +0000 | |||
515 | +++ hooks/charmhelpers/core/services/helpers.py 2015-11-18 20:55:22 +0000 | |||
516 | @@ -249,16 +249,18 @@ | |||
517 | 249 | :param int perms: The permissions of the rendered file | 249 | :param int perms: The permissions of the rendered file |
518 | 250 | :param partial on_change_action: functools partial to be executed when | 250 | :param partial on_change_action: functools partial to be executed when |
519 | 251 | rendered file changes | 251 | rendered file changes |
520 | 252 | :param jinja2 loader template_loader: A jinja2 template loader | ||
521 | 252 | """ | 253 | """ |
522 | 253 | def __init__(self, source, target, | 254 | def __init__(self, source, target, |
523 | 254 | owner='root', group='root', perms=0o444, | 255 | owner='root', group='root', perms=0o444, |
525 | 255 | on_change_action=None): | 256 | on_change_action=None, template_loader=None): |
526 | 256 | self.source = source | 257 | self.source = source |
527 | 257 | self.target = target | 258 | self.target = target |
528 | 258 | self.owner = owner | 259 | self.owner = owner |
529 | 259 | self.group = group | 260 | self.group = group |
530 | 260 | self.perms = perms | 261 | self.perms = perms |
531 | 261 | self.on_change_action = on_change_action | 262 | self.on_change_action = on_change_action |
532 | 263 | self.template_loader = template_loader | ||
533 | 262 | 264 | ||
534 | 263 | def __call__(self, manager, service_name, event_name): | 265 | def __call__(self, manager, service_name, event_name): |
535 | 264 | pre_checksum = '' | 266 | pre_checksum = '' |
536 | @@ -269,7 +271,8 @@ | |||
537 | 269 | for ctx in service.get('required_data', []): | 271 | for ctx in service.get('required_data', []): |
538 | 270 | context.update(ctx) | 272 | context.update(ctx) |
539 | 271 | templating.render(self.source, self.target, context, | 273 | templating.render(self.source, self.target, context, |
541 | 272 | self.owner, self.group, self.perms) | 274 | self.owner, self.group, self.perms, |
542 | 275 | template_loader=self.template_loader) | ||
543 | 273 | if self.on_change_action: | 276 | if self.on_change_action: |
544 | 274 | if pre_checksum == host.file_hash(self.target): | 277 | if pre_checksum == host.file_hash(self.target): |
545 | 275 | hookenv.log( | 278 | hookenv.log( |
546 | 276 | 279 | ||
547 | === modified file 'hooks/charmhelpers/core/strutils.py' | |||
548 | --- hooks/charmhelpers/core/strutils.py 2015-04-16 21:32:48 +0000 | |||
549 | +++ hooks/charmhelpers/core/strutils.py 2015-11-18 20:55:22 +0000 | |||
550 | @@ -18,6 +18,7 @@ | |||
551 | 18 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. | 18 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
552 | 19 | 19 | ||
553 | 20 | import six | 20 | import six |
554 | 21 | import re | ||
555 | 21 | 22 | ||
556 | 22 | 23 | ||
557 | 23 | def bool_from_string(value): | 24 | def bool_from_string(value): |
558 | @@ -40,3 +41,32 @@ | |||
559 | 40 | 41 | ||
560 | 41 | msg = "Unable to interpret string value '%s' as boolean" % (value) | 42 | msg = "Unable to interpret string value '%s' as boolean" % (value) |
561 | 42 | raise ValueError(msg) | 43 | raise ValueError(msg) |
562 | 44 | |||
563 | 45 | |||
564 | 46 | def bytes_from_string(value): | ||
565 | 47 | """Interpret human readable string value as bytes. | ||
566 | 48 | |||
567 | 49 | Returns int | ||
568 | 50 | """ | ||
569 | 51 | BYTE_POWER = { | ||
570 | 52 | 'K': 1, | ||
571 | 53 | 'KB': 1, | ||
572 | 54 | 'M': 2, | ||
573 | 55 | 'MB': 2, | ||
574 | 56 | 'G': 3, | ||
575 | 57 | 'GB': 3, | ||
576 | 58 | 'T': 4, | ||
577 | 59 | 'TB': 4, | ||
578 | 60 | 'P': 5, | ||
579 | 61 | 'PB': 5, | ||
580 | 62 | } | ||
581 | 63 | if isinstance(value, six.string_types): | ||
582 | 64 | value = six.text_type(value) | ||
583 | 65 | else: | ||
584 | 66 | msg = "Unable to interpret non-string value '%s' as boolean" % (value) | ||
585 | 67 | raise ValueError(msg) | ||
586 | 68 | matches = re.match("([0-9]+)([a-zA-Z]+)", value) | ||
587 | 69 | if not matches: | ||
588 | 70 | msg = "Unable to interpret string value '%s' as bytes" % (value) | ||
589 | 71 | raise ValueError(msg) | ||
590 | 72 | return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) | ||
591 | 43 | 73 | ||
592 | === modified file 'hooks/charmhelpers/core/templating.py' | |||
593 | --- hooks/charmhelpers/core/templating.py 2015-02-26 13:37:18 +0000 | |||
594 | +++ hooks/charmhelpers/core/templating.py 2015-11-18 20:55:22 +0000 | |||
595 | @@ -21,7 +21,7 @@ | |||
596 | 21 | 21 | ||
597 | 22 | 22 | ||
598 | 23 | def render(source, target, context, owner='root', group='root', | 23 | def render(source, target, context, owner='root', group='root', |
600 | 24 | perms=0o444, templates_dir=None, encoding='UTF-8'): | 24 | perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None): |
601 | 25 | """ | 25 | """ |
602 | 26 | Render a template. | 26 | Render a template. |
603 | 27 | 27 | ||
604 | @@ -52,17 +52,24 @@ | |||
605 | 52 | apt_install('python-jinja2', fatal=True) | 52 | apt_install('python-jinja2', fatal=True) |
606 | 53 | from jinja2 import FileSystemLoader, Environment, exceptions | 53 | from jinja2 import FileSystemLoader, Environment, exceptions |
607 | 54 | 54 | ||
611 | 55 | if templates_dir is None: | 55 | if template_loader: |
612 | 56 | templates_dir = os.path.join(hookenv.charm_dir(), 'templates') | 56 | template_env = Environment(loader=template_loader) |
613 | 57 | loader = Environment(loader=FileSystemLoader(templates_dir)) | 57 | else: |
614 | 58 | if templates_dir is None: | ||
615 | 59 | templates_dir = os.path.join(hookenv.charm_dir(), 'templates') | ||
616 | 60 | template_env = Environment(loader=FileSystemLoader(templates_dir)) | ||
617 | 58 | try: | 61 | try: |
618 | 59 | source = source | 62 | source = source |
620 | 60 | template = loader.get_template(source) | 63 | template = template_env.get_template(source) |
621 | 61 | except exceptions.TemplateNotFound as e: | 64 | except exceptions.TemplateNotFound as e: |
622 | 62 | hookenv.log('Could not load template %s from %s.' % | 65 | hookenv.log('Could not load template %s from %s.' % |
623 | 63 | (source, templates_dir), | 66 | (source, templates_dir), |
624 | 64 | level=hookenv.ERROR) | 67 | level=hookenv.ERROR) |
625 | 65 | raise e | 68 | raise e |
626 | 66 | content = template.render(context) | 69 | content = template.render(context) |
628 | 67 | host.mkdir(os.path.dirname(target), owner, group, perms=0o755) | 70 | target_dir = os.path.dirname(target) |
629 | 71 | if not os.path.exists(target_dir): | ||
630 | 72 | # This is a terrible default directory permission, as the file | ||
631 | 73 | # or its siblings will often contain secrets. | ||
632 | 74 | host.mkdir(os.path.dirname(target), owner, group, perms=0o755) | ||
633 | 68 | host.write_file(target, content.encode(encoding), owner, group, perms) | 75 | host.write_file(target, content.encode(encoding), owner, group, perms) |
634 | 69 | 76 | ||
635 | === modified file 'hooks/charmhelpers/fetch/__init__.py' | |||
636 | --- hooks/charmhelpers/fetch/__init__.py 2015-08-19 13:50:42 +0000 | |||
637 | +++ hooks/charmhelpers/fetch/__init__.py 2015-11-18 20:55:22 +0000 | |||
638 | @@ -225,12 +225,12 @@ | |||
639 | 225 | 225 | ||
640 | 226 | def apt_mark(packages, mark, fatal=False): | 226 | def apt_mark(packages, mark, fatal=False): |
641 | 227 | """Flag one or more packages using apt-mark""" | 227 | """Flag one or more packages using apt-mark""" |
642 | 228 | log("Marking {} as {}".format(packages, mark)) | ||
643 | 228 | cmd = ['apt-mark', mark] | 229 | cmd = ['apt-mark', mark] |
644 | 229 | if isinstance(packages, six.string_types): | 230 | if isinstance(packages, six.string_types): |
645 | 230 | cmd.append(packages) | 231 | cmd.append(packages) |
646 | 231 | else: | 232 | else: |
647 | 232 | cmd.extend(packages) | 233 | cmd.extend(packages) |
648 | 233 | log("Holding {}".format(packages)) | ||
649 | 234 | 234 | ||
650 | 235 | if fatal: | 235 | if fatal: |
651 | 236 | subprocess.check_call(cmd, universal_newlines=True) | 236 | subprocess.check_call(cmd, universal_newlines=True) |
652 | 237 | 237 | ||
653 | === added symlink 'hooks/osd-devices-storage-attached' | |||
654 | === target is u'./ceph_hooks.py' | |||
655 | === added symlink 'hooks/osd-devices-storage-detaching' | |||
656 | === target is u'./ceph_hooks.py' | |||
657 | === modified file 'metadata.yaml' | |||
658 | --- metadata.yaml 2015-11-18 10:30:34 +0000 | |||
659 | +++ metadata.yaml 2015-11-18 20:55:22 +0000 | |||
660 | @@ -19,3 +19,8 @@ | |||
661 | 19 | requires: | 19 | requires: |
662 | 20 | mon: | 20 | mon: |
663 | 21 | interface: ceph-osd | 21 | interface: ceph-osd |
664 | 22 | storage: | ||
665 | 23 | osd-devices: | ||
666 | 24 | type: block | ||
667 | 25 | multiple: | ||
668 | 26 | range: 0+ | ||
669 | 22 | \ No newline at end of file | 27 | \ No newline at end of file |
670 | 23 | 28 | ||
671 | === modified file 'tests/charmhelpers/contrib/amulet/deployment.py' | |||
672 | --- tests/charmhelpers/contrib/amulet/deployment.py 2015-01-26 11:51:28 +0000 | |||
673 | +++ tests/charmhelpers/contrib/amulet/deployment.py 2015-11-18 20:55:22 +0000 | |||
674 | @@ -51,7 +51,8 @@ | |||
675 | 51 | if 'units' not in this_service: | 51 | if 'units' not in this_service: |
676 | 52 | this_service['units'] = 1 | 52 | this_service['units'] = 1 |
677 | 53 | 53 | ||
679 | 54 | self.d.add(this_service['name'], units=this_service['units']) | 54 | self.d.add(this_service['name'], units=this_service['units'], |
680 | 55 | constraints=this_service.get('constraints')) | ||
681 | 55 | 56 | ||
682 | 56 | for svc in other_services: | 57 | for svc in other_services: |
683 | 57 | if 'location' in svc: | 58 | if 'location' in svc: |
684 | @@ -64,7 +65,8 @@ | |||
685 | 64 | if 'units' not in svc: | 65 | if 'units' not in svc: |
686 | 65 | svc['units'] = 1 | 66 | svc['units'] = 1 |
687 | 66 | 67 | ||
689 | 67 | self.d.add(svc['name'], charm=branch_location, units=svc['units']) | 68 | self.d.add(svc['name'], charm=branch_location, units=svc['units'], |
690 | 69 | constraints=svc.get('constraints')) | ||
691 | 68 | 70 | ||
692 | 69 | def _add_relations(self, relations): | 71 | def _add_relations(self, relations): |
693 | 70 | """Add all of the relations for the services.""" | 72 | """Add all of the relations for the services.""" |
694 | 71 | 73 | ||
695 | === modified file 'tests/charmhelpers/contrib/amulet/utils.py' | |||
696 | --- tests/charmhelpers/contrib/amulet/utils.py 2015-08-19 13:50:42 +0000 | |||
697 | +++ tests/charmhelpers/contrib/amulet/utils.py 2015-11-18 20:55:22 +0000 | |||
698 | @@ -19,9 +19,11 @@ | |||
699 | 19 | import logging | 19 | import logging |
700 | 20 | import os | 20 | import os |
701 | 21 | import re | 21 | import re |
702 | 22 | import socket | ||
703 | 22 | import subprocess | 23 | import subprocess |
704 | 23 | import sys | 24 | import sys |
705 | 24 | import time | 25 | import time |
706 | 26 | import uuid | ||
707 | 25 | 27 | ||
708 | 26 | import amulet | 28 | import amulet |
709 | 27 | import distro_info | 29 | import distro_info |
710 | @@ -114,7 +116,7 @@ | |||
711 | 114 | # /!\ DEPRECATION WARNING (beisner): | 116 | # /!\ DEPRECATION WARNING (beisner): |
712 | 115 | # New and existing tests should be rewritten to use | 117 | # New and existing tests should be rewritten to use |
713 | 116 | # validate_services_by_name() as it is aware of init systems. | 118 | # validate_services_by_name() as it is aware of init systems. |
715 | 117 | self.log.warn('/!\\ DEPRECATION WARNING: use ' | 119 | self.log.warn('DEPRECATION WARNING: use ' |
716 | 118 | 'validate_services_by_name instead of validate_services ' | 120 | 'validate_services_by_name instead of validate_services ' |
717 | 119 | 'due to init system differences.') | 121 | 'due to init system differences.') |
718 | 120 | 122 | ||
719 | @@ -269,33 +271,52 @@ | |||
720 | 269 | """Get last modification time of directory.""" | 271 | """Get last modification time of directory.""" |
721 | 270 | return sentry_unit.directory_stat(directory)['mtime'] | 272 | return sentry_unit.directory_stat(directory)['mtime'] |
722 | 271 | 273 | ||
741 | 272 | def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False): | 274 | def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None): |
742 | 273 | """Get process' start time. | 275 | """Get start time of a process based on the last modification time |
743 | 274 | 276 | of the /proc/pid directory. | |
744 | 275 | Determine start time of the process based on the last modification | 277 | |
745 | 276 | time of the /proc/pid directory. If pgrep_full is True, the process | 278 | :sentry_unit: The sentry unit to check for the service on |
746 | 277 | name is matched against the full command line. | 279 | :service: service name to look for in process table |
747 | 278 | """ | 280 | :pgrep_full: [Deprecated] Use full command line search mode with pgrep |
748 | 279 | if pgrep_full: | 281 | :returns: epoch time of service process start |
749 | 280 | cmd = 'pgrep -o -f {}'.format(service) | 282 | :param commands: list of bash commands |
750 | 281 | else: | 283 | :param sentry_units: list of sentry unit pointers |
751 | 282 | cmd = 'pgrep -o {}'.format(service) | 284 | :returns: None if successful; Failure message otherwise |
752 | 283 | cmd = cmd + ' | grep -v pgrep || exit 0' | 285 | """ |
753 | 284 | cmd_out = sentry_unit.run(cmd) | 286 | if pgrep_full is not None: |
754 | 285 | self.log.debug('CMDout: ' + str(cmd_out)) | 287 | # /!\ DEPRECATION WARNING (beisner): |
755 | 286 | if cmd_out[0]: | 288 | # No longer implemented, as pidof is now used instead of pgrep. |
756 | 287 | self.log.debug('Pid for %s %s' % (service, str(cmd_out[0]))) | 289 | # https://bugs.launchpad.net/charm-helpers/+bug/1474030 |
757 | 288 | proc_dir = '/proc/{}'.format(cmd_out[0].strip()) | 290 | self.log.warn('DEPRECATION WARNING: pgrep_full bool is no ' |
758 | 289 | return self._get_dir_mtime(sentry_unit, proc_dir) | 291 | 'longer implemented re: lp 1474030.') |
759 | 292 | |||
760 | 293 | pid_list = self.get_process_id_list(sentry_unit, service) | ||
761 | 294 | pid = pid_list[0] | ||
762 | 295 | proc_dir = '/proc/{}'.format(pid) | ||
763 | 296 | self.log.debug('Pid for {} on {}: {}'.format( | ||
764 | 297 | service, sentry_unit.info['unit_name'], pid)) | ||
765 | 298 | |||
766 | 299 | return self._get_dir_mtime(sentry_unit, proc_dir) | ||
767 | 290 | 300 | ||
768 | 291 | def service_restarted(self, sentry_unit, service, filename, | 301 | def service_restarted(self, sentry_unit, service, filename, |
770 | 292 | pgrep_full=False, sleep_time=20): | 302 | pgrep_full=None, sleep_time=20): |
771 | 293 | """Check if service was restarted. | 303 | """Check if service was restarted. |
772 | 294 | 304 | ||
773 | 295 | Compare a service's start time vs a file's last modification time | 305 | Compare a service's start time vs a file's last modification time |
774 | 296 | (such as a config file for that service) to determine if the service | 306 | (such as a config file for that service) to determine if the service |
775 | 297 | has been restarted. | 307 | has been restarted. |
776 | 298 | """ | 308 | """ |
777 | 309 | # /!\ DEPRECATION WARNING (beisner): | ||
778 | 310 | # This method is prone to races in that no before-time is known. | ||
779 | 311 | # Use validate_service_config_changed instead. | ||
780 | 312 | |||
781 | 313 | # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now | ||
782 | 314 | # used instead of pgrep. pgrep_full is still passed through to ensure | ||
783 | 315 | # deprecation WARNS. lp1474030 | ||
784 | 316 | self.log.warn('DEPRECATION WARNING: use ' | ||
785 | 317 | 'validate_service_config_changed instead of ' | ||
786 | 318 | 'service_restarted due to known races.') | ||
787 | 319 | |||
788 | 299 | time.sleep(sleep_time) | 320 | time.sleep(sleep_time) |
789 | 300 | if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >= | 321 | if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >= |
790 | 301 | self._get_file_mtime(sentry_unit, filename)): | 322 | self._get_file_mtime(sentry_unit, filename)): |
791 | @@ -304,78 +325,122 @@ | |||
792 | 304 | return False | 325 | return False |
793 | 305 | 326 | ||
794 | 306 | def service_restarted_since(self, sentry_unit, mtime, service, | 327 | def service_restarted_since(self, sentry_unit, mtime, service, |
797 | 307 | pgrep_full=False, sleep_time=20, | 328 | pgrep_full=None, sleep_time=20, |
798 | 308 | retry_count=2): | 329 | retry_count=30, retry_sleep_time=10): |
799 | 309 | """Check if service was been started after a given time. | 330 | """Check if service was been started after a given time. |
800 | 310 | 331 | ||
801 | 311 | Args: | 332 | Args: |
802 | 312 | sentry_unit (sentry): The sentry unit to check for the service on | 333 | sentry_unit (sentry): The sentry unit to check for the service on |
803 | 313 | mtime (float): The epoch time to check against | 334 | mtime (float): The epoch time to check against |
804 | 314 | service (string): service name to look for in process table | 335 | service (string): service name to look for in process table |
808 | 315 | pgrep_full (boolean): Use full command line search mode with pgrep | 336 | pgrep_full: [Deprecated] Use full command line search mode with pgrep |
809 | 316 | sleep_time (int): Seconds to sleep before looking for process | 337 | sleep_time (int): Initial sleep time (s) before looking for file |
810 | 317 | retry_count (int): If service is not found, how many times to retry | 338 | retry_sleep_time (int): Time (s) to sleep between retries |
811 | 339 | retry_count (int): If file is not found, how many times to retry | ||
812 | 318 | 340 | ||
813 | 319 | Returns: | 341 | Returns: |
814 | 320 | bool: True if service found and its start time it newer than mtime, | 342 | bool: True if service found and its start time it newer than mtime, |
815 | 321 | False if service is older than mtime or if service was | 343 | False if service is older than mtime or if service was |
816 | 322 | not found. | 344 | not found. |
817 | 323 | """ | 345 | """ |
819 | 324 | self.log.debug('Checking %s restarted since %s' % (service, mtime)) | 346 | # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now |
820 | 347 | # used instead of pgrep. pgrep_full is still passed through to ensure | ||
821 | 348 | # deprecation WARNS. lp1474030 | ||
822 | 349 | |||
823 | 350 | unit_name = sentry_unit.info['unit_name'] | ||
824 | 351 | self.log.debug('Checking that %s service restarted since %s on ' | ||
825 | 352 | '%s' % (service, mtime, unit_name)) | ||
826 | 325 | time.sleep(sleep_time) | 353 | time.sleep(sleep_time) |
836 | 326 | proc_start_time = self._get_proc_start_time(sentry_unit, service, | 354 | proc_start_time = None |
837 | 327 | pgrep_full) | 355 | tries = 0 |
838 | 328 | while retry_count > 0 and not proc_start_time: | 356 | while tries <= retry_count and not proc_start_time: |
839 | 329 | self.log.debug('No pid file found for service %s, will retry %i ' | 357 | try: |
840 | 330 | 'more times' % (service, retry_count)) | 358 | proc_start_time = self._get_proc_start_time(sentry_unit, |
841 | 331 | time.sleep(30) | 359 | service, |
842 | 332 | proc_start_time = self._get_proc_start_time(sentry_unit, service, | 360 | pgrep_full) |
843 | 333 | pgrep_full) | 361 | self.log.debug('Attempt {} to get {} proc start time on {} ' |
844 | 334 | retry_count = retry_count - 1 | 362 | 'OK'.format(tries, service, unit_name)) |
845 | 363 | except IOError as e: | ||
846 | 364 | # NOTE(beisner) - race avoidance, proc may not exist yet. | ||
847 | 365 | # https://bugs.launchpad.net/charm-helpers/+bug/1474030 | ||
848 | 366 | self.log.debug('Attempt {} to get {} proc start time on {} ' | ||
849 | 367 | 'failed\n{}'.format(tries, service, | ||
850 | 368 | unit_name, e)) | ||
851 | 369 | time.sleep(retry_sleep_time) | ||
852 | 370 | tries += 1 | ||
853 | 335 | 371 | ||
854 | 336 | if not proc_start_time: | 372 | if not proc_start_time: |
855 | 337 | self.log.warn('No proc start time found, assuming service did ' | 373 | self.log.warn('No proc start time found, assuming service did ' |
856 | 338 | 'not start') | 374 | 'not start') |
857 | 339 | return False | 375 | return False |
858 | 340 | if proc_start_time >= mtime: | 376 | if proc_start_time >= mtime: |
861 | 341 | self.log.debug('proc start time is newer than provided mtime' | 377 | self.log.debug('Proc start time is newer than provided mtime' |
862 | 342 | '(%s >= %s)' % (proc_start_time, mtime)) | 378 | '(%s >= %s) on %s (OK)' % (proc_start_time, |
863 | 379 | mtime, unit_name)) | ||
864 | 343 | return True | 380 | return True |
865 | 344 | else: | 381 | else: |
869 | 345 | self.log.warn('proc start time (%s) is older than provided mtime ' | 382 | self.log.warn('Proc start time (%s) is older than provided mtime ' |
870 | 346 | '(%s), service did not restart' % (proc_start_time, | 383 | '(%s) on %s, service did not ' |
871 | 347 | mtime)) | 384 | 'restart' % (proc_start_time, mtime, unit_name)) |
872 | 348 | return False | 385 | return False |
873 | 349 | 386 | ||
874 | 350 | def config_updated_since(self, sentry_unit, filename, mtime, | 387 | def config_updated_since(self, sentry_unit, filename, mtime, |
876 | 351 | sleep_time=20): | 388 | sleep_time=20, retry_count=30, |
877 | 389 | retry_sleep_time=10): | ||
878 | 352 | """Check if file was modified after a given time. | 390 | """Check if file was modified after a given time. |
879 | 353 | 391 | ||
880 | 354 | Args: | 392 | Args: |
881 | 355 | sentry_unit (sentry): The sentry unit to check the file mtime on | 393 | sentry_unit (sentry): The sentry unit to check the file mtime on |
882 | 356 | filename (string): The file to check mtime of | 394 | filename (string): The file to check mtime of |
883 | 357 | mtime (float): The epoch time to check against | 395 | mtime (float): The epoch time to check against |
885 | 358 | sleep_time (int): Seconds to sleep before looking for process | 396 | sleep_time (int): Initial sleep time (s) before looking for file |
886 | 397 | retry_sleep_time (int): Time (s) to sleep between retries | ||
887 | 398 | retry_count (int): If file is not found, how many times to retry | ||
888 | 359 | 399 | ||
889 | 360 | Returns: | 400 | Returns: |
890 | 361 | bool: True if file was modified more recently than mtime, False if | 401 | bool: True if file was modified more recently than mtime, False if |
892 | 362 | file was modified before mtime, | 402 | file was modified before mtime, or if file not found. |
893 | 363 | """ | 403 | """ |
895 | 364 | self.log.debug('Checking %s updated since %s' % (filename, mtime)) | 404 | unit_name = sentry_unit.info['unit_name'] |
896 | 405 | self.log.debug('Checking that %s updated since %s on ' | ||
897 | 406 | '%s' % (filename, mtime, unit_name)) | ||
898 | 365 | time.sleep(sleep_time) | 407 | time.sleep(sleep_time) |
900 | 366 | file_mtime = self._get_file_mtime(sentry_unit, filename) | 408 | file_mtime = None |
901 | 409 | tries = 0 | ||
902 | 410 | while tries <= retry_count and not file_mtime: | ||
903 | 411 | try: | ||
904 | 412 | file_mtime = self._get_file_mtime(sentry_unit, filename) | ||
905 | 413 | self.log.debug('Attempt {} to get {} file mtime on {} ' | ||
906 | 414 | 'OK'.format(tries, filename, unit_name)) | ||
907 | 415 | except IOError as e: | ||
908 | 416 | # NOTE(beisner) - race avoidance, file may not exist yet. | ||
909 | 417 | # https://bugs.launchpad.net/charm-helpers/+bug/1474030 | ||
910 | 418 | self.log.debug('Attempt {} to get {} file mtime on {} ' | ||
911 | 419 | 'failed\n{}'.format(tries, filename, | ||
912 | 420 | unit_name, e)) | ||
913 | 421 | time.sleep(retry_sleep_time) | ||
914 | 422 | tries += 1 | ||
915 | 423 | |||
916 | 424 | if not file_mtime: | ||
917 | 425 | self.log.warn('Could not determine file mtime, assuming ' | ||
918 | 426 | 'file does not exist') | ||
919 | 427 | return False | ||
920 | 428 | |||
921 | 367 | if file_mtime >= mtime: | 429 | if file_mtime >= mtime: |
922 | 368 | self.log.debug('File mtime is newer than provided mtime ' | 430 | self.log.debug('File mtime is newer than provided mtime ' |
924 | 369 | '(%s >= %s)' % (file_mtime, mtime)) | 431 | '(%s >= %s) on %s (OK)' % (file_mtime, |
925 | 432 | mtime, unit_name)) | ||
926 | 370 | return True | 433 | return True |
927 | 371 | else: | 434 | else: |
930 | 372 | self.log.warn('File mtime %s is older than provided mtime %s' | 435 | self.log.warn('File mtime is older than provided mtime' |
931 | 373 | % (file_mtime, mtime)) | 436 | '(%s < on %s) on %s' % (file_mtime, |
932 | 437 | mtime, unit_name)) | ||
933 | 374 | return False | 438 | return False |
934 | 375 | 439 | ||
935 | 376 | def validate_service_config_changed(self, sentry_unit, mtime, service, | 440 | def validate_service_config_changed(self, sentry_unit, mtime, service, |
938 | 377 | filename, pgrep_full=False, | 441 | filename, pgrep_full=None, |
939 | 378 | sleep_time=20, retry_count=2): | 442 | sleep_time=20, retry_count=30, |
940 | 443 | retry_sleep_time=10): | ||
941 | 379 | """Check service and file were updated after mtime | 444 | """Check service and file were updated after mtime |
942 | 380 | 445 | ||
943 | 381 | Args: | 446 | Args: |
944 | @@ -383,9 +448,10 @@ | |||
945 | 383 | mtime (float): The epoch time to check against | 448 | mtime (float): The epoch time to check against |
946 | 384 | service (string): service name to look for in process table | 449 | service (string): service name to look for in process table |
947 | 385 | filename (string): The file to check mtime of | 450 | filename (string): The file to check mtime of |
950 | 386 | pgrep_full (boolean): Use full command line search mode with pgrep | 451 | pgrep_full: [Deprecated] Use full command line search mode with pgrep |
951 | 387 | sleep_time (int): Seconds to sleep before looking for process | 452 | sleep_time (int): Initial sleep in seconds to pass to test helpers |
952 | 388 | retry_count (int): If service is not found, how many times to retry | 453 | retry_count (int): If service is not found, how many times to retry |
953 | 454 | retry_sleep_time (int): Time in seconds to wait between retries | ||
954 | 389 | 455 | ||
955 | 390 | Typical Usage: | 456 | Typical Usage: |
956 | 391 | u = OpenStackAmuletUtils(ERROR) | 457 | u = OpenStackAmuletUtils(ERROR) |
957 | @@ -402,15 +468,27 @@ | |||
958 | 402 | mtime, False if service is older than mtime or if service was | 468 | mtime, False if service is older than mtime or if service was |
959 | 403 | not found or if filename was modified before mtime. | 469 | not found or if filename was modified before mtime. |
960 | 404 | """ | 470 | """ |
970 | 405 | self.log.debug('Checking %s restarted since %s' % (service, mtime)) | 471 | |
971 | 406 | time.sleep(sleep_time) | 472 | # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now |
972 | 407 | service_restart = self.service_restarted_since(sentry_unit, mtime, | 473 | # used instead of pgrep. pgrep_full is still passed through to ensure |
973 | 408 | service, | 474 | # deprecation WARNS. lp1474030 |
974 | 409 | pgrep_full=pgrep_full, | 475 | |
975 | 410 | sleep_time=0, | 476 | service_restart = self.service_restarted_since( |
976 | 411 | retry_count=retry_count) | 477 | sentry_unit, mtime, |
977 | 412 | config_update = self.config_updated_since(sentry_unit, filename, mtime, | 478 | service, |
978 | 413 | sleep_time=0) | 479 | pgrep_full=pgrep_full, |
979 | 480 | sleep_time=sleep_time, | ||
980 | 481 | retry_count=retry_count, | ||
981 | 482 | retry_sleep_time=retry_sleep_time) | ||
982 | 483 | |||
983 | 484 | config_update = self.config_updated_since( | ||
984 | 485 | sentry_unit, | ||
985 | 486 | filename, | ||
986 | 487 | mtime, | ||
987 | 488 | sleep_time=sleep_time, | ||
988 | 489 | retry_count=retry_count, | ||
989 | 490 | retry_sleep_time=retry_sleep_time) | ||
990 | 491 | |||
991 | 414 | return service_restart and config_update | 492 | return service_restart and config_update |
992 | 415 | 493 | ||
993 | 416 | def get_sentry_time(self, sentry_unit): | 494 | def get_sentry_time(self, sentry_unit): |
994 | @@ -428,7 +506,6 @@ | |||
995 | 428 | """Return a list of all Ubuntu releases in order of release.""" | 506 | """Return a list of all Ubuntu releases in order of release.""" |
996 | 429 | _d = distro_info.UbuntuDistroInfo() | 507 | _d = distro_info.UbuntuDistroInfo() |
997 | 430 | _release_list = _d.all | 508 | _release_list = _d.all |
998 | 431 | self.log.debug('Ubuntu release list: {}'.format(_release_list)) | ||
999 | 432 | return _release_list | 509 | return _release_list |
1000 | 433 | 510 | ||
1001 | 434 | def file_to_url(self, file_rel_path): | 511 | def file_to_url(self, file_rel_path): |
1002 | @@ -568,6 +645,142 @@ | |||
1003 | 568 | 645 | ||
1004 | 569 | return None | 646 | return None |
1005 | 570 | 647 | ||
1006 | 648 | def validate_sectionless_conf(self, file_contents, expected): | ||
1007 | 649 | """A crude conf parser. Useful to inspect configuration files which | ||
1008 | 650 | do not have section headers (as would be necessary in order to use | ||
1009 | 651 | the configparser). Such as openstack-dashboard or rabbitmq confs.""" | ||
1010 | 652 | for line in file_contents.split('\n'): | ||
1011 | 653 | if '=' in line: | ||
1012 | 654 | args = line.split('=') | ||
1013 | 655 | if len(args) <= 1: | ||
1014 | 656 | continue | ||
1015 | 657 | key = args[0].strip() | ||
1016 | 658 | value = args[1].strip() | ||
1017 | 659 | if key in expected.keys(): | ||
1018 | 660 | if expected[key] != value: | ||
1019 | 661 | msg = ('Config mismatch. Expected, actual: {}, ' | ||
1020 | 662 | '{}'.format(expected[key], value)) | ||
1021 | 663 | amulet.raise_status(amulet.FAIL, msg=msg) | ||
1022 | 664 | |||
1023 | 665 | def get_unit_hostnames(self, units): | ||
1024 | 666 | """Return a dict of juju unit names to hostnames.""" | ||
1025 | 667 | host_names = {} | ||
1026 | 668 | for unit in units: | ||
1027 | 669 | host_names[unit.info['unit_name']] = \ | ||
1028 | 670 | str(unit.file_contents('/etc/hostname').strip()) | ||
1029 | 671 | self.log.debug('Unit host names: {}'.format(host_names)) | ||
1030 | 672 | return host_names | ||
1031 | 673 | |||
1032 | 674 | def run_cmd_unit(self, sentry_unit, cmd): | ||
1033 | 675 | """Run a command on a unit, return the output and exit code.""" | ||
1034 | 676 | output, code = sentry_unit.run(cmd) | ||
1035 | 677 | if code == 0: | ||
1036 | 678 | self.log.debug('{} `{}` command returned {} ' | ||
1037 | 679 | '(OK)'.format(sentry_unit.info['unit_name'], | ||
1038 | 680 | cmd, code)) | ||
1039 | 681 | else: | ||
1040 | 682 | msg = ('{} `{}` command returned {} ' | ||
1041 | 683 | '{}'.format(sentry_unit.info['unit_name'], | ||
1042 | 684 | cmd, code, output)) | ||
1043 | 685 | amulet.raise_status(amulet.FAIL, msg=msg) | ||
1044 | 686 | return str(output), code | ||
1045 | 687 | |||
1046 | 688 | def file_exists_on_unit(self, sentry_unit, file_name): | ||
1047 | 689 | """Check if a file exists on a unit.""" | ||
1048 | 690 | try: | ||
1049 | 691 | sentry_unit.file_stat(file_name) | ||
1050 | 692 | return True | ||
1051 | 693 | except IOError: | ||
1052 | 694 | return False | ||
1053 | 695 | except Exception as e: | ||
1054 | 696 | msg = 'Error checking file {}: {}'.format(file_name, e) | ||
1055 | 697 | amulet.raise_status(amulet.FAIL, msg=msg) | ||
1056 | 698 | |||
1057 | 699 | def file_contents_safe(self, sentry_unit, file_name, | ||
1058 | 700 | max_wait=60, fatal=False): | ||
1059 | 701 | """Get file contents from a sentry unit. Wrap amulet file_contents | ||
1060 | 702 | with retry logic to address races where a file checks as existing, | ||
1061 | 703 | but no longer exists by the time file_contents is called. | ||
1062 | 704 | Return None if file not found. Optionally raise if fatal is True.""" | ||
1063 | 705 | unit_name = sentry_unit.info['unit_name'] | ||
1064 | 706 | file_contents = False | ||
1065 | 707 | tries = 0 | ||
1066 | 708 | while not file_contents and tries < (max_wait / 4): | ||
1067 | 709 | try: | ||
1068 | 710 | file_contents = sentry_unit.file_contents(file_name) | ||
1069 | 711 | except IOError: | ||
1070 | 712 | self.log.debug('Attempt {} to open file {} from {} ' | ||
1071 | 713 | 'failed'.format(tries, file_name, | ||
1072 | 714 | unit_name)) | ||
1073 | 715 | time.sleep(4) | ||
1074 | 716 | tries += 1 | ||
1075 | 717 | |||
1076 | 718 | if file_contents: | ||
1077 | 719 | return file_contents | ||
1078 | 720 | elif not fatal: | ||
1079 | 721 | return None | ||
1080 | 722 | elif fatal: | ||
1081 | 723 | msg = 'Failed to get file contents from unit.' | ||
1082 | 724 | amulet.raise_status(amulet.FAIL, msg) | ||
1083 | 725 | |||
1084 | 726 | def port_knock_tcp(self, host="localhost", port=22, timeout=15): | ||
1085 | 727 | """Open a TCP socket to check for a listening sevice on a host. | ||
1086 | 728 | |||
1087 | 729 | :param host: host name or IP address, default to localhost | ||
1088 | 730 | :param port: TCP port number, default to 22 | ||
1089 | 731 | :param timeout: Connect timeout, default to 15 seconds | ||
1090 | 732 | :returns: True if successful, False if connect failed | ||
1091 | 733 | """ | ||
1092 | 734 | |||
1093 | 735 | # Resolve host name if possible | ||
1094 | 736 | try: | ||
1095 | 737 | connect_host = socket.gethostbyname(host) | ||
1096 | 738 | host_human = "{} ({})".format(connect_host, host) | ||
1097 | 739 | except socket.error as e: | ||
1098 | 740 | self.log.warn('Unable to resolve address: ' | ||
1099 | 741 | '{} ({}) Trying anyway!'.format(host, e)) | ||
1100 | 742 | connect_host = host | ||
1101 | 743 | host_human = connect_host | ||
1102 | 744 | |||
1103 | 745 | # Attempt socket connection | ||
1104 | 746 | try: | ||
1105 | 747 | knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | ||
1106 | 748 | knock.settimeout(timeout) | ||
1107 | 749 | knock.connect((connect_host, port)) | ||
1108 | 750 | knock.close() | ||
1109 | 751 | self.log.debug('Socket connect OK for host ' | ||
1110 | 752 | '{} on port {}.'.format(host_human, port)) | ||
1111 | 753 | return True | ||
1112 | 754 | except socket.error as e: | ||
1113 | 755 | self.log.debug('Socket connect FAIL for' | ||
1114 | 756 | ' {} port {} ({})'.format(host_human, port, e)) | ||
1115 | 757 | return False | ||
1116 | 758 | |||
1117 | 759 | def port_knock_units(self, sentry_units, port=22, | ||
1118 | 760 | timeout=15, expect_success=True): | ||
1119 | 761 | """Open a TCP socket to check for a listening sevice on each | ||
1120 | 762 | listed juju unit. | ||
1121 | 763 | |||
1122 | 764 | :param sentry_units: list of sentry unit pointers | ||
1123 | 765 | :param port: TCP port number, default to 22 | ||
1124 | 766 | :param timeout: Connect timeout, default to 15 seconds | ||
1125 | 767 | :expect_success: True by default, set False to invert logic | ||
1126 | 768 | :returns: None if successful, Failure message otherwise | ||
1127 | 769 | """ | ||
1128 | 770 | for unit in sentry_units: | ||
1129 | 771 | host = unit.info['public-address'] | ||
1130 | 772 | connected = self.port_knock_tcp(host, port, timeout) | ||
1131 | 773 | if not connected and expect_success: | ||
1132 | 774 | return 'Socket connect failed.' | ||
1133 | 775 | elif connected and not expect_success: | ||
1134 | 776 | return 'Socket connected unexpectedly.' | ||
1135 | 777 | |||
1136 | 778 | def get_uuid_epoch_stamp(self): | ||
1137 | 779 | """Returns a stamp string based on uuid4 and epoch time. Useful in | ||
1138 | 780 | generating test messages which need to be unique-ish.""" | ||
1139 | 781 | return '[{}-{}]'.format(uuid.uuid4(), time.time()) | ||
1140 | 782 | |||
1141 | 783 | # amulet juju action helpers: | ||
1142 | 571 | def run_action(self, unit_sentry, action, | 784 | def run_action(self, unit_sentry, action, |
1143 | 572 | _check_output=subprocess.check_output): | 785 | _check_output=subprocess.check_output): |
1144 | 573 | """Run the named action on a given unit sentry. | 786 | """Run the named action on a given unit sentry. |
1145 | @@ -594,3 +807,12 @@ | |||
1146 | 594 | output = _check_output(command, universal_newlines=True) | 807 | output = _check_output(command, universal_newlines=True) |
1147 | 595 | data = json.loads(output) | 808 | data = json.loads(output) |
1148 | 596 | return data.get(u"status") == "completed" | 809 | return data.get(u"status") == "completed" |
1149 | 810 | |||
1150 | 811 | def status_get(self, unit): | ||
1151 | 812 | """Return the current service status of this unit.""" | ||
1152 | 813 | raw_status, return_code = unit.run( | ||
1153 | 814 | "status-get --format=json --include-data") | ||
1154 | 815 | if return_code != 0: | ||
1155 | 816 | return ("unknown", "") | ||
1156 | 817 | status = json.loads(raw_status) | ||
1157 | 818 | return (status["status"], status["message"]) | ||
1158 | 597 | 819 | ||
1159 | === modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py' | |||
1160 | --- tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-08-19 13:50:42 +0000 | |||
1161 | +++ tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-11-18 20:55:22 +0000 | |||
1162 | @@ -14,12 +14,18 @@ | |||
1163 | 14 | # You should have received a copy of the GNU Lesser General Public License | 14 | # You should have received a copy of the GNU Lesser General Public License |
1164 | 15 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. | 15 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
1165 | 16 | 16 | ||
1166 | 17 | import logging | ||
1167 | 18 | import re | ||
1168 | 19 | import sys | ||
1169 | 17 | import six | 20 | import six |
1170 | 18 | from collections import OrderedDict | 21 | from collections import OrderedDict |
1171 | 19 | from charmhelpers.contrib.amulet.deployment import ( | 22 | from charmhelpers.contrib.amulet.deployment import ( |
1172 | 20 | AmuletDeployment | 23 | AmuletDeployment |
1173 | 21 | ) | 24 | ) |
1174 | 22 | 25 | ||
1175 | 26 | DEBUG = logging.DEBUG | ||
1176 | 27 | ERROR = logging.ERROR | ||
1177 | 28 | |||
1178 | 23 | 29 | ||
1179 | 24 | class OpenStackAmuletDeployment(AmuletDeployment): | 30 | class OpenStackAmuletDeployment(AmuletDeployment): |
1180 | 25 | """OpenStack amulet deployment. | 31 | """OpenStack amulet deployment. |
1181 | @@ -28,9 +34,12 @@ | |||
1182 | 28 | that is specifically for use by OpenStack charms. | 34 | that is specifically for use by OpenStack charms. |
1183 | 29 | """ | 35 | """ |
1184 | 30 | 36 | ||
1186 | 31 | def __init__(self, series=None, openstack=None, source=None, stable=True): | 37 | def __init__(self, series=None, openstack=None, source=None, |
1187 | 38 | stable=True, log_level=DEBUG): | ||
1188 | 32 | """Initialize the deployment environment.""" | 39 | """Initialize the deployment environment.""" |
1189 | 33 | super(OpenStackAmuletDeployment, self).__init__(series) | 40 | super(OpenStackAmuletDeployment, self).__init__(series) |
1190 | 41 | self.log = self.get_logger(level=log_level) | ||
1191 | 42 | self.log.info('OpenStackAmuletDeployment: init') | ||
1192 | 34 | self.openstack = openstack | 43 | self.openstack = openstack |
1193 | 35 | self.source = source | 44 | self.source = source |
1194 | 36 | self.stable = stable | 45 | self.stable = stable |
1195 | @@ -38,26 +47,55 @@ | |||
1196 | 38 | # out. | 47 | # out. |
1197 | 39 | self.current_next = "trusty" | 48 | self.current_next = "trusty" |
1198 | 40 | 49 | ||
1199 | 50 | def get_logger(self, name="deployment-logger", level=logging.DEBUG): | ||
1200 | 51 | """Get a logger object that will log to stdout.""" | ||
1201 | 52 | log = logging | ||
1202 | 53 | logger = log.getLogger(name) | ||
1203 | 54 | fmt = log.Formatter("%(asctime)s %(funcName)s " | ||
1204 | 55 | "%(levelname)s: %(message)s") | ||
1205 | 56 | |||
1206 | 57 | handler = log.StreamHandler(stream=sys.stdout) | ||
1207 | 58 | handler.setLevel(level) | ||
1208 | 59 | handler.setFormatter(fmt) | ||
1209 | 60 | |||
1210 | 61 | logger.addHandler(handler) | ||
1211 | 62 | logger.setLevel(level) | ||
1212 | 63 | |||
1213 | 64 | return logger | ||
1214 | 65 | |||
1215 | 41 | def _determine_branch_locations(self, other_services): | 66 | def _determine_branch_locations(self, other_services): |
1216 | 42 | """Determine the branch locations for the other services. | 67 | """Determine the branch locations for the other services. |
1217 | 43 | 68 | ||
1218 | 44 | Determine if the local branch being tested is derived from its | 69 | Determine if the local branch being tested is derived from its |
1219 | 45 | stable or next (dev) branch, and based on this, use the corresonding | 70 | stable or next (dev) branch, and based on this, use the corresonding |
1220 | 46 | stable or next branches for the other_services.""" | 71 | stable or next branches for the other_services.""" |
1221 | 72 | |||
1222 | 73 | self.log.info('OpenStackAmuletDeployment: determine branch locations') | ||
1223 | 74 | |||
1224 | 75 | # Charms outside the lp:~openstack-charmers namespace | ||
1225 | 47 | base_charms = ['mysql', 'mongodb', 'nrpe'] | 76 | base_charms = ['mysql', 'mongodb', 'nrpe'] |
1226 | 48 | 77 | ||
1227 | 78 | # Force these charms to current series even when using an older series. | ||
1228 | 79 | # ie. Use trusty/nrpe even when series is precise, as the P charm | ||
1229 | 80 | # does not possess the necessary external master config and hooks. | ||
1230 | 81 | force_series_current = ['nrpe'] | ||
1231 | 82 | |||
1232 | 49 | if self.series in ['precise', 'trusty']: | 83 | if self.series in ['precise', 'trusty']: |
1233 | 50 | base_series = self.series | 84 | base_series = self.series |
1234 | 51 | else: | 85 | else: |
1235 | 52 | base_series = self.current_next | 86 | base_series = self.current_next |
1236 | 53 | 87 | ||
1239 | 54 | if self.stable: | 88 | for svc in other_services: |
1240 | 55 | for svc in other_services: | 89 | if svc['name'] in force_series_current: |
1241 | 90 | base_series = self.current_next | ||
1242 | 91 | # If a location has been explicitly set, use it | ||
1243 | 92 | if svc.get('location'): | ||
1244 | 93 | continue | ||
1245 | 94 | if self.stable: | ||
1246 | 56 | temp = 'lp:charms/{}/{}' | 95 | temp = 'lp:charms/{}/{}' |
1247 | 57 | svc['location'] = temp.format(base_series, | 96 | svc['location'] = temp.format(base_series, |
1248 | 58 | svc['name']) | 97 | svc['name']) |
1251 | 59 | else: | 98 | else: |
1250 | 60 | for svc in other_services: | ||
1252 | 61 | if svc['name'] in base_charms: | 99 | if svc['name'] in base_charms: |
1253 | 62 | temp = 'lp:charms/{}/{}' | 100 | temp = 'lp:charms/{}/{}' |
1254 | 63 | svc['location'] = temp.format(base_series, | 101 | svc['location'] = temp.format(base_series, |
1255 | @@ -66,10 +104,13 @@ | |||
1256 | 66 | temp = 'lp:~openstack-charmers/charms/{}/{}/next' | 104 | temp = 'lp:~openstack-charmers/charms/{}/{}/next' |
1257 | 67 | svc['location'] = temp.format(self.current_next, | 105 | svc['location'] = temp.format(self.current_next, |
1258 | 68 | svc['name']) | 106 | svc['name']) |
1259 | 107 | |||
1260 | 69 | return other_services | 108 | return other_services |
1261 | 70 | 109 | ||
1262 | 71 | def _add_services(self, this_service, other_services): | 110 | def _add_services(self, this_service, other_services): |
1263 | 72 | """Add services to the deployment and set openstack-origin/source.""" | 111 | """Add services to the deployment and set openstack-origin/source.""" |
1264 | 112 | self.log.info('OpenStackAmuletDeployment: adding services') | ||
1265 | 113 | |||
1266 | 73 | other_services = self._determine_branch_locations(other_services) | 114 | other_services = self._determine_branch_locations(other_services) |
1267 | 74 | 115 | ||
1268 | 75 | super(OpenStackAmuletDeployment, self)._add_services(this_service, | 116 | super(OpenStackAmuletDeployment, self)._add_services(this_service, |
1269 | @@ -77,29 +118,101 @@ | |||
1270 | 77 | 118 | ||
1271 | 78 | services = other_services | 119 | services = other_services |
1272 | 79 | services.append(this_service) | 120 | services.append(this_service) |
1273 | 121 | |||
1274 | 122 | # Charms which should use the source config option | ||
1275 | 80 | use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', | 123 | use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', |
1276 | 81 | 'ceph-osd', 'ceph-radosgw'] | 124 | 'ceph-osd', 'ceph-radosgw'] |
1280 | 82 | # Most OpenStack subordinate charms do not expose an origin option | 125 | |
1281 | 83 | # as that is controlled by the principle. | 126 | # Charms which can not use openstack-origin, ie. many subordinates |
1282 | 84 | ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe'] | 127 | no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe'] |
1283 | 85 | 128 | ||
1284 | 86 | if self.openstack: | 129 | if self.openstack: |
1285 | 87 | for svc in services: | 130 | for svc in services: |
1287 | 88 | if svc['name'] not in use_source + ignore: | 131 | if svc['name'] not in use_source + no_origin: |
1288 | 89 | config = {'openstack-origin': self.openstack} | 132 | config = {'openstack-origin': self.openstack} |
1289 | 90 | self.d.configure(svc['name'], config) | 133 | self.d.configure(svc['name'], config) |
1290 | 91 | 134 | ||
1291 | 92 | if self.source: | 135 | if self.source: |
1292 | 93 | for svc in services: | 136 | for svc in services: |
1294 | 94 | if svc['name'] in use_source and svc['name'] not in ignore: | 137 | if svc['name'] in use_source and svc['name'] not in no_origin: |
1295 | 95 | config = {'source': self.source} | 138 | config = {'source': self.source} |
1296 | 96 | self.d.configure(svc['name'], config) | 139 | self.d.configure(svc['name'], config) |
1297 | 97 | 140 | ||
1298 | 98 | def _configure_services(self, configs): | 141 | def _configure_services(self, configs): |
1299 | 99 | """Configure all of the services.""" | 142 | """Configure all of the services.""" |
1300 | 143 | self.log.info('OpenStackAmuletDeployment: configure services') | ||
1301 | 100 | for service, config in six.iteritems(configs): | 144 | for service, config in six.iteritems(configs): |
1302 | 101 | self.d.configure(service, config) | 145 | self.d.configure(service, config) |
1303 | 102 | 146 | ||
1304 | 147 | def _auto_wait_for_status(self, message=None, exclude_services=None, | ||
1305 | 148 | include_only=None, timeout=1800): | ||
1306 | 149 | """Wait for all units to have a specific extended status, except | ||
1307 | 150 | for any defined as excluded. Unless specified via message, any | ||
1308 | 151 | status containing any case of 'ready' will be considered a match. | ||
1309 | 152 | |||
1310 | 153 | Examples of message usage: | ||
1311 | 154 | |||
1312 | 155 | Wait for all unit status to CONTAIN any case of 'ready' or 'ok': | ||
1313 | 156 | message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE) | ||
1314 | 157 | |||
1315 | 158 | Wait for all units to reach this status (exact match): | ||
1316 | 159 | message = re.compile('^Unit is ready and clustered$') | ||
1317 | 160 | |||
1318 | 161 | Wait for all units to reach any one of these (exact match): | ||
1319 | 162 | message = re.compile('Unit is ready|OK|Ready') | ||
1320 | 163 | |||
1321 | 164 | Wait for at least one unit to reach this status (exact match): | ||
1322 | 165 | message = {'ready'} | ||
1323 | 166 | |||
1324 | 167 | See Amulet's sentry.wait_for_messages() for message usage detail. | ||
1325 | 168 | https://github.com/juju/amulet/blob/master/amulet/sentry.py | ||
1326 | 169 | |||
1327 | 170 | :param message: Expected status match | ||
1328 | 171 | :param exclude_services: List of juju service names to ignore, | ||
1329 | 172 | not to be used in conjuction with include_only. | ||
1330 | 173 | :param include_only: List of juju service names to exclusively check, | ||
1331 | 174 | not to be used in conjuction with exclude_services. | ||
1332 | 175 | :param timeout: Maximum time in seconds to wait for status match | ||
1333 | 176 | :returns: None. Raises if timeout is hit. | ||
1334 | 177 | """ | ||
1335 | 178 | self.log.info('Waiting for extended status on units...') | ||
1336 | 179 | |||
1337 | 180 | all_services = self.d.services.keys() | ||
1338 | 181 | |||
1339 | 182 | if exclude_services and include_only: | ||
1340 | 183 | raise ValueError('exclude_services can not be used ' | ||
1341 | 184 | 'with include_only') | ||
1342 | 185 | |||
1343 | 186 | if message: | ||
1344 | 187 | if isinstance(message, re._pattern_type): | ||
1345 | 188 | match = message.pattern | ||
1346 | 189 | else: | ||
1347 | 190 | match = message | ||
1348 | 191 | |||
1349 | 192 | self.log.debug('Custom extended status wait match: ' | ||
1350 | 193 | '{}'.format(match)) | ||
1351 | 194 | else: | ||
1352 | 195 | self.log.debug('Default extended status wait match: contains ' | ||
1353 | 196 | 'READY (case-insensitive)') | ||
1354 | 197 | message = re.compile('.*ready.*', re.IGNORECASE) | ||
1355 | 198 | |||
1356 | 199 | if exclude_services: | ||
1357 | 200 | self.log.debug('Excluding services from extended status match: ' | ||
1358 | 201 | '{}'.format(exclude_services)) | ||
1359 | 202 | else: | ||
1360 | 203 | exclude_services = [] | ||
1361 | 204 | |||
1362 | 205 | if include_only: | ||
1363 | 206 | services = include_only | ||
1364 | 207 | else: | ||
1365 | 208 | services = list(set(all_services) - set(exclude_services)) | ||
1366 | 209 | |||
1367 | 210 | self.log.debug('Waiting up to {}s for extended status on services: ' | ||
1368 | 211 | '{}'.format(timeout, services)) | ||
1369 | 212 | service_messages = {service: message for service in services} | ||
1370 | 213 | self.d.sentry.wait_for_messages(service_messages, timeout=timeout) | ||
1371 | 214 | self.log.info('OK') | ||
1372 | 215 | |||
1373 | 103 | def _get_openstack_release(self): | 216 | def _get_openstack_release(self): |
1374 | 104 | """Get openstack release. | 217 | """Get openstack release. |
1375 | 105 | 218 | ||
1376 | 106 | 219 | ||
1377 | === modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py' | |||
1378 | --- tests/charmhelpers/contrib/openstack/amulet/utils.py 2015-06-29 14:24:05 +0000 | |||
1379 | +++ tests/charmhelpers/contrib/openstack/amulet/utils.py 2015-11-18 20:55:22 +0000 | |||
1380 | @@ -18,6 +18,7 @@ | |||
1381 | 18 | import json | 18 | import json |
1382 | 19 | import logging | 19 | import logging |
1383 | 20 | import os | 20 | import os |
1384 | 21 | import re | ||
1385 | 21 | import six | 22 | import six |
1386 | 22 | import time | 23 | import time |
1387 | 23 | import urllib | 24 | import urllib |
1388 | @@ -27,6 +28,7 @@ | |||
1389 | 27 | import heatclient.v1.client as heat_client | 28 | import heatclient.v1.client as heat_client |
1390 | 28 | import keystoneclient.v2_0 as keystone_client | 29 | import keystoneclient.v2_0 as keystone_client |
1391 | 29 | import novaclient.v1_1.client as nova_client | 30 | import novaclient.v1_1.client as nova_client |
1392 | 31 | import pika | ||
1393 | 30 | import swiftclient | 32 | import swiftclient |
1394 | 31 | 33 | ||
1395 | 32 | from charmhelpers.contrib.amulet.utils import ( | 34 | from charmhelpers.contrib.amulet.utils import ( |
1396 | @@ -602,3 +604,382 @@ | |||
1397 | 602 | self.log.debug('Ceph {} samples (OK): ' | 604 | self.log.debug('Ceph {} samples (OK): ' |
1398 | 603 | '{}'.format(sample_type, samples)) | 605 | '{}'.format(sample_type, samples)) |
1399 | 604 | return None | 606 | return None |
1400 | 607 | |||
1401 | 608 | # rabbitmq/amqp specific helpers: | ||
1402 | 609 | |||
1403 | 610 | def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200): | ||
1404 | 611 | """Wait for rmq units extended status to show cluster readiness, | ||
1405 | 612 | after an optional initial sleep period. Initial sleep is likely | ||
1406 | 613 | necessary to be effective following a config change, as status | ||
1407 | 614 | message may not instantly update to non-ready.""" | ||
1408 | 615 | |||
1409 | 616 | if init_sleep: | ||
1410 | 617 | time.sleep(init_sleep) | ||
1411 | 618 | |||
1412 | 619 | message = re.compile('^Unit is ready and clustered$') | ||
1413 | 620 | deployment._auto_wait_for_status(message=message, | ||
1414 | 621 | timeout=timeout, | ||
1415 | 622 | include_only=['rabbitmq-server']) | ||
1416 | 623 | |||
1417 | 624 | def add_rmq_test_user(self, sentry_units, | ||
1418 | 625 | username="testuser1", password="changeme"): | ||
1419 | 626 | """Add a test user via the first rmq juju unit, check connection as | ||
1420 | 627 | the new user against all sentry units. | ||
1421 | 628 | |||
1422 | 629 | :param sentry_units: list of sentry unit pointers | ||
1423 | 630 | :param username: amqp user name, default to testuser1 | ||
1424 | 631 | :param password: amqp user password | ||
1425 | 632 | :returns: None if successful. Raise on error. | ||
1426 | 633 | """ | ||
1427 | 634 | self.log.debug('Adding rmq user ({})...'.format(username)) | ||
1428 | 635 | |||
1429 | 636 | # Check that user does not already exist | ||
1430 | 637 | cmd_user_list = 'rabbitmqctl list_users' | ||
1431 | 638 | output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list) | ||
1432 | 639 | if username in output: | ||
1433 | 640 | self.log.warning('User ({}) already exists, returning ' | ||
1434 | 641 | 'gracefully.'.format(username)) | ||
1435 | 642 | return | ||
1436 | 643 | |||
1437 | 644 | perms = '".*" ".*" ".*"' | ||
1438 | 645 | cmds = ['rabbitmqctl add_user {} {}'.format(username, password), | ||
1439 | 646 | 'rabbitmqctl set_permissions {} {}'.format(username, perms)] | ||
1440 | 647 | |||
1441 | 648 | # Add user via first unit | ||
1442 | 649 | for cmd in cmds: | ||
1443 | 650 | output, _ = self.run_cmd_unit(sentry_units[0], cmd) | ||
1444 | 651 | |||
1445 | 652 | # Check connection against the other sentry_units | ||
1446 | 653 | self.log.debug('Checking user connect against units...') | ||
1447 | 654 | for sentry_unit in sentry_units: | ||
1448 | 655 | connection = self.connect_amqp_by_unit(sentry_unit, ssl=False, | ||
1449 | 656 | username=username, | ||
1450 | 657 | password=password) | ||
1451 | 658 | connection.close() | ||
1452 | 659 | |||
1453 | 660 | def delete_rmq_test_user(self, sentry_units, username="testuser1"): | ||
1454 | 661 | """Delete a rabbitmq user via the first rmq juju unit. | ||
1455 | 662 | |||
1456 | 663 | :param sentry_units: list of sentry unit pointers | ||
1457 | 664 | :param username: amqp user name, default to testuser1 | ||
1458 | 665 | :param password: amqp user password | ||
1459 | 666 | :returns: None if successful or no such user. | ||
1460 | 667 | """ | ||
1461 | 668 | self.log.debug('Deleting rmq user ({})...'.format(username)) | ||
1462 | 669 | |||
1463 | 670 | # Check that the user exists | ||
1464 | 671 | cmd_user_list = 'rabbitmqctl list_users' | ||
1465 | 672 | output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list) | ||
1466 | 673 | |||
1467 | 674 | if username not in output: | ||
1468 | 675 | self.log.warning('User ({}) does not exist, returning ' | ||
1469 | 676 | 'gracefully.'.format(username)) | ||
1470 | 677 | return | ||
1471 | 678 | |||
1472 | 679 | # Delete the user | ||
1473 | 680 | cmd_user_del = 'rabbitmqctl delete_user {}'.format(username) | ||
1474 | 681 | output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del) | ||
1475 | 682 | |||
1476 | 683 | def get_rmq_cluster_status(self, sentry_unit): | ||
1477 | 684 | """Execute rabbitmq cluster status command on a unit and return | ||
1478 | 685 | the full output. | ||
1479 | 686 | |||
1480 | 687 | :param unit: sentry unit | ||
1481 | 688 | :returns: String containing console output of cluster status command | ||
1482 | 689 | """ | ||
1483 | 690 | cmd = 'rabbitmqctl cluster_status' | ||
1484 | 691 | output, _ = self.run_cmd_unit(sentry_unit, cmd) | ||
1485 | 692 | self.log.debug('{} cluster_status:\n{}'.format( | ||
1486 | 693 | sentry_unit.info['unit_name'], output)) | ||
1487 | 694 | return str(output) | ||
1488 | 695 | |||
1489 | 696 | def get_rmq_cluster_running_nodes(self, sentry_unit): | ||
1490 | 697 | """Parse rabbitmqctl cluster_status output string, return list of | ||
1491 | 698 | running rabbitmq cluster nodes. | ||
1492 | 699 | |||
1493 | 700 | :param unit: sentry unit | ||
1494 | 701 | :returns: List containing node names of running nodes | ||
1495 | 702 | """ | ||
1496 | 703 | # NOTE(beisner): rabbitmqctl cluster_status output is not | ||
1497 | 704 | # json-parsable, do string chop foo, then json.loads that. | ||
1498 | 705 | str_stat = self.get_rmq_cluster_status(sentry_unit) | ||
1499 | 706 | if 'running_nodes' in str_stat: | ||
1500 | 707 | pos_start = str_stat.find("{running_nodes,") + 15 | ||
1501 | 708 | pos_end = str_stat.find("]},", pos_start) + 1 | ||
1502 | 709 | str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"') | ||
1503 | 710 | run_nodes = json.loads(str_run_nodes) | ||
1504 | 711 | return run_nodes | ||
1505 | 712 | else: | ||
1506 | 713 | return [] | ||
1507 | 714 | |||
1508 | 715 | def validate_rmq_cluster_running_nodes(self, sentry_units): | ||
1509 | 716 | """Check that all rmq unit hostnames are represented in the | ||
1510 | 717 | cluster_status output of all units. | ||
1511 | 718 | |||
1512 | 719 | :param host_names: dict of juju unit names to host names | ||
1513 | 720 | :param units: list of sentry unit pointers (all rmq units) | ||
1514 | 721 | :returns: None if successful, otherwise return error message | ||
1515 | 722 | """ | ||
1516 | 723 | host_names = self.get_unit_hostnames(sentry_units) | ||
1517 | 724 | errors = [] | ||
1518 | 725 | |||
1519 | 726 | # Query every unit for cluster_status running nodes | ||
1520 | 727 | for query_unit in sentry_units: | ||
1521 | 728 | query_unit_name = query_unit.info['unit_name'] | ||
1522 | 729 | running_nodes = self.get_rmq_cluster_running_nodes(query_unit) | ||
1523 | 730 | |||
1524 | 731 | # Confirm that every unit is represented in the queried unit's | ||
1525 | 732 | # cluster_status running nodes output. | ||
1526 | 733 | for validate_unit in sentry_units: | ||
1527 | 734 | val_host_name = host_names[validate_unit.info['unit_name']] | ||
1528 | 735 | val_node_name = 'rabbit@{}'.format(val_host_name) | ||
1529 | 736 | |||
1530 | 737 | if val_node_name not in running_nodes: | ||
1531 | 738 | errors.append('Cluster member check failed on {}: {} not ' | ||
1532 | 739 | 'in {}\n'.format(query_unit_name, | ||
1533 | 740 | val_node_name, | ||
1534 | 741 | running_nodes)) | ||
1535 | 742 | if errors: | ||
1536 | 743 | return ''.join(errors) | ||
1537 | 744 | |||
1538 | 745 | def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None): | ||
1539 | 746 | """Check a single juju rmq unit for ssl and port in the config file.""" | ||
1540 | 747 | host = sentry_unit.info['public-address'] | ||
1541 | 748 | unit_name = sentry_unit.info['unit_name'] | ||
1542 | 749 | |||
1543 | 750 | conf_file = '/etc/rabbitmq/rabbitmq.config' | ||
1544 | 751 | conf_contents = str(self.file_contents_safe(sentry_unit, | ||
1545 | 752 | conf_file, max_wait=16)) | ||
1546 | 753 | # Checks | ||
1547 | 754 | conf_ssl = 'ssl' in conf_contents | ||
1548 | 755 | conf_port = str(port) in conf_contents | ||
1549 | 756 | |||
1550 | 757 | # Port explicitly checked in config | ||
1551 | 758 | if port and conf_port and conf_ssl: | ||
1552 | 759 | self.log.debug('SSL is enabled @{}:{} ' | ||
1553 | 760 | '({})'.format(host, port, unit_name)) | ||
1554 | 761 | return True | ||
1555 | 762 | elif port and not conf_port and conf_ssl: | ||
1556 | 763 | self.log.debug('SSL is enabled @{} but not on port {} ' | ||
1557 | 764 | '({})'.format(host, port, unit_name)) | ||
1558 | 765 | return False | ||
1559 | 766 | # Port not checked (useful when checking that ssl is disabled) | ||
1560 | 767 | elif not port and conf_ssl: | ||
1561 | 768 | self.log.debug('SSL is enabled @{}:{} ' | ||
1562 | 769 | '({})'.format(host, port, unit_name)) | ||
1563 | 770 | return True | ||
1564 | 771 | elif not conf_ssl: | ||
1565 | 772 | self.log.debug('SSL not enabled @{}:{} ' | ||
1566 | 773 | '({})'.format(host, port, unit_name)) | ||
1567 | 774 | return False | ||
1568 | 775 | else: | ||
1569 | 776 | msg = ('Unknown condition when checking SSL status @{}:{} ' | ||
1570 | 777 | '({})'.format(host, port, unit_name)) | ||
1571 | 778 | amulet.raise_status(amulet.FAIL, msg) | ||
1572 | 779 | |||
1573 | 780 | def validate_rmq_ssl_enabled_units(self, sentry_units, port=None): | ||
1574 | 781 | """Check that ssl is enabled on rmq juju sentry units. | ||
1575 | 782 | |||
1576 | 783 | :param sentry_units: list of all rmq sentry units | ||
1577 | 784 | :param port: optional ssl port override to validate | ||
1578 | 785 | :returns: None if successful, otherwise return error message | ||
1579 | 786 | """ | ||
1580 | 787 | for sentry_unit in sentry_units: | ||
1581 | 788 | if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port): | ||
1582 | 789 | return ('Unexpected condition: ssl is disabled on unit ' | ||
1583 | 790 | '({})'.format(sentry_unit.info['unit_name'])) | ||
1584 | 791 | return None | ||
1585 | 792 | |||
1586 | 793 | def validate_rmq_ssl_disabled_units(self, sentry_units): | ||
1587 | 794 | """Check that ssl is enabled on listed rmq juju sentry units. | ||
1588 | 795 | |||
1589 | 796 | :param sentry_units: list of all rmq sentry units | ||
1590 | 797 | :returns: True if successful. Raise on error. | ||
1591 | 798 | """ | ||
1592 | 799 | for sentry_unit in sentry_units: | ||
1593 | 800 | if self.rmq_ssl_is_enabled_on_unit(sentry_unit): | ||
1594 | 801 | return ('Unexpected condition: ssl is enabled on unit ' | ||
1595 | 802 | '({})'.format(sentry_unit.info['unit_name'])) | ||
1596 | 803 | return None | ||
1597 | 804 | |||
1598 | 805 | def configure_rmq_ssl_on(self, sentry_units, deployment, | ||
1599 | 806 | port=None, max_wait=60): | ||
1600 | 807 | """Turn ssl charm config option on, with optional non-default | ||
1601 | 808 | ssl port specification. Confirm that it is enabled on every | ||
1602 | 809 | unit. | ||
1603 | 810 | |||
1604 | 811 | :param sentry_units: list of sentry units | ||
1605 | 812 | :param deployment: amulet deployment object pointer | ||
1606 | 813 | :param port: amqp port, use defaults if None | ||
1607 | 814 | :param max_wait: maximum time to wait in seconds to confirm | ||
1608 | 815 | :returns: None if successful. Raise on error. | ||
1609 | 816 | """ | ||
1610 | 817 | self.log.debug('Setting ssl charm config option: on') | ||
1611 | 818 | |||
1612 | 819 | # Enable RMQ SSL | ||
1613 | 820 | config = {'ssl': 'on'} | ||
1614 | 821 | if port: | ||
1615 | 822 | config['ssl_port'] = port | ||
1616 | 823 | |||
1617 | 824 | deployment.d.configure('rabbitmq-server', config) | ||
1618 | 825 | |||
1619 | 826 | # Wait for unit status | ||
1620 | 827 | self.rmq_wait_for_cluster(deployment) | ||
1621 | 828 | |||
1622 | 829 | # Confirm | ||
1623 | 830 | tries = 0 | ||
1624 | 831 | ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port) | ||
1625 | 832 | while ret and tries < (max_wait / 4): | ||
1626 | 833 | time.sleep(4) | ||
1627 | 834 | self.log.debug('Attempt {}: {}'.format(tries, ret)) | ||
1628 | 835 | ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port) | ||
1629 | 836 | tries += 1 | ||
1630 | 837 | |||
1631 | 838 | if ret: | ||
1632 | 839 | amulet.raise_status(amulet.FAIL, ret) | ||
1633 | 840 | |||
1634 | 841 | def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60): | ||
1635 | 842 | """Turn ssl charm config option off, confirm that it is disabled | ||
1636 | 843 | on every unit. | ||
1637 | 844 | |||
1638 | 845 | :param sentry_units: list of sentry units | ||
1639 | 846 | :param deployment: amulet deployment object pointer | ||
1640 | 847 | :param max_wait: maximum time to wait in seconds to confirm | ||
1641 | 848 | :returns: None if successful. Raise on error. | ||
1642 | 849 | """ | ||
1643 | 850 | self.log.debug('Setting ssl charm config option: off') | ||
1644 | 851 | |||
1645 | 852 | # Disable RMQ SSL | ||
1646 | 853 | config = {'ssl': 'off'} | ||
1647 | 854 | deployment.d.configure('rabbitmq-server', config) | ||
1648 | 855 | |||
1649 | 856 | # Wait for unit status | ||
1650 | 857 | self.rmq_wait_for_cluster(deployment) | ||
1651 | 858 | |||
1652 | 859 | # Confirm | ||
1653 | 860 | tries = 0 | ||
1654 | 861 | ret = self.validate_rmq_ssl_disabled_units(sentry_units) | ||
1655 | 862 | while ret and tries < (max_wait / 4): | ||
1656 | 863 | time.sleep(4) | ||
1657 | 864 | self.log.debug('Attempt {}: {}'.format(tries, ret)) | ||
1658 | 865 | ret = self.validate_rmq_ssl_disabled_units(sentry_units) | ||
1659 | 866 | tries += 1 | ||
1660 | 867 | |||
1661 | 868 | if ret: | ||
1662 | 869 | amulet.raise_status(amulet.FAIL, ret) | ||
1663 | 870 | |||
1664 | 871 | def connect_amqp_by_unit(self, sentry_unit, ssl=False, | ||
1665 | 872 | port=None, fatal=True, | ||
1666 | 873 | username="testuser1", password="changeme"): | ||
1667 | 874 | """Establish and return a pika amqp connection to the rabbitmq service | ||
1668 | 875 | running on a rmq juju unit. | ||
1669 | 876 | |||
1670 | 877 | :param sentry_unit: sentry unit pointer | ||
1671 | 878 | :param ssl: boolean, default to False | ||
1672 | 879 | :param port: amqp port, use defaults if None | ||
1673 | 880 | :param fatal: boolean, default to True (raises on connect error) | ||
1674 | 881 | :param username: amqp user name, default to testuser1 | ||
1675 | 882 | :param password: amqp user password | ||
1676 | 883 | :returns: pika amqp connection pointer or None if failed and non-fatal | ||
1677 | 884 | """ | ||
1678 | 885 | host = sentry_unit.info['public-address'] | ||
1679 | 886 | unit_name = sentry_unit.info['unit_name'] | ||
1680 | 887 | |||
1681 | 888 | # Default port logic if port is not specified | ||
1682 | 889 | if ssl and not port: | ||
1683 | 890 | port = 5671 | ||
1684 | 891 | elif not ssl and not port: | ||
1685 | 892 | port = 5672 | ||
1686 | 893 | |||
1687 | 894 | self.log.debug('Connecting to amqp on {}:{} ({}) as ' | ||
1688 | 895 | '{}...'.format(host, port, unit_name, username)) | ||
1689 | 896 | |||
1690 | 897 | try: | ||
1691 | 898 | credentials = pika.PlainCredentials(username, password) | ||
1692 | 899 | parameters = pika.ConnectionParameters(host=host, port=port, | ||
1693 | 900 | credentials=credentials, | ||
1694 | 901 | ssl=ssl, | ||
1695 | 902 | connection_attempts=3, | ||
1696 | 903 | retry_delay=5, | ||
1697 | 904 | socket_timeout=1) | ||
1698 | 905 | connection = pika.BlockingConnection(parameters) | ||
1699 | 906 | assert connection.server_properties['product'] == 'RabbitMQ' | ||
1700 | 907 | self.log.debug('Connect OK') | ||
1701 | 908 | return connection | ||
1702 | 909 | except Exception as e: | ||
1703 | 910 | msg = ('amqp connection failed to {}:{} as ' | ||
1704 | 911 | '{} ({})'.format(host, port, username, str(e))) | ||
1705 | 912 | if fatal: | ||
1706 | 913 | amulet.raise_status(amulet.FAIL, msg) | ||
1707 | 914 | else: | ||
1708 | 915 | self.log.warn(msg) | ||
1709 | 916 | return None | ||
1710 | 917 | |||
1711 | 918 | def publish_amqp_message_by_unit(self, sentry_unit, message, | ||
1712 | 919 | queue="test", ssl=False, | ||
1713 | 920 | username="testuser1", | ||
1714 | 921 | password="changeme", | ||
1715 | 922 | port=None): | ||
1716 | 923 | """Publish an amqp message to a rmq juju unit. | ||
1717 | 924 | |||
1718 | 925 | :param sentry_unit: sentry unit pointer | ||
1719 | 926 | :param message: amqp message string | ||
1720 | 927 | :param queue: message queue, default to test | ||
1721 | 928 | :param username: amqp user name, default to testuser1 | ||
1722 | 929 | :param password: amqp user password | ||
1723 | 930 | :param ssl: boolean, default to False | ||
1724 | 931 | :param port: amqp port, use defaults if None | ||
1725 | 932 | :returns: None. Raises exception if publish failed. | ||
1726 | 933 | """ | ||
1727 | 934 | self.log.debug('Publishing message to {} queue:\n{}'.format(queue, | ||
1728 | 935 | message)) | ||
1729 | 936 | connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl, | ||
1730 | 937 | port=port, | ||
1731 | 938 | username=username, | ||
1732 | 939 | password=password) | ||
1733 | 940 | |||
1734 | 941 | # NOTE(beisner): extra debug here re: pika hang potential: | ||
1735 | 942 | # https://github.com/pika/pika/issues/297 | ||
1736 | 943 | # https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw | ||
1737 | 944 | self.log.debug('Defining channel...') | ||
1738 | 945 | channel = connection.channel() | ||
1739 | 946 | self.log.debug('Declaring queue...') | ||
1740 | 947 | channel.queue_declare(queue=queue, auto_delete=False, durable=True) | ||
1741 | 948 | self.log.debug('Publishing message...') | ||
1742 | 949 | channel.basic_publish(exchange='', routing_key=queue, body=message) | ||
1743 | 950 | self.log.debug('Closing channel...') | ||
1744 | 951 | channel.close() | ||
1745 | 952 | self.log.debug('Closing connection...') | ||
1746 | 953 | connection.close() | ||
1747 | 954 | |||
1748 | 955 | def get_amqp_message_by_unit(self, sentry_unit, queue="test", | ||
1749 | 956 | username="testuser1", | ||
1750 | 957 | password="changeme", | ||
1751 | 958 | ssl=False, port=None): | ||
1752 | 959 | """Get an amqp message from a rmq juju unit. | ||
1753 | 960 | |||
1754 | 961 | :param sentry_unit: sentry unit pointer | ||
1755 | 962 | :param queue: message queue, default to test | ||
1756 | 963 | :param username: amqp user name, default to testuser1 | ||
1757 | 964 | :param password: amqp user password | ||
1758 | 965 | :param ssl: boolean, default to False | ||
1759 | 966 | :param port: amqp port, use defaults if None | ||
1760 | 967 | :returns: amqp message body as string. Raise if get fails. | ||
1761 | 968 | """ | ||
1762 | 969 | connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl, | ||
1763 | 970 | port=port, | ||
1764 | 971 | username=username, | ||
1765 | 972 | password=password) | ||
1766 | 973 | channel = connection.channel() | ||
1767 | 974 | method_frame, _, body = channel.basic_get(queue) | ||
1768 | 975 | |||
1769 | 976 | if method_frame: | ||
1770 | 977 | self.log.debug('Retreived message from {} queue:\n{}'.format(queue, | ||
1771 | 978 | body)) | ||
1772 | 979 | channel.basic_ack(method_frame.delivery_tag) | ||
1773 | 980 | channel.close() | ||
1774 | 981 | connection.close() | ||
1775 | 982 | return body | ||
1776 | 983 | else: | ||
1777 | 984 | msg = 'No message retrieved.' | ||
1778 | 985 | amulet.raise_status(amulet.FAIL, msg) |
charm_unit_test #12559 ceph-osd-next for chris.macnaughton mp277135
UNIT OK: passed
Build: http:// 10.245. 162.77: 8080/job/ charm_unit_ test/12559/