Merge lp:~1chb1n/charms/trusty/ceph-osd/next-ch-sync-mitaka into lp:~openstack-charmers-archive/charms/trusty/ceph-osd/next
- Trusty Tahr (14.04)
- next-ch-sync-mitaka
- Merge into next
Proposed by
Ryan Beisner
Status: | Merged |
---|---|
Merged at revision: | 59 |
Proposed branch: | lp:~1chb1n/charms/trusty/ceph-osd/next-ch-sync-mitaka |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/ceph-osd/next |
Diff against target: |
2384 lines (+1314/-241) 18 files modified
hooks/charmhelpers/cli/__init__.py (+3/-3) hooks/charmhelpers/contrib/charmsupport/nrpe.py (+52/-14) hooks/charmhelpers/contrib/network/ip.py (+26/-22) hooks/charmhelpers/core/hookenv.py (+84/-4) hooks/charmhelpers/core/host.py (+153/-50) hooks/charmhelpers/core/hugepage.py (+10/-1) hooks/charmhelpers/core/kernel.py (+68/-0) hooks/charmhelpers/core/services/helpers.py (+14/-5) hooks/charmhelpers/core/strutils.py (+30/-0) hooks/charmhelpers/core/templating.py (+21/-8) hooks/charmhelpers/fetch/__init__.py (+10/-2) hooks/charmhelpers/fetch/archiveurl.py (+1/-1) hooks/charmhelpers/fetch/bzrurl.py (+22/-32) hooks/charmhelpers/fetch/giturl.py (+20/-23) tests/charmhelpers/contrib/amulet/deployment.py (+4/-2) tests/charmhelpers/contrib/amulet/utils.py (+284/-62) tests/charmhelpers/contrib/openstack/amulet/deployment.py (+131/-12) tests/charmhelpers/contrib/openstack/amulet/utils.py (+381/-0) |
To merge this branch: | bzr merge lp:~1chb1n/charms/trusty/ceph-osd/next-ch-sync-mitaka |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Corey Bryant (community) | Approve | ||
Review via email: mp+283704@code.launchpad.net |
Commit message
sync charm helpers for mitaka cloud archive recognition
Description of the change
sync charm helpers for mitaka cloud archive recognition
To post a comment you must log in.
- 60. By Ryan Beisner
-
enable liberty amulet test targets
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #17960 ceph-osd-next for 1chb1n mp283704
LINT OK: passed
Build: http://
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #8982 ceph-osd-next for 1chb1n mp283704
AMULET OK: passed
Build: http://
Revision history for this message
Corey Bryant (corey.bryant) : | # |
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'hooks/charmhelpers/cli/__init__.py' | |||
2 | --- hooks/charmhelpers/cli/__init__.py 2015-08-19 13:50:42 +0000 | |||
3 | +++ hooks/charmhelpers/cli/__init__.py 2016-01-22 22:32:27 +0000 | |||
4 | @@ -20,7 +20,7 @@ | |||
5 | 20 | 20 | ||
6 | 21 | from six.moves import zip | 21 | from six.moves import zip |
7 | 22 | 22 | ||
9 | 23 | from charmhelpers.core import unitdata | 23 | import charmhelpers.core.unitdata |
10 | 24 | 24 | ||
11 | 25 | 25 | ||
12 | 26 | class OutputFormatter(object): | 26 | class OutputFormatter(object): |
13 | @@ -163,8 +163,8 @@ | |||
14 | 163 | if getattr(arguments.func, '_cli_no_output', False): | 163 | if getattr(arguments.func, '_cli_no_output', False): |
15 | 164 | output = '' | 164 | output = '' |
16 | 165 | self.formatter.format_output(output, arguments.format) | 165 | self.formatter.format_output(output, arguments.format) |
19 | 166 | if unitdata._KV: | 166 | if charmhelpers.core.unitdata._KV: |
20 | 167 | unitdata._KV.flush() | 167 | charmhelpers.core.unitdata._KV.flush() |
21 | 168 | 168 | ||
22 | 169 | 169 | ||
23 | 170 | cmdline = CommandLine() | 170 | cmdline = CommandLine() |
24 | 171 | 171 | ||
25 | === modified file 'hooks/charmhelpers/contrib/charmsupport/nrpe.py' | |||
26 | --- hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-06-17 16:59:15 +0000 | |||
27 | +++ hooks/charmhelpers/contrib/charmsupport/nrpe.py 2016-01-22 22:32:27 +0000 | |||
28 | @@ -148,6 +148,13 @@ | |||
29 | 148 | self.description = description | 148 | self.description = description |
30 | 149 | self.check_cmd = self._locate_cmd(check_cmd) | 149 | self.check_cmd = self._locate_cmd(check_cmd) |
31 | 150 | 150 | ||
32 | 151 | def _get_check_filename(self): | ||
33 | 152 | return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command)) | ||
34 | 153 | |||
35 | 154 | def _get_service_filename(self, hostname): | ||
36 | 155 | return os.path.join(NRPE.nagios_exportdir, | ||
37 | 156 | 'service__{}_{}.cfg'.format(hostname, self.command)) | ||
38 | 157 | |||
39 | 151 | def _locate_cmd(self, check_cmd): | 158 | def _locate_cmd(self, check_cmd): |
40 | 152 | search_path = ( | 159 | search_path = ( |
41 | 153 | '/usr/lib/nagios/plugins', | 160 | '/usr/lib/nagios/plugins', |
42 | @@ -163,9 +170,21 @@ | |||
43 | 163 | log('Check command not found: {}'.format(parts[0])) | 170 | log('Check command not found: {}'.format(parts[0])) |
44 | 164 | return '' | 171 | return '' |
45 | 165 | 172 | ||
46 | 173 | def _remove_service_files(self): | ||
47 | 174 | if not os.path.exists(NRPE.nagios_exportdir): | ||
48 | 175 | return | ||
49 | 176 | for f in os.listdir(NRPE.nagios_exportdir): | ||
50 | 177 | if f.endswith('_{}.cfg'.format(self.command)): | ||
51 | 178 | os.remove(os.path.join(NRPE.nagios_exportdir, f)) | ||
52 | 179 | |||
53 | 180 | def remove(self, hostname): | ||
54 | 181 | nrpe_check_file = self._get_check_filename() | ||
55 | 182 | if os.path.exists(nrpe_check_file): | ||
56 | 183 | os.remove(nrpe_check_file) | ||
57 | 184 | self._remove_service_files() | ||
58 | 185 | |||
59 | 166 | def write(self, nagios_context, hostname, nagios_servicegroups): | 186 | def write(self, nagios_context, hostname, nagios_servicegroups): |
62 | 167 | nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( | 187 | nrpe_check_file = self._get_check_filename() |
61 | 168 | self.command) | ||
63 | 169 | with open(nrpe_check_file, 'w') as nrpe_check_config: | 188 | with open(nrpe_check_file, 'w') as nrpe_check_config: |
64 | 170 | nrpe_check_config.write("# check {}\n".format(self.shortname)) | 189 | nrpe_check_config.write("# check {}\n".format(self.shortname)) |
65 | 171 | nrpe_check_config.write("command[{}]={}\n".format( | 190 | nrpe_check_config.write("command[{}]={}\n".format( |
66 | @@ -180,9 +199,7 @@ | |||
67 | 180 | 199 | ||
68 | 181 | def write_service_config(self, nagios_context, hostname, | 200 | def write_service_config(self, nagios_context, hostname, |
69 | 182 | nagios_servicegroups): | 201 | nagios_servicegroups): |
73 | 183 | for f in os.listdir(NRPE.nagios_exportdir): | 202 | self._remove_service_files() |
71 | 184 | if re.search('.*{}.cfg'.format(self.command), f): | ||
72 | 185 | os.remove(os.path.join(NRPE.nagios_exportdir, f)) | ||
74 | 186 | 203 | ||
75 | 187 | templ_vars = { | 204 | templ_vars = { |
76 | 188 | 'nagios_hostname': hostname, | 205 | 'nagios_hostname': hostname, |
77 | @@ -192,8 +209,7 @@ | |||
78 | 192 | 'command': self.command, | 209 | 'command': self.command, |
79 | 193 | } | 210 | } |
80 | 194 | nrpe_service_text = Check.service_template.format(**templ_vars) | 211 | nrpe_service_text = Check.service_template.format(**templ_vars) |
83 | 195 | nrpe_service_file = '{}/service__{}_{}.cfg'.format( | 212 | nrpe_service_file = self._get_service_filename(hostname) |
82 | 196 | NRPE.nagios_exportdir, hostname, self.command) | ||
84 | 197 | with open(nrpe_service_file, 'w') as nrpe_service_config: | 213 | with open(nrpe_service_file, 'w') as nrpe_service_config: |
85 | 198 | nrpe_service_config.write(str(nrpe_service_text)) | 214 | nrpe_service_config.write(str(nrpe_service_text)) |
86 | 199 | 215 | ||
87 | @@ -218,12 +234,32 @@ | |||
88 | 218 | if hostname: | 234 | if hostname: |
89 | 219 | self.hostname = hostname | 235 | self.hostname = hostname |
90 | 220 | else: | 236 | else: |
92 | 221 | self.hostname = "{}-{}".format(self.nagios_context, self.unit_name) | 237 | nagios_hostname = get_nagios_hostname() |
93 | 238 | if nagios_hostname: | ||
94 | 239 | self.hostname = nagios_hostname | ||
95 | 240 | else: | ||
96 | 241 | self.hostname = "{}-{}".format(self.nagios_context, self.unit_name) | ||
97 | 222 | self.checks = [] | 242 | self.checks = [] |
98 | 223 | 243 | ||
99 | 224 | def add_check(self, *args, **kwargs): | 244 | def add_check(self, *args, **kwargs): |
100 | 225 | self.checks.append(Check(*args, **kwargs)) | 245 | self.checks.append(Check(*args, **kwargs)) |
101 | 226 | 246 | ||
102 | 247 | def remove_check(self, *args, **kwargs): | ||
103 | 248 | if kwargs.get('shortname') is None: | ||
104 | 249 | raise ValueError('shortname of check must be specified') | ||
105 | 250 | |||
106 | 251 | # Use sensible defaults if they're not specified - these are not | ||
107 | 252 | # actually used during removal, but they're required for constructing | ||
108 | 253 | # the Check object; check_disk is chosen because it's part of the | ||
109 | 254 | # nagios-plugins-basic package. | ||
110 | 255 | if kwargs.get('check_cmd') is None: | ||
111 | 256 | kwargs['check_cmd'] = 'check_disk' | ||
112 | 257 | if kwargs.get('description') is None: | ||
113 | 258 | kwargs['description'] = '' | ||
114 | 259 | |||
115 | 260 | check = Check(*args, **kwargs) | ||
116 | 261 | check.remove(self.hostname) | ||
117 | 262 | |||
118 | 227 | def write(self): | 263 | def write(self): |
119 | 228 | try: | 264 | try: |
120 | 229 | nagios_uid = pwd.getpwnam('nagios').pw_uid | 265 | nagios_uid = pwd.getpwnam('nagios').pw_uid |
121 | @@ -260,7 +296,7 @@ | |||
122 | 260 | :param str relation_name: Name of relation nrpe sub joined to | 296 | :param str relation_name: Name of relation nrpe sub joined to |
123 | 261 | """ | 297 | """ |
124 | 262 | for rel in relations_of_type(relation_name): | 298 | for rel in relations_of_type(relation_name): |
126 | 263 | if 'nagios_hostname' in rel: | 299 | if 'nagios_host_context' in rel: |
127 | 264 | return rel['nagios_host_context'] | 300 | return rel['nagios_host_context'] |
128 | 265 | 301 | ||
129 | 266 | 302 | ||
130 | @@ -301,11 +337,13 @@ | |||
131 | 301 | upstart_init = '/etc/init/%s.conf' % svc | 337 | upstart_init = '/etc/init/%s.conf' % svc |
132 | 302 | sysv_init = '/etc/init.d/%s' % svc | 338 | sysv_init = '/etc/init.d/%s' % svc |
133 | 303 | if os.path.exists(upstart_init): | 339 | if os.path.exists(upstart_init): |
139 | 304 | nrpe.add_check( | 340 | # Don't add a check for these services from neutron-gateway |
140 | 305 | shortname=svc, | 341 | if svc not in ['ext-port', 'os-charm-phy-nic-mtu']: |
141 | 306 | description='process check {%s}' % unit_name, | 342 | nrpe.add_check( |
142 | 307 | check_cmd='check_upstart_job %s' % svc | 343 | shortname=svc, |
143 | 308 | ) | 344 | description='process check {%s}' % unit_name, |
144 | 345 | check_cmd='check_upstart_job %s' % svc | ||
145 | 346 | ) | ||
146 | 309 | elif os.path.exists(sysv_init): | 347 | elif os.path.exists(sysv_init): |
147 | 310 | cronpath = '/etc/cron.d/nagios-service-check-%s' % svc | 348 | cronpath = '/etc/cron.d/nagios-service-check-%s' % svc |
148 | 311 | cron_file = ('*/5 * * * * root ' | 349 | cron_file = ('*/5 * * * * root ' |
149 | 312 | 350 | ||
150 | === modified file 'hooks/charmhelpers/contrib/network/ip.py' | |||
151 | --- hooks/charmhelpers/contrib/network/ip.py 2015-09-03 09:42:18 +0000 | |||
152 | +++ hooks/charmhelpers/contrib/network/ip.py 2016-01-22 22:32:27 +0000 | |||
153 | @@ -23,7 +23,7 @@ | |||
154 | 23 | from functools import partial | 23 | from functools import partial |
155 | 24 | 24 | ||
156 | 25 | from charmhelpers.core.hookenv import unit_get | 25 | from charmhelpers.core.hookenv import unit_get |
158 | 26 | from charmhelpers.fetch import apt_install | 26 | from charmhelpers.fetch import apt_install, apt_update |
159 | 27 | from charmhelpers.core.hookenv import ( | 27 | from charmhelpers.core.hookenv import ( |
160 | 28 | log, | 28 | log, |
161 | 29 | WARNING, | 29 | WARNING, |
162 | @@ -32,13 +32,15 @@ | |||
163 | 32 | try: | 32 | try: |
164 | 33 | import netifaces | 33 | import netifaces |
165 | 34 | except ImportError: | 34 | except ImportError: |
167 | 35 | apt_install('python-netifaces') | 35 | apt_update(fatal=True) |
168 | 36 | apt_install('python-netifaces', fatal=True) | ||
169 | 36 | import netifaces | 37 | import netifaces |
170 | 37 | 38 | ||
171 | 38 | try: | 39 | try: |
172 | 39 | import netaddr | 40 | import netaddr |
173 | 40 | except ImportError: | 41 | except ImportError: |
175 | 41 | apt_install('python-netaddr') | 42 | apt_update(fatal=True) |
176 | 43 | apt_install('python-netaddr', fatal=True) | ||
177 | 42 | import netaddr | 44 | import netaddr |
178 | 43 | 45 | ||
179 | 44 | 46 | ||
180 | @@ -51,7 +53,7 @@ | |||
181 | 51 | 53 | ||
182 | 52 | 54 | ||
183 | 53 | def no_ip_found_error_out(network): | 55 | def no_ip_found_error_out(network): |
185 | 54 | errmsg = ("No IP address found in network: %s" % network) | 56 | errmsg = ("No IP address found in network(s): %s" % network) |
186 | 55 | raise ValueError(errmsg) | 57 | raise ValueError(errmsg) |
187 | 56 | 58 | ||
188 | 57 | 59 | ||
189 | @@ -59,7 +61,7 @@ | |||
190 | 59 | """Get an IPv4 or IPv6 address within the network from the host. | 61 | """Get an IPv4 or IPv6 address within the network from the host. |
191 | 60 | 62 | ||
192 | 61 | :param network (str): CIDR presentation format. For example, | 63 | :param network (str): CIDR presentation format. For example, |
194 | 62 | '192.168.1.0/24'. | 64 | '192.168.1.0/24'. Supports multiple networks as a space-delimited list. |
195 | 63 | :param fallback (str): If no address is found, return fallback. | 65 | :param fallback (str): If no address is found, return fallback. |
196 | 64 | :param fatal (boolean): If no address is found, fallback is not | 66 | :param fatal (boolean): If no address is found, fallback is not |
197 | 65 | set and fatal is True then exit(1). | 67 | set and fatal is True then exit(1). |
198 | @@ -73,24 +75,26 @@ | |||
199 | 73 | else: | 75 | else: |
200 | 74 | return None | 76 | return None |
201 | 75 | 77 | ||
212 | 76 | _validate_cidr(network) | 78 | networks = network.split() or [network] |
213 | 77 | network = netaddr.IPNetwork(network) | 79 | for network in networks: |
214 | 78 | for iface in netifaces.interfaces(): | 80 | _validate_cidr(network) |
215 | 79 | addresses = netifaces.ifaddresses(iface) | 81 | network = netaddr.IPNetwork(network) |
216 | 80 | if network.version == 4 and netifaces.AF_INET in addresses: | 82 | for iface in netifaces.interfaces(): |
217 | 81 | addr = addresses[netifaces.AF_INET][0]['addr'] | 83 | addresses = netifaces.ifaddresses(iface) |
218 | 82 | netmask = addresses[netifaces.AF_INET][0]['netmask'] | 84 | if network.version == 4 and netifaces.AF_INET in addresses: |
219 | 83 | cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) | 85 | addr = addresses[netifaces.AF_INET][0]['addr'] |
220 | 84 | if cidr in network: | 86 | netmask = addresses[netifaces.AF_INET][0]['netmask'] |
221 | 85 | return str(cidr.ip) | 87 | cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask)) |
222 | 88 | if cidr in network: | ||
223 | 89 | return str(cidr.ip) | ||
224 | 86 | 90 | ||
232 | 87 | if network.version == 6 and netifaces.AF_INET6 in addresses: | 91 | if network.version == 6 and netifaces.AF_INET6 in addresses: |
233 | 88 | for addr in addresses[netifaces.AF_INET6]: | 92 | for addr in addresses[netifaces.AF_INET6]: |
234 | 89 | if not addr['addr'].startswith('fe80'): | 93 | if not addr['addr'].startswith('fe80'): |
235 | 90 | cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], | 94 | cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], |
236 | 91 | addr['netmask'])) | 95 | addr['netmask'])) |
237 | 92 | if cidr in network: | 96 | if cidr in network: |
238 | 93 | return str(cidr.ip) | 97 | return str(cidr.ip) |
239 | 94 | 98 | ||
240 | 95 | if fallback is not None: | 99 | if fallback is not None: |
241 | 96 | return fallback | 100 | return fallback |
242 | 97 | 101 | ||
243 | === modified file 'hooks/charmhelpers/core/hookenv.py' | |||
244 | --- hooks/charmhelpers/core/hookenv.py 2015-09-03 09:42:18 +0000 | |||
245 | +++ hooks/charmhelpers/core/hookenv.py 2016-01-22 22:32:27 +0000 | |||
246 | @@ -491,6 +491,19 @@ | |||
247 | 491 | 491 | ||
248 | 492 | 492 | ||
249 | 493 | @cached | 493 | @cached |
250 | 494 | def peer_relation_id(): | ||
251 | 495 | '''Get the peers relation id if a peers relation has been joined, else None.''' | ||
252 | 496 | md = metadata() | ||
253 | 497 | section = md.get('peers') | ||
254 | 498 | if section: | ||
255 | 499 | for key in section: | ||
256 | 500 | relids = relation_ids(key) | ||
257 | 501 | if relids: | ||
258 | 502 | return relids[0] | ||
259 | 503 | return None | ||
260 | 504 | |||
261 | 505 | |||
262 | 506 | @cached | ||
263 | 494 | def relation_to_interface(relation_name): | 507 | def relation_to_interface(relation_name): |
264 | 495 | """ | 508 | """ |
265 | 496 | Given the name of a relation, return the interface that relation uses. | 509 | Given the name of a relation, return the interface that relation uses. |
266 | @@ -504,12 +517,12 @@ | |||
267 | 504 | def relation_to_role_and_interface(relation_name): | 517 | def relation_to_role_and_interface(relation_name): |
268 | 505 | """ | 518 | """ |
269 | 506 | Given the name of a relation, return the role and the name of the interface | 519 | Given the name of a relation, return the role and the name of the interface |
271 | 507 | that relation uses (where role is one of ``provides``, ``requires``, or ``peer``). | 520 | that relation uses (where role is one of ``provides``, ``requires``, or ``peers``). |
272 | 508 | 521 | ||
273 | 509 | :returns: A tuple containing ``(role, interface)``, or ``(None, None)``. | 522 | :returns: A tuple containing ``(role, interface)``, or ``(None, None)``. |
274 | 510 | """ | 523 | """ |
275 | 511 | _metadata = metadata() | 524 | _metadata = metadata() |
277 | 512 | for role in ('provides', 'requires', 'peer'): | 525 | for role in ('provides', 'requires', 'peers'): |
278 | 513 | interface = _metadata.get(role, {}).get(relation_name, {}).get('interface') | 526 | interface = _metadata.get(role, {}).get(relation_name, {}).get('interface') |
279 | 514 | if interface: | 527 | if interface: |
280 | 515 | return role, interface | 528 | return role, interface |
281 | @@ -521,7 +534,7 @@ | |||
282 | 521 | """ | 534 | """ |
283 | 522 | Given a role and interface name, return a list of relation names for the | 535 | Given a role and interface name, return a list of relation names for the |
284 | 523 | current charm that use that interface under that role (where role is one | 536 | current charm that use that interface under that role (where role is one |
286 | 524 | of ``provides``, ``requires``, or ``peer``). | 537 | of ``provides``, ``requires``, or ``peers``). |
287 | 525 | 538 | ||
288 | 526 | :returns: A list of relation names. | 539 | :returns: A list of relation names. |
289 | 527 | """ | 540 | """ |
290 | @@ -542,7 +555,7 @@ | |||
291 | 542 | :returns: A list of relation names. | 555 | :returns: A list of relation names. |
292 | 543 | """ | 556 | """ |
293 | 544 | results = [] | 557 | results = [] |
295 | 545 | for role in ('provides', 'requires', 'peer'): | 558 | for role in ('provides', 'requires', 'peers'): |
296 | 546 | results.extend(role_and_interface_to_relations(role, interface_name)) | 559 | results.extend(role_and_interface_to_relations(role, interface_name)) |
297 | 547 | return results | 560 | return results |
298 | 548 | 561 | ||
299 | @@ -623,6 +636,38 @@ | |||
300 | 623 | return unit_get('private-address') | 636 | return unit_get('private-address') |
301 | 624 | 637 | ||
302 | 625 | 638 | ||
303 | 639 | @cached | ||
304 | 640 | def storage_get(attribute=None, storage_id=None): | ||
305 | 641 | """Get storage attributes""" | ||
306 | 642 | _args = ['storage-get', '--format=json'] | ||
307 | 643 | if storage_id: | ||
308 | 644 | _args.extend(('-s', storage_id)) | ||
309 | 645 | if attribute: | ||
310 | 646 | _args.append(attribute) | ||
311 | 647 | try: | ||
312 | 648 | return json.loads(subprocess.check_output(_args).decode('UTF-8')) | ||
313 | 649 | except ValueError: | ||
314 | 650 | return None | ||
315 | 651 | |||
316 | 652 | |||
317 | 653 | @cached | ||
318 | 654 | def storage_list(storage_name=None): | ||
319 | 655 | """List the storage IDs for the unit""" | ||
320 | 656 | _args = ['storage-list', '--format=json'] | ||
321 | 657 | if storage_name: | ||
322 | 658 | _args.append(storage_name) | ||
323 | 659 | try: | ||
324 | 660 | return json.loads(subprocess.check_output(_args).decode('UTF-8')) | ||
325 | 661 | except ValueError: | ||
326 | 662 | return None | ||
327 | 663 | except OSError as e: | ||
328 | 664 | import errno | ||
329 | 665 | if e.errno == errno.ENOENT: | ||
330 | 666 | # storage-list does not exist | ||
331 | 667 | return [] | ||
332 | 668 | raise | ||
333 | 669 | |||
334 | 670 | |||
335 | 626 | class UnregisteredHookError(Exception): | 671 | class UnregisteredHookError(Exception): |
336 | 627 | """Raised when an undefined hook is called""" | 672 | """Raised when an undefined hook is called""" |
337 | 628 | pass | 673 | pass |
338 | @@ -788,6 +833,7 @@ | |||
339 | 788 | 833 | ||
340 | 789 | def translate_exc(from_exc, to_exc): | 834 | def translate_exc(from_exc, to_exc): |
341 | 790 | def inner_translate_exc1(f): | 835 | def inner_translate_exc1(f): |
342 | 836 | @wraps(f) | ||
343 | 791 | def inner_translate_exc2(*args, **kwargs): | 837 | def inner_translate_exc2(*args, **kwargs): |
344 | 792 | try: | 838 | try: |
345 | 793 | return f(*args, **kwargs) | 839 | return f(*args, **kwargs) |
346 | @@ -832,6 +878,40 @@ | |||
347 | 832 | subprocess.check_call(cmd) | 878 | subprocess.check_call(cmd) |
348 | 833 | 879 | ||
349 | 834 | 880 | ||
350 | 881 | @translate_exc(from_exc=OSError, to_exc=NotImplementedError) | ||
351 | 882 | def payload_register(ptype, klass, pid): | ||
352 | 883 | """ is used while a hook is running to let Juju know that a | ||
353 | 884 | payload has been started.""" | ||
354 | 885 | cmd = ['payload-register'] | ||
355 | 886 | for x in [ptype, klass, pid]: | ||
356 | 887 | cmd.append(x) | ||
357 | 888 | subprocess.check_call(cmd) | ||
358 | 889 | |||
359 | 890 | |||
360 | 891 | @translate_exc(from_exc=OSError, to_exc=NotImplementedError) | ||
361 | 892 | def payload_unregister(klass, pid): | ||
362 | 893 | """ is used while a hook is running to let Juju know | ||
363 | 894 | that a payload has been manually stopped. The <class> and <id> provided | ||
364 | 895 | must match a payload that has been previously registered with juju using | ||
365 | 896 | payload-register.""" | ||
366 | 897 | cmd = ['payload-unregister'] | ||
367 | 898 | for x in [klass, pid]: | ||
368 | 899 | cmd.append(x) | ||
369 | 900 | subprocess.check_call(cmd) | ||
370 | 901 | |||
371 | 902 | |||
372 | 903 | @translate_exc(from_exc=OSError, to_exc=NotImplementedError) | ||
373 | 904 | def payload_status_set(klass, pid, status): | ||
374 | 905 | """is used to update the current status of a registered payload. | ||
375 | 906 | The <class> and <id> provided must match a payload that has been previously | ||
376 | 907 | registered with juju using payload-register. The <status> must be one of the | ||
377 | 908 | follow: starting, started, stopping, stopped""" | ||
378 | 909 | cmd = ['payload-status-set'] | ||
379 | 910 | for x in [klass, pid, status]: | ||
380 | 911 | cmd.append(x) | ||
381 | 912 | subprocess.check_call(cmd) | ||
382 | 913 | |||
383 | 914 | |||
384 | 835 | @cached | 915 | @cached |
385 | 836 | def juju_version(): | 916 | def juju_version(): |
386 | 837 | """Full version string (eg. '1.23.3.1-trusty-amd64')""" | 917 | """Full version string (eg. '1.23.3.1-trusty-amd64')""" |
387 | 838 | 918 | ||
388 | === modified file 'hooks/charmhelpers/core/host.py' | |||
389 | --- hooks/charmhelpers/core/host.py 2015-08-19 13:50:42 +0000 | |||
390 | +++ hooks/charmhelpers/core/host.py 2016-01-22 22:32:27 +0000 | |||
391 | @@ -63,55 +63,86 @@ | |||
392 | 63 | return service_result | 63 | return service_result |
393 | 64 | 64 | ||
394 | 65 | 65 | ||
396 | 66 | def service_pause(service_name, init_dir=None): | 66 | def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"): |
397 | 67 | """Pause a system service. | 67 | """Pause a system service. |
398 | 68 | 68 | ||
399 | 69 | Stop it, and prevent it from starting again at boot.""" | 69 | Stop it, and prevent it from starting again at boot.""" |
408 | 70 | if init_dir is None: | 70 | stopped = True |
409 | 71 | init_dir = "/etc/init" | 71 | if service_running(service_name): |
410 | 72 | stopped = service_stop(service_name) | 72 | stopped = service_stop(service_name) |
411 | 73 | # XXX: Support systemd too | 73 | upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) |
412 | 74 | override_path = os.path.join( | 74 | sysv_file = os.path.join(initd_dir, service_name) |
413 | 75 | init_dir, '{}.override'.format(service_name)) | 75 | if init_is_systemd(): |
414 | 76 | with open(override_path, 'w') as fh: | 76 | service('disable', service_name) |
415 | 77 | fh.write("manual\n") | 77 | elif os.path.exists(upstart_file): |
416 | 78 | override_path = os.path.join( | ||
417 | 79 | init_dir, '{}.override'.format(service_name)) | ||
418 | 80 | with open(override_path, 'w') as fh: | ||
419 | 81 | fh.write("manual\n") | ||
420 | 82 | elif os.path.exists(sysv_file): | ||
421 | 83 | subprocess.check_call(["update-rc.d", service_name, "disable"]) | ||
422 | 84 | else: | ||
423 | 85 | raise ValueError( | ||
424 | 86 | "Unable to detect {0} as SystemD, Upstart {1} or" | ||
425 | 87 | " SysV {2}".format( | ||
426 | 88 | service_name, upstart_file, sysv_file)) | ||
427 | 78 | return stopped | 89 | return stopped |
428 | 79 | 90 | ||
429 | 80 | 91 | ||
431 | 81 | def service_resume(service_name, init_dir=None): | 92 | def service_resume(service_name, init_dir="/etc/init", |
432 | 93 | initd_dir="/etc/init.d"): | ||
433 | 82 | """Resume a system service. | 94 | """Resume a system service. |
434 | 83 | 95 | ||
435 | 84 | Reenable starting again at boot. Start the service""" | 96 | Reenable starting again at boot. Start the service""" |
444 | 85 | # XXX: Support systemd too | 97 | upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) |
445 | 86 | if init_dir is None: | 98 | sysv_file = os.path.join(initd_dir, service_name) |
446 | 87 | init_dir = "/etc/init" | 99 | if init_is_systemd(): |
447 | 88 | override_path = os.path.join( | 100 | service('enable', service_name) |
448 | 89 | init_dir, '{}.override'.format(service_name)) | 101 | elif os.path.exists(upstart_file): |
449 | 90 | if os.path.exists(override_path): | 102 | override_path = os.path.join( |
450 | 91 | os.unlink(override_path) | 103 | init_dir, '{}.override'.format(service_name)) |
451 | 92 | started = service_start(service_name) | 104 | if os.path.exists(override_path): |
452 | 105 | os.unlink(override_path) | ||
453 | 106 | elif os.path.exists(sysv_file): | ||
454 | 107 | subprocess.check_call(["update-rc.d", service_name, "enable"]) | ||
455 | 108 | else: | ||
456 | 109 | raise ValueError( | ||
457 | 110 | "Unable to detect {0} as SystemD, Upstart {1} or" | ||
458 | 111 | " SysV {2}".format( | ||
459 | 112 | service_name, upstart_file, sysv_file)) | ||
460 | 113 | |||
461 | 114 | started = service_running(service_name) | ||
462 | 115 | if not started: | ||
463 | 116 | started = service_start(service_name) | ||
464 | 93 | return started | 117 | return started |
465 | 94 | 118 | ||
466 | 95 | 119 | ||
467 | 96 | def service(action, service_name): | 120 | def service(action, service_name): |
468 | 97 | """Control a system service""" | 121 | """Control a system service""" |
470 | 98 | cmd = ['service', service_name, action] | 122 | if init_is_systemd(): |
471 | 123 | cmd = ['systemctl', action, service_name] | ||
472 | 124 | else: | ||
473 | 125 | cmd = ['service', service_name, action] | ||
474 | 99 | return subprocess.call(cmd) == 0 | 126 | return subprocess.call(cmd) == 0 |
475 | 100 | 127 | ||
476 | 101 | 128 | ||
478 | 102 | def service_running(service): | 129 | def service_running(service_name): |
479 | 103 | """Determine whether a system service is running""" | 130 | """Determine whether a system service is running""" |
486 | 104 | try: | 131 | if init_is_systemd(): |
487 | 105 | output = subprocess.check_output( | 132 | return service('is-active', service_name) |
482 | 106 | ['service', service, 'status'], | ||
483 | 107 | stderr=subprocess.STDOUT).decode('UTF-8') | ||
484 | 108 | except subprocess.CalledProcessError: | ||
485 | 109 | return False | ||
488 | 110 | else: | 133 | else: |
492 | 111 | if ("start/running" in output or "is running" in output): | 134 | try: |
493 | 112 | return True | 135 | output = subprocess.check_output( |
494 | 113 | else: | 136 | ['service', service_name, 'status'], |
495 | 137 | stderr=subprocess.STDOUT).decode('UTF-8') | ||
496 | 138 | except subprocess.CalledProcessError: | ||
497 | 114 | return False | 139 | return False |
498 | 140 | else: | ||
499 | 141 | if ("start/running" in output or "is running" in output or | ||
500 | 142 | "up and running" in output): | ||
501 | 143 | return True | ||
502 | 144 | else: | ||
503 | 145 | return False | ||
504 | 115 | 146 | ||
505 | 116 | 147 | ||
506 | 117 | def service_available(service_name): | 148 | def service_available(service_name): |
507 | @@ -126,8 +157,29 @@ | |||
508 | 126 | return True | 157 | return True |
509 | 127 | 158 | ||
510 | 128 | 159 | ||
513 | 129 | def adduser(username, password=None, shell='/bin/bash', system_user=False): | 160 | SYSTEMD_SYSTEM = '/run/systemd/system' |
514 | 130 | """Add a user to the system""" | 161 | |
515 | 162 | |||
516 | 163 | def init_is_systemd(): | ||
517 | 164 | """Return True if the host system uses systemd, False otherwise.""" | ||
518 | 165 | return os.path.isdir(SYSTEMD_SYSTEM) | ||
519 | 166 | |||
520 | 167 | |||
521 | 168 | def adduser(username, password=None, shell='/bin/bash', system_user=False, | ||
522 | 169 | primary_group=None, secondary_groups=None): | ||
523 | 170 | """Add a user to the system. | ||
524 | 171 | |||
525 | 172 | Will log but otherwise succeed if the user already exists. | ||
526 | 173 | |||
527 | 174 | :param str username: Username to create | ||
528 | 175 | :param str password: Password for user; if ``None``, create a system user | ||
529 | 176 | :param str shell: The default shell for the user | ||
530 | 177 | :param bool system_user: Whether to create a login or system user | ||
531 | 178 | :param str primary_group: Primary group for user; defaults to username | ||
532 | 179 | :param list secondary_groups: Optional list of additional groups | ||
533 | 180 | |||
534 | 181 | :returns: The password database entry struct, as returned by `pwd.getpwnam` | ||
535 | 182 | """ | ||
536 | 131 | try: | 183 | try: |
537 | 132 | user_info = pwd.getpwnam(username) | 184 | user_info = pwd.getpwnam(username) |
538 | 133 | log('user {0} already exists!'.format(username)) | 185 | log('user {0} already exists!'.format(username)) |
539 | @@ -142,6 +194,16 @@ | |||
540 | 142 | '--shell', shell, | 194 | '--shell', shell, |
541 | 143 | '--password', password, | 195 | '--password', password, |
542 | 144 | ]) | 196 | ]) |
543 | 197 | if not primary_group: | ||
544 | 198 | try: | ||
545 | 199 | grp.getgrnam(username) | ||
546 | 200 | primary_group = username # avoid "group exists" error | ||
547 | 201 | except KeyError: | ||
548 | 202 | pass | ||
549 | 203 | if primary_group: | ||
550 | 204 | cmd.extend(['-g', primary_group]) | ||
551 | 205 | if secondary_groups: | ||
552 | 206 | cmd.extend(['-G', ','.join(secondary_groups)]) | ||
553 | 145 | cmd.append(username) | 207 | cmd.append(username) |
554 | 146 | subprocess.check_call(cmd) | 208 | subprocess.check_call(cmd) |
555 | 147 | user_info = pwd.getpwnam(username) | 209 | user_info = pwd.getpwnam(username) |
556 | @@ -239,14 +301,12 @@ | |||
557 | 239 | 301 | ||
558 | 240 | 302 | ||
559 | 241 | def fstab_remove(mp): | 303 | def fstab_remove(mp): |
562 | 242 | """Remove the given mountpoint entry from /etc/fstab | 304 | """Remove the given mountpoint entry from /etc/fstab""" |
561 | 243 | """ | ||
563 | 244 | return Fstab.remove_by_mountpoint(mp) | 305 | return Fstab.remove_by_mountpoint(mp) |
564 | 245 | 306 | ||
565 | 246 | 307 | ||
566 | 247 | def fstab_add(dev, mp, fs, options=None): | 308 | def fstab_add(dev, mp, fs, options=None): |
569 | 248 | """Adds the given device entry to the /etc/fstab file | 309 | """Adds the given device entry to the /etc/fstab file""" |
568 | 249 | """ | ||
570 | 250 | return Fstab.add(dev, mp, fs, options=options) | 310 | return Fstab.add(dev, mp, fs, options=options) |
571 | 251 | 311 | ||
572 | 252 | 312 | ||
573 | @@ -302,8 +362,7 @@ | |||
574 | 302 | 362 | ||
575 | 303 | 363 | ||
576 | 304 | def file_hash(path, hash_type='md5'): | 364 | def file_hash(path, hash_type='md5'): |
579 | 305 | """ | 365 | """Generate a hash checksum of the contents of 'path' or None if not found. |
578 | 306 | Generate a hash checksum of the contents of 'path' or None if not found. | ||
580 | 307 | 366 | ||
581 | 308 | :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`, | 367 | :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`, |
582 | 309 | such as md5, sha1, sha256, sha512, etc. | 368 | such as md5, sha1, sha256, sha512, etc. |
583 | @@ -318,10 +377,9 @@ | |||
584 | 318 | 377 | ||
585 | 319 | 378 | ||
586 | 320 | def path_hash(path): | 379 | def path_hash(path): |
591 | 321 | """ | 380 | """Generate a hash checksum of all files matching 'path'. Standard |
592 | 322 | Generate a hash checksum of all files matching 'path'. Standard wildcards | 381 | wildcards like '*' and '?' are supported, see documentation for the 'glob' |
593 | 323 | like '*' and '?' are supported, see documentation for the 'glob' module for | 382 | module for more information. |
590 | 324 | more information. | ||
594 | 325 | 383 | ||
595 | 326 | :return: dict: A { filename: hash } dictionary for all matched files. | 384 | :return: dict: A { filename: hash } dictionary for all matched files. |
596 | 327 | Empty if none found. | 385 | Empty if none found. |
597 | @@ -333,8 +391,7 @@ | |||
598 | 333 | 391 | ||
599 | 334 | 392 | ||
600 | 335 | def check_hash(path, checksum, hash_type='md5'): | 393 | def check_hash(path, checksum, hash_type='md5'): |
603 | 336 | """ | 394 | """Validate a file using a cryptographic checksum. |
602 | 337 | Validate a file using a cryptographic checksum. | ||
604 | 338 | 395 | ||
605 | 339 | :param str checksum: Value of the checksum used to validate the file. | 396 | :param str checksum: Value of the checksum used to validate the file. |
606 | 340 | :param str hash_type: Hash algorithm used to generate `checksum`. | 397 | :param str hash_type: Hash algorithm used to generate `checksum`. |
607 | @@ -349,6 +406,7 @@ | |||
608 | 349 | 406 | ||
609 | 350 | 407 | ||
610 | 351 | class ChecksumError(ValueError): | 408 | class ChecksumError(ValueError): |
611 | 409 | """A class derived from Value error to indicate the checksum failed.""" | ||
612 | 352 | pass | 410 | pass |
613 | 353 | 411 | ||
614 | 354 | 412 | ||
615 | @@ -454,7 +512,7 @@ | |||
616 | 454 | 512 | ||
617 | 455 | 513 | ||
618 | 456 | def list_nics(nic_type=None): | 514 | def list_nics(nic_type=None): |
620 | 457 | '''Return a list of nics of given type(s)''' | 515 | """Return a list of nics of given type(s)""" |
621 | 458 | if isinstance(nic_type, six.string_types): | 516 | if isinstance(nic_type, six.string_types): |
622 | 459 | int_types = [nic_type] | 517 | int_types = [nic_type] |
623 | 460 | else: | 518 | else: |
624 | @@ -496,12 +554,13 @@ | |||
625 | 496 | 554 | ||
626 | 497 | 555 | ||
627 | 498 | def set_nic_mtu(nic, mtu): | 556 | def set_nic_mtu(nic, mtu): |
629 | 499 | '''Set MTU on a network interface''' | 557 | """Set the Maximum Transmission Unit (MTU) on a network interface.""" |
630 | 500 | cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] | 558 | cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] |
631 | 501 | subprocess.check_call(cmd) | 559 | subprocess.check_call(cmd) |
632 | 502 | 560 | ||
633 | 503 | 561 | ||
634 | 504 | def get_nic_mtu(nic): | 562 | def get_nic_mtu(nic): |
635 | 563 | """Return the Maximum Transmission Unit (MTU) for a network interface.""" | ||
636 | 505 | cmd = ['ip', 'addr', 'show', nic] | 564 | cmd = ['ip', 'addr', 'show', nic] |
637 | 506 | ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') | 565 | ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') |
638 | 507 | mtu = "" | 566 | mtu = "" |
639 | @@ -513,6 +572,7 @@ | |||
640 | 513 | 572 | ||
641 | 514 | 573 | ||
642 | 515 | def get_nic_hwaddr(nic): | 574 | def get_nic_hwaddr(nic): |
643 | 575 | """Return the Media Access Control (MAC) for a network interface.""" | ||
644 | 516 | cmd = ['ip', '-o', '-0', 'addr', 'show', nic] | 576 | cmd = ['ip', '-o', '-0', 'addr', 'show', nic] |
645 | 517 | ip_output = subprocess.check_output(cmd).decode('UTF-8') | 577 | ip_output = subprocess.check_output(cmd).decode('UTF-8') |
646 | 518 | hwaddr = "" | 578 | hwaddr = "" |
647 | @@ -523,7 +583,7 @@ | |||
648 | 523 | 583 | ||
649 | 524 | 584 | ||
650 | 525 | def cmp_pkgrevno(package, revno, pkgcache=None): | 585 | def cmp_pkgrevno(package, revno, pkgcache=None): |
652 | 526 | '''Compare supplied revno with the revno of the installed package | 586 | """Compare supplied revno with the revno of the installed package |
653 | 527 | 587 | ||
654 | 528 | * 1 => Installed revno is greater than supplied arg | 588 | * 1 => Installed revno is greater than supplied arg |
655 | 529 | * 0 => Installed revno is the same as supplied arg | 589 | * 0 => Installed revno is the same as supplied arg |
656 | @@ -532,7 +592,7 @@ | |||
657 | 532 | This function imports apt_cache function from charmhelpers.fetch if | 592 | This function imports apt_cache function from charmhelpers.fetch if |
658 | 533 | the pkgcache argument is None. Be sure to add charmhelpers.fetch if | 593 | the pkgcache argument is None. Be sure to add charmhelpers.fetch if |
659 | 534 | you call this function, or pass an apt_pkg.Cache() instance. | 594 | you call this function, or pass an apt_pkg.Cache() instance. |
661 | 535 | ''' | 595 | """ |
662 | 536 | import apt_pkg | 596 | import apt_pkg |
663 | 537 | if not pkgcache: | 597 | if not pkgcache: |
664 | 538 | from charmhelpers.fetch import apt_cache | 598 | from charmhelpers.fetch import apt_cache |
665 | @@ -542,15 +602,30 @@ | |||
666 | 542 | 602 | ||
667 | 543 | 603 | ||
668 | 544 | @contextmanager | 604 | @contextmanager |
670 | 545 | def chdir(d): | 605 | def chdir(directory): |
671 | 606 | """Change the current working directory to a different directory for a code | ||
672 | 607 | block and return the previous directory after the block exits. Useful to | ||
673 | 608 | run commands from a specificed directory. | ||
674 | 609 | |||
675 | 610 | :param str directory: The directory path to change to for this context. | ||
676 | 611 | """ | ||
677 | 546 | cur = os.getcwd() | 612 | cur = os.getcwd() |
678 | 547 | try: | 613 | try: |
680 | 548 | yield os.chdir(d) | 614 | yield os.chdir(directory) |
681 | 549 | finally: | 615 | finally: |
682 | 550 | os.chdir(cur) | 616 | os.chdir(cur) |
683 | 551 | 617 | ||
684 | 552 | 618 | ||
686 | 553 | def chownr(path, owner, group, follow_links=True): | 619 | def chownr(path, owner, group, follow_links=True, chowntopdir=False): |
687 | 620 | """Recursively change user and group ownership of files and directories | ||
688 | 621 | in given path. Doesn't chown path itself by default, only its children. | ||
689 | 622 | |||
690 | 623 | :param str path: The string path to start changing ownership. | ||
691 | 624 | :param str owner: The owner string to use when looking up the uid. | ||
692 | 625 | :param str group: The group string to use when looking up the gid. | ||
693 | 626 | :param bool follow_links: Also Chown links if True | ||
694 | 627 | :param bool chowntopdir: Also chown path itself if True | ||
695 | 628 | """ | ||
696 | 554 | uid = pwd.getpwnam(owner).pw_uid | 629 | uid = pwd.getpwnam(owner).pw_uid |
697 | 555 | gid = grp.getgrnam(group).gr_gid | 630 | gid = grp.getgrnam(group).gr_gid |
698 | 556 | if follow_links: | 631 | if follow_links: |
699 | @@ -558,6 +633,10 @@ | |||
700 | 558 | else: | 633 | else: |
701 | 559 | chown = os.lchown | 634 | chown = os.lchown |
702 | 560 | 635 | ||
703 | 636 | if chowntopdir: | ||
704 | 637 | broken_symlink = os.path.lexists(path) and not os.path.exists(path) | ||
705 | 638 | if not broken_symlink: | ||
706 | 639 | chown(path, uid, gid) | ||
707 | 561 | for root, dirs, files in os.walk(path): | 640 | for root, dirs, files in os.walk(path): |
708 | 562 | for name in dirs + files: | 641 | for name in dirs + files: |
709 | 563 | full = os.path.join(root, name) | 642 | full = os.path.join(root, name) |
710 | @@ -567,4 +646,28 @@ | |||
711 | 567 | 646 | ||
712 | 568 | 647 | ||
713 | 569 | def lchownr(path, owner, group): | 648 | def lchownr(path, owner, group): |
714 | 649 | """Recursively change user and group ownership of files and directories | ||
715 | 650 | in a given path, not following symbolic links. See the documentation for | ||
716 | 651 | 'os.lchown' for more information. | ||
717 | 652 | |||
718 | 653 | :param str path: The string path to start changing ownership. | ||
719 | 654 | :param str owner: The owner string to use when looking up the uid. | ||
720 | 655 | :param str group: The group string to use when looking up the gid. | ||
721 | 656 | """ | ||
722 | 570 | chownr(path, owner, group, follow_links=False) | 657 | chownr(path, owner, group, follow_links=False) |
723 | 658 | |||
724 | 659 | |||
725 | 660 | def get_total_ram(): | ||
726 | 661 | """The total amount of system RAM in bytes. | ||
727 | 662 | |||
728 | 663 | This is what is reported by the OS, and may be overcommitted when | ||
729 | 664 | there are multiple containers hosted on the same machine. | ||
730 | 665 | """ | ||
731 | 666 | with open('/proc/meminfo', 'r') as f: | ||
732 | 667 | for line in f.readlines(): | ||
733 | 668 | if line: | ||
734 | 669 | key, value, unit = line.split() | ||
735 | 670 | if key == 'MemTotal:': | ||
736 | 671 | assert unit == 'kB', 'Unknown unit' | ||
737 | 672 | return int(value) * 1024 # Classic, not KiB. | ||
738 | 673 | raise NotImplementedError() | ||
739 | 571 | 674 | ||
740 | === modified file 'hooks/charmhelpers/core/hugepage.py' | |||
741 | --- hooks/charmhelpers/core/hugepage.py 2015-08-19 13:50:42 +0000 | |||
742 | +++ hooks/charmhelpers/core/hugepage.py 2016-01-22 22:32:27 +0000 | |||
743 | @@ -25,11 +25,13 @@ | |||
744 | 25 | fstab_mount, | 25 | fstab_mount, |
745 | 26 | mkdir, | 26 | mkdir, |
746 | 27 | ) | 27 | ) |
747 | 28 | from charmhelpers.core.strutils import bytes_from_string | ||
748 | 29 | from subprocess import check_output | ||
749 | 28 | 30 | ||
750 | 29 | 31 | ||
751 | 30 | def hugepage_support(user, group='hugetlb', nr_hugepages=256, | 32 | def hugepage_support(user, group='hugetlb', nr_hugepages=256, |
752 | 31 | max_map_count=65536, mnt_point='/run/hugepages/kvm', | 33 | max_map_count=65536, mnt_point='/run/hugepages/kvm', |
754 | 32 | pagesize='2MB', mount=True): | 34 | pagesize='2MB', mount=True, set_shmmax=False): |
755 | 33 | """Enable hugepages on system. | 35 | """Enable hugepages on system. |
756 | 34 | 36 | ||
757 | 35 | Args: | 37 | Args: |
758 | @@ -44,11 +46,18 @@ | |||
759 | 44 | group_info = add_group(group) | 46 | group_info = add_group(group) |
760 | 45 | gid = group_info.gr_gid | 47 | gid = group_info.gr_gid |
761 | 46 | add_user_to_group(user, group) | 48 | add_user_to_group(user, group) |
762 | 49 | if max_map_count < 2 * nr_hugepages: | ||
763 | 50 | max_map_count = 2 * nr_hugepages | ||
764 | 47 | sysctl_settings = { | 51 | sysctl_settings = { |
765 | 48 | 'vm.nr_hugepages': nr_hugepages, | 52 | 'vm.nr_hugepages': nr_hugepages, |
766 | 49 | 'vm.max_map_count': max_map_count, | 53 | 'vm.max_map_count': max_map_count, |
767 | 50 | 'vm.hugetlb_shm_group': gid, | 54 | 'vm.hugetlb_shm_group': gid, |
768 | 51 | } | 55 | } |
769 | 56 | if set_shmmax: | ||
770 | 57 | shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax'])) | ||
771 | 58 | shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages | ||
772 | 59 | if shmmax_minsize > shmmax_current: | ||
773 | 60 | sysctl_settings['kernel.shmmax'] = shmmax_minsize | ||
774 | 52 | sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf') | 61 | sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf') |
775 | 53 | mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False) | 62 | mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False) |
776 | 54 | lfstab = fstab.Fstab() | 63 | lfstab = fstab.Fstab() |
777 | 55 | 64 | ||
778 | === added file 'hooks/charmhelpers/core/kernel.py' | |||
779 | --- hooks/charmhelpers/core/kernel.py 1970-01-01 00:00:00 +0000 | |||
780 | +++ hooks/charmhelpers/core/kernel.py 2016-01-22 22:32:27 +0000 | |||
781 | @@ -0,0 +1,68 @@ | |||
782 | 1 | #!/usr/bin/env python | ||
783 | 2 | # -*- coding: utf-8 -*- | ||
784 | 3 | |||
785 | 4 | # Copyright 2014-2015 Canonical Limited. | ||
786 | 5 | # | ||
787 | 6 | # This file is part of charm-helpers. | ||
788 | 7 | # | ||
789 | 8 | # charm-helpers is free software: you can redistribute it and/or modify | ||
790 | 9 | # it under the terms of the GNU Lesser General Public License version 3 as | ||
791 | 10 | # published by the Free Software Foundation. | ||
792 | 11 | # | ||
793 | 12 | # charm-helpers is distributed in the hope that it will be useful, | ||
794 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
795 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
796 | 15 | # GNU Lesser General Public License for more details. | ||
797 | 16 | # | ||
798 | 17 | # You should have received a copy of the GNU Lesser General Public License | ||
799 | 18 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. | ||
800 | 19 | |||
801 | 20 | __author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>" | ||
802 | 21 | |||
803 | 22 | from charmhelpers.core.hookenv import ( | ||
804 | 23 | log, | ||
805 | 24 | INFO | ||
806 | 25 | ) | ||
807 | 26 | |||
808 | 27 | from subprocess import check_call, check_output | ||
809 | 28 | import re | ||
810 | 29 | |||
811 | 30 | |||
812 | 31 | def modprobe(module, persist=True): | ||
813 | 32 | """Load a kernel module and configure for auto-load on reboot.""" | ||
814 | 33 | cmd = ['modprobe', module] | ||
815 | 34 | |||
816 | 35 | log('Loading kernel module %s' % module, level=INFO) | ||
817 | 36 | |||
818 | 37 | check_call(cmd) | ||
819 | 38 | if persist: | ||
820 | 39 | with open('/etc/modules', 'r+') as modules: | ||
821 | 40 | if module not in modules.read(): | ||
822 | 41 | modules.write(module) | ||
823 | 42 | |||
824 | 43 | |||
825 | 44 | def rmmod(module, force=False): | ||
826 | 45 | """Remove a module from the linux kernel""" | ||
827 | 46 | cmd = ['rmmod'] | ||
828 | 47 | if force: | ||
829 | 48 | cmd.append('-f') | ||
830 | 49 | cmd.append(module) | ||
831 | 50 | log('Removing kernel module %s' % module, level=INFO) | ||
832 | 51 | return check_call(cmd) | ||
833 | 52 | |||
834 | 53 | |||
835 | 54 | def lsmod(): | ||
836 | 55 | """Shows what kernel modules are currently loaded""" | ||
837 | 56 | return check_output(['lsmod'], | ||
838 | 57 | universal_newlines=True) | ||
839 | 58 | |||
840 | 59 | |||
841 | 60 | def is_module_loaded(module): | ||
842 | 61 | """Checks if a kernel module is already loaded""" | ||
843 | 62 | matches = re.findall('^%s[ ]+' % module, lsmod(), re.M) | ||
844 | 63 | return len(matches) > 0 | ||
845 | 64 | |||
846 | 65 | |||
847 | 66 | def update_initramfs(version='all'): | ||
848 | 67 | """Updates an initramfs image""" | ||
849 | 68 | return check_call(["update-initramfs", "-k", version, "-u"]) | ||
850 | 0 | 69 | ||
851 | === modified file 'hooks/charmhelpers/core/services/helpers.py' | |||
852 | --- hooks/charmhelpers/core/services/helpers.py 2015-08-19 13:50:42 +0000 | |||
853 | +++ hooks/charmhelpers/core/services/helpers.py 2016-01-22 22:32:27 +0000 | |||
854 | @@ -243,33 +243,40 @@ | |||
855 | 243 | :param str source: The template source file, relative to | 243 | :param str source: The template source file, relative to |
856 | 244 | `$CHARM_DIR/templates` | 244 | `$CHARM_DIR/templates` |
857 | 245 | 245 | ||
859 | 246 | :param str target: The target to write the rendered template to | 246 | :param str target: The target to write the rendered template to (or None) |
860 | 247 | :param str owner: The owner of the rendered file | 247 | :param str owner: The owner of the rendered file |
861 | 248 | :param str group: The group of the rendered file | 248 | :param str group: The group of the rendered file |
862 | 249 | :param int perms: The permissions of the rendered file | 249 | :param int perms: The permissions of the rendered file |
863 | 250 | :param partial on_change_action: functools partial to be executed when | 250 | :param partial on_change_action: functools partial to be executed when |
864 | 251 | rendered file changes | 251 | rendered file changes |
865 | 252 | :param jinja2 loader template_loader: A jinja2 template loader | ||
866 | 253 | |||
867 | 254 | :return str: The rendered template | ||
868 | 252 | """ | 255 | """ |
869 | 253 | def __init__(self, source, target, | 256 | def __init__(self, source, target, |
870 | 254 | owner='root', group='root', perms=0o444, | 257 | owner='root', group='root', perms=0o444, |
872 | 255 | on_change_action=None): | 258 | on_change_action=None, template_loader=None): |
873 | 256 | self.source = source | 259 | self.source = source |
874 | 257 | self.target = target | 260 | self.target = target |
875 | 258 | self.owner = owner | 261 | self.owner = owner |
876 | 259 | self.group = group | 262 | self.group = group |
877 | 260 | self.perms = perms | 263 | self.perms = perms |
878 | 261 | self.on_change_action = on_change_action | 264 | self.on_change_action = on_change_action |
879 | 265 | self.template_loader = template_loader | ||
880 | 262 | 266 | ||
881 | 263 | def __call__(self, manager, service_name, event_name): | 267 | def __call__(self, manager, service_name, event_name): |
882 | 264 | pre_checksum = '' | 268 | pre_checksum = '' |
883 | 265 | if self.on_change_action and os.path.isfile(self.target): | 269 | if self.on_change_action and os.path.isfile(self.target): |
884 | 266 | pre_checksum = host.file_hash(self.target) | 270 | pre_checksum = host.file_hash(self.target) |
885 | 267 | service = manager.get_service(service_name) | 271 | service = manager.get_service(service_name) |
887 | 268 | context = {} | 272 | context = {'ctx': {}} |
888 | 269 | for ctx in service.get('required_data', []): | 273 | for ctx in service.get('required_data', []): |
889 | 270 | context.update(ctx) | 274 | context.update(ctx) |
892 | 271 | templating.render(self.source, self.target, context, | 275 | context['ctx'].update(ctx) |
893 | 272 | self.owner, self.group, self.perms) | 276 | |
894 | 277 | result = templating.render(self.source, self.target, context, | ||
895 | 278 | self.owner, self.group, self.perms, | ||
896 | 279 | template_loader=self.template_loader) | ||
897 | 273 | if self.on_change_action: | 280 | if self.on_change_action: |
898 | 274 | if pre_checksum == host.file_hash(self.target): | 281 | if pre_checksum == host.file_hash(self.target): |
899 | 275 | hookenv.log( | 282 | hookenv.log( |
900 | @@ -278,6 +285,8 @@ | |||
901 | 278 | else: | 285 | else: |
902 | 279 | self.on_change_action() | 286 | self.on_change_action() |
903 | 280 | 287 | ||
904 | 288 | return result | ||
905 | 289 | |||
906 | 281 | 290 | ||
907 | 282 | # Convenience aliases for templates | 291 | # Convenience aliases for templates |
908 | 283 | render_template = template = TemplateCallback | 292 | render_template = template = TemplateCallback |
909 | 284 | 293 | ||
910 | === modified file 'hooks/charmhelpers/core/strutils.py' | |||
911 | --- hooks/charmhelpers/core/strutils.py 2015-04-16 21:32:48 +0000 | |||
912 | +++ hooks/charmhelpers/core/strutils.py 2016-01-22 22:32:27 +0000 | |||
913 | @@ -18,6 +18,7 @@ | |||
914 | 18 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. | 18 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
915 | 19 | 19 | ||
916 | 20 | import six | 20 | import six |
917 | 21 | import re | ||
918 | 21 | 22 | ||
919 | 22 | 23 | ||
920 | 23 | def bool_from_string(value): | 24 | def bool_from_string(value): |
921 | @@ -40,3 +41,32 @@ | |||
922 | 40 | 41 | ||
923 | 41 | msg = "Unable to interpret string value '%s' as boolean" % (value) | 42 | msg = "Unable to interpret string value '%s' as boolean" % (value) |
924 | 42 | raise ValueError(msg) | 43 | raise ValueError(msg) |
925 | 44 | |||
926 | 45 | |||
927 | 46 | def bytes_from_string(value): | ||
928 | 47 | """Interpret human readable string value as bytes. | ||
929 | 48 | |||
930 | 49 | Returns int | ||
931 | 50 | """ | ||
932 | 51 | BYTE_POWER = { | ||
933 | 52 | 'K': 1, | ||
934 | 53 | 'KB': 1, | ||
935 | 54 | 'M': 2, | ||
936 | 55 | 'MB': 2, | ||
937 | 56 | 'G': 3, | ||
938 | 57 | 'GB': 3, | ||
939 | 58 | 'T': 4, | ||
940 | 59 | 'TB': 4, | ||
941 | 60 | 'P': 5, | ||
942 | 61 | 'PB': 5, | ||
943 | 62 | } | ||
944 | 63 | if isinstance(value, six.string_types): | ||
945 | 64 | value = six.text_type(value) | ||
946 | 65 | else: | ||
947 | 66 | msg = "Unable to interpret non-string value '%s' as boolean" % (value) | ||
948 | 67 | raise ValueError(msg) | ||
949 | 68 | matches = re.match("([0-9]+)([a-zA-Z]+)", value) | ||
950 | 69 | if not matches: | ||
951 | 70 | msg = "Unable to interpret string value '%s' as bytes" % (value) | ||
952 | 71 | raise ValueError(msg) | ||
953 | 72 | return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) | ||
954 | 43 | 73 | ||
955 | === modified file 'hooks/charmhelpers/core/templating.py' | |||
956 | --- hooks/charmhelpers/core/templating.py 2015-02-26 13:37:18 +0000 | |||
957 | +++ hooks/charmhelpers/core/templating.py 2016-01-22 22:32:27 +0000 | |||
958 | @@ -21,13 +21,14 @@ | |||
959 | 21 | 21 | ||
960 | 22 | 22 | ||
961 | 23 | def render(source, target, context, owner='root', group='root', | 23 | def render(source, target, context, owner='root', group='root', |
963 | 24 | perms=0o444, templates_dir=None, encoding='UTF-8'): | 24 | perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None): |
964 | 25 | """ | 25 | """ |
965 | 26 | Render a template. | 26 | Render a template. |
966 | 27 | 27 | ||
967 | 28 | The `source` path, if not absolute, is relative to the `templates_dir`. | 28 | The `source` path, if not absolute, is relative to the `templates_dir`. |
968 | 29 | 29 | ||
970 | 30 | The `target` path should be absolute. | 30 | The `target` path should be absolute. It can also be `None`, in which |
971 | 31 | case no file will be written. | ||
972 | 31 | 32 | ||
973 | 32 | The context should be a dict containing the values to be replaced in the | 33 | The context should be a dict containing the values to be replaced in the |
974 | 33 | template. | 34 | template. |
975 | @@ -36,6 +37,9 @@ | |||
976 | 36 | 37 | ||
977 | 37 | If omitted, `templates_dir` defaults to the `templates` folder in the charm. | 38 | If omitted, `templates_dir` defaults to the `templates` folder in the charm. |
978 | 38 | 39 | ||
979 | 40 | The rendered template will be written to the file as well as being returned | ||
980 | 41 | as a string. | ||
981 | 42 | |||
982 | 39 | Note: Using this requires python-jinja2; if it is not installed, calling | 43 | Note: Using this requires python-jinja2; if it is not installed, calling |
983 | 40 | this will attempt to use charmhelpers.fetch.apt_install to install it. | 44 | this will attempt to use charmhelpers.fetch.apt_install to install it. |
984 | 41 | """ | 45 | """ |
985 | @@ -52,17 +56,26 @@ | |||
986 | 52 | apt_install('python-jinja2', fatal=True) | 56 | apt_install('python-jinja2', fatal=True) |
987 | 53 | from jinja2 import FileSystemLoader, Environment, exceptions | 57 | from jinja2 import FileSystemLoader, Environment, exceptions |
988 | 54 | 58 | ||
992 | 55 | if templates_dir is None: | 59 | if template_loader: |
993 | 56 | templates_dir = os.path.join(hookenv.charm_dir(), 'templates') | 60 | template_env = Environment(loader=template_loader) |
994 | 57 | loader = Environment(loader=FileSystemLoader(templates_dir)) | 61 | else: |
995 | 62 | if templates_dir is None: | ||
996 | 63 | templates_dir = os.path.join(hookenv.charm_dir(), 'templates') | ||
997 | 64 | template_env = Environment(loader=FileSystemLoader(templates_dir)) | ||
998 | 58 | try: | 65 | try: |
999 | 59 | source = source | 66 | source = source |
1001 | 60 | template = loader.get_template(source) | 67 | template = template_env.get_template(source) |
1002 | 61 | except exceptions.TemplateNotFound as e: | 68 | except exceptions.TemplateNotFound as e: |
1003 | 62 | hookenv.log('Could not load template %s from %s.' % | 69 | hookenv.log('Could not load template %s from %s.' % |
1004 | 63 | (source, templates_dir), | 70 | (source, templates_dir), |
1005 | 64 | level=hookenv.ERROR) | 71 | level=hookenv.ERROR) |
1006 | 65 | raise e | 72 | raise e |
1007 | 66 | content = template.render(context) | 73 | content = template.render(context) |
1010 | 67 | host.mkdir(os.path.dirname(target), owner, group, perms=0o755) | 74 | if target is not None: |
1011 | 68 | host.write_file(target, content.encode(encoding), owner, group, perms) | 75 | target_dir = os.path.dirname(target) |
1012 | 76 | if not os.path.exists(target_dir): | ||
1013 | 77 | # This is a terrible default directory permission, as the file | ||
1014 | 78 | # or its siblings will often contain secrets. | ||
1015 | 79 | host.mkdir(os.path.dirname(target), owner, group, perms=0o755) | ||
1016 | 80 | host.write_file(target, content.encode(encoding), owner, group, perms) | ||
1017 | 81 | return content | ||
1018 | 69 | 82 | ||
1019 | === modified file 'hooks/charmhelpers/fetch/__init__.py' | |||
1020 | --- hooks/charmhelpers/fetch/__init__.py 2015-08-19 13:50:42 +0000 | |||
1021 | +++ hooks/charmhelpers/fetch/__init__.py 2016-01-22 22:32:27 +0000 | |||
1022 | @@ -98,6 +98,14 @@ | |||
1023 | 98 | 'liberty/proposed': 'trusty-proposed/liberty', | 98 | 'liberty/proposed': 'trusty-proposed/liberty', |
1024 | 99 | 'trusty-liberty/proposed': 'trusty-proposed/liberty', | 99 | 'trusty-liberty/proposed': 'trusty-proposed/liberty', |
1025 | 100 | 'trusty-proposed/liberty': 'trusty-proposed/liberty', | 100 | 'trusty-proposed/liberty': 'trusty-proposed/liberty', |
1026 | 101 | # Mitaka | ||
1027 | 102 | 'mitaka': 'trusty-updates/mitaka', | ||
1028 | 103 | 'trusty-mitaka': 'trusty-updates/mitaka', | ||
1029 | 104 | 'trusty-mitaka/updates': 'trusty-updates/mitaka', | ||
1030 | 105 | 'trusty-updates/mitaka': 'trusty-updates/mitaka', | ||
1031 | 106 | 'mitaka/proposed': 'trusty-proposed/mitaka', | ||
1032 | 107 | 'trusty-mitaka/proposed': 'trusty-proposed/mitaka', | ||
1033 | 108 | 'trusty-proposed/mitaka': 'trusty-proposed/mitaka', | ||
1034 | 101 | } | 109 | } |
1035 | 102 | 110 | ||
1036 | 103 | # The order of this list is very important. Handlers should be listed in from | 111 | # The order of this list is very important. Handlers should be listed in from |
1037 | @@ -225,12 +233,12 @@ | |||
1038 | 225 | 233 | ||
1039 | 226 | def apt_mark(packages, mark, fatal=False): | 234 | def apt_mark(packages, mark, fatal=False): |
1040 | 227 | """Flag one or more packages using apt-mark""" | 235 | """Flag one or more packages using apt-mark""" |
1041 | 236 | log("Marking {} as {}".format(packages, mark)) | ||
1042 | 228 | cmd = ['apt-mark', mark] | 237 | cmd = ['apt-mark', mark] |
1043 | 229 | if isinstance(packages, six.string_types): | 238 | if isinstance(packages, six.string_types): |
1044 | 230 | cmd.append(packages) | 239 | cmd.append(packages) |
1045 | 231 | else: | 240 | else: |
1046 | 232 | cmd.extend(packages) | 241 | cmd.extend(packages) |
1047 | 233 | log("Holding {}".format(packages)) | ||
1048 | 234 | 242 | ||
1049 | 235 | if fatal: | 243 | if fatal: |
1050 | 236 | subprocess.check_call(cmd, universal_newlines=True) | 244 | subprocess.check_call(cmd, universal_newlines=True) |
1051 | @@ -411,7 +419,7 @@ | |||
1052 | 411 | importlib.import_module(package), | 419 | importlib.import_module(package), |
1053 | 412 | classname) | 420 | classname) |
1054 | 413 | plugin_list.append(handler_class()) | 421 | plugin_list.append(handler_class()) |
1056 | 414 | except (ImportError, AttributeError): | 422 | except NotImplementedError: |
1057 | 415 | # Skip missing plugins so that they can be ommitted from | 423 | # Skip missing plugins so that they can be ommitted from |
1058 | 416 | # installation if desired | 424 | # installation if desired |
1059 | 417 | log("FetchHandler {} not found, skipping plugin".format( | 425 | log("FetchHandler {} not found, skipping plugin".format( |
1060 | 418 | 426 | ||
1061 | === modified file 'hooks/charmhelpers/fetch/archiveurl.py' | |||
1062 | --- hooks/charmhelpers/fetch/archiveurl.py 2015-08-03 14:53:05 +0000 | |||
1063 | +++ hooks/charmhelpers/fetch/archiveurl.py 2016-01-22 22:32:27 +0000 | |||
1064 | @@ -108,7 +108,7 @@ | |||
1065 | 108 | install_opener(opener) | 108 | install_opener(opener) |
1066 | 109 | response = urlopen(source) | 109 | response = urlopen(source) |
1067 | 110 | try: | 110 | try: |
1069 | 111 | with open(dest, 'w') as dest_file: | 111 | with open(dest, 'wb') as dest_file: |
1070 | 112 | dest_file.write(response.read()) | 112 | dest_file.write(response.read()) |
1071 | 113 | except Exception as e: | 113 | except Exception as e: |
1072 | 114 | if os.path.isfile(dest): | 114 | if os.path.isfile(dest): |
1073 | 115 | 115 | ||
1074 | === modified file 'hooks/charmhelpers/fetch/bzrurl.py' | |||
1075 | --- hooks/charmhelpers/fetch/bzrurl.py 2015-01-26 11:51:28 +0000 | |||
1076 | +++ hooks/charmhelpers/fetch/bzrurl.py 2016-01-22 22:32:27 +0000 | |||
1077 | @@ -15,60 +15,50 @@ | |||
1078 | 15 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. | 15 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
1079 | 16 | 16 | ||
1080 | 17 | import os | 17 | import os |
1081 | 18 | from subprocess import check_call | ||
1082 | 18 | from charmhelpers.fetch import ( | 19 | from charmhelpers.fetch import ( |
1083 | 19 | BaseFetchHandler, | 20 | BaseFetchHandler, |
1085 | 20 | UnhandledSource | 21 | UnhandledSource, |
1086 | 22 | filter_installed_packages, | ||
1087 | 23 | apt_install, | ||
1088 | 21 | ) | 24 | ) |
1089 | 22 | from charmhelpers.core.host import mkdir | 25 | from charmhelpers.core.host import mkdir |
1090 | 23 | 26 | ||
1091 | 24 | import six | ||
1092 | 25 | if six.PY3: | ||
1093 | 26 | raise ImportError('bzrlib does not support Python3') | ||
1094 | 27 | 27 | ||
1103 | 28 | try: | 28 | if filter_installed_packages(['bzr']) != []: |
1104 | 29 | from bzrlib.branch import Branch | 29 | apt_install(['bzr']) |
1105 | 30 | from bzrlib import bzrdir, workingtree, errors | 30 | if filter_installed_packages(['bzr']) != []: |
1106 | 31 | except ImportError: | 31 | raise NotImplementedError('Unable to install bzr') |
1099 | 32 | from charmhelpers.fetch import apt_install | ||
1100 | 33 | apt_install("python-bzrlib") | ||
1101 | 34 | from bzrlib.branch import Branch | ||
1102 | 35 | from bzrlib import bzrdir, workingtree, errors | ||
1107 | 36 | 32 | ||
1108 | 37 | 33 | ||
1109 | 38 | class BzrUrlFetchHandler(BaseFetchHandler): | 34 | class BzrUrlFetchHandler(BaseFetchHandler): |
1110 | 39 | """Handler for bazaar branches via generic and lp URLs""" | 35 | """Handler for bazaar branches via generic and lp URLs""" |
1111 | 40 | def can_handle(self, source): | 36 | def can_handle(self, source): |
1112 | 41 | url_parts = self.parse_url(source) | 37 | url_parts = self.parse_url(source) |
1114 | 42 | if url_parts.scheme not in ('bzr+ssh', 'lp'): | 38 | if url_parts.scheme not in ('bzr+ssh', 'lp', ''): |
1115 | 43 | return False | 39 | return False |
1116 | 40 | elif not url_parts.scheme: | ||
1117 | 41 | return os.path.exists(os.path.join(source, '.bzr')) | ||
1118 | 44 | else: | 42 | else: |
1119 | 45 | return True | 43 | return True |
1120 | 46 | 44 | ||
1121 | 47 | def branch(self, source, dest): | 45 | def branch(self, source, dest): |
1122 | 48 | url_parts = self.parse_url(source) | ||
1123 | 49 | # If we use lp:branchname scheme we need to load plugins | ||
1124 | 50 | if not self.can_handle(source): | 46 | if not self.can_handle(source): |
1125 | 51 | raise UnhandledSource("Cannot handle {}".format(source)) | 47 | raise UnhandledSource("Cannot handle {}".format(source)) |
1140 | 52 | if url_parts.scheme == "lp": | 48 | if os.path.exists(dest): |
1141 | 53 | from bzrlib.plugin import load_plugins | 49 | check_call(['bzr', 'pull', '--overwrite', '-d', dest, source]) |
1142 | 54 | load_plugins() | 50 | else: |
1143 | 55 | try: | 51 | check_call(['bzr', 'branch', source, dest]) |
1130 | 56 | local_branch = bzrdir.BzrDir.create_branch_convenience(dest) | ||
1131 | 57 | except errors.AlreadyControlDirError: | ||
1132 | 58 | local_branch = Branch.open(dest) | ||
1133 | 59 | try: | ||
1134 | 60 | remote_branch = Branch.open(source) | ||
1135 | 61 | remote_branch.push(local_branch) | ||
1136 | 62 | tree = workingtree.WorkingTree.open(dest) | ||
1137 | 63 | tree.update() | ||
1138 | 64 | except Exception as e: | ||
1139 | 65 | raise e | ||
1144 | 66 | 52 | ||
1146 | 67 | def install(self, source): | 53 | def install(self, source, dest=None): |
1147 | 68 | url_parts = self.parse_url(source) | 54 | url_parts = self.parse_url(source) |
1148 | 69 | branch_name = url_parts.path.strip("/").split("/")[-1] | 55 | branch_name = url_parts.path.strip("/").split("/")[-1] |
1151 | 70 | dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", | 56 | if dest: |
1152 | 71 | branch_name) | 57 | dest_dir = os.path.join(dest, branch_name) |
1153 | 58 | else: | ||
1154 | 59 | dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", | ||
1155 | 60 | branch_name) | ||
1156 | 61 | |||
1157 | 72 | if not os.path.exists(dest_dir): | 62 | if not os.path.exists(dest_dir): |
1158 | 73 | mkdir(dest_dir, perms=0o755) | 63 | mkdir(dest_dir, perms=0o755) |
1159 | 74 | try: | 64 | try: |
1160 | 75 | 65 | ||
1161 | === modified file 'hooks/charmhelpers/fetch/giturl.py' | |||
1162 | --- hooks/charmhelpers/fetch/giturl.py 2015-08-03 14:53:05 +0000 | |||
1163 | +++ hooks/charmhelpers/fetch/giturl.py 2016-01-22 22:32:27 +0000 | |||
1164 | @@ -15,24 +15,18 @@ | |||
1165 | 15 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. | 15 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
1166 | 16 | 16 | ||
1167 | 17 | import os | 17 | import os |
1168 | 18 | from subprocess import check_call, CalledProcessError | ||
1169 | 18 | from charmhelpers.fetch import ( | 19 | from charmhelpers.fetch import ( |
1170 | 19 | BaseFetchHandler, | 20 | BaseFetchHandler, |
1172 | 20 | UnhandledSource | 21 | UnhandledSource, |
1173 | 22 | filter_installed_packages, | ||
1174 | 23 | apt_install, | ||
1175 | 21 | ) | 24 | ) |
1190 | 22 | from charmhelpers.core.host import mkdir | 25 | |
1191 | 23 | 26 | if filter_installed_packages(['git']) != []: | |
1192 | 24 | import six | 27 | apt_install(['git']) |
1193 | 25 | if six.PY3: | 28 | if filter_installed_packages(['git']) != []: |
1194 | 26 | raise ImportError('GitPython does not support Python 3') | 29 | raise NotImplementedError('Unable to install git') |
1181 | 27 | |||
1182 | 28 | try: | ||
1183 | 29 | from git import Repo | ||
1184 | 30 | except ImportError: | ||
1185 | 31 | from charmhelpers.fetch import apt_install | ||
1186 | 32 | apt_install("python-git") | ||
1187 | 33 | from git import Repo | ||
1188 | 34 | |||
1189 | 35 | from git.exc import GitCommandError # noqa E402 | ||
1195 | 36 | 30 | ||
1196 | 37 | 31 | ||
1197 | 38 | class GitUrlFetchHandler(BaseFetchHandler): | 32 | class GitUrlFetchHandler(BaseFetchHandler): |
1198 | @@ -40,19 +34,24 @@ | |||
1199 | 40 | def can_handle(self, source): | 34 | def can_handle(self, source): |
1200 | 41 | url_parts = self.parse_url(source) | 35 | url_parts = self.parse_url(source) |
1201 | 42 | # TODO (mattyw) no support for ssh git@ yet | 36 | # TODO (mattyw) no support for ssh git@ yet |
1203 | 43 | if url_parts.scheme not in ('http', 'https', 'git'): | 37 | if url_parts.scheme not in ('http', 'https', 'git', ''): |
1204 | 44 | return False | 38 | return False |
1205 | 39 | elif not url_parts.scheme: | ||
1206 | 40 | return os.path.exists(os.path.join(source, '.git')) | ||
1207 | 45 | else: | 41 | else: |
1208 | 46 | return True | 42 | return True |
1209 | 47 | 43 | ||
1211 | 48 | def clone(self, source, dest, branch, depth=None): | 44 | def clone(self, source, dest, branch="master", depth=None): |
1212 | 49 | if not self.can_handle(source): | 45 | if not self.can_handle(source): |
1213 | 50 | raise UnhandledSource("Cannot handle {}".format(source)) | 46 | raise UnhandledSource("Cannot handle {}".format(source)) |
1214 | 51 | 47 | ||
1217 | 52 | if depth: | 48 | if os.path.exists(dest): |
1218 | 53 | Repo.clone_from(source, dest, branch=branch, depth=depth) | 49 | cmd = ['git', '-C', dest, 'pull', source, branch] |
1219 | 54 | else: | 50 | else: |
1221 | 55 | Repo.clone_from(source, dest, branch=branch) | 51 | cmd = ['git', 'clone', source, dest, '--branch', branch] |
1222 | 52 | if depth: | ||
1223 | 53 | cmd.extend(['--depth', depth]) | ||
1224 | 54 | check_call(cmd) | ||
1225 | 56 | 55 | ||
1226 | 57 | def install(self, source, branch="master", dest=None, depth=None): | 56 | def install(self, source, branch="master", dest=None, depth=None): |
1227 | 58 | url_parts = self.parse_url(source) | 57 | url_parts = self.parse_url(source) |
1228 | @@ -62,11 +61,9 @@ | |||
1229 | 62 | else: | 61 | else: |
1230 | 63 | dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", | 62 | dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", |
1231 | 64 | branch_name) | 63 | branch_name) |
1232 | 65 | if not os.path.exists(dest_dir): | ||
1233 | 66 | mkdir(dest_dir, perms=0o755) | ||
1234 | 67 | try: | 64 | try: |
1235 | 68 | self.clone(source, dest_dir, branch, depth) | 65 | self.clone(source, dest_dir, branch, depth) |
1237 | 69 | except GitCommandError as e: | 66 | except CalledProcessError as e: |
1238 | 70 | raise UnhandledSource(e) | 67 | raise UnhandledSource(e) |
1239 | 71 | except OSError as e: | 68 | except OSError as e: |
1240 | 72 | raise UnhandledSource(e.strerror) | 69 | raise UnhandledSource(e.strerror) |
1241 | 73 | 70 | ||
1242 | === modified file 'tests/018-basic-trusty-liberty' (properties changed: -x to +x) | |||
1243 | === modified file 'tests/020-basic-wily-liberty' (properties changed: -x to +x) | |||
1244 | === modified file 'tests/charmhelpers/contrib/amulet/deployment.py' | |||
1245 | --- tests/charmhelpers/contrib/amulet/deployment.py 2015-01-26 11:51:28 +0000 | |||
1246 | +++ tests/charmhelpers/contrib/amulet/deployment.py 2016-01-22 22:32:27 +0000 | |||
1247 | @@ -51,7 +51,8 @@ | |||
1248 | 51 | if 'units' not in this_service: | 51 | if 'units' not in this_service: |
1249 | 52 | this_service['units'] = 1 | 52 | this_service['units'] = 1 |
1250 | 53 | 53 | ||
1252 | 54 | self.d.add(this_service['name'], units=this_service['units']) | 54 | self.d.add(this_service['name'], units=this_service['units'], |
1253 | 55 | constraints=this_service.get('constraints')) | ||
1254 | 55 | 56 | ||
1255 | 56 | for svc in other_services: | 57 | for svc in other_services: |
1256 | 57 | if 'location' in svc: | 58 | if 'location' in svc: |
1257 | @@ -64,7 +65,8 @@ | |||
1258 | 64 | if 'units' not in svc: | 65 | if 'units' not in svc: |
1259 | 65 | svc['units'] = 1 | 66 | svc['units'] = 1 |
1260 | 66 | 67 | ||
1262 | 67 | self.d.add(svc['name'], charm=branch_location, units=svc['units']) | 68 | self.d.add(svc['name'], charm=branch_location, units=svc['units'], |
1263 | 69 | constraints=svc.get('constraints')) | ||
1264 | 68 | 70 | ||
1265 | 69 | def _add_relations(self, relations): | 71 | def _add_relations(self, relations): |
1266 | 70 | """Add all of the relations for the services.""" | 72 | """Add all of the relations for the services.""" |
1267 | 71 | 73 | ||
1268 | === modified file 'tests/charmhelpers/contrib/amulet/utils.py' | |||
1269 | --- tests/charmhelpers/contrib/amulet/utils.py 2015-08-19 13:50:42 +0000 | |||
1270 | +++ tests/charmhelpers/contrib/amulet/utils.py 2016-01-22 22:32:27 +0000 | |||
1271 | @@ -19,9 +19,11 @@ | |||
1272 | 19 | import logging | 19 | import logging |
1273 | 20 | import os | 20 | import os |
1274 | 21 | import re | 21 | import re |
1275 | 22 | import socket | ||
1276 | 22 | import subprocess | 23 | import subprocess |
1277 | 23 | import sys | 24 | import sys |
1278 | 24 | import time | 25 | import time |
1279 | 26 | import uuid | ||
1280 | 25 | 27 | ||
1281 | 26 | import amulet | 28 | import amulet |
1282 | 27 | import distro_info | 29 | import distro_info |
1283 | @@ -114,7 +116,7 @@ | |||
1284 | 114 | # /!\ DEPRECATION WARNING (beisner): | 116 | # /!\ DEPRECATION WARNING (beisner): |
1285 | 115 | # New and existing tests should be rewritten to use | 117 | # New and existing tests should be rewritten to use |
1286 | 116 | # validate_services_by_name() as it is aware of init systems. | 118 | # validate_services_by_name() as it is aware of init systems. |
1288 | 117 | self.log.warn('/!\\ DEPRECATION WARNING: use ' | 119 | self.log.warn('DEPRECATION WARNING: use ' |
1289 | 118 | 'validate_services_by_name instead of validate_services ' | 120 | 'validate_services_by_name instead of validate_services ' |
1290 | 119 | 'due to init system differences.') | 121 | 'due to init system differences.') |
1291 | 120 | 122 | ||
1292 | @@ -269,33 +271,52 @@ | |||
1293 | 269 | """Get last modification time of directory.""" | 271 | """Get last modification time of directory.""" |
1294 | 270 | return sentry_unit.directory_stat(directory)['mtime'] | 272 | return sentry_unit.directory_stat(directory)['mtime'] |
1295 | 271 | 273 | ||
1314 | 272 | def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False): | 274 | def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None): |
1315 | 273 | """Get process' start time. | 275 | """Get start time of a process based on the last modification time |
1316 | 274 | 276 | of the /proc/pid directory. | |
1317 | 275 | Determine start time of the process based on the last modification | 277 | |
1318 | 276 | time of the /proc/pid directory. If pgrep_full is True, the process | 278 | :sentry_unit: The sentry unit to check for the service on |
1319 | 277 | name is matched against the full command line. | 279 | :service: service name to look for in process table |
1320 | 278 | """ | 280 | :pgrep_full: [Deprecated] Use full command line search mode with pgrep |
1321 | 279 | if pgrep_full: | 281 | :returns: epoch time of service process start |
1322 | 280 | cmd = 'pgrep -o -f {}'.format(service) | 282 | :param commands: list of bash commands |
1323 | 281 | else: | 283 | :param sentry_units: list of sentry unit pointers |
1324 | 282 | cmd = 'pgrep -o {}'.format(service) | 284 | :returns: None if successful; Failure message otherwise |
1325 | 283 | cmd = cmd + ' | grep -v pgrep || exit 0' | 285 | """ |
1326 | 284 | cmd_out = sentry_unit.run(cmd) | 286 | if pgrep_full is not None: |
1327 | 285 | self.log.debug('CMDout: ' + str(cmd_out)) | 287 | # /!\ DEPRECATION WARNING (beisner): |
1328 | 286 | if cmd_out[0]: | 288 | # No longer implemented, as pidof is now used instead of pgrep. |
1329 | 287 | self.log.debug('Pid for %s %s' % (service, str(cmd_out[0]))) | 289 | # https://bugs.launchpad.net/charm-helpers/+bug/1474030 |
1330 | 288 | proc_dir = '/proc/{}'.format(cmd_out[0].strip()) | 290 | self.log.warn('DEPRECATION WARNING: pgrep_full bool is no ' |
1331 | 289 | return self._get_dir_mtime(sentry_unit, proc_dir) | 291 | 'longer implemented re: lp 1474030.') |
1332 | 292 | |||
1333 | 293 | pid_list = self.get_process_id_list(sentry_unit, service) | ||
1334 | 294 | pid = pid_list[0] | ||
1335 | 295 | proc_dir = '/proc/{}'.format(pid) | ||
1336 | 296 | self.log.debug('Pid for {} on {}: {}'.format( | ||
1337 | 297 | service, sentry_unit.info['unit_name'], pid)) | ||
1338 | 298 | |||
1339 | 299 | return self._get_dir_mtime(sentry_unit, proc_dir) | ||
1340 | 290 | 300 | ||
1341 | 291 | def service_restarted(self, sentry_unit, service, filename, | 301 | def service_restarted(self, sentry_unit, service, filename, |
1343 | 292 | pgrep_full=False, sleep_time=20): | 302 | pgrep_full=None, sleep_time=20): |
1344 | 293 | """Check if service was restarted. | 303 | """Check if service was restarted. |
1345 | 294 | 304 | ||
1346 | 295 | Compare a service's start time vs a file's last modification time | 305 | Compare a service's start time vs a file's last modification time |
1347 | 296 | (such as a config file for that service) to determine if the service | 306 | (such as a config file for that service) to determine if the service |
1348 | 297 | has been restarted. | 307 | has been restarted. |
1349 | 298 | """ | 308 | """ |
1350 | 309 | # /!\ DEPRECATION WARNING (beisner): | ||
1351 | 310 | # This method is prone to races in that no before-time is known. | ||
1352 | 311 | # Use validate_service_config_changed instead. | ||
1353 | 312 | |||
1354 | 313 | # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now | ||
1355 | 314 | # used instead of pgrep. pgrep_full is still passed through to ensure | ||
1356 | 315 | # deprecation WARNS. lp1474030 | ||
1357 | 316 | self.log.warn('DEPRECATION WARNING: use ' | ||
1358 | 317 | 'validate_service_config_changed instead of ' | ||
1359 | 318 | 'service_restarted due to known races.') | ||
1360 | 319 | |||
1361 | 299 | time.sleep(sleep_time) | 320 | time.sleep(sleep_time) |
1362 | 300 | if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >= | 321 | if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >= |
1363 | 301 | self._get_file_mtime(sentry_unit, filename)): | 322 | self._get_file_mtime(sentry_unit, filename)): |
1364 | @@ -304,78 +325,122 @@ | |||
1365 | 304 | return False | 325 | return False |
1366 | 305 | 326 | ||
1367 | 306 | def service_restarted_since(self, sentry_unit, mtime, service, | 327 | def service_restarted_since(self, sentry_unit, mtime, service, |
1370 | 307 | pgrep_full=False, sleep_time=20, | 328 | pgrep_full=None, sleep_time=20, |
1371 | 308 | retry_count=2): | 329 | retry_count=30, retry_sleep_time=10): |
1372 | 309 | """Check if service was been started after a given time. | 330 | """Check if service was been started after a given time. |
1373 | 310 | 331 | ||
1374 | 311 | Args: | 332 | Args: |
1375 | 312 | sentry_unit (sentry): The sentry unit to check for the service on | 333 | sentry_unit (sentry): The sentry unit to check for the service on |
1376 | 313 | mtime (float): The epoch time to check against | 334 | mtime (float): The epoch time to check against |
1377 | 314 | service (string): service name to look for in process table | 335 | service (string): service name to look for in process table |
1381 | 315 | pgrep_full (boolean): Use full command line search mode with pgrep | 336 | pgrep_full: [Deprecated] Use full command line search mode with pgrep |
1382 | 316 | sleep_time (int): Seconds to sleep before looking for process | 337 | sleep_time (int): Initial sleep time (s) before looking for file |
1383 | 317 | retry_count (int): If service is not found, how many times to retry | 338 | retry_sleep_time (int): Time (s) to sleep between retries |
1384 | 339 | retry_count (int): If file is not found, how many times to retry | ||
1385 | 318 | 340 | ||
1386 | 319 | Returns: | 341 | Returns: |
1387 | 320 | bool: True if service found and its start time it newer than mtime, | 342 | bool: True if service found and its start time it newer than mtime, |
1388 | 321 | False if service is older than mtime or if service was | 343 | False if service is older than mtime or if service was |
1389 | 322 | not found. | 344 | not found. |
1390 | 323 | """ | 345 | """ |
1392 | 324 | self.log.debug('Checking %s restarted since %s' % (service, mtime)) | 346 | # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now |
1393 | 347 | # used instead of pgrep. pgrep_full is still passed through to ensure | ||
1394 | 348 | # deprecation WARNS. lp1474030 | ||
1395 | 349 | |||
1396 | 350 | unit_name = sentry_unit.info['unit_name'] | ||
1397 | 351 | self.log.debug('Checking that %s service restarted since %s on ' | ||
1398 | 352 | '%s' % (service, mtime, unit_name)) | ||
1399 | 325 | time.sleep(sleep_time) | 353 | time.sleep(sleep_time) |
1409 | 326 | proc_start_time = self._get_proc_start_time(sentry_unit, service, | 354 | proc_start_time = None |
1410 | 327 | pgrep_full) | 355 | tries = 0 |
1411 | 328 | while retry_count > 0 and not proc_start_time: | 356 | while tries <= retry_count and not proc_start_time: |
1412 | 329 | self.log.debug('No pid file found for service %s, will retry %i ' | 357 | try: |
1413 | 330 | 'more times' % (service, retry_count)) | 358 | proc_start_time = self._get_proc_start_time(sentry_unit, |
1414 | 331 | time.sleep(30) | 359 | service, |
1415 | 332 | proc_start_time = self._get_proc_start_time(sentry_unit, service, | 360 | pgrep_full) |
1416 | 333 | pgrep_full) | 361 | self.log.debug('Attempt {} to get {} proc start time on {} ' |
1417 | 334 | retry_count = retry_count - 1 | 362 | 'OK'.format(tries, service, unit_name)) |
1418 | 363 | except IOError as e: | ||
1419 | 364 | # NOTE(beisner) - race avoidance, proc may not exist yet. | ||
1420 | 365 | # https://bugs.launchpad.net/charm-helpers/+bug/1474030 | ||
1421 | 366 | self.log.debug('Attempt {} to get {} proc start time on {} ' | ||
1422 | 367 | 'failed\n{}'.format(tries, service, | ||
1423 | 368 | unit_name, e)) | ||
1424 | 369 | time.sleep(retry_sleep_time) | ||
1425 | 370 | tries += 1 | ||
1426 | 335 | 371 | ||
1427 | 336 | if not proc_start_time: | 372 | if not proc_start_time: |
1428 | 337 | self.log.warn('No proc start time found, assuming service did ' | 373 | self.log.warn('No proc start time found, assuming service did ' |
1429 | 338 | 'not start') | 374 | 'not start') |
1430 | 339 | return False | 375 | return False |
1431 | 340 | if proc_start_time >= mtime: | 376 | if proc_start_time >= mtime: |
1434 | 341 | self.log.debug('proc start time is newer than provided mtime' | 377 | self.log.debug('Proc start time is newer than provided mtime' |
1435 | 342 | '(%s >= %s)' % (proc_start_time, mtime)) | 378 | '(%s >= %s) on %s (OK)' % (proc_start_time, |
1436 | 379 | mtime, unit_name)) | ||
1437 | 343 | return True | 380 | return True |
1438 | 344 | else: | 381 | else: |
1442 | 345 | self.log.warn('proc start time (%s) is older than provided mtime ' | 382 | self.log.warn('Proc start time (%s) is older than provided mtime ' |
1443 | 346 | '(%s), service did not restart' % (proc_start_time, | 383 | '(%s) on %s, service did not ' |
1444 | 347 | mtime)) | 384 | 'restart' % (proc_start_time, mtime, unit_name)) |
1445 | 348 | return False | 385 | return False |
1446 | 349 | 386 | ||
1447 | 350 | def config_updated_since(self, sentry_unit, filename, mtime, | 387 | def config_updated_since(self, sentry_unit, filename, mtime, |
1449 | 351 | sleep_time=20): | 388 | sleep_time=20, retry_count=30, |
1450 | 389 | retry_sleep_time=10): | ||
1451 | 352 | """Check if file was modified after a given time. | 390 | """Check if file was modified after a given time. |
1452 | 353 | 391 | ||
1453 | 354 | Args: | 392 | Args: |
1454 | 355 | sentry_unit (sentry): The sentry unit to check the file mtime on | 393 | sentry_unit (sentry): The sentry unit to check the file mtime on |
1455 | 356 | filename (string): The file to check mtime of | 394 | filename (string): The file to check mtime of |
1456 | 357 | mtime (float): The epoch time to check against | 395 | mtime (float): The epoch time to check against |
1458 | 358 | sleep_time (int): Seconds to sleep before looking for process | 396 | sleep_time (int): Initial sleep time (s) before looking for file |
1459 | 397 | retry_sleep_time (int): Time (s) to sleep between retries | ||
1460 | 398 | retry_count (int): If file is not found, how many times to retry | ||
1461 | 359 | 399 | ||
1462 | 360 | Returns: | 400 | Returns: |
1463 | 361 | bool: True if file was modified more recently than mtime, False if | 401 | bool: True if file was modified more recently than mtime, False if |
1465 | 362 | file was modified before mtime, | 402 | file was modified before mtime, or if file not found. |
1466 | 363 | """ | 403 | """ |
1468 | 364 | self.log.debug('Checking %s updated since %s' % (filename, mtime)) | 404 | unit_name = sentry_unit.info['unit_name'] |
1469 | 405 | self.log.debug('Checking that %s updated since %s on ' | ||
1470 | 406 | '%s' % (filename, mtime, unit_name)) | ||
1471 | 365 | time.sleep(sleep_time) | 407 | time.sleep(sleep_time) |
1473 | 366 | file_mtime = self._get_file_mtime(sentry_unit, filename) | 408 | file_mtime = None |
1474 | 409 | tries = 0 | ||
1475 | 410 | while tries <= retry_count and not file_mtime: | ||
1476 | 411 | try: | ||
1477 | 412 | file_mtime = self._get_file_mtime(sentry_unit, filename) | ||
1478 | 413 | self.log.debug('Attempt {} to get {} file mtime on {} ' | ||
1479 | 414 | 'OK'.format(tries, filename, unit_name)) | ||
1480 | 415 | except IOError as e: | ||
1481 | 416 | # NOTE(beisner) - race avoidance, file may not exist yet. | ||
1482 | 417 | # https://bugs.launchpad.net/charm-helpers/+bug/1474030 | ||
1483 | 418 | self.log.debug('Attempt {} to get {} file mtime on {} ' | ||
1484 | 419 | 'failed\n{}'.format(tries, filename, | ||
1485 | 420 | unit_name, e)) | ||
1486 | 421 | time.sleep(retry_sleep_time) | ||
1487 | 422 | tries += 1 | ||
1488 | 423 | |||
1489 | 424 | if not file_mtime: | ||
1490 | 425 | self.log.warn('Could not determine file mtime, assuming ' | ||
1491 | 426 | 'file does not exist') | ||
1492 | 427 | return False | ||
1493 | 428 | |||
1494 | 367 | if file_mtime >= mtime: | 429 | if file_mtime >= mtime: |
1495 | 368 | self.log.debug('File mtime is newer than provided mtime ' | 430 | self.log.debug('File mtime is newer than provided mtime ' |
1497 | 369 | '(%s >= %s)' % (file_mtime, mtime)) | 431 | '(%s >= %s) on %s (OK)' % (file_mtime, |
1498 | 432 | mtime, unit_name)) | ||
1499 | 370 | return True | 433 | return True |
1500 | 371 | else: | 434 | else: |
1503 | 372 | self.log.warn('File mtime %s is older than provided mtime %s' | 435 | self.log.warn('File mtime is older than provided mtime' |
1504 | 373 | % (file_mtime, mtime)) | 436 | '(%s < on %s) on %s' % (file_mtime, |
1505 | 437 | mtime, unit_name)) | ||
1506 | 374 | return False | 438 | return False |
1507 | 375 | 439 | ||
1508 | 376 | def validate_service_config_changed(self, sentry_unit, mtime, service, | 440 | def validate_service_config_changed(self, sentry_unit, mtime, service, |
1511 | 377 | filename, pgrep_full=False, | 441 | filename, pgrep_full=None, |
1512 | 378 | sleep_time=20, retry_count=2): | 442 | sleep_time=20, retry_count=30, |
1513 | 443 | retry_sleep_time=10): | ||
1514 | 379 | """Check service and file were updated after mtime | 444 | """Check service and file were updated after mtime |
1515 | 380 | 445 | ||
1516 | 381 | Args: | 446 | Args: |
1517 | @@ -383,9 +448,10 @@ | |||
1518 | 383 | mtime (float): The epoch time to check against | 448 | mtime (float): The epoch time to check against |
1519 | 384 | service (string): service name to look for in process table | 449 | service (string): service name to look for in process table |
1520 | 385 | filename (string): The file to check mtime of | 450 | filename (string): The file to check mtime of |
1523 | 386 | pgrep_full (boolean): Use full command line search mode with pgrep | 451 | pgrep_full: [Deprecated] Use full command line search mode with pgrep |
1524 | 387 | sleep_time (int): Seconds to sleep before looking for process | 452 | sleep_time (int): Initial sleep in seconds to pass to test helpers |
1525 | 388 | retry_count (int): If service is not found, how many times to retry | 453 | retry_count (int): If service is not found, how many times to retry |
1526 | 454 | retry_sleep_time (int): Time in seconds to wait between retries | ||
1527 | 389 | 455 | ||
1528 | 390 | Typical Usage: | 456 | Typical Usage: |
1529 | 391 | u = OpenStackAmuletUtils(ERROR) | 457 | u = OpenStackAmuletUtils(ERROR) |
1530 | @@ -402,15 +468,27 @@ | |||
1531 | 402 | mtime, False if service is older than mtime or if service was | 468 | mtime, False if service is older than mtime or if service was |
1532 | 403 | not found or if filename was modified before mtime. | 469 | not found or if filename was modified before mtime. |
1533 | 404 | """ | 470 | """ |
1543 | 405 | self.log.debug('Checking %s restarted since %s' % (service, mtime)) | 471 | |
1544 | 406 | time.sleep(sleep_time) | 472 | # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now |
1545 | 407 | service_restart = self.service_restarted_since(sentry_unit, mtime, | 473 | # used instead of pgrep. pgrep_full is still passed through to ensure |
1546 | 408 | service, | 474 | # deprecation WARNS. lp1474030 |
1547 | 409 | pgrep_full=pgrep_full, | 475 | |
1548 | 410 | sleep_time=0, | 476 | service_restart = self.service_restarted_since( |
1549 | 411 | retry_count=retry_count) | 477 | sentry_unit, mtime, |
1550 | 412 | config_update = self.config_updated_since(sentry_unit, filename, mtime, | 478 | service, |
1551 | 413 | sleep_time=0) | 479 | pgrep_full=pgrep_full, |
1552 | 480 | sleep_time=sleep_time, | ||
1553 | 481 | retry_count=retry_count, | ||
1554 | 482 | retry_sleep_time=retry_sleep_time) | ||
1555 | 483 | |||
1556 | 484 | config_update = self.config_updated_since( | ||
1557 | 485 | sentry_unit, | ||
1558 | 486 | filename, | ||
1559 | 487 | mtime, | ||
1560 | 488 | sleep_time=sleep_time, | ||
1561 | 489 | retry_count=retry_count, | ||
1562 | 490 | retry_sleep_time=retry_sleep_time) | ||
1563 | 491 | |||
1564 | 414 | return service_restart and config_update | 492 | return service_restart and config_update |
1565 | 415 | 493 | ||
1566 | 416 | def get_sentry_time(self, sentry_unit): | 494 | def get_sentry_time(self, sentry_unit): |
1567 | @@ -428,7 +506,6 @@ | |||
1568 | 428 | """Return a list of all Ubuntu releases in order of release.""" | 506 | """Return a list of all Ubuntu releases in order of release.""" |
1569 | 429 | _d = distro_info.UbuntuDistroInfo() | 507 | _d = distro_info.UbuntuDistroInfo() |
1570 | 430 | _release_list = _d.all | 508 | _release_list = _d.all |
1571 | 431 | self.log.debug('Ubuntu release list: {}'.format(_release_list)) | ||
1572 | 432 | return _release_list | 509 | return _release_list |
1573 | 433 | 510 | ||
1574 | 434 | def file_to_url(self, file_rel_path): | 511 | def file_to_url(self, file_rel_path): |
1575 | @@ -568,6 +645,142 @@ | |||
1576 | 568 | 645 | ||
1577 | 569 | return None | 646 | return None |
1578 | 570 | 647 | ||
1579 | 648 | def validate_sectionless_conf(self, file_contents, expected): | ||
1580 | 649 | """A crude conf parser. Useful to inspect configuration files which | ||
1581 | 650 | do not have section headers (as would be necessary in order to use | ||
1582 | 651 | the configparser). Such as openstack-dashboard or rabbitmq confs.""" | ||
1583 | 652 | for line in file_contents.split('\n'): | ||
1584 | 653 | if '=' in line: | ||
1585 | 654 | args = line.split('=') | ||
1586 | 655 | if len(args) <= 1: | ||
1587 | 656 | continue | ||
1588 | 657 | key = args[0].strip() | ||
1589 | 658 | value = args[1].strip() | ||
1590 | 659 | if key in expected.keys(): | ||
1591 | 660 | if expected[key] != value: | ||
1592 | 661 | msg = ('Config mismatch. Expected, actual: {}, ' | ||
1593 | 662 | '{}'.format(expected[key], value)) | ||
1594 | 663 | amulet.raise_status(amulet.FAIL, msg=msg) | ||
1595 | 664 | |||
1596 | 665 | def get_unit_hostnames(self, units): | ||
1597 | 666 | """Return a dict of juju unit names to hostnames.""" | ||
1598 | 667 | host_names = {} | ||
1599 | 668 | for unit in units: | ||
1600 | 669 | host_names[unit.info['unit_name']] = \ | ||
1601 | 670 | str(unit.file_contents('/etc/hostname').strip()) | ||
1602 | 671 | self.log.debug('Unit host names: {}'.format(host_names)) | ||
1603 | 672 | return host_names | ||
1604 | 673 | |||
1605 | 674 | def run_cmd_unit(self, sentry_unit, cmd): | ||
1606 | 675 | """Run a command on a unit, return the output and exit code.""" | ||
1607 | 676 | output, code = sentry_unit.run(cmd) | ||
1608 | 677 | if code == 0: | ||
1609 | 678 | self.log.debug('{} `{}` command returned {} ' | ||
1610 | 679 | '(OK)'.format(sentry_unit.info['unit_name'], | ||
1611 | 680 | cmd, code)) | ||
1612 | 681 | else: | ||
1613 | 682 | msg = ('{} `{}` command returned {} ' | ||
1614 | 683 | '{}'.format(sentry_unit.info['unit_name'], | ||
1615 | 684 | cmd, code, output)) | ||
1616 | 685 | amulet.raise_status(amulet.FAIL, msg=msg) | ||
1617 | 686 | return str(output), code | ||
1618 | 687 | |||
1619 | 688 | def file_exists_on_unit(self, sentry_unit, file_name): | ||
1620 | 689 | """Check if a file exists on a unit.""" | ||
1621 | 690 | try: | ||
1622 | 691 | sentry_unit.file_stat(file_name) | ||
1623 | 692 | return True | ||
1624 | 693 | except IOError: | ||
1625 | 694 | return False | ||
1626 | 695 | except Exception as e: | ||
1627 | 696 | msg = 'Error checking file {}: {}'.format(file_name, e) | ||
1628 | 697 | amulet.raise_status(amulet.FAIL, msg=msg) | ||
1629 | 698 | |||
1630 | 699 | def file_contents_safe(self, sentry_unit, file_name, | ||
1631 | 700 | max_wait=60, fatal=False): | ||
1632 | 701 | """Get file contents from a sentry unit. Wrap amulet file_contents | ||
1633 | 702 | with retry logic to address races where a file checks as existing, | ||
1634 | 703 | but no longer exists by the time file_contents is called. | ||
1635 | 704 | Return None if file not found. Optionally raise if fatal is True.""" | ||
1636 | 705 | unit_name = sentry_unit.info['unit_name'] | ||
1637 | 706 | file_contents = False | ||
1638 | 707 | tries = 0 | ||
1639 | 708 | while not file_contents and tries < (max_wait / 4): | ||
1640 | 709 | try: | ||
1641 | 710 | file_contents = sentry_unit.file_contents(file_name) | ||
1642 | 711 | except IOError: | ||
1643 | 712 | self.log.debug('Attempt {} to open file {} from {} ' | ||
1644 | 713 | 'failed'.format(tries, file_name, | ||
1645 | 714 | unit_name)) | ||
1646 | 715 | time.sleep(4) | ||
1647 | 716 | tries += 1 | ||
1648 | 717 | |||
1649 | 718 | if file_contents: | ||
1650 | 719 | return file_contents | ||
1651 | 720 | elif not fatal: | ||
1652 | 721 | return None | ||
1653 | 722 | elif fatal: | ||
1654 | 723 | msg = 'Failed to get file contents from unit.' | ||
1655 | 724 | amulet.raise_status(amulet.FAIL, msg) | ||
1656 | 725 | |||
1657 | 726 | def port_knock_tcp(self, host="localhost", port=22, timeout=15): | ||
1658 | 727 | """Open a TCP socket to check for a listening sevice on a host. | ||
1659 | 728 | |||
1660 | 729 | :param host: host name or IP address, default to localhost | ||
1661 | 730 | :param port: TCP port number, default to 22 | ||
1662 | 731 | :param timeout: Connect timeout, default to 15 seconds | ||
1663 | 732 | :returns: True if successful, False if connect failed | ||
1664 | 733 | """ | ||
1665 | 734 | |||
1666 | 735 | # Resolve host name if possible | ||
1667 | 736 | try: | ||
1668 | 737 | connect_host = socket.gethostbyname(host) | ||
1669 | 738 | host_human = "{} ({})".format(connect_host, host) | ||
1670 | 739 | except socket.error as e: | ||
1671 | 740 | self.log.warn('Unable to resolve address: ' | ||
1672 | 741 | '{} ({}) Trying anyway!'.format(host, e)) | ||
1673 | 742 | connect_host = host | ||
1674 | 743 | host_human = connect_host | ||
1675 | 744 | |||
1676 | 745 | # Attempt socket connection | ||
1677 | 746 | try: | ||
1678 | 747 | knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) | ||
1679 | 748 | knock.settimeout(timeout) | ||
1680 | 749 | knock.connect((connect_host, port)) | ||
1681 | 750 | knock.close() | ||
1682 | 751 | self.log.debug('Socket connect OK for host ' | ||
1683 | 752 | '{} on port {}.'.format(host_human, port)) | ||
1684 | 753 | return True | ||
1685 | 754 | except socket.error as e: | ||
1686 | 755 | self.log.debug('Socket connect FAIL for' | ||
1687 | 756 | ' {} port {} ({})'.format(host_human, port, e)) | ||
1688 | 757 | return False | ||
1689 | 758 | |||
1690 | 759 | def port_knock_units(self, sentry_units, port=22, | ||
1691 | 760 | timeout=15, expect_success=True): | ||
1692 | 761 | """Open a TCP socket to check for a listening sevice on each | ||
1693 | 762 | listed juju unit. | ||
1694 | 763 | |||
1695 | 764 | :param sentry_units: list of sentry unit pointers | ||
1696 | 765 | :param port: TCP port number, default to 22 | ||
1697 | 766 | :param timeout: Connect timeout, default to 15 seconds | ||
1698 | 767 | :expect_success: True by default, set False to invert logic | ||
1699 | 768 | :returns: None if successful, Failure message otherwise | ||
1700 | 769 | """ | ||
1701 | 770 | for unit in sentry_units: | ||
1702 | 771 | host = unit.info['public-address'] | ||
1703 | 772 | connected = self.port_knock_tcp(host, port, timeout) | ||
1704 | 773 | if not connected and expect_success: | ||
1705 | 774 | return 'Socket connect failed.' | ||
1706 | 775 | elif connected and not expect_success: | ||
1707 | 776 | return 'Socket connected unexpectedly.' | ||
1708 | 777 | |||
1709 | 778 | def get_uuid_epoch_stamp(self): | ||
1710 | 779 | """Returns a stamp string based on uuid4 and epoch time. Useful in | ||
1711 | 780 | generating test messages which need to be unique-ish.""" | ||
1712 | 781 | return '[{}-{}]'.format(uuid.uuid4(), time.time()) | ||
1713 | 782 | |||
1714 | 783 | # amulet juju action helpers: | ||
1715 | 571 | def run_action(self, unit_sentry, action, | 784 | def run_action(self, unit_sentry, action, |
1716 | 572 | _check_output=subprocess.check_output): | 785 | _check_output=subprocess.check_output): |
1717 | 573 | """Run the named action on a given unit sentry. | 786 | """Run the named action on a given unit sentry. |
1718 | @@ -594,3 +807,12 @@ | |||
1719 | 594 | output = _check_output(command, universal_newlines=True) | 807 | output = _check_output(command, universal_newlines=True) |
1720 | 595 | data = json.loads(output) | 808 | data = json.loads(output) |
1721 | 596 | return data.get(u"status") == "completed" | 809 | return data.get(u"status") == "completed" |
1722 | 810 | |||
1723 | 811 | def status_get(self, unit): | ||
1724 | 812 | """Return the current service status of this unit.""" | ||
1725 | 813 | raw_status, return_code = unit.run( | ||
1726 | 814 | "status-get --format=json --include-data") | ||
1727 | 815 | if return_code != 0: | ||
1728 | 816 | return ("unknown", "") | ||
1729 | 817 | status = json.loads(raw_status) | ||
1730 | 818 | return (status["status"], status["message"]) | ||
1731 | 597 | 819 | ||
1732 | === modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py' | |||
1733 | --- tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-08-19 13:50:42 +0000 | |||
1734 | +++ tests/charmhelpers/contrib/openstack/amulet/deployment.py 2016-01-22 22:32:27 +0000 | |||
1735 | @@ -14,12 +14,18 @@ | |||
1736 | 14 | # You should have received a copy of the GNU Lesser General Public License | 14 | # You should have received a copy of the GNU Lesser General Public License |
1737 | 15 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. | 15 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
1738 | 16 | 16 | ||
1739 | 17 | import logging | ||
1740 | 18 | import re | ||
1741 | 19 | import sys | ||
1742 | 17 | import six | 20 | import six |
1743 | 18 | from collections import OrderedDict | 21 | from collections import OrderedDict |
1744 | 19 | from charmhelpers.contrib.amulet.deployment import ( | 22 | from charmhelpers.contrib.amulet.deployment import ( |
1745 | 20 | AmuletDeployment | 23 | AmuletDeployment |
1746 | 21 | ) | 24 | ) |
1747 | 22 | 25 | ||
1748 | 26 | DEBUG = logging.DEBUG | ||
1749 | 27 | ERROR = logging.ERROR | ||
1750 | 28 | |||
1751 | 23 | 29 | ||
1752 | 24 | class OpenStackAmuletDeployment(AmuletDeployment): | 30 | class OpenStackAmuletDeployment(AmuletDeployment): |
1753 | 25 | """OpenStack amulet deployment. | 31 | """OpenStack amulet deployment. |
1754 | @@ -28,9 +34,12 @@ | |||
1755 | 28 | that is specifically for use by OpenStack charms. | 34 | that is specifically for use by OpenStack charms. |
1756 | 29 | """ | 35 | """ |
1757 | 30 | 36 | ||
1759 | 31 | def __init__(self, series=None, openstack=None, source=None, stable=True): | 37 | def __init__(self, series=None, openstack=None, source=None, |
1760 | 38 | stable=True, log_level=DEBUG): | ||
1761 | 32 | """Initialize the deployment environment.""" | 39 | """Initialize the deployment environment.""" |
1762 | 33 | super(OpenStackAmuletDeployment, self).__init__(series) | 40 | super(OpenStackAmuletDeployment, self).__init__(series) |
1763 | 41 | self.log = self.get_logger(level=log_level) | ||
1764 | 42 | self.log.info('OpenStackAmuletDeployment: init') | ||
1765 | 34 | self.openstack = openstack | 43 | self.openstack = openstack |
1766 | 35 | self.source = source | 44 | self.source = source |
1767 | 36 | self.stable = stable | 45 | self.stable = stable |
1768 | @@ -38,26 +47,55 @@ | |||
1769 | 38 | # out. | 47 | # out. |
1770 | 39 | self.current_next = "trusty" | 48 | self.current_next = "trusty" |
1771 | 40 | 49 | ||
1772 | 50 | def get_logger(self, name="deployment-logger", level=logging.DEBUG): | ||
1773 | 51 | """Get a logger object that will log to stdout.""" | ||
1774 | 52 | log = logging | ||
1775 | 53 | logger = log.getLogger(name) | ||
1776 | 54 | fmt = log.Formatter("%(asctime)s %(funcName)s " | ||
1777 | 55 | "%(levelname)s: %(message)s") | ||
1778 | 56 | |||
1779 | 57 | handler = log.StreamHandler(stream=sys.stdout) | ||
1780 | 58 | handler.setLevel(level) | ||
1781 | 59 | handler.setFormatter(fmt) | ||
1782 | 60 | |||
1783 | 61 | logger.addHandler(handler) | ||
1784 | 62 | logger.setLevel(level) | ||
1785 | 63 | |||
1786 | 64 | return logger | ||
1787 | 65 | |||
1788 | 41 | def _determine_branch_locations(self, other_services): | 66 | def _determine_branch_locations(self, other_services): |
1789 | 42 | """Determine the branch locations for the other services. | 67 | """Determine the branch locations for the other services. |
1790 | 43 | 68 | ||
1791 | 44 | Determine if the local branch being tested is derived from its | 69 | Determine if the local branch being tested is derived from its |
1792 | 45 | stable or next (dev) branch, and based on this, use the corresonding | 70 | stable or next (dev) branch, and based on this, use the corresonding |
1793 | 46 | stable or next branches for the other_services.""" | 71 | stable or next branches for the other_services.""" |
1794 | 72 | |||
1795 | 73 | self.log.info('OpenStackAmuletDeployment: determine branch locations') | ||
1796 | 74 | |||
1797 | 75 | # Charms outside the lp:~openstack-charmers namespace | ||
1798 | 47 | base_charms = ['mysql', 'mongodb', 'nrpe'] | 76 | base_charms = ['mysql', 'mongodb', 'nrpe'] |
1799 | 48 | 77 | ||
1800 | 78 | # Force these charms to current series even when using an older series. | ||
1801 | 79 | # ie. Use trusty/nrpe even when series is precise, as the P charm | ||
1802 | 80 | # does not possess the necessary external master config and hooks. | ||
1803 | 81 | force_series_current = ['nrpe'] | ||
1804 | 82 | |||
1805 | 49 | if self.series in ['precise', 'trusty']: | 83 | if self.series in ['precise', 'trusty']: |
1806 | 50 | base_series = self.series | 84 | base_series = self.series |
1807 | 51 | else: | 85 | else: |
1808 | 52 | base_series = self.current_next | 86 | base_series = self.current_next |
1809 | 53 | 87 | ||
1812 | 54 | if self.stable: | 88 | for svc in other_services: |
1813 | 55 | for svc in other_services: | 89 | if svc['name'] in force_series_current: |
1814 | 90 | base_series = self.current_next | ||
1815 | 91 | # If a location has been explicitly set, use it | ||
1816 | 92 | if svc.get('location'): | ||
1817 | 93 | continue | ||
1818 | 94 | if self.stable: | ||
1819 | 56 | temp = 'lp:charms/{}/{}' | 95 | temp = 'lp:charms/{}/{}' |
1820 | 57 | svc['location'] = temp.format(base_series, | 96 | svc['location'] = temp.format(base_series, |
1821 | 58 | svc['name']) | 97 | svc['name']) |
1824 | 59 | else: | 98 | else: |
1823 | 60 | for svc in other_services: | ||
1825 | 61 | if svc['name'] in base_charms: | 99 | if svc['name'] in base_charms: |
1826 | 62 | temp = 'lp:charms/{}/{}' | 100 | temp = 'lp:charms/{}/{}' |
1827 | 63 | svc['location'] = temp.format(base_series, | 101 | svc['location'] = temp.format(base_series, |
1828 | @@ -66,10 +104,13 @@ | |||
1829 | 66 | temp = 'lp:~openstack-charmers/charms/{}/{}/next' | 104 | temp = 'lp:~openstack-charmers/charms/{}/{}/next' |
1830 | 67 | svc['location'] = temp.format(self.current_next, | 105 | svc['location'] = temp.format(self.current_next, |
1831 | 68 | svc['name']) | 106 | svc['name']) |
1832 | 107 | |||
1833 | 69 | return other_services | 108 | return other_services |
1834 | 70 | 109 | ||
1835 | 71 | def _add_services(self, this_service, other_services): | 110 | def _add_services(self, this_service, other_services): |
1836 | 72 | """Add services to the deployment and set openstack-origin/source.""" | 111 | """Add services to the deployment and set openstack-origin/source.""" |
1837 | 112 | self.log.info('OpenStackAmuletDeployment: adding services') | ||
1838 | 113 | |||
1839 | 73 | other_services = self._determine_branch_locations(other_services) | 114 | other_services = self._determine_branch_locations(other_services) |
1840 | 74 | 115 | ||
1841 | 75 | super(OpenStackAmuletDeployment, self)._add_services(this_service, | 116 | super(OpenStackAmuletDeployment, self)._add_services(this_service, |
1842 | @@ -77,29 +118,103 @@ | |||
1843 | 77 | 118 | ||
1844 | 78 | services = other_services | 119 | services = other_services |
1845 | 79 | services.append(this_service) | 120 | services.append(this_service) |
1846 | 121 | |||
1847 | 122 | # Charms which should use the source config option | ||
1848 | 80 | use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', | 123 | use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', |
1849 | 81 | 'ceph-osd', 'ceph-radosgw'] | 124 | 'ceph-osd', 'ceph-radosgw'] |
1853 | 82 | # Most OpenStack subordinate charms do not expose an origin option | 125 | |
1854 | 83 | # as that is controlled by the principle. | 126 | # Charms which can not use openstack-origin, ie. many subordinates |
1855 | 84 | ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe'] | 127 | no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe', |
1856 | 128 | 'openvswitch-odl', 'neutron-api-odl', 'odl-controller', | ||
1857 | 129 | 'cinder-backup'] | ||
1858 | 85 | 130 | ||
1859 | 86 | if self.openstack: | 131 | if self.openstack: |
1860 | 87 | for svc in services: | 132 | for svc in services: |
1862 | 88 | if svc['name'] not in use_source + ignore: | 133 | if svc['name'] not in use_source + no_origin: |
1863 | 89 | config = {'openstack-origin': self.openstack} | 134 | config = {'openstack-origin': self.openstack} |
1864 | 90 | self.d.configure(svc['name'], config) | 135 | self.d.configure(svc['name'], config) |
1865 | 91 | 136 | ||
1866 | 92 | if self.source: | 137 | if self.source: |
1867 | 93 | for svc in services: | 138 | for svc in services: |
1869 | 94 | if svc['name'] in use_source and svc['name'] not in ignore: | 139 | if svc['name'] in use_source and svc['name'] not in no_origin: |
1870 | 95 | config = {'source': self.source} | 140 | config = {'source': self.source} |
1871 | 96 | self.d.configure(svc['name'], config) | 141 | self.d.configure(svc['name'], config) |
1872 | 97 | 142 | ||
1873 | 98 | def _configure_services(self, configs): | 143 | def _configure_services(self, configs): |
1874 | 99 | """Configure all of the services.""" | 144 | """Configure all of the services.""" |
1875 | 145 | self.log.info('OpenStackAmuletDeployment: configure services') | ||
1876 | 100 | for service, config in six.iteritems(configs): | 146 | for service, config in six.iteritems(configs): |
1877 | 101 | self.d.configure(service, config) | 147 | self.d.configure(service, config) |
1878 | 102 | 148 | ||
1879 | 149 | def _auto_wait_for_status(self, message=None, exclude_services=None, | ||
1880 | 150 | include_only=None, timeout=1800): | ||
1881 | 151 | """Wait for all units to have a specific extended status, except | ||
1882 | 152 | for any defined as excluded. Unless specified via message, any | ||
1883 | 153 | status containing any case of 'ready' will be considered a match. | ||
1884 | 154 | |||
1885 | 155 | Examples of message usage: | ||
1886 | 156 | |||
1887 | 157 | Wait for all unit status to CONTAIN any case of 'ready' or 'ok': | ||
1888 | 158 | message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE) | ||
1889 | 159 | |||
1890 | 160 | Wait for all units to reach this status (exact match): | ||
1891 | 161 | message = re.compile('^Unit is ready and clustered$') | ||
1892 | 162 | |||
1893 | 163 | Wait for all units to reach any one of these (exact match): | ||
1894 | 164 | message = re.compile('Unit is ready|OK|Ready') | ||
1895 | 165 | |||
1896 | 166 | Wait for at least one unit to reach this status (exact match): | ||
1897 | 167 | message = {'ready'} | ||
1898 | 168 | |||
1899 | 169 | See Amulet's sentry.wait_for_messages() for message usage detail. | ||
1900 | 170 | https://github.com/juju/amulet/blob/master/amulet/sentry.py | ||
1901 | 171 | |||
1902 | 172 | :param message: Expected status match | ||
1903 | 173 | :param exclude_services: List of juju service names to ignore, | ||
1904 | 174 | not to be used in conjuction with include_only. | ||
1905 | 175 | :param include_only: List of juju service names to exclusively check, | ||
1906 | 176 | not to be used in conjuction with exclude_services. | ||
1907 | 177 | :param timeout: Maximum time in seconds to wait for status match | ||
1908 | 178 | :returns: None. Raises if timeout is hit. | ||
1909 | 179 | """ | ||
1910 | 180 | self.log.info('Waiting for extended status on units...') | ||
1911 | 181 | |||
1912 | 182 | all_services = self.d.services.keys() | ||
1913 | 183 | |||
1914 | 184 | if exclude_services and include_only: | ||
1915 | 185 | raise ValueError('exclude_services can not be used ' | ||
1916 | 186 | 'with include_only') | ||
1917 | 187 | |||
1918 | 188 | if message: | ||
1919 | 189 | if isinstance(message, re._pattern_type): | ||
1920 | 190 | match = message.pattern | ||
1921 | 191 | else: | ||
1922 | 192 | match = message | ||
1923 | 193 | |||
1924 | 194 | self.log.debug('Custom extended status wait match: ' | ||
1925 | 195 | '{}'.format(match)) | ||
1926 | 196 | else: | ||
1927 | 197 | self.log.debug('Default extended status wait match: contains ' | ||
1928 | 198 | 'READY (case-insensitive)') | ||
1929 | 199 | message = re.compile('.*ready.*', re.IGNORECASE) | ||
1930 | 200 | |||
1931 | 201 | if exclude_services: | ||
1932 | 202 | self.log.debug('Excluding services from extended status match: ' | ||
1933 | 203 | '{}'.format(exclude_services)) | ||
1934 | 204 | else: | ||
1935 | 205 | exclude_services = [] | ||
1936 | 206 | |||
1937 | 207 | if include_only: | ||
1938 | 208 | services = include_only | ||
1939 | 209 | else: | ||
1940 | 210 | services = list(set(all_services) - set(exclude_services)) | ||
1941 | 211 | |||
1942 | 212 | self.log.debug('Waiting up to {}s for extended status on services: ' | ||
1943 | 213 | '{}'.format(timeout, services)) | ||
1944 | 214 | service_messages = {service: message for service in services} | ||
1945 | 215 | self.d.sentry.wait_for_messages(service_messages, timeout=timeout) | ||
1946 | 216 | self.log.info('OK') | ||
1947 | 217 | |||
1948 | 103 | def _get_openstack_release(self): | 218 | def _get_openstack_release(self): |
1949 | 104 | """Get openstack release. | 219 | """Get openstack release. |
1950 | 105 | 220 | ||
1951 | @@ -111,7 +226,8 @@ | |||
1952 | 111 | self.precise_havana, self.precise_icehouse, | 226 | self.precise_havana, self.precise_icehouse, |
1953 | 112 | self.trusty_icehouse, self.trusty_juno, self.utopic_juno, | 227 | self.trusty_icehouse, self.trusty_juno, self.utopic_juno, |
1954 | 113 | self.trusty_kilo, self.vivid_kilo, self.trusty_liberty, | 228 | self.trusty_kilo, self.vivid_kilo, self.trusty_liberty, |
1956 | 114 | self.wily_liberty) = range(12) | 229 | self.wily_liberty, self.trusty_mitaka, |
1957 | 230 | self.xenial_mitaka) = range(14) | ||
1958 | 115 | 231 | ||
1959 | 116 | releases = { | 232 | releases = { |
1960 | 117 | ('precise', None): self.precise_essex, | 233 | ('precise', None): self.precise_essex, |
1961 | @@ -123,9 +239,11 @@ | |||
1962 | 123 | ('trusty', 'cloud:trusty-juno'): self.trusty_juno, | 239 | ('trusty', 'cloud:trusty-juno'): self.trusty_juno, |
1963 | 124 | ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo, | 240 | ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo, |
1964 | 125 | ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty, | 241 | ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty, |
1965 | 242 | ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka, | ||
1966 | 126 | ('utopic', None): self.utopic_juno, | 243 | ('utopic', None): self.utopic_juno, |
1967 | 127 | ('vivid', None): self.vivid_kilo, | 244 | ('vivid', None): self.vivid_kilo, |
1969 | 128 | ('wily', None): self.wily_liberty} | 245 | ('wily', None): self.wily_liberty, |
1970 | 246 | ('xenial', None): self.xenial_mitaka} | ||
1971 | 129 | return releases[(self.series, self.openstack)] | 247 | return releases[(self.series, self.openstack)] |
1972 | 130 | 248 | ||
1973 | 131 | def _get_openstack_release_string(self): | 249 | def _get_openstack_release_string(self): |
1974 | @@ -142,6 +260,7 @@ | |||
1975 | 142 | ('utopic', 'juno'), | 260 | ('utopic', 'juno'), |
1976 | 143 | ('vivid', 'kilo'), | 261 | ('vivid', 'kilo'), |
1977 | 144 | ('wily', 'liberty'), | 262 | ('wily', 'liberty'), |
1978 | 263 | ('xenial', 'mitaka'), | ||
1979 | 145 | ]) | 264 | ]) |
1980 | 146 | if self.openstack: | 265 | if self.openstack: |
1981 | 147 | os_origin = self.openstack.split(':')[1] | 266 | os_origin = self.openstack.split(':')[1] |
1982 | 148 | 267 | ||
1983 | === modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py' | |||
1984 | --- tests/charmhelpers/contrib/openstack/amulet/utils.py 2015-06-29 14:24:05 +0000 | |||
1985 | +++ tests/charmhelpers/contrib/openstack/amulet/utils.py 2016-01-22 22:32:27 +0000 | |||
1986 | @@ -18,6 +18,7 @@ | |||
1987 | 18 | import json | 18 | import json |
1988 | 19 | import logging | 19 | import logging |
1989 | 20 | import os | 20 | import os |
1990 | 21 | import re | ||
1991 | 21 | import six | 22 | import six |
1992 | 22 | import time | 23 | import time |
1993 | 23 | import urllib | 24 | import urllib |
1994 | @@ -27,6 +28,7 @@ | |||
1995 | 27 | import heatclient.v1.client as heat_client | 28 | import heatclient.v1.client as heat_client |
1996 | 28 | import keystoneclient.v2_0 as keystone_client | 29 | import keystoneclient.v2_0 as keystone_client |
1997 | 29 | import novaclient.v1_1.client as nova_client | 30 | import novaclient.v1_1.client as nova_client |
1998 | 31 | import pika | ||
1999 | 30 | import swiftclient | 32 | import swiftclient |
2000 | 31 | 33 | ||
2001 | 32 | from charmhelpers.contrib.amulet.utils import ( | 34 | from charmhelpers.contrib.amulet.utils import ( |
2002 | @@ -602,3 +604,382 @@ | |||
2003 | 602 | self.log.debug('Ceph {} samples (OK): ' | 604 | self.log.debug('Ceph {} samples (OK): ' |
2004 | 603 | '{}'.format(sample_type, samples)) | 605 | '{}'.format(sample_type, samples)) |
2005 | 604 | return None | 606 | return None |
2006 | 607 | |||
2007 | 608 | # rabbitmq/amqp specific helpers: | ||
2008 | 609 | |||
2009 | 610 | def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200): | ||
2010 | 611 | """Wait for rmq units extended status to show cluster readiness, | ||
2011 | 612 | after an optional initial sleep period. Initial sleep is likely | ||
2012 | 613 | necessary to be effective following a config change, as status | ||
2013 | 614 | message may not instantly update to non-ready.""" | ||
2014 | 615 | |||
2015 | 616 | if init_sleep: | ||
2016 | 617 | time.sleep(init_sleep) | ||
2017 | 618 | |||
2018 | 619 | message = re.compile('^Unit is ready and clustered$') | ||
2019 | 620 | deployment._auto_wait_for_status(message=message, | ||
2020 | 621 | timeout=timeout, | ||
2021 | 622 | include_only=['rabbitmq-server']) | ||
2022 | 623 | |||
2023 | 624 | def add_rmq_test_user(self, sentry_units, | ||
2024 | 625 | username="testuser1", password="changeme"): | ||
2025 | 626 | """Add a test user via the first rmq juju unit, check connection as | ||
2026 | 627 | the new user against all sentry units. | ||
2027 | 628 | |||
2028 | 629 | :param sentry_units: list of sentry unit pointers | ||
2029 | 630 | :param username: amqp user name, default to testuser1 | ||
2030 | 631 | :param password: amqp user password | ||
2031 | 632 | :returns: None if successful. Raise on error. | ||
2032 | 633 | """ | ||
2033 | 634 | self.log.debug('Adding rmq user ({})...'.format(username)) | ||
2034 | 635 | |||
2035 | 636 | # Check that user does not already exist | ||
2036 | 637 | cmd_user_list = 'rabbitmqctl list_users' | ||
2037 | 638 | output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list) | ||
2038 | 639 | if username in output: | ||
2039 | 640 | self.log.warning('User ({}) already exists, returning ' | ||
2040 | 641 | 'gracefully.'.format(username)) | ||
2041 | 642 | return | ||
2042 | 643 | |||
2043 | 644 | perms = '".*" ".*" ".*"' | ||
2044 | 645 | cmds = ['rabbitmqctl add_user {} {}'.format(username, password), | ||
2045 | 646 | 'rabbitmqctl set_permissions {} {}'.format(username, perms)] | ||
2046 | 647 | |||
2047 | 648 | # Add user via first unit | ||
2048 | 649 | for cmd in cmds: | ||
2049 | 650 | output, _ = self.run_cmd_unit(sentry_units[0], cmd) | ||
2050 | 651 | |||
2051 | 652 | # Check connection against the other sentry_units | ||
2052 | 653 | self.log.debug('Checking user connect against units...') | ||
2053 | 654 | for sentry_unit in sentry_units: | ||
2054 | 655 | connection = self.connect_amqp_by_unit(sentry_unit, ssl=False, | ||
2055 | 656 | username=username, | ||
2056 | 657 | password=password) | ||
2057 | 658 | connection.close() | ||
2058 | 659 | |||
2059 | 660 | def delete_rmq_test_user(self, sentry_units, username="testuser1"): | ||
2060 | 661 | """Delete a rabbitmq user via the first rmq juju unit. | ||
2061 | 662 | |||
2062 | 663 | :param sentry_units: list of sentry unit pointers | ||
2063 | 664 | :param username: amqp user name, default to testuser1 | ||
2064 | 665 | :param password: amqp user password | ||
2065 | 666 | :returns: None if successful or no such user. | ||
2066 | 667 | """ | ||
2067 | 668 | self.log.debug('Deleting rmq user ({})...'.format(username)) | ||
2068 | 669 | |||
2069 | 670 | # Check that the user exists | ||
2070 | 671 | cmd_user_list = 'rabbitmqctl list_users' | ||
2071 | 672 | output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list) | ||
2072 | 673 | |||
2073 | 674 | if username not in output: | ||
2074 | 675 | self.log.warning('User ({}) does not exist, returning ' | ||
2075 | 676 | 'gracefully.'.format(username)) | ||
2076 | 677 | return | ||
2077 | 678 | |||
2078 | 679 | # Delete the user | ||
2079 | 680 | cmd_user_del = 'rabbitmqctl delete_user {}'.format(username) | ||
2080 | 681 | output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del) | ||
2081 | 682 | |||
2082 | 683 | def get_rmq_cluster_status(self, sentry_unit): | ||
2083 | 684 | """Execute rabbitmq cluster status command on a unit and return | ||
2084 | 685 | the full output. | ||
2085 | 686 | |||
2086 | 687 | :param unit: sentry unit | ||
2087 | 688 | :returns: String containing console output of cluster status command | ||
2088 | 689 | """ | ||
2089 | 690 | cmd = 'rabbitmqctl cluster_status' | ||
2090 | 691 | output, _ = self.run_cmd_unit(sentry_unit, cmd) | ||
2091 | 692 | self.log.debug('{} cluster_status:\n{}'.format( | ||
2092 | 693 | sentry_unit.info['unit_name'], output)) | ||
2093 | 694 | return str(output) | ||
2094 | 695 | |||
2095 | 696 | def get_rmq_cluster_running_nodes(self, sentry_unit): | ||
2096 | 697 | """Parse rabbitmqctl cluster_status output string, return list of | ||
2097 | 698 | running rabbitmq cluster nodes. | ||
2098 | 699 | |||
2099 | 700 | :param unit: sentry unit | ||
2100 | 701 | :returns: List containing node names of running nodes | ||
2101 | 702 | """ | ||
2102 | 703 | # NOTE(beisner): rabbitmqctl cluster_status output is not | ||
2103 | 704 | # json-parsable, do string chop foo, then json.loads that. | ||
2104 | 705 | str_stat = self.get_rmq_cluster_status(sentry_unit) | ||
2105 | 706 | if 'running_nodes' in str_stat: | ||
2106 | 707 | pos_start = str_stat.find("{running_nodes,") + 15 | ||
2107 | 708 | pos_end = str_stat.find("]},", pos_start) + 1 | ||
2108 | 709 | str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"') | ||
2109 | 710 | run_nodes = json.loads(str_run_nodes) | ||
2110 | 711 | return run_nodes | ||
2111 | 712 | else: | ||
2112 | 713 | return [] | ||
2113 | 714 | |||
2114 | 715 | def validate_rmq_cluster_running_nodes(self, sentry_units): | ||
2115 | 716 | """Check that all rmq unit hostnames are represented in the | ||
2116 | 717 | cluster_status output of all units. | ||
2117 | 718 | |||
2118 | 719 | :param host_names: dict of juju unit names to host names | ||
2119 | 720 | :param units: list of sentry unit pointers (all rmq units) | ||
2120 | 721 | :returns: None if successful, otherwise return error message | ||
2121 | 722 | """ | ||
2122 | 723 | host_names = self.get_unit_hostnames(sentry_units) | ||
2123 | 724 | errors = [] | ||
2124 | 725 | |||
2125 | 726 | # Query every unit for cluster_status running nodes | ||
2126 | 727 | for query_unit in sentry_units: | ||
2127 | 728 | query_unit_name = query_unit.info['unit_name'] | ||
2128 | 729 | running_nodes = self.get_rmq_cluster_running_nodes(query_unit) | ||
2129 | 730 | |||
2130 | 731 | # Confirm that every unit is represented in the queried unit's | ||
2131 | 732 | # cluster_status running nodes output. | ||
2132 | 733 | for validate_unit in sentry_units: | ||
2133 | 734 | val_host_name = host_names[validate_unit.info['unit_name']] | ||
2134 | 735 | val_node_name = 'rabbit@{}'.format(val_host_name) | ||
2135 | 736 | |||
2136 | 737 | if val_node_name not in running_nodes: | ||
2137 | 738 | errors.append('Cluster member check failed on {}: {} not ' | ||
2138 | 739 | 'in {}\n'.format(query_unit_name, | ||
2139 | 740 | val_node_name, | ||
2140 | 741 | running_nodes)) | ||
2141 | 742 | if errors: | ||
2142 | 743 | return ''.join(errors) | ||
2143 | 744 | |||
2144 | 745 | def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None): | ||
2145 | 746 | """Check a single juju rmq unit for ssl and port in the config file.""" | ||
2146 | 747 | host = sentry_unit.info['public-address'] | ||
2147 | 748 | unit_name = sentry_unit.info['unit_name'] | ||
2148 | 749 | |||
2149 | 750 | conf_file = '/etc/rabbitmq/rabbitmq.config' | ||
2150 | 751 | conf_contents = str(self.file_contents_safe(sentry_unit, | ||
2151 | 752 | conf_file, max_wait=16)) | ||
2152 | 753 | # Checks | ||
2153 | 754 | conf_ssl = 'ssl' in conf_contents | ||
2154 | 755 | conf_port = str(port) in conf_contents | ||
2155 | 756 | |||
2156 | 757 | # Port explicitly checked in config | ||
2157 | 758 | if port and conf_port and conf_ssl: | ||
2158 | 759 | self.log.debug('SSL is enabled @{}:{} ' | ||
2159 | 760 | '({})'.format(host, port, unit_name)) | ||
2160 | 761 | return True | ||
2161 | 762 | elif port and not conf_port and conf_ssl: | ||
2162 | 763 | self.log.debug('SSL is enabled @{} but not on port {} ' | ||
2163 | 764 | '({})'.format(host, port, unit_name)) | ||
2164 | 765 | return False | ||
2165 | 766 | # Port not checked (useful when checking that ssl is disabled) | ||
2166 | 767 | elif not port and conf_ssl: | ||
2167 | 768 | self.log.debug('SSL is enabled @{}:{} ' | ||
2168 | 769 | '({})'.format(host, port, unit_name)) | ||
2169 | 770 | return True | ||
2170 | 771 | elif not conf_ssl: | ||
2171 | 772 | self.log.debug('SSL not enabled @{}:{} ' | ||
2172 | 773 | '({})'.format(host, port, unit_name)) | ||
2173 | 774 | return False | ||
2174 | 775 | else: | ||
2175 | 776 | msg = ('Unknown condition when checking SSL status @{}:{} ' | ||
2176 | 777 | '({})'.format(host, port, unit_name)) | ||
2177 | 778 | amulet.raise_status(amulet.FAIL, msg) | ||
2178 | 779 | |||
2179 | 780 | def validate_rmq_ssl_enabled_units(self, sentry_units, port=None): | ||
2180 | 781 | """Check that ssl is enabled on rmq juju sentry units. | ||
2181 | 782 | |||
2182 | 783 | :param sentry_units: list of all rmq sentry units | ||
2183 | 784 | :param port: optional ssl port override to validate | ||
2184 | 785 | :returns: None if successful, otherwise return error message | ||
2185 | 786 | """ | ||
2186 | 787 | for sentry_unit in sentry_units: | ||
2187 | 788 | if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port): | ||
2188 | 789 | return ('Unexpected condition: ssl is disabled on unit ' | ||
2189 | 790 | '({})'.format(sentry_unit.info['unit_name'])) | ||
2190 | 791 | return None | ||
2191 | 792 | |||
2192 | 793 | def validate_rmq_ssl_disabled_units(self, sentry_units): | ||
2193 | 794 | """Check that ssl is enabled on listed rmq juju sentry units. | ||
2194 | 795 | |||
2195 | 796 | :param sentry_units: list of all rmq sentry units | ||
2196 | 797 | :returns: True if successful. Raise on error. | ||
2197 | 798 | """ | ||
2198 | 799 | for sentry_unit in sentry_units: | ||
2199 | 800 | if self.rmq_ssl_is_enabled_on_unit(sentry_unit): | ||
2200 | 801 | return ('Unexpected condition: ssl is enabled on unit ' | ||
2201 | 802 | '({})'.format(sentry_unit.info['unit_name'])) | ||
2202 | 803 | return None | ||
2203 | 804 | |||
2204 | 805 | def configure_rmq_ssl_on(self, sentry_units, deployment, | ||
2205 | 806 | port=None, max_wait=60): | ||
2206 | 807 | """Turn ssl charm config option on, with optional non-default | ||
2207 | 808 | ssl port specification. Confirm that it is enabled on every | ||
2208 | 809 | unit. | ||
2209 | 810 | |||
2210 | 811 | :param sentry_units: list of sentry units | ||
2211 | 812 | :param deployment: amulet deployment object pointer | ||
2212 | 813 | :param port: amqp port, use defaults if None | ||
2213 | 814 | :param max_wait: maximum time to wait in seconds to confirm | ||
2214 | 815 | :returns: None if successful. Raise on error. | ||
2215 | 816 | """ | ||
2216 | 817 | self.log.debug('Setting ssl charm config option: on') | ||
2217 | 818 | |||
2218 | 819 | # Enable RMQ SSL | ||
2219 | 820 | config = {'ssl': 'on'} | ||
2220 | 821 | if port: | ||
2221 | 822 | config['ssl_port'] = port | ||
2222 | 823 | |||
2223 | 824 | deployment.d.configure('rabbitmq-server', config) | ||
2224 | 825 | |||
2225 | 826 | # Wait for unit status | ||
2226 | 827 | self.rmq_wait_for_cluster(deployment) | ||
2227 | 828 | |||
2228 | 829 | # Confirm | ||
2229 | 830 | tries = 0 | ||
2230 | 831 | ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port) | ||
2231 | 832 | while ret and tries < (max_wait / 4): | ||
2232 | 833 | time.sleep(4) | ||
2233 | 834 | self.log.debug('Attempt {}: {}'.format(tries, ret)) | ||
2234 | 835 | ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port) | ||
2235 | 836 | tries += 1 | ||
2236 | 837 | |||
2237 | 838 | if ret: | ||
2238 | 839 | amulet.raise_status(amulet.FAIL, ret) | ||
2239 | 840 | |||
2240 | 841 | def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60): | ||
2241 | 842 | """Turn ssl charm config option off, confirm that it is disabled | ||
2242 | 843 | on every unit. | ||
2243 | 844 | |||
2244 | 845 | :param sentry_units: list of sentry units | ||
2245 | 846 | :param deployment: amulet deployment object pointer | ||
2246 | 847 | :param max_wait: maximum time to wait in seconds to confirm | ||
2247 | 848 | :returns: None if successful. Raise on error. | ||
2248 | 849 | """ | ||
2249 | 850 | self.log.debug('Setting ssl charm config option: off') | ||
2250 | 851 | |||
2251 | 852 | # Disable RMQ SSL | ||
2252 | 853 | config = {'ssl': 'off'} | ||
2253 | 854 | deployment.d.configure('rabbitmq-server', config) | ||
2254 | 855 | |||
2255 | 856 | # Wait for unit status | ||
2256 | 857 | self.rmq_wait_for_cluster(deployment) | ||
2257 | 858 | |||
2258 | 859 | # Confirm | ||
2259 | 860 | tries = 0 | ||
2260 | 861 | ret = self.validate_rmq_ssl_disabled_units(sentry_units) | ||
2261 | 862 | while ret and tries < (max_wait / 4): | ||
2262 | 863 | time.sleep(4) | ||
2263 | 864 | self.log.debug('Attempt {}: {}'.format(tries, ret)) | ||
2264 | 865 | ret = self.validate_rmq_ssl_disabled_units(sentry_units) | ||
2265 | 866 | tries += 1 | ||
2266 | 867 | |||
2267 | 868 | if ret: | ||
2268 | 869 | amulet.raise_status(amulet.FAIL, ret) | ||
2269 | 870 | |||
2270 | 871 | def connect_amqp_by_unit(self, sentry_unit, ssl=False, | ||
2271 | 872 | port=None, fatal=True, | ||
2272 | 873 | username="testuser1", password="changeme"): | ||
2273 | 874 | """Establish and return a pika amqp connection to the rabbitmq service | ||
2274 | 875 | running on a rmq juju unit. | ||
2275 | 876 | |||
2276 | 877 | :param sentry_unit: sentry unit pointer | ||
2277 | 878 | :param ssl: boolean, default to False | ||
2278 | 879 | :param port: amqp port, use defaults if None | ||
2279 | 880 | :param fatal: boolean, default to True (raises on connect error) | ||
2280 | 881 | :param username: amqp user name, default to testuser1 | ||
2281 | 882 | :param password: amqp user password | ||
2282 | 883 | :returns: pika amqp connection pointer or None if failed and non-fatal | ||
2283 | 884 | """ | ||
2284 | 885 | host = sentry_unit.info['public-address'] | ||
2285 | 886 | unit_name = sentry_unit.info['unit_name'] | ||
2286 | 887 | |||
2287 | 888 | # Default port logic if port is not specified | ||
2288 | 889 | if ssl and not port: | ||
2289 | 890 | port = 5671 | ||
2290 | 891 | elif not ssl and not port: | ||
2291 | 892 | port = 5672 | ||
2292 | 893 | |||
2293 | 894 | self.log.debug('Connecting to amqp on {}:{} ({}) as ' | ||
2294 | 895 | '{}...'.format(host, port, unit_name, username)) | ||
2295 | 896 | |||
2296 | 897 | try: | ||
2297 | 898 | credentials = pika.PlainCredentials(username, password) | ||
2298 | 899 | parameters = pika.ConnectionParameters(host=host, port=port, | ||
2299 | 900 | credentials=credentials, | ||
2300 | 901 | ssl=ssl, | ||
2301 | 902 | connection_attempts=3, | ||
2302 | 903 | retry_delay=5, | ||
2303 | 904 | socket_timeout=1) | ||
2304 | 905 | connection = pika.BlockingConnection(parameters) | ||
2305 | 906 | assert connection.server_properties['product'] == 'RabbitMQ' | ||
2306 | 907 | self.log.debug('Connect OK') | ||
2307 | 908 | return connection | ||
2308 | 909 | except Exception as e: | ||
2309 | 910 | msg = ('amqp connection failed to {}:{} as ' | ||
2310 | 911 | '{} ({})'.format(host, port, username, str(e))) | ||
2311 | 912 | if fatal: | ||
2312 | 913 | amulet.raise_status(amulet.FAIL, msg) | ||
2313 | 914 | else: | ||
2314 | 915 | self.log.warn(msg) | ||
2315 | 916 | return None | ||
2316 | 917 | |||
2317 | 918 | def publish_amqp_message_by_unit(self, sentry_unit, message, | ||
2318 | 919 | queue="test", ssl=False, | ||
2319 | 920 | username="testuser1", | ||
2320 | 921 | password="changeme", | ||
2321 | 922 | port=None): | ||
2322 | 923 | """Publish an amqp message to a rmq juju unit. | ||
2323 | 924 | |||
2324 | 925 | :param sentry_unit: sentry unit pointer | ||
2325 | 926 | :param message: amqp message string | ||
2326 | 927 | :param queue: message queue, default to test | ||
2327 | 928 | :param username: amqp user name, default to testuser1 | ||
2328 | 929 | :param password: amqp user password | ||
2329 | 930 | :param ssl: boolean, default to False | ||
2330 | 931 | :param port: amqp port, use defaults if None | ||
2331 | 932 | :returns: None. Raises exception if publish failed. | ||
2332 | 933 | """ | ||
2333 | 934 | self.log.debug('Publishing message to {} queue:\n{}'.format(queue, | ||
2334 | 935 | message)) | ||
2335 | 936 | connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl, | ||
2336 | 937 | port=port, | ||
2337 | 938 | username=username, | ||
2338 | 939 | password=password) | ||
2339 | 940 | |||
2340 | 941 | # NOTE(beisner): extra debug here re: pika hang potential: | ||
2341 | 942 | # https://github.com/pika/pika/issues/297 | ||
2342 | 943 | # https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw | ||
2343 | 944 | self.log.debug('Defining channel...') | ||
2344 | 945 | channel = connection.channel() | ||
2345 | 946 | self.log.debug('Declaring queue...') | ||
2346 | 947 | channel.queue_declare(queue=queue, auto_delete=False, durable=True) | ||
2347 | 948 | self.log.debug('Publishing message...') | ||
2348 | 949 | channel.basic_publish(exchange='', routing_key=queue, body=message) | ||
2349 | 950 | self.log.debug('Closing channel...') | ||
2350 | 951 | channel.close() | ||
2351 | 952 | self.log.debug('Closing connection...') | ||
2352 | 953 | connection.close() | ||
2353 | 954 | |||
2354 | 955 | def get_amqp_message_by_unit(self, sentry_unit, queue="test", | ||
2355 | 956 | username="testuser1", | ||
2356 | 957 | password="changeme", | ||
2357 | 958 | ssl=False, port=None): | ||
2358 | 959 | """Get an amqp message from a rmq juju unit. | ||
2359 | 960 | |||
2360 | 961 | :param sentry_unit: sentry unit pointer | ||
2361 | 962 | :param queue: message queue, default to test | ||
2362 | 963 | :param username: amqp user name, default to testuser1 | ||
2363 | 964 | :param password: amqp user password | ||
2364 | 965 | :param ssl: boolean, default to False | ||
2365 | 966 | :param port: amqp port, use defaults if None | ||
2366 | 967 | :returns: amqp message body as string. Raise if get fails. | ||
2367 | 968 | """ | ||
2368 | 969 | connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl, | ||
2369 | 970 | port=port, | ||
2370 | 971 | username=username, | ||
2371 | 972 | password=password) | ||
2372 | 973 | channel = connection.channel() | ||
2373 | 974 | method_frame, _, body = channel.basic_get(queue) | ||
2374 | 975 | |||
2375 | 976 | if method_frame: | ||
2376 | 977 | self.log.debug('Retreived message from {} queue:\n{}'.format(queue, | ||
2377 | 978 | body)) | ||
2378 | 979 | channel.basic_ack(method_frame.delivery_tag) | ||
2379 | 980 | channel.close() | ||
2380 | 981 | connection.close() | ||
2381 | 982 | return body | ||
2382 | 983 | else: | ||
2383 | 984 | msg = 'No message retrieved.' | ||
2384 | 985 | amulet.raise_status(amulet.FAIL, msg) |
charm_unit_test #16782 ceph-osd-next for 1chb1n mp283704
UNIT OK: passed
Build: http:// 10.245. 162.77: 8080/job/ charm_unit_ test/16782/