Merge lp:~james-page/charms/trusty/heat/lp1531102-trunk into lp:~openstack-charmers-archive/charms/trusty/heat/trunk

Proposed by James Page on 2016-01-07
Status: Merged
Merged at revision: 48
Proposed branch: lp:~james-page/charms/trusty/heat/lp1531102-trunk
Merge into: lp:~openstack-charmers-archive/charms/trusty/heat/trunk
Diff against target: 1024 lines (+506/-61)
13 files modified
hooks/charmhelpers/contrib/openstack/amulet/deployment.py (+100/-1)
hooks/charmhelpers/contrib/openstack/amulet/utils.py (+26/-4)
hooks/charmhelpers/contrib/openstack/context.py (+40/-13)
hooks/charmhelpers/contrib/openstack/neutron.py (+17/-3)
hooks/charmhelpers/contrib/openstack/templates/ceph.conf (+6/-0)
hooks/charmhelpers/contrib/openstack/utils.py (+30/-18)
hooks/charmhelpers/core/hookenv.py (+32/-0)
hooks/charmhelpers/core/host.py (+12/-1)
hooks/charmhelpers/core/hugepage.py (+2/-0)
hooks/charmhelpers/core/kernel.py (+68/-0)
tests/charmhelpers/contrib/amulet/utils.py (+47/-16)
tests/charmhelpers/contrib/openstack/amulet/deployment.py (+100/-1)
tests/charmhelpers/contrib/openstack/amulet/utils.py (+26/-4)
To merge this branch: bzr merge lp:~james-page/charms/trusty/heat/lp1531102-trunk
Reviewer Review Type Date Requested Status
Liam Young 2016-01-07 Approve on 2016-01-12
Review via email: mp+281869@code.launchpad.net

Commit message

Resync helpers

Description of the change

Resync stable helpers to resolve issues with liberty point release detection.

To post a comment you must log in.

charm_unit_test #15627 heat for james-page mp281869
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/15627/

charm_lint_check #16745 heat for james-page mp281869
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/16745/

charm_amulet_test #8565 heat for james-page mp281869
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/8565/

Liam Young (gnuoy) wrote :

Approved

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
2--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2015-10-22 13:20:40 +0000
3+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2016-01-07 14:27:41 +0000
4@@ -14,12 +14,18 @@
5 # You should have received a copy of the GNU Lesser General Public License
6 # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
7
8+import logging
9+import re
10+import sys
11 import six
12 from collections import OrderedDict
13 from charmhelpers.contrib.amulet.deployment import (
14 AmuletDeployment
15 )
16
17+DEBUG = logging.DEBUG
18+ERROR = logging.ERROR
19+
20
21 class OpenStackAmuletDeployment(AmuletDeployment):
22 """OpenStack amulet deployment.
23@@ -28,9 +34,12 @@
24 that is specifically for use by OpenStack charms.
25 """
26
27- def __init__(self, series=None, openstack=None, source=None, stable=True):
28+ def __init__(self, series=None, openstack=None, source=None,
29+ stable=True, log_level=DEBUG):
30 """Initialize the deployment environment."""
31 super(OpenStackAmuletDeployment, self).__init__(series)
32+ self.log = self.get_logger(level=log_level)
33+ self.log.info('OpenStackAmuletDeployment: init')
34 self.openstack = openstack
35 self.source = source
36 self.stable = stable
37@@ -38,6 +47,22 @@
38 # out.
39 self.current_next = "trusty"
40
41+ def get_logger(self, name="deployment-logger", level=logging.DEBUG):
42+ """Get a logger object that will log to stdout."""
43+ log = logging
44+ logger = log.getLogger(name)
45+ fmt = log.Formatter("%(asctime)s %(funcName)s "
46+ "%(levelname)s: %(message)s")
47+
48+ handler = log.StreamHandler(stream=sys.stdout)
49+ handler.setLevel(level)
50+ handler.setFormatter(fmt)
51+
52+ logger.addHandler(handler)
53+ logger.setLevel(level)
54+
55+ return logger
56+
57 def _determine_branch_locations(self, other_services):
58 """Determine the branch locations for the other services.
59
60@@ -45,6 +70,8 @@
61 stable or next (dev) branch, and based on this, use the corresonding
62 stable or next branches for the other_services."""
63
64+ self.log.info('OpenStackAmuletDeployment: determine branch locations')
65+
66 # Charms outside the lp:~openstack-charmers namespace
67 base_charms = ['mysql', 'mongodb', 'nrpe']
68
69@@ -82,6 +109,8 @@
70
71 def _add_services(self, this_service, other_services):
72 """Add services to the deployment and set openstack-origin/source."""
73+ self.log.info('OpenStackAmuletDeployment: adding services')
74+
75 other_services = self._determine_branch_locations(other_services)
76
77 super(OpenStackAmuletDeployment, self)._add_services(this_service,
78@@ -111,9 +140,79 @@
79
80 def _configure_services(self, configs):
81 """Configure all of the services."""
82+ self.log.info('OpenStackAmuletDeployment: configure services')
83 for service, config in six.iteritems(configs):
84 self.d.configure(service, config)
85
86+ def _auto_wait_for_status(self, message=None, exclude_services=None,
87+ include_only=None, timeout=1800):
88+ """Wait for all units to have a specific extended status, except
89+ for any defined as excluded. Unless specified via message, any
90+ status containing any case of 'ready' will be considered a match.
91+
92+ Examples of message usage:
93+
94+ Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
95+ message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
96+
97+ Wait for all units to reach this status (exact match):
98+ message = re.compile('^Unit is ready and clustered$')
99+
100+ Wait for all units to reach any one of these (exact match):
101+ message = re.compile('Unit is ready|OK|Ready')
102+
103+ Wait for at least one unit to reach this status (exact match):
104+ message = {'ready'}
105+
106+ See Amulet's sentry.wait_for_messages() for message usage detail.
107+ https://github.com/juju/amulet/blob/master/amulet/sentry.py
108+
109+ :param message: Expected status match
110+ :param exclude_services: List of juju service names to ignore,
111+ not to be used in conjuction with include_only.
112+ :param include_only: List of juju service names to exclusively check,
113+ not to be used in conjuction with exclude_services.
114+ :param timeout: Maximum time in seconds to wait for status match
115+ :returns: None. Raises if timeout is hit.
116+ """
117+ self.log.info('Waiting for extended status on units...')
118+
119+ all_services = self.d.services.keys()
120+
121+ if exclude_services and include_only:
122+ raise ValueError('exclude_services can not be used '
123+ 'with include_only')
124+
125+ if message:
126+ if isinstance(message, re._pattern_type):
127+ match = message.pattern
128+ else:
129+ match = message
130+
131+ self.log.debug('Custom extended status wait match: '
132+ '{}'.format(match))
133+ else:
134+ self.log.debug('Default extended status wait match: contains '
135+ 'READY (case-insensitive)')
136+ message = re.compile('.*ready.*', re.IGNORECASE)
137+
138+ if exclude_services:
139+ self.log.debug('Excluding services from extended status match: '
140+ '{}'.format(exclude_services))
141+ else:
142+ exclude_services = []
143+
144+ if include_only:
145+ services = include_only
146+ else:
147+ services = list(set(all_services) - set(exclude_services))
148+
149+ self.log.debug('Waiting up to {}s for extended status on services: '
150+ '{}'.format(timeout, services))
151+ service_messages = {service: message for service in services}
152+ self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
153+ self.log.info('OK')
154+
155 def _get_openstack_release(self):
156 """Get openstack release.
157
158
159=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
160--- hooks/charmhelpers/contrib/openstack/amulet/utils.py 2015-10-22 13:20:40 +0000
161+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py 2016-01-07 14:27:41 +0000
162@@ -18,6 +18,7 @@
163 import json
164 import logging
165 import os
166+import re
167 import six
168 import time
169 import urllib
170@@ -604,7 +605,22 @@
171 '{}'.format(sample_type, samples))
172 return None
173
174-# rabbitmq/amqp specific helpers:
175+ # rabbitmq/amqp specific helpers:
176+
177+ def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
178+ """Wait for rmq units extended status to show cluster readiness,
179+ after an optional initial sleep period. Initial sleep is likely
180+ necessary to be effective following a config change, as status
181+ message may not instantly update to non-ready."""
182+
183+ if init_sleep:
184+ time.sleep(init_sleep)
185+
186+ message = re.compile('^Unit is ready and clustered$')
187+ deployment._auto_wait_for_status(message=message,
188+ timeout=timeout,
189+ include_only=['rabbitmq-server'])
190+
191 def add_rmq_test_user(self, sentry_units,
192 username="testuser1", password="changeme"):
193 """Add a test user via the first rmq juju unit, check connection as
194@@ -752,7 +768,7 @@
195 self.log.debug('SSL is enabled @{}:{} '
196 '({})'.format(host, port, unit_name))
197 return True
198- elif not port and not conf_ssl:
199+ elif not conf_ssl:
200 self.log.debug('SSL not enabled @{}:{} '
201 '({})'.format(host, port, unit_name))
202 return False
203@@ -805,7 +821,10 @@
204 if port:
205 config['ssl_port'] = port
206
207- deployment.configure('rabbitmq-server', config)
208+ deployment.d.configure('rabbitmq-server', config)
209+
210+ # Wait for unit status
211+ self.rmq_wait_for_cluster(deployment)
212
213 # Confirm
214 tries = 0
215@@ -832,7 +851,10 @@
216
217 # Disable RMQ SSL
218 config = {'ssl': 'off'}
219- deployment.configure('rabbitmq-server', config)
220+ deployment.d.configure('rabbitmq-server', config)
221+
222+ # Wait for unit status
223+ self.rmq_wait_for_cluster(deployment)
224
225 # Confirm
226 tries = 0
227
228=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
229--- hooks/charmhelpers/contrib/openstack/context.py 2015-10-22 13:20:40 +0000
230+++ hooks/charmhelpers/contrib/openstack/context.py 2016-01-07 14:27:41 +0000
231@@ -14,6 +14,7 @@
232 # You should have received a copy of the GNU Lesser General Public License
233 # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
234
235+import glob
236 import json
237 import os
238 import re
239@@ -951,6 +952,19 @@
240 'config': config}
241 return ovs_ctxt
242
243+ def midonet_ctxt(self):
244+ driver = neutron_plugin_attribute(self.plugin, 'driver',
245+ self.network_manager)
246+ midonet_config = neutron_plugin_attribute(self.plugin, 'config',
247+ self.network_manager)
248+ mido_ctxt = {'core_plugin': driver,
249+ 'neutron_plugin': 'midonet',
250+ 'neutron_security_groups': self.neutron_security_groups,
251+ 'local_ip': unit_private_ip(),
252+ 'config': midonet_config}
253+
254+ return mido_ctxt
255+
256 def __call__(self):
257 if self.network_manager not in ['quantum', 'neutron']:
258 return {}
259@@ -972,6 +986,8 @@
260 ctxt.update(self.nuage_ctxt())
261 elif self.plugin == 'plumgrid':
262 ctxt.update(self.pg_ctxt())
263+ elif self.plugin == 'midonet':
264+ ctxt.update(self.midonet_ctxt())
265
266 alchemy_flags = config('neutron-alchemy-flags')
267 if alchemy_flags:
268@@ -1104,7 +1120,7 @@
269
270 ctxt = {
271 ... other context ...
272- 'subordinate_config': {
273+ 'subordinate_configuration': {
274 'DEFAULT': {
275 'key1': 'value1',
276 },
277@@ -1145,22 +1161,23 @@
278 try:
279 sub_config = json.loads(sub_config)
280 except:
281- log('Could not parse JSON from subordinate_config '
282- 'setting from %s' % rid, level=ERROR)
283+ log('Could not parse JSON from '
284+ 'subordinate_configuration setting from %s'
285+ % rid, level=ERROR)
286 continue
287
288 for service in self.services:
289 if service not in sub_config:
290- log('Found subordinate_config on %s but it contained'
291- 'nothing for %s service' % (rid, service),
292- level=INFO)
293+ log('Found subordinate_configuration on %s but it '
294+ 'contained nothing for %s service'
295+ % (rid, service), level=INFO)
296 continue
297
298 sub_config = sub_config[service]
299 if self.config_file not in sub_config:
300- log('Found subordinate_config on %s but it contained'
301- 'nothing for %s' % (rid, self.config_file),
302- level=INFO)
303+ log('Found subordinate_configuration on %s but it '
304+ 'contained nothing for %s'
305+ % (rid, self.config_file), level=INFO)
306 continue
307
308 sub_config = sub_config[self.config_file]
309@@ -1363,7 +1380,7 @@
310 normalized.update({port: port for port in resolved
311 if port in ports})
312 if resolved:
313- return {bridge: normalized[port] for port, bridge in
314+ return {normalized[port]: bridge for port, bridge in
315 six.iteritems(portmap) if port in normalized.keys()}
316
317 return None
318@@ -1374,12 +1391,22 @@
319 def __call__(self):
320 ctxt = {}
321 mappings = super(PhyNICMTUContext, self).__call__()
322- if mappings and mappings.values():
323- ports = mappings.values()
324+ if mappings and mappings.keys():
325+ ports = sorted(mappings.keys())
326 napi_settings = NeutronAPIContext()()
327 mtu = napi_settings.get('network_device_mtu')
328+ all_ports = set()
329+ # If any of ports is a vlan device, its underlying device must have
330+ # mtu applied first.
331+ for port in ports:
332+ for lport in glob.glob("/sys/class/net/%s/lower_*" % port):
333+ lport = os.path.basename(lport)
334+ all_ports.add(lport.split('_')[1])
335+
336+ all_ports = list(all_ports)
337+ all_ports.extend(ports)
338 if mtu:
339- ctxt["devs"] = '\\n'.join(ports)
340+ ctxt["devs"] = '\\n'.join(all_ports)
341 ctxt['mtu'] = mtu
342
343 return ctxt
344
345=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
346--- hooks/charmhelpers/contrib/openstack/neutron.py 2015-10-22 13:20:40 +0000
347+++ hooks/charmhelpers/contrib/openstack/neutron.py 2016-01-07 14:27:41 +0000
348@@ -209,6 +209,20 @@
349 'server_packages': ['neutron-server',
350 'neutron-plugin-plumgrid'],
351 'server_services': ['neutron-server']
352+ },
353+ 'midonet': {
354+ 'config': '/etc/neutron/plugins/midonet/midonet.ini',
355+ 'driver': 'midonet.neutron.plugin.MidonetPluginV2',
356+ 'contexts': [
357+ context.SharedDBContext(user=config('neutron-database-user'),
358+ database=config('neutron-database'),
359+ relation_prefix='neutron',
360+ ssl_dir=NEUTRON_CONF_DIR)],
361+ 'services': [],
362+ 'packages': [[headers_package()] + determine_dkms_package()],
363+ 'server_packages': ['neutron-server',
364+ 'python-neutron-plugin-midonet'],
365+ 'server_services': ['neutron-server']
366 }
367 }
368 if release >= 'icehouse':
369@@ -310,10 +324,10 @@
370 def parse_data_port_mappings(mappings, default_bridge='br-data'):
371 """Parse data port mappings.
372
373- Mappings must be a space-delimited list of port:bridge mappings.
374+ Mappings must be a space-delimited list of bridge:port.
375
376- Returns dict of the form {port:bridge} where port may be an mac address or
377- interface name.
378+ Returns dict of the form {port:bridge} where ports may be mac addresses or
379+ interface names.
380 """
381
382 # NOTE(dosaboy): we use rvalue for key to allow multiple values to be
383
384=== modified file 'hooks/charmhelpers/contrib/openstack/templates/ceph.conf'
385--- hooks/charmhelpers/contrib/openstack/templates/ceph.conf 2015-08-10 16:35:15 +0000
386+++ hooks/charmhelpers/contrib/openstack/templates/ceph.conf 2016-01-07 14:27:41 +0000
387@@ -13,3 +13,9 @@
388 err to syslog = {{ use_syslog }}
389 clog to syslog = {{ use_syslog }}
390
391+[client]
392+{% if rbd_client_cache_settings -%}
393+{% for key, value in rbd_client_cache_settings.iteritems() -%}
394+{{ key }} = {{ value }}
395+{% endfor -%}
396+{%- endif %}
397\ No newline at end of file
398
399=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
400--- hooks/charmhelpers/contrib/openstack/utils.py 2015-10-22 13:20:40 +0000
401+++ hooks/charmhelpers/contrib/openstack/utils.py 2016-01-07 14:27:41 +0000
402@@ -54,7 +54,8 @@
403 )
404
405 from charmhelpers.contrib.network.ip import (
406- get_ipv6_addr
407+ get_ipv6_addr,
408+ is_ipv6,
409 )
410
411 from charmhelpers.contrib.python.packages import (
412@@ -120,36 +121,37 @@
413 ('2.2.2', 'kilo'),
414 ('2.3.0', 'liberty'),
415 ('2.4.0', 'liberty'),
416+ ('2.5.0', 'liberty'),
417 ])
418
419 # >= Liberty version->codename mapping
420 PACKAGE_CODENAMES = {
421 'nova-common': OrderedDict([
422- ('12.0.0', 'liberty'),
423+ ('12.0', 'liberty'),
424 ]),
425 'neutron-common': OrderedDict([
426- ('7.0.0', 'liberty'),
427+ ('7.0', 'liberty'),
428 ]),
429 'cinder-common': OrderedDict([
430- ('7.0.0', 'liberty'),
431+ ('7.0', 'liberty'),
432 ]),
433 'keystone': OrderedDict([
434- ('8.0.0', 'liberty'),
435+ ('8.0', 'liberty'),
436 ]),
437 'horizon-common': OrderedDict([
438- ('8.0.0', 'liberty'),
439+ ('8.0', 'liberty'),
440 ]),
441 'ceilometer-common': OrderedDict([
442- ('5.0.0', 'liberty'),
443+ ('5.0', 'liberty'),
444 ]),
445 'heat-common': OrderedDict([
446- ('5.0.0', 'liberty'),
447+ ('5.0', 'liberty'),
448 ]),
449 'glance-common': OrderedDict([
450- ('11.0.0', 'liberty'),
451+ ('11.0', 'liberty'),
452 ]),
453 'openstack-dashboard': OrderedDict([
454- ('8.0.0', 'liberty'),
455+ ('8.0', 'liberty'),
456 ]),
457 }
458
459@@ -236,7 +238,14 @@
460 error_out(e)
461
462 vers = apt.upstream_version(pkg.current_ver.ver_str)
463- match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
464+ if 'swift' in pkg.name:
465+ # Fully x.y.z match for swift versions
466+ match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
467+ else:
468+ # x.y match only for 20XX.X
469+ # and ignore patch level for other packages
470+ match = re.match('^(\d+)\.(\d+)', vers)
471+
472 if match:
473 vers = match.group(0)
474
475@@ -248,13 +257,8 @@
476 # < Liberty co-ordinated project versions
477 try:
478 if 'swift' in pkg.name:
479- swift_vers = vers[:5]
480- if swift_vers not in SWIFT_CODENAMES:
481- # Deal with 1.10.0 upward
482- swift_vers = vers[:6]
483- return SWIFT_CODENAMES[swift_vers]
484+ return SWIFT_CODENAMES[vers]
485 else:
486- vers = vers[:6]
487 return OPENSTACK_CODENAMES[vers]
488 except KeyError:
489 if not fatal:
490@@ -519,6 +523,12 @@
491 relation_prefix=None):
492 hosts = get_ipv6_addr(dynamic_only=False)
493
494+ if config('vip'):
495+ vips = config('vip').split()
496+ for vip in vips:
497+ if vip and is_ipv6(vip):
498+ hosts.append(vip)
499+
500 kwargs = {'database': database,
501 'username': database_user,
502 'hostname': json.dumps(hosts)}
503@@ -851,7 +861,9 @@
504 if charm_state != 'active' and charm_state != 'unknown':
505 state = workload_state_compare(state, charm_state)
506 if message:
507- message = "{} {}".format(message, charm_message)
508+ charm_message = charm_message.replace("Incomplete relations: ",
509+ "")
510+ message = "{}, {}".format(message, charm_message)
511 else:
512 message = charm_message
513
514
515=== modified file 'hooks/charmhelpers/core/hookenv.py'
516--- hooks/charmhelpers/core/hookenv.py 2015-10-22 13:20:40 +0000
517+++ hooks/charmhelpers/core/hookenv.py 2016-01-07 14:27:41 +0000
518@@ -623,6 +623,38 @@
519 return unit_get('private-address')
520
521
522+@cached
523+def storage_get(attribute="", storage_id=""):
524+ """Get storage attributes"""
525+ _args = ['storage-get', '--format=json']
526+ if storage_id:
527+ _args.extend(('-s', storage_id))
528+ if attribute:
529+ _args.append(attribute)
530+ try:
531+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
532+ except ValueError:
533+ return None
534+
535+
536+@cached
537+def storage_list(storage_name=""):
538+ """List the storage IDs for the unit"""
539+ _args = ['storage-list', '--format=json']
540+ if storage_name:
541+ _args.append(storage_name)
542+ try:
543+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
544+ except ValueError:
545+ return None
546+ except OSError as e:
547+ import errno
548+ if e.errno == errno.ENOENT:
549+ # storage-list does not exist
550+ return []
551+ raise
552+
553+
554 class UnregisteredHookError(Exception):
555 """Raised when an undefined hook is called"""
556 pass
557
558=== modified file 'hooks/charmhelpers/core/host.py'
559--- hooks/charmhelpers/core/host.py 2015-10-22 13:20:40 +0000
560+++ hooks/charmhelpers/core/host.py 2016-01-07 14:27:41 +0000
561@@ -566,7 +566,14 @@
562 os.chdir(cur)
563
564
565-def chownr(path, owner, group, follow_links=True):
566+def chownr(path, owner, group, follow_links=True, chowntopdir=False):
567+ """
568+ Recursively change user and group ownership of files and directories
569+ in given path. Doesn't chown path itself by default, only its children.
570+
571+ :param bool follow_links: Also Chown links if True
572+ :param bool chowntopdir: Also chown path itself if True
573+ """
574 uid = pwd.getpwnam(owner).pw_uid
575 gid = grp.getgrnam(group).gr_gid
576 if follow_links:
577@@ -574,6 +581,10 @@
578 else:
579 chown = os.lchown
580
581+ if chowntopdir:
582+ broken_symlink = os.path.lexists(path) and not os.path.exists(path)
583+ if not broken_symlink:
584+ chown(path, uid, gid)
585 for root, dirs, files in os.walk(path):
586 for name in dirs + files:
587 full = os.path.join(root, name)
588
589=== modified file 'hooks/charmhelpers/core/hugepage.py'
590--- hooks/charmhelpers/core/hugepage.py 2015-10-22 13:20:40 +0000
591+++ hooks/charmhelpers/core/hugepage.py 2016-01-07 14:27:41 +0000
592@@ -46,6 +46,8 @@
593 group_info = add_group(group)
594 gid = group_info.gr_gid
595 add_user_to_group(user, group)
596+ if max_map_count < 2 * nr_hugepages:
597+ max_map_count = 2 * nr_hugepages
598 sysctl_settings = {
599 'vm.nr_hugepages': nr_hugepages,
600 'vm.max_map_count': max_map_count,
601
602=== added file 'hooks/charmhelpers/core/kernel.py'
603--- hooks/charmhelpers/core/kernel.py 1970-01-01 00:00:00 +0000
604+++ hooks/charmhelpers/core/kernel.py 2016-01-07 14:27:41 +0000
605@@ -0,0 +1,68 @@
606+#!/usr/bin/env python
607+# -*- coding: utf-8 -*-
608+
609+# Copyright 2014-2015 Canonical Limited.
610+#
611+# This file is part of charm-helpers.
612+#
613+# charm-helpers is free software: you can redistribute it and/or modify
614+# it under the terms of the GNU Lesser General Public License version 3 as
615+# published by the Free Software Foundation.
616+#
617+# charm-helpers is distributed in the hope that it will be useful,
618+# but WITHOUT ANY WARRANTY; without even the implied warranty of
619+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
620+# GNU Lesser General Public License for more details.
621+#
622+# You should have received a copy of the GNU Lesser General Public License
623+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
624+
625+__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
626+
627+from charmhelpers.core.hookenv import (
628+ log,
629+ INFO
630+)
631+
632+from subprocess import check_call, check_output
633+import re
634+
635+
636+def modprobe(module, persist=True):
637+ """Load a kernel module and configure for auto-load on reboot."""
638+ cmd = ['modprobe', module]
639+
640+ log('Loading kernel module %s' % module, level=INFO)
641+
642+ check_call(cmd)
643+ if persist:
644+ with open('/etc/modules', 'r+') as modules:
645+ if module not in modules.read():
646+ modules.write(module)
647+
648+
649+def rmmod(module, force=False):
650+ """Remove a module from the linux kernel"""
651+ cmd = ['rmmod']
652+ if force:
653+ cmd.append('-f')
654+ cmd.append(module)
655+ log('Removing kernel module %s' % module, level=INFO)
656+ return check_call(cmd)
657+
658+
659+def lsmod():
660+ """Shows what kernel modules are currently loaded"""
661+ return check_output(['lsmod'],
662+ universal_newlines=True)
663+
664+
665+def is_module_loaded(module):
666+ """Checks if a kernel module is already loaded"""
667+ matches = re.findall('^%s[ ]+' % module, lsmod(), re.M)
668+ return len(matches) > 0
669+
670+
671+def update_initramfs(version='all'):
672+ """Updates an initramfs image"""
673+ return check_call(["update-initramfs", "-k", version, "-u"])
674
675=== modified file 'tests/charmhelpers/contrib/amulet/utils.py'
676--- tests/charmhelpers/contrib/amulet/utils.py 2015-10-22 13:20:40 +0000
677+++ tests/charmhelpers/contrib/amulet/utils.py 2016-01-07 14:27:41 +0000
678@@ -326,7 +326,7 @@
679
680 def service_restarted_since(self, sentry_unit, mtime, service,
681 pgrep_full=None, sleep_time=20,
682- retry_count=2, retry_sleep_time=30):
683+ retry_count=30, retry_sleep_time=10):
684 """Check if service was been started after a given time.
685
686 Args:
687@@ -334,8 +334,9 @@
688 mtime (float): The epoch time to check against
689 service (string): service name to look for in process table
690 pgrep_full: [Deprecated] Use full command line search mode with pgrep
691- sleep_time (int): Seconds to sleep before looking for process
692- retry_count (int): If service is not found, how many times to retry
693+ sleep_time (int): Initial sleep time (s) before looking for file
694+ retry_sleep_time (int): Time (s) to sleep between retries
695+ retry_count (int): If file is not found, how many times to retry
696
697 Returns:
698 bool: True if service found and its start time it newer than mtime,
699@@ -359,11 +360,12 @@
700 pgrep_full)
701 self.log.debug('Attempt {} to get {} proc start time on {} '
702 'OK'.format(tries, service, unit_name))
703- except IOError:
704+ except IOError as e:
705 # NOTE(beisner) - race avoidance, proc may not exist yet.
706 # https://bugs.launchpad.net/charm-helpers/+bug/1474030
707 self.log.debug('Attempt {} to get {} proc start time on {} '
708- 'failed'.format(tries, service, unit_name))
709+ 'failed\n{}'.format(tries, service,
710+ unit_name, e))
711 time.sleep(retry_sleep_time)
712 tries += 1
713
714@@ -383,35 +385,62 @@
715 return False
716
717 def config_updated_since(self, sentry_unit, filename, mtime,
718- sleep_time=20):
719+ sleep_time=20, retry_count=30,
720+ retry_sleep_time=10):
721 """Check if file was modified after a given time.
722
723 Args:
724 sentry_unit (sentry): The sentry unit to check the file mtime on
725 filename (string): The file to check mtime of
726 mtime (float): The epoch time to check against
727- sleep_time (int): Seconds to sleep before looking for process
728+ sleep_time (int): Initial sleep time (s) before looking for file
729+ retry_sleep_time (int): Time (s) to sleep between retries
730+ retry_count (int): If file is not found, how many times to retry
731
732 Returns:
733 bool: True if file was modified more recently than mtime, False if
734- file was modified before mtime,
735+ file was modified before mtime, or if file not found.
736 """
737- self.log.debug('Checking %s updated since %s' % (filename, mtime))
738+ unit_name = sentry_unit.info['unit_name']
739+ self.log.debug('Checking that %s updated since %s on '
740+ '%s' % (filename, mtime, unit_name))
741 time.sleep(sleep_time)
742- file_mtime = self._get_file_mtime(sentry_unit, filename)
743+ file_mtime = None
744+ tries = 0
745+ while tries <= retry_count and not file_mtime:
746+ try:
747+ file_mtime = self._get_file_mtime(sentry_unit, filename)
748+ self.log.debug('Attempt {} to get {} file mtime on {} '
749+ 'OK'.format(tries, filename, unit_name))
750+ except IOError as e:
751+ # NOTE(beisner) - race avoidance, file may not exist yet.
752+ # https://bugs.launchpad.net/charm-helpers/+bug/1474030
753+ self.log.debug('Attempt {} to get {} file mtime on {} '
754+ 'failed\n{}'.format(tries, filename,
755+ unit_name, e))
756+ time.sleep(retry_sleep_time)
757+ tries += 1
758+
759+ if not file_mtime:
760+ self.log.warn('Could not determine file mtime, assuming '
761+ 'file does not exist')
762+ return False
763+
764 if file_mtime >= mtime:
765 self.log.debug('File mtime is newer than provided mtime '
766- '(%s >= %s)' % (file_mtime, mtime))
767+ '(%s >= %s) on %s (OK)' % (file_mtime,
768+ mtime, unit_name))
769 return True
770 else:
771- self.log.warn('File mtime %s is older than provided mtime %s'
772- % (file_mtime, mtime))
773+ self.log.warn('File mtime is older than provided mtime'
774+ '(%s < on %s) on %s' % (file_mtime,
775+ mtime, unit_name))
776 return False
777
778 def validate_service_config_changed(self, sentry_unit, mtime, service,
779 filename, pgrep_full=None,
780- sleep_time=20, retry_count=2,
781- retry_sleep_time=30):
782+ sleep_time=20, retry_count=30,
783+ retry_sleep_time=10):
784 """Check service and file were updated after mtime
785
786 Args:
787@@ -456,7 +485,9 @@
788 sentry_unit,
789 filename,
790 mtime,
791- sleep_time=0)
792+ sleep_time=sleep_time,
793+ retry_count=retry_count,
794+ retry_sleep_time=retry_sleep_time)
795
796 return service_restart and config_update
797
798
799=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
800--- tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-10-22 13:20:40 +0000
801+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py 2016-01-07 14:27:41 +0000
802@@ -14,12 +14,18 @@
803 # You should have received a copy of the GNU Lesser General Public License
804 # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
805
806+import logging
807+import re
808+import sys
809 import six
810 from collections import OrderedDict
811 from charmhelpers.contrib.amulet.deployment import (
812 AmuletDeployment
813 )
814
815+DEBUG = logging.DEBUG
816+ERROR = logging.ERROR
817+
818
819 class OpenStackAmuletDeployment(AmuletDeployment):
820 """OpenStack amulet deployment.
821@@ -28,9 +34,12 @@
822 that is specifically for use by OpenStack charms.
823 """
824
825- def __init__(self, series=None, openstack=None, source=None, stable=True):
826+ def __init__(self, series=None, openstack=None, source=None,
827+ stable=True, log_level=DEBUG):
828 """Initialize the deployment environment."""
829 super(OpenStackAmuletDeployment, self).__init__(series)
830+ self.log = self.get_logger(level=log_level)
831+ self.log.info('OpenStackAmuletDeployment: init')
832 self.openstack = openstack
833 self.source = source
834 self.stable = stable
835@@ -38,6 +47,22 @@
836 # out.
837 self.current_next = "trusty"
838
839+ def get_logger(self, name="deployment-logger", level=logging.DEBUG):
840+ """Get a logger object that will log to stdout."""
841+ log = logging
842+ logger = log.getLogger(name)
843+ fmt = log.Formatter("%(asctime)s %(funcName)s "
844+ "%(levelname)s: %(message)s")
845+
846+ handler = log.StreamHandler(stream=sys.stdout)
847+ handler.setLevel(level)
848+ handler.setFormatter(fmt)
849+
850+ logger.addHandler(handler)
851+ logger.setLevel(level)
852+
853+ return logger
854+
855 def _determine_branch_locations(self, other_services):
856 """Determine the branch locations for the other services.
857
858@@ -45,6 +70,8 @@
859 stable or next (dev) branch, and based on this, use the corresonding
860 stable or next branches for the other_services."""
861
862+ self.log.info('OpenStackAmuletDeployment: determine branch locations')
863+
864 # Charms outside the lp:~openstack-charmers namespace
865 base_charms = ['mysql', 'mongodb', 'nrpe']
866
867@@ -82,6 +109,8 @@
868
869 def _add_services(self, this_service, other_services):
870 """Add services to the deployment and set openstack-origin/source."""
871+ self.log.info('OpenStackAmuletDeployment: adding services')
872+
873 other_services = self._determine_branch_locations(other_services)
874
875 super(OpenStackAmuletDeployment, self)._add_services(this_service,
876@@ -111,9 +140,79 @@
877
878 def _configure_services(self, configs):
879 """Configure all of the services."""
880+ self.log.info('OpenStackAmuletDeployment: configure services')
881 for service, config in six.iteritems(configs):
882 self.d.configure(service, config)
883
884+ def _auto_wait_for_status(self, message=None, exclude_services=None,
885+ include_only=None, timeout=1800):
886+ """Wait for all units to have a specific extended status, except
887+ for any defined as excluded. Unless specified via message, any
888+ status containing any case of 'ready' will be considered a match.
889+
890+ Examples of message usage:
891+
892+ Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
893+ message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
894+
895+ Wait for all units to reach this status (exact match):
896+ message = re.compile('^Unit is ready and clustered$')
897+
898+ Wait for all units to reach any one of these (exact match):
899+ message = re.compile('Unit is ready|OK|Ready')
900+
901+ Wait for at least one unit to reach this status (exact match):
902+ message = {'ready'}
903+
904+ See Amulet's sentry.wait_for_messages() for message usage detail.
905+ https://github.com/juju/amulet/blob/master/amulet/sentry.py
906+
907+ :param message: Expected status match
908+ :param exclude_services: List of juju service names to ignore,
909+ not to be used in conjuction with include_only.
910+ :param include_only: List of juju service names to exclusively check,
911+ not to be used in conjuction with exclude_services.
912+ :param timeout: Maximum time in seconds to wait for status match
913+ :returns: None. Raises if timeout is hit.
914+ """
915+ self.log.info('Waiting for extended status on units...')
916+
917+ all_services = self.d.services.keys()
918+
919+ if exclude_services and include_only:
920+ raise ValueError('exclude_services can not be used '
921+ 'with include_only')
922+
923+ if message:
924+ if isinstance(message, re._pattern_type):
925+ match = message.pattern
926+ else:
927+ match = message
928+
929+ self.log.debug('Custom extended status wait match: '
930+ '{}'.format(match))
931+ else:
932+ self.log.debug('Default extended status wait match: contains '
933+ 'READY (case-insensitive)')
934+ message = re.compile('.*ready.*', re.IGNORECASE)
935+
936+ if exclude_services:
937+ self.log.debug('Excluding services from extended status match: '
938+ '{}'.format(exclude_services))
939+ else:
940+ exclude_services = []
941+
942+ if include_only:
943+ services = include_only
944+ else:
945+ services = list(set(all_services) - set(exclude_services))
946+
947+ self.log.debug('Waiting up to {}s for extended status on services: '
948+ '{}'.format(timeout, services))
949+ service_messages = {service: message for service in services}
950+ self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
951+ self.log.info('OK')
952+
953 def _get_openstack_release(self):
954 """Get openstack release.
955
956
957=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
958--- tests/charmhelpers/contrib/openstack/amulet/utils.py 2015-10-22 13:20:40 +0000
959+++ tests/charmhelpers/contrib/openstack/amulet/utils.py 2016-01-07 14:27:41 +0000
960@@ -18,6 +18,7 @@
961 import json
962 import logging
963 import os
964+import re
965 import six
966 import time
967 import urllib
968@@ -604,7 +605,22 @@
969 '{}'.format(sample_type, samples))
970 return None
971
972-# rabbitmq/amqp specific helpers:
973+ # rabbitmq/amqp specific helpers:
974+
975+ def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
976+ """Wait for rmq units extended status to show cluster readiness,
977+ after an optional initial sleep period. Initial sleep is likely
978+ necessary to be effective following a config change, as status
979+ message may not instantly update to non-ready."""
980+
981+ if init_sleep:
982+ time.sleep(init_sleep)
983+
984+ message = re.compile('^Unit is ready and clustered$')
985+ deployment._auto_wait_for_status(message=message,
986+ timeout=timeout,
987+ include_only=['rabbitmq-server'])
988+
989 def add_rmq_test_user(self, sentry_units,
990 username="testuser1", password="changeme"):
991 """Add a test user via the first rmq juju unit, check connection as
992@@ -752,7 +768,7 @@
993 self.log.debug('SSL is enabled @{}:{} '
994 '({})'.format(host, port, unit_name))
995 return True
996- elif not port and not conf_ssl:
997+ elif not conf_ssl:
998 self.log.debug('SSL not enabled @{}:{} '
999 '({})'.format(host, port, unit_name))
1000 return False
1001@@ -805,7 +821,10 @@
1002 if port:
1003 config['ssl_port'] = port
1004
1005- deployment.configure('rabbitmq-server', config)
1006+ deployment.d.configure('rabbitmq-server', config)
1007+
1008+ # Wait for unit status
1009+ self.rmq_wait_for_cluster(deployment)
1010
1011 # Confirm
1012 tries = 0
1013@@ -832,7 +851,10 @@
1014
1015 # Disable RMQ SSL
1016 config = {'ssl': 'off'}
1017- deployment.configure('rabbitmq-server', config)
1018+ deployment.d.configure('rabbitmq-server', config)
1019+
1020+ # Wait for unit status
1021+ self.rmq_wait_for_cluster(deployment)
1022
1023 # Confirm
1024 tries = 0

Subscribers

People subscribed via source and target branches