Merge lp:~james-page/charms/trusty/neutron-api/lp1531102-trunk into lp:~openstack-charmers-archive/charms/trusty/neutron-api/trunk

Proposed by James Page on 2016-01-07
Status: Merged
Merged at revision: 121
Proposed branch: lp:~james-page/charms/trusty/neutron-api/lp1531102-trunk
Merge into: lp:~openstack-charmers-archive/charms/trusty/neutron-api/trunk
Diff against target: 606 lines (+275/-39)
7 files modified
hooks/charmhelpers/contrib/openstack/amulet/deployment.py (+67/-8)
hooks/charmhelpers/contrib/openstack/amulet/utils.py (+25/-3)
hooks/charmhelpers/contrib/openstack/utils.py (+21/-17)
hooks/charmhelpers/core/hugepage.py (+2/-0)
hooks/charmhelpers/core/kernel.py (+68/-0)
tests/charmhelpers/contrib/openstack/amulet/deployment.py (+67/-8)
tests/charmhelpers/contrib/openstack/amulet/utils.py (+25/-3)
To merge this branch: bzr merge lp:~james-page/charms/trusty/neutron-api/lp1531102-trunk
Reviewer Review Type Date Requested Status
Liam Young 2016-01-07 Approve on 2016-01-12
Review via email: mp+281862@code.launchpad.net

Commit message

Resync helpers

Description of the change

Resync helpers to resolve issues with liberty point releases.

To post a comment you must log in.

charm_lint_check #16740 neutron-api for james-page mp281862
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/16740/

charm_unit_test #15636 neutron-api for james-page mp281862
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/15636/

charm_amulet_test #8560 neutron-api for james-page mp281862
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/8560/

Liam Young (gnuoy) wrote :

Approved

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
2--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2015-10-22 13:21:43 +0000
3+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2016-01-07 14:15:21 +0000
4@@ -14,13 +14,18 @@
5 # You should have received a copy of the GNU Lesser General Public License
6 # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
7
8+import logging
9 import re
10+import sys
11 import six
12 from collections import OrderedDict
13 from charmhelpers.contrib.amulet.deployment import (
14 AmuletDeployment
15 )
16
17+DEBUG = logging.DEBUG
18+ERROR = logging.ERROR
19+
20
21 class OpenStackAmuletDeployment(AmuletDeployment):
22 """OpenStack amulet deployment.
23@@ -29,9 +34,12 @@
24 that is specifically for use by OpenStack charms.
25 """
26
27- def __init__(self, series=None, openstack=None, source=None, stable=True):
28+ def __init__(self, series=None, openstack=None, source=None,
29+ stable=True, log_level=DEBUG):
30 """Initialize the deployment environment."""
31 super(OpenStackAmuletDeployment, self).__init__(series)
32+ self.log = self.get_logger(level=log_level)
33+ self.log.info('OpenStackAmuletDeployment: init')
34 self.openstack = openstack
35 self.source = source
36 self.stable = stable
37@@ -39,6 +47,22 @@
38 # out.
39 self.current_next = "trusty"
40
41+ def get_logger(self, name="deployment-logger", level=logging.DEBUG):
42+ """Get a logger object that will log to stdout."""
43+ log = logging
44+ logger = log.getLogger(name)
45+ fmt = log.Formatter("%(asctime)s %(funcName)s "
46+ "%(levelname)s: %(message)s")
47+
48+ handler = log.StreamHandler(stream=sys.stdout)
49+ handler.setLevel(level)
50+ handler.setFormatter(fmt)
51+
52+ logger.addHandler(handler)
53+ logger.setLevel(level)
54+
55+ return logger
56+
57 def _determine_branch_locations(self, other_services):
58 """Determine the branch locations for the other services.
59
60@@ -46,6 +70,8 @@
61 stable or next (dev) branch, and based on this, use the corresonding
62 stable or next branches for the other_services."""
63
64+ self.log.info('OpenStackAmuletDeployment: determine branch locations')
65+
66 # Charms outside the lp:~openstack-charmers namespace
67 base_charms = ['mysql', 'mongodb', 'nrpe']
68
69@@ -83,6 +109,8 @@
70
71 def _add_services(self, this_service, other_services):
72 """Add services to the deployment and set openstack-origin/source."""
73+ self.log.info('OpenStackAmuletDeployment: adding services')
74+
75 other_services = self._determine_branch_locations(other_services)
76
77 super(OpenStackAmuletDeployment, self)._add_services(this_service,
78@@ -112,11 +140,12 @@
79
80 def _configure_services(self, configs):
81 """Configure all of the services."""
82+ self.log.info('OpenStackAmuletDeployment: configure services')
83 for service, config in six.iteritems(configs):
84 self.d.configure(service, config)
85
86 def _auto_wait_for_status(self, message=None, exclude_services=None,
87- timeout=1800):
88+ include_only=None, timeout=1800):
89 """Wait for all units to have a specific extended status, except
90 for any defined as excluded. Unless specified via message, any
91 status containing any case of 'ready' will be considered a match.
92@@ -127,7 +156,7 @@
93 message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
94
95 Wait for all units to reach this status (exact match):
96- message = 'Unit is ready'
97+ message = re.compile('^Unit is ready and clustered$')
98
99 Wait for all units to reach any one of these (exact match):
100 message = re.compile('Unit is ready|OK|Ready')
101@@ -139,20 +168,50 @@
102 https://github.com/juju/amulet/blob/master/amulet/sentry.py
103
104 :param message: Expected status match
105- :param exclude_services: List of juju service names to ignore
106+ :param exclude_services: List of juju service names to ignore,
107+ not to be used in conjuction with include_only.
108+ :param include_only: List of juju service names to exclusively check,
109+ not to be used in conjuction with exclude_services.
110 :param timeout: Maximum time in seconds to wait for status match
111 :returns: None. Raises if timeout is hit.
112 """
113-
114- if not message:
115+ self.log.info('Waiting for extended status on units...')
116+
117+ all_services = self.d.services.keys()
118+
119+ if exclude_services and include_only:
120+ raise ValueError('exclude_services can not be used '
121+ 'with include_only')
122+
123+ if message:
124+ if isinstance(message, re._pattern_type):
125+ match = message.pattern
126+ else:
127+ match = message
128+
129+ self.log.debug('Custom extended status wait match: '
130+ '{}'.format(match))
131+ else:
132+ self.log.debug('Default extended status wait match: contains '
133+ 'READY (case-insensitive)')
134 message = re.compile('.*ready.*', re.IGNORECASE)
135
136- if not exclude_services:
137+ if exclude_services:
138+ self.log.debug('Excluding services from extended status match: '
139+ '{}'.format(exclude_services))
140+ else:
141 exclude_services = []
142
143- services = list(set(self.d.services.keys()) - set(exclude_services))
144+ if include_only:
145+ services = include_only
146+ else:
147+ services = list(set(all_services) - set(exclude_services))
148+
149+ self.log.debug('Waiting up to {}s for extended status on services: '
150+ '{}'.format(timeout, services))
151 service_messages = {service: message for service in services}
152 self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
153+ self.log.info('OK')
154
155 def _get_openstack_release(self):
156 """Get openstack release.
157
158=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
159--- hooks/charmhelpers/contrib/openstack/amulet/utils.py 2015-10-22 13:21:43 +0000
160+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py 2016-01-07 14:15:21 +0000
161@@ -18,6 +18,7 @@
162 import json
163 import logging
164 import os
165+import re
166 import six
167 import time
168 import urllib
169@@ -604,7 +605,22 @@
170 '{}'.format(sample_type, samples))
171 return None
172
173-# rabbitmq/amqp specific helpers:
174+ # rabbitmq/amqp specific helpers:
175+
176+ def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
177+ """Wait for rmq units extended status to show cluster readiness,
178+ after an optional initial sleep period. Initial sleep is likely
179+ necessary to be effective following a config change, as status
180+ message may not instantly update to non-ready."""
181+
182+ if init_sleep:
183+ time.sleep(init_sleep)
184+
185+ message = re.compile('^Unit is ready and clustered$')
186+ deployment._auto_wait_for_status(message=message,
187+ timeout=timeout,
188+ include_only=['rabbitmq-server'])
189+
190 def add_rmq_test_user(self, sentry_units,
191 username="testuser1", password="changeme"):
192 """Add a test user via the first rmq juju unit, check connection as
193@@ -805,7 +821,10 @@
194 if port:
195 config['ssl_port'] = port
196
197- deployment.configure('rabbitmq-server', config)
198+ deployment.d.configure('rabbitmq-server', config)
199+
200+ # Wait for unit status
201+ self.rmq_wait_for_cluster(deployment)
202
203 # Confirm
204 tries = 0
205@@ -832,7 +851,10 @@
206
207 # Disable RMQ SSL
208 config = {'ssl': 'off'}
209- deployment.configure('rabbitmq-server', config)
210+ deployment.d.configure('rabbitmq-server', config)
211+
212+ # Wait for unit status
213+ self.rmq_wait_for_cluster(deployment)
214
215 # Confirm
216 tries = 0
217
218=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
219--- hooks/charmhelpers/contrib/openstack/utils.py 2015-10-22 13:21:43 +0000
220+++ hooks/charmhelpers/contrib/openstack/utils.py 2016-01-07 14:15:21 +0000
221@@ -127,31 +127,31 @@
222 # >= Liberty version->codename mapping
223 PACKAGE_CODENAMES = {
224 'nova-common': OrderedDict([
225- ('12.0.0', 'liberty'),
226+ ('12.0', 'liberty'),
227 ]),
228 'neutron-common': OrderedDict([
229- ('7.0.0', 'liberty'),
230+ ('7.0', 'liberty'),
231 ]),
232 'cinder-common': OrderedDict([
233- ('7.0.0', 'liberty'),
234+ ('7.0', 'liberty'),
235 ]),
236 'keystone': OrderedDict([
237- ('8.0.0', 'liberty'),
238+ ('8.0', 'liberty'),
239 ]),
240 'horizon-common': OrderedDict([
241- ('8.0.0', 'liberty'),
242+ ('8.0', 'liberty'),
243 ]),
244 'ceilometer-common': OrderedDict([
245- ('5.0.0', 'liberty'),
246+ ('5.0', 'liberty'),
247 ]),
248 'heat-common': OrderedDict([
249- ('5.0.0', 'liberty'),
250+ ('5.0', 'liberty'),
251 ]),
252 'glance-common': OrderedDict([
253- ('11.0.0', 'liberty'),
254+ ('11.0', 'liberty'),
255 ]),
256 'openstack-dashboard': OrderedDict([
257- ('8.0.0', 'liberty'),
258+ ('8.0', 'liberty'),
259 ]),
260 }
261
262@@ -238,7 +238,14 @@
263 error_out(e)
264
265 vers = apt.upstream_version(pkg.current_ver.ver_str)
266- match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
267+ if 'swift' in pkg.name:
268+ # Fully x.y.z match for swift versions
269+ match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
270+ else:
271+ # x.y match only for 20XX.X
272+ # and ignore patch level for other packages
273+ match = re.match('^(\d+)\.(\d+)', vers)
274+
275 if match:
276 vers = match.group(0)
277
278@@ -250,13 +257,8 @@
279 # < Liberty co-ordinated project versions
280 try:
281 if 'swift' in pkg.name:
282- swift_vers = vers[:5]
283- if swift_vers not in SWIFT_CODENAMES:
284- # Deal with 1.10.0 upward
285- swift_vers = vers[:6]
286- return SWIFT_CODENAMES[swift_vers]
287+ return SWIFT_CODENAMES[vers]
288 else:
289- vers = vers[:6]
290 return OPENSTACK_CODENAMES[vers]
291 except KeyError:
292 if not fatal:
293@@ -859,7 +861,9 @@
294 if charm_state != 'active' and charm_state != 'unknown':
295 state = workload_state_compare(state, charm_state)
296 if message:
297- message = "{} {}".format(message, charm_message)
298+ charm_message = charm_message.replace("Incomplete relations: ",
299+ "")
300+ message = "{}, {}".format(message, charm_message)
301 else:
302 message = charm_message
303
304
305=== modified file 'hooks/charmhelpers/core/hugepage.py'
306--- hooks/charmhelpers/core/hugepage.py 2015-10-22 13:21:43 +0000
307+++ hooks/charmhelpers/core/hugepage.py 2016-01-07 14:15:21 +0000
308@@ -46,6 +46,8 @@
309 group_info = add_group(group)
310 gid = group_info.gr_gid
311 add_user_to_group(user, group)
312+ if max_map_count < 2 * nr_hugepages:
313+ max_map_count = 2 * nr_hugepages
314 sysctl_settings = {
315 'vm.nr_hugepages': nr_hugepages,
316 'vm.max_map_count': max_map_count,
317
318=== added file 'hooks/charmhelpers/core/kernel.py'
319--- hooks/charmhelpers/core/kernel.py 1970-01-01 00:00:00 +0000
320+++ hooks/charmhelpers/core/kernel.py 2016-01-07 14:15:21 +0000
321@@ -0,0 +1,68 @@
322+#!/usr/bin/env python
323+# -*- coding: utf-8 -*-
324+
325+# Copyright 2014-2015 Canonical Limited.
326+#
327+# This file is part of charm-helpers.
328+#
329+# charm-helpers is free software: you can redistribute it and/or modify
330+# it under the terms of the GNU Lesser General Public License version 3 as
331+# published by the Free Software Foundation.
332+#
333+# charm-helpers is distributed in the hope that it will be useful,
334+# but WITHOUT ANY WARRANTY; without even the implied warranty of
335+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
336+# GNU Lesser General Public License for more details.
337+#
338+# You should have received a copy of the GNU Lesser General Public License
339+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
340+
341+__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
342+
343+from charmhelpers.core.hookenv import (
344+ log,
345+ INFO
346+)
347+
348+from subprocess import check_call, check_output
349+import re
350+
351+
352+def modprobe(module, persist=True):
353+ """Load a kernel module and configure for auto-load on reboot."""
354+ cmd = ['modprobe', module]
355+
356+ log('Loading kernel module %s' % module, level=INFO)
357+
358+ check_call(cmd)
359+ if persist:
360+ with open('/etc/modules', 'r+') as modules:
361+ if module not in modules.read():
362+ modules.write(module)
363+
364+
365+def rmmod(module, force=False):
366+ """Remove a module from the linux kernel"""
367+ cmd = ['rmmod']
368+ if force:
369+ cmd.append('-f')
370+ cmd.append(module)
371+ log('Removing kernel module %s' % module, level=INFO)
372+ return check_call(cmd)
373+
374+
375+def lsmod():
376+ """Shows what kernel modules are currently loaded"""
377+ return check_output(['lsmod'],
378+ universal_newlines=True)
379+
380+
381+def is_module_loaded(module):
382+ """Checks if a kernel module is already loaded"""
383+ matches = re.findall('^%s[ ]+' % module, lsmod(), re.M)
384+ return len(matches) > 0
385+
386+
387+def update_initramfs(version='all'):
388+ """Updates an initramfs image"""
389+ return check_call(["update-initramfs", "-k", version, "-u"])
390
391=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
392--- tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-10-22 13:21:43 +0000
393+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py 2016-01-07 14:15:21 +0000
394@@ -14,13 +14,18 @@
395 # You should have received a copy of the GNU Lesser General Public License
396 # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
397
398+import logging
399 import re
400+import sys
401 import six
402 from collections import OrderedDict
403 from charmhelpers.contrib.amulet.deployment import (
404 AmuletDeployment
405 )
406
407+DEBUG = logging.DEBUG
408+ERROR = logging.ERROR
409+
410
411 class OpenStackAmuletDeployment(AmuletDeployment):
412 """OpenStack amulet deployment.
413@@ -29,9 +34,12 @@
414 that is specifically for use by OpenStack charms.
415 """
416
417- def __init__(self, series=None, openstack=None, source=None, stable=True):
418+ def __init__(self, series=None, openstack=None, source=None,
419+ stable=True, log_level=DEBUG):
420 """Initialize the deployment environment."""
421 super(OpenStackAmuletDeployment, self).__init__(series)
422+ self.log = self.get_logger(level=log_level)
423+ self.log.info('OpenStackAmuletDeployment: init')
424 self.openstack = openstack
425 self.source = source
426 self.stable = stable
427@@ -39,6 +47,22 @@
428 # out.
429 self.current_next = "trusty"
430
431+ def get_logger(self, name="deployment-logger", level=logging.DEBUG):
432+ """Get a logger object that will log to stdout."""
433+ log = logging
434+ logger = log.getLogger(name)
435+ fmt = log.Formatter("%(asctime)s %(funcName)s "
436+ "%(levelname)s: %(message)s")
437+
438+ handler = log.StreamHandler(stream=sys.stdout)
439+ handler.setLevel(level)
440+ handler.setFormatter(fmt)
441+
442+ logger.addHandler(handler)
443+ logger.setLevel(level)
444+
445+ return logger
446+
447 def _determine_branch_locations(self, other_services):
448 """Determine the branch locations for the other services.
449
450@@ -46,6 +70,8 @@
451 stable or next (dev) branch, and based on this, use the corresonding
452 stable or next branches for the other_services."""
453
454+ self.log.info('OpenStackAmuletDeployment: determine branch locations')
455+
456 # Charms outside the lp:~openstack-charmers namespace
457 base_charms = ['mysql', 'mongodb', 'nrpe']
458
459@@ -83,6 +109,8 @@
460
461 def _add_services(self, this_service, other_services):
462 """Add services to the deployment and set openstack-origin/source."""
463+ self.log.info('OpenStackAmuletDeployment: adding services')
464+
465 other_services = self._determine_branch_locations(other_services)
466
467 super(OpenStackAmuletDeployment, self)._add_services(this_service,
468@@ -112,11 +140,12 @@
469
470 def _configure_services(self, configs):
471 """Configure all of the services."""
472+ self.log.info('OpenStackAmuletDeployment: configure services')
473 for service, config in six.iteritems(configs):
474 self.d.configure(service, config)
475
476 def _auto_wait_for_status(self, message=None, exclude_services=None,
477- timeout=1800):
478+ include_only=None, timeout=1800):
479 """Wait for all units to have a specific extended status, except
480 for any defined as excluded. Unless specified via message, any
481 status containing any case of 'ready' will be considered a match.
482@@ -127,7 +156,7 @@
483 message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
484
485 Wait for all units to reach this status (exact match):
486- message = 'Unit is ready'
487+ message = re.compile('^Unit is ready and clustered$')
488
489 Wait for all units to reach any one of these (exact match):
490 message = re.compile('Unit is ready|OK|Ready')
491@@ -139,20 +168,50 @@
492 https://github.com/juju/amulet/blob/master/amulet/sentry.py
493
494 :param message: Expected status match
495- :param exclude_services: List of juju service names to ignore
496+ :param exclude_services: List of juju service names to ignore,
497+ not to be used in conjuction with include_only.
498+ :param include_only: List of juju service names to exclusively check,
499+ not to be used in conjuction with exclude_services.
500 :param timeout: Maximum time in seconds to wait for status match
501 :returns: None. Raises if timeout is hit.
502 """
503-
504- if not message:
505+ self.log.info('Waiting for extended status on units...')
506+
507+ all_services = self.d.services.keys()
508+
509+ if exclude_services and include_only:
510+ raise ValueError('exclude_services can not be used '
511+ 'with include_only')
512+
513+ if message:
514+ if isinstance(message, re._pattern_type):
515+ match = message.pattern
516+ else:
517+ match = message
518+
519+ self.log.debug('Custom extended status wait match: '
520+ '{}'.format(match))
521+ else:
522+ self.log.debug('Default extended status wait match: contains '
523+ 'READY (case-insensitive)')
524 message = re.compile('.*ready.*', re.IGNORECASE)
525
526- if not exclude_services:
527+ if exclude_services:
528+ self.log.debug('Excluding services from extended status match: '
529+ '{}'.format(exclude_services))
530+ else:
531 exclude_services = []
532
533- services = list(set(self.d.services.keys()) - set(exclude_services))
534+ if include_only:
535+ services = include_only
536+ else:
537+ services = list(set(all_services) - set(exclude_services))
538+
539+ self.log.debug('Waiting up to {}s for extended status on services: '
540+ '{}'.format(timeout, services))
541 service_messages = {service: message for service in services}
542 self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
543+ self.log.info('OK')
544
545 def _get_openstack_release(self):
546 """Get openstack release.
547
548=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
549--- tests/charmhelpers/contrib/openstack/amulet/utils.py 2015-10-22 13:21:43 +0000
550+++ tests/charmhelpers/contrib/openstack/amulet/utils.py 2016-01-07 14:15:21 +0000
551@@ -18,6 +18,7 @@
552 import json
553 import logging
554 import os
555+import re
556 import six
557 import time
558 import urllib
559@@ -604,7 +605,22 @@
560 '{}'.format(sample_type, samples))
561 return None
562
563-# rabbitmq/amqp specific helpers:
564+ # rabbitmq/amqp specific helpers:
565+
566+ def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
567+ """Wait for rmq units extended status to show cluster readiness,
568+ after an optional initial sleep period. Initial sleep is likely
569+ necessary to be effective following a config change, as status
570+ message may not instantly update to non-ready."""
571+
572+ if init_sleep:
573+ time.sleep(init_sleep)
574+
575+ message = re.compile('^Unit is ready and clustered$')
576+ deployment._auto_wait_for_status(message=message,
577+ timeout=timeout,
578+ include_only=['rabbitmq-server'])
579+
580 def add_rmq_test_user(self, sentry_units,
581 username="testuser1", password="changeme"):
582 """Add a test user via the first rmq juju unit, check connection as
583@@ -805,7 +821,10 @@
584 if port:
585 config['ssl_port'] = port
586
587- deployment.configure('rabbitmq-server', config)
588+ deployment.d.configure('rabbitmq-server', config)
589+
590+ # Wait for unit status
591+ self.rmq_wait_for_cluster(deployment)
592
593 # Confirm
594 tries = 0
595@@ -832,7 +851,10 @@
596
597 # Disable RMQ SSL
598 config = {'ssl': 'off'}
599- deployment.configure('rabbitmq-server', config)
600+ deployment.d.configure('rabbitmq-server', config)
601+
602+ # Wait for unit status
603+ self.rmq_wait_for_cluster(deployment)
604
605 # Confirm
606 tries = 0

Subscribers

People subscribed via source and target branches