Merge lp:~gnuoy/charms/trusty/nova-compute/chsync-stable into lp:~openstack-charmers-archive/charms/trusty/nova-compute/trunk

Proposed by Liam Young on 2015-10-22
Status: Merged
Merged at revision: 141
Proposed branch: lp:~gnuoy/charms/trusty/nova-compute/chsync-stable
Merge into: lp:~openstack-charmers-archive/charms/trusty/nova-compute/trunk
Diff against target: 546 lines (+245/-26)
8 files modified
hooks/charmhelpers/contrib/openstack/amulet/deployment.py (+100/-1)
hooks/charmhelpers/contrib/openstack/amulet/utils.py (+25/-3)
hooks/charmhelpers/contrib/openstack/context.py (+10/-9)
hooks/charmhelpers/contrib/openstack/utils.py (+4/-1)
hooks/charmhelpers/core/host.py (+12/-1)
hooks/charmhelpers/core/hugepage.py (+2/-0)
tests/charmhelpers/contrib/openstack/amulet/deployment.py (+67/-8)
tests/charmhelpers/contrib/openstack/amulet/utils.py (+25/-3)
To merge this branch: bzr merge lp:~gnuoy/charms/trusty/nova-compute/chsync-stable
Reviewer Review Type Date Requested Status
Marco Ceppi (community) Approve on 2015-10-23
David Ames 2015-10-22 Approve on 2015-10-22
Review via email: mp+275371@code.launchpad.net
To post a comment you must log in.

charm_lint_check #12350 nova-compute for gnuoy mp275371
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/12350/

charm_lint_check #12414 nova-compute for gnuoy mp275371
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/12414/

charm_unit_test #11518 nova-compute for gnuoy mp275371
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/11518/

charm_amulet_test #7507 nova-compute for gnuoy mp275371
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/7507/

Marco Ceppi (marcoceppi) wrote :

LGTM +1 NOOP ch sync

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
2--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2015-10-22 13:23:10 +0000
3+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2015-10-22 15:18:37 +0000
4@@ -14,12 +14,18 @@
5 # You should have received a copy of the GNU Lesser General Public License
6 # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
7
8+import logging
9+import re
10+import sys
11 import six
12 from collections import OrderedDict
13 from charmhelpers.contrib.amulet.deployment import (
14 AmuletDeployment
15 )
16
17+DEBUG = logging.DEBUG
18+ERROR = logging.ERROR
19+
20
21 class OpenStackAmuletDeployment(AmuletDeployment):
22 """OpenStack amulet deployment.
23@@ -28,9 +34,12 @@
24 that is specifically for use by OpenStack charms.
25 """
26
27- def __init__(self, series=None, openstack=None, source=None, stable=True):
28+ def __init__(self, series=None, openstack=None, source=None,
29+ stable=True, log_level=DEBUG):
30 """Initialize the deployment environment."""
31 super(OpenStackAmuletDeployment, self).__init__(series)
32+ self.log = self.get_logger(level=log_level)
33+ self.log.info('OpenStackAmuletDeployment: init')
34 self.openstack = openstack
35 self.source = source
36 self.stable = stable
37@@ -38,6 +47,22 @@
38 # out.
39 self.current_next = "trusty"
40
41+ def get_logger(self, name="deployment-logger", level=logging.DEBUG):
42+ """Get a logger object that will log to stdout."""
43+ log = logging
44+ logger = log.getLogger(name)
45+ fmt = log.Formatter("%(asctime)s %(funcName)s "
46+ "%(levelname)s: %(message)s")
47+
48+ handler = log.StreamHandler(stream=sys.stdout)
49+ handler.setLevel(level)
50+ handler.setFormatter(fmt)
51+
52+ logger.addHandler(handler)
53+ logger.setLevel(level)
54+
55+ return logger
56+
57 def _determine_branch_locations(self, other_services):
58 """Determine the branch locations for the other services.
59
60@@ -45,6 +70,8 @@
61 stable or next (dev) branch, and based on this, use the corresonding
62 stable or next branches for the other_services."""
63
64+ self.log.info('OpenStackAmuletDeployment: determine branch locations')
65+
66 # Charms outside the lp:~openstack-charmers namespace
67 base_charms = ['mysql', 'mongodb', 'nrpe']
68
69@@ -82,6 +109,8 @@
70
71 def _add_services(self, this_service, other_services):
72 """Add services to the deployment and set openstack-origin/source."""
73+ self.log.info('OpenStackAmuletDeployment: adding services')
74+
75 other_services = self._determine_branch_locations(other_services)
76
77 super(OpenStackAmuletDeployment, self)._add_services(this_service,
78@@ -111,9 +140,79 @@
79
80 def _configure_services(self, configs):
81 """Configure all of the services."""
82+ self.log.info('OpenStackAmuletDeployment: configure services')
83 for service, config in six.iteritems(configs):
84 self.d.configure(service, config)
85
86+ def _auto_wait_for_status(self, message=None, exclude_services=None,
87+ include_only=None, timeout=1800):
88+ """Wait for all units to have a specific extended status, except
89+ for any defined as excluded. Unless specified via message, any
90+ status containing any case of 'ready' will be considered a match.
91+
92+ Examples of message usage:
93+
94+ Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
95+ message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
96+
97+ Wait for all units to reach this status (exact match):
98+ message = re.compile('^Unit is ready and clustered$')
99+
100+ Wait for all units to reach any one of these (exact match):
101+ message = re.compile('Unit is ready|OK|Ready')
102+
103+ Wait for at least one unit to reach this status (exact match):
104+ message = {'ready'}
105+
106+ See Amulet's sentry.wait_for_messages() for message usage detail.
107+ https://github.com/juju/amulet/blob/master/amulet/sentry.py
108+
109+ :param message: Expected status match
110+ :param exclude_services: List of juju service names to ignore,
111+ not to be used in conjuction with include_only.
112+ :param include_only: List of juju service names to exclusively check,
113+ not to be used in conjuction with exclude_services.
114+ :param timeout: Maximum time in seconds to wait for status match
115+ :returns: None. Raises if timeout is hit.
116+ """
117+ self.log.info('Waiting for extended status on units...')
118+
119+ all_services = self.d.services.keys()
120+
121+ if exclude_services and include_only:
122+ raise ValueError('exclude_services can not be used '
123+ 'with include_only')
124+
125+ if message:
126+ if isinstance(message, re._pattern_type):
127+ match = message.pattern
128+ else:
129+ match = message
130+
131+ self.log.debug('Custom extended status wait match: '
132+ '{}'.format(match))
133+ else:
134+ self.log.debug('Default extended status wait match: contains '
135+ 'READY (case-insensitive)')
136+ message = re.compile('.*ready.*', re.IGNORECASE)
137+
138+ if exclude_services:
139+ self.log.debug('Excluding services from extended status match: '
140+ '{}'.format(exclude_services))
141+ else:
142+ exclude_services = []
143+
144+ if include_only:
145+ services = include_only
146+ else:
147+ services = list(set(all_services) - set(exclude_services))
148+
149+ self.log.debug('Waiting up to {}s for extended status on services: '
150+ '{}'.format(timeout, services))
151+ service_messages = {service: message for service in services}
152+ self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
153+ self.log.info('OK')
154+
155 def _get_openstack_release(self):
156 """Get openstack release.
157
158
159=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
160--- hooks/charmhelpers/contrib/openstack/amulet/utils.py 2015-10-22 13:23:10 +0000
161+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py 2015-10-22 15:18:37 +0000
162@@ -18,6 +18,7 @@
163 import json
164 import logging
165 import os
166+import re
167 import six
168 import time
169 import urllib
170@@ -604,7 +605,22 @@
171 '{}'.format(sample_type, samples))
172 return None
173
174-# rabbitmq/amqp specific helpers:
175+ # rabbitmq/amqp specific helpers:
176+
177+ def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
178+ """Wait for rmq units extended status to show cluster readiness,
179+ after an optional initial sleep period. Initial sleep is likely
180+ necessary to be effective following a config change, as status
181+ message may not instantly update to non-ready."""
182+
183+ if init_sleep:
184+ time.sleep(init_sleep)
185+
186+ message = re.compile('^Unit is ready and clustered$')
187+ deployment._auto_wait_for_status(message=message,
188+ timeout=timeout,
189+ include_only=['rabbitmq-server'])
190+
191 def add_rmq_test_user(self, sentry_units,
192 username="testuser1", password="changeme"):
193 """Add a test user via the first rmq juju unit, check connection as
194@@ -805,7 +821,10 @@
195 if port:
196 config['ssl_port'] = port
197
198- deployment.configure('rabbitmq-server', config)
199+ deployment.d.configure('rabbitmq-server', config)
200+
201+ # Wait for unit status
202+ self.rmq_wait_for_cluster(deployment)
203
204 # Confirm
205 tries = 0
206@@ -832,7 +851,10 @@
207
208 # Disable RMQ SSL
209 config = {'ssl': 'off'}
210- deployment.configure('rabbitmq-server', config)
211+ deployment.d.configure('rabbitmq-server', config)
212+
213+ # Wait for unit status
214+ self.rmq_wait_for_cluster(deployment)
215
216 # Confirm
217 tries = 0
218
219=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
220--- hooks/charmhelpers/contrib/openstack/context.py 2015-10-22 13:23:10 +0000
221+++ hooks/charmhelpers/contrib/openstack/context.py 2015-10-22 15:18:37 +0000
222@@ -1120,7 +1120,7 @@
223
224 ctxt = {
225 ... other context ...
226- 'subordinate_config': {
227+ 'subordinate_configuration': {
228 'DEFAULT': {
229 'key1': 'value1',
230 },
231@@ -1161,22 +1161,23 @@
232 try:
233 sub_config = json.loads(sub_config)
234 except:
235- log('Could not parse JSON from subordinate_config '
236- 'setting from %s' % rid, level=ERROR)
237+ log('Could not parse JSON from '
238+ 'subordinate_configuration setting from %s'
239+ % rid, level=ERROR)
240 continue
241
242 for service in self.services:
243 if service not in sub_config:
244- log('Found subordinate_config on %s but it contained'
245- 'nothing for %s service' % (rid, service),
246- level=INFO)
247+ log('Found subordinate_configuration on %s but it '
248+ 'contained nothing for %s service'
249+ % (rid, service), level=INFO)
250 continue
251
252 sub_config = sub_config[service]
253 if self.config_file not in sub_config:
254- log('Found subordinate_config on %s but it contained'
255- 'nothing for %s' % (rid, self.config_file),
256- level=INFO)
257+ log('Found subordinate_configuration on %s but it '
258+ 'contained nothing for %s'
259+ % (rid, self.config_file), level=INFO)
260 continue
261
262 sub_config = sub_config[self.config_file]
263
264=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
265--- hooks/charmhelpers/contrib/openstack/utils.py 2015-10-22 13:23:10 +0000
266+++ hooks/charmhelpers/contrib/openstack/utils.py 2015-10-22 15:18:37 +0000
267@@ -121,6 +121,7 @@
268 ('2.2.2', 'kilo'),
269 ('2.3.0', 'liberty'),
270 ('2.4.0', 'liberty'),
271+ ('2.5.0', 'liberty'),
272 ])
273
274 # >= Liberty version->codename mapping
275@@ -858,7 +859,9 @@
276 if charm_state != 'active' and charm_state != 'unknown':
277 state = workload_state_compare(state, charm_state)
278 if message:
279- message = "{} {}".format(message, charm_message)
280+ charm_message = charm_message.replace("Incomplete relations: ",
281+ "")
282+ message = "{}, {}".format(message, charm_message)
283 else:
284 message = charm_message
285
286
287=== modified file 'hooks/charmhelpers/core/host.py'
288--- hooks/charmhelpers/core/host.py 2015-10-22 13:23:10 +0000
289+++ hooks/charmhelpers/core/host.py 2015-10-22 15:18:37 +0000
290@@ -566,7 +566,14 @@
291 os.chdir(cur)
292
293
294-def chownr(path, owner, group, follow_links=True):
295+def chownr(path, owner, group, follow_links=True, chowntopdir=False):
296+ """
297+ Recursively change user and group ownership of files and directories
298+ in given path. Doesn't chown path itself by default, only its children.
299+
300+ :param bool follow_links: Also Chown links if True
301+ :param bool chowntopdir: Also chown path itself if True
302+ """
303 uid = pwd.getpwnam(owner).pw_uid
304 gid = grp.getgrnam(group).gr_gid
305 if follow_links:
306@@ -574,6 +581,10 @@
307 else:
308 chown = os.lchown
309
310+ if chowntopdir:
311+ broken_symlink = os.path.lexists(path) and not os.path.exists(path)
312+ if not broken_symlink:
313+ chown(path, uid, gid)
314 for root, dirs, files in os.walk(path):
315 for name in dirs + files:
316 full = os.path.join(root, name)
317
318=== modified file 'hooks/charmhelpers/core/hugepage.py'
319--- hooks/charmhelpers/core/hugepage.py 2015-10-22 13:23:10 +0000
320+++ hooks/charmhelpers/core/hugepage.py 2015-10-22 15:18:37 +0000
321@@ -46,6 +46,8 @@
322 group_info = add_group(group)
323 gid = group_info.gr_gid
324 add_user_to_group(user, group)
325+ if max_map_count < 2 * nr_hugepages:
326+ max_map_count = 2 * nr_hugepages
327 sysctl_settings = {
328 'vm.nr_hugepages': nr_hugepages,
329 'vm.max_map_count': max_map_count,
330
331=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
332--- tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-10-22 13:23:10 +0000
333+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-10-22 15:18:37 +0000
334@@ -14,13 +14,18 @@
335 # You should have received a copy of the GNU Lesser General Public License
336 # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
337
338+import logging
339 import re
340+import sys
341 import six
342 from collections import OrderedDict
343 from charmhelpers.contrib.amulet.deployment import (
344 AmuletDeployment
345 )
346
347+DEBUG = logging.DEBUG
348+ERROR = logging.ERROR
349+
350
351 class OpenStackAmuletDeployment(AmuletDeployment):
352 """OpenStack amulet deployment.
353@@ -29,9 +34,12 @@
354 that is specifically for use by OpenStack charms.
355 """
356
357- def __init__(self, series=None, openstack=None, source=None, stable=True):
358+ def __init__(self, series=None, openstack=None, source=None,
359+ stable=True, log_level=DEBUG):
360 """Initialize the deployment environment."""
361 super(OpenStackAmuletDeployment, self).__init__(series)
362+ self.log = self.get_logger(level=log_level)
363+ self.log.info('OpenStackAmuletDeployment: init')
364 self.openstack = openstack
365 self.source = source
366 self.stable = stable
367@@ -39,6 +47,22 @@
368 # out.
369 self.current_next = "trusty"
370
371+ def get_logger(self, name="deployment-logger", level=logging.DEBUG):
372+ """Get a logger object that will log to stdout."""
373+ log = logging
374+ logger = log.getLogger(name)
375+ fmt = log.Formatter("%(asctime)s %(funcName)s "
376+ "%(levelname)s: %(message)s")
377+
378+ handler = log.StreamHandler(stream=sys.stdout)
379+ handler.setLevel(level)
380+ handler.setFormatter(fmt)
381+
382+ logger.addHandler(handler)
383+ logger.setLevel(level)
384+
385+ return logger
386+
387 def _determine_branch_locations(self, other_services):
388 """Determine the branch locations for the other services.
389
390@@ -46,6 +70,8 @@
391 stable or next (dev) branch, and based on this, use the corresonding
392 stable or next branches for the other_services."""
393
394+ self.log.info('OpenStackAmuletDeployment: determine branch locations')
395+
396 # Charms outside the lp:~openstack-charmers namespace
397 base_charms = ['mysql', 'mongodb', 'nrpe']
398
399@@ -83,6 +109,8 @@
400
401 def _add_services(self, this_service, other_services):
402 """Add services to the deployment and set openstack-origin/source."""
403+ self.log.info('OpenStackAmuletDeployment: adding services')
404+
405 other_services = self._determine_branch_locations(other_services)
406
407 super(OpenStackAmuletDeployment, self)._add_services(this_service,
408@@ -112,11 +140,12 @@
409
410 def _configure_services(self, configs):
411 """Configure all of the services."""
412+ self.log.info('OpenStackAmuletDeployment: configure services')
413 for service, config in six.iteritems(configs):
414 self.d.configure(service, config)
415
416 def _auto_wait_for_status(self, message=None, exclude_services=None,
417- timeout=1800):
418+ include_only=None, timeout=1800):
419 """Wait for all units to have a specific extended status, except
420 for any defined as excluded. Unless specified via message, any
421 status containing any case of 'ready' will be considered a match.
422@@ -127,7 +156,7 @@
423 message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
424
425 Wait for all units to reach this status (exact match):
426- message = 'Unit is ready'
427+ message = re.compile('^Unit is ready and clustered$')
428
429 Wait for all units to reach any one of these (exact match):
430 message = re.compile('Unit is ready|OK|Ready')
431@@ -139,20 +168,50 @@
432 https://github.com/juju/amulet/blob/master/amulet/sentry.py
433
434 :param message: Expected status match
435- :param exclude_services: List of juju service names to ignore
436+ :param exclude_services: List of juju service names to ignore,
437+ not to be used in conjuction with include_only.
438+ :param include_only: List of juju service names to exclusively check,
439+ not to be used in conjuction with exclude_services.
440 :param timeout: Maximum time in seconds to wait for status match
441 :returns: None. Raises if timeout is hit.
442 """
443-
444- if not message:
445+ self.log.info('Waiting for extended status on units...')
446+
447+ all_services = self.d.services.keys()
448+
449+ if exclude_services and include_only:
450+ raise ValueError('exclude_services can not be used '
451+ 'with include_only')
452+
453+ if message:
454+ if isinstance(message, re._pattern_type):
455+ match = message.pattern
456+ else:
457+ match = message
458+
459+ self.log.debug('Custom extended status wait match: '
460+ '{}'.format(match))
461+ else:
462+ self.log.debug('Default extended status wait match: contains '
463+ 'READY (case-insensitive)')
464 message = re.compile('.*ready.*', re.IGNORECASE)
465
466- if not exclude_services:
467+ if exclude_services:
468+ self.log.debug('Excluding services from extended status match: '
469+ '{}'.format(exclude_services))
470+ else:
471 exclude_services = []
472
473- services = list(set(self.d.services.keys()) - set(exclude_services))
474+ if include_only:
475+ services = include_only
476+ else:
477+ services = list(set(all_services) - set(exclude_services))
478+
479+ self.log.debug('Waiting up to {}s for extended status on services: '
480+ '{}'.format(timeout, services))
481 service_messages = {service: message for service in services}
482 self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
483+ self.log.info('OK')
484
485 def _get_openstack_release(self):
486 """Get openstack release.
487
488=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
489--- tests/charmhelpers/contrib/openstack/amulet/utils.py 2015-10-22 13:23:10 +0000
490+++ tests/charmhelpers/contrib/openstack/amulet/utils.py 2015-10-22 15:18:37 +0000
491@@ -18,6 +18,7 @@
492 import json
493 import logging
494 import os
495+import re
496 import six
497 import time
498 import urllib
499@@ -604,7 +605,22 @@
500 '{}'.format(sample_type, samples))
501 return None
502
503-# rabbitmq/amqp specific helpers:
504+ # rabbitmq/amqp specific helpers:
505+
506+ def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
507+ """Wait for rmq units extended status to show cluster readiness,
508+ after an optional initial sleep period. Initial sleep is likely
509+ necessary to be effective following a config change, as status
510+ message may not instantly update to non-ready."""
511+
512+ if init_sleep:
513+ time.sleep(init_sleep)
514+
515+ message = re.compile('^Unit is ready and clustered$')
516+ deployment._auto_wait_for_status(message=message,
517+ timeout=timeout,
518+ include_only=['rabbitmq-server'])
519+
520 def add_rmq_test_user(self, sentry_units,
521 username="testuser1", password="changeme"):
522 """Add a test user via the first rmq juju unit, check connection as
523@@ -805,7 +821,10 @@
524 if port:
525 config['ssl_port'] = port
526
527- deployment.configure('rabbitmq-server', config)
528+ deployment.d.configure('rabbitmq-server', config)
529+
530+ # Wait for unit status
531+ self.rmq_wait_for_cluster(deployment)
532
533 # Confirm
534 tries = 0
535@@ -832,7 +851,10 @@
536
537 # Disable RMQ SSL
538 config = {'ssl': 'off'}
539- deployment.configure('rabbitmq-server', config)
540+ deployment.d.configure('rabbitmq-server', config)
541+
542+ # Wait for unit status
543+ self.rmq_wait_for_cluster(deployment)
544
545 # Confirm
546 tries = 0

Subscribers

People subscribed via source and target branches