Merge lp:~james-page/charms/trusty/swift-proxy/lp1531102-trunk into lp:~openstack-charmers-archive/charms/trusty/swift-proxy/trunk

Proposed by James Page on 2016-01-07
Status: Merged
Merged at revision: 107
Proposed branch: lp:~james-page/charms/trusty/swift-proxy/lp1531102-trunk
Merge into: lp:~openstack-charmers-archive/charms/trusty/swift-proxy/trunk
Diff against target: 566 lines (+285/-26)
7 files modified
charmhelpers/contrib/openstack/amulet/deployment.py (+100/-1)
charmhelpers/contrib/openstack/amulet/utils.py (+25/-3)
charmhelpers/contrib/openstack/utils.py (+21/-17)
charmhelpers/core/host.py (+12/-1)
charmhelpers/core/hugepage.py (+2/-0)
tests/charmhelpers/contrib/openstack/amulet/deployment.py (+100/-1)
tests/charmhelpers/contrib/openstack/amulet/utils.py (+25/-3)
To merge this branch: bzr merge lp:~james-page/charms/trusty/swift-proxy/lp1531102-trunk
Reviewer Review Type Date Requested Status
Liam Young 2016-01-07 Approve on 2016-01-08
Review via email: mp+281872@code.launchpad.net

Commit message

Resync helpers

Description of the change

Resync stable helpers to resolve liberty point release issues.

To post a comment you must log in.

charm_lint_check #16739 swift-proxy for james-page mp281872
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/16739/

charm_unit_test #15637 swift-proxy for james-page mp281872
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/15637/

charm_amulet_test #8563 swift-proxy for james-page mp281872
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/8563/

Liam Young (gnuoy) wrote :

approve

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'charmhelpers/contrib/openstack/amulet/deployment.py'
2--- charmhelpers/contrib/openstack/amulet/deployment.py 2015-10-22 13:24:57 +0000
3+++ charmhelpers/contrib/openstack/amulet/deployment.py 2016-01-07 14:33:08 +0000
4@@ -14,12 +14,18 @@
5 # You should have received a copy of the GNU Lesser General Public License
6 # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
7
8+import logging
9+import re
10+import sys
11 import six
12 from collections import OrderedDict
13 from charmhelpers.contrib.amulet.deployment import (
14 AmuletDeployment
15 )
16
17+DEBUG = logging.DEBUG
18+ERROR = logging.ERROR
19+
20
21 class OpenStackAmuletDeployment(AmuletDeployment):
22 """OpenStack amulet deployment.
23@@ -28,9 +34,12 @@
24 that is specifically for use by OpenStack charms.
25 """
26
27- def __init__(self, series=None, openstack=None, source=None, stable=True):
28+ def __init__(self, series=None, openstack=None, source=None,
29+ stable=True, log_level=DEBUG):
30 """Initialize the deployment environment."""
31 super(OpenStackAmuletDeployment, self).__init__(series)
32+ self.log = self.get_logger(level=log_level)
33+ self.log.info('OpenStackAmuletDeployment: init')
34 self.openstack = openstack
35 self.source = source
36 self.stable = stable
37@@ -38,6 +47,22 @@
38 # out.
39 self.current_next = "trusty"
40
41+ def get_logger(self, name="deployment-logger", level=logging.DEBUG):
42+ """Get a logger object that will log to stdout."""
43+ log = logging
44+ logger = log.getLogger(name)
45+ fmt = log.Formatter("%(asctime)s %(funcName)s "
46+ "%(levelname)s: %(message)s")
47+
48+ handler = log.StreamHandler(stream=sys.stdout)
49+ handler.setLevel(level)
50+ handler.setFormatter(fmt)
51+
52+ logger.addHandler(handler)
53+ logger.setLevel(level)
54+
55+ return logger
56+
57 def _determine_branch_locations(self, other_services):
58 """Determine the branch locations for the other services.
59
60@@ -45,6 +70,8 @@
61 stable or next (dev) branch, and based on this, use the corresonding
62 stable or next branches for the other_services."""
63
64+ self.log.info('OpenStackAmuletDeployment: determine branch locations')
65+
66 # Charms outside the lp:~openstack-charmers namespace
67 base_charms = ['mysql', 'mongodb', 'nrpe']
68
69@@ -82,6 +109,8 @@
70
71 def _add_services(self, this_service, other_services):
72 """Add services to the deployment and set openstack-origin/source."""
73+ self.log.info('OpenStackAmuletDeployment: adding services')
74+
75 other_services = self._determine_branch_locations(other_services)
76
77 super(OpenStackAmuletDeployment, self)._add_services(this_service,
78@@ -111,9 +140,79 @@
79
80 def _configure_services(self, configs):
81 """Configure all of the services."""
82+ self.log.info('OpenStackAmuletDeployment: configure services')
83 for service, config in six.iteritems(configs):
84 self.d.configure(service, config)
85
86+ def _auto_wait_for_status(self, message=None, exclude_services=None,
87+ include_only=None, timeout=1800):
88+ """Wait for all units to have a specific extended status, except
89+ for any defined as excluded. Unless specified via message, any
90+ status containing any case of 'ready' will be considered a match.
91+
92+ Examples of message usage:
93+
94+ Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
95+ message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
96+
97+ Wait for all units to reach this status (exact match):
98+ message = re.compile('^Unit is ready and clustered$')
99+
100+ Wait for all units to reach any one of these (exact match):
101+ message = re.compile('Unit is ready|OK|Ready')
102+
103+ Wait for at least one unit to reach this status (exact match):
104+ message = {'ready'}
105+
106+ See Amulet's sentry.wait_for_messages() for message usage detail.
107+ https://github.com/juju/amulet/blob/master/amulet/sentry.py
108+
109+ :param message: Expected status match
110+ :param exclude_services: List of juju service names to ignore,
111+ not to be used in conjuction with include_only.
112+ :param include_only: List of juju service names to exclusively check,
113+ not to be used in conjuction with exclude_services.
114+ :param timeout: Maximum time in seconds to wait for status match
115+ :returns: None. Raises if timeout is hit.
116+ """
117+ self.log.info('Waiting for extended status on units...')
118+
119+ all_services = self.d.services.keys()
120+
121+ if exclude_services and include_only:
122+ raise ValueError('exclude_services can not be used '
123+ 'with include_only')
124+
125+ if message:
126+ if isinstance(message, re._pattern_type):
127+ match = message.pattern
128+ else:
129+ match = message
130+
131+ self.log.debug('Custom extended status wait match: '
132+ '{}'.format(match))
133+ else:
134+ self.log.debug('Default extended status wait match: contains '
135+ 'READY (case-insensitive)')
136+ message = re.compile('.*ready.*', re.IGNORECASE)
137+
138+ if exclude_services:
139+ self.log.debug('Excluding services from extended status match: '
140+ '{}'.format(exclude_services))
141+ else:
142+ exclude_services = []
143+
144+ if include_only:
145+ services = include_only
146+ else:
147+ services = list(set(all_services) - set(exclude_services))
148+
149+ self.log.debug('Waiting up to {}s for extended status on services: '
150+ '{}'.format(timeout, services))
151+ service_messages = {service: message for service in services}
152+ self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
153+ self.log.info('OK')
154+
155 def _get_openstack_release(self):
156 """Get openstack release.
157
158
159=== modified file 'charmhelpers/contrib/openstack/amulet/utils.py'
160--- charmhelpers/contrib/openstack/amulet/utils.py 2015-10-22 13:24:57 +0000
161+++ charmhelpers/contrib/openstack/amulet/utils.py 2016-01-07 14:33:08 +0000
162@@ -18,6 +18,7 @@
163 import json
164 import logging
165 import os
166+import re
167 import six
168 import time
169 import urllib
170@@ -604,7 +605,22 @@
171 '{}'.format(sample_type, samples))
172 return None
173
174-# rabbitmq/amqp specific helpers:
175+ # rabbitmq/amqp specific helpers:
176+
177+ def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
178+ """Wait for rmq units extended status to show cluster readiness,
179+ after an optional initial sleep period. Initial sleep is likely
180+ necessary to be effective following a config change, as status
181+ message may not instantly update to non-ready."""
182+
183+ if init_sleep:
184+ time.sleep(init_sleep)
185+
186+ message = re.compile('^Unit is ready and clustered$')
187+ deployment._auto_wait_for_status(message=message,
188+ timeout=timeout,
189+ include_only=['rabbitmq-server'])
190+
191 def add_rmq_test_user(self, sentry_units,
192 username="testuser1", password="changeme"):
193 """Add a test user via the first rmq juju unit, check connection as
194@@ -805,7 +821,10 @@
195 if port:
196 config['ssl_port'] = port
197
198- deployment.configure('rabbitmq-server', config)
199+ deployment.d.configure('rabbitmq-server', config)
200+
201+ # Wait for unit status
202+ self.rmq_wait_for_cluster(deployment)
203
204 # Confirm
205 tries = 0
206@@ -832,7 +851,10 @@
207
208 # Disable RMQ SSL
209 config = {'ssl': 'off'}
210- deployment.configure('rabbitmq-server', config)
211+ deployment.d.configure('rabbitmq-server', config)
212+
213+ # Wait for unit status
214+ self.rmq_wait_for_cluster(deployment)
215
216 # Confirm
217 tries = 0
218
219=== modified file 'charmhelpers/contrib/openstack/utils.py'
220--- charmhelpers/contrib/openstack/utils.py 2015-10-22 13:24:57 +0000
221+++ charmhelpers/contrib/openstack/utils.py 2016-01-07 14:33:08 +0000
222@@ -127,31 +127,31 @@
223 # >= Liberty version->codename mapping
224 PACKAGE_CODENAMES = {
225 'nova-common': OrderedDict([
226- ('12.0.0', 'liberty'),
227+ ('12.0', 'liberty'),
228 ]),
229 'neutron-common': OrderedDict([
230- ('7.0.0', 'liberty'),
231+ ('7.0', 'liberty'),
232 ]),
233 'cinder-common': OrderedDict([
234- ('7.0.0', 'liberty'),
235+ ('7.0', 'liberty'),
236 ]),
237 'keystone': OrderedDict([
238- ('8.0.0', 'liberty'),
239+ ('8.0', 'liberty'),
240 ]),
241 'horizon-common': OrderedDict([
242- ('8.0.0', 'liberty'),
243+ ('8.0', 'liberty'),
244 ]),
245 'ceilometer-common': OrderedDict([
246- ('5.0.0', 'liberty'),
247+ ('5.0', 'liberty'),
248 ]),
249 'heat-common': OrderedDict([
250- ('5.0.0', 'liberty'),
251+ ('5.0', 'liberty'),
252 ]),
253 'glance-common': OrderedDict([
254- ('11.0.0', 'liberty'),
255+ ('11.0', 'liberty'),
256 ]),
257 'openstack-dashboard': OrderedDict([
258- ('8.0.0', 'liberty'),
259+ ('8.0', 'liberty'),
260 ]),
261 }
262
263@@ -238,7 +238,14 @@
264 error_out(e)
265
266 vers = apt.upstream_version(pkg.current_ver.ver_str)
267- match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
268+ if 'swift' in pkg.name:
269+ # Fully x.y.z match for swift versions
270+ match = re.match('^(\d+)\.(\d+)\.(\d+)', vers)
271+ else:
272+ # x.y match only for 20XX.X
273+ # and ignore patch level for other packages
274+ match = re.match('^(\d+)\.(\d+)', vers)
275+
276 if match:
277 vers = match.group(0)
278
279@@ -250,13 +257,8 @@
280 # < Liberty co-ordinated project versions
281 try:
282 if 'swift' in pkg.name:
283- swift_vers = vers[:5]
284- if swift_vers not in SWIFT_CODENAMES:
285- # Deal with 1.10.0 upward
286- swift_vers = vers[:6]
287- return SWIFT_CODENAMES[swift_vers]
288+ return SWIFT_CODENAMES[vers]
289 else:
290- vers = vers[:6]
291 return OPENSTACK_CODENAMES[vers]
292 except KeyError:
293 if not fatal:
294@@ -859,7 +861,9 @@
295 if charm_state != 'active' and charm_state != 'unknown':
296 state = workload_state_compare(state, charm_state)
297 if message:
298- message = "{} {}".format(message, charm_message)
299+ charm_message = charm_message.replace("Incomplete relations: ",
300+ "")
301+ message = "{}, {}".format(message, charm_message)
302 else:
303 message = charm_message
304
305
306=== modified file 'charmhelpers/core/host.py'
307--- charmhelpers/core/host.py 2015-10-22 13:24:57 +0000
308+++ charmhelpers/core/host.py 2016-01-07 14:33:08 +0000
309@@ -566,7 +566,14 @@
310 os.chdir(cur)
311
312
313-def chownr(path, owner, group, follow_links=True):
314+def chownr(path, owner, group, follow_links=True, chowntopdir=False):
315+ """
316+ Recursively change user and group ownership of files and directories
317+ in given path. Doesn't chown path itself by default, only its children.
318+
319+ :param bool follow_links: Also Chown links if True
320+ :param bool chowntopdir: Also chown path itself if True
321+ """
322 uid = pwd.getpwnam(owner).pw_uid
323 gid = grp.getgrnam(group).gr_gid
324 if follow_links:
325@@ -574,6 +581,10 @@
326 else:
327 chown = os.lchown
328
329+ if chowntopdir:
330+ broken_symlink = os.path.lexists(path) and not os.path.exists(path)
331+ if not broken_symlink:
332+ chown(path, uid, gid)
333 for root, dirs, files in os.walk(path):
334 for name in dirs + files:
335 full = os.path.join(root, name)
336
337=== modified file 'charmhelpers/core/hugepage.py'
338--- charmhelpers/core/hugepage.py 2015-10-22 13:24:57 +0000
339+++ charmhelpers/core/hugepage.py 2016-01-07 14:33:08 +0000
340@@ -46,6 +46,8 @@
341 group_info = add_group(group)
342 gid = group_info.gr_gid
343 add_user_to_group(user, group)
344+ if max_map_count < 2 * nr_hugepages:
345+ max_map_count = 2 * nr_hugepages
346 sysctl_settings = {
347 'vm.nr_hugepages': nr_hugepages,
348 'vm.max_map_count': max_map_count,
349
350=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
351--- tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-10-22 13:24:57 +0000
352+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py 2016-01-07 14:33:08 +0000
353@@ -14,12 +14,18 @@
354 # You should have received a copy of the GNU Lesser General Public License
355 # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
356
357+import logging
358+import re
359+import sys
360 import six
361 from collections import OrderedDict
362 from charmhelpers.contrib.amulet.deployment import (
363 AmuletDeployment
364 )
365
366+DEBUG = logging.DEBUG
367+ERROR = logging.ERROR
368+
369
370 class OpenStackAmuletDeployment(AmuletDeployment):
371 """OpenStack amulet deployment.
372@@ -28,9 +34,12 @@
373 that is specifically for use by OpenStack charms.
374 """
375
376- def __init__(self, series=None, openstack=None, source=None, stable=True):
377+ def __init__(self, series=None, openstack=None, source=None,
378+ stable=True, log_level=DEBUG):
379 """Initialize the deployment environment."""
380 super(OpenStackAmuletDeployment, self).__init__(series)
381+ self.log = self.get_logger(level=log_level)
382+ self.log.info('OpenStackAmuletDeployment: init')
383 self.openstack = openstack
384 self.source = source
385 self.stable = stable
386@@ -38,6 +47,22 @@
387 # out.
388 self.current_next = "trusty"
389
390+ def get_logger(self, name="deployment-logger", level=logging.DEBUG):
391+ """Get a logger object that will log to stdout."""
392+ log = logging
393+ logger = log.getLogger(name)
394+ fmt = log.Formatter("%(asctime)s %(funcName)s "
395+ "%(levelname)s: %(message)s")
396+
397+ handler = log.StreamHandler(stream=sys.stdout)
398+ handler.setLevel(level)
399+ handler.setFormatter(fmt)
400+
401+ logger.addHandler(handler)
402+ logger.setLevel(level)
403+
404+ return logger
405+
406 def _determine_branch_locations(self, other_services):
407 """Determine the branch locations for the other services.
408
409@@ -45,6 +70,8 @@
410 stable or next (dev) branch, and based on this, use the corresonding
411 stable or next branches for the other_services."""
412
413+ self.log.info('OpenStackAmuletDeployment: determine branch locations')
414+
415 # Charms outside the lp:~openstack-charmers namespace
416 base_charms = ['mysql', 'mongodb', 'nrpe']
417
418@@ -82,6 +109,8 @@
419
420 def _add_services(self, this_service, other_services):
421 """Add services to the deployment and set openstack-origin/source."""
422+ self.log.info('OpenStackAmuletDeployment: adding services')
423+
424 other_services = self._determine_branch_locations(other_services)
425
426 super(OpenStackAmuletDeployment, self)._add_services(this_service,
427@@ -111,9 +140,79 @@
428
429 def _configure_services(self, configs):
430 """Configure all of the services."""
431+ self.log.info('OpenStackAmuletDeployment: configure services')
432 for service, config in six.iteritems(configs):
433 self.d.configure(service, config)
434
435+ def _auto_wait_for_status(self, message=None, exclude_services=None,
436+ include_only=None, timeout=1800):
437+ """Wait for all units to have a specific extended status, except
438+ for any defined as excluded. Unless specified via message, any
439+ status containing any case of 'ready' will be considered a match.
440+
441+ Examples of message usage:
442+
443+ Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
444+ message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
445+
446+ Wait for all units to reach this status (exact match):
447+ message = re.compile('^Unit is ready and clustered$')
448+
449+ Wait for all units to reach any one of these (exact match):
450+ message = re.compile('Unit is ready|OK|Ready')
451+
452+ Wait for at least one unit to reach this status (exact match):
453+ message = {'ready'}
454+
455+ See Amulet's sentry.wait_for_messages() for message usage detail.
456+ https://github.com/juju/amulet/blob/master/amulet/sentry.py
457+
458+ :param message: Expected status match
459+ :param exclude_services: List of juju service names to ignore,
460+ not to be used in conjuction with include_only.
461+ :param include_only: List of juju service names to exclusively check,
462+ not to be used in conjuction with exclude_services.
463+ :param timeout: Maximum time in seconds to wait for status match
464+ :returns: None. Raises if timeout is hit.
465+ """
466+ self.log.info('Waiting for extended status on units...')
467+
468+ all_services = self.d.services.keys()
469+
470+ if exclude_services and include_only:
471+ raise ValueError('exclude_services can not be used '
472+ 'with include_only')
473+
474+ if message:
475+ if isinstance(message, re._pattern_type):
476+ match = message.pattern
477+ else:
478+ match = message
479+
480+ self.log.debug('Custom extended status wait match: '
481+ '{}'.format(match))
482+ else:
483+ self.log.debug('Default extended status wait match: contains '
484+ 'READY (case-insensitive)')
485+ message = re.compile('.*ready.*', re.IGNORECASE)
486+
487+ if exclude_services:
488+ self.log.debug('Excluding services from extended status match: '
489+ '{}'.format(exclude_services))
490+ else:
491+ exclude_services = []
492+
493+ if include_only:
494+ services = include_only
495+ else:
496+ services = list(set(all_services) - set(exclude_services))
497+
498+ self.log.debug('Waiting up to {}s for extended status on services: '
499+ '{}'.format(timeout, services))
500+ service_messages = {service: message for service in services}
501+ self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
502+ self.log.info('OK')
503+
504 def _get_openstack_release(self):
505 """Get openstack release.
506
507
508=== modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py'
509--- tests/charmhelpers/contrib/openstack/amulet/utils.py 2015-10-22 13:24:57 +0000
510+++ tests/charmhelpers/contrib/openstack/amulet/utils.py 2016-01-07 14:33:08 +0000
511@@ -18,6 +18,7 @@
512 import json
513 import logging
514 import os
515+import re
516 import six
517 import time
518 import urllib
519@@ -604,7 +605,22 @@
520 '{}'.format(sample_type, samples))
521 return None
522
523-# rabbitmq/amqp specific helpers:
524+ # rabbitmq/amqp specific helpers:
525+
526+ def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
527+ """Wait for rmq units extended status to show cluster readiness,
528+ after an optional initial sleep period. Initial sleep is likely
529+ necessary to be effective following a config change, as status
530+ message may not instantly update to non-ready."""
531+
532+ if init_sleep:
533+ time.sleep(init_sleep)
534+
535+ message = re.compile('^Unit is ready and clustered$')
536+ deployment._auto_wait_for_status(message=message,
537+ timeout=timeout,
538+ include_only=['rabbitmq-server'])
539+
540 def add_rmq_test_user(self, sentry_units,
541 username="testuser1", password="changeme"):
542 """Add a test user via the first rmq juju unit, check connection as
543@@ -805,7 +821,10 @@
544 if port:
545 config['ssl_port'] = port
546
547- deployment.configure('rabbitmq-server', config)
548+ deployment.d.configure('rabbitmq-server', config)
549+
550+ # Wait for unit status
551+ self.rmq_wait_for_cluster(deployment)
552
553 # Confirm
554 tries = 0
555@@ -832,7 +851,10 @@
556
557 # Disable RMQ SSL
558 config = {'ssl': 'off'}
559- deployment.configure('rabbitmq-server', config)
560+ deployment.d.configure('rabbitmq-server', config)
561+
562+ # Wait for unit status
563+ self.rmq_wait_for_cluster(deployment)
564
565 # Confirm
566 tries = 0

Subscribers

People subscribed via source and target branches