Merge lp:~le-charmers/charms/trusty/neutron-api/leadership-election into lp:~openstack-charmers-archive/charms/trusty/neutron-api/next
- Trusty Tahr (14.04)
- leadership-election
- Merge into next
Proposed by
Edward Hope-Morley
Status: | Merged |
---|---|
Merged at revision: | 116 |
Proposed branch: | lp:~le-charmers/charms/trusty/neutron-api/leadership-election |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/neutron-api/next |
Diff against target: |
463 lines (+204/-40) 6 files modified
hooks/charmhelpers/contrib/hahelpers/cluster.py (+12/-2) hooks/charmhelpers/contrib/openstack/utils.py (+65/-18) hooks/charmhelpers/contrib/python/packages.py (+28/-5) hooks/charmhelpers/core/hookenv.py (+62/-1) hooks/charmhelpers/core/services/base.py (+30/-9) hooks/charmhelpers/fetch/giturl.py (+7/-5) |
To merge this branch: | bzr merge lp:~le-charmers/charms/trusty/neutron-api/leadership-election |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
OpenStack Charmers | Pending | ||
Review via email: mp+255010@code.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
- 88. By Edward Hope-Morley
-
synced next
- 89. By Liam Young
-
Merged trunk in + LE charmhelper sync
- 90. By Liam Young
-
Resync le charm helpers
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py' |
2 | --- hooks/charmhelpers/contrib/hahelpers/cluster.py 2015-05-20 19:42:53 +0000 |
3 | +++ hooks/charmhelpers/contrib/hahelpers/cluster.py 2015-06-04 08:45:02 +0000 |
4 | @@ -44,6 +44,7 @@ |
5 | ERROR, |
6 | WARNING, |
7 | unit_get, |
8 | + is_leader as juju_is_leader |
9 | ) |
10 | from charmhelpers.core.decorators import ( |
11 | retry_on_exception, |
12 | @@ -68,12 +69,21 @@ |
13 | Returns True if the charm executing this is the elected cluster leader. |
14 | |
15 | It relies on two mechanisms to determine leadership: |
16 | - 1. If the charm is part of a corosync cluster, call corosync to |
17 | + 1. If juju is sufficiently new and leadership election is supported, |
18 | + the is_leader command will be used. |
19 | + 2. If the charm is part of a corosync cluster, call corosync to |
20 | determine leadership. |
21 | - 2. If the charm is not part of a corosync cluster, the leader is |
22 | + 3. If the charm is not part of a corosync cluster, the leader is |
23 | determined as being "the alive unit with the lowest unit numer". In |
24 | other words, the oldest surviving unit. |
25 | """ |
26 | + try: |
27 | + return juju_is_leader() |
28 | + except NotImplementedError: |
29 | + log('Juju leadership election feature not enabled' |
30 | + ', using fallback support', |
31 | + level=WARNING) |
32 | + |
33 | if is_clustered(): |
34 | if not is_crm_leader(resource): |
35 | log('Deferring action to CRM leader.', level=INFO) |
36 | |
37 | === modified file 'hooks/charmhelpers/contrib/openstack/utils.py' |
38 | --- hooks/charmhelpers/contrib/openstack/utils.py 2015-04-16 19:58:18 +0000 |
39 | +++ hooks/charmhelpers/contrib/openstack/utils.py 2015-06-04 08:45:02 +0000 |
40 | @@ -53,9 +53,13 @@ |
41 | get_ipv6_addr |
42 | ) |
43 | |
44 | +from charmhelpers.contrib.python.packages import ( |
45 | + pip_create_virtualenv, |
46 | + pip_install, |
47 | +) |
48 | + |
49 | from charmhelpers.core.host import lsb_release, mounts, umount |
50 | from charmhelpers.fetch import apt_install, apt_cache, install_remote |
51 | -from charmhelpers.contrib.python.packages import pip_install |
52 | from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk |
53 | from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device |
54 | |
55 | @@ -497,7 +501,17 @@ |
56 | requirements_dir = None |
57 | |
58 | |
59 | -def git_clone_and_install(projects_yaml, core_project): |
60 | +def _git_yaml_load(projects_yaml): |
61 | + """ |
62 | + Load the specified yaml into a dictionary. |
63 | + """ |
64 | + if not projects_yaml: |
65 | + return None |
66 | + |
67 | + return yaml.load(projects_yaml) |
68 | + |
69 | + |
70 | +def git_clone_and_install(projects_yaml, core_project, depth=1): |
71 | """ |
72 | Clone/install all specified OpenStack repositories. |
73 | |
74 | @@ -510,23 +524,22 @@ |
75 | repository: 'git://git.openstack.org/openstack/requirements.git', |
76 | branch: 'stable/icehouse'} |
77 | directory: /mnt/openstack-git |
78 | - http_proxy: http://squid.internal:3128 |
79 | - https_proxy: https://squid.internal:3128 |
80 | + http_proxy: squid-proxy-url |
81 | + https_proxy: squid-proxy-url |
82 | |
83 | The directory, http_proxy, and https_proxy keys are optional. |
84 | """ |
85 | global requirements_dir |
86 | parent_dir = '/mnt/openstack-git' |
87 | - |
88 | - if not projects_yaml: |
89 | - return |
90 | - |
91 | - projects = yaml.load(projects_yaml) |
92 | + http_proxy = None |
93 | + |
94 | + projects = _git_yaml_load(projects_yaml) |
95 | _git_validate_projects_yaml(projects, core_project) |
96 | |
97 | old_environ = dict(os.environ) |
98 | |
99 | if 'http_proxy' in projects.keys(): |
100 | + http_proxy = projects['http_proxy'] |
101 | os.environ['http_proxy'] = projects['http_proxy'] |
102 | if 'https_proxy' in projects.keys(): |
103 | os.environ['https_proxy'] = projects['https_proxy'] |
104 | @@ -534,15 +547,19 @@ |
105 | if 'directory' in projects.keys(): |
106 | parent_dir = projects['directory'] |
107 | |
108 | + pip_create_virtualenv(os.path.join(parent_dir, 'venv')) |
109 | + |
110 | for p in projects['repositories']: |
111 | repo = p['repository'] |
112 | branch = p['branch'] |
113 | if p['name'] == 'requirements': |
114 | - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, |
115 | + repo_dir = _git_clone_and_install_single(repo, branch, depth, |
116 | + parent_dir, http_proxy, |
117 | update_requirements=False) |
118 | requirements_dir = repo_dir |
119 | else: |
120 | - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, |
121 | + repo_dir = _git_clone_and_install_single(repo, branch, depth, |
122 | + parent_dir, http_proxy, |
123 | update_requirements=True) |
124 | |
125 | os.environ = old_environ |
126 | @@ -574,7 +591,8 @@ |
127 | error_out('openstack-origin-git key \'{}\' is missing'.format(key)) |
128 | |
129 | |
130 | -def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): |
131 | +def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, |
132 | + update_requirements): |
133 | """ |
134 | Clone and install a single git repository. |
135 | """ |
136 | @@ -587,7 +605,8 @@ |
137 | |
138 | if not os.path.exists(dest_dir): |
139 | juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) |
140 | - repo_dir = install_remote(repo, dest=parent_dir, branch=branch) |
141 | + repo_dir = install_remote(repo, dest=parent_dir, branch=branch, |
142 | + depth=depth) |
143 | else: |
144 | repo_dir = dest_dir |
145 | |
146 | @@ -598,7 +617,12 @@ |
147 | _git_update_requirements(repo_dir, requirements_dir) |
148 | |
149 | juju_log('Installing git repo from dir: {}'.format(repo_dir)) |
150 | - pip_install(repo_dir) |
151 | + if http_proxy: |
152 | + pip_install(repo_dir, proxy=http_proxy, |
153 | + venv=os.path.join(parent_dir, 'venv')) |
154 | + else: |
155 | + pip_install(repo_dir, |
156 | + venv=os.path.join(parent_dir, 'venv')) |
157 | |
158 | return repo_dir |
159 | |
160 | @@ -621,16 +645,27 @@ |
161 | os.chdir(orig_dir) |
162 | |
163 | |
164 | +def git_pip_venv_dir(projects_yaml): |
165 | + """ |
166 | + Return the pip virtualenv path. |
167 | + """ |
168 | + parent_dir = '/mnt/openstack-git' |
169 | + |
170 | + projects = _git_yaml_load(projects_yaml) |
171 | + |
172 | + if 'directory' in projects.keys(): |
173 | + parent_dir = projects['directory'] |
174 | + |
175 | + return os.path.join(parent_dir, 'venv') |
176 | + |
177 | + |
178 | def git_src_dir(projects_yaml, project): |
179 | """ |
180 | Return the directory where the specified project's source is located. |
181 | """ |
182 | parent_dir = '/mnt/openstack-git' |
183 | |
184 | - if not projects_yaml: |
185 | - return |
186 | - |
187 | - projects = yaml.load(projects_yaml) |
188 | + projects = _git_yaml_load(projects_yaml) |
189 | |
190 | if 'directory' in projects.keys(): |
191 | parent_dir = projects['directory'] |
192 | @@ -640,3 +675,15 @@ |
193 | return os.path.join(parent_dir, os.path.basename(p['repository'])) |
194 | |
195 | return None |
196 | + |
197 | + |
198 | +def git_yaml_value(projects_yaml, key): |
199 | + """ |
200 | + Return the value in projects_yaml for the specified key. |
201 | + """ |
202 | + projects = _git_yaml_load(projects_yaml) |
203 | + |
204 | + if key in projects.keys(): |
205 | + return projects[key] |
206 | + |
207 | + return None |
208 | |
209 | === modified file 'hooks/charmhelpers/contrib/python/packages.py' |
210 | --- hooks/charmhelpers/contrib/python/packages.py 2015-03-16 14:16:02 +0000 |
211 | +++ hooks/charmhelpers/contrib/python/packages.py 2015-06-04 08:45:02 +0000 |
212 | @@ -17,8 +17,11 @@ |
213 | # You should have received a copy of the GNU Lesser General Public License |
214 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
215 | |
216 | +import os |
217 | +import subprocess |
218 | + |
219 | from charmhelpers.fetch import apt_install, apt_update |
220 | -from charmhelpers.core.hookenv import log |
221 | +from charmhelpers.core.hookenv import charm_dir, log |
222 | |
223 | try: |
224 | from pip import main as pip_execute |
225 | @@ -51,11 +54,15 @@ |
226 | pip_execute(command) |
227 | |
228 | |
229 | -def pip_install(package, fatal=False, upgrade=False, **options): |
230 | +def pip_install(package, fatal=False, upgrade=False, venv=None, **options): |
231 | """Install a python package""" |
232 | - command = ["install"] |
233 | + if venv: |
234 | + venv_python = os.path.join(venv, 'bin/pip') |
235 | + command = [venv_python, "install"] |
236 | + else: |
237 | + command = ["install"] |
238 | |
239 | - available_options = ('proxy', 'src', 'log', "index-url", ) |
240 | + available_options = ('proxy', 'src', 'log', 'index-url', ) |
241 | for option in parse_options(options, available_options): |
242 | command.append(option) |
243 | |
244 | @@ -69,7 +76,10 @@ |
245 | |
246 | log("Installing {} package with options: {}".format(package, |
247 | command)) |
248 | - pip_execute(command) |
249 | + if venv: |
250 | + subprocess.check_call(command) |
251 | + else: |
252 | + pip_execute(command) |
253 | |
254 | |
255 | def pip_uninstall(package, **options): |
256 | @@ -94,3 +104,16 @@ |
257 | """Returns the list of current python installed packages |
258 | """ |
259 | return pip_execute(["list"]) |
260 | + |
261 | + |
262 | +def pip_create_virtualenv(path=None): |
263 | + """Create an isolated Python environment.""" |
264 | + apt_install('python-virtualenv') |
265 | + |
266 | + if path: |
267 | + venv_path = path |
268 | + else: |
269 | + venv_path = os.path.join(charm_dir(), 'venv') |
270 | + |
271 | + if not os.path.exists(venv_path): |
272 | + subprocess.check_call(['virtualenv', venv_path]) |
273 | |
274 | === modified file 'hooks/charmhelpers/core/hookenv.py' |
275 | --- hooks/charmhelpers/core/hookenv.py 2015-05-21 15:02:07 +0000 |
276 | +++ hooks/charmhelpers/core/hookenv.py 2015-06-04 08:45:02 +0000 |
277 | @@ -364,11 +364,16 @@ |
278 | relation_settings = relation_settings if relation_settings else {} |
279 | relation_cmd_line = ['relation-set'] |
280 | accepts_file = "--file" in subprocess.check_output( |
281 | - relation_cmd_line + ["--help"]) |
282 | + relation_cmd_line + ["--help"], universal_newlines=True) |
283 | if relation_id is not None: |
284 | relation_cmd_line.extend(('-r', relation_id)) |
285 | settings = relation_settings.copy() |
286 | settings.update(kwargs) |
287 | + for key, value in settings.items(): |
288 | + # Force value to be a string: it always should, but some call |
289 | + # sites pass in things like dicts or numbers. |
290 | + if value is not None: |
291 | + settings[key] = "{}".format(value) |
292 | if accepts_file: |
293 | # --file was introduced in Juju 1.23.2. Use it by default if |
294 | # available, since otherwise we'll break if the relation data is |
295 | @@ -390,6 +395,17 @@ |
296 | flush(local_unit()) |
297 | |
298 | |
299 | +def relation_clear(r_id=None): |
300 | + ''' Clears any relation data already set on relation r_id ''' |
301 | + settings = relation_get(rid=r_id, |
302 | + unit=local_unit()) |
303 | + for setting in settings: |
304 | + if setting not in ['public-address', 'private-address']: |
305 | + settings[setting] = None |
306 | + relation_set(relation_id=r_id, |
307 | + **settings) |
308 | + |
309 | + |
310 | @cached |
311 | def relation_ids(reltype=None): |
312 | """A list of relation_ids""" |
313 | @@ -681,3 +697,48 @@ |
314 | return 'unknown' |
315 | else: |
316 | raise |
317 | + |
318 | + |
319 | +def translate_exc(from_exc, to_exc): |
320 | + def inner_translate_exc1(f): |
321 | + def inner_translate_exc2(*args, **kwargs): |
322 | + try: |
323 | + return f(*args, **kwargs) |
324 | + except from_exc: |
325 | + raise to_exc |
326 | + |
327 | + return inner_translate_exc2 |
328 | + |
329 | + return inner_translate_exc1 |
330 | + |
331 | + |
332 | +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) |
333 | +def is_leader(): |
334 | + """Does the current unit hold the juju leadership |
335 | + |
336 | + Uses juju to determine whether the current unit is the leader of its peers |
337 | + """ |
338 | + cmd = ['is-leader', '--format=json'] |
339 | + return json.loads(subprocess.check_output(cmd).decode('UTF-8')) |
340 | + |
341 | + |
342 | +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) |
343 | +def leader_get(attribute=None): |
344 | + """Juju leader get value(s)""" |
345 | + cmd = ['leader-get', '--format=json'] + [attribute or '-'] |
346 | + return json.loads(subprocess.check_output(cmd).decode('UTF-8')) |
347 | + |
348 | + |
349 | +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) |
350 | +def leader_set(settings=None, **kwargs): |
351 | + """Juju leader set value(s)""" |
352 | + log("Juju leader-set '%s'" % (settings), level=DEBUG) |
353 | + cmd = ['leader-set'] |
354 | + settings = settings or {} |
355 | + settings.update(kwargs) |
356 | + for k, v in settings.iteritems(): |
357 | + if v is None: |
358 | + cmd.append('{}='.format(k)) |
359 | + else: |
360 | + cmd.append('{}={}'.format(k, v)) |
361 | + subprocess.check_call(cmd) |
362 | |
363 | === modified file 'hooks/charmhelpers/core/services/base.py' |
364 | --- hooks/charmhelpers/core/services/base.py 2015-05-21 15:02:07 +0000 |
365 | +++ hooks/charmhelpers/core/services/base.py 2015-06-04 08:45:02 +0000 |
366 | @@ -15,8 +15,8 @@ |
367 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
368 | |
369 | import os |
370 | -import re |
371 | import json |
372 | +from inspect import getargspec |
373 | from collections import Iterable, OrderedDict |
374 | |
375 | from charmhelpers.core import host |
376 | @@ -132,8 +132,8 @@ |
377 | if hook_name == 'stop': |
378 | self.stop_services() |
379 | else: |
380 | + self.reconfigure_services() |
381 | self.provide_data() |
382 | - self.reconfigure_services() |
383 | cfg = hookenv.config() |
384 | if cfg.implicit_save: |
385 | cfg.save() |
386 | @@ -145,15 +145,36 @@ |
387 | A provider must have a `name` attribute, which indicates which relation |
388 | to set data on, and a `provide_data()` method, which returns a dict of |
389 | data to set. |
390 | + |
391 | + The `provide_data()` method can optionally accept two parameters: |
392 | + |
393 | + * ``remote_service`` The name of the remote service that the data will |
394 | + be provided to. The `provide_data()` method will be called once |
395 | + for each connected service (not unit). This allows the method to |
396 | + tailor its data to the given service. |
397 | + * ``service_ready`` Whether or not the service definition had all of |
398 | + its requirements met, and thus the ``data_ready`` callbacks run. |
399 | + |
400 | + Note that the ``provided_data`` methods are now called **after** the |
401 | + ``data_ready`` callbacks are run. This gives the ``data_ready`` callbacks |
402 | + a chance to generate any data necessary for the providing to the remote |
403 | + services. |
404 | """ |
405 | - hook_name = hookenv.hook_name() |
406 | - for service in self.services.values(): |
407 | + for service_name, service in self.services.items(): |
408 | + service_ready = self.is_ready(service_name) |
409 | for provider in service.get('provided_data', []): |
410 | - if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name): |
411 | - data = provider.provide_data() |
412 | - _ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data |
413 | - if _ready: |
414 | - hookenv.relation_set(None, data) |
415 | + for relid in hookenv.relation_ids(provider.name): |
416 | + units = hookenv.related_units(relid) |
417 | + if not units: |
418 | + continue |
419 | + remote_service = units[0].split('/')[0] |
420 | + argspec = getargspec(provider.provide_data) |
421 | + if len(argspec.args) > 1: |
422 | + data = provider.provide_data(remote_service, service_ready) |
423 | + else: |
424 | + data = provider.provide_data() |
425 | + if data: |
426 | + hookenv.relation_set(relid, data) |
427 | |
428 | def reconfigure_services(self, *service_names): |
429 | """ |
430 | |
431 | === modified file 'hooks/charmhelpers/fetch/giturl.py' |
432 | --- hooks/charmhelpers/fetch/giturl.py 2015-03-16 14:16:02 +0000 |
433 | +++ hooks/charmhelpers/fetch/giturl.py 2015-06-04 08:45:02 +0000 |
434 | @@ -45,14 +45,16 @@ |
435 | else: |
436 | return True |
437 | |
438 | - def clone(self, source, dest, branch): |
439 | + def clone(self, source, dest, branch, depth=None): |
440 | if not self.can_handle(source): |
441 | raise UnhandledSource("Cannot handle {}".format(source)) |
442 | |
443 | - repo = Repo.clone_from(source, dest) |
444 | - repo.git.checkout(branch) |
445 | + if depth: |
446 | + Repo.clone_from(source, dest, branch=branch, depth=depth) |
447 | + else: |
448 | + Repo.clone_from(source, dest, branch=branch) |
449 | |
450 | - def install(self, source, branch="master", dest=None): |
451 | + def install(self, source, branch="master", dest=None, depth=None): |
452 | url_parts = self.parse_url(source) |
453 | branch_name = url_parts.path.strip("/").split("/")[-1] |
454 | if dest: |
455 | @@ -63,7 +65,7 @@ |
456 | if not os.path.exists(dest_dir): |
457 | mkdir(dest_dir, perms=0o755) |
458 | try: |
459 | - self.clone(source, dest_dir, branch) |
460 | + self.clone(source, dest_dir, branch, depth) |
461 | except GitCommandError as e: |
462 | raise UnhandledSource(e.message) |
463 | except OSError as e: |