Merge lp:~billy-olsen/charms/trusty/keystone/public-endpoint-host into lp:~openstack-charmers-archive/charms/trusty/keystone/next
- Trusty Tahr (14.04)
- public-endpoint-host
- Merge into next
Proposed by
Billy Olsen
Status: | Merged |
---|---|
Merged at revision: | 154 |
Proposed branch: | lp:~billy-olsen/charms/trusty/keystone/public-endpoint-host |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/keystone/next |
Diff against target: |
1011 lines (+420/-110) 14 files modified
config.yaml (+12/-0) hooks/charmhelpers/contrib/hahelpers/cluster.py (+25/-0) hooks/charmhelpers/contrib/openstack/ip.py (+49/-44) hooks/charmhelpers/contrib/openstack/neutron.py (+10/-5) hooks/charmhelpers/contrib/openstack/utils.py (+65/-18) hooks/charmhelpers/contrib/peerstorage/__init__.py (+2/-0) hooks/charmhelpers/contrib/python/packages.py (+28/-5) hooks/charmhelpers/contrib/unison/__init__.py (+5/-4) hooks/charmhelpers/core/hookenv.py (+147/-10) hooks/charmhelpers/core/host.py (+1/-1) hooks/charmhelpers/core/services/base.py (+32/-11) hooks/charmhelpers/fetch/__init__.py (+1/-1) hooks/charmhelpers/fetch/giturl.py (+7/-5) unit_tests/test_keystone_utils.py (+36/-6) |
To merge this branch: | bzr merge lp:~billy-olsen/charms/trusty/keystone/public-endpoint-host |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Corey Bryant (community) | Approve | ||
Review via email: mp+261008@code.launchpad.net |
Commit message
Description of the change
Provides a config option which allows the user to specify the public hostname used to advertise to keystone when creating endpoints.
To post a comment you must log in.
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #4726 keystone-next for billy-olsen mp261008
UNIT OK: passed
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #4452 keystone-next for billy-olsen mp261008
AMULET OK: passed
Build: http://
- 153. By Billy Olsen
-
c-h sync
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #5070 keystone-next for billy-olsen mp261008
LINT OK: passed
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #4749 keystone-next for billy-olsen mp261008
UNIT OK: passed
Revision history for this message
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #4477 keystone-next for billy-olsen mp261008
AMULET OK: passed
Build: http://
Revision history for this message
Corey Bryant (corey.bryant) wrote : | # |
Looks good! One really minor comment below in config.yaml description.
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'config.yaml' |
2 | --- config.yaml 2015-04-01 15:15:49 +0000 |
3 | +++ config.yaml 2015-06-04 23:25:07 +0000 |
4 | @@ -235,6 +235,18 @@ |
5 | 192.168.0.0/24) |
6 | . |
7 | This network will be used for public endpoints. |
8 | + os-public-hostname: |
9 | + type: string |
10 | + default: |
11 | + description: | |
12 | + The hostname or address of the public endpoints created for keystone |
13 | + inside itself. |
14 | + . |
15 | + This value will be used for public endpoints. For example, an |
16 | + os-public-hostname set to 'keystone.example.com' with ssl enabled will |
17 | + create a public endpoint for keystone as: |
18 | + . |
19 | + https://keystone.example.com:5000/v2.0 |
20 | prefer-ipv6: |
21 | type: boolean |
22 | default: False |
23 | |
24 | === modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py' |
25 | --- hooks/charmhelpers/contrib/hahelpers/cluster.py 2015-03-18 18:59:03 +0000 |
26 | +++ hooks/charmhelpers/contrib/hahelpers/cluster.py 2015-06-04 23:25:07 +0000 |
27 | @@ -52,6 +52,8 @@ |
28 | bool_from_string, |
29 | ) |
30 | |
31 | +DC_RESOURCE_NAME = 'DC' |
32 | + |
33 | |
34 | class HAIncompleteConfig(Exception): |
35 | pass |
36 | @@ -95,6 +97,27 @@ |
37 | return False |
38 | |
39 | |
40 | +def is_crm_dc(): |
41 | + """ |
42 | + Determine leadership by querying the pacemaker Designated Controller |
43 | + """ |
44 | + cmd = ['crm', 'status'] |
45 | + try: |
46 | + status = subprocess.check_output(cmd, stderr=subprocess.STDOUT) |
47 | + if not isinstance(status, six.text_type): |
48 | + status = six.text_type(status, "utf-8") |
49 | + except subprocess.CalledProcessError: |
50 | + return False |
51 | + current_dc = '' |
52 | + for line in status.split('\n'): |
53 | + if line.startswith('Current DC'): |
54 | + # Current DC: juju-lytrusty-machine-2 (168108163) - partition with quorum |
55 | + current_dc = line.split(':')[1].split()[0] |
56 | + if current_dc == get_unit_hostname(): |
57 | + return True |
58 | + return False |
59 | + |
60 | + |
61 | @retry_on_exception(5, base_delay=2, exc_type=CRMResourceNotFound) |
62 | def is_crm_leader(resource, retry=False): |
63 | """ |
64 | @@ -104,6 +127,8 @@ |
65 | We allow this operation to be retried to avoid the possibility of getting a |
66 | false negative. See LP #1396246 for more info. |
67 | """ |
68 | + if resource == DC_RESOURCE_NAME: |
69 | + return is_crm_dc() |
70 | cmd = ['crm', 'resource', 'show', resource] |
71 | try: |
72 | status = subprocess.check_output(cmd, stderr=subprocess.STDOUT) |
73 | |
74 | === modified file 'hooks/charmhelpers/contrib/openstack/ip.py' |
75 | --- hooks/charmhelpers/contrib/openstack/ip.py 2015-03-11 11:45:09 +0000 |
76 | +++ hooks/charmhelpers/contrib/openstack/ip.py 2015-06-04 23:25:07 +0000 |
77 | @@ -17,6 +17,7 @@ |
78 | from charmhelpers.core.hookenv import ( |
79 | config, |
80 | unit_get, |
81 | + service_name, |
82 | ) |
83 | from charmhelpers.contrib.network.ip import ( |
84 | get_address_in_network, |
85 | @@ -26,8 +27,6 @@ |
86 | ) |
87 | from charmhelpers.contrib.hahelpers.cluster import is_clustered |
88 | |
89 | -from functools import partial |
90 | - |
91 | PUBLIC = 'public' |
92 | INTERNAL = 'int' |
93 | ADMIN = 'admin' |
94 | @@ -35,15 +34,18 @@ |
95 | ADDRESS_MAP = { |
96 | PUBLIC: { |
97 | 'config': 'os-public-network', |
98 | - 'fallback': 'public-address' |
99 | + 'fallback': 'public-address', |
100 | + 'override': 'os-public-hostname', |
101 | }, |
102 | INTERNAL: { |
103 | 'config': 'os-internal-network', |
104 | - 'fallback': 'private-address' |
105 | + 'fallback': 'private-address', |
106 | + 'override': 'os-internal-hostname', |
107 | }, |
108 | ADMIN: { |
109 | 'config': 'os-admin-network', |
110 | - 'fallback': 'private-address' |
111 | + 'fallback': 'private-address', |
112 | + 'override': 'os-admin-hostname', |
113 | } |
114 | } |
115 | |
116 | @@ -57,15 +59,50 @@ |
117 | :param endpoint_type: str endpoint type to resolve. |
118 | :param returns: str base URL for services on the current service unit. |
119 | """ |
120 | - scheme = 'http' |
121 | - if 'https' in configs.complete_contexts(): |
122 | - scheme = 'https' |
123 | + scheme = _get_scheme(configs) |
124 | + |
125 | address = resolve_address(endpoint_type) |
126 | if is_ipv6(address): |
127 | address = "[{}]".format(address) |
128 | + |
129 | return '%s://%s' % (scheme, address) |
130 | |
131 | |
132 | +def _get_scheme(configs): |
133 | + """Returns the scheme to use for the url (either http or https) |
134 | + depending upon whether https is in the configs value. |
135 | + |
136 | + :param configs: OSTemplateRenderer config templating object to inspect |
137 | + for a complete https context. |
138 | + :returns: either 'http' or 'https' depending on whether https is |
139 | + configured within the configs context. |
140 | + """ |
141 | + scheme = 'http' |
142 | + if configs and 'https' in configs.complete_contexts(): |
143 | + scheme = 'https' |
144 | + return scheme |
145 | + |
146 | + |
147 | +def _get_address_override(endpoint_type=PUBLIC): |
148 | + """Returns any address overrides that the user has defined based on the |
149 | + endpoint type. |
150 | + |
151 | + Note: this function allows for the service name to be inserted into the |
152 | + address if the user specifies {service_name}.somehost.org. |
153 | + |
154 | + :param endpoint_type: the type of endpoint to retrieve the override |
155 | + value for. |
156 | + :returns: any endpoint address or hostname that the user has overridden |
157 | + or None if an override is not present. |
158 | + """ |
159 | + override_key = ADDRESS_MAP[endpoint_type]['override'] |
160 | + addr_override = config(override_key) |
161 | + if not addr_override: |
162 | + return None |
163 | + else: |
164 | + return addr_override.format(service_name=service_name()) |
165 | + |
166 | + |
167 | def resolve_address(endpoint_type=PUBLIC): |
168 | """Return unit address depending on net config. |
169 | |
170 | @@ -77,7 +114,10 @@ |
171 | |
172 | :param endpoint_type: Network endpoing type |
173 | """ |
174 | - resolved_address = None |
175 | + resolved_address = _get_address_override(endpoint_type) |
176 | + if resolved_address: |
177 | + return resolved_address |
178 | + |
179 | vips = config('vip') |
180 | if vips: |
181 | vips = vips.split() |
182 | @@ -109,38 +149,3 @@ |
183 | "clustered=%s)" % (net_type, clustered)) |
184 | |
185 | return resolved_address |
186 | - |
187 | - |
188 | -def endpoint_url(configs, url_template, port, endpoint_type=PUBLIC, |
189 | - override=None): |
190 | - """Returns the correct endpoint URL to advertise to Keystone. |
191 | - |
192 | - This method provides the correct endpoint URL which should be advertised to |
193 | - the keystone charm for endpoint creation. This method allows for the url to |
194 | - be overridden to force a keystone endpoint to have specific URL for any of |
195 | - the defined scopes (admin, internal, public). |
196 | - |
197 | - :param configs: OSTemplateRenderer config templating object to inspect |
198 | - for a complete https context. |
199 | - :param url_template: str format string for creating the url template. Only |
200 | - two values will be passed - the scheme+hostname |
201 | - returned by the canonical_url and the port. |
202 | - :param endpoint_type: str endpoint type to resolve. |
203 | - :param override: str the name of the config option which overrides the |
204 | - endpoint URL defined by the charm itself. None will |
205 | - disable any overrides (default). |
206 | - """ |
207 | - if override: |
208 | - # Return any user-defined overrides for the keystone endpoint URL. |
209 | - user_value = config(override) |
210 | - if user_value: |
211 | - return user_value.strip() |
212 | - |
213 | - return url_template % (canonical_url(configs, endpoint_type), port) |
214 | - |
215 | - |
216 | -public_endpoint = partial(endpoint_url, endpoint_type=PUBLIC) |
217 | - |
218 | -internal_endpoint = partial(endpoint_url, endpoint_type=INTERNAL) |
219 | - |
220 | -admin_endpoint = partial(endpoint_url, endpoint_type=ADMIN) |
221 | |
222 | === modified file 'hooks/charmhelpers/contrib/openstack/neutron.py' |
223 | --- hooks/charmhelpers/contrib/openstack/neutron.py 2015-04-16 19:55:16 +0000 |
224 | +++ hooks/charmhelpers/contrib/openstack/neutron.py 2015-06-04 23:25:07 +0000 |
225 | @@ -256,11 +256,14 @@ |
226 | def parse_mappings(mappings): |
227 | parsed = {} |
228 | if mappings: |
229 | - mappings = mappings.split(' ') |
230 | + mappings = mappings.split() |
231 | for m in mappings: |
232 | p = m.partition(':') |
233 | - if p[1] == ':': |
234 | - parsed[p[0].strip()] = p[2].strip() |
235 | + key = p[0].strip() |
236 | + if p[1]: |
237 | + parsed[key] = p[2].strip() |
238 | + else: |
239 | + parsed[key] = '' |
240 | |
241 | return parsed |
242 | |
243 | @@ -283,13 +286,13 @@ |
244 | Returns dict of the form {bridge:port}. |
245 | """ |
246 | _mappings = parse_mappings(mappings) |
247 | - if not _mappings: |
248 | + if not _mappings or list(_mappings.values()) == ['']: |
249 | if not mappings: |
250 | return {} |
251 | |
252 | # For backwards-compatibility we need to support port-only provided in |
253 | # config. |
254 | - _mappings = {default_bridge: mappings.split(' ')[0]} |
255 | + _mappings = {default_bridge: mappings.split()[0]} |
256 | |
257 | bridges = _mappings.keys() |
258 | ports = _mappings.values() |
259 | @@ -309,6 +312,8 @@ |
260 | |
261 | Mappings must be a space-delimited list of provider:start:end mappings. |
262 | |
263 | + The start:end range is optional and may be omitted. |
264 | + |
265 | Returns dict of the form {provider: (start, end)}. |
266 | """ |
267 | _mappings = parse_mappings(mappings) |
268 | |
269 | === modified file 'hooks/charmhelpers/contrib/openstack/utils.py' |
270 | --- hooks/charmhelpers/contrib/openstack/utils.py 2015-04-16 14:09:47 +0000 |
271 | +++ hooks/charmhelpers/contrib/openstack/utils.py 2015-06-04 23:25:07 +0000 |
272 | @@ -53,9 +53,13 @@ |
273 | get_ipv6_addr |
274 | ) |
275 | |
276 | +from charmhelpers.contrib.python.packages import ( |
277 | + pip_create_virtualenv, |
278 | + pip_install, |
279 | +) |
280 | + |
281 | from charmhelpers.core.host import lsb_release, mounts, umount |
282 | from charmhelpers.fetch import apt_install, apt_cache, install_remote |
283 | -from charmhelpers.contrib.python.packages import pip_install |
284 | from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk |
285 | from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device |
286 | |
287 | @@ -497,7 +501,17 @@ |
288 | requirements_dir = None |
289 | |
290 | |
291 | -def git_clone_and_install(projects_yaml, core_project): |
292 | +def _git_yaml_load(projects_yaml): |
293 | + """ |
294 | + Load the specified yaml into a dictionary. |
295 | + """ |
296 | + if not projects_yaml: |
297 | + return None |
298 | + |
299 | + return yaml.load(projects_yaml) |
300 | + |
301 | + |
302 | +def git_clone_and_install(projects_yaml, core_project, depth=1): |
303 | """ |
304 | Clone/install all specified OpenStack repositories. |
305 | |
306 | @@ -510,23 +524,22 @@ |
307 | repository: 'git://git.openstack.org/openstack/requirements.git', |
308 | branch: 'stable/icehouse'} |
309 | directory: /mnt/openstack-git |
310 | - http_proxy: http://squid.internal:3128 |
311 | - https_proxy: https://squid.internal:3128 |
312 | + http_proxy: squid-proxy-url |
313 | + https_proxy: squid-proxy-url |
314 | |
315 | The directory, http_proxy, and https_proxy keys are optional. |
316 | """ |
317 | global requirements_dir |
318 | parent_dir = '/mnt/openstack-git' |
319 | - |
320 | - if not projects_yaml: |
321 | - return |
322 | - |
323 | - projects = yaml.load(projects_yaml) |
324 | + http_proxy = None |
325 | + |
326 | + projects = _git_yaml_load(projects_yaml) |
327 | _git_validate_projects_yaml(projects, core_project) |
328 | |
329 | old_environ = dict(os.environ) |
330 | |
331 | if 'http_proxy' in projects.keys(): |
332 | + http_proxy = projects['http_proxy'] |
333 | os.environ['http_proxy'] = projects['http_proxy'] |
334 | if 'https_proxy' in projects.keys(): |
335 | os.environ['https_proxy'] = projects['https_proxy'] |
336 | @@ -534,15 +547,19 @@ |
337 | if 'directory' in projects.keys(): |
338 | parent_dir = projects['directory'] |
339 | |
340 | + pip_create_virtualenv(os.path.join(parent_dir, 'venv')) |
341 | + |
342 | for p in projects['repositories']: |
343 | repo = p['repository'] |
344 | branch = p['branch'] |
345 | if p['name'] == 'requirements': |
346 | - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, |
347 | + repo_dir = _git_clone_and_install_single(repo, branch, depth, |
348 | + parent_dir, http_proxy, |
349 | update_requirements=False) |
350 | requirements_dir = repo_dir |
351 | else: |
352 | - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, |
353 | + repo_dir = _git_clone_and_install_single(repo, branch, depth, |
354 | + parent_dir, http_proxy, |
355 | update_requirements=True) |
356 | |
357 | os.environ = old_environ |
358 | @@ -574,7 +591,8 @@ |
359 | error_out('openstack-origin-git key \'{}\' is missing'.format(key)) |
360 | |
361 | |
362 | -def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): |
363 | +def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, |
364 | + update_requirements): |
365 | """ |
366 | Clone and install a single git repository. |
367 | """ |
368 | @@ -587,7 +605,8 @@ |
369 | |
370 | if not os.path.exists(dest_dir): |
371 | juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) |
372 | - repo_dir = install_remote(repo, dest=parent_dir, branch=branch) |
373 | + repo_dir = install_remote(repo, dest=parent_dir, branch=branch, |
374 | + depth=depth) |
375 | else: |
376 | repo_dir = dest_dir |
377 | |
378 | @@ -598,7 +617,12 @@ |
379 | _git_update_requirements(repo_dir, requirements_dir) |
380 | |
381 | juju_log('Installing git repo from dir: {}'.format(repo_dir)) |
382 | - pip_install(repo_dir) |
383 | + if http_proxy: |
384 | + pip_install(repo_dir, proxy=http_proxy, |
385 | + venv=os.path.join(parent_dir, 'venv')) |
386 | + else: |
387 | + pip_install(repo_dir, |
388 | + venv=os.path.join(parent_dir, 'venv')) |
389 | |
390 | return repo_dir |
391 | |
392 | @@ -621,16 +645,27 @@ |
393 | os.chdir(orig_dir) |
394 | |
395 | |
396 | +def git_pip_venv_dir(projects_yaml): |
397 | + """ |
398 | + Return the pip virtualenv path. |
399 | + """ |
400 | + parent_dir = '/mnt/openstack-git' |
401 | + |
402 | + projects = _git_yaml_load(projects_yaml) |
403 | + |
404 | + if 'directory' in projects.keys(): |
405 | + parent_dir = projects['directory'] |
406 | + |
407 | + return os.path.join(parent_dir, 'venv') |
408 | + |
409 | + |
410 | def git_src_dir(projects_yaml, project): |
411 | """ |
412 | Return the directory where the specified project's source is located. |
413 | """ |
414 | parent_dir = '/mnt/openstack-git' |
415 | |
416 | - if not projects_yaml: |
417 | - return |
418 | - |
419 | - projects = yaml.load(projects_yaml) |
420 | + projects = _git_yaml_load(projects_yaml) |
421 | |
422 | if 'directory' in projects.keys(): |
423 | parent_dir = projects['directory'] |
424 | @@ -640,3 +675,15 @@ |
425 | return os.path.join(parent_dir, os.path.basename(p['repository'])) |
426 | |
427 | return None |
428 | + |
429 | + |
430 | +def git_yaml_value(projects_yaml, key): |
431 | + """ |
432 | + Return the value in projects_yaml for the specified key. |
433 | + """ |
434 | + projects = _git_yaml_load(projects_yaml) |
435 | + |
436 | + if key in projects.keys(): |
437 | + return projects[key] |
438 | + |
439 | + return None |
440 | |
441 | === modified file 'hooks/charmhelpers/contrib/peerstorage/__init__.py' |
442 | --- hooks/charmhelpers/contrib/peerstorage/__init__.py 2015-03-11 11:45:09 +0000 |
443 | +++ hooks/charmhelpers/contrib/peerstorage/__init__.py 2015-06-04 23:25:07 +0000 |
444 | @@ -73,6 +73,8 @@ |
445 | exc_list = exc_list if exc_list else [] |
446 | peerdb_settings = peer_retrieve('-', relation_name=relation_name) |
447 | matched = {} |
448 | + if peerdb_settings is None: |
449 | + return matched |
450 | for k, v in peerdb_settings.items(): |
451 | full_prefix = prefix + delimiter |
452 | if k.startswith(full_prefix): |
453 | |
454 | === modified file 'hooks/charmhelpers/contrib/python/packages.py' |
455 | --- hooks/charmhelpers/contrib/python/packages.py 2015-03-11 11:45:09 +0000 |
456 | +++ hooks/charmhelpers/contrib/python/packages.py 2015-06-04 23:25:07 +0000 |
457 | @@ -17,8 +17,11 @@ |
458 | # You should have received a copy of the GNU Lesser General Public License |
459 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
460 | |
461 | +import os |
462 | +import subprocess |
463 | + |
464 | from charmhelpers.fetch import apt_install, apt_update |
465 | -from charmhelpers.core.hookenv import log |
466 | +from charmhelpers.core.hookenv import charm_dir, log |
467 | |
468 | try: |
469 | from pip import main as pip_execute |
470 | @@ -51,11 +54,15 @@ |
471 | pip_execute(command) |
472 | |
473 | |
474 | -def pip_install(package, fatal=False, upgrade=False, **options): |
475 | +def pip_install(package, fatal=False, upgrade=False, venv=None, **options): |
476 | """Install a python package""" |
477 | - command = ["install"] |
478 | + if venv: |
479 | + venv_python = os.path.join(venv, 'bin/pip') |
480 | + command = [venv_python, "install"] |
481 | + else: |
482 | + command = ["install"] |
483 | |
484 | - available_options = ('proxy', 'src', 'log', "index-url", ) |
485 | + available_options = ('proxy', 'src', 'log', 'index-url', ) |
486 | for option in parse_options(options, available_options): |
487 | command.append(option) |
488 | |
489 | @@ -69,7 +76,10 @@ |
490 | |
491 | log("Installing {} package with options: {}".format(package, |
492 | command)) |
493 | - pip_execute(command) |
494 | + if venv: |
495 | + subprocess.check_call(command) |
496 | + else: |
497 | + pip_execute(command) |
498 | |
499 | |
500 | def pip_uninstall(package, **options): |
501 | @@ -94,3 +104,16 @@ |
502 | """Returns the list of current python installed packages |
503 | """ |
504 | return pip_execute(["list"]) |
505 | + |
506 | + |
507 | +def pip_create_virtualenv(path=None): |
508 | + """Create an isolated Python environment.""" |
509 | + apt_install('python-virtualenv') |
510 | + |
511 | + if path: |
512 | + venv_path = path |
513 | + else: |
514 | + venv_path = os.path.join(charm_dir(), 'venv') |
515 | + |
516 | + if not os.path.exists(venv_path): |
517 | + subprocess.check_call(['virtualenv', venv_path]) |
518 | |
519 | === modified file 'hooks/charmhelpers/contrib/unison/__init__.py' |
520 | --- hooks/charmhelpers/contrib/unison/__init__.py 2015-03-18 18:59:03 +0000 |
521 | +++ hooks/charmhelpers/contrib/unison/__init__.py 2015-06-04 23:25:07 +0000 |
522 | @@ -63,6 +63,7 @@ |
523 | from charmhelpers.core.host import ( |
524 | adduser, |
525 | add_user_to_group, |
526 | + pwgen, |
527 | ) |
528 | |
529 | from charmhelpers.core.hookenv import ( |
530 | @@ -140,7 +141,7 @@ |
531 | ssh_dir = os.path.join(home_dir, '.ssh') |
532 | auth_keys = os.path.join(ssh_dir, 'authorized_keys') |
533 | log('Syncing authorized_keys @ %s.' % auth_keys) |
534 | - with open(auth_keys, 'wb') as out: |
535 | + with open(auth_keys, 'w') as out: |
536 | for k in keys: |
537 | out.write('%s\n' % k) |
538 | |
539 | @@ -152,16 +153,16 @@ |
540 | khosts = [] |
541 | for host in hosts: |
542 | cmd = ['ssh-keyscan', '-H', '-t', 'rsa', host] |
543 | - remote_key = check_output(cmd).strip() |
544 | + remote_key = check_output(cmd, universal_newlines=True).strip() |
545 | khosts.append(remote_key) |
546 | log('Syncing known_hosts @ %s.' % known_hosts) |
547 | - with open(known_hosts, 'wb') as out: |
548 | + with open(known_hosts, 'w') as out: |
549 | for host in khosts: |
550 | out.write('%s\n' % host) |
551 | |
552 | |
553 | def ensure_user(user, group=None): |
554 | - adduser(user) |
555 | + adduser(user, pwgen()) |
556 | if group: |
557 | add_user_to_group(user, group) |
558 | |
559 | |
560 | === modified file 'hooks/charmhelpers/core/hookenv.py' |
561 | --- hooks/charmhelpers/core/hookenv.py 2015-04-15 15:21:50 +0000 |
562 | +++ hooks/charmhelpers/core/hookenv.py 2015-06-04 23:25:07 +0000 |
563 | @@ -21,12 +21,14 @@ |
564 | # Charm Helpers Developers <juju@lists.ubuntu.com> |
565 | |
566 | from __future__ import print_function |
567 | +from functools import wraps |
568 | import os |
569 | import json |
570 | import yaml |
571 | import subprocess |
572 | import sys |
573 | import errno |
574 | +import tempfile |
575 | from subprocess import CalledProcessError |
576 | |
577 | import six |
578 | @@ -58,15 +60,17 @@ |
579 | |
580 | will cache the result of unit_get + 'test' for future calls. |
581 | """ |
582 | + @wraps(func) |
583 | def wrapper(*args, **kwargs): |
584 | global cache |
585 | key = str((func, args, kwargs)) |
586 | try: |
587 | return cache[key] |
588 | except KeyError: |
589 | - res = func(*args, **kwargs) |
590 | - cache[key] = res |
591 | - return res |
592 | + pass # Drop out of the exception handler scope. |
593 | + res = func(*args, **kwargs) |
594 | + cache[key] = res |
595 | + return res |
596 | return wrapper |
597 | |
598 | |
599 | @@ -178,7 +182,7 @@ |
600 | |
601 | def remote_unit(): |
602 | """The remote unit for the current relation hook""" |
603 | - return os.environ['JUJU_REMOTE_UNIT'] |
604 | + return os.environ.get('JUJU_REMOTE_UNIT', None) |
605 | |
606 | |
607 | def service_name(): |
608 | @@ -250,6 +254,12 @@ |
609 | except KeyError: |
610 | return (self._prev_dict or {})[key] |
611 | |
612 | + def get(self, key, default=None): |
613 | + try: |
614 | + return self[key] |
615 | + except KeyError: |
616 | + return default |
617 | + |
618 | def keys(self): |
619 | prev_keys = [] |
620 | if self._prev_dict is not None: |
621 | @@ -353,18 +363,49 @@ |
622 | """Set relation information for the current unit""" |
623 | relation_settings = relation_settings if relation_settings else {} |
624 | relation_cmd_line = ['relation-set'] |
625 | + accepts_file = "--file" in subprocess.check_output( |
626 | + relation_cmd_line + ["--help"], universal_newlines=True) |
627 | if relation_id is not None: |
628 | relation_cmd_line.extend(('-r', relation_id)) |
629 | - for k, v in (list(relation_settings.items()) + list(kwargs.items())): |
630 | - if v is None: |
631 | - relation_cmd_line.append('{}='.format(k)) |
632 | - else: |
633 | - relation_cmd_line.append('{}={}'.format(k, v)) |
634 | - subprocess.check_call(relation_cmd_line) |
635 | + settings = relation_settings.copy() |
636 | + settings.update(kwargs) |
637 | + for key, value in settings.items(): |
638 | + # Force value to be a string: it always should, but some call |
639 | + # sites pass in things like dicts or numbers. |
640 | + if value is not None: |
641 | + settings[key] = "{}".format(value) |
642 | + if accepts_file: |
643 | + # --file was introduced in Juju 1.23.2. Use it by default if |
644 | + # available, since otherwise we'll break if the relation data is |
645 | + # too big. Ideally we should tell relation-set to read the data from |
646 | + # stdin, but that feature is broken in 1.23.2: Bug #1454678. |
647 | + with tempfile.NamedTemporaryFile(delete=False) as settings_file: |
648 | + settings_file.write(yaml.safe_dump(settings).encode("utf-8")) |
649 | + subprocess.check_call( |
650 | + relation_cmd_line + ["--file", settings_file.name]) |
651 | + os.remove(settings_file.name) |
652 | + else: |
653 | + for key, value in settings.items(): |
654 | + if value is None: |
655 | + relation_cmd_line.append('{}='.format(key)) |
656 | + else: |
657 | + relation_cmd_line.append('{}={}'.format(key, value)) |
658 | + subprocess.check_call(relation_cmd_line) |
659 | # Flush cache of any relation-gets for local unit |
660 | flush(local_unit()) |
661 | |
662 | |
663 | +def relation_clear(r_id=None): |
664 | + ''' Clears any relation data already set on relation r_id ''' |
665 | + settings = relation_get(rid=r_id, |
666 | + unit=local_unit()) |
667 | + for setting in settings: |
668 | + if setting not in ['public-address', 'private-address']: |
669 | + settings[setting] = None |
670 | + relation_set(relation_id=r_id, |
671 | + **settings) |
672 | + |
673 | + |
674 | @cached |
675 | def relation_ids(reltype=None): |
676 | """A list of relation_ids""" |
677 | @@ -509,6 +550,11 @@ |
678 | return None |
679 | |
680 | |
681 | +def unit_public_ip(): |
682 | + """Get this unit's public IP address""" |
683 | + return unit_get('public-address') |
684 | + |
685 | + |
686 | def unit_private_ip(): |
687 | """Get this unit's private IP address""" |
688 | return unit_get('private-address') |
689 | @@ -605,3 +651,94 @@ |
690 | |
691 | The results set by action_set are preserved.""" |
692 | subprocess.check_call(['action-fail', message]) |
693 | + |
694 | + |
695 | +def status_set(workload_state, message): |
696 | + """Set the workload state with a message |
697 | + |
698 | + Use status-set to set the workload state with a message which is visible |
699 | + to the user via juju status. If the status-set command is not found then |
700 | + assume this is juju < 1.23 and juju-log the message unstead. |
701 | + |
702 | + workload_state -- valid juju workload state. |
703 | + message -- status update message |
704 | + """ |
705 | + valid_states = ['maintenance', 'blocked', 'waiting', 'active'] |
706 | + if workload_state not in valid_states: |
707 | + raise ValueError( |
708 | + '{!r} is not a valid workload state'.format(workload_state) |
709 | + ) |
710 | + cmd = ['status-set', workload_state, message] |
711 | + try: |
712 | + ret = subprocess.call(cmd) |
713 | + if ret == 0: |
714 | + return |
715 | + except OSError as e: |
716 | + if e.errno != errno.ENOENT: |
717 | + raise |
718 | + log_message = 'status-set failed: {} {}'.format(workload_state, |
719 | + message) |
720 | + log(log_message, level='INFO') |
721 | + |
722 | + |
723 | +def status_get(): |
724 | + """Retrieve the previously set juju workload state |
725 | + |
726 | + If the status-set command is not found then assume this is juju < 1.23 and |
727 | + return 'unknown' |
728 | + """ |
729 | + cmd = ['status-get'] |
730 | + try: |
731 | + raw_status = subprocess.check_output(cmd, universal_newlines=True) |
732 | + status = raw_status.rstrip() |
733 | + return status |
734 | + except OSError as e: |
735 | + if e.errno == errno.ENOENT: |
736 | + return 'unknown' |
737 | + else: |
738 | + raise |
739 | + |
740 | + |
741 | +def translate_exc(from_exc, to_exc): |
742 | + def inner_translate_exc1(f): |
743 | + def inner_translate_exc2(*args, **kwargs): |
744 | + try: |
745 | + return f(*args, **kwargs) |
746 | + except from_exc: |
747 | + raise to_exc |
748 | + |
749 | + return inner_translate_exc2 |
750 | + |
751 | + return inner_translate_exc1 |
752 | + |
753 | + |
754 | +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) |
755 | +def is_leader(): |
756 | + """Does the current unit hold the juju leadership |
757 | + |
758 | + Uses juju to determine whether the current unit is the leader of its peers |
759 | + """ |
760 | + cmd = ['is-leader', '--format=json'] |
761 | + return json.loads(subprocess.check_output(cmd).decode('UTF-8')) |
762 | + |
763 | + |
764 | +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) |
765 | +def leader_get(attribute=None): |
766 | + """Juju leader get value(s)""" |
767 | + cmd = ['leader-get', '--format=json'] + [attribute or '-'] |
768 | + return json.loads(subprocess.check_output(cmd).decode('UTF-8')) |
769 | + |
770 | + |
771 | +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) |
772 | +def leader_set(settings=None, **kwargs): |
773 | + """Juju leader set value(s)""" |
774 | + log("Juju leader-set '%s'" % (settings), level=DEBUG) |
775 | + cmd = ['leader-set'] |
776 | + settings = settings or {} |
777 | + settings.update(kwargs) |
778 | + for k, v in settings.iteritems(): |
779 | + if v is None: |
780 | + cmd.append('{}='.format(k)) |
781 | + else: |
782 | + cmd.append('{}={}'.format(k, v)) |
783 | + subprocess.check_call(cmd) |
784 | |
785 | === modified file 'hooks/charmhelpers/core/host.py' |
786 | --- hooks/charmhelpers/core/host.py 2015-03-30 11:43:06 +0000 |
787 | +++ hooks/charmhelpers/core/host.py 2015-06-04 23:25:07 +0000 |
788 | @@ -90,7 +90,7 @@ |
789 | ['service', service_name, 'status'], |
790 | stderr=subprocess.STDOUT).decode('UTF-8') |
791 | except subprocess.CalledProcessError as e: |
792 | - return 'unrecognized service' not in e.output |
793 | + return b'unrecognized service' not in e.output |
794 | else: |
795 | return True |
796 | |
797 | |
798 | === modified file 'hooks/charmhelpers/core/services/base.py' |
799 | --- hooks/charmhelpers/core/services/base.py 2015-03-11 11:45:09 +0000 |
800 | +++ hooks/charmhelpers/core/services/base.py 2015-06-04 23:25:07 +0000 |
801 | @@ -15,9 +15,9 @@ |
802 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
803 | |
804 | import os |
805 | -import re |
806 | import json |
807 | -from collections import Iterable |
808 | +from inspect import getargspec |
809 | +from collections import Iterable, OrderedDict |
810 | |
811 | from charmhelpers.core import host |
812 | from charmhelpers.core import hookenv |
813 | @@ -119,7 +119,7 @@ |
814 | """ |
815 | self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json') |
816 | self._ready = None |
817 | - self.services = {} |
818 | + self.services = OrderedDict() |
819 | for service in services or []: |
820 | service_name = service['service'] |
821 | self.services[service_name] = service |
822 | @@ -132,8 +132,8 @@ |
823 | if hook_name == 'stop': |
824 | self.stop_services() |
825 | else: |
826 | + self.reconfigure_services() |
827 | self.provide_data() |
828 | - self.reconfigure_services() |
829 | cfg = hookenv.config() |
830 | if cfg.implicit_save: |
831 | cfg.save() |
832 | @@ -145,15 +145,36 @@ |
833 | A provider must have a `name` attribute, which indicates which relation |
834 | to set data on, and a `provide_data()` method, which returns a dict of |
835 | data to set. |
836 | + |
837 | + The `provide_data()` method can optionally accept two parameters: |
838 | + |
839 | + * ``remote_service`` The name of the remote service that the data will |
840 | + be provided to. The `provide_data()` method will be called once |
841 | + for each connected service (not unit). This allows the method to |
842 | + tailor its data to the given service. |
843 | + * ``service_ready`` Whether or not the service definition had all of |
844 | + its requirements met, and thus the ``data_ready`` callbacks run. |
845 | + |
846 | + Note that the ``provided_data`` methods are now called **after** the |
847 | + ``data_ready`` callbacks are run. This gives the ``data_ready`` callbacks |
848 | + a chance to generate any data necessary for the providing to the remote |
849 | + services. |
850 | """ |
851 | - hook_name = hookenv.hook_name() |
852 | - for service in self.services.values(): |
853 | + for service_name, service in self.services.items(): |
854 | + service_ready = self.is_ready(service_name) |
855 | for provider in service.get('provided_data', []): |
856 | - if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name): |
857 | - data = provider.provide_data() |
858 | - _ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data |
859 | - if _ready: |
860 | - hookenv.relation_set(None, data) |
861 | + for relid in hookenv.relation_ids(provider.name): |
862 | + units = hookenv.related_units(relid) |
863 | + if not units: |
864 | + continue |
865 | + remote_service = units[0].split('/')[0] |
866 | + argspec = getargspec(provider.provide_data) |
867 | + if len(argspec.args) > 1: |
868 | + data = provider.provide_data(remote_service, service_ready) |
869 | + else: |
870 | + data = provider.provide_data() |
871 | + if data: |
872 | + hookenv.relation_set(relid, data) |
873 | |
874 | def reconfigure_services(self, *service_names): |
875 | """ |
876 | |
877 | === modified file 'hooks/charmhelpers/fetch/__init__.py' |
878 | --- hooks/charmhelpers/fetch/__init__.py 2015-03-11 11:45:09 +0000 |
879 | +++ hooks/charmhelpers/fetch/__init__.py 2015-06-04 23:25:07 +0000 |
880 | @@ -158,7 +158,7 @@ |
881 | |
882 | def apt_cache(in_memory=True): |
883 | """Build and return an apt cache""" |
884 | - import apt_pkg |
885 | + from apt import apt_pkg |
886 | apt_pkg.init() |
887 | if in_memory: |
888 | apt_pkg.config.set("Dir::Cache::pkgcache", "") |
889 | |
890 | === modified file 'hooks/charmhelpers/fetch/giturl.py' |
891 | --- hooks/charmhelpers/fetch/giturl.py 2015-03-11 11:45:09 +0000 |
892 | +++ hooks/charmhelpers/fetch/giturl.py 2015-06-04 23:25:07 +0000 |
893 | @@ -45,14 +45,16 @@ |
894 | else: |
895 | return True |
896 | |
897 | - def clone(self, source, dest, branch): |
898 | + def clone(self, source, dest, branch, depth=None): |
899 | if not self.can_handle(source): |
900 | raise UnhandledSource("Cannot handle {}".format(source)) |
901 | |
902 | - repo = Repo.clone_from(source, dest) |
903 | - repo.git.checkout(branch) |
904 | + if depth: |
905 | + Repo.clone_from(source, dest, branch=branch, depth=depth) |
906 | + else: |
907 | + Repo.clone_from(source, dest, branch=branch) |
908 | |
909 | - def install(self, source, branch="master", dest=None): |
910 | + def install(self, source, branch="master", dest=None, depth=None): |
911 | url_parts = self.parse_url(source) |
912 | branch_name = url_parts.path.strip("/").split("/")[-1] |
913 | if dest: |
914 | @@ -63,7 +65,7 @@ |
915 | if not os.path.exists(dest_dir): |
916 | mkdir(dest_dir, perms=0o755) |
917 | try: |
918 | - self.clone(source, dest_dir, branch) |
919 | + self.clone(source, dest_dir, branch, depth) |
920 | except GitCommandError as e: |
921 | raise UnhandledSource(e.message) |
922 | except OSError as e: |
923 | |
924 | === modified file 'unit_tests/test_keystone_utils.py' |
925 | --- unit_tests/test_keystone_utils.py 2015-04-17 12:10:54 +0000 |
926 | +++ unit_tests/test_keystone_utils.py 2015-06-04 23:25:07 +0000 |
927 | @@ -49,8 +49,6 @@ |
928 | 'subprocess', |
929 | 'time', |
930 | 'pwgen', |
931 | - # openstack.ip |
932 | - 'resolve_address', |
933 | ] |
934 | |
935 | openstack_origin_git = \ |
936 | @@ -173,12 +171,13 @@ |
937 | self.subprocess.check_output.assert_called_with(cmd) |
938 | self.service_start.assert_called_wkth('keystone') |
939 | |
940 | + @patch.object(utils, 'resolve_address') |
941 | @patch.object(utils, 'b64encode') |
942 | def test_add_service_to_keystone_clustered_https_none_values( |
943 | - self, b64encode): |
944 | + self, b64encode, _resolve_address): |
945 | relation_id = 'identity-service:0' |
946 | remote_unit = 'unit/0' |
947 | - self.resolve_address.return_value = '10.10.10.10' |
948 | + _resolve_address.return_value = '10.10.10.10' |
949 | self.https.return_value = True |
950 | self.test_config.set('https-service-endpoints', 'True') |
951 | self.test_config.set('vip', '10.10.10.10') |
952 | @@ -211,11 +210,13 @@ |
953 | self.peer_store_and_set.assert_called_with(relation_id=relation_id, |
954 | **relation_data) |
955 | |
956 | + @patch.object(utils, 'resolve_address') |
957 | @patch.object(utils, 'ensure_valid_service') |
958 | @patch.object(utils, 'add_endpoint') |
959 | @patch.object(manager, 'KeystoneManager') |
960 | def test_add_service_to_keystone_no_clustered_no_https_complete_values( |
961 | - self, KeystoneManager, add_endpoint, ensure_valid_service): |
962 | + self, KeystoneManager, add_endpoint, ensure_valid_service, |
963 | + _resolve_address): |
964 | relation_id = 'identity-service:0' |
965 | remote_unit = 'unit/0' |
966 | self.get_admin_token.return_value = 'token' |
967 | @@ -223,7 +224,7 @@ |
968 | self.test_config.set('service-tenant', 'tenant') |
969 | self.test_config.set('admin-role', 'admin') |
970 | self.get_requested_roles.return_value = ['role1', ] |
971 | - self.resolve_address.return_value = '10.0.0.3' |
972 | + _resolve_address.return_value = '10.0.0.3' |
973 | self.test_config.set('admin-port', 80) |
974 | self.test_config.set('service-port', 81) |
975 | self.https.return_value = False |
976 | @@ -587,6 +588,35 @@ |
977 | self.assertFalse(utils.ensure_ssl_cert_master()) |
978 | self.assertFalse(self.relation_set.called) |
979 | |
980 | + @patch('charmhelpers.contrib.openstack.ip.unit_get') |
981 | + @patch('charmhelpers.contrib.openstack.ip.is_clustered') |
982 | + @patch('charmhelpers.contrib.openstack.ip.config') |
983 | + @patch.object(utils, 'create_keystone_endpoint') |
984 | + @patch.object(utils, 'create_tenant') |
985 | + @patch.object(utils, 'create_user_credentials') |
986 | + @patch.object(utils, 'create_service_entry') |
987 | + def test_ensure_initial_admin_public_name(self, |
988 | + _create_service_entry, |
989 | + _create_user_creds, |
990 | + _create_tenant, |
991 | + _create_keystone_endpoint, |
992 | + _ip_config, |
993 | + _is_clustered, |
994 | + _unit_get): |
995 | + _is_clustered.return_value = False |
996 | + _ip_config.side_effect = self.test_config.get |
997 | + _unit_get.return_value = '10.0.0.1' |
998 | + self.test_config.set('os-public-hostname', 'keystone.example.com') |
999 | + utils.ensure_initial_admin(self.config) |
1000 | + _create_keystone_endpoint.assert_called_with( |
1001 | + public_ip='keystone.example.com', |
1002 | + service_port=5000, |
1003 | + internal_ip='10.0.0.1', |
1004 | + admin_ip='10.0.0.1', |
1005 | + auth_port=35357, |
1006 | + region='RegionOne' |
1007 | + ) |
1008 | + |
1009 | @patch.object(utils, 'peer_units') |
1010 | @patch.object(utils, 'is_ssl_enabled') |
1011 | def test_ensure_ssl_cert_master_is_leader_bad_votes(self, |
charm_lint_check #5046 keystone-next for billy-olsen mp261008
LINT OK: passed
Build: http:// 10.245. 162.77: 8080/job/ charm_lint_ check/5046/