Merge lp:~corey.bryant/charms/trusty/keystone/amulet-updates into lp:~openstack-charmers-archive/charms/trusty/keystone/next
- Trusty Tahr (14.04)
- amulet-updates
- Merge into next
Proposed by
Corey Bryant
Status: | Merged |
---|---|
Merged at revision: | 83 |
Proposed branch: | lp:~corey.bryant/charms/trusty/keystone/amulet-updates |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/keystone/next |
Diff against target: |
1435 lines (+639/-179) 24 files modified
Makefile (+2/-1) hooks/charmhelpers/contrib/hahelpers/apache.py (+10/-3) hooks/charmhelpers/contrib/hahelpers/cluster.py (+1/-2) hooks/charmhelpers/contrib/network/ip.py (+100/-16) hooks/charmhelpers/contrib/openstack/amulet/deployment.py (+35/-7) hooks/charmhelpers/contrib/openstack/amulet/utils.py (+5/-4) hooks/charmhelpers/contrib/openstack/context.py (+75/-24) hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg (+9/-0) hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend (+9/-8) hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf (+9/-8) hooks/charmhelpers/contrib/openstack/utils.py (+1/-0) hooks/charmhelpers/contrib/peerstorage/__init__.py (+27/-29) hooks/charmhelpers/core/hookenv.py (+42/-16) hooks/charmhelpers/core/host.py (+30/-5) hooks/charmhelpers/core/services/base.py (+3/-0) hooks/charmhelpers/core/services/helpers.py (+119/-5) hooks/charmhelpers/fetch/__init__.py (+19/-5) hooks/charmhelpers/fetch/archiveurl.py (+49/-4) tests/00-setup (+4/-2) tests/README (+6/-0) tests/basic_deployment.py (+24/-16) tests/charmhelpers/contrib/amulet/deployment.py (+19/-13) tests/charmhelpers/contrib/openstack/amulet/deployment.py (+36/-7) tests/charmhelpers/contrib/openstack/amulet/utils.py (+5/-4) |
To merge this branch: | bzr merge lp:~corey.bryant/charms/trusty/keystone/amulet-updates |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
OpenStack Charmers | Pending | ||
Review via email: mp+236271@code.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
- 82. By Corey Bryant
-
Force restart test to run last rather than re-authenticate everywhere.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'Makefile' |
2 | --- Makefile 2014-07-02 08:19:37 +0000 |
3 | +++ Makefile 2014-09-29 21:16:33 +0000 |
4 | @@ -14,7 +14,8 @@ |
5 | # coreycb note: The -v should only be temporary until Amulet sends |
6 | # raise_status() messages to stderr: |
7 | # https://bugs.launchpad.net/amulet/+bug/1320357 |
8 | - @juju test -v -p AMULET_HTTP_PROXY |
9 | + @juju test -v -p AMULET_HTTP_PROXY --timeout 900 \ |
10 | + 00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse |
11 | |
12 | bin/charm_helpers_sync.py: |
13 | @mkdir -p bin |
14 | |
15 | === modified file 'hooks/charmhelpers/contrib/hahelpers/apache.py' |
16 | --- hooks/charmhelpers/contrib/hahelpers/apache.py 2014-03-27 10:54:38 +0000 |
17 | +++ hooks/charmhelpers/contrib/hahelpers/apache.py 2014-09-29 21:16:33 +0000 |
18 | @@ -20,20 +20,27 @@ |
19 | ) |
20 | |
21 | |
22 | -def get_cert(): |
23 | +def get_cert(cn=None): |
24 | + # TODO: deal with multiple https endpoints via charm config |
25 | cert = config_get('ssl_cert') |
26 | key = config_get('ssl_key') |
27 | if not (cert and key): |
28 | log("Inspecting identity-service relations for SSL certificate.", |
29 | level=INFO) |
30 | cert = key = None |
31 | + if cn: |
32 | + ssl_cert_attr = 'ssl_cert_{}'.format(cn) |
33 | + ssl_key_attr = 'ssl_key_{}'.format(cn) |
34 | + else: |
35 | + ssl_cert_attr = 'ssl_cert' |
36 | + ssl_key_attr = 'ssl_key' |
37 | for r_id in relation_ids('identity-service'): |
38 | for unit in relation_list(r_id): |
39 | if not cert: |
40 | - cert = relation_get('ssl_cert', |
41 | + cert = relation_get(ssl_cert_attr, |
42 | rid=r_id, unit=unit) |
43 | if not key: |
44 | - key = relation_get('ssl_key', |
45 | + key = relation_get(ssl_key_attr, |
46 | rid=r_id, unit=unit) |
47 | return (cert, key) |
48 | |
49 | |
50 | === modified file 'hooks/charmhelpers/contrib/hahelpers/cluster.py' |
51 | --- hooks/charmhelpers/contrib/hahelpers/cluster.py 2014-08-13 13:11:49 +0000 |
52 | +++ hooks/charmhelpers/contrib/hahelpers/cluster.py 2014-09-29 21:16:33 +0000 |
53 | @@ -139,10 +139,9 @@ |
54 | return True |
55 | for r_id in relation_ids('identity-service'): |
56 | for unit in relation_list(r_id): |
57 | + # TODO - needs fixing for new helper as ssl_cert/key suffixes with CN |
58 | rel_state = [ |
59 | relation_get('https_keystone', rid=r_id, unit=unit), |
60 | - relation_get('ssl_cert', rid=r_id, unit=unit), |
61 | - relation_get('ssl_key', rid=r_id, unit=unit), |
62 | relation_get('ca_cert', rid=r_id, unit=unit), |
63 | ] |
64 | # NOTE: works around (LP: #1203241) |
65 | |
66 | === modified file 'hooks/charmhelpers/contrib/network/ip.py' |
67 | --- hooks/charmhelpers/contrib/network/ip.py 2014-08-13 13:11:49 +0000 |
68 | +++ hooks/charmhelpers/contrib/network/ip.py 2014-09-29 21:16:33 +0000 |
69 | @@ -1,10 +1,11 @@ |
70 | +import glob |
71 | import sys |
72 | |
73 | from functools import partial |
74 | |
75 | from charmhelpers.fetch import apt_install |
76 | from charmhelpers.core.hookenv import ( |
77 | - ERROR, log, config, |
78 | + ERROR, log, |
79 | ) |
80 | |
81 | try: |
82 | @@ -156,19 +157,102 @@ |
83 | get_netmask_for_address = partial(_get_for_address, key='netmask') |
84 | |
85 | |
86 | -def get_ipv6_addr(iface="eth0"): |
87 | +def format_ipv6_addr(address): |
88 | + """ |
89 | + IPv6 needs to be wrapped with [] in url link to parse correctly. |
90 | + """ |
91 | + if is_ipv6(address): |
92 | + address = "[%s]" % address |
93 | + else: |
94 | + log("Not an valid ipv6 address: %s" % address, |
95 | + level=ERROR) |
96 | + address = None |
97 | + return address |
98 | + |
99 | + |
100 | +def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, fatal=True, exc_list=None): |
101 | + """ |
102 | + Return the assigned IP address for a given interface, if any, or []. |
103 | + """ |
104 | + # Extract nic if passed /dev/ethX |
105 | + if '/' in iface: |
106 | + iface = iface.split('/')[-1] |
107 | + if not exc_list: |
108 | + exc_list = [] |
109 | try: |
110 | - iface_addrs = netifaces.ifaddresses(iface) |
111 | - if netifaces.AF_INET6 not in iface_addrs: |
112 | - raise Exception("Interface '%s' doesn't have an ipv6 address." % iface) |
113 | - |
114 | - addresses = netifaces.ifaddresses(iface)[netifaces.AF_INET6] |
115 | - ipv6_addr = [a['addr'] for a in addresses if not a['addr'].startswith('fe80') |
116 | - and config('vip') != a['addr']] |
117 | - if not ipv6_addr: |
118 | - raise Exception("Interface '%s' doesn't have global ipv6 address." % iface) |
119 | - |
120 | - return ipv6_addr[0] |
121 | - |
122 | - except ValueError: |
123 | - raise ValueError("Invalid interface '%s'" % iface) |
124 | + inet_num = getattr(netifaces, inet_type) |
125 | + except AttributeError: |
126 | + raise Exception('Unknown inet type ' + str(inet_type)) |
127 | + |
128 | + interfaces = netifaces.interfaces() |
129 | + if inc_aliases: |
130 | + ifaces = [] |
131 | + for _iface in interfaces: |
132 | + if iface == _iface or _iface.split(':')[0] == iface: |
133 | + ifaces.append(_iface) |
134 | + if fatal and not ifaces: |
135 | + raise Exception("Invalid interface '%s'" % iface) |
136 | + ifaces.sort() |
137 | + else: |
138 | + if iface not in interfaces: |
139 | + if fatal: |
140 | + raise Exception("%s not found " % (iface)) |
141 | + else: |
142 | + return [] |
143 | + else: |
144 | + ifaces = [iface] |
145 | + |
146 | + addresses = [] |
147 | + for netiface in ifaces: |
148 | + net_info = netifaces.ifaddresses(netiface) |
149 | + if inet_num in net_info: |
150 | + for entry in net_info[inet_num]: |
151 | + if 'addr' in entry and entry['addr'] not in exc_list: |
152 | + addresses.append(entry['addr']) |
153 | + if fatal and not addresses: |
154 | + raise Exception("Interface '%s' doesn't have any %s addresses." % (iface, inet_type)) |
155 | + return addresses |
156 | + |
157 | +get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET') |
158 | + |
159 | + |
160 | +def get_ipv6_addr(iface='eth0', inc_aliases=False, fatal=True, exc_list=None): |
161 | + """ |
162 | + Return the assigned IPv6 address for a given interface, if any, or []. |
163 | + """ |
164 | + addresses = get_iface_addr(iface=iface, inet_type='AF_INET6', |
165 | + inc_aliases=inc_aliases, fatal=fatal, |
166 | + exc_list=exc_list) |
167 | + remotly_addressable = [] |
168 | + for address in addresses: |
169 | + if not address.startswith('fe80'): |
170 | + remotly_addressable.append(address) |
171 | + if fatal and not remotly_addressable: |
172 | + raise Exception("Interface '%s' doesn't have global ipv6 address." % iface) |
173 | + return remotly_addressable |
174 | + |
175 | + |
176 | +def get_bridges(vnic_dir='/sys/devices/virtual/net'): |
177 | + """ |
178 | + Return a list of bridges on the system or [] |
179 | + """ |
180 | + b_rgex = vnic_dir + '/*/bridge' |
181 | + return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_rgex)] |
182 | + |
183 | + |
184 | +def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'): |
185 | + """ |
186 | + Return a list of nics comprising a given bridge on the system or [] |
187 | + """ |
188 | + brif_rgex = "%s/%s/brif/*" % (vnic_dir, bridge) |
189 | + return [x.split('/')[-1] for x in glob.glob(brif_rgex)] |
190 | + |
191 | + |
192 | +def is_bridge_member(nic): |
193 | + """ |
194 | + Check if a given nic is a member of a bridge |
195 | + """ |
196 | + for bridge in get_bridges(): |
197 | + if nic in get_bridge_nics(bridge): |
198 | + return True |
199 | + return False |
200 | |
201 | === modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py' |
202 | --- hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2014-07-30 15:11:18 +0000 |
203 | +++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2014-09-29 21:16:33 +0000 |
204 | @@ -10,32 +10,60 @@ |
205 | that is specifically for use by OpenStack charms. |
206 | """ |
207 | |
208 | - def __init__(self, series=None, openstack=None, source=None): |
209 | + def __init__(self, series=None, openstack=None, source=None, stable=True): |
210 | """Initialize the deployment environment.""" |
211 | super(OpenStackAmuletDeployment, self).__init__(series) |
212 | self.openstack = openstack |
213 | self.source = source |
214 | + self.stable = stable |
215 | + # Note(coreycb): this needs to be changed when new next branches come out. |
216 | + self.current_next = "trusty" |
217 | + |
218 | + def _determine_branch_locations(self, other_services): |
219 | + """Determine the branch locations for the other services. |
220 | + |
221 | + Determine if the local branch being tested is derived from its |
222 | + stable or next (dev) branch, and based on this, use the corresonding |
223 | + stable or next branches for the other_services.""" |
224 | + base_charms = ['mysql', 'mongodb', 'rabbitmq-server'] |
225 | + |
226 | + if self.stable: |
227 | + for svc in other_services: |
228 | + temp = 'lp:charms/{}' |
229 | + svc['location'] = temp.format(svc['name']) |
230 | + else: |
231 | + for svc in other_services: |
232 | + if svc['name'] in base_charms: |
233 | + temp = 'lp:charms/{}' |
234 | + svc['location'] = temp.format(svc['name']) |
235 | + else: |
236 | + temp = 'lp:~openstack-charmers/charms/{}/{}/next' |
237 | + svc['location'] = temp.format(self.current_next, |
238 | + svc['name']) |
239 | + return other_services |
240 | |
241 | def _add_services(self, this_service, other_services): |
242 | - """Add services to the deployment and set openstack-origin.""" |
243 | + """Add services to the deployment and set openstack-origin/source.""" |
244 | + other_services = self._determine_branch_locations(other_services) |
245 | + |
246 | super(OpenStackAmuletDeployment, self)._add_services(this_service, |
247 | other_services) |
248 | - name = 0 |
249 | + |
250 | services = other_services |
251 | services.append(this_service) |
252 | use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph'] |
253 | |
254 | if self.openstack: |
255 | for svc in services: |
256 | - if svc[name] not in use_source: |
257 | + if svc['name'] not in use_source: |
258 | config = {'openstack-origin': self.openstack} |
259 | - self.d.configure(svc[name], config) |
260 | + self.d.configure(svc['name'], config) |
261 | |
262 | if self.source: |
263 | for svc in services: |
264 | - if svc[name] in use_source: |
265 | + if svc['name'] in use_source: |
266 | config = {'source': self.source} |
267 | - self.d.configure(svc[name], config) |
268 | + self.d.configure(svc['name'], config) |
269 | |
270 | def _configure_services(self, configs): |
271 | """Configure all of the services.""" |
272 | |
273 | === modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py' |
274 | --- hooks/charmhelpers/contrib/openstack/amulet/utils.py 2014-07-30 15:11:18 +0000 |
275 | +++ hooks/charmhelpers/contrib/openstack/amulet/utils.py 2014-09-29 21:16:33 +0000 |
276 | @@ -187,15 +187,16 @@ |
277 | |
278 | f = opener.open("http://download.cirros-cloud.net/version/released") |
279 | version = f.read().strip() |
280 | - cirros_img = "tests/cirros-{}-x86_64-disk.img".format(version) |
281 | + cirros_img = "cirros-{}-x86_64-disk.img".format(version) |
282 | + local_path = os.path.join('tests', cirros_img) |
283 | |
284 | - if not os.path.exists(cirros_img): |
285 | + if not os.path.exists(local_path): |
286 | cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net", |
287 | version, cirros_img) |
288 | - opener.retrieve(cirros_url, cirros_img) |
289 | + opener.retrieve(cirros_url, local_path) |
290 | f.close() |
291 | |
292 | - with open(cirros_img) as f: |
293 | + with open(local_path) as f: |
294 | image = glance.images.create(name=image_name, is_public=True, |
295 | disk_format='qcow2', |
296 | container_format='bare', data=f) |
297 | |
298 | === modified file 'hooks/charmhelpers/contrib/openstack/context.py' |
299 | --- hooks/charmhelpers/contrib/openstack/context.py 2014-08-13 13:11:49 +0000 |
300 | +++ hooks/charmhelpers/contrib/openstack/context.py 2014-09-29 21:16:33 +0000 |
301 | @@ -8,7 +8,6 @@ |
302 | check_call |
303 | ) |
304 | |
305 | - |
306 | from charmhelpers.fetch import ( |
307 | apt_install, |
308 | filter_installed_packages, |
309 | @@ -28,6 +27,11 @@ |
310 | INFO |
311 | ) |
312 | |
313 | +from charmhelpers.core.host import ( |
314 | + mkdir, |
315 | + write_file |
316 | +) |
317 | + |
318 | from charmhelpers.contrib.hahelpers.cluster import ( |
319 | determine_apache_port, |
320 | determine_api_port, |
321 | @@ -38,6 +42,7 @@ |
322 | from charmhelpers.contrib.hahelpers.apache import ( |
323 | get_cert, |
324 | get_ca_cert, |
325 | + install_ca_cert, |
326 | ) |
327 | |
328 | from charmhelpers.contrib.openstack.neutron import ( |
329 | @@ -47,6 +52,7 @@ |
330 | from charmhelpers.contrib.network.ip import ( |
331 | get_address_in_network, |
332 | get_ipv6_addr, |
333 | + is_address_in_network |
334 | ) |
335 | |
336 | CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt' |
337 | @@ -421,6 +427,11 @@ |
338 | 'units': cluster_hosts, |
339 | } |
340 | |
341 | + if config('haproxy-server-timeout'): |
342 | + ctxt['haproxy_server_timeout'] = config('haproxy-server-timeout') |
343 | + if config('haproxy-client-timeout'): |
344 | + ctxt['haproxy_client_timeout'] = config('haproxy-client-timeout') |
345 | + |
346 | if config('prefer-ipv6'): |
347 | ctxt['local_host'] = 'ip6-localhost' |
348 | ctxt['haproxy_host'] = '::' |
349 | @@ -490,22 +501,36 @@ |
350 | cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http'] |
351 | check_call(cmd) |
352 | |
353 | - def configure_cert(self): |
354 | - if not os.path.isdir('/etc/apache2/ssl'): |
355 | - os.mkdir('/etc/apache2/ssl') |
356 | + def configure_cert(self, cn=None): |
357 | ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace) |
358 | - if not os.path.isdir(ssl_dir): |
359 | - os.mkdir(ssl_dir) |
360 | - cert, key = get_cert() |
361 | - with open(os.path.join(ssl_dir, 'cert'), 'w') as cert_out: |
362 | - cert_out.write(b64decode(cert)) |
363 | - with open(os.path.join(ssl_dir, 'key'), 'w') as key_out: |
364 | - key_out.write(b64decode(key)) |
365 | + mkdir(path=ssl_dir) |
366 | + cert, key = get_cert(cn) |
367 | + if cn: |
368 | + cert_filename = 'cert_{}'.format(cn) |
369 | + key_filename = 'key_{}'.format(cn) |
370 | + else: |
371 | + cert_filename = 'cert' |
372 | + key_filename = 'key' |
373 | + write_file(path=os.path.join(ssl_dir, cert_filename), |
374 | + content=b64decode(cert)) |
375 | + write_file(path=os.path.join(ssl_dir, key_filename), |
376 | + content=b64decode(key)) |
377 | + |
378 | + def configure_ca(self): |
379 | ca_cert = get_ca_cert() |
380 | if ca_cert: |
381 | - with open(CA_CERT_PATH, 'w') as ca_out: |
382 | - ca_out.write(b64decode(ca_cert)) |
383 | - check_call(['update-ca-certificates']) |
384 | + install_ca_cert(b64decode(ca_cert)) |
385 | + |
386 | + def canonical_names(self): |
387 | + '''Figure out which canonical names clients will access this service''' |
388 | + cns = [] |
389 | + for r_id in relation_ids('identity-service'): |
390 | + for unit in related_units(r_id): |
391 | + rdata = relation_get(rid=r_id, unit=unit) |
392 | + for k in rdata: |
393 | + if k.startswith('ssl_key_'): |
394 | + cns.append(k.lstrip('ssl_key_')) |
395 | + return list(set(cns)) |
396 | |
397 | def __call__(self): |
398 | if isinstance(self.external_ports, basestring): |
399 | @@ -513,21 +538,47 @@ |
400 | if (not self.external_ports or not https()): |
401 | return {} |
402 | |
403 | - self.configure_cert() |
404 | + self.configure_ca() |
405 | self.enable_modules() |
406 | |
407 | ctxt = { |
408 | 'namespace': self.service_namespace, |
409 | - 'private_address': unit_get('private-address'), |
410 | - 'endpoints': [] |
411 | + 'endpoints': [], |
412 | + 'ext_ports': [] |
413 | } |
414 | - if is_clustered(): |
415 | - ctxt['private_address'] = config('vip') |
416 | - for api_port in self.external_ports: |
417 | - ext_port = determine_apache_port(api_port) |
418 | - int_port = determine_api_port(api_port) |
419 | - portmap = (int(ext_port), int(int_port)) |
420 | - ctxt['endpoints'].append(portmap) |
421 | + |
422 | + for cn in self.canonical_names(): |
423 | + self.configure_cert(cn) |
424 | + |
425 | + addresses = [] |
426 | + vips = [] |
427 | + if config('vip'): |
428 | + vips = config('vip').split() |
429 | + |
430 | + for network_type in ['os-internal-network', |
431 | + 'os-admin-network', |
432 | + 'os-public-network']: |
433 | + address = get_address_in_network(config(network_type), |
434 | + unit_get('private-address')) |
435 | + if len(vips) > 0 and is_clustered(): |
436 | + for vip in vips: |
437 | + if is_address_in_network(config(network_type), |
438 | + vip): |
439 | + addresses.append((address, vip)) |
440 | + break |
441 | + elif is_clustered(): |
442 | + addresses.append((address, config('vip'))) |
443 | + else: |
444 | + addresses.append((address, address)) |
445 | + |
446 | + for address, endpoint in set(addresses): |
447 | + for api_port in self.external_ports: |
448 | + ext_port = determine_apache_port(api_port) |
449 | + int_port = determine_api_port(api_port) |
450 | + portmap = (address, endpoint, int(ext_port), int(int_port)) |
451 | + ctxt['endpoints'].append(portmap) |
452 | + ctxt['ext_ports'].append(int(ext_port)) |
453 | + ctxt['ext_ports'] = list(set(ctxt['ext_ports'])) |
454 | return ctxt |
455 | |
456 | |
457 | |
458 | === modified file 'hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg' |
459 | --- hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg 2014-08-13 13:11:49 +0000 |
460 | +++ hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg 2014-09-29 21:16:33 +0000 |
461 | @@ -14,8 +14,17 @@ |
462 | retries 3 |
463 | timeout queue 1000 |
464 | timeout connect 1000 |
465 | +{% if haproxy_client_timeout -%} |
466 | + timeout client {{ haproxy_client_timeout }} |
467 | +{% else -%} |
468 | timeout client 30000 |
469 | +{% endif -%} |
470 | + |
471 | +{% if haproxy_server_timeout -%} |
472 | + timeout server {{ haproxy_server_timeout }} |
473 | +{% else -%} |
474 | timeout server 30000 |
475 | +{% endif -%} |
476 | |
477 | listen stats {{ stat_port }} |
478 | mode http |
479 | |
480 | === modified file 'hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend' |
481 | --- hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend 2014-03-27 10:54:38 +0000 |
482 | +++ hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend 2014-09-29 21:16:33 +0000 |
483 | @@ -1,16 +1,18 @@ |
484 | {% if endpoints -%} |
485 | -{% for ext, int in endpoints -%} |
486 | -Listen {{ ext }} |
487 | -NameVirtualHost *:{{ ext }} |
488 | -<VirtualHost *:{{ ext }}> |
489 | - ServerName {{ private_address }} |
490 | +{% for ext_port in ext_ports -%} |
491 | +Listen {{ ext_port }} |
492 | +{% endfor -%} |
493 | +{% for address, endpoint, ext, int in endpoints -%} |
494 | +<VirtualHost {{ address }}:{{ ext }}> |
495 | + ServerName {{ endpoint }} |
496 | SSLEngine on |
497 | - SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert |
498 | - SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key |
499 | + SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }} |
500 | + SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }} |
501 | ProxyPass / http://localhost:{{ int }}/ |
502 | ProxyPassReverse / http://localhost:{{ int }}/ |
503 | ProxyPreserveHost on |
504 | </VirtualHost> |
505 | +{% endfor -%} |
506 | <Proxy *> |
507 | Order deny,allow |
508 | Allow from all |
509 | @@ -19,5 +21,4 @@ |
510 | Order allow,deny |
511 | Allow from all |
512 | </Location> |
513 | -{% endfor -%} |
514 | {% endif -%} |
515 | |
516 | === modified file 'hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf' |
517 | --- hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf 2014-03-27 10:54:38 +0000 |
518 | +++ hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf 2014-09-29 21:16:33 +0000 |
519 | @@ -1,16 +1,18 @@ |
520 | {% if endpoints -%} |
521 | -{% for ext, int in endpoints -%} |
522 | -Listen {{ ext }} |
523 | -NameVirtualHost *:{{ ext }} |
524 | -<VirtualHost *:{{ ext }}> |
525 | - ServerName {{ private_address }} |
526 | +{% for ext_port in ext_ports -%} |
527 | +Listen {{ ext_port }} |
528 | +{% endfor -%} |
529 | +{% for address, endpoint, ext, int in endpoints -%} |
530 | +<VirtualHost {{ address }}:{{ ext }}> |
531 | + ServerName {{ endpoint }} |
532 | SSLEngine on |
533 | - SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert |
534 | - SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key |
535 | + SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }} |
536 | + SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }} |
537 | ProxyPass / http://localhost:{{ int }}/ |
538 | ProxyPassReverse / http://localhost:{{ int }}/ |
539 | ProxyPreserveHost on |
540 | </VirtualHost> |
541 | +{% endfor -%} |
542 | <Proxy *> |
543 | Order deny,allow |
544 | Allow from all |
545 | @@ -19,5 +21,4 @@ |
546 | Order allow,deny |
547 | Allow from all |
548 | </Location> |
549 | -{% endfor -%} |
550 | {% endif -%} |
551 | |
552 | === modified file 'hooks/charmhelpers/contrib/openstack/utils.py' |
553 | --- hooks/charmhelpers/contrib/openstack/utils.py 2014-08-26 13:27:50 +0000 |
554 | +++ hooks/charmhelpers/contrib/openstack/utils.py 2014-09-29 21:16:33 +0000 |
555 | @@ -70,6 +70,7 @@ |
556 | ('1.13.0', 'icehouse'), |
557 | ('1.12.0', 'icehouse'), |
558 | ('1.11.0', 'icehouse'), |
559 | + ('2.0.0', 'juno'), |
560 | ]) |
561 | |
562 | DEFAULT_LOOPBACK_SIZE = '5G' |
563 | |
564 | === modified file 'hooks/charmhelpers/contrib/peerstorage/__init__.py' |
565 | --- hooks/charmhelpers/contrib/peerstorage/__init__.py 2014-08-26 13:27:50 +0000 |
566 | +++ hooks/charmhelpers/contrib/peerstorage/__init__.py 2014-09-29 21:16:33 +0000 |
567 | @@ -7,40 +7,38 @@ |
568 | relation_set, |
569 | ) |
570 | |
571 | + |
572 | """ |
573 | This helper provides functions to support use of a peer relation |
574 | for basic key/value storage, with the added benefit that all storage |
575 | -can be replicated across peer units, so this is really useful for |
576 | -services that issue usernames/passwords to remote services. |
577 | - |
578 | -def shared_db_changed() |
579 | - # Only the lead unit should create passwords |
580 | - if not is_leader(): |
581 | - return |
582 | - username = relation_get('username') |
583 | - key = '{}.password'.format(username) |
584 | - # Attempt to retrieve any existing password for this user |
585 | - password = peer_retrieve(key) |
586 | - if password is None: |
587 | - # New user, create password and store |
588 | - password = pwgen(length=64) |
589 | - peer_store(key, password) |
590 | - create_access(username, password) |
591 | - relation_set(password=password) |
592 | - |
593 | - |
594 | -def cluster_changed() |
595 | - # Echo any relation data other that *-address |
596 | - # back onto the peer relation so all units have |
597 | - # all *.password keys stored on their local relation |
598 | - # for later retrieval. |
599 | +can be replicated across peer units. |
600 | + |
601 | +Requirement to use: |
602 | + |
603 | +To use this, the "peer_echo()" method has to be called form the peer |
604 | +relation's relation-changed hook: |
605 | + |
606 | +@hooks.hook("cluster-relation-changed") # Adapt the to your peer relation name |
607 | +def cluster_relation_changed(): |
608 | peer_echo() |
609 | |
610 | +Once this is done, you can use peer storage from anywhere: |
611 | + |
612 | +@hooks.hook("some-hook") |
613 | +def some_hook(): |
614 | + # You can store and retrieve key/values this way: |
615 | + if is_relation_made("cluster"): # from charmhelpers.core.hookenv |
616 | + # There are peers available so we can work with peer storage |
617 | + peer_store("mykey", "myvalue") |
618 | + value = peer_retrieve("mykey") |
619 | + print value |
620 | + else: |
621 | + print "No peers joind the relation, cannot share key/values :(" |
622 | """ |
623 | |
624 | |
625 | def peer_retrieve(key, relation_name='cluster'): |
626 | - """ Retrieve a named key from peer relation relation_name """ |
627 | + """Retrieve a named key from peer relation `relation_name`.""" |
628 | cluster_rels = relation_ids(relation_name) |
629 | if len(cluster_rels) > 0: |
630 | cluster_rid = cluster_rels[0] |
631 | @@ -70,7 +68,7 @@ |
632 | |
633 | |
634 | def peer_store(key, value, relation_name='cluster'): |
635 | - """ Store the key/value pair on the named peer relation relation_name """ |
636 | + """Store the key/value pair on the named peer relation `relation_name`.""" |
637 | cluster_rels = relation_ids(relation_name) |
638 | if len(cluster_rels) > 0: |
639 | cluster_rid = cluster_rels[0] |
640 | @@ -82,10 +80,10 @@ |
641 | |
642 | |
643 | def peer_echo(includes=None): |
644 | - """Echo filtered attributes back onto the same relation for storage |
645 | + """Echo filtered attributes back onto the same relation for storage. |
646 | |
647 | - Note that this helper must only be called within a peer relation |
648 | - changed hook |
649 | + This is a requirement to use the peerstorage module - it needs to be called |
650 | + from the peer relation's changed hook. |
651 | """ |
652 | rdata = relation_get() |
653 | echo_data = {} |
654 | |
655 | === modified file 'hooks/charmhelpers/core/hookenv.py' |
656 | --- hooks/charmhelpers/core/hookenv.py 2014-08-26 13:27:50 +0000 |
657 | +++ hooks/charmhelpers/core/hookenv.py 2014-09-29 21:16:33 +0000 |
658 | @@ -156,12 +156,15 @@ |
659 | |
660 | |
661 | class Config(dict): |
662 | - """A Juju charm config dictionary that can write itself to |
663 | - disk (as json) and track which values have changed since |
664 | - the previous hook invocation. |
665 | - |
666 | - Do not instantiate this object directly - instead call |
667 | - ``hookenv.config()`` |
668 | + """A dictionary representation of the charm's config.yaml, with some |
669 | + extra features: |
670 | + |
671 | + - See which values in the dictionary have changed since the previous hook. |
672 | + - For values that have changed, see what the previous value was. |
673 | + - Store arbitrary data for use in a later hook. |
674 | + |
675 | + NOTE: Do not instantiate this object directly - instead call |
676 | + ``hookenv.config()``, which will return an instance of :class:`Config`. |
677 | |
678 | Example usage:: |
679 | |
680 | @@ -170,8 +173,8 @@ |
681 | >>> config = hookenv.config() |
682 | >>> config['foo'] |
683 | 'bar' |
684 | + >>> # store a new key/value for later use |
685 | >>> config['mykey'] = 'myval' |
686 | - >>> config.save() |
687 | |
688 | |
689 | >>> # user runs `juju set mycharm foo=baz` |
690 | @@ -188,22 +191,34 @@ |
691 | >>> # keys/values that we add are preserved across hooks |
692 | >>> config['mykey'] |
693 | 'myval' |
694 | - >>> # don't forget to save at the end of hook! |
695 | - >>> config.save() |
696 | |
697 | """ |
698 | CONFIG_FILE_NAME = '.juju-persistent-config' |
699 | |
700 | def __init__(self, *args, **kw): |
701 | super(Config, self).__init__(*args, **kw) |
702 | + self.implicit_save = True |
703 | self._prev_dict = None |
704 | self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) |
705 | if os.path.exists(self.path): |
706 | self.load_previous() |
707 | |
708 | + def __getitem__(self, key): |
709 | + """For regular dict lookups, check the current juju config first, |
710 | + then the previous (saved) copy. This ensures that user-saved values |
711 | + will be returned by a dict lookup. |
712 | + |
713 | + """ |
714 | + try: |
715 | + return dict.__getitem__(self, key) |
716 | + except KeyError: |
717 | + return (self._prev_dict or {})[key] |
718 | + |
719 | def load_previous(self, path=None): |
720 | - """Load previous copy of config from disk so that current values |
721 | - can be compared to previous values. |
722 | + """Load previous copy of config from disk. |
723 | + |
724 | + In normal usage you don't need to call this method directly - it |
725 | + is called automatically at object initialization. |
726 | |
727 | :param path: |
728 | |
729 | @@ -218,8 +233,8 @@ |
730 | self._prev_dict = json.load(f) |
731 | |
732 | def changed(self, key): |
733 | - """Return true if the value for this key has changed since |
734 | - the last save. |
735 | + """Return True if the current value for this key is different from |
736 | + the previous value. |
737 | |
738 | """ |
739 | if self._prev_dict is None: |
740 | @@ -228,7 +243,7 @@ |
741 | |
742 | def previous(self, key): |
743 | """Return previous value for this key, or None if there |
744 | - is no "previous" value. |
745 | + is no previous value. |
746 | |
747 | """ |
748 | if self._prev_dict: |
749 | @@ -238,7 +253,13 @@ |
750 | def save(self): |
751 | """Save this config to disk. |
752 | |
753 | - Preserves items in _prev_dict that do not exist in self. |
754 | + If the charm is using the :mod:`Services Framework <services.base>` |
755 | + or :meth:'@hook <Hooks.hook>' decorator, this |
756 | + is called automatically at the end of successful hook execution. |
757 | + Otherwise, it should be called directly by user code. |
758 | + |
759 | + To disable automatic saves, set ``implicit_save=False`` on this |
760 | + instance. |
761 | |
762 | """ |
763 | if self._prev_dict: |
764 | @@ -465,9 +486,10 @@ |
765 | hooks.execute(sys.argv) |
766 | """ |
767 | |
768 | - def __init__(self): |
769 | + def __init__(self, config_save=True): |
770 | super(Hooks, self).__init__() |
771 | self._hooks = {} |
772 | + self._config_save = config_save |
773 | |
774 | def register(self, name, function): |
775 | """Register a hook""" |
776 | @@ -478,6 +500,10 @@ |
777 | hook_name = os.path.basename(args[0]) |
778 | if hook_name in self._hooks: |
779 | self._hooks[hook_name]() |
780 | + if self._config_save: |
781 | + cfg = config() |
782 | + if cfg.implicit_save: |
783 | + cfg.save() |
784 | else: |
785 | raise UnregisteredHookError(hook_name) |
786 | |
787 | |
788 | === modified file 'hooks/charmhelpers/core/host.py' |
789 | --- hooks/charmhelpers/core/host.py 2014-08-26 13:27:50 +0000 |
790 | +++ hooks/charmhelpers/core/host.py 2014-09-29 21:16:33 +0000 |
791 | @@ -68,8 +68,8 @@ |
792 | """Determine whether a system service is available""" |
793 | try: |
794 | subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT) |
795 | - except subprocess.CalledProcessError: |
796 | - return False |
797 | + except subprocess.CalledProcessError as e: |
798 | + return 'unrecognized service' not in e.output |
799 | else: |
800 | return True |
801 | |
802 | @@ -209,10 +209,15 @@ |
803 | return system_mounts |
804 | |
805 | |
806 | -def file_hash(path): |
807 | - """Generate a md5 hash of the contents of 'path' or None if not found """ |
808 | +def file_hash(path, hash_type='md5'): |
809 | + """ |
810 | + Generate a hash checksum of the contents of 'path' or None if not found. |
811 | + |
812 | + :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`, |
813 | + such as md5, sha1, sha256, sha512, etc. |
814 | + """ |
815 | if os.path.exists(path): |
816 | - h = hashlib.md5() |
817 | + h = getattr(hashlib, hash_type)() |
818 | with open(path, 'r') as source: |
819 | h.update(source.read()) # IGNORE:E1101 - it does have update |
820 | return h.hexdigest() |
821 | @@ -220,6 +225,26 @@ |
822 | return None |
823 | |
824 | |
825 | +def check_hash(path, checksum, hash_type='md5'): |
826 | + """ |
827 | + Validate a file using a cryptographic checksum. |
828 | + |
829 | + :param str checksum: Value of the checksum used to validate the file. |
830 | + :param str hash_type: Hash algorithm used to generate `checksum`. |
831 | + Can be any hash alrgorithm supported by :mod:`hashlib`, |
832 | + such as md5, sha1, sha256, sha512, etc. |
833 | + :raises ChecksumError: If the file fails the checksum |
834 | + |
835 | + """ |
836 | + actual_checksum = file_hash(path, hash_type) |
837 | + if checksum != actual_checksum: |
838 | + raise ChecksumError("'%s' != '%s'" % (checksum, actual_checksum)) |
839 | + |
840 | + |
841 | +class ChecksumError(ValueError): |
842 | + pass |
843 | + |
844 | + |
845 | def restart_on_change(restart_map, stopstart=False): |
846 | """Restart services based on configuration files changing |
847 | |
848 | |
849 | === modified file 'hooks/charmhelpers/core/services/base.py' |
850 | --- hooks/charmhelpers/core/services/base.py 2014-08-26 13:27:50 +0000 |
851 | +++ hooks/charmhelpers/core/services/base.py 2014-09-29 21:16:33 +0000 |
852 | @@ -118,6 +118,9 @@ |
853 | else: |
854 | self.provide_data() |
855 | self.reconfigure_services() |
856 | + cfg = hookenv.config() |
857 | + if cfg.implicit_save: |
858 | + cfg.save() |
859 | |
860 | def provide_data(self): |
861 | """ |
862 | |
863 | === modified file 'hooks/charmhelpers/core/services/helpers.py' |
864 | --- hooks/charmhelpers/core/services/helpers.py 2014-08-13 13:11:49 +0000 |
865 | +++ hooks/charmhelpers/core/services/helpers.py 2014-09-29 21:16:33 +0000 |
866 | @@ -1,3 +1,5 @@ |
867 | +import os |
868 | +import yaml |
869 | from charmhelpers.core import hookenv |
870 | from charmhelpers.core import templating |
871 | |
872 | @@ -19,15 +21,21 @@ |
873 | the `name` attribute that are complete will used to populate the dictionary |
874 | values (see `get_data`, below). |
875 | |
876 | - The generated context will be namespaced under the interface type, to prevent |
877 | - potential naming conflicts. |
878 | + The generated context will be namespaced under the relation :attr:`name`, |
879 | + to prevent potential naming conflicts. |
880 | + |
881 | + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm |
882 | + :param list additional_required_keys: Extend the list of :attr:`required_keys` |
883 | """ |
884 | name = None |
885 | interface = None |
886 | required_keys = [] |
887 | |
888 | - def __init__(self, *args, **kwargs): |
889 | - super(RelationContext, self).__init__(*args, **kwargs) |
890 | + def __init__(self, name=None, additional_required_keys=None): |
891 | + if name is not None: |
892 | + self.name = name |
893 | + if additional_required_keys is not None: |
894 | + self.required_keys.extend(additional_required_keys) |
895 | self.get_data() |
896 | |
897 | def __bool__(self): |
898 | @@ -101,9 +109,115 @@ |
899 | return {} |
900 | |
901 | |
902 | +class MysqlRelation(RelationContext): |
903 | + """ |
904 | + Relation context for the `mysql` interface. |
905 | + |
906 | + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm |
907 | + :param list additional_required_keys: Extend the list of :attr:`required_keys` |
908 | + """ |
909 | + name = 'db' |
910 | + interface = 'mysql' |
911 | + required_keys = ['host', 'user', 'password', 'database'] |
912 | + |
913 | + |
914 | +class HttpRelation(RelationContext): |
915 | + """ |
916 | + Relation context for the `http` interface. |
917 | + |
918 | + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm |
919 | + :param list additional_required_keys: Extend the list of :attr:`required_keys` |
920 | + """ |
921 | + name = 'website' |
922 | + interface = 'http' |
923 | + required_keys = ['host', 'port'] |
924 | + |
925 | + def provide_data(self): |
926 | + return { |
927 | + 'host': hookenv.unit_get('private-address'), |
928 | + 'port': 80, |
929 | + } |
930 | + |
931 | + |
932 | +class RequiredConfig(dict): |
933 | + """ |
934 | + Data context that loads config options with one or more mandatory options. |
935 | + |
936 | + Once the required options have been changed from their default values, all |
937 | + config options will be available, namespaced under `config` to prevent |
938 | + potential naming conflicts (for example, between a config option and a |
939 | + relation property). |
940 | + |
941 | + :param list *args: List of options that must be changed from their default values. |
942 | + """ |
943 | + |
944 | + def __init__(self, *args): |
945 | + self.required_options = args |
946 | + self['config'] = hookenv.config() |
947 | + with open(os.path.join(hookenv.charm_dir(), 'config.yaml')) as fp: |
948 | + self.config = yaml.load(fp).get('options', {}) |
949 | + |
950 | + def __bool__(self): |
951 | + for option in self.required_options: |
952 | + if option not in self['config']: |
953 | + return False |
954 | + current_value = self['config'][option] |
955 | + default_value = self.config[option].get('default') |
956 | + if current_value == default_value: |
957 | + return False |
958 | + if current_value in (None, '') and default_value in (None, ''): |
959 | + return False |
960 | + return True |
961 | + |
962 | + def __nonzero__(self): |
963 | + return self.__bool__() |
964 | + |
965 | + |
966 | +class StoredContext(dict): |
967 | + """ |
968 | + A data context that always returns the data that it was first created with. |
969 | + |
970 | + This is useful to do a one-time generation of things like passwords, that |
971 | + will thereafter use the same value that was originally generated, instead |
972 | + of generating a new value each time it is run. |
973 | + """ |
974 | + def __init__(self, file_name, config_data): |
975 | + """ |
976 | + If the file exists, populate `self` with the data from the file. |
977 | + Otherwise, populate with the given data and persist it to the file. |
978 | + """ |
979 | + if os.path.exists(file_name): |
980 | + self.update(self.read_context(file_name)) |
981 | + else: |
982 | + self.store_context(file_name, config_data) |
983 | + self.update(config_data) |
984 | + |
985 | + def store_context(self, file_name, config_data): |
986 | + if not os.path.isabs(file_name): |
987 | + file_name = os.path.join(hookenv.charm_dir(), file_name) |
988 | + with open(file_name, 'w') as file_stream: |
989 | + os.fchmod(file_stream.fileno(), 0600) |
990 | + yaml.dump(config_data, file_stream) |
991 | + |
992 | + def read_context(self, file_name): |
993 | + if not os.path.isabs(file_name): |
994 | + file_name = os.path.join(hookenv.charm_dir(), file_name) |
995 | + with open(file_name, 'r') as file_stream: |
996 | + data = yaml.load(file_stream) |
997 | + if not data: |
998 | + raise OSError("%s is empty" % file_name) |
999 | + return data |
1000 | + |
1001 | + |
1002 | class TemplateCallback(ManagerCallback): |
1003 | """ |
1004 | - Callback class that will render a template, for use as a ready action. |
1005 | + Callback class that will render a Jinja2 template, for use as a ready action. |
1006 | + |
1007 | + :param str source: The template source file, relative to `$CHARM_DIR/templates` |
1008 | + :param str target: The target to write the rendered template to |
1009 | + :param str owner: The owner of the rendered file |
1010 | + :param str group: The group of the rendered file |
1011 | + :param int perms: The permissions of the rendered file |
1012 | """ |
1013 | def __init__(self, source, target, owner='root', group='root', perms=0444): |
1014 | self.source = source |
1015 | |
1016 | === modified file 'hooks/charmhelpers/fetch/__init__.py' |
1017 | --- hooks/charmhelpers/fetch/__init__.py 2014-08-26 13:27:50 +0000 |
1018 | +++ hooks/charmhelpers/fetch/__init__.py 2014-09-29 21:16:33 +0000 |
1019 | @@ -208,7 +208,8 @@ |
1020 | """Add a package source to this system. |
1021 | |
1022 | @param source: a URL or sources.list entry, as supported by |
1023 | - add-apt-repository(1). Examples: |
1024 | + add-apt-repository(1). Examples:: |
1025 | + |
1026 | ppa:charmers/example |
1027 | deb https://stub:key@private.example.com/ubuntu trusty main |
1028 | |
1029 | @@ -311,22 +312,35 @@ |
1030 | apt_update(fatal=True) |
1031 | |
1032 | |
1033 | -def install_remote(source): |
1034 | +def install_remote(source, *args, **kwargs): |
1035 | """ |
1036 | Install a file tree from a remote source |
1037 | |
1038 | The specified source should be a url of the form: |
1039 | scheme://[host]/path[#[option=value][&...]] |
1040 | |
1041 | - Schemes supported are based on this modules submodules |
1042 | - Options supported are submodule-specific""" |
1043 | + Schemes supported are based on this modules submodules. |
1044 | + Options supported are submodule-specific. |
1045 | + Additional arguments are passed through to the submodule. |
1046 | + |
1047 | + For example:: |
1048 | + |
1049 | + dest = install_remote('http://example.com/archive.tgz', |
1050 | + checksum='deadbeef', |
1051 | + hash_type='sha1') |
1052 | + |
1053 | + This will download `archive.tgz`, validate it using SHA1 and, if |
1054 | + the file is ok, extract it and return the directory in which it |
1055 | + was extracted. If the checksum fails, it will raise |
1056 | + :class:`charmhelpers.core.host.ChecksumError`. |
1057 | + """ |
1058 | # We ONLY check for True here because can_handle may return a string |
1059 | # explaining why it can't handle a given source. |
1060 | handlers = [h for h in plugins() if h.can_handle(source) is True] |
1061 | installed_to = None |
1062 | for handler in handlers: |
1063 | try: |
1064 | - installed_to = handler.install(source) |
1065 | + installed_to = handler.install(source, *args, **kwargs) |
1066 | except UnhandledSource: |
1067 | pass |
1068 | if not installed_to: |
1069 | |
1070 | === modified file 'hooks/charmhelpers/fetch/archiveurl.py' |
1071 | --- hooks/charmhelpers/fetch/archiveurl.py 2014-03-27 10:54:38 +0000 |
1072 | +++ hooks/charmhelpers/fetch/archiveurl.py 2014-09-29 21:16:33 +0000 |
1073 | @@ -1,6 +1,8 @@ |
1074 | import os |
1075 | import urllib2 |
1076 | +from urllib import urlretrieve |
1077 | import urlparse |
1078 | +import hashlib |
1079 | |
1080 | from charmhelpers.fetch import ( |
1081 | BaseFetchHandler, |
1082 | @@ -10,11 +12,19 @@ |
1083 | get_archive_handler, |
1084 | extract, |
1085 | ) |
1086 | -from charmhelpers.core.host import mkdir |
1087 | +from charmhelpers.core.host import mkdir, check_hash |
1088 | |
1089 | |
1090 | class ArchiveUrlFetchHandler(BaseFetchHandler): |
1091 | - """Handler for archives via generic URLs""" |
1092 | + """ |
1093 | + Handler to download archive files from arbitrary URLs. |
1094 | + |
1095 | + Can fetch from http, https, ftp, and file URLs. |
1096 | + |
1097 | + Can install either tarballs (.tar, .tgz, .tbz2, etc) or zip files. |
1098 | + |
1099 | + Installs the contents of the archive in $CHARM_DIR/fetched/. |
1100 | + """ |
1101 | def can_handle(self, source): |
1102 | url_parts = self.parse_url(source) |
1103 | if url_parts.scheme not in ('http', 'https', 'ftp', 'file'): |
1104 | @@ -24,6 +34,12 @@ |
1105 | return False |
1106 | |
1107 | def download(self, source, dest): |
1108 | + """ |
1109 | + Download an archive file. |
1110 | + |
1111 | + :param str source: URL pointing to an archive file. |
1112 | + :param str dest: Local path location to download archive file to. |
1113 | + """ |
1114 | # propogate all exceptions |
1115 | # URLError, OSError, etc |
1116 | proto, netloc, path, params, query, fragment = urlparse.urlparse(source) |
1117 | @@ -48,7 +64,30 @@ |
1118 | os.unlink(dest) |
1119 | raise e |
1120 | |
1121 | - def install(self, source): |
1122 | + # Mandatory file validation via Sha1 or MD5 hashing. |
1123 | + def download_and_validate(self, url, hashsum, validate="sha1"): |
1124 | + tempfile, headers = urlretrieve(url) |
1125 | + check_hash(tempfile, hashsum, validate) |
1126 | + return tempfile |
1127 | + |
1128 | + def install(self, source, dest=None, checksum=None, hash_type='sha1'): |
1129 | + """ |
1130 | + Download and install an archive file, with optional checksum validation. |
1131 | + |
1132 | + The checksum can also be given on the `source` URL's fragment. |
1133 | + For example:: |
1134 | + |
1135 | + handler.install('http://example.com/file.tgz#sha1=deadbeef') |
1136 | + |
1137 | + :param str source: URL pointing to an archive file. |
1138 | + :param str dest: Local destination path to install to. If not given, |
1139 | + installs to `$CHARM_DIR/archives/archive_file_name`. |
1140 | + :param str checksum: If given, validate the archive file after download. |
1141 | + :param str hash_type: Algorithm used to generate `checksum`. |
1142 | + Can be any hash alrgorithm supported by :mod:`hashlib`, |
1143 | + such as md5, sha1, sha256, sha512, etc. |
1144 | + |
1145 | + """ |
1146 | url_parts = self.parse_url(source) |
1147 | dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched') |
1148 | if not os.path.exists(dest_dir): |
1149 | @@ -60,4 +99,10 @@ |
1150 | raise UnhandledSource(e.reason) |
1151 | except OSError as e: |
1152 | raise UnhandledSource(e.strerror) |
1153 | - return extract(dld_file) |
1154 | + options = urlparse.parse_qs(url_parts.fragment) |
1155 | + for key, value in options.items(): |
1156 | + if key in hashlib.algorithms: |
1157 | + check_hash(dld_file, value, key) |
1158 | + if checksum: |
1159 | + check_hash(dld_file, checksum, hash_type) |
1160 | + return extract(dld_file, dest) |
1161 | |
1162 | === modified file 'tests/00-setup' |
1163 | --- tests/00-setup 2014-06-24 17:11:36 +0000 |
1164 | +++ tests/00-setup 2014-09-29 21:16:33 +0000 |
1165 | @@ -4,5 +4,7 @@ |
1166 | |
1167 | sudo add-apt-repository --yes ppa:juju/stable |
1168 | sudo apt-get update --yes |
1169 | -sudo apt-get install --yes python-amulet |
1170 | -sudo apt-get install --yes python-keystoneclient |
1171 | +sudo apt-get install --yes python-amulet \ |
1172 | + python-keystoneclient \ |
1173 | + python-glanceclient \ |
1174 | + python-novaclient |
1175 | |
1176 | === modified file 'tests/README' |
1177 | --- tests/README 2014-07-11 14:07:51 +0000 |
1178 | +++ tests/README 2014-09-29 21:16:33 +0000 |
1179 | @@ -1,6 +1,12 @@ |
1180 | This directory provides Amulet tests that focus on verification of Keystone |
1181 | deployments. |
1182 | |
1183 | +In order to run tests, you'll need charm-tools installed (in addition to |
1184 | +juju, of course): |
1185 | + sudo add-apt-repository ppa:juju/stable |
1186 | + sudo apt-get update |
1187 | + sudo apt-get install charm-tools |
1188 | + |
1189 | If you use a web proxy server to access the web, you'll need to set the |
1190 | AMULET_HTTP_PROXY environment variable to the http URL of the proxy server. |
1191 | |
1192 | |
1193 | === modified file 'tests/basic_deployment.py' |
1194 | --- tests/basic_deployment.py 2014-07-13 02:44:06 +0000 |
1195 | +++ tests/basic_deployment.py 2014-09-29 21:16:33 +0000 |
1196 | @@ -19,9 +19,9 @@ |
1197 | class KeystoneBasicDeployment(OpenStackAmuletDeployment): |
1198 | """Amulet tests on a basic keystone deployment.""" |
1199 | |
1200 | - def __init__(self, series=None, openstack=None, source=None): |
1201 | + def __init__(self, series=None, openstack=None, source=None, stable=False): |
1202 | """Deploy the entire test environment.""" |
1203 | - super(KeystoneBasicDeployment, self).__init__(series, openstack, source) |
1204 | + super(KeystoneBasicDeployment, self).__init__(series, openstack, source, stable) |
1205 | self._add_services() |
1206 | self._add_relations() |
1207 | self._configure_services() |
1208 | @@ -29,11 +29,14 @@ |
1209 | self._initialize_tests() |
1210 | |
1211 | def _add_services(self): |
1212 | - """Add the services that we're testing, including the number of units, |
1213 | - where keystone is local, and mysql and cinder are from the charm |
1214 | - store.""" |
1215 | - this_service = ('keystone', 1) |
1216 | - other_services = [('mysql', 1), ('cinder', 1)] |
1217 | + """Add services |
1218 | + |
1219 | + Add the services that we're testing, where keystone is local, |
1220 | + and the rest of the service are from lp branches that are |
1221 | + compatible with the local charm (e.g. stable or next). |
1222 | + """ |
1223 | + this_service = {'name': 'keystone'} |
1224 | + other_services = [{'name': 'mysql'}, {'name': 'cinder'}] |
1225 | super(KeystoneBasicDeployment, self)._add_services(this_service, |
1226 | other_services) |
1227 | |
1228 | @@ -61,7 +64,7 @@ |
1229 | self.keystone_sentry = self.d.sentry.unit['keystone/0'] |
1230 | self.cinder_sentry = self.d.sentry.unit['cinder/0'] |
1231 | |
1232 | - # Authenticate admin with keystone |
1233 | + # Authenticate keystone admin |
1234 | self.keystone = u.authenticate_keystone_admin(self.keystone_sentry, |
1235 | user='admin', |
1236 | password='openstack', |
1237 | @@ -80,7 +83,7 @@ |
1238 | tenant_id=tenant.id, |
1239 | email='demo@demo.com') |
1240 | |
1241 | - # Authenticate demo user with keystone |
1242 | + # Authenticate keystone demo |
1243 | self.keystone_demo = u.authenticate_keystone_user(self.keystone, |
1244 | user=self.demo_user, |
1245 | password='password', |
1246 | @@ -123,10 +126,8 @@ |
1247 | def test_roles(self): |
1248 | """Verify all existing roles.""" |
1249 | role1 = {'name': 'demoRole', 'id': u.not_null} |
1250 | - role2 = {'name': 'KeystoneAdmin', 'id': u.not_null} |
1251 | - role3 = {'name': 'KeystoneServiceAdmin', 'id': u.not_null} |
1252 | - role4 = {'name': 'Admin', 'id': u.not_null} |
1253 | - expected = [role1, role2, role3, role4] |
1254 | + role2 = {'name': 'Admin', 'id': u.not_null} |
1255 | + expected = [role1, role2] |
1256 | actual = self.keystone.roles.list() |
1257 | |
1258 | ret = u.validate_role_data(expected, actual) |
1259 | @@ -280,11 +281,18 @@ |
1260 | message = u.relation_error('cinder identity-service', ret) |
1261 | amulet.raise_status(amulet.FAIL, msg=message) |
1262 | |
1263 | - def test_restart_on_config_change(self): |
1264 | - """Verify that keystone is restarted when the config is changed.""" |
1265 | + def test_z_restart_on_config_change(self): |
1266 | + """Verify that keystone is restarted when the config is changed. |
1267 | + |
1268 | + Note(coreycb): The method name with the _z_ is a little odd |
1269 | + but it forces the test to run last. It just makes things |
1270 | + easier because restarting services requires re-authorization. |
1271 | + """ |
1272 | self.d.configure('keystone', {'verbose': 'True'}) |
1273 | if not u.service_restarted(self.keystone_sentry, 'keystone-all', |
1274 | - '/etc/keystone/keystone.conf', sleep_time=10): |
1275 | + '/etc/keystone/keystone.conf', |
1276 | + sleep_time=30): |
1277 | + self.d.configure('keystone', {'verbose': 'False'}) |
1278 | message = "keystone service didn't restart after config change" |
1279 | amulet.raise_status(amulet.FAIL, msg=message) |
1280 | self.d.configure('keystone', {'verbose': 'False'}) |
1281 | |
1282 | === modified file 'tests/charmhelpers/contrib/amulet/deployment.py' |
1283 | --- tests/charmhelpers/contrib/amulet/deployment.py 2014-07-30 15:11:18 +0000 |
1284 | +++ tests/charmhelpers/contrib/amulet/deployment.py 2014-09-29 21:16:33 +0000 |
1285 | @@ -24,25 +24,31 @@ |
1286 | """Add services. |
1287 | |
1288 | Add services to the deployment where this_service is the local charm |
1289 | - that we're focused on testing and other_services are the other |
1290 | - charms that come from the charm store. |
1291 | + that we're testing and other_services are the other services that |
1292 | + are being used in the local amulet tests. |
1293 | """ |
1294 | - name, units = range(2) |
1295 | - |
1296 | - if this_service[name] != os.path.basename(os.getcwd()): |
1297 | - s = this_service[name] |
1298 | + if this_service['name'] != os.path.basename(os.getcwd()): |
1299 | + s = this_service['name'] |
1300 | msg = "The charm's root directory name needs to be {}".format(s) |
1301 | amulet.raise_status(amulet.FAIL, msg=msg) |
1302 | |
1303 | - self.d.add(this_service[name], units=this_service[units]) |
1304 | + if 'units' not in this_service: |
1305 | + this_service['units'] = 1 |
1306 | + |
1307 | + self.d.add(this_service['name'], units=this_service['units']) |
1308 | |
1309 | for svc in other_services: |
1310 | - if self.series: |
1311 | - self.d.add(svc[name], |
1312 | - charm='cs:{}/{}'.format(self.series, svc[name]), |
1313 | - units=svc[units]) |
1314 | + if 'location' in svc: |
1315 | + branch_location = svc['location'] |
1316 | + elif self.series: |
1317 | + branch_location = 'cs:{}/{}'.format(self.series, svc['name']), |
1318 | else: |
1319 | - self.d.add(svc[name], units=svc[units]) |
1320 | + branch_location = None |
1321 | + |
1322 | + if 'units' not in svc: |
1323 | + svc['units'] = 1 |
1324 | + |
1325 | + self.d.add(svc['name'], charm=branch_location, units=svc['units']) |
1326 | |
1327 | def _add_relations(self, relations): |
1328 | """Add all of the relations for the services.""" |
1329 | @@ -57,7 +63,7 @@ |
1330 | def _deploy(self): |
1331 | """Deploy environment and wait for all hooks to finish executing.""" |
1332 | try: |
1333 | - self.d.setup() |
1334 | + self.d.setup(timeout=900) |
1335 | self.d.sentry.wait(timeout=900) |
1336 | except amulet.helpers.TimeoutError: |
1337 | amulet.raise_status(amulet.FAIL, msg="Deployment timed out") |
1338 | |
1339 | === modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py' |
1340 | --- tests/charmhelpers/contrib/openstack/amulet/deployment.py 2014-07-30 15:11:18 +0000 |
1341 | +++ tests/charmhelpers/contrib/openstack/amulet/deployment.py 2014-09-29 21:16:33 +0000 |
1342 | @@ -10,32 +10,61 @@ |
1343 | that is specifically for use by OpenStack charms. |
1344 | """ |
1345 | |
1346 | - def __init__(self, series=None, openstack=None, source=None): |
1347 | + def __init__(self, series=None, openstack=None, source=None, stable=True): |
1348 | """Initialize the deployment environment.""" |
1349 | super(OpenStackAmuletDeployment, self).__init__(series) |
1350 | self.openstack = openstack |
1351 | self.source = source |
1352 | + self.stable = stable |
1353 | + # Note(coreycb): this needs to be changed when new next branches come |
1354 | + # out. |
1355 | + self.current_next = "trusty" |
1356 | + |
1357 | + def _determine_branch_locations(self, other_services): |
1358 | + """Determine the branch locations for the other services. |
1359 | + |
1360 | + Determine if the local branch being tested is derived from its |
1361 | + stable or next (dev) branch, and based on this, use the corresonding |
1362 | + stable or next branches for the other_services.""" |
1363 | + base_charms = ['mysql', 'mongodb', 'rabbitmq-server'] |
1364 | + |
1365 | + if self.stable: |
1366 | + for svc in other_services: |
1367 | + temp = 'lp:charms/{}' |
1368 | + svc['location'] = temp.format(svc['name']) |
1369 | + else: |
1370 | + for svc in other_services: |
1371 | + if svc['name'] in base_charms: |
1372 | + temp = 'lp:charms/{}' |
1373 | + svc['location'] = temp.format(svc['name']) |
1374 | + else: |
1375 | + temp = 'lp:~openstack-charmers/charms/{}/{}/next' |
1376 | + svc['location'] = temp.format(self.current_next, |
1377 | + svc['name']) |
1378 | + return other_services |
1379 | |
1380 | def _add_services(self, this_service, other_services): |
1381 | - """Add services to the deployment and set openstack-origin.""" |
1382 | + """Add services to the deployment and set openstack-origin/source.""" |
1383 | + other_services = self._determine_branch_locations(other_services) |
1384 | + |
1385 | super(OpenStackAmuletDeployment, self)._add_services(this_service, |
1386 | other_services) |
1387 | - name = 0 |
1388 | + |
1389 | services = other_services |
1390 | services.append(this_service) |
1391 | use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph'] |
1392 | |
1393 | if self.openstack: |
1394 | for svc in services: |
1395 | - if svc[name] not in use_source: |
1396 | + if svc['name'] not in use_source: |
1397 | config = {'openstack-origin': self.openstack} |
1398 | - self.d.configure(svc[name], config) |
1399 | + self.d.configure(svc['name'], config) |
1400 | |
1401 | if self.source: |
1402 | for svc in services: |
1403 | - if svc[name] in use_source: |
1404 | + if svc['name'] in use_source: |
1405 | config = {'source': self.source} |
1406 | - self.d.configure(svc[name], config) |
1407 | + self.d.configure(svc['name'], config) |
1408 | |
1409 | def _configure_services(self, configs): |
1410 | """Configure all of the services.""" |
1411 | |
1412 | === modified file 'tests/charmhelpers/contrib/openstack/amulet/utils.py' |
1413 | --- tests/charmhelpers/contrib/openstack/amulet/utils.py 2014-07-30 15:11:18 +0000 |
1414 | +++ tests/charmhelpers/contrib/openstack/amulet/utils.py 2014-09-29 21:16:33 +0000 |
1415 | @@ -187,15 +187,16 @@ |
1416 | |
1417 | f = opener.open("http://download.cirros-cloud.net/version/released") |
1418 | version = f.read().strip() |
1419 | - cirros_img = "tests/cirros-{}-x86_64-disk.img".format(version) |
1420 | + cirros_img = "cirros-{}-x86_64-disk.img".format(version) |
1421 | + local_path = os.path.join('tests', cirros_img) |
1422 | |
1423 | - if not os.path.exists(cirros_img): |
1424 | + if not os.path.exists(local_path): |
1425 | cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net", |
1426 | version, cirros_img) |
1427 | - opener.retrieve(cirros_url, cirros_img) |
1428 | + opener.retrieve(cirros_url, local_path) |
1429 | f.close() |
1430 | |
1431 | - with open(cirros_img) as f: |
1432 | + with open(local_path) as f: |
1433 | image = glance.images.create(name=image_name, is_public=True, |
1434 | disk_format='qcow2', |
1435 | container_format='bare', data=f) |