Merge lp:~bloodearnest/charms/trusty/apache2/update-charm-helpers into lp:charms/trusty/apache2
- Trusty Tahr (14.04)
- update-charm-helpers
- Merge into trunk
Proposed by
Simon Davy
Status: | Merged |
---|---|
Merged at revision: | 63 |
Proposed branch: | lp:~bloodearnest/charms/trusty/apache2/update-charm-helpers |
Merge into: | lp:charms/trusty/apache2 |
Diff against target: |
1974 lines (+1090/-276) 16 files modified
Makefile (+1/-1) charm-helpers.yaml (+2/-2) config-manager.txt (+1/-1) hooks/charmhelpers/__init__.py (+38/-0) hooks/charmhelpers/contrib/__init__.py (+15/-0) hooks/charmhelpers/contrib/charmsupport/__init__.py (+15/-0) hooks/charmhelpers/contrib/charmsupport/nrpe.py (+150/-10) hooks/charmhelpers/contrib/charmsupport/volumes.py (+21/-2) hooks/charmhelpers/core/__init__.py (+15/-0) hooks/charmhelpers/core/hookenv.py (+269/-41) hooks/charmhelpers/fetch/__init__.py (+309/-79) hooks/charmhelpers/fetch/archiveurl.py (+121/-8) hooks/charmhelpers/fetch/bzrurl.py (+39/-5) hooks/charmhelpers/fetch/giturl.py (+71/-0) hooks/tests/test_create_vhost.py (+1/-1) hooks/tests/test_nrpe_hooks.py (+22/-126) |
To merge this branch: | bzr merge lp:~bloodearnest/charms/trusty/apache2/update-charm-helpers |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Tom Haddon | Approve | ||
Review via email: mp+252331@code.launchpad.net |
Commit message
Update charm-helpers to latest rev.
Description of the change
Update charm-helpers to latest rev.
Replace old nrpe tests with new simpler tests that actually test the code in this char, not test the implementation details of the charm-helpers library
Fix a lint bug, duplicate test method.
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'Makefile' |
2 | --- Makefile 2014-11-20 00:06:41 +0000 |
3 | +++ Makefile 2015-03-09 16:35:13 +0000 |
4 | @@ -24,7 +24,7 @@ |
5 | |
6 | test: .venv |
7 | @echo Starting tests... |
8 | - @CHARM_DIR=$(CHARM_DIR) $(TEST_PREFIX) .venv/bin/nosetests $(TEST_DIR) |
9 | + @CHARM_DIR=$(CHARM_DIR) $(TEST_PREFIX) .venv/bin/nosetests -s $(TEST_DIR) |
10 | |
11 | lint: |
12 | @echo Checking for Python syntax... |
13 | |
14 | === modified file 'charm-helpers.yaml' |
15 | --- charm-helpers.yaml 2013-10-10 22:47:57 +0000 |
16 | +++ charm-helpers.yaml 2015-03-09 16:35:13 +0000 |
17 | @@ -1,4 +1,4 @@ |
18 | include: |
19 | - - core |
20 | + - core.hookenv |
21 | - fetch |
22 | - - contrib.charmsupport |
23 | \ No newline at end of file |
24 | + - contrib.charmsupport |
25 | |
26 | === modified file 'config-manager.txt' |
27 | --- config-manager.txt 2013-10-10 22:47:57 +0000 |
28 | +++ config-manager.txt 2015-03-09 16:35:13 +0000 |
29 | @@ -3,4 +3,4 @@ |
30 | # |
31 | # make sourcedeps |
32 | |
33 | -./build/charm-helpers lp:charm-helpers;revno=70 |
34 | +./build/charm-helpers lp:charm-helpers;revno=330 |
35 | |
36 | === modified file 'hooks/charmhelpers/__init__.py' |
37 | --- hooks/charmhelpers/__init__.py 2013-10-10 22:47:57 +0000 |
38 | +++ hooks/charmhelpers/__init__.py 2015-03-09 16:35:13 +0000 |
39 | @@ -0,0 +1,38 @@ |
40 | +# Copyright 2014-2015 Canonical Limited. |
41 | +# |
42 | +# This file is part of charm-helpers. |
43 | +# |
44 | +# charm-helpers is free software: you can redistribute it and/or modify |
45 | +# it under the terms of the GNU Lesser General Public License version 3 as |
46 | +# published by the Free Software Foundation. |
47 | +# |
48 | +# charm-helpers is distributed in the hope that it will be useful, |
49 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
50 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
51 | +# GNU Lesser General Public License for more details. |
52 | +# |
53 | +# You should have received a copy of the GNU Lesser General Public License |
54 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
55 | + |
56 | +# Bootstrap charm-helpers, installing its dependencies if necessary using |
57 | +# only standard libraries. |
58 | +import subprocess |
59 | +import sys |
60 | + |
61 | +try: |
62 | + import six # flake8: noqa |
63 | +except ImportError: |
64 | + if sys.version_info.major == 2: |
65 | + subprocess.check_call(['apt-get', 'install', '-y', 'python-six']) |
66 | + else: |
67 | + subprocess.check_call(['apt-get', 'install', '-y', 'python3-six']) |
68 | + import six # flake8: noqa |
69 | + |
70 | +try: |
71 | + import yaml # flake8: noqa |
72 | +except ImportError: |
73 | + if sys.version_info.major == 2: |
74 | + subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml']) |
75 | + else: |
76 | + subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) |
77 | + import yaml # flake8: noqa |
78 | |
79 | === modified file 'hooks/charmhelpers/contrib/__init__.py' |
80 | --- hooks/charmhelpers/contrib/__init__.py 2013-10-10 22:47:57 +0000 |
81 | +++ hooks/charmhelpers/contrib/__init__.py 2015-03-09 16:35:13 +0000 |
82 | @@ -0,0 +1,15 @@ |
83 | +# Copyright 2014-2015 Canonical Limited. |
84 | +# |
85 | +# This file is part of charm-helpers. |
86 | +# |
87 | +# charm-helpers is free software: you can redistribute it and/or modify |
88 | +# it under the terms of the GNU Lesser General Public License version 3 as |
89 | +# published by the Free Software Foundation. |
90 | +# |
91 | +# charm-helpers is distributed in the hope that it will be useful, |
92 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
93 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
94 | +# GNU Lesser General Public License for more details. |
95 | +# |
96 | +# You should have received a copy of the GNU Lesser General Public License |
97 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
98 | |
99 | === modified file 'hooks/charmhelpers/contrib/charmsupport/__init__.py' |
100 | --- hooks/charmhelpers/contrib/charmsupport/__init__.py 2013-10-10 22:47:57 +0000 |
101 | +++ hooks/charmhelpers/contrib/charmsupport/__init__.py 2015-03-09 16:35:13 +0000 |
102 | @@ -0,0 +1,15 @@ |
103 | +# Copyright 2014-2015 Canonical Limited. |
104 | +# |
105 | +# This file is part of charm-helpers. |
106 | +# |
107 | +# charm-helpers is free software: you can redistribute it and/or modify |
108 | +# it under the terms of the GNU Lesser General Public License version 3 as |
109 | +# published by the Free Software Foundation. |
110 | +# |
111 | +# charm-helpers is distributed in the hope that it will be useful, |
112 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
113 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
114 | +# GNU Lesser General Public License for more details. |
115 | +# |
116 | +# You should have received a copy of the GNU Lesser General Public License |
117 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
118 | |
119 | === modified file 'hooks/charmhelpers/contrib/charmsupport/nrpe.py' |
120 | --- hooks/charmhelpers/contrib/charmsupport/nrpe.py 2013-10-10 22:47:57 +0000 |
121 | +++ hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-03-09 16:35:13 +0000 |
122 | @@ -1,3 +1,19 @@ |
123 | +# Copyright 2014-2015 Canonical Limited. |
124 | +# |
125 | +# This file is part of charm-helpers. |
126 | +# |
127 | +# charm-helpers is free software: you can redistribute it and/or modify |
128 | +# it under the terms of the GNU Lesser General Public License version 3 as |
129 | +# published by the Free Software Foundation. |
130 | +# |
131 | +# charm-helpers is distributed in the hope that it will be useful, |
132 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
133 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
134 | +# GNU Lesser General Public License for more details. |
135 | +# |
136 | +# You should have received a copy of the GNU Lesser General Public License |
137 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
138 | + |
139 | """Compatibility with the nrpe-external-master charm""" |
140 | # Copyright 2012 Canonical Ltd. |
141 | # |
142 | @@ -8,6 +24,8 @@ |
143 | import pwd |
144 | import grp |
145 | import os |
146 | +import glob |
147 | +import shutil |
148 | import re |
149 | import shlex |
150 | import yaml |
151 | @@ -18,6 +36,7 @@ |
152 | log, |
153 | relation_ids, |
154 | relation_set, |
155 | + relations_of_type, |
156 | ) |
157 | |
158 | from charmhelpers.core.host import service |
159 | @@ -54,6 +73,12 @@ |
160 | # juju-myservice-0 |
161 | # If you're running multiple environments with the same services in them |
162 | # this allows you to differentiate between them. |
163 | +# nagios_servicegroups: |
164 | +# default: "" |
165 | +# type: string |
166 | +# description: | |
167 | +# A comma-separated list of nagios servicegroups. |
168 | +# If left empty, the nagios_context will be used as the servicegroup |
169 | # |
170 | # 3. Add custom checks (Nagios plugins) to files/nrpe-external-master |
171 | # |
172 | @@ -125,10 +150,8 @@ |
173 | |
174 | def _locate_cmd(self, check_cmd): |
175 | search_path = ( |
176 | - '/', |
177 | - os.path.join(os.environ['CHARM_DIR'], |
178 | - 'files/nrpe-external-master'), |
179 | '/usr/lib/nagios/plugins', |
180 | + '/usr/local/lib/nagios/plugins', |
181 | ) |
182 | parts = shlex.split(check_cmd) |
183 | for path in search_path: |
184 | @@ -140,7 +163,7 @@ |
185 | log('Check command not found: {}'.format(parts[0])) |
186 | return '' |
187 | |
188 | - def write(self, nagios_context, hostname): |
189 | + def write(self, nagios_context, hostname, nagios_servicegroups): |
190 | nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( |
191 | self.command) |
192 | with open(nrpe_check_file, 'w') as nrpe_check_config: |
193 | @@ -152,16 +175,18 @@ |
194 | log('Not writing service config as {} is not accessible'.format( |
195 | NRPE.nagios_exportdir)) |
196 | else: |
197 | - self.write_service_config(nagios_context, hostname) |
198 | + self.write_service_config(nagios_context, hostname, |
199 | + nagios_servicegroups) |
200 | |
201 | - def write_service_config(self, nagios_context, hostname): |
202 | + def write_service_config(self, nagios_context, hostname, |
203 | + nagios_servicegroups): |
204 | for f in os.listdir(NRPE.nagios_exportdir): |
205 | if re.search('.*{}.cfg'.format(self.command), f): |
206 | os.remove(os.path.join(NRPE.nagios_exportdir, f)) |
207 | |
208 | templ_vars = { |
209 | 'nagios_hostname': hostname, |
210 | - 'nagios_servicegroup': nagios_context, |
211 | + 'nagios_servicegroup': nagios_servicegroups, |
212 | 'description': self.description, |
213 | 'shortname': self.shortname, |
214 | 'command': self.command, |
215 | @@ -181,12 +206,19 @@ |
216 | nagios_exportdir = '/var/lib/nagios/export' |
217 | nrpe_confdir = '/etc/nagios/nrpe.d' |
218 | |
219 | - def __init__(self): |
220 | + def __init__(self, hostname=None): |
221 | super(NRPE, self).__init__() |
222 | self.config = config() |
223 | self.nagios_context = self.config['nagios_context'] |
224 | + if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: |
225 | + self.nagios_servicegroups = self.config['nagios_servicegroups'] |
226 | + else: |
227 | + self.nagios_servicegroups = self.nagios_context |
228 | self.unit_name = local_unit().replace('/', '-') |
229 | - self.hostname = "{}-{}".format(self.nagios_context, self.unit_name) |
230 | + if hostname: |
231 | + self.hostname = hostname |
232 | + else: |
233 | + self.hostname = "{}-{}".format(self.nagios_context, self.unit_name) |
234 | self.checks = [] |
235 | |
236 | def add_check(self, *args, **kwargs): |
237 | @@ -207,7 +239,8 @@ |
238 | nrpe_monitors = {} |
239 | monitors = {"monitors": {"remote": {"nrpe": nrpe_monitors}}} |
240 | for nrpecheck in self.checks: |
241 | - nrpecheck.write(self.nagios_context, self.hostname) |
242 | + nrpecheck.write(self.nagios_context, self.hostname, |
243 | + self.nagios_servicegroups) |
244 | nrpe_monitors[nrpecheck.shortname] = { |
245 | "command": nrpecheck.command, |
246 | } |
247 | @@ -216,3 +249,110 @@ |
248 | |
249 | for rid in relation_ids("local-monitors"): |
250 | relation_set(relation_id=rid, monitors=yaml.dump(monitors)) |
251 | + |
252 | + |
253 | +def get_nagios_hostcontext(relation_name='nrpe-external-master'): |
254 | + """ |
255 | + Query relation with nrpe subordinate, return the nagios_host_context |
256 | + |
257 | + :param str relation_name: Name of relation nrpe sub joined to |
258 | + """ |
259 | + for rel in relations_of_type(relation_name): |
260 | + if 'nagios_hostname' in rel: |
261 | + return rel['nagios_host_context'] |
262 | + |
263 | + |
264 | +def get_nagios_hostname(relation_name='nrpe-external-master'): |
265 | + """ |
266 | + Query relation with nrpe subordinate, return the nagios_hostname |
267 | + |
268 | + :param str relation_name: Name of relation nrpe sub joined to |
269 | + """ |
270 | + for rel in relations_of_type(relation_name): |
271 | + if 'nagios_hostname' in rel: |
272 | + return rel['nagios_hostname'] |
273 | + |
274 | + |
275 | +def get_nagios_unit_name(relation_name='nrpe-external-master'): |
276 | + """ |
277 | + Return the nagios unit name prepended with host_context if needed |
278 | + |
279 | + :param str relation_name: Name of relation nrpe sub joined to |
280 | + """ |
281 | + host_context = get_nagios_hostcontext(relation_name) |
282 | + if host_context: |
283 | + unit = "%s:%s" % (host_context, local_unit()) |
284 | + else: |
285 | + unit = local_unit() |
286 | + return unit |
287 | + |
288 | + |
289 | +def add_init_service_checks(nrpe, services, unit_name): |
290 | + """ |
291 | + Add checks for each service in list |
292 | + |
293 | + :param NRPE nrpe: NRPE object to add check to |
294 | + :param list services: List of services to check |
295 | + :param str unit_name: Unit name to use in check description |
296 | + """ |
297 | + for svc in services: |
298 | + upstart_init = '/etc/init/%s.conf' % svc |
299 | + sysv_init = '/etc/init.d/%s' % svc |
300 | + if os.path.exists(upstart_init): |
301 | + nrpe.add_check( |
302 | + shortname=svc, |
303 | + description='process check {%s}' % unit_name, |
304 | + check_cmd='check_upstart_job %s' % svc |
305 | + ) |
306 | + elif os.path.exists(sysv_init): |
307 | + cronpath = '/etc/cron.d/nagios-service-check-%s' % svc |
308 | + cron_file = ('*/5 * * * * root ' |
309 | + '/usr/local/lib/nagios/plugins/check_exit_status.pl ' |
310 | + '-s /etc/init.d/%s status > ' |
311 | + '/var/lib/nagios/service-check-%s.txt\n' % (svc, |
312 | + svc) |
313 | + ) |
314 | + f = open(cronpath, 'w') |
315 | + f.write(cron_file) |
316 | + f.close() |
317 | + nrpe.add_check( |
318 | + shortname=svc, |
319 | + description='process check {%s}' % unit_name, |
320 | + check_cmd='check_status_file.py -f ' |
321 | + '/var/lib/nagios/service-check-%s.txt' % svc, |
322 | + ) |
323 | + |
324 | + |
325 | +def copy_nrpe_checks(): |
326 | + """ |
327 | + Copy the nrpe checks into place |
328 | + |
329 | + """ |
330 | + NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' |
331 | + nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks', |
332 | + 'charmhelpers', 'contrib', 'openstack', |
333 | + 'files') |
334 | + |
335 | + if not os.path.exists(NAGIOS_PLUGINS): |
336 | + os.makedirs(NAGIOS_PLUGINS) |
337 | + for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): |
338 | + if os.path.isfile(fname): |
339 | + shutil.copy2(fname, |
340 | + os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) |
341 | + |
342 | + |
343 | +def add_haproxy_checks(nrpe, unit_name): |
344 | + """ |
345 | + Add checks for each service in list |
346 | + |
347 | + :param NRPE nrpe: NRPE object to add check to |
348 | + :param str unit_name: Unit name to use in check description |
349 | + """ |
350 | + nrpe.add_check( |
351 | + shortname='haproxy_servers', |
352 | + description='Check HAProxy {%s}' % unit_name, |
353 | + check_cmd='check_haproxy.sh') |
354 | + nrpe.add_check( |
355 | + shortname='haproxy_queue', |
356 | + description='Check HAProxy queue depth {%s}' % unit_name, |
357 | + check_cmd='check_haproxy_queue_depth.sh') |
358 | |
359 | === modified file 'hooks/charmhelpers/contrib/charmsupport/volumes.py' |
360 | --- hooks/charmhelpers/contrib/charmsupport/volumes.py 2013-10-10 22:47:57 +0000 |
361 | +++ hooks/charmhelpers/contrib/charmsupport/volumes.py 2015-03-09 16:35:13 +0000 |
362 | @@ -1,8 +1,25 @@ |
363 | +# Copyright 2014-2015 Canonical Limited. |
364 | +# |
365 | +# This file is part of charm-helpers. |
366 | +# |
367 | +# charm-helpers is free software: you can redistribute it and/or modify |
368 | +# it under the terms of the GNU Lesser General Public License version 3 as |
369 | +# published by the Free Software Foundation. |
370 | +# |
371 | +# charm-helpers is distributed in the hope that it will be useful, |
372 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
373 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
374 | +# GNU Lesser General Public License for more details. |
375 | +# |
376 | +# You should have received a copy of the GNU Lesser General Public License |
377 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
378 | + |
379 | ''' |
380 | Functions for managing volumes in juju units. One volume is supported per unit. |
381 | Subordinates may have their own storage, provided it is on its own partition. |
382 | |
383 | -Configuration stanzas: |
384 | +Configuration stanzas:: |
385 | + |
386 | volume-ephemeral: |
387 | type: boolean |
388 | default: true |
389 | @@ -20,7 +37,8 @@ |
390 | is 'true' and no volume-map value is set. Use 'juju set' to set a |
391 | value and 'juju resolved' to complete configuration. |
392 | |
393 | -Usage: |
394 | +Usage:: |
395 | + |
396 | from charmsupport.volumes import configure_volume, VolumeConfigurationError |
397 | from charmsupport.hookenv import log, ERROR |
398 | def post_mount_hook(): |
399 | @@ -34,6 +52,7 @@ |
400 | after_change=post_mount_hook) |
401 | except VolumeConfigurationError: |
402 | log('Storage could not be configured', ERROR) |
403 | + |
404 | ''' |
405 | |
406 | # XXX: Known limitations |
407 | |
408 | === modified file 'hooks/charmhelpers/core/__init__.py' |
409 | --- hooks/charmhelpers/core/__init__.py 2013-10-10 22:47:57 +0000 |
410 | +++ hooks/charmhelpers/core/__init__.py 2015-03-09 16:35:13 +0000 |
411 | @@ -0,0 +1,15 @@ |
412 | +# Copyright 2014-2015 Canonical Limited. |
413 | +# |
414 | +# This file is part of charm-helpers. |
415 | +# |
416 | +# charm-helpers is free software: you can redistribute it and/or modify |
417 | +# it under the terms of the GNU Lesser General Public License version 3 as |
418 | +# published by the Free Software Foundation. |
419 | +# |
420 | +# charm-helpers is distributed in the hope that it will be useful, |
421 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
422 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
423 | +# GNU Lesser General Public License for more details. |
424 | +# |
425 | +# You should have received a copy of the GNU Lesser General Public License |
426 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
427 | |
428 | === modified file 'hooks/charmhelpers/core/hookenv.py' |
429 | --- hooks/charmhelpers/core/hookenv.py 2013-10-10 22:47:57 +0000 |
430 | +++ hooks/charmhelpers/core/hookenv.py 2015-03-09 16:35:13 +0000 |
431 | @@ -1,3 +1,19 @@ |
432 | +# Copyright 2014-2015 Canonical Limited. |
433 | +# |
434 | +# This file is part of charm-helpers. |
435 | +# |
436 | +# charm-helpers is free software: you can redistribute it and/or modify |
437 | +# it under the terms of the GNU Lesser General Public License version 3 as |
438 | +# published by the Free Software Foundation. |
439 | +# |
440 | +# charm-helpers is distributed in the hope that it will be useful, |
441 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
442 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
443 | +# GNU Lesser General Public License for more details. |
444 | +# |
445 | +# You should have received a copy of the GNU Lesser General Public License |
446 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
447 | + |
448 | "Interactions with the Juju environment" |
449 | # Copyright 2013 Canonical Ltd. |
450 | # |
451 | @@ -8,7 +24,14 @@ |
452 | import json |
453 | import yaml |
454 | import subprocess |
455 | -import UserDict |
456 | +import sys |
457 | +from subprocess import CalledProcessError |
458 | + |
459 | +import six |
460 | +if not six.PY3: |
461 | + from UserDict import UserDict |
462 | +else: |
463 | + from collections import UserDict |
464 | |
465 | CRITICAL = "CRITICAL" |
466 | ERROR = "ERROR" |
467 | @@ -21,9 +44,9 @@ |
468 | |
469 | |
470 | def cached(func): |
471 | - ''' Cache return values for multiple executions of func + args |
472 | + """Cache return values for multiple executions of func + args |
473 | |
474 | - For example: |
475 | + For example:: |
476 | |
477 | @cached |
478 | def unit_get(attribute): |
479 | @@ -32,7 +55,7 @@ |
480 | unit_get('test') |
481 | |
482 | will cache the result of unit_get + 'test' for future calls. |
483 | - ''' |
484 | + """ |
485 | def wrapper(*args, **kwargs): |
486 | global cache |
487 | key = str((func, args, kwargs)) |
488 | @@ -46,8 +69,8 @@ |
489 | |
490 | |
491 | def flush(key): |
492 | - ''' Flushes any entries from function cache where the |
493 | - key is found in the function+args ''' |
494 | + """Flushes any entries from function cache where the |
495 | + key is found in the function+args """ |
496 | flush_list = [] |
497 | for item in cache: |
498 | if key in item: |
499 | @@ -57,20 +80,22 @@ |
500 | |
501 | |
502 | def log(message, level=None): |
503 | - "Write a message to the juju log" |
504 | + """Write a message to the juju log""" |
505 | command = ['juju-log'] |
506 | if level: |
507 | command += ['-l', level] |
508 | + if not isinstance(message, six.string_types): |
509 | + message = repr(message) |
510 | command += [message] |
511 | subprocess.call(command) |
512 | |
513 | |
514 | -class Serializable(UserDict.IterableUserDict): |
515 | - "Wrapper, an object that can be serialized to yaml or json" |
516 | +class Serializable(UserDict): |
517 | + """Wrapper, an object that can be serialized to yaml or json""" |
518 | |
519 | def __init__(self, obj): |
520 | # wrap the object |
521 | - UserDict.IterableUserDict.__init__(self) |
522 | + UserDict.__init__(self) |
523 | self.data = obj |
524 | |
525 | def __getattr__(self, attr): |
526 | @@ -96,11 +121,11 @@ |
527 | self.data = state |
528 | |
529 | def json(self): |
530 | - "Serialize the object to json" |
531 | + """Serialize the object to json""" |
532 | return json.dumps(self.data) |
533 | |
534 | def yaml(self): |
535 | - "Serialize the object to yaml" |
536 | + """Serialize the object to yaml""" |
537 | return yaml.dump(self.data) |
538 | |
539 | |
540 | @@ -119,50 +144,181 @@ |
541 | |
542 | |
543 | def in_relation_hook(): |
544 | - "Determine whether we're running in a relation hook" |
545 | + """Determine whether we're running in a relation hook""" |
546 | return 'JUJU_RELATION' in os.environ |
547 | |
548 | |
549 | def relation_type(): |
550 | - "The scope for the current relation hook" |
551 | + """The scope for the current relation hook""" |
552 | return os.environ.get('JUJU_RELATION', None) |
553 | |
554 | |
555 | def relation_id(): |
556 | - "The relation ID for the current relation hook" |
557 | + """The relation ID for the current relation hook""" |
558 | return os.environ.get('JUJU_RELATION_ID', None) |
559 | |
560 | |
561 | def local_unit(): |
562 | - "Local unit ID" |
563 | + """Local unit ID""" |
564 | return os.environ['JUJU_UNIT_NAME'] |
565 | |
566 | |
567 | def remote_unit(): |
568 | - "The remote unit for the current relation hook" |
569 | + """The remote unit for the current relation hook""" |
570 | return os.environ['JUJU_REMOTE_UNIT'] |
571 | |
572 | |
573 | def service_name(): |
574 | - "The name service group this unit belongs to" |
575 | + """The name service group this unit belongs to""" |
576 | return local_unit().split('/')[0] |
577 | |
578 | |
579 | +def hook_name(): |
580 | + """The name of the currently executing hook""" |
581 | + return os.path.basename(sys.argv[0]) |
582 | + |
583 | + |
584 | +class Config(dict): |
585 | + """A dictionary representation of the charm's config.yaml, with some |
586 | + extra features: |
587 | + |
588 | + - See which values in the dictionary have changed since the previous hook. |
589 | + - For values that have changed, see what the previous value was. |
590 | + - Store arbitrary data for use in a later hook. |
591 | + |
592 | + NOTE: Do not instantiate this object directly - instead call |
593 | + ``hookenv.config()``, which will return an instance of :class:`Config`. |
594 | + |
595 | + Example usage:: |
596 | + |
597 | + >>> # inside a hook |
598 | + >>> from charmhelpers.core import hookenv |
599 | + >>> config = hookenv.config() |
600 | + >>> config['foo'] |
601 | + 'bar' |
602 | + >>> # store a new key/value for later use |
603 | + >>> config['mykey'] = 'myval' |
604 | + |
605 | + |
606 | + >>> # user runs `juju set mycharm foo=baz` |
607 | + >>> # now we're inside subsequent config-changed hook |
608 | + >>> config = hookenv.config() |
609 | + >>> config['foo'] |
610 | + 'baz' |
611 | + >>> # test to see if this val has changed since last hook |
612 | + >>> config.changed('foo') |
613 | + True |
614 | + >>> # what was the previous value? |
615 | + >>> config.previous('foo') |
616 | + 'bar' |
617 | + >>> # keys/values that we add are preserved across hooks |
618 | + >>> config['mykey'] |
619 | + 'myval' |
620 | + |
621 | + """ |
622 | + CONFIG_FILE_NAME = '.juju-persistent-config' |
623 | + |
624 | + def __init__(self, *args, **kw): |
625 | + super(Config, self).__init__(*args, **kw) |
626 | + self.implicit_save = True |
627 | + self._prev_dict = None |
628 | + self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) |
629 | + if os.path.exists(self.path): |
630 | + self.load_previous() |
631 | + |
632 | + def __getitem__(self, key): |
633 | + """For regular dict lookups, check the current juju config first, |
634 | + then the previous (saved) copy. This ensures that user-saved values |
635 | + will be returned by a dict lookup. |
636 | + |
637 | + """ |
638 | + try: |
639 | + return dict.__getitem__(self, key) |
640 | + except KeyError: |
641 | + return (self._prev_dict or {})[key] |
642 | + |
643 | + def keys(self): |
644 | + prev_keys = [] |
645 | + if self._prev_dict is not None: |
646 | + prev_keys = self._prev_dict.keys() |
647 | + return list(set(prev_keys + list(dict.keys(self)))) |
648 | + |
649 | + def load_previous(self, path=None): |
650 | + """Load previous copy of config from disk. |
651 | + |
652 | + In normal usage you don't need to call this method directly - it |
653 | + is called automatically at object initialization. |
654 | + |
655 | + :param path: |
656 | + |
657 | + File path from which to load the previous config. If `None`, |
658 | + config is loaded from the default location. If `path` is |
659 | + specified, subsequent `save()` calls will write to the same |
660 | + path. |
661 | + |
662 | + """ |
663 | + self.path = path or self.path |
664 | + with open(self.path) as f: |
665 | + self._prev_dict = json.load(f) |
666 | + |
667 | + def changed(self, key): |
668 | + """Return True if the current value for this key is different from |
669 | + the previous value. |
670 | + |
671 | + """ |
672 | + if self._prev_dict is None: |
673 | + return True |
674 | + return self.previous(key) != self.get(key) |
675 | + |
676 | + def previous(self, key): |
677 | + """Return previous value for this key, or None if there |
678 | + is no previous value. |
679 | + |
680 | + """ |
681 | + if self._prev_dict: |
682 | + return self._prev_dict.get(key) |
683 | + return None |
684 | + |
685 | + def save(self): |
686 | + """Save this config to disk. |
687 | + |
688 | + If the charm is using the :mod:`Services Framework <services.base>` |
689 | + or :meth:'@hook <Hooks.hook>' decorator, this |
690 | + is called automatically at the end of successful hook execution. |
691 | + Otherwise, it should be called directly by user code. |
692 | + |
693 | + To disable automatic saves, set ``implicit_save=False`` on this |
694 | + instance. |
695 | + |
696 | + """ |
697 | + if self._prev_dict: |
698 | + for k, v in six.iteritems(self._prev_dict): |
699 | + if k not in self: |
700 | + self[k] = v |
701 | + with open(self.path, 'w') as f: |
702 | + json.dump(self, f) |
703 | + |
704 | + |
705 | @cached |
706 | def config(scope=None): |
707 | - "Juju charm configuration" |
708 | + """Juju charm configuration""" |
709 | config_cmd_line = ['config-get'] |
710 | if scope is not None: |
711 | config_cmd_line.append(scope) |
712 | config_cmd_line.append('--format=json') |
713 | try: |
714 | - return json.loads(subprocess.check_output(config_cmd_line)) |
715 | + config_data = json.loads( |
716 | + subprocess.check_output(config_cmd_line).decode('UTF-8')) |
717 | + if scope is not None: |
718 | + return config_data |
719 | + return Config(config_data) |
720 | except ValueError: |
721 | return None |
722 | |
723 | |
724 | @cached |
725 | def relation_get(attribute=None, unit=None, rid=None): |
726 | + """Get relation information""" |
727 | _args = ['relation-get', '--format=json'] |
728 | if rid: |
729 | _args.append('-r') |
730 | @@ -171,16 +327,22 @@ |
731 | if unit: |
732 | _args.append(unit) |
733 | try: |
734 | - return json.loads(subprocess.check_output(_args)) |
735 | + return json.loads(subprocess.check_output(_args).decode('UTF-8')) |
736 | except ValueError: |
737 | return None |
738 | - |
739 | - |
740 | -def relation_set(relation_id=None, relation_settings={}, **kwargs): |
741 | + except CalledProcessError as e: |
742 | + if e.returncode == 2: |
743 | + return None |
744 | + raise |
745 | + |
746 | + |
747 | +def relation_set(relation_id=None, relation_settings=None, **kwargs): |
748 | + """Set relation information for the current unit""" |
749 | + relation_settings = relation_settings if relation_settings else {} |
750 | relation_cmd_line = ['relation-set'] |
751 | if relation_id is not None: |
752 | relation_cmd_line.extend(('-r', relation_id)) |
753 | - for k, v in (relation_settings.items() + kwargs.items()): |
754 | + for k, v in (list(relation_settings.items()) + list(kwargs.items())): |
755 | if v is None: |
756 | relation_cmd_line.append('{}='.format(k)) |
757 | else: |
758 | @@ -192,28 +354,30 @@ |
759 | |
760 | @cached |
761 | def relation_ids(reltype=None): |
762 | - "A list of relation_ids" |
763 | + """A list of relation_ids""" |
764 | reltype = reltype or relation_type() |
765 | relid_cmd_line = ['relation-ids', '--format=json'] |
766 | if reltype is not None: |
767 | relid_cmd_line.append(reltype) |
768 | - return json.loads(subprocess.check_output(relid_cmd_line)) or [] |
769 | + return json.loads( |
770 | + subprocess.check_output(relid_cmd_line).decode('UTF-8')) or [] |
771 | return [] |
772 | |
773 | |
774 | @cached |
775 | def related_units(relid=None): |
776 | - "A list of related units" |
777 | + """A list of related units""" |
778 | relid = relid or relation_id() |
779 | units_cmd_line = ['relation-list', '--format=json'] |
780 | if relid is not None: |
781 | units_cmd_line.extend(('-r', relid)) |
782 | - return json.loads(subprocess.check_output(units_cmd_line)) or [] |
783 | + return json.loads( |
784 | + subprocess.check_output(units_cmd_line).decode('UTF-8')) or [] |
785 | |
786 | |
787 | @cached |
788 | def relation_for_unit(unit=None, rid=None): |
789 | - "Get the json represenation of a unit's relation" |
790 | + """Get the json represenation of a unit's relation""" |
791 | unit = unit or remote_unit() |
792 | relation = relation_get(unit=unit, rid=rid) |
793 | for key in relation: |
794 | @@ -225,7 +389,7 @@ |
795 | |
796 | @cached |
797 | def relations_for_id(relid=None): |
798 | - "Get relations of a specific relation ID" |
799 | + """Get relations of a specific relation ID""" |
800 | relation_data = [] |
801 | relid = relid or relation_ids() |
802 | for unit in related_units(relid): |
803 | @@ -237,7 +401,7 @@ |
804 | |
805 | @cached |
806 | def relations_of_type(reltype=None): |
807 | - "Get relations of a specific type" |
808 | + """Get relations of a specific type""" |
809 | relation_data = [] |
810 | reltype = reltype or relation_type() |
811 | for relid in relation_ids(reltype): |
812 | @@ -248,22 +412,33 @@ |
813 | |
814 | |
815 | @cached |
816 | +def metadata(): |
817 | + """Get the current charm metadata.yaml contents as a python object""" |
818 | + with open(os.path.join(charm_dir(), 'metadata.yaml')) as md: |
819 | + return yaml.safe_load(md) |
820 | + |
821 | + |
822 | +@cached |
823 | def relation_types(): |
824 | - "Get a list of relation types supported by this charm" |
825 | - charmdir = os.environ.get('CHARM_DIR', '') |
826 | - mdf = open(os.path.join(charmdir, 'metadata.yaml')) |
827 | - md = yaml.safe_load(mdf) |
828 | + """Get a list of relation types supported by this charm""" |
829 | rel_types = [] |
830 | + md = metadata() |
831 | for key in ('provides', 'requires', 'peers'): |
832 | section = md.get(key) |
833 | if section: |
834 | rel_types.extend(section.keys()) |
835 | - mdf.close() |
836 | return rel_types |
837 | |
838 | |
839 | @cached |
840 | +def charm_name(): |
841 | + """Get the name of the current charm as is specified on metadata.yaml""" |
842 | + return metadata().get('name') |
843 | + |
844 | + |
845 | +@cached |
846 | def relations(): |
847 | + """Get a nested dictionary of relation data for all related units""" |
848 | rels = {} |
849 | for reltype in relation_types(): |
850 | relids = {} |
851 | @@ -277,15 +452,35 @@ |
852 | return rels |
853 | |
854 | |
855 | +@cached |
856 | +def is_relation_made(relation, keys='private-address'): |
857 | + ''' |
858 | + Determine whether a relation is established by checking for |
859 | + presence of key(s). If a list of keys is provided, they |
860 | + must all be present for the relation to be identified as made |
861 | + ''' |
862 | + if isinstance(keys, str): |
863 | + keys = [keys] |
864 | + for r_id in relation_ids(relation): |
865 | + for unit in related_units(r_id): |
866 | + context = {} |
867 | + for k in keys: |
868 | + context[k] = relation_get(k, rid=r_id, |
869 | + unit=unit) |
870 | + if None not in context.values(): |
871 | + return True |
872 | + return False |
873 | + |
874 | + |
875 | def open_port(port, protocol="TCP"): |
876 | - "Open a service network port" |
877 | + """Open a service network port""" |
878 | _args = ['open-port'] |
879 | _args.append('{}/{}'.format(port, protocol)) |
880 | subprocess.check_call(_args) |
881 | |
882 | |
883 | def close_port(port, protocol="TCP"): |
884 | - "Close a service network port" |
885 | + """Close a service network port""" |
886 | _args = ['close-port'] |
887 | _args.append('{}/{}'.format(port, protocol)) |
888 | subprocess.check_call(_args) |
889 | @@ -293,37 +488,69 @@ |
890 | |
891 | @cached |
892 | def unit_get(attribute): |
893 | + """Get the unit ID for the remote unit""" |
894 | _args = ['unit-get', '--format=json', attribute] |
895 | try: |
896 | - return json.loads(subprocess.check_output(_args)) |
897 | + return json.loads(subprocess.check_output(_args).decode('UTF-8')) |
898 | except ValueError: |
899 | return None |
900 | |
901 | |
902 | def unit_private_ip(): |
903 | + """Get this unit's private IP address""" |
904 | return unit_get('private-address') |
905 | |
906 | |
907 | class UnregisteredHookError(Exception): |
908 | + """Raised when an undefined hook is called""" |
909 | pass |
910 | |
911 | |
912 | class Hooks(object): |
913 | - def __init__(self): |
914 | + """A convenient handler for hook functions. |
915 | + |
916 | + Example:: |
917 | + |
918 | + hooks = Hooks() |
919 | + |
920 | + # register a hook, taking its name from the function name |
921 | + @hooks.hook() |
922 | + def install(): |
923 | + pass # your code here |
924 | + |
925 | + # register a hook, providing a custom hook name |
926 | + @hooks.hook("config-changed") |
927 | + def config_changed(): |
928 | + pass # your code here |
929 | + |
930 | + if __name__ == "__main__": |
931 | + # execute a hook based on the name the program is called by |
932 | + hooks.execute(sys.argv) |
933 | + """ |
934 | + |
935 | + def __init__(self, config_save=True): |
936 | super(Hooks, self).__init__() |
937 | self._hooks = {} |
938 | + self._config_save = config_save |
939 | |
940 | def register(self, name, function): |
941 | + """Register a hook""" |
942 | self._hooks[name] = function |
943 | |
944 | def execute(self, args): |
945 | + """Execute a registered hook based on args[0]""" |
946 | hook_name = os.path.basename(args[0]) |
947 | if hook_name in self._hooks: |
948 | self._hooks[hook_name]() |
949 | + if self._config_save: |
950 | + cfg = config() |
951 | + if cfg.implicit_save: |
952 | + cfg.save() |
953 | else: |
954 | raise UnregisteredHookError(hook_name) |
955 | |
956 | def hook(self, *hook_names): |
957 | + """Decorator, registering them as hooks""" |
958 | def wrapper(decorated): |
959 | for hook_name in hook_names: |
960 | self.register(hook_name, decorated) |
961 | @@ -337,4 +564,5 @@ |
962 | |
963 | |
964 | def charm_dir(): |
965 | + """Return the root directory of the current charm""" |
966 | return os.environ.get('CHARM_DIR') |
967 | |
968 | === modified file 'hooks/charmhelpers/fetch/__init__.py' |
969 | --- hooks/charmhelpers/fetch/__init__.py 2013-10-10 22:47:57 +0000 |
970 | +++ hooks/charmhelpers/fetch/__init__.py 2015-03-09 16:35:13 +0000 |
971 | @@ -1,18 +1,39 @@ |
972 | +# Copyright 2014-2015 Canonical Limited. |
973 | +# |
974 | +# This file is part of charm-helpers. |
975 | +# |
976 | +# charm-helpers is free software: you can redistribute it and/or modify |
977 | +# it under the terms of the GNU Lesser General Public License version 3 as |
978 | +# published by the Free Software Foundation. |
979 | +# |
980 | +# charm-helpers is distributed in the hope that it will be useful, |
981 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
982 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
983 | +# GNU Lesser General Public License for more details. |
984 | +# |
985 | +# You should have received a copy of the GNU Lesser General Public License |
986 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
987 | + |
988 | import importlib |
989 | +from tempfile import NamedTemporaryFile |
990 | +import time |
991 | from yaml import safe_load |
992 | from charmhelpers.core.host import ( |
993 | lsb_release |
994 | ) |
995 | -from urlparse import ( |
996 | - urlparse, |
997 | - urlunparse, |
998 | -) |
999 | import subprocess |
1000 | from charmhelpers.core.hookenv import ( |
1001 | config, |
1002 | log, |
1003 | ) |
1004 | -import apt_pkg |
1005 | +import os |
1006 | + |
1007 | +import six |
1008 | +if six.PY3: |
1009 | + from urllib.parse import urlparse, urlunparse |
1010 | +else: |
1011 | + from urlparse import urlparse, urlunparse |
1012 | + |
1013 | |
1014 | CLOUD_ARCHIVE = """# Ubuntu Cloud Archive |
1015 | deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main |
1016 | @@ -20,12 +41,109 @@ |
1017 | PROPOSED_POCKET = """# Proposed |
1018 | deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted |
1019 | """ |
1020 | +CLOUD_ARCHIVE_POCKETS = { |
1021 | + # Folsom |
1022 | + 'folsom': 'precise-updates/folsom', |
1023 | + 'precise-folsom': 'precise-updates/folsom', |
1024 | + 'precise-folsom/updates': 'precise-updates/folsom', |
1025 | + 'precise-updates/folsom': 'precise-updates/folsom', |
1026 | + 'folsom/proposed': 'precise-proposed/folsom', |
1027 | + 'precise-folsom/proposed': 'precise-proposed/folsom', |
1028 | + 'precise-proposed/folsom': 'precise-proposed/folsom', |
1029 | + # Grizzly |
1030 | + 'grizzly': 'precise-updates/grizzly', |
1031 | + 'precise-grizzly': 'precise-updates/grizzly', |
1032 | + 'precise-grizzly/updates': 'precise-updates/grizzly', |
1033 | + 'precise-updates/grizzly': 'precise-updates/grizzly', |
1034 | + 'grizzly/proposed': 'precise-proposed/grizzly', |
1035 | + 'precise-grizzly/proposed': 'precise-proposed/grizzly', |
1036 | + 'precise-proposed/grizzly': 'precise-proposed/grizzly', |
1037 | + # Havana |
1038 | + 'havana': 'precise-updates/havana', |
1039 | + 'precise-havana': 'precise-updates/havana', |
1040 | + 'precise-havana/updates': 'precise-updates/havana', |
1041 | + 'precise-updates/havana': 'precise-updates/havana', |
1042 | + 'havana/proposed': 'precise-proposed/havana', |
1043 | + 'precise-havana/proposed': 'precise-proposed/havana', |
1044 | + 'precise-proposed/havana': 'precise-proposed/havana', |
1045 | + # Icehouse |
1046 | + 'icehouse': 'precise-updates/icehouse', |
1047 | + 'precise-icehouse': 'precise-updates/icehouse', |
1048 | + 'precise-icehouse/updates': 'precise-updates/icehouse', |
1049 | + 'precise-updates/icehouse': 'precise-updates/icehouse', |
1050 | + 'icehouse/proposed': 'precise-proposed/icehouse', |
1051 | + 'precise-icehouse/proposed': 'precise-proposed/icehouse', |
1052 | + 'precise-proposed/icehouse': 'precise-proposed/icehouse', |
1053 | + # Juno |
1054 | + 'juno': 'trusty-updates/juno', |
1055 | + 'trusty-juno': 'trusty-updates/juno', |
1056 | + 'trusty-juno/updates': 'trusty-updates/juno', |
1057 | + 'trusty-updates/juno': 'trusty-updates/juno', |
1058 | + 'juno/proposed': 'trusty-proposed/juno', |
1059 | + 'trusty-juno/proposed': 'trusty-proposed/juno', |
1060 | + 'trusty-proposed/juno': 'trusty-proposed/juno', |
1061 | + # Kilo |
1062 | + 'kilo': 'trusty-updates/kilo', |
1063 | + 'trusty-kilo': 'trusty-updates/kilo', |
1064 | + 'trusty-kilo/updates': 'trusty-updates/kilo', |
1065 | + 'trusty-updates/kilo': 'trusty-updates/kilo', |
1066 | + 'kilo/proposed': 'trusty-proposed/kilo', |
1067 | + 'trusty-kilo/proposed': 'trusty-proposed/kilo', |
1068 | + 'trusty-proposed/kilo': 'trusty-proposed/kilo', |
1069 | +} |
1070 | + |
1071 | +# The order of this list is very important. Handlers should be listed in from |
1072 | +# least- to most-specific URL matching. |
1073 | +FETCH_HANDLERS = ( |
1074 | + 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler', |
1075 | + 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler', |
1076 | + 'charmhelpers.fetch.giturl.GitUrlFetchHandler', |
1077 | +) |
1078 | + |
1079 | +APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. |
1080 | +APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks. |
1081 | +APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. |
1082 | + |
1083 | + |
1084 | +class SourceConfigError(Exception): |
1085 | + pass |
1086 | + |
1087 | + |
1088 | +class UnhandledSource(Exception): |
1089 | + pass |
1090 | + |
1091 | + |
1092 | +class AptLockError(Exception): |
1093 | + pass |
1094 | + |
1095 | + |
1096 | +class BaseFetchHandler(object): |
1097 | + |
1098 | + """Base class for FetchHandler implementations in fetch plugins""" |
1099 | + |
1100 | + def can_handle(self, source): |
1101 | + """Returns True if the source can be handled. Otherwise returns |
1102 | + a string explaining why it cannot""" |
1103 | + return "Wrong source type" |
1104 | + |
1105 | + def install(self, source): |
1106 | + """Try to download and unpack the source. Return the path to the |
1107 | + unpacked files or raise UnhandledSource.""" |
1108 | + raise UnhandledSource("Wrong source type {}".format(source)) |
1109 | + |
1110 | + def parse_url(self, url): |
1111 | + return urlparse(url) |
1112 | + |
1113 | + def base_url(self, url): |
1114 | + """Return url without querystring or fragment""" |
1115 | + parts = list(self.parse_url(url)) |
1116 | + parts[4:] = ['' for i in parts[4:]] |
1117 | + return urlunparse(parts) |
1118 | |
1119 | |
1120 | def filter_installed_packages(packages): |
1121 | """Returns a list of packages that require installation""" |
1122 | - apt_pkg.init() |
1123 | - cache = apt_pkg.Cache() |
1124 | + cache = apt_cache() |
1125 | _pkgs = [] |
1126 | for package in packages: |
1127 | try: |
1128 | @@ -38,41 +156,74 @@ |
1129 | return _pkgs |
1130 | |
1131 | |
1132 | +def apt_cache(in_memory=True): |
1133 | + """Build and return an apt cache""" |
1134 | + import apt_pkg |
1135 | + apt_pkg.init() |
1136 | + if in_memory: |
1137 | + apt_pkg.config.set("Dir::Cache::pkgcache", "") |
1138 | + apt_pkg.config.set("Dir::Cache::srcpkgcache", "") |
1139 | + return apt_pkg.Cache() |
1140 | + |
1141 | + |
1142 | def apt_install(packages, options=None, fatal=False): |
1143 | """Install one or more packages""" |
1144 | - options = options or [] |
1145 | - cmd = ['apt-get', '-y'] |
1146 | + if options is None: |
1147 | + options = ['--option=Dpkg::Options::=--force-confold'] |
1148 | + |
1149 | + cmd = ['apt-get', '--assume-yes'] |
1150 | cmd.extend(options) |
1151 | cmd.append('install') |
1152 | - if isinstance(packages, basestring): |
1153 | + if isinstance(packages, six.string_types): |
1154 | cmd.append(packages) |
1155 | else: |
1156 | cmd.extend(packages) |
1157 | log("Installing {} with options: {}".format(packages, |
1158 | options)) |
1159 | - if fatal: |
1160 | - subprocess.check_call(cmd) |
1161 | + _run_apt_command(cmd, fatal) |
1162 | + |
1163 | + |
1164 | +def apt_upgrade(options=None, fatal=False, dist=False): |
1165 | + """Upgrade all packages""" |
1166 | + if options is None: |
1167 | + options = ['--option=Dpkg::Options::=--force-confold'] |
1168 | + |
1169 | + cmd = ['apt-get', '--assume-yes'] |
1170 | + cmd.extend(options) |
1171 | + if dist: |
1172 | + cmd.append('dist-upgrade') |
1173 | else: |
1174 | - subprocess.call(cmd) |
1175 | + cmd.append('upgrade') |
1176 | + log("Upgrading with options: {}".format(options)) |
1177 | + _run_apt_command(cmd, fatal) |
1178 | |
1179 | |
1180 | def apt_update(fatal=False): |
1181 | """Update local apt cache""" |
1182 | cmd = ['apt-get', 'update'] |
1183 | - if fatal: |
1184 | - subprocess.check_call(cmd) |
1185 | - else: |
1186 | - subprocess.call(cmd) |
1187 | + _run_apt_command(cmd, fatal) |
1188 | |
1189 | |
1190 | def apt_purge(packages, fatal=False): |
1191 | """Purge one or more packages""" |
1192 | - cmd = ['apt-get', '-y', 'purge'] |
1193 | - if isinstance(packages, basestring): |
1194 | + cmd = ['apt-get', '--assume-yes', 'purge'] |
1195 | + if isinstance(packages, six.string_types): |
1196 | cmd.append(packages) |
1197 | else: |
1198 | cmd.extend(packages) |
1199 | log("Purging {}".format(packages)) |
1200 | + _run_apt_command(cmd, fatal) |
1201 | + |
1202 | + |
1203 | +def apt_hold(packages, fatal=False): |
1204 | + """Hold one or more packages""" |
1205 | + cmd = ['apt-mark', 'hold'] |
1206 | + if isinstance(packages, six.string_types): |
1207 | + cmd.append(packages) |
1208 | + else: |
1209 | + cmd.extend(packages) |
1210 | + log("Holding {}".format(packages)) |
1211 | + |
1212 | if fatal: |
1213 | subprocess.check_call(cmd) |
1214 | else: |
1215 | @@ -80,84 +231,145 @@ |
1216 | |
1217 | |
1218 | def add_source(source, key=None): |
1219 | - if ((source.startswith('ppa:') or |
1220 | - source.startswith('http:'))): |
1221 | + """Add a package source to this system. |
1222 | + |
1223 | + @param source: a URL or sources.list entry, as supported by |
1224 | + add-apt-repository(1). Examples:: |
1225 | + |
1226 | + ppa:charmers/example |
1227 | + deb https://stub:key@private.example.com/ubuntu trusty main |
1228 | + |
1229 | + In addition: |
1230 | + 'proposed:' may be used to enable the standard 'proposed' |
1231 | + pocket for the release. |
1232 | + 'cloud:' may be used to activate official cloud archive pockets, |
1233 | + such as 'cloud:icehouse' |
1234 | + 'distro' may be used as a noop |
1235 | + |
1236 | + @param key: A key to be added to the system's APT keyring and used |
1237 | + to verify the signatures on packages. Ideally, this should be an |
1238 | + ASCII format GPG public key including the block headers. A GPG key |
1239 | + id may also be used, but be aware that only insecure protocols are |
1240 | + available to retrieve the actual public key from a public keyserver |
1241 | + placing your Juju environment at risk. ppa and cloud archive keys |
1242 | + are securely added automtically, so sould not be provided. |
1243 | + """ |
1244 | + if source is None: |
1245 | + log('Source is not present. Skipping') |
1246 | + return |
1247 | + |
1248 | + if (source.startswith('ppa:') or |
1249 | + source.startswith('http') or |
1250 | + source.startswith('deb ') or |
1251 | + source.startswith('cloud-archive:')): |
1252 | subprocess.check_call(['add-apt-repository', '--yes', source]) |
1253 | elif source.startswith('cloud:'): |
1254 | apt_install(filter_installed_packages(['ubuntu-cloud-keyring']), |
1255 | fatal=True) |
1256 | pocket = source.split(':')[-1] |
1257 | + if pocket not in CLOUD_ARCHIVE_POCKETS: |
1258 | + raise SourceConfigError( |
1259 | + 'Unsupported cloud: source option %s' % |
1260 | + pocket) |
1261 | + actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] |
1262 | with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: |
1263 | - apt.write(CLOUD_ARCHIVE.format(pocket)) |
1264 | + apt.write(CLOUD_ARCHIVE.format(actual_pocket)) |
1265 | elif source == 'proposed': |
1266 | release = lsb_release()['DISTRIB_CODENAME'] |
1267 | with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: |
1268 | apt.write(PROPOSED_POCKET.format(release)) |
1269 | + elif source == 'distro': |
1270 | + pass |
1271 | + else: |
1272 | + log("Unknown source: {!r}".format(source)) |
1273 | + |
1274 | if key: |
1275 | - subprocess.check_call(['apt-key', 'import', key]) |
1276 | - |
1277 | - |
1278 | -class SourceConfigError(Exception): |
1279 | - pass |
1280 | + if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key: |
1281 | + with NamedTemporaryFile('w+') as key_file: |
1282 | + key_file.write(key) |
1283 | + key_file.flush() |
1284 | + key_file.seek(0) |
1285 | + subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file) |
1286 | + else: |
1287 | + # Note that hkp: is in no way a secure protocol. Using a |
1288 | + # GPG key id is pointless from a security POV unless you |
1289 | + # absolutely trust your network and DNS. |
1290 | + subprocess.check_call(['apt-key', 'adv', '--keyserver', |
1291 | + 'hkp://keyserver.ubuntu.com:80', '--recv', |
1292 | + key]) |
1293 | |
1294 | |
1295 | def configure_sources(update=False, |
1296 | sources_var='install_sources', |
1297 | keys_var='install_keys'): |
1298 | """ |
1299 | - Configure multiple sources from charm configuration |
1300 | + Configure multiple sources from charm configuration. |
1301 | + |
1302 | + The lists are encoded as yaml fragments in the configuration. |
1303 | + The frament needs to be included as a string. Sources and their |
1304 | + corresponding keys are of the types supported by add_source(). |
1305 | |
1306 | Example config: |
1307 | - install_sources: |
1308 | + install_sources: | |
1309 | - "ppa:foo" |
1310 | - "http://example.com/repo precise main" |
1311 | - install_keys: |
1312 | + install_keys: | |
1313 | - null |
1314 | - "a1b2c3d4" |
1315 | |
1316 | Note that 'null' (a.k.a. None) should not be quoted. |
1317 | """ |
1318 | - sources = safe_load(config(sources_var)) |
1319 | - keys = safe_load(config(keys_var)) |
1320 | - if isinstance(sources, basestring) and isinstance(keys, basestring): |
1321 | - add_source(sources, keys) |
1322 | + sources = safe_load((config(sources_var) or '').strip()) or [] |
1323 | + keys = safe_load((config(keys_var) or '').strip()) or None |
1324 | + |
1325 | + if isinstance(sources, six.string_types): |
1326 | + sources = [sources] |
1327 | + |
1328 | + if keys is None: |
1329 | + for source in sources: |
1330 | + add_source(source, None) |
1331 | else: |
1332 | - if not len(sources) == len(keys): |
1333 | - msg = 'Install sources and keys lists are different lengths' |
1334 | - raise SourceConfigError(msg) |
1335 | - for src_num in range(len(sources)): |
1336 | - add_source(sources[src_num], keys[src_num]) |
1337 | + if isinstance(keys, six.string_types): |
1338 | + keys = [keys] |
1339 | + |
1340 | + if len(sources) != len(keys): |
1341 | + raise SourceConfigError( |
1342 | + 'Install sources and keys lists are different lengths') |
1343 | + for source, key in zip(sources, keys): |
1344 | + add_source(source, key) |
1345 | if update: |
1346 | apt_update(fatal=True) |
1347 | |
1348 | -# The order of this list is very important. Handlers should be listed in from |
1349 | -# least- to most-specific URL matching. |
1350 | -FETCH_HANDLERS = ( |
1351 | - 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler', |
1352 | - 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler', |
1353 | -) |
1354 | - |
1355 | - |
1356 | -class UnhandledSource(Exception): |
1357 | - pass |
1358 | - |
1359 | - |
1360 | -def install_remote(source): |
1361 | + |
1362 | +def install_remote(source, *args, **kwargs): |
1363 | """ |
1364 | Install a file tree from a remote source |
1365 | |
1366 | The specified source should be a url of the form: |
1367 | scheme://[host]/path[#[option=value][&...]] |
1368 | |
1369 | - Schemes supported are based on this modules submodules |
1370 | - Options supported are submodule-specific""" |
1371 | + Schemes supported are based on this modules submodules. |
1372 | + Options supported are submodule-specific. |
1373 | + Additional arguments are passed through to the submodule. |
1374 | + |
1375 | + For example:: |
1376 | + |
1377 | + dest = install_remote('http://example.com/archive.tgz', |
1378 | + checksum='deadbeef', |
1379 | + hash_type='sha1') |
1380 | + |
1381 | + This will download `archive.tgz`, validate it using SHA1 and, if |
1382 | + the file is ok, extract it and return the directory in which it |
1383 | + was extracted. If the checksum fails, it will raise |
1384 | + :class:`charmhelpers.core.host.ChecksumError`. |
1385 | + """ |
1386 | # We ONLY check for True here because can_handle may return a string |
1387 | # explaining why it can't handle a given source. |
1388 | handlers = [h for h in plugins() if h.can_handle(source) is True] |
1389 | installed_to = None |
1390 | for handler in handlers: |
1391 | try: |
1392 | - installed_to = handler.install(source) |
1393 | + installed_to = handler.install(source, *args, **kwargs) |
1394 | except UnhandledSource: |
1395 | pass |
1396 | if not installed_to: |
1397 | @@ -171,28 +383,6 @@ |
1398 | return install_remote(source) |
1399 | |
1400 | |
1401 | -class BaseFetchHandler(object): |
1402 | - """Base class for FetchHandler implementations in fetch plugins""" |
1403 | - def can_handle(self, source): |
1404 | - """Returns True if the source can be handled. Otherwise returns |
1405 | - a string explaining why it cannot""" |
1406 | - return "Wrong source type" |
1407 | - |
1408 | - def install(self, source): |
1409 | - """Try to download and unpack the source. Return the path to the |
1410 | - unpacked files or raise UnhandledSource.""" |
1411 | - raise UnhandledSource("Wrong source type {}".format(source)) |
1412 | - |
1413 | - def parse_url(self, url): |
1414 | - return urlparse(url) |
1415 | - |
1416 | - def base_url(self, url): |
1417 | - """Return url without querystring or fragment""" |
1418 | - parts = list(self.parse_url(url)) |
1419 | - parts[4:] = ['' for i in parts[4:]] |
1420 | - return urlunparse(parts) |
1421 | - |
1422 | - |
1423 | def plugins(fetch_handlers=None): |
1424 | if not fetch_handlers: |
1425 | fetch_handlers = FETCH_HANDLERS |
1426 | @@ -200,10 +390,50 @@ |
1427 | for handler_name in fetch_handlers: |
1428 | package, classname = handler_name.rsplit('.', 1) |
1429 | try: |
1430 | - handler_class = getattr(importlib.import_module(package), classname) |
1431 | + handler_class = getattr( |
1432 | + importlib.import_module(package), |
1433 | + classname) |
1434 | plugin_list.append(handler_class()) |
1435 | except (ImportError, AttributeError): |
1436 | # Skip missing plugins so that they can be ommitted from |
1437 | # installation if desired |
1438 | - log("FetchHandler {} not found, skipping plugin".format(handler_name)) |
1439 | + log("FetchHandler {} not found, skipping plugin".format( |
1440 | + handler_name)) |
1441 | return plugin_list |
1442 | + |
1443 | + |
1444 | +def _run_apt_command(cmd, fatal=False): |
1445 | + """ |
1446 | + Run an APT command, checking output and retrying if the fatal flag is set |
1447 | + to True. |
1448 | + |
1449 | + :param: cmd: str: The apt command to run. |
1450 | + :param: fatal: bool: Whether the command's output should be checked and |
1451 | + retried. |
1452 | + """ |
1453 | + env = os.environ.copy() |
1454 | + |
1455 | + if 'DEBIAN_FRONTEND' not in env: |
1456 | + env['DEBIAN_FRONTEND'] = 'noninteractive' |
1457 | + |
1458 | + if fatal: |
1459 | + retry_count = 0 |
1460 | + result = None |
1461 | + |
1462 | + # If the command is considered "fatal", we need to retry if the apt |
1463 | + # lock was not acquired. |
1464 | + |
1465 | + while result is None or result == APT_NO_LOCK: |
1466 | + try: |
1467 | + result = subprocess.check_call(cmd, env=env) |
1468 | + except subprocess.CalledProcessError as e: |
1469 | + retry_count = retry_count + 1 |
1470 | + if retry_count > APT_NO_LOCK_RETRY_COUNT: |
1471 | + raise |
1472 | + result = e.returncode |
1473 | + log("Couldn't acquire DPKG lock. Will retry in {} seconds." |
1474 | + "".format(APT_NO_LOCK_RETRY_DELAY)) |
1475 | + time.sleep(APT_NO_LOCK_RETRY_DELAY) |
1476 | + |
1477 | + else: |
1478 | + subprocess.call(cmd, env=env) |
1479 | |
1480 | === modified file 'hooks/charmhelpers/fetch/archiveurl.py' |
1481 | --- hooks/charmhelpers/fetch/archiveurl.py 2013-10-10 22:47:57 +0000 |
1482 | +++ hooks/charmhelpers/fetch/archiveurl.py 2015-03-09 16:35:13 +0000 |
1483 | @@ -1,5 +1,23 @@ |
1484 | +# Copyright 2014-2015 Canonical Limited. |
1485 | +# |
1486 | +# This file is part of charm-helpers. |
1487 | +# |
1488 | +# charm-helpers is free software: you can redistribute it and/or modify |
1489 | +# it under the terms of the GNU Lesser General Public License version 3 as |
1490 | +# published by the Free Software Foundation. |
1491 | +# |
1492 | +# charm-helpers is distributed in the hope that it will be useful, |
1493 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
1494 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
1495 | +# GNU Lesser General Public License for more details. |
1496 | +# |
1497 | +# You should have received a copy of the GNU Lesser General Public License |
1498 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
1499 | + |
1500 | import os |
1501 | -import urllib2 |
1502 | +import hashlib |
1503 | +import re |
1504 | + |
1505 | from charmhelpers.fetch import ( |
1506 | BaseFetchHandler, |
1507 | UnhandledSource |
1508 | @@ -8,11 +26,54 @@ |
1509 | get_archive_handler, |
1510 | extract, |
1511 | ) |
1512 | -from charmhelpers.core.host import mkdir |
1513 | +from charmhelpers.core.host import mkdir, check_hash |
1514 | + |
1515 | +import six |
1516 | +if six.PY3: |
1517 | + from urllib.request import ( |
1518 | + build_opener, install_opener, urlopen, urlretrieve, |
1519 | + HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, |
1520 | + ) |
1521 | + from urllib.parse import urlparse, urlunparse, parse_qs |
1522 | + from urllib.error import URLError |
1523 | +else: |
1524 | + from urllib import urlretrieve |
1525 | + from urllib2 import ( |
1526 | + build_opener, install_opener, urlopen, |
1527 | + HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, |
1528 | + URLError |
1529 | + ) |
1530 | + from urlparse import urlparse, urlunparse, parse_qs |
1531 | + |
1532 | + |
1533 | +def splituser(host): |
1534 | + '''urllib.splituser(), but six's support of this seems broken''' |
1535 | + _userprog = re.compile('^(.*)@(.*)$') |
1536 | + match = _userprog.match(host) |
1537 | + if match: |
1538 | + return match.group(1, 2) |
1539 | + return None, host |
1540 | + |
1541 | + |
1542 | +def splitpasswd(user): |
1543 | + '''urllib.splitpasswd(), but six's support of this is missing''' |
1544 | + _passwdprog = re.compile('^([^:]*):(.*)$', re.S) |
1545 | + match = _passwdprog.match(user) |
1546 | + if match: |
1547 | + return match.group(1, 2) |
1548 | + return user, None |
1549 | |
1550 | |
1551 | class ArchiveUrlFetchHandler(BaseFetchHandler): |
1552 | - """Handler for archives via generic URLs""" |
1553 | + """ |
1554 | + Handler to download archive files from arbitrary URLs. |
1555 | + |
1556 | + Can fetch from http, https, ftp, and file URLs. |
1557 | + |
1558 | + Can install either tarballs (.tar, .tgz, .tbz2, etc) or zip files. |
1559 | + |
1560 | + Installs the contents of the archive in $CHARM_DIR/fetched/. |
1561 | + """ |
1562 | def can_handle(self, source): |
1563 | url_parts = self.parse_url(source) |
1564 | if url_parts.scheme not in ('http', 'https', 'ftp', 'file'): |
1565 | @@ -22,9 +83,28 @@ |
1566 | return False |
1567 | |
1568 | def download(self, source, dest): |
1569 | + """ |
1570 | + Download an archive file. |
1571 | + |
1572 | + :param str source: URL pointing to an archive file. |
1573 | + :param str dest: Local path location to download archive file to. |
1574 | + """ |
1575 | # propogate all exceptions |
1576 | # URLError, OSError, etc |
1577 | - response = urllib2.urlopen(source) |
1578 | + proto, netloc, path, params, query, fragment = urlparse(source) |
1579 | + if proto in ('http', 'https'): |
1580 | + auth, barehost = splituser(netloc) |
1581 | + if auth is not None: |
1582 | + source = urlunparse((proto, barehost, path, params, query, fragment)) |
1583 | + username, password = splitpasswd(auth) |
1584 | + passman = HTTPPasswordMgrWithDefaultRealm() |
1585 | + # Realm is set to None in add_password to force the username and password |
1586 | + # to be used whatever the realm |
1587 | + passman.add_password(None, source, username, password) |
1588 | + authhandler = HTTPBasicAuthHandler(passman) |
1589 | + opener = build_opener(authhandler) |
1590 | + install_opener(opener) |
1591 | + response = urlopen(source) |
1592 | try: |
1593 | with open(dest, 'w') as dest_file: |
1594 | dest_file.write(response.read()) |
1595 | @@ -33,16 +113,49 @@ |
1596 | os.unlink(dest) |
1597 | raise e |
1598 | |
1599 | - def install(self, source): |
1600 | + # Mandatory file validation via Sha1 or MD5 hashing. |
1601 | + def download_and_validate(self, url, hashsum, validate="sha1"): |
1602 | + tempfile, headers = urlretrieve(url) |
1603 | + check_hash(tempfile, hashsum, validate) |
1604 | + return tempfile |
1605 | + |
1606 | + def install(self, source, dest=None, checksum=None, hash_type='sha1'): |
1607 | + """ |
1608 | + Download and install an archive file, with optional checksum validation. |
1609 | + |
1610 | + The checksum can also be given on the `source` URL's fragment. |
1611 | + For example:: |
1612 | + |
1613 | + handler.install('http://example.com/file.tgz#sha1=deadbeef') |
1614 | + |
1615 | + :param str source: URL pointing to an archive file. |
1616 | + :param str dest: Local destination path to install to. If not given, |
1617 | + installs to `$CHARM_DIR/archives/archive_file_name`. |
1618 | + :param str checksum: If given, validate the archive file after download. |
1619 | + :param str hash_type: Algorithm used to generate `checksum`. |
1620 | + Can be any hash alrgorithm supported by :mod:`hashlib`, |
1621 | + such as md5, sha1, sha256, sha512, etc. |
1622 | + |
1623 | + """ |
1624 | url_parts = self.parse_url(source) |
1625 | dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched') |
1626 | if not os.path.exists(dest_dir): |
1627 | - mkdir(dest_dir, perms=0755) |
1628 | + mkdir(dest_dir, perms=0o755) |
1629 | dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path)) |
1630 | try: |
1631 | self.download(source, dld_file) |
1632 | - except urllib2.URLError as e: |
1633 | + except URLError as e: |
1634 | raise UnhandledSource(e.reason) |
1635 | except OSError as e: |
1636 | raise UnhandledSource(e.strerror) |
1637 | - return extract(dld_file) |
1638 | + options = parse_qs(url_parts.fragment) |
1639 | + for key, value in options.items(): |
1640 | + if not six.PY3: |
1641 | + algorithms = hashlib.algorithms |
1642 | + else: |
1643 | + algorithms = hashlib.algorithms_available |
1644 | + if key in algorithms: |
1645 | + check_hash(dld_file, value, key) |
1646 | + if checksum: |
1647 | + check_hash(dld_file, checksum, hash_type) |
1648 | + return extract(dld_file, dest) |
1649 | |
1650 | === modified file 'hooks/charmhelpers/fetch/bzrurl.py' |
1651 | --- hooks/charmhelpers/fetch/bzrurl.py 2013-10-10 22:47:57 +0000 |
1652 | +++ hooks/charmhelpers/fetch/bzrurl.py 2015-03-09 16:35:13 +0000 |
1653 | @@ -1,11 +1,39 @@ |
1654 | +# Copyright 2014-2015 Canonical Limited. |
1655 | +# |
1656 | +# This file is part of charm-helpers. |
1657 | +# |
1658 | +# charm-helpers is free software: you can redistribute it and/or modify |
1659 | +# it under the terms of the GNU Lesser General Public License version 3 as |
1660 | +# published by the Free Software Foundation. |
1661 | +# |
1662 | +# charm-helpers is distributed in the hope that it will be useful, |
1663 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
1664 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
1665 | +# GNU Lesser General Public License for more details. |
1666 | +# |
1667 | +# You should have received a copy of the GNU Lesser General Public License |
1668 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
1669 | + |
1670 | import os |
1671 | -from bzrlib.branch import Branch |
1672 | from charmhelpers.fetch import ( |
1673 | BaseFetchHandler, |
1674 | UnhandledSource |
1675 | ) |
1676 | from charmhelpers.core.host import mkdir |
1677 | |
1678 | +import six |
1679 | +if six.PY3: |
1680 | + raise ImportError('bzrlib does not support Python3') |
1681 | + |
1682 | +try: |
1683 | + from bzrlib.branch import Branch |
1684 | + from bzrlib import bzrdir, workingtree, errors |
1685 | +except ImportError: |
1686 | + from charmhelpers.fetch import apt_install |
1687 | + apt_install("python-bzrlib") |
1688 | + from bzrlib.branch import Branch |
1689 | + from bzrlib import bzrdir, workingtree, errors |
1690 | + |
1691 | |
1692 | class BzrUrlFetchHandler(BaseFetchHandler): |
1693 | """Handler for bazaar branches via generic and lp URLs""" |
1694 | @@ -25,20 +53,26 @@ |
1695 | from bzrlib.plugin import load_plugins |
1696 | load_plugins() |
1697 | try: |
1698 | + local_branch = bzrdir.BzrDir.create_branch_convenience(dest) |
1699 | + except errors.AlreadyControlDirError: |
1700 | + local_branch = Branch.open(dest) |
1701 | + try: |
1702 | remote_branch = Branch.open(source) |
1703 | - remote_branch.bzrdir.sprout(dest).open_branch() |
1704 | + remote_branch.push(local_branch) |
1705 | + tree = workingtree.WorkingTree.open(dest) |
1706 | + tree.update() |
1707 | except Exception as e: |
1708 | raise e |
1709 | |
1710 | def install(self, source): |
1711 | url_parts = self.parse_url(source) |
1712 | branch_name = url_parts.path.strip("/").split("/")[-1] |
1713 | - dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name) |
1714 | + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", |
1715 | + branch_name) |
1716 | if not os.path.exists(dest_dir): |
1717 | - mkdir(dest_dir, perms=0755) |
1718 | + mkdir(dest_dir, perms=0o755) |
1719 | try: |
1720 | self.branch(source, dest_dir) |
1721 | except OSError as e: |
1722 | raise UnhandledSource(e.strerror) |
1723 | return dest_dir |
1724 | - |
1725 | |
1726 | === added file 'hooks/charmhelpers/fetch/giturl.py' |
1727 | --- hooks/charmhelpers/fetch/giturl.py 1970-01-01 00:00:00 +0000 |
1728 | +++ hooks/charmhelpers/fetch/giturl.py 2015-03-09 16:35:13 +0000 |
1729 | @@ -0,0 +1,71 @@ |
1730 | +# Copyright 2014-2015 Canonical Limited. |
1731 | +# |
1732 | +# This file is part of charm-helpers. |
1733 | +# |
1734 | +# charm-helpers is free software: you can redistribute it and/or modify |
1735 | +# it under the terms of the GNU Lesser General Public License version 3 as |
1736 | +# published by the Free Software Foundation. |
1737 | +# |
1738 | +# charm-helpers is distributed in the hope that it will be useful, |
1739 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
1740 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
1741 | +# GNU Lesser General Public License for more details. |
1742 | +# |
1743 | +# You should have received a copy of the GNU Lesser General Public License |
1744 | +# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
1745 | + |
1746 | +import os |
1747 | +from charmhelpers.fetch import ( |
1748 | + BaseFetchHandler, |
1749 | + UnhandledSource |
1750 | +) |
1751 | +from charmhelpers.core.host import mkdir |
1752 | + |
1753 | +import six |
1754 | +if six.PY3: |
1755 | + raise ImportError('GitPython does not support Python 3') |
1756 | + |
1757 | +try: |
1758 | + from git import Repo |
1759 | +except ImportError: |
1760 | + from charmhelpers.fetch import apt_install |
1761 | + apt_install("python-git") |
1762 | + from git import Repo |
1763 | + |
1764 | +from git.exc import GitCommandError # noqa E402 |
1765 | + |
1766 | + |
1767 | +class GitUrlFetchHandler(BaseFetchHandler): |
1768 | + """Handler for git branches via generic and github URLs""" |
1769 | + def can_handle(self, source): |
1770 | + url_parts = self.parse_url(source) |
1771 | + # TODO (mattyw) no support for ssh git@ yet |
1772 | + if url_parts.scheme not in ('http', 'https', 'git'): |
1773 | + return False |
1774 | + else: |
1775 | + return True |
1776 | + |
1777 | + def clone(self, source, dest, branch): |
1778 | + if not self.can_handle(source): |
1779 | + raise UnhandledSource("Cannot handle {}".format(source)) |
1780 | + |
1781 | + repo = Repo.clone_from(source, dest) |
1782 | + repo.git.checkout(branch) |
1783 | + |
1784 | + def install(self, source, branch="master", dest=None): |
1785 | + url_parts = self.parse_url(source) |
1786 | + branch_name = url_parts.path.strip("/").split("/")[-1] |
1787 | + if dest: |
1788 | + dest_dir = os.path.join(dest, branch_name) |
1789 | + else: |
1790 | + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", |
1791 | + branch_name) |
1792 | + if not os.path.exists(dest_dir): |
1793 | + mkdir(dest_dir, perms=0o755) |
1794 | + try: |
1795 | + self.clone(source, dest_dir, branch) |
1796 | + except GitCommandError as e: |
1797 | + raise UnhandledSource(e.message) |
1798 | + except OSError as e: |
1799 | + raise UnhandledSource(e.strerror) |
1800 | + return dest_dir |
1801 | |
1802 | === modified file 'hooks/tests/test_create_vhost.py' |
1803 | --- hooks/tests/test_create_vhost.py 2015-02-27 13:48:16 +0000 |
1804 | +++ hooks/tests/test_create_vhost.py 2015-03-09 16:35:13 +0000 |
1805 | @@ -109,7 +109,7 @@ |
1806 | @patch('hooks.site_filename') |
1807 | @patch('hooks.open_port') |
1808 | @patch('hooks.subprocess.call') |
1809 | - def test_create_vhost_template_config( |
1810 | + def test_create_vhost_template_config_template_vars( |
1811 | self, mock_call, mock_open_port, mock_site_filename, |
1812 | mock_close_port): |
1813 | """Template passed in as config setting.""" |
1814 | |
1815 | === modified file 'hooks/tests/test_nrpe_hooks.py' |
1816 | --- hooks/tests/test_nrpe_hooks.py 2014-11-20 00:06:41 +0000 |
1817 | +++ hooks/tests/test_nrpe_hooks.py 2015-03-09 16:35:13 +0000 |
1818 | @@ -1,134 +1,30 @@ |
1819 | -import os |
1820 | -import grp |
1821 | -import pwd |
1822 | -import subprocess |
1823 | from testtools import TestCase |
1824 | -from mock import patch, call |
1825 | +from mock import patch |
1826 | |
1827 | import hooks |
1828 | -from charmhelpers.contrib.charmsupport import nrpe |
1829 | -from charmhelpers.core.hookenv import Serializable |
1830 | |
1831 | |
1832 | class NRPERelationTest(TestCase): |
1833 | - """Tests for the update_nrpe_checks hook. |
1834 | - |
1835 | - Half of this is already tested in the tests for charmsupport.nrpe, but |
1836 | - as the hook in the charm pre-dates that, the tests are left here to ensure |
1837 | - backwards-compatibility. |
1838 | - |
1839 | - """ |
1840 | - patches = { |
1841 | - 'config': {'object': nrpe}, |
1842 | - 'log': {'object': nrpe}, |
1843 | - 'getpwnam': {'object': pwd}, |
1844 | - 'getgrnam': {'object': grp}, |
1845 | - 'mkdir': {'object': os}, |
1846 | - 'chown': {'object': os}, |
1847 | - 'exists': {'object': os.path}, |
1848 | - 'listdir': {'object': os}, |
1849 | - 'remove': {'object': os}, |
1850 | - 'open': {'object': nrpe, 'create': True}, |
1851 | - 'isfile': {'object': os.path}, |
1852 | - 'call': {'object': subprocess}, |
1853 | - 'relation_ids': {'object': nrpe}, |
1854 | - 'relation_set': {'object': nrpe}, |
1855 | - } |
1856 | - |
1857 | - def setUp(self): |
1858 | - super(NRPERelationTest, self).setUp() |
1859 | - self.patched = {} |
1860 | - # Mock the universe. |
1861 | - for attr, data in self.patches.items(): |
1862 | - create = data.get('create', False) |
1863 | - patcher = patch.object(data['object'], attr, create=create) |
1864 | - self.patched[attr] = patcher.start() |
1865 | - self.addCleanup(patcher.stop) |
1866 | - if 'JUJU_UNIT_NAME' not in os.environ: |
1867 | - os.environ['JUJU_UNIT_NAME'] = 'test' |
1868 | - |
1869 | - def check_call_counts(self, **kwargs): |
1870 | - for attr, expected in kwargs.items(): |
1871 | - patcher = self.patched[attr] |
1872 | - self.assertEqual(expected, patcher.call_count, attr) |
1873 | - |
1874 | - def test_update_nrpe_no_nagios_bails(self): |
1875 | - config = {'nagios_context': 'test'} |
1876 | - self.patched['config'].return_value = Serializable(config) |
1877 | - self.patched['getpwnam'].side_effect = KeyError |
1878 | - |
1879 | - self.assertEqual(None, hooks.update_nrpe_checks()) |
1880 | - |
1881 | - expected = 'Nagios user not set up, nrpe checks not updated' |
1882 | - self.patched['log'].assert_called_once_with(expected) |
1883 | - self.check_call_counts(log=1, config=1, getpwnam=1) |
1884 | - |
1885 | - def test_update_nrpe_removes_existing_config(self): |
1886 | - config = { |
1887 | - 'nagios_context': 'test', |
1888 | - 'nagios_check_http_params': '-u http://example.com/url', |
1889 | - } |
1890 | - self.patched['config'].return_value = Serializable(config) |
1891 | - self.patched['exists'].return_value = True |
1892 | - self.patched['listdir'].return_value = [ |
1893 | - 'foo', 'bar.cfg', 'check_vhost.cfg'] |
1894 | - |
1895 | - self.assertEqual(None, hooks.update_nrpe_checks()) |
1896 | - |
1897 | - expected = '/var/lib/nagios/export/check_vhost.cfg' |
1898 | - self.patched['remove'].assert_called_once_with(expected) |
1899 | - self.check_call_counts(config=1, getpwnam=1, getgrnam=1, |
1900 | - exists=3, remove=1, open=2, listdir=1) |
1901 | - |
1902 | - def test_update_nrpe_with_check_url(self): |
1903 | - config = { |
1904 | - 'nagios_context': 'test', |
1905 | + """Tests for the update_nrpe_checks hook.""" |
1906 | + |
1907 | + @patch('hooks.nrpe.NRPE') |
1908 | + def test_update_nrpe_with_check(self, mock_nrpe): |
1909 | + nrpe = mock_nrpe.return_value |
1910 | + nrpe.config = { |
1911 | 'nagios_check_http_params': '-u foo -H bar', |
1912 | } |
1913 | - self.patched['config'].return_value = Serializable(config) |
1914 | - self.patched['exists'].return_value = True |
1915 | - self.patched['isfile'].return_value = False |
1916 | - |
1917 | - self.assertEqual(None, hooks.update_nrpe_checks()) |
1918 | - self.assertEqual(2, self.patched['open'].call_count) |
1919 | - filename = 'check_vhost.cfg' |
1920 | - |
1921 | - service_file_contents = """ |
1922 | -#--------------------------------------------------- |
1923 | -# This file is Juju managed |
1924 | -#--------------------------------------------------- |
1925 | -define service { |
1926 | - use active-service |
1927 | - host_name test-test |
1928 | - service_description test-test[vhost] Check Virtual Host |
1929 | - check_command check_nrpe!check_vhost |
1930 | - servicegroups test |
1931 | -} |
1932 | -""" |
1933 | - self.patched['open'].assert_has_calls( |
1934 | - [call('/etc/nagios/nrpe.d/%s' % filename, 'w'), |
1935 | - call('/var/lib/nagios/export/service__test-test_%s' % |
1936 | - filename, 'w'), |
1937 | - call().__enter__().write(service_file_contents), |
1938 | - call().__enter__().write('# check vhost\n'), |
1939 | - call().__enter__().write( |
1940 | - 'command[check_vhost]=/check_http -u foo -H bar\n')], |
1941 | - any_order=True) |
1942 | - |
1943 | - self.check_call_counts(config=1, getpwnam=1, getgrnam=1, |
1944 | - exists=3, open=2, listdir=1) |
1945 | - |
1946 | - def test_update_nrpe_restarts_service(self): |
1947 | - config = { |
1948 | - 'nagios_context': 'test', |
1949 | - 'nagios_check_http_params': '-u foo -p 3128' |
1950 | - } |
1951 | - self.patched['config'].return_value = Serializable(config) |
1952 | - self.patched['exists'].return_value = True |
1953 | - |
1954 | - self.assertEqual(None, hooks.update_nrpe_checks()) |
1955 | - |
1956 | - expected = ['service', 'nagios-nrpe-server', 'restart'] |
1957 | - self.assertEqual(expected, self.patched['call'].call_args[0][0]) |
1958 | - self.check_call_counts(config=1, getpwnam=1, getgrnam=1, |
1959 | - exists=3, open=2, listdir=1, call=1) |
1960 | + hooks.update_nrpe_checks() |
1961 | + nrpe.add_check.assert_called_once_with( |
1962 | + shortname='vhost', |
1963 | + description='Check Virtual Host', |
1964 | + check_cmd='check_http -u foo -H bar' |
1965 | + ) |
1966 | + nrpe.write.assert_called_once_with() |
1967 | + |
1968 | + @patch('hooks.nrpe.NRPE') |
1969 | + def test_update_nrpe_no_check(self, mock_nrpe): |
1970 | + nrpe = mock_nrpe.return_value |
1971 | + nrpe.config = {} |
1972 | + hooks.update_nrpe_checks() |
1973 | + self.assertFalse(nrpe.add_check.called) |
1974 | + nrpe.write.assert_called_once_with() |
Looks like a pretty self-contained update, and tests still pass, so I'll approve and merge