Merge ~barryprice/charm-nrpe/+git/nrpe-charm:barryprice into ~nrpe-charmers/charm-nrpe:master
- Git
- lp:~barryprice/charm-nrpe/+git/nrpe-charm
- barryprice
- Merge into master
Proposed by
Barry Price
Status: | Merged |
---|---|
Approved by: | Paul Gear |
Approved revision: | 0ac9aba714497c262e47a6f8c2516685b67e0bed |
Merged at revision: | 1eca699520ad627d89ba08fce9ca5094b4f1df17 |
Proposed branch: | ~barryprice/charm-nrpe/+git/nrpe-charm:barryprice |
Merge into: | ~nrpe-charmers/charm-nrpe:master |
Diff against target: |
884 lines (+202/-387) 8 files modified
hooks/charmhelpers/core/hookenv.py (+61/-0) hooks/charmhelpers/core/host.py (+75/-64) hooks/charmhelpers/core/kernel.py (+21/-15) hooks/charmhelpers/fetch/__init__.py (+27/-297) hooks/charmhelpers/fetch/bzrurl.py (+4/-3) hooks/charmhelpers/fetch/giturl.py (+4/-3) tests/charmhelpers/contrib/amulet/deployment.py (+7/-3) tests/charmhelpers/contrib/amulet/utils.py (+3/-2) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Paul Gear (community) | Approve | ||
Review via email: mp+316423@code.launchpad.net |
Commit message
Charmhelpers sync
Description of the change
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py |
2 | index 48b2b9d..e44e22b 100644 |
3 | --- a/hooks/charmhelpers/core/hookenv.py |
4 | +++ b/hooks/charmhelpers/core/hookenv.py |
5 | @@ -332,6 +332,8 @@ def config(scope=None): |
6 | config_cmd_line = ['config-get'] |
7 | if scope is not None: |
8 | config_cmd_line.append(scope) |
9 | + else: |
10 | + config_cmd_line.append('--all') |
11 | config_cmd_line.append('--format=json') |
12 | try: |
13 | config_data = json.loads( |
14 | @@ -614,6 +616,20 @@ def close_port(port, protocol="TCP"): |
15 | subprocess.check_call(_args) |
16 | |
17 | |
18 | +def open_ports(start, end, protocol="TCP"): |
19 | + """Opens a range of service network ports""" |
20 | + _args = ['open-port'] |
21 | + _args.append('{}-{}/{}'.format(start, end, protocol)) |
22 | + subprocess.check_call(_args) |
23 | + |
24 | + |
25 | +def close_ports(start, end, protocol="TCP"): |
26 | + """Close a range of service network ports""" |
27 | + _args = ['close-port'] |
28 | + _args.append('{}-{}/{}'.format(start, end, protocol)) |
29 | + subprocess.check_call(_args) |
30 | + |
31 | + |
32 | @cached |
33 | def unit_get(attribute): |
34 | """Get the unit ID for the remote unit""" |
35 | @@ -843,6 +859,20 @@ def translate_exc(from_exc, to_exc): |
36 | return inner_translate_exc1 |
37 | |
38 | |
39 | +def application_version_set(version): |
40 | + """Charm authors may trigger this command from any hook to output what |
41 | + version of the application is running. This could be a package version, |
42 | + for instance postgres version 9.5. It could also be a build number or |
43 | + version control revision identifier, for instance git sha 6fb7ba68. """ |
44 | + |
45 | + cmd = ['application-version-set'] |
46 | + cmd.append(version) |
47 | + try: |
48 | + subprocess.check_call(cmd) |
49 | + except OSError: |
50 | + log("Application Version: {}".format(version)) |
51 | + |
52 | + |
53 | @translate_exc(from_exc=OSError, to_exc=NotImplementedError) |
54 | def is_leader(): |
55 | """Does the current unit hold the juju leadership |
56 | @@ -1005,3 +1035,34 @@ def network_get_primary_address(binding): |
57 | ''' |
58 | cmd = ['network-get', '--primary-address', binding] |
59 | return subprocess.check_output(cmd).decode('UTF-8').strip() |
60 | + |
61 | + |
62 | +def add_metric(*args, **kwargs): |
63 | + """Add metric values. Values may be expressed with keyword arguments. For |
64 | + metric names containing dashes, these may be expressed as one or more |
65 | + 'key=value' positional arguments. May only be called from the collect-metrics |
66 | + hook.""" |
67 | + _args = ['add-metric'] |
68 | + _kvpairs = [] |
69 | + _kvpairs.extend(args) |
70 | + _kvpairs.extend(['{}={}'.format(k, v) for k, v in kwargs.items()]) |
71 | + _args.extend(sorted(_kvpairs)) |
72 | + try: |
73 | + subprocess.check_call(_args) |
74 | + return |
75 | + except EnvironmentError as e: |
76 | + if e.errno != errno.ENOENT: |
77 | + raise |
78 | + log_message = 'add-metric failed: {}'.format(' '.join(_kvpairs)) |
79 | + log(log_message, level='INFO') |
80 | + |
81 | + |
82 | +def meter_status(): |
83 | + """Get the meter status, if running in the meter-status-changed hook.""" |
84 | + return os.environ.get('JUJU_METER_STATUS') |
85 | + |
86 | + |
87 | +def meter_info(): |
88 | + """Get the meter status information, if running in the meter-status-changed |
89 | + hook.""" |
90 | + return os.environ.get('JUJU_METER_INFO') |
91 | diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py |
92 | index 5306859..3638e65 100644 |
93 | --- a/hooks/charmhelpers/core/host.py |
94 | +++ b/hooks/charmhelpers/core/host.py |
95 | @@ -30,14 +30,31 @@ import subprocess |
96 | import hashlib |
97 | import functools |
98 | import itertools |
99 | -from contextlib import contextmanager |
100 | -from collections import OrderedDict |
101 | - |
102 | import six |
103 | |
104 | +from contextlib import contextmanager |
105 | +from collections import OrderedDict |
106 | from .hookenv import log |
107 | from .fstab import Fstab |
108 | - |
109 | +from charmhelpers.osplatform import get_platform |
110 | + |
111 | +__platform__ = get_platform() |
112 | +if __platform__ == "ubuntu": |
113 | + from charmhelpers.core.host_factory.ubuntu import ( |
114 | + service_available, |
115 | + add_new_group, |
116 | + lsb_release, |
117 | + cmp_pkgrevno, |
118 | + ) # flake8: noqa -- ignore F401 for this import |
119 | +elif __platform__ == "centos": |
120 | + from charmhelpers.core.host_factory.centos import ( |
121 | + service_available, |
122 | + add_new_group, |
123 | + lsb_release, |
124 | + cmp_pkgrevno, |
125 | + ) # flake8: noqa -- ignore F401 for this import |
126 | + |
127 | +UPDATEDB_PATH = '/etc/updatedb.conf' |
128 | |
129 | def service_start(service_name): |
130 | """Start a system service""" |
131 | @@ -144,8 +161,11 @@ def service_running(service_name): |
132 | return False |
133 | else: |
134 | # This works for upstart scripts where the 'service' command |
135 | - # returns a consistent string to represent running 'start/running' |
136 | - if "start/running" in output: |
137 | + # returns a consistent string to represent running |
138 | + # 'start/running' |
139 | + if ("start/running" in output or |
140 | + "is running" in output or |
141 | + "up and running" in output): |
142 | return True |
143 | elif os.path.exists(_INIT_D_CONF.format(service_name)): |
144 | # Check System V scripts init script return codes |
145 | @@ -153,18 +173,6 @@ def service_running(service_name): |
146 | return False |
147 | |
148 | |
149 | -def service_available(service_name): |
150 | - """Determine whether a system service is available""" |
151 | - try: |
152 | - subprocess.check_output( |
153 | - ['service', service_name, 'status'], |
154 | - stderr=subprocess.STDOUT).decode('UTF-8') |
155 | - except subprocess.CalledProcessError as e: |
156 | - return b'unrecognized service' not in e.output |
157 | - else: |
158 | - return True |
159 | - |
160 | - |
161 | SYSTEMD_SYSTEM = '/run/systemd/system' |
162 | |
163 | |
164 | @@ -173,8 +181,9 @@ def init_is_systemd(): |
165 | return os.path.isdir(SYSTEMD_SYSTEM) |
166 | |
167 | |
168 | -def adduser(username, password=None, shell='/bin/bash', system_user=False, |
169 | - primary_group=None, secondary_groups=None, uid=None, home_dir=None): |
170 | +def adduser(username, password=None, shell='/bin/bash', |
171 | + system_user=False, primary_group=None, |
172 | + secondary_groups=None, uid=None, home_dir=None): |
173 | """Add a user to the system. |
174 | |
175 | Will log but otherwise succeed if the user already exists. |
176 | @@ -286,17 +295,7 @@ def add_group(group_name, system_group=False, gid=None): |
177 | log('group with gid {0} already exists!'.format(gid)) |
178 | except KeyError: |
179 | log('creating group {0}'.format(group_name)) |
180 | - cmd = ['addgroup'] |
181 | - if gid: |
182 | - cmd.extend(['--gid', str(gid)]) |
183 | - if system_group: |
184 | - cmd.append('--system') |
185 | - else: |
186 | - cmd.extend([ |
187 | - '--group', |
188 | - ]) |
189 | - cmd.append(group_name) |
190 | - subprocess.check_call(cmd) |
191 | + add_new_group(group_name, system_group, gid) |
192 | group_info = grp.getgrnam(group_name) |
193 | return group_info |
194 | |
195 | @@ -308,15 +307,17 @@ def add_user_to_group(username, group): |
196 | subprocess.check_call(cmd) |
197 | |
198 | |
199 | -def rsync(from_path, to_path, flags='-r', options=None): |
200 | +def rsync(from_path, to_path, flags='-r', options=None, timeout=None): |
201 | """Replicate the contents of a path""" |
202 | options = options or ['--delete', '--executability'] |
203 | cmd = ['/usr/bin/rsync', flags] |
204 | + if timeout: |
205 | + cmd = ['timeout', str(timeout)] + cmd |
206 | cmd.extend(options) |
207 | cmd.append(from_path) |
208 | cmd.append(to_path) |
209 | log(" ".join(cmd)) |
210 | - return subprocess.check_output(cmd).decode('UTF-8').strip() |
211 | + return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip() |
212 | |
213 | |
214 | def symlink(source, destination): |
215 | @@ -541,16 +542,6 @@ def restart_on_change_helper(lambda_f, restart_map, stopstart=False, |
216 | return r |
217 | |
218 | |
219 | -def lsb_release(): |
220 | - """Return /etc/lsb-release in a dict""" |
221 | - d = {} |
222 | - with open('/etc/lsb-release', 'r') as lsb: |
223 | - for l in lsb: |
224 | - k, v = l.split('=') |
225 | - d[k.strip()] = v.strip() |
226 | - return d |
227 | - |
228 | - |
229 | def pwgen(length=None): |
230 | """Generate a random pasword.""" |
231 | if length is None: |
232 | @@ -674,25 +665,6 @@ def get_nic_hwaddr(nic): |
233 | return hwaddr |
234 | |
235 | |
236 | -def cmp_pkgrevno(package, revno, pkgcache=None): |
237 | - """Compare supplied revno with the revno of the installed package |
238 | - |
239 | - * 1 => Installed revno is greater than supplied arg |
240 | - * 0 => Installed revno is the same as supplied arg |
241 | - * -1 => Installed revno is less than supplied arg |
242 | - |
243 | - This function imports apt_cache function from charmhelpers.fetch if |
244 | - the pkgcache argument is None. Be sure to add charmhelpers.fetch if |
245 | - you call this function, or pass an apt_pkg.Cache() instance. |
246 | - """ |
247 | - import apt_pkg |
248 | - if not pkgcache: |
249 | - from charmhelpers.fetch import apt_cache |
250 | - pkgcache = apt_cache() |
251 | - pkg = pkgcache[package] |
252 | - return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) |
253 | - |
254 | - |
255 | @contextmanager |
256 | def chdir(directory): |
257 | """Change the current working directory to a different directory for a code |
258 | @@ -715,7 +687,7 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False): |
259 | :param str path: The string path to start changing ownership. |
260 | :param str owner: The owner string to use when looking up the uid. |
261 | :param str group: The group string to use when looking up the gid. |
262 | - :param bool follow_links: Also Chown links if True |
263 | + :param bool follow_links: Also follow and chown links if True |
264 | :param bool chowntopdir: Also chown path itself if True |
265 | """ |
266 | uid = pwd.getpwnam(owner).pw_uid |
267 | @@ -729,7 +701,7 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False): |
268 | broken_symlink = os.path.lexists(path) and not os.path.exists(path) |
269 | if not broken_symlink: |
270 | chown(path, uid, gid) |
271 | - for root, dirs, files in os.walk(path): |
272 | + for root, dirs, files in os.walk(path, followlinks=follow_links): |
273 | for name in dirs + files: |
274 | full = os.path.join(root, name) |
275 | broken_symlink = os.path.lexists(full) and not os.path.exists(full) |
276 | @@ -763,3 +735,42 @@ def get_total_ram(): |
277 | assert unit == 'kB', 'Unknown unit' |
278 | return int(value) * 1024 # Classic, not KiB. |
279 | raise NotImplementedError() |
280 | + |
281 | + |
282 | +UPSTART_CONTAINER_TYPE = '/run/container_type' |
283 | + |
284 | + |
285 | +def is_container(): |
286 | + """Determine whether unit is running in a container |
287 | + |
288 | + @return: boolean indicating if unit is in a container |
289 | + """ |
290 | + if init_is_systemd(): |
291 | + # Detect using systemd-detect-virt |
292 | + return subprocess.call(['systemd-detect-virt', |
293 | + '--container']) == 0 |
294 | + else: |
295 | + # Detect using upstart container file marker |
296 | + return os.path.exists(UPSTART_CONTAINER_TYPE) |
297 | + |
298 | + |
299 | +def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH): |
300 | + with open(updatedb_path, 'r+') as f_id: |
301 | + updatedb_text = f_id.read() |
302 | + output = updatedb(updatedb_text, path) |
303 | + f_id.seek(0) |
304 | + f_id.write(output) |
305 | + f_id.truncate() |
306 | + |
307 | + |
308 | +def updatedb(updatedb_text, new_path): |
309 | + lines = [line for line in updatedb_text.split("\n")] |
310 | + for i, line in enumerate(lines): |
311 | + if line.startswith("PRUNEPATHS="): |
312 | + paths_line = line.split("=")[1].replace('"', '') |
313 | + paths = paths_line.split(" ") |
314 | + if new_path not in paths: |
315 | + paths.append(new_path) |
316 | + lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths)) |
317 | + output = "\n".join(lines) |
318 | + return output |
319 | diff --git a/hooks/charmhelpers/core/kernel.py b/hooks/charmhelpers/core/kernel.py |
320 | index b166efe..2d40452 100644 |
321 | --- a/hooks/charmhelpers/core/kernel.py |
322 | +++ b/hooks/charmhelpers/core/kernel.py |
323 | @@ -15,15 +15,28 @@ |
324 | # See the License for the specific language governing permissions and |
325 | # limitations under the License. |
326 | |
327 | -__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>" |
328 | +import re |
329 | +import subprocess |
330 | |
331 | +from charmhelpers.osplatform import get_platform |
332 | from charmhelpers.core.hookenv import ( |
333 | log, |
334 | INFO |
335 | ) |
336 | |
337 | -from subprocess import check_call, check_output |
338 | -import re |
339 | +__platform__ = get_platform() |
340 | +if __platform__ == "ubuntu": |
341 | + from charmhelpers.core.kernel_factory.ubuntu import ( |
342 | + persistent_modprobe, |
343 | + update_initramfs, |
344 | + ) # flake8: noqa -- ignore F401 for this import |
345 | +elif __platform__ == "centos": |
346 | + from charmhelpers.core.kernel_factory.centos import ( |
347 | + persistent_modprobe, |
348 | + update_initramfs, |
349 | + ) # flake8: noqa -- ignore F401 for this import |
350 | + |
351 | +__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>" |
352 | |
353 | |
354 | def modprobe(module, persist=True): |
355 | @@ -32,11 +45,9 @@ def modprobe(module, persist=True): |
356 | |
357 | log('Loading kernel module %s' % module, level=INFO) |
358 | |
359 | - check_call(cmd) |
360 | + subprocess.check_call(cmd) |
361 | if persist: |
362 | - with open('/etc/modules', 'r+') as modules: |
363 | - if module not in modules.read(): |
364 | - modules.write(module) |
365 | + persistent_modprobe(module) |
366 | |
367 | |
368 | def rmmod(module, force=False): |
369 | @@ -46,21 +57,16 @@ def rmmod(module, force=False): |
370 | cmd.append('-f') |
371 | cmd.append(module) |
372 | log('Removing kernel module %s' % module, level=INFO) |
373 | - return check_call(cmd) |
374 | + return subprocess.check_call(cmd) |
375 | |
376 | |
377 | def lsmod(): |
378 | """Shows what kernel modules are currently loaded""" |
379 | - return check_output(['lsmod'], |
380 | - universal_newlines=True) |
381 | + return subprocess.check_output(['lsmod'], |
382 | + universal_newlines=True) |
383 | |
384 | |
385 | def is_module_loaded(module): |
386 | """Checks if a kernel module is already loaded""" |
387 | matches = re.findall('^%s[ ]+' % module, lsmod(), re.M) |
388 | return len(matches) > 0 |
389 | - |
390 | - |
391 | -def update_initramfs(version='all'): |
392 | - """Updates an initramfs image""" |
393 | - return check_call(["update-initramfs", "-k", version, "-u"]) |
394 | diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py |
395 | index 52eaf82..ec5e0fe 100644 |
396 | --- a/hooks/charmhelpers/fetch/__init__.py |
397 | +++ b/hooks/charmhelpers/fetch/__init__.py |
398 | @@ -13,18 +13,12 @@ |
399 | # limitations under the License. |
400 | |
401 | import importlib |
402 | -from tempfile import NamedTemporaryFile |
403 | -import time |
404 | +from charmhelpers.osplatform import get_platform |
405 | from yaml import safe_load |
406 | -from charmhelpers.core.host import ( |
407 | - lsb_release |
408 | -) |
409 | -import subprocess |
410 | from charmhelpers.core.hookenv import ( |
411 | config, |
412 | log, |
413 | ) |
414 | -import os |
415 | |
416 | import six |
417 | if six.PY3: |
418 | @@ -33,87 +27,6 @@ else: |
419 | from urlparse import urlparse, urlunparse |
420 | |
421 | |
422 | -CLOUD_ARCHIVE = """# Ubuntu Cloud Archive |
423 | -deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main |
424 | -""" |
425 | -PROPOSED_POCKET = """# Proposed |
426 | -deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted |
427 | -""" |
428 | -CLOUD_ARCHIVE_POCKETS = { |
429 | - # Folsom |
430 | - 'folsom': 'precise-updates/folsom', |
431 | - 'precise-folsom': 'precise-updates/folsom', |
432 | - 'precise-folsom/updates': 'precise-updates/folsom', |
433 | - 'precise-updates/folsom': 'precise-updates/folsom', |
434 | - 'folsom/proposed': 'precise-proposed/folsom', |
435 | - 'precise-folsom/proposed': 'precise-proposed/folsom', |
436 | - 'precise-proposed/folsom': 'precise-proposed/folsom', |
437 | - # Grizzly |
438 | - 'grizzly': 'precise-updates/grizzly', |
439 | - 'precise-grizzly': 'precise-updates/grizzly', |
440 | - 'precise-grizzly/updates': 'precise-updates/grizzly', |
441 | - 'precise-updates/grizzly': 'precise-updates/grizzly', |
442 | - 'grizzly/proposed': 'precise-proposed/grizzly', |
443 | - 'precise-grizzly/proposed': 'precise-proposed/grizzly', |
444 | - 'precise-proposed/grizzly': 'precise-proposed/grizzly', |
445 | - # Havana |
446 | - 'havana': 'precise-updates/havana', |
447 | - 'precise-havana': 'precise-updates/havana', |
448 | - 'precise-havana/updates': 'precise-updates/havana', |
449 | - 'precise-updates/havana': 'precise-updates/havana', |
450 | - 'havana/proposed': 'precise-proposed/havana', |
451 | - 'precise-havana/proposed': 'precise-proposed/havana', |
452 | - 'precise-proposed/havana': 'precise-proposed/havana', |
453 | - # Icehouse |
454 | - 'icehouse': 'precise-updates/icehouse', |
455 | - 'precise-icehouse': 'precise-updates/icehouse', |
456 | - 'precise-icehouse/updates': 'precise-updates/icehouse', |
457 | - 'precise-updates/icehouse': 'precise-updates/icehouse', |
458 | - 'icehouse/proposed': 'precise-proposed/icehouse', |
459 | - 'precise-icehouse/proposed': 'precise-proposed/icehouse', |
460 | - 'precise-proposed/icehouse': 'precise-proposed/icehouse', |
461 | - # Juno |
462 | - 'juno': 'trusty-updates/juno', |
463 | - 'trusty-juno': 'trusty-updates/juno', |
464 | - 'trusty-juno/updates': 'trusty-updates/juno', |
465 | - 'trusty-updates/juno': 'trusty-updates/juno', |
466 | - 'juno/proposed': 'trusty-proposed/juno', |
467 | - 'trusty-juno/proposed': 'trusty-proposed/juno', |
468 | - 'trusty-proposed/juno': 'trusty-proposed/juno', |
469 | - # Kilo |
470 | - 'kilo': 'trusty-updates/kilo', |
471 | - 'trusty-kilo': 'trusty-updates/kilo', |
472 | - 'trusty-kilo/updates': 'trusty-updates/kilo', |
473 | - 'trusty-updates/kilo': 'trusty-updates/kilo', |
474 | - 'kilo/proposed': 'trusty-proposed/kilo', |
475 | - 'trusty-kilo/proposed': 'trusty-proposed/kilo', |
476 | - 'trusty-proposed/kilo': 'trusty-proposed/kilo', |
477 | - # Liberty |
478 | - 'liberty': 'trusty-updates/liberty', |
479 | - 'trusty-liberty': 'trusty-updates/liberty', |
480 | - 'trusty-liberty/updates': 'trusty-updates/liberty', |
481 | - 'trusty-updates/liberty': 'trusty-updates/liberty', |
482 | - 'liberty/proposed': 'trusty-proposed/liberty', |
483 | - 'trusty-liberty/proposed': 'trusty-proposed/liberty', |
484 | - 'trusty-proposed/liberty': 'trusty-proposed/liberty', |
485 | - # Mitaka |
486 | - 'mitaka': 'trusty-updates/mitaka', |
487 | - 'trusty-mitaka': 'trusty-updates/mitaka', |
488 | - 'trusty-mitaka/updates': 'trusty-updates/mitaka', |
489 | - 'trusty-updates/mitaka': 'trusty-updates/mitaka', |
490 | - 'mitaka/proposed': 'trusty-proposed/mitaka', |
491 | - 'trusty-mitaka/proposed': 'trusty-proposed/mitaka', |
492 | - 'trusty-proposed/mitaka': 'trusty-proposed/mitaka', |
493 | - # Newton |
494 | - 'newton': 'xenial-updates/newton', |
495 | - 'xenial-newton': 'xenial-updates/newton', |
496 | - 'xenial-newton/updates': 'xenial-updates/newton', |
497 | - 'xenial-updates/newton': 'xenial-updates/newton', |
498 | - 'newton/proposed': 'xenial-proposed/newton', |
499 | - 'xenial-newton/proposed': 'xenial-proposed/newton', |
500 | - 'xenial-proposed/newton': 'xenial-proposed/newton', |
501 | -} |
502 | - |
503 | # The order of this list is very important. Handlers should be listed in from |
504 | # least- to most-specific URL matching. |
505 | FETCH_HANDLERS = ( |
506 | @@ -122,10 +35,6 @@ FETCH_HANDLERS = ( |
507 | 'charmhelpers.fetch.giturl.GitUrlFetchHandler', |
508 | ) |
509 | |
510 | -APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. |
511 | -APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks. |
512 | -APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. |
513 | - |
514 | |
515 | class SourceConfigError(Exception): |
516 | pass |
517 | @@ -163,180 +72,38 @@ class BaseFetchHandler(object): |
518 | return urlunparse(parts) |
519 | |
520 | |
521 | -def filter_installed_packages(packages): |
522 | - """Returns a list of packages that require installation""" |
523 | - cache = apt_cache() |
524 | - _pkgs = [] |
525 | - for package in packages: |
526 | - try: |
527 | - p = cache[package] |
528 | - p.current_ver or _pkgs.append(package) |
529 | - except KeyError: |
530 | - log('Package {} has no installation candidate.'.format(package), |
531 | - level='WARNING') |
532 | - _pkgs.append(package) |
533 | - return _pkgs |
534 | - |
535 | - |
536 | -def apt_cache(in_memory=True, progress=None): |
537 | - """Build and return an apt cache""" |
538 | - from apt import apt_pkg |
539 | - apt_pkg.init() |
540 | - if in_memory: |
541 | - apt_pkg.config.set("Dir::Cache::pkgcache", "") |
542 | - apt_pkg.config.set("Dir::Cache::srcpkgcache", "") |
543 | - return apt_pkg.Cache(progress) |
544 | - |
545 | - |
546 | -def apt_install(packages, options=None, fatal=False): |
547 | - """Install one or more packages""" |
548 | - if options is None: |
549 | - options = ['--option=Dpkg::Options::=--force-confold'] |
550 | - |
551 | - cmd = ['apt-get', '--assume-yes'] |
552 | - cmd.extend(options) |
553 | - cmd.append('install') |
554 | - if isinstance(packages, six.string_types): |
555 | - cmd.append(packages) |
556 | - else: |
557 | - cmd.extend(packages) |
558 | - log("Installing {} with options: {}".format(packages, |
559 | - options)) |
560 | - _run_apt_command(cmd, fatal) |
561 | - |
562 | - |
563 | -def apt_upgrade(options=None, fatal=False, dist=False): |
564 | - """Upgrade all packages""" |
565 | - if options is None: |
566 | - options = ['--option=Dpkg::Options::=--force-confold'] |
567 | - |
568 | - cmd = ['apt-get', '--assume-yes'] |
569 | - cmd.extend(options) |
570 | - if dist: |
571 | - cmd.append('dist-upgrade') |
572 | - else: |
573 | - cmd.append('upgrade') |
574 | - log("Upgrading with options: {}".format(options)) |
575 | - _run_apt_command(cmd, fatal) |
576 | - |
577 | - |
578 | -def apt_update(fatal=False): |
579 | - """Update local apt cache""" |
580 | - cmd = ['apt-get', 'update'] |
581 | - _run_apt_command(cmd, fatal) |
582 | - |
583 | - |
584 | -def apt_purge(packages, fatal=False): |
585 | - """Purge one or more packages""" |
586 | - cmd = ['apt-get', '--assume-yes', 'purge'] |
587 | - if isinstance(packages, six.string_types): |
588 | - cmd.append(packages) |
589 | - else: |
590 | - cmd.extend(packages) |
591 | - log("Purging {}".format(packages)) |
592 | - _run_apt_command(cmd, fatal) |
593 | - |
594 | - |
595 | -def apt_mark(packages, mark, fatal=False): |
596 | - """Flag one or more packages using apt-mark""" |
597 | - log("Marking {} as {}".format(packages, mark)) |
598 | - cmd = ['apt-mark', mark] |
599 | - if isinstance(packages, six.string_types): |
600 | - cmd.append(packages) |
601 | - else: |
602 | - cmd.extend(packages) |
603 | - |
604 | - if fatal: |
605 | - subprocess.check_call(cmd, universal_newlines=True) |
606 | - else: |
607 | - subprocess.call(cmd, universal_newlines=True) |
608 | - |
609 | - |
610 | -def apt_hold(packages, fatal=False): |
611 | - return apt_mark(packages, 'hold', fatal=fatal) |
612 | - |
613 | - |
614 | -def apt_unhold(packages, fatal=False): |
615 | - return apt_mark(packages, 'unhold', fatal=fatal) |
616 | - |
617 | +__platform__ = get_platform() |
618 | +module = "charmhelpers.fetch.%s" % __platform__ |
619 | +fetch = importlib.import_module(module) |
620 | |
621 | -def add_source(source, key=None): |
622 | - """Add a package source to this system. |
623 | +filter_installed_packages = fetch.filter_installed_packages |
624 | +install = fetch.install |
625 | +upgrade = fetch.upgrade |
626 | +update = fetch.update |
627 | +purge = fetch.purge |
628 | +add_source = fetch.add_source |
629 | |
630 | - @param source: a URL or sources.list entry, as supported by |
631 | - add-apt-repository(1). Examples:: |
632 | - |
633 | - ppa:charmers/example |
634 | - deb https://stub:key@private.example.com/ubuntu trusty main |
635 | - |
636 | - In addition: |
637 | - 'proposed:' may be used to enable the standard 'proposed' |
638 | - pocket for the release. |
639 | - 'cloud:' may be used to activate official cloud archive pockets, |
640 | - such as 'cloud:icehouse' |
641 | - 'distro' may be used as a noop |
642 | - |
643 | - @param key: A key to be added to the system's APT keyring and used |
644 | - to verify the signatures on packages. Ideally, this should be an |
645 | - ASCII format GPG public key including the block headers. A GPG key |
646 | - id may also be used, but be aware that only insecure protocols are |
647 | - available to retrieve the actual public key from a public keyserver |
648 | - placing your Juju environment at risk. ppa and cloud archive keys |
649 | - are securely added automtically, so sould not be provided. |
650 | - """ |
651 | - if source is None: |
652 | - log('Source is not present. Skipping') |
653 | - return |
654 | - |
655 | - if (source.startswith('ppa:') or |
656 | - source.startswith('http') or |
657 | - source.startswith('deb ') or |
658 | - source.startswith('cloud-archive:')): |
659 | - subprocess.check_call(['add-apt-repository', '--yes', source]) |
660 | - elif source.startswith('cloud:'): |
661 | - apt_install(filter_installed_packages(['ubuntu-cloud-keyring']), |
662 | - fatal=True) |
663 | - pocket = source.split(':')[-1] |
664 | - if pocket not in CLOUD_ARCHIVE_POCKETS: |
665 | - raise SourceConfigError( |
666 | - 'Unsupported cloud: source option %s' % |
667 | - pocket) |
668 | - actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] |
669 | - with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: |
670 | - apt.write(CLOUD_ARCHIVE.format(actual_pocket)) |
671 | - elif source == 'proposed': |
672 | - release = lsb_release()['DISTRIB_CODENAME'] |
673 | - with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: |
674 | - apt.write(PROPOSED_POCKET.format(release)) |
675 | - elif source == 'distro': |
676 | - pass |
677 | - else: |
678 | - log("Unknown source: {!r}".format(source)) |
679 | - |
680 | - if key: |
681 | - if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key: |
682 | - with NamedTemporaryFile('w+') as key_file: |
683 | - key_file.write(key) |
684 | - key_file.flush() |
685 | - key_file.seek(0) |
686 | - subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file) |
687 | - else: |
688 | - # Note that hkp: is in no way a secure protocol. Using a |
689 | - # GPG key id is pointless from a security POV unless you |
690 | - # absolutely trust your network and DNS. |
691 | - subprocess.check_call(['apt-key', 'adv', '--keyserver', |
692 | - 'hkp://keyserver.ubuntu.com:80', '--recv', |
693 | - key]) |
694 | +if __platform__ == "ubuntu": |
695 | + apt_cache = fetch.apt_cache |
696 | + apt_install = fetch.install |
697 | + apt_update = fetch.update |
698 | + apt_upgrade = fetch.upgrade |
699 | + apt_purge = fetch.purge |
700 | + apt_mark = fetch.apt_mark |
701 | + apt_hold = fetch.apt_hold |
702 | + apt_unhold = fetch.apt_unhold |
703 | + get_upstream_version = fetch.get_upstream_version |
704 | +elif __platform__ == "centos": |
705 | + yum_search = fetch.yum_search |
706 | |
707 | |
708 | def configure_sources(update=False, |
709 | sources_var='install_sources', |
710 | keys_var='install_keys'): |
711 | - """ |
712 | - Configure multiple sources from charm configuration. |
713 | + """Configure multiple sources from charm configuration. |
714 | |
715 | The lists are encoded as yaml fragments in the configuration. |
716 | - The frament needs to be included as a string. Sources and their |
717 | + The fragment needs to be included as a string. Sources and their |
718 | corresponding keys are of the types supported by add_source(). |
719 | |
720 | Example config: |
721 | @@ -368,12 +135,11 @@ def configure_sources(update=False, |
722 | for source, key in zip(sources, keys): |
723 | add_source(source, key) |
724 | if update: |
725 | - apt_update(fatal=True) |
726 | + fetch.update(fatal=True) |
727 | |
728 | |
729 | def install_remote(source, *args, **kwargs): |
730 | - """ |
731 | - Install a file tree from a remote source |
732 | + """Install a file tree from a remote source. |
733 | |
734 | The specified source should be a url of the form: |
735 | scheme://[host]/path[#[option=value][&...]] |
736 | @@ -406,6 +172,7 @@ def install_remote(source, *args, **kwargs): |
737 | |
738 | |
739 | def install_from_config(config_var_name): |
740 | + """Install a file from config.""" |
741 | charm_config = config() |
742 | source = charm_config[config_var_name] |
743 | return install_remote(source) |
744 | @@ -428,40 +195,3 @@ def plugins(fetch_handlers=None): |
745 | log("FetchHandler {} not found, skipping plugin".format( |
746 | handler_name)) |
747 | return plugin_list |
748 | - |
749 | - |
750 | -def _run_apt_command(cmd, fatal=False): |
751 | - """ |
752 | - Run an APT command, checking output and retrying if the fatal flag is set |
753 | - to True. |
754 | - |
755 | - :param: cmd: str: The apt command to run. |
756 | - :param: fatal: bool: Whether the command's output should be checked and |
757 | - retried. |
758 | - """ |
759 | - env = os.environ.copy() |
760 | - |
761 | - if 'DEBIAN_FRONTEND' not in env: |
762 | - env['DEBIAN_FRONTEND'] = 'noninteractive' |
763 | - |
764 | - if fatal: |
765 | - retry_count = 0 |
766 | - result = None |
767 | - |
768 | - # If the command is considered "fatal", we need to retry if the apt |
769 | - # lock was not acquired. |
770 | - |
771 | - while result is None or result == APT_NO_LOCK: |
772 | - try: |
773 | - result = subprocess.check_call(cmd, env=env) |
774 | - except subprocess.CalledProcessError as e: |
775 | - retry_count = retry_count + 1 |
776 | - if retry_count > APT_NO_LOCK_RETRY_COUNT: |
777 | - raise |
778 | - result = e.returncode |
779 | - log("Couldn't acquire DPKG lock. Will retry in {} seconds." |
780 | - "".format(APT_NO_LOCK_RETRY_DELAY)) |
781 | - time.sleep(APT_NO_LOCK_RETRY_DELAY) |
782 | - |
783 | - else: |
784 | - subprocess.call(cmd, env=env) |
785 | diff --git a/hooks/charmhelpers/fetch/bzrurl.py b/hooks/charmhelpers/fetch/bzrurl.py |
786 | index b3404d8..07cd029 100644 |
787 | --- a/hooks/charmhelpers/fetch/bzrurl.py |
788 | +++ b/hooks/charmhelpers/fetch/bzrurl.py |
789 | @@ -18,19 +18,20 @@ from charmhelpers.fetch import ( |
790 | BaseFetchHandler, |
791 | UnhandledSource, |
792 | filter_installed_packages, |
793 | - apt_install, |
794 | + install, |
795 | ) |
796 | from charmhelpers.core.host import mkdir |
797 | |
798 | |
799 | if filter_installed_packages(['bzr']) != []: |
800 | - apt_install(['bzr']) |
801 | + install(['bzr']) |
802 | if filter_installed_packages(['bzr']) != []: |
803 | raise NotImplementedError('Unable to install bzr') |
804 | |
805 | |
806 | class BzrUrlFetchHandler(BaseFetchHandler): |
807 | - """Handler for bazaar branches via generic and lp URLs""" |
808 | + """Handler for bazaar branches via generic and lp URLs.""" |
809 | + |
810 | def can_handle(self, source): |
811 | url_parts = self.parse_url(source) |
812 | if url_parts.scheme not in ('bzr+ssh', 'lp', ''): |
813 | diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py |
814 | index f708d1e..4cf21bc 100644 |
815 | --- a/hooks/charmhelpers/fetch/giturl.py |
816 | +++ b/hooks/charmhelpers/fetch/giturl.py |
817 | @@ -18,17 +18,18 @@ from charmhelpers.fetch import ( |
818 | BaseFetchHandler, |
819 | UnhandledSource, |
820 | filter_installed_packages, |
821 | - apt_install, |
822 | + install, |
823 | ) |
824 | |
825 | if filter_installed_packages(['git']) != []: |
826 | - apt_install(['git']) |
827 | + install(['git']) |
828 | if filter_installed_packages(['git']) != []: |
829 | raise NotImplementedError('Unable to install git') |
830 | |
831 | |
832 | class GitUrlFetchHandler(BaseFetchHandler): |
833 | - """Handler for git branches via generic and github URLs""" |
834 | + """Handler for git branches via generic and github URLs.""" |
835 | + |
836 | def can_handle(self, source): |
837 | url_parts = self.parse_url(source) |
838 | # TODO (mattyw) no support for ssh git@ yet |
839 | diff --git a/tests/charmhelpers/contrib/amulet/deployment.py b/tests/charmhelpers/contrib/amulet/deployment.py |
840 | index 0146236..9c65518 100644 |
841 | --- a/tests/charmhelpers/contrib/amulet/deployment.py |
842 | +++ b/tests/charmhelpers/contrib/amulet/deployment.py |
843 | @@ -78,11 +78,15 @@ class AmuletDeployment(object): |
844 | |
845 | def _deploy(self): |
846 | """Deploy environment and wait for all hooks to finish executing.""" |
847 | + timeout = int(os.environ.get('AMULET_SETUP_TIMEOUT', 900)) |
848 | try: |
849 | - self.d.setup(timeout=900) |
850 | - self.d.sentry.wait(timeout=900) |
851 | + self.d.setup(timeout=timeout) |
852 | + self.d.sentry.wait(timeout=timeout) |
853 | except amulet.helpers.TimeoutError: |
854 | - amulet.raise_status(amulet.FAIL, msg="Deployment timed out") |
855 | + amulet.raise_status( |
856 | + amulet.FAIL, |
857 | + msg="Deployment timed out ({}s)".format(timeout) |
858 | + ) |
859 | except Exception: |
860 | raise |
861 | |
862 | diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py |
863 | index a39ed4c..f9e4c3a 100644 |
864 | --- a/tests/charmhelpers/contrib/amulet/utils.py |
865 | +++ b/tests/charmhelpers/contrib/amulet/utils.py |
866 | @@ -148,7 +148,8 @@ class AmuletUtils(object): |
867 | |
868 | for service_name in services_list: |
869 | if (self.ubuntu_releases.index(release) >= systemd_switch or |
870 | - service_name in ['rabbitmq-server', 'apache2']): |
871 | + service_name in ['rabbitmq-server', 'apache2', |
872 | + 'memcached']): |
873 | # init is systemd (or regular sysv) |
874 | cmd = 'sudo service {} status'.format(service_name) |
875 | output, code = sentry_unit.run(cmd) |
876 | @@ -546,7 +547,7 @@ class AmuletUtils(object): |
877 | raise if it is present. |
878 | :returns: List of process IDs |
879 | """ |
880 | - cmd = 'pidof -x {}'.format(process_name) |
881 | + cmd = 'pidof -x "{}"'.format(process_name) |
882 | if not expect_success: |
883 | cmd += " || exit 0 && exit 1" |
884 | output, code = sentry_unit.run(cmd) |
LGTM