Merge lp:~xavpaice/charms/trusty/thruk-master/trunk into lp:~canonical-bootstack/charms/trusty/thruk-master/trunk

Proposed by Xav Paice on 2017-07-25
Status: Merged
Merged at revision: 23
Proposed branch: lp:~xavpaice/charms/trusty/thruk-master/trunk
Merge into: lp:~canonical-bootstack/charms/trusty/thruk-master/trunk
Diff against target: 4794 lines (+3049/-823)
33 files modified
charm-helpers.yaml (+1/-0)
config.yaml (+21/-3)
hooks/actions.py (+10/-12)
hooks/charmhelpers/__init__.py (+72/-13)
hooks/charmhelpers/core/__init__.py (+11/-13)
hooks/charmhelpers/core/decorators.py (+11/-13)
hooks/charmhelpers/core/files.py (+43/-0)
hooks/charmhelpers/core/fstab.py (+11/-13)
hooks/charmhelpers/core/hookenv.py (+530/-56)
hooks/charmhelpers/core/host.py (+629/-155)
hooks/charmhelpers/core/host_factory/centos.py (+72/-0)
hooks/charmhelpers/core/host_factory/ubuntu.py (+89/-0)
hooks/charmhelpers/core/hugepage.py (+69/-0)
hooks/charmhelpers/core/kernel.py (+72/-0)
hooks/charmhelpers/core/kernel_factory/centos.py (+17/-0)
hooks/charmhelpers/core/kernel_factory/ubuntu.py (+13/-0)
hooks/charmhelpers/core/services/__init__.py (+11/-13)
hooks/charmhelpers/core/services/base.py (+54/-32)
hooks/charmhelpers/core/services/helpers.py (+42/-19)
hooks/charmhelpers/core/strutils.py (+96/-15)
hooks/charmhelpers/core/sysctl.py (+11/-13)
hooks/charmhelpers/core/templating.py (+40/-24)
hooks/charmhelpers/core/unitdata.py (+72/-31)
hooks/charmhelpers/fetch/__init__.py (+54/-288)
hooks/charmhelpers/fetch/archiveurl.py (+19/-15)
hooks/charmhelpers/fetch/bzrurl.py (+48/-50)
hooks/charmhelpers/fetch/centos.py (+171/-0)
hooks/charmhelpers/fetch/giturl.py (+37/-39)
hooks/charmhelpers/fetch/snap.py (+122/-0)
hooks/charmhelpers/fetch/ubuntu.py (+568/-0)
hooks/charmhelpers/osplatform.py (+25/-0)
hooks/install.real (+4/-6)
templates/thruk_local.conf (+4/-0)
To merge this branch: bzr merge lp:~xavpaice/charms/trusty/thruk-master/trunk
Reviewer Review Type Date Requested Status
James Hebden (community) 2017-07-25 Approve on 2017-08-16
Review via email: mp+328013@code.launchpad.net

Description of the change

Update Thruk to PPA version 2.14 and use LMD

To post a comment you must log in.
James Troup (elmo) wrote :

Have you not made a PPA mandatory with this change? That's fine if it's deliberate, but config.yaml should be updated to not claim 'source' is optional.

James Hebden (ec0) :
review: Approve
23. By James Hebden on 2017-08-16

[jhebden, r=elmo] Fixed working around PPAs, merged changes for 328013

James Hebden (ec0) wrote :

Merging, but fixing the wording as pointed out by elmo.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'charm-helpers.yaml'
2--- charm-helpers.yaml 2015-04-09 06:46:35 +0000
3+++ charm-helpers.yaml 2017-07-25 09:17:37 +0000
4@@ -3,3 +3,4 @@
5 include:
6 - core
7 - fetch
8+ - osplatform
9
10=== modified file 'config.yaml'
11--- config.yaml 2015-08-31 04:46:54 +0000
12+++ config.yaml 2017-07-25 09:17:37 +0000
13@@ -1,8 +1,22 @@
14 options:
15 source:
16 type: string
17- default: "ppa:brad-marshall/thruk"
18- description: "PPA to install thruk"
19+ default: ppa:canonical-bootstack/thruk
20+ description: |
21+ Optional configuration to support use of additional sources such as:
22+
23+ - ppa:myteam/ppa
24+ - cloud:trusty-proposed/kilo
25+ - http://my.archive.com/ubuntu main
26+
27+ The last option should be used in conjunction with the key configuration
28+ option.
29+ key:
30+ type: string
31+ default: 7A207542
32+ description: |
33+ Key ID to import to the apt keyring to support use with arbitary source
34+ configuration from outside of Launchpad archives or PPA's.
35 url:
36 type: string
37 default: "nagios.example.com"
38@@ -55,4 +69,8 @@
39 description: >
40 Enables debug logging to stderr for thruk - in the default case, this ends
41 up in the apache error logs.
42-
43+ enable-lmd:
44+ default: true
45+ type: boolean
46+ description: >
47+ If enabled, make use of the LMD - Livestatus Multitool Daemon
48
49=== modified file 'hooks/actions.py'
50--- hooks/actions.py 2017-03-21 04:14:41 +0000
51+++ hooks/actions.py 2017-07-25 09:17:37 +0000
52@@ -1,6 +1,6 @@
53 from charmhelpers.core import hookenv
54 from charmhelpers.fetch import (
55- apt_install, apt_update, add_source
56+ apt_install, apt_update, add_source, apt_purge
57 )
58 import subprocess
59 import os
60@@ -17,17 +17,15 @@
61
62 def update_ppa(service_name):
63 config = hookenv.config()
64-
65- if config.changed('source'):
66- prev_ppa = config.previous('source')
67- if prev_ppa is not None:
68- subprocess.check_call(['add-apt-repository',
69- '--yes', '--remove', prev_ppa])
70- ppa = config.get('source')
71- if ppa is not None:
72- add_source(ppa)
73- apt_update()
74- apt_install(["thruk"])
75+ new_source = config.get('source')
76+ prev_source = config.previous('source')
77+ if prev_source is not None and prev_source != new_source:
78+ subprocess.check_call(['add-apt-repository',
79+ '--yes', '--remove', prev_source])
80+ add_source(config.get('source'), config.get('key', None))
81+ apt_update(fatal=True)
82+ package_list = ["thruk", "pwgen", "apache2-utils"]
83+ apt_install(packages=package_list, fatal=True)
84
85
86 def thruk_set_password(service_name):
87
88=== modified file 'hooks/charmhelpers/__init__.py'
89--- hooks/charmhelpers/__init__.py 2015-04-09 06:46:35 +0000
90+++ hooks/charmhelpers/__init__.py 2017-07-25 09:17:37 +0000
91@@ -1,21 +1,24 @@
92 # Copyright 2014-2015 Canonical Limited.
93 #
94-# This file is part of charm-helpers.
95-#
96-# charm-helpers is free software: you can redistribute it and/or modify
97-# it under the terms of the GNU Lesser General Public License version 3 as
98-# published by the Free Software Foundation.
99-#
100-# charm-helpers is distributed in the hope that it will be useful,
101-# but WITHOUT ANY WARRANTY; without even the implied warranty of
102-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
103-# GNU Lesser General Public License for more details.
104-#
105-# You should have received a copy of the GNU Lesser General Public License
106-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
107+# Licensed under the Apache License, Version 2.0 (the "License");
108+# you may not use this file except in compliance with the License.
109+# You may obtain a copy of the License at
110+#
111+# http://www.apache.org/licenses/LICENSE-2.0
112+#
113+# Unless required by applicable law or agreed to in writing, software
114+# distributed under the License is distributed on an "AS IS" BASIS,
115+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
116+# See the License for the specific language governing permissions and
117+# limitations under the License.
118
119 # Bootstrap charm-helpers, installing its dependencies if necessary using
120 # only standard libraries.
121+from __future__ import print_function
122+from __future__ import absolute_import
123+
124+import functools
125+import inspect
126 import subprocess
127 import sys
128
129@@ -36,3 +39,59 @@
130 else:
131 subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
132 import yaml # flake8: noqa
133+
134+
135+# Holds a list of mapping of mangled function names that have been deprecated
136+# using the @deprecate decorator below. This is so that the warning is only
137+# printed once for each usage of the function.
138+__deprecated_functions = {}
139+
140+
141+def deprecate(warning, date=None, log=None):
142+ """Add a deprecation warning the first time the function is used.
143+ The date, which is a string in semi-ISO8660 format indicate the year-month
144+ that the function is officially going to be removed.
145+
146+ usage:
147+
148+ @deprecate('use core/fetch/add_source() instead', '2017-04')
149+ def contributed_add_source_thing(...):
150+ ...
151+
152+ And it then prints to the log ONCE that the function is deprecated.
153+ The reason for passing the logging function (log) is so that hookenv.log
154+ can be used for a charm if needed.
155+
156+ :param warning: String to indicat where it has moved ot.
157+ :param date: optional sting, in YYYY-MM format to indicate when the
158+ function will definitely (probably) be removed.
159+ :param log: The log function to call to log. If not, logs to stdout
160+ """
161+ def wrap(f):
162+
163+ @functools.wraps(f)
164+ def wrapped_f(*args, **kwargs):
165+ try:
166+ module = inspect.getmodule(f)
167+ file = inspect.getsourcefile(f)
168+ lines = inspect.getsourcelines(f)
169+ f_name = "{}-{}-{}..{}-{}".format(
170+ module.__name__, file, lines[0], lines[-1], f.__name__)
171+ except (IOError, TypeError):
172+ # assume it was local, so just use the name of the function
173+ f_name = f.__name__
174+ if f_name not in __deprecated_functions:
175+ __deprecated_functions[f_name] = True
176+ s = "DEPRECATION WARNING: Function {} is being removed".format(
177+ f.__name__)
178+ if date:
179+ s = "{} on/around {}".format(s, date)
180+ if warning:
181+ s = "{} : {}".format(s, warning)
182+ if log:
183+ log(s)
184+ else:
185+ print(s)
186+ return f(*args, **kwargs)
187+ return wrapped_f
188+ return wrap
189
190=== modified file 'hooks/charmhelpers/core/__init__.py'
191--- hooks/charmhelpers/core/__init__.py 2015-04-09 06:46:35 +0000
192+++ hooks/charmhelpers/core/__init__.py 2017-07-25 09:17:37 +0000
193@@ -1,15 +1,13 @@
194 # Copyright 2014-2015 Canonical Limited.
195 #
196-# This file is part of charm-helpers.
197-#
198-# charm-helpers is free software: you can redistribute it and/or modify
199-# it under the terms of the GNU Lesser General Public License version 3 as
200-# published by the Free Software Foundation.
201-#
202-# charm-helpers is distributed in the hope that it will be useful,
203-# but WITHOUT ANY WARRANTY; without even the implied warranty of
204-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
205-# GNU Lesser General Public License for more details.
206-#
207-# You should have received a copy of the GNU Lesser General Public License
208-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
209+# Licensed under the Apache License, Version 2.0 (the "License");
210+# you may not use this file except in compliance with the License.
211+# You may obtain a copy of the License at
212+#
213+# http://www.apache.org/licenses/LICENSE-2.0
214+#
215+# Unless required by applicable law or agreed to in writing, software
216+# distributed under the License is distributed on an "AS IS" BASIS,
217+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
218+# See the License for the specific language governing permissions and
219+# limitations under the License.
220
221=== modified file 'hooks/charmhelpers/core/decorators.py'
222--- hooks/charmhelpers/core/decorators.py 2015-04-09 06:46:35 +0000
223+++ hooks/charmhelpers/core/decorators.py 2017-07-25 09:17:37 +0000
224@@ -1,18 +1,16 @@
225 # Copyright 2014-2015 Canonical Limited.
226 #
227-# This file is part of charm-helpers.
228-#
229-# charm-helpers is free software: you can redistribute it and/or modify
230-# it under the terms of the GNU Lesser General Public License version 3 as
231-# published by the Free Software Foundation.
232-#
233-# charm-helpers is distributed in the hope that it will be useful,
234-# but WITHOUT ANY WARRANTY; without even the implied warranty of
235-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
236-# GNU Lesser General Public License for more details.
237-#
238-# You should have received a copy of the GNU Lesser General Public License
239-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
240+# Licensed under the Apache License, Version 2.0 (the "License");
241+# you may not use this file except in compliance with the License.
242+# You may obtain a copy of the License at
243+#
244+# http://www.apache.org/licenses/LICENSE-2.0
245+#
246+# Unless required by applicable law or agreed to in writing, software
247+# distributed under the License is distributed on an "AS IS" BASIS,
248+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
249+# See the License for the specific language governing permissions and
250+# limitations under the License.
251
252 #
253 # Copyright 2014 Canonical Ltd.
254
255=== added file 'hooks/charmhelpers/core/files.py'
256--- hooks/charmhelpers/core/files.py 1970-01-01 00:00:00 +0000
257+++ hooks/charmhelpers/core/files.py 2017-07-25 09:17:37 +0000
258@@ -0,0 +1,43 @@
259+#!/usr/bin/env python
260+# -*- coding: utf-8 -*-
261+
262+# Copyright 2014-2015 Canonical Limited.
263+#
264+# Licensed under the Apache License, Version 2.0 (the "License");
265+# you may not use this file except in compliance with the License.
266+# You may obtain a copy of the License at
267+#
268+# http://www.apache.org/licenses/LICENSE-2.0
269+#
270+# Unless required by applicable law or agreed to in writing, software
271+# distributed under the License is distributed on an "AS IS" BASIS,
272+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
273+# See the License for the specific language governing permissions and
274+# limitations under the License.
275+
276+__author__ = 'Jorge Niedbalski <niedbalski@ubuntu.com>'
277+
278+import os
279+import subprocess
280+
281+
282+def sed(filename, before, after, flags='g'):
283+ """
284+ Search and replaces the given pattern on filename.
285+
286+ :param filename: relative or absolute file path.
287+ :param before: expression to be replaced (see 'man sed')
288+ :param after: expression to replace with (see 'man sed')
289+ :param flags: sed-compatible regex flags in example, to make
290+ the search and replace case insensitive, specify ``flags="i"``.
291+ The ``g`` flag is always specified regardless, so you do not
292+ need to remember to include it when overriding this parameter.
293+ :returns: If the sed command exit code was zero then return,
294+ otherwise raise CalledProcessError.
295+ """
296+ expression = r's/{0}/{1}/{2}'.format(before,
297+ after, flags)
298+
299+ return subprocess.check_call(["sed", "-i", "-r", "-e",
300+ expression,
301+ os.path.expanduser(filename)])
302
303=== modified file 'hooks/charmhelpers/core/fstab.py'
304--- hooks/charmhelpers/core/fstab.py 2015-04-09 06:46:35 +0000
305+++ hooks/charmhelpers/core/fstab.py 2017-07-25 09:17:37 +0000
306@@ -3,19 +3,17 @@
307
308 # Copyright 2014-2015 Canonical Limited.
309 #
310-# This file is part of charm-helpers.
311-#
312-# charm-helpers is free software: you can redistribute it and/or modify
313-# it under the terms of the GNU Lesser General Public License version 3 as
314-# published by the Free Software Foundation.
315-#
316-# charm-helpers is distributed in the hope that it will be useful,
317-# but WITHOUT ANY WARRANTY; without even the implied warranty of
318-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
319-# GNU Lesser General Public License for more details.
320-#
321-# You should have received a copy of the GNU Lesser General Public License
322-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
323+# Licensed under the Apache License, Version 2.0 (the "License");
324+# you may not use this file except in compliance with the License.
325+# You may obtain a copy of the License at
326+#
327+# http://www.apache.org/licenses/LICENSE-2.0
328+#
329+# Unless required by applicable law or agreed to in writing, software
330+# distributed under the License is distributed on an "AS IS" BASIS,
331+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
332+# See the License for the specific language governing permissions and
333+# limitations under the License.
334
335 import io
336 import os
337
338=== modified file 'hooks/charmhelpers/core/hookenv.py'
339--- hooks/charmhelpers/core/hookenv.py 2015-04-09 06:46:35 +0000
340+++ hooks/charmhelpers/core/hookenv.py 2017-07-25 09:17:37 +0000
341@@ -1,18 +1,16 @@
342 # Copyright 2014-2015 Canonical Limited.
343 #
344-# This file is part of charm-helpers.
345-#
346-# charm-helpers is free software: you can redistribute it and/or modify
347-# it under the terms of the GNU Lesser General Public License version 3 as
348-# published by the Free Software Foundation.
349-#
350-# charm-helpers is distributed in the hope that it will be useful,
351-# but WITHOUT ANY WARRANTY; without even the implied warranty of
352-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
353-# GNU Lesser General Public License for more details.
354-#
355-# You should have received a copy of the GNU Lesser General Public License
356-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
357+# Licensed under the Apache License, Version 2.0 (the "License");
358+# you may not use this file except in compliance with the License.
359+# You may obtain a copy of the License at
360+#
361+# http://www.apache.org/licenses/LICENSE-2.0
362+#
363+# Unless required by applicable law or agreed to in writing, software
364+# distributed under the License is distributed on an "AS IS" BASIS,
365+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
366+# See the License for the specific language governing permissions and
367+# limitations under the License.
368
369 "Interactions with the Juju environment"
370 # Copyright 2013 Canonical Ltd.
371@@ -20,11 +18,18 @@
372 # Authors:
373 # Charm Helpers Developers <juju@lists.ubuntu.com>
374
375+from __future__ import print_function
376+import copy
377+from distutils.version import LooseVersion
378+from functools import wraps
379+import glob
380 import os
381 import json
382 import yaml
383 import subprocess
384 import sys
385+import errno
386+import tempfile
387 from subprocess import CalledProcessError
388
389 import six
390@@ -56,15 +61,18 @@
391
392 will cache the result of unit_get + 'test' for future calls.
393 """
394+ @wraps(func)
395 def wrapper(*args, **kwargs):
396 global cache
397 key = str((func, args, kwargs))
398 try:
399 return cache[key]
400 except KeyError:
401- res = func(*args, **kwargs)
402- cache[key] = res
403- return res
404+ pass # Drop out of the exception handler scope.
405+ res = func(*args, **kwargs)
406+ cache[key] = res
407+ return res
408+ wrapper._wrapped = func
409 return wrapper
410
411
412@@ -87,7 +95,18 @@
413 if not isinstance(message, six.string_types):
414 message = repr(message)
415 command += [message]
416- subprocess.call(command)
417+ # Missing juju-log should not cause failures in unit tests
418+ # Send log output to stderr
419+ try:
420+ subprocess.call(command)
421+ except OSError as e:
422+ if e.errno == errno.ENOENT:
423+ if level:
424+ message = "{}: {}".format(level, message)
425+ message = "juju-log: {}".format(message)
426+ print(message, file=sys.stderr)
427+ else:
428+ raise
429
430
431 class Serializable(UserDict):
432@@ -153,9 +172,19 @@
433 return os.environ.get('JUJU_RELATION', None)
434
435
436-def relation_id():
437- """The relation ID for the current relation hook"""
438- return os.environ.get('JUJU_RELATION_ID', None)
439+@cached
440+def relation_id(relation_name=None, service_or_unit=None):
441+ """The relation ID for the current or a specified relation"""
442+ if not relation_name and not service_or_unit:
443+ return os.environ.get('JUJU_RELATION_ID', None)
444+ elif relation_name and service_or_unit:
445+ service_name = service_or_unit.split('/')[0]
446+ for relid in relation_ids(relation_name):
447+ remote_service = remote_service_name(relid)
448+ if remote_service == service_name:
449+ return relid
450+ else:
451+ raise ValueError('Must specify neither or both of relation_name and service_or_unit')
452
453
454 def local_unit():
455@@ -165,7 +194,7 @@
456
457 def remote_unit():
458 """The remote unit for the current relation hook"""
459- return os.environ['JUJU_REMOTE_UNIT']
460+ return os.environ.get('JUJU_REMOTE_UNIT', None)
461
462
463 def service_name():
464@@ -173,9 +202,20 @@
465 return local_unit().split('/')[0]
466
467
468+@cached
469+def remote_service_name(relid=None):
470+ """The remote service name for a given relation-id (or the current relation)"""
471+ if relid is None:
472+ unit = remote_unit()
473+ else:
474+ units = related_units(relid)
475+ unit = units[0] if units else None
476+ return unit.split('/')[0] if unit else None
477+
478+
479 def hook_name():
480 """The name of the currently executing hook"""
481- return os.path.basename(sys.argv[0])
482+ return os.environ.get('JUJU_HOOK_NAME', os.path.basename(sys.argv[0]))
483
484
485 class Config(dict):
486@@ -225,23 +265,7 @@
487 self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
488 if os.path.exists(self.path):
489 self.load_previous()
490-
491- def __getitem__(self, key):
492- """For regular dict lookups, check the current juju config first,
493- then the previous (saved) copy. This ensures that user-saved values
494- will be returned by a dict lookup.
495-
496- """
497- try:
498- return dict.__getitem__(self, key)
499- except KeyError:
500- return (self._prev_dict or {})[key]
501-
502- def keys(self):
503- prev_keys = []
504- if self._prev_dict is not None:
505- prev_keys = self._prev_dict.keys()
506- return list(set(prev_keys + list(dict.keys(self))))
507+ atexit(self._implicit_save)
508
509 def load_previous(self, path=None):
510 """Load previous copy of config from disk.
511@@ -260,6 +284,9 @@
512 self.path = path or self.path
513 with open(self.path) as f:
514 self._prev_dict = json.load(f)
515+ for k, v in copy.deepcopy(self._prev_dict).items():
516+ if k not in self:
517+ self[k] = v
518
519 def changed(self, key):
520 """Return True if the current value for this key is different from
521@@ -291,13 +318,13 @@
522 instance.
523
524 """
525- if self._prev_dict:
526- for k, v in six.iteritems(self._prev_dict):
527- if k not in self:
528- self[k] = v
529 with open(self.path, 'w') as f:
530 json.dump(self, f)
531
532+ def _implicit_save(self):
533+ if self.implicit_save:
534+ self.save()
535+
536
537 @cached
538 def config(scope=None):
539@@ -305,6 +332,8 @@
540 config_cmd_line = ['config-get']
541 if scope is not None:
542 config_cmd_line.append(scope)
543+ else:
544+ config_cmd_line.append('--all')
545 config_cmd_line.append('--format=json')
546 try:
547 config_data = json.loads(
548@@ -340,18 +369,49 @@
549 """Set relation information for the current unit"""
550 relation_settings = relation_settings if relation_settings else {}
551 relation_cmd_line = ['relation-set']
552+ accepts_file = "--file" in subprocess.check_output(
553+ relation_cmd_line + ["--help"], universal_newlines=True)
554 if relation_id is not None:
555 relation_cmd_line.extend(('-r', relation_id))
556- for k, v in (list(relation_settings.items()) + list(kwargs.items())):
557- if v is None:
558- relation_cmd_line.append('{}='.format(k))
559- else:
560- relation_cmd_line.append('{}={}'.format(k, v))
561- subprocess.check_call(relation_cmd_line)
562+ settings = relation_settings.copy()
563+ settings.update(kwargs)
564+ for key, value in settings.items():
565+ # Force value to be a string: it always should, but some call
566+ # sites pass in things like dicts or numbers.
567+ if value is not None:
568+ settings[key] = "{}".format(value)
569+ if accepts_file:
570+ # --file was introduced in Juju 1.23.2. Use it by default if
571+ # available, since otherwise we'll break if the relation data is
572+ # too big. Ideally we should tell relation-set to read the data from
573+ # stdin, but that feature is broken in 1.23.2: Bug #1454678.
574+ with tempfile.NamedTemporaryFile(delete=False) as settings_file:
575+ settings_file.write(yaml.safe_dump(settings).encode("utf-8"))
576+ subprocess.check_call(
577+ relation_cmd_line + ["--file", settings_file.name])
578+ os.remove(settings_file.name)
579+ else:
580+ for key, value in settings.items():
581+ if value is None:
582+ relation_cmd_line.append('{}='.format(key))
583+ else:
584+ relation_cmd_line.append('{}={}'.format(key, value))
585+ subprocess.check_call(relation_cmd_line)
586 # Flush cache of any relation-gets for local unit
587 flush(local_unit())
588
589
590+def relation_clear(r_id=None):
591+ ''' Clears any relation data already set on relation r_id '''
592+ settings = relation_get(rid=r_id,
593+ unit=local_unit())
594+ for setting in settings:
595+ if setting not in ['public-address', 'private-address']:
596+ settings[setting] = None
597+ relation_set(relation_id=r_id,
598+ **settings)
599+
600+
601 @cached
602 def relation_ids(reltype=None):
603 """A list of relation_ids"""
604@@ -431,6 +491,76 @@
605
606
607 @cached
608+def peer_relation_id():
609+ '''Get the peers relation id if a peers relation has been joined, else None.'''
610+ md = metadata()
611+ section = md.get('peers')
612+ if section:
613+ for key in section:
614+ relids = relation_ids(key)
615+ if relids:
616+ return relids[0]
617+ return None
618+
619+
620+@cached
621+def relation_to_interface(relation_name):
622+ """
623+ Given the name of a relation, return the interface that relation uses.
624+
625+ :returns: The interface name, or ``None``.
626+ """
627+ return relation_to_role_and_interface(relation_name)[1]
628+
629+
630+@cached
631+def relation_to_role_and_interface(relation_name):
632+ """
633+ Given the name of a relation, return the role and the name of the interface
634+ that relation uses (where role is one of ``provides``, ``requires``, or ``peers``).
635+
636+ :returns: A tuple containing ``(role, interface)``, or ``(None, None)``.
637+ """
638+ _metadata = metadata()
639+ for role in ('provides', 'requires', 'peers'):
640+ interface = _metadata.get(role, {}).get(relation_name, {}).get('interface')
641+ if interface:
642+ return role, interface
643+ return None, None
644+
645+
646+@cached
647+def role_and_interface_to_relations(role, interface_name):
648+ """
649+ Given a role and interface name, return a list of relation names for the
650+ current charm that use that interface under that role (where role is one
651+ of ``provides``, ``requires``, or ``peers``).
652+
653+ :returns: A list of relation names.
654+ """
655+ _metadata = metadata()
656+ results = []
657+ for relation_name, relation in _metadata.get(role, {}).items():
658+ if relation['interface'] == interface_name:
659+ results.append(relation_name)
660+ return results
661+
662+
663+@cached
664+def interface_to_relations(interface_name):
665+ """
666+ Given an interface, return a list of relation names for the current
667+ charm that use that interface.
668+
669+ :returns: A list of relation names.
670+ """
671+ results = []
672+ for role in ('provides', 'requires', 'peers'):
673+ results.extend(role_and_interface_to_relations(role, interface_name))
674+ return results
675+
676+
677+@cached
678 def charm_name():
679 """Get the name of the current charm as is specified on metadata.yaml"""
680 return metadata().get('name')
681@@ -486,6 +616,20 @@
682 subprocess.check_call(_args)
683
684
685+def open_ports(start, end, protocol="TCP"):
686+ """Opens a range of service network ports"""
687+ _args = ['open-port']
688+ _args.append('{}-{}/{}'.format(start, end, protocol))
689+ subprocess.check_call(_args)
690+
691+
692+def close_ports(start, end, protocol="TCP"):
693+ """Close a range of service network ports"""
694+ _args = ['close-port']
695+ _args.append('{}-{}/{}'.format(start, end, protocol))
696+ subprocess.check_call(_args)
697+
698+
699 @cached
700 def unit_get(attribute):
701 """Get the unit ID for the remote unit"""
702@@ -496,11 +640,48 @@
703 return None
704
705
706+def unit_public_ip():
707+ """Get this unit's public IP address"""
708+ return unit_get('public-address')
709+
710+
711 def unit_private_ip():
712 """Get this unit's private IP address"""
713 return unit_get('private-address')
714
715
716+@cached
717+def storage_get(attribute=None, storage_id=None):
718+ """Get storage attributes"""
719+ _args = ['storage-get', '--format=json']
720+ if storage_id:
721+ _args.extend(('-s', storage_id))
722+ if attribute:
723+ _args.append(attribute)
724+ try:
725+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
726+ except ValueError:
727+ return None
728+
729+
730+@cached
731+def storage_list(storage_name=None):
732+ """List the storage IDs for the unit"""
733+ _args = ['storage-list', '--format=json']
734+ if storage_name:
735+ _args.append(storage_name)
736+ try:
737+ return json.loads(subprocess.check_output(_args).decode('UTF-8'))
738+ except ValueError:
739+ return None
740+ except OSError as e:
741+ import errno
742+ if e.errno == errno.ENOENT:
743+ # storage-list does not exist
744+ return []
745+ raise
746+
747+
748 class UnregisteredHookError(Exception):
749 """Raised when an undefined hook is called"""
750 pass
751@@ -528,10 +709,14 @@
752 hooks.execute(sys.argv)
753 """
754
755- def __init__(self, config_save=True):
756+ def __init__(self, config_save=None):
757 super(Hooks, self).__init__()
758 self._hooks = {}
759- self._config_save = config_save
760+
761+ # For unknown reasons, we allow the Hooks constructor to override
762+ # config().implicit_save.
763+ if config_save is not None:
764+ config().implicit_save = config_save
765
766 def register(self, name, function):
767 """Register a hook"""
768@@ -539,13 +724,16 @@
769
770 def execute(self, args):
771 """Execute a registered hook based on args[0]"""
772+ _run_atstart()
773 hook_name = os.path.basename(args[0])
774 if hook_name in self._hooks:
775- self._hooks[hook_name]()
776- if self._config_save:
777- cfg = config()
778- if cfg.implicit_save:
779- cfg.save()
780+ try:
781+ self._hooks[hook_name]()
782+ except SystemExit as x:
783+ if x.code is None or x.code == 0:
784+ _run_atexit()
785+ raise
786+ _run_atexit()
787 else:
788 raise UnregisteredHookError(hook_name)
789
790@@ -592,3 +780,289 @@
791
792 The results set by action_set are preserved."""
793 subprocess.check_call(['action-fail', message])
794+
795+
796+def action_name():
797+ """Get the name of the currently executing action."""
798+ return os.environ.get('JUJU_ACTION_NAME')
799+
800+
801+def action_uuid():
802+ """Get the UUID of the currently executing action."""
803+ return os.environ.get('JUJU_ACTION_UUID')
804+
805+
806+def action_tag():
807+ """Get the tag for the currently executing action."""
808+ return os.environ.get('JUJU_ACTION_TAG')
809+
810+
811+def status_set(workload_state, message):
812+ """Set the workload state with a message
813+
814+ Use status-set to set the workload state with a message which is visible
815+ to the user via juju status. If the status-set command is not found then
816+ assume this is juju < 1.23 and juju-log the message unstead.
817+
818+ workload_state -- valid juju workload state.
819+ message -- status update message
820+ """
821+ valid_states = ['maintenance', 'blocked', 'waiting', 'active']
822+ if workload_state not in valid_states:
823+ raise ValueError(
824+ '{!r} is not a valid workload state'.format(workload_state)
825+ )
826+ cmd = ['status-set', workload_state, message]
827+ try:
828+ ret = subprocess.call(cmd)
829+ if ret == 0:
830+ return
831+ except OSError as e:
832+ if e.errno != errno.ENOENT:
833+ raise
834+ log_message = 'status-set failed: {} {}'.format(workload_state,
835+ message)
836+ log(log_message, level='INFO')
837+
838+
839+def status_get():
840+ """Retrieve the previously set juju workload state and message
841+
842+ If the status-get command is not found then assume this is juju < 1.23 and
843+ return 'unknown', ""
844+
845+ """
846+ cmd = ['status-get', "--format=json", "--include-data"]
847+ try:
848+ raw_status = subprocess.check_output(cmd)
849+ except OSError as e:
850+ if e.errno == errno.ENOENT:
851+ return ('unknown', "")
852+ else:
853+ raise
854+ else:
855+ status = json.loads(raw_status.decode("UTF-8"))
856+ return (status["status"], status["message"])
857+
858+
859+def translate_exc(from_exc, to_exc):
860+ def inner_translate_exc1(f):
861+ @wraps(f)
862+ def inner_translate_exc2(*args, **kwargs):
863+ try:
864+ return f(*args, **kwargs)
865+ except from_exc:
866+ raise to_exc
867+
868+ return inner_translate_exc2
869+
870+ return inner_translate_exc1
871+
872+
873+def application_version_set(version):
874+ """Charm authors may trigger this command from any hook to output what
875+ version of the application is running. This could be a package version,
876+ for instance postgres version 9.5. It could also be a build number or
877+ version control revision identifier, for instance git sha 6fb7ba68. """
878+
879+ cmd = ['application-version-set']
880+ cmd.append(version)
881+ try:
882+ subprocess.check_call(cmd)
883+ except OSError:
884+ log("Application Version: {}".format(version))
885+
886+
887+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
888+def is_leader():
889+ """Does the current unit hold the juju leadership
890+
891+ Uses juju to determine whether the current unit is the leader of its peers
892+ """
893+ cmd = ['is-leader', '--format=json']
894+ return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
895+
896+
897+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
898+def leader_get(attribute=None):
899+ """Juju leader get value(s)"""
900+ cmd = ['leader-get', '--format=json'] + [attribute or '-']
901+ return json.loads(subprocess.check_output(cmd).decode('UTF-8'))
902+
903+
904+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
905+def leader_set(settings=None, **kwargs):
906+ """Juju leader set value(s)"""
907+ # Don't log secrets.
908+ # log("Juju leader-set '%s'" % (settings), level=DEBUG)
909+ cmd = ['leader-set']
910+ settings = settings or {}
911+ settings.update(kwargs)
912+ for k, v in settings.items():
913+ if v is None:
914+ cmd.append('{}='.format(k))
915+ else:
916+ cmd.append('{}={}'.format(k, v))
917+ subprocess.check_call(cmd)
918+
919+
920+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
921+def payload_register(ptype, klass, pid):
922+ """ is used while a hook is running to let Juju know that a
923+ payload has been started."""
924+ cmd = ['payload-register']
925+ for x in [ptype, klass, pid]:
926+ cmd.append(x)
927+ subprocess.check_call(cmd)
928+
929+
930+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
931+def payload_unregister(klass, pid):
932+ """ is used while a hook is running to let Juju know
933+ that a payload has been manually stopped. The <class> and <id> provided
934+ must match a payload that has been previously registered with juju using
935+ payload-register."""
936+ cmd = ['payload-unregister']
937+ for x in [klass, pid]:
938+ cmd.append(x)
939+ subprocess.check_call(cmd)
940+
941+
942+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
943+def payload_status_set(klass, pid, status):
944+ """is used to update the current status of a registered payload.
945+ The <class> and <id> provided must match a payload that has been previously
946+ registered with juju using payload-register. The <status> must be one of the
947+ follow: starting, started, stopping, stopped"""
948+ cmd = ['payload-status-set']
949+ for x in [klass, pid, status]:
950+ cmd.append(x)
951+ subprocess.check_call(cmd)
952+
953+
954+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
955+def resource_get(name):
956+ """used to fetch the resource path of the given name.
957+
958+ <name> must match a name of defined resource in metadata.yaml
959+
960+ returns either a path or False if resource not available
961+ """
962+ if not name:
963+ return False
964+
965+ cmd = ['resource-get', name]
966+ try:
967+ return subprocess.check_output(cmd).decode('UTF-8')
968+ except subprocess.CalledProcessError:
969+ return False
970+
971+
972+@cached
973+def juju_version():
974+ """Full version string (eg. '1.23.3.1-trusty-amd64')"""
975+ # Per https://bugs.launchpad.net/juju-core/+bug/1455368/comments/1
976+ jujud = glob.glob('/var/lib/juju/tools/machine-*/jujud')[0]
977+ return subprocess.check_output([jujud, 'version'],
978+ universal_newlines=True).strip()
979+
980+
981+@cached
982+def has_juju_version(minimum_version):
983+ """Return True if the Juju version is at least the provided version"""
984+ return LooseVersion(juju_version()) >= LooseVersion(minimum_version)
985+
986+
987+_atexit = []
988+_atstart = []
989+
990+
991+def atstart(callback, *args, **kwargs):
992+ '''Schedule a callback to run before the main hook.
993+
994+ Callbacks are run in the order they were added.
995+
996+ This is useful for modules and classes to perform initialization
997+ and inject behavior. In particular:
998+
999+ - Run common code before all of your hooks, such as logging
1000+ the hook name or interesting relation data.
1001+ - Defer object or module initialization that requires a hook
1002+ context until we know there actually is a hook context,
1003+ making testing easier.
1004+ - Rather than requiring charm authors to include boilerplate to
1005+ invoke your helper's behavior, have it run automatically if
1006+ your object is instantiated or module imported.
1007+
1008+ This is not at all useful after your hook framework as been launched.
1009+ '''
1010+ global _atstart
1011+ _atstart.append((callback, args, kwargs))
1012+
1013+
1014+def atexit(callback, *args, **kwargs):
1015+ '''Schedule a callback to run on successful hook completion.
1016+
1017+ Callbacks are run in the reverse order that they were added.'''
1018+ _atexit.append((callback, args, kwargs))
1019+
1020+
1021+def _run_atstart():
1022+ '''Hook frameworks must invoke this before running the main hook body.'''
1023+ global _atstart
1024+ for callback, args, kwargs in _atstart:
1025+ callback(*args, **kwargs)
1026+ del _atstart[:]
1027+
1028+
1029+def _run_atexit():
1030+ '''Hook frameworks must invoke this after the main hook body has
1031+ successfully completed. Do not invoke it if the hook fails.'''
1032+ global _atexit
1033+ for callback, args, kwargs in reversed(_atexit):
1034+ callback(*args, **kwargs)
1035+ del _atexit[:]
1036+
1037+
1038+@translate_exc(from_exc=OSError, to_exc=NotImplementedError)
1039+def network_get_primary_address(binding):
1040+ '''
1041+ Retrieve the primary network address for a named binding
1042+
1043+ :param binding: string. The name of a relation of extra-binding
1044+ :return: string. The primary IP address for the named binding
1045+ :raise: NotImplementedError if run on Juju < 2.0
1046+ '''
1047+ cmd = ['network-get', '--primary-address', binding]
1048+ return subprocess.check_output(cmd).decode('UTF-8').strip()
1049+
1050+
1051+def add_metric(*args, **kwargs):
1052+ """Add metric values. Values may be expressed with keyword arguments. For
1053+ metric names containing dashes, these may be expressed as one or more
1054+ 'key=value' positional arguments. May only be called from the collect-metrics
1055+ hook."""
1056+ _args = ['add-metric']
1057+ _kvpairs = []
1058+ _kvpairs.extend(args)
1059+ _kvpairs.extend(['{}={}'.format(k, v) for k, v in kwargs.items()])
1060+ _args.extend(sorted(_kvpairs))
1061+ try:
1062+ subprocess.check_call(_args)
1063+ return
1064+ except EnvironmentError as e:
1065+ if e.errno != errno.ENOENT:
1066+ raise
1067+ log_message = 'add-metric failed: {}'.format(' '.join(_kvpairs))
1068+ log(log_message, level='INFO')
1069+
1070+
1071+def meter_status():
1072+ """Get the meter status, if running in the meter-status-changed hook."""
1073+ return os.environ.get('JUJU_METER_STATUS')
1074+
1075+
1076+def meter_info():
1077+ """Get the meter status information, if running in the meter-status-changed
1078+ hook."""
1079+ return os.environ.get('JUJU_METER_INFO')
1080
1081=== modified file 'hooks/charmhelpers/core/host.py'
1082--- hooks/charmhelpers/core/host.py 2015-04-09 06:46:35 +0000
1083+++ hooks/charmhelpers/core/host.py 2017-07-25 09:17:37 +0000
1084@@ -1,18 +1,16 @@
1085 # Copyright 2014-2015 Canonical Limited.
1086 #
1087-# This file is part of charm-helpers.
1088-#
1089-# charm-helpers is free software: you can redistribute it and/or modify
1090-# it under the terms of the GNU Lesser General Public License version 3 as
1091-# published by the Free Software Foundation.
1092-#
1093-# charm-helpers is distributed in the hope that it will be useful,
1094-# but WITHOUT ANY WARRANTY; without even the implied warranty of
1095-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1096-# GNU Lesser General Public License for more details.
1097-#
1098-# You should have received a copy of the GNU Lesser General Public License
1099-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
1100+# Licensed under the Apache License, Version 2.0 (the "License");
1101+# you may not use this file except in compliance with the License.
1102+# You may obtain a copy of the License at
1103+#
1104+# http://www.apache.org/licenses/LICENSE-2.0
1105+#
1106+# Unless required by applicable law or agreed to in writing, software
1107+# distributed under the License is distributed on an "AS IS" BASIS,
1108+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1109+# See the License for the specific language governing permissions and
1110+# limitations under the License.
1111
1112 """Tools for working with the host system"""
1113 # Copyright 2012 Canonical Ltd.
1114@@ -24,85 +22,330 @@
1115 import os
1116 import re
1117 import pwd
1118+import glob
1119 import grp
1120 import random
1121 import string
1122 import subprocess
1123 import hashlib
1124+import functools
1125+import itertools
1126+import six
1127+
1128 from contextlib import contextmanager
1129 from collections import OrderedDict
1130-
1131-import six
1132-
1133 from .hookenv import log
1134 from .fstab import Fstab
1135-
1136-
1137-def service_start(service_name):
1138- """Start a system service"""
1139- return service('start', service_name)
1140-
1141-
1142-def service_stop(service_name):
1143- """Stop a system service"""
1144- return service('stop', service_name)
1145-
1146-
1147-def service_restart(service_name):
1148- """Restart a system service"""
1149+from charmhelpers.osplatform import get_platform
1150+
1151+__platform__ = get_platform()
1152+if __platform__ == "ubuntu":
1153+ from charmhelpers.core.host_factory.ubuntu import (
1154+ service_available,
1155+ add_new_group,
1156+ lsb_release,
1157+ cmp_pkgrevno,
1158+ CompareHostReleases,
1159+ ) # flake8: noqa -- ignore F401 for this import
1160+elif __platform__ == "centos":
1161+ from charmhelpers.core.host_factory.centos import (
1162+ service_available,
1163+ add_new_group,
1164+ lsb_release,
1165+ cmp_pkgrevno,
1166+ CompareHostReleases,
1167+ ) # flake8: noqa -- ignore F401 for this import
1168+
1169+UPDATEDB_PATH = '/etc/updatedb.conf'
1170+
1171+def service_start(service_name, **kwargs):
1172+ """Start a system service.
1173+
1174+ The specified service name is managed via the system level init system.
1175+ Some init systems (e.g. upstart) require that additional arguments be
1176+ provided in order to directly control service instances whereas other init
1177+ systems allow for addressing instances of a service directly by name (e.g.
1178+ systemd).
1179+
1180+ The kwargs allow for the additional parameters to be passed to underlying
1181+ init systems for those systems which require/allow for them. For example,
1182+ the ceph-osd upstart script requires the id parameter to be passed along
1183+ in order to identify which running daemon should be reloaded. The follow-
1184+ ing example stops the ceph-osd service for instance id=4:
1185+
1186+ service_stop('ceph-osd', id=4)
1187+
1188+ :param service_name: the name of the service to stop
1189+ :param **kwargs: additional parameters to pass to the init system when
1190+ managing services. These will be passed as key=value
1191+ parameters to the init system's commandline. kwargs
1192+ are ignored for systemd enabled systems.
1193+ """
1194+ return service('start', service_name, **kwargs)
1195+
1196+
1197+def service_stop(service_name, **kwargs):
1198+ """Stop a system service.
1199+
1200+ The specified service name is managed via the system level init system.
1201+ Some init systems (e.g. upstart) require that additional arguments be
1202+ provided in order to directly control service instances whereas other init
1203+ systems allow for addressing instances of a service directly by name (e.g.
1204+ systemd).
1205+
1206+ The kwargs allow for the additional parameters to be passed to underlying
1207+ init systems for those systems which require/allow for them. For example,
1208+ the ceph-osd upstart script requires the id parameter to be passed along
1209+ in order to identify which running daemon should be reloaded. The follow-
1210+ ing example stops the ceph-osd service for instance id=4:
1211+
1212+ service_stop('ceph-osd', id=4)
1213+
1214+ :param service_name: the name of the service to stop
1215+ :param **kwargs: additional parameters to pass to the init system when
1216+ managing services. These will be passed as key=value
1217+ parameters to the init system's commandline. kwargs
1218+ are ignored for systemd enabled systems.
1219+ """
1220+ return service('stop', service_name, **kwargs)
1221+
1222+
1223+def service_restart(service_name, **kwargs):
1224+ """Restart a system service.
1225+
1226+ The specified service name is managed via the system level init system.
1227+ Some init systems (e.g. upstart) require that additional arguments be
1228+ provided in order to directly control service instances whereas other init
1229+ systems allow for addressing instances of a service directly by name (e.g.
1230+ systemd).
1231+
1232+ The kwargs allow for the additional parameters to be passed to underlying
1233+ init systems for those systems which require/allow for them. For example,
1234+ the ceph-osd upstart script requires the id parameter to be passed along
1235+ in order to identify which running daemon should be restarted. The follow-
1236+ ing example restarts the ceph-osd service for instance id=4:
1237+
1238+ service_restart('ceph-osd', id=4)
1239+
1240+ :param service_name: the name of the service to restart
1241+ :param **kwargs: additional parameters to pass to the init system when
1242+ managing services. These will be passed as key=value
1243+ parameters to the init system's commandline. kwargs
1244+ are ignored for init systems not allowing additional
1245+ parameters via the commandline (systemd).
1246+ """
1247 return service('restart', service_name)
1248
1249
1250-def service_reload(service_name, restart_on_failure=False):
1251+def service_reload(service_name, restart_on_failure=False, **kwargs):
1252 """Reload a system service, optionally falling back to restart if
1253- reload fails"""
1254- service_result = service('reload', service_name)
1255+ reload fails.
1256+
1257+ The specified service name is managed via the system level init system.
1258+ Some init systems (e.g. upstart) require that additional arguments be
1259+ provided in order to directly control service instances whereas other init
1260+ systems allow for addressing instances of a service directly by name (e.g.
1261+ systemd).
1262+
1263+ The kwargs allow for the additional parameters to be passed to underlying
1264+ init systems for those systems which require/allow for them. For example,
1265+ the ceph-osd upstart script requires the id parameter to be passed along
1266+ in order to identify which running daemon should be reloaded. The follow-
1267+ ing example restarts the ceph-osd service for instance id=4:
1268+
1269+ service_reload('ceph-osd', id=4)
1270+
1271+ :param service_name: the name of the service to reload
1272+ :param restart_on_failure: boolean indicating whether to fallback to a
1273+ restart if the reload fails.
1274+ :param **kwargs: additional parameters to pass to the init system when
1275+ managing services. These will be passed as key=value
1276+ parameters to the init system's commandline. kwargs
1277+ are ignored for init systems not allowing additional
1278+ parameters via the commandline (systemd).
1279+ """
1280+ service_result = service('reload', service_name, **kwargs)
1281 if not service_result and restart_on_failure:
1282- service_result = service('restart', service_name)
1283+ service_result = service('restart', service_name, **kwargs)
1284 return service_result
1285
1286
1287-def service(action, service_name):
1288- """Control a system service"""
1289- cmd = ['service', service_name, action]
1290+def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d",
1291+ **kwargs):
1292+ """Pause a system service.
1293+
1294+ Stop it, and prevent it from starting again at boot.
1295+
1296+ :param service_name: the name of the service to pause
1297+ :param init_dir: path to the upstart init directory
1298+ :param initd_dir: path to the sysv init directory
1299+ :param **kwargs: additional parameters to pass to the init system when
1300+ managing services. These will be passed as key=value
1301+ parameters to the init system's commandline. kwargs
1302+ are ignored for init systems which do not support
1303+ key=value arguments via the commandline.
1304+ """
1305+ stopped = True
1306+ if service_running(service_name, **kwargs):
1307+ stopped = service_stop(service_name, **kwargs)
1308+ upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
1309+ sysv_file = os.path.join(initd_dir, service_name)
1310+ if init_is_systemd():
1311+ service('disable', service_name)
1312+ service('mask', service_name)
1313+ elif os.path.exists(upstart_file):
1314+ override_path = os.path.join(
1315+ init_dir, '{}.override'.format(service_name))
1316+ with open(override_path, 'w') as fh:
1317+ fh.write("manual\n")
1318+ elif os.path.exists(sysv_file):
1319+ subprocess.check_call(["update-rc.d", service_name, "disable"])
1320+ else:
1321+ raise ValueError(
1322+ "Unable to detect {0} as SystemD, Upstart {1} or"
1323+ " SysV {2}".format(
1324+ service_name, upstart_file, sysv_file))
1325+ return stopped
1326+
1327+
1328+def service_resume(service_name, init_dir="/etc/init",
1329+ initd_dir="/etc/init.d", **kwargs):
1330+ """Resume a system service.
1331+
1332+ Reenable starting again at boot. Start the service.
1333+
1334+ :param service_name: the name of the service to resume
1335+ :param init_dir: the path to the init dir
1336+ :param initd dir: the path to the initd dir
1337+ :param **kwargs: additional parameters to pass to the init system when
1338+ managing services. These will be passed as key=value
1339+ parameters to the init system's commandline. kwargs
1340+ are ignored for systemd enabled systems.
1341+ """
1342+ upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
1343+ sysv_file = os.path.join(initd_dir, service_name)
1344+ if init_is_systemd():
1345+ service('unmask', service_name)
1346+ service('enable', service_name)
1347+ elif os.path.exists(upstart_file):
1348+ override_path = os.path.join(
1349+ init_dir, '{}.override'.format(service_name))
1350+ if os.path.exists(override_path):
1351+ os.unlink(override_path)
1352+ elif os.path.exists(sysv_file):
1353+ subprocess.check_call(["update-rc.d", service_name, "enable"])
1354+ else:
1355+ raise ValueError(
1356+ "Unable to detect {0} as SystemD, Upstart {1} or"
1357+ " SysV {2}".format(
1358+ service_name, upstart_file, sysv_file))
1359+ started = service_running(service_name, **kwargs)
1360+
1361+ if not started:
1362+ started = service_start(service_name, **kwargs)
1363+ return started
1364+
1365+
1366+def service(action, service_name, **kwargs):
1367+ """Control a system service.
1368+
1369+ :param action: the action to take on the service
1370+ :param service_name: the name of the service to perform th action on
1371+ :param **kwargs: additional params to be passed to the service command in
1372+ the form of key=value.
1373+ """
1374+ if init_is_systemd():
1375+ cmd = ['systemctl', action, service_name]
1376+ else:
1377+ cmd = ['service', service_name, action]
1378+ for key, value in six.iteritems(kwargs):
1379+ parameter = '%s=%s' % (key, value)
1380+ cmd.append(parameter)
1381 return subprocess.call(cmd) == 0
1382
1383
1384-def service_running(service):
1385- """Determine whether a system service is running"""
1386- try:
1387- output = subprocess.check_output(
1388- ['service', service, 'status'],
1389- stderr=subprocess.STDOUT).decode('UTF-8')
1390- except subprocess.CalledProcessError:
1391- return False
1392- else:
1393- if ("start/running" in output or "is running" in output):
1394- return True
1395- else:
1396- return False
1397-
1398-
1399-def service_available(service_name):
1400- """Determine whether a system service is available"""
1401- try:
1402- subprocess.check_output(
1403- ['service', service_name, 'status'],
1404- stderr=subprocess.STDOUT).decode('UTF-8')
1405- except subprocess.CalledProcessError as e:
1406- return 'unrecognized service' not in e.output
1407- else:
1408- return True
1409-
1410-
1411-def adduser(username, password=None, shell='/bin/bash', system_user=False):
1412- """Add a user to the system"""
1413+_UPSTART_CONF = "/etc/init/{}.conf"
1414+_INIT_D_CONF = "/etc/init.d/{}"
1415+
1416+
1417+def service_running(service_name, **kwargs):
1418+ """Determine whether a system service is running.
1419+
1420+ :param service_name: the name of the service
1421+ :param **kwargs: additional args to pass to the service command. This is
1422+ used to pass additional key=value arguments to the
1423+ service command line for managing specific instance
1424+ units (e.g. service ceph-osd status id=2). The kwargs
1425+ are ignored in systemd services.
1426+ """
1427+ if init_is_systemd():
1428+ return service('is-active', service_name)
1429+ else:
1430+ if os.path.exists(_UPSTART_CONF.format(service_name)):
1431+ try:
1432+ cmd = ['status', service_name]
1433+ for key, value in six.iteritems(kwargs):
1434+ parameter = '%s=%s' % (key, value)
1435+ cmd.append(parameter)
1436+ output = subprocess.check_output(cmd,
1437+ stderr=subprocess.STDOUT).decode('UTF-8')
1438+ except subprocess.CalledProcessError:
1439+ return False
1440+ else:
1441+ # This works for upstart scripts where the 'service' command
1442+ # returns a consistent string to represent running
1443+ # 'start/running'
1444+ if ("start/running" in output or
1445+ "is running" in output or
1446+ "up and running" in output):
1447+ return True
1448+ elif os.path.exists(_INIT_D_CONF.format(service_name)):
1449+ # Check System V scripts init script return codes
1450+ return service('status', service_name)
1451+ return False
1452+
1453+
1454+SYSTEMD_SYSTEM = '/run/systemd/system'
1455+
1456+
1457+def init_is_systemd():
1458+ """Return True if the host system uses systemd, False otherwise."""
1459+ if lsb_release()['DISTRIB_CODENAME'] == 'trusty':
1460+ return False
1461+ return os.path.isdir(SYSTEMD_SYSTEM)
1462+
1463+
1464+def adduser(username, password=None, shell='/bin/bash',
1465+ system_user=False, primary_group=None,
1466+ secondary_groups=None, uid=None, home_dir=None):
1467+ """Add a user to the system.
1468+
1469+ Will log but otherwise succeed if the user already exists.
1470+
1471+ :param str username: Username to create
1472+ :param str password: Password for user; if ``None``, create a system user
1473+ :param str shell: The default shell for the user
1474+ :param bool system_user: Whether to create a login or system user
1475+ :param str primary_group: Primary group for user; defaults to username
1476+ :param list secondary_groups: Optional list of additional groups
1477+ :param int uid: UID for user being created
1478+ :param str home_dir: Home directory for user
1479+
1480+ :returns: The password database entry struct, as returned by `pwd.getpwnam`
1481+ """
1482 try:
1483 user_info = pwd.getpwnam(username)
1484 log('user {0} already exists!'.format(username))
1485+ if uid:
1486+ user_info = pwd.getpwuid(int(uid))
1487+ log('user with uid {0} already exists!'.format(uid))
1488 except KeyError:
1489 log('creating user {0}'.format(username))
1490 cmd = ['useradd']
1491+ if uid:
1492+ cmd.extend(['--uid', str(uid)])
1493+ if home_dir:
1494+ cmd.extend(['--home', str(home_dir)])
1495 if system_user or password is None:
1496 cmd.append('--system')
1497 else:
1498@@ -111,52 +354,104 @@
1499 '--shell', shell,
1500 '--password', password,
1501 ])
1502+ if not primary_group:
1503+ try:
1504+ grp.getgrnam(username)
1505+ primary_group = username # avoid "group exists" error
1506+ except KeyError:
1507+ pass
1508+ if primary_group:
1509+ cmd.extend(['-g', primary_group])
1510+ if secondary_groups:
1511+ cmd.extend(['-G', ','.join(secondary_groups)])
1512 cmd.append(username)
1513 subprocess.check_call(cmd)
1514 user_info = pwd.getpwnam(username)
1515 return user_info
1516
1517
1518-def add_group(group_name, system_group=False):
1519- """Add a group to the system"""
1520+def user_exists(username):
1521+ """Check if a user exists"""
1522+ try:
1523+ pwd.getpwnam(username)
1524+ user_exists = True
1525+ except KeyError:
1526+ user_exists = False
1527+ return user_exists
1528+
1529+
1530+def uid_exists(uid):
1531+ """Check if a uid exists"""
1532+ try:
1533+ pwd.getpwuid(uid)
1534+ uid_exists = True
1535+ except KeyError:
1536+ uid_exists = False
1537+ return uid_exists
1538+
1539+
1540+def group_exists(groupname):
1541+ """Check if a group exists"""
1542+ try:
1543+ grp.getgrnam(groupname)
1544+ group_exists = True
1545+ except KeyError:
1546+ group_exists = False
1547+ return group_exists
1548+
1549+
1550+def gid_exists(gid):
1551+ """Check if a gid exists"""
1552+ try:
1553+ grp.getgrgid(gid)
1554+ gid_exists = True
1555+ except KeyError:
1556+ gid_exists = False
1557+ return gid_exists
1558+
1559+
1560+def add_group(group_name, system_group=False, gid=None):
1561+ """Add a group to the system
1562+
1563+ Will log but otherwise succeed if the group already exists.
1564+
1565+ :param str group_name: group to create
1566+ :param bool system_group: Create system group
1567+ :param int gid: GID for user being created
1568+
1569+ :returns: The password database entry struct, as returned by `grp.getgrnam`
1570+ """
1571 try:
1572 group_info = grp.getgrnam(group_name)
1573 log('group {0} already exists!'.format(group_name))
1574+ if gid:
1575+ group_info = grp.getgrgid(gid)
1576+ log('group with gid {0} already exists!'.format(gid))
1577 except KeyError:
1578 log('creating group {0}'.format(group_name))
1579- cmd = ['addgroup']
1580- if system_group:
1581- cmd.append('--system')
1582- else:
1583- cmd.extend([
1584- '--group',
1585- ])
1586- cmd.append(group_name)
1587- subprocess.check_call(cmd)
1588+ add_new_group(group_name, system_group, gid)
1589 group_info = grp.getgrnam(group_name)
1590 return group_info
1591
1592
1593 def add_user_to_group(username, group):
1594 """Add a user to a group"""
1595- cmd = [
1596- 'gpasswd', '-a',
1597- username,
1598- group
1599- ]
1600+ cmd = ['gpasswd', '-a', username, group]
1601 log("Adding user {} to group {}".format(username, group))
1602 subprocess.check_call(cmd)
1603
1604
1605-def rsync(from_path, to_path, flags='-r', options=None):
1606+def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
1607 """Replicate the contents of a path"""
1608 options = options or ['--delete', '--executability']
1609 cmd = ['/usr/bin/rsync', flags]
1610+ if timeout:
1611+ cmd = ['timeout', str(timeout)] + cmd
1612 cmd.extend(options)
1613 cmd.append(from_path)
1614 cmd.append(to_path)
1615 log(" ".join(cmd))
1616- return subprocess.check_output(cmd).decode('UTF-8').strip()
1617+ return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip()
1618
1619
1620 def symlink(source, destination):
1621@@ -202,14 +497,12 @@
1622
1623
1624 def fstab_remove(mp):
1625- """Remove the given mountpoint entry from /etc/fstab
1626- """
1627+ """Remove the given mountpoint entry from /etc/fstab"""
1628 return Fstab.remove_by_mountpoint(mp)
1629
1630
1631 def fstab_add(dev, mp, fs, options=None):
1632- """Adds the given device entry to the /etc/fstab file
1633- """
1634+ """Adds the given device entry to the /etc/fstab file"""
1635 return Fstab.add(dev, mp, fs, options=options)
1636
1637
1638@@ -253,9 +546,19 @@
1639 return system_mounts
1640
1641
1642+def fstab_mount(mountpoint):
1643+ """Mount filesystem using fstab"""
1644+ cmd_args = ['mount', mountpoint]
1645+ try:
1646+ subprocess.check_output(cmd_args)
1647+ except subprocess.CalledProcessError as e:
1648+ log('Error unmounting {}\n{}'.format(mountpoint, e.output))
1649+ return False
1650+ return True
1651+
1652+
1653 def file_hash(path, hash_type='md5'):
1654- """
1655- Generate a hash checksum of the contents of 'path' or None if not found.
1656+ """Generate a hash checksum of the contents of 'path' or None if not found.
1657
1658 :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`,
1659 such as md5, sha1, sha256, sha512, etc.
1660@@ -269,9 +572,22 @@
1661 return None
1662
1663
1664+def path_hash(path):
1665+ """Generate a hash checksum of all files matching 'path'. Standard
1666+ wildcards like '*' and '?' are supported, see documentation for the 'glob'
1667+ module for more information.
1668+
1669+ :return: dict: A { filename: hash } dictionary for all matched files.
1670+ Empty if none found.
1671+ """
1672+ return {
1673+ filename: file_hash(filename)
1674+ for filename in glob.iglob(path)
1675+ }
1676+
1677+
1678 def check_hash(path, checksum, hash_type='md5'):
1679- """
1680- Validate a file using a cryptographic checksum.
1681+ """Validate a file using a cryptographic checksum.
1682
1683 :param str checksum: Value of the checksum used to validate the file.
1684 :param str hash_type: Hash algorithm used to generate `checksum`.
1685@@ -286,54 +602,78 @@
1686
1687
1688 class ChecksumError(ValueError):
1689+ """A class derived from Value error to indicate the checksum failed."""
1690 pass
1691
1692
1693-def restart_on_change(restart_map, stopstart=False):
1694+def restart_on_change(restart_map, stopstart=False, restart_functions=None):
1695 """Restart services based on configuration files changing
1696
1697 This function is used a decorator, for example::
1698
1699 @restart_on_change({
1700 '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
1701+ '/etc/apache/sites-enabled/*': [ 'apache2' ]
1702 })
1703- def ceph_client_changed():
1704+ def config_changed():
1705 pass # your code here
1706
1707 In this example, the cinder-api and cinder-volume services
1708 would be restarted if /etc/ceph/ceph.conf is changed by the
1709- ceph_client_changed function.
1710+ ceph_client_changed function. The apache2 service would be
1711+ restarted if any file matching the pattern got changed, created
1712+ or removed. Standard wildcards are supported, see documentation
1713+ for the 'glob' module for more information.
1714+
1715+ @param restart_map: {path_file_name: [service_name, ...]
1716+ @param stopstart: DEFAULT false; whether to stop, start OR restart
1717+ @param restart_functions: nonstandard functions to use to restart services
1718+ {svc: func, ...}
1719+ @returns result from decorated function
1720 """
1721 def wrap(f):
1722+ @functools.wraps(f)
1723 def wrapped_f(*args, **kwargs):
1724- checksums = {}
1725- for path in restart_map:
1726- checksums[path] = file_hash(path)
1727- f(*args, **kwargs)
1728- restarts = []
1729- for path in restart_map:
1730- if checksums[path] != file_hash(path):
1731- restarts += restart_map[path]
1732- services_list = list(OrderedDict.fromkeys(restarts))
1733- if not stopstart:
1734- for service_name in services_list:
1735- service('restart', service_name)
1736- else:
1737- for action in ['stop', 'start']:
1738- for service_name in services_list:
1739- service(action, service_name)
1740+ return restart_on_change_helper(
1741+ (lambda: f(*args, **kwargs)), restart_map, stopstart,
1742+ restart_functions)
1743 return wrapped_f
1744 return wrap
1745
1746
1747-def lsb_release():
1748- """Return /etc/lsb-release in a dict"""
1749- d = {}
1750- with open('/etc/lsb-release', 'r') as lsb:
1751- for l in lsb:
1752- k, v = l.split('=')
1753- d[k.strip()] = v.strip()
1754- return d
1755+def restart_on_change_helper(lambda_f, restart_map, stopstart=False,
1756+ restart_functions=None):
1757+ """Helper function to perform the restart_on_change function.
1758+
1759+ This is provided for decorators to restart services if files described
1760+ in the restart_map have changed after an invocation of lambda_f().
1761+
1762+ @param lambda_f: function to call.
1763+ @param restart_map: {file: [service, ...]}
1764+ @param stopstart: whether to stop, start or restart a service
1765+ @param restart_functions: nonstandard functions to use to restart services
1766+ {svc: func, ...}
1767+ @returns result of lambda_f()
1768+ """
1769+ if restart_functions is None:
1770+ restart_functions = {}
1771+ checksums = {path: path_hash(path) for path in restart_map}
1772+ r = lambda_f()
1773+ # create a list of lists of the services to restart
1774+ restarts = [restart_map[path]
1775+ for path in restart_map
1776+ if path_hash(path) != checksums[path]]
1777+ # create a flat list of ordered services without duplicates from lists
1778+ services_list = list(OrderedDict.fromkeys(itertools.chain(*restarts)))
1779+ if services_list:
1780+ actions = ('stop', 'start') if stopstart else ('restart',)
1781+ for service_name in services_list:
1782+ if service_name in restart_functions:
1783+ restart_functions[service_name](service_name)
1784+ else:
1785+ for action in actions:
1786+ service(action, service_name)
1787+ return r
1788
1789
1790 def pwgen(length=None):
1791@@ -352,36 +692,92 @@
1792 return(''.join(random_chars))
1793
1794
1795-def list_nics(nic_type):
1796- '''Return a list of nics of given type(s)'''
1797+def is_phy_iface(interface):
1798+ """Returns True if interface is not virtual, otherwise False."""
1799+ if interface:
1800+ sys_net = '/sys/class/net'
1801+ if os.path.isdir(sys_net):
1802+ for iface in glob.glob(os.path.join(sys_net, '*')):
1803+ if '/virtual/' in os.path.realpath(iface):
1804+ continue
1805+
1806+ if interface == os.path.basename(iface):
1807+ return True
1808+
1809+ return False
1810+
1811+
1812+def get_bond_master(interface):
1813+ """Returns bond master if interface is bond slave otherwise None.
1814+
1815+ NOTE: the provided interface is expected to be physical
1816+ """
1817+ if interface:
1818+ iface_path = '/sys/class/net/%s' % (interface)
1819+ if os.path.exists(iface_path):
1820+ if '/virtual/' in os.path.realpath(iface_path):
1821+ return None
1822+
1823+ master = os.path.join(iface_path, 'master')
1824+ if os.path.exists(master):
1825+ master = os.path.realpath(master)
1826+ # make sure it is a bond master
1827+ if os.path.exists(os.path.join(master, 'bonding')):
1828+ return os.path.basename(master)
1829+
1830+ return None
1831+
1832+
1833+def list_nics(nic_type=None):
1834+ """Return a list of nics of given type(s)"""
1835 if isinstance(nic_type, six.string_types):
1836 int_types = [nic_type]
1837 else:
1838 int_types = nic_type
1839+
1840 interfaces = []
1841- for int_type in int_types:
1842- cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
1843+ if nic_type:
1844+ for int_type in int_types:
1845+ cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
1846+ ip_output = subprocess.check_output(cmd).decode('UTF-8')
1847+ ip_output = ip_output.split('\n')
1848+ ip_output = (line for line in ip_output if line)
1849+ for line in ip_output:
1850+ if line.split()[1].startswith(int_type):
1851+ matched = re.search('.*: (' + int_type +
1852+ r'[0-9]+\.[0-9]+)@.*', line)
1853+ if matched:
1854+ iface = matched.groups()[0]
1855+ else:
1856+ iface = line.split()[1].replace(":", "")
1857+
1858+ if iface not in interfaces:
1859+ interfaces.append(iface)
1860+ else:
1861+ cmd = ['ip', 'a']
1862 ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
1863- ip_output = (line for line in ip_output if line)
1864+ ip_output = (line.strip() for line in ip_output if line)
1865+
1866+ key = re.compile('^[0-9]+:\s+(.+):')
1867 for line in ip_output:
1868- if line.split()[1].startswith(int_type):
1869- matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line)
1870- if matched:
1871- interface = matched.groups()[0]
1872- else:
1873- interface = line.split()[1].replace(":", "")
1874- interfaces.append(interface)
1875+ matched = re.search(key, line)
1876+ if matched:
1877+ iface = matched.group(1)
1878+ iface = iface.partition("@")[0]
1879+ if iface not in interfaces:
1880+ interfaces.append(iface)
1881
1882 return interfaces
1883
1884
1885 def set_nic_mtu(nic, mtu):
1886- '''Set MTU on a network interface'''
1887+ """Set the Maximum Transmission Unit (MTU) on a network interface."""
1888 cmd = ['ip', 'link', 'set', nic, 'mtu', mtu]
1889 subprocess.check_call(cmd)
1890
1891
1892 def get_nic_mtu(nic):
1893+ """Return the Maximum Transmission Unit (MTU) for a network interface."""
1894 cmd = ['ip', 'addr', 'show', nic]
1895 ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
1896 mtu = ""
1897@@ -393,6 +789,7 @@
1898
1899
1900 def get_nic_hwaddr(nic):
1901+ """Return the Media Access Control (MAC) for a network interface."""
1902 cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
1903 ip_output = subprocess.check_output(cmd).decode('UTF-8')
1904 hwaddr = ""
1905@@ -402,35 +799,31 @@
1906 return hwaddr
1907
1908
1909-def cmp_pkgrevno(package, revno, pkgcache=None):
1910- '''Compare supplied revno with the revno of the installed package
1911-
1912- * 1 => Installed revno is greater than supplied arg
1913- * 0 => Installed revno is the same as supplied arg
1914- * -1 => Installed revno is less than supplied arg
1915-
1916- This function imports apt_cache function from charmhelpers.fetch if
1917- the pkgcache argument is None. Be sure to add charmhelpers.fetch if
1918- you call this function, or pass an apt_pkg.Cache() instance.
1919- '''
1920- import apt_pkg
1921- if not pkgcache:
1922- from charmhelpers.fetch import apt_cache
1923- pkgcache = apt_cache()
1924- pkg = pkgcache[package]
1925- return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
1926-
1927-
1928 @contextmanager
1929-def chdir(d):
1930+def chdir(directory):
1931+ """Change the current working directory to a different directory for a code
1932+ block and return the previous directory after the block exits. Useful to
1933+ run commands from a specificed directory.
1934+
1935+ :param str directory: The directory path to change to for this context.
1936+ """
1937 cur = os.getcwd()
1938 try:
1939- yield os.chdir(d)
1940+ yield os.chdir(directory)
1941 finally:
1942 os.chdir(cur)
1943
1944
1945-def chownr(path, owner, group, follow_links=True):
1946+def chownr(path, owner, group, follow_links=True, chowntopdir=False):
1947+ """Recursively change user and group ownership of files and directories
1948+ in given path. Doesn't chown path itself by default, only its children.
1949+
1950+ :param str path: The string path to start changing ownership.
1951+ :param str owner: The owner string to use when looking up the uid.
1952+ :param str group: The group string to use when looking up the gid.
1953+ :param bool follow_links: Also follow and chown links if True
1954+ :param bool chowntopdir: Also chown path itself if True
1955+ """
1956 uid = pwd.getpwnam(owner).pw_uid
1957 gid = grp.getgrnam(group).gr_gid
1958 if follow_links:
1959@@ -438,7 +831,11 @@
1960 else:
1961 chown = os.lchown
1962
1963- for root, dirs, files in os.walk(path):
1964+ if chowntopdir:
1965+ broken_symlink = os.path.lexists(path) and not os.path.exists(path)
1966+ if not broken_symlink:
1967+ chown(path, uid, gid)
1968+ for root, dirs, files in os.walk(path, followlinks=follow_links):
1969 for name in dirs + files:
1970 full = os.path.join(root, name)
1971 broken_symlink = os.path.lexists(full) and not os.path.exists(full)
1972@@ -447,4 +844,81 @@
1973
1974
1975 def lchownr(path, owner, group):
1976+ """Recursively change user and group ownership of files and directories
1977+ in a given path, not following symbolic links. See the documentation for
1978+ 'os.lchown' for more information.
1979+
1980+ :param str path: The string path to start changing ownership.
1981+ :param str owner: The owner string to use when looking up the uid.
1982+ :param str group: The group string to use when looking up the gid.
1983+ """
1984 chownr(path, owner, group, follow_links=False)
1985+
1986+
1987+def owner(path):
1988+ """Returns a tuple containing the username & groupname owning the path.
1989+
1990+ :param str path: the string path to retrieve the ownership
1991+ :return tuple(str, str): A (username, groupname) tuple containing the
1992+ name of the user and group owning the path.
1993+ :raises OSError: if the specified path does not exist
1994+ """
1995+ stat = os.stat(path)
1996+ username = pwd.getpwuid(stat.st_uid)[0]
1997+ groupname = grp.getgrgid(stat.st_gid)[0]
1998+ return username, groupname
1999+
2000+
2001+def get_total_ram():
2002+ """The total amount of system RAM in bytes.
2003+
2004+ This is what is reported by the OS, and may be overcommitted when
2005+ there are multiple containers hosted on the same machine.
2006+ """
2007+ with open('/proc/meminfo', 'r') as f:
2008+ for line in f.readlines():
2009+ if line:
2010+ key, value, unit = line.split()
2011+ if key == 'MemTotal:':
2012+ assert unit == 'kB', 'Unknown unit'
2013+ return int(value) * 1024 # Classic, not KiB.
2014+ raise NotImplementedError()
2015+
2016+
2017+UPSTART_CONTAINER_TYPE = '/run/container_type'
2018+
2019+
2020+def is_container():
2021+ """Determine whether unit is running in a container
2022+
2023+ @return: boolean indicating if unit is in a container
2024+ """
2025+ if init_is_systemd():
2026+ # Detect using systemd-detect-virt
2027+ return subprocess.call(['systemd-detect-virt',
2028+ '--container']) == 0
2029+ else:
2030+ # Detect using upstart container file marker
2031+ return os.path.exists(UPSTART_CONTAINER_TYPE)
2032+
2033+
2034+def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH):
2035+ with open(updatedb_path, 'r+') as f_id:
2036+ updatedb_text = f_id.read()
2037+ output = updatedb(updatedb_text, path)
2038+ f_id.seek(0)
2039+ f_id.write(output)
2040+ f_id.truncate()
2041+
2042+
2043+def updatedb(updatedb_text, new_path):
2044+ lines = [line for line in updatedb_text.split("\n")]
2045+ for i, line in enumerate(lines):
2046+ if line.startswith("PRUNEPATHS="):
2047+ paths_line = line.split("=")[1].replace('"', '')
2048+ paths = paths_line.split(" ")
2049+ if new_path not in paths:
2050+ paths.append(new_path)
2051+ lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths))
2052+ output = "\n".join(lines)
2053+ return output
2054
2055=== added directory 'hooks/charmhelpers/core/host_factory'
2056=== added file 'hooks/charmhelpers/core/host_factory/__init__.py'
2057=== added file 'hooks/charmhelpers/core/host_factory/centos.py'
2058--- hooks/charmhelpers/core/host_factory/centos.py 1970-01-01 00:00:00 +0000
2059+++ hooks/charmhelpers/core/host_factory/centos.py 2017-07-25 09:17:37 +0000
2060@@ -0,0 +1,72 @@
2061+import subprocess
2062+import yum
2063+import os
2064+
2065+from charmhelpers.core.strutils import BasicStringComparator
2066+
2067+
2068+class CompareHostReleases(BasicStringComparator):
2069+ """Provide comparisons of Host releases.
2070+
2071+ Use in the form of
2072+
2073+ if CompareHostReleases(release) > 'trusty':
2074+ # do something with mitaka
2075+ """
2076+
2077+ def __init__(self, item):
2078+ raise NotImplementedError(
2079+ "CompareHostReleases() is not implemented for CentOS")
2080+
2081+
2082+def service_available(service_name):
2083+ # """Determine whether a system service is available."""
2084+ if os.path.isdir('/run/systemd/system'):
2085+ cmd = ['systemctl', 'is-enabled', service_name]
2086+ else:
2087+ cmd = ['service', service_name, 'is-enabled']
2088+ return subprocess.call(cmd) == 0
2089+
2090+
2091+def add_new_group(group_name, system_group=False, gid=None):
2092+ cmd = ['groupadd']
2093+ if gid:
2094+ cmd.extend(['--gid', str(gid)])
2095+ if system_group:
2096+ cmd.append('-r')
2097+ cmd.append(group_name)
2098+ subprocess.check_call(cmd)
2099+
2100+
2101+def lsb_release():
2102+ """Return /etc/os-release in a dict."""
2103+ d = {}
2104+ with open('/etc/os-release', 'r') as lsb:
2105+ for l in lsb:
2106+ s = l.split('=')
2107+ if len(s) != 2:
2108+ continue
2109+ d[s[0].strip()] = s[1].strip()
2110+ return d
2111+
2112+
2113+def cmp_pkgrevno(package, revno, pkgcache=None):
2114+ """Compare supplied revno with the revno of the installed package.
2115+
2116+ * 1 => Installed revno is greater than supplied arg
2117+ * 0 => Installed revno is the same as supplied arg
2118+ * -1 => Installed revno is less than supplied arg
2119+
2120+ This function imports YumBase function if the pkgcache argument
2121+ is None.
2122+ """
2123+ if not pkgcache:
2124+ y = yum.YumBase()
2125+ packages = y.doPackageLists()
2126+ pkgcache = {i.Name: i.version for i in packages['installed']}
2127+ pkg = pkgcache[package]
2128+ if pkg > revno:
2129+ return 1
2130+ if pkg < revno:
2131+ return -1
2132+ return 0
2133
2134=== added file 'hooks/charmhelpers/core/host_factory/ubuntu.py'
2135--- hooks/charmhelpers/core/host_factory/ubuntu.py 1970-01-01 00:00:00 +0000
2136+++ hooks/charmhelpers/core/host_factory/ubuntu.py 2017-07-25 09:17:37 +0000
2137@@ -0,0 +1,89 @@
2138+import subprocess
2139+
2140+from charmhelpers.core.strutils import BasicStringComparator
2141+
2142+
2143+UBUNTU_RELEASES = (
2144+ 'lucid',
2145+ 'maverick',
2146+ 'natty',
2147+ 'oneiric',
2148+ 'precise',
2149+ 'quantal',
2150+ 'raring',
2151+ 'saucy',
2152+ 'trusty',
2153+ 'utopic',
2154+ 'vivid',
2155+ 'wily',
2156+ 'xenial',
2157+ 'yakkety',
2158+ 'zesty',
2159+ 'artful',
2160+)
2161+
2162+
2163+class CompareHostReleases(BasicStringComparator):
2164+ """Provide comparisons of Ubuntu releases.
2165+
2166+ Use in the form of
2167+
2168+ if CompareHostReleases(release) > 'trusty':
2169+ # do something with mitaka
2170+ """
2171+ _list = UBUNTU_RELEASES
2172+
2173+
2174+def service_available(service_name):
2175+ """Determine whether a system service is available"""
2176+ try:
2177+ subprocess.check_output(
2178+ ['service', service_name, 'status'],
2179+ stderr=subprocess.STDOUT).decode('UTF-8')
2180+ except subprocess.CalledProcessError as e:
2181+ return b'unrecognized service' not in e.output
2182+ else:
2183+ return True
2184+
2185+
2186+def add_new_group(group_name, system_group=False, gid=None):
2187+ cmd = ['addgroup']
2188+ if gid:
2189+ cmd.extend(['--gid', str(gid)])
2190+ if system_group:
2191+ cmd.append('--system')
2192+ else:
2193+ cmd.extend([
2194+ '--group',
2195+ ])
2196+ cmd.append(group_name)
2197+ subprocess.check_call(cmd)
2198+
2199+
2200+def lsb_release():
2201+ """Return /etc/lsb-release in a dict"""
2202+ d = {}
2203+ with open('/etc/lsb-release', 'r') as lsb:
2204+ for l in lsb:
2205+ k, v = l.split('=')
2206+ d[k.strip()] = v.strip()
2207+ return d
2208+
2209+
2210+def cmp_pkgrevno(package, revno, pkgcache=None):
2211+ """Compare supplied revno with the revno of the installed package.
2212+
2213+ * 1 => Installed revno is greater than supplied arg
2214+ * 0 => Installed revno is the same as supplied arg
2215+ * -1 => Installed revno is less than supplied arg
2216+
2217+ This function imports apt_cache function from charmhelpers.fetch if
2218+ the pkgcache argument is None. Be sure to add charmhelpers.fetch if
2219+ you call this function, or pass an apt_pkg.Cache() instance.
2220+ """
2221+ import apt_pkg
2222+ if not pkgcache:
2223+ from charmhelpers.fetch import apt_cache
2224+ pkgcache = apt_cache()
2225+ pkg = pkgcache[package]
2226+ return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
2227
2228=== added file 'hooks/charmhelpers/core/hugepage.py'
2229--- hooks/charmhelpers/core/hugepage.py 1970-01-01 00:00:00 +0000
2230+++ hooks/charmhelpers/core/hugepage.py 2017-07-25 09:17:37 +0000
2231@@ -0,0 +1,69 @@
2232+# -*- coding: utf-8 -*-
2233+
2234+# Copyright 2014-2015 Canonical Limited.
2235+#
2236+# Licensed under the Apache License, Version 2.0 (the "License");
2237+# you may not use this file except in compliance with the License.
2238+# You may obtain a copy of the License at
2239+#
2240+# http://www.apache.org/licenses/LICENSE-2.0
2241+#
2242+# Unless required by applicable law or agreed to in writing, software
2243+# distributed under the License is distributed on an "AS IS" BASIS,
2244+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2245+# See the License for the specific language governing permissions and
2246+# limitations under the License.
2247+
2248+import yaml
2249+from charmhelpers.core import fstab
2250+from charmhelpers.core import sysctl
2251+from charmhelpers.core.host import (
2252+ add_group,
2253+ add_user_to_group,
2254+ fstab_mount,
2255+ mkdir,
2256+)
2257+from charmhelpers.core.strutils import bytes_from_string
2258+from subprocess import check_output
2259+
2260+
2261+def hugepage_support(user, group='hugetlb', nr_hugepages=256,
2262+ max_map_count=65536, mnt_point='/run/hugepages/kvm',
2263+ pagesize='2MB', mount=True, set_shmmax=False):
2264+ """Enable hugepages on system.
2265+
2266+ Args:
2267+ user (str) -- Username to allow access to hugepages to
2268+ group (str) -- Group name to own hugepages
2269+ nr_hugepages (int) -- Number of pages to reserve
2270+ max_map_count (int) -- Number of Virtual Memory Areas a process can own
2271+ mnt_point (str) -- Directory to mount hugepages on
2272+ pagesize (str) -- Size of hugepages
2273+ mount (bool) -- Whether to Mount hugepages
2274+ """
2275+ group_info = add_group(group)
2276+ gid = group_info.gr_gid
2277+ add_user_to_group(user, group)
2278+ if max_map_count < 2 * nr_hugepages:
2279+ max_map_count = 2 * nr_hugepages
2280+ sysctl_settings = {
2281+ 'vm.nr_hugepages': nr_hugepages,
2282+ 'vm.max_map_count': max_map_count,
2283+ 'vm.hugetlb_shm_group': gid,
2284+ }
2285+ if set_shmmax:
2286+ shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax']))
2287+ shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages
2288+ if shmmax_minsize > shmmax_current:
2289+ sysctl_settings['kernel.shmmax'] = shmmax_minsize
2290+ sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf')
2291+ mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False)
2292+ lfstab = fstab.Fstab()
2293+ fstab_entry = lfstab.get_entry_by_attr('mountpoint', mnt_point)
2294+ if fstab_entry:
2295+ lfstab.remove_entry(fstab_entry)
2296+ entry = lfstab.Entry('nodev', mnt_point, 'hugetlbfs',
2297+ 'mode=1770,gid={},pagesize={}'.format(gid, pagesize), 0, 0)
2298+ lfstab.add_entry(entry)
2299+ if mount:
2300+ fstab_mount(mnt_point)
2301
2302=== added file 'hooks/charmhelpers/core/kernel.py'
2303--- hooks/charmhelpers/core/kernel.py 1970-01-01 00:00:00 +0000
2304+++ hooks/charmhelpers/core/kernel.py 2017-07-25 09:17:37 +0000
2305@@ -0,0 +1,72 @@
2306+#!/usr/bin/env python
2307+# -*- coding: utf-8 -*-
2308+
2309+# Copyright 2014-2015 Canonical Limited.
2310+#
2311+# Licensed under the Apache License, Version 2.0 (the "License");
2312+# you may not use this file except in compliance with the License.
2313+# You may obtain a copy of the License at
2314+#
2315+# http://www.apache.org/licenses/LICENSE-2.0
2316+#
2317+# Unless required by applicable law or agreed to in writing, software
2318+# distributed under the License is distributed on an "AS IS" BASIS,
2319+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2320+# See the License for the specific language governing permissions and
2321+# limitations under the License.
2322+
2323+import re
2324+import subprocess
2325+
2326+from charmhelpers.osplatform import get_platform
2327+from charmhelpers.core.hookenv import (
2328+ log,
2329+ INFO
2330+)
2331+
2332+__platform__ = get_platform()
2333+if __platform__ == "ubuntu":
2334+ from charmhelpers.core.kernel_factory.ubuntu import (
2335+ persistent_modprobe,
2336+ update_initramfs,
2337+ ) # flake8: noqa -- ignore F401 for this import
2338+elif __platform__ == "centos":
2339+ from charmhelpers.core.kernel_factory.centos import (
2340+ persistent_modprobe,
2341+ update_initramfs,
2342+ ) # flake8: noqa -- ignore F401 for this import
2343+
2344+__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
2345+
2346+
2347+def modprobe(module, persist=True):
2348+ """Load a kernel module and configure for auto-load on reboot."""
2349+ cmd = ['modprobe', module]
2350+
2351+ log('Loading kernel module %s' % module, level=INFO)
2352+
2353+ subprocess.check_call(cmd)
2354+ if persist:
2355+ persistent_modprobe(module)
2356+
2357+
2358+def rmmod(module, force=False):
2359+ """Remove a module from the linux kernel"""
2360+ cmd = ['rmmod']
2361+ if force:
2362+ cmd.append('-f')
2363+ cmd.append(module)
2364+ log('Removing kernel module %s' % module, level=INFO)
2365+ return subprocess.check_call(cmd)
2366+
2367+
2368+def lsmod():
2369+ """Shows what kernel modules are currently loaded"""
2370+ return subprocess.check_output(['lsmod'],
2371+ universal_newlines=True)
2372+
2373+
2374+def is_module_loaded(module):
2375+ """Checks if a kernel module is already loaded"""
2376+ matches = re.findall('^%s[ ]+' % module, lsmod(), re.M)
2377+ return len(matches) > 0
2378
2379=== added directory 'hooks/charmhelpers/core/kernel_factory'
2380=== added file 'hooks/charmhelpers/core/kernel_factory/__init__.py'
2381=== added file 'hooks/charmhelpers/core/kernel_factory/centos.py'
2382--- hooks/charmhelpers/core/kernel_factory/centos.py 1970-01-01 00:00:00 +0000
2383+++ hooks/charmhelpers/core/kernel_factory/centos.py 2017-07-25 09:17:37 +0000
2384@@ -0,0 +1,17 @@
2385+import subprocess
2386+import os
2387+
2388+
2389+def persistent_modprobe(module):
2390+ """Load a kernel module and configure for auto-load on reboot."""
2391+ if not os.path.exists('/etc/rc.modules'):
2392+ open('/etc/rc.modules', 'a')
2393+ os.chmod('/etc/rc.modules', 111)
2394+ with open('/etc/rc.modules', 'r+') as modules:
2395+ if module not in modules.read():
2396+ modules.write('modprobe %s\n' % module)
2397+
2398+
2399+def update_initramfs(version='all'):
2400+ """Updates an initramfs image."""
2401+ return subprocess.check_call(["dracut", "-f", version])
2402
2403=== added file 'hooks/charmhelpers/core/kernel_factory/ubuntu.py'
2404--- hooks/charmhelpers/core/kernel_factory/ubuntu.py 1970-01-01 00:00:00 +0000
2405+++ hooks/charmhelpers/core/kernel_factory/ubuntu.py 2017-07-25 09:17:37 +0000
2406@@ -0,0 +1,13 @@
2407+import subprocess
2408+
2409+
2410+def persistent_modprobe(module):
2411+ """Load a kernel module and configure for auto-load on reboot."""
2412+ with open('/etc/modules', 'r+') as modules:
2413+ if module not in modules.read():
2414+ modules.write(module + "\n")
2415+
2416+
2417+def update_initramfs(version='all'):
2418+ """Updates an initramfs image."""
2419+ return subprocess.check_call(["update-initramfs", "-k", version, "-u"])
2420
2421=== modified file 'hooks/charmhelpers/core/services/__init__.py'
2422--- hooks/charmhelpers/core/services/__init__.py 2015-04-09 06:46:35 +0000
2423+++ hooks/charmhelpers/core/services/__init__.py 2017-07-25 09:17:37 +0000
2424@@ -1,18 +1,16 @@
2425 # Copyright 2014-2015 Canonical Limited.
2426 #
2427-# This file is part of charm-helpers.
2428-#
2429-# charm-helpers is free software: you can redistribute it and/or modify
2430-# it under the terms of the GNU Lesser General Public License version 3 as
2431-# published by the Free Software Foundation.
2432-#
2433-# charm-helpers is distributed in the hope that it will be useful,
2434-# but WITHOUT ANY WARRANTY; without even the implied warranty of
2435-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2436-# GNU Lesser General Public License for more details.
2437-#
2438-# You should have received a copy of the GNU Lesser General Public License
2439-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2440+# Licensed under the Apache License, Version 2.0 (the "License");
2441+# you may not use this file except in compliance with the License.
2442+# You may obtain a copy of the License at
2443+#
2444+# http://www.apache.org/licenses/LICENSE-2.0
2445+#
2446+# Unless required by applicable law or agreed to in writing, software
2447+# distributed under the License is distributed on an "AS IS" BASIS,
2448+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2449+# See the License for the specific language governing permissions and
2450+# limitations under the License.
2451
2452 from .base import * # NOQA
2453 from .helpers import * # NOQA
2454
2455=== modified file 'hooks/charmhelpers/core/services/base.py'
2456--- hooks/charmhelpers/core/services/base.py 2015-04-09 06:46:35 +0000
2457+++ hooks/charmhelpers/core/services/base.py 2017-07-25 09:17:37 +0000
2458@@ -1,23 +1,21 @@
2459 # Copyright 2014-2015 Canonical Limited.
2460 #
2461-# This file is part of charm-helpers.
2462-#
2463-# charm-helpers is free software: you can redistribute it and/or modify
2464-# it under the terms of the GNU Lesser General Public License version 3 as
2465-# published by the Free Software Foundation.
2466-#
2467-# charm-helpers is distributed in the hope that it will be useful,
2468-# but WITHOUT ANY WARRANTY; without even the implied warranty of
2469-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2470-# GNU Lesser General Public License for more details.
2471-#
2472-# You should have received a copy of the GNU Lesser General Public License
2473-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2474+# Licensed under the Apache License, Version 2.0 (the "License");
2475+# you may not use this file except in compliance with the License.
2476+# You may obtain a copy of the License at
2477+#
2478+# http://www.apache.org/licenses/LICENSE-2.0
2479+#
2480+# Unless required by applicable law or agreed to in writing, software
2481+# distributed under the License is distributed on an "AS IS" BASIS,
2482+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2483+# See the License for the specific language governing permissions and
2484+# limitations under the License.
2485
2486 import os
2487-import re
2488 import json
2489-from collections import Iterable
2490+from inspect import getargspec
2491+from collections import Iterable, OrderedDict
2492
2493 from charmhelpers.core import host
2494 from charmhelpers.core import hookenv
2495@@ -119,7 +117,7 @@
2496 """
2497 self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json')
2498 self._ready = None
2499- self.services = {}
2500+ self.services = OrderedDict()
2501 for service in services or []:
2502 service_name = service['service']
2503 self.services[service_name] = service
2504@@ -128,15 +126,18 @@
2505 """
2506 Handle the current hook by doing The Right Thing with the registered services.
2507 """
2508- hook_name = hookenv.hook_name()
2509- if hook_name == 'stop':
2510- self.stop_services()
2511- else:
2512- self.provide_data()
2513- self.reconfigure_services()
2514- cfg = hookenv.config()
2515- if cfg.implicit_save:
2516- cfg.save()
2517+ hookenv._run_atstart()
2518+ try:
2519+ hook_name = hookenv.hook_name()
2520+ if hook_name == 'stop':
2521+ self.stop_services()
2522+ else:
2523+ self.reconfigure_services()
2524+ self.provide_data()
2525+ except SystemExit as x:
2526+ if x.code is None or x.code == 0:
2527+ hookenv._run_atexit()
2528+ hookenv._run_atexit()
2529
2530 def provide_data(self):
2531 """
2532@@ -145,15 +146,36 @@
2533 A provider must have a `name` attribute, which indicates which relation
2534 to set data on, and a `provide_data()` method, which returns a dict of
2535 data to set.
2536+
2537+ The `provide_data()` method can optionally accept two parameters:
2538+
2539+ * ``remote_service`` The name of the remote service that the data will
2540+ be provided to. The `provide_data()` method will be called once
2541+ for each connected service (not unit). This allows the method to
2542+ tailor its data to the given service.
2543+ * ``service_ready`` Whether or not the service definition had all of
2544+ its requirements met, and thus the ``data_ready`` callbacks run.
2545+
2546+ Note that the ``provided_data`` methods are now called **after** the
2547+ ``data_ready`` callbacks are run. This gives the ``data_ready`` callbacks
2548+ a chance to generate any data necessary for the providing to the remote
2549+ services.
2550 """
2551- hook_name = hookenv.hook_name()
2552- for service in self.services.values():
2553+ for service_name, service in self.services.items():
2554+ service_ready = self.is_ready(service_name)
2555 for provider in service.get('provided_data', []):
2556- if re.match(r'{}-relation-(joined|changed)'.format(provider.name), hook_name):
2557- data = provider.provide_data()
2558- _ready = provider._is_ready(data) if hasattr(provider, '_is_ready') else data
2559- if _ready:
2560- hookenv.relation_set(None, data)
2561+ for relid in hookenv.relation_ids(provider.name):
2562+ units = hookenv.related_units(relid)
2563+ if not units:
2564+ continue
2565+ remote_service = units[0].split('/')[0]
2566+ argspec = getargspec(provider.provide_data)
2567+ if len(argspec.args) > 1:
2568+ data = provider.provide_data(remote_service, service_ready)
2569+ else:
2570+ data = provider.provide_data()
2571+ if data:
2572+ hookenv.relation_set(relid, data)
2573
2574 def reconfigure_services(self, *service_names):
2575 """
2576
2577=== modified file 'hooks/charmhelpers/core/services/helpers.py'
2578--- hooks/charmhelpers/core/services/helpers.py 2015-04-09 06:46:35 +0000
2579+++ hooks/charmhelpers/core/services/helpers.py 2017-07-25 09:17:37 +0000
2580@@ -1,22 +1,22 @@
2581 # Copyright 2014-2015 Canonical Limited.
2582 #
2583-# This file is part of charm-helpers.
2584-#
2585-# charm-helpers is free software: you can redistribute it and/or modify
2586-# it under the terms of the GNU Lesser General Public License version 3 as
2587-# published by the Free Software Foundation.
2588-#
2589-# charm-helpers is distributed in the hope that it will be useful,
2590-# but WITHOUT ANY WARRANTY; without even the implied warranty of
2591-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2592-# GNU Lesser General Public License for more details.
2593-#
2594-# You should have received a copy of the GNU Lesser General Public License
2595-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2596+# Licensed under the Apache License, Version 2.0 (the "License");
2597+# you may not use this file except in compliance with the License.
2598+# You may obtain a copy of the License at
2599+#
2600+# http://www.apache.org/licenses/LICENSE-2.0
2601+#
2602+# Unless required by applicable law or agreed to in writing, software
2603+# distributed under the License is distributed on an "AS IS" BASIS,
2604+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2605+# See the License for the specific language governing permissions and
2606+# limitations under the License.
2607
2608 import os
2609 import yaml
2610+
2611 from charmhelpers.core import hookenv
2612+from charmhelpers.core import host
2613 from charmhelpers.core import templating
2614
2615 from charmhelpers.core.services.base import ManagerCallback
2616@@ -239,28 +239,51 @@
2617 action.
2618
2619 :param str source: The template source file, relative to
2620- `$CHARM_DIR/templates`
2621+ `$CHARM_DIR/templates`
2622
2623- :param str target: The target to write the rendered template to
2624+ :param str target: The target to write the rendered template to (or None)
2625 :param str owner: The owner of the rendered file
2626 :param str group: The group of the rendered file
2627 :param int perms: The permissions of the rendered file
2628+ :param partial on_change_action: functools partial to be executed when
2629+ rendered file changes
2630+ :param jinja2 loader template_loader: A jinja2 template loader
2631+
2632+ :return str: The rendered template
2633 """
2634 def __init__(self, source, target,
2635- owner='root', group='root', perms=0o444):
2636+ owner='root', group='root', perms=0o444,
2637+ on_change_action=None, template_loader=None):
2638 self.source = source
2639 self.target = target
2640 self.owner = owner
2641 self.group = group
2642 self.perms = perms
2643+ self.on_change_action = on_change_action
2644+ self.template_loader = template_loader
2645
2646 def __call__(self, manager, service_name, event_name):
2647+ pre_checksum = ''
2648+ if self.on_change_action and os.path.isfile(self.target):
2649+ pre_checksum = host.file_hash(self.target)
2650 service = manager.get_service(service_name)
2651- context = {}
2652+ context = {'ctx': {}}
2653 for ctx in service.get('required_data', []):
2654 context.update(ctx)
2655- templating.render(self.source, self.target, context,
2656- self.owner, self.group, self.perms)
2657+ context['ctx'].update(ctx)
2658+
2659+ result = templating.render(self.source, self.target, context,
2660+ self.owner, self.group, self.perms,
2661+ template_loader=self.template_loader)
2662+ if self.on_change_action:
2663+ if pre_checksum == host.file_hash(self.target):
2664+ hookenv.log(
2665+ 'No change detected: {}'.format(self.target),
2666+ hookenv.DEBUG)
2667+ else:
2668+ self.on_change_action()
2669+
2670+ return result
2671
2672
2673 # Convenience aliases for templates
2674
2675=== modified file 'hooks/charmhelpers/core/strutils.py'
2676--- hooks/charmhelpers/core/strutils.py 2015-04-09 06:46:35 +0000
2677+++ hooks/charmhelpers/core/strutils.py 2017-07-25 09:17:37 +0000
2678@@ -3,21 +3,20 @@
2679
2680 # Copyright 2014-2015 Canonical Limited.
2681 #
2682-# This file is part of charm-helpers.
2683-#
2684-# charm-helpers is free software: you can redistribute it and/or modify
2685-# it under the terms of the GNU Lesser General Public License version 3 as
2686-# published by the Free Software Foundation.
2687-#
2688-# charm-helpers is distributed in the hope that it will be useful,
2689-# but WITHOUT ANY WARRANTY; without even the implied warranty of
2690-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2691-# GNU Lesser General Public License for more details.
2692-#
2693-# You should have received a copy of the GNU Lesser General Public License
2694-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2695+# Licensed under the Apache License, Version 2.0 (the "License");
2696+# you may not use this file except in compliance with the License.
2697+# You may obtain a copy of the License at
2698+#
2699+# http://www.apache.org/licenses/LICENSE-2.0
2700+#
2701+# Unless required by applicable law or agreed to in writing, software
2702+# distributed under the License is distributed on an "AS IS" BASIS,
2703+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2704+# See the License for the specific language governing permissions and
2705+# limitations under the License.
2706
2707 import six
2708+import re
2709
2710
2711 def bool_from_string(value):
2712@@ -33,10 +32,92 @@
2713
2714 value = value.strip().lower()
2715
2716- if value in ['y', 'yes', 'true', 't']:
2717+ if value in ['y', 'yes', 'true', 't', 'on']:
2718 return True
2719- elif value in ['n', 'no', 'false', 'f']:
2720+ elif value in ['n', 'no', 'false', 'f', 'off']:
2721 return False
2722
2723 msg = "Unable to interpret string value '%s' as boolean" % (value)
2724 raise ValueError(msg)
2725+
2726+
2727+def bytes_from_string(value):
2728+ """Interpret human readable string value as bytes.
2729+
2730+ Returns int
2731+ """
2732+ BYTE_POWER = {
2733+ 'K': 1,
2734+ 'KB': 1,
2735+ 'M': 2,
2736+ 'MB': 2,
2737+ 'G': 3,
2738+ 'GB': 3,
2739+ 'T': 4,
2740+ 'TB': 4,
2741+ 'P': 5,
2742+ 'PB': 5,
2743+ }
2744+ if isinstance(value, six.string_types):
2745+ value = six.text_type(value)
2746+ else:
2747+ msg = "Unable to interpret non-string value '%s' as boolean" % (value)
2748+ raise ValueError(msg)
2749+ matches = re.match("([0-9]+)([a-zA-Z]+)", value)
2750+ if not matches:
2751+ msg = "Unable to interpret string value '%s' as bytes" % (value)
2752+ raise ValueError(msg)
2753+ return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
2754+
2755+
2756+class BasicStringComparator(object):
2757+ """Provides a class that will compare strings from an iterator type object.
2758+ Used to provide > and < comparisons on strings that may not necessarily be
2759+ alphanumerically ordered. e.g. OpenStack or Ubuntu releases AFTER the
2760+ z-wrap.
2761+ """
2762+
2763+ _list = None
2764+
2765+ def __init__(self, item):
2766+ if self._list is None:
2767+ raise Exception("Must define the _list in the class definition!")
2768+ try:
2769+ self.index = self._list.index(item)
2770+ except Exception:
2771+ raise KeyError("Item '{}' is not in list '{}'"
2772+ .format(item, self._list))
2773+
2774+ def __eq__(self, other):
2775+ assert isinstance(other, str) or isinstance(other, self.__class__)
2776+ return self.index == self._list.index(other)
2777+
2778+ def __ne__(self, other):
2779+ return not self.__eq__(other)
2780+
2781+ def __lt__(self, other):
2782+ assert isinstance(other, str) or isinstance(other, self.__class__)
2783+ return self.index < self._list.index(other)
2784+
2785+ def __ge__(self, other):
2786+ return not self.__lt__(other)
2787+
2788+ def __gt__(self, other):
2789+ assert isinstance(other, str) or isinstance(other, self.__class__)
2790+ return self.index > self._list.index(other)
2791+
2792+ def __le__(self, other):
2793+ return not self.__gt__(other)
2794+
2795+ def __str__(self):
2796+ """Always give back the item at the index so it can be used in
2797+ comparisons like:
2798+
2799+ s_mitaka = CompareOpenStack('mitaka')
2800+ s_newton = CompareOpenstack('newton')
2801+
2802+ assert s_newton > s_mitaka
2803+
2804+ @returns: <string>
2805+ """
2806+ return self._list[self.index]
2807
2808=== modified file 'hooks/charmhelpers/core/sysctl.py'
2809--- hooks/charmhelpers/core/sysctl.py 2015-04-09 06:46:35 +0000
2810+++ hooks/charmhelpers/core/sysctl.py 2017-07-25 09:17:37 +0000
2811@@ -3,19 +3,17 @@
2812
2813 # Copyright 2014-2015 Canonical Limited.
2814 #
2815-# This file is part of charm-helpers.
2816-#
2817-# charm-helpers is free software: you can redistribute it and/or modify
2818-# it under the terms of the GNU Lesser General Public License version 3 as
2819-# published by the Free Software Foundation.
2820-#
2821-# charm-helpers is distributed in the hope that it will be useful,
2822-# but WITHOUT ANY WARRANTY; without even the implied warranty of
2823-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2824-# GNU Lesser General Public License for more details.
2825-#
2826-# You should have received a copy of the GNU Lesser General Public License
2827-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2828+# Licensed under the Apache License, Version 2.0 (the "License");
2829+# you may not use this file except in compliance with the License.
2830+# You may obtain a copy of the License at
2831+#
2832+# http://www.apache.org/licenses/LICENSE-2.0
2833+#
2834+# Unless required by applicable law or agreed to in writing, software
2835+# distributed under the License is distributed on an "AS IS" BASIS,
2836+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2837+# See the License for the specific language governing permissions and
2838+# limitations under the License.
2839
2840 import yaml
2841
2842
2843=== modified file 'hooks/charmhelpers/core/templating.py'
2844--- hooks/charmhelpers/core/templating.py 2015-04-09 06:46:35 +0000
2845+++ hooks/charmhelpers/core/templating.py 2017-07-25 09:17:37 +0000
2846@@ -1,33 +1,33 @@
2847 # Copyright 2014-2015 Canonical Limited.
2848 #
2849-# This file is part of charm-helpers.
2850-#
2851-# charm-helpers is free software: you can redistribute it and/or modify
2852-# it under the terms of the GNU Lesser General Public License version 3 as
2853-# published by the Free Software Foundation.
2854-#
2855-# charm-helpers is distributed in the hope that it will be useful,
2856-# but WITHOUT ANY WARRANTY; without even the implied warranty of
2857-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2858-# GNU Lesser General Public License for more details.
2859-#
2860-# You should have received a copy of the GNU Lesser General Public License
2861-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2862+# Licensed under the Apache License, Version 2.0 (the "License");
2863+# you may not use this file except in compliance with the License.
2864+# You may obtain a copy of the License at
2865+#
2866+# http://www.apache.org/licenses/LICENSE-2.0
2867+#
2868+# Unless required by applicable law or agreed to in writing, software
2869+# distributed under the License is distributed on an "AS IS" BASIS,
2870+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2871+# See the License for the specific language governing permissions and
2872+# limitations under the License.
2873
2874 import os
2875+import sys
2876
2877 from charmhelpers.core import host
2878 from charmhelpers.core import hookenv
2879
2880
2881 def render(source, target, context, owner='root', group='root',
2882- perms=0o444, templates_dir=None, encoding='UTF-8'):
2883+ perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None):
2884 """
2885 Render a template.
2886
2887 The `source` path, if not absolute, is relative to the `templates_dir`.
2888
2889- The `target` path should be absolute.
2890+ The `target` path should be absolute. It can also be `None`, in which
2891+ case no file will be written.
2892
2893 The context should be a dict containing the values to be replaced in the
2894 template.
2895@@ -36,8 +36,12 @@
2896
2897 If omitted, `templates_dir` defaults to the `templates` folder in the charm.
2898
2899- Note: Using this requires python-jinja2; if it is not installed, calling
2900- this will attempt to use charmhelpers.fetch.apt_install to install it.
2901+ The rendered template will be written to the file as well as being returned
2902+ as a string.
2903+
2904+ Note: Using this requires python-jinja2 or python3-jinja2; if it is not
2905+ installed, calling this will attempt to use charmhelpers.fetch.apt_install
2906+ to install it.
2907 """
2908 try:
2909 from jinja2 import FileSystemLoader, Environment, exceptions
2910@@ -49,20 +53,32 @@
2911 'charmhelpers.fetch to install it',
2912 level=hookenv.ERROR)
2913 raise
2914- apt_install('python-jinja2', fatal=True)
2915+ if sys.version_info.major == 2:
2916+ apt_install('python-jinja2', fatal=True)
2917+ else:
2918+ apt_install('python3-jinja2', fatal=True)
2919 from jinja2 import FileSystemLoader, Environment, exceptions
2920
2921- if templates_dir is None:
2922- templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
2923- loader = Environment(loader=FileSystemLoader(templates_dir))
2924+ if template_loader:
2925+ template_env = Environment(loader=template_loader)
2926+ else:
2927+ if templates_dir is None:
2928+ templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
2929+ template_env = Environment(loader=FileSystemLoader(templates_dir))
2930 try:
2931 source = source
2932- template = loader.get_template(source)
2933+ template = template_env.get_template(source)
2934 except exceptions.TemplateNotFound as e:
2935 hookenv.log('Could not load template %s from %s.' %
2936 (source, templates_dir),
2937 level=hookenv.ERROR)
2938 raise e
2939 content = template.render(context)
2940- host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
2941- host.write_file(target, content.encode(encoding), owner, group, perms)
2942+ if target is not None:
2943+ target_dir = os.path.dirname(target)
2944+ if not os.path.exists(target_dir):
2945+ # This is a terrible default directory permission, as the file
2946+ # or its siblings will often contain secrets.
2947+ host.mkdir(os.path.dirname(target), owner, group, perms=0o755)
2948+ host.write_file(target, content.encode(encoding), owner, group, perms)
2949+ return content
2950
2951=== modified file 'hooks/charmhelpers/core/unitdata.py'
2952--- hooks/charmhelpers/core/unitdata.py 2015-04-09 06:46:35 +0000
2953+++ hooks/charmhelpers/core/unitdata.py 2017-07-25 09:17:37 +0000
2954@@ -3,20 +3,17 @@
2955 #
2956 # Copyright 2014-2015 Canonical Limited.
2957 #
2958-# This file is part of charm-helpers.
2959-#
2960-# charm-helpers is free software: you can redistribute it and/or modify
2961-# it under the terms of the GNU Lesser General Public License version 3 as
2962-# published by the Free Software Foundation.
2963-#
2964-# charm-helpers is distributed in the hope that it will be useful,
2965-# but WITHOUT ANY WARRANTY; without even the implied warranty of
2966-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2967-# GNU Lesser General Public License for more details.
2968-#
2969-# You should have received a copy of the GNU Lesser General Public License
2970-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2971-#
2972+# Licensed under the Apache License, Version 2.0 (the "License");
2973+# you may not use this file except in compliance with the License.
2974+# You may obtain a copy of the License at
2975+#
2976+# http://www.apache.org/licenses/LICENSE-2.0
2977+#
2978+# Unless required by applicable law or agreed to in writing, software
2979+# distributed under the License is distributed on an "AS IS" BASIS,
2980+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2981+# See the License for the specific language governing permissions and
2982+# limitations under the License.
2983 #
2984 # Authors:
2985 # Kapil Thangavelu <kapil.foss@gmail.com>
2986@@ -152,6 +149,7 @@
2987 import collections
2988 import contextlib
2989 import datetime
2990+import itertools
2991 import json
2992 import os
2993 import pprint
2994@@ -164,8 +162,7 @@
2995 class Storage(object):
2996 """Simple key value database for local unit state within charms.
2997
2998- Modifications are automatically committed at hook exit. That's
2999- currently regardless of exit code.
3000+ Modifications are not persisted unless :meth:`flush` is called.
3001
3002 To support dicts, lists, integer, floats, and booleans values
3003 are automatically json encoded/decoded.
3004@@ -173,8 +170,11 @@
3005 def __init__(self, path=None):
3006 self.db_path = path
3007 if path is None:
3008- self.db_path = os.path.join(
3009- os.environ.get('CHARM_DIR', ''), '.unit-state.db')
3010+ if 'UNIT_STATE_DB' in os.environ:
3011+ self.db_path = os.environ['UNIT_STATE_DB']
3012+ else:
3013+ self.db_path = os.path.join(
3014+ os.environ.get('CHARM_DIR', ''), '.unit-state.db')
3015 self.conn = sqlite3.connect('%s' % self.db_path)
3016 self.cursor = self.conn.cursor()
3017 self.revision = None
3018@@ -189,15 +189,8 @@
3019 self.conn.close()
3020 self._closed = True
3021
3022- def _scoped_query(self, stmt, params=None):
3023- if params is None:
3024- params = []
3025- return stmt, params
3026-
3027 def get(self, key, default=None, record=False):
3028- self.cursor.execute(
3029- *self._scoped_query(
3030- 'select data from kv where key=?', [key]))
3031+ self.cursor.execute('select data from kv where key=?', [key])
3032 result = self.cursor.fetchone()
3033 if not result:
3034 return default
3035@@ -206,33 +199,81 @@
3036 return json.loads(result[0])
3037
3038 def getrange(self, key_prefix, strip=False):
3039- stmt = "select key, data from kv where key like '%s%%'" % key_prefix
3040- self.cursor.execute(*self._scoped_query(stmt))
3041+ """
3042+ Get a range of keys starting with a common prefix as a mapping of
3043+ keys to values.
3044+
3045+ :param str key_prefix: Common prefix among all keys
3046+ :param bool strip: Optionally strip the common prefix from the key
3047+ names in the returned dict
3048+ :return dict: A (possibly empty) dict of key-value mappings
3049+ """
3050+ self.cursor.execute("select key, data from kv where key like ?",
3051+ ['%s%%' % key_prefix])
3052 result = self.cursor.fetchall()
3053
3054 if not result:
3055- return None
3056+ return {}
3057 if not strip:
3058 key_prefix = ''
3059 return dict([
3060 (k[len(key_prefix):], json.loads(v)) for k, v in result])
3061
3062 def update(self, mapping, prefix=""):
3063+ """
3064+ Set the values of multiple keys at once.
3065+
3066+ :param dict mapping: Mapping of keys to values
3067+ :param str prefix: Optional prefix to apply to all keys in `mapping`
3068+ before setting
3069+ """
3070 for k, v in mapping.items():
3071 self.set("%s%s" % (prefix, k), v)
3072
3073 def unset(self, key):
3074+ """
3075+ Remove a key from the database entirely.
3076+ """
3077 self.cursor.execute('delete from kv where key=?', [key])
3078 if self.revision and self.cursor.rowcount:
3079 self.cursor.execute(
3080 'insert into kv_revisions values (?, ?, ?)',
3081 [key, self.revision, json.dumps('DELETED')])
3082
3083+ def unsetrange(self, keys=None, prefix=""):
3084+ """
3085+ Remove a range of keys starting with a common prefix, from the database
3086+ entirely.
3087+
3088+ :param list keys: List of keys to remove.
3089+ :param str prefix: Optional prefix to apply to all keys in ``keys``
3090+ before removing.
3091+ """
3092+ if keys is not None:
3093+ keys = ['%s%s' % (prefix, key) for key in keys]
3094+ self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys)
3095+ if self.revision and self.cursor.rowcount:
3096+ self.cursor.execute(
3097+ 'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)),
3098+ list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys)))
3099+ else:
3100+ self.cursor.execute('delete from kv where key like ?',
3101+ ['%s%%' % prefix])
3102+ if self.revision and self.cursor.rowcount:
3103+ self.cursor.execute(
3104+ 'insert into kv_revisions values (?, ?, ?)',
3105+ ['%s%%' % prefix, self.revision, json.dumps('DELETED')])
3106+
3107 def set(self, key, value):
3108+ """
3109+ Set a value in the database.
3110+
3111+ :param str key: Key to set the value for
3112+ :param value: Any JSON-serializable value to be set
3113+ """
3114 serialized = json.dumps(value)
3115
3116- self.cursor.execute(
3117- 'select data from kv where key=?', [key])
3118+ self.cursor.execute('select data from kv where key=?', [key])
3119 exists = self.cursor.fetchone()
3120
3121 # Skip mutations to the same value
3122
3123=== modified file 'hooks/charmhelpers/fetch/__init__.py'
3124--- hooks/charmhelpers/fetch/__init__.py 2015-04-09 06:46:35 +0000
3125+++ hooks/charmhelpers/fetch/__init__.py 2017-07-25 09:17:37 +0000
3126@@ -1,32 +1,24 @@
3127 # Copyright 2014-2015 Canonical Limited.
3128 #
3129-# This file is part of charm-helpers.
3130-#
3131-# charm-helpers is free software: you can redistribute it and/or modify
3132-# it under the terms of the GNU Lesser General Public License version 3 as
3133-# published by the Free Software Foundation.
3134-#
3135-# charm-helpers is distributed in the hope that it will be useful,
3136-# but WITHOUT ANY WARRANTY; without even the implied warranty of
3137-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3138-# GNU Lesser General Public License for more details.
3139-#
3140-# You should have received a copy of the GNU Lesser General Public License
3141-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3142+# Licensed under the Apache License, Version 2.0 (the "License");
3143+# you may not use this file except in compliance with the License.
3144+# You may obtain a copy of the License at
3145+#
3146+# http://www.apache.org/licenses/LICENSE-2.0
3147+#
3148+# Unless required by applicable law or agreed to in writing, software
3149+# distributed under the License is distributed on an "AS IS" BASIS,
3150+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3151+# See the License for the specific language governing permissions and
3152+# limitations under the License.
3153
3154 import importlib
3155-from tempfile import NamedTemporaryFile
3156-import time
3157+from charmhelpers.osplatform import get_platform
3158 from yaml import safe_load
3159-from charmhelpers.core.host import (
3160- lsb_release
3161-)
3162-import subprocess
3163 from charmhelpers.core.hookenv import (
3164 config,
3165 log,
3166 )
3167-import os
3168
3169 import six
3170 if six.PY3:
3171@@ -35,63 +27,6 @@
3172 from urlparse import urlparse, urlunparse
3173
3174
3175-CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
3176-deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
3177-"""
3178-PROPOSED_POCKET = """# Proposed
3179-deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
3180-"""
3181-CLOUD_ARCHIVE_POCKETS = {
3182- # Folsom
3183- 'folsom': 'precise-updates/folsom',
3184- 'precise-folsom': 'precise-updates/folsom',
3185- 'precise-folsom/updates': 'precise-updates/folsom',
3186- 'precise-updates/folsom': 'precise-updates/folsom',
3187- 'folsom/proposed': 'precise-proposed/folsom',
3188- 'precise-folsom/proposed': 'precise-proposed/folsom',
3189- 'precise-proposed/folsom': 'precise-proposed/folsom',
3190- # Grizzly
3191- 'grizzly': 'precise-updates/grizzly',
3192- 'precise-grizzly': 'precise-updates/grizzly',
3193- 'precise-grizzly/updates': 'precise-updates/grizzly',
3194- 'precise-updates/grizzly': 'precise-updates/grizzly',
3195- 'grizzly/proposed': 'precise-proposed/grizzly',
3196- 'precise-grizzly/proposed': 'precise-proposed/grizzly',
3197- 'precise-proposed/grizzly': 'precise-proposed/grizzly',
3198- # Havana
3199- 'havana': 'precise-updates/havana',
3200- 'precise-havana': 'precise-updates/havana',
3201- 'precise-havana/updates': 'precise-updates/havana',
3202- 'precise-updates/havana': 'precise-updates/havana',
3203- 'havana/proposed': 'precise-proposed/havana',
3204- 'precise-havana/proposed': 'precise-proposed/havana',
3205- 'precise-proposed/havana': 'precise-proposed/havana',
3206- # Icehouse
3207- 'icehouse': 'precise-updates/icehouse',
3208- 'precise-icehouse': 'precise-updates/icehouse',
3209- 'precise-icehouse/updates': 'precise-updates/icehouse',
3210- 'precise-updates/icehouse': 'precise-updates/icehouse',
3211- 'icehouse/proposed': 'precise-proposed/icehouse',
3212- 'precise-icehouse/proposed': 'precise-proposed/icehouse',
3213- 'precise-proposed/icehouse': 'precise-proposed/icehouse',
3214- # Juno
3215- 'juno': 'trusty-updates/juno',
3216- 'trusty-juno': 'trusty-updates/juno',
3217- 'trusty-juno/updates': 'trusty-updates/juno',
3218- 'trusty-updates/juno': 'trusty-updates/juno',
3219- 'juno/proposed': 'trusty-proposed/juno',
3220- 'trusty-juno/proposed': 'trusty-proposed/juno',
3221- 'trusty-proposed/juno': 'trusty-proposed/juno',
3222- # Kilo
3223- 'kilo': 'trusty-updates/kilo',
3224- 'trusty-kilo': 'trusty-updates/kilo',
3225- 'trusty-kilo/updates': 'trusty-updates/kilo',
3226- 'trusty-updates/kilo': 'trusty-updates/kilo',
3227- 'kilo/proposed': 'trusty-proposed/kilo',
3228- 'trusty-kilo/proposed': 'trusty-proposed/kilo',
3229- 'trusty-proposed/kilo': 'trusty-proposed/kilo',
3230-}
3231-
3232 # The order of this list is very important. Handlers should be listed in from
3233 # least- to most-specific URL matching.
3234 FETCH_HANDLERS = (
3235@@ -100,10 +35,6 @@
3236 'charmhelpers.fetch.giturl.GitUrlFetchHandler',
3237 )
3238
3239-APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
3240-APT_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
3241-APT_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
3242-
3243
3244 class SourceConfigError(Exception):
3245 pass
3246@@ -117,6 +48,13 @@
3247 pass
3248
3249
3250+class GPGKeyError(Exception):
3251+ """Exception occurs when a GPG key cannot be fetched or used. The message
3252+ indicates what the problem is.
3253+ """
3254+ pass
3255+
3256+
3257 class BaseFetchHandler(object):
3258
3259 """Base class for FetchHandler implementations in fetch plugins"""
3260@@ -141,172 +79,39 @@
3261 return urlunparse(parts)
3262
3263
3264-def filter_installed_packages(packages):
3265- """Returns a list of packages that require installation"""
3266- cache = apt_cache()
3267- _pkgs = []
3268- for package in packages:
3269- try:
3270- p = cache[package]
3271- p.current_ver or _pkgs.append(package)
3272- except KeyError:
3273- log('Package {} has no installation candidate.'.format(package),
3274- level='WARNING')
3275- _pkgs.append(package)
3276- return _pkgs
3277-
3278-
3279-def apt_cache(in_memory=True):
3280- """Build and return an apt cache"""
3281- import apt_pkg
3282- apt_pkg.init()
3283- if in_memory:
3284- apt_pkg.config.set("Dir::Cache::pkgcache", "")
3285- apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
3286- return apt_pkg.Cache()
3287-
3288-
3289-def apt_install(packages, options=None, fatal=False):
3290- """Install one or more packages"""
3291- if options is None:
3292- options = ['--option=Dpkg::Options::=--force-confold']
3293-
3294- cmd = ['apt-get', '--assume-yes']
3295- cmd.extend(options)
3296- cmd.append('install')
3297- if isinstance(packages, six.string_types):
3298- cmd.append(packages)
3299- else:
3300- cmd.extend(packages)
3301- log("Installing {} with options: {}".format(packages,
3302- options))
3303- _run_apt_command(cmd, fatal)
3304-
3305-
3306-def apt_upgrade(options=None, fatal=False, dist=False):
3307- """Upgrade all packages"""
3308- if options is None:
3309- options = ['--option=Dpkg::Options::=--force-confold']
3310-
3311- cmd = ['apt-get', '--assume-yes']
3312- cmd.extend(options)
3313- if dist:
3314- cmd.append('dist-upgrade')
3315- else:
3316- cmd.append('upgrade')
3317- log("Upgrading with options: {}".format(options))
3318- _run_apt_command(cmd, fatal)
3319-
3320-
3321-def apt_update(fatal=False):
3322- """Update local apt cache"""
3323- cmd = ['apt-get', 'update']
3324- _run_apt_command(cmd, fatal)
3325-
3326-
3327-def apt_purge(packages, fatal=False):
3328- """Purge one or more packages"""
3329- cmd = ['apt-get', '--assume-yes', 'purge']
3330- if isinstance(packages, six.string_types):
3331- cmd.append(packages)
3332- else:
3333- cmd.extend(packages)
3334- log("Purging {}".format(packages))
3335- _run_apt_command(cmd, fatal)
3336-
3337-
3338-def apt_hold(packages, fatal=False):
3339- """Hold one or more packages"""
3340- cmd = ['apt-mark', 'hold']
3341- if isinstance(packages, six.string_types):
3342- cmd.append(packages)
3343- else:
3344- cmd.extend(packages)
3345- log("Holding {}".format(packages))
3346-
3347- if fatal:
3348- subprocess.check_call(cmd)
3349- else:
3350- subprocess.call(cmd)
3351-
3352-
3353-def add_source(source, key=None):
3354- """Add a package source to this system.
3355-
3356- @param source: a URL or sources.list entry, as supported by
3357- add-apt-repository(1). Examples::
3358-
3359- ppa:charmers/example
3360- deb https://stub:key@private.example.com/ubuntu trusty main
3361-
3362- In addition:
3363- 'proposed:' may be used to enable the standard 'proposed'
3364- pocket for the release.
3365- 'cloud:' may be used to activate official cloud archive pockets,
3366- such as 'cloud:icehouse'
3367- 'distro' may be used as a noop
3368-
3369- @param key: A key to be added to the system's APT keyring and used
3370- to verify the signatures on packages. Ideally, this should be an
3371- ASCII format GPG public key including the block headers. A GPG key
3372- id may also be used, but be aware that only insecure protocols are
3373- available to retrieve the actual public key from a public keyserver
3374- placing your Juju environment at risk. ppa and cloud archive keys
3375- are securely added automtically, so sould not be provided.
3376- """
3377- if source is None:
3378- log('Source is not present. Skipping')
3379- return
3380-
3381- if (source.startswith('ppa:') or
3382- source.startswith('http') or
3383- source.startswith('deb ') or
3384- source.startswith('cloud-archive:')):
3385- subprocess.check_call(['add-apt-repository', '--yes', source])
3386- elif source.startswith('cloud:'):
3387- apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
3388- fatal=True)
3389- pocket = source.split(':')[-1]
3390- if pocket not in CLOUD_ARCHIVE_POCKETS:
3391- raise SourceConfigError(
3392- 'Unsupported cloud: source option %s' %
3393- pocket)
3394- actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
3395- with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
3396- apt.write(CLOUD_ARCHIVE.format(actual_pocket))
3397- elif source == 'proposed':
3398- release = lsb_release()['DISTRIB_CODENAME']
3399- with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
3400- apt.write(PROPOSED_POCKET.format(release))
3401- elif source == 'distro':
3402- pass
3403- else:
3404- log("Unknown source: {!r}".format(source))
3405-
3406- if key:
3407- if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
3408- with NamedTemporaryFile('w+') as key_file:
3409- key_file.write(key)
3410- key_file.flush()
3411- key_file.seek(0)
3412- subprocess.check_call(['apt-key', 'add', '-'], stdin=key_file)
3413- else:
3414- # Note that hkp: is in no way a secure protocol. Using a
3415- # GPG key id is pointless from a security POV unless you
3416- # absolutely trust your network and DNS.
3417- subprocess.check_call(['apt-key', 'adv', '--keyserver',
3418- 'hkp://keyserver.ubuntu.com:80', '--recv',
3419- key])
3420+__platform__ = get_platform()
3421+module = "charmhelpers.fetch.%s" % __platform__
3422+fetch = importlib.import_module(module)
3423+
3424+filter_installed_packages = fetch.filter_installed_packages
3425+install = fetch.apt_install
3426+upgrade = fetch.apt_upgrade
3427+update = _fetch_update = fetch.apt_update
3428+purge = fetch.apt_purge
3429+add_source = fetch.add_source
3430+
3431+if __platform__ == "ubuntu":
3432+ apt_cache = fetch.apt_cache
3433+ apt_install = fetch.apt_install
3434+ apt_update = fetch.apt_update
3435+ apt_upgrade = fetch.apt_upgrade
3436+ apt_purge = fetch.apt_purge
3437+ apt_mark = fetch.apt_mark
3438+ apt_hold = fetch.apt_hold
3439+ apt_unhold = fetch.apt_unhold
3440+ import_key = fetch.import_key
3441+ get_upstream_version = fetch.get_upstream_version
3442+elif __platform__ == "centos":
3443+ yum_search = fetch.yum_search
3444
3445
3446 def configure_sources(update=False,
3447 sources_var='install_sources',
3448 keys_var='install_keys'):
3449- """
3450- Configure multiple sources from charm configuration.
3451+ """Configure multiple sources from charm configuration.
3452
3453 The lists are encoded as yaml fragments in the configuration.
3454- The frament needs to be included as a string. Sources and their
3455+ The fragment needs to be included as a string. Sources and their
3456 corresponding keys are of the types supported by add_source().
3457
3458 Example config:
3459@@ -338,12 +143,11 @@
3460 for source, key in zip(sources, keys):
3461 add_source(source, key)
3462 if update:
3463- apt_update(fatal=True)
3464+ _fetch_update(fatal=True)
3465
3466
3467 def install_remote(source, *args, **kwargs):
3468- """
3469- Install a file tree from a remote source
3470+ """Install a file tree from a remote source.
3471
3472 The specified source should be a url of the form:
3473 scheme://[host]/path[#[option=value][&...]]
3474@@ -366,18 +170,17 @@
3475 # We ONLY check for True here because can_handle may return a string
3476 # explaining why it can't handle a given source.
3477 handlers = [h for h in plugins() if h.can_handle(source) is True]
3478- installed_to = None
3479 for handler in handlers:
3480 try:
3481- installed_to = handler.install(source, *args, **kwargs)
3482- except UnhandledSource:
3483- pass
3484- if not installed_to:
3485- raise UnhandledSource("No handler found for source {}".format(source))
3486- return installed_to
3487+ return handler.install(source, *args, **kwargs)
3488+ except UnhandledSource as e:
3489+ log('Install source attempt unsuccessful: {}'.format(e),
3490+ level='WARNING')
3491+ raise UnhandledSource("No handler found for source {}".format(source))
3492
3493
3494 def install_from_config(config_var_name):
3495+ """Install a file from config."""
3496 charm_config = config()
3497 source = charm_config[config_var_name]
3498 return install_remote(source)
3499@@ -394,46 +197,9 @@
3500 importlib.import_module(package),
3501 classname)
3502 plugin_list.append(handler_class())
3503- except (ImportError, AttributeError):
3504+ except NotImplementedError:
3505 # Skip missing plugins so that they can be ommitted from
3506 # installation if desired
3507 log("FetchHandler {} not found, skipping plugin".format(
3508 handler_name))
3509 return plugin_list
3510-
3511-
3512-def _run_apt_command(cmd, fatal=False):
3513- """
3514- Run an APT command, checking output and retrying if the fatal flag is set
3515- to True.
3516-
3517- :param: cmd: str: The apt command to run.
3518- :param: fatal: bool: Whether the command's output should be checked and
3519- retried.
3520- """
3521- env = os.environ.copy()
3522-
3523- if 'DEBIAN_FRONTEND' not in env:
3524- env['DEBIAN_FRONTEND'] = 'noninteractive'
3525-
3526- if fatal:
3527- retry_count = 0
3528- result = None
3529-
3530- # If the command is considered "fatal", we need to retry if the apt
3531- # lock was not acquired.
3532-
3533- while result is None or result == APT_NO_LOCK:
3534- try:
3535- result = subprocess.check_call(cmd, env=env)
3536- except subprocess.CalledProcessError as e:
3537- retry_count = retry_count + 1
3538- if retry_count > APT_NO_LOCK_RETRY_COUNT:
3539- raise
3540- result = e.returncode
3541- log("Couldn't acquire DPKG lock. Will retry in {} seconds."
3542- "".format(APT_NO_LOCK_RETRY_DELAY))
3543- time.sleep(APT_NO_LOCK_RETRY_DELAY)
3544-
3545- else:
3546- subprocess.call(cmd, env=env)
3547
3548=== modified file 'hooks/charmhelpers/fetch/archiveurl.py'
3549--- hooks/charmhelpers/fetch/archiveurl.py 2015-04-09 06:46:35 +0000
3550+++ hooks/charmhelpers/fetch/archiveurl.py 2017-07-25 09:17:37 +0000
3551@@ -1,18 +1,16 @@
3552 # Copyright 2014-2015 Canonical Limited.
3553 #
3554-# This file is part of charm-helpers.
3555-#
3556-# charm-helpers is free software: you can redistribute it and/or modify
3557-# it under the terms of the GNU Lesser General Public License version 3 as
3558-# published by the Free Software Foundation.
3559-#
3560-# charm-helpers is distributed in the hope that it will be useful,
3561-# but WITHOUT ANY WARRANTY; without even the implied warranty of
3562-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3563-# GNU Lesser General Public License for more details.
3564-#
3565-# You should have received a copy of the GNU Lesser General Public License
3566-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3567+# Licensed under the Apache License, Version 2.0 (the "License");
3568+# you may not use this file except in compliance with the License.
3569+# You may obtain a copy of the License at
3570+#
3571+# http://www.apache.org/licenses/LICENSE-2.0
3572+#
3573+# Unless required by applicable law or agreed to in writing, software
3574+# distributed under the License is distributed on an "AS IS" BASIS,
3575+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3576+# See the License for the specific language governing permissions and
3577+# limitations under the License.
3578
3579 import os
3580 import hashlib
3581@@ -77,6 +75,8 @@
3582 def can_handle(self, source):
3583 url_parts = self.parse_url(source)
3584 if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
3585+ # XXX: Why is this returning a boolean and a string? It's
3586+ # doomed to fail since "bool(can_handle('foo://'))" will be True.
3587 return "Wrong source type"
3588 if get_archive_handler(self.base_url(source)):
3589 return True
3590@@ -106,7 +106,7 @@
3591 install_opener(opener)
3592 response = urlopen(source)
3593 try:
3594- with open(dest, 'w') as dest_file:
3595+ with open(dest, 'wb') as dest_file:
3596 dest_file.write(response.read())
3597 except Exception as e:
3598 if os.path.isfile(dest):
3599@@ -155,7 +155,11 @@
3600 else:
3601 algorithms = hashlib.algorithms_available
3602 if key in algorithms:
3603- check_hash(dld_file, value, key)
3604+ if len(value) != 1:
3605+ raise TypeError(
3606+ "Expected 1 hash value, not %d" % len(value))
3607+ expected = value[0]
3608+ check_hash(dld_file, expected, key)
3609 if checksum:
3610 check_hash(dld_file, checksum, hash_type)
3611 return extract(dld_file, dest)
3612
3613=== modified file 'hooks/charmhelpers/fetch/bzrurl.py'
3614--- hooks/charmhelpers/fetch/bzrurl.py 2015-04-09 06:46:35 +0000
3615+++ hooks/charmhelpers/fetch/bzrurl.py 2017-07-25 09:17:37 +0000
3616@@ -1,78 +1,76 @@
3617 # Copyright 2014-2015 Canonical Limited.
3618 #
3619-# This file is part of charm-helpers.
3620-#
3621-# charm-helpers is free software: you can redistribute it and/or modify
3622-# it under the terms of the GNU Lesser General Public License version 3 as
3623-# published by the Free Software Foundation.
3624-#
3625-# charm-helpers is distributed in the hope that it will be useful,
3626-# but WITHOUT ANY WARRANTY; without even the implied warranty of
3627-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3628-# GNU Lesser General Public License for more details.
3629-#
3630-# You should have received a copy of the GNU Lesser General Public License
3631-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3632+# Licensed under the Apache License, Version 2.0 (the "License");
3633+# you may not use this file except in compliance with the License.
3634+# You may obtain a copy of the License at
3635+#
3636+# http://www.apache.org/licenses/LICENSE-2.0
3637+#
3638+# Unless required by applicable law or agreed to in writing, software
3639+# distributed under the License is distributed on an "AS IS" BASIS,
3640+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3641+# See the License for the specific language governing permissions and
3642+# limitations under the License.
3643
3644 import os
3645+from subprocess import check_call
3646 from charmhelpers.fetch import (
3647 BaseFetchHandler,
3648- UnhandledSource
3649+ UnhandledSource,
3650+ filter_installed_packages,
3651+ install,
3652 )
3653 from charmhelpers.core.host import mkdir
3654
3655-import six
3656-if six.PY3:
3657- raise ImportError('bzrlib does not support Python3')
3658
3659-try:
3660- from bzrlib.branch import Branch
3661- from bzrlib import bzrdir, workingtree, errors
3662-except ImportError:
3663- from charmhelpers.fetch import apt_install
3664- apt_install("python-bzrlib")
3665- from bzrlib.branch import Branch
3666- from bzrlib import bzrdir, workingtree, errors
3667+if filter_installed_packages(['bzr']) != []:
3668+ install(['bzr'])
3669+ if filter_installed_packages(['bzr']) != []:
3670+ raise NotImplementedError('Unable to install bzr')
3671
3672
3673 class BzrUrlFetchHandler(BaseFetchHandler):
3674- """Handler for bazaar branches via generic and lp URLs"""
3675+ """Handler for bazaar branches via generic and lp URLs."""
3676+
3677 def can_handle(self, source):
3678 url_parts = self.parse_url(source)
3679- if url_parts.scheme not in ('bzr+ssh', 'lp'):
3680+ if url_parts.scheme not in ('bzr+ssh', 'lp', ''):
3681 return False
3682+ elif not url_parts.scheme:
3683+ return os.path.exists(os.path.join(source, '.bzr'))
3684 else:
3685 return True
3686
3687- def branch(self, source, dest):
3688- url_parts = self.parse_url(source)
3689- # If we use lp:branchname scheme we need to load plugins
3690+ def branch(self, source, dest, revno=None):
3691 if not self.can_handle(source):
3692 raise UnhandledSource("Cannot handle {}".format(source))
3693- if url_parts.scheme == "lp":
3694- from bzrlib.plugin import load_plugins
3695- load_plugins()
3696- try:
3697- local_branch = bzrdir.BzrDir.create_branch_convenience(dest)
3698- except errors.AlreadyControlDirError:
3699- local_branch = Branch.open(dest)
3700- try:
3701- remote_branch = Branch.open(source)
3702- remote_branch.push(local_branch)
3703- tree = workingtree.WorkingTree.open(dest)
3704- tree.update()
3705- except Exception as e:
3706- raise e
3707+ cmd_opts = []
3708+ if revno:
3709+ cmd_opts += ['-r', str(revno)]
3710+ if os.path.exists(dest):
3711+ cmd = ['bzr', 'pull']
3712+ cmd += cmd_opts
3713+ cmd += ['--overwrite', '-d', dest, source]
3714+ else:
3715+ cmd = ['bzr', 'branch']
3716+ cmd += cmd_opts
3717+ cmd += [source, dest]
3718+ check_call(cmd)
3719
3720- def install(self, source):
3721+ def install(self, source, dest=None, revno=None):
3722 url_parts = self.parse_url(source)
3723 branch_name = url_parts.path.strip("/").split("/")[-1]
3724- dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
3725- branch_name)
3726- if not os.path.exists(dest_dir):
3727- mkdir(dest_dir, perms=0o755)
3728+ if dest:
3729+ dest_dir = os.path.join(dest, branch_name)
3730+ else:
3731+ dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
3732+ branch_name)
3733+
3734+ if dest and not os.path.exists(dest):
3735+ mkdir(dest, perms=0o755)
3736+
3737 try:
3738- self.branch(source, dest_dir)
3739+ self.branch(source, dest_dir, revno)
3740 except OSError as e:
3741 raise UnhandledSource(e.strerror)
3742 return dest_dir
3743
3744=== added file 'hooks/charmhelpers/fetch/centos.py'
3745--- hooks/charmhelpers/fetch/centos.py 1970-01-01 00:00:00 +0000
3746+++ hooks/charmhelpers/fetch/centos.py 2017-07-25 09:17:37 +0000
3747@@ -0,0 +1,171 @@
3748+# Copyright 2014-2015 Canonical Limited.
3749+#
3750+# Licensed under the Apache License, Version 2.0 (the "License");
3751+# you may not use this file except in compliance with the License.
3752+# You may obtain a copy of the License at
3753+#
3754+# http://www.apache.org/licenses/LICENSE-2.0
3755+#
3756+# Unless required by applicable law or agreed to in writing, software
3757+# distributed under the License is distributed on an "AS IS" BASIS,
3758+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3759+# See the License for the specific language governing permissions and
3760+# limitations under the License.
3761+
3762+import subprocess
3763+import os
3764+import time
3765+import six
3766+import yum
3767+
3768+from tempfile import NamedTemporaryFile
3769+from charmhelpers.core.hookenv import log
3770+
3771+YUM_NO_LOCK = 1 # The return code for "couldn't acquire lock" in YUM.
3772+YUM_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
3773+YUM_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
3774+
3775+
3776+def filter_installed_packages(packages):
3777+ """Return a list of packages that require installation."""
3778+ yb = yum.YumBase()
3779+ package_list = yb.doPackageLists()
3780+ temp_cache = {p.base_package_name: 1 for p in package_list['installed']}
3781+
3782+ _pkgs = [p for p in packages if not temp_cache.get(p, False)]
3783+ return _pkgs
3784+
3785+
3786+def install(packages, options=None, fatal=False):
3787+ """Install one or more packages."""
3788+ cmd = ['yum', '--assumeyes']
3789+ if options is not None:
3790+ cmd.extend(options)
3791+ cmd.append('install')
3792+ if isinstance(packages, six.string_types):
3793+ cmd.append(packages)
3794+ else:
3795+ cmd.extend(packages)
3796+ log("Installing {} with options: {}".format(packages,
3797+ options))
3798+ _run_yum_command(cmd, fatal)
3799+
3800+
3801+def upgrade(options=None, fatal=False, dist=False):
3802+ """Upgrade all packages."""
3803+ cmd = ['yum', '--assumeyes']
3804+ if options is not None:
3805+ cmd.extend(options)
3806+ cmd.append('upgrade')
3807+ log("Upgrading with options: {}".format(options))
3808+ _run_yum_command(cmd, fatal)
3809+
3810+
3811+def update(fatal=False):
3812+ """Update local yum cache."""
3813+ cmd = ['yum', '--assumeyes', 'update']
3814+ log("Update with fatal: {}".format(fatal))
3815+ _run_yum_command(cmd, fatal)
3816+
3817+
3818+def purge(packages, fatal=False):
3819+ """Purge one or more packages."""
3820+ cmd = ['yum', '--assumeyes', 'remove']
3821+ if isinstance(packages, six.string_types):
3822+ cmd.append(packages)
3823+ else:
3824+ cmd.extend(packages)
3825+ log("Purging {}".format(packages))
3826+ _run_yum_command(cmd, fatal)
3827+
3828+
3829+def yum_search(packages):
3830+ """Search for a package."""
3831+ output = {}
3832+ cmd = ['yum', 'search']
3833+ if isinstance(packages, six.string_types):
3834+ cmd.append(packages)
3835+ else:
3836+ cmd.extend(packages)
3837+ log("Searching for {}".format(packages))
3838+ result = subprocess.check_output(cmd)
3839+ for package in list(packages):
3840+ output[package] = package in result
3841+ return output
3842+
3843+
3844+def add_source(source, key=None):
3845+ """Add a package source to this system.
3846+
3847+ @param source: a URL with a rpm package
3848+
3849+ @param key: A key to be added to the system's keyring and used
3850+ to verify the signatures on packages. Ideally, this should be an
3851+ ASCII format GPG public key including the block headers. A GPG key
3852+ id may also be used, but be aware that only insecure protocols are
3853+ available to retrieve the actual public key from a public keyserver
3854+ placing your Juju environment at risk.
3855+ """
3856+ if source is None:
3857+ log('Source is not present. Skipping')
3858+ return
3859+
3860+ if source.startswith('http'):
3861+ directory = '/etc/yum.repos.d/'
3862+ for filename in os.listdir(directory):
3863+ with open(directory + filename, 'r') as rpm_file:
3864+ if source in rpm_file.read():
3865+ break
3866+ else:
3867+ log("Add source: {!r}".format(source))
3868+ # write in the charms.repo
3869+ with open(directory + 'Charms.repo', 'a') as rpm_file:
3870+ rpm_file.write('[%s]\n' % source[7:].replace('/', '_'))
3871+ rpm_file.write('name=%s\n' % source[7:])
3872+ rpm_file.write('baseurl=%s\n\n' % source)
3873+ else:
3874+ log("Unknown source: {!r}".format(source))
3875+
3876+ if key:
3877+ if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
3878+ with NamedTemporaryFile('w+') as key_file:
3879+ key_file.write(key)
3880+ key_file.flush()
3881+ key_file.seek(0)
3882+ subprocess.check_call(['rpm', '--import', key_file.name])
3883+ else:
3884+ subprocess.check_call(['rpm', '--import', key])
3885+
3886+
3887+def _run_yum_command(cmd, fatal=False):
3888+ """Run an YUM command.
3889+
3890+ Checks the output and retry if the fatal flag is set to True.
3891+
3892+ :param: cmd: str: The yum command to run.
3893+ :param: fatal: bool: Whether the command's output should be checked and
3894+ retried.
3895+ """
3896+ env = os.environ.copy()
3897+
3898+ if fatal:
3899+ retry_count = 0
3900+ result = None
3901+
3902+ # If the command is considered "fatal", we need to retry if the yum
3903+ # lock was not acquired.
3904+
3905+ while result is None or result == YUM_NO_LOCK:
3906+ try:
3907+ result = subprocess.check_call(cmd, env=env)
3908+ except subprocess.CalledProcessError as e:
3909+ retry_count = retry_count + 1
3910+ if retry_count > YUM_NO_LOCK_RETRY_COUNT:
3911+ raise
3912+ result = e.returncode
3913+ log("Couldn't acquire YUM lock. Will retry in {} seconds."
3914+ "".format(YUM_NO_LOCK_RETRY_DELAY))
3915+ time.sleep(YUM_NO_LOCK_RETRY_DELAY)
3916+
3917+ else:
3918+ subprocess.call(cmd, env=env)
3919
3920=== modified file 'hooks/charmhelpers/fetch/giturl.py'
3921--- hooks/charmhelpers/fetch/giturl.py 2015-04-09 06:46:35 +0000
3922+++ hooks/charmhelpers/fetch/giturl.py 2017-07-25 09:17:37 +0000
3923@@ -1,58 +1,58 @@
3924 # Copyright 2014-2015 Canonical Limited.
3925 #
3926-# This file is part of charm-helpers.
3927-#
3928-# charm-helpers is free software: you can redistribute it and/or modify
3929-# it under the terms of the GNU Lesser General Public License version 3 as
3930-# published by the Free Software Foundation.
3931-#
3932-# charm-helpers is distributed in the hope that it will be useful,
3933-# but WITHOUT ANY WARRANTY; without even the implied warranty of
3934-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3935-# GNU Lesser General Public License for more details.
3936-#
3937-# You should have received a copy of the GNU Lesser General Public License
3938-# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3939+# Licensed under the Apache License, Version 2.0 (the "License");
3940+# you may not use this file except in compliance with the License.
3941+# You may obtain a copy of the License at
3942+#
3943+# http://www.apache.org/licenses/LICENSE-2.0
3944+#
3945+# Unless required by applicable law or agreed to in writing, software
3946+# distributed under the License is distributed on an "AS IS" BASIS,
3947+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3948+# See the License for the specific language governing permissions and
3949+# limitations under the License.
3950
3951 import os
3952+from subprocess import check_call, CalledProcessError
3953 from charmhelpers.fetch import (
3954 BaseFetchHandler,
3955- UnhandledSource
3956+ UnhandledSource,
3957+ filter_installed_packages,
3958+ install,
3959 )
3960-from charmhelpers.core.host import mkdir
3961-
3962-import six
3963-if six.PY3:
3964- raise ImportError('GitPython does not support Python 3')
3965-
3966-try:
3967- from git import Repo
3968-except ImportError:
3969- from charmhelpers.fetch import apt_install
3970- apt_install("python-git")
3971- from git import Repo
3972-
3973-from git.exc import GitCommandError # noqa E402
3974+
3975+if filter_installed_packages(['git']) != []:
3976+ install(['git'])
3977+ if filter_installed_packages(['git']) != []:
3978+ raise NotImplementedError('Unable to install git')
3979
3980
3981 class GitUrlFetchHandler(BaseFetchHandler):
3982- """Handler for git branches via generic and github URLs"""
3983+ """Handler for git branches via generic and github URLs."""
3984+
3985 def can_handle(self, source):
3986 url_parts = self.parse_url(source)
3987 # TODO (mattyw) no support for ssh git@ yet
3988- if url_parts.scheme not in ('http', 'https', 'git'):
3989+ if url_parts.scheme not in ('http', 'https', 'git', ''):
3990 return False
3991+ elif not url_parts.scheme:
3992+ return os.path.exists(os.path.join(source, '.git'))
3993 else:
3994 return True
3995
3996- def clone(self, source, dest, branch):
3997+ def clone(self, source, dest, branch="master", depth=None):
3998 if not self.can_handle(source):
3999 raise UnhandledSource("Cannot handle {}".format(source))
4000
4001- repo = Repo.clone_from(source, dest)
4002- repo.git.checkout(branch)
4003+ if os.path.exists(dest):
4004+ cmd = ['git', '-C', dest, 'pull', source, branch]
4005+ else:
4006+ cmd = ['git', 'clone', source, dest, '--branch', branch]
4007+ if depth:
4008+ cmd.extend(['--depth', depth])
4009+ check_call(cmd)
4010
4011- def install(self, source, branch="master", dest=None):
4012+ def install(self, source, branch="master", dest=None, depth=None):
4013 url_parts = self.parse_url(source)
4014 branch_name = url_parts.path.strip("/").split("/")[-1]
4015 if dest:
4016@@ -60,12 +60,10 @@
4017 else:
4018 dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
4019 branch_name)
4020- if not os.path.exists(dest_dir):
4021- mkdir(dest_dir, perms=0o755)
4022 try:
4023- self.clone(source, dest_dir, branch)
4024- except GitCommandError as e:
4025- raise UnhandledSource(e.message)
4026+ self.clone(source, dest_dir, branch, depth)
4027+ except CalledProcessError as e:
4028+ raise UnhandledSource(e)
4029 except OSError as e:
4030 raise UnhandledSource(e.strerror)
4031 return dest_dir
4032
4033=== added file 'hooks/charmhelpers/fetch/snap.py'
4034--- hooks/charmhelpers/fetch/snap.py 1970-01-01 00:00:00 +0000
4035+++ hooks/charmhelpers/fetch/snap.py 2017-07-25 09:17:37 +0000
4036@@ -0,0 +1,122 @@
4037+# Copyright 2014-2017 Canonical Limited.
4038+#
4039+# Licensed under the Apache License, Version 2.0 (the "License");
4040+# you may not use this file except in compliance with the License.
4041+# You may obtain a copy of the License at
4042+#
4043+# http://www.apache.org/licenses/LICENSE-2.0
4044+#
4045+# Unless required by applicable law or agreed to in writing, software
4046+# distributed under the License is distributed on an "AS IS" BASIS,
4047+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4048+# See the License for the specific language governing permissions and
4049+# limitations under the License.
4050+"""
4051+Charm helpers snap for classic charms.
4052+
4053+If writing reactive charms, use the snap layer:
4054+https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html
4055+"""
4056+import subprocess
4057+from os import environ
4058+from time import sleep
4059+from charmhelpers.core.hookenv import log
4060+
4061+__author__ = 'Joseph Borg <joseph.borg@canonical.com>'
4062+
4063+SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved).
4064+SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks.
4065+SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
4066+
4067+
4068+class CouldNotAcquireLockException(Exception):
4069+ pass
4070+
4071+
4072+def _snap_exec(commands):
4073+ """
4074+ Execute snap commands.
4075+
4076+ :param commands: List commands
4077+ :return: Integer exit code
4078+ """
4079+ assert type(commands) == list
4080+
4081+ retry_count = 0
4082+ return_code = None
4083+
4084+ while return_code is None or return_code == SNAP_NO_LOCK:
4085+ try:
4086+ return_code = subprocess.check_call(['snap'] + commands, env=environ)
4087+ except subprocess.CalledProcessError as e:
4088+ retry_count += + 1
4089+ if retry_count > SNAP_NO_LOCK_RETRY_COUNT:
4090+ raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT)
4091+ return_code = e.returncode
4092+ log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN')
4093+ sleep(SNAP_NO_LOCK_RETRY_DELAY)
4094+
4095+ return return_code
4096+
4097+
4098+def snap_install(packages, *flags):
4099+ """
4100+ Install a snap package.
4101+
4102+ :param packages: String or List String package name
4103+ :param flags: List String flags to pass to install command
4104+ :return: Integer return code from snap
4105+ """
4106+ if type(packages) is not list:
4107+ packages = [packages]
4108+
4109+ flags = list(flags)
4110+
4111+ message = 'Installing snap(s) "%s"' % ', '.join(packages)
4112+ if flags:
4113+ message += ' with option(s) "%s"' % ', '.join(flags)
4114+
4115+ log(message, level='INFO')
4116+ return _snap_exec(['install'] + flags + packages)
4117+
4118+
4119+def snap_remove(packages, *flags):
4120+ """
4121+ Remove a snap package.
4122+
4123+ :param packages: String or List String package name
4124+ :param flags: List String flags to pass to remove command
4125+ :return: Integer return code from snap
4126+ """
4127+ if type(packages) is not list:
4128+ packages = [packages]
4129+
4130+ flags = list(flags)
4131+
4132+ message = 'Removing snap(s) "%s"' % ', '.join(packages)
4133+ if flags:
4134+ message += ' with options "%s"' % ', '.join(flags)
4135+
4136+ log(message, level='INFO')
4137+ return _snap_exec(['remove'] + flags + packages)
4138+
4139+
4140+def snap_refresh(packages, *flags):
4141+ """
4142+ Refresh / Update snap package.
4143+
4144+ :param packages: String or List String package name
4145+ :param flags: List String flags to pass to refresh command
4146+ :return: Integer return code from snap
4147+ """
4148+ if type(packages) is not list:
4149+ packages = [packages]
4150+
4151+ flags = list(flags)
4152+
4153+ message = 'Refreshing snap(s) "%s"' % ', '.join(packages)
4154+ if flags:
4155+ message += ' with options "%s"' % ', '.join(flags)
4156+
4157+ log(message, level='INFO')
4158+ return _snap_exec(['refresh'] + flags + packages)
4159
4160=== added file 'hooks/charmhelpers/fetch/ubuntu.py'
4161--- hooks/charmhelpers/fetch/ubuntu.py 1970-01-01 00:00:00 +0000
4162+++ hooks/charmhelpers/fetch/ubuntu.py 2017-07-25 09:17:37 +0000
4163@@ -0,0 +1,568 @@
4164+# Copyright 2014-2015 Canonical Limited.
4165+#
4166+# Licensed under the Apache License, Version 2.0 (the "License");
4167+# you may not use this file except in compliance with the License.
4168+# You may obtain a copy of the License at
4169+#
4170+# http://www.apache.org/licenses/LICENSE-2.0
4171+#
4172+# Unless required by applicable law or agreed to in writing, software
4173+# distributed under the License is distributed on an "AS IS" BASIS,
4174+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4175+# See the License for the specific language governing permissions and
4176+# limitations under the License.
4177+
4178+from collections import OrderedDict
4179+import os
4180+import platform
4181+import re
4182+import six
4183+import time
4184+import subprocess
4185+from tempfile import NamedTemporaryFile
4186+
4187+from charmhelpers.core.host import (
4188+ lsb_release
4189+)
4190+from charmhelpers.core.hookenv import (
4191+ log,
4192+ DEBUG,
4193+)
4194+from charmhelpers.fetch import SourceConfigError, GPGKeyError
4195+
4196+PROPOSED_POCKET = (
4197+ "# Proposed\n"
4198+ "deb http://archive.ubuntu.com/ubuntu {}-proposed main universe "
4199+ "multiverse restricted\n")
4200+PROPOSED_PORTS_POCKET = (
4201+ "# Proposed\n"
4202+ "deb http://ports.ubuntu.com/ubuntu-ports {}-proposed main universe "
4203+ "multiverse restricted\n")
4204+# Only supports 64bit and ppc64 at the moment.
4205+ARCH_TO_PROPOSED_POCKET = {
4206+ 'x86_64': PROPOSED_POCKET,
4207+ 'ppc64le': PROPOSED_PORTS_POCKET,
4208+ 'aarch64': PROPOSED_PORTS_POCKET,
4209+}
4210+CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
4211+CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
4212+CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
4213+deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
4214+"""
4215+CLOUD_ARCHIVE_POCKETS = {
4216+ # Folsom
4217+ 'folsom': 'precise-updates/folsom',
4218+ 'folsom/updates': 'precise-updates/folsom',
4219+ 'precise-folsom': 'precise-updates/folsom',
4220+ 'precise-folsom/updates': 'precise-updates/folsom',
4221+ 'precise-updates/folsom': 'precise-updates/folsom',
4222+ 'folsom/proposed': 'precise-proposed/folsom',
4223+ 'precise-folsom/proposed': 'precise-proposed/folsom',
4224+ 'precise-proposed/folsom': 'precise-proposed/folsom',
4225+ # Grizzly
4226+ 'grizzly': 'precise-updates/grizzly',
4227+ 'grizzly/updates': 'precise-updates/grizzly',
4228+ 'precise-grizzly': 'precise-updates/grizzly',
4229+ 'precise-grizzly/updates': 'precise-updates/grizzly',
4230+ 'precise-updates/grizzly': 'precise-updates/grizzly',
4231+ 'grizzly/proposed': 'precise-proposed/grizzly',
4232+ 'precise-grizzly/proposed': 'precise-proposed/grizzly',
4233+ 'precise-proposed/grizzly': 'precise-proposed/grizzly',
4234+ # Havana
4235+ 'havana': 'precise-updates/havana',
4236+ 'havana/updates': 'precise-updates/havana',
4237+ 'precise-havana': 'precise-updates/havana',
4238+ 'precise-havana/updates': 'precise-updates/havana',
4239+ 'precise-updates/havana': 'precise-updates/havana',
4240+ 'havana/proposed': 'precise-proposed/havana',
4241+ 'precise-havana/proposed': 'precise-proposed/havana',
4242+ 'precise-proposed/havana': 'precise-proposed/havana',
4243+ # Icehouse
4244+ 'icehouse': 'precise-updates/icehouse',
4245+ 'icehouse/updates': 'precise-updates/icehouse',
4246+ 'precise-icehouse': 'precise-updates/icehouse',
4247+ 'precise-icehouse/updates': 'precise-updates/icehouse',
4248+ 'precise-updates/icehouse': 'precise-updates/icehouse',
4249+ 'icehouse/proposed': 'precise-proposed/icehouse',
4250+ 'precise-icehouse/proposed': 'precise-proposed/icehouse',
4251+ 'precise-proposed/icehouse': 'precise-proposed/icehouse',
4252+ # Juno
4253+ 'juno': 'trusty-updates/juno',
4254+ 'juno/updates': 'trusty-updates/juno',
4255+ 'trusty-juno': 'trusty-updates/juno',
4256+ 'trusty-juno/updates': 'trusty-updates/juno',
4257+ 'trusty-updates/juno': 'trusty-updates/juno',
4258+ 'juno/proposed': 'trusty-proposed/juno',
4259+ 'trusty-juno/proposed': 'trusty-proposed/juno',
4260+ 'trusty-proposed/juno': 'trusty-proposed/juno',
4261+ # Kilo
4262+ 'kilo': 'trusty-updates/kilo',
4263+ 'kilo/updates': 'trusty-updates/kilo',
4264+ 'trusty-kilo': 'trusty-updates/kilo',
4265+ 'trusty-kilo/updates': 'trusty-updates/kilo',
4266+ 'trusty-updates/kilo': 'trusty-updates/kilo',
4267+ 'kilo/proposed': 'trusty-proposed/kilo',
4268+ 'trusty-kilo/proposed': 'trusty-proposed/kilo',
4269+ 'trusty-proposed/kilo': 'trusty-proposed/kilo',
4270+ # Liberty
4271+ 'liberty': 'trusty-updates/liberty',
4272+ 'liberty/updates': 'trusty-updates/liberty',
4273+ 'trusty-liberty': 'trusty-updates/liberty',
4274+ 'trusty-liberty/updates': 'trusty-updates/liberty',
4275+ 'trusty-updates/liberty': 'trusty-updates/liberty',
4276+ 'liberty/proposed': 'trusty-proposed/liberty',
4277+ 'trusty-liberty/proposed': 'trusty-proposed/liberty',
4278+ 'trusty-proposed/liberty': 'trusty-proposed/liberty',
4279+ # Mitaka
4280+ 'mitaka': 'trusty-updates/mitaka',
4281+ 'mitaka/updates': 'trusty-updates/mitaka',
4282+ 'trusty-mitaka': 'trusty-updates/mitaka',
4283+ 'trusty-mitaka/updates': 'trusty-updates/mitaka',
4284+ 'trusty-updates/mitaka': 'trusty-updates/mitaka',
4285+ 'mitaka/proposed': 'trusty-proposed/mitaka',
4286+ 'trusty-mitaka/proposed': 'trusty-proposed/mitaka',
4287+ 'trusty-proposed/mitaka': 'trusty-proposed/mitaka',
4288+ # Newton
4289+ 'newton': 'xenial-updates/newton',
4290+ 'newton/updates': 'xenial-updates/newton',
4291+ 'xenial-newton': 'xenial-updates/newton',
4292+ 'xenial-newton/updates': 'xenial-updates/newton',
4293+ 'xenial-updates/newton': 'xenial-updates/newton',
4294+ 'newton/proposed': 'xenial-proposed/newton',
4295+ 'xenial-newton/proposed': 'xenial-proposed/newton',
4296+ 'xenial-proposed/newton': 'xenial-proposed/newton',
4297+ # Ocata
4298+ 'ocata': 'xenial-updates/ocata',
4299+ 'ocata/updates': 'xenial-updates/ocata',
4300+ 'xenial-ocata': 'xenial-updates/ocata',
4301+ 'xenial-ocata/updates': 'xenial-updates/ocata',
4302+ 'xenial-updates/ocata': 'xenial-updates/ocata',
4303+ 'ocata/proposed': 'xenial-proposed/ocata',
4304+ 'xenial-ocata/proposed': 'xenial-proposed/ocata',
4305+ 'xenial-ocata/newton': 'xenial-proposed/ocata',
4306+ # Pike
4307+ 'pike': 'xenial-updates/pike',
4308+ 'xenial-pike': 'xenial-updates/pike',
4309+ 'xenial-pike/updates': 'xenial-updates/pike',
4310+ 'xenial-updates/pike': 'xenial-updates/pike',
4311+ 'pike/proposed': 'xenial-proposed/pike',
4312+ 'xenial-pike/proposed': 'xenial-proposed/pike',
4313+ 'xenial-pike/newton': 'xenial-proposed/pike',
4314+ # Queens
4315+ 'queens': 'xenial-updates/queens',
4316+ 'xenial-queens': 'xenial-updates/queens',
4317+ 'xenial-queens/updates': 'xenial-updates/queens',
4318+ 'xenial-updates/queens': 'xenial-updates/queens',
4319+ 'queens/proposed': 'xenial-proposed/queens',
4320+ 'xenial-queens/proposed': 'xenial-proposed/queens',
4321+ 'xenial-queens/newton': 'xenial-proposed/queens',
4322+}
4323+
4324+
4325+APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
4326+CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries.
4327+CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times.
4328+
4329+
4330+def filter_installed_packages(packages):
4331+ """Return a list of packages that require installation."""
4332+ cache = apt_cache()
4333+ _pkgs = []
4334+ for package in packages:
4335+ try:
4336+ p = cache[package]
4337+ p.current_ver or _pkgs.append(package)
4338+ except KeyError:
4339+ log('Package {} has no installation candidate.'.format(package),
4340+ level='WARNING')
4341+ _pkgs.append(package)
4342+ return _pkgs
4343+
4344+
4345+def apt_cache(in_memory=True, progress=None):
4346+ """Build and return an apt cache."""
4347+ from apt import apt_pkg
4348+ apt_pkg.init()
4349+ if in_memory:
4350+ apt_pkg.config.set("Dir::Cache::pkgcache", "")
4351+ apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
4352+ return apt_pkg.Cache(progress)
4353+
4354+
4355+def apt_install(packages, options=None, fatal=False):
4356+ """Install one or more packages."""
4357+ if options is None:
4358+ options = ['--option=Dpkg::Options::=--force-confold']
4359+
4360+ cmd = ['apt-get', '--assume-yes']
4361+ cmd.extend(options)
4362+ cmd.append('install')
4363+ if isinstance(packages, six.string_types):
4364+ cmd.append(packages)
4365+ else:
4366+ cmd.extend(packages)
4367+ log("Installing {} with options: {}".format(packages,
4368+ options))
4369+ _run_apt_command(cmd, fatal)
4370+
4371+
4372+def apt_upgrade(options=None, fatal=False, dist=False):
4373+ """Upgrade all packages."""
4374+ if options is None:
4375+ options = ['--option=Dpkg::Options::=--force-confold']
4376+
4377+ cmd = ['apt-get', '--assume-yes']
4378+ cmd.extend(options)
4379+ if dist:
4380+ cmd.append('dist-upgrade')
4381+ else:
4382+ cmd.append('upgrade')
4383+ log("Upgrading with options: {}".format(options))
4384+ _run_apt_command(cmd, fatal)
4385+
4386+
4387+def apt_update(fatal=False):
4388+ """Update local apt cache."""
4389+ cmd = ['apt-get', 'update']
4390+ _run_apt_command(cmd, fatal)
4391+
4392+
4393+def apt_purge(packages, fatal=False):
4394+ """Purge one or more packages."""
4395+ cmd = ['apt-get', '--assume-yes', 'purge']
4396+ if isinstance(packages, six.string_types):
4397+ cmd.append(packages)
4398+ else:
4399+ cmd.extend(packages)
4400+ log("Purging {}".format(packages))
4401+ _run_apt_command(cmd, fatal)
4402+
4403+
4404+def apt_mark(packages, mark, fatal=False):
4405+ """Flag one or more packages using apt-mark."""
4406+ log("Marking {} as {}".format(packages, mark))
4407+ cmd = ['apt-mark', mark]
4408+ if isinstance(packages, six.string_types):
4409+ cmd.append(packages)
4410+ else:
4411+ cmd.extend(packages)
4412+
4413+ if fatal:
4414+ subprocess.check_call(cmd, universal_newlines=True)
4415+ else:
4416+ subprocess.call(cmd, universal_newlines=True)
4417+
4418+
4419+def apt_hold(packages, fatal=False):
4420+ return apt_mark(packages, 'hold', fatal=fatal)
4421+
4422+
4423+def apt_unhold(packages, fatal=False):
4424+ return apt_mark(packages, 'unhold', fatal=fatal)
4425+
4426+
4427+def import_key(keyid):
4428+ """Import a key in either ASCII Armor or Radix64 format.
4429+
4430+ `keyid` is either the keyid to fetch from a PGP server, or
4431+ the key in ASCII armor foramt.
4432+
4433+ :param keyid: String of key (or key id).
4434+ :raises: GPGKeyError if the key could not be imported
4435+ """
4436+ key = keyid.strip()
4437+ if (key.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----') and
4438+ key.endswith('-----END PGP PUBLIC KEY BLOCK-----')):
4439+ log("PGP key found (looks like ASCII Armor format)", level=DEBUG)
4440+ log("Importing ASCII Armor PGP key", level=DEBUG)
4441+ with NamedTemporaryFile() as keyfile:
4442+ with open(keyfile.name, 'w') as fd:
4443+ fd.write(key)
4444+ fd.write("\n")
4445+ cmd = ['apt-key', 'add', keyfile.name]
4446+ try:
4447+ subprocess.check_call(cmd)
4448+ except subprocess.CalledProcessError:
4449+ error = "Error importing PGP key '{}'".format(key)
4450+ log(error)
4451+ raise GPGKeyError(error)
4452+ else:
4453+ log("PGP key found (looks like Radix64 format)", level=DEBUG)
4454+ log("Importing PGP key from keyserver", level=DEBUG)
4455+ cmd = ['apt-key', 'adv', '--keyserver',
4456+ 'hkp://keyserver.ubuntu.com:80', '--recv-keys', key]
4457+ try:
4458+ subprocess.check_call(cmd)
4459+ except subprocess.CalledProcessError:
4460+ error = "Error importing PGP key '{}'".format(key)
4461+ log(error)
4462+ raise GPGKeyError(error)
4463+
4464+
4465+def add_source(source, key=None, fail_invalid=False):
4466+ """Add a package source to this system.
4467+
4468+ @param source: a URL or sources.list entry, as supported by
4469+ add-apt-repository(1). Examples::
4470+
4471+ ppa:charmers/example
4472+ deb https://stub:key@private.example.com/ubuntu trusty main
4473+
4474+ In addition:
4475+ 'proposed:' may be used to enable the standard 'proposed'
4476+ pocket for the release.
4477+ 'cloud:' may be used to activate official cloud archive pockets,
4478+ such as 'cloud:icehouse'
4479+ 'distro' may be used as a noop
4480+
4481+ Full list of source specifications supported by the function are:
4482+
4483+ 'distro': A NOP; i.e. it has no effect.
4484+ 'proposed': the proposed deb spec [2] is wrtten to
4485+ /etc/apt/sources.list/proposed
4486+ 'distro-proposed': adds <version>-proposed to the debs [2]
4487+ 'ppa:<ppa-name>': add-apt-repository --yes <ppa_name>
4488+ 'deb <deb-spec>': add-apt-repository --yes deb <deb-spec>
4489+ 'http://....': add-apt-repository --yes http://...
4490+ 'cloud-archive:<spec>': add-apt-repository -yes cloud-archive:<spec>
4491+ 'cloud:<release>[-staging]': specify a Cloud Archive pocket <release> with
4492+ optional staging version. If staging is used then the staging PPA [2]
4493+ with be used. If staging is NOT used then the cloud archive [3] will be
4494+ added, and the 'ubuntu-cloud-keyring' package will be added for the
4495+ current distro.
4496+
4497+ Otherwise the source is not recognised and this is logged to the juju log.
4498+ However, no error is raised, unless sys_error_on_exit is True.
4499+
4500+ [1] deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
4501+ where {} is replaced with the derived pocket name.
4502+ [2] deb http://archive.ubuntu.com/ubuntu {}-proposed \
4503+ main universe multiverse restricted
4504+ where {} is replaced with the lsb_release codename (e.g. xenial)
4505+ [3] deb http://ubuntu-cloud.archive.canonical.com/ubuntu <pocket>
4506+ to /etc/apt/sources.list.d/cloud-archive-list
4507+
4508+ @param key: A key to be added to the system's APT keyring and used
4509+ to verify the signatures on packages. Ideally, this should be an
4510+ ASCII format GPG public key including the block headers. A GPG key
4511+ id may also be used, but be aware that only insecure protocols are
4512+ available to retrieve the actual public key from a public keyserver
4513+ placing your Juju environment at risk. ppa and cloud archive keys
4514+ are securely added automtically, so sould not be provided.
4515+
4516+ @param fail_invalid: (boolean) if True, then the function raises a
4517+ SourceConfigError is there is no matching installation source.
4518+
4519+ @raises SourceConfigError() if for cloud:<pocket>, the <pocket> is not a
4520+ valid pocket in CLOUD_ARCHIVE_POCKETS
4521+ """
4522+ _mapping = OrderedDict([
4523+ (r"^distro$", lambda: None), # This is a NOP
4524+ (r"^(?:proposed|distro-proposed)$", _add_proposed),
4525+ (r"^cloud-archive:(.*)$", _add_apt_repository),
4526+ (r"^((?:deb |http:|https:|ppa:).*)$", _add_apt_repository),
4527+ (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging),
4528+ (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check),
4529+ (r"^cloud:(.*)$", _add_cloud_pocket),
4530+ ])
4531+ if source is None:
4532+ source = ''
4533+ for r, fn in six.iteritems(_mapping):
4534+ m = re.match(r, source)
4535+ if m:
4536+ # call the assoicated function with the captured groups
4537+ # raises SourceConfigError on error.
4538+ fn(*m.groups())
4539+ if key:
4540+ try:
4541+ import_key(key)
4542+ except GPGKeyError as e:
4543+ raise SourceConfigError(str(e))
4544+ break
4545+ else:
4546+ # nothing matched. log an error and maybe sys.exit
4547+ err = "Unknown source: {!r}".format(source)
4548+ log(err)
4549+ if fail_invalid:
4550+ raise SourceConfigError(err)
4551+
4552+
4553+def _add_proposed():
4554+ """Add the PROPOSED_POCKET as /etc/apt/source.list.d/proposed.list
4555+
4556+ Uses lsb_release()['DISTRIB_CODENAME'] to determine the correct staza for
4557+ the deb line.
4558+
4559+ For intel architecutres PROPOSED_POCKET is used for the release, but for
4560+ other architectures PROPOSED_PORTS_POCKET is used for the release.
4561+ """
4562+ release = lsb_release()['DISTRIB_CODENAME']
4563+ arch = platform.machine()
4564+ if arch not in six.iterkeys(ARCH_TO_PROPOSED_POCKET):
4565+ raise SourceConfigError("Arch {} not supported for (distro-)proposed"
4566+ .format(arch))
4567+ with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
4568+ apt.write(ARCH_TO_PROPOSED_POCKET[arch].format(release))
4569+
4570+
4571+def _add_apt_repository(spec):
4572+ """Add the spec using add_apt_repository
4573+
4574+ :param spec: the parameter to pass to add_apt_repository
4575+ """
4576+ _run_with_retries(['add-apt-repository', '--yes', spec])
4577+
4578+
4579+def _add_cloud_pocket(pocket):
4580+ """Add a cloud pocket as /etc/apt/sources.d/cloud-archive.list
4581+
4582+ Note that this overwrites the existing file if there is one.
4583+
4584+ This function also converts the simple pocket in to the actual pocket using
4585+ the CLOUD_ARCHIVE_POCKETS mapping.
4586+
4587+ :param pocket: string representing the pocket to add a deb spec for.
4588+ :raises: SourceConfigError if the cloud pocket doesn't exist or the
4589+ requested release doesn't match the current distro version.
4590+ """
4591+ apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
4592+ fatal=True)
4593+ if pocket not in CLOUD_ARCHIVE_POCKETS:
4594+ raise SourceConfigError(
4595+ 'Unsupported cloud: source option %s' %
4596+ pocket)
4597+ actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
4598+ with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
4599+ apt.write(CLOUD_ARCHIVE.format(actual_pocket))
4600+
4601+
4602+def _add_cloud_staging(cloud_archive_release, openstack_release):
4603+ """Add the cloud staging repository which is in
4604+ ppa:ubuntu-cloud-archive/<openstack_release>-staging
4605+
4606+ This function checks that the cloud_archive_release matches the current
4607+ codename for the distro that charm is being installed on.
4608+
4609+ :param cloud_archive_release: string, codename for the release.
4610+ :param openstack_release: String, codename for the openstack release.
4611+ :raises: SourceConfigError if the cloud_archive_release doesn't match the
4612+ current version of the os.
4613+ """
4614+ _verify_is_ubuntu_rel(cloud_archive_release, openstack_release)
4615+ ppa = 'ppa:ubuntu-cloud-archive/{}-staging'.format(openstack_release)
4616+ cmd = 'add-apt-repository -y {}'.format(ppa)
4617+ _run_with_retries(cmd.split(' '))
4618+
4619+
4620+def _add_cloud_distro_check(cloud_archive_release, openstack_release):
4621+ """Add the cloud pocket, but also check the cloud_archive_release against
4622+ the current distro, and use the openstack_release as the full lookup.
4623+
4624+ This just calls _add_cloud_pocket() with the openstack_release as pocket
4625+ to get the correct cloud-archive.list for dpkg to work with.
4626+
4627+ :param cloud_archive_release:String, codename for the distro release.
4628+ :param openstack_release: String, spec for the release to look up in the
4629+ CLOUD_ARCHIVE_POCKETS
4630+ :raises: SourceConfigError if this is the wrong distro, or the pocket spec
4631+ doesn't exist.
4632+ """
4633+ _verify_is_ubuntu_rel(cloud_archive_release, openstack_release)
4634+ _add_cloud_pocket("{}-{}".format(cloud_archive_release, openstack_release))
4635+
4636+
4637+def _verify_is_ubuntu_rel(release, os_release):
4638+ """Verify that the release is in the same as the current ubuntu release.
4639+
4640+ :param release: String, lowercase for the release.
4641+ :param os_release: String, the os_release being asked for
4642+ :raises: SourceConfigError if the release is not the same as the ubuntu
4643+ release.
4644+ """
4645+ ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
4646+ if release != ubuntu_rel:
4647+ raise SourceConfigError(
4648+ 'Invalid Cloud Archive release specified: {}-{} on this Ubuntu'
4649+ 'version ({})'.format(release, os_release, ubuntu_rel))
4650+
4651+
4652+def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,),
4653+ retry_message="", cmd_env=None):
4654+ """Run a command and retry until success or max_retries is reached.
4655+
4656+ :param: cmd: str: The apt command to run.
4657+ :param: max_retries: int: The number of retries to attempt on a fatal
4658+ command. Defaults to CMD_RETRY_COUNT.
4659+ :param: retry_exitcodes: tuple: Optional additional exit codes to retry.
4660+ Defaults to retry on exit code 1.
4661+ :param: retry_message: str: Optional log prefix emitted during retries.
4662+ :param: cmd_env: dict: Environment variables to add to the command run.
4663+ """
4664+
4665+ env = None
4666+ kwargs = {}
4667+ if cmd_env:
4668+ env = os.environ.copy()
4669+ env.update(cmd_env)
4670+ kwargs['env'] = env
4671+
4672+ if not retry_message:
4673+ retry_message = "Failed executing '{}'".format(" ".join(cmd))
4674+ retry_message += ". Will retry in {} seconds".format(CMD_RETRY_DELAY)
4675+
4676+ retry_count = 0
4677+ result = None
4678+
4679+ retry_results = (None,) + retry_exitcodes
4680+ while result in retry_results:
4681+ try:
4682+ # result = subprocess.check_call(cmd, env=env)
4683+ result = subprocess.check_call(cmd, **kwargs)
4684+ except subprocess.CalledProcessError as e:
4685+ retry_count = retry_count + 1
4686+ if retry_count > max_retries:
4687+ raise
4688+ result = e.returncode
4689+ log(retry_message)
4690+ time.sleep(CMD_RETRY_DELAY)
4691+
4692+
4693+def _run_apt_command(cmd, fatal=False):
4694+ """Run an apt command with optional retries.
4695+
4696+ :param: cmd: str: The apt command to run.
4697+ :param: fatal: bool: Whether the command's output should be checked and
4698+ retried.
4699+ """
4700+ # Provide DEBIAN_FRONTEND=noninteractive if not present in the environment.
4701+ cmd_env = {
4702+ 'DEBIAN_FRONTEND': os.environ.get('DEBIAN_FRONTEND', 'noninteractive')}
4703+
4704+ if fatal:
4705+ _run_with_retries(
4706+ cmd, cmd_env=cmd_env, retry_exitcodes=(1, APT_NO_LOCK,),
4707+ retry_message="Couldn't acquire DPKG lock")
4708+ else:
4709+ env = os.environ.copy()
4710+ env.update(cmd_env)
4711+ subprocess.call(cmd, env=env)
4712+
4713+
4714+def get_upstream_version(package):
4715+ """Determine upstream version based on installed package
4716+
4717+ @returns None (if not installed) or the upstream version
4718+ """
4719+ import apt_pkg
4720+ cache = apt_cache()
4721+ try:
4722+ pkg = cache[package]
4723+ except:
4724+ # the package is unknown to the current apt cache.
4725+ return None
4726+
4727+ if not pkg.current_ver:
4728+ # package is known, but no version is currently installed.
4729+ return None
4730+
4731+ return apt_pkg.upstream_version(pkg.current_ver.ver_str)
4732
4733=== added file 'hooks/charmhelpers/osplatform.py'
4734--- hooks/charmhelpers/osplatform.py 1970-01-01 00:00:00 +0000
4735+++ hooks/charmhelpers/osplatform.py 2017-07-25 09:17:37 +0000
4736@@ -0,0 +1,25 @@
4737+import platform
4738+
4739+
4740+def get_platform():
4741+ """Return the current OS platform.
4742+
4743+ For example: if current os platform is Ubuntu then a string "ubuntu"
4744+ will be returned (which is the name of the module).
4745+ This string is used to decide which platform module should be imported.
4746+ """
4747+ # linux_distribution is deprecated and will be removed in Python 3.7
4748+ # Warings *not* disabled, as we certainly need to fix this.
4749+ tuple_platform = platform.linux_distribution()
4750+ current_platform = tuple_platform[0]
4751+ if "Ubuntu" in current_platform:
4752+ return "ubuntu"
4753+ elif "CentOS" in current_platform:
4754+ return "centos"
4755+ elif "debian" in current_platform:
4756+ # Stock Python does not detect Ubuntu and instead returns debian.
4757+ # Or at least it does in some build environments like Travis CI
4758+ return "ubuntu"
4759+ else:
4760+ raise RuntimeError("This module is not supported on {}."
4761+ .format(current_platform))
4762
4763=== modified file 'hooks/install.real'
4764--- hooks/install.real 2017-03-21 04:14:41 +0000
4765+++ hooks/install.real 2017-07-25 09:17:37 +0000
4766@@ -15,12 +15,10 @@
4767 # e.g.: from charmhelpers import fetch
4768 # fetch.apt_install(fetch.filter_installed_packages(['nginx']))
4769 config = hookenv.config()
4770- ppa = config.get('source')
4771- if ppa is not None:
4772- add_source(ppa)
4773- apt_update()
4774-
4775- apt_install(["thruk", "pwgen", "apache2-utils"])
4776+ add_source(config.get('source'), config.get('key', None))
4777+ apt_update(fatal=True)
4778+ package_list = ["thruk", "pwgen", "apache2-utils"]
4779+ apt_install(packages=package_list, fatal=True)
4780
4781
4782 if __name__ == "__main__":
4783
4784=== modified file 'templates/thruk_local.conf'
4785--- templates/thruk_local.conf 2015-04-28 05:49:19 +0000
4786+++ templates/thruk_local.conf 2017-07-25 09:17:37 +0000
4787@@ -13,3 +13,7 @@
4788 {% endfor -%}
4789 </Component>
4790
4791+{% if config['enable-lmd'] -%}
4792+use_lmd_core=1
4793+lmd_core_bin=/usr/bin/lmd
4794+{% endif -%}

Subscribers

People subscribed via source and target branches