Merge ~barryprice/charm-nrpe/+git/nrpe-charm:master into ~nrpe-charmers/charm-nrpe:master

[Barry Price]

Freshen charm_helpers_sync.py, make sync, remove yakkety and zesty support (EOL), add artful & bionic support

[Junien F (axino)]

+1

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision f2747e0ca46b6acd170929458192bf8f6538a19b

diff --git a/bin/charm_helpers_sync.py b/bin/charm_helpers_sync.py
index f67fdb9..e3f0e74 100644
--- a/bin/charm_helpers_sync.py
+++ b/bin/charm_helpers_sync.py
@@ -2,19 +2,17 @@
 
 # Copyright 2014-2015 Canonical Limited.
 #
-# This file is part of charm-helpers.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# charm-helpers is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License version 3 as
-# published by the Free Software Foundation.
+#  http://www.apache.org/licenses/LICENSE-2.0
 #
-# charm-helpers is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 # Authors:
 #   Adam Gandelman <[e-mail]>
@@ -31,7 +29,7 @@ from fnmatch import fnmatch
 
 import six
 
-CHARM_HELPERS_BRANCH = 'lp:charm-helpers'
+CHARM_HELPERS_REPO = 'https://github.com/juju/charm-helpers'
 
 
 def parse_config(conf_file):
@@ -41,10 +39,16 @@ def parse_config(conf_file):
     return yaml.load(open(conf_file).read())
 
 
-def clone_helpers(work_dir, branch):
+def clone_helpers(work_dir, repo):
     dest = os.path.join(work_dir, 'charm-helpers')
-    logging.info('Checking out %s to %s.' % (branch, dest))
-    cmd = ['bzr', 'checkout', '--lightweight', branch, dest]
+    logging.info('Cloning out %s to %s.' % (repo, dest))
+    branch = None
+    if '@' in repo:
+        repo, branch = repo.split('@', 1)
+    cmd = ['git', 'clone', '--depth=1']
+    if branch is not None:
+        cmd += ['--branch', branch]
+    cmd += [repo, dest]
     subprocess.check_call(cmd)
     return dest
 
@@ -193,14 +197,15 @@ def sync_helpers(include, src, dest, options=None):
                         inc, opts = extract_options(m, global_options)
                         sync(src, dest, '%s.%s' % (k, inc), opts)
 
+
 if __name__ == '__main__':
     parser = optparse.OptionParser()
     parser.add_option('-c', '--config', action='store', dest='config',
                       default=None, help='helper config file')
     parser.add_option('-D', '--debug', action='store_true', dest='debug',
                       default=False, help='debug')
-    parser.add_option('-b', '--branch', action='store', dest='branch',
-                      help='charm-helpers bzr branch (overrides config)')
+    parser.add_option('-r', '--repository', action='store', dest='repo',
+                      help='charm-helpers git repository (overrides config)')
     parser.add_option('-d', '--destination', action='store', dest='dest_dir',
                       help='sync destination dir (overrides config)')
     (opts, args) = parser.parse_args()
@@ -219,10 +224,10 @@ if __name__ == '__main__':
     else:
         config = {}
 
-    if 'branch' not in config:
-        config['branch'] = CHARM_HELPERS_BRANCH
-    if opts.branch:
-        config['branch'] = opts.branch
+    if 'repo' not in config:
+        config['repo'] = CHARM_HELPERS_REPO
+    if opts.repo:
+        config['repo'] = opts.repo
     if opts.dest_dir:
         config['destination'] = opts.dest_dir
 
@@ -242,7 +247,7 @@ if __name__ == '__main__':
         sync_options = config['options']
     tmpd = tempfile.mkdtemp()
     try:
-        checkout = clone_helpers(tmpd, config['branch'])
+        checkout = clone_helpers(tmpd, config['repo'])
         sync_helpers(config['include'], checkout, config['destination'],
                      options=sync_options)
     except Exception as e:
diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py
index c7feeaf..7ed1cc4 100644
--- a/hooks/charmhelpers/core/hookenv.py
+++ b/hooks/charmhelpers/core/hookenv.py
@@ -22,6 +22,7 @@ from __future__ import print_function
 import copy
 from distutils.version import LooseVersion
 from functools import wraps
+from collections import namedtuple
 import glob
 import os
 import json
@@ -38,6 +39,7 @@ if not six.PY3:
 else:
     from collections import UserDict
 
+
 CRITICAL = "CRITICAL"
 ERROR = "ERROR"
 WARNING = "WARNING"
@@ -343,6 +345,7 @@ class Config(dict):
 
         """
         with open(self.path, 'w') as f:
+            os.fchmod(f.fileno(), 0o600)
             json.dump(self, f)
 
     def _implicit_save(self):
@@ -654,7 +657,7 @@ def _port_op(op_name, port, protocol="TCP"):
         _args.append('{}/{}'.format(port, protocol))
     try:
         subprocess.check_call(_args)
-    except:
+    except subprocess.CalledProcessError:
         # Older Juju pre 2.3 doesn't support ICMP
         # so treat it as a no-op if it fails.
         if not icmp:
@@ -685,6 +688,17 @@ def close_ports(start, end, protocol="TCP"):
     subprocess.check_call(_args)
 
 
+def opened_ports():
+    """Get the opened ports
+
+    *Note that this will only show ports opened in a previous hook*
+
+    :returns: Opened ports as a list of strings: ``['8080/tcp', '8081-8083/tcp']``
+    """
+    _args = ['opened-ports', '--format=json']
+    return json.loads(subprocess.check_output(_args).decode('UTF-8'))
+
+
 @cached
 def unit_get(attribute):
     """Get the unit ID for the remote unit"""
@@ -806,6 +820,10 @@ class Hooks(object):
         return wrapper
 
 
+class NoNetworkBinding(Exception):
+    pass
+
+
 def charm_dir():
     """Return the root directory of the current charm"""
     d = os.environ.get('JUJU_CHARM_DIR')
@@ -1092,7 +1110,17 @@ def network_get_primary_address(binding):
     :raise: NotImplementedError if run on Juju < 2.0
     '''
     cmd = ['network-get', '--primary-address', binding]
-    return subprocess.check_output(cmd).decode('UTF-8').strip()
+    try:
+        response = subprocess.check_output(
+            cmd,
+            stderr=subprocess.STDOUT).decode('UTF-8').strip()
+    except CalledProcessError as e:
+        if 'no network config found for binding' in e.output.decode('UTF-8'):
+            raise NoNetworkBinding("No network binding for {}"
+                                   .format(binding))
+        else:
+            raise
+    return response
 
 
 @translate_exc(from_exc=OSError, to_exc=NotImplementedError)
@@ -1153,3 +1181,42 @@ def meter_info():
     """Get the meter status information, if running in the meter-status-changed
     hook."""
     return os.environ.get('JUJU_METER_INFO')
+
+
+def iter_units_for_relation_name(relation_name):
+    """Iterate through all units in a relation
+
+    Generator that iterates through all the units in a relation and yields
+    a named tuple with rid and unit field names.
+
+    Usage:
+    data = [(u.rid, u.unit)
+            for u in iter_units_for_relation_name(relation_name)]
+
+    :param relation_name: string relation name
+    :yield: Named Tuple with rid and unit field names
+    """
+    RelatedUnit = namedtuple('RelatedUnit', 'rid, unit')
+    for rid in relation_ids(relation_name):
+        for unit in related_units(rid):
+            yield RelatedUnit(rid, unit)
+
+
+def ingress_address(rid=None, unit=None):
+    """
+    Retrieve the ingress-address from a relation when available. Otherwise,
+    return the private-address. This function is to be used on the consuming
+    side of the relation.
+
+    Usage:
+    addresses = [ingress_address(rid=u.rid, unit=u.unit)
+                 for u in iter_units_for_relation_name(relation_name)]
+
+    :param rid: string relation id
+    :param unit: string unit name
+    :side effect: calls relation_get
+    :return: string IP address
+    """
+    settings = relation_get(rid=rid, unit=unit)
+    return (settings.get('ingress-address') or
+            settings.get('private-address'))
diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py
index 5656e2f..fd14d60 100644
--- a/hooks/charmhelpers/core/host.py
+++ b/hooks/charmhelpers/core/host.py
@@ -34,7 +34,7 @@ import six
 
 from contextlib import contextmanager
 from collections import OrderedDict
-from .hookenv import log, DEBUG
+from .hookenv import log, DEBUG, local_unit
 from .fstab import Fstab
 from charmhelpers.osplatform import get_platform
 
@@ -441,6 +441,49 @@ def add_user_to_group(username, group):
     subprocess.check_call(cmd)
 
 
+def chage(username, lastday=None, expiredate=None, inactive=None,
+           mindays=None, maxdays=None, root=None, warndays=None):
+    """Change user password expiry information
+
+    :param str username: User to update
+    :param str lastday: Set when password was changed in YYYY-MM-DD format
+    :param str expiredate: Set when user's account will no longer be
+                           accessible in YYYY-MM-DD format.
+                           -1 will remove an account expiration date.
+    :param str inactive: Set the number of days of inactivity after a password
+                         has expired before the account is locked.
+                         -1 will remove an account's inactivity.
+    :param str mindays: Set the minimum number of days between password
+                        changes to MIN_DAYS.
+                        0 indicates the password can be changed anytime.
+    :param str maxdays: Set the maximum number of days during which a
+                        password is valid.
+                        -1 as MAX_DAYS will remove checking maxdays
+    :param str root: Apply changes in the CHROOT_DIR directory
+    :param str warndays: Set the number of days of warning before a password
+                         change is required
+    :raises subprocess.CalledProcessError: if call to chage fails
+    """
+    cmd = ['chage']
+    if root:
+        cmd.extend(['--root', root])
+    if lastday:
+        cmd.extend(['--lastday', lastday])
+    if expiredate:
+        cmd.extend(['--expiredate', expiredate])
+    if inactive:
+        cmd.extend(['--inactive', inactive])
+    if mindays:
+        cmd.extend(['--mindays', mindays])
+    if maxdays:
+        cmd.extend(['--maxdays', maxdays])
+    if warndays:
+        cmd.extend(['--warndays', warndays])
+    cmd.append(username)
+    subprocess.check_call(cmd)
+
+remove_password_expiry = functools.partial(chage, expiredate='-1', inactive='-1', mindays='0', maxdays='-1')
+
 def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
     """Replicate the contents of a path"""
     options = options or ['--delete', '--executability']
@@ -506,6 +549,8 @@ def write_file(path, content, owner='root', group='root', perms=0o444):
         with open(path, 'wb') as target:
             os.fchown(target.fileno(), uid, gid)
             os.fchmod(target.fileno(), perms)
+            if six.PY3 and isinstance(content, six.string_types):
+                content = content.encode('UTF-8')
             target.write(content)
         return
     # the contents were the same, but we might still need to change the
@@ -946,3 +991,31 @@ def updatedb(updatedb_text, new_path):
                 lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths))
     output = "\n".join(lines)
     return output
+
+
+def modulo_distribution(modulo=3, wait=30):
+    """ Modulo distribution
+
+    This helper uses the unit number, a modulo value and a constant wait time
+    to produce a calculated wait time distribution. This is useful in large
+    scale deployments to distribute load during an expensive operation such as
+    service restarts.
+
+    If you have 1000 nodes that need to restart 100 at a time 1 minute at a
+    time:
+
+      time.wait(modulo_distribution(modulo=100, wait=60))
+      restart()
+
+    If you need restarts to happen serially set modulo to the exact number of
+    nodes and set a high constant wait time:
+
+      time.wait(modulo_distribution(modulo=10, wait=120))
+      restart()
+
+    @param modulo: int The modulo number creates the group distribution
+    @param wait: int The constant time wait value
+    @return: int Calculated time to wait for unit operation
+    """
+    unit_number = int(local_unit().split('/')[1])
+    return (unit_number % modulo) * wait
diff --git a/hooks/charmhelpers/core/host_factory/ubuntu.py b/hooks/charmhelpers/core/host_factory/ubuntu.py
index d8dc378..99451b5 100644
--- a/hooks/charmhelpers/core/host_factory/ubuntu.py
+++ b/hooks/charmhelpers/core/host_factory/ubuntu.py
@@ -20,6 +20,7 @@ UBUNTU_RELEASES = (
     'yakkety',
     'zesty',
     'artful',
+    'bionic',
 )
 
 
diff --git a/hooks/charmhelpers/core/services/base.py b/hooks/charmhelpers/core/services/base.py
index 345b60d..ca9dc99 100644
--- a/hooks/charmhelpers/core/services/base.py
+++ b/hooks/charmhelpers/core/services/base.py
@@ -313,26 +313,17 @@ class PortManagerCallback(ManagerCallback):
             with open(port_file) as fp:
                 old_ports = fp.read().split(',')
             for old_port in old_ports:
-                if bool(old_port) and not self.ports_contains(old_port, new_ports):
-                    hookenv.close_port(old_port)
+                if bool(old_port):
+                    old_port = int(old_port)
+                    if old_port not in new_ports:
+                        hookenv.close_port(old_port)
         with open(port_file, 'w') as fp:
             fp.write(','.join(str(port) for port in new_ports))
         for port in new_ports:
-            # A port is either a number or 'ICMP'
-            protocol = 'TCP'
-            if str(port).upper() == 'ICMP':
-                protocol = 'ICMP'
             if event_name == 'start':
-                hookenv.open_port(port, protocol)
+                hookenv.open_port(port)
             elif event_name == 'stop':
-                hookenv.close_port(port, protocol)
-
-    def ports_contains(self, port, ports):
-        if not bool(port):
-            return False
-        if str(port).upper() != 'ICMP':
-            port = int(port)
-        return port in ports
+                hookenv.close_port(port)
 
 
 def service_stop(service_name):
diff --git a/hooks/charmhelpers/core/strutils.py b/hooks/charmhelpers/core/strutils.py
index 685dabd..e8df045 100644
--- a/hooks/charmhelpers/core/strutils.py
+++ b/hooks/charmhelpers/core/strutils.py
@@ -61,13 +61,19 @@ def bytes_from_string(value):
     if isinstance(value, six.string_types):
         value = six.text_type(value)
     else:
-        msg = "Unable to interpret non-string value '%s' as boolean" % (value)
+        msg = "Unable to interpret non-string value '%s' as bytes" % (value)
         raise ValueError(msg)
     matches = re.match("([0-9]+)([a-zA-Z]+)", value)
-    if not matches:
-        msg = "Unable to interpret string value '%s' as bytes" % (value)
-        raise ValueError(msg)
-    return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
+    if matches:
+        size = int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
+    else:
+        # Assume that value passed in is bytes
+        try:
+            size = int(value)
+        except ValueError:
+            msg = "Unable to interpret string value '%s' as bytes" % (value)
+            raise ValueError(msg)
+    return size
 
 
 class BasicStringComparator(object):
diff --git a/hooks/charmhelpers/core/templating.py b/hooks/charmhelpers/core/templating.py
index 7b801a3..9014015 100644
--- a/hooks/charmhelpers/core/templating.py
+++ b/hooks/charmhelpers/core/templating.py
@@ -20,7 +20,8 @@ from charmhelpers.core import hookenv
 
 
 def render(source, target, context, owner='root', group='root',
-           perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None):
+           perms=0o444, templates_dir=None, encoding='UTF-8',
+           template_loader=None, config_template=None):
     """
     Render a template.
 
@@ -32,6 +33,9 @@ def render(source, target, context, owner='root', group='root',
     The context should be a dict containing the values to be replaced in the
     template.
 
+    config_template may be provided to render from a provided template instead
+    of loading from a file.
+
     The `owner`, `group`, and `perms` options will be passed to `write_file`.
 
     If omitted, `templates_dir` defaults to the `templates` folder in the charm.
@@ -65,14 +69,19 @@ def render(source, target, context, owner='root', group='root',
         if templates_dir is None:
             templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
         template_env = Environment(loader=FileSystemLoader(templates_dir))
-    try:
-        source = source
-        template = template_env.get_template(source)
-    except exceptions.TemplateNotFound as e:
-        hookenv.log('Could not load template %s from %s.' %
-                    (source, templates_dir),
-                    level=hookenv.ERROR)
-        raise e
+
+    # load from a string if provided explicitly
+    if config_template is not None:
+        template = template_env.from_string(config_template)
+    else:
+        try:
+            source = source
+            template = template_env.get_template(source)
+        except exceptions.TemplateNotFound as e:
+            hookenv.log('Could not load template %s from %s.' %
+                        (source, templates_dir),
+                        level=hookenv.ERROR)
+            raise e
     content = template.render(context)
     if target is not None:
         target_dir = os.path.dirname(target)
diff --git a/hooks/charmhelpers/core/unitdata.py b/hooks/charmhelpers/core/unitdata.py
index 54ec969..6d7b494 100644
--- a/hooks/charmhelpers/core/unitdata.py
+++ b/hooks/charmhelpers/core/unitdata.py
@@ -175,6 +175,8 @@ class Storage(object):
             else:
                 self.db_path = os.path.join(
                     os.environ.get('CHARM_DIR', ''), '.unit-state.db')
+        with open(self.db_path, 'a') as f:
+            os.fchmod(f.fileno(), 0o600)
         self.conn = sqlite3.connect('%s' % self.db_path)
         self.cursor = self.conn.cursor()
         self.revision = None
@@ -358,7 +360,7 @@ class Storage(object):
         try:
             yield self.revision
             self.revision = None
-        except:
+        except Exception:
             self.flush(False)
             self.revision = None
             raise
diff --git a/hooks/charmhelpers/fetch/snap.py b/hooks/charmhelpers/fetch/snap.py
index 112a54c..395836c 100644
--- a/hooks/charmhelpers/fetch/snap.py
+++ b/hooks/charmhelpers/fetch/snap.py
@@ -41,6 +41,10 @@ class CouldNotAcquireLockException(Exception):
     pass
 
 
+class InvalidSnapChannel(Exception):
+    pass
+
+
 def _snap_exec(commands):
     """
     Execute snap commands.
@@ -132,3 +136,15 @@ def snap_refresh(packages, *flags):
 
     log(message, level='INFO')
     return _snap_exec(['refresh'] + flags + packages)
+
+
+def valid_snap_channel(channel):
+    """ Validate snap channel exists
+
+    :raises InvalidSnapChannel: When channel does not exist
+    :return: Boolean
+    """
+    if channel.lower() in SNAP_CHANNELS:
+        return True
+    else:
+        raise InvalidSnapChannel("Invalid Snap Channel: {}".format(channel))
diff --git a/hooks/charmhelpers/fetch/ubuntu.py b/hooks/charmhelpers/fetch/ubuntu.py
index 40e1cb5..910e96a 100644
--- a/hooks/charmhelpers/fetch/ubuntu.py
+++ b/hooks/charmhelpers/fetch/ubuntu.py
@@ -572,7 +572,7 @@ def get_upstream_version(package):
     cache = apt_cache()
     try:
         pkg = cache[package]
-    except:
+    except Exception:
         # the package is unknown to the current apt cache.
         return None
 
diff --git a/metadata.yaml b/metadata.yaml
index d0f196f..ace705f 100644
--- a/metadata.yaml
+++ b/metadata.yaml
@@ -29,6 +29,5 @@ requires:
 series:
   - xenial
   - trusty
-  - zesty
-  - yakkety
   - artful
+  - bionic