Merge lp:~tribaal/charms/trusty/ceph/sync-charm-helpers into lp:~openstack-charmers-archive/charms/trusty/ceph/trunk

[Chris Glass]

This branch syncs back the changes from charm-helpers as of http://bazaar.launchpad.net/~charm-helpers/charm-helpers/devel/revision/153?start_revid=153 into the charm.

Furthermore, the charm-helpers "is_device_mounted" function is used to prevent using mounted disks.

=== modified file 'hooks/ceph.py'
--- hooks/ceph.py	2013-09-23 10:19:06 +0000
+++ hooks/ceph.py	2014-05-19 12:52:02 +0000
@@ -23,6 +23,7 @@
 from charmhelpers.contrib.storage.linux.utils import (
     zap_disk,
     is_block_device,
+    is_device_mounted
 )
 from utils import (
     get_unit_hostname,
@@ -353,7 +354,7 @@
         log('Looks like {} is already an OSD, skipping.'.format(dev))
         return
 
-    if device_mounted(dev):
+    if is_device_mounted(dev):
         log('Looks like {} is in use, skipping.'.format(dev))
         return
 
@@ -396,9 +397,5 @@
     subprocess.check_call(cmd)
 
 
-def device_mounted(dev):
-    return subprocess.call(['grep', '-wqs', dev + '1', '/proc/mounts']) == 0
-
-
 def filesystem_mounted(fs):
     return subprocess.call(['grep', '-wqs', fs, '/proc/mounts']) == 0

=== modified file 'hooks/charmhelpers/contrib/storage/linux/utils.py'
--- hooks/charmhelpers/contrib/storage/linux/utils.py	2014-04-16 08:14:54 +0000
+++ hooks/charmhelpers/contrib/storage/linux/utils.py	2014-05-19 12:52:02 +0000
@@ -1,4 +1,5 @@
-from os import stat
+import os
+import re
 from stat import S_ISBLK
 
 from subprocess import (
@@ -14,7 +15,9 @@
 
     :returns: boolean: True if path is a block device, False if not.
     '''
-    return S_ISBLK(stat(path).st_mode)
+    if not os.path.exists(path):
+        return False
+    return S_ISBLK(os.stat(path).st_mode)
 
 
 def zap_disk(block_device):
@@ -29,7 +32,18 @@
           '--clear', block_device])
     dev_end = check_output(['blockdev', '--getsz', block_device])
     gpt_end = int(dev_end.split()[0]) - 100
-    check_call(['dd', 'if=/dev/zero', 'of=%s'%(block_device),
+    check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
                 'bs=1M', 'count=1'])
-    check_call(['dd', 'if=/dev/zero', 'of=%s'%(block_device),
-                'bs=512', 'count=100', 'seek=%s'%(gpt_end)])
+    check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
+                'bs=512', 'count=100', 'seek=%s' % (gpt_end)])
+
+def is_device_mounted(device):
+    '''Given a device path, return True if that device is mounted, and False
+    if it isn't.
+
+    :param device: str: Full path of the device to check.
+    :returns: boolean: True if the path represents a mounted device, False if
+        it doesn't.
+    '''
+    out = check_output(['mount'])
+    return bool(re.search(device + r"[0-9]+\b", out))

=== modified file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py	2014-04-16 08:14:54 +0000
+++ hooks/charmhelpers/core/hookenv.py	2014-05-19 12:52:02 +0000
@@ -155,6 +155,100 @@
     return os.path.basename(sys.argv[0])
 
 
+class Config(dict):
+    """A Juju charm config dictionary that can write itself to
+    disk (as json) and track which values have changed since
+    the previous hook invocation.
+
+    Do not instantiate this object directly - instead call
+    ``hookenv.config()``
+
+    Example usage::
+
+        >>> # inside a hook
+        >>> from charmhelpers.core import hookenv
+        >>> config = hookenv.config()
+        >>> config['foo']
+        'bar'
+        >>> config['mykey'] = 'myval'
+        >>> config.save()
+
+
+        >>> # user runs `juju set mycharm foo=baz`
+        >>> # now we're inside subsequent config-changed hook
+        >>> config = hookenv.config()
+        >>> config['foo']
+        'baz'
+        >>> # test to see if this val has changed since last hook
+        >>> config.changed('foo')
+        True
+        >>> # what was the previous value?
+        >>> config.previous('foo')
+        'bar'
+        >>> # keys/values that we add are preserved across hooks
+        >>> config['mykey']
+        'myval'
+        >>> # don't forget to save at the end of hook!
+        >>> config.save()
+
+    """
+    CONFIG_FILE_NAME = '.juju-persistent-config'
+
+    def __init__(self, *args, **kw):
+        super(Config, self).__init__(*args, **kw)
+        self._prev_dict = None
+        self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
+        if os.path.exists(self.path):
+            self.load_previous()
+
+    def load_previous(self, path=None):
+        """Load previous copy of config from disk so that current values
+        can be compared to previous values.
+
+        :param path:
+
+            File path from which to load the previous config. If `None`,
+            config is loaded from the default location. If `path` is
+            specified, subsequent `save()` calls will write to the same
+            path.
+
+        """
+        self.path = path or self.path
+        with open(self.path) as f:
+            self._prev_dict = json.load(f)
+
+    def changed(self, key):
+        """Return true if the value for this key has changed since
+        the last save.
+
+        """
+        if self._prev_dict is None:
+            return True
+        return self.previous(key) != self.get(key)
+
+    def previous(self, key):
+        """Return previous value for this key, or None if there
+        is no "previous" value.
+
+        """
+        if self._prev_dict:
+            return self._prev_dict.get(key)
+        return None
+
+    def save(self):
+        """Save this config to disk.
+
+        Preserves items in _prev_dict that do not exist in self.
+
+        """
+        if self._prev_dict:
+            for k, v in self._prev_dict.iteritems():
+                if k not in self:
+                    self[k] = v
+        with open(self.path, 'w') as f:
+            json.dump(self, f)
+
+
 @cached
 def config(scope=None):
     """Juju charm configuration"""
@@ -163,7 +257,10 @@
         config_cmd_line.append(scope)
     config_cmd_line.append('--format=json')
     try:
-        return json.loads(subprocess.check_output(config_cmd_line))
+        config_data = json.loads(subprocess.check_output(config_cmd_line))
+        if scope is not None:
+            return config_data
+        return Config(config_data)
     except ValueError:
         return None
 

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2014-04-16 08:14:54 +0000
+++ hooks/charmhelpers/core/host.py	2014-05-19 12:52:02 +0000
@@ -12,6 +12,7 @@
 import string
 import subprocess
 import hashlib
+import apt_pkg
 
 from collections import OrderedDict
 
@@ -295,3 +296,16 @@
     if 'link/ether' in words:
         hwaddr = words[words.index('link/ether') + 1]
     return hwaddr
+
+
+def cmp_pkgrevno(package, revno, pkgcache=None):
+    '''Compare supplied revno with the revno of the installed package
+       1 => Installed revno is greater than supplied arg
+       0 => Installed revno is the same as supplied arg
+      -1 => Installed revno is less than supplied arg
+    '''
+    if not pkgcache:
+        apt_pkg.init()
+        pkgcache = apt_pkg.Cache()
+    pkg = pkgcache[package]
+    return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)

=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2014-04-24 17:28:21 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2014-05-19 12:52:02 +0000
@@ -1,4 +1,5 @@
 import importlib
+import time
 from yaml import safe_load
 from charmhelpers.core.host import (
     lsb_release
@@ -15,6 +16,7 @@
 import apt_pkg
 import os
 
+
 CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
 deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
 """
@@ -56,10 +58,62 @@
     'precise-proposed/icehouse': 'precise-proposed/icehouse',
 }
 
+# The order of this list is very important. Handlers should be listed in from
+# least- to most-specific URL matching.
+FETCH_HANDLERS = (
+    'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
+    'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
+)
+
+APT_NO_LOCK = 100  # The return code for "couldn't acquire lock" in APT.
+APT_NO_LOCK_RETRY_DELAY = 10  # Wait 10 seconds between apt lock checks.
+APT_NO_LOCK_RETRY_COUNT = 30  # Retry to acquire the lock X times.
+
+
+class SourceConfigError(Exception):
+    pass
+
+
+class UnhandledSource(Exception):
+    pass
+
+
+class AptLockError(Exception):
+    pass
+
+
+class BaseFetchHandler(object):
+
+    """Base class for FetchHandler implementations in fetch plugins"""
+
+    def can_handle(self, source):
+        """Returns True if the source can be handled. Otherwise returns
+        a string explaining why it cannot"""
+        return "Wrong source type"
+
+    def install(self, source):
+        """Try to download and unpack the source. Return the path to the
+        unpacked files or raise UnhandledSource."""
+        raise UnhandledSource("Wrong source type {}".format(source))
+
+    def parse_url(self, url):
+        return urlparse(url)
+
+    def base_url(self, url):
+        """Return url without querystring or fragment"""
+        parts = list(self.parse_url(url))
+        parts[4:] = ['' for i in parts[4:]]
+        return urlunparse(parts)
+
 
 def filter_installed_packages(packages):
     """Returns a list of packages that require installation"""
     apt_pkg.init()
+
+    # Tell apt to build an in-memory cache to prevent race conditions (if
+    # another process is already building the cache).
+    apt_pkg.config.set("Dir::Cache::pkgcache", "")
+
     cache = apt_pkg.Cache()
     _pkgs = []
     for package in packages:
@@ -87,14 +141,7 @@
         cmd.extend(packages)
     log("Installing {} with options: {}".format(packages,
                                                 options))
-    env = os.environ.copy()
-    if 'DEBIAN_FRONTEND' not in env:
-        env['DEBIAN_FRONTEND'] = 'noninteractive'
-
-    if fatal:
-        subprocess.check_call(cmd, env=env)
-    else:
-        subprocess.call(cmd, env=env)
+    _run_apt_command(cmd, fatal)
 
 
 def apt_upgrade(options=None, fatal=False, dist=False):
@@ -109,24 +156,13 @@
     else:
         cmd.append('upgrade')
     log("Upgrading with options: {}".format(options))
-
-    env = os.environ.copy()
-    if 'DEBIAN_FRONTEND' not in env:
-        env['DEBIAN_FRONTEND'] = 'noninteractive'
-
-    if fatal:
-        subprocess.check_call(cmd, env=env)
-    else:
-        subprocess.call(cmd, env=env)
+    _run_apt_command(cmd, fatal)
 
 
 def apt_update(fatal=False):
     """Update local apt cache"""
     cmd = ['apt-get', 'update']
-    if fatal:
-        subprocess.check_call(cmd)
-    else:
-        subprocess.call(cmd)
+    _run_apt_command(cmd, fatal)
 
 
 def apt_purge(packages, fatal=False):
@@ -137,10 +173,7 @@
     else:
         cmd.extend(packages)
     log("Purging {}".format(packages))
-    if fatal:
-        subprocess.check_call(cmd)
-    else:
-        subprocess.call(cmd)
+    _run_apt_command(cmd, fatal)
 
 
 def apt_hold(packages, fatal=False):
@@ -151,6 +184,7 @@
     else:
         cmd.extend(packages)
     log("Holding {}".format(packages))
+
     if fatal:
         subprocess.check_call(cmd)
     else:
@@ -188,10 +222,6 @@
                                key])
 
 
-class SourceConfigError(Exception):
-    pass
-
-
 def configure_sources(update=False,
                       sources_var='install_sources',
                       keys_var='install_keys'):
@@ -224,17 +254,6 @@
     if update:
         apt_update(fatal=True)
 
-# The order of this list is very important. Handlers should be listed in from
-# least- to most-specific URL matching.
-FETCH_HANDLERS = (
-    'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
-    'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
-)
-
-
-class UnhandledSource(Exception):
-    pass
-
 
 def install_remote(source):
     """
@@ -265,30 +284,6 @@
     return install_remote(source)
 
 
-class BaseFetchHandler(object):
-
-    """Base class for FetchHandler implementations in fetch plugins"""
-
-    def can_handle(self, source):
-        """Returns True if the source can be handled. Otherwise returns
-        a string explaining why it cannot"""
-        return "Wrong source type"
-
-    def install(self, source):
-        """Try to download and unpack the source. Return the path to the
-        unpacked files or raise UnhandledSource."""
-        raise UnhandledSource("Wrong source type {}".format(source))
-
-    def parse_url(self, url):
-        return urlparse(url)
-
-    def base_url(self, url):
-        """Return url without querystring or fragment"""
-        parts = list(self.parse_url(url))
-        parts[4:] = ['' for i in parts[4:]]
-        return urlunparse(parts)
-
-
 def plugins(fetch_handlers=None):
     if not fetch_handlers:
         fetch_handlers = FETCH_HANDLERS
@@ -306,3 +301,40 @@
             log("FetchHandler {} not found, skipping plugin".format(
                 handler_name))
     return plugin_list
+
+
+def _run_apt_command(cmd, fatal=False):
+    """
+    Run an APT command, checking output and retrying if the fatal flag is set
+    to True.
+
+    :param: cmd: str: The apt command to run.
+    :param: fatal: bool: Whether the command's output should be checked and
+        retried.
+    """
+    env = os.environ.copy()
+
+    if 'DEBIAN_FRONTEND' not in env:
+        env['DEBIAN_FRONTEND'] = 'noninteractive'
+
+    if fatal:
+        retry_count = 0
+        result = None
+
+        # If the command is considered "fatal", we need to retry if the apt
+        # lock was not acquired.
+
+        while result is None or result == APT_NO_LOCK:
+            try:
+                result = subprocess.check_call(cmd, env=env)
+            except subprocess.CalledProcessError, e:
+                retry_count = retry_count + 1
+                if retry_count > APT_NO_LOCK_RETRY_COUNT:
+                    raise
+                result = e.returncode
+                log("Couldn't acquire DPKG lock. Will retry in {} seconds."
+                    "".format(APT_NO_LOCK_RETRY_DELAY))
+                time.sleep(APT_NO_LOCK_RETRY_DELAY)
+
+    else:
+        subprocess.call(cmd, env=env)