Merge lp:~tribaal/charms/trusty/ceph/backport-apt-cache-fixes into lp:~openstack-charmers-archive/charms/trusty/ceph/trunk

[Chris Glass]

This branch backports changes fixing https://bugs.launchpad.net/charms/+source/ceph/+bug/1346489 into the "main" branch from the "next" branch.

It also refreshes charm-helpers.

[James Page (james-page)]

I think this or an equiv was already been merged.

review: Approve

=== added file '.bzrignore'
--- .bzrignore	1970-01-01 00:00:00 +0000
+++ .bzrignore	2014-09-26 07:27:46 +0000
@@ -0,0 +1,1 @@
+bin/

=== modified file 'Makefile'
--- Makefile	2014-05-21 10:09:21 +0000
+++ Makefile	2014-09-26 07:27:46 +0000
@@ -1,11 +1,17 @@
 #!/usr/bin/make
+PYTHON := /usr/bin/env python
 
 lint:
 	@flake8 --exclude hooks/charmhelpers hooks
 	@charm proof
 
-sync:
-	@charm-helper-sync -c charm-helpers-sync.yaml
+bin/charm_helpers_sync.py:
+	@mkdir -p bin
+	@bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \
+        > bin/charm_helpers_sync.py
+
+sync: bin/charm_helpers_sync.py
+	$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-sync.yaml
 
 publish: lint
 	bzr push lp:charms/ceph

=== modified file 'hooks/ceph.py'
--- hooks/ceph.py	2014-05-19 12:15:38 +0000
+++ hooks/ceph.py	2014-09-26 07:27:46 +0000
@@ -11,19 +11,20 @@
 import subprocess
 import time
 import os
-import apt_pkg as apt
 from charmhelpers.core.host import (
     mkdir,
     service_restart,
+    cmp_pkgrevno,
 )
 from charmhelpers.core.hookenv import (
     log,
     ERROR,
+    WARNING,
 )
 from charmhelpers.contrib.storage.linux.utils import (
     zap_disk,
     is_block_device,
-    is_device_mounted
+    is_device_mounted,
 )
 from utils import (
     get_unit_hostname,
@@ -126,7 +127,7 @@
 def start_osds(devices):
     # Scan for ceph block devices
     rescan_osd_devices()
-    if get_ceph_version() >= "0.56.6":
+    if cmp_pkgrevno('ceph', "0.56.6") >= 0:
         # Use ceph-disk-activate for directory based OSD's
         for dev_or_path in devices:
             if os.path.exists(dev_or_path) and os.path.isdir(dev_or_path):
@@ -284,7 +285,7 @@
         log('bootstrap_monitor_cluster: mon already initialized.')
     else:
         # Ceph >= 0.61.3 needs this for ceph-mon fs creation
-        mkdir('/var/run/ceph', perms=0755)
+        mkdir('/var/run/ceph', perms=0o755)
         mkdir(path)
         # end changes for Ceph >= 0.61.3
         try:
@@ -309,20 +310,6 @@
             os.unlink(keyring)
 
 
-def get_ceph_version():
-    apt.init()
-    cache = apt.Cache()
-    pkg = cache['ceph']
-    if pkg.current_ver:
-        return apt.upstream_version(pkg.current_ver.ver_str)
-    else:
-        return None
-
-
-def version_compare(a, b):
-    return apt.version_compare(a, b)
-
-
 def update_monfs():
     hostname = get_unit_hostname()
     monfs = '/var/lib/ceph/mon/ceph-{}'.format(hostname)
@@ -334,14 +321,16 @@
             pass
 
 
-def osdize(dev, osd_format, osd_journal, reformat_osd=False):
+def osdize(dev, osd_format, osd_journal, reformat_osd=False,
+           ignore_errors=False):
     if dev.startswith('/dev'):
-        osdize_dev(dev, osd_format, osd_journal, reformat_osd)
+        osdize_dev(dev, osd_format, osd_journal, reformat_osd, ignore_errors)
     else:
         osdize_dir(dev)
 
 
-def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False):
+def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False,
+               ignore_errors=False):
     if not os.path.exists(dev):
         log('Path {} does not exist - bailing'.format(dev))
         return
@@ -360,10 +349,12 @@
 
     cmd = ['ceph-disk-prepare']
     # Later versions of ceph support more options
-    if get_ceph_version() >= "0.48.3":
+    if cmp_pkgrevno('ceph', '0.48.3') >= 0:
         if osd_format:
             cmd.append('--fs-type')
             cmd.append(osd_format)
+        if reformat_osd:
+            cmd.append('--zap-disk')
         cmd.append(dev)
         if osd_journal and os.path.exists(osd_journal):
             cmd.append(osd_journal)
@@ -371,11 +362,17 @@
         # Just provide the device - no other options
         # for older versions of ceph
         cmd.append(dev)
-
-    if reformat_osd:
-        zap_disk(dev)
-
-    subprocess.check_call(cmd)
+        if reformat_osd:
+            zap_disk(dev)
+
+    try:
+        subprocess.check_call(cmd)
+    except subprocess.CalledProcessError as e:
+        if ignore_errors:
+            log('Unable to initialize device: {}'.format(dev), WARNING)
+        else:
+            log('Unable to initialize device: {}'.format(dev), ERROR)
+            raise e
 
 
 def osdize_dir(path):
@@ -383,7 +380,7 @@
         log('Path {} is already configured as an OSD - bailing'.format(path))
         return
 
-    if get_ceph_version() < "0.56.6":
+    if cmp_pkgrevno('ceph', "0.56.6") < 0:
         log('Unable to use directories for OSDs with ceph < 0.56.6',
             level=ERROR)
         raise

=== modified file 'hooks/charmhelpers/contrib/storage/linux/utils.py'
--- hooks/charmhelpers/contrib/storage/linux/utils.py	2014-05-19 11:38:45 +0000
+++ hooks/charmhelpers/contrib/storage/linux/utils.py	2014-09-26 07:27:46 +0000
@@ -37,6 +37,7 @@
     check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
                 'bs=512', 'count=100', 'seek=%s' % (gpt_end)])
 
+
 def is_device_mounted(device):
     '''Given a device path, return True if that device is mounted, and False
     if it isn't.
@@ -45,5 +46,8 @@
     :returns: boolean: True if the path represents a mounted device, False if
         it doesn't.
     '''
+    is_partition = bool(re.search(r".*[0-9]+\b", device))
     out = check_output(['mount'])
+    if is_partition:
+        return bool(re.search(device + r"\b", out))
     return bool(re.search(device + r"[0-9]+\b", out))

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2014-07-24 09:59:35 +0000
+++ hooks/charmhelpers/core/host.py	2014-09-26 07:27:46 +0000
@@ -318,12 +318,8 @@
        0 => Installed revno is the same as supplied arg
       -1 => Installed revno is less than supplied arg
     '''
+    from charmhelpers.fetch import apt_cache
     if not pkgcache:
-        apt_pkg.init()
-        # Force Apt to build its cache in memory. That way we avoid race
-        # conditions with other applications building the cache in the same
-        # place.
-        apt_pkg.config.set("Dir::Cache::pkgcache", "")
-        pkgcache = apt_pkg.Cache()
+        pkgcache = apt_cache()
     pkg = pkgcache[package]
     return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)

=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2014-05-19 11:38:45 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2014-09-26 07:27:46 +0000
@@ -108,13 +108,8 @@
 
 def filter_installed_packages(packages):
     """Returns a list of packages that require installation"""
-    apt_pkg.init()
-
-    # Tell apt to build an in-memory cache to prevent race conditions (if
-    # another process is already building the cache).
-    apt_pkg.config.set("Dir::Cache::pkgcache", "")
-
-    cache = apt_pkg.Cache()
+
+    cache = apt_cache()
     _pkgs = []
     for package in packages:
         try:
@@ -127,6 +122,15 @@
     return _pkgs
 
 
+def apt_cache(in_memory=True):
+    """Build and return an apt cache"""
+    apt_pkg.init()
+    if in_memory:
+        apt_pkg.config.set("Dir::Cache::pkgcache", "")
+        apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
+    return apt_pkg.Cache()
+
+
 def apt_install(packages, options=None, fatal=False):
     """Install one or more packages"""
     if options is None: