Merge lp:~gnuoy/charms/trusty/glance/1453940-stable into lp:~openstack-charmers-archive/charms/trusty/glance/trunk

[Liam Young]

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #10410 glance for gnuoy mp271262
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/10410/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #6555 glance for gnuoy mp271262
    AMULET FAIL: amulet-test failed

AMULET Results (max last 2 lines):
make: *** [functional_test] Error 1
ERROR:root:Make target returned non-zero.

Full amulet test output: http://paste.ubuntu.com/12514441/
Build: http://10.245.162.77:8080/job/charm_amulet_test/6555/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #9626 glance for gnuoy mp271262
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/9626/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #10437 glance for gnuoy mp271262
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/10437/

[lp:~gnuoy/charms/trusty/glance/1453940-stable]

125. By Liam Young on 2015-09-22

Empty commit to trigger osci

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #10458 glance for gnuoy mp271262
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/10458/

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #9648 glance for gnuoy mp271262
    UNIT OK: passed

Build: http://10.245.162.77:8080/job/charm_unit_test/9648/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #6582 glance for gnuoy mp271262
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/6582/

[Chris Glass (tribaal)]

Looks good! +1

review: Approve

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #6602 glance for gnuoy mp271262
    AMULET OK: passed

Build: http://10.245.162.77:8080/job/charm_amulet_test/6602/

=== modified file 'hooks/charmhelpers/cli/__init__.py'
--- hooks/charmhelpers/cli/__init__.py	2015-08-10 16:34:51 +0000
+++ hooks/charmhelpers/cli/__init__.py	2015-09-22 15:21:44 +0000
@@ -152,15 +152,11 @@
         arguments = self.argument_parser.parse_args()
         argspec = inspect.getargspec(arguments.func)
         vargs = []
-        kwargs = {}
         for arg in argspec.args:
             vargs.append(getattr(arguments, arg))
         if argspec.varargs:
             vargs.extend(getattr(arguments, argspec.varargs))
-        if argspec.keywords:
-            for kwarg in argspec.keywords.items():
-                kwargs[kwarg] = getattr(arguments, kwarg)
-        output = arguments.func(*vargs, **kwargs)
+        output = arguments.func(*vargs)
         if getattr(arguments.func, '_cli_test_command', False):
             self.exit_code = 0 if output else 1
             output = ''

=== modified file 'hooks/charmhelpers/cli/commands.py'
--- hooks/charmhelpers/cli/commands.py	2015-08-10 16:34:51 +0000
+++ hooks/charmhelpers/cli/commands.py	2015-09-22 15:21:44 +0000
@@ -26,7 +26,7 @@
 """
 Import the sub-modules which have decorated subcommands to register with chlp.
 """
-import host  # noqa
-import benchmark  # noqa
-import unitdata  # noqa
-from charmhelpers.core import hookenv  # noqa
+from . import host  # noqa
+from . import benchmark  # noqa
+from . import unitdata  # noqa
+from . import hookenv  # noqa

=== added file 'hooks/charmhelpers/cli/hookenv.py'
--- hooks/charmhelpers/cli/hookenv.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/cli/hookenv.py	2015-09-22 15:21:44 +0000
@@ -0,0 +1,23 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+from . import cmdline
+from charmhelpers.core import hookenv
+
+
+cmdline.subcommand('relation-id')(hookenv.relation_id._wrapped)
+cmdline.subcommand('service-name')(hookenv.service_name)
+cmdline.subcommand('remote-service-name')(hookenv.remote_service_name._wrapped)

=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2015-08-10 16:34:51 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2015-09-22 15:21:44 +0000
@@ -483,13 +483,15 @@
 
         log('Generating template context for ceph', level=DEBUG)
         mon_hosts = []
-        auth = None
-        key = None
-        use_syslog = str(config('use-syslog')).lower()
+        ctxt = {
+            'use_syslog': str(config('use-syslog')).lower()
+        }
         for rid in relation_ids('ceph'):
             for unit in related_units(rid):
-                auth = relation_get('auth', rid=rid, unit=unit)
-                key = relation_get('key', rid=rid, unit=unit)
+                if not ctxt.get('auth'):
+                    ctxt['auth'] = relation_get('auth', rid=rid, unit=unit)
+                if not ctxt.get('key'):
+                    ctxt['key'] = relation_get('key', rid=rid, unit=unit)
                 ceph_pub_addr = relation_get('ceph-public-address', rid=rid,
                                              unit=unit)
                 unit_priv_addr = relation_get('private-address', rid=rid,
@@ -498,10 +500,7 @@
                 ceph_addr = format_ipv6_addr(ceph_addr) or ceph_addr
                 mon_hosts.append(ceph_addr)
 
-        ctxt = {'mon_hosts': ' '.join(sorted(mon_hosts)),
-                'auth': auth,
-                'key': key,
-                'use_syslog': use_syslog}
+        ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts))
 
         if not os.path.isdir('/etc/ceph'):
             os.mkdir('/etc/ceph')

=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	2015-08-10 16:34:51 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2015-09-22 15:21:44 +0000
@@ -1,5 +1,3 @@
-#!/usr/bin/python
-
 # Copyright 2014-2015 Canonical Limited.
 #
 # This file is part of charm-helpers.
@@ -167,9 +165,9 @@
         error_out(e)
 
 
-def get_os_version_codename(codename):
+def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES):
     '''Determine OpenStack version number from codename.'''
-    for k, v in six.iteritems(OPENSTACK_CODENAMES):
+    for k, v in six.iteritems(version_map):
         if v == codename:
             return k
     e = 'Could not derive OpenStack version for '\
@@ -392,7 +390,11 @@
     import apt_pkg as apt
     src = config('openstack-origin')
     cur_vers = get_os_version_package(package)
-    available_vers = get_os_version_install_source(src)
+    if "swift" in package:
+        codename = get_os_codename_install_source(src)
+        available_vers = get_os_version_codename(codename, SWIFT_CODENAMES)
+    else:
+        available_vers = get_os_version_install_source(src)
     apt.init()
     return apt.version_compare(available_vers, cur_vers) == 1
 

=== modified file 'hooks/charmhelpers/contrib/storage/linux/ceph.py'
--- hooks/charmhelpers/contrib/storage/linux/ceph.py	2015-08-10 16:34:51 +0000
+++ hooks/charmhelpers/contrib/storage/linux/ceph.py	2015-09-22 15:21:44 +0000
@@ -28,6 +28,7 @@
 import shutil
 import json
 import time
+import uuid
 
 from subprocess import (
     check_call,
@@ -35,8 +36,10 @@
     CalledProcessError,
 )
 from charmhelpers.core.hookenv import (
+    local_unit,
     relation_get,
     relation_ids,
+    relation_set,
     related_units,
     log,
     DEBUG,
@@ -411,17 +414,52 @@
 
     The API is versioned and defaults to version 1.
     """
-    def __init__(self, api_version=1):
+    def __init__(self, api_version=1, request_id=None):
         self.api_version = api_version
+        if request_id:
+            self.request_id = request_id
+        else:
+            self.request_id = str(uuid.uuid1())
         self.ops = []
 
     def add_op_create_pool(self, name, replica_count=3):
         self.ops.append({'op': 'create-pool', 'name': name,
                          'replicas': replica_count})
 
+    def set_ops(self, ops):
+        """Set request ops to provided value.
+
+        Useful for injecting ops that come from a previous request
+        to allow comparisons to ensure validity.
+        """
+        self.ops = ops
+
     @property
     def request(self):
-        return json.dumps({'api-version': self.api_version, 'ops': self.ops})
+        return json.dumps({'api-version': self.api_version, 'ops': self.ops,
+                           'request-id': self.request_id})
+
+    def _ops_equal(self, other):
+        if len(self.ops) == len(other.ops):
+            for req_no in range(0, len(self.ops)):
+                for key in ['replicas', 'name', 'op']:
+                    if self.ops[req_no][key] != other.ops[req_no][key]:
+                        return False
+        else:
+            return False
+        return True
+
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            return False
+        if self.api_version == other.api_version and \
+                self._ops_equal(other):
+            return True
+        else:
+            return False
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
 
 
 class CephBrokerRsp(object):
@@ -431,14 +469,198 @@
 
     The API is versioned and defaults to version 1.
     """
+
     def __init__(self, encoded_rsp):
         self.api_version = None
         self.rsp = json.loads(encoded_rsp)
 
     @property
+    def request_id(self):
+        return self.rsp.get('request-id')
+
+    @property
     def exit_code(self):
         return self.rsp.get('exit-code')
 
     @property
     def exit_msg(self):
         return self.rsp.get('stderr')
+
+
+# Ceph Broker Conversation:
+# If a charm needs an action to be taken by ceph it can create a CephBrokerRq
+# and send that request to ceph via the ceph relation. The CephBrokerRq has a
+# unique id so that the client can identity which CephBrokerRsp is associated
+# with the request. Ceph will also respond to each client unit individually
+# creating a response key per client unit eg glance/0 will get a CephBrokerRsp
+# via key broker-rsp-glance-0
+#
+# To use this the charm can just do something like:
+#
+# from charmhelpers.contrib.storage.linux.ceph import (
+#     send_request_if_needed,
+#     is_request_complete,
+#     CephBrokerRq,
+# )
+#
+# @hooks.hook('ceph-relation-changed')
+# def ceph_changed():
+#     rq = CephBrokerRq()
+#     rq.add_op_create_pool(name='poolname', replica_count=3)
+#
+#     if is_request_complete(rq):
+#         <Request complete actions>
+#     else:
+#         send_request_if_needed(get_ceph_request())
+#
+# CephBrokerRq and CephBrokerRsp are serialized into JSON. Below is an example
+# of glance having sent a request to ceph which ceph has successfully processed
+#  'ceph:8': {
+#      'ceph/0': {
+#          'auth': 'cephx',
+#          'broker-rsp-glance-0': '{"request-id": "0bc7dc54", "exit-code": 0}',
+#          'broker_rsp': '{"request-id": "0da543b8", "exit-code": 0}',
+#          'ceph-public-address': '10.5.44.103',
+#          'key': 'AQCLDttVuHXINhAAvI144CB09dYchhHyTUY9BQ==',
+#          'private-address': '10.5.44.103',
+#      },
+#      'glance/0': {
+#          'broker_req': ('{"api-version": 1, "request-id": "0bc7dc54", '
+#                         '"ops": [{"replicas": 3, "name": "glance", '
+#                         '"op": "create-pool"}]}'),
+#          'private-address': '10.5.44.109',
+#      },
+#  }
+
+def get_previous_request(rid):
+    """Return the last ceph broker request sent on a given relation
+
+    @param rid: Relation id to query for request
+    """
+    request = None
+    broker_req = relation_get(attribute='broker_req', rid=rid,
+                              unit=local_unit())
+    if broker_req:
+        request_data = json.loads(broker_req)
+        request = CephBrokerRq(api_version=request_data['api-version'],
+                               request_id=request_data['request-id'])
+        request.set_ops(request_data['ops'])
+
+    return request
+
+
+def get_request_states(request):
+    """Return a dict of requests per relation id with their corresponding
+       completion state.
+
+    This allows a charm, which has a request for ceph, to see whether there is
+    an equivalent request already being processed and if so what state that
+    request is in.
+
+    @param request: A CephBrokerRq object
+    """
+    complete = []
+    requests = {}
+    for rid in relation_ids('ceph'):
+        complete = False
+        previous_request = get_previous_request(rid)
+        if request == previous_request:
+            sent = True
+            complete = is_request_complete_for_rid(previous_request, rid)
+        else:
+            sent = False
+            complete = False
+
+        requests[rid] = {
+            'sent': sent,
+            'complete': complete,
+        }
+
+    return requests
+
+
+def is_request_sent(request):
+    """Check to see if a functionally equivalent request has already been sent
+
+    Returns True if a similair request has been sent
+
+    @param request: A CephBrokerRq object
+    """
+    states = get_request_states(request)
+    for rid in states.keys():
+        if not states[rid]['sent']:
+            return False
+
+    return True
+
+
+def is_request_complete(request):
+    """Check to see if a functionally equivalent request has already been
+    completed
+
+    Returns True if a similair request has been completed
+
+    @param request: A CephBrokerRq object
+    """
+    states = get_request_states(request)
+    for rid in states.keys():
+        if not states[rid]['complete']:
+            return False
+
+    return True
+
+
+def is_request_complete_for_rid(request, rid):
+    """Check if a given request has been completed on the given relation
+
+    @param request: A CephBrokerRq object
+    @param rid: Relation ID
+    """
+    broker_key = get_broker_rsp_key()
+    for unit in related_units(rid):
+        rdata = relation_get(rid=rid, unit=unit)
+        if rdata.get(broker_key):
+            rsp = CephBrokerRsp(rdata.get(broker_key))
+            if rsp.request_id == request.request_id:
+                if not rsp.exit_code:
+                    return True
+        else:
+            # The remote unit sent no reply targeted at this unit so either the
+            # remote ceph cluster does not support unit targeted replies or it
+            # has not processed our request yet.
+            if rdata.get('broker_rsp'):
+                request_data = json.loads(rdata['broker_rsp'])
+                if request_data.get('request-id'):
+                    log('Ignoring legacy broker_rsp without unit key as remote '
+                        'service supports unit specific replies', level=DEBUG)
+                else:
+                    log('Using legacy broker_rsp as remote service does not '
+                        'supports unit specific replies', level=DEBUG)
+                    rsp = CephBrokerRsp(rdata['broker_rsp'])
+                    if not rsp.exit_code:
+                        return True
+
+    return False
+
+
+def get_broker_rsp_key():
+    """Return broker response key for this unit
+
+    This is the key that ceph is going to use to pass request status
+    information back to this unit
+    """
+    return 'broker-rsp-' + local_unit().replace('/', '-')
+
+
+def send_request_if_needed(request):
+    """Send broker request if an equivalent request has not already been sent
+
+    @param request: A CephBrokerRq object
+    """
+    if is_request_sent(request):
+        log('Request already sent but not complete, not sending new request',
+            level=DEBUG)
+    else:
+        for rid in relation_ids('ceph'):
+            log('Sending request {}'.format(request.request_id), level=DEBUG)
+            relation_set(relation_id=rid, broker_req=request.request)

=== modified file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py	2015-08-10 16:34:51 +0000
+++ hooks/charmhelpers/core/hookenv.py	2015-09-22 15:21:44 +0000
@@ -34,23 +34,6 @@
 import tempfile
 from subprocess import CalledProcessError
 
-try:
-    from charmhelpers.cli import cmdline
-except ImportError as e:
-    # due to the anti-pattern of partially synching charmhelpers directly
-    # into charms, it's possible that charmhelpers.cli is not available;
-    # if that's the case, they don't really care about using the cli anyway,
-    # so mock it out
-    if str(e) == 'No module named cli':
-        class cmdline(object):
-            @classmethod
-            def subcommand(cls, *args, **kwargs):
-                def _wrap(func):
-                    return func
-                return _wrap
-    else:
-        raise
-
 import six
 if not six.PY3:
     from UserDict import UserDict
@@ -91,6 +74,7 @@
         res = func(*args, **kwargs)
         cache[key] = res
         return res
+    wrapper._wrapped = func
     return wrapper
 
 
@@ -190,7 +174,6 @@
     return os.environ.get('JUJU_RELATION', None)
 
 
[e-mail]()
 @cached
 def relation_id(relation_name=None, service_or_unit=None):
     """The relation ID for the current or a specified relation"""
@@ -216,13 +199,11 @@
     return os.environ.get('JUJU_REMOTE_UNIT', None)
 
 
[e-mail]()
 def service_name():
     """The name service group this unit belongs to"""
     return local_unit().split('/')[0]
 
 
[e-mail]()
 @cached
 def remote_service_name(relid=None):
     """The remote service name for a given relation-id (or the current relation)"""

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2015-08-10 16:34:51 +0000
+++ hooks/charmhelpers/core/host.py	2015-09-22 15:21:44 +0000
@@ -72,7 +72,7 @@
     stopped = service_stop(service_name)
     # XXX: Support systemd too
     override_path = os.path.join(
-        init_dir, '{}.conf.override'.format(service_name))
+        init_dir, '{}.override'.format(service_name))
     with open(override_path, 'w') as fh:
         fh.write("manual\n")
     return stopped
@@ -86,7 +86,7 @@
     if init_dir is None:
         init_dir = "/etc/init"
     override_path = os.path.join(
-        init_dir, '{}.conf.override'.format(service_name))
+        init_dir, '{}.override'.format(service_name))
     if os.path.exists(override_path):
         os.unlink(override_path)
     started = service_start(service_name)

=== modified file 'hooks/glance_relations.py'
--- hooks/glance_relations.py	2015-08-10 16:34:51 +0000
+++ hooks/glance_relations.py	2015-09-22 15:21:44 +0000
@@ -29,7 +29,6 @@
     config,
     Hooks,
     log as juju_log,
-    INFO,
     ERROR,
     open_port,
     is_relation_made,
@@ -66,9 +65,10 @@
     sync_db_with_multi_ipv6_addresses,
 )
 from charmhelpers.contrib.storage.linux.ceph import (
+    send_request_if_needed,
+    is_request_complete,
     ensure_ceph_keyring,
     CephBrokerRq,
-    CephBrokerRsp,
     delete_keyring,
 )
 from charmhelpers.payload.execd import (
@@ -240,6 +240,14 @@
     apt_install(['ceph-common', 'python-ceph'])
 
 
+def get_ceph_request():
+    service = service_name()
+    rq = CephBrokerRq()
+    replicas = config('ceph-osd-replication-count')
+    rq.add_op_create_pool(name=service, replica_count=replicas)
+    return rq
+
+
 @hooks.hook('ceph-relation-changed')
 @restart_on_change(restart_map())
 def ceph_changed():
@@ -253,29 +261,15 @@
         juju_log('Could not create ceph keyring: peer not ready?')
         return
 
-    settings = relation_get()
-    if settings and 'broker_rsp' in settings:
-        rsp = CephBrokerRsp(settings['broker_rsp'])
-        # Non-zero return code implies failure
-        if rsp.exit_code:
-            juju_log("Ceph broker request failed (rc=%s, msg=%s)" %
-                     (rsp.exit_code, rsp.exit_msg), level=ERROR)
-            return
-
-        juju_log("Ceph broker request succeeded (rc=%s, msg=%s)" %
-                 (rsp.exit_code, rsp.exit_msg), level=INFO)
+    if is_request_complete(get_ceph_request()):
+        juju_log('Request complete')
         CONFIGS.write(GLANCE_API_CONF)
         CONFIGS.write(ceph_config_file())
         # Ensure that glance-api is restarted since only now can we
         # guarantee that ceph resources are ready.
         service_restart('glance-api')
     else:
-        rq = CephBrokerRq()
-        replicas = config('ceph-osd-replication-count')
-        rq.add_op_create_pool(name=service, replica_count=replicas)
-        for rid in relation_ids('ceph'):
-            relation_set(relation_id=rid, broker_req=rq.request)
-            juju_log("Request(s) sent to Ceph broker (rid=%s)" % (rid))
+        send_request_if_needed(get_ceph_request())
 
 
 @hooks.hook('ceph-relation-broken')

=== modified file 'templates/ceph.conf'
--- templates/ceph.conf	2014-03-25 18:44:22 +0000
+++ templates/ceph.conf	2015-09-22 15:21:44 +0000
@@ -10,7 +10,8 @@
  keyring = /etc/ceph/ceph.$name.keyring
  mon host = {{ mon_hosts }}
 {% endif -%}
+{% if use_syslog -%}
  log to syslog = {{ use_syslog }}
  err to syslog = {{ use_syslog }}
  clog to syslog = {{ use_syslog }}
-
+{% endif -%}

=== modified file 'unit_tests/test_glance_relations.py'
--- unit_tests/test_glance_relations.py	2015-08-10 16:34:51 +0000
+++ unit_tests/test_glance_relations.py	2015-09-22 15:21:44 +0000
@@ -68,6 +68,9 @@
     'get_ipv6_addr',
     'sync_db_with_multi_ipv6_addresses',
     'delete_keyring',
+    'is_request_complete',
+    'send_request_if_needed',
+    'CephBrokerRq',
 ]
 
 
@@ -398,24 +401,24 @@
             'Could not create ceph keyring: peer not ready?'
         )
 
+    @patch("glance_relations.get_ceph_request")
     @patch("glance_relations.relation_set")
     @patch("glance_relations.relation_get")
     @patch.object(relations, 'CONFIGS')
     def test_ceph_changed_broker_send_rq(self, configs, mock_relation_get,
-                                         mock_relation_set):
+                                         mock_relation_set,
+                                         mock_get_ceph_request):
+        mock_get_ceph_request.return_value = "CephRequest"
         configs.complete_contexts.return_value = ['ceph']
         self.service_name.return_value = 'glance'
         self.ensure_ceph_keyring.return_value = True
         self.relation_ids.return_value = ['ceph:0']
+        self.is_request_complete.return_value = False
         relations.hooks.execute(['hooks/ceph-relation-changed'])
         self.ensure_ceph_keyring.assert_called_with(service='glance',
                                                     user='glance',
                                                     group='glance')
-        req = {'api-version': 1,
-               'ops': [{"op": "create-pool", "name": "glance", "replicas": 3}]}
-        broker_dict = json.dumps(req)
-        mock_relation_set.assert_called_with(broker_req=broker_dict,
-                                             relation_id='ceph:0')
+        self.send_request_if_needed.assert_called_with("CephRequest")
         for c in [call('/etc/glance/glance.conf')]:
             self.assertNotIn(c, configs.write.call_args_list)