Merge lp:~johnsca/charm-helpers/action-metadata-cli-helpers into lp:charm-helpers

[Cory Johns]

Added some new helpers for actions, metadata introspection, and CLI, that became useful when working on charms.reactive.

[lp:~johnsca/charm-helpers/action-metadata-cli-helpers]

418. By Cory Johns on 2015-07-29

Made import of cli subcommands from hookenv explicit

419. By Cory Johns on 2015-07-29

Add ability to override unitdata DB file via env, and added unsetrange

[Marco Ceppi (marcoceppi)]

LGTM

review: Approve

=== modified file 'charmhelpers/cli/__init__.py'
--- charmhelpers/cli/__init__.py	2015-01-22 06:06:03 +0000
+++ charmhelpers/cli/__init__.py	2015-07-29 16:32:12 +0000
@@ -20,6 +20,8 @@
 
 from six.moves import zip
 
+from charmhelpers.core import unitdata
+
 
 class OutputFormatter(object):
     def __init__(self, outfile=sys.stdout):
@@ -53,6 +55,8 @@
 
     def raw(self, output):
         """Output data as raw string (default)"""
+        if isinstance(output, (list, tuple)):
+            output = '\n'.join(map(str, output))
         self.outfile.write(str(output))
 
     def py(self, output):
@@ -91,6 +95,7 @@
     argument_parser = None
     subparsers = None
     formatter = None
+    exit_code = 0
 
     def __init__(self):
         if not self.argument_parser:
@@ -115,6 +120,21 @@
             return decorated
         return wrapper
 
+    def test_command(self, decorated):
+        """
+        Subcommand is a boolean test function, so bool return values should be
+        converted to a 0/1 exit code.
+        """
+        decorated._cli_test_command = True
+        return decorated
+
+    def no_output(self, decorated):
+        """
+        Subcommand is not expected to return a value, so don't print a spurious None.
+        """
+        decorated._cli_no_output = True
+        return decorated
+
     def subcommand_builder(self, command_name, description=None):
         """
         Decorate a function that builds a subcommand. Builders should accept a
@@ -133,11 +153,22 @@
         argspec = inspect.getargspec(arguments.func)
         vargs = []
         kwargs = {}
+        for arg in argspec.args:
+            vargs.append(getattr(arguments, arg))
         if argspec.varargs:
-            vargs = getattr(arguments, argspec.varargs)
-        for arg in argspec.args:
-            kwargs[arg] = getattr(arguments, arg)
-        self.formatter.format_output(arguments.func(*vargs, **kwargs), arguments.format)
+            vargs.extend(getattr(arguments, argspec.varargs))
+        if argspec.keywords:
+            for kwarg in argspec.keywords.items():
+                kwargs[kwarg] = getattr(arguments, kwarg)
+        output = arguments.func(*vargs, **kwargs)
+        if getattr(arguments.func, '_cli_test_command', False):
+            self.exit_code = 0 if output else 1
+            output = ''
+        if getattr(arguments.func, '_cli_no_output', False):
+            output = ''
+        self.formatter.format_output(output, arguments.format)
+        if unitdata._KV:
+            unitdata._KV.flush()
 
 
 cmdline = CommandLine()

=== modified file 'charmhelpers/cli/commands.py'
--- charmhelpers/cli/commands.py	2015-05-13 20:44:19 +0000
+++ charmhelpers/cli/commands.py	2015-07-29 16:32:12 +0000
@@ -24,8 +24,9 @@
 from . import CommandLine  # noqa
 
 """
-Import the sub-modules to be included by chlp.
+Import the sub-modules which have decorated subcommands to register with chlp.
 """
 import host  # noqa
 import benchmark  # noqa
 import unitdata  # noqa
+from charmhelpers.core import hookenv  # noqa

=== modified file 'charmhelpers/core/hookenv.py'
--- charmhelpers/core/hookenv.py	2015-07-22 05:58:46 +0000
+++ charmhelpers/core/hookenv.py	2015-07-29 16:32:12 +0000
@@ -34,6 +34,8 @@
 import tempfile
 from subprocess import CalledProcessError
 
+from charmhelpers.cli import cmdline
+
 import six
 if not six.PY3:
     from UserDict import UserDict
@@ -173,9 +175,20 @@
     return os.environ.get('JUJU_RELATION', None)
 
 
-def relation_id():
-    """The relation ID for the current relation hook"""
-    return os.environ.get('JUJU_RELATION_ID', None)
[e-mail]()
+@cached
+def relation_id(relation_name=None, service_or_unit=None):
+    """The relation ID for the current or a specified relation"""
+    if not relation_name and not service_or_unit:
+        return os.environ.get('JUJU_RELATION_ID', None)
+    elif relation_name and service_or_unit:
+        service_name = service_or_unit.split('/')[0]
+        for relid in relation_ids(relation_name):
+            remote_service = remote_service_name(relid)
+            if remote_service == service_name:
+                return relid
+    else:
+        raise ValueError('Must specify neither or both of relation_name and service_or_unit')
 
 
 def local_unit():
@@ -188,14 +201,27 @@
     return os.environ.get('JUJU_REMOTE_UNIT', None)
 
 
[e-mail]()
 def service_name():
     """The name service group this unit belongs to"""
     return local_unit().split('/')[0]
 
 
[e-mail]()
+@cached
+def remote_service_name(relid=None):
+    """The remote service name for a given relation-id (or the current relation)"""
+    if relid is None:
+        unit = remote_unit()
+    else:
+        units = related_units(relid)
+        unit = units[0] if units else None
+    return unit.split('/')[0] if unit else None
+
+
 def hook_name():
     """The name of the currently executing hook"""
-    return os.path.basename(sys.argv[0])
+    return os.environ.get('JUJU_HOOK_NAME', os.path.basename(sys.argv[0]))
 
 
 class Config(dict):
@@ -469,6 +495,63 @@
 
 
 @cached
+def relation_to_interface(relation_name):
+    """
+    Given the name of a relation, return the interface that relation uses.
+
+    :returns: The interface name, or ``None``.
+    """
+    return relation_to_role_and_interface(relation_name)[1]
+
+
+@cached
+def relation_to_role_and_interface(relation_name):
+    """
+    Given the name of a relation, return the role and the name of the interface
+    that relation uses (where role is one of ``provides``, ``requires``, or ``peer``).
+
+    :returns: A tuple containing ``(role, interface)``, or ``(None, None)``.
+    """
+    _metadata = metadata()
+    for role in ('provides', 'requires', 'peer'):
+        interface = _metadata.get(role, {}).get(relation_name, {}).get('interface')
+        if interface:
+            return role, interface
+    return None, None
+
+
+@cached
+def role_and_interface_to_relations(role, interface_name):
+    """
+    Given a role and interface name, return a list of relation names for the
+    current charm that use that interface under that role (where role is one
+    of ``provides``, ``requires``, or ``peer``).
+
+    :returns: A list of relation names.
+    """
+    _metadata = metadata()
+    results = []
+    for relation_name, relation in _metadata.get(role, {}).items():
+        if relation['interface'] == interface_name:
+            results.append(relation_name)
+    return results
+
+
+@cached
+def interface_to_relations(interface_name):
+    """
+    Given an interface, return a list of relation names for the current
+    charm that use that interface.
+
+    :returns: A list of relation names.
+    """
+    results = []
+    for role in ('provides', 'requires', 'peer'):
+        results.extend(role_and_interface_to_relations(role, interface_name))
+    return results
+
+
+@cached
 def charm_name():
     """Get the name of the current charm as is specified on metadata.yaml"""
     return metadata().get('name')
@@ -644,6 +727,21 @@
     subprocess.check_call(['action-fail', message])
 
 
+def action_name():
+    """Get the name of the currently executing action."""
+    return os.environ.get('JUJU_ACTION_NAME')
+
+
+def action_uuid():
+    """Get the UUID of the currently executing action."""
+    return os.environ.get('JUJU_ACTION_UUID')
+
+
+def action_tag():
+    """Get the tag for the currently executing action."""
+    return os.environ.get('JUJU_ACTION_TAG')
+
+
 def status_set(workload_state, message):
     """Set the workload state with a message
 

=== modified file 'charmhelpers/core/unitdata.py'
--- charmhelpers/core/unitdata.py	2015-03-18 15:51:22 +0000
+++ charmhelpers/core/unitdata.py	2015-07-29 16:32:12 +0000
@@ -152,6 +152,7 @@
 import collections
 import contextlib
 import datetime
+import itertools
 import json
 import os
 import pprint
@@ -164,8 +165,7 @@
 class Storage(object):
     """Simple key value database for local unit state within charms.
 
-    Modifications are automatically committed at hook exit. That's
-    currently regardless of exit code.
+    Modifications are not persisted unless :meth:`flush` is called.
 
     To support dicts, lists, integer, floats, and booleans values
     are automatically json encoded/decoded.
@@ -173,8 +173,11 @@
     def __init__(self, path=None):
         self.db_path = path
         if path is None:
-            self.db_path = os.path.join(
-                os.environ.get('CHARM_DIR', ''), '.unit-state.db')
+            if 'UNIT_STATE_DB' in os.environ:
+                self.db_path = os.environ['UNIT_STATE_DB']
+            else:
+                self.db_path = os.path.join(
+                    os.environ.get('CHARM_DIR', ''), '.unit-state.db')
         self.conn = sqlite3.connect('%s' % self.db_path)
         self.cursor = self.conn.cursor()
         self.revision = None
@@ -189,15 +192,8 @@
         self.conn.close()
         self._closed = True
 
-    def _scoped_query(self, stmt, params=None):
-        if params is None:
-            params = []
-        return stmt, params
-
     def get(self, key, default=None, record=False):
-        self.cursor.execute(
-            *self._scoped_query(
-                'select data from kv where key=?', [key]))
+        self.cursor.execute('select data from kv where key=?', [key])
         result = self.cursor.fetchone()
         if not result:
             return default
@@ -206,33 +202,81 @@
         return json.loads(result[0])
 
     def getrange(self, key_prefix, strip=False):
-        stmt = "select key, data from kv where key like '%s%%'" % key_prefix
-        self.cursor.execute(*self._scoped_query(stmt))
+        """
+        Get a range of keys starting with a common prefix as a mapping of
+        keys to values.
+
+        :param str key_prefix: Common prefix among all keys
+        :param bool strip: Optionally strip the common prefix from the key
+            names in the returned dict
+        :return dict: A (possibly empty) dict of key-value mappings
+        """
+        self.cursor.execute("select key, data from kv where key like ?",
+                            ['%s%%' % key_prefix])
         result = self.cursor.fetchall()
 
         if not result:
-            return None
+            return {}
         if not strip:
             key_prefix = ''
         return dict([
             (k[len(key_prefix):], json.loads(v)) for k, v in result])
 
     def update(self, mapping, prefix=""):
+        """
+        Set the values of multiple keys at once.
+
+        :param dict mapping: Mapping of keys to values
+        :param str prefix: Optional prefix to apply to all keys in `mapping`
+            before setting
+        """
         for k, v in mapping.items():
             self.set("%s%s" % (prefix, k), v)
 
     def unset(self, key):
+        """
+        Remove a key from the database entirely.
+        """
         self.cursor.execute('delete from kv where key=?', [key])
         if self.revision and self.cursor.rowcount:
             self.cursor.execute(
                 'insert into kv_revisions values (?, ?, ?)',
                 [key, self.revision, json.dumps('DELETED')])
 
+    def unsetrange(self, keys=None, prefix=""):
+        """
+        Remove a range of keys starting with a common prefix, from the database
+        entirely.
+
+        :param list keys: List of keys to remove.
+        :param str prefix: Optional prefix to apply to all keys in ``keys``
+            before removing.
+        """
+        if keys is not None:
+            keys = ['%s%s' % (prefix, key) for key in keys]
+            self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys)
+            if self.revision and self.cursor.rowcount:
+                self.cursor.execute(
+                    'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)),
+                    list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys)))
+        else:
+            self.cursor.execute('delete from kv where key like ?',
+                                ['%s%%' % prefix])
+            if self.revision and self.cursor.rowcount:
+                self.cursor.execute(
+                    'insert into kv_revisions values (?, ?, ?)',
+                    ['%s%%' % prefix, self.revision, json.dumps('DELETED')])
+
     def set(self, key, value):
+        """
+        Set a value in the database.
+
+        :param str key: Key to set the value for
+        :param value: Any JSON-serializable value to be set
+        """
         serialized = json.dumps(value)
 
-        self.cursor.execute(
-            'select data from kv where key=?', [key])
+        self.cursor.execute('select data from kv where key=?', [key])
         exists = self.cursor.fetchone()
 
         # Skip mutations to the same value

=== modified file 'tests/cli/test_cmdline.py'
--- tests/cli/test_cmdline.py	2014-11-25 15:04:52 +0000
+++ tests/cli/test_cmdline.py	2015-07-29 16:32:12 +0000
@@ -5,6 +5,7 @@
 from mock import (
     patch,
     MagicMock,
+    ANY,
 )
 import json
 from pprint import pformat
@@ -87,15 +88,61 @@
         @self.cl.subcommand()
         def bar(x, y=None, *vargs):
             "A function that does work."
-            self.bar_called = True
-            return "qux"
-
-        args = ['foo', 'bar', 'baz']
-        self.cl.formatter = MagicMock()
-        with patch("sys.argv", args):
-            self.cl.run()
-        self.assertTrue(self.bar_called)
-        self.assertTrue(self.cl.formatter.format_output.called)
+            self.assertEqual(x, 'baz')
+            self.assertEqual(y, 'why')
+            self.assertEqual(vargs, ('mux', 'zob'))
+            self.bar_called = True
+            return "qux"
+
+        args = ['chlp', 'bar', '--y', 'why', 'baz', 'mux', 'zob']
+        self.cl.formatter = MagicMock()
+        with patch("sys.argv", args):
+            with patch("charmhelpers.core.unitdata._KV") as _KV:
+                self.cl.run()
+                assert _KV.flush.called
+        self.assertTrue(self.bar_called)
+        self.cl.formatter.format_output.assert_called_once_with('qux', ANY)
+
+    def test_no_output(self):
+        self.bar_called = False
+
+        @self.cl.subcommand()
+        @self.cl.no_output
+        def bar(x, y=None, *vargs):
+            "A function that does work."
+            self.bar_called = True
+            return "qux"
+
+        args = ['foo', 'bar', 'baz']
+        self.cl.formatter = MagicMock()
+        with patch("sys.argv", args):
+            self.cl.run()
+        self.assertTrue(self.bar_called)
+        self.cl.formatter.format_output.assert_called_once_with('', ANY)
+
+    def test_test_command(self):
+        self.bar_called = False
+        self.bar_result = True
+
+        @self.cl.subcommand()
+        @self.cl.test_command
+        def bar(x, y=None, *vargs):
+            "A function that does work."
+            self.bar_called = True
+            return self.bar_result
+
+        args = ['foo', 'bar', 'baz']
+        self.cl.formatter = MagicMock()
+        with patch("sys.argv", args):
+            self.cl.run()
+        self.assertTrue(self.bar_called)
+        self.assertEqual(self.cl.exit_code, 0)
+        self.cl.formatter.format_output.assert_called_once_with('', ANY)
+
+        self.bar_result = False
+        with patch("sys.argv", args):
+            self.cl.run()
+        self.assertEqual(self.cl.exit_code, 1)
 
 
 class OutputFormatterTest(TestCase):

=== modified file 'tests/core/test_hookenv.py'
--- tests/core/test_hookenv.py	2015-07-22 05:58:46 +0000
+++ tests/core/test_hookenv.py	2015-07-29 16:32:12 +0000
@@ -854,14 +854,27 @@
             'env': 'some-environment',
         })
 
+    @patch('charmhelpers.core.hookenv.remote_service_name')
+    @patch('charmhelpers.core.hookenv.relation_ids')
     @patch('charmhelpers.core.hookenv.os')
-    def test_gets_the_relation_id(self, os_):
+    def test_gets_the_relation_id(self, os_, relation_ids, remote_service_name):
         os_.environ = {
             'JUJU_RELATION_ID': 'foo',
         }
 
         self.assertEqual(hookenv.relation_id(), 'foo')
 
+        relation_ids.return_value = ['r:1', 'r:2']
+        remote_service_name.side_effect = ['other', 'service']
+        self.assertEqual(hookenv.relation_id('rel', 'service/0'), 'r:2')
+        relation_ids.assert_called_once_with('rel')
+        self.assertEqual(remote_service_name.call_args_list, [
+            call('r:1'),
+            call('r:2'),
+        ])
+        remote_service_name.side_effect = ['other', 'service']
+        self.assertEqual(hookenv.relation_id('rel', 'service'), 'r:2')
+
     @patch('charmhelpers.core.hookenv.os')
     def test_relation_id_none_if_no_env(self, os_):
         os_.environ = {}
@@ -993,6 +1006,7 @@
         hookenv.relation_set(foo=None)
         check_call_.assert_called_with(['relation-set', 'foo='])
 
+    @patch('charmhelpers.core.hookenv.local_unit', MagicMock())
     @patch('os.remove')
     @patch('subprocess.check_output')
     @patch('subprocess.check_call')
@@ -1021,6 +1035,7 @@
             self.assertEqual("{foo: bar}", f.read().strip())
         remove.assert_called_with(temp_file)
 
+    @patch('charmhelpers.core.hookenv.local_unit', MagicMock())
     @patch('os.remove')
     @patch('subprocess.check_output')
     @patch('subprocess.check_call')
@@ -1265,6 +1280,20 @@
         _unit.return_value = 'mysql/3'
         self.assertEqual(hookenv.service_name(), 'mysql')
 
+    @patch('charmhelpers.core.hookenv.related_units')
+    @patch('charmhelpers.core.hookenv.remote_unit')
+    def test_gets_remote_service_name(self, remote_unit, related_units):
+        remote_unit.return_value = 'mysql/3'
+        related_units.return_value = ['pgsql/0', 'pgsql/1']
+        self.assertEqual(hookenv.remote_service_name(), 'mysql')
+        self.assertEqual(hookenv.remote_service_name('pgsql:1'), 'pgsql')
+
+    def test_gets_hook_name(self):
+        with patch.dict(os.environ, JUJU_HOOK_NAME='hook'):
+            self.assertEqual(hookenv.hook_name(), 'hook')
+        with patch('sys.argv', ['other-hook']):
+            self.assertEqual(hookenv.hook_name(), 'other-hook')
+
     @patch('subprocess.check_output')
     def test_action_get_with_key(self, check_output):
         action_data = 'bar'
@@ -1369,3 +1398,103 @@
         self.assertTrue(hookenv.has_juju_version('1.24-beta5'))
         self.assertTrue(hookenv.has_juju_version('1.24-beta5.1'))
         self.assertTrue(hookenv.has_juju_version('1.18-backport6'))
+
+    @patch.object(hookenv, 'relation_to_role_and_interface')
+    def test_relation_to_interface(self, rtri):
+        rtri.return_value = (None, 'foo')
+        self.assertEqual(hookenv.relation_to_interface('rel'), 'foo')
+
+    @patch.object(hookenv, 'metadata')
+    def test_relation_to_role_and_interface(self, metadata):
+        metadata.return_value = {
+            'provides': {
+                'pro-rel': {
+                    'interface': 'pro-int',
+                },
+                'pro-rel2': {
+                    'interface': 'pro-int',
+                },
+            },
+            'requires': {
+                'req-rel': {
+                    'interface': 'req-int',
+                },
+            },
+            'peer': {
+                'pee-rel': {
+                    'interface': 'pee-int',
+                },
+            },
+        }
+        rtri = hookenv.relation_to_role_and_interface
+        self.assertEqual(rtri('pro-rel'), ('provides', 'pro-int'))
+        self.assertEqual(rtri('req-rel'), ('requires', 'req-int'))
+        self.assertEqual(rtri('pee-rel'), ('peer', 'pee-int'))
+
+    @patch.object(hookenv, 'metadata')
+    def test_role_and_interface_to_relations(self, metadata):
+        metadata.return_value = {
+            'provides': {
+                'pro-rel': {
+                    'interface': 'pro-int',
+                },
+                'pro-rel2': {
+                    'interface': 'pro-int',
+                },
+            },
+            'requires': {
+                'req-rel': {
+                    'interface': 'int',
+                },
+            },
+            'peer': {
+                'pee-rel': {
+                    'interface': 'int',
+                },
+            },
+        }
+        ritr = hookenv.role_and_interface_to_relations
+        assertItemsEqual = getattr(self, 'assertItemsEqual', getattr(self, 'assertCountEqual', None))
+        assertItemsEqual(ritr('provides', 'pro-int'), ['pro-rel', 'pro-rel2'])
+        assertItemsEqual(ritr('requires', 'int'), ['req-rel'])
+        assertItemsEqual(ritr('peer', 'int'), ['pee-rel'])
+
+    @patch.object(hookenv, 'metadata')
+    def test_interface_to_relations(self, metadata):
+        metadata.return_value = {
+            'provides': {
+                'pro-rel': {
+                    'interface': 'pro-int',
+                },
+                'pro-rel2': {
+                    'interface': 'pro-int',
+                },
+            },
+            'requires': {
+                'req-rel': {
+                    'interface': 'req-int',
+                },
+            },
+            'peer': {
+                'pee-rel': {
+                    'interface': 'pee-int',
+                },
+            },
+        }
+        itr = hookenv.interface_to_relations
+        assertItemsEqual = getattr(self, 'assertItemsEqual', getattr(self, 'assertCountEqual', None))
+        assertItemsEqual(itr('pro-int'), ['pro-rel', 'pro-rel2'])
+        assertItemsEqual(itr('req-int'), ['req-rel'])
+        assertItemsEqual(itr('pee-int'), ['pee-rel'])
+
+    def test_action_name(self):
+        with patch.dict('os.environ', JUJU_ACTION_NAME='action-jack'):
+            self.assertEqual(hookenv.action_name(), 'action-jack')
+
+    def test_action_uuid(self):
+        with patch.dict('os.environ', JUJU_ACTION_UUID='action-jack'):
+            self.assertEqual(hookenv.action_uuid(), 'action-jack')
+
+    def test_action_tag(self):
+        with patch.dict('os.environ', JUJU_ACTION_TAG='action-jack'):
+            self.assertEqual(hookenv.action_tag(), 'action-jack')