Merge lp:~michael.nelson/charm-helpers/ansible-support into lp:charm-helpers

[Michael Nelson]

Add support for using ansible within charms.

Adds ansible support to charm-helpers.

I've converted a (private - sorry) charm to test out this support [1] - all works fine.

The juju config data, as well as any relation data is available to the ansible playbooks and templates using standard jinja2 syntax: {{ instance_type }}, where relation data is namespaced by the relation type, using django-like syntax: {{ solr__hostname }}.

One question I have is whether there's a better place for the juju_state_to_yaml() helper - it's now being used by both the saltstack and ansible support modules, and I assume it would be useful outside of those contexts also (to cache the juju config+relation data etc).

[1] https://code.launchpad.net/~michael.nelson/canonical-marshal/click-package-index-ansible/+merge/176972

[Matthew Wedgwood (mew)]

Hey, Michael,

This looks awesome, thanks! I can't wait to try it out.

Regarding juju_state_to_yaml(), I'd say it belongs in core.hookenv. Bonus points for including a larger chunk of execution_environment().

-Matthew

review: Approve

=== added directory 'charmhelpers/contrib/ansible'
=== added file 'charmhelpers/contrib/ansible/__init__.py'
--- charmhelpers/contrib/ansible/__init__.py	1970-01-01 00:00:00 +0000
+++ charmhelpers/contrib/ansible/__init__.py	2013-07-25 15:55:36 +0000
@@ -0,0 +1,101 @@
+# Copyright 2013 Canonical Ltd.
+#
+# Authors:
+#  Charm Helpers Developers <[e-mail]>
+"""Charm Helpers ansible - declare the state of your machines.
+
+This helper enables you to declare your machine state, rather than
+program it procedurally (and have to test each change to your procedures).
+Your install hook can be as simple as:
+
+{{{
+import charmhelpers.contrib.ansible
+
+
+def install():
+    charmhelpers.contrib.ansible.install_ansible_support()
+    charmhelpers.contrib.ansible.apply_playbook('playbooks/install.yaml')
+}}}
+
+and won't need to change (nor will its tests) when you change the machine
+state. 
+
+All of your juju config and relation-data are available as template
+variables within your playbooks and templates. An install playbook looks
+something like:
+
+{{{
+---
+- hosts: localhost 
+  user: root
+ 
+  tasks:
+    - name: Add private repositories.
+      template:
+        src: ../templates/private-repositories.list.jinja2
+        dest: /etc/apt/sources.list.d/private.list
+
+    - name: Update the cache.
+      apt: update_cache=yes
+
+    - name: Install dependencies.
+      apt: pkg={{ item }}
+      with_items:
+        - python-mimeparse
+        - python-webob
+        - sunburnt
+
+    - name: Setup groups.
+      group: name={{ item.name }} gid={{ item.gid }}
+      with_items:
+        - { name: 'deploy_user', gid: 1800 }
+        - { name: 'service_user', gid: 1500 }
+
+  ...
+}}}
+
+Read more online about playbooks[1] and standard ansible modules[2].
+
+[1] http://www.ansibleworks.com/docs/playbooks.html
+[2] http://www.ansibleworks.com/docs/modules.html
+"""
+import os
+import subprocess
+
+import charmhelpers.contrib.saltstack
+import charmhelpers.core.host
+import charmhelpers.core.hookenv
+import charmhelpers.fetch
+
+
+charm_dir = os.environ.get('CHARM_DIR', '')
+ansible_hosts_path = '/etc/ansible/hosts'
+# Ansible will automatically include any vars in the following
+# file in its inventory when run locally.
+ansible_vars_path = '/etc/ansible/host_vars/localhost'
+
+
+def install_ansible_support(from_ppa=True):
+    """Installs the ansible package.
+
+    By default it is installed from the PPA [1] linked from
+    the ansible website [2].
+
+    [1] https://launchpad.net/~rquillo/+archive/ansible
+    [2] http://www.ansibleworks.com/docs/gettingstarted.html#ubuntu-and-debian
+
+    If from_ppa is false, you must ensure that the package is available
+    from a configured repository.
+    """
+    if from_ppa:
+        charmhelpers.fetch.add_source('ppa:rquillo/ansible')
+        charmhelpers.core.host.apt_update(fatal=True)
+    charmhelpers.core.host.apt_install('ansible')
+    with open(ansible_hosts_path, 'w+') as hosts_file:
+        hosts_file.write('localhost ansible_connection=local')
+
+
+def apply_playbook(playbook):
+    charmhelpers.contrib.saltstack.juju_state_to_yaml(
+        ansible_vars_path, namespace_separator='__')
+    subprocess.check_call(['ansible-playbook', '-c', 'local', playbook])
 
=== modified file 'charmhelpers/contrib/saltstack/__init__.py'
--- charmhelpers/contrib/saltstack/__init__.py	2013-07-11 13:38:36 +0000
+++ charmhelpers/contrib/saltstack/__init__.py	2013-07-25 15:55:36 +0000
@@ -94,7 +94,7 @@
 
 def update_machine_state(state_path):
     """Update the machine state using the provided state declaration."""
-    juju_config_2_grains()
+    juju_state_to_yaml(salt_grains_path)
     subprocess.check_call([
         'salt-call',
         '--local',
@@ -103,8 +103,8 @@
     ])
 
 
-def juju_config_2_grains():
-    """Insert the juju config as salt grains for use in state templates.
+def juju_state_to_yaml(yaml_path, namespace_separator=':'):
+    """Update the juju config and state in a yaml file.
 
     This includes any current relation-get data, and the charm
     directory.
@@ -121,7 +121,10 @@
     if relation_type is not None:
         relation_data = charmhelpers.core.hookenv.relation_get()
         relation_data = dict(
-            ("{}:{}".format(relation_type, key), val)
+            ("{relation_type}{namespace_separator}{key}".format(
+                relation_type=relation_type.replace('-', '_'),
+                key=key,
+                namespace_separator=namespace_separator), val)
             for key, val in relation_data.items())
         config.update(relation_data)
 
@@ -131,16 +134,16 @@
                          value: dumper.represent_scalar(
                              u'tag:yaml.org,2002:str', value))
 
-    grains_dir = os.path.dirname(salt_grains_path)
-    if not os.path.exists(grains_dir):
-        os.makedirs(grains_dir)
+    yaml_dir = os.path.dirname(yaml_path)
+    if not os.path.exists(yaml_dir):
+        os.makedirs(yaml_dir)
 
-    if os.path.exists(salt_grains_path):
-        with open(salt_grains_path, "r") as grain_file:
-            grains = yaml.load(grain_file.read())
+    if os.path.exists(yaml_path):
+        with open(yaml_path, "r") as existing_vars_file:
+            existing_vars = yaml.load(existing_vars_file.read())
     else:
-        grains = {}
+        existing_vars = {}
 
-    grains.update(config)
-    with open(salt_grains_path, "w+") as fp:
-        fp.write(yaml.dump(grains))
+    existing_vars.update(config)
+    with open(yaml_path, "w+") as fp:
+        fp.write(yaml.dump(existing_vars))
 
=== modified file 'charmhelpers/fetch/__init__.py'
--- charmhelpers/fetch/__init__.py	2013-07-03 11:42:37 +0000
+++ charmhelpers/fetch/__init__.py	2013-07-25 15:55:36 +0000
@@ -27,7 +27,7 @@
 def add_source(source, key=None):
     if ((source.startswith('ppa:') or
          source.startswith('http:'))):
-        subprocess.check_call(['add-apt-repository', source])
+        subprocess.check_call(['add-apt-repository', '--yes', source])
     elif source.startswith('cloud:'):
         apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
                     fatal=True)
 
=== modified file 'setup.py'
--- setup.py	2013-06-19 09:54:19 +0000
+++ setup.py	2013-07-25 15:55:36 +0000
@@ -20,6 +20,7 @@
         "charmhelpers.fetch",
         "charmhelpers.payload",
         "charmhelpers.contrib",
+        "charmhelpers.contrib.ansible",
         "charmhelpers.contrib.charmhelpers",
         "charmhelpers.contrib.charmsupport",
         "charmhelpers.contrib.saltstack",
 
=== added directory 'tests/contrib/ansible'
=== added file 'tests/contrib/ansible/__init__.py'
=== added file 'tests/contrib/ansible/test_ansible.py'
--- tests/contrib/ansible/test_ansible.py	1970-01-01 00:00:00 +0000
+++ tests/contrib/ansible/test_ansible.py	2013-07-25 15:55:36 +0000
@@ -0,0 +1,136 @@
+# Copyright 2013 Canonical Ltd.
+#
+# Authors:
+#  Charm Helpers Developers <[e-mail]>
+import mock
+import os
+import shutil
+import tempfile
+import unittest
+import yaml
+
+
+import charmhelpers.contrib.ansible
+
+
+class InstallAnsibleSupportTestCase(unittest.TestCase):
+
+    def setUp(self):
+        super(InstallAnsibleSupportTestCase, self).setUp()
+
+        patcher = mock.patch('charmhelpers.fetch')
+        self.mock_fetch = patcher.start()
+        self.addCleanup(patcher.stop)
+
+        patcher = mock.patch('charmhelpers.core')
+        self.mock_core = patcher.start()
+        self.addCleanup(patcher.stop)
+
+
+        hosts_file = tempfile.NamedTemporaryFile()
+        self.ansible_hosts_path = hosts_file.name
+        self.addCleanup(hosts_file.close)
+        patcher = mock.patch.object(charmhelpers.contrib.ansible,
+                                    'ansible_hosts_path',
+                                    self.ansible_hosts_path)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def test_adds_ppa_by_default(self):
+        charmhelpers.contrib.ansible.install_ansible_support()
+
+        self.mock_fetch.add_source.assert_called_once_with(
+            'ppa:rquillo/ansible')
+        self.mock_core.host.apt_update.assert_called_once_with(fatal=True)
+        self.mock_core.host.apt_install.assert_called_once_with(
+            'ansible')
+
+    def test_no_ppa(self):
+        charmhelpers.contrib.ansible.install_ansible_support(
+            from_ppa=False)
+
+        self.assertEqual(self.mock_fetch.add_source.call_count, 0)
+        self.mock_core.host.apt_install.assert_called_once_with(
+            'ansible')
+
+    def test_writes_ansible_hosts(self):
+        with open(self.ansible_hosts_path) as hosts_file:
+            self.assertEqual(hosts_file.read(), '')
+
+        charmhelpers.contrib.ansible.install_ansible_support()
+
+        with open(self.ansible_hosts_path) as hosts_file:
+            self.assertEqual(hosts_file.read(),
+                             'localhost ansible_connection=local')
+
+
+class ApplyPlaybookTestCases(unittest.TestCase):
+
+    def setUp(self):
+        super(ApplyPlaybookTestCases, self).setUp()
+
+        # Hookenv patches (a single patch to hookenv doesn't work):
+        patcher = mock.patch('charmhelpers.core.hookenv.config')
+        self.mock_config = patcher.start()
+        self.addCleanup(patcher.stop)
+        Serializable = charmhelpers.core.hookenv.Serializable
+        self.mock_config.return_value = Serializable({})
+        patcher = mock.patch('charmhelpers.core.hookenv.relation_get')
+        self.mock_relation_get = patcher.start()
+        self.mock_relation_get.return_value = {}
+        self.addCleanup(patcher.stop)
+        patcher = mock.patch('charmhelpers.core.hookenv.relation_type')
+        self.mock_relation_type = patcher.start()
+        self.mock_relation_type.return_value = None
+        self.addCleanup(patcher.stop)
+        patcher = mock.patch('charmhelpers.core.hookenv.local_unit')
+        self.mock_local_unit = patcher.start()
+        self.addCleanup(patcher.stop)
+        self.mock_local_unit.return_value = {}
+
+        patcher = mock.patch('charmhelpers.contrib.ansible.subprocess')
+        self.mock_subprocess = patcher.start()
+        self.addCleanup(patcher.stop)
+
+        etc_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, etc_dir)
+        self.vars_path = os.path.join(etc_dir, 'ansible', 'vars.yaml')
+        patcher = mock.patch.object(charmhelpers.contrib.ansible,
+                                    'ansible_vars_path', self.vars_path)
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+
+    def test_calls_ansible_playbook(self):
+        charmhelpers.contrib.ansible.apply_playbook(
+            'playbooks/dependencies.yaml')
+
+        self.mock_subprocess.check_call.assert_called_once_with([
+            'ansible-playbook', '-c', 'local', 'playbooks/dependencies.yaml'])
+
+    def test_writes_vars_file(self):
+        self.assertFalse(os.path.exists(self.vars_path))
+        self.mock_config.return_value = charmhelpers.core.hookenv.Serializable({
+            'group_code_owner': 'webops_deploy',
+            'user_code_runner': 'ubunet',
+        })
+        self.mock_relation_type.return_value = 'wsgi-file'
+        self.mock_relation_get.return_value = {
+            'relation_key1': 'relation_value1',
+            'relation_key2': 'relation_value2',
+        }
+
+        charmhelpers.contrib.ansible.apply_playbook(
+            'playbooks/dependencies.yaml')
+
+        self.assertTrue(os.path.exists(self.vars_path))
+        with open(self.vars_path, 'r') as vars_file:
+            result = yaml.load(vars_file.read())
+            self.assertEqual({
+                "group_code_owner": "webops_deploy",
+                "user_code_runner": "ubunet",
+                "charm_dir": "",
+                "local_unit": {},
+                "wsgi_file__relation_key1": "relation_value1",
+                "wsgi_file__relation_key2": "relation_value2",
+            }, result)
 
=== modified file 'tests/contrib/saltstack/test_saltstates.py'
--- tests/contrib/saltstack/test_saltstates.py	2013-07-11 13:38:36 +0000
+++ tests/contrib/saltstack/test_saltstates.py	2013-07-25 15:55:36 +0000
@@ -9,7 +9,6 @@
 import unittest
 import yaml
 
-import charmhelpers.core.hookenv
 import charmhelpers.contrib.saltstack
 
 
@@ -62,7 +61,7 @@
         self.addCleanup(patcher.stop)
 
         patcher = mock.patch('charmhelpers.contrib.saltstack.'
-                             'juju_config_2_grains')
+                             'juju_state_to_yaml')
         self.mock_config_2_grains = patcher.start()
         self.addCleanup(patcher.stop)
 
@@ -81,7 +80,7 @@
         charmhelpers.contrib.saltstack.update_machine_state(
             'states/install.yaml')
 
-        self.mock_config_2_grains.assert_called_once_with()
+        self.mock_config_2_grains.assert_called_once_with('/etc/salt/grains')
 
 
 class JujuConfig2GrainsTestCase(unittest.TestCase):
@@ -108,56 +107,98 @@
         etc_dir = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, etc_dir)
         self.grain_path = os.path.join(etc_dir, 'salt', 'grains')
-        patcher = mock.patch.object(charmhelpers.contrib.saltstack,
-                                    'salt_grains_path', self.grain_path)
-        patcher.start()
-        self.addCleanup(patcher.stop)
 
         patcher = mock.patch.object(charmhelpers.contrib.saltstack,
                                     'charm_dir', '/tmp/charm_dir')
         patcher.start()
         self.addCleanup(patcher.stop)
 
-    def test_output_without_relation(self):
-        self.mock_config.return_value = charmhelpers.core.hookenv.Serializable({
-            'group_code_owner': 'webops_deploy',
-            'user_code_runner': 'ubunet',
-        })
-        self.mock_local_unit.return_value = "click-index/3"
-
-        charmhelpers.contrib.saltstack.juju_config_2_grains()
-
-        with open(self.grain_path, 'r') as grain_file:
-            result = yaml.load(grain_file.read())
-            self.assertEqual({
-                "charm_dir": "/tmp/charm_dir",
-                "group_code_owner": "webops_deploy",
-                "user_code_runner": "ubunet",
-                "local_unit": "click-index/3",
-            }, result)
+    def test_output_with_empty_relation(self):
+        self.mock_config.return_value = {
+            'group_code_owner': 'webops_deploy',
+            'user_code_runner': 'ubunet',
+        }
+        self.mock_local_unit.return_value = "click-index/3"
+
+        charmhelpers.contrib.saltstack.juju_state_to_yaml(self.grain_path)
+
+        with open(self.grain_path, 'r') as grain_file:
+            result = yaml.load(grain_file.read())
+            self.assertEqual({
+                "charm_dir": "/tmp/charm_dir",
+                "group_code_owner": "webops_deploy",
+                "user_code_runner": "ubunet",
+                "local_unit": "click-index/3",
+            }, result)
+
+    def test_output_with_no_relation(self):
+        self.mock_config.return_value = {
+            'group_code_owner': 'webops_deploy',
+            'user_code_runner': 'ubunet',
+        }
+        self.mock_local_unit.return_value = "click-index/3"
+        self.mock_relation_get.return_value = None
+
+        charmhelpers.contrib.saltstack.juju_state_to_yaml(self.grain_path)
+
+        with open(self.grain_path, 'r') as grain_file:
+            result = yaml.load(grain_file.read())
+            self.assertEqual({
+                "charm_dir": "/tmp/charm_dir",
+                "group_code_owner": "webops_deploy",
+                "user_code_runner": "ubunet",
+                "local_unit": "click-index/3",
+            }, result)
+
 
     def test_output_with_relation(self):
-        self.mock_config.return_value = charmhelpers.core.hookenv.Serializable({
-            'group_code_owner': 'webops_deploy',
-            'user_code_runner': 'ubunet',
-        })
-        self.mock_relation_type.return_value = 'wsgi-file'
-        self.mock_relation_get.return_value = {
-            'relation_key1': 'relation_value1',
-            'relation_key2': 'relation_value2',
-        }
-        self.mock_local_unit.return_value = "click-index/3"
-
-        charmhelpers.contrib.saltstack.juju_config_2_grains()
-
-        with open(self.grain_path, 'r') as grain_file:
-            result = yaml.load(grain_file.read())
-            self.assertEqual({
-                "charm_dir": "/tmp/charm_dir",
-                "group_code_owner": "webops_deploy",
-                "user_code_runner": "ubunet",
-                "wsgi-file:relation_key1": "relation_value1",
-                "wsgi-file:relation_key2": "relation_value2",
+        self.mock_config.return_value = {
+            'group_code_owner': 'webops_deploy',
+            'user_code_runner': 'ubunet',
+        }
+        self.mock_relation_type.return_value = 'wsgi-file'
+        self.mock_relation_get.return_value = {
+            'relation_key1': 'relation_value1',
+            'relation_key2': 'relation_value2',
+        }
+        self.mock_local_unit.return_value = "click-index/3"
+
+        charmhelpers.contrib.saltstack.juju_state_to_yaml(self.grain_path)
+
+        with open(self.grain_path, 'r') as grain_file:
+            result = yaml.load(grain_file.read())
+            self.assertEqual({
+                "charm_dir": "/tmp/charm_dir",
+                "group_code_owner": "webops_deploy",
+                "user_code_runner": "ubunet",
+                "wsgi_file:relation_key1": "relation_value1",
+                "wsgi_file:relation_key2": "relation_value2",
+                "local_unit": "click-index/3",
+            }, result)
+
+    def test_relation_with_separator(self):
+        self.mock_config.return_value = {
+            'group_code_owner': 'webops_deploy',
+            'user_code_runner': 'ubunet',
+        }
+        self.mock_relation_type.return_value = 'wsgi-file'
+        self.mock_relation_get.return_value = {
+            'relation_key1': 'relation_value1',
+            'relation_key2': 'relation_value2',
+        }
+        self.mock_local_unit.return_value = "click-index/3"
+
+        charmhelpers.contrib.saltstack.juju_state_to_yaml(
+            self.grain_path, namespace_separator='__')
+
+        with open(self.grain_path, 'r') as grain_file:
+            result = yaml.load(grain_file.read())
+            self.assertEqual({
+                "charm_dir": "/tmp/charm_dir",
+                "group_code_owner": "webops_deploy",
+                "user_code_runner": "ubunet",
+                "wsgi_file__relation_key1": "relation_value1",
+                "wsgi_file__relation_key2": "relation_value2",
                 "local_unit": "click-index/3",
             }, result)
 
@@ -180,7 +221,7 @@
         })
         self.mock_local_unit.return_value = "click-index/3"
 
-        charmhelpers.contrib.saltstack.juju_config_2_grains()
+        charmhelpers.contrib.saltstack.juju_state_to_yaml(self.grain_path)
 
         with open(self.grain_path, 'r') as grain_file:
             result = yaml.load(grain_file.read())
 
=== modified file 'tests/fetch/test_fetch.py'
--- tests/fetch/test_fetch.py	2013-07-03 11:42:37 +0000
+++ tests/fetch/test_fetch.py	2013-07-25 15:55:36 +0000
@@ -34,6 +34,7 @@
         source = "ppa:test-ppa"
         fetch.add_source(source=source)
         check_call.assert_called_with(['add-apt-repository',
+                                       '--yes',
                                        source])
 
     @patch('subprocess.check_call')
@@ -41,6 +42,7 @@
         source = "http://archive.ubuntu.com/ubuntu raring-backports main"
         fetch.add_source(source=source)
         check_call.assert_called_with(['add-apt-repository',
+                                       '--yes',
                                        source])
 
     @patch.object(fetch, 'filter_installed_packages')
@@ -72,7 +74,7 @@
         key = "akey"
         fetch.add_source(source=source, key=key)
         check_call.assert_has_calls([
-            call(['add-apt-repository', source]),
+            call(['add-apt-repository', '--yes', source]),
             call(['apt-key', 'import', key])
         ])