Merge lp:~james-page/juju-deployer/fixup-to-for-strings into lp:~gandelman-a/juju-deployer/trunk

[James Page]

Support use of string based machine identifiers in force-machine and
terminate-machine calls

Unmerged revisions

118. By James Page on 2013-09-30

Fixup terminate-machine to deal with string based machine ids

117. By James Page on 2013-09-30

Support strings when specifing to/force-machine to enable support for lxc machines

116. By Kapil Thangavelu on 2013-09-13

merge frankban guiserver

115. By Kapil Thangavelu on 2013-09-13

merge gnuoy fix-force-machine

=== modified file '.bzrignore'
--- .bzrignore	2013-05-16 03:17:28 +0000
+++ .bzrignore	2013-09-30 10:52:03 +0000
@@ -1,7 +1,11 @@
 deployer.sublime-project
 deployer.sublime-workspace
 tmp
+jujuclient-0.0.9-py2.7.egg/
 juju_deployer.egg-info
 .emacs.desktop
 .emacs.desktop.lock
 _build
+dist/
+juju-deployer.sublime-workspace
+juju-deployer

=== modified file 'deployer/env/base.py'
--- deployer/env/base.py	2013-07-30 23:39:51 +0000
+++ deployer/env/base.py	2013-09-30 10:52:03 +0000
@@ -82,7 +82,7 @@
                     repo = "."
                 params.extend(["--repository=%s" % repo])
             if force_machine is not None:
-                params.extend["--force-machine=%d" % force_machine]
+                params.extend(["--to=%s" % force_machine])
 
             params.extend([charm_url, name])
             self._check_call(
@@ -96,7 +96,7 @@
         delete the machine (ie units have finished executing stop hooks and are
         removed)
         """
-        if int(mid) == 0:
+        if isinstance(mid, int) and int(mid) == 0:
             raise RuntimeError("Can't terminate machine 0")
         params = self._named_env(["juju", "terminate-machine"])
         params.append(mid)

=== added file 'deployer/env/gui.py'
--- deployer/env/gui.py	1970-01-01 00:00:00 +0000
+++ deployer/env/gui.py	2013-09-30 10:52:03 +0000
@@ -0,0 +1,57 @@
+"""GUI server environment implementation.
+
+The environment defined here is intended to be used by the Juju GUI server.
+See <https://code.launchpad.net/~juju-gui/charms/precise/juju-gui/trunk>.
+"""
+
+from .go import GoEnvironment
+
+
+class GUIEnvironment(GoEnvironment):
+    """A Juju environment for the juju-deployer.
+
+    Add support for deployments via the Juju API and for authenticating with
+    the provided password.
+    """
+
+    def __init__(self, endpoint, password):
+        super(GUIEnvironment, self).__init__('gui', endpoint=endpoint)
+        self._password = password
+
+    def _get_token(self):
+        """Return the stored password.
+
+        This method is overridden so that the juju-deployer does not try to
+        parse the environments.yaml file in order to retrieve the admin-secret.
+        """
+        return self._password
+
+    def connect(self):
+        """Connect the API client to the Juju backend.
+
+        This method is overridden so that a call to connect is a no-op if the
+        client is already connected.
+        """
+        if self.client is None:
+            super(GUIEnvironment, self).connect()
+
+    def close(self):
+        """Close the API connection.
+
+        Also set the client attribute to None after the disconnection.
+        """
+        super(GUIEnvironment, self).close()
+        self.client = None
+
+    def deploy(
+            self, name, charm_url, config=None, constraints=None, num_units=1,
+            *args, **kwargs):
+        """Deploy a service using the API.
+
+        Using the API in place of the command line introduces some limitations:
+          - it is not possible to use a local charm/repository;
+          - it is not possible to deploy to a specific machine.
+        """
+        self.client.deploy(
+            name, charm_url, config=config, constraints=constraints,
+            num_units=num_units)

=== added file 'deployer/guiserver.py'
--- deployer/guiserver.py	1970-01-01 00:00:00 +0000
+++ deployer/guiserver.py	2013-09-30 10:52:03 +0000
@@ -0,0 +1,70 @@
+"""Juju GUI server bundles deployment support.
+
+The following functions are used by the Juju GUI server to validate and start
+bundle deployments. The validate and import_bundle operations represents the
+public API: they are directly called in the GUI server bundles support code,
+which also takes care of handling any exception they can raise.
+Those functions are blocking, and therefore the GUI server executes them in
+separate processes.
+See <https://code.launchpad.net/~juju-gui/charms/precise/juju-gui/trunk>.
+"""
+
+import os
+
+from deployer.action.importer import Importer
+from deployer.deployment import Deployment
+from deployer.env.gui import GUIEnvironment
+from deployer.utils import mkdir
+
+
+# This value is used by the juju-deployer Importer object to store charms.
+# This directory is usually created in the machine where the Juju GUI charm is
+# deployed the first time a bundle deployment is requested.
+JUJU_HOME = '/var/lib/juju-gui/juju-home'
+
+
+def _validate(env, bundle):
+    """Bundle validation logic, used by both validate and import_bundle.
+
+    This function receives a connected environment and the bundle as a YAML
+    decoded object.
+    """
+    # Retrieve the services deployed in the Juju environment.
+    env_status = env.status()
+    env_services = set(env_status['services'].keys())
+    # Retrieve the services in the bundle.
+    bundle_services = set(bundle.get('services', {}).keys())
+    # Calculate overlapping services.
+    overlapping = env_services.intersection(bundle_services)
+    if overlapping:
+        services = ', '.join(overlapping)
+        error = 'service(s) already in the environment: {}'.format(services)
+        raise ValueError(error)
+
+
+def validate(apiurl, password, bundle):
+    """Validate a bundle."""
+    env = GUIEnvironment(apiurl, password)
+    env.connect()
+    try:
+        _validate(env, bundle)
+    finally:
+        env.close()
+
+
+def import_bundle(apiurl, password, name, bundle, options):
+    """Import a bundle."""
+    env = GUIEnvironment(apiurl, password)
+    deployment = Deployment(name, bundle, [])
+    importer = Importer(env, deployment, options)
+    env.connect()
+    # The Importer tries to retrieve the Juju home from the JUJU_HOME
+    # environment variable: create a customized directory (if required) and
+    # set up the environment context for the Importer.
+    mkdir(JUJU_HOME)
+    os.environ['JUJU_HOME'] = JUJU_HOME
+    try:
+        _validate(env, bundle)
+        importer.run()
+    finally:
+        env.close()

=== modified file 'deployer/service.py'
--- deployer/service.py	2013-07-22 15:29:31 +0000
+++ deployer/service.py	2013-09-30 10:52:03 +0000
@@ -18,7 +18,8 @@
 
     @property
     def force_machine(self):
-        return self.svc_data.get('force-machine')
+        return self.svc_data.get('to') or self.svc_data.get(
+            'force-machine')
 
     @property
     def expose(self):

=== added file 'deployer/tests/test_guienv.py'
--- deployer/tests/test_guienv.py	1970-01-01 00:00:00 +0000
+++ deployer/tests/test_guienv.py	2013-09-30 10:52:03 +0000
@@ -0,0 +1,51 @@
+"""Tests for the GUIEnvironment."""
+
+import unittest
+
+import mock
+
+from deployer.env.gui import GUIEnvironment
+
+
[e-mail]('deployer.env.go.EnvironmentClient')
+class TestGUIEnvironment(unittest.TestCase):
+
+    endpoint = 'wss://api.example.com:17070'
+    password = 'Secret!'
+
+    def setUp(self):
+        self.env = GUIEnvironment(self.endpoint, self.password)
+
+    def test_connect(self, mock_client):
+        # The environment uses the provided endpoint and password to connect
+        # to the Juju API server.
+        self.env.connect()
+        mock_client.assert_called_once_with(self.endpoint)
+        mock_client().login.assert_called_once_with(self.password)
+
+    def test_multiple_connections(self, mock_client):
+        # The environment does not attempt a second connection if it is already
+        # connected to the API backend.
+        self.env.connect()
+        self.env.connect()
+        self.assertEqual(1, mock_client.call_count)
+
+    def test_close(self, mock_client):
+        # The client attribute is set to None when the connection is closed.
+        self.env.connect()
+        self.env.close()
+        self.assertIsNone(self.env.client)
+
+    def test_deploy(self, mock_client):
+        # The environment uses the API to deploy charms.
+        self.env.connect()
+        config = {'foo': 'bar'}
+        constraints = {'cpu': 4}
+        # Deploy a service: the last two arguments (force_machine and repo) are
+        # ignored.
+        self.env.deploy(
+            'myservice', 'cs:precise/service-42', config=config,
+            constraints=constraints, num_units=2, force_machine=1, repo='/tmp')
+        mock_client().deploy.assert_called_once_with(
+            'myservice', 'cs:precise/service-42', config=config,
+            constraints=constraints, num_units=2)

=== added file 'deployer/tests/test_guiserver.py'
--- deployer/tests/test_guiserver.py	1970-01-01 00:00:00 +0000
+++ deployer/tests/test_guiserver.py	2013-09-30 10:52:03 +0000
@@ -0,0 +1,136 @@
+"""Tests for the GUI server bundles deployment support."""
+
+from contextlib import contextmanager
+import os
+import shutil
+import tempfile
+import unittest
+
+import mock
+
+from deployer import guiserver
+from deployer.deployment import Deployment
+
+
+class DeployerFunctionsTestMixin(object):
+    """Base set up for the functions that make use of the juju-deployer."""
+
+    apiurl = 'wss://api.example.com:17070'
+    password = 'Secret!'
+    name = 'mybundle'
+    bundle = {'services': {'wordpress': {}, 'mysql': {}}}
+
+    def check_environment_life(self, mock_environment):
+        """Check the calls executed on the given mock environment.
+
+        Ensure that, in order to retrieve the list of currently deployed
+        services, the environment is instantiated, connected, env.status is
+        called and then the connection is closed.
+        """
+        mock_environment.assert_called_once_with(self.apiurl, self.password)
+        mock_env_instance = mock_environment()
+        mock_env_instance.connect.assert_called_once_with()
+        mock_env_instance.status.assert_called_once_with()
+        mock_env_instance.close.assert_called_once_with()
+
+    @contextmanager
+    def assert_overlapping_services(self, mock_environment):
+        """Ensure a ValueError is raised in the context manager block.
+
+        The given mock environment object is set up so that its status
+        simulates an existing service. The name of this service overlaps with
+        the name of one of the services in the bundle.
+        """
+        mock_env_instance = mock_environment()
+        mock_env_instance.status.return_value = {'services': {'mysql': {}}}
+        # Ensure a ValueError is raised by the code in the context block.
+        with self.assertRaises(ValueError) as context_manager:
+            yield
+        # The error reflects the overlapping service name.
+        error = str(context_manager.exception)
+        self.assertEqual('service(s) already in the environment: mysql', error)
+        # Even if an error occurs, the environment connection is closed.
+        mock_env_instance.close.assert_called_once_with()
+
+
[e-mail]('deployer.guiserver.GUIEnvironment')
+class TestValidate(DeployerFunctionsTestMixin, unittest.TestCase):
+
+    def test_validation(self, mock_environment):
+        # The validation is correctly run.
+        guiserver.validate(self.apiurl, self.password, self.bundle)
+        # The environment is correctly instantiated and used.
+        self.check_environment_life(mock_environment)
+
+    def test_overlapping_services(self, mock_environment):
+        # The validation fails if the bundle includes a service name already
+        # present in the Juju environment.
+        with self.assert_overlapping_services(mock_environment):
+            guiserver.validate(self.apiurl, self.password, self.bundle)
+
+
[e-mail]('deployer.guiserver.GUIEnvironment')
+class TestImportBundle(DeployerFunctionsTestMixin, unittest.TestCase):
+
+    # The options attribute simulates the options passed to the Importer.
+    options = 'mock options'
+
+    @contextmanager
+    def patch_juju_home(self):
+        """Patch the value used by the bundle importer as Juju home."""
+        base_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, base_dir)
+        juju_home = os.path.join(base_dir, 'juju-home')
+        with mock.patch('deployer.guiserver.JUJU_HOME', juju_home):
+            try:
+                yield juju_home
+            finally:
+                del os.environ['JUJU_HOME']
+
+    def import_bundle(self):
+        """Call the import_bundle function."""
+        guiserver.import_bundle(
+            self.apiurl, self.password, self.name, self.bundle, self.options)
+
+    @mock.patch('deployer.guiserver.Importer')
+    def test_importing_bundle(self, mock_importer, mock_environment):
+        # The juju-deployer importer is correctly set up and run.
+        with self.patch_juju_home():
+            self.import_bundle()
+        # The environment is correctly instantiated and used.
+        self.check_environment_life(mock_environment)
+        # The importer is correctly instantiated.
+        self.assertEqual(1, mock_importer.call_count)
+        importer_args = mock_importer.call_args[0]
+        self.assertEqual(3, len(importer_args))
+        env, deployment, options = importer_args
+        # The first argument passed to the importer is the environment.
+        self.assertIs(mock_environment(), env)
+        # The second argument is the deployment object.
+        self.assertIsInstance(deployment, Deployment)
+        self.assertEqual(self.name, deployment.name)
+        self.assertEqual(self.bundle, deployment.data)
+        # The third and last argument is the options object.
+        self.assertIs(self.options, options)
+        # The importer is started.
+        mock_importer().run.assert_called_once_with()
+
+    def test_overlapping_services(self, mock_environment):
+        # The import fails if the bundle includes a service name already
+        # present in the Juju environment.
+        with self.assert_overlapping_services(mock_environment):
+            with self.patch_juju_home():
+                self.import_bundle()
+
+    @mock.patch('deployer.guiserver.Importer')
+    def test_juju_home(self, mock_importer, mock_environment):
+        # A customized Juju home is created and used during the import process.
+        with self.patch_juju_home() as juju_home:
+            assert not os.path.isdir(juju_home), 'directory should not exist'
+            # Ensure JUJU_HOME is included in the context when the Importer
+            # instance is run.
+            run = lambda: self.assertEqual(juju_home, os.getenv('JUJU_HOME'))
+            mock_importer().run = run
+            self.import_bundle()
+        # The JUJU_HOME directory has been created.
+        self.assertTrue(os.path.isdir(juju_home))

=== modified file 'deployer/tests/test_utils.py'
--- deployer/tests/test_utils.py	2013-07-30 23:39:51 +0000
+++ deployer/tests/test_utils.py	2013-09-30 10:52:03 +0000
@@ -1,7 +1,15 @@
+import os
+from subprocess import CalledProcessError
+
 from mock import patch, MagicMock
-from subprocess import CalledProcessError
+
 from .base import Base
-from deployer.utils import dict_merge, _check_call, ErrorExit
+from deployer.utils import (
+    _check_call,
+    dict_merge,
+    ErrorExit,
+    mkdir,
+)
 
 
 class UtilTests(Base):
@@ -48,3 +56,50 @@
         self.assertEquals(output, 'good')
         # 1 failure + 3 retries
         self.assertEquals(len(check_output.call_args_list), 3)
+
+
+class TestMkdir(Base):
+
+    def setUp(self):
+        self.playground = self.mkdir()
+
+    def test_create_dir(self):
+        # A directory is correctly created.
+        path = os.path.join(self.playground, 'foo')
+        mkdir(path)
+        self.assertTrue(os.path.isdir(path))
+
+    def test_intermediate_dirs(self):
+        # All intermediate directories are created.
+        path = os.path.join(self.playground, 'foo', 'bar', 'leaf')
+        mkdir(path)
+        self.assertTrue(os.path.isdir(path))
+
+    def test_expand_user(self):
+        # The ~ construction is expanded.
+        with patch('os.environ', {'HOME': self.playground}):
+            mkdir('~/in/my/home')
+        path = os.path.join(self.playground, 'in', 'my', 'home')
+        self.assertTrue(os.path.isdir(path))
+
+    def test_existing_dir(self):
+        # The function exits without errors if the target directory exists.
+        path = os.path.join(self.playground, 'foo')
+        os.mkdir(path)
+        mkdir(path)
+
+    def test_existing_file(self):
+        # An OSError is raised if a file already exists in the target path.
+        path = os.path.join(self.playground, 'foo')
+        with open(path, 'w'):
+            with self.assertRaises(OSError):
+                mkdir(path)
+
+    def test_failure(self):
+        # Errors are correctly re-raised.
+        path = os.path.join(self.playground, 'foo')
+        os.chmod(self.playground, 0000)
+        self.addCleanup(os.chmod, self.playground, 0700)
+        with self.assertRaises(OSError):
+            mkdir(os.path.join(path))
+        self.assertFalse(os.path.exists(path))

=== modified file 'deployer/utils.py'
--- deployer/utils.py	2013-07-30 23:39:51 +0000
+++ deployer/utils.py	2013-09-30 10:52:03 +0000
@@ -1,13 +1,19 @@
 from copy import deepcopy
 from contextlib import contextmanager
 
+import errno
 import logging
 from logging.config import dictConfig as logConfig
+
 import os
-
-from os.path import abspath, isabs
-from os.path import join as path_join
-from os.path import exists as path_exists
+from os.path import (
+    abspath,
+    expanduser,
+    isabs,
+    isdir,
+    join as path_join,
+    exists as path_exists,
+)
 
 import stat
 import subprocess
@@ -221,3 +227,18 @@
             return full_path
 
     return None
+
+
+def mkdir(path):
+    """Create a leaf directory and all intermediate ones.
+
+    Also expand ~ and ~user constructions.
+    If path exists and it's a directory, return without errors.
+    """
+    path = expanduser(path)
+    try:
+        os.makedirs(path)
+    except OSError as err:
+        # Re-raise the error if the target path exists but it is not a dir.
+        if (err.errno != errno.EEXIST) or (not isdir(path)):
+            raise