Merge lp:~daniel-thewatkins/cloud-init/cloudstack-passwords into lp:~cloud-init-dev/cloud-init/trunk

[Dan Watkins]

CloudStack supports setting passwords; this MP introduces support for fetching these from the virtual router.

[lp:~daniel-thewatkins/cloud-init/cloudstack-passwords]

1060. By Dan Watkins on 2015-02-17

Clean up imports in DataSourceCloudStack.py.

1061. By Dan Watkins on 2015-02-17

Fetch and use passwords from CloudStack virtual router.

1062. By Dan Watkins on 2015-02-17

Add explanatory comment.

1063. By Dan Watkins on 2015-02-18

Failing to fetch a CloudStack password should never fail the whole DS.

There might be some CloudStack deployments without the :8080 password
server, and there's no reason the rest of the data source can't be used
for them.

1064. By Dan Watkins on 2015-02-18

Set an explicit timeout when fetching CloudStack passwords.

[lp:~daniel-thewatkins/cloud-init/cloudstack-passwords]

1065. By Dan Watkins on 2015-02-20

Add automated tests for CloudStack passwords.

1066. By Dan Watkins on 2015-02-20

Minor formatting clean-up in CloudStack DS.

1067. By Dan Watkins on 2015-02-20

Split CloudStack password handling out to separate class.

[Joshua Harlow (harlowja)]

Just a few more adjusments and looks ok to me.

[lp:~daniel-thewatkins/cloud-init/cloudstack-passwords]

1068. By Dan Watkins on 2015-02-23

Always close the password server connection, even on failure.

1069. By Dan Watkins on 2015-02-23

Add documentation about upstream CloudStack HTTP fix.

[Joshua Harlow (harlowja)]

Looks ok to me, cloudstack is scary, ha.

[Joshua Harlow (harlowja)]

Scary code commit linked :-P

=== modified file 'cloudinit/sources/DataSourceCloudStack.py'
--- cloudinit/sources/DataSourceCloudStack.py	2014-08-21 17:46:41 +0000
+++ cloudinit/sources/DataSourceCloudStack.py	2015-02-23 09:36:47 +0000
@@ -26,18 +26,67 @@
 
 import os
 import time
+from socket import inet_ntoa
+from struct import pack
+
+from six.moves import http_client
 
 from cloudinit import ec2_utils as ec2
 from cloudinit import log as logging
-from cloudinit import sources
 from cloudinit import url_helper as uhelp
-from cloudinit import util
-from socket import inet_ntoa
-from struct import pack
+from cloudinit import sources, util
 
 LOG = logging.getLogger(__name__)
 
 
+class CloudStackPasswordServerClient(object):
+    """
+    Implements password fetching from the CloudStack password server.
+
+    http://cloudstack-administration.readthedocs.org/en/latest/templates.html#adding-password-management-to-your-templates
+    has documentation about the system.  This implementation is following that
+    found at
+    https://github.com/shankerbalan/cloudstack-scripts/blob/master/cloud-set-guest-password-debian
+
+    The CloudStack password server is, essentially, a broken HTTP
+    server. It requires us to provide a valid HTTP request (including a
+    DomU_Request header, which is the meat of the request), but just
+    writes the text of its response on to the socket, without a status
+    line or any HTTP headers.  This makes HTTP libraries sad, which
+    explains the screwiness of the implementation of this class.
+
+    This should be fixed in CloudStack by commit
+    a72f14ea9cb832faaac946b3cf9f56856b50142a in December 2014.
+    """
+
+    def __init__(self, virtual_router_address):
+        self.virtual_router_address = virtual_router_address
+
+    def _do_request(self, domu_request):
+        # We have to provide a valid HTTP request, but a valid HTTP
+        # response is not returned. This means that getresponse() chokes,
+        # so we use the socket directly to read off the response.
+        # Because we're reading off the socket directly, we can't re-use the
+        # connection.
+        conn = http_client.HTTPConnection(self.virtual_router_address, 8080)
+        try:
+            conn.request('GET', '', headers={'DomU_Request': domu_request})
+            conn.sock.settimeout(30)
+            output = conn.sock.recv(1024).decode('utf-8').strip()
+        finally:
+            conn.close()
+        return output
+
+    def get_password(self):
+        password = self._do_request('send_my_password')
+        if password in ['', 'saved_password']:
+            return None
+        if password == 'bad_request':
+            raise RuntimeError('Error when attempting to fetch root password.')
+        self._do_request('saved_password')
+        return password
+
+
 class DataSourceCloudStack(sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
@@ -45,10 +94,11 @@
         # Cloudstack has its metadata/userdata URLs located at
         # http://<virtual-router-ip>/latest/
         self.api_ver = 'latest'
-        vr_addr = get_vr_address()
-        if not vr_addr:
+        self.vr_addr = get_vr_address()
+        if not self.vr_addr:
             raise RuntimeError("No virtual router found!")
-        self.metadata_address = "http://%s/" % (vr_addr)
+        self.metadata_address = "http://%s/" % (self.vr_addr,)
+        self.cfg = {}
 
     def _get_url_settings(self):
         mcfg = self.ds_cfg
@@ -82,17 +132,20 @@
                                   'latest/meta-data/instance-id')]
         start_time = time.time()
         url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
-                                timeout=timeout, status_cb=LOG.warn)
+                                 timeout=timeout, status_cb=LOG.warn)
 
         if url:
             LOG.debug("Using metadata source: '%s'", url)
         else:
             LOG.critical(("Giving up on waiting for the metadata from %s"
                           " after %s seconds"),
-                          urls, int(time.time() - start_time))
+                         urls, int(time.time() - start_time))
 
         return bool(url)
 
+    def get_config_obj(self):
+        return self.cfg
+
     def get_data(self):
         seed_ret = {}
         if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")):
@@ -104,12 +157,28 @@
             if not self.wait_for_metadata_service():
                 return False
             start_time = time.time()
-            self.userdata_raw = ec2.get_instance_userdata(self.api_ver,
-                self.metadata_address)
+            self.userdata_raw = ec2.get_instance_userdata(
+                self.api_ver, self.metadata_address)
             self.metadata = ec2.get_instance_metadata(self.api_ver,
                                                       self.metadata_address)
             LOG.debug("Crawl of metadata service took %s seconds",
                       int(time.time() - start_time))
+            password_client = CloudStackPasswordServerClient(self.vr_addr)
+            try:
+                set_password = password_client.get_password()
+            except Exception:
+                util.logexc(LOG,
+                            'Failed to fetch password from virtual router %s',
+                            self.vr_addr)
+            else:
+                if set_password:
+                    self.cfg = {
+                        'ssh_pwauth': True,
+                        'password': set_password,
+                        'chpasswd': {
+                            'expire': False,
+                        },
+                    }
             return True
         except Exception:
             util.logexc(LOG, 'Failed fetching from metadata service %s',
@@ -192,7 +261,7 @@
 
 # Used to match classes to dependencies
 datasources = [
-  (DataSourceCloudStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+    (DataSourceCloudStack, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
 ]
 
 

=== added file 'tests/unittests/test_datasource/test_cloudstack.py'
--- tests/unittests/test_datasource/test_cloudstack.py	1970-01-01 00:00:00 +0000
+++ tests/unittests/test_datasource/test_cloudstack.py	2015-02-23 09:36:47 +0000
@@ -0,0 +1,86 @@
+from cloudinit import helpers
+from cloudinit.sources.DataSourceCloudStack import DataSourceCloudStack
+from ..helpers import TestCase
+
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+try:
+    from contextlib import ExitStack
+except ImportError:
+    from contextlib2 import ExitStack
+
+
+class TestCloudStackPasswordFetching(TestCase):
+
+    def setUp(self):
+        super(TestCloudStackPasswordFetching, self).setUp()
+        self.patches = ExitStack()
+        self.addCleanup(self.patches.close)
+        mod_name = 'cloudinit.sources.DataSourceCloudStack'
+        self.patches.enter_context(mock.patch('{0}.ec2'.format(mod_name)))
+        self.patches.enter_context(mock.patch('{0}.uhelp'.format(mod_name)))
+
+    def _set_password_server_response(self, response_string):
+        http_client = mock.MagicMock()
+        http_client.HTTPConnection.return_value.sock.recv.return_value = \
+            response_string.encode('utf-8')
+        self.patches.enter_context(
+            mock.patch('cloudinit.sources.DataSourceCloudStack.http_client',
+                       http_client))
+        return http_client
+
+    def test_empty_password_doesnt_create_config(self):
+        self._set_password_server_response('')
+        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds.get_data()
+        self.assertEqual({}, ds.get_config_obj())
+
+    def test_saved_password_doesnt_create_config(self):
+        self._set_password_server_response('saved_password')
+        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds.get_data()
+        self.assertEqual({}, ds.get_config_obj())
+
+    def test_password_sets_password(self):
+        password = 'SekritSquirrel'
+        self._set_password_server_response(password)
+        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds.get_data()
+        self.assertEqual(password, ds.get_config_obj()['password'])
+
+    def test_bad_request_doesnt_stop_ds_from_working(self):
+        self._set_password_server_response('bad_request')
+        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        self.assertTrue(ds.get_data())
+
+    def assertRequestTypesSent(self, http_client, expected_request_types):
+        request_types = [
+            kwargs['headers']['DomU_Request']
+            for _, kwargs
+            in http_client.HTTPConnection.return_value.request.call_args_list]
+        self.assertEqual(expected_request_types, request_types)
+
+    def test_valid_response_means_password_marked_as_saved(self):
+        password = 'SekritSquirrel'
+        http_client = self._set_password_server_response(password)
+        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds.get_data()
+        self.assertRequestTypesSent(http_client,
+                                    ['send_my_password', 'saved_password'])
+
+    def _check_password_not_saved_for(self, response_string):
+        http_client = self._set_password_server_response(response_string)
+        ds = DataSourceCloudStack({}, None, helpers.Paths({}))
+        ds.get_data()
+        self.assertRequestTypesSent(http_client, ['send_my_password'])
+
+    def test_password_not_saved_if_empty(self):
+        self._check_password_not_saved_for('')
+
+    def test_password_not_saved_if_already_saved(self):
+        self._check_password_not_saved_for('saved_password')
+
+    def test_password_not_saved_if_bad_request(self):
+        self._check_password_not_saved_for('bad_request')