Merge lp:~kiril-vladimiroff/cloud-init/cloudsigma-data-source into lp:~cloud-init-dev/cloud-init/trunk

[Kiril Vladimiroff]

Add support for the CloudSigma server context.

[lp:~kiril-vladimiroff/cloud-init/cloudsigma-data-source]

940. By Scott Moser on 2014-02-12

Add 'unverified_modules' config option and skip unverified modules

Config modules are able to declare distros that they were verified
to run on by setting 'distros' as a list in the config module.

Previously, if a module was configured to run and the running distro was not
listed as supported, it would run anyway, and a warning would be written.

Now, we change the behavior to skip those modules.

The distro (or user) can specify that a given list of modules should run anyway
by declaring the 'unverified_modules' config variable.

run_once modules will be run without this filter (ie, expecting that the user
explicitly wanted to run it).

941. By Kiril Vladimiroff on 2014-02-12

Add support for the CloudSigma server context.

=== added file 'cloudinit/cs_utils.py'
--- cloudinit/cs_utils.py	1970-01-01 00:00:00 +0000
+++ cloudinit/cs_utils.py	2014-02-12 10:49:46 +0000
@@ -0,0 +1,99 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2014 CloudSigma
+#
+#    Author: Kiril Vladimiroff <[e-mail]>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+cepko implements easy-to-use communication with CloudSigma's VMs through
+a virtual serial port without bothering with formatting the messages
+properly nor parsing the output with the specific and sometimes
+confusing shell tools for that purpose.
+
+Having the server definition accessible by the VM can ve useful in various
+ways. For example it is possible to easily determine from within the VM,
+which network interfaces are connected to public and which to private network.
+Another use is to pass some data to initial VM setup scripts, like setting the
+hostname to the VM name or passing ssh public keys through server meta.
+
+For more information take a look at the Server Context section of CloudSigma
+API Docs: http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html
+"""
+import json
+import platform
+
+import serial
+
+SERIAL_PORT = '/dev/ttyS1'
+if platform.system() == 'Windows':
+    SERIAL_PORT = 'COM2'
+
+
+class Cepko(object):
+    """
+    One instance of that object could be use for one or more
+    queries to the serial port.
+    """
+    request_pattern = "<\n{}\n>"
+
+    def get(self, key="", request_pattern=None):
+        if request_pattern is None:
+            request_pattern = self.request_pattern
+        return CepkoResult(request_pattern.format(key))
+
+    def all(self):
+        return self.get()
+
+    def meta(self, key=""):
+        request_pattern = self.request_pattern.format("/meta/{}")
+        return self.get(key, request_pattern)
+
+    def global_context(self, key=""):
+        request_pattern = self.request_pattern.format("/global_context/{}")
+        return self.get(key, request_pattern)
+
+
+class CepkoResult(object):
+    """
+    CepkoResult executes the request to the virtual serial port as soon
+    as the instance is initialized and stores the result in both raw and
+    marshalled format.
+    """
+    def __init__(self, request):
+        self.request = request
+        self.raw_result = self._execute()
+        self.result = self._marshal(self.raw_result)
+
+    def _execute(self):
+        connection = serial.Serial(SERIAL_PORT)
+        connection.write(self.request)
+        return connection.readline().strip('\x04\n')
+
+    def _marshal(self, raw_result):
+        try:
+            return json.loads(raw_result)
+        except ValueError:
+            return raw_result
+
+    def __len__(self):
+        return self.result.__len__()
+
+    def __getitem__(self, key):
+        return self.result.__getitem__(key)
+
+    def __contains__(self, item):
+        return self.result.__contains__(item)
+
+    def __iter__(self):
+        return self.result.__iter__()

=== modified file 'cloudinit/settings.py'
--- cloudinit/settings.py	2014-01-16 21:53:21 +0000
+++ cloudinit/settings.py	2014-02-12 10:49:46 +0000
@@ -37,6 +37,7 @@
         'OVF',
         'MAAS',
         'Ec2',
+        'CloudSigma',
         'CloudStack',
         'SmartOS',
         # At the end to act as a 'catch' when none of the above work...

=== added file 'cloudinit/sources/DataSourceCloudSigma.py'
--- cloudinit/sources/DataSourceCloudSigma.py	1970-01-01 00:00:00 +0000
+++ cloudinit/sources/DataSourceCloudSigma.py	2014-02-12 10:49:46 +0000
@@ -0,0 +1,91 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2014 CloudSigma
+#
+#    Author: Kiril Vladimiroff <[e-mail]>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+import re
+
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import util
+from cloudinit.cs_utils import Cepko
+
+LOG = logging.getLogger(__name__)
+
+VALID_DSMODES = ("local", "net", "disabled")
+
+
+class DataSourceCloudSigma(sources.DataSource):
+    """
+    Uses cepko in order to gather the server context from the VM.
+
+    For more information about CloudSigma's Server Context:
+    http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html
+    """
+    def __init__(self, sys_cfg, distro, paths):
+        self.dsmode = 'local'
+        self.cepko = Cepko()
+        self.ssh_public_key = ''
+        sources.DataSource.__init__(self, sys_cfg, distro, paths)
+
+    def get_data(self):
+        """
+        Metadata is the whole server context and /meta/cloud-config is used
+        as userdata.
+        """
+        try:
+            server_context = self.cepko.all().result
+            server_meta = server_context['meta']
+            self.userdata_raw = server_meta.get('cloudinit-user-data', "")
+            self.metadata = server_context
+            self.ssh_public_key = server_meta['ssh_public_key']
+
+            if server_meta.get('cloudinit-dsmode') in VALID_DSMODES:
+                self.dsmode = server_meta['cloudinit-dsmode']
+        except:
+            util.logexc(LOG, "Failed reading from the serial port")
+            return False
+        return True
+
+    def get_hostname(self, fqdn=False, resolve_ip=False):
+        """
+        Cleans up and uses the server's name if the latter is set. Otherwise
+        the first part from uuid is being used.
+        """
+        if re.match(r'^[A-Za-z0-9 -_\.]+$', self.metadata['name']):
+            return self.metadata['name'][:61]
+        else:
+            return self.metadata['uuid'].split('-')[0]
+
+    def get_public_ssh_keys(self):
+        return [self.ssh_public_key]
+
+    def get_instance_id(self):
+        return self.metadata['uuid']
+
+
+# Used to match classes to dependencies. Since this datasource uses the serial
+# port network is not really required, so it's okay to load without it, too.
+datasources = [
+    (DataSourceCloudSigma, (sources.DEP_FILESYSTEM)),
+    (DataSourceCloudSigma, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+]
+
+
+def get_datasource_list(depends):
+    """
+    Return a list of data sources that match this set of dependencies
+    """
+    return sources.list_from_depends(depends, datasources)

=== added directory 'doc/sources/cloudsigma'
=== added file 'doc/sources/cloudsigma/README.rst'
--- doc/sources/cloudsigma/README.rst	1970-01-01 00:00:00 +0000
+++ doc/sources/cloudsigma/README.rst	2014-02-12 10:49:46 +0000
@@ -0,0 +1,34 @@
+=====================
+CloudSigma Datasource
+=====================
+
+This datasource finds metadata and user-data from the `CloudSigma`_ cloud platform.
+Data transfer occurs through a virtual serial port of the `CloudSigma`_'s VM and the
+presence of network adapter is **NOT** a requirement,
+
+  See `server context`_ in the public documentation for more information.
+
+
+Setting a hostname
+~~~~~~~~~~~~~~~~~~
+
+By default the name of the server will be applied as a hostname on the first boot.
+
+
+Providing user-data
+~~~~~~~~~~~~~~~~~~~
+
+You can provide user-data to the VM using the dedicated `meta field`_ in the `server context`_
+``cloudinit-user-data``. By default *cloud-config* format is expected there and the ``#cloud-config``
+header could be omitted. However since this is a raw-text field you could provide any of the valid
+`config formats`_.
+
+If your user-data needs an internet connection you have to create a `meta field`_ in the `server context`_
+``cloudinit-dsmode`` and set "net" as value. If this field does not exist the default value is "local".
+
+
+
+.. _CloudSigma: http://cloudsigma.com/
+.. _server context: http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html
+.. _meta field: http://cloudsigma-docs.readthedocs.org/en/latest/meta.html
+.. _config formats: http://cloudinit.readthedocs.org/en/latest/topics/format.html

=== modified file 'requirements.txt'
--- requirements.txt	2014-01-18 07:46:19 +0000
+++ requirements.txt	2014-02-12 10:49:46 +0000
@@ -10,8 +10,8 @@
 # datasource is removed, this is no longer needed
 oauth
 
-# This one is currently used only by the SmartOS datasource. If that
-# datasource is removed, this is no longer needed
+# This one is currently used only by the CloudSigma and SmartOS datasources.
+# If these datasources are removed, this is no longer needed
 pyserial
 
 # This is only needed for places where we need to support configs in a manner

=== added file 'tests/unittests/test_cs_util.py'
--- tests/unittests/test_cs_util.py	1970-01-01 00:00:00 +0000
+++ tests/unittests/test_cs_util.py	2014-02-12 10:49:46 +0000
@@ -0,0 +1,65 @@
+from mocker import MockerTestCase
+
+from cloudinit.cs_utils import Cepko
+
+
+SERVER_CONTEXT = {
+    "cpu": 1000,
+    "cpus_instead_of_cores": False,
+    "global_context": {"some_global_key": "some_global_val"},
+    "mem": 1073741824,
+    "meta": {"ssh_public_key": "ssh-rsa AAAAB3NzaC1yc2E.../hQ5D5 john@doe"},
+    "name": "test_server",
+    "requirements": [],
+    "smp": 1,
+    "tags": ["much server", "very performance"],
+    "uuid": "65b2fb23-8c03-4187-a3ba-8b7c919e889",
+    "vnc_password": "9e84d6cb49e46379"
+}
+
+
+class CepkoMock(Cepko):
+    def all(self):
+        return SERVER_CONTEXT
+
+    def get(self, key="", request_pattern=None):
+        return SERVER_CONTEXT['tags']
+
+
+class CepkoResultTests(MockerTestCase):
+    def setUp(self):
+        self.mocked = self.mocker.replace("cloudinit.cs_utils.Cepko",
+                            spec=CepkoMock,
+                            count=False,
+                            passthrough=False)
+        self.mocked()
+        self.mocker.result(CepkoMock())
+        self.mocker.replay()
+        self.c = Cepko()
+
+    def test_getitem(self):
+        result = self.c.all()
+        self.assertEqual("65b2fb23-8c03-4187-a3ba-8b7c919e889", result['uuid'])
+        self.assertEqual([], result['requirements'])
+        self.assertEqual("much server", result['tags'][0])
+        self.assertEqual(1, result['smp'])
+
+    def test_len(self):
+        self.assertEqual(len(SERVER_CONTEXT), len(self.c.all()))
+
+    def test_contains(self):
+        result = self.c.all()
+        self.assertTrue('uuid' in result)
+        self.assertFalse('uid' in result)
+        self.assertTrue('meta' in result)
+        self.assertFalse('ssh_public_key' in result)
+
+    def test_iter(self):
+        self.assertEqual(sorted(SERVER_CONTEXT.keys()),
+                         sorted([key for key in self.c.all()]))
+
+    def test_with_list_as_result(self):
+        result = self.c.get('tags')
+        self.assertEqual('much server', result[0])
+        self.assertTrue('very performance' in result)
+        self.assertEqual(2, len(result))

=== added file 'tests/unittests/test_datasource/test_cloudsigma.py'
--- tests/unittests/test_datasource/test_cloudsigma.py	1970-01-01 00:00:00 +0000
+++ tests/unittests/test_datasource/test_cloudsigma.py	2014-02-12 10:49:46 +0000
@@ -0,0 +1,59 @@
+# coding: utf-8
+from unittest import TestCase
+
+from cloudinit.cs_utils import Cepko
+from cloudinit.sources import DataSourceCloudSigma
+
+
+SERVER_CONTEXT = {
+    "cpu": 1000,
+    "cpus_instead_of_cores": False,
+    "global_context": {"some_global_key": "some_global_val"},
+    "mem": 1073741824,
+    "meta": {
+        "ssh_public_key": "ssh-rsa AAAAB3NzaC1yc2E.../hQ5D5 john@doe",
+        "cloudinit-user-data": "#cloud-config\n\n...",
+    },
+    "name": "test_server",
+    "requirements": [],
+    "smp": 1,
+    "tags": ["much server", "very performance"],
+    "uuid": "65b2fb23-8c03-4187-a3ba-8b7c919e8890",
+    "vnc_password": "9e84d6cb49e46379"
+}
+
+
+class CepkoMock(Cepko):
+    result = SERVER_CONTEXT
+
+    def all(self):
+        return self
+
+
+class DataSourceCloudSigmaTest(TestCase):
+    def setUp(self):
+        self.datasource = DataSourceCloudSigma.DataSourceCloudSigma("", "", "")
+        self.datasource.cepko = CepkoMock()
+        self.datasource.get_data()
+
+    def test_get_hostname(self):
+        self.assertEqual("test_server", self.datasource.get_hostname())
+        self.datasource.metadata['name'] = ''
+        self.assertEqual("65b2fb23", self.datasource.get_hostname())
+        self.datasource.metadata['name'] = u'тест'
+        self.assertEqual("65b2fb23", self.datasource.get_hostname())
+
+    def test_get_public_ssh_keys(self):
+        self.assertEqual([SERVER_CONTEXT['meta']['ssh_public_key']],
+                         self.datasource.get_public_ssh_keys())
+
+    def test_get_instance_id(self):
+        self.assertEqual(SERVER_CONTEXT['uuid'],
+                         self.datasource.get_instance_id())
+
+    def test_metadata(self):
+        self.assertEqual(self.datasource.metadata, SERVER_CONTEXT)
+
+    def test_user_data(self):
+        self.assertEqual(self.datasource.userdata_raw,
+                         SERVER_CONTEXT['meta']['cloudinit-user-data'])