Merge lp:~facundo/locolander/ns2df into lp:locolander

[Facundo Batista]

The transformer from locolander config to docker stuff

The transformer from locolander config to docker stuff

[Loco Lander (locolander)]

l. 31-33: this should be

run pip install --download=/var/cache/locolander/pip --no-install foobar
run ip link set dev eth0 down
run pip install --find-links=file:///var/cache/locolander/pip --no-index foobar

[Loco Lander (locolander)]

l. 20: I'd use pip instead of pypi
l. 29: I'd split each dependency into it's own line to make better use of the underlying caching algorithm of docker
l. 29: use apt-get install -q -y to avoid apt-get asking questions interactively
l. 30: no longer needed, part of the base image (assume /var/cache/locolander exists and will be used as the pip cache)

[lp:~facundo/locolander/ns2df]

3. By Facundo Batista on 2013-06-22

Some changes.

4. By Facundo Batista on 2013-06-22

More changes.

5. By Facundo Batista on 2013-06-22

Merged trunk in.

[Loco Lander (locolander)]

LGTM

review: Approve

=== added file 'ns2df.py'
--- ns2df.py	1970-01-01 00:00:00 +0000
+++ ns2df.py	2013-06-22 23:34:26 +0000
@@ -0,0 +1,129 @@
+"""Nessita Syntax to Docker File.
+
+Example:
+
+    precise:
+        apt:
+            apache2: 3.3-4
+            bzr:
+        pip:
+            foobar:
+    metadata:
+        test_script: ./test
+
+
+will be converted to
+
+    from ubuntu:precise
+    run apt-get -q -y install apache2=3.3-4
+    run apt-get -q -y install bzr
+    run pip install --download=/tmp/pipcache --no-install foobar
+    run ip link set dev eth0 down
+    run pip install --find-links=file:///tmp/pipcache --no-index foobar
+
+and also the system will use "./test" as script
+"""
+
+import yaml
+
+# the conversion between our nice base names and those that are
+# needed by docker
+BASE_TRANSLATIONS = {
+    'precise': 'ubuntu:precise',
+}
+
+def _get_base(config, **params):
+    """Process the system base stuff."""
+    # support only one base for now
+    if len(config) > 1:
+        raise NotImplementedError("Too many base machines.")
+    base = config.keys()[0]
+    return ["from " + BASE_TRANSLATIONS[base]]
+
+
+def _get_depends_prev(config, **params):
+    """All the dependencies before securing machine."""
+    # support only one base for now
+    config = config.values()[0]
+    dependencies = []
+
+    # apt
+    items = config.get('apt', [])
+    if items:
+        for name, ver in sorted(items.items()):
+            if ver is None:
+                p = name
+            else:
+                p = "%s=%s" % (name, ver)
+            dep = "run apt-get -q -y install " + p
+            dependencies.append(dep)
+
+    # pip
+    items = config.get('pip', [])
+    pip_cache_dir = params["pip_cache_dir"]
+    if items:
+        for name, ver in sorted(items.items()):
+            if ver is None:
+                p = name
+            else:
+                p = "%s==%s" % (name, ver)
+            dep = "run pip install --download=%s --no-install %s" % (
+                pip_cache_dir, p)
+            dependencies.append(dep)
+
+    return dependencies
+
+def _get_secure(config, **params):
+    """All that needs to be done to secure the machine."""
+    return ["run ip link set dev eth0 down"]
+
+def _get_depends_post(config, **params):
+    """All the dependencies stuff after the machine was secured."""
+    # support only one base for now
+    config = config.values()[0]
+    dependencies = []
+
+    # pip
+    items = config.get('pip', [])
+    pip_cache_dir = params["pip_cache_dir"]
+    if items:
+        for name, ver in sorted(items.items()):
+            if ver is None:
+                p = name
+            else:
+                p = "%s==%s" % (name, ver)
+            dep = "run pip install --find-links=file://%s --no-index %s" % (
+                    pip_cache_dir, p)
+            dependencies.append(dep)
+
+    return dependencies
+
+
+def _get_rest(config, **params):
+    """The final stuff."""
+    return []
+
+
+def parse(config_text, pip_cache_dir):
+    """Convert Nessita Syntax and return the text for Docker."""
+    config = yaml.load(config_text)
+    metadata = config.pop("metadata")
+
+    # the order for calling these functions is VERY important
+    functions = [
+        _get_base,
+        _get_depends_prev,
+        _get_secure,
+        _get_depends_post,
+        _get_rest,
+    ]
+    data = []
+    for func in functions:
+        items = func(config, pip_cache_dir=pip_cache_dir)
+        data.extend(items)
+    docker = "\n".join(data)
+
+    # get stuff from the metadata
+    script = metadata["test_script"]
+
+    return script, docker

=== added directory 'tests'
=== added file 'tests/__init__.py'
=== added file 'tests/test_ns2df.py'
--- tests/test_ns2df.py	1970-01-01 00:00:00 +0000
+++ tests/test_ns2df.py	2013-06-22 23:34:26 +0000
@@ -0,0 +1,168 @@
+"""Tests for the ns2df convertor."""
+
+import unittest
+
+import ns2df
+
+class DependenciesTestCase(unittest.TestCase):
+    """Tests for the dependencies transformer."""
+
+    def test_no_dependencies(self):
+        config = dict(somebase={})
+        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        self.assertEqual(res, [])
+        res = ns2df._get_depends_post(config, pip_cache_dir="/tmp/pipcache")
+        self.assertEqual(res, [])
+
+    def test_apt_one(self):
+        config = dict(somebase={
+            'apt': {'bzr': None}
+        })
+        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        self.assertEqual(res, ["run apt-get -q -y install bzr"])
+
+    def test_apt_several(self):
+        config = dict(somebase={
+            'apt': {'bzr': None, 'apache2': None}
+        })
+        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        self.assertEqual(res, [
+            "run apt-get -q -y install apache2",
+            "run apt-get -q -y install bzr",
+        ])
+
+    def test_apt_versions(self):
+        config = dict(somebase={
+            'apt': {'bzr': None, 'apache2': '3.3-4'}
+        })
+        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        self.assertEqual(res, [
+            "run apt-get -q -y install apache2=3.3-4",
+            "run apt-get -q -y install bzr",
+        ])
+
+    def test_pip_prev_one(self):
+        config = dict(somebase={
+            'pip': {'foo': None}
+        })
+        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        self.assertEqual(res, [
+            "run pip install --download=/tmp/pipcache --no-install foo",
+        ])
+
+    def test_pip_prev_several(self):
+        config = dict(somebase={
+            'pip': {'foo': None, 'bar': None}
+        })
+        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        self.assertEqual(res, [
+            "run pip install --download=/tmp/pipcache --no-install bar",
+            "run pip install --download=/tmp/pipcache --no-install foo",
+        ])
+
+    def test_pip_prev_versions(self):
+        config = dict(somebase={
+            'pip': {'foo': None, 'bar': '2.1'}
+        })
+        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        self.assertEqual(res, [
+            "run pip install --download=/tmp/pipcache --no-install bar==2.1",
+            "run pip install --download=/tmp/pipcache --no-install foo",
+        ])
+
+    def test_pip_post_one(self):
+        config = dict(somebase={
+            'pip': {'foo': None}
+        })
+        res = ns2df._get_depends_post(config, pip_cache_dir="/tmp/pipcache")
+        self.assertEqual(res, [
+            "run pip install --find-links=file:///tmp/pipcache --no-index foo"
+        ])
+
+    def test_pip_post_several(self):
+        config = dict(somebase={
+            'pip': {'foo': None, 'bar': None}
+        })
+        res = ns2df._get_depends_post(config, pip_cache_dir="/tmp/pipcache")
+        self.assertEqual(res, [
+            "run pip install --find-links=file:///tmp/pipcache --no-index bar",
+            "run pip install --find-links=file:///tmp/pipcache --no-index foo",
+        ])
+
+    def test_pip_post_versions(self):
+        config = dict(somebase={
+            'pip': {'foo': None, 'bar': '2.1'}
+        })
+        res = ns2df._get_depends_post(config, pip_cache_dir="/tmcache")
+        self.assertEqual(res, [
+            "run pip install --find-links=file:///tmcache --no-index bar==2.1",
+            "run pip install --find-links=file:///tmcache --no-index foo",
+        ])
+
+    def test_mixed_prev(self):
+        config = dict(somebase={
+            'apt': {'bzr': None, 'apache2': '3.3-4'},
+            'pip': {'foo': None, 'bar': None},
+        })
+        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        self.assertEqual(res, [
+            "run apt-get -q -y install apache2=3.3-4",
+            "run apt-get -q -y install bzr",
+            "run pip install --download=/tmp/pipcache --no-install bar",
+            "run pip install --download=/tmp/pipcache --no-install foo",
+        ])
+
+    def test_mixed_post(self):
+        config = dict(somebase={
+            'apt': {'bzr': None, 'apache2': '3.3-4'},
+            'pip': {'foo': None, 'bar': None},
+        })
+        res = ns2df._get_depends_post(config, pip_cache_dir="/tmp/pipcache")
+        self.assertEqual(res, [
+            "run pip install --find-links=file:///tmp/pipcache --no-index bar",
+            "run pip install --find-links=file:///tmp/pipcache --no-index foo",
+        ])
+
+
+class BoilerplateTestCase(unittest.TestCase):
+    """Tests for the setup and the rest of commands."""
+
+    def test_one_base(self):
+        config = dict(precise={})
+        res = ns2df._get_base(config)
+        self.assertEqual(res, ["from ubuntu:precise"])
+
+    def test_several_bases(self):
+        config = dict(base1={}, base2={})
+        self.assertRaises(NotImplementedError, ns2df._get_base, config)
+
+    def test_secure_actions(self):
+        res = ns2df._get_secure({})
+        self.assertEqual(res, ["run ip link set dev eth0 down"])
+
+    def test_rest(self):
+        res = ns2df._get_rest({})
+        self.assertEqual(res, [])
+
+    def test_all_mixed(self):
+        config_text = """
+            precise:
+                apt:
+                    apache2: 3.3-4
+                    bzr:
+                pip:
+                    foobar:
+            metadata:
+                test_script: foo
+        """
+        script, docker = ns2df.parse(config_text, "/tmp/pipcache")
+        self.assertEqual(script, "foo")
+        self.assertEqual(docker,
+            "from ubuntu:precise\n"
+            "run apt-get -q -y install apache2=3.3-4\n"
+            "run apt-get -q -y install bzr\n"
+            "run pip install --download=/tmp/pipcache --no-install foobar\n"
+            "run ip link set dev eth0 down\n"
+            "run pip install --find-links=file:///tmp/pipcache "
+                "--no-index foobar"
+        )