Merge lp:~ricardokirkner/locolander/requirements into lp:locolander

[Ricardo Kirkner]

pre-requisites to make locolander able to land itself

- updated requirements
- updated locolander config
- updated ns2df.py script to create Dockerfile from locolander config

[lp:~ricardokirkner/locolander/requirements]

12. By Ricardo Kirkner on 2013-06-29

fixed test script path

13. By Ricardo Kirkner on 2013-07-09

improvements to ns2df.py

- allow using a requirements file
- use locolander base images
- install dependencies using a single command for improved caching
- allow ns2df.py to be called from the commandline to create a Dockerfile
- build Dockerfile so that it includes the command to run tests
- build Dockerfile so that it times out the test run

14. By Ricardo Kirkner on 2013-07-09

include ns2df tests in test script

15. By Ricardo Kirkner on 2013-07-09

added fabfile for bootstrapping and running tests

[Facundo Batista (facundo)]

I see you're changing back to put all apt-get installations in the same line... you told me that we should put them all in different lines because of docker image caching... did that change?

The rest looks ok!

review: Needs Information

[Ricardo Kirkner (ricardokirkner-f)]

I did tell you that, but I have changed my mind. The reason being that
while each new command gives us a different cached state (which was the
original reason for having it one per line), I now think that the overload
of having to invoke pip for each dependency is just too high. Also having a
single line for all dependencies should make rebuilding the image from
cache much faster than having to iterate over each intermediate state (plus
avoiding unnecessary wasted disk space by not-really reusable containers).

Hope this makes sense.

On Wed, Jul 10, 2013 at 8:15 PM, Facundo Batista <email address hidden>wrote:

> Review: Needs Information
>
> I see you're changing back to put all apt-get installations in the same
> line... you told me that we should put them all in different lines because
> of docker image caching... did that change?
>
> The rest looks ok!
> --
>
> https://code.launchpad.net/~ricardokirkner/locolander/requirements/+merge/172214
> Your team LocoLanderos is subscribed to branch lp:locolander.
>

[Facundo Batista (facundo)]

Good enough!

review: Approve

=== added file '.locolander.yml'
--- .locolander.yml	1970-01-01 00:00:00 +0000
+++ .locolander.yml	2013-07-09 18:45:42 +0000
@@ -0,0 +1,26 @@
+precise:
+    pip:
+        distribute: 0.6.34
+        Django: 1.5.1
+        github3.py: 0.7.0
+        httplib2: 0.8.0
+        keyring: 1.5.0
+        launchpadlib: 1.10.2
+        lazr.authentication: 0.1.2
+        lazr.restfulclient: 0.13.3
+        lazr.uri: 1.0.3
+        mock: 1.0.1
+        oauth: 1.0.1
+        pep8: 1.4.5
+        pyflakes: 0.7.2
+        PyYAML: 3.10.0
+        requests: 1.2.3
+        simplejson: 3.3.0
+        South: 0.8.1
+        testresources: 0.2.7
+        wadllib: 1.3.2
+        wsgi-intercept: 0.5.1
+        zope.interface: 4.0.5
+
+metadata:
+    test_script: ./run_tests.sh

=== added file 'fabfile.py'
--- fabfile.py	1970-01-01 00:00:00 +0000
+++ fabfile.py	2013-07-09 18:45:42 +0000
@@ -0,0 +1,26 @@
+from fabric.api import local
+from fabric.context_managers import prefix
+
+
+def virtualenv(func):
+    def wrapped(*args):
+        with prefix('. .env/bin/activate'):
+            return func(*args)
+    return wrapped
+
+
+def bootstrap():
+    local('virtualenv -p /usr/bin/python2 .env')
+    local('.env/bin/pip install -r requirements.txt')
+
+
+@virtualenv
+def manage(*args):
+    cmd = ['python locolander/manage.py']
+    cmd.extend(args)
+    local(' '.join(cmd))
+
+
+@virtualenv
+def test():
+    local('./run_tests.sh')

=== modified file 'ns2df.py'
--- ns2df.py	2013-06-22 23:29:05 +0000
+++ ns2df.py	2013-07-09 18:45:42 +0000
@@ -14,7 +14,7 @@
 
 will be converted to
 
-    from ubuntu:precise
+    from locolander:precise
     run apt-get -q -y install apache2=3.3-4
     run apt-get -q -y install bzr
     run pip install --download=/tmp/pipcache --no-install foobar
@@ -24,14 +24,30 @@
 and also the system will use "./test" as script
 """
 
+import os.path
+
 import yaml
 
 # the conversion between our nice base names and those that are
 # needed by docker
 BASE_TRANSLATIONS = {
-    'precise': 'ubuntu:precise',
+    'precise': 'locolander:precise',
 }
 
+
+def _get_pip_packages(items):
+    if isinstance(items, basestring):
+        return items
+    packages = []
+    for name, ver in sorted(items.items()):
+        if ver is None:
+            p = name
+        else:
+            p = "%s==%s" % (name, ver)
+        packages.append(p)
+    return ' '.join(packages)
+
+
 def _get_base(config, **params):
     """Process the system base stuff."""
     # support only one base for now
@@ -41,6 +57,13 @@
     return ["from " + BASE_TRANSLATIONS[base]]
 
 
+def _get_code(config, **params):
+    return ['run mkdir -p {target_path}'.format(**params),
+            'add . {target_path}'.format(**params),
+            'run chown -R locolander.locolander {target_path}'.format(
+                **params)]
+
+
 def _get_depends_prev(config, **params):
     """All the dependencies before securing machine."""
     # support only one base for now
@@ -50,26 +73,26 @@
     # apt
     items = config.get('apt', [])
     if items:
+        packages = []
         for name, ver in sorted(items.items()):
             if ver is None:
                 p = name
             else:
                 p = "%s=%s" % (name, ver)
-            dep = "run apt-get -q -y install " + p
-            dependencies.append(dep)
+            packages.append(p)
+        params['packages'] = ' '.join(packages)
+        dependencies.append('run apt-get -q -y install {packages}'.format(**params))
 
     # pip
     items = config.get('pip', [])
-    pip_cache_dir = params["pip_cache_dir"]
     if items:
-        for name, ver in sorted(items.items()):
-            if ver is None:
-                p = name
-            else:
-                p = "%s==%s" % (name, ver)
-            dep = "run pip install --download=%s --no-install %s" % (
-                pip_cache_dir, p)
-            dependencies.append(dep)
+        packages = _get_pip_packages(items)
+        pip_cache_dir = params['pip_cache_dir']
+        dependencies.append(
+            'run cd {target_path} && pip install --download={pip_cache_dir} '
+            '--no-install {packages}'.format(
+                pip_cache_dir=pip_cache_dir, packages=packages,
+                target_path=params['target_path']))
 
     return dependencies
 
@@ -85,33 +108,39 @@
 
     # pip
     items = config.get('pip', [])
-    pip_cache_dir = params["pip_cache_dir"]
     if items:
-        for name, ver in sorted(items.items()):
-            if ver is None:
-                p = name
-            else:
-                p = "%s==%s" % (name, ver)
-            dep = "run pip install --find-links=file://%s --no-index %s" % (
-                    pip_cache_dir, p)
-            dependencies.append(dep)
+        packages = _get_pip_packages(items)
+        pip_cache_dir = params['pip_cache_dir']
+        dependencies.append(
+            'run cd {target_path} && pip install '
+            '--find-links=file://{pip_cache_dir} --no-index '
+            '{packages}'.format(pip_cache_dir=pip_cache_dir,
+                                packages=packages,
+                                target_path=params['target_path']))
 
     return dependencies
 
 
 def _get_rest(config, **params):
     """The final stuff."""
-    return []
-
-
-def parse(config_text, pip_cache_dir):
+    return ['cmd cd {target_path} && timeout 300 {test_cmd}'.format(**params)]
+
+
+def parse(config_text, pip_cache_dir, project):
     """Convert Nessita Syntax and return the text for Docker."""
     config = yaml.load(config_text)
     metadata = config.pop("metadata")
 
+    params = {
+        'test_cmd': metadata['test_script'],
+        'target_path': os.path.join('/srv/locolander', project),
+        'pip_cache_dir': pip_cache_dir,
+    }
+
     # the order for calling these functions is VERY important
     functions = [
         _get_base,
+        _get_code,
         _get_depends_prev,
         _get_secure,
         _get_depends_post,
@@ -119,11 +148,20 @@
     ]
     data = []
     for func in functions:
-        items = func(config, pip_cache_dir=pip_cache_dir)
+        items = func(config, **params)
         data.extend(items)
     docker = "\n".join(data)
 
-    # get stuff from the metadata
-    script = metadata["test_script"]
-
-    return script, docker
+    return docker
+
+
+if __name__ == '__main__':
+    import sys
+
+    project = sys.argv[1]
+    infile, outfile = sys.argv[2:]
+
+    config = open(infile, 'r').read()
+    data = parse(config, '/var/cache/locolander', project)
+    with open(outfile, 'w') as dockerfile:
+        dockerfile.write(data + '\n')

=== modified file 'requirements.txt'
--- requirements.txt	2013-06-22 19:53:22 +0000
+++ requirements.txt	2013-07-09 18:45:42 +0000
@@ -1,7 +1,21 @@
+distribute==0.6.34
 Django==1.5.1
+github3.py==0.7.0
+httplib2==0.8
+keyring==1.5
+launchpadlib==1.10.2
+lazr.authentication==0.1.2
+lazr.restfulclient==0.13.3
+lazr.uri==1.0.3
 mock==1.0.1
+oauth==1.0.1
 pep8==1.4.5
 pyflakes==0.7.2
+PyYAML==3.10
+requests==1.2.3
+simplejson==3.3.0
 South==0.8.1
-github3.py==0.7.0
-launchpadlib==1.10.2
+testresources==0.2.7
+wadllib==1.3.2
+wsgi-intercept==0.5.1
+zope.interface==4.0.5

=== modified file 'run_tests.sh'
--- run_tests.sh	2013-06-22 20:51:04 +0000
+++ run_tests.sh	2013-07-09 18:45:42 +0000
@@ -1,7 +1,7 @@
 #! /bin/bash
 
 set -e
-set -x
 
 python locolander/manage.py test locolanderweb
+python -m unittest discover
 PYTHONPATH=locolander python -m unittest discover -s locolander/repos/

=== modified file 'tests/test_ns2df.py'
--- tests/test_ns2df.py	2013-06-22 23:29:05 +0000
+++ tests/test_ns2df.py	2013-07-09 18:45:42 +0000
@@ -7,96 +7,120 @@
 class DependenciesTestCase(unittest.TestCase):
     """Tests for the dependencies transformer."""
 
+    def setUp(self):
+        super(DependenciesTestCase, self).setUp()
+        self.params = {
+            'pip_cache_dir': '/tmp/pipcache',
+            'target_path': '/srv/locolander/project',
+        }
+
     def test_no_dependencies(self):
         config = dict(somebase={})
-        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        res = ns2df._get_depends_prev(config, **self.params)
         self.assertEqual(res, [])
-        res = ns2df._get_depends_post(config, pip_cache_dir="/tmp/pipcache")
+        res = ns2df._get_depends_post(config, **self.params)
         self.assertEqual(res, [])
 
     def test_apt_one(self):
         config = dict(somebase={
             'apt': {'bzr': None}
         })
-        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        res = ns2df._get_depends_prev(config, **self.params)
         self.assertEqual(res, ["run apt-get -q -y install bzr"])
 
     def test_apt_several(self):
         config = dict(somebase={
             'apt': {'bzr': None, 'apache2': None}
         })
-        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        res = ns2df._get_depends_prev(config, **self.params)
         self.assertEqual(res, [
-            "run apt-get -q -y install apache2",
-            "run apt-get -q -y install bzr",
+            "run apt-get -q -y install apache2 bzr",
         ])
 
     def test_apt_versions(self):
         config = dict(somebase={
             'apt': {'bzr': None, 'apache2': '3.3-4'}
         })
-        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        res = ns2df._get_depends_prev(config, **self.params)
         self.assertEqual(res, [
-            "run apt-get -q -y install apache2=3.3-4",
-            "run apt-get -q -y install bzr",
+            "run apt-get -q -y install apache2=3.3-4 bzr",
         ])
 
     def test_pip_prev_one(self):
         config = dict(somebase={
             'pip': {'foo': None}
         })
-        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        res = ns2df._get_depends_prev(config, **self.params)
         self.assertEqual(res, [
-            "run pip install --download=/tmp/pipcache --no-install foo",
+            "run cd /srv/locolander/project && pip install --download=/tmp/pipcache --no-install foo",
         ])
 
     def test_pip_prev_several(self):
         config = dict(somebase={
             'pip': {'foo': None, 'bar': None}
         })
-        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        res = ns2df._get_depends_prev(config, **self.params)
         self.assertEqual(res, [
-            "run pip install --download=/tmp/pipcache --no-install bar",
-            "run pip install --download=/tmp/pipcache --no-install foo",
+            "run cd /srv/locolander/project && pip install --download=/tmp/pipcache --no-install bar foo",
         ])
 
     def test_pip_prev_versions(self):
         config = dict(somebase={
             'pip': {'foo': None, 'bar': '2.1'}
         })
-        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
-        self.assertEqual(res, [
-            "run pip install --download=/tmp/pipcache --no-install bar==2.1",
-            "run pip install --download=/tmp/pipcache --no-install foo",
+        res = ns2df._get_depends_prev(config, **self.params)
+        self.assertEqual(res, [
+            ("run cd /srv/locolander/project && pip install --download=/tmp/pipcache --no-install "
+             "bar==2.1 foo"),
+        ])
+
+    def test_pip_prev_requirements(self):
+        config = dict(somebase={
+            'pip': '-r requirements.txt'
+        })
+        res = ns2df._get_depends_prev(config, **self.params)
+        self.assertEqual(res, [
+            ("run cd /srv/locolander/project && pip install --download=/tmp/pipcache --no-install "
+             "-r requirements.txt"),
         ])
 
     def test_pip_post_one(self):
         config = dict(somebase={
             'pip': {'foo': None}
         })
-        res = ns2df._get_depends_post(config, pip_cache_dir="/tmp/pipcache")
+        res = ns2df._get_depends_post(config, **self.params)
         self.assertEqual(res, [
-            "run pip install --find-links=file:///tmp/pipcache --no-index foo"
+            "run cd /srv/locolander/project && pip install --find-links=file:///tmp/pipcache --no-index foo"
         ])
 
     def test_pip_post_several(self):
         config = dict(somebase={
             'pip': {'foo': None, 'bar': None}
         })
-        res = ns2df._get_depends_post(config, pip_cache_dir="/tmp/pipcache")
+        res = ns2df._get_depends_post(config, **self.params)
         self.assertEqual(res, [
-            "run pip install --find-links=file:///tmp/pipcache --no-index bar",
-            "run pip install --find-links=file:///tmp/pipcache --no-index foo",
+            ("run cd /srv/locolander/project && pip install --find-links=file:///tmp/pipcache --no-index "
+             "bar foo"),
         ])
 
     def test_pip_post_versions(self):
         config = dict(somebase={
             'pip': {'foo': None, 'bar': '2.1'}
         })
-        res = ns2df._get_depends_post(config, pip_cache_dir="/tmcache")
-        self.assertEqual(res, [
-            "run pip install --find-links=file:///tmcache --no-index bar==2.1",
-            "run pip install --find-links=file:///tmcache --no-index foo",
+        res = ns2df._get_depends_post(config, **self.params)
+        self.assertEqual(res, [
+            ("run cd /srv/locolander/project && pip install --find-links=file:///tmp/pipcache --no-index "
+             "bar==2.1 foo"),
+        ])
+
+    def test_pip_post_requirements(self):
+        config = dict(somebase={
+            'pip': '-r requirements.txt'
+        })
+        res = ns2df._get_depends_post(config, **self.params)
+        self.assertEqual(res, [
+            ("run cd /srv/locolander/project && pip install --find-links=file:///tmp/pipcache --no-index "
+             "-r requirements.txt"),
         ])
 
     def test_mixed_prev(self):
@@ -104,12 +128,10 @@
             'apt': {'bzr': None, 'apache2': '3.3-4'},
             'pip': {'foo': None, 'bar': None},
         })
-        res = ns2df._get_depends_prev(config, pip_cache_dir="/tmp/pipcache")
+        res = ns2df._get_depends_prev(config, **self.params)
         self.assertEqual(res, [
-            "run apt-get -q -y install apache2=3.3-4",
-            "run apt-get -q -y install bzr",
-            "run pip install --download=/tmp/pipcache --no-install bar",
-            "run pip install --download=/tmp/pipcache --no-install foo",
+            "run apt-get -q -y install apache2=3.3-4 bzr",
+            "run cd /srv/locolander/project && pip install --download=/tmp/pipcache --no-install bar foo",
         ])
 
     def test_mixed_post(self):
@@ -117,10 +139,10 @@
             'apt': {'bzr': None, 'apache2': '3.3-4'},
             'pip': {'foo': None, 'bar': None},
         })
-        res = ns2df._get_depends_post(config, pip_cache_dir="/tmp/pipcache")
+        res = ns2df._get_depends_post(config, **self.params)
         self.assertEqual(res, [
-            "run pip install --find-links=file:///tmp/pipcache --no-index bar",
-            "run pip install --find-links=file:///tmp/pipcache --no-index foo",
+            ("run cd /srv/locolander/project && pip install --find-links=file:///tmp/pipcache --no-index "
+             "bar foo"),
         ])
 
 
@@ -130,7 +152,7 @@
     def test_one_base(self):
         config = dict(precise={})
         res = ns2df._get_base(config)
-        self.assertEqual(res, ["from ubuntu:precise"])
+        self.assertEqual(res, ["from locolander:precise"])
 
     def test_several_bases(self):
         config = dict(base1={}, base2={})
@@ -141,8 +163,10 @@
         self.assertEqual(res, ["run ip link set dev eth0 down"])
 
     def test_rest(self):
-        res = ns2df._get_rest({})
-        self.assertEqual(res, [])
+        res = ns2df._get_rest({},
+                              target_path='/srv/locolander',
+                              test_cmd='./run_tests.sh')
+        self.assertEqual(res, ["cmd cd /srv/locolander && timeout 300 ./run_tests.sh"])
 
     def test_all_mixed(self):
         config_text = """
@@ -155,14 +179,16 @@
             metadata:
                 test_script: foo
         """
-        script, docker = ns2df.parse(config_text, "/tmp/pipcache")
-        self.assertEqual(script, "foo")
+        docker = ns2df.parse(config_text, "/tmp/pipcache", 'project')
         self.assertEqual(docker,
-            "from ubuntu:precise\n"
-            "run apt-get -q -y install apache2=3.3-4\n"
-            "run apt-get -q -y install bzr\n"
-            "run pip install --download=/tmp/pipcache --no-install foobar\n"
+            "from locolander:precise\n"
+            "run mkdir -p /srv/locolander/project\n"
+            "add . /srv/locolander/project\n"
+            "run chown -R locolander.locolander /srv/locolander/project\n"
+            "run apt-get -q -y install apache2=3.3-4 bzr\n"
+            "run cd /srv/locolander/project && pip install --download=/tmp/pipcache --no-install foobar\n"
             "run ip link set dev eth0 down\n"
-            "run pip install --find-links=file:///tmp/pipcache "
-                "--no-index foobar"
+            "run cd /srv/locolander/project && pip install --find-links=file:///tmp/pipcache "
+                "--no-index foobar\n"
+            "cmd cd /srv/locolander/project && timeout 300 foo"
         )