Merge lp:~ricardokirkner/locolander/django-supervisor into lp:locolander

[Ricardo Kirkner]

simplified fabfile for supervisor tasks

[lp:~ricardokirkner/locolander/django-supervisor]

34. By Ricardo Kirkner on 2013-07-29

better output from script

35. By Ricardo Kirkner on 2013-08-17

added config for docker process

36. By Ricardo Kirkner on 2013-08-17

merged trunk

Unmerged revisions

36. By Ricardo Kirkner on 2013-08-17

merged trunk

35. By Ricardo Kirkner on 2013-08-17

added config for docker process

34. By Ricardo Kirkner on 2013-07-29

better output from script

33. By Ricardo Kirkner on 2013-07-29

replace subprocess with envoy

use envoy instead of subprocess, because the latter fails to run properly
when celeryd is started under supervisor.

32. By Ricardo Kirkner on 2013-07-29

customized model repr strings

31. By Ricardo Kirkner on 2013-07-29

use absolute path for database file

30. By Ricardo Kirkner on 2013-07-29

better output handling

29. By Ricardo Kirkner on 2013-07-29

disable autoreload

28. By Ricardo Kirkner on 2013-07-29

added envoy and django-supervisor as dependencies

27. By Ricardo Kirkner on 2013-07-27

merged branch lp:~ricardokirkner/locolander/lean-output

=== modified file '.locolander.yml'
--- .locolander.yml	2013-07-23 23:55:56 +0000
+++ .locolander.yml	2013-08-17 00:46:15 +0000
@@ -5,9 +5,6 @@
         libxml2-dev:
         libxslt1-dev:
     pip:
-        Django: 1.5.1
-        PyYAML: 3.10.0
-        South: 0.8.1
         amqp: 1.0.12
         anyjson: 0.3.3
         argparse: 1.2.1
@@ -16,8 +13,11 @@
         celery: 3.0.21
         cssselect: 0.8
         distribute: 0.6.34
+        Django: 1.5.1
         django-allauth: 0.12.0
         django-celery: 3.0.17
+        django-supervisor: 0.3.1
+        envoy: 0.0.2
         flower: 0.5.2
         github3.py: 0.7.0
         gunicorn: 17.5
@@ -33,6 +33,7 @@
         mock: 1.0.1
         oauth: 1.0.1
         oauthlib: 0.5.0
+        pathtools: 0.1.2
         pep8: 1.4.6
         psycopg2: 2.5.1
         pyflakes: 0.7.3
@@ -40,15 +41,18 @@
         python-dateutil: 2.1
         python-openid: 2.2.5
         pytz: 2013b
+        PyYAML: 3.10.0
         redis: 2.7.6
         requests: 1.2.3
         requests-oauthlib: 0.3.2
         simplejson: 3.3.0
         six: 1.3.0
+        South: 0.8.1
         supervisor: 3.0b2
         testresources: 0.2.7
         tornado: 3.1
         wadllib: 1.3.2
+        watchdog: 0.6.0
         wsgi-intercept: 0.5.1
         wsgiref: 0.1.2
         zope.interface: 4.0.5

=== removed file 'config/supervisord.conf'
--- config/supervisord.conf	2013-07-13 17:01:27 +0000
+++ config/supervisord.conf	1970-01-01 00:00:00 +0000
@@ -1,37 +0,0 @@
-; Sample supervisor config file.
-
-[supervisord]
-nodaemon = true
-loglevel = info
-logfile = supervisord.log
-
-
-; service configuration sections
-
-[program:gunicorn]
-command = gunicorn_django locolander/locolander/settings.py
-autostart = true
-autorestart = true
-stdout_logfile = supervisord.log
-redirect_stderr = true
-
-[program:redis]
-command = redis-server
-autostart = true
-autorestart = true
-stdout_logfile = supervisord.log
-redirect_stderr = true
-
-[program:celeryd]
-command = python locolander/manage.py celeryd
-autostart = true
-autorestart = true
-stdout_logfile = supervisord.log
-redirect_stderr = true
-
-[program:flower]
-command = python locolander/manage.py celery flower
-autostart = true
-autorestart = true
-stdout_logfile = supervisord.log
-redirect_stderr = true

=== modified file 'docker/Dockerfile'
--- docker/Dockerfile	2013-06-21 21:48:36 +0000
+++ docker/Dockerfile	2013-08-17 00:46:15 +0000
@@ -24,10 +24,6 @@
 # add locolander user
 run adduser locolander
 
-# install main locolander script
-add ./scripts/run_tests.sh /usr/local/bin/run_tests.sh
-run chown locolander.locolander /usr/local/bin/run_tests.sh
-
 # configure ssh
 run mkdir /home/locolander/.ssh
 add ./files/ssh/id_rsa /home/locolander/.ssh/id_rsa
@@ -40,3 +36,8 @@
 
 # ensure proper ownership
 run chown -R locolander.locolander /home/locolander/
+
+# install main locolander script
+add ./scripts/locolander /usr/local/bin/locolander
+run chown locolander.locolander /usr/local/bin/locolander
+

=== modified file 'docker/scripts/locolander'
--- docker/scripts/locolander	2013-07-27 00:04:40 +0000
+++ docker/scripts/locolander	2013-08-17 00:46:15 +0000
@@ -56,8 +56,8 @@
 OUTPUT=$(python ns2df.py $PROJECT .locolander.yml Dockerfile 2>&1)
 OUTPUT=$(docker build -t $PROJECT . 2>&1)
 if [ -z $? -o "`echo $OUTPUT | grep 'Error'`" != "" ]; then
-    echo "FAILURE - Build failed:"
     echo $OUTPUT
+    echo "FAILURE - Build failed"
     exit 1
 fi
 
@@ -67,9 +67,8 @@
 
 # 4. run test script using image
 echo "Running tests ..."
-docker run $PROJECT
-
-if [ -z $? -o "`echo $OUTPUT | grep 'Error'`" != "" ]; then
+docker run $PROJECT 2>&1
+if [ -z $? ]; then
     echo "FAILURE - Tests failed"
     exit 1
 fi
@@ -87,7 +86,7 @@
 echo "Pushing merge ..."
 OUTPUT=$(bzr push $TARGET 2>&1)
 
-echo "SUCCESS"
-
 # cleanup
 rm -rf $TMPFOLDER
+
+echo "SUCCESS"

=== modified file 'fabfile.py'
--- fabfile.py	2013-07-17 13:33:49 +0000
+++ fabfile.py	2013-08-17 00:46:15 +0000
@@ -23,30 +23,16 @@
 
 
 @virtualenv
+def run():
+    manage('supervisor')
+
+
+@virtualenv
 def test():
     local('./run_tests.sh')
 
 
 @virtualenv
-def start_redis():
-    local('redis-server')
-
-
-@virtualenv
-def start_gunicorn():
-    local('gunicorn_django locolander/locolander/settings.py')
-
-
-@virtualenv
-def start_celery():
-    manage('celeryd')
-
-
-@virtualenv
-def start_flower():
-    manage('celery flower')
-
-
-@virtualenv
-def start_supervisor():
-    local('supervisord -c config/supervisord.conf -n')
+def docker_build():
+    local('python ns2df.py locolander .locolander.yml Dockerfile')
+    local('docker build -t locolander .')

=== modified file 'locolander/locolander/settings.py'
--- locolander/locolander/settings.py	2013-07-21 03:05:23 +0000
+++ locolander/locolander/settings.py	2013-08-17 00:46:15 +0000
@@ -16,7 +16,7 @@
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': 'locolander.db',
+        'NAME': os.path.abspath(os.path.join(os.curdir, 'locolander.db')),
         # The following settings are not used with sqlite3:
         'USER': '',
         'PASSWORD': '',
@@ -126,9 +126,10 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django.contrib.admin',
-    'djcelery',
     'locolanderweb',
     'south',
+    'djcelery',
+    'djsupervisor',
     'allauth',
     'allauth.account',
     'allauth.socialaccount',

=== modified file 'locolander/locolanderweb/models.py'
--- locolander/locolanderweb/models.py	2013-07-13 18:44:18 +0000
+++ locolander/locolanderweb/models.py	2013-08-17 00:46:15 +0000
@@ -33,6 +33,9 @@
     name = models.TextField()
     url = models.CharField(max_length=256, validators=[service_url_validator])
 
+    def __repr__(self):
+        return u"<Project: %s>" % self.name
+
     @property
     def service(self):
         return get_service_from_url(self.url)
@@ -56,6 +59,9 @@
     date_created = models.DateTimeField()
     date_completed = models.DateTimeField(null=True, blank=True)
 
+    def __repr__(self):
+        return u"<MergeRequest: %s -> %s>" % (self.source, self.target)
+
     def complete(self, error=False):
         self.date_completed = datetime.utcnow()
         self.status = STATUS_MERGED if not error else STATUS_ERROR
@@ -71,3 +77,6 @@
 
     class Meta:
         get_latest_by = 'timestamp'
+
+    def __repr__(self):
+        return u"<RunLog: %r at %s>" % (self.merge_request, self.timestamp)

=== modified file 'locolander/locolanderweb/tasks.py'
--- locolander/locolanderweb/tasks.py	2013-08-03 17:06:41 +0000
+++ locolander/locolanderweb/tasks.py	2013-08-17 00:46:15 +0000
@@ -1,7 +1,7 @@
 import os
-import subprocess
 from datetime import datetime
 
+import envoy
 from celery import task
 
 from locolanderweb.models import MergeRequest, RunLog
@@ -21,23 +21,16 @@
     author = request.author
     message = request.commit_message
 
-    cmd = [
-        os.path.realpath(os.path.join(
-            os.path.abspath('.'), 'docker/scripts/locolander')),
-        project.name,
-        source,
-        target,
-        message,
-        author,
-    ]
-    try:
-        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-        return_code = 0
-        error = False
-    except subprocess.CalledProcessError as err:
-        output = err.output
-        return_code = err.returncode
-        error = True
+    cmd = '{script} {project} {source} {target} "{message}" "{author}"'.format(
+        script=os.path.join(os.path.abspath(os.curdir),
+                            'docker', 'scripts', 'locolander'),
+        project=project.name, source=source, target=target, message=message,
+        author=author)
+
+    r = envoy.run(cmd)
+    output = r.std_out
+    return_code = r.status_code
+    error = return_code != 0
 
     RunLog.objects.create(merge_request=request, raw=output,
                           return_code=return_code)

=== modified file 'locolander/locolanderweb/tests/test_models.py'
--- locolander/locolanderweb/tests/test_models.py	2013-07-20 21:10:37 +0000
+++ locolander/locolanderweb/tests/test_models.py	2013-08-17 00:46:15 +0000
@@ -96,6 +96,10 @@
         result = self.obj.approved_requests()
         self.assertEqual(len(result), 1)
 
+    def test_repr(self):
+        expected = u'<Project: foo>'
+        self.assertEqual(repr(self.obj), expected)
+
 
 class MergeRequestTestCase(BaseTestCase):
 
@@ -156,14 +160,20 @@
         self.assertEqual(self.obj.status, STATUS_ERROR)
         self.assertEqual(self.obj.date_completed, completed)
 
+    def test_repr(self):
+        expected = u'<MergeRequest: source -> target>'
+        self.assertEqual(repr(self.obj), expected)
+
 
 class RunLogTestCase(BaseTestCase):
 
     def setUp(self):
         super(RunLogTestCase, self).setUp()
         self.merge_request = self.factory.make_merge_request()
+        self.timestamp = datetime.utcnow()
         self.obj = RunLog.objects.create(
-            merge_request=self.merge_request, return_code=0)
+            merge_request=self.merge_request, return_code=0,
+            timestamp=self.timestamp)
 
     def test_merge_request(self):
         self.assertEqual(self.obj.merge_request, self.merge_request)
@@ -187,3 +197,7 @@
             merge_request=self.merge_request, return_code=0,
             timestamp=self.obj.timestamp + timedelta(seconds=1))
         self.assertEqual(RunLog.objects.latest(), other)
+
+    def test_repr(self):
+        expected = u"<RunLog: %r at %s>" % (self.merge_request, self.timestamp)
+        self.assertEqual(repr(self.obj), expected)

=== modified file 'locolander/locolanderweb/tests/test_tasks.py'
--- locolander/locolanderweb/tests/test_tasks.py	2013-07-20 21:07:18 +0000
+++ locolander/locolanderweb/tests/test_tasks.py	2013-08-17 00:46:15 +0000
@@ -1,7 +1,7 @@
-import subprocess
+import os
 from datetime import datetime
 
-from mock import ANY, patch
+from mock import patch
 
 from locolanderweb.models import (
     STATUS_ERROR,
@@ -55,29 +55,28 @@
 
     def setUp(self):
         super(RunRequestTestCase, self).setUp()
-        patcher = patch('locolanderweb.tasks.subprocess.check_output')
-        self.mock_check_output = patcher.start()
+        patcher = patch('locolanderweb.tasks.envoy.run')
+        self.mock_run = patcher.start()
         self.addCleanup(patcher.stop)
 
     def test_run_request(self):
-        self.mock_check_output.return_value = 'SUCCESS'
+        self.mock_run.return_value.std_out = 'SUCCESS'
+        self.mock_run.return_value.status_code = 0
 
         merge_request = self.factory.make_merge_request()
 
         run_request(merge_request)
 
-        cmd = [
-            ANY,
-            merge_request.project.name,
-            merge_request.source,
-            merge_request.target,
-            merge_request.commit_message,
-            merge_request.author,
-            ]
-        self.mock_check_output.assert_called_once_with(
-            cmd, stderr=subprocess.STDOUT)
-        real_cmd = self.mock_check_output.call_args[0][0][0]
-        self.assertTrue(real_cmd.endswith('docker/scripts/locolander'))
+        cmd = ('{script} {project} {source} {target} '
+               '"{message}" "{author}"').format(
+            script=os.path.abspath('./docker/scripts/locolander'),
+            project=merge_request.project.name,
+            source=merge_request.source,
+            target=merge_request.target,
+            message=merge_request.commit_message,
+            author=merge_request.author
+        )
+        self.mock_run.assert_called_once_with(cmd)
 
         self.assertEqual(RunLog.objects.all().count(), 1)
         run_log = RunLog.objects.get(merge_request=merge_request)
@@ -91,23 +90,21 @@
     def test_run_request_with_error(self):
         merge_request = self.factory.make_merge_request()
 
-        cmd = [
-            ANY,
-            merge_request.project.name,
-            merge_request.source,
-            merge_request.target,
-            merge_request.commit_message,
-            merge_request.author,
-            ]
-        error = subprocess.CalledProcessError(1, cmd, output='FAILURE')
-        self.mock_check_output.side_effect = error
+        cmd = ('{script} {project} {source} {target} '
+               '"{message}" "{author}"').format(
+            script=os.path.abspath('./docker/scripts/locolander'),
+            project=merge_request.project.name,
+            source=merge_request.source,
+            target=merge_request.target,
+            message=merge_request.commit_message,
+            author=merge_request.author
+        )
+        self.mock_run.return_value.std_out = 'FAILURE'
+        self.mock_run.return_value.status_code = 1
 
         run_request(merge_request)
 
-        self.mock_check_output.assert_called_once_with(
-            cmd, stderr=subprocess.STDOUT)
-        real_cmd = self.mock_check_output.call_args[0][0][0]
-        self.assertTrue(real_cmd.endswith('docker/scripts/locolander'))
+        self.mock_run.assert_called_once_with(cmd)
 
         self.assertEqual(RunLog.objects.all().count(), 1)
         run_log = RunLog.objects.get(merge_request=merge_request)

=== added file 'locolander/supervisord.conf'
--- locolander/supervisord.conf	1970-01-01 00:00:00 +0000
+++ locolander/supervisord.conf	2013-08-17 00:46:15 +0000
@@ -0,0 +1,38 @@
+[program:gunicorn]
+command = gunicorn -b 0.0.0.0:8000 locolander.wsgi:application
+directory = {{ PROJECT_DIR }}
+autostart = true
+autorestart = true
+stdout_logfile = supervisord.log
+redirect_stderr = true
+
+[program:redis]
+command = redis-server
+autostart = true
+autorestart = true
+stdout_logfile = supervisord.log
+redirect_stderr = true
+
+[program:docker]
+command = /usr/bin/docker
+autostart = false
+autorestart = true
+stdout_logfile = supervisord.log
+redirect_stderr = true
+
+[program:celeryd]
+command = {{ PYTHON }} {{ PROJECT_DIR }}/manage.py celeryd
+autostart = true
+autorestart = true
+stdout_logfile = supervisord.log
+redirect_stderr = true
+
+[program:flower]
+command = {{ PYTHON }} {{ PROJECT_DIR }}/manage.py celery flower
+autostart = true
+autorestart = true
+stdout_logfile = supervisord.log
+redirect_stderr = true
+
+[program:autoreload]
+exclude = true

=== modified file 'requirements.txt'
--- requirements.txt	2013-07-20 23:52:03 +0000
+++ requirements.txt	2013-08-17 00:46:15 +0000
@@ -1,16 +1,17 @@
-Django==1.5.1
-PyYAML==3.10
-South==0.8.1
 amqp==1.0.12
 anyjson==0.3.3
+argh==0.23.2
 argparse==1.2.1
 billiard==2.7.3.31
 bzr==2.6b2
 celery==3.0.21
 cssselect==0.8
 distribute==0.6.34
+Django==1.5.1
 django-allauth==0.12.0
 django-celery==3.0.17
+django-supervisor==0.3.1
+envoy==0.0.2
 flower==0.5.2
 github3.py==0.7.0
 gunicorn==17.5
@@ -26,6 +27,7 @@
 mock==1.0.1
 oauth==1.0.1
 oauthlib==0.5.0
+pathtools==0.1.2
 pep8==1.4.6
 psycopg2==2.5.1
 pyflakes==0.7.3
@@ -33,15 +35,18 @@
 python-dateutil==2.1
 python-openid==2.2.5
 pytz==2013b
+PyYAML==3.10
 redis==2.7.6
 requests==1.2.3
 requests-oauthlib==0.3.2
 simplejson==3.3.0
 six==1.3.0
+South==0.8.1
 supervisor==3.0b2
 testresources==0.2.7
 tornado==3.1
 wadllib==1.3.2
+watchdog==0.6.0
 wsgi-intercept==0.5.1
 wsgiref==0.1.2
 zope.interface==4.0.5