Merge lp:~ricardokirkner/locolander/celery-tasks into lp:locolander

[Ricardo Kirkner]

added celery-based tasks

- run_project: processes all outstanding merge requests for a project
- run_request: runs tests and merges a merge requests
- scan_project: finds approved requests for a project

[lp:~ricardokirkner/locolander/celery-tasks]

24. By Ricardo Kirkner on 2013-07-14

merged add-author

25. By Ricardo Kirkner on 2013-07-20

fixes per review

Unmerged revisions

=== added file 'locolander/locolanderweb/migrations/0003_auto__add_field_mergerequest_author.py'
--- locolander/locolanderweb/migrations/0003_auto__add_field_mergerequest_author.py	1970-01-01 00:00:00 +0000
+++ locolander/locolanderweb/migrations/0003_auto__add_field_mergerequest_author.py	2013-07-14 15:26:27 +0000
@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+from south.db import db
+from south.v2 import SchemaMigration
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'MergeRequest.author'
+        db.add_column(u'locolanderweb_mergerequest', 'author',
+                      self.gf('django.db.models.fields.TextField')(default=''),
+                      keep_default=False)
+
+
+    def backwards(self, orm):
+        # Deleting field 'MergeRequest.author'
+        db.delete_column(u'locolanderweb_mergerequest', 'author')
+
+
+    models = {
+        u'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        u'auth.permission': {
+            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        u'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        u'locolanderweb.mergerequest': {
+            'Meta': {'object_name': 'MergeRequest'},
+            'author': ('django.db.models.fields.TextField', [], {'default': "''"}),
+            'commit_message': ('django.db.models.fields.TextField', [], {}),
+            'date_completed': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'date_created': ('django.db.models.fields.DateTimeField', [], {}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['locolanderweb.Project']"}),
+            'reviewers': ('django.db.models.fields.TextField', [], {}),
+            'source': ('django.db.models.fields.TextField', [], {}),
+            'status': ('django.db.models.fields.CharField', [], {'default': "'pending'", 'max_length': '128'}),
+            'target': ('django.db.models.fields.TextField', [], {})
+        },
+        u'locolanderweb.project': {
+            'Meta': {'object_name': 'Project'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.TextField', [], {}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['auth.User']"}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '200'})
+        },
+        u'locolanderweb.runlog': {
+            'Meta': {'object_name': 'RunLog'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'merge_request': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['locolanderweb.MergeRequest']"}),
+            'raw': ('django.db.models.fields.TextField', [], {}),
+            'return_code': ('django.db.models.fields.IntegerField', [], {}),
+            'timestamp': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'})
+        }
+    }
+
+    complete_apps = ['locolanderweb']

=== modified file 'locolander/locolanderweb/models.py'
--- locolander/locolanderweb/models.py	2013-07-14 15:26:27 +0000
+++ locolander/locolanderweb/models.py	2013-07-14 15:26:27 +0000
@@ -39,10 +39,6 @@
 
     def approved_requests(self):
         """Return approved merge proposals."""
-        mrs = self.service.approved_requests_for_project_url(self.url)
-        for mr in mrs:
-            MergeRequest.objects.get_or_create(project=self, **mr)
-
         return MergeRequest.objects.filter(project=self, status=STATUS_PENDING)
 
 

=== added file 'locolander/locolanderweb/tasks.py'
--- locolander/locolanderweb/tasks.py	1970-01-01 00:00:00 +0000
+++ locolander/locolanderweb/tasks.py	2013-07-14 15:26:27 +0000
@@ -0,0 +1,52 @@
+import os.path
+import subprocess
+
+from celery import task
+
+from locolanderweb.models import MergeRequest, RunLog
+
+
+@task
+def run_project(project):
+    for request in project.approved_requests():
+        run_request(request)
+
+
+@task
+def run_request(request):
+    project = request.project
+    source = request.source
+    target = request.target
+    author = request.author
+    message = request.commit_message
+
+    cmd = [
+        os.path.realpath(os.path.join(
+            os.path.abspath('.'), 'docker/scripts/locolander')),
+        project.name,
+        source,
+        target,
+        message,
+        author,
+    ]
+    try:
+        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+        return_code = 0
+        error = False
+    except subprocess.CalledProcessError as err:
+        output = err.output
+        return_code = err.returncode
+        error = True
+
+    RunLog.objects.create(merge_request=request, raw=output,
+                          return_code=return_code)
+
+    # update status on merge request
+    request.complete(error=error)
+
+
+@task
+def scan_project(project):
+    mrs = project.service.approved_requests_for_project_url(project.url)
+    for mr in mrs:
+        MergeRequest.objects.get_or_create(project=project, **mr)

=== modified file 'locolander/locolanderweb/tests/__init__.py'
--- locolander/locolanderweb/tests/__init__.py	2013-06-21 15:34:22 +0000
+++ locolander/locolanderweb/tests/__init__.py	2013-07-14 15:26:27 +0000
@@ -1,4 +1,5 @@
-from locolanderweb.tests.test_code_style import *
-from locolanderweb.tests.test_forms import *
-from locolanderweb.tests.test_models import *
-from locolanderweb.tests.test_views import *
+from .test_code_style import *
+from .test_forms import *
+from .test_models import *
+from .test_tasks import *
+from .test_views import *

=== modified file 'locolander/locolanderweb/tests/test_base.py'
--- locolander/locolanderweb/tests/test_base.py	2013-07-14 15:26:27 +0000
+++ locolander/locolanderweb/tests/test_base.py	2013-07-14 15:26:27 +0000
@@ -36,7 +36,7 @@
 
     def make_project(self, name=None, user=None, url=None):
         if name is None:
-            name = self.make_random_string(prefix='project-')
+            name = self.make_random_string(prefix='project')
         if user is None:
             user = self.make_user()
         if url is None:

=== modified file 'locolander/locolanderweb/tests/test_models.py'
--- locolander/locolanderweb/tests/test_models.py	2013-07-14 15:26:27 +0000
+++ locolander/locolanderweb/tests/test_models.py	2013-07-14 15:26:27 +0000
@@ -15,20 +15,6 @@
 )
 from locolanderweb.tests.test_base import BaseTestCase
 
-MERGE_REQUESTS = [
-    dict(
-        source='source1', target='target1', reviewers='a,b,c',
-        date_created=datetime(2013, 4, 23), commit_message='something 1',
-    ),
-    dict(
-        source='source2', target='target2', reviewers='pepe',
-        date_created=datetime(2012, 2, 29), commit_message='something else',
-    ),
-    dict(
-        source='source3', target='target3', reviewers='pepe y pepa',
-        date_created=datetime.utcnow(), commit_message='',
-    ),
-]
 
 INVALID_URLS = (
     'locolander',
@@ -99,16 +85,16 @@
         self.get_service_from_url.assert_called_once_with(self.obj.url)
 
     def test_approved_requests(self):
-        assert MergeRequest.objects.all().count() == 0
-
-        service = self.get_service_from_url.return_value
-        service.approved_requests_for_project_url.return_value = MERGE_REQUESTS
-
-        result = self.obj.approved_requests()
-        self.assertEqual(MergeRequest.objects.all().count(), len(result))
-        for mr in MERGE_REQUESTS:
-            self.assertEqual(
-                MergeRequest.objects.filter(**mr).count(), 1)
+        assert MergeRequest.objects.filter(project=self.obj).count() == 0
+
+        result = self.obj.approved_requests()
+        self.assertEqual(len(result), 0)
+
+        # Create an approved merge request
+        self.factory.make_merge_request(project=self.obj)
+
+        result = self.obj.approved_requests()
+        self.assertEqual(len(result), 1)
 
 
 class MergeRequestTestCase(BaseTestCase):

=== added file 'locolander/locolanderweb/tests/test_tasks.py'
--- locolander/locolanderweb/tests/test_tasks.py	1970-01-01 00:00:00 +0000
+++ locolander/locolanderweb/tests/test_tasks.py	2013-07-14 15:26:27 +0000
@@ -0,0 +1,138 @@
+import subprocess
+from datetime import datetime
+from mock import ANY, patch
+
+from locolanderweb.models import (
+    STATUS_ERROR,
+    STATUS_MERGED,
+    MergeRequest,
+    RunLog,
+)
+from locolanderweb.tasks import run_project, run_request, scan_project
+from locolanderweb.tests.test_base import BaseTestCase
+
+
+MERGE_REQUESTS = [
+    dict(
+        source='source1', target='target1', reviewers='a,b,c',
+        date_created=datetime(2013, 4, 23), commit_message='something 1',
+    ),
+    dict(
+        source='source2', target='target2', reviewers='pepe',
+        date_created=datetime(2012, 2, 29), commit_message='something else',
+    ),
+    dict(
+        source='source3', target='target3', reviewers='pepe y pepa',
+        date_created=datetime.utcnow(), commit_message='',
+    ),
+]
+
+
+class RunProjectTestCase(BaseTestCase):
+
+    @patch('locolanderweb.tasks.run_request')
+    def test_run_project(self, mock_run_request):
+        project = self.factory.make_project()
+        merge_request = self.factory.make_merge_request(project=project)
+
+        run_project(project)
+        mock_run_request.assert_called_once_with(merge_request)
+
+    @patch('locolanderweb.tasks.run_request')
+    def test_run_project_with_no_approved_requests(self, mock_run_request):
+        project = self.factory.make_project()
+
+        run_project(project)
+        self.assertFalse(mock_run_request.called)
+
+
+class RunRequestTestCase(BaseTestCase):
+
+    @patch('locolanderweb.tasks.subprocess.check_output')
+    def test_run_request(self, mock_check_output):
+        mock_check_output.return_value = 'SUCCESS'
+
+        merge_request = self.factory.make_merge_request()
+
+        run_request(merge_request)
+
+        cmd = [
+            ANY,
+            merge_request.project.name,
+            merge_request.source,
+            merge_request.target,
+            merge_request.commit_message,
+            merge_request.author,
+            ]
+        mock_check_output.assert_called_once_with(
+            cmd, stderr=subprocess.STDOUT)
+        real_cmd = mock_check_output.call_args[0][0][0]
+        self.assertTrue(real_cmd.endswith('docker/scripts/locolander'))
+
+        self.assertEqual(RunLog.objects.all().count(), 1)
+        run_log = RunLog.objects.get(merge_request=merge_request)
+        self.assertEqual(run_log.raw, 'SUCCESS')
+        self.assertEqual(run_log.return_code, 0)
+
+        # refresh MR object
+        mr = MergeRequest.objects.get(pk=merge_request.pk)
+        self.assertEqual(mr.status, STATUS_MERGED)
+
+    @patch('locolanderweb.tasks.subprocess.check_output')
+    def test_run_request_with_error(self, mock_check_output):
+        merge_request = self.factory.make_merge_request()
+
+        cmd = [
+            ANY,
+            merge_request.project.name,
+            merge_request.source,
+            merge_request.target,
+            merge_request.commit_message,
+            merge_request.author,
+            ]
+        error = subprocess.CalledProcessError(1, cmd, output='FAILURE')
+        mock_check_output.side_effect = error
+
+        run_request(merge_request)
+
+        mock_check_output.assert_called_once_with(
+            cmd, stderr=subprocess.STDOUT)
+        real_cmd = mock_check_output.call_args[0][0][0]
+        self.assertTrue(real_cmd.endswith('docker/scripts/locolander'))
+
+        self.assertEqual(RunLog.objects.all().count(), 1)
+        run_log = RunLog.objects.get(merge_request=merge_request)
+        self.assertEqual(run_log.raw, 'FAILURE')
+        self.assertEqual(run_log.return_code, 1)
+
+        # refresh MR object
+        mr = MergeRequest.objects.get(pk=merge_request.pk)
+        self.assertEqual(mr.status, STATUS_ERROR)
+
+
+class ScanProjectTestCase(BaseTestCase):
+
+    def test_scan_project(self):
+        project = self.factory.make_project()
+
+        service = self.get_service_from_url.return_value
+        service.approved_requests_for_project_url.return_value = MERGE_REQUESTS
+
+        scan_project(project)
+
+        self.assertEqual(MergeRequest.objects.filter(project=project).count(),
+                         len(MERGE_REQUESTS))
+        for mr in MERGE_REQUESTS:
+            self.assertEqual(
+                MergeRequest.objects.filter(**mr).count(), 1)
+
+    def test_scan_project_with_no_approved_request(self):
+        project = self.factory.make_project()
+
+        service = self.get_service_from_url.return_value
+        service.approved_requests_for_project_url.return_value = []
+
+        scan_project(project)
+
+        self.assertEqual(MergeRequest.objects.filter(project=project).count(),
+                         0)