Merge lp:~ricardokirkner/locolander/celery-stack into lp:locolander

[Ricardo Kirkner]

added celery stack for async tasks

[lp:~ricardokirkner/locolander/celery-stack]

17. By Ricardo Kirkner on 2013-07-13

disable TZ-aware datetime objects

[Matias Bordese (matiasb)]

LGTM

review: Approve

=== modified file '.bzrignore'
--- .bzrignore	2013-06-21 22:23:29 +0000
+++ .bzrignore	2013-07-13 17:24:24 +0000
@@ -1,2 +1,3 @@
 docker/files/ssh
 locolander.db
+supervisord.log

=== modified file '.locolander.yml'
--- .locolander.yml	2013-06-29 22:14:32 +0000
+++ .locolander.yml	2013-07-13 17:24:24 +0000
@@ -1,26 +1,5 @@
 precise:
-    pip:
-        distribute: 0.6.34
-        Django: 1.5.1
-        github3.py: 0.7.0
-        httplib2: 0.8.0
-        keyring: 1.5.0
-        launchpadlib: 1.10.2
-        lazr.authentication: 0.1.2
-        lazr.restfulclient: 0.13.3
-        lazr.uri: 1.0.3
-        mock: 1.0.1
-        oauth: 1.0.1
-        pep8: 1.4.5
-        pyflakes: 0.7.2
-        PyYAML: 3.10.0
-        requests: 1.2.3
-        simplejson: 3.3.0
-        South: 0.8.1
-        testresources: 0.2.7
-        wadllib: 1.3.2
-        wsgi-intercept: 0.5.1
-        zope.interface: 4.0.5
+    pip: -r requirements.txt
 
 metadata:
     test_script: ./run_tests.sh

=== modified file 'README'
--- README	2013-06-21 20:26:50 +0000
+++ README	2013-07-13 17:24:24 +0000
@@ -1,50 +1,40 @@
 
+==========
 LocoLander
 ==========
 
-.. image:: locolander/locolanderweb/media/locolander.png
-
-
-To Do
-=====
-
-* automate docker bootstrap
-
-  - install docker
-  - install squid
-  - setup squid as a global cache
-
-* generate Dockerfile
-
-  - parse custom project file
-  - write Dockerfile for project image
-  - include "env http_proxy http://172.16.42.1:3128" in Dockerfile to use global squid cache
-
-* build project images
-
-  - use generated Dockerfile to run 'docker build'
-
-    - step1: install base image and system packages
-    - step2: download pip dependencies from PyPI
-    - step3: cut internet access
-    - step4: install pip dependencies from locally downloaded cache
-
-  - tag built image with project name
-
-* main script
-
-  - get target branch (trunk)
-  - merge source branch into target
-  - run tests
-  - capture output
-  - commit merge and push to target upstream
-
-* main process
-
-  - build project image
-  - run main script in image
-  - extract output from container
-
-* prevent inbound/outbound network access inside of container
-
-* implement quotas (disk, cpu, mem)
+.. image::- locolander/locolanderweb/media/locolander.png
+
+
+Installing
+==========
+
+As redis cannot be installed in a virtualenv, you'll need to install it from
+the system packages
+::
+
+    sudo apt-get install redis-server
+
+Everything else can be installed in a virtualenv
+::
+
+    sudo apt-get install fabric
+    fab bootstrap
+
+
+Running
+=======
+
+Once everything is installed, you can start the full stack easily
+::
+
+    fab start_supervisor
+
+
+Developing
+==========
+
+In order to run the tests, you can just do
+::
+
+    fab test

=== added file 'TODO'
--- TODO	1970-01-01 00:00:00 +0000
+++ TODO	2013-07-13 17:24:24 +0000
@@ -0,0 +1,43 @@
+To Do
+=====
+
+* automate docker bootstrap
+
+  - install docker
+  - install squid
+  - setup squid as a global cache
+
+* generate Dockerfile
+
+  - parse custom project file
+  - write Dockerfile for project image
+  - include "env http_proxy http://172.16.42.1:3128" in Dockerfile to use global squid cache
+
+* build project images
+
+  - use generated Dockerfile to run 'docker build'
+
+    - step1: install base image and system packages
+    - step2: download pip dependencies from PyPI
+    - step3: cut internet access
+    - step4: install pip dependencies from locally downloaded cache
+
+  - tag built image with project name
+
+* main script
+
+  - get target branch (trunk)
+  - merge source branch into target
+  - run tests
+  - capture output
+  - commit merge and push to target upstream
+
+* main process
+
+  - build project image
+  - run main script in image
+  - extract output from container
+
+* prevent inbound/outbound network access inside of container
+
+* implement quotas (disk, cpu, mem)

=== added directory 'config'
=== added file 'config/supervisord.conf'
--- config/supervisord.conf	1970-01-01 00:00:00 +0000
+++ config/supervisord.conf	2013-07-13 17:24:24 +0000
@@ -0,0 +1,37 @@
+; Sample supervisor config file.
+
+[supervisord]
+nodaemon = true
+loglevel = info
+logfile = supervisord.log
+
+
+; service configuration sections
+
+[program:gunicorn]
+command = gunicorn_django locolander/locolander/settings.py
+autostart = true
+autorestart = true
+stdout_logfile = supervisord.log
+redirect_stderr = true
+
+[program:redis]
+command = redis-server
+autostart = true
+autorestart = true
+stdout_logfile = supervisord.log
+redirect_stderr = true
+
+[program:celeryd]
+command = python locolander/manage.py celeryd
+autostart = true
+autorestart = true
+stdout_logfile = supervisord.log
+redirect_stderr = true
+
+[program:flower]
+command = python locolander/manage.py celery flower
+autostart = true
+autorestart = true
+stdout_logfile = supervisord.log
+redirect_stderr = true

=== modified file 'fabfile.py'
--- fabfile.py	2013-07-09 18:42:01 +0000
+++ fabfile.py	2013-07-13 17:24:24 +0000
@@ -24,3 +24,28 @@
 @virtualenv
 def test():
     local('./run_tests.sh')
+
+
+@virtualenv
+def start_redis():
+    local('redis-server')
+
+
+@virtualenv
+def start_gunicorn():
+    local('gunicorn_django locolander/locolander/settings.py')
+
+
+@virtualenv
+def start_celery():
+    manage('celeryd')
+
+
+@virtualenv
+def start_flower():
+    manage('celery flower')
+
+
+@virtualenv
+def start_supervisor():
+    local('supervisord -c config/supervisord.conf -n')

=== modified file 'locolander/locolander/settings.py'
--- locolander/locolander/settings.py	2013-06-23 00:00:56 +0000
+++ locolander/locolander/settings.py	2013-07-13 17:24:24 +0000
@@ -2,9 +2,6 @@
 
 import os
 
-from django.core.urlresolvers import reverse
-
-
 PROJECT_DIR = os.path.abspath(os.path.dirname(__file__))
 
 DEBUG = True
@@ -134,6 +131,7 @@
     # 'django.contrib.admindocs',
     'locolanderweb',
     'south',
+    'djcelery',
 )
 
 # A sample logging configuration. The only tangible logging
@@ -165,10 +163,11 @@
     }
 }
 
-LOGIN_URL = reverse('login')
-LOGIN_REDIRECT_URL = reverse('home')
-
-try:
-    from local_setting import *
-except:
-    pass
+# settings for celery/redis
+import djcelery
+djcelery.setup_loader()
+
+BROKER_URL = 'redis://localhost:6379/0'
+
+LOGIN_URL = '/login/'
+LOGIN_REDIRECT_URL = '/'

=== modified file 'locolander/locolander/urls.py'
--- locolander/locolander/urls.py	2013-06-23 00:00:56 +0000
+++ locolander/locolander/urls.py	2013-07-13 17:24:24 +0000
@@ -1,5 +1,6 @@
 from django.conf.urls import patterns, include, url
 from django.contrib import admin
+from django.contrib.staticfiles.urls import staticfiles_urlpatterns
 admin.autodiscover()
 
 
@@ -14,3 +15,6 @@
     # Uncomment the next line to enable the admin:
     url(r'^admin/', include(admin.site.urls)),
 )
+
+# enable staticfiles for gunicorn
+urlpatterns += staticfiles_urlpatterns()

=== modified file 'requirements.txt'
--- requirements.txt	2013-07-13 15:38:09 +0000
+++ requirements.txt	2013-07-13 17:24:24 +0000
@@ -1,22 +1,40 @@
-distribute==0.6.34
+amqp==1.0.12
+anyjson==0.3.3
+billiard==2.7.3.31
+bzr==2.6b2
+celery==3.0.21
+cssselect==0.8
 Django==1.5.1
+django-celery==3.0.17
+flower==0.5.2
 github3.py==0.7.0
+gunicorn==17.5
 httplib2==0.8
 keyring==1.5
+kombu==2.5.12
 launchpadlib==1.10.2
 lazr.authentication==0.1.2
 lazr.restfulclient==0.13.3
 lazr.uri==1.0.3
+lxml==3.2.1
+meld3==0.6.10
 mock==1.0.1
 oauth==1.0.1
-pep8==1.4.5
-pyflakes==0.7.2
+pep8==1.4.6
+pyflakes==0.7.3
 pyquery==1.2.4
+python-dateutil==2.1
+pytz==2013b
 PyYAML==3.10
+redis==2.7.6
 requests==1.2.3
 simplejson==3.3.0
+six==1.3.0
 South==0.8.1
+supervisor==3.0b2
 testresources==0.2.7
+tornado==3.1
 wadllib==1.3.2
 wsgi-intercept==0.5.1
+wsgiref==0.1.2
 zope.interface==4.0.5