Merge lp:~ricardokirkner/isitdeployable/docker into lp:isitdeployable

[Ricardo Kirkner]

added tooling to build and run application using docker

[Guillermo Gonzalez (verterok)]

besides the weird missing modules errors, it worked!

review: Approve

[Otto Co-Pilot (otto-copilot)]

Project setup failed
https://jenkins.ols.canonical.com/online-services/job/isitdeployable/17/

[Otto Co-Pilot (otto-copilot)]

Project setup failed
https://jenkins.ols.canonical.com/online-services/job/isitdeployable/19/

[Otto Co-Pilot (otto-copilot)]

Project setup failed
https://jenkins.ols.canonical.com/online-services/job/isitdeployable/21/

[Otto Co-Pilot (otto-copilot)]

Project setup failed
https://jenkins.ols.canonical.com/online-services/job/isitdeployable/22/

[Otto Co-Pilot (otto-copilot)]

Project setup failed
https://jenkins.ols.canonical.com/online-services/job/isitdeployable/23/

[Otto Co-Pilot (otto-copilot)]

Project setup failed
https://jenkins.ols.canonical.com/online-services/job/isitdeployable/24/

[Otto Co-Pilot (otto-copilot)]

Project setup failed
https://jenkins.ols.canonical.com/online-services/job/isitdeployable/25/

[lp:~ricardokirkner/isitdeployable/docker]

225. By Ricardo Kirkner on 2017-08-01

do not override default $(ENV) target in docker Makefile

=== modified file '.bzrignore'
--- .bzrignore	2016-07-14 21:54:18 +0000
+++ .bzrignore	2017-08-01 11:41:20 +0000
@@ -15,7 +15,7 @@
 ./tmp/supervisor-childlogs/*
 ./.sass-cache
 ./revtracker/static/css/*
-./virtualenv
+./env
 ./lib/versioninfo.py
 ./static
 ./logs/*
 
=== added file '.dockerignore'
--- .dockerignore	1970-01-01 00:00:00 +0000
+++ .dockerignore	2017-08-01 11:41:20 +0000
@@ -0,0 +1,2 @@
+.bzr
+env
 
=== added file 'Dockerfile'
--- Dockerfile	1970-01-01 00:00:00 +0000
+++ Dockerfile	2017-08-01 11:41:20 +0000
@@ -0,0 +1,26 @@
+FROM ubuntu:trusty
+
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+
+# install system dependencies (debs)
+COPY dependencies.txt dependencies-devel.txt /usr/src/app/
+RUN apt-get update && apt-get upgrade -y \
+    && apt-get install -y software-properties-common \
+    && apt-add-repository --yes ppa:mojo-maintainers \
+    && apt-add-repository --yes ppa:ubuntuone/isitdeployable-dependencies \
+    && apt-get update \
+    && cat dependencies.txt dependencies-devel.txt | xargs apt-get install -y --no-install-recommends \
+    && apt-get install -y python-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+# install python dependencies (wheels)
+COPY branches/wheels branches/wheels
+COPY requirements.txt /usr/src/app
+RUN pip install --find-links=branches/wheels --no-index -r requirements.txt
+
+# add app source code
+COPY . /usr/src/app
+
+# start container
+CMD ["make", "app-start"]
 
=== added file 'Dockerfile.test'
--- Dockerfile.test	1970-01-01 00:00:00 +0000
+++ Dockerfile.test	2017-08-01 11:41:20 +0000
@@ -0,0 +1,4 @@
+FROM deployable:dev
+
+COPY requirements-devel.txt /usr/src/app
+RUN pip install --find-links=branches/wheels --no-index -r requirements-devel.txt
 
=== modified file 'Makefile'
--- Makefile	2016-07-27 14:51:28 +0000
+++ Makefile	2017-08-01 11:41:20 +0000
@@ -17,28 +17,20 @@
 LOCAL_SETTINGS_DIR ?= ../local_config
 LOCAL_SETTINGS_PATH = $(LOCAL_SETTINGS_DIR)/settings.py
 
-HERE := $(shell pwd)
-VIRTUALENV_DIR ?= ${HERE}/virtualenv
-VIRTUALENV_BIN := ${VIRTUALENV_DIR}/bin
+ENV = $(CURDIR)/env
 PYTHONPATH := ${HERE}/revtracker:${HERE}/lib:${HERE}:${PYTHONPATH}
 
 CM ?= /usr/lib/config-manager/cm.py
 CONFIGMANAGER = config-manager.txt
-PYTHON = $(VIRTUALENV_DIR)/bin/python
-# use env's python to run pip in order to overcome shebang's max line length limitation
-PIP = $(PYTHON) $(VIRTUALENV_DIR)/bin/pip
+GUNICORN ?= $(ENV)/bin/gunicorn
+PYTHON ?= $(ENV)/bin/python
+PIP ?= $(ENV)/bin/pip
 
 # Several targets assume (wrongly) that $(ENV) already exists. As a stop-gap
 # we ifdef them with ENV_CREATED to avoid spurious
 # '/bin/sh: 1: /.../env/bin/python: not found' errors
 ENV_CREATED = $(shell test -e $(PYTHON) && echo "yes" || echo "no")
 
-include makefiles/css.mk
-include makefiles/gem.mk
-include makefiles/js.mk
-ifeq (yes,$(ENV_CREATED))
-include Makefile.db
-endif
 WHEELS_BRANCH_URL ?= lp:isitdeployable-dependencies
 WHEELS_DIR = branches/wheels
 
@@ -78,13 +70,13 @@
 	cp django_project/settings.py.example $(LOCAL_SETTINGS_PATH)
 
 
-$(VIRTUALENV_DIR):
+$(ENV):
 	echo 'creating new virtualenv...'
-	virtualenv --clear --system-site-packages $(VIRTUALENV_DIR)
+	virtualenv --clear --system-site-packages $(ENV)
 
 clean-virtualenv:
 	echo 'cleaning virtualenv...'
-	rm -rf $(VIRTUALENV_DIR)
+	rm -rf $(ENV)
 
 version:
 	bzr version-info --format=python > lib/versioninfo.py
@@ -101,7 +93,7 @@
 
 sourcedeps: fetch-sourcedeps install-wheels
 
-bootstrap: $(LOCAL_SETTINGS_PATH) $(VIRTUALENV_DIR) sourcedeps install-wheels-dev
+bootstrap: $(LOCAL_SETTINGS_PATH) $(ENV) sourcedeps install-wheels-dev
 
 
 # Management
@@ -115,7 +107,7 @@
 run: ARGS=0.0.0.0:8000
 run:  collectstatic
 	$(MAKE) manage ARGS="loaddata fixtures/group.json"
-	$(VIRTUALENV_DIR)/bin/gunicorn django_project.wsgi:application --workers=2 --reload --pid=logs/gunicorn.pid --bind=$(ARGS) --timeout=99999
+	$(GUNICORN) django_project.wsgi:application --workers=2 --reload --pid=logs/gunicorn.pid --bind=$(ARGS) --timeout=99999
 
 start-supervisor: logs
 	@echo Starting supervised processes
@@ -141,14 +133,14 @@
 	[ -d $(WHEELS_DIR)  ] && (cd $(WHEELS_DIR) && bzr pull) || (bzr branch $(WHEELS_BRANCH_URL) $(WHEELS_DIR))
 
 update-wheels: ARGS=-r requirements-devel.txt
-update-wheels: $(VIRTUALENV_DIR)
+update-wheels: $(ENV)
 	$(PIP) wheel -w $(WHEELS_DIR) -f $(WHEELS_DIR) $(ARGS)
 
 install-wheels: ARGS=-r requirements.txt
-install-wheels: $(VIRTUALENV_DIR)
+install-wheels: $(ENV)
 	$(PIP) install --find-links=$(WHEELS_DIR) --no-index $(ARGS)
 
-install-wheels-dev: $(VIRTUALENV_DIR) install-wheels
+install-wheels-dev: $(ENV) install-wheels
 	$(MAKE) install-wheels ARGS="-r requirements-devel.txt --pre"
 
 
@@ -162,7 +154,7 @@
 REV_DIR=$(STATIC_ROOT)/combo/rev-r$$(cat $(STATIC_ROOT)/combo/revno)
 
 collectstatic: js css
-	DJANGO_SETTINGS_MODULE=django_project.settings_base $(DJANGO_MANAGE) collectstatic --noinput \
+	DJANGO_SETTINGS_MODULE=django_project.settings_build $(DJANGO_MANAGE) collectstatic --noinput \
 		> logs/collectstatic.log 2>&1 || (cat logs/collectstatic.log && false)
 	cd $(STATIC_ROOT)/combo
 	[ -L $(REV_DIR) ] && (echo 'existing rev symbolink, removing' &&  rm $(REV_DIR)) \
@@ -187,4 +179,15 @@
 	conn-check $(CONN_CHECK_CONFIG_PATH)
 
 
+# static assets related targets
+include makefiles/css.mk
+include makefiles/gem.mk
+include makefiles/js.mk
+# database related targets
+ifeq (yes,$(ENV_CREATED))
+include Makefile.db
+endif
+# deployment related targets
+include Makefile.docker
+
 .PHONY: build-tarball install-wheels
 
=== added file 'Makefile.docker'
--- Makefile.docker	1970-01-01 00:00:00 +0000
+++ Makefile.docker	2017-08-01 11:41:20 +0000
@@ -0,0 +1,62 @@
+
+PROJECT_NAME = django_project
+
+# define docker related variables
+DOCKER_COMPOSE_FILES ?= -f docker-compose.yml -f docker-compose.dev.yml
+DOCKER_COMPOSE ?= $(ENV)/bin/docker-compose
+TAG ?= deployable:dev
+
+# pre-process arguments
+# if the first argument is "special" (docker-compose, docker-test)...
+ifneq (,$(filter $(firstword $(MAKECMDGOALS)),docker-compose docker-test))
+# use the rest as sub-arguments
+ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
+# ...and turn them into do-nothing targets
+$(eval $(ARGS):;@:)
+endif
+
+
+docker-bootstrap: $(ENV) fetch-sourcedeps
+	$(ENV)/bin/pip install --find-links=$(WHEELS_DIR) --no-index -r requirements-docker.txt
+
+docker-build: version  ## Build images
+	$(MAKE) docker-compose ARGS="build"
+
+docker-run:  ## Run stack in foreground
+	DOCKER_COMPOSE_FILES="$(DOCKER_COMPOSE_FILES) -f docker-compose.secrets.yml" $(MAKE) docker-compose ARGS="up"
+
+docker-start:  ## Run stack in background
+	DOCKER_COMPOSE_FILES="$(DOCKER_COMPOSE_FILES) -f docker-compose.secrets.yml" $(MAKE) docker-compose ARGS="up -d"
+
+docker-stop:  ## Stop containers
+	DOCKER_COMPOSE_FILES="$(DOCKER_COMPOSE_FILES) -f docker-compose.secrets.yml" $(MAKE) docker-compose ARGS="down"
+
+docker-clean: docker-stop  ## Remove images
+	docker rmi $(TAG)
+
+docker-compose: $(ENV)
+	$(DOCKER_COMPOSE) $(DOCKER_COMPOSE_FILES) $(ARGS)
+
+docker-test:  ## Run tests using containers
+	DOCKER_COMPOSE_FILES='-f docker-compose.yml -f docker-compose.test.yml' $(MAKE) docker-compose ARGS="up --build -d"
+	DOCKER_COMPOSE_FILES='-f docker-compose.yml -f docker-compose.test.yml' $(MAKE) docker-compose ARGS="run app make test $(ARGS)"
+	DOCKER_COMPOSE_FILES='-f docker-compose.yml -f docker-compose.test.yml' $(MAKE) docker-compose ARGS="down"
+	docker rmi deployable:test
+
+### targets for in-container use
+
+app-update-db:
+	$(MAKE) manage ARGS='syncdb --noinput' PYTHON=python
+	$(MAKE) manage ARGS='migrate --noinput' PYTHON=python
+
+app-start: BIND=0.0.0.0:8000
+app-start: WORKERS=3
+app-start: app-update-db collectstatic
+	$(MAKE) manage ARGS="loaddata fixtures/group.json"
+	gunicorn --workers=$(WORKERS) --bind=$(BIND) $(PROJECT_NAME).wsgi:application
+
+worker-worker:
+	python $(PROJECT_NAME)/celeryapp.py worker
+
+worker-beat:
+	python $(PROJECT_NAME)/celeryapp.py beat
 
=== added file 'README.docker'
--- README.docker	1970-01-01 00:00:00 +0000
+++ README.docker	2017-08-01 11:41:20 +0000
@@ -0,0 +1,35 @@
+------
+Docker
+------
+
+It's as simple as:
+
+    $ sudo apt install docker.io config-manager
+    $ make docker-bootstrap
+    $ make docker-build
+    $ make docker-run
+
+.. important:
+
+    Notice that docker.io can't be run inside a container easily, thus the suggested way to run it is to
+    install it on your host and run it from there directly (or run everything inside a virtual machine like VirtualBox or KVM).
+
+.. important:
+
+    Before running any docker commands you'll need to add yourself to the `docker` group. Otherwise all docker commands will
+    have to be run as root.
+
+    sudo groupadd -a -G docker $USER
+    sg docker bash
+
+.. note:
+
+    You only need to run the `sg` command in the current terminal. The next time you login with your user, you should
+    already be a part of the docker group.
+
+You can now interact with the service by opening http://localhost:8000 in your browser of choice.
+
+To run the tests simply do:
+
+    $ make docker-test
+
 
=== added directory 'branches/wait-for-it'
=== added file 'branches/wait-for-it/LICENSE'
--- branches/wait-for-it/LICENSE	1970-01-01 00:00:00 +0000
+++ branches/wait-for-it/LICENSE	2017-08-01 11:41:20 +0000
@@ -0,0 +1,20 @@
+The MIT License (MIT)
+Copyright (c) 2016 Giles Hall
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
 
=== added file 'branches/wait-for-it/README.md'
--- branches/wait-for-it/README.md	1970-01-01 00:00:00 +0000
+++ branches/wait-for-it/README.md	2017-08-01 11:41:20 +0000
@@ -0,0 +1,69 @@
+# HELP NEEDED
+
+Hi there!  I wrote `wait-for-it` in order to help me orchestrate containers I operate at my day job.  I thought it was a neat little script, so I published it.  I assumed I would be its only user, but that's not what happened!  `wait-for-it` has received more stars then all of my other public repositories put together.  I had no idea this tool would solicit such an audience, and I was equally unprepared to carve out the time required to address my user's issues and patches.  I would like to solicit a volunteer from the community who would be willing to be a co-maintainer of this repository.  If this is something you might be interested in, please email me at `[e-mail]`.  Thanks!
+
+## wait-for-it
+
+`wait-for-it.sh` is a pure bash script that will wait on the availability of a host and TCP port.  It is useful for synchronizing the spin-up of interdependent services, such as linked docker containers.  Since it is a pure bash script, it does not have any external dependencies.
+
+## Usage
+
+```
+wait-for-it.sh host:port [-s] [-t timeout] [-- command args]
+-h HOST | --host=HOST       Host or IP under test
+-p PORT | --port=PORT       TCP port under test
+                            Alternatively, you specify the host and port as host:port
+-s | --strict               Only execute subcommand if the test succeeds
+-q | --quiet                Don't output any status messages
+-t TIMEOUT | --timeout=TIMEOUT
+                            Timeout in seconds, zero for no timeout
+-- COMMAND ARGS             Execute command with args after the test finishes
+```
+
+## Examples
+
+For example, let's test to see if we can access port 80 on www.google.com, and if it is available, echo the message `google is up`.
+
+```
+$ ./wait-for-it.sh www.google.com:80 -- echo "google is up"
+wait-for-it.sh: waiting 15 seconds for www.google.com:80
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+google is up
+```
+
+You can set your own timeout with the `-t` or `--timeout=` option.  Setting the timeout value to 0 will disable the timeout:
+
+```
+$ ./wait-for-it.sh -t 0 www.google.com:80 -- echo "google is up"
+wait-for-it.sh: waiting for www.google.com:80 without a timeout
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+google is up
+```
+
+The subcommand will be executed regardless if the service is up or not.  If you wish to execute the subcommand only if the service is up, add the `--strict` argument. In this example, we will test port 81 on www.google.com which will fail:
+
+```
+$ ./wait-for-it.sh www.google.com:81 --timeout=1 --strict -- echo "google is up"
+wait-for-it.sh: waiting 1 seconds for www.google.com:81
+wait-for-it.sh: timeout occurred after waiting 1 seconds for www.google.com:81
+wait-for-it.sh: strict mode, refusing to execute subprocess
+```
+
+If you don't want to execute a subcommand, leave off the `--` argument.  This way, you can test the exit condition of `wait-for-it.sh` in your own scripts, and determine how to proceed:
+
+```
+$ ./wait-for-it.sh www.google.com:80
+wait-for-it.sh: waiting 15 seconds for www.google.com:80
+wait-for-it.sh: www.google.com:80 is available after 0 seconds
+$ echo $?
+0
+$ ./wait-for-it.sh www.google.com:81
+wait-for-it.sh: waiting 15 seconds for www.google.com:81
+wait-for-it.sh: timeout occurred after waiting 15 seconds for www.google.com:81
+$ echo $?
+124
+```
+
+## Thanks
+
+I wrote this script for my employer, [Ginkgo Bioworks](http://www.ginkgobioworks.com/), who was kind enough to let me release it as an open source tool.  We are always looking to [hire](https://jobs.lever.co/ginkgobioworks) talented folks who are interested in working in the field of synthetic biology.
 
=== added file 'branches/wait-for-it/wait-for-it.sh'
--- branches/wait-for-it/wait-for-it.sh	1970-01-01 00:00:00 +0000
+++ branches/wait-for-it/wait-for-it.sh	2017-08-01 11:41:20 +0000
@@ -0,0 +1,161 @@
+#!/usr/bin/env bash
+#   Use this script to test if a given TCP host/port are available
+
+cmdname=$(basename $0)
+
+echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $TIMEOUT -gt 0 ]]; then
+        echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT"
+    else
+        echoerr "$cmdname: waiting for $HOST:$PORT without a timeout"
+    fi
+    start_ts=$(date +%s)
+    while :
+    do
+        (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1
+        result=$?
+        if [[ $result -eq 0 ]]; then
+            end_ts=$(date +%s)
+            echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $QUIET -eq 1 ]]; then
+        timeout $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
+    else
+        timeout $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
+    fi
+    PID=$!
+    trap "kill -INT -$PID" INT
+    wait $PID
+    RESULT=$?
+    if [[ $RESULT -ne 0 ]]; then
+        echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT"
+    fi
+    return $RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        hostport=(${1//:/ })
+        HOST=${hostport[0]}
+        PORT=${hostport[1]}
+        shift 1
+        ;;
+        --child)
+        CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        STRICT=1
+        shift 1
+        ;;
+        -h)
+        HOST="$2"
+        if [[ $HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        PORT="$2"
+        if [[ $PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        TIMEOUT="$2"
+        if [[ $TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        CLI="$@"
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$HOST" == "" || "$PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+TIMEOUT=${TIMEOUT:-15}
+STRICT=${STRICT:-0}
+CHILD=${CHILD:-0}
+QUIET=${QUIET:-0}
+
+if [[ $CHILD -gt 0 ]]; then
+    wait_for
+    RESULT=$?
+    exit $RESULT
+else
+    if [[ $TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        RESULT=$?
+    else
+        wait_for
+        RESULT=$?
+    fi
+fi
+
+if [[ $CLI != "" ]]; then
+    if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then
+        echoerr "$cmdname: strict mode, refusing to execute subprocess"
+        exit $RESULT
+    fi
+    exec $CLI
+else
+    exit $RESULT
+fi
 
=== modified file 'django_project/settings_base.py'
--- django_project/settings_base.py	2016-11-03 21:48:25 +0000
+++ django_project/settings_base.py	2017-08-01 11:41:20 +0000
@@ -2,120 +2,47 @@
 
 import os
 import pwd
-import sys
+from datetime import timedelta
+
+import dj_database_url
+import djcelery
+from getenv import env
 
 # Needed to have celery work from management commands
-import djcelery
 djcelery.setup_loader()
 
-from datetime import timedelta
-
-DEBUG = True
-TEMPLATE_DEBUG = DEBUG
-
-
-BASE_DIR = os.path.dirname(os.path.dirname(__file__))
-SRC_DIR = os.path.join(BASE_DIR, 'revtracker')
+
+# base directories
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_LOGS_DIR = os.path.abspath(
+    os.path.join(BASE_DIR, os.pardir, os.pardir, 'logs'))
+LOGS_DIR = os.path.abspath(os.getenv('DPA_LOGS_DIR', _LOGS_DIR))
 HOST_DIR = os.path.abspath(
     os.getenv('DPA_LP_CACHE_HOST_DIR', os.path.abspath(
         os.path.join(BASE_DIR, os.path.pardir, os.path.pardir))))
-LOGS_DIR = os.path.abspath(
-    os.getenv('DPA_LOGS_DIR', os.path.join(BASE_DIR, 'logs')))
 OOPS_DIR = os.path.join(LOGS_DIR, 'www-oops')
-
-
-ADMINS = (
-    # ('Your Name', '[e-mail]'),
+SRC_DIR = os.path.join(BASE_DIR, 'revtracker')
+
+# django settings
+
+ADMIN_MEDIA_PREFIX = '/static/admin/'
+ALLOWED_HOSTS = env('ALLOWED_HOSTS', '[]')
+AUTHENTICATION_BACKENDS = (
+    'django_openid_auth.auth.OpenIDBackend',
+    'django.contrib.auth.backends.ModelBackend',
 )
-
-MANAGERS = ADMINS
-
+CSRF_COOKIE_DOMAIN = None
+CSRF_COOKIE_HTTPONLY = False
+CSRF_COOKIE_NAME = 'csrftoken'
+CSRF_COOKIE_PATH = '/'
+CSRF_COOKIE_SECURE = False
+CSRF_FAILURE_VIEW = 'django.views.csrf.csrf_failure'
 DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        'ATOMIC_REQUESTS': False,
-        'AUTOCOMMIT': True,
-        'CONN_MAX_AGE': 600,
-        'HOST': 'localhost',
-        'NAME': 'revtracker',
-        'OPTIONS': {},
-        'PORT': '',
-        'TEST': {
-            'NAME': None,
-            'MIRROR': None,
-            'CHARSET': None,
-            'COLLATION': None,
-        },
-        'TIME_ZONE': 'UTC',
-        'USER': 'postgres',
-    },
+        'default': dj_database_url.config(
+            default='postgres://postgres@localhost/revtracker')
 }
-
-TIME_ZONE = 'UTC'
-
-LANGUAGE_CODE = 'en-us'
-
-SITE_ID = 1
-
-USE_I18N = True
-
-USE_L10N = True
-
-MEDIA_ROOT = ''
-
-MEDIA_URL = ''
-
-STATIC_ROOT = 'static'
-
-STATIC_URL = '/static/'
-
-ADMIN_MEDIA_PREFIX = '/static/admin/'
-
-STATICFILES_DIRS = (
-    os.path.join(BASE_DIR, 'build/revtracker/js'),
-)
-
-STATICFILES_FINDERS = (
-    'django.contrib.staticfiles.finders.FileSystemFinder',
-    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
-#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
-)
-
-SECRET_KEY = '6s!*-&#t8mks^8+7_2w%!oh8wh9uzzm-vzh5)l+7r2403+tm^!'
-
-TEMPLATE_LOADERS = (
-    'django.template.loaders.filesystem.Loader',
-    'django.template.loaders.app_directories.Loader',
-    'django.template.loaders.eggs.Loader',
-)
-
-MIDDLEWARE_CLASSES = (
-    'django.middleware.common.CommonMiddleware',
-    'django.contrib.sessions.middleware.SessionMiddleware',
-    'django.middleware.csrf.CsrfViewMiddleware',
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.contrib.messages.middleware.MessageMiddleware',
-    'waffle.middleware.WaffleMiddleware',
-    #'debug_toolbar.middleware.DebugToolbarMiddleware',
-)
-
-INTERNAL_IPS = ('127.0.0.1',)
-
-ROOT_URLCONF = 'django_project.urls'
-
-TEMPLATE_DIRS = (
-    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
-    # Always use forward slashes, even on Windows.
-    # Don't forget to use absolute paths, not relative paths.
-)
-
-TEMPLATE_CONTEXT_PROCESSORS = (
-    "django.contrib.auth.context_processors.auth",
-    "django.core.context_processors.csrf",
-    "django.core.context_processors.static",
-    "django.core.context_processors.request",
-)
-
+DEBUG = env('DEBUG', '').lower() == 'true'
 INSTALLED_APPS = (
     'django.contrib.auth',
     'django.contrib.contenttypes',
@@ -135,7 +62,8 @@
     'pgtools',
     #'debug_toolbar',
 )
-
+INTERNAL_IPS = ('127.0.0.1',)
+LANGUAGE_CODE = 'en-us'
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,
@@ -174,13 +102,57 @@
          },
     }
 }
+LOGIN_URL = '/openid/login'
+LOGIN_REDIRECT_URL = '/'
+MIDDLEWARE_CLASSES = (
+    'django.middleware.common.CommonMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'waffle.middleware.WaffleMiddleware',
+    #'debug_toolbar.middleware.DebugToolbarMiddleware',
+)
+ROOT_URLCONF = 'django_project.urls'
+SECRET_KEY = env('SECRET_KEY', '')
+SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
+STATIC_ROOT = 'static'
+STATIC_URL = '/static/'
+STATICFILES_DIRS = (
+    os.path.join(BASE_DIR, 'build/revtracker/js'),
+)
+STATICFILES_FINDERS = (
+    'django.contrib.staticfiles.finders.FileSystemFinder',
+    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+)
+TEMPLATE_CONTEXT_PROCESSORS = (
+    "django.contrib.auth.context_processors.auth",
+    "django.core.context_processors.csrf",
+    "django.core.context_processors.static",
+    "django.core.context_processors.request",
+)
+TEMPLATE_LOADERS = (
+    'django.template.loaders.filesystem.Loader',
+    'django.template.loaders.app_directories.Loader',
+    'django.template.loaders.eggs.Loader',
+)
+TEST_RUNNER = 'django.test.runner.DiscoverRunner'
+TIME_ZONE = 'UTC'
+USE_I18N = True
+USE_L10N = True
+
+# debug-toolbar settings
 
 DEBUG_TOOLBAR_CONFIG = {
     'INTERCEPT_REDIRECTS': False,
 }
 
-BRANCH_CACHE_DIR = "branches"
+# celery settings
 
+CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL', 'amqp://')
+CELERY_ACCEPT_CONTENT = ['json']
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
 CELERYBEAT_SCHEDULE = {
     "update_all_projects": {
         "task": "revtracker.tasks.update_all_projects",
@@ -192,14 +164,25 @@
 CELERYD_PREFETCH_MULTIPLIER = 1
 CELERYBEAT_SCHEDULE_FILENAME = 'tmp/celerybeat-schedule'
 
-LOGIN_URL = '/openid/login'
-LOGIN_REDIRECT_URL = '/'
-
-AUTHENTICATION_BACKENDS = (
-    'django_openid_auth.auth.OpenIDBackend',
-    'django.contrib.auth.backends.ModelBackend',
-    )
-
+# django-openid-auth settings
+
+OPENID_CREATE_USERS = True
+OPENID_LAUNCHPAD_TEAMS_MAPPING_AUTO = True
+OPENID_UPDATE_DETAILS_FROM_SREG = True
+OPENID_SREG_REQUIRED_FIELDS = ['nickname']
+OPENID_SSO_SERVER_URL = 'https://login.ubuntu.com/'
+
+# revtracker settings
+
+BRANCH_CACHE_DIR = "branches"
+COMBO_URL = env('COMBO_URL', 'http://127.0.0.1:8001/')
+CONVOY_ROOT = os.path.join(STATIC_ROOT, 'combo')
+GIT_SSH_DEFAULT_USERNAME = pwd.getpwuid(os.getuid()).pw_name
+GIT_SSH_DEFAULT_PORT = None
+GIT_SSH_PRIVATE_KEY = os.path.expanduser('~/.ssh/id_rsa')
+GIT_SSH_PUBLIC_KEY = os.path.expanduser('~/.ssh/id_rsa.pub')
+LP_INSTANCE = 'production'
+MULTILANG_OP_ENDPOINTS = False
 OOPSES = {
     'publishers': [{
         'inherit_id': False,
@@ -209,40 +192,6 @@
     }],
     'template': {'reporter': 'DEFAULT'},
 }
-
-USE_HTTPS = True
-
-OPENID_SSO_SERVER_URL = 'https://login.ubuntu.com/'
-OPENID_CREATE_USERS = True
-OPENID_UPDATE_DETAILS_FROM_SREG = True
-OPENID_SREG_REQUIRED_FIELDS = ['nickname']
-OPENID_LAUNCHPAD_TEAMS_MAPPING_AUTO = True
-
-MULTILANG_OP_ENDPOINTS = False
-
-COMBO_URL = 'http://127.0.0.1:8001/'
-
-CONVOY_ROOT = os.path.join(STATIC_ROOT, 'combo')
-
+SOFT_TIMEOUT_MILLISECONDS = 10000
+USE_HTTPS = env('USE_HTTPS', 'true').lower() == 'true'
 WAFFLE_OVERRIDE = True
-SOFT_TIMEOUT_MILLISECONDS = 10000
-
-CSRF_COOKIE_DOMAIN = None
-CSRF_COOKIE_HTTPONLY = False
-CSRF_COOKIE_NAME = 'csrftoken'
-CSRF_COOKIE_PATH = '/'
-CSRF_COOKIE_SECURE = False
-CSRF_FAILURE_VIEW = 'django.views.csrf.csrf_failure'
-
-TEST_RUNNER = 'django.test.runner.DiscoverRunner'
-SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
-
-LP_INSTANCE = 'production'
-
-GIT_SSH_DEFAULT_USERNAME = pwd.getpwuid(os.getuid()).pw_name
-GIT_SSH_DEFAULT_PORT = None
-GIT_SSH_PRIVATE_KEY = os.path.expanduser('~/.ssh/id_rsa')
-GIT_SSH_PUBLIC_KEY = os.path.expanduser('~/.ssh/id_rsa.pub')
-
-
-sys.path.insert(0, os.path.dirname(__file__))
 
=== added file 'django_project/settings_build.py'
--- django_project/settings_build.py	1970-01-01 00:00:00 +0000
+++ django_project/settings_build.py	2017-08-01 11:41:20 +0000
@@ -0,0 +1,12 @@
+import os
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+os.environ.setdefault(
+    'DPA_LOGS_DIR',
+    os.path.abspath(os.path.join(BASE_DIR, 'logs')))
+
+from django_project.settings_base import *  # noqa
+
+DATABASES = {}
+SECRET_KEY = 'build-secret-key'
 
=== modified file 'django_project/settings_devel.py'
--- django_project/settings_devel.py	2016-07-25 20:52:24 +0000
+++ django_project/settings_devel.py	2017-08-01 11:41:20 +0000
@@ -1,8 +1,20 @@
-from django_project.settings_base import *  # noqa
-
-DATABASES['default']['HOST'] = '/dev/shm/pg_deployable'
-USE_HTTPS = False
+import os
+
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+os.environ.setdefault('ALLOWED_HOSTS', "['*']")
 # Serve combo data from a relative url - served by project's gunicorn
 # directly.
-COMBO_URL = '/combo/'
+os.environ.setdefault('COMBO_URL', '/combo/')
+os.environ.setdefault(
+    'DATABASE_URL',
+    'postgres://postgres@%2Fdev%2Fshm%2Fpg_deployable/revtracker')
+os.environ.setdefault('DEBUG', 'true')
+os.environ.setdefault(
+    'DPA_LOGS_DIR',
+    os.path.abspath(os.path.join(BASE_DIR, 'logs')))
+os.environ.setdefault(
+    'SECRET_KEY', '6s!*-&#t8mks^8+7_2w%!oh8wh9uzzm-vzh5)l+7r2403+tm^!')
+os.environ.setdefault('USE_HTTPS', 'false')
 
+from django_project.settings_base import *  # noqa
 
=== added file 'docker-compose.dev.yml'
--- docker-compose.dev.yml	1970-01-01 00:00:00 +0000
+++ docker-compose.dev.yml	2017-08-01 11:41:20 +0000
@@ -0,0 +1,24 @@
+version: '2'
+services:
+    app:
+        restart: always
+        build: .
+        image: deployable:dev
+        command: ["./branches/wait-for-it/wait-for-it.sh", "db:5432", "--", "make", "app-start"]
+        volumes:
+          - .:/usr/src/app
+        ports:
+            - "8000:8000"
+        environment:
+            CELERY_BROKER_URL: amqp://amqp
+            DATABASE_URL: postgres://postgres@db/postgres
+            DJANGO_SETTINGS_MODULE: "django_project.settings_devel"
+    worker:
+        restart: always
+        image: deployable:dev
+        volumes:
+          - .:/usr/src/app
+        environment:
+            CELERY_BROKER_URL: amqp://amqp
+            DATABASE_URL: postgres://postgres@db/postgres
+            DJANGO_SETTINGS_MODULE: "django_project.settings_devel"
 
=== added file 'docker-compose.secrets.yml'
--- docker-compose.secrets.yml	1970-01-01 00:00:00 +0000
+++ docker-compose.secrets.yml	2017-08-01 11:41:20 +0000
@@ -0,0 +1,5 @@
+version: '2'
+services:
+    app:
+#        environment:
+#            SECRET_KEY: 'changeme'
 
=== added file 'docker-compose.test.yml'
--- docker-compose.test.yml	1970-01-01 00:00:00 +0000
+++ docker-compose.test.yml	2017-08-01 11:41:20 +0000
@@ -0,0 +1,9 @@
+version: '2'
+services:
+    app:
+        extends:
+            file: docker-compose.dev.yml
+            service: app
+        build:
+            dockerfile: Dockerfile.test
+        image: deployable:test
 
=== added file 'docker-compose.yml'
--- docker-compose.yml	1970-01-01 00:00:00 +0000
+++ docker-compose.yml	2017-08-01 11:41:20 +0000
@@ -0,0 +1,19 @@
+version: '2'
+services:
+    db:
+        image: postgres
+        expose:
+            - "5432"
+    amqp:
+        image: rabbitmq
+    app:
+        build: .
+        image: deployable
+        expose:
+            - "8000"
+        depends_on:
+            - db
+            - amqp
+    worker:
+        image: deployable
+        command: ["make", "worker-worker", "worker-beat"]
 
=== added file 'requirements-docker.txt'
--- requirements-docker.txt	1970-01-01 00:00:00 +0000
+++ requirements-docker.txt	2017-08-01 11:41:20 +0000
@@ -0,0 +1,19 @@
+# direct dependencies
+docker_compose==1.14.0
+
+# indirect dependencies
+PyYAML==3.12
+backports.ssl_match_hostname==3.5.0.1
+cached_property==1.3.0
+colorama==0.3.9
+docker==2.4.2
+docker_pycreds==0.2.1
+dockerpty==0.4.1
+docopt==0.6.2
+enum34==1.1.6
+functools32==3.2.3-2
+ipaddress==1.0.18
+jsonschema==2.6.0
+requests==2.11.1
+texttable==0.8.8
+websocket_client==0.44.0
 
=== modified file 'requirements.txt'
--- requirements.txt	2016-09-02 22:32:27 +0000
+++ requirements.txt	2017-08-01 11:41:20 +0000
@@ -11,6 +11,7 @@
 django-debug-toolbar==0.10.0
 django-devserver==0.7.0
 django-factory==0.10
+django-getenv==1.3.1
 django-kombu==0.9.4
 django-openid-auth==0.9
 # A checkout of https://github.com/bfirsh/django-ordered-model at 2998740eeda348473d8a8965b40b2292c31efd85
@@ -18,6 +19,7 @@
 django-picklefield==0.2.1
 django-pgtools==0.2
 django-waffle==0.10.1
+dj-database-url==0.4.2
 gorun==1.7
 gunicorn==19.3.0
 html5lib==0.9999999