Merge lp:~nataliabidart/magicicada-server/use-venv-dependencies into lp:magicicada-server

[Natalia Bidart]

- Use devel and non-devel dependencies from a virtualenv.
- Clean former usage of revno from versioninfo.
- Provide travis and docker files so tests can be run in Travis using a xenial image.

Tests passing!!! https://travis-ci.org/chicharreros/magicicada-server/builds/362843128

[Facundo Batista (facundo)]

Looks fine

review: Approve

[lp:~nataliabidart/magicicada-server/use-venv-dependencies]

92. By Natalia Bidart on 2018-04-12

One extra iteration of cleaning:
- use the storageprotocol from the wheel (dropping build deps for it)
- check the readme.rst with a Makefile file without needing a bash script

=== modified file '.bzrignore'
--- .bzrignore	2016-07-13 22:18:34 +0000
+++ .bzrignore	2018-04-12 02:08:49 +0000
@@ -1,5 +1,4 @@
-env
-lib/versioninfo.py
+.env
 tmp/*
 twistd.pid
 .sourcecode/*
 
=== modified file '.travis.yml'
--- .travis.yml	2016-09-01 17:10:04 +0000
+++ .travis.yml	2018-04-12 02:08:49 +0000
@@ -1,13 +1,13 @@
-addons:
-    postgresql: "9.5"
+sudo: required
+dist: trusty
+language: bash
 
 services:
-    - postgresql
+    - docker
 
 before_install:
-    - sudo apt-get update
-    - make bootstrap
+    - docker pull ubuntu:16.04
+    - docker build -t magicicada-test-run .
 
 script:
-    - make clean-sourcedeps
-    - make test SOURCEDEPS_DIR=sourcedeps
+    - docker run magicicada-test-run make test SOURCEDEPS_DIR=sourcedeps
 
=== added file 'Dockerfile'
--- Dockerfile	1970-01-01 00:00:00 +0000
+++ Dockerfile	2018-04-12 02:08:49 +0000
@@ -0,0 +1,15 @@
+FROM ubuntu:16.04
+
+ADD . /home/ubuntu/magicicada
+COPY . /home/ubuntu/magicicada
+WORKDIR /home/ubuntu/magicicada
+
+RUN apt-get update && apt-get install make -y
+RUN make docker-bootstrap
+
+RUN useradd -ms /bin/bash ubuntu
+RUN chown -R ubuntu:ubuntu /home/ubuntu
+
+USER ubuntu
+ENV HOME /home/ubuntu
+ENV PG_HOST /home/ubuntu/pg_data
 
=== modified file 'Makefile'
--- Makefile	2018-04-05 20:57:19 +0000
+++ Makefile	2018-04-12 02:08:49 +0000
@@ -17,11 +17,11 @@
 # For further info, check  http://launchpad.net/magicicada-server
 
 DJANGO_SETTINGS_MODULE ?= magicicada.settings
-FLAKE8 = flake8
-PYTHON = python
+ENV = $(CURDIR)/.env
+PYTHON = $(ENV)/bin/python
 SRC_DIR = $(CURDIR)/magicicada
 LIB_DIR = $(CURDIR)/lib
-ENV = $(CURDIR)/env
+PATH := $(ENV)/bin:$(PATH)
 PYTHONPATH := $(SRC_DIR):$(LIB_DIR):$(CURDIR):$(PYTHONPATH)
 DJANGO_ADMIN = $(LIB_DIR)/django/bin/django-admin.py
 DJANGO_MANAGE = $(PYTHON) manage.py
@@ -32,7 +32,9 @@
 PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2
 
 START_SUPERVISORD = lib/ubuntuone/supervisor/start-supervisord.py
+SUPERVISOR_CTL = $(ENV)/bin/supervisorctl
 
+export PATH
 export PYTHONPATH
 export DJANGO_SETTINGS_MODULE
 export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION
@@ -44,8 +46,6 @@
 SOURCEDEPS_SOURCECODE_DIR = $(SOURCEDEPS_DIR)/sourcecode
 TARGET_SOURCECODE_DIR = $(CURDIR)/.sourcecode
 
-BUILD_DEPLOY_SOURCEDEPS=magicicada-protocol
-
 TESTFLAGS=
 
 TAR_EXTRA = --exclude 'tmp/*' --exclude tags
@@ -65,53 +65,49 @@
 ifndef EXPORT_FROM_BZR
 	$(MAKE) link-sourcedeps
 endif
-	$(MAKE) build-sourcedeps
 	touch $(SOURCEDEPS_TAG)
 
-build: link-sourcedeps build-sourcedeps version
-
 link-sourcedeps:
 	@echo "Checking out external source dependencies..."
 	dev-scripts/link-external-sourcecode -p $(SOURCEDEPS_SOURCECODE_DIR)/ \
 		-t $(TARGET_SOURCECODE_DIR) -c config-manager.txt
 
-# no need to link sourcedeps before building them, as rollout process
-# handles config-manager.txt automatically
-build-for-deployment: build-deploy-sourcedeps version
-
-build-sourcedeps: build-deploy-sourcedeps
+build-clientdefs:
 	@echo "Building client clientdefs.py"
-	@cd .sourcecode/magicicada-client/ubuntuone/ && sed \
+	@cd $(TARGET_SOURCECODE_DIR)/magicicada-client/ubuntuone/ && sed \
 		-e 's|\@localedir\@|/usr/local/share/locale|g' \
 		-e 's|\@libexecdir\@|/usr/local/libexec|g' \
 		-e 's|\@GETTEXT_PACKAGE\@|ubuntuone-client|g' \
 		-e 's|\@VERSION\@|0.0.0|g' < clientdefs.py.in > clientdefs.py
 
-build-deploy-sourcedeps:
-	@echo "Building Python extensions"
-
-	@for sourcedep in $(BUILD_DEPLOY_SOURCEDEPS) ; do \
-            d=".sourcecode/$$sourcedep" ; \
-            if test -e "$$d/setup.py" ; then \
-	        (cd "$$d" && $(PYTHON) \
-	        setup.py build build_ext --inplace > /dev/null) ; \
-            fi ; \
-	done
-
-	@echo "Generating twistd plugin cache"
-	@$(PYTHON) -c "from twisted.plugin import IPlugin, getPlugins; list(getPlugins(IPlugin));"
-
-tarball: build-for-deployment
-	tar czf ../filesync-server.tgz $(TAR_EXTRA) .
-
 bootstrap:
 	cat dependencies.txt | sudo xargs apt-get install -y --no-install-recommends
 	cat dependencies-devel.txt | sudo xargs apt-get install -y --no-install-recommends
+	$(MAKE) $(ENV)
+	$(MAKE) sourcedeps build-clientdefs
+	mkdir -p tmp
+
+docker-bootstrap: clean
+	cat dependencies.txt | xargs apt-get install -y --no-install-recommends
+	cat dependencies-devel.txt | xargs apt-get install -y --no-install-recommends
+	$(MAKE) $(ENV)
+	$(MAKE) sourcedeps build-clientdefs
+	mkdir -p tmp
+
+$(ENV): $(ENV)/bin/activate
+
+# only runs when requirements.txt or requirements-devel.txt changes
+$(ENV)/bin/activate: requirements.txt requirements-devel.txt
+	test -d $(ENV) || virtualenv $(ENV)
+	$(ENV)/bin/pip install -Ur requirements.txt
+	$(ENV)/bin/pip install -Ur requirements-devel.txt
+	$(ENV)/bin/pip install ubuntuone-storageprotocol --no-deps -t $(TARGET_SOURCECODE_DIR)
+	touch $(ENV)/bin/activate
 
 raw-test:
-	./test $(TESTFLAGS)
+	$(PYTHON) test $(TESTFLAGS)
 
-test: lint sourcedeps clean version start-db start-base start-dbus raw-test stop
+test: lint sourcedeps start-db start-base start-dbus raw-test stop
 
 ci-test:
 	$(MAKE) test TESTFLAGS="-1 $(TESTFLAGS)"
@@ -119,16 +115,13 @@
 clean:
 	rm -rf tmp/* _trial_temp $(ENV)
 
-lint:
-	virtualenv $(ENV)
-	$(ENV)/bin/pip install flake8 rst2html5
+check-readme:
+	$(ENV)/bin/rst2html5 README.rst  --exit-status=warning > /dev/null && echo "README.rst OK"|| ( echo "ERROR: README.rst format is incorrect!!!!!" && exit 1)
+
+lint: $(ENV) check-readme
 	$(ENV)/bin/flake8 --filename='*.py' --exclude='migrations' $(SRC_DIR)
-	dev-scripts/check_readme.sh
-
-version:
-	bzr version-info --format=python > lib/versioninfo.py || true
-
-start: build start-base start-filesync-server-group publish-api-port
+
+start: $(ENV) start-base start-filesync-server-group publish-api-port
 
 resume: start-base start-filesync-server-group
 
@@ -138,7 +131,7 @@
 start-base:
 	$(MAKE) start-supervisor && $(MAKE) start-dbus || ( $(MAKE) stop ; exit 1 )
 
-stop: stop-filesync-dummy-group stop-supervisor stop-dbus
+stop: stop-supervisor stop-dbus
 
 start-dbus:
 	dev-scripts/start-dbus.sh
@@ -147,28 +140,28 @@
 	dev-scripts/stop-dbus.sh
 
 start-supervisor:
-	@python dev-scripts/supervisor-config-dev.py
+	$(PYTHON) dev-scripts/supervisor-config-dev.py
 	-@$(START_SUPERVISORD) dev-scripts/supervisor-dev.conf.tpl
 
 stop-supervisor:
-	-@dev-scripts/supervisorctl-dev shutdown
+	$(SUPERVISOR_CTL) -c $(CURDIR)/tmp/supervisor-dev.conf shutdown
 
 start-%-group:
-	-@dev-scripts/supervisorctl-dev start $*:
+	$(SUPERVISOR_CTL) -c $(CURDIR)/tmp/supervisor-dev.conf start $*:
 
 stop-%-group:
-	-@dev-scripts/supervisorctl-dev stop $*:
+	$(SUPERVISOR_CTL) -c $(CURDIR)/tmp/supervisor-dev.conf stop $*:
 
 start-%:
-	-@dev-scripts/supervisorctl-dev start $*
+	$(SUPERVISOR_CTL) -c $(CURDIR)/tmp/supervisor-dev.conf start $*
 
 stop-%:
-	-@dev-scripts/supervisorctl-dev stop $*
+	$(SUPERVISOR_CTL) -c $(CURDIR)/tmp/supervisor-dev.conf stop $*
 
 publish-api-port:
-	python -c 'from magicicada import settings; print >> file("tmp/filesyncserver.port", "w"), settings.TCP_PORT'
-	python -c 'from magicicada import settings; print >> file("tmp/filesyncserver.port.ssl", "w"), settings.SSL_PORT'
-	python -c 'from magicicada import settings; print >> file("tmp/filesyncserver-status.port", "w"), settings.API_STATUS_PORT'
+	$(PYTHON) -c 'from magicicada import settings; print >> file("tmp/filesyncserver.port", "w"), settings.TCP_PORT'
+	$(PYTHON) -c 'from magicicada import settings; print >> file("tmp/filesyncserver.port.ssl", "w"), settings.SSL_PORT'
+	$(PYTHON) -c 'from magicicada import settings; print >> file("tmp/filesyncserver-status.port", "w"), settings.API_STATUS_PORT'
 
 shell:
 	$(DJANGO_MANAGE) shell
@@ -179,7 +172,6 @@
 admin:
 	$(DJANGO_ADMIN) $(ARGS)
 
-.PHONY: sourcedeps link-sourcedeps build-sourcedeps build-deploy-sourcedeps \
-	build clean version lint test ci-test build-for-deployment \
-	clean-sourcedeps tarball start stop publish-api-port start-supervisor \
-	stop-supervisor start-dbus stop-dbus start-heapy
+.PHONY: sourcedeps link-sourcedeps clean lint test ci-test clean-sourcedeps \
+	start stop publish-api-port start-supervisor stop-supervisor \
+	start-dbus stop-dbus start-heapy check-readme
 
=== modified file 'README.rst'
--- README.rst	2017-04-22 21:07:36 +0000
+++ README.rst	2018-04-12 02:08:49 +0000
@@ -75,12 +75,6 @@
 access)::
 
     make bootstrap
-    make sourcedeps
-
-If you are in Xenial, please also install virtualenv (didn't include it in
-the previous step as it doesn't exist as a separate package in older systems)::
-
-    sudo apt-get install virtualenv
 
 Ensure the files 'privkey.pem' and 'cacert.pem' produced in the "Before server
 or client" section are copied into the ~/magicicada/magicicada-server/certs
 
=== modified file 'config-manager.txt'
--- config-manager.txt	2018-04-05 20:57:19 +0000
+++ config-manager.txt	2018-04-12 02:08:49 +0000
@@ -21,9 +21,5 @@
 #
 #   make clean-sourcedeps sourcedeps
 
-./.sourcecode/configglue                  ~configglue/configglue/trunk;revno=1
-./.sourcecode/dirspec                     ~ubuntuone-control-tower/dirspec/trunk;revno=14
-./.sourcecode/django                      ~ubuntuone-pqm-team/django/stable;revno=17
 ./.sourcecode/u1sync                      ~facundo/u1sync/opensourcing;revno=10
 ./.sourcecode/magicicada-client           ~chicharreros/magicicada-client/trunk;revno=1440
-./.sourcecode/magicicada-protocol         ~chicharreros/magicicada-protocol/trunk;revno=169
 
=== modified file 'dependencies-devel.txt'
--- dependencies-devel.txt	2017-04-22 21:07:36 +0000
+++ dependencies-devel.txt	2018-04-12 02:08:49 +0000
@@ -1,12 +1,7 @@
 dbus
-openssl
+libcairo2-dev
+libdbus-1-dev
+libdbus-glib-1-dev
+libgirepository1.0-dev
+pkg-config
 postgresql-client-9.5
-python-dbus
-python-gobject
-python-httplib2
-python-mock
-python-mocker
-python-pyinotify
-python-testtools
-python-twisted-names
-python-virtualenv
 
=== modified file 'dependencies.txt'
--- dependencies.txt	2017-04-22 21:07:36 +0000
+++ dependencies.txt	2018-04-12 02:08:49 +0000
@@ -4,19 +4,5 @@
 postgresql-9.5
 postgresql-contrib
 postgresql-plpython-9.5
-protobuf-compiler
-python-boto
-python-bson
 python-dev
-python-iso8601
-python-meliae
-python-openssl
-python-protobuf
-python-psutil
-python-psycopg2
-python-setuptools
-python-twisted-web
-python-tz
-python-virtualenv
-python-yaml
-supervisor
+virtualenv
 
=== removed file 'dev-scripts/check_readme.sh'
--- dev-scripts/check_readme.sh	2015-12-27 22:18:59 +0000
+++ dev-scripts/check_readme.sh	1970-01-01 00:00:00 +0000
@@ -1,8 +0,0 @@
-#/bin/bash
-
-OUTPUT=$(env/bin/rst2html5 README.rst 2>&1 >/dev/null)
-if [ -n "$OUTPUT" ]; then
-    echo -e "README.rst format is incorrect!!!!!\n"
-    echo -e "Errors: \n$OUTPUT"
-    exit 1
-fi
 
=== modified file 'dev-scripts/cmd_client.py'
--- dev-scripts/cmd_client.py	2018-04-05 20:57:19 +0000
+++ dev-scripts/cmd_client.py	2018-04-12 02:08:49 +0000
@@ -33,12 +33,8 @@
 from threading import Thread
 from optparse import OptionParser
 
-try:
-    from twisted.internet import gireactor
-    gireactor.install()
-except ImportError:
-    from twisted.internet import glib2reactor
-    glib2reactor.install()
+from twisted.internet import gireactor
+gireactor.install()
 
 from dbus.mainloop.glib import DBusGMainLoop
 DBusGMainLoop(set_as_default=True)
 
=== removed file 'dev-scripts/dev_launcher.py'
--- dev-scripts/dev_launcher.py	2018-04-05 20:58:20 +0000
+++ dev-scripts/dev_launcher.py	1970-01-01 00:00:00 +0000
@@ -1,87 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2008-2015 Canonical
-# Copyright 2015-2018 Chicharreros (https://launchpad.net/~chicharreros)
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-# For further info, check  http://launchpad.net/magicicada-server
-
-"""Script to execute client stuff in dev environment."""
-
-import os
-import sys
-
-import _pythonpath
-
-# fix environment before further imports
-os.environ["DJANGO_SETTINGS_MODULE"] = "magicicada.settings"
-
-from utilities import dev_launcher
-
-
-def usage():
-    """Return the usage."""
-    return ("dev_launcher PROGNAME USERNAME [--machine=N] [args*]\n"
-            "runs PROGNAME with args plus:\n"
-            "     --port with the local server port\n"
-            "     --host localhost\n"
-            "     --auth username and password for user USERNAME\n")
-
-
-def main(args):
-    """run devlauncher"""
-    from optparse import OptionParser
-    if len(args) < 3:
-        print usage()
-        return
-
-    parser = OptionParser(usage=usage())
-    parser.add_option('-p', '--port', dest='port',
-                      help="The port to where connect")
-    parser.add_option('-t', '--host', dest='host',
-                      help="The host to where connect")
-    parser.add_option('-a', '--auth', dest='auth',
-                      help="Auth data")
-    parser.add_option('-m', '--machine', dest='machine',
-                      help="Machine number (to have multiple launchers with "
-                      "the same user)")
-
-    # check to see if we have extra params for the command to execute
-    if "--" in args:
-        pos = args.index("--")
-        params = args[pos + 1:]
-        args = args[:pos]
-    else:
-        params = []
-
-    (options, pargs) = parser.parse_args(args[1:])
-    progname, username = pargs
-    if options.machine:
-        machine_suffix = "_" + options.machine
-    else:
-        machine_suffix = None
-
-    lib_dir = _pythonpath.get_lib_dir()
-    d = dict(
-        PYTHONPATH=lib_dir,
-        XDG_CACHE_HOME="tmp/xdg_cache",
-    )
-
-    dev_launcher.launch(progname, username, params, environ=d, verbose=True,
-                        machine_suffix=machine_suffix, host=options.host,
-                        api_port=options.port, auth_data=options.auth)
-
-if __name__ == "__main__":
-    main(sys.argv)
 
=== removed file 'dev-scripts/sd_test.sh'
--- dev-scripts/sd_test.sh	2018-04-05 20:57:19 +0000
+++ dev-scripts/sd_test.sh	1970-01-01 00:00:00 +0000
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# Copyright 2008-2015 Canonical
-# Copyright 2015-2018 Chicharreros (https://launchpad.net/~chicharreros)
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-# For further info, check  http://launchpad.net/magicicada-server
-
-# run the minimal infraestructure to run the sdtests isolated from the current session
-ROOTDIR=${ROOTDIR:-`bzr root`}
-ADDRESS_FILE="${ROOTDIR}/tmp/dbus.address"
-make start-dbus
-env DEBUG="file $DEBUG" SERVER_DEBUG="$SERVER_DEBUG" ROOTDIR=$ROOTDIR DBUS_SESSION_BUS_ADDRESS=$(cat $ADDRESS_FILE) XDG_CACHE_HOME="${ROOTDIR}/tmp/xdg_cache" \
-${PYTHON:-python} -Wignore ./test $@
-EXIT_VAL=$?
-make stop-dbus
-exit $EXIT_VAL
 
=== modified file 'dev-scripts/start-dbus.sh'
--- dev-scripts/start-dbus.sh	2018-04-05 20:57:19 +0000
+++ dev-scripts/start-dbus.sh	2018-04-12 02:08:49 +0000
@@ -41,4 +41,3 @@
 exec 4>&-
 
 echo "DBUS_SESSION_BUS_ADDRESS=$(cat $ADDRESS_FILE)"
-
 
=== modified file 'dev-scripts/stop-dbus.sh'
--- dev-scripts/stop-dbus.sh	2018-04-05 20:57:19 +0000
+++ dev-scripts/stop-dbus.sh	2018-04-12 02:08:49 +0000
@@ -23,7 +23,6 @@
 ADDRESS_FILE="${ROOTDIR}/tmp/dbus.address"
 
 /sbin/start-stop-daemon -v --stop --oknodo --pidfile $PID_FILE
-kill `cat $PID_FILE`
+kill `cat $PID_FILE` || true
 # cleanup the files created on start
 rm -f $PID_FILE $ADDRESS_FILE
-
 
=== modified file 'dev-scripts/supervisor-config-dev.py'
--- dev-scripts/supervisor-config-dev.py	2018-04-05 20:57:19 +0000
+++ dev-scripts/supervisor-config-dev.py	2018-04-12 02:08:49 +0000
@@ -20,24 +20,68 @@
 
 import os
 import socket
+import sys
 
 import _pythonpath  # NOQA
 
 from ubuntuone.supervisor.config_helpers import generate_server_config
-from utilities import utils
-
-from supervisor_templates import TEMPLATES
-
-ROOTDIR = utils.get_rootdir()
-TMPDIR = os.path.join(ROOTDIR, 'tmp')
-if not os.path.exists(TMPDIR):
-    os.mkdir(TMPDIR)
-
+from utilities.utils import get_rootdir, get_tmpdir
+
+
+ROOT_DIR = get_rootdir()
+TMP_DIR = get_tmpdir()
+
+
+FILESYNC_TEMPLATE = (
+    '[program:%(instance_name)s]\n'
+    'command=%(basepath)s/dev-scripts/set_rlimits.sh -d %(fs_memory_limit)s '
+    '-m %(fs_memory_limit)s -v %(fs_memory_limit)s -n %(open_fds)s '
+    '%(basepath)s/.env/bin/twistd --pidfile %(pid_folder)s/fsync_slave_%(instance)s.pid '
+    '-n -y %(basepath)s/magicicada/server/server.tac '
+    '--reactor=epoll\n'
+    'environment=DJANGO_SETTINGS_MODULE="magicicada.settings",'
+    'PYTHONPATH="%(pythonpath)s",'
+    'FSYNC_INSTANCE_ID=%(instance)03d,'
+    'CONFIG="%(basepath)s/configs/%(config)s"%(environment_vars)s\n'
+    'autostart=false\n'
+    'stdout_capture_maxbytes=%(stdout_capture_maxbytes)s\n'
+    'stopsignal=INT\n'
+)
+
+SSL_PROXY_TEMPLATE = (
+    '[program:%(instance_name)s]\n'
+    'command=%(basepath)s/dev-scripts/set_rlimits.sh -d %(ssl_memory_limit)s '
+    '-m %(ssl_memory_limit)s -v %(ssl_memory_limit)s -n %(open_fds)s '
+    '%(basepath)s/.env/bin/twistd --pidfile %(pid_folder)s/ssl-proxy-%(instance)s.pid '
+    '-n -y %(basepath)s/magicicada/server/ssl_proxy.tac '
+    '--reactor=epoll\n'
+    'environment=DJANGO_SETTINGS_MODULE="magicicada.settings",'
+    'PYTHONPATH="%(pythonpath)s",'
+    'FSYNC_INSTANCE_ID=%(instance)03d,'
+    'CONFIG="%(basepath)s/configs/%(config)s",'
+    'FSYNC_SERVICE_NAME="ssl-proxy"%(environment_vars)s\n'
+    'autostart=false\n'
+    'stdout_capture_maxbytes=%(stdout_capture_maxbytes)s\n'
+    'stopsignal=INT\n'
+)
+
+
+TEMPLATES = {
+    'filesync': {
+        'name': "filesync-worker-%(instance)d",
+        'config': FILESYNC_TEMPLATE,
+    },
+    'ssl-proxy': {
+        'name': "ssl-proxy-%(instance)d",
+        'config': SSL_PROXY_TEMPLATE,
+    },
+}
 
 config_spec = {
-    "basepath": ROOTDIR,
-    "log_folder": TMPDIR,
-    "pid_folder": TMPDIR,
+    "basepath": ROOT_DIR,
+    "log_folder": TMP_DIR,
+    "pid_folder": TMP_DIR,
+    "pythonpath": ':'.join(filter(None, sys.path)),
 }
 
 workers = {}
@@ -52,5 +96,5 @@
     config_content = generate_server_config(
         hostname, services, config_spec, TEMPLATES, None,
         with_heartbeat=False, with_header=False)
-    with open(os.path.join(TMPDIR, 'services-supervisor.conf'), 'w') as f:
+    with open(os.path.join(TMP_DIR, 'services-supervisor.conf'), 'w') as f:
         f.write(config_content)
 
=== modified file 'dev-scripts/supervisor-dev.conf.tpl'
--- dev-scripts/supervisor-dev.conf.tpl	2017-04-22 21:07:36 +0000
+++ dev-scripts/supervisor-dev.conf.tpl	2018-04-12 02:08:49 +0000
@@ -25,20 +25,20 @@
 
 [eventlistener:heartbeat]
 command=python %(basepath)s/lib/ubuntuone/supervisor/heartbeat_listener.py --interval=10 --timeout=20 --log_level=DEBUG --log_file=%(tmp_dir)s/heartbeat.log --groups=filesync-server
-environment=PYTHONPATH="%(basepath)s:%(basepath)s/lib"
+environment=PYTHONPATH="%(pythonpath)s"
 events=PROCESS_COMMUNICATION,TICK_5
 buffer_size=42
 
 [program:filesync]
-command=/usr/bin/twistd --pidfile %(tmp_dir)s/filesync.pid -n -y %(basepath)s/magicicada/server/server.tac --reactor=epoll
-environment=PYTHONPATH="%(basepath)s:%(basepath)s/lib",DJANGO_SETTINGS_MODULE="magicicada.settings",PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
+command=%(basepath)s/.env/bin/twistd --pidfile %(tmp_dir)s/filesync.pid -n -y %(basepath)s/magicicada/server/server.tac --reactor=epoll
+environment=PYTHONPATH="%(pythonpath)s",DJANGO_SETTINGS_MODULE="magicicada.settings",PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
 stdout_capture_maxbytes=16384
 autostart=false
 stopsignal=INT
 
 [program:ssl-proxy]
-command=/usr/bin/twistd --pidfile %(tmp_dir)s/ssl-proxy.pid -n -y %(basepath)s/magicicada/server/ssl_proxy.tac  --reactor=epoll
-environment=PYTHONPATH="%(basepath)s:%(basepath)s/lib",DJANGO_SETTINGS_MODULE="magicicada.settings"
+command=%(basepath)s/.env/bin/twistd --pidfile %(tmp_dir)s/ssl-proxy.pid -n -y %(basepath)s/magicicada/server/ssl_proxy.tac  --reactor=epoll
+environment=PYTHONPATH="%(pythonpath)s",DJANGO_SETTINGS_MODULE="magicicada.settings"
 stdout_capture_maxbytes=16384
 autostart=false
 stopsignal=INT
 
=== removed file 'dev-scripts/supervisor_templates.py'
--- dev-scripts/supervisor_templates.py	2018-04-05 20:57:19 +0000
+++ dev-scripts/supervisor_templates.py	1970-01-01 00:00:00 +0000
@@ -1,65 +0,0 @@
-# Copyright 2008-2015 Canonical
-# Copyright 2015-2018 Chicharreros (https://launchpad.net/~chicharreros)
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-# For further info, check  http://launchpad.net/magicicada-server
-
-"""Templates to generate supervisor config."""
-
-FILESYNC_TEMPLATE = (
-    '[program:%(instance_name)s]\n'
-    'command=%(basepath)s/dev-scripts/set_rlimits.sh -d %(fs_memory_limit)s '
-    '-m %(fs_memory_limit)s -v %(fs_memory_limit)s -n %(open_fds)s '
-    '/usr/bin/twistd --pidfile %(pid_folder)s/fsync_slave_%(instance)s.pid '
-    '-n -y %(basepath)s/magicicada/server/server.tac '
-    '--reactor=epoll\n'
-    'environment=DJANGO_SETTINGS_MODULE="magicicada.settings",'
-    'PYTHONPATH="%(basepath)s/lib",'
-    'FSYNC_INSTANCE_ID=%(instance)03d,'
-    'CONFIG="%(basepath)s/configs/%(config)s"%(environment_vars)s\n'
-    'autostart=false\n'
-    'stdout_capture_maxbytes=%(stdout_capture_maxbytes)s\n'
-    'stopsignal=INT\n'
-)
-
-SSL_PROXY_TEMPLATE = (
-    '[program:%(instance_name)s]\n'
-    'command=%(basepath)s/dev-scripts/set_rlimits.sh -d %(ssl_memory_limit)s '
-    '-m %(ssl_memory_limit)s -v %(ssl_memory_limit)s -n %(open_fds)s '
-    '/usr/bin/twistd '
-    '--pidfile %(pid_folder)s/ssl-proxy-%(instance)s.pid '
-    '-n -y %(basepath)s/magicicada/server/ssl_proxy.tac '
-    '--reactor=epoll\n'
-    'environment=DJANGO_SETTINGS_MODULE="magicicada.settings",'
-    'PYTHONPATH="%(basepath)s/lib",'
-    'FSYNC_INSTANCE_ID=%(instance)03d,'
-    'CONFIG="%(basepath)s/configs/%(config)s",'
-    'FSYNC_SERVICE_NAME="ssl-proxy"%(environment_vars)s\n'
-    'autostart=false\n'
-    'stdout_capture_maxbytes=%(stdout_capture_maxbytes)s\n'
-    'stopsignal=INT\n'
-)
-
-
-TEMPLATES = {
-    'filesync': {
-        'name': "filesync-worker-%(instance)d",
-        'config': FILESYNC_TEMPLATE,
-    },
-    'ssl-proxy': {
-        'name': "ssl-proxy-%(instance)d",
-        'config': SSL_PROXY_TEMPLATE,
-    },
-}
 
=== removed file 'dev-scripts/supervisorctl-dev'
--- dev-scripts/supervisorctl-dev	2015-08-05 13:10:02 +0000
+++ dev-scripts/supervisorctl-dev	1970-01-01 00:00:00 +0000
@@ -1,6 +0,0 @@
-#!/bin/bash
-ROOT=$(dirname $0)/..
-NAME=$(basename $0)
-
-export PYTHONPATH="$ROOT/lib"
-/usr/bin/supervisorctl -c $ROOT/tmp/${NAME/supervisorctl/supervisor}.conf $*
 
=== removed symlink 'lib/configglue'
=== target was u'../.sourcecode/configglue/configglue/'
=== removed symlink 'lib/dirspec'
=== target was u'../.sourcecode/dirspec/dirspec'
=== removed symlink 'lib/django'
=== target was u'../.sourcecode/django/django'
=== removed symlink 'lib/mock.py'
=== target was u'../.sourcecode/mock/mock.py'
=== removed symlink 'lib/oauthlib'
=== target was u'../.sourcecode/oauthlib/oauthlib/'
=== removed symlink 'lib/requests'
=== target was u'../.sourcecode/requests/requests'
=== removed symlink 'lib/requests_oauthlib'
=== target was u'../.sourcecode/requests_oauthlib/requests_oauthlib'
=== removed symlink 'lib/ubuntuone/status'
=== target was u'../../.sourcecode/magicicada-client/ubuntuone/status/'
=== modified symlink 'lib/ubuntuone/storageprotocol'
=== target changed u'../../.sourcecode/magicicada-protocol/ubuntuone/storageprotocol' => u'../../.sourcecode/ubuntuone/storageprotocol'
=== modified file 'lib/ubuntuone/supervisor/config_helpers.py'
--- lib/ubuntuone/supervisor/config_helpers.py	2018-04-05 20:57:19 +0000
+++ lib/ubuntuone/supervisor/config_helpers.py	2018-04-12 02:08:49 +0000
@@ -20,6 +20,7 @@
 
 import os
 import platform
+import sys
 
 from utilities.utils import get_rootdir
 
@@ -34,7 +35,7 @@
 HEARTBEAT_LISTENER_TEMPLATE = '\n'.join((
     "[eventlistener:heartbeat]",
     "command=python %(basepath)s/lib/ubuntuone/supervisor/heartbeat_listener.py --interval=%(interval)s --timeout=%(timeout)s --log_level=%(log_level)s --log_file=%(log_folder)s/heartbeat_listener.log --groups=%(groups)s %(processes)s",  # NOQA
-    'environment=PYTHONPATH="%(basepath)s:%(basepath)s/lib",DJANGO_SETTINGS_MODULE="magicicada.settings"',  # NOQA
+    'environment=PYTHONPATH="%(pythonpath)s",DJANGO_SETTINGS_MODULE="magicicada.settings"',  # NOQA
     "events=PROCESS_COMMUNICATION,TICK_5",
     "buffer_size=%(buffer_size)s",
 ))
@@ -68,6 +69,7 @@
     "log_folder": "/srv/%(base_dir)s/%(env)s-logs",
     "pid_folder": "/srv/%(base_dir)s/var",
     "stdout_capture_maxbytes": 16384,
+    "pythonpath": ':'.join(filter(None, sys.path)),
 }
 
 base_heartbeat_listener_spec = {
 
=== modified file 'lib/ubuntuone/supervisor/heartbeat_listener.py'
--- lib/ubuntuone/supervisor/heartbeat_listener.py	2018-04-05 20:57:19 +0000
+++ lib/ubuntuone/supervisor/heartbeat_listener.py	2018-04-12 02:08:49 +0000
@@ -219,7 +219,7 @@
 
 # main entry point, parse args and start the service.
 if __name__ == '__main__':
-    from configglue import configglue
+    from configglue.inischema.glue import configglue
     from StringIO import StringIO
     config_file = StringIO()
     config_file.write(default_config)
 
=== modified file 'lib/ubuntuone/supervisor/start-supervisord.py'
--- lib/ubuntuone/supervisor/start-supervisord.py	2018-04-05 20:57:19 +0000
+++ lib/ubuntuone/supervisor/start-supervisord.py	2018-04-12 02:08:49 +0000
@@ -36,10 +36,12 @@
     """Write out filled in conf template in tmp."""
     template = open(os.path.join(ROOT_DIR, tpl)).read()
 
-    template_vars = {}
-    template_vars['inet_http_server_port'] = inet_http_server_port
-    template_vars['basepath'] = ROOT_DIR
-    template_vars['tmp_dir'] = TMP_DIR
+    template_vars = {
+        'inet_http_server_port': inet_http_server_port,
+        'basepath': ROOT_DIR,
+        'tmp_dir': TMP_DIR,
+        'pythonpath': ':'.join(filter(None, sys.path)),
+    }
 
     conf = template % template_vars
 
@@ -79,7 +81,7 @@
     except OSError:
         pass
 
-    os_exec("/usr/bin/supervisord", "-c", conf_file_path)
+    os_exec(".env/bin/supervisord", "-c", conf_file_path)
 
 
 if __name__ == "__main__":
 
=== removed file 'magicicada/metrics/services.py'
--- magicicada/metrics/services.py	2018-04-05 20:57:19 +0000
+++ magicicada/metrics/services.py	1970-01-01 00:00:00 +0000
@@ -1,34 +0,0 @@
-# Copyright 2008-2015 Canonical
-# Copyright 2015-2018 Chicharreros (https://launchpad.net/~chicharreros)
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-# For further info, check  http://launchpad.net/magicicada-server
-
-"""Metric services."""
-
-from __future__ import unicode_literals
-
-from metrics import get_meter
-try:
-    from versioninfo import version_info
-except ImportError:
-    version_info = {'revno': 'Undefined'}
-
-meter = get_meter('service')
-
-
-def revno():
-    """Trigger a service revision number update."""
-    meter.gauge('revno', version_info['revno'])
 
=== removed file 'magicicada/metrics/tests/test_services.py'
--- magicicada/metrics/tests/test_services.py	2018-04-05 20:57:19 +0000
+++ magicicada/metrics/tests/test_services.py	1970-01-01 00:00:00 +0000
@@ -1,38 +0,0 @@
-# Copyright 2008-2015 Canonical
-# Copyright 2015-2018 Chicharreros (https://launchpad.net/~chicharreros)
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-# For further info, check  http://launchpad.net/magicicada-server
-
-"""Tests for metric services."""
-
-from __future__ import unicode_literals
-
-from mock import patch
-from testtools import TestCase
-
-from metrics import get_meter
-from metrics.services import revno, version_info
-
-
-class ServicesTest(TestCase):
-    """Tests for metric services."""
-
-    def test_meters_revno_with_gauge_meter(self):
-        """The service is able to meter a revision by gauge meter."""
-        service_meter = get_meter('service')
-        with patch.object(service_meter, 'gauge') as gauge:
-            revno()
-            gauge.assert_called_with('revno', version_info['revno'])
 
=== modified file 'magicicada/server/server.py'
--- magicicada/server/server.py	2018-04-05 20:57:19 +0000
+++ magicicada/server/server.py	2018-04-12 02:08:49 +0000
@@ -41,7 +41,6 @@
 import twisted.web.error
 
 import metrics
-import metrics.services
 
 from twisted.application.service import MultiService, Service
 from twisted.application.internet import TCPServer
@@ -60,14 +59,6 @@
 from magicicada.server import auth, content, errors, stats
 from magicicada.server.diskstorage import DiskStorage
 
-try:
-    from versioninfo import version_info
-except ImportError:
-    version_info = {
-        'branch_nick': 'Unavailable',
-        'revno': 'Unavailable',
-    }
-
 
 # this is the minimal cap we support (to avoid hardcoding it in the code)
 MIN_CAP = frozenset(['no-content', 'account-info', 'resumable-uploads',
@@ -85,6 +76,8 @@
     # frozenset(['example2']): dict(srv_record='_https._tcp.fs.server.com')
 }
 
+FILESYNC_STATUS_MSG = 'filesync server'
+
 logger = logging.getLogger(__name__)
 
 
@@ -381,8 +374,7 @@
         request.RequestHandler.connectionMade(self)
         self.factory.protocols.append(self)
         self.log.info('Connection Made')
-        msg = '%d filesync server revision %s.\r\n' % (
-            self.PROTOCOL_VERSION, version_info['revno'])
+        msg = '%d %s.\r\n' % (self.PROTOCOL_VERSION, FILESYNC_STATUS_MSG)
         self.transport.write(msg)
         self.ping_loop.start()
         self.factory.metrics.meter('connection_made', 1)
@@ -2685,7 +2677,6 @@
         self.factory.rpc_dal = self.rpc_dal
         self.metrics.meter('server_start')
         self.metrics.increment('services_active')
-        metrics.services.revno()
 
         self._reactor_inspector.start()
         # only start the HeartbeatWriter if the interval is > 0
 
=== modified file 'magicicada/server/ssl_proxy.py'
--- magicicada/server/ssl_proxy.py	2018-04-05 20:57:19 +0000
+++ magicicada/server/ssl_proxy.py	2018-04-12 02:08:49 +0000
@@ -34,7 +34,7 @@
 import metrics
 
 from magicicada import settings
-from magicicada.server.server import get_service_port
+from magicicada.server.server import FILESYNC_STATUS_MSG, get_service_port
 from magicicada.server.ssl import disable_ssl_compression
 from ubuntuone.supervisor import utils as supervisor_utils
 
@@ -164,7 +164,7 @@
 
     def lineReceived(self, line):
         """Handle a single line received."""
-        if "filesync server revision" in line:
+        if FILESYNC_STATUS_MSG in line:
             self.factory.deferred.callback(line)
         else:
             self.factory.deferred.errback(ValueError(line))
 
=== modified file 'magicicada/server/tests/test_basic.py'
--- magicicada/server/tests/test_basic.py	2018-04-05 20:57:19 +0000
+++ magicicada/server/tests/test_basic.py	2018-04-12 02:08:49 +0000
@@ -35,9 +35,6 @@
 from ubuntuone.storageprotocol import request
 from ubuntuone.supervisor import utils as supervisor_utils
 
-import metrics
-import metrics.services
-
 from magicicada.filesync.models import StorageUser
 from magicicada.server.server import logger
 from magicicada.server.testing.testcase import TestWithDatabase
@@ -120,9 +117,6 @@
         service_meter = mocker.mock(name='meter')
         self.service.metrics = service_meter
 
-        revno = mocker.mock(name='revno')
-        self.patch(metrics.services, 'revno', revno)
-
         service_meter.meter('server_stop')
         service_meter.decrement('services_active')
         service_meter.meter('server_start')
@@ -131,9 +125,6 @@
         service_meter.timing("busy.ping", ANY)
         mocker.count(0, None)
 
-        revno()
-        mocker.count(0, None)
-
         with mocker:
             yield self.service.stopService()
             yield self.service.startService()
 
=== added file 'requirements-devel.txt'
--- requirements-devel.txt	1970-01-01 00:00:00 +0000
+++ requirements-devel.txt	2018-04-12 02:08:49 +0000
@@ -0,0 +1,8 @@
+flake8
+dbus-python
+pygobject
+meliae
+mock
+mocker
+pyinotify
+rst2html5
 
=== added file 'requirements.txt'
--- requirements.txt	1970-01-01 00:00:00 +0000
+++ requirements.txt	2018-04-12 02:08:49 +0000
@@ -0,0 +1,17 @@
+Django<1.10
+bzr
+configglue
+cython
+pyOpenSSL
+protobuf
+psycopg2-binary
+# From twisted UserWarning: You do not have a working installation of the
+# service_identity module: 'No module named service_identity'.  Please install
+# it from <https://pypi.python.org/pypi/service_identity> and make sure all of
+# its dependencies are satisfied.  Without the service_identity module, Twisted
+# can perform only rudimentary TLS client hostname verification.  Many valid
+# certificate/hostname mappings may be rejected.
+service_identity
+supervisor
+twisted
+https://launchpad.net/dirspec/stable-13-10/13.10/+download/dirspec-13.10.tar.gz
 
=== modified file 'test'
--- test	2018-04-05 20:57:19 +0000
+++ test	2018-04-12 02:08:49 +0000
@@ -54,10 +54,10 @@
         with open(dbus_address_file) as fh:
             os.environ["DBUS_SESSION_BUS_ADDRESS"] = fh.read().strip()
 
-    # install the glib2reactor before any import of the reactor to avoid
+    # install the twisted reactor before any import of the reactor to avoid
     # using the default SelectReactor and be able to run the dbus tests
-    from twisted.internet import glib2reactor
-    glib2reactor.install()
+    from twisted.internet import gireactor
+    gireactor.install()
 
 
 if __name__ == "__main__":