Merge lp:~vila/ubuntu-ci-services-itself/run-workers-fix into lp:ubuntu-ci-services-itself

[Vincent Ladeuil]

fix broken run-worker change

revno(315) caused a regression because all the run-worker scripts
were dealing with sys.path in order to import things properly. This
patch adds a new "run-python" script that will launch the worker
scripts with the proper PYTHONPATH.

Fix pep8 issue blocking https://code.launchpad.net/~doanac/ubuntu-ci-services-itself/run-workers-fix/+merge/209583 landing.

I'm not using Andy's branch as a pre-requisite because doing so would block the landing. Full credit to Andy anyway.

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:319
No commit message was specified in the merge proposal. Click on the following link and set the commit message (if you want a jenkins rebuild you need to trigger it yourself):
https://code.launchpad.net/~vila/ubuntu-ci-services-itself/run-workers-fix/+merge/209623/+edit-commit-message

http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/305/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/uci-engine-ci/305/rebuild

review: Needs Fixing (continuous-integration)

[Evan (ev)]

I've done a full deploy to test this, checking that all the rabbit workers were operational.

+1

review: Approve

=== modified file 'branch-source-builder/bsbuilder/run_worker.py'
--- branch-source-builder/bsbuilder/run_worker.py	2014-03-05 11:41:47 +0000
+++ branch-source-builder/bsbuilder/run_worker.py	2014-03-06 09:35:24 +0000
@@ -16,20 +16,16 @@
 
 import json
 import logging
-import os
-import sys
 import time
 
 import upload_package
 import watch_ppa
 
+from ci_utils import amqp_utils, dump_stack
+
 logging.basicConfig(level=logging.INFO)
 log = logging.getLogger(__name__)
 
-# the worker might not have installed this module, so determine the path
-# and add it, so we can always safely import stuff
-sys.path.append(os.path.join(os.path.dirname(__file__), '../ci-utils'))
-from ci_utils import amqp_utils, dump_stack
 TIME_BETWEEN_CHECKS = 60
 
 

=== modified file 'image-builder/imagebuilder/run_worker.py'
--- image-builder/imagebuilder/run_worker.py	2014-03-05 11:20:09 +0000
+++ image-builder/imagebuilder/run_worker.py	2014-03-06 09:35:24 +0000
@@ -16,7 +16,6 @@
 
 import json
 import logging
-import os
 import sys
 from cStringIO import StringIO
 
@@ -31,9 +30,6 @@
 logstream.setLevel(logging.INFO)
 log.addHandler(logstream)
 
-# the worker might not have installed this module, so determine the path
-# and add it, so we can always safely import stuff
-sys.path.append(os.path.join(os.path.dirname(__file__), '../ci-utils'))
 from ci_utils import amqp_utils
 from ci_utils import dump_stack
 from ci_utils import data_store

=== modified file 'juju-deployer/branch-source-builder.yaml.tmpl'
--- juju-deployer/branch-source-builder.yaml.tmpl	2014-03-05 11:41:47 +0000
+++ juju-deployer/branch-source-builder.yaml.tmpl	2014-03-06 09:35:24 +0000
@@ -19,7 +19,7 @@
         bsb-worker:
             charm: rabbitmq-worker
             options:
-                main: ./branch-source-builder/bsbuilder/run_worker.py
+                main: ./run-python ./branch-source-builder/bsbuilder/run_worker.py
                 vcs: ${CI_CODE_SOURCE}
                 branch: ${CI_BRANCH}
                 tarball: ${CI_PAYLOAD_URL}

=== modified file 'juju-deployer/image-builder.yaml.tmpl'
--- juju-deployer/image-builder.yaml.tmpl	2014-03-05 11:20:09 +0000
+++ juju-deployer/image-builder.yaml.tmpl	2014-03-06 09:35:24 +0000
@@ -20,7 +20,7 @@
             charm: rabbitmq-worker
             constraints: "root-disk=8192M"
             options:
-                main: ./image-builder/imagebuilder/run_worker.py
+                main: ./run-python ./image-builder/imagebuilder/run_worker.py
                 vcs: ${CI_CODE_SOURCE}
                 branch: ${CI_BRANCH}
                 tarball: ${CI_PAYLOAD_URL}

=== modified file 'juju-deployer/lander.yaml.tmpl'
--- juju-deployer/lander.yaml.tmpl	2014-03-05 11:20:09 +0000
+++ juju-deployer/lander.yaml.tmpl	2014-03-06 09:35:24 +0000
@@ -37,7 +37,7 @@
                 packages: "python-amqplib python-swiftclient lazr.enum"
                 plugins: "parameterized-trigger:2.14"
                 unit-config: include-base64://configs/unit_config.yaml
-                main: ./lander/lander/run_worker.py --service-name ts-django --service-port 8080 --delay 60
+                main: ./run-python ./lander/lander/run_worker.py --service-name ts-django --service-port 8080 --delay 60
         rabbit:
             branch: lp:~canonical-ci-engineering/charms/precise/ubuntu-ci-services-itself/rabbitmq-server@46
             charm: rabbitmq

=== modified file 'juju-deployer/test-runner.yaml.tmpl'
--- juju-deployer/test-runner.yaml.tmpl	2014-03-05 11:20:09 +0000
+++ juju-deployer/test-runner.yaml.tmpl	2014-03-06 09:35:24 +0000
@@ -27,7 +27,7 @@
             charm: rabbitmq-worker
             options:
                 # for tests branch: lp:~vila/ubuntu-ci-services-itself/testy
-                main: ./test_runner/tstrun/run_worker.py
+                main: ./run-python ./test_runner/tstrun/run_worker.py
                 vcs: ${CI_CODE_SOURCE}
                 branch: ${CI_BRANCH}
                 tarball: ${CI_PAYLOAD_URL}

=== modified file 'lander/lander/run_worker.py'
--- lander/lander/run_worker.py	2014-03-05 11:20:09 +0000
+++ lander/lander/run_worker.py	2014-03-06 09:35:24 +0000
@@ -23,14 +23,11 @@
 import time
 import urllib2
 
-# the worker might not have installed this module, so determine the path
-# and add it, so we can always safely import stuff
-sys.path.append(os.path.join(os.path.dirname(__file__), '../ci-utils'))
 from ci_utils import jenkins_utils
 from ci_utils import dump_stack
 
 MAX_RETRIES = 15
-SERVICE_PATH = os.path.join(os.path.dirname(__file__), '../juju-relations')
+SERVICE_PATH = os.path.join(os.path.dirname(__file__), '../../juju-relations')
 
 # set up a root logger for everything
 stdout_handler = logging.StreamHandler(stream=sys.stdout)

=== added file 'run-python'
--- run-python	1970-01-01 00:00:00 +0000
+++ run-python	2014-03-06 09:35:24 +0000
@@ -0,0 +1,46 @@
+#!/usr/bin/python
+
+import os
+import sys
+
+
+def _usage():
+    print('Usage: %s script [arg [arg...]]' % sys.argv[0])
+    print('Sets up a PYTHONPATH with our ci-utils so that python scripts '
+          'can be launched properly')
+    exit(0)
+
+
+def _parse_args(args):
+    if not args or args[0] in ['-h', '--help', '-help']:
+        _usage()
+    return args[0], args
+
+
+def _component_path(basedir, script):
+    '''return the path to the component of the script'''
+    script = os.path.abspath(script)
+    if basedir not in script:
+        exit('ERROR: scripts must reside in source tree')
+    script = script[len(basedir) + 1:]
+    return script.split('/')[0]
+
+
+def main(args):
+    script, args = _parse_args(args)
+
+    here = os.path.abspath(os.path.dirname(__file__))
+    ci_utils = os.path.join(here, 'ci-utils')
+    comp = _component_path(here, script)
+
+    python_path = os.environ.get('PYTHONPATH')
+    if python_path:
+        python_path = '{}:{}:{}'.format(ci_utils, comp, python_path)
+    else:
+        python_path = '{}:{}'.format(ci_utils, comp)
+    os.environ['PYTHONPATH'] = python_path
+
+    os.execv(script, args)
+
+if __name__ == '__main__':
+    exit(main(sys.argv[1:]))

=== modified file 'test_runner/tstrun/run_worker.py'
--- test_runner/tstrun/run_worker.py	2014-03-05 11:20:09 +0000
+++ test_runner/tstrun/run_worker.py	2014-03-06 09:35:24 +0000
@@ -16,22 +16,16 @@
 
 import json
 import logging
-import os
-import sys
-
-
-logging.basicConfig(level=logging.INFO)
-log = logging.getLogger(__name__)
-
-# the worker might not have installed this module, so determine the path
-# and add it, so we can always safely import stuff
-sys.path.append(os.path.join(os.path.dirname(__file__), '../ci-utils'))
+
 from ci_utils import amqp_utils
 from ci_utils import dump_stack
 from ci_utils import data_store
 import tstrun
 from tstrun import testbed
 
+logging.basicConfig(level=logging.INFO)
+log = logging.getLogger(__name__)
+
 
 def on_message(msg):
     log.info('on_message: %s', msg.body)