Merge ~morphis/snappy-hwe-snaps/+git/build-scripts:feature/parallel-spread into ~snappy-hwe-team/snappy-hwe-snaps/+git/build-scripts:master

[Simon Fels]

Allow multiple spread jobs running in parallel by copying the spread binary into a job specific location.

[Jim Hodapp (jhodapp)]

One question inline below.

review: Needs Information

[Jim Hodapp (jhodapp)]

review: Approve

[System Enablement Bot (system-enablement-ci-bot)]

PASSED: Continuous integration, rev:b346d5b3c233536c453671c60c6f4f4edd84f5d2
https://jenkins.canonical.com/system-enablement/job/generic-build-snap/798/
Executed test runs:
    SUCCESS: https://jenkins.canonical.com/system-enablement/job/generic-run-snap-spread-tests/498
    None: https://jenkins.canonical.com/system-enablement/job/generic-update-snap-mp/706/console

Click here to trigger a rebuild:
https://jenkins.canonical.com/system-enablement/job/generic-build-snap/798/rebuild

review: Approve (continuous-integration)

[System Enablement Bot (system-enablement-ci-bot)]

PASSED: Continuous integration, rev:3f4f5f652e5f10f84f580c2db3c0d85b92b6a024
https://jenkins.canonical.com/system-enablement/job/generic-build-snap/802/
Executed test runs:
    SUCCESS: https://jenkins.canonical.com/system-enablement/job/generic-run-snap-spread-tests/501
    None: https://jenkins.canonical.com/system-enablement/job/generic-update-snap-mp/708/console

Click here to trigger a rebuild:
https://jenkins.canonical.com/system-enablement/job/generic-build-snap/802/rebuild

review: Approve (continuous-integration)

[System Enablement Bot (system-enablement-ci-bot)]

PASSED: Continuous integration, rev:e5c8acc9f901ecc9afc0b553a3d61b2916c5badf
https://jenkins.canonical.com/system-enablement/job/generic-build-snap/801/
Executed test runs:
    SUCCESS: https://jenkins.canonical.com/system-enablement/job/generic-run-snap-spread-tests/500
    None: https://jenkins.canonical.com/system-enablement/job/generic-update-snap-mp/710/console

Click here to trigger a rebuild:
https://jenkins.canonical.com/system-enablement/job/generic-build-snap/801/rebuild

review: Approve (continuous-integration)

[System Enablement Bot (system-enablement-ci-bot)]

PASSED: Continuous integration, rev:3f4f5f652e5f10f84f580c2db3c0d85b92b6a024
https://jenkins.canonical.com/system-enablement/job/generic-build-snap/809/
Executed test runs:
    SUCCESS: https://jenkins.canonical.com/system-enablement/job/generic-run-snap-spread-tests/505
    None: https://jenkins.canonical.com/system-enablement/job/generic-update-snap-mp/717/console

Click here to trigger a rebuild:
https://jenkins.canonical.com/system-enablement/job/generic-build-snap/809/rebuild

review: Approve (continuous-integration)

diff --git a/jobs/generic-run-snap-spread-tests b/jobs/generic-run-snap-spread-tests
index ae9bd4d..5d3d025 100644
--- a/jobs/generic-run-snap-spread-tests
+++ b/jobs/generic-run-snap-spread-tests
@@ -20,18 +20,11 @@ set -e
 SSH_PATH="${JENKINS_HOME}/.ssh/"
 SSH_KEY_PATH="${SSH_PATH}/bazaar.launchpad.net/system-enablement-ci-bot"
 
-JENKINS_WORK_DIR=${JENKINS_HOME}/build
-
-if  [ -e $JENKINS_WORK_DIR ] ; then
-    rm -rf $JENKINS_WORK_DIR
-    mkdir -p $JENKINS_WORK_DIR
-fi
-
-cd $JENKINS_WORK_DIR
-
 SSH="ssh -i $SSH_KEY_PATH/id_rsa $REMOTE_USER@$REMOTE_WORKER"
 SCP="scp -i $SSH_KEY_PATH/id_rsa"
 
+REMOTE_WORKSPACE=/home/$REMOTE_USER/$BUILD_TAG
+
 # Do a quick check if tests are maybe disabled for the repository
 # if not applicable.
 tmpdir=`mktemp -d`
@@ -41,7 +34,9 @@ if [ -e $tmpdir/src/.ci_tests_disabled ]; then
     exit 0
 fi
 
-cat << EOF > build-spread.sh
+$SSH mkdir -p $REMOTE_WORKSPACE/bin
+
+cat << EOF > $WORKSPACE/build-spread.sh
 #!/bin/bash
 set -x
 set -e
@@ -57,31 +52,32 @@ export DEBIAN_FRONTEND=noninteractive
 export GOPATH=`mktemp -d`
 go get -d -v github.com/snapcore/spread/...
 go build github.com/snapcore/spread/cmd/spread
-cp spread /usr/local/bin
-which spread
+cp spread $REMOTE_WORKSPACE/bin
+# Remove old executables of spread
+if [ -e /usr/local/bin/spread ]; then
+    rm /usr/local/bin/spread
+fi
+PATH=$REMOTE_WORKSPACE/bin:$PATH which spread
 rm -rf $GOPATH
 EOF
 
-# Make sure that we don't conflict with any other instances of this
-# build job running at the same time.
-if [ ! `lsof | grep spread$` ] ; then
-    $SCP build-spread.sh $REMOTE_USER@$REMOTE_WORKER:
-    $SSH chmod u+x build-spread.sh
-    $SSH sudo systemd-run --scope ./build-spread.sh
-fi
+$SCP $WORKSPACE/build-spread.sh $REMOTE_USER@$REMOTE_WORKER:
+$SSH chmod u+x build-spread.sh
+$SSH sudo systemd-run --scope ./build-spread.sh
 
 if [ $($SSH snap list | grep ubuntu-image | wc -l) -eq 0 ] ; then
     $SSH sudo snap install --edge --devmode ubuntu-image
 fi
 $SSH sudo apt install -y --force-yes kpartx qemu-kvm
 
-cat << EOF > run-spread-tests.sh
+cat << EOF > $WORKSPACE/run-spread-tests.sh
 #!/bin/bash
 set -x
 set -e
 
 export TERM=linux
 export DEBIAN_FRONTEND=noninteractive
+export PATH=$REMOTE_WORKSPACE/bin:$PATH
 
 if [ -e "src_branch" ] ; then
     rm -rf src_branch
@@ -111,6 +107,8 @@ fi
 rm -rf src_branch
 EOF
 
-$SCP run-spread-tests.sh $REMOTE_USER@$REMOTE_WORKER:
+$SCP $WORKSPACE/run-spread-tests.sh $REMOTE_USER@$REMOTE_WORKER:
 $SSH chmod u+x run-spread-tests.sh
 $SSH sudo systemd-run --scope ./run-spread-tests.sh
+
+$SSH rm -rf $REMOTE_WORKSPACE