Merge lp:~canonical-platform-qa/qa-jenkins-jobs/update-chroot-builder into lp:qa-jenkins-jobs

[Max Brustkern]

Update pbuilder chroot creation and make it idempotent

These changes update the pbuilder chroot setup with new requirements and make it one idempotent job.

[platform-qa-bot (platform-qa-bot)]

PASSED: Continuous integration, rev:220
https://platform-qa-jenkins.ubuntu.com/job/qa-jenkins-jobs-ci/146/
Executed test runs:
    SUCCESS: https://platform-qa-jenkins.ubuntu.com/job/build-xenial-amd64-package/504
        deb: https://platform-qa-jenkins.ubuntu.com/job/build-xenial-amd64-package/504/artifact/work/output/*zip*/output.zip
    SUCCESS: https://platform-qa-jenkins.ubuntu.com/job/build-xenial-i386-package/502
        deb: https://platform-qa-jenkins.ubuntu.com/job/build-xenial-i386-package/502/artifact/work/output/*zip*/output.zip
    None: https://platform-qa-jenkins.ubuntu.com/job/generic-update-mp/1995/console

Click here to trigger a rebuild:
https://platform-qa-jenkins.ubuntu.com/job/qa-jenkins-jobs-ci/146/rebuild

review: Approve (continuous-integration)

Unmerged revisions

220. By Max Brustkern on 2017-02-24

Updated pbuilder chroot setup to be idempotent job that fits new requirements

219. By Max Brustkern on 2017-02-24

Add daily spread tests for ubuntu core.

Approved by platform-qa-bot, Jean-Baptiste Lallement.

218. By Sergio Cazzolato on 2017-02-24

adding aggregation to the performance tests execution.

Approved by platform-qa-bot.

217. By Max Brustkern on 2017-02-24

Updating exit status

216. By Max Brustkern on 2017-02-24

Removing old hooks

=== modified file 'jobs/mp-ci-autoland/ci-jobs.yaml'
--- jobs/mp-ci-autoland/ci-jobs.yaml	2017-02-24 16:37:18 +0000
+++ jobs/mp-ci-autoland/ci-jobs.yaml	2017-02-24 20:24:18 +0000
@@ -21,16 +21,9 @@
 # should be tested there as well, as noted below.
 
 - builder:
-    name: create-chroot
-    # This and the update-chroot builder share a lot of code. TODO: Dedupe/be
-    # smarter about this.
+    name: setup-chroot
     builders:
         - shell: |
-            # TODO: Use the production PPA
-            sudo add-apt-repository -y ppa:jenkaas-hackers/tools
-            sudo apt-get update
-            sudo apt-get install -y --assume-yes sphinx-common python3-setuptools pbuilder pbuilderjenkins bzr-builddeb devscripts dh-translations
-
             # Only proceed if the config options are set for building chroots
             if [ -z "$distributions" ] || [ -z "$arches" ]; then
                 echo "Missing vales for distributions and arches,"
@@ -40,65 +33,27 @@
 
             COMPONENTS="main restricted universe multiverse"
             MIRROR="http://archive.ubuntu.com/ubuntu/"
-            EXTRA_PACKAGES="apt-transport-https ca-certificates devscripts eatmydata gnupg patchutils pbuilder vim-tiny"
+            EXTRA_PACKAGES="apt-transport-https bzr ca-certificates devscripts eatmydata gnupg patchutils pbuilder vim-tiny"
             for dist in $distributions; do
                 for arch in $arches; do
+                    OTHER_MIRROR="deb $MIRROR $dist-updates main restricted universe multiverse|deb-src $MIRROR $dist-updates main restricted universe multiverse|deb-src $MIRROR $dist main restricted universe multiverse"
+                    TARBALL="/var/cache/pbuilder/$dist-$arch.tgz"
                     if [ ! -f "/var/cache/pbuilder/$dist-$arch.tgz" ]; then
-                        OTHER_MIRROR="deb $MIRROR $dist-updates main restricted universe multiverse|deb-src $MIRROR $dist-updates main restricted universe multiverse|deb-src $MIRROR $dist main restricted universe multiverse"
                         sudo /usr/sbin/pbuilder --create \
                             --distribution "$dist" \
                             --architecture "$arch" \
-                            --basetgz "/var/cache/pbuilder/$dist-$arch.tgz" \
+                            --basetgz "$TARBALL" \
                             --debootstrapopts \
                             --variant=buildd
-                        sudo /usr/sbin/pbuilder --update \
-                            --distribution "$dist" \
-                            --basetgz "/var/cache/pbuilder/$dist-$arch.tgz" \
-                            --components "$COMPONENTS" \
-                            --mirror "$MIRROR" \
-                            --othermirror "$OTHER_MIRROR" \
-                            --override-config \
-                            --extrapackages "$EXTRA_PACKAGES"
-                    else
-                        echo "Skipping creation of /var/cache/pbuilder/$dist-$arch.tgz, it already exists."
-                    fi
-                done
-            done
-
-
-- builder:
-    name: update-chroot
-    builders:
-        - shell: |
-            # Only proceed if the config options are set for building chroots
-            if [ -z "$distributions" ] || [ -z "$arches" ]; then
-                echo "Missing values for distributions and arches,"
-                echo "  skipping pbuilder chroot creation."
-                exit 1
-            fi
-
-            COMPONENTS="main restricted universe multiverse"
-            MIRROR="http://archive.ubuntu.com/ubuntu/"
-
-            for dist in "$distributions"; do
-                for arch in "$arches"; do
-                    if [ -f "/var/cache/pbuilder/$dist-$arch.tgz" ]; then
-                        OTHER_MIRROR="deb $MIRROR $dist-updates $COMPONENTS|deb-src $MIRROR $dist-updates $COMPONENTS|deb-src $MIRROR $dist $COMPONENTS"
-                        sudo /usr/sbin/pbuilder --update \
-                            --basetgz "/var/cache/pbuilder/$dist-$arch.tgz" \
-                            --distribution "$dist" \
-                            --components "$COMPONENTS" \
-                            --mirror "$MIRROR" \
-                            --othermirror "$OTHER_MIRROR" \
-                            --override-config
-                        if [ -n "$extra_packages" ]; then
-                            sudo /usr/sbin/pbuilder --update \
-                                --basetgz "/var/cache/pbuilder/$dist-$arch.tgz" \
-                                --extrapackages "$extra_packages"
-                        fi
-                    else
-                        echo "Chroot /var/cache/pbuilder/$dist-$arch.tgz not found, skipping update."
-                    fi
+                    fi
+                    sudo /usr/sbin/pbuilder --update \
+                        --distribution "$dist" \
+                        --basetgz "$TARBALL" \
+                        --components "$COMPONENTS" \
+                        --mirror "$MIRROR" \
+                        --othermirror "$OTHER_MIRROR" \
+                        --override-config \
+                        --extrapackages "$EXTRA_PACKAGES"
                 done
             done
 
@@ -281,9 +236,9 @@
             description: The merge proposal to process
 
 - job-template:
-    name: 'create-debian-chroot'
+    name: 'setup-debian-chroot'
     description: |
-        Create the required chroots for pbuilder.
+        Create or update the required chroots for pbuilder.
     node: 'jenkins-slave-2'
     parameters:
         - string:
@@ -296,25 +251,12 @@
             description: 'Distributions to create machines of'
     builders:
         - clear-artifacts
-        - create-chroot
+        - shell: |
+            sudo add-apt-repository -y ppa:jenkaas-hackers/tools
+            sudo apt-get update
+            sudo apt-get install -y --assume-yes sphinx-common python3-setuptools pbuilder pbuilderjenkins bzr-builddeb devscripts dh-translations
 
-- job-template:
-    name: 'update-debian-chroot'
-    description: |
-        Update the required chroots for pbuilder.
-    node: 'jenkins-slave-2'
-    parameters:
-        - string:
-            name: arches
-            default: 'amd64 i386'
-            description: 'Architectures to create machines of'
-        - string:
-            name: distributions
-            default: 'xenial'
-            description: 'Distributions to create machines of'
-    builders:
-        - clear-artifacts
-        - update-chroot
+        - setup-chroot
 
 - job-template:
     name: 'build-{dist}-{arch}-package'