Merge lp:~adeuring/vmbuilder/jenkins_kvm-optionally-use-juju-test-packages into lp:~ubuntu-on-ec2/vmbuilder/jenkins_kvm

[Abel Deuring]

The Juju QA team wants to build and test Vagrant boxes with debian packages that are regularily built by the team's CI infrastructure.

This branch adds two new environment variables, JUJU_CORE_PKG and JUJU_LOCAL_PKG. If they are defined, launch_kvm.sh adds the package files specificied by the variables to the cloud-init data and img-juju.tmpl installs them.

[Ben Howard (darkmuggle-deactivatedaccount)]

I'm okay with changing launch_kvm, however, changes must be generic and not specific. launch_kvm is used to buid a variety of images.

review: Disapprove

[lp:~adeuring/vmbuilder/jenkins_kvm-optionally-use-juju-test-packages]

455. By Abel Deuring on 2014-08-06

Pass optional could-init files to launch_kvm.sh and standalone.sh via command a line parameter; add these parameter in build-juju-local.sh when then env variables JUJU_CORE_PKG or JUJU_LOCAL_PKG are set.

[Abel Deuring (adeuring)]

> I'm okay with changing launch_kvm, however, changes must be generic and not
> specific. launch_kvm is used to buid a variety of images.

I've changed launch_kvm.sh and standalone.sh to accept accept another parameter, --cloud-init-file. The value(s) of this parameter is(are) passed to write-mime-multipart.

review: Needs Resubmitting

[lp:~adeuring/vmbuilder/jenkins_kvm-optionally-use-juju-test-packages]

456. By Abel Deuring on 2014-08-14

standalone.sh: Prevent processing of --fetch_new consuming the next parameter.

=== modified file 'build-juju-local.sh'
--- build-juju-local.sh	2014-07-25 21:29:07 +0000
+++ build-juju-local.sh	2014-08-14 07:58:38 +0000
@@ -37,7 +37,10 @@
 test_cmd_exists sstream-query simplestreams
 
 # This defines what gets built
-build_for=${BUILD_FOR:-trusty:amd64 precise:amd64}
+build_for=(${BUILD_FOR:-trusty:amd64 precise:amd64})
+[ -n "${JUJU_CORE_PKG}" -o -n "${JUJU_LOCAL_PKG}" ] && \
+    [ ${#build_for[@]} -ge 2 ] && \
+    fail "JUJU_CORE_PKG and JUJU_LOCAL_PKG can be specified only for a single build target."
 
 for build in ${build_for[@]};
 do
@@ -62,6 +65,18 @@
         )
 
     [ ! -e "${builder_img}" ] && cmd+=("--fetch_new")
+    if [ -n "${JUJU_CORE_PKG}" -o -n "${JUJU_LOCAL_PKG}" ]; then
+        cmd+=("--cloud-init-file ${mydir}/handle-xdeb.py:text/part-handler")
+        if [ -n "${JUJU_CORE_PKG}" ]; then
+            cmd+=("--cloud-init-file ${JUJU_CORE_PKG}:application/x-deb")
+            echo "JUJU_CORE_PKG=$(basename $JUJU_CORE_PKG)" > ${tmp_dir}/juju-sources.sh
+        fi
+        if [ -n "${JUJU_LOCAL_PKG}" ]; then
+            cmd+=("--cloud-init-file ${JUJU_LOCAL_PKG}:application/x-deb")
+            echo "JUJU_LOCAL_PKG=$(basename $JUJU_LOCAL_PKG)" >> ${tmp_dir}/juju-sources.sh
+        fi
+        cmd+=("--cloud-init-file ${tmp_dir}/juju-sources.sh:application/x-shellscript")
+    fi
 
     [ -e "${results_d_arch}/${suite}-server-cloudimg-${arch}-juju-vagrant-disk1.img" ] ||
         ( cd ${results_d_arch} && ${cmd[@]} )

=== added file 'handle-xdeb.py'
--- handle-xdeb.py	1970-01-01 00:00:00 +0000
+++ handle-xdeb.py	2014-08-14 07:58:38 +0000
@@ -0,0 +1,15 @@
+handler_version = 2
+
+
+def list_types():
+    return(["application/x-deb", "application/x-shellscript"])
+
+
+def handle_part(data, ctype, filename, payload, frequency):
+    import os
+    if ctype in ("__begin__", "__end__"):
+       return
+
+    dest = os.path.join(data.get_ipath(), filename)
+    with open(dest, 'w') as out:
+        out.write(payload)

=== modified file 'launch_kvm.sh'
--- launch_kvm.sh	2014-06-24 19:40:10 +0000
+++ launch_kvm.sh	2014-08-14 07:58:38 +0000
@@ -17,12 +17,13 @@
     --raw-disk <ARG>     Name of RAW disk to create and attach.
     --raw-size <ARG>     Size of RAW disk in GB.
     --extra-disk <ARG>   Add an extra disk, starting with /dev/vdd
+    --cloud-init-file <ARG> Additional file for the cloud-init data
 EOF
 exit 1
 }
 
 short_opts="h"
-long_opts="id:,ssh_port,disk-gb:,mem:,bzr-automated-ec2-builds:,cloud-config:,user-data:,kernel-url:,img-url:,raw-disk:,raw-size:,smp:,extra-disk:,help"
+long_opts="id:,ssh_port,disk-gb:,mem:,bzr-automated-ec2-builds:,cloud-config:,user-data:,kernel-url:,img-url:,raw-disk:,raw-size:,smp:,extra-disk:,cloud-init-file:,help"
 getopt_out=$(getopt --name "${0##*/}" \
     --options "${short_opts}" --long "${long_opts}" -- "$@") &&
     eval set -- "${getopt_out}" ||
@@ -52,6 +53,7 @@
     --raw-size)                 raw_size="$2"; shift;;
     --smp)                      smp_opts="$2"; shift;;
     --extra-disk)               [ -z "${extra_disk}" ] && extra_disk=$2 || extra_disk="${extra_disk} $2"; shift;;
+    --cloud-init-file)          [ -z "${cloud_init_files}" ] && cloud_init_files=$2 || cloud_init_files="${cloud_init_files} $2"; shift;;
     -h|--help)                  usage; exit 0;;
     --) shift; break;;
   esac
@@ -78,8 +80,12 @@
 [ -z "${ccloud}" ] && fail "Must define cloud-config script via --cloud-config"
 
 debug "Creating Cloud-Init configuration..."
-write-mime-multipart -o "${work_d}/user-data.txt" "${ccloud}" "${ud}" ||
-    fail "Unable to create user-data"
+write_mime_args=(
+    -o "${work_d}/user-data.txt"
+    "${ccloud}"
+    "${ud}")
+write_mime_args+=(${cloud_init_files[@]})
+write-mime-multipart ${write_mime_args[@]} || fail "Unable to create user-data"
 
 echo "instance-id: $(uuidgen)" > "${work_d}/meta-data"
 echo "local-hostname: builder" >> "${work_d}/meta-data"

=== modified file 'standalone.sh'
--- standalone.sh	2014-07-10 21:50:47 +0000
+++ standalone.sh	2014-08-14 07:58:38 +0000
@@ -71,7 +71,7 @@
 }
 
 short_opts="h"
-long_opts="suite:,use_img:,install,streams:,template:,addin:,arch:,resize:,config:,final_img:,cloud_cfg:,fetch_new,resize_final:"
+long_opts="suite:,use_img:,install,streams:,template:,addin:,arch:,resize:,config:,final_img:,cloud_cfg:,fetch_new,resize_final:,cloud-init-file:"
 getopt_out=$(getopt --name "${0##*/}" \
     --options "${short_opts}" --long "${long_opts}" -- "$@") &&
     eval set -- "${getopt_out}" ||
@@ -105,8 +105,9 @@
         --config)       config="${2}"; shift;;
         --cloud_cfg)    cloud_cfg="${2}"; shift;;
         --final_img)    final_img="${2}"; shift;;
-        --fetch_new)    fetch_new=1; shift;;
+        --fetch_new)    fetch_new=1;;
         --resize_final)   resize_final="${2}"; shift;;
+        --cloud-init-file) [ -z "${cloud_init_files}" ] && cloud_init_files="--cloud-init-file ${2}" || cloud_init_files="${cloud_init_files} --cloud-init-file  ${2}"; shift;;
         -h|--help)      usage; exit 0;;
         --) shift; break;;
     esac
@@ -256,6 +257,7 @@
     "--raw-disk $(readlink -f $raw_f)"
     "--raw-size ${raw_size:-20}"
     "--img-url $(readlink -f $mod_img)"
+    "${cloud_init_files[@]}"
 )
 
 # Run the command

=== modified file 'templates/img-juju.tmpl'
--- templates/img-juju.tmpl	2014-08-08 07:48:18 +0000
+++ templates/img-juju.tmpl	2014-08-14 07:58:38 +0000
@@ -316,7 +316,6 @@
     byobu
     lxc
     cgroup-lite
-    juju-core
     bzr
     git
     mongodb-server
@@ -324,9 +323,13 @@
 	python-pip
 	libnss-myhostname
 	rsyslog-gnutls
-	juju-local
     )
 
+instance_dir=$(dirname $(dirname $(readlink -f $0)))
+[ -e ${instance_dir}/juju-sources.sh ] && source ${instance_dir}/juju-sources.sh
+[ -z ${JUJU_CORE_PKG} ] && pkgs+=(juju-core)
+[ -z ${JUJU_LOCAL_PKG} ] && pkgs+=(juju-local)
+
 if [ "${suite}" == "precise" ]; then
 	pkgs+=(linux-headers-generic-lts-raring
            linux-generic-lts-raring)
@@ -355,6 +358,17 @@
 xchroot apt-get -y update
 xchroot update-grub
 
+if [ -n "${JUJU_CORE_PKG}" ] ; then
+  cp ${instance_dir}/${JUJU_CORE_PKG} ${mp}/tmp/
+  xchroot dpkg -i /tmp/${JUJU_CORE_PKG}
+fi
+
+if [ -n "${JUJU_LOCAL_PKG}" ] ; then
+  cp ${instance_dir}/${JUJU_LOCAL_PKG} ${mp}/tmp/
+  xchroot apt-get install -y cpu-checker juju-mongodb
+  xchroot dpkg -i /tmp/${JUJU_LOCAL_PKG}
+fi
+
 # Gets rid of an annoying message for shell provisioner
 xchroot sed -i "s|mesg n|tty -s \&\& mesg n|g" /root/.profile