Merge ~smoser/curtin:fix/curtainer-try-harder-for-source into curtin:master

[Scott Moser]

tools/curtainer: Try harder to get source at binary version.

When we test curtin from the daily PPA, sometimes the source version
that is available is newer than the binary version available. That
causes mismatch of vmtest:code.

If 'apt-get source curtin/<version>' fails, then ask apt
what the url to the deb for 'curtin' at 'version' is.
Then replace _all.deb with .dsc and try dget on that url. This
should work because:
a.) curtin is a binary and a source package.
b.) source are generally published along side binaries.
c.) in the ppa case, the files are likely still available but only
    the most current is published in the Sources for the archive.

If this proves to not work, then the next solution is to actually
provide a binary package with the files needed for vmtest.

see commit message

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:ebe72b8c539b37f937bd3a33790fc3adebb7ed2c
https://jenkins.ubuntu.com/server/job/curtin-ci/833/
Executed test runs:
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-amd64/833
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-arm64/833
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-ppc64el/833
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-s390x/833

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/curtin-ci/833/rebuild

review: Approve (continuous-integration)

e33e302... by Scott Moser on 2018-03-01

better search for directory

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:e33e302b3a92c8be8d32a0b2f5f558184c5a478f
https://jenkins.ubuntu.com/server/job/curtin-ci/857/
Executed test runs:
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-amd64/857
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-arm64/857
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-ppc64el/857
    SUCCESS: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-s390x/857

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/curtin-ci/857/rebuild

review: Approve (continuous-integration)

[Chad Smith (chad.smith)]

Approved, thanks for this fix and the additional thoughts on error content w/ the ls output.

review: Approve

diff --git a/tools/curtainer b/tools/curtainer
index 6c0628f..ca698c6 100755
--- a/tools/curtainer
+++ b/tools/curtainer
@@ -79,6 +79,39 @@ debug() {
     error "${@}"
 }
 
+get_source() {
+    local target="$1" pkg="$2" ver="$3"
+    local tmpd="" x=""
+    tmpd=$(mktemp -d) || fail "failed to mktemp"
+    mkdir "$tmpd/extract"
+    cd "$tmpd/extract"
+    debug 1 "Inside. getting source for $pkg${ver:+=${ver}}"
+    if ! apt-get source "${pkg}${ver:+=${ver}}"; then
+        [ -n "$ver" ] || fail "Failed to get source for $pkg"
+        # Getting the specific version failed.
+        # Assume 'pkg' is a binary package and source package.
+        # Ask apt for the url to the binary, and assume source in same dir.
+        debug 1 "Failed to apt-get source ${pkg}=$ver. Trying workaround."
+        url=$(apt-get -qq download --print-uris "$pkg=${ver}" |
+            awk '{ gsub(/'\''/, ""); print $1}')
+        local dsc_url="${url%_*.deb}.dsc"
+        debug 1 "Binary package came from $url."
+        debug 1 "Trying dsc from $dsc_url"
+        dget --allow-unauthenticated "$dsc_url" || fail "Failed dget $dsc_url"
+    fi
+
+    # dget or apt-get source of pkg/ver produces pkg-<upstream-ver>/
+    x="${pkg}-${ver%-*}"
+    [ -d "$x" ] || {
+        error "getting source for '$pkg/$ver' did not produce directory '$x'"
+        error "ls -l:"
+        ls -l 1>&2
+        fail
+    }
+    cp -a "$x" "$target" || fail "failed copying $x to $target"
+    rm -Rf "$tmpd"
+}
+
 main() {
     local short_opts="hv"
     local long_opts="help,daily,proposed,source:,verbose"
@@ -158,7 +191,7 @@ main() {
         inside "$name" sed -i '/^deb-src/s/^/#/' /etc/apt/sources.list ||
             error "failed to disable deb-src entries"
     else
-        pkgs="${pkgs} dpkg-dev"
+        pkgs="${pkgs} dpkg-dev devscripts"
     fi
 
     inside "$name" $eatmydata apt-get -q update ||
@@ -172,18 +205,9 @@ main() {
     if [ "${getsource}" != "none" ]; then
         local isrcd="/tmp/curtin-source"
         debug 1 "getting source for curtin at $pkg_ver to $getsource"
-        inside "$name" $eatmydata sh -ec '
-            target="$1"
-            pkg="$2"
-            ver="$3"
-            d=$(mktemp -d)
-            cd "$d"
-            apt-get source "${pkg}${ver:+=${ver}}"
-            for x in *; do [ -d "$x" ] && break; done
-            [ -d "$x" ] || { echo no source dir found.; exit 1; }
-            cp -a $x "$target"
-            rm -Rf "$d"
-            ' -- "$isrcd" "curtin" "$pkg_ver"
+        inside "$name" - $eatmydata /bin/bash -s \
+            get_source "$isrcd" "curtin" "$pkg_ver" < "$0" ||
+            fail "Failed getting source in $name"
         mkdir "$getsource" || fail "failed to create dir '$getsource'"
         inside "$name" tar -C "$isrcd" -cf - . |
             tar -C "$getsource" -xf - ||
@@ -202,6 +226,12 @@ main() {
     CONTAINER=""
 }
 
-main "$@"
+
+if [ "$1" = "get_source" ]; then
+    shift
+    get_source "$@"
+else
+    main "$@"
+fi
 
 # vi: ts=4 expandtab syntax=sh