Merge ~raharper/curtin:fix-tools-build-package into curtin:master

[Ryan Harper]

Fix tools/build-deb

- Import some tools from cloud-init
  - tools/read-version
    - Updated to mention curtin instead
    - Added --shell which emits key=value pairs
    - Added --long which emits just the version_long value
  - tools/make-tarball
    - Updated to mention/use curtin instead
- Removed tools/export-tarball
- debian/changelog.trunk
  - Modified for easier replacement for the version string
- Removed all of the bzr command and revision parsing logic
  and now use read-version to extract the long and short
  version values and feed them to make-tarball

This now works and writes out debs like:
     curtin_17.1-2-g757848bf-1~bddeb_all.deb
     curtin-common_17.1-2-g757848bf-1~bddeb_all.deb
     python3-curtin_17.1-2-g757848bf-1~bddeb_all.deb
     python-curtin_17.1-2-g757848bf-1~bddeb_all.deb

[Server Team CI bot (server-team-bot)]

FAILED: Continuous integration, rev:d27ca0c0214d107a072e871b3dffa7443ce9efbd
https://jenkins.ubuntu.com/server/job/curtin-ci/708/
Executed test runs:
    FAILURE: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-amd64/708/console
    FAILURE: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-arm64/708/console
    FAILURE: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-ppc64el/708/console
    FAILURE: https://jenkins.ubuntu.com/server/job/curtin-ci/nodes=metal-s390x/708/console

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/curtin-ci/708/rebuild

review: Needs Fixing (continuous-integration)

[Scott Moser (smoser)]

talked with ryan, the mp that i had up did the same basic thing as this.
took some input on it and merged in 8108efbd9377 and c5cb03e197f0

calling this MP 'merged'.

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..7f909cf
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,12 @@
+build
+curtin.egg-info
+dist
+*.pyc
+__pycache__
+.tox
+.coverage
+doc/rtd_html
+parts
+prime
+stage
+*.snap
diff --git a/debian/changelog.trunk b/debian/changelog.trunk
index ec1e294..c8c6e08 100644
--- a/debian/changelog.trunk
+++ b/debian/changelog.trunk
@@ -1,4 +1,4 @@
-curtin (17.1~bzrREVNO-0ubuntu1) UNRELEASED; urgency=low
+curtin (VERSION_LONG-1~bddeb) UNRELEASED; urgency=low
 
   * Initial release
 
diff --git a/tools/build-deb b/tools/build-deb
index 685473b..68d77c8 100755
--- a/tools/build-deb
+++ b/tools/build-deb
@@ -38,46 +38,31 @@ top_d=$(cd "$(dirname "${0}")"/.. && pwd)
 # resulting in something like: 0.1.0~bzrREVNO-1~trunk1
 clogver_o=$(sed -n '1s,.*(\([^)]*\)).*,\1,p' debian/changelog.trunk)
 
-revno=$(bzr revno) || fail "failed to get revno"
-clogver_upstream=${clogver_o%%-*}
-clogver_debian=${clogver_o#*-}
-
-# uver_bzr gets 17.1~bzr<revno>
-uver_bzr=$(echo "${clogver_upstream}" | sed "s,REVNO,$revno,")
-# uver_rel gets '17.1'
-uver_rel=$(echo "${clogver_upstream}" | sed 's,~bzrREVNO,,')
-
-tag_revno=$(bzr tags | awk '$1 == v { print $2 }' v="${uver_rel}")
-if [ "${tag_revno}" = "$revno" ]; then
-    # if this is a tag, then drop ~bzrXXX from the changelog version.
-    echo "Building a tag of upstream version ($uver_rel -> $revno)" 1>&2
-    clogver_new="${uver}-${clogver_debian}"
-    uver="${uver_rel}"
-    export_string="tag:${uver_rel}"
-else
-    uver="${uver_bzr}"
-    export_string="$revno"
-fi
-clogver_new="${uver}-${clogver_debian}"
+
+revno=$(git describe --abbrev=8 --match=[0-9]*) || fail "failed to get revno"
+version_long=$(./tools/read-version --long)
+version_short=$(./tools/read-version)
+
+
 
 TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${bname}.XXXXXX")
 
 trap cleanup EXIT
 
-echo "building upstream version $uver, debian ver=${clogver_debian}"
+echo "building upstream version $version_long, debian ver=${version_short}"
 
-dir="${sourcename}-$uver"
-tarball="${sourcename}_$uver.orig.tar.gz"
+dir="${sourcename}-$version_long"
+tarball="${sourcename}_$version_long.orig.tar.gz"
 
 myd=$(dirname "$0")
-"$myd/export-tarball" "${export_string}" "${TEMP_D}/$tarball"
+"$myd/make-tarball" --long "${version_long}" --output="${TEMP_D}/$tarball"
 echo "created ${tarball}"
 
 cd "${TEMP_D}"
 tar xzf "$tarball" || fail "failed extract tarball"
 
 if [ ! -d "$dir" ]; then
-    # export-tarball will create the directory name based on the
+    # make-tarball will create the directory name based on the
     # contents of debian/changelog.trunk in the version provided.
     # if that differs from what is here, then user has changes.
     for d in ${sourcename}*; do
@@ -85,13 +70,13 @@ if [ ! -d "$dir" ]; then
     done
     if [ -d "$d" ]; then
         {
-        echo "WARNING: bzr at '${export_string}' had different version"
+        echo "WARNING: git at '${export_string}' had different version"
         echo "  in debian/changelog.trunk than your tree. version there"
         echo "  is '$d' working directory had $uver"
         } 1>&2
         dir=$d
     else
-        echo "did not find a directory created by export-tarball. sorry." 1>&2
+        echo "did not find a directory created by make-tarball. sorry." 1>&2
         exit
     fi
 fi
@@ -106,7 +91,7 @@ done
 # first line of debian/changelog looks like
 #   curtin (<version>) UNRELEASED; urgency=low
 # fix the version and UNRELEASED
-sed -i -e "1s,([^)]*),(${clogver_new})," \
+sed -i -e "1s,VERSION_LONG,${version_long}," \
        -e "1s,UNRELEASED,${RELEASE}," debian/changelog ||
    fail "failed to write debian/changelog"
 debuild "$@" || fail "debuild failed"
diff --git a/tools/export-tarball b/tools/export-tarball
deleted file mode 100755
index 1b9c17d..0000000
--- a/tools/export-tarball
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/bin/sh
-
-set -e
-sourcename="curtin"
-
-Usage() {
-   cat <<EOF
-Usage: [revno [output.tar.gz]]
-
-  Create a tarball of revno (or tag) in output.tar.gz
-  revno defaults to \`bzr revno\`.
-  output filename defaults to:
-   curtin-X.Y~bzrREVNO.tar.gz
-  or, if a tag is given:
-   curtin-TAG.tar.gz
-
-  if UNCOMMITTED is set to non '0' in environment
-  then uncommitted changes will be kept in the tarball.
-EOF
-}
-
-TEMP_D=""
-fail() { echo "$@" 1>&2; exit 1; }
-cleanup() {
-   [ -z "$TEMP_D" ] || rm -Rf "$TEMP_D"
-}
-
-export_uncommitted=""
-if [ "${UNCOMMITTED:-0}" != "0" ]; then
-   export_uncommitted="--uncommitted"
-fi
-
-[ "$1" = "-h" -o "$1" = "--help" ] && { Usage; exit 0; }
-
-TEMP_D=$(mktemp -d)
-trap cleanup EXIT
-
-case "${1:-HEAD}" in
-   tag:*)  version="${1#tag:}";;
-   HEAD)   revno="$(bzr revno)"; revargs="-r $revno";;
-   [0-9]*) revno="$1"          ; revargs="-r $1";;
-esac
-output="$2"
-
-if [ -z "$version" ]; then
-   bzr cat $revargs debian/changelog.trunk > "$TEMP_D/clog" ||
-      fail "failed to extract debian/change.log.trunk at $revargs"
-
-   clogver_o=$(sed -n '1s,.*(\([^)]*\)).*,\1,p' $TEMP_D/clog)
-   clogver_upstream=${clogver_o%%-*}
-   mmm=${clogver_o%%~*}
-   version="$mmm~bzr$revno"
-fi
-
-if [ -z "$output" ]; then
-   output="$sourcename-$version.tar.gz"
-fi
-
-bzr export ${export_uncommitted} \
-   --format=tgz --root="$sourcename-${version}" $revargs $output
-echo "wrote $output"
diff --git a/tools/make-tarball b/tools/make-tarball
new file mode 100755
index 0000000..60ede68
--- /dev/null
+++ b/tools/make-tarball
@@ -0,0 +1,63 @@
+#!/bin/sh
+set -e
+
+TEMP_D=""
+cleanup() {
+    [ -z "$TEMP_D" ] || rm -Rf "${TEMP_D}"
+}
+trap cleanup EXIT
+
+Usage() {
+    cat <<EOF
+Usage: ${0##*/} [revision]
+    create a tarball of revision (default HEAD)
+
+    options:
+      -o | --output FILE   write to file
+EOF
+}
+
+short_opts="ho:v"
+long_opts="help,output:,long,verbose"
+getopt_out=$(getopt --name "${0##*/}" \
+    --options "${short_opts}" --long "${long_opts}" -- "$@") &&
+    eval set -- "${getopt_out}" || { Usage 1>&2; exit 1; }
+
+long_opt=""
+while [ $# -ne 0 ]; do
+    cur=$1; next=$2
+    case "$cur" in
+        -o|--output) output=$next; shift;;
+           --long) long_opt="--long";;
+        --) shift; break;;
+    esac
+    shift;
+done
+
+rev=${1:-HEAD}
+version=$(git describe --abbrev=8 "--match=[0-9]*" ${long_opt} $rev)
+
+archive_base="curtin-$version"
+if [ -z "$output" ]; then
+    output="$archive_base.tar.gz"
+fi
+
+# when building an archiving from HEAD, ensure that there aren't any
+# uncomitted changes in the working directory (because these would not
+# end up in the archive).
+if [ "$rev" = HEAD ] && ! git diff-index --quiet HEAD --; then
+    if [ -z "$SKIP_UNCOMITTED_CHANGES_CHECK" ]; then
+        echo "ERROR: There are uncommitted changes in your working directory." >&2
+        exit 1
+    else
+        echo "WARNING: There are uncommitted changes in your working directory." >&2
+        echo "         This changes will not be included in the archive." >&2
+    fi
+fi
+
+TEMP_D=$(mktemp -d)
+tar=${output##*/}
+tar="$TEMP_D/${tar%.gz}"
+git archive --format=tar --prefix="$archive_base/" "$rev" > "$tar"
+gzip -9 -c "$tar" > "$output"
+echo "$output"
diff --git a/tools/read-version b/tools/read-version
new file mode 100755
index 0000000..8ad89d2
--- /dev/null
+++ b/tools/read-version
@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+
+import os
+import json
+import subprocess
+import sys
+
+if "avoid-pep8-E402-import-not-top-of-file":
+    _tdir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+    sys.path.insert(0, _tdir)
+    from curtin import version as ci_version
+
+
+def tiny_p(cmd, capture=True):
+    # python 2.6 doesn't have check_output
+    stdout = subprocess.PIPE
+    stderr = subprocess.PIPE
+    sp = subprocess.Popen(cmd, stdout=stdout,
+                          stderr=stderr, stdin=None,
+                          universal_newlines=True)
+    (out, err) = sp.communicate()
+    ret = sp.returncode
+    if ret not in [0]:
+        raise RuntimeError("Failed running %s [rc=%s] (%s, %s)" %
+                           (cmd, ret, out, err))
+    return out
+
+
+def which(program):
+    # Return path of program for execution if found in path
+    def is_exe(fpath):
+        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+    _fpath, _ = os.path.split(program)
+    if _fpath:
+        if is_exe(program):
+            return program
+    else:
+        for path in os.environ.get("PATH", "").split(os.pathsep):
+            path = path.strip('"')
+            exe_file = os.path.join(path, program)
+            if is_exe(exe_file):
+                return exe_file
+
+    return None
+
+
+use_long = '--long' in sys.argv or os.environ.get('CI_RV_LONG')
+use_tags = '--tags' in sys.argv or os.environ.get('CI_RV_TAGS')
+output_json = '--json' in sys.argv
+output_shell = '--shell' in sys.argv
+
+src_version = ci_version.version_string()
+version_long = None
+
+if os.path.isdir(os.path.join(_tdir, ".git")) and which("git"):
+    flags = []
+    if use_tags:
+        flags = ['--tags']
+    cmd = ['git', 'describe', '--abbrev=8', '--match=[0-9]*'] + flags
+
+    version = tiny_p(cmd).strip()
+
+    if not version.startswith(src_version):
+        sys.stderr.write("git describe version (%s) differs from "
+                         "curtin.version (%s)\n" % (version, src_version))
+        sys.exit(1)
+
+    version_long = tiny_p(cmd + ["--long"]).strip()
+else:
+    version = src_version
+    version_long = None
+
+# version is X.Y.Z[+xxx.gHASH]
+# version_long is None or X.Y.Z-xxx-gHASH
+release = version.partition("-")[0]
+extra = None
+commit = None
+distance = None
+
+if version_long:
+    info = version_long.partition("-")[2]
+    extra = "-" + info
+    distance, commit = info.split("-")
+    # remove the 'g' from gHASH
+    commit = commit[1:]
+
+data = {
+    'release': release,
+    'version': version,
+    'version_long': version_long,
+    'extra': extra,
+    'commit': commit,
+    'distance': distance,
+}
+
+if output_json:
+    sys.stdout.write(json.dumps(data, indent=1) + "\n")
+elif output_shell:
+    for (k, v) in data.items():
+        sys.stdout.write('%s="%s"\n' % (k, v))
+elif use_long:
+    sys.stdout.write(data['version_long'] + "\n")
+else:
+    sys.stdout.write(release + "\n")
+
+sys.exit(0)
+
+# vi: ts=4 expandtab