Merge ~kissiel/cc-mini-ci-legacy:python-snapcraft-test into cc-mini-ci-legacy:master

[Maciej Kisielewski]

this MR switches ci to use python version of snapcraft test

[Paul Larson (pwlars)]

A couple of questions:
1. Should we use pastebinit? Is there a risk that we could expose something in some private checkbox branch related to a non-public project?
2. Any intention to add parameters for things like dont_clean? It looked like that might be where you were headed with it, but we'd need to modify the script to do things like that now.

review: Needs Information

[Maciej Kisielewski (kissiel)]

1. It's meant for testing checkbox-snappy, non-public pastes would require OAuth/OpenID magic, or dropping bins altogether.
2. I wanted to have it in the final version from the get-go. There are many ways that stuff can go wrong, and then it's easier to debug. I also though that the next person looking closer to this won't hesitate to skim it over and find it handy.

[Paul Larson (pwlars)]

That makes sense. I'd say let's land this so we can get moving with the other mini-ci plans

review: Approve

diff --git a/src/www/snapcraft/run-snapcraft-test-launcher.sh b/src/www/snapcraft/run-snapcraft-test-launcher.sh
index 1d76ffe..d60bb3f 100755
--- a/src/www/snapcraft/run-snapcraft-test-launcher.sh
+++ b/src/www/snapcraft/run-snapcraft-test-launcher.sh
@@ -5,12 +5,12 @@ minici_snapcraft_log="/home/ubuntu/minici-snapcraft/minici-snapcraft.log"
 sleep 60
 # run the real test and fetch the log
 cd /home/ubuntu/minici-snapcraft
-./run-snapcraft-test.sh  > ./minici-snapcraft.log 2>&1
+./run_snapcraft_test.py > ./minici-snapcraft.log 2>&1
 execute_status=$?
 
 # TODO: submit the hostname to cgi
 
-curl -F snapcraft_subject="./run-snapcraft-test.sh" -F snapcraft_output=@${minici_snapcraft_log} -F snapcraft_status="$execute_status" $cgi_url
+curl -F snapcraft_subject="./run_snapcraft_test.py" -F snapcraft_output=@${minici_snapcraft_log} -F snapcraft_status="$execute_status" $cgi_url
 echo "snapcraft_status=$execute_status"
 curl -d "email_mini_ci_snapcraft=yes" -d "snapcraft_status=$execute_status" $cgi_url
 # protect the test machine and polar bears!
diff --git a/src/www/snapcraft/run-snapcraft-test.sh b/src/www/snapcraft/run-snapcraft-test.sh
deleted file mode 100755
index c54522c..0000000
--- a/src/www/snapcraft/run-snapcraft-test.sh
+++ /dev/null
@@ -1,99 +0,0 @@
-#!/usr/bin/env bash
-# This file is part of Checkbox.
-#
-# Copyright 2016-2017 Canonical Ltd.
-# Written by:
-#   Maciej Kisielewski <[e-mail]>
-#
-# Checkbox is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3,
-# as published by the Free Software Foundation.
-
-SAVED_DIR=$PWD
-WORK_DIR=`mktemp -d`
-
-cd $WORK_DIR
-mkdir -p logs
-LOG_DIR=$WORK_DIR/logs
-
-function clean_up {
-    cd $SAVED_DIR
-    rm -rf $WORK_DIR
-}
-
-function time_it {
-    /usr/bin/time -f "%E elapsed. %P CPU" -o $LOG_DIR/timing.dat $@
-}
-
-function pastebin_logs {
-    if [ -s $LOG_DIR/stdout ]; then
-        echo stdout: $(pastebinit $LOG_DIR/stdout)
-    else
-        echo stdout: empty
-    fi
-    if [ -s $LOG_DIR/stderr ]; then
-        echo stderr: $(pastebinit $LOG_DIR/stderr)
-    else
-        echo stderr: empty
-    fi
-}
-
-function run_step {
-    echo -n Step \"$1\"
-    if ! time_it $2 >>$LOG_DIR/stdout 2>>$LOG_DIR/stderr; then
-        echo -n "- FAIL - "
-        outcome=1
-    else
-        echo -n "- PASS - "
-        outcome=0
-    fi
-    echo -n elapsed: ; cat $LOG_DIR/timing.dat
-    if [ $outcome -ne 0 ]; then
-        pastebin_logs
-        return 1
-    fi
-}
-
-# master branch test
-function test_master {
-    run_step "git-clone" "git clone -b master https://git.launchpad.net/checkbox-snappy packaging" || return 1
-    run_step "update-snapcraft" "snapcraft update" || return 1
-    (cd packaging && run_step "run-snapcraft" "snapcraft snap -o ../checkbox-snappy.snap") || return 1
-    run_step "verify-build" "ls -al checkbox-snappy.snap" || return 1
-    run_step "install-checkbox-snappy" "sudo snap install --devmode ./checkbox-snappy.snap" || return 1
-    (cd && run_step "run smoke-test" "checkbox-snappy.smoke-test") || return 1
-    return 0
-}
-test_master
-master_result=$?
-if [[ $master_result -eq 1 ]]; then
-    echo "Testing Checkbox snap for branch master failed!"
-fi
-pastebin_logs
-
-# beta branch test
-function test_beta {
-    run_step "git-clone" "git clone -b beta https://git.launchpad.net/checkbox-snappy packaging-beta" || return 1
-    run_step "update-snapcraft" "snapcraft update" || return 1
-    (cd packaging-beta && run_step "run-snapcraft" "snapcraft snap -o ../checkbox-snappy-beta.snap") || return 1
-    run_step "verify-build" "ls -al checkbox-snappy-beta.snap" || return 1
-    run_step "install-checkbox-snappy" "sudo snap install --devmode ./checkbox-snappy-beta.snap" || return 1
-    (cd && run_step "run smoke-test" "checkbox-snappy.smoke-test") || return 1
-    return 0
-}
-test_beta
-beta_result=$?
-if [[ $beta_result -eq 1 ]]; then
-    echo "Testing Checkbox snap for branch beta failed!"
-fi
-pastebin_logs
-
-clean_up
-echo "$master_result"
-echo "$beta_result"
-if [[ $master_result -eq 0 && $beta_result -eq 0 ]]; then
-    echo "ALL STEPS PASSED"
-    exit 0
-else
-    exit 1
-fi
diff --git a/src/www/snapcraft/run_snapcraft_test.py b/src/www/snapcraft/run_snapcraft_test.py
new file mode 100755
index 0000000..6f00589
--- /dev/null
+++ b/src/www/snapcraft/run_snapcraft_test.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+# This file is part of Checkbox.
+#
+# Copyright 2017 Canonical Ltd.
+# Written by:
+#   Maciej Kisielewski <[e-mail]>
+#
+# Checkbox is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3,
+# as published by the Free Software Foundation.
+
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+
+
+"""
+This program checks if checkbox-snappy can be snapped and whether "Smoke Tests"
+test plan passes after installing that snap.
+"""
+
+"""
+Have snapcraft, git installed
+"""
+
+REPO = "https://git.launchpad.net/checkbox-snappy"
+
+BRANCHES = ['master', 'beta']
+
+
+def update_snapcraft():
+    subprocess.run(['snapcraft', 'update'])
+
+
+class Snap:
+    def __init__(self, repo, branch, dont_clean=False):
+        self._repo = repo
+        self._branch = branch
+        self._dont_clean = dont_clean
+        self._work_dir = ''
+        self._start_dir = os.path.abspath(os.curdir)
+        self._log_out = ''
+        self._log_err = ''
+
+    def __enter__(self):
+        self._work_dir = tempfile.mkdtemp('-checkbox-snappy-' + self._branch)
+        os.chdir(self._work_dir)
+        return self
+
+    def clone(self):
+        if self._run_cmd(
+                ['git', 'clone', '-b', self._branch, self._repo, '.']) != 0:
+            raise Exception('Failed to git-clone. Repo: {}. Branch: {}'.format(
+                self._repo, self._branch))
+        print('cloned into {}'.format(self._work_dir))
+    def snap(self):
+        if self._run_cmd(['snapcraft']) != 0:
+            raise Exception('Failed to snap checkbox-snappy')
+        # the last line in the log should contain "Snapped $SNAPNAME"
+        self._snap_path = self._tail_out().split()[1]
+        print('{} snapped.'.format(self._snap_path))
+        
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if not self._dont_clean:
+            shutil.rmtree(self._work_dir)
+        os.chdir(self._start_dir)
+
+    def _run_cmd(self, args):
+        cp = subprocess.run(
+            args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        self._log_out += cp.stdout.decode(sys.stdout.encoding)
+        self._log_err += cp.stderr.decode(sys.stderr.encoding)
+        if cp.returncode != 0:
+            print('Command `{}` failed with return code {}'.format(
+                ' '.join(args), cp.returncode), file=sys.stderr)
+        return cp.returncode
+
+    def _tail_out(self, n=1):
+        return '\n'.join(self._log_out.strip().split('\n')[-n:])
+    def _tail_err(self, n=1):
+        return '\n'.join(self._log_err.strip().split('\n')[-n:])
+        
+
+    def install(self):
+        if self._run_cmd(
+                ['sudo', 'snap', 'install', '--devmode', self._snap_path]) != 0:
+            raise Exception('Failed to install checkbox-snappy')
+
+    def smoke_test(self):
+        if self._run_cmd(['checkbox-snappy.smoke-test']) != 0:
+            raise Exception('Smoke test failed')
+
+    def print_tails(self):
+        print('Last 10 lines of stdout:')
+        print(self._tail_out(10))
+        print('Last 10 lines of stderr:')
+        print(self._tail_err(10))
+
+    def submit_logs(self):
+        with open('stdout.log', 'wt') as f:
+            f.write(self._log_out)
+        with open('stderr.log', 'wt') as f:
+            f.write(self._log_err)
+        if self._log_out:
+            self._run_cmd(['pastebinit', 'stdout.log'])
+            print("stdout: {}".format(self._tail_out()))
+        else:
+            print("stdout: empty")
+        if self._log_err:
+            self._run_cmd(['pastebinit', 'stderr.log'])
+            print("stderr: {}".format(self._tail_out()))
+        else:
+            print("stderr: empty")
+
+def main():
+    failed = False
+    for branch in BRANCHES:
+        with Snap(REPO, branch) as snap:
+            try:
+                snap.clone()
+                snap.snap()
+                snap.install()
+                snap.smoke_test()
+            except Exception as exc:
+                failed = True
+                print(exc, file=sys.stderr)
+                snap.print_tails()
+            finally:
+                snap.submit_logs()
+
+    return 1 if failed else 0
+
+if __name__ == '__main__':
+    main()