Merge lp:~mvo/snappy/selftest-failover into lp:~mvo/snappy/snappy-selftest-snap

[Michael Vogt]

Test that failover from a broken partition to a good partition works as expected, ugly but should be a good starting point.

To test run:
$ adt-run upgrade-test/ --- ssh -s snappy -- -i ~/path/to/snappy.img

There is too much code duplication between 80_, 90_, 91_ right now so best is probably to use this as the starting point for a cleanup branch.

[lp:~mvo/snappy/selftest-failover]

45. By Michael Vogt on 2015-05-09

tests/07_test_install_framework: revert incorrect debug change (thanks Steve!)

46. By Michael Vogt on 2015-05-21

README: fix README

47. By Michael Vogt on 2015-05-21

extract common code into common.sh and cleanup

48. By Michael Vogt on 2015-05-21

improve debug output if upgrade/downgrade/failover fails

49. By Michael Vogt on 2015-05-21

merged lp:snappy/selftest

50. By Michael Vogt on 2015-05-21

prepare for BBB auto tests

51. By Michael Vogt on 2015-05-21

tests/common.sh: typo fix

52. By Michael Vogt on 2015-05-21

README: document how to run on a BBB

Unmerged revisions

52. By Michael Vogt on 2015-05-21

README: document how to run on a BBB

51. By Michael Vogt on 2015-05-21

tests/common.sh: typo fix

50. By Michael Vogt on 2015-05-21

prepare for BBB auto tests

49. By Michael Vogt on 2015-05-21

merged lp:snappy/selftest

48. By Michael Vogt on 2015-05-21

improve debug output if upgrade/downgrade/failover fails

47. By Michael Vogt on 2015-05-21

extract common code into common.sh and cleanup

46. By Michael Vogt on 2015-05-21

README: fix README

45. By Michael Vogt on 2015-05-09

tests/07_test_install_framework: revert incorrect debug change (thanks Steve!)

44. By Michael Vogt on 2015-05-08

add basic test for failover

43. By Michael Vogt on 2015-05-08

add upgrade test for stable->edge

=== modified file 'tests/07_test_install_framework'
--- tests/07_test_install_framework	2015-05-08 15:54:20 +0000
+++ tests/07_test_install_framework	2015-05-08 15:54:20 +0000
@@ -1,7 +1,9 @@
 test() {
     if after_reboot; then
         # ensure the service comes up after a reboot
-        services=$(systemctl | grep "${framework}_.*.service")
+        services="$(systemctl)"
+        echo "$services"
+        echo "$services" | grep "${framework}_.*.service"
         [ -n "$services" ] || fail "$framework provided no services"
         return
     fi

=== added file 'tests/80_test_failover'
--- tests/80_test_failover	1970-01-01 00:00:00 +0000
+++ tests/80_test_failover	2015-05-08 15:54:20 +0000
@@ -0,0 +1,70 @@
+# Test that we rollback on failed upgrades
+# (needs autopkgtest with a supporting runner).
+
+test() {
+    can_reboot || { echo "SKIP: cannot reboot testbed"; return; }
+
+    versions=$(snappy list)
+    current=$(echo "$versions" | awk '/^ubuntu-core/ {print $3}')
+    echo "original current snappy version: $current"
+
+    # debug
+    echo "Boot order"
+    grep root=LABEL /boot/grub/grub.cfg
+    cat /boot/grub/grubenv
+
+    if after_reboot; then
+        # reboot: ensure we booted back into original
+        orig_current=$(cat $ADT_ARTIFACTS/current)
+        if [ "$orig_current" != "$current" ]; then
+            fail "did not failover to good version (\"$orig_current\" != \"$current\")"
+        fi
+
+        # FIXME: grep log(other partition) for the crash to ensure it
+        #        really worked
+        # FIXME2: reboot again as regression test for #1449904
+
+        # we booted, cleanup the panic for the next tests to work
+        sudo mount -o remount,rw /writable/cache/system
+        sudo rm /writable/cache/system/etc/rc.local
+        return
+    fi
+
+    # fake new available version by doing a current--
+    sudo mount -o remount,rw /
+    sudo sed -i "s/build_number: $current/build_number: $((current-1))/" /etc/system-image/channel.ini
+    sudo mount -o remount,ro /
+
+    if [ -e /writable/cache/system/etc/system-image/channel.ini ]; then
+        sudo mount -o remount,rw /writable/cache/system/
+        sudo sed -i 's/build_number: $current/build_number: $((current-1))/' /writable/cache/system/etc/system-image/channel.ini
+        sudo mount -o remount,ro /writable/cache/system/
+    fi
+
+    # we should have something now :)
+    versions=$(snappy list)
+    current=$(echo "$versions" | awk '/^ubuntu-core/ {print $3}')
+    echo "faked current snappy version: $current"
+
+    versions=$(snappy list -u)
+    avail=$(echo "$versions" | awk '/^ubuntu-core/ {print $3}')
+    echo "available snappy version to upgrade: $avail"
+    
+    # skip if not numeric, e. g. "-"
+    [ "$avail" != "${avail#[0-9]}" ] || { echo "SKIP: no image to upgrade to"; return; }
+
+    [ $avail -gt $current ] || fail "$avail is not newer than $current"
+
+    # save version for post-upgrade test
+    echo "$avail" > $ADT_ARTIFACTS/avail
+    echo "$current" > $ADT_ARTIFACTS/current
+
+    echo "upgrading..."
+    sudo snappy update
+
+    # break the upgrade
+    sudo mount -o remount,rw /writable/cache/system/
+    printf "#!/bin/sh\nprintf c > /proc/sysrq-trigger" | sudo tee /writable/cache/system/etc/rc.local
+    
+    reboot
+}

=== modified file 'tests/90_test_upgrade'
--- tests/90_test_upgrade	2015-05-08 15:54:20 +0000
+++ tests/90_test_upgrade	2015-05-08 15:54:20 +0000
@@ -6,7 +6,7 @@
     can_reboot || { echo "SKIP: cannot reboot testbed"; return; }
 
     versions=$(snappy list)
-    current=$(echo "$versions" | awk '/ubuntu-core/ {print $3}')
+    current=$(echo "$versions" | awk '/^ubuntu-core/ {print $3}')
     echo "original current snappy version: $current"
 
     # debug
@@ -45,11 +45,11 @@
 
     # we should have something now :)
     versions=$(snappy list)
-    current=$(echo "$versions" | awk '/ubuntu-core/ {print $3}')
+    current=$(echo "$versions" | awk '/^ubuntu-core/ {print $3}')
     echo "faked current snappy version: $current"
 
     versions=$(snappy list -u)
-    avail=$(echo "$versions" | awk '/ubuntu-core/ {print $3}')
+    avail=$(echo "$versions" | awk '/^ubuntu-core/ {print $3}')
     echo "available snappy version to upgrade: $avail"
     
     # skip if not numeric, e. g. "-"

=== modified file 'tests/91_test_upgrade_from_stable_image'
--- tests/91_test_upgrade_from_stable_image	2015-05-08 15:54:20 +0000
+++ tests/91_test_upgrade_from_stable_image	2015-05-08 15:54:20 +0000
@@ -12,7 +12,7 @@
     can_reboot || { echo "SKIP: cannot reboot testbed"; return; }
 
     versions=$(snappy list)
-    current=$(echo "$versions" | awk '/ubuntu-core/ {print $3}')
+    current=$(echo "$versions" | awk '/^ubuntu-core/ {print $3}')
     echo "original current snappy version: $current"
 
     # debug
@@ -51,7 +51,7 @@
 
     # now upgrade to latest edge from stable
     versions=$(snappy list -u)
-    avail=$(echo "$versions" | awk '/ubuntu-core/ {print $3}')
+    avail=$(echo "$versions" | awk '/^ubuntu-core/ {print $3}')
     echo "available snappy version to upgrade: $avail"
     
     # skip if not numeric, e. g. "-"