Merge ~jocave/plainbox-provider-snappy:modify-for-refresh-control into plainbox-provider-snappy:master

[Jonathan Cave]

Add copies of manual jobs that should work on devices that are refresh controlled.

[Paul Larson (pwlars)]

I was thinking there may be some points in time where it could fail if the version in edge hasn't been updated yet. However, I think their release process has been mostly ensuring there's always a newer version in edge for the most part, so probably not a big enough concern to worry about.
+1

review: Approve

diff --git a/plainbox-provider-snappy/bin/snap_tests b/plainbox-provider-snappy/bin/snap_tests
new file mode 100755
index 0000000..955e535
--- /dev/null
+++ b/plainbox-provider-snappy/bin/snap_tests
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+# Copyright 2015-2017 Canonical Ltd.
+# All rights reserved.
+#
+# Written by:
+#    Authors: Jonathan Cave <[e-mail]>
+
+
+import subprocess
+
+from guacamole import Command
+
+
+class SnapList(Command):
+
+    """snap list sub-command."""
+
+    def invoked(self, ctx):
+        """snap list should show the core package is installed."""
+        cmd = ['snap', 'list']
+        output = subprocess.check_output(cmd)
+        if 'core' not in str(output):
+            print("String not found")
+            return 1
+
+
+class SnapSearch(Command):
+
+    """snap search sub-command."""
+
+    def invoked(self, ctx):
+        """snap search for 'hello'."""
+        cmd = ['snap', 'find', 'hello']
+        output = subprocess.check_output(cmd)
+        if 'hello-world' not in str(output):
+            print("String not found")
+            return 1
+
+
+class SnapInstall(Command):
+
+    """snap install sub-command."""
+
+    def invoked(self, ctx):
+        """Test install of test-snapd-tools snap."""
+        cmd = ['sudo', 'snap', 'install', 'test-snapd-tools']
+        subprocess.check_call(cmd)
+        cmd = ['snap', 'list']
+        output = subprocess.check_output(cmd)
+        if 'test-snapd-tools' not in str(output):
+            print("Didn't find package name")
+            return 1
+
+
+class SnapRemove(Command):
+
+    """snap remove sub-command."""
+
+    def invoked(self, ctx):
+        """Test remove of test-snapd-tools snap."""
+        cmd = ['sudo', 'snap', 'remove', 'test-snapd-tools']
+        subprocess.check_call(cmd)
+        cmd = ['snap', 'list']
+        output = subprocess.check_output(cmd)
+        if 'test-snapd-tools' in str(output):
+            print("Found package name")
+            return 1
+
+
+class Snap(Command):
+
+    """Fake snap like command."""
+
+    sub_commands = (
+        ('list', SnapList),
+        ('search', SnapSearch),
+        ('install', SnapInstall),
+        ('remove', SnapRemove),
+    )
+
+
+if __name__ == '__main__':
+    Snap().main()
diff --git a/plainbox-provider-snappy/bin/snappy_tests b/plainbox-provider-snappy/bin/snappy_tests
deleted file mode 100755
index 6c21f40..0000000
--- a/plainbox-provider-snappy/bin/snappy_tests
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2015 Canonical Ltd.
-# All rights reserved.
-#
-# Written by:
-#    Authors: Jonathan Cave <[e-mail]>
-
-
-import subprocess
-
-from guacamole import Command
-
-
-class SnappyList(Command):
-
-    """snappy list sub-command."""
-
-    def invoked(self, ctx):
-        """Snappy list should show the ubuntu-core package is installed."""
-        cmd = ['snap', 'list']
-        output = subprocess.check_output(cmd)
-        if 'core' not in str(output):
-            print("String not found")
-            return 1
-
-
-class SnappySearch(Command):
-
-    """snappy search sub-command."""
-
-    def invoked(self, ctx):
-        """Snappy search for 'hello'."""
-        cmd = ['snap', 'find', 'hello']
-        output = subprocess.check_output(cmd)
-        if 'hello-world' not in str(output):
-            print("String not found")
-            return 1
-
-
-class SnappyInstall(Command):
-
-    """snappy install sub-command."""
-
-    def invoked(self, ctx):
-        """Test install of test-snapd-tools snap."""
-        cmd = ['sudo', 'snap', 'install', 'test-snapd-tools']
-        subprocess.check_call(cmd)
-        cmd = ['snap', 'list']
-        output = subprocess.check_output(cmd)
-        if 'test-snapd-tools' not in str(output):
-            print("Didn't find package name")
-            return 1
-
-
-class SnappyRemove(Command):
-
-    """snappy remove sub-command."""
-
-    def invoked(self, ctx):
-        """Test remove of test-snapd-tools snap."""
-        cmd = ['sudo', 'snap', 'remove', 'test-snapd-tools']
-        subprocess.check_call(cmd)
-        cmd = ['snap', 'list']
-        output = subprocess.check_output(cmd)
-        if 'test-snapd-tools' in str(output):
-            print("Found package name")
-            return 1
-
-
-class Snappy(Command):
-
-    """Fake snappy like command."""
-
-    sub_commands = (
-        ('list', SnappyList),
-        ('search', SnappySearch),
-        ('install', SnappyInstall),
-        ('remove', SnappyRemove),
-    )
-
-if __name__ == '__main__':
-    Snappy().main()
diff --git a/plainbox-provider-snappy/units/snappy/snappy.pxu b/plainbox-provider-snappy/units/snappy/snappy.pxu
index 313b0d5..774c2d2 100644
--- a/plainbox-provider-snappy/units/snappy/snappy.pxu
+++ b/plainbox-provider-snappy/units/snappy/snappy.pxu
@@ -3,7 +3,7 @@ _summary: Test that the snap list command is working.
 _purpose: If snap list command is working then should at least find the
  ubuntu-core package.
 plugin: shell
-command: snappy_tests list
+command: snap_tests list
 category_id: snappy
 estimated_duration: 10s
 flags: preserve-locale
@@ -13,7 +13,7 @@ _summary: Test that the snap find command is working.
 _purpose: If snap find command is working then should find hello-world
  in the store.
 plugin: shell
-command: snappy_tests search
+command: snap_tests search
 category_id: snappy
 estimated_duration: 10s
 flags: preserve-locale
@@ -24,7 +24,7 @@ _purpose:
  The store should contain the basic test-snapd-tools snap makes sure this
  can be downloaded and installed on the system.
 plugin: shell
-command: snappy_tests install
+command: snap_tests install
 category_id: snappy
 estimated_duration: 10s
 flags: preserve-locale
@@ -33,7 +33,7 @@ id: snappy/snap-remove
 _summary: Test the snap remove command is able to remove the test-snapd-tools snap.
 _purpose: After having installed the test-snapd-tools snap, check it can removed.
 plugin: shell
-command: snappy_tests remove
+command: snap_tests remove
 category_id: snappy
 estimated_duration: 10s
 depends: snappy/snap-install
@@ -109,7 +109,25 @@ _steps:
 _verification:
  Check core version is newer using the edge channel
 plugin: manual
-after: snappy/snap-reupdate
+category_id: snappy
+estimated_duration: 400
+
+id: snappy/os-refresh-with-refresh-control
+_summary: Refresh the system using the snap tool
+_purpose:
+ Check "core" can be refreshed by snap refresh
+_steps:
+ 1. Check version number
+    snap list core
+ 2. Update
+    snap refresh core --edge --ignore-validation
+ 3. Reboot the system and log in
+    sudo reboot
+ 4. Check version number
+    snap list core
+_verification:
+ Check core version is newer using the edge channel
+plugin: manual
 category_id: snappy
 estimated_duration: 400
 
@@ -133,6 +151,26 @@ depends: snappy/os-refresh
 category_id: snappy
 estimated_duration: 400
 
+id: snappy/os-revert-with-refresh-control
+_summary: Rollback system update using the snap tool
+_purpose:
+ Check core can be reverted by snap revert
+_steps:
+ 1. Check version number
+    snap list core
+ 2. Revert
+    snap revert core
+ 3. Reboot the system and log in
+    sudo reboot
+ 4. Check version number
+    snap list core
+_verification:
+ Check core version is back to its stable version
+plugin: manual
+depends: snappy/os-refresh-with-refresh-control
+category_id: snappy
+estimated_duration: 400
+
 id: snappy/os-fail-boot
 _summary: Automatically rollback after failed boot after upgrade
 _purpose:
@@ -157,6 +195,30 @@ category_id: snappy
 depends: snappy/os-revert
 estimated_duration: 500
 
+id: snappy/os-fail-boot-with-refresh-control
+_summary: Automatically rollback after failed boot after upgrade
+_purpose:
+ Check system will rollback to original core snap if failed to boot the updated one
+_steps:
+ 1. Remove reverted version (and associated data)
+    snap remove core --revision=<edge_revision>
+ 2. Check that the edge revision is back in the refresh list
+    snap refresh --list core
+ 3. Update
+    snap refresh core --edge --ignore-validation
+ 4. Modify the GRUB Environment Block to simulate a failed boot
+    sudo /usr/bin/grub-editenv /boot/grub/grubenv set snap_mode=trying
+ 5. Reboot the system and log in
+    sudo reboot
+ 6. Check version number
+    snap list core
+_verification:
+ Check system is currently booting the stable core version
+plugin: manual
+category_id: snappy
+depends: snappy/os-revert-with-refresh-control
+estimated_duration: 500
+
 id: snappy/sshd
 _summary: SSH is enabled and operational
 _purpose:
diff --git a/plainbox-provider-snappy/units/snappy/test-plan.pxu b/plainbox-provider-snappy/units/snappy/test-plan.pxu
index 3aeb033..fb03450 100644
--- a/plainbox-provider-snappy/units/snappy/test-plan.pxu
+++ b/plainbox-provider-snappy/units/snappy/test-plan.pxu
@@ -10,6 +10,18 @@ nested_part:
     snappy-snap-manual
     snappy-snap-automated
 
+id: snappy-snap-full-with-refresh-control
+unit: test plan
+_name: Tests for snap command
+_description:
+ QA test plan that includes generic tests for the snap command for Snappy
+ Ubuntu Core devices.
+estimated_duration: 5m
+include:
+nested_part:
+    snappy-snap-manual-with-refresh-control
+    snappy-snap-automated
+
 id: snappy-snap-manual
 unit: test plan
 _name: QA tests for snap command
@@ -30,6 +42,26 @@ include:
 mandatory_include:
     snap
 
+id: snappy-snap-manual-with-refresh-control
+unit: test plan
+_name: QA tests for snap command on refresh controlled stores
+_description:
+ QA test that includes manual tests for the snap command for Snappy Ubuntu
+ Core devices.
+include:
+    snappy/snap-refresh
+    snappy/snap-revert
+    snappy/snap-reupdate
+    snappy/os-refresh-with-refresh-control
+    snappy/os-revert-with-refresh-control
+    snappy/os-fail-boot-with-refresh-control
+    snappy/sshd
+    snappy/snapweb
+    snappy/snapweb-search
+    snappy/snapweb-install
+mandatory_include:
+    snap
+
 id: snappy-snap-automated
 unit: test plan
 _name: Automated tests for snap command