Merge lp:~le-chi-thu/lava-dispatcher/add-deploy-retry into lp:lava-dispatcher

[Le Chi Thu]

Added retry when deploy image.

[Michael Hudson-Doyle (mwhudson)]

Looks good, thanks for this.

review: Approve

=== modified file 'doc/changes.rst'
--- doc/changes.rst	2012-05-23 21:49:59 +0000
+++ doc/changes.rst	2012-05-24 21:49:18 +0000
@@ -3,6 +3,7 @@
 
 * Fixed reboot issues
 * Skip raising exception on the home screen has not displayed for health check jobs
+* Retry deployment if failed.
 
 .. _version_0_7_1:
 

=== modified file 'lava_dispatcher/client/master.py'
--- lava_dispatcher/client/master.py	2012-05-22 13:03:31 +0000
+++ lava_dispatcher/client/master.py	2012-05-24 21:49:18 +0000
@@ -67,16 +67,43 @@
         if rc:
             raise RuntimeError("Failed to create tarball: %s" % tarfile)
 
-def _deploy_tarball_to_board(session, tarball_url, dest, timeout=-1):
+def _deploy_tarball_to_board(session, tarball_url, dest, timeout=-1, num_retry=2):
     decompression_char = ''
     if tarball_url.endswith('.gz') or tarball_url.endswith('.tgz'):
         decompression_char = 'z'
     elif tarball_url.endswith('.bz2'):
         decompression_char = 'j'
-    session.run(
-        'wget --no-proxy --connect-timeout=30 -S -O- %s --progress=dot -e dotbytes=2M | tar --numeric-owner -C %s -x%sf -' % (
-            tarball_url, dest, decompression_char),
-        timeout=timeout)
+
+    deploy_ok=False
+
+    while (num_retry > 0 and not deploy_ok):
+        try:
+            session._empty_pexpect_buffer()
+            session._client.proc.sendline(
+                'wget --no-proxy --connect-timeout=30 -S -O- %s --progress=dot -e dotbytes=2M |'
+                'tar --numeric-owner -C %s -x%sf -'
+                % (tarball_url, dest, decompression_char))
+
+            match_id = session._client.proc.expect(["tar: Error", "unexpected end of file",
+                                                    "tar: Unexpected EOF in archive",session._prompt_str], timeout=timeout)
+            if match_id != 3:
+                logging.warning("Deploy %s failed. %d retry left." %(tarball_url, num_retry-1))
+            else:
+                deploy_ok=True
+        except:
+            logging.warning("Deploy %s failed. %d retry left." %(tarball_url, num_retry-1))
+        # do some delay before retry
+        if (num_retry > 1 and not deploy_ok):
+            sleep_time=5*60
+            logging.info("Wait %d second before retry" % sleep_time)
+            time.sleep(sleep_time)
+            # send CTRL C to exit last command.
+            session._client.proc.sendline("\003")
+            session._client.proc.sendline("echo 'retry left %s'" % num_retry-1)
+        num_retry = num_retry - 1
+
+    if not deploy_ok:
+        raise Exception("Deploy tarball (%s) to board failed" % tarball_url);
 
 def _deploy_linaro_rootfs(session, rootfs):
     logging.info("Deploying linaro image")
@@ -754,6 +781,7 @@
             self.proc.sendline("hardreset")
         # after hardreset empty the pexpect buffer
         self._empty_pexpect_buffer()
+
     def _empty_pexpect_buffer(self):
         """Make sure there is nothing in the pexpect buffer."""
         index = 0