Merge lp:~blake-rouse/curtin/fix-1453869 into lp:~curtin-dev/curtin/trunk

[Blake Rouse]

Retry apt-get update and apt-get install command a maximum of 3 times to make sure no transient issues prevent the deployment from succeeding.

[lp:~blake-rouse/curtin/fix-1453869]

212. By Blake Rouse on 2015-06-05

Use sleeps between retries. Remove retries on apt-get install.

=== modified file 'curtin/deps/install.py'
--- curtin/deps/install.py	2015-03-20 20:20:14 +0000
+++ curtin/deps/install.py	2015-06-05 17:05:49 +0000
@@ -23,6 +23,7 @@
 if __name__ == '__main__':
     import subprocess
     import sys
+    import time
     if sys.version_info[0] == 2:
         pkgs = ['python-yaml']
     else:
@@ -32,8 +33,20 @@
 
     cmds = [apt_update, apt_install + pkgs]
     for cmd in cmds:
-        try:
-            subprocess.check_call(cmd)
-        except subprocess.CalledProcessError as e:
-            sys.exit(e.returncode)
+        # Retry each command with a wait between. The final wait time in this
+        # list is zero because we don't need to wait at the end of the last
+        # call.
+        wait_times = [0.5, 1, 2, 0]
+        for i, wait in enumerate(wait_times):
+            try:
+                subprocess.check_call(cmd)
+                returncode = 0
+            except subprocess.CalledProcessError as e:
+                returncode = e.returncode
+            if returncode == 0:
+                break
+            else:
+                time.sleep(wait)
+        if returncode != 0:
+            sys.exit(returncode)
     sys.exit(0)

=== modified file 'curtin/util.py'
--- curtin/util.py	2015-06-05 13:46:46 +0000
+++ curtin/util.py	2015-06-05 17:05:49 +0000
@@ -30,8 +30,8 @@
 _INSTALLED_MAIN = '/usr/bin/curtin'
 
 
-def subp(args, data=None, rcs=None, env=None, capture=False, shell=False,
-         logstring=False):
+def _subp(args, data=None, rcs=None, env=None, capture=False, shell=False,
+          logstring=False):
     if rcs is None:
         rcs = [0]
 
@@ -73,6 +73,25 @@
     return (out, err)
 
 
+def subp(*args, **kwargs):
+    retries = []
+    if "retries" in kwargs:
+        retries = kwargs.pop("retries")
+
+    # Add the final wait time at the end. It is zero because we want
+    # to execute the final command and don't want it to wait.
+    retries.append(0)
+
+    # Retry with waits between the retried command.
+    for i, wait in enumerate(retries):
+        try:
+            return _subp(*args, **kwargs)
+        except ProcessExecutionError:
+            if i == len(retries) - 1:
+                raise
+            time.sleep(wait)
+
+
 def load_command_environment(env=os.environ, strict=False):
 
     mapping = {'scratch': 'WORKING_DIR', 'fstab': 'OUTPUT_FSTAB',
@@ -431,7 +450,8 @@
         return False
 
 
-def apt_update(target=None, env=None, force=False, comment=None):
+def apt_update(target=None, env=None, force=False, comment=None,
+               retries=[0.5, 1, 2]):
     marker = "tmp/curtin.aptupdate"
     if target is None:
         target = "/"
@@ -451,7 +471,7 @@
 
     apt_update = ['apt-get', 'update', '--quiet']
     with RunInChroot(target) as inchroot:
-        inchroot(apt_update, env=env)
+        inchroot(apt_update, env=env, retries=retries)
 
     with open(marker, "w") as fp:
         fp.write(comment + "\n")
@@ -479,7 +499,8 @@
 
     apt_update(target, comment=' '.join(pkglist))
     with RunInChroot(target) as inchroot:
-        return inchroot(emd + apt_inst_cmd + list(pkglist), env=env)
+        return inchroot(
+            emd + apt_inst_cmd + list(pkglist), env=env)
 
 
 def is_uefi_bootable():