Merge ~paelzer/ubuntu/+source/open-iscsi:retry-boot-tests-cosmic into ubuntu/+source/open-iscsi:ubuntu/cosmic-devel

[Christian Ehrhardt ]

Unmerged commits

4d42110... by Christian Ehrhardt  on 2018-07-18

changelog: retry test being flaky on LP

Signed-off-by: Christian Ehrhardt <email address hidden>

8a38bb5... by Christian Ehrhardt  on 2018-07-18

retry test being flaky on LP

Signed-off-by: Christian Ehrhardt <email address hidden>

diff --git a/debian/changelog b/debian/changelog
index 4474c35..1997399 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,4 @@
+<<<<<<< debian/changelog
 open-iscsi (2.0.874-5ubuntu4) cosmic; urgency=medium
 
   * debian/tests/install: ignore the potential stderr of the probing command
@@ -20,6 +21,19 @@ open-iscsi (2.0.874-5ubuntu3) cosmic; urgency=medium
     - debian/tests/install: fix tests to work with socket activation
 
  -- Christian Ehrhardt <[e-mail]>  Wed, 23 May 2018 15:50:01 +0200
+=======
+open-iscsi (2.0.874-5ubuntu3) cosmic; urgency=medium
+
+  * debian/tests/tgt-boot-test: The nested kvm based boot is test is known to
+    be flaky (on LP infrastructure only), so retry it.
+    The issue usually is after the actually-to-be-tested iscsi boot itself,
+    initialization does not fully run to cloud-init to shut it down.
+    Sucessful runs usually take ~230-450 seconds, so set timeout down from 60
+    to 10 minutes and retry 6 times; That is approximately the same worst case
+    runtime for a six times better chance to get a good run.
+
+ -- Christian Ehrhardt <[e-mail]>  Wed, 18 Jul 2018 11:00:25 +0200
+>>>>>>> debian/changelog
 
 open-iscsi (2.0.874-5ubuntu2) bionic; urgency=medium
 
diff --git a/debian/tests/tgt-boot-test b/debian/tests/tgt-boot-test
index 2f8bff3..c098186 100755
--- a/debian/tests/tgt-boot-test
+++ b/debian/tests/tgt-boot-test
@@ -511,20 +511,36 @@ main() {
         netdev_args[${#netdev_args[@]}]="--netdev=$i"
     done
 
-    local mem="512" start="$SECONDS" ret=""
+    local mem="512" ret=""
     local cmd=""
     cmd=(
-        timeout --kill-after=1m --signal=TERM ${BOOT_TIMEOUT:-60m}
+        timeout --kill-after=1m --signal=TERM ${BOOT_TIMEOUT:-10m}
         xkvm  "${netdev_args[@]}" ${overlay_drive_xkvm} "${pt[@]}" --
         -echr 0x5
         -m $mem ${serial_log:+-serial "file:$serial_log"} -nographic
         -kernel "${t_kernel}" -initrd "${t_initrd}"
         -append "${cmdline}"
     )
-    debug 1 "executing: ${cmd[*]}"
-    "${cmd[@]}"
-    ret=$?
-    debug 1 "xkvm returned $ret in $((SECONDS-start))s"
+    # The nested kvm based boot is test is known to be flaky (on LP
+    # infrastructure only), so retry it.
+    # The issue usually is after the actually-to-be-tested iscsi boot itself,
+    # initialization does not fully run to cloud-init to shut it down.
+    # Sucessful runs usually take ~230-450 seconds, so set timeout down from 60
+    # to 10 minutes and retry 6 times; That is approximately the same worst case
+    # runtime for a six times better chance to get a good run.
+    local retry=0
+    local maxretries=6
+    until [ ${retry} -ge ${maxretries} ]; do
+        ((retry++))
+        debug 1 "Try ${retry}/${maxretries}: executing: ${cmd[*]}"
+        local starttime="$SECONDS"
+        "${cmd[@]}"
+        ret=$?
+        debug 1 "Try ${retry}/${maxretries}: xkvm returned $ret in $((SECONDS-starttime))s"
+        if [ ${ret} -eq 0 ]; then
+            break
+        fi
+    done
     return $ret
 }