Merge lp:~gl-az/percona-xtrabackup/2.1-testrun-summary into lp:percona-xtrabackup/2.1

[George Ormond Lorch III]

Fixed issue in testrun.c where specifying testsuite would find the tests in the corredt directory but would still try to execute them out of ./t

Added test status tracking and summary report at end of test run cycle to make it wasy to see what passed, skipped and failed at a glance, similar to MTR.

Changed testrun.sh slightly to return same code out of testrun.sh as the test that was executed when only a single test was run.

[Laurynas Biveinis (laurynas-biveinis)]

Same comments as for the 2.0 MP.

review: Needs Fixing

[Laurynas Biveinis (laurynas-biveinis)]

Same comments as for 2.0 MP.

review: Needs Fixing

[Laurynas Biveinis (laurynas-biveinis)]

review: Approve

[Alexey Kopytov (akopytov)]

Superseded with https://code.launchpad.net/~akopytov/percona-xtrabackup/test-suite-cleanups-2.1/+merge/172963

Unmerged revisions

610. By George Ormond Lorch III on 2013-06-06

Fixed issue in testrun.c where specifying testsuite would find the tests in the corredt directory but would still try to execute them out of ./t

Added test status tracking and summary report at end of test run cycle to make it wasy to see what passed, skipped and failed at a glance, similar to MTR.

Changed testrun.sh slightly to return same code out of testrun.sh as the test that was executed when only a single test was run.

=== modified file 'test/testrun.c'
--- test/testrun.c	2013-01-11 13:00:50 +0000
+++ test/testrun.c	2013-06-06 05:54:39 +0000
@@ -41,6 +41,7 @@
   char *buf;
   size_t end_of_buf;
   size_t bufsz;
+  int status;
 };
 
 pid_t *childpid;
@@ -96,6 +97,7 @@
     (*cases)[i].buf= NULL;
     (*cases)[i].end_of_buf= 0;
     (*cases)[i].bufsz= 0;
+    (*cases)[i].status= 0;
     free(namelist[i]);
   }
 
@@ -113,7 +115,41 @@
   free(cases);
 }
 
-static int run_testcase_in_child(int nr, struct testcase *t, pid_t *cpid, const char* xbtarget)
+static void report_status(struct testcase *cases, int n)
+{
+  int i;
+  int failed=0;
+  int skipped=0;
+  for (i=0; i<n; i++)
+    if (WEXITSTATUS(cases[i].status) != 0)
+      if (WEXITSTATUS(cases[i].status) == 200)
+        skipped++;
+      else
+        failed++;
+
+  printf("SUMMARY: %d run, %d successful, %d skipped, %d failed\n", n, n - failed - skipped, skipped, failed);
+  if (skipped)
+  {
+    printf("Skipped cases:");
+    for (i=0; i<n; i++) 
+      if (WEXITSTATUS(cases[i].status) == 200)
+        printf(" %s", cases[i].name);
+    printf("\n");
+  }
+
+  if (failed)
+  {
+    printf("Failing cases:");
+    for (i=0; i<n; i++) 
+      if (WEXITSTATUS(cases[i].status) != 0 && WEXITSTATUS(cases[i].status) != 200)
+        printf(" %s", cases[i].name);
+    printf("\n");
+  }
+}
+
+static int run_testcase_in_child(const char* suitedir, int nr,
+				 struct testcase *t, pid_t *cpid,
+				 const char* xbtarget)
 {
   int fd[2];
 
@@ -132,7 +168,7 @@
     close(fd[0]);
 
     char tname[500];
-    snprintf(tname, sizeof(tname), "t/%s",t->name);
+    snprintf(tname, sizeof(tname), "%s/%s",suitedir, t->name);
 
     char basedir[PATH_MAX];
     char cwd[PATH_MAX];
@@ -195,14 +231,14 @@
   free(buf);
 }
 
-static void run_testcases(struct testcase *testcases, int nrcases,
-			  int njobs, int timeout, const char* xbtarget)
+static void run_testcases(const char* suitedir, struct testcase *testcases,
+			  int nrcases, int njobs, int timeout,
+			  const char* xbtarget)
 {
   int childfd[njobs];
   int nfds= 0;
   int retval;
   pid_t chpid[njobs];
-  int status;
   int next_testcase= 0;
   int i;
   fd_set rfds;
@@ -223,7 +259,7 @@
   for(i=0; i<njobs; i++)
   {
     childtest[i]=next_testcase++;
-    childfd[i]= run_testcase_in_child(i, &testcases[childtest[i]], &childpid[i], xbtarget);
+    childfd[i]= run_testcase_in_child(suitedir, i, &testcases[childtest[i]], &childpid[i], xbtarget);
   }
 
   fflush(stdout);
@@ -277,8 +313,8 @@
 	    t->end_of_buf+= r;
 	} while(r>0);
 
-	pid_t waited= waitpid(childpid[i], &status, WNOHANG);
-	if (!(WIFEXITED(status) || WIFSIGNALED(status)))
+	pid_t waited= waitpid(childpid[i], &testcases[childtest[i]].status, WNOHANG);
+	if (!(WIFEXITED(testcases[childtest[i]].status) || WIFSIGNALED(testcases[childtest[i]].status)))
 	  continue;
 
 	if (waited != childpid[i])
@@ -289,14 +325,14 @@
 
 	close(childfd[i]);
 	printf("[%d] completed %s status %d\n",
-	       i, testcases[childtest[i]].name, WEXITSTATUS(status));
+	       i, testcases[childtest[i]].name, WEXITSTATUS(testcases[childtest[i]].status));
 	childfd[i]=-1;
 	nchildren--;
 
 	if (next_testcase < nrcases)
 	{
 	  childtest[i]=next_testcase++;
-	  childfd[i]= run_testcase_in_child(i, &testcases[childtest[i]], &childpid[i], xbtarget);
+	  childfd[i]= run_testcase_in_child(suitedir, i, &testcases[childtest[i]], &childpid[i], xbtarget);
 	  nfds= (childfd[i] > nfds)? childfd[i] : nfds;
 	  nchildren++;
 	}
@@ -397,7 +433,9 @@
 
   printf("Found %d testcases\n", nrcases);
 
-  run_testcases(testcases, nrcases, njobs, timeout, xbtarget);
+  run_testcases(suitedir, testcases, nrcases, njobs, timeout, xbtarget);
+
+  report_status(testcases, nrcases);
 
   free_testcases(testcases, nrcases);
 
 
=== modified file 'test/testrun.sh'
--- test/testrun.sh	2013-05-17 11:07:26 +0000
+++ test/testrun.sh	2013-06-06 05:54:39 +0000
@@ -281,6 +281,7 @@
 
 failed_count=0
 failed_tests=
+skipped_count=0
 total_count=0
 
 export OUTFILE="$PWD/results/setup"
@@ -334,6 +335,7 @@
        test -r $SKIPPED_REASON && sreason=`cat $SKIPPED_REASON`
        echo "[skipped]    $sreason"
        subunit_skip_test $t >> $SUBUNIT_OUT
+       skipped_count=`expr $skipped_count + 1`
    else
        echo "[failed]"
 
@@ -345,7 +347,7 @@
 
        failed_count=$((failed_count+1))
        failed_tests="$failed_tests $t"
-       result=1
+       result=$rc
        if [ -z "$force" ]
        then
 	   break;
@@ -356,10 +358,18 @@
 echo "========================================================================"
 echo
 
-if [ $result -eq 1 ]
+if [ $result -ne 0 ]
 then
     echo
     echo "$failed_count/$total_count tests have failed: $failed_tests"
     echo "See results/ for detailed output"
-    exit -1
+    if [ $total_count -gt 1 ]
+    then
+      exit -1
+    else
+      exit $result
+    fi
+elif [ $total_count -eq 1 ] && [ $skipped_count -eq 1 ]
+then
+    exit $SKIPPED_EXIT_CODE
 fi