Merge lp:~akopytov/percona-xtrabackup/test-suite-cleanups-2.1 into lp:percona-xtrabackup/2.1

[Alexey Kopytov]

[Alexey Kopytov (akopytov)]

http://jenkins.percona.com/view/XtraBackup/job/percona-xtrabackup-2.1-param/360/

Same comments on remaining failures as in the 2.0 MP.

[Alexey Kopytov (akopytov)]

Would be nice to have this merged sooner rather than later, as I have a pile of other fixes which depends on this branch.

[George Ormond Lorch III (gl-az)]

Same comments as 2.0 MP, nice work!

review: Approve (g2)

=== modified file 'patches/innodb56.patch'
--- patches/innodb56.patch	2013-07-01 12:21:20 +0000
+++ patches/innodb56.patch	2013-07-04 07:51:31 +0000
@@ -1431,4 +1431,20 @@
      int main()
      {
        extern void __attribute__((weak)) foo(void);
-
+--- a/mysys/my_thr_init.c
++++ b/mysys/my_thr_init.c
+@@ -462,8 +462,12 @@
+ 
+ extern void **my_thread_var_dbug()
+ {
+-  struct st_my_thread_var *tmp=
+-    my_pthread_getspecific(struct st_my_thread_var*,THR_KEY_mysys);
++  struct st_my_thread_var *tmp;
++
++  if (!my_thread_global_init_done)
++    return NULL;
++
++  tmp= my_pthread_getspecific(struct st_my_thread_var*,THR_KEY_mysys);
+   return tmp && tmp->init ? &tmp->dbug : 0;
+ }
+ #endif /* DBUG_OFF */
 
=== modified file 'src/xtrabackup.cc'
--- src/xtrabackup.cc	2013-06-18 14:49:01 +0000
+++ src/xtrabackup.cc	2013-07-04 07:51:31 +0000
@@ -3835,6 +3835,8 @@
 
 	ulint	fold;
 
+	bool	checkpoint_found;
+
 	max_no = ut_dulint_zero;
 
 	if (!xb_init_log_block_size()) {
@@ -3949,6 +3951,8 @@
 		//		' ', 4);
 	}
 
+	checkpoint_found = false;
+
 	/* read last checkpoint lsn */
 	for (field = LOG_CHECKPOINT_1; field <= LOG_CHECKPOINT_2;
 			field += LOG_CHECKPOINT_2 - LOG_CHECKPOINT_1) {
@@ -3961,25 +3965,13 @@
 		if (ut_dulint_cmp(checkpoint_no, max_no) >= 0) {
 			max_no = checkpoint_no;
 			max_lsn = MACH_READ_64(log_buf + field + LOG_CHECKPOINT_LSN);
-/*
-			mach_write_to_4(log_buf + field + LOG_CHECKPOINT_OFFSET,
-					LOG_FILE_HDR_SIZE + ut_dulint_minus(max_lsn,
-					ut_dulint_align_down(max_lsn,OS_FILE_LOG_BLOCK_SIZE)));
-
-			ulint	fold;
-			fold = ut_fold_binary(log_buf + field, LOG_CHECKPOINT_CHECKSUM_1);
-			mach_write_to_4(log_buf + field + LOG_CHECKPOINT_CHECKSUM_1, fold);
-
-			fold = ut_fold_binary(log_buf + field + LOG_CHECKPOINT_LSN,
-				LOG_CHECKPOINT_CHECKSUM_2 - LOG_CHECKPOINT_LSN);
-			mach_write_to_4(log_buf + field + LOG_CHECKPOINT_CHECKSUM_2, fold);
-*/
+			checkpoint_found = true;
 		}
 not_consistent:
 		;
 	}
 
-	if (ut_dulint_cmp(max_no, ut_dulint_zero) == 0) {
+	if (!checkpoint_found) {
 		msg("xtrabackup: No valid checkpoint found.\n");
 		goto error;
 	}
 
=== modified file 'test/inc/common.sh'
--- test/inc/common.sh	2013-06-28 15:23:23 +0000
+++ test/inc/common.sh	2013-07-04 07:51:31 +0000
@@ -16,9 +16,12 @@
 }
 
 
-function clean()
+########################################################################
+# Remove server var* directories from the worker's var root
+########################################################################
+function remove_var_dirs()
 {
-    rm -rf ${TEST_BASEDIR}/var[0-9]
+    rm -rf ${TEST_VAR_ROOT}/var[0-9]
 }
 
 function die()
@@ -34,29 +37,33 @@
 	$MYSQL_INSTALL_DB --defaults-file=${MYSQLD_VARDIR}/my.cnf \
                     --basedir=${MYSQL_BASEDIR} \
                     ${MYSQLD_EXTRA_ARGS}
-	cd -
+	cd - >/dev/null 2>&1
 }
 
 ########################################################################
-# Checks whether MySQL is alive
+# Checks whether MySQL with the PID specified as an argument is alive
 ########################################################################
 function mysql_ping()
 {
-    $MYSQLADMIN $MYSQL_ARGS --wait=100 ping >/dev/null 2>&1 || return 1
-}
+    local pid=$1
+    local attempts=60
+    local i
 
-function kill_leftovers()
-{
-    local file
-    for file in ${TEST_BASEDIR}/mysqld*.pid
+    for ((i=1; i<=attempts; i++))
     do
-	if [ -f $file ]
-	then
-	    vlog "Found a leftover mysqld processes with PID `cat $file`, stopping it"
-	    kill -9 `cat $file` 2>/dev/null || true
-	    rm -f $file
-	fi
+        $MYSQLADMIN $MYSQL_ARGS ping >/dev/null 2>&1 && return 0
+        sleep 1
+        # Is the server process still alive?
+        if ! kill -0 $pid >/dev/null 2>&1
+        then
+            vlog "Server process PID=$pid died."
+            wait $pid
+            return 1
+        fi
+        vlog "Made $i attempts to connect to server"
     done
+
+    return 1
 }
 
 function run_cmd()
@@ -123,21 +130,14 @@
     local port
     local lockfile
 
-    for (( port=PORT_BASE+id; port < 65535; port++))
+    for (( port=3307 + RANDOM; port < 65535; port++))
     do
 	lockfile="/tmp/xtrabackup_port_lock.$port"
 	# Try to atomically lock the current port number
-	if ! set -C > $lockfile
+	if ! (set -C; > $lockfile)
 	then
-	    set +C
-	    # check if the process still exists
-	    if kill -0 "`cat $lockfile`"
-	    then
-		continue;
-	    fi
-	    # stale file, overwrite it with the current PID
+	    continue;
 	fi
-	set +C
 	echo $$ > $lockfile
 	if ! nc -z -w1 localhost $port >/dev/null 2>&1
 	then
@@ -173,14 +173,14 @@
     fi
 
     SRV_MYSQLD_IDS[$id]="$id"
-    local vardir="${TEST_BASEDIR}/var${id}"
+    local vardir="${TEST_VAR_ROOT}/var${id}"
     SRV_MYSQLD_VARDIR[$id]="$vardir"
     SRV_MYSQLD_DATADIR[$id]="$vardir/data"
     SRV_MYSQLD_TMPDIR[$id]="$vardir/tmp"
-    SRV_MYSQLD_PIDFILE[$id]="${TEST_BASEDIR}/mysqld${id}.pid"
+    SRV_MYSQLD_PIDFILE[$id]="${TEST_VAR_ROOT}/mysqld${id}.pid"
     SRV_MYSQLD_ERRFILE[$id]="$vardir/data/mysqld${id}.err"
     SRV_MYSQLD_PORT[$id]=`get_free_port $id`
-    SRV_MYSQLD_SOCKET[$id]=`mktemp -t xtrabackup.mysql.sock.XXXXXX`
+    SRV_MYSQLD_SOCKET[$id]=`mktemp -t mysql.sock.XXXXXX`
 }
 
 ########################################################################
@@ -245,27 +245,36 @@
 function start_server_with_id()
 {
     local id=$1
+    local attempts=0
+    local max_attempts=5
     shift
 
     vlog "Starting server with id=$id..."
 
-    init_server_variables $id
-    switch_server $id
-
-    if [ ! -d "$MYSQLD_VARDIR" ]
-    then
-	vlog "Creating server root directory: $MYSQLD_VARDIR"
-	mkdir "$MYSQLD_VARDIR"
-    fi
-    if [ ! -d "$MYSQLD_TMPDIR" ]
-    then
-	vlog "Creating server temporary directory: $MYSQLD_TMPDIR"
-	mkdir "$MYSQLD_TMPDIR"
-    fi
-
-    # Create the configuration file used by mysql_install_db, the server
-    # and the xtrabackup binary
-    cat > ${MYSQLD_VARDIR}/my.cnf <<EOF
+    while true
+    do
+        init_server_variables $id
+        switch_server $id
+
+        if [ ! -d "$MYSQLD_VARDIR" ]
+        then
+	    vlog "Creating server root directory: $MYSQLD_VARDIR"
+	    mkdir "$MYSQLD_VARDIR"
+        fi
+        if [ ! -d "$MYSQLD_TMPDIR" ]
+        then
+	    vlog "Creating server temporary directory: $MYSQLD_TMPDIR"
+	    mkdir "$MYSQLD_TMPDIR"
+        fi
+
+        if [ -f "$MYSQLD_ERRFILE" ]
+        then
+            rm "$MYSQLD_ERRFILE"
+        fi
+
+        # Create the configuration file used by mysql_install_db, the server
+        # and the xtrabackup binary
+        cat > ${MYSQLD_VARDIR}/my.cnf <<EOF
 [mysqld]
 socket=${MYSQLD_SOCKET}
 port=${MYSQLD_PORT}
@@ -285,22 +294,42 @@
 user=root
 EOF
 
-    # Create datadir and call mysql_install_db if it doesn't exist
-    if [ ! -d "$MYSQLD_DATADIR" ]
-    then
-	vlog "Creating server data directory: $MYSQLD_DATADIR"
-	mkdir -p "$MYSQLD_DATADIR"
-	call_mysql_install_db
-    fi
-
-    # Start the server
-    echo "Starting ${MYSQLD} ${MYSQLD_ARGS} $* "
-    ${MYSQLD} ${MYSQLD_ARGS} $* &
-    if ! mysql_ping
-    then
-        die "Can't start the server!"
-    fi
-    vlog "Server with id=$id has been started on port $MYSQLD_PORT, \
+        # Create datadir and call mysql_install_db if it doesn't exist
+        if [ ! -d "$MYSQLD_DATADIR" ]
+        then
+            vlog "Creating server data directory: $MYSQLD_DATADIR"
+            mkdir -p "$MYSQLD_DATADIR"
+            call_mysql_install_db
+        fi
+
+        # Start the server
+        echo "Starting ${MYSQLD} ${MYSQLD_ARGS} $* "
+        ${MYSQLD} ${MYSQLD_ARGS} $* &
+        if ! mysql_ping $!
+        then
+            if grep "another mysqld server running on port" $MYSQLD_ERRFILE
+            then
+                if ((++attempts < max_attempts))
+                then
+                    vlog "Made $attempts attempts to find a free port"
+                    reset_server_variables $id
+                    free_reserved_port $MYSQLD_PORT
+                    continue
+                else
+                    vlog "Failed to find a free port after $attempts attempts"
+                fi
+            fi
+            vlog "Can't start the server. Server log (if exists):"
+            vlog "----------------"
+            cat ${MYSQLD_ERRFILE} >&2 || true
+            vlog "----------------"
+            exit -1
+        else
+            break
+        fi
+    done
+
+     vlog "Server with id=$id has been started on port $MYSQLD_PORT, \
 socket $MYSQLD_SOCKET"
 }
 
@@ -317,7 +346,9 @@
 
     if [ -f "${MYSQLD_PIDFILE}" ]
     then
-        kill -9 `cat ${MYSQLD_PIDFILE}`
+        local pid=`cat $MYSQLD_PIDFILE`
+        kill -9 $pid
+        wait $pid || true
         rm -f ${MYSQLD_PIDFILE}
     else
         vlog "Server PID file '${MYSQLD_PIDFILE}' doesn't exist!"
@@ -484,7 +515,8 @@
 ##########################################################################
 function record_db_state()
 {
-    $MYSQLDUMP $MYSQL_ARGS -t --compact $1 >"$topdir/tmp/$1_old.sql"
+    $MYSQLDUMP $MYSQL_ARGS -t --compact --skip-extended-insert \
+        $1 >"$topdir/tmp/$1_old.sql"
 }
 
 
@@ -494,7 +526,8 @@
 ##########################################################################
 function verify_db_state()
 {
-    $MYSQLDUMP $MYSQL_ARGS -t --compact $1 >"$topdir/tmp/$1_new.sql"
+    $MYSQLDUMP $MYSQL_ARGS -t --compact --skip-extended-insert \
+        $1 >"$topdir/tmp/$1_new.sql"
     diff -u "$topdir/tmp/$1_old.sql" "$topdir/tmp/$1_new.sql"
 }
 
@@ -514,14 +547,14 @@
     return ${PIPESTATUS[0]}
 }
 
-readonly xb_performed_bmp_inc_backup="xtrabackup: using the full scan for incremental backup"
-readonly xb_performed_full_scan_inc_backup="xtrabackup: using the changed page bitmap"
-
 ####################################################
 # Helper functions for testing incremental backups #
 ####################################################
 function check_full_scan_inc_backup()
 {
+    local xb_performed_bmp_inc_backup="xtrabackup: using the full scan for incremental backup"
+    local xb_performed_full_scan_inc_backup="xtrabackup: using the changed page bitmap"
+
     if ! grep -q "$xb_performed_bmp_inc_backup" $OUTFILE ;
     then
         vlog "xtrabackup did not perform a full scan for the incremental backup."
@@ -536,6 +569,9 @@
 
 function check_bitmap_inc_backup()
 {
+    local xb_performed_bmp_inc_backup="xtrabackup: using the full scan for incremental backup"
+    local xb_performed_full_scan_inc_backup="xtrabackup: using the changed page bitmap"
+
     if ! grep -q "$xb_performed_full_scan_inc_backup" $OUTFILE ;
     then
         vlog "xtrabackup did not use bitmaps for the incremental backup."
 
=== removed file 'test/run.sh'
--- test/run.sh	2012-11-30 04:28:06 +0000
+++ test/run.sh	1970-01-01 00:00:00 +0000
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-XB_BUILD="autodetect"
-while getopts "fc:" options; do
-	case $options in
-	        c ) XB_BUILD="$OPTARG";;
-		f ) ;; # ignored
-	esac
-done
-
-rm -rf results/ var/ test_results.subunit 
-
-CFLAGS=-g make testrun
-./testrun -c $XB_BUILD
 
=== renamed file 'test/testrun.sh' => 'test/run.sh'
--- test/testrun.sh	2013-05-17 11:07:26 +0000
+++ test/run.sh	2013-07-04 07:51:31 +0000
@@ -2,14 +2,39 @@
 
 export DEBUG=
 
+shopt -s extglob
+
+set +e
+
 . inc/common.sh
-
-trap kill_leftovers 1 2 3 15
-
-set +e
+. subunit.sh
+
+trap cleanup_on_exit EXIT
+trap terminate SIGHUP SIGINT SIGQUIT SIGTERM
 
 result=0
 
+# Default test timeout in seconds
+TEST_TIMEOUT=600
+
+# Magic exit code to indicate a skipped test
+export SKIPPED_EXIT_CODE=200
+
+# Default server installation directory (-d option)
+MYSQL_BASEDIR=${MYSQL_BASEDIR:-"$PWD/server"}
+
+TEST_BASEDIR="$PWD"
+TEST_VAR_ROOT="$TEST_BASEDIR/var"
+
+# Global statistics
+FAILED_COUNT=0
+FAILED_TESTS=
+SKIPPED_COUNT=0
+SKIPPED_TESTS=
+SUCCESSFUL_COUNT=0
+TOTAL_COUNT=0
+TOTAL_TIME=0
+
 function usage()
 {
 cat <<EOF
@@ -24,7 +49,183 @@
 -c conf     XtraBackup build configuration as specified to build.sh.
             Default is 'autodetect', i.e. xtrabackup binary is determined based
             on the MySQL version.
-EOF
+-j N        Run tests in N parallel processes.
+-T seconds  Test timeout (default is $TEST_TIMEOUT seconds).
+EOF
+}
+
+###############################################################################
+# Calculate the number of parallel workers automatically based on the number of
+# available cores
+###############################################################################
+function autocalc_nworkers()
+{
+    if [ -r /proc/cpuinfo ]
+    then
+        NWORKERS=`grep processor /proc/cpuinfo | wc -l`
+    elif which sysctl >/dev/null 2>&1
+    then
+        NWORKERS=`sysctl -n hw.ncpu`
+    fi
+
+    if [[ ! $NWORKERS =~ ^[0-9]+$ || $NWORKERS < 1 ]]
+    then
+        echo "Cannot determine the number of available CPU cures!"
+        exit -1
+    fi
+
+    if [ "$NWORKERS" -gt 16 ]
+    then
+        echo "Autodetected number of cores: $NWORKERS"
+        echo "Limiting to 16 to avoid excessive resource consumption"
+
+        NWORKERS=16
+    fi
+}
+
+###############################################################################
+# Kill a specified worker
+###############################################################################
+function kill_worker()
+{
+    local worker=$1
+    local pid=${worker_pids[$worker]}
+
+    worker_pids[$worker]=""
+
+    if ! kill -0 $pid >/dev/null 2>&1
+    then
+        wait $pid >/dev/null 2>&1 || true
+        return 0
+    fi
+
+    # First send SIGTERM to let worker exit gracefully
+    kill -SIGTERM $pid >/dev/null 2>&1 || true
+
+    sleep 1
+
+    if ! kill -0 $pid >/dev/null 2>&1
+    then
+        wait $pid >/dev/null 2>&1 || true
+        return 0
+    fi
+
+    # Now kill with SIGKILL
+    kill -SIGKILL $pid >/dev/null 2>&1 || true
+    wait $pid >/dev/null 2>&1 || true
+    release_port_locks $pid
+}
+
+########################################################################
+# Kill all running workers
+########################################################################
+function kill_all_workers()
+{
+    while true
+    do
+        found=""
+
+        # First send SIGTERM to let workers exit gracefully
+        for ((i = 1; i <= NWORKERS; i++))
+        do
+            [ -z ${worker_pids[$i]:-""} ] && continue
+
+            found="yes"
+
+           if ! kill -0 ${worker_pids[$i]} >/dev/null 2>&1
+           then
+               worker_pids[$i]=""
+               continue
+           fi
+
+            kill -SIGTERM ${worker_pids[$i]} >/dev/null 2>&1 || true
+        done
+
+        [ -z "$found" ] && break
+
+        sleep 1
+
+        # Now kill with SIGKILL
+        for ((i = 1; i <= NWORKERS; i++))
+        do
+            [ -z ${worker_pids[$i]:-""} ] && continue
+
+            if ! kill -0 ${worker_pids[$i]} >/dev/null 2>&1
+            then
+                wait ${worker_pids[$i]} >/dev/null 2>&1 || true
+                worker_pids[$i]=""
+                continue
+            fi
+
+            kill -SIGKILL ${worker_pids[$i]} >/dev/null 2>&1 || true
+            wait ${worker_pids[$i]} >/dev/null 2>&1 || true
+            release_port_locks ${worker_pids[$i]}
+            worker_pids[$i]=""
+        done
+    done
+}
+
+########################################################################
+# Handler called from a fatal signal handler trap
+########################################################################
+function terminate()
+{
+    echo "Terminated, cleaning up..."
+
+    # The following will call cleanup_on_exit()
+    exit 2
+}
+
+########################################################################
+# Display the test run summary and exit
+########################################################################
+function print_status_and_exit()
+{
+    local test_time=$((`now` - TEST_START_TIME))
+    cat <<EOF
+==============================================================================
+Spent $TOTAL_TIME of $test_time seconds executing testcases
+
+SUMMARY: $TOTAL_COUNT run, $SUCCESSFUL_COUNT successful, $SKIPPED_COUNT skipped, $FAILED_COUNT failed
+
+EOF
+
+    if [ -n "$SKIPPED_TESTS" ]
+    then
+        echo "Skipped tests: $SKIPPED_TESTS"
+        echo
+    fi
+
+    if [ -n "$FAILED_TESTS" ]
+    then
+        echo "Failed tests: $FAILED_TESTS"
+        echo
+    fi
+
+    echo "See results/ for detailed output"
+
+    if [ "$FAILED_COUNT" = 0 ]
+    then
+        exit 0
+    fi
+
+    exit 1
+}
+
+########################################################################
+# Cleanup procedure invoked on process exit
+########################################################################
+function cleanup_on_exit()
+{
+    kill_servers $TEST_VAR_ROOT
+
+    remove_var_dirs
+
+    release_port_locks $$
+
+    kill_all_workers
+
+    cleanup_all_workers
 }
 
 function find_program()
@@ -52,12 +253,11 @@
     fi
 }
 
+########################################################################
+# Explore environment and setup global variables
+########################################################################
 function set_vars()
 {
-    TEST_BASEDIR=${TEST_BASEDIR:-"$PWD"}
-    MYSQL_BASEDIR=${MYSQL_BASEDIR:-"$PWD/server"}
-    PORT_BASE=$((3306 + $RANDOM))
-
     if gnutar --version > /dev/null 2>&1
     then
 	TAR=gnutar
@@ -88,7 +288,7 @@
     fi
     DYLD_LIBRARY_PATH="$LD_LIBRARY_PATH"
 
-    export TEST_BASEDIR PORT_BASE TAR MYSQL_BASEDIR MYSQL MYSQLD MYSQLADMIN \
+    export TAR MYSQL_BASEDIR MYSQL MYSQLD MYSQLADMIN \
 MYSQL_INSTALL_DB PATH LD_LIBRARY_PATH DYLD_LIBRARY_PATH MYSQLDUMP
 }
 
@@ -229,7 +429,7 @@
 	return 1
     fi
 
-    stop_server
+    stop_server >>$OUTFILE 2>&1
 
     export MYSQL_VERSION MYSQL_VERSION_COMMENT MYSQL_FLAVOR \
 	INNODB_VERSION XTRADB_VERSION INNODB_FLAVOR \
@@ -237,39 +437,362 @@
         DEFAULT_IBDATA_SIZE XB_BUILD
 }
 
-export SKIPPED_EXIT_CODE=200
+###########################################################################
+# Kill all server processes started by a worker specified with its var root
+# directory
+###########################################################################
+function kill_servers()
+{
+    local var_root=$1
+    local file
+
+    [ -d $var_root ] || return 0
+
+    cd $var_root
+
+    for file in mysqld*.pid
+    do
+        if [ -f $file ]
+        then
+            vlog "Found a leftover mysqld processes with PID `cat $file`, \
+stopping it"
+            kill -9 `cat $file` 2>/dev/null || true
+            rm -f $file
+        fi
+    done
+
+    cd - >/dev/null 2>&1
+}
+
+###########################################################################
+# Kill all server processes started by a worker specified with its number
+###########################################################################
+function kill_servers_for_worker()
+{
+    local worker=$1
+
+    kill_servers $TEST_BASEDIR/var/w$worker
+}
+
+################################################################################
+# Clean up all workers (except DEBUG_WORKER if set) We can't use worker* arrays
+# here and examine the directory structure, because the same functions is called
+# on startup when we want to cleanup all workers started on previous invokations
+################################################################################
+function cleanup_all_workers() { local worker
+
+    for worker_dir in $TEST_BASEDIR/var/w+([0-9])
+    do
+        [ -d $worker_dir ] || continue
+        [[ $worker_dir =~ w([0-9]+)$ ]] || continue
+
+        worker=${BASH_REMATCH[1]}
+
+        if [ "$worker" = "$DEBUG_WORKER" ]
+        then
+            echo
+            echo "Skipping cleanup for worker #$worker due to debug mode."
+            echo "You can do post-mortem analysis by examining test data in \
+$worker_dir"
+            echo "and the server process if it was running at the failure time."
+            continue
+        fi
+
+        cleanup_worker $worker
+    done
+}
+########################################################################
+# Clean up a specified worker
+########################################################################
+function cleanup_worker()
+{
+    local worker=$1
+    local tmpdir
+
+    kill_servers_for_worker $worker >>$OUTFILE 2>&1
+
+    # It is possible that a file is created while rm is in progress
+    # which results in "rm: cannot remove ...: Directory not empty
+    # hence the loop below
+    while true
+    do
+        # Fix permissions as some tests modify them so the following 'rm' fails
+        chmod -R 0700 $TEST_BASEDIR/var/w$worker >/dev/null 2>&1
+        rm -rf $TEST_BASEDIR/var/w$worker && break
+    done
+
+    tmpdir=${worker_tmpdirs[$worker]:-""}
+    if [ -n "$tmpdir" ]
+    then
+        rm -rf $tmpdir
+    fi
+}
+
+########################################################################
+# Return the number of seconds since the Epoch, UTC
+########################################################################
+function now()
+{
+    date '+%s'
+}
+
+########################################################################
+# Process the exit code of a specified worker
+########################################################################
+function reap_worker()
+{
+    local worker=$1
+    local pid=${worker_pids[$worker]}
+    local skip_file=${worker_skip_files[$worker]}
+    local tpath=${worker_names[$worker]}
+    local tname=`basename $tpath .sh`
+    local test_time=$((`now` - worker_stime[$worker]))
+    local status_file=${worker_status_files[$worker]}
+    local rc
+
+    if [ -f "$status_file" ]
+    then
+        rc=`cat $status_file`
+        # Assume exit code 1 if status file is empty
+        rc=${rc:-"1"}
+    else
+        # Assume exit code 1 if status file does not exist
+        rc="1"
+    fi
+
+    printf "%-40s w%d\t" $tname $worker
+
+    ((TOTAL_TIME+=test_time))
+
+    # Have to call subunit_start_test here, as currently tests cannot be
+    # interleaved in the subunit output
+    subunit_start_test $tpath "${worker_stime_txt[$worker]}" >> $SUBUNIT_OUT
+
+    if [ $rc -eq 0 ]
+    then
+        echo "[passed]    $test_time"
+
+        SUCCESSFUL_COUNT=$((SUCCESSFUL_COUNT + 1))
+        subunit_pass_test $tpath >> $SUBUNIT_OUT
+
+        cleanup_worker $worker
+    elif [ $rc -eq $SKIPPED_EXIT_CODE ]
+    then
+        sreason=""
+        test -r $skip_file && sreason=`cat $skip_file`
+
+        SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
+        SKIPPED_TESTS="$SKIPPED_TESTS $tname"
+
+        echo "[skipped]   $sreason"
+
+        subunit_skip_test $tpath >> $SUBUNIT_OUT
+
+        cleanup_worker $worker
+    else
+        echo "[failed]    $test_time"
+
+        (
+            (echo "Something went wrong running $tpath. Exited with $rc";
+                echo; echo; cat ${worker_outfiles[$worker]}
+            ) | subunit_fail_test $tpath
+        ) >> $SUBUNIT_OUT
+
+       FAILED_COUNT=$((FAILED_COUNT + 1))
+       FAILED_TESTS="$FAILED_TESTS $tname"
+
+       # Return 0 on failed tests in the -f mode
+       if [ -z "$force" ]
+       then
+
+           if [ -z "$DEBUG" ]
+           then
+               cleanup_worker $worker
+           else
+               DEBUG_WORKER=$worker
+           fi
+
+           return 1
+       else
+           cleanup_worker $worker
+
+           return 0
+       fi
+    fi
+
+}
+
+#############################################################################
+# Check if a specified worker has exceed TEST_TIMEOUT and if so, terminate it
+#############################################################################
+function check_timeout_for_worker()
+{
+    local worker=$1
+    local tpath=${worker_names[$worker]}
+    local tname=`basename $tpath .sh`
+
+    if (( `now` - worker_stime[$worker] > TEST_TIMEOUT ))
+    then
+        kill_worker $worker
+        printf "%-40s w%d\t" $tname $worker
+        echo "[failed]    Timed out after $TEST_TIMEOUT seconds."
+
+        (
+            # Have to call subunit_start_test here, as currently tests cannot be
+            # interleaved in the subunit output
+            subunit_start_test $tpath "${worker_stime_txt[$worker]}"
+
+            (echo "Timeout exceeded running $tpath.";
+                echo; echo; cat ${worker_outfiles[$worker]}
+            ) | subunit_fail_test $tpath
+        ) >> $SUBUNIT_OUT
+
+        FAILED_COUNT=$((FAILED_COUNT + 1))
+        FAILED_TESTS="$FAILED_TESTS $tname"
+
+        # Return 0 on failed tests in the -f mode
+        if [ -z "$force" ]
+        then
+
+            if [ -z "$DEBUG" ]
+           then
+                cleanup_worker $worker
+            else
+               DEBUG_WORKER=$worker
+           fi
+
+            return 1
+        else
+            cleanup_worker $worker
+
+            return 0
+       fi
+    fi
+}
+#########################################################################
+# Wait for all currently running worker to finish
+#########################################################################
+function reap_all_workers()
+{
+    while true
+    do
+        found=""
+
+        for ((i = 1; i <= NWORKERS; i++))
+        do
+            [ -z ${worker_pids[$i]:-""} ] && continue
+
+            found="yes"
+
+            # Check if it's alive
+            if kill -0 ${worker_pids[$i]} >/dev/null 2>&1
+            then
+                check_timeout_for_worker $i || print_status_and_exit
+                continue
+            fi
+
+            reap_worker $i || print_status_and_exit
+
+            worker_pids[$i]=""
+        done
+
+        [ -z "$found" ] && break
+
+        sleep 1
+    done
+}
+
+########################################################################
+# Release all port locks reserved by the current process
+# Used by the EXIT trap (in both normal and abnormal shell termination)
+########################################################################
+function release_port_locks()
+{
+    local process=$1
+    local lockfile
+
+    # Suppress errors when no port lock files are found
+    shopt -s nullglob
+
+    for lockfile in /tmp/xtrabackup_port_lock.*
+    do
+        if [ "`cat $lockfile 2>/dev/null`" = $process ]
+        then
+            rm -rf $lockfile
+        fi
+    done
+
+    shopt -u nullglob
+}
+
+########################################################################
+# Report status and release port locks on exit
+########################################################################
+function cleanup_on_test_exit()
+{
+    local rc=$?
+
+    echo $rc > $STATUS_FILE
+
+    release_port_locks $$
+}
+
+########################################################################
+# Script body
+########################################################################
+
+TEST_START_TIME=`now`
 
 tname=""
 XTRACE_OPTION=""
 XB_BUILD="autodetect"
 force=""
-KEEP_RESULTS=0
 SUBUNIT_OUT=test_results.subunit
-
-while getopts "fgh?:t:s:d:c:b:nr:" options; do
-	case $options in
-	        f ) force="yes";;
-		t ) tname="$OPTARG";;
-		g ) XTRACE_OPTION="-x"; DEBUG=on;;
-		h ) usage; exit;;
-		s ) tname="$OPTARG/*.sh";;
-	        d ) export MYSQL_BASEDIR="$OPTARG";;
-	        b ) TEST_BASEDIR="$OPTARG";;
-	        c ) XB_BUILD="$OPTARG";;
-	        n ) KEEP_RESULTS=1;;
-	        r ) SUBUNIT_OUT="$OPTARG";;
-		? ) echo "Use \`$0 -h' for the list of available options."
+NWORKERS=
+DEBUG_WORKER=""
+
+while getopts "fgh?:t:s:d:c:j:T:" options; do
+        case $options in
+            f ) force="yes";;
+            t )
+
+                tname="$OPTARG";
+                if ! [ -r "$tname" ]
+                then
+                    echo "Cannot find test $tname."
+                    exit -1
+                fi
+                ;;
+
+            g ) DEBUG=on;;
+            h ) usage; exit;;
+            s ) tname="$OPTARG/*.sh";;
+            d ) export MYSQL_BASEDIR="$OPTARG";;
+            c ) XB_BUILD="$OPTARG";;
+            j )
+
+                if [[ ! $OPTARG =~ ^[0-9]+$ || $NWORKERS < 1 ]]
+                then
+                    echo "Wrong -j argument: $OPTARG"
+                    exit -1
+                fi
+                NWORKERS="$OPTARG"
+                ;;
+
+            T )
+                if [[ ! $OPTARG =~ ^[0-9]+$ ]]
+                then
+                    echo "Wrong -T argument: $OPTARG"
+                    exit -1
+                fi
+                TEST_TIMEOUT="$OPTARG"
+                ;;
+
+            ? ) echo "Use \`$0 -h' for the list of available options."
                     exit -1;;
-	esac
+        esac
 done
 
-if [ $KEEP_RESULTS -eq 0 ];
-then
-    rm -rf results
-    rm -f $SUBUNIT_OUT
-fi
-mkdir results
-
 set_vars
 
 if [ -n "$tname" ]
@@ -279,13 +802,17 @@
    tests="t/*.sh"
 fi
 
-failed_count=0
-failed_tests=
-total_count=0
-
 export OUTFILE="$PWD/results/setup"
 
-clean >>$OUTFILE 2>&1
+rm -rf results
+mkdir results
+
+cleanup_all_workers >>$OUTFILE 2>&1
+
+rm -rf var test_results.subunit
+mkdir var
+
+echo "Detecting server version..." | tee -a $OUTFILE
 
 if ! get_version_info
 then
@@ -297,69 +824,99 @@
   tee -a $OUTFILE
 
 echo "Using '`basename $XB_BIN`' as xtrabackup binary" | tee -a $OUTFILE
+
+[ -z "$NWORKERS" ] && autocalc_nworkers
+
+if [ "$NWORKERS" -gt 1 ]
+then
+    echo "Using $NWORKERS parallel workers" | tee -a $OUTFILE
+fi
 echo | tee -a $OUTFILE
 
-kill_leftovers >>$OUTFILE 2>&1
-clean >>$OUTFILE 2>&1
-
-source subunit.sh
-
-echo "========================================================================"
+cat <<EOF
+==============================================================================
+TEST                                   WORKER    RESULT     TIME(s) or COMMENT
+------------------------------------------------------------------------------
+EOF
 
 for t in $tests
 do
-   total_count=$((total_count+1))
-
-   printf "%-40s" $t
-   subunit_start_test $t >> $SUBUNIT_OUT
+   # Check if we have available workers
+   found=""
+
+   while [ -z "$found" ]
+   do
+       for ((i = 1; i <= NWORKERS; i++))
+       do
+           if [ -z ${worker_pids[$i]:-""} ]
+           then
+               found="yes"
+               break
+           else
+               # Check if it's alive
+               if kill -0 ${worker_pids[$i]} >/dev/null 2>&1
+               then
+                   check_timeout_for_worker $i || print_status_and_exit
+                   continue
+               fi
+               reap_worker $i || print_status_and_exit
+
+               worker_pids[$i]=""
+               found="yes"
+
+               break
+           fi
+       done
+
+       if [ -z "$found" ]
+       then
+           sleep 1
+       fi
+   done
+
+   worker=$i
+
+   TOTAL_COUNT=$((TOTAL_COUNT+1))
+
    name=`basename $t .sh`
-   export OUTFILE="$PWD/results/$name"
-   export SKIPPED_REASON="$PWD/results/$name.skipped"
-   bash $XTRACE_OPTION $t > $OUTFILE 2>&1
-   rc=$?
-
-   if [[ -z "$DEBUG" || -n "$force" ]]
-   then
-       kill_leftovers >>$OUTFILE 2>&1
-       clean >>$OUTFILE 2>&1
-   fi
-
-   if [ $rc -eq 0 ]
-   then
-       echo "[passed]"
-       subunit_pass_test $t >> $SUBUNIT_OUT
-   elif [ $rc -eq $SKIPPED_EXIT_CODE ]
-   then
-       sreason=""
-       test -r $SKIPPED_REASON && sreason=`cat $SKIPPED_REASON`
-       echo "[skipped]    $sreason"
-       subunit_skip_test $t >> $SUBUNIT_OUT
-   else
-       echo "[failed]"
-
-       (
-	   (echo "Something went wrong running $t. Exited with $rc";
-	       echo; echo; cat $OUTFILE
-	   ) | subunit_fail_test $t
-       ) >> $SUBUNIT_OUT
-
-       failed_count=$((failed_count+1))
-       failed_tests="$failed_tests $t"
-       result=1
-       if [ -z "$force" ]
+   worker_names[$worker]=$t
+   worker_outfiles[$worker]="$PWD/results/$name"
+   worker_skip_files[$worker]="$PWD/results/$name.skipped"
+   worker_status_files[$worker]="$PWD/results/$name.status"
+   # Create a unique TMPDIR for each worker so that it can be removed as a part
+   # of the cleanup procedure. Server socket files will also be created there.
+   worker_tmpdirs[$worker]="`mktemp -d -t xbtemp.XXXXXX`"
+
+   (
+       set -eu
+       if [ -n "$DEBUG" ]
        then
-	   break;
+           set -x
        fi
-   fi       
+
+       trap "cleanup_on_test_exit" EXIT
+
+       . inc/common.sh
+
+       export OUTFILE=${worker_outfiles[$worker]}
+       export SKIPPED_REASON=${worker_skip_files[$worker]}
+       export TEST_VAR_ROOT=$TEST_BASEDIR/var/w$worker
+       export TMPDIR=${worker_tmpdirs[$worker]}
+       export STATUS_FILE=${worker_status_files[$worker]}
+
+       mkdir $TEST_VAR_ROOT
+
+       . $t
+   ) > ${worker_outfiles[$worker]} 2>&1 &
+
+   worker_pids[$worker]=$!
+   worker_stime[$worker]="`now`"
+   # Used in subunit reports
+   worker_stime_txt[$worker]="`date -u '+%Y-%m-%d %H:%M:%S'`"
 done
 
-echo "========================================================================"
-echo
-
-if [ $result -eq 1 ]
-then
-    echo
-    echo "$failed_count/$total_count tests have failed: $failed_tests"
-    echo "See results/ for detailed output"
-    exit -1
-fi
+# Wait for in-progress workers to finish
+reap_all_workers
+
+print_status_and_exit
+
 
=== modified file 'test/subunit.sh'
--- test/subunit.sh	2011-06-12 09:57:00 +0000
+++ test/subunit.sh	2013-07-04 07:51:31 +0000
@@ -15,8 +15,11 @@
 #
 
 subunit_start_test () {
+  local stime=$2
   # emit the current protocol start-marker for test $1
-  echo "time: `date -u '+%Y-%m-%d %H:%M:%S'`"
+  # adjust time by the number of seconds to execute the test to get the actual
+  # start time
+  echo "time: $stime"
   echo "test: $1"
 }
 
 
=== modified file 'test/t/bug1085099.sh'
--- test/t/bug1085099.sh	2013-03-06 10:04:13 +0000
+++ test/t/bug1085099.sh	2013-07-04 07:51:31 +0000
@@ -13,9 +13,7 @@
 # Create a new tmpdir
 mkdir $topdir/new_tmpdir
 
-# Make the default tmpdir inaccessible and make sure permissions are restored on
-# failure
-trap "chmod 755 $MYSQLD_TMPDIR" INT TERM EXIT
+# Make the default tmpdir inaccessible
 chmod 000 $MYSQLD_TMPDIR
 
 innobackupex --tmpdir=$topdir/new_tmpdir --no-timestamp $topdir/backup
 
=== modified file 'test/t/bug664986.sh'
--- test/t/bug664986.sh	2013-03-03 11:01:19 +0000
+++ test/t/bug664986.sh	2013-07-04 07:51:31 +0000
@@ -15,14 +15,6 @@
 
 EOF
 
-# Make sure we restore permissions to not leave stale files in Jenkins
-# on a test failure
-trap "vlog restoring directories and files permissions ; \
-  chmod 777 $MYSQLD_DATADIR/test_bug664986_innodb; \
-  chmod 777 $MYSQLD_DATADIR/test_bug664986_myisam; \
-  chmod 644 $MYSQLD_DATADIR/test_bug664986_innodb/*; \
-  chmod 644 $MYSQLD_DATADIR/test_bug664986_myisam/*" INT TERM EXIT
-
 # Test that wrong directory permissions result in a backup failure
 # for both InnoDB and non-InnoDB files
 chmod 000 $MYSQLD_DATADIR/test_bug664986_innodb
 
=== modified file 'test/t/bug996493.sh'
--- test/t/bug996493.sh	2012-11-29 04:37:56 +0000
+++ test/t/bug996493.sh	2013-07-04 07:51:31 +0000
@@ -20,9 +20,6 @@
 vlog "Remove my.cnf"
 mv $topdir/my.cnf $topdir/my.cnf.bak
 
-trap "vlog restoring $topdir/my.cnf ; \
-mv $topdir/my.cnf.bak $topdir/my.cnf" INT TERM EXIT
-
 vlog "Apply log"
 # Do not run innobackupex, because it pass option --defaults-file
 # which we should avoid. Our goal is to test that backup-my.cnf
@@ -31,7 +28,6 @@
 
 vlog "Get my.cnf back"
 mv $topdir/my.cnf.bak $topdir/my.cnf
-trap "vlog test exit" INT TERM EXIT
 
 vlog "Copy back"
 innobackupex --copy-back $backup_dir
 
=== modified file 'test/t/compact_compressed.sh'
--- test/t/compact_compressed.sh	2013-01-14 11:02:59 +0000
+++ test/t/compact_compressed.sh	2013-07-04 07:51:31 +0000
@@ -11,8 +11,13 @@
 
 # Use innodb_strict_mode so that failure to use compression results in an 
 # error rather than a warning
-mysqld_additional_args="--innodb_strict_mode --innodb_file_per_table \
---innodb_file_format=Barracuda"
+MYSQLD_EXTRA_MY_CNF_OPTS="
+innodb_strict_mode
+innodb_file_per_table
+innodb_file_format=Barracuda
+# Set the minimal log file size to minimize xtrabackup_logfile
+# Recovering compressed pages is extremely slow with UNIV_ZIP_DEBUG
+innodb_log_file_size=1M"
 
 #
 # Test compact backup of a compressed tablespace with a specific page size
@@ -26,7 +31,7 @@
   echo "Testing compact backup with compressed page size=${page_size}KB "
   echo "************************************************************************"
 
-  start_server ${mysqld_additional_args}
+  start_server
 
   load_dbase_schema sakila
   load_dbase_data sakila
@@ -52,7 +57,7 @@
 
   rm -r $mysql_datadir
 
-  innobackupex --apply-log --rebuild-indexes $backup_dir
+  innobackupex --apply-log --rebuild-indexes --rebuild-threads=2 $backup_dir
 
   vlog "Restoring MySQL datadir"
   mkdir -p $mysql_datadir
@@ -63,7 +68,7 @@
   verify_db_state sakila
 
   stop_server
-  clean
+  remove_var_dirs
 }
 
 
 
=== modified file 'test/t/ib_stream_incremental.sh'
--- test/t/ib_stream_incremental.sh	2012-06-05 12:35:33 +0000
+++ test/t/ib_stream_incremental.sh	2013-07-04 07:51:31 +0000
@@ -98,7 +98,7 @@
 	exit -1
     fi
     
-    clean
+    remove_var_dirs
 }
 
 stream_format="xbstream"
 
=== modified file 'test/t/remote_tablespaces.sh'
--- test/t/remote_tablespaces.sh	2013-04-23 06:38:06 +0000
+++ test/t/remote_tablespaces.sh	2013-07-04 07:51:31 +0000
@@ -12,7 +12,7 @@
 
 start_server --innodb_file_per_table
 
-remote_dir=$TEST_BASEDIR/var1/remote_dir
+remote_dir=$TEST_VAR_ROOT/var1/remote_dir
 
 $MYSQL $MYSQL_ARGS test <<EOF
 CREATE TABLE t(id INT AUTO_INCREMENT PRIMARY KEY, c INT)
 
=== modified file 'test/t/undo_tablespaces.sh'
--- test/t/undo_tablespaces.sh	2013-04-27 18:46:54 +0000
+++ test/t/undo_tablespaces.sh	2013-07-04 07:51:31 +0000
@@ -23,7 +23,7 @@
 EOF
 }
 
-undo_directory=$TEST_BASEDIR/var1/undo_dir
+undo_directory=$TEST_VAR_ROOT/var1/undo_dir
 
 MYSQLD_EXTRA_MY_CNF_OPTS="
 innodb_file_per_table=1
 
=== modified file 'test/t/xb_encrypt.sh'
--- test/t/xb_encrypt.sh	2013-03-11 19:11:29 +0000
+++ test/t/xb_encrypt.sh	2013-07-04 07:51:31 +0000
@@ -5,8 +5,6 @@
 encrypt_algo="AES256"
 encrypt_key="percona_xtrabackup_is_awesome___"
 
-echo -n "Percona XtraBackup is awesome!!!" > $encrypt_key_file
-
 innobackupex_options="--encrypt=$encrypt_algo --encrypt-key=$encrypt_key --encrypt-threads=4 --encrypt-chunk-size=8K"
 data_decrypt_cmd="for i in *.xbcrypt; do \
 xbcrypt -d -a $encrypt_algo -k $encrypt_key < \$i > \${i:0:\${#i}-8}; \
 
=== modified file 'test/t/xb_part_range.sh'
--- test/t/xb_part_range.sh	2012-07-06 05:13:15 +0000
+++ test/t/xb_part_range.sh	2013-07-04 07:51:31 +0000
@@ -1,19 +1,9 @@
 . inc/common.sh
+. inc/ib_part.sh
 
 start_server
 
-function check_partitioning()
-{
-   $MYSQL $MYSQL_ARGS -Ns -e "show variables like 'have_partitioning'"
-}
-
-PARTITION_CHECK=`check_partitioning`
-
-if [ -z "$PARTITION_CHECK" ]; then
-    echo "Requires Partitioning." > $SKIPPED_REASON
-    stop_server
-    exit $SKIPPED_EXIT_CODE
-fi
+require_partitioning
 
 run_cmd $MYSQL $MYSQL_ARGS test <<EOF
 CREATE TABLE test (
 
=== modified file 'test/t/xb_perm_basic.sh'
--- test/t/xb_perm_basic.sh	2012-10-15 16:14:59 +0000
+++ test/t/xb_perm_basic.sh	2013-07-04 07:51:31 +0000
@@ -4,6 +4,4 @@
 start_server
 
 chmod -R 500 $mysql_datadir
-trap "vlog restoring permissions on datadir $mysql_datadir ; \
-chmod -R 700 $mysql_datadir" INT TERM EXIT
 innobackupex  --no-timestamp $topdir/backup
 
=== modified file 'test/t/xb_perm_stream.sh'
--- test/t/xb_perm_stream.sh	2012-10-15 16:14:59 +0000
+++ test/t/xb_perm_stream.sh	2013-07-04 07:51:31 +0000
@@ -5,7 +5,5 @@
 
 # Take backup
 chmod -R 500 $mysql_datadir
-trap "vlog restoring permissions on datadir $mysql_datadir ; \
-chmod -R 700 $mysql_datadir" INT TERM EXIT
 mkdir -p $topdir/backup
 innobackupex --stream=tar $topdir/backup > $topdir/backup/out.tar
 
=== modified file 'test/t/xb_stats_datadir.sh'
--- test/t/xb_stats_datadir.sh	2013-05-02 13:05:59 +0000
+++ test/t/xb_stats_datadir.sh	2013-07-04 07:51:31 +0000
@@ -5,12 +5,11 @@
 
 . inc/common.sh
 
-mkdir ${TEST_BASEDIR}/logs
-
-trap "rm -rf ${TEST_BASEDIR}/logs" INT TERM EXIT
+logdir=${TEST_VAR_ROOT}/logs
+mkdir $logdir
 
 MYSQLD_EXTRA_MY_CNF_OPTS="
-innodb_log_group_home_dir=${TEST_BASEDIR}/logs
+innodb_log_group_home_dir=$logdir
 "
 
 start_server
@@ -28,6 +27,6 @@
 # we pass all necessary options as an arguments, so if someday this
 # will be changed, test still will work
 xtrabackup --stats --datadir=${MYSQLD_DATADIR} \
-	--innodb_log_group_home_dir=${TEST_BASEDIR}/logs
+        --innodb_log_group_home_dir=$logdir
 
 vlog "stats did not fail"
 
=== removed file 'test/testrun.c'
--- test/testrun.c	2013-06-14 00:59:24 +0000
+++ test/testrun.c	1970-01-01 00:00:00 +0000
@@ -1,405 +0,0 @@
-/*
- * testrun.c - a parallel test runner for the XtraBackup test suite
- */
-/* BEGIN LICENSE
- * Copyright (C) 2012 Percona Ireland Ltd.
- *
- * Written by Stewart Smith
- *
- * This program is free software: you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2, as published
- * by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranties of
- * MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
- * PURPOSE.  See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program.  If not, see <http://www.gnu.org/licenses/>.
- * END LICENSE */
-
-#include <stdio.h>
-#include <unistd.h>
-#include <dirent.h>
-#include <string.h>
-#include <limits.h>
-#include <stdlib.h>
-#include <sys/select.h>
-
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <fcntl.h>
-#include <assert.h>
-
-#include <signal.h>
-
-struct testcase
-{
-  char *name;
-  char *buf;
-  size_t end_of_buf;
-  size_t bufsz;
-};
-
-pid_t *childpid;
-int nchildpid=0;
-
-static void kill_children(int sig)
-{
-  int i;
-  int status;
-
-  (void)sig;
-
-  fprintf(stderr, "Killing child processes...\n");
-
-  for(i=0; i<nchildpid; i++)
-    if(childpid[i] > 0)
-    {
-      kill(childpid[i], SIGKILL);
-      wait(&status);
-    }
-
-  exit(EXIT_SUCCESS);
-}
-
-int collect_testcases_filter(const struct dirent *a)
-{
-  int l;
-
-  if (a->d_name[0] == '.')
-    return 0;
-
-  l= strlen(a->d_name);
-
-  if (l > 2 && strncmp(a->d_name + l - 3, ".sh", 3)==0)
-    return 1;
-
-  return 0;
-}
-
-static int collect_testcases(const char* suitedir, struct testcase **cases)
-{
-  struct dirent **namelist;
-  int n;
-  int i;
-
-  n= scandir(suitedir, &namelist, collect_testcases_filter, alphasort);
-
-  *cases= (struct testcase*) malloc(sizeof(struct testcase)*n);
-
-  for(i=0; i<n; i++)
-  {
-    (*cases)[i].name= strdup(namelist[i]->d_name);
-    (*cases)[i].buf= NULL;
-    (*cases)[i].end_of_buf= 0;
-    (*cases)[i].bufsz= 0;
-    free(namelist[i]);
-  }
-
-  free(namelist);
-
-  return n;
-}
-
-static void free_testcases(struct testcase *cases, int n)
-{
-  while (n>0)
-  {
-    free(cases[--n].name);
-  }
-  free(cases);
-}
-
-static int run_testcase_in_child(int nr, struct testcase *t, pid_t *cpid, const char* xbtarget)
-{
-  int fd[2];
-
-  printf("[%d] LAUNCHING - %s\n", nr, t->name);
-
-  if (pipe(fd) == -1)
-  {
-    perror("pipe");
-    exit(EXIT_FAILURE);
-  }
-
-  *cpid= fork();
-  if (*cpid == 0)
-  {
-    /* child */
-    close(fd[0]);
-
-    char tname[500];
-    snprintf(tname, sizeof(tname), "t/%s",t->name);
-
-    char basedir[PATH_MAX];
-    char cwd[PATH_MAX];
-    snprintf(basedir, sizeof(basedir), "%s/var/%d", getcwd(cwd,sizeof(cwd)), nr);
-
-    mkdir("var",0700);
-    mkdir(basedir,0700);
-
-    char logname[PATH_MAX];
-    snprintf(logname, sizeof(logname), "%s/var/%d.log", getcwd(cwd,sizeof(cwd)), nr);
-
-    int logfd= open(logname, O_WRONLY|O_APPEND|O_CREAT, 0600);
-    dup2(logfd, STDOUT_FILENO);
-    dup2(logfd, STDERR_FILENO);
-    close(logfd);
-
-    char subunitfd[50];
-    snprintf(subunitfd, sizeof(subunitfd), "/dev/fd/%d", fd[1]);
-
-    char* xbtarget_param;
-    if (xbtarget)
-        xbtarget_param= strdup(xbtarget);
-    else
-        xbtarget_param= NULL; 
-
-    char *const newargv[] = {"testrun.sh", "-n",
-		       "-t", tname,
-		       "-b", basedir,
-		       "-r", subunitfd,
-		       (xbtarget)? "-c" : NULL, xbtarget_param,
-		       NULL };
-    char *newenviron[] = { NULL };
-    execve(newargv[0], newargv, newenviron);
-    perror("execve");
-    exit(EXIT_FAILURE);
-  }
-  else
-  {
-    /* parent */
-    close(fd[1]);
-    fcntl(fd[0], F_SETFL, O_NONBLOCK);
-    return fd[0];
-  }
-}
-
-static inline void subunit_progress_sign(int fd, int n, char *sign)
-{
-  char *buf;
-  const char* fmt= "progress: %s%d\n";
-  size_t sz= 1+strlen(fmt)+100;
-  size_t l;
-
-  buf= (char*)malloc(sz);
-
-  l= snprintf(buf, sz, fmt, sign, n);
-  assert(l < sz);
-
-  write(fd, buf, l);
-
-  free(buf);
-}
-
-static void run_testcases(struct testcase *testcases, int nrcases,
-			  int njobs, int timeout, const char* xbtarget)
-{
-  int childfd[njobs];
-  int nfds= 0;
-  int retval;
-  pid_t chpid[njobs];
-  int status;
-  int next_testcase= 0;
-  int i;
-  fd_set rfds;
-  fd_set efds;
-  struct timeval tv;
-  int nchildren;
-  int childtest[njobs];
-
-  int subunitfd= open("test_results.subunit", O_TRUNC|O_WRONLY|O_APPEND|O_CREAT, 0600);
-  subunit_progress_sign(subunitfd, nrcases, "");
-
-  if (nrcases < njobs)
-    njobs= nrcases;
-
-  childpid= chpid;
-  nchildpid= njobs;
-
-  for(i=0; i<njobs; i++)
-  {
-    childtest[i]=next_testcase++;
-    childfd[i]= run_testcase_in_child(i, &testcases[childtest[i]], &childpid[i], xbtarget);
-  }
-
-  fflush(stdout);
-
-loop:
-  FD_ZERO(&efds);
-  FD_ZERO(&rfds);
-
-  nchildren=0;
-
-  for (i=0; i<njobs; i++)
-  {
-    if (childfd[i] != -1)
-    {
-      FD_SET(childfd[i], &efds);
-      FD_SET(childfd[i], &rfds);
-      nfds= (childfd[i] > nfds)? childfd[i] : nfds;
-      nchildren++;
-    }
-  }
-
-  tv.tv_sec= timeout;
-  tv.tv_usec= 0;
-
-  retval = select(nfds+1, &rfds, NULL, &efds, &tv);
-
-  if (retval == -1)
-    perror("select()");
-  else if (retval)
-  {
-    int childexited=0;
-
-    for (i=0; i<njobs; i++)
-    {
-      if(childfd[i] != -1
-	 && (FD_ISSET(childfd[i], &efds) || FD_ISSET(childfd[i], &rfds)))
-      {
-	ssize_t r=0;
-
-	do {
-	  struct testcase *t= &testcases[childtest[i]];
-
-	  if(t->bufsz == t->end_of_buf)
-	  {
-	    t->bufsz+=4000;
-	    t->buf= (char*)realloc(t->buf, t->bufsz);
-	  }
-
-	  r= read(childfd[i], t->buf+t->end_of_buf, t->bufsz - t->end_of_buf);
-	  if (r>0)
-	    t->end_of_buf+= r;
-	} while(r>0);
-
-	pid_t waited= waitpid(childpid[i], &status, WNOHANG);
-	if (!(WIFEXITED(status) || WIFSIGNALED(status)))
-	  continue;
-
-	if (waited != childpid[i])
-	  continue;
-
-	write(subunitfd, testcases[childtest[i]].buf,
-	      testcases[childtest[i]].end_of_buf);
-
-	close(childfd[i]);
-	printf("[%d] completed %s status %d\n",
-	       i, testcases[childtest[i]].name, WEXITSTATUS(status));
-	childfd[i]=-1;
-	nchildren--;
-
-	if (next_testcase < nrcases)
-	{
-	  childtest[i]=next_testcase++;
-	  childfd[i]= run_testcase_in_child(i, &testcases[childtest[i]], &childpid[i], xbtarget);
-	  nfds= (childfd[i] > nfds)? childfd[i] : nfds;
-	  nchildren++;
-	}
-	printf("\nnrchildren= %d, %d tests remaining\n",
-	       nchildren, nrcases-next_testcase);
-	childexited=1;
-	fflush(stdout);
-      }
-    }
-    if (childexited)
-    {
-      printf ("Running: ");
-      for(i=0; i<njobs; i++)
-	if (childfd[i] != -1)
-	  printf("%s ",testcases[childtest[i]].name);
-      printf("\n");
-    }
-  }
-  else
-  {
-    printf("Timeout\n");
-    kill_children(SIGKILL);
-    exit(EXIT_FAILURE);
-  }
-
-  if (nchildren==0)
-    goto end;
-
-  goto loop;
-
-end:
-
-  close(subunitfd);
-  return;
-}
-
-int main(int argc, char* argv[])
-{
-  const char* suitedir= "t/";
-  int njobs= 4;
-  int opt;
-  int nrcases;
-  struct testcase *testcases;
-  int timeout= 600;
-  const char* xbtarget= NULL;
-
-  struct sigaction sa;
-
-  sa.sa_flags= 0;
-  sigemptyset(&sa.sa_mask);
-  sa.sa_handler = kill_children;
-  sigaction(SIGHUP, &sa, NULL);
-  sigaction(SIGINT, &sa, NULL);
-  sigaction(SIGQUIT, &sa, NULL);
-
-#ifdef  _SC_NPROCESSORS_ONLN
-  njobs= sysconf(_SC_NPROCESSORS_ONLN);
-#endif
-
-#ifdef _SC_PHYS_PAGES
-  long nrpages= sysconf(_SC_PHYS_PAGES);
-  long pagesize= sysconf(_SC_PAGESIZE);
-  long pages_per_job= (128*(1 << 20)) / pagesize;
-  nrpages= nrpages/2;
-  if ((pages_per_job * njobs) > nrpages)
-    njobs= nrpages / pages_per_job;
-#endif
-
-  if (njobs == 0)
-    njobs= 1;
-
-  while ((opt = getopt(argc, argv, "j:s:t:c:")) != -1)
-  {
-    switch (opt) {
-    case 'c':
-      xbtarget= optarg;
-      break;
-    case 's':
-      suitedir= optarg;
-      break;
-    case 'j':
-      njobs= atoi(optarg);
-      break;
-    case 't':
-      timeout= atoi(optarg);
-      break;
-    default:
-      fprintf(stderr, "Usage: %s [-s suite] [-j parallel] [-t timeout]\n",
-              argv[0]);
-      exit(EXIT_FAILURE);
-    }
-  }
-
-  printf("%s running with: -s %s -j %d -t %d\n\n",
-	 argv[0], suitedir,njobs, timeout);
-
-  nrcases= collect_testcases(suitedir, &testcases);
-
-  printf("Found %d testcases\n", nrcases);
-
-  run_testcases(testcases, nrcases, njobs, timeout, xbtarget);
-
-  free_testcases(testcases, nrcases);
-
-  return 0;
-}