Merge lp:~akopytov/percona-xtrabackup/test-suite-cleanups-2.1 into lp:percona-xtrabackup/2.1

Proposed by Alexey Kopytov on 2013-07-04
Status: Merged
Approved by: Alexey Kopytov on 2013-07-10
Approved revision: 620
Merged at revision: 625
Proposed branch: lp:~akopytov/percona-xtrabackup/test-suite-cleanups-2.1
Merge into: lp:percona-xtrabackup/2.1
Diff against target: 1868 lines (+804/-631)
18 files modified
patches/innodb56.patch (+17/-1)
src/xtrabackup.cc (+6/-14)
test/inc/common.sh (+104/-68)
test/run.sh (+653/-96)
test/subunit.sh (+4/-1)
test/t/bug1085099.sh (+1/-3)
test/t/bug664986.sh (+0/-8)
test/t/bug996493.sh (+0/-4)
test/t/compact_compressed.sh (+10/-5)
test/t/ib_stream_incremental.sh (+1/-1)
test/t/remote_tablespaces.sh (+1/-1)
test/t/undo_tablespaces.sh (+1/-1)
test/t/xb_encrypt.sh (+0/-2)
test/t/xb_part_range.sh (+2/-12)
test/t/xb_perm_basic.sh (+0/-2)
test/t/xb_perm_stream.sh (+0/-2)
test/t/xb_stats_datadir.sh (+4/-5)
test/testrun.c (+0/-405)
To merge this branch: bzr merge lp:~akopytov/percona-xtrabackup/test-suite-cleanups-2.1
Reviewer Review Type Date Requested Status
George Ormond Lorch III g2 2013-07-04 Approve on 2013-07-09
Stewart Smith 2013-07-04 Pending
Review via email: mp+172963@code.launchpad.net
To post a comment you must log in.
Alexey Kopytov (akopytov) wrote :

http://jenkins.percona.com/view/XtraBackup/job/percona-xtrabackup-2.1-param/360/

Same comments on remaining failures as in the 2.0 MP.

Alexey Kopytov (akopytov) wrote :

Would be nice to have this merged sooner rather than later, as I have a pile of other fixes which depends on this branch.

George Ormond Lorch III (gl-az) wrote :

Same comments as 2.0 MP, nice work!

review: Approve (g2)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'patches/innodb56.patch'
2--- patches/innodb56.patch 2013-07-01 12:21:20 +0000
3+++ patches/innodb56.patch 2013-07-04 07:51:31 +0000
4@@ -1431,4 +1431,20 @@
5 int main()
6 {
7 extern void __attribute__((weak)) foo(void);
8-
9+--- a/mysys/my_thr_init.c
10++++ b/mysys/my_thr_init.c
11+@@ -462,8 +462,12 @@
12+
13+ extern void **my_thread_var_dbug()
14+ {
15+- struct st_my_thread_var *tmp=
16+- my_pthread_getspecific(struct st_my_thread_var*,THR_KEY_mysys);
17++ struct st_my_thread_var *tmp;
18++
19++ if (!my_thread_global_init_done)
20++ return NULL;
21++
22++ tmp= my_pthread_getspecific(struct st_my_thread_var*,THR_KEY_mysys);
23+ return tmp && tmp->init ? &tmp->dbug : 0;
24+ }
25+ #endif /* DBUG_OFF */
26
27=== modified file 'src/xtrabackup.cc'
28--- src/xtrabackup.cc 2013-06-18 14:49:01 +0000
29+++ src/xtrabackup.cc 2013-07-04 07:51:31 +0000
30@@ -3835,6 +3835,8 @@
31
32 ulint fold;
33
34+ bool checkpoint_found;
35+
36 max_no = ut_dulint_zero;
37
38 if (!xb_init_log_block_size()) {
39@@ -3949,6 +3951,8 @@
40 // ' ', 4);
41 }
42
43+ checkpoint_found = false;
44+
45 /* read last checkpoint lsn */
46 for (field = LOG_CHECKPOINT_1; field <= LOG_CHECKPOINT_2;
47 field += LOG_CHECKPOINT_2 - LOG_CHECKPOINT_1) {
48@@ -3961,25 +3965,13 @@
49 if (ut_dulint_cmp(checkpoint_no, max_no) >= 0) {
50 max_no = checkpoint_no;
51 max_lsn = MACH_READ_64(log_buf + field + LOG_CHECKPOINT_LSN);
52-/*
53- mach_write_to_4(log_buf + field + LOG_CHECKPOINT_OFFSET,
54- LOG_FILE_HDR_SIZE + ut_dulint_minus(max_lsn,
55- ut_dulint_align_down(max_lsn,OS_FILE_LOG_BLOCK_SIZE)));
56-
57- ulint fold;
58- fold = ut_fold_binary(log_buf + field, LOG_CHECKPOINT_CHECKSUM_1);
59- mach_write_to_4(log_buf + field + LOG_CHECKPOINT_CHECKSUM_1, fold);
60-
61- fold = ut_fold_binary(log_buf + field + LOG_CHECKPOINT_LSN,
62- LOG_CHECKPOINT_CHECKSUM_2 - LOG_CHECKPOINT_LSN);
63- mach_write_to_4(log_buf + field + LOG_CHECKPOINT_CHECKSUM_2, fold);
64-*/
65+ checkpoint_found = true;
66 }
67 not_consistent:
68 ;
69 }
70
71- if (ut_dulint_cmp(max_no, ut_dulint_zero) == 0) {
72+ if (!checkpoint_found) {
73 msg("xtrabackup: No valid checkpoint found.\n");
74 goto error;
75 }
76
77=== modified file 'test/inc/common.sh'
78--- test/inc/common.sh 2013-06-28 15:23:23 +0000
79+++ test/inc/common.sh 2013-07-04 07:51:31 +0000
80@@ -16,9 +16,12 @@
81 }
82
83
84-function clean()
85+########################################################################
86+# Remove server var* directories from the worker's var root
87+########################################################################
88+function remove_var_dirs()
89 {
90- rm -rf ${TEST_BASEDIR}/var[0-9]
91+ rm -rf ${TEST_VAR_ROOT}/var[0-9]
92 }
93
94 function die()
95@@ -34,29 +37,33 @@
96 $MYSQL_INSTALL_DB --defaults-file=${MYSQLD_VARDIR}/my.cnf \
97 --basedir=${MYSQL_BASEDIR} \
98 ${MYSQLD_EXTRA_ARGS}
99- cd -
100+ cd - >/dev/null 2>&1
101 }
102
103 ########################################################################
104-# Checks whether MySQL is alive
105+# Checks whether MySQL with the PID specified as an argument is alive
106 ########################################################################
107 function mysql_ping()
108 {
109- $MYSQLADMIN $MYSQL_ARGS --wait=100 ping >/dev/null 2>&1 || return 1
110-}
111+ local pid=$1
112+ local attempts=60
113+ local i
114
115-function kill_leftovers()
116-{
117- local file
118- for file in ${TEST_BASEDIR}/mysqld*.pid
119+ for ((i=1; i<=attempts; i++))
120 do
121- if [ -f $file ]
122- then
123- vlog "Found a leftover mysqld processes with PID `cat $file`, stopping it"
124- kill -9 `cat $file` 2>/dev/null || true
125- rm -f $file
126- fi
127+ $MYSQLADMIN $MYSQL_ARGS ping >/dev/null 2>&1 && return 0
128+ sleep 1
129+ # Is the server process still alive?
130+ if ! kill -0 $pid >/dev/null 2>&1
131+ then
132+ vlog "Server process PID=$pid died."
133+ wait $pid
134+ return 1
135+ fi
136+ vlog "Made $i attempts to connect to server"
137 done
138+
139+ return 1
140 }
141
142 function run_cmd()
143@@ -123,21 +130,14 @@
144 local port
145 local lockfile
146
147- for (( port=PORT_BASE+id; port < 65535; port++))
148+ for (( port=3307 + RANDOM; port < 65535; port++))
149 do
150 lockfile="/tmp/xtrabackup_port_lock.$port"
151 # Try to atomically lock the current port number
152- if ! set -C > $lockfile
153+ if ! (set -C; > $lockfile)
154 then
155- set +C
156- # check if the process still exists
157- if kill -0 "`cat $lockfile`"
158- then
159- continue;
160- fi
161- # stale file, overwrite it with the current PID
162+ continue;
163 fi
164- set +C
165 echo $$ > $lockfile
166 if ! nc -z -w1 localhost $port >/dev/null 2>&1
167 then
168@@ -173,14 +173,14 @@
169 fi
170
171 SRV_MYSQLD_IDS[$id]="$id"
172- local vardir="${TEST_BASEDIR}/var${id}"
173+ local vardir="${TEST_VAR_ROOT}/var${id}"
174 SRV_MYSQLD_VARDIR[$id]="$vardir"
175 SRV_MYSQLD_DATADIR[$id]="$vardir/data"
176 SRV_MYSQLD_TMPDIR[$id]="$vardir/tmp"
177- SRV_MYSQLD_PIDFILE[$id]="${TEST_BASEDIR}/mysqld${id}.pid"
178+ SRV_MYSQLD_PIDFILE[$id]="${TEST_VAR_ROOT}/mysqld${id}.pid"
179 SRV_MYSQLD_ERRFILE[$id]="$vardir/data/mysqld${id}.err"
180 SRV_MYSQLD_PORT[$id]=`get_free_port $id`
181- SRV_MYSQLD_SOCKET[$id]=`mktemp -t xtrabackup.mysql.sock.XXXXXX`
182+ SRV_MYSQLD_SOCKET[$id]=`mktemp -t mysql.sock.XXXXXX`
183 }
184
185 ########################################################################
186@@ -245,27 +245,36 @@
187 function start_server_with_id()
188 {
189 local id=$1
190+ local attempts=0
191+ local max_attempts=5
192 shift
193
194 vlog "Starting server with id=$id..."
195
196- init_server_variables $id
197- switch_server $id
198-
199- if [ ! -d "$MYSQLD_VARDIR" ]
200- then
201- vlog "Creating server root directory: $MYSQLD_VARDIR"
202- mkdir "$MYSQLD_VARDIR"
203- fi
204- if [ ! -d "$MYSQLD_TMPDIR" ]
205- then
206- vlog "Creating server temporary directory: $MYSQLD_TMPDIR"
207- mkdir "$MYSQLD_TMPDIR"
208- fi
209-
210- # Create the configuration file used by mysql_install_db, the server
211- # and the xtrabackup binary
212- cat > ${MYSQLD_VARDIR}/my.cnf <<EOF
213+ while true
214+ do
215+ init_server_variables $id
216+ switch_server $id
217+
218+ if [ ! -d "$MYSQLD_VARDIR" ]
219+ then
220+ vlog "Creating server root directory: $MYSQLD_VARDIR"
221+ mkdir "$MYSQLD_VARDIR"
222+ fi
223+ if [ ! -d "$MYSQLD_TMPDIR" ]
224+ then
225+ vlog "Creating server temporary directory: $MYSQLD_TMPDIR"
226+ mkdir "$MYSQLD_TMPDIR"
227+ fi
228+
229+ if [ -f "$MYSQLD_ERRFILE" ]
230+ then
231+ rm "$MYSQLD_ERRFILE"
232+ fi
233+
234+ # Create the configuration file used by mysql_install_db, the server
235+ # and the xtrabackup binary
236+ cat > ${MYSQLD_VARDIR}/my.cnf <<EOF
237 [mysqld]
238 socket=${MYSQLD_SOCKET}
239 port=${MYSQLD_PORT}
240@@ -285,22 +294,42 @@
241 user=root
242 EOF
243
244- # Create datadir and call mysql_install_db if it doesn't exist
245- if [ ! -d "$MYSQLD_DATADIR" ]
246- then
247- vlog "Creating server data directory: $MYSQLD_DATADIR"
248- mkdir -p "$MYSQLD_DATADIR"
249- call_mysql_install_db
250- fi
251-
252- # Start the server
253- echo "Starting ${MYSQLD} ${MYSQLD_ARGS} $* "
254- ${MYSQLD} ${MYSQLD_ARGS} $* &
255- if ! mysql_ping
256- then
257- die "Can't start the server!"
258- fi
259- vlog "Server with id=$id has been started on port $MYSQLD_PORT, \
260+ # Create datadir and call mysql_install_db if it doesn't exist
261+ if [ ! -d "$MYSQLD_DATADIR" ]
262+ then
263+ vlog "Creating server data directory: $MYSQLD_DATADIR"
264+ mkdir -p "$MYSQLD_DATADIR"
265+ call_mysql_install_db
266+ fi
267+
268+ # Start the server
269+ echo "Starting ${MYSQLD} ${MYSQLD_ARGS} $* "
270+ ${MYSQLD} ${MYSQLD_ARGS} $* &
271+ if ! mysql_ping $!
272+ then
273+ if grep "another mysqld server running on port" $MYSQLD_ERRFILE
274+ then
275+ if ((++attempts < max_attempts))
276+ then
277+ vlog "Made $attempts attempts to find a free port"
278+ reset_server_variables $id
279+ free_reserved_port $MYSQLD_PORT
280+ continue
281+ else
282+ vlog "Failed to find a free port after $attempts attempts"
283+ fi
284+ fi
285+ vlog "Can't start the server. Server log (if exists):"
286+ vlog "----------------"
287+ cat ${MYSQLD_ERRFILE} >&2 || true
288+ vlog "----------------"
289+ exit -1
290+ else
291+ break
292+ fi
293+ done
294+
295+ vlog "Server with id=$id has been started on port $MYSQLD_PORT, \
296 socket $MYSQLD_SOCKET"
297 }
298
299@@ -317,7 +346,9 @@
300
301 if [ -f "${MYSQLD_PIDFILE}" ]
302 then
303- kill -9 `cat ${MYSQLD_PIDFILE}`
304+ local pid=`cat $MYSQLD_PIDFILE`
305+ kill -9 $pid
306+ wait $pid || true
307 rm -f ${MYSQLD_PIDFILE}
308 else
309 vlog "Server PID file '${MYSQLD_PIDFILE}' doesn't exist!"
310@@ -484,7 +515,8 @@
311 ##########################################################################
312 function record_db_state()
313 {
314- $MYSQLDUMP $MYSQL_ARGS -t --compact $1 >"$topdir/tmp/$1_old.sql"
315+ $MYSQLDUMP $MYSQL_ARGS -t --compact --skip-extended-insert \
316+ $1 >"$topdir/tmp/$1_old.sql"
317 }
318
319
320@@ -494,7 +526,8 @@
321 ##########################################################################
322 function verify_db_state()
323 {
324- $MYSQLDUMP $MYSQL_ARGS -t --compact $1 >"$topdir/tmp/$1_new.sql"
325+ $MYSQLDUMP $MYSQL_ARGS -t --compact --skip-extended-insert \
326+ $1 >"$topdir/tmp/$1_new.sql"
327 diff -u "$topdir/tmp/$1_old.sql" "$topdir/tmp/$1_new.sql"
328 }
329
330@@ -514,14 +547,14 @@
331 return ${PIPESTATUS[0]}
332 }
333
334-readonly xb_performed_bmp_inc_backup="xtrabackup: using the full scan for incremental backup"
335-readonly xb_performed_full_scan_inc_backup="xtrabackup: using the changed page bitmap"
336-
337 ####################################################
338 # Helper functions for testing incremental backups #
339 ####################################################
340 function check_full_scan_inc_backup()
341 {
342+ local xb_performed_bmp_inc_backup="xtrabackup: using the full scan for incremental backup"
343+ local xb_performed_full_scan_inc_backup="xtrabackup: using the changed page bitmap"
344+
345 if ! grep -q "$xb_performed_bmp_inc_backup" $OUTFILE ;
346 then
347 vlog "xtrabackup did not perform a full scan for the incremental backup."
348@@ -536,6 +569,9 @@
349
350 function check_bitmap_inc_backup()
351 {
352+ local xb_performed_bmp_inc_backup="xtrabackup: using the full scan for incremental backup"
353+ local xb_performed_full_scan_inc_backup="xtrabackup: using the changed page bitmap"
354+
355 if ! grep -q "$xb_performed_full_scan_inc_backup" $OUTFILE ;
356 then
357 vlog "xtrabackup did not use bitmaps for the incremental backup."
358
359=== removed file 'test/run.sh'
360--- test/run.sh 2012-11-30 04:28:06 +0000
361+++ test/run.sh 1970-01-01 00:00:00 +0000
362@@ -1,14 +0,0 @@
363-#!/bin/bash
364-
365-XB_BUILD="autodetect"
366-while getopts "fc:" options; do
367- case $options in
368- c ) XB_BUILD="$OPTARG";;
369- f ) ;; # ignored
370- esac
371-done
372-
373-rm -rf results/ var/ test_results.subunit
374-
375-CFLAGS=-g make testrun
376-./testrun -c $XB_BUILD
377
378=== renamed file 'test/testrun.sh' => 'test/run.sh'
379--- test/testrun.sh 2013-05-17 11:07:26 +0000
380+++ test/run.sh 2013-07-04 07:51:31 +0000
381@@ -2,14 +2,39 @@
382
383 export DEBUG=
384
385+shopt -s extglob
386+
387+set +e
388+
389 . inc/common.sh
390-
391-trap kill_leftovers 1 2 3 15
392-
393-set +e
394+. subunit.sh
395+
396+trap cleanup_on_exit EXIT
397+trap terminate SIGHUP SIGINT SIGQUIT SIGTERM
398
399 result=0
400
401+# Default test timeout in seconds
402+TEST_TIMEOUT=600
403+
404+# Magic exit code to indicate a skipped test
405+export SKIPPED_EXIT_CODE=200
406+
407+# Default server installation directory (-d option)
408+MYSQL_BASEDIR=${MYSQL_BASEDIR:-"$PWD/server"}
409+
410+TEST_BASEDIR="$PWD"
411+TEST_VAR_ROOT="$TEST_BASEDIR/var"
412+
413+# Global statistics
414+FAILED_COUNT=0
415+FAILED_TESTS=
416+SKIPPED_COUNT=0
417+SKIPPED_TESTS=
418+SUCCESSFUL_COUNT=0
419+TOTAL_COUNT=0
420+TOTAL_TIME=0
421+
422 function usage()
423 {
424 cat <<EOF
425@@ -24,7 +49,183 @@
426 -c conf XtraBackup build configuration as specified to build.sh.
427 Default is 'autodetect', i.e. xtrabackup binary is determined based
428 on the MySQL version.
429-EOF
430+-j N Run tests in N parallel processes.
431+-T seconds Test timeout (default is $TEST_TIMEOUT seconds).
432+EOF
433+}
434+
435+###############################################################################
436+# Calculate the number of parallel workers automatically based on the number of
437+# available cores
438+###############################################################################
439+function autocalc_nworkers()
440+{
441+ if [ -r /proc/cpuinfo ]
442+ then
443+ NWORKERS=`grep processor /proc/cpuinfo | wc -l`
444+ elif which sysctl >/dev/null 2>&1
445+ then
446+ NWORKERS=`sysctl -n hw.ncpu`
447+ fi
448+
449+ if [[ ! $NWORKERS =~ ^[0-9]+$ || $NWORKERS < 1 ]]
450+ then
451+ echo "Cannot determine the number of available CPU cures!"
452+ exit -1
453+ fi
454+
455+ if [ "$NWORKERS" -gt 16 ]
456+ then
457+ echo "Autodetected number of cores: $NWORKERS"
458+ echo "Limiting to 16 to avoid excessive resource consumption"
459+
460+ NWORKERS=16
461+ fi
462+}
463+
464+###############################################################################
465+# Kill a specified worker
466+###############################################################################
467+function kill_worker()
468+{
469+ local worker=$1
470+ local pid=${worker_pids[$worker]}
471+
472+ worker_pids[$worker]=""
473+
474+ if ! kill -0 $pid >/dev/null 2>&1
475+ then
476+ wait $pid >/dev/null 2>&1 || true
477+ return 0
478+ fi
479+
480+ # First send SIGTERM to let worker exit gracefully
481+ kill -SIGTERM $pid >/dev/null 2>&1 || true
482+
483+ sleep 1
484+
485+ if ! kill -0 $pid >/dev/null 2>&1
486+ then
487+ wait $pid >/dev/null 2>&1 || true
488+ return 0
489+ fi
490+
491+ # Now kill with SIGKILL
492+ kill -SIGKILL $pid >/dev/null 2>&1 || true
493+ wait $pid >/dev/null 2>&1 || true
494+ release_port_locks $pid
495+}
496+
497+########################################################################
498+# Kill all running workers
499+########################################################################
500+function kill_all_workers()
501+{
502+ while true
503+ do
504+ found=""
505+
506+ # First send SIGTERM to let workers exit gracefully
507+ for ((i = 1; i <= NWORKERS; i++))
508+ do
509+ [ -z ${worker_pids[$i]:-""} ] && continue
510+
511+ found="yes"
512+
513+ if ! kill -0 ${worker_pids[$i]} >/dev/null 2>&1
514+ then
515+ worker_pids[$i]=""
516+ continue
517+ fi
518+
519+ kill -SIGTERM ${worker_pids[$i]} >/dev/null 2>&1 || true
520+ done
521+
522+ [ -z "$found" ] && break
523+
524+ sleep 1
525+
526+ # Now kill with SIGKILL
527+ for ((i = 1; i <= NWORKERS; i++))
528+ do
529+ [ -z ${worker_pids[$i]:-""} ] && continue
530+
531+ if ! kill -0 ${worker_pids[$i]} >/dev/null 2>&1
532+ then
533+ wait ${worker_pids[$i]} >/dev/null 2>&1 || true
534+ worker_pids[$i]=""
535+ continue
536+ fi
537+
538+ kill -SIGKILL ${worker_pids[$i]} >/dev/null 2>&1 || true
539+ wait ${worker_pids[$i]} >/dev/null 2>&1 || true
540+ release_port_locks ${worker_pids[$i]}
541+ worker_pids[$i]=""
542+ done
543+ done
544+}
545+
546+########################################################################
547+# Handler called from a fatal signal handler trap
548+########################################################################
549+function terminate()
550+{
551+ echo "Terminated, cleaning up..."
552+
553+ # The following will call cleanup_on_exit()
554+ exit 2
555+}
556+
557+########################################################################
558+# Display the test run summary and exit
559+########################################################################
560+function print_status_and_exit()
561+{
562+ local test_time=$((`now` - TEST_START_TIME))
563+ cat <<EOF
564+==============================================================================
565+Spent $TOTAL_TIME of $test_time seconds executing testcases
566+
567+SUMMARY: $TOTAL_COUNT run, $SUCCESSFUL_COUNT successful, $SKIPPED_COUNT skipped, $FAILED_COUNT failed
568+
569+EOF
570+
571+ if [ -n "$SKIPPED_TESTS" ]
572+ then
573+ echo "Skipped tests: $SKIPPED_TESTS"
574+ echo
575+ fi
576+
577+ if [ -n "$FAILED_TESTS" ]
578+ then
579+ echo "Failed tests: $FAILED_TESTS"
580+ echo
581+ fi
582+
583+ echo "See results/ for detailed output"
584+
585+ if [ "$FAILED_COUNT" = 0 ]
586+ then
587+ exit 0
588+ fi
589+
590+ exit 1
591+}
592+
593+########################################################################
594+# Cleanup procedure invoked on process exit
595+########################################################################
596+function cleanup_on_exit()
597+{
598+ kill_servers $TEST_VAR_ROOT
599+
600+ remove_var_dirs
601+
602+ release_port_locks $$
603+
604+ kill_all_workers
605+
606+ cleanup_all_workers
607 }
608
609 function find_program()
610@@ -52,12 +253,11 @@
611 fi
612 }
613
614+########################################################################
615+# Explore environment and setup global variables
616+########################################################################
617 function set_vars()
618 {
619- TEST_BASEDIR=${TEST_BASEDIR:-"$PWD"}
620- MYSQL_BASEDIR=${MYSQL_BASEDIR:-"$PWD/server"}
621- PORT_BASE=$((3306 + $RANDOM))
622-
623 if gnutar --version > /dev/null 2>&1
624 then
625 TAR=gnutar
626@@ -88,7 +288,7 @@
627 fi
628 DYLD_LIBRARY_PATH="$LD_LIBRARY_PATH"
629
630- export TEST_BASEDIR PORT_BASE TAR MYSQL_BASEDIR MYSQL MYSQLD MYSQLADMIN \
631+ export TAR MYSQL_BASEDIR MYSQL MYSQLD MYSQLADMIN \
632 MYSQL_INSTALL_DB PATH LD_LIBRARY_PATH DYLD_LIBRARY_PATH MYSQLDUMP
633 }
634
635@@ -229,7 +429,7 @@
636 return 1
637 fi
638
639- stop_server
640+ stop_server >>$OUTFILE 2>&1
641
642 export MYSQL_VERSION MYSQL_VERSION_COMMENT MYSQL_FLAVOR \
643 INNODB_VERSION XTRADB_VERSION INNODB_FLAVOR \
644@@ -237,39 +437,362 @@
645 DEFAULT_IBDATA_SIZE XB_BUILD
646 }
647
648-export SKIPPED_EXIT_CODE=200
649+###########################################################################
650+# Kill all server processes started by a worker specified with its var root
651+# directory
652+###########################################################################
653+function kill_servers()
654+{
655+ local var_root=$1
656+ local file
657+
658+ [ -d $var_root ] || return 0
659+
660+ cd $var_root
661+
662+ for file in mysqld*.pid
663+ do
664+ if [ -f $file ]
665+ then
666+ vlog "Found a leftover mysqld processes with PID `cat $file`, \
667+stopping it"
668+ kill -9 `cat $file` 2>/dev/null || true
669+ rm -f $file
670+ fi
671+ done
672+
673+ cd - >/dev/null 2>&1
674+}
675+
676+###########################################################################
677+# Kill all server processes started by a worker specified with its number
678+###########################################################################
679+function kill_servers_for_worker()
680+{
681+ local worker=$1
682+
683+ kill_servers $TEST_BASEDIR/var/w$worker
684+}
685+
686+################################################################################
687+# Clean up all workers (except DEBUG_WORKER if set) We can't use worker* arrays
688+# here and examine the directory structure, because the same functions is called
689+# on startup when we want to cleanup all workers started on previous invokations
690+################################################################################
691+function cleanup_all_workers() { local worker
692+
693+ for worker_dir in $TEST_BASEDIR/var/w+([0-9])
694+ do
695+ [ -d $worker_dir ] || continue
696+ [[ $worker_dir =~ w([0-9]+)$ ]] || continue
697+
698+ worker=${BASH_REMATCH[1]}
699+
700+ if [ "$worker" = "$DEBUG_WORKER" ]
701+ then
702+ echo
703+ echo "Skipping cleanup for worker #$worker due to debug mode."
704+ echo "You can do post-mortem analysis by examining test data in \
705+$worker_dir"
706+ echo "and the server process if it was running at the failure time."
707+ continue
708+ fi
709+
710+ cleanup_worker $worker
711+ done
712+}
713+########################################################################
714+# Clean up a specified worker
715+########################################################################
716+function cleanup_worker()
717+{
718+ local worker=$1
719+ local tmpdir
720+
721+ kill_servers_for_worker $worker >>$OUTFILE 2>&1
722+
723+ # It is possible that a file is created while rm is in progress
724+ # which results in "rm: cannot remove ...: Directory not empty
725+ # hence the loop below
726+ while true
727+ do
728+ # Fix permissions as some tests modify them so the following 'rm' fails
729+ chmod -R 0700 $TEST_BASEDIR/var/w$worker >/dev/null 2>&1
730+ rm -rf $TEST_BASEDIR/var/w$worker && break
731+ done
732+
733+ tmpdir=${worker_tmpdirs[$worker]:-""}
734+ if [ -n "$tmpdir" ]
735+ then
736+ rm -rf $tmpdir
737+ fi
738+}
739+
740+########################################################################
741+# Return the number of seconds since the Epoch, UTC
742+########################################################################
743+function now()
744+{
745+ date '+%s'
746+}
747+
748+########################################################################
749+# Process the exit code of a specified worker
750+########################################################################
751+function reap_worker()
752+{
753+ local worker=$1
754+ local pid=${worker_pids[$worker]}
755+ local skip_file=${worker_skip_files[$worker]}
756+ local tpath=${worker_names[$worker]}
757+ local tname=`basename $tpath .sh`
758+ local test_time=$((`now` - worker_stime[$worker]))
759+ local status_file=${worker_status_files[$worker]}
760+ local rc
761+
762+ if [ -f "$status_file" ]
763+ then
764+ rc=`cat $status_file`
765+ # Assume exit code 1 if status file is empty
766+ rc=${rc:-"1"}
767+ else
768+ # Assume exit code 1 if status file does not exist
769+ rc="1"
770+ fi
771+
772+ printf "%-40s w%d\t" $tname $worker
773+
774+ ((TOTAL_TIME+=test_time))
775+
776+ # Have to call subunit_start_test here, as currently tests cannot be
777+ # interleaved in the subunit output
778+ subunit_start_test $tpath "${worker_stime_txt[$worker]}" >> $SUBUNIT_OUT
779+
780+ if [ $rc -eq 0 ]
781+ then
782+ echo "[passed] $test_time"
783+
784+ SUCCESSFUL_COUNT=$((SUCCESSFUL_COUNT + 1))
785+ subunit_pass_test $tpath >> $SUBUNIT_OUT
786+
787+ cleanup_worker $worker
788+ elif [ $rc -eq $SKIPPED_EXIT_CODE ]
789+ then
790+ sreason=""
791+ test -r $skip_file && sreason=`cat $skip_file`
792+
793+ SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
794+ SKIPPED_TESTS="$SKIPPED_TESTS $tname"
795+
796+ echo "[skipped] $sreason"
797+
798+ subunit_skip_test $tpath >> $SUBUNIT_OUT
799+
800+ cleanup_worker $worker
801+ else
802+ echo "[failed] $test_time"
803+
804+ (
805+ (echo "Something went wrong running $tpath. Exited with $rc";
806+ echo; echo; cat ${worker_outfiles[$worker]}
807+ ) | subunit_fail_test $tpath
808+ ) >> $SUBUNIT_OUT
809+
810+ FAILED_COUNT=$((FAILED_COUNT + 1))
811+ FAILED_TESTS="$FAILED_TESTS $tname"
812+
813+ # Return 0 on failed tests in the -f mode
814+ if [ -z "$force" ]
815+ then
816+
817+ if [ -z "$DEBUG" ]
818+ then
819+ cleanup_worker $worker
820+ else
821+ DEBUG_WORKER=$worker
822+ fi
823+
824+ return 1
825+ else
826+ cleanup_worker $worker
827+
828+ return 0
829+ fi
830+ fi
831+
832+}
833+
834+#############################################################################
835+# Check if a specified worker has exceed TEST_TIMEOUT and if so, terminate it
836+#############################################################################
837+function check_timeout_for_worker()
838+{
839+ local worker=$1
840+ local tpath=${worker_names[$worker]}
841+ local tname=`basename $tpath .sh`
842+
843+ if (( `now` - worker_stime[$worker] > TEST_TIMEOUT ))
844+ then
845+ kill_worker $worker
846+ printf "%-40s w%d\t" $tname $worker
847+ echo "[failed] Timed out after $TEST_TIMEOUT seconds."
848+
849+ (
850+ # Have to call subunit_start_test here, as currently tests cannot be
851+ # interleaved in the subunit output
852+ subunit_start_test $tpath "${worker_stime_txt[$worker]}"
853+
854+ (echo "Timeout exceeded running $tpath.";
855+ echo; echo; cat ${worker_outfiles[$worker]}
856+ ) | subunit_fail_test $tpath
857+ ) >> $SUBUNIT_OUT
858+
859+ FAILED_COUNT=$((FAILED_COUNT + 1))
860+ FAILED_TESTS="$FAILED_TESTS $tname"
861+
862+ # Return 0 on failed tests in the -f mode
863+ if [ -z "$force" ]
864+ then
865+
866+ if [ -z "$DEBUG" ]
867+ then
868+ cleanup_worker $worker
869+ else
870+ DEBUG_WORKER=$worker
871+ fi
872+
873+ return 1
874+ else
875+ cleanup_worker $worker
876+
877+ return 0
878+ fi
879+ fi
880+}
881+#########################################################################
882+# Wait for all currently running worker to finish
883+#########################################################################
884+function reap_all_workers()
885+{
886+ while true
887+ do
888+ found=""
889+
890+ for ((i = 1; i <= NWORKERS; i++))
891+ do
892+ [ -z ${worker_pids[$i]:-""} ] && continue
893+
894+ found="yes"
895+
896+ # Check if it's alive
897+ if kill -0 ${worker_pids[$i]} >/dev/null 2>&1
898+ then
899+ check_timeout_for_worker $i || print_status_and_exit
900+ continue
901+ fi
902+
903+ reap_worker $i || print_status_and_exit
904+
905+ worker_pids[$i]=""
906+ done
907+
908+ [ -z "$found" ] && break
909+
910+ sleep 1
911+ done
912+}
913+
914+########################################################################
915+# Release all port locks reserved by the current process
916+# Used by the EXIT trap (in both normal and abnormal shell termination)
917+########################################################################
918+function release_port_locks()
919+{
920+ local process=$1
921+ local lockfile
922+
923+ # Suppress errors when no port lock files are found
924+ shopt -s nullglob
925+
926+ for lockfile in /tmp/xtrabackup_port_lock.*
927+ do
928+ if [ "`cat $lockfile 2>/dev/null`" = $process ]
929+ then
930+ rm -rf $lockfile
931+ fi
932+ done
933+
934+ shopt -u nullglob
935+}
936+
937+########################################################################
938+# Report status and release port locks on exit
939+########################################################################
940+function cleanup_on_test_exit()
941+{
942+ local rc=$?
943+
944+ echo $rc > $STATUS_FILE
945+
946+ release_port_locks $$
947+}
948+
949+########################################################################
950+# Script body
951+########################################################################
952+
953+TEST_START_TIME=`now`
954
955 tname=""
956 XTRACE_OPTION=""
957 XB_BUILD="autodetect"
958 force=""
959-KEEP_RESULTS=0
960 SUBUNIT_OUT=test_results.subunit
961-
962-while getopts "fgh?:t:s:d:c:b:nr:" options; do
963- case $options in
964- f ) force="yes";;
965- t ) tname="$OPTARG";;
966- g ) XTRACE_OPTION="-x"; DEBUG=on;;
967- h ) usage; exit;;
968- s ) tname="$OPTARG/*.sh";;
969- d ) export MYSQL_BASEDIR="$OPTARG";;
970- b ) TEST_BASEDIR="$OPTARG";;
971- c ) XB_BUILD="$OPTARG";;
972- n ) KEEP_RESULTS=1;;
973- r ) SUBUNIT_OUT="$OPTARG";;
974- ? ) echo "Use \`$0 -h' for the list of available options."
975+NWORKERS=
976+DEBUG_WORKER=""
977+
978+while getopts "fgh?:t:s:d:c:j:T:" options; do
979+ case $options in
980+ f ) force="yes";;
981+ t )
982+
983+ tname="$OPTARG";
984+ if ! [ -r "$tname" ]
985+ then
986+ echo "Cannot find test $tname."
987+ exit -1
988+ fi
989+ ;;
990+
991+ g ) DEBUG=on;;
992+ h ) usage; exit;;
993+ s ) tname="$OPTARG/*.sh";;
994+ d ) export MYSQL_BASEDIR="$OPTARG";;
995+ c ) XB_BUILD="$OPTARG";;
996+ j )
997+
998+ if [[ ! $OPTARG =~ ^[0-9]+$ || $NWORKERS < 1 ]]
999+ then
1000+ echo "Wrong -j argument: $OPTARG"
1001+ exit -1
1002+ fi
1003+ NWORKERS="$OPTARG"
1004+ ;;
1005+
1006+ T )
1007+ if [[ ! $OPTARG =~ ^[0-9]+$ ]]
1008+ then
1009+ echo "Wrong -T argument: $OPTARG"
1010+ exit -1
1011+ fi
1012+ TEST_TIMEOUT="$OPTARG"
1013+ ;;
1014+
1015+ ? ) echo "Use \`$0 -h' for the list of available options."
1016 exit -1;;
1017- esac
1018+ esac
1019 done
1020
1021-if [ $KEEP_RESULTS -eq 0 ];
1022-then
1023- rm -rf results
1024- rm -f $SUBUNIT_OUT
1025-fi
1026-mkdir results
1027-
1028 set_vars
1029
1030 if [ -n "$tname" ]
1031@@ -279,13 +802,17 @@
1032 tests="t/*.sh"
1033 fi
1034
1035-failed_count=0
1036-failed_tests=
1037-total_count=0
1038-
1039 export OUTFILE="$PWD/results/setup"
1040
1041-clean >>$OUTFILE 2>&1
1042+rm -rf results
1043+mkdir results
1044+
1045+cleanup_all_workers >>$OUTFILE 2>&1
1046+
1047+rm -rf var test_results.subunit
1048+mkdir var
1049+
1050+echo "Detecting server version..." | tee -a $OUTFILE
1051
1052 if ! get_version_info
1053 then
1054@@ -297,69 +824,99 @@
1055 tee -a $OUTFILE
1056
1057 echo "Using '`basename $XB_BIN`' as xtrabackup binary" | tee -a $OUTFILE
1058+
1059+[ -z "$NWORKERS" ] && autocalc_nworkers
1060+
1061+if [ "$NWORKERS" -gt 1 ]
1062+then
1063+ echo "Using $NWORKERS parallel workers" | tee -a $OUTFILE
1064+fi
1065 echo | tee -a $OUTFILE
1066
1067-kill_leftovers >>$OUTFILE 2>&1
1068-clean >>$OUTFILE 2>&1
1069-
1070-source subunit.sh
1071-
1072-echo "========================================================================"
1073+cat <<EOF
1074+==============================================================================
1075+TEST WORKER RESULT TIME(s) or COMMENT
1076+------------------------------------------------------------------------------
1077+EOF
1078
1079 for t in $tests
1080 do
1081- total_count=$((total_count+1))
1082-
1083- printf "%-40s" $t
1084- subunit_start_test $t >> $SUBUNIT_OUT
1085+ # Check if we have available workers
1086+ found=""
1087+
1088+ while [ -z "$found" ]
1089+ do
1090+ for ((i = 1; i <= NWORKERS; i++))
1091+ do
1092+ if [ -z ${worker_pids[$i]:-""} ]
1093+ then
1094+ found="yes"
1095+ break
1096+ else
1097+ # Check if it's alive
1098+ if kill -0 ${worker_pids[$i]} >/dev/null 2>&1
1099+ then
1100+ check_timeout_for_worker $i || print_status_and_exit
1101+ continue
1102+ fi
1103+ reap_worker $i || print_status_and_exit
1104+
1105+ worker_pids[$i]=""
1106+ found="yes"
1107+
1108+ break
1109+ fi
1110+ done
1111+
1112+ if [ -z "$found" ]
1113+ then
1114+ sleep 1
1115+ fi
1116+ done
1117+
1118+ worker=$i
1119+
1120+ TOTAL_COUNT=$((TOTAL_COUNT+1))
1121+
1122 name=`basename $t .sh`
1123- export OUTFILE="$PWD/results/$name"
1124- export SKIPPED_REASON="$PWD/results/$name.skipped"
1125- bash $XTRACE_OPTION $t > $OUTFILE 2>&1
1126- rc=$?
1127-
1128- if [[ -z "$DEBUG" || -n "$force" ]]
1129- then
1130- kill_leftovers >>$OUTFILE 2>&1
1131- clean >>$OUTFILE 2>&1
1132- fi
1133-
1134- if [ $rc -eq 0 ]
1135- then
1136- echo "[passed]"
1137- subunit_pass_test $t >> $SUBUNIT_OUT
1138- elif [ $rc -eq $SKIPPED_EXIT_CODE ]
1139- then
1140- sreason=""
1141- test -r $SKIPPED_REASON && sreason=`cat $SKIPPED_REASON`
1142- echo "[skipped] $sreason"
1143- subunit_skip_test $t >> $SUBUNIT_OUT
1144- else
1145- echo "[failed]"
1146-
1147- (
1148- (echo "Something went wrong running $t. Exited with $rc";
1149- echo; echo; cat $OUTFILE
1150- ) | subunit_fail_test $t
1151- ) >> $SUBUNIT_OUT
1152-
1153- failed_count=$((failed_count+1))
1154- failed_tests="$failed_tests $t"
1155- result=1
1156- if [ -z "$force" ]
1157+ worker_names[$worker]=$t
1158+ worker_outfiles[$worker]="$PWD/results/$name"
1159+ worker_skip_files[$worker]="$PWD/results/$name.skipped"
1160+ worker_status_files[$worker]="$PWD/results/$name.status"
1161+ # Create a unique TMPDIR for each worker so that it can be removed as a part
1162+ # of the cleanup procedure. Server socket files will also be created there.
1163+ worker_tmpdirs[$worker]="`mktemp -d -t xbtemp.XXXXXX`"
1164+
1165+ (
1166+ set -eu
1167+ if [ -n "$DEBUG" ]
1168 then
1169- break;
1170+ set -x
1171 fi
1172- fi
1173+
1174+ trap "cleanup_on_test_exit" EXIT
1175+
1176+ . inc/common.sh
1177+
1178+ export OUTFILE=${worker_outfiles[$worker]}
1179+ export SKIPPED_REASON=${worker_skip_files[$worker]}
1180+ export TEST_VAR_ROOT=$TEST_BASEDIR/var/w$worker
1181+ export TMPDIR=${worker_tmpdirs[$worker]}
1182+ export STATUS_FILE=${worker_status_files[$worker]}
1183+
1184+ mkdir $TEST_VAR_ROOT
1185+
1186+ . $t
1187+ ) > ${worker_outfiles[$worker]} 2>&1 &
1188+
1189+ worker_pids[$worker]=$!
1190+ worker_stime[$worker]="`now`"
1191+ # Used in subunit reports
1192+ worker_stime_txt[$worker]="`date -u '+%Y-%m-%d %H:%M:%S'`"
1193 done
1194
1195-echo "========================================================================"
1196-echo
1197-
1198-if [ $result -eq 1 ]
1199-then
1200- echo
1201- echo "$failed_count/$total_count tests have failed: $failed_tests"
1202- echo "See results/ for detailed output"
1203- exit -1
1204-fi
1205+# Wait for in-progress workers to finish
1206+reap_all_workers
1207+
1208+print_status_and_exit
1209+
1210
1211=== modified file 'test/subunit.sh'
1212--- test/subunit.sh 2011-06-12 09:57:00 +0000
1213+++ test/subunit.sh 2013-07-04 07:51:31 +0000
1214@@ -15,8 +15,11 @@
1215 #
1216
1217 subunit_start_test () {
1218+ local stime=$2
1219 # emit the current protocol start-marker for test $1
1220- echo "time: `date -u '+%Y-%m-%d %H:%M:%S'`"
1221+ # adjust time by the number of seconds to execute the test to get the actual
1222+ # start time
1223+ echo "time: $stime"
1224 echo "test: $1"
1225 }
1226
1227
1228=== modified file 'test/t/bug1085099.sh'
1229--- test/t/bug1085099.sh 2013-03-06 10:04:13 +0000
1230+++ test/t/bug1085099.sh 2013-07-04 07:51:31 +0000
1231@@ -13,9 +13,7 @@
1232 # Create a new tmpdir
1233 mkdir $topdir/new_tmpdir
1234
1235-# Make the default tmpdir inaccessible and make sure permissions are restored on
1236-# failure
1237-trap "chmod 755 $MYSQLD_TMPDIR" INT TERM EXIT
1238+# Make the default tmpdir inaccessible
1239 chmod 000 $MYSQLD_TMPDIR
1240
1241 innobackupex --tmpdir=$topdir/new_tmpdir --no-timestamp $topdir/backup
1242
1243=== modified file 'test/t/bug664986.sh'
1244--- test/t/bug664986.sh 2013-03-03 11:01:19 +0000
1245+++ test/t/bug664986.sh 2013-07-04 07:51:31 +0000
1246@@ -15,14 +15,6 @@
1247
1248 EOF
1249
1250-# Make sure we restore permissions to not leave stale files in Jenkins
1251-# on a test failure
1252-trap "vlog restoring directories and files permissions ; \
1253- chmod 777 $MYSQLD_DATADIR/test_bug664986_innodb; \
1254- chmod 777 $MYSQLD_DATADIR/test_bug664986_myisam; \
1255- chmod 644 $MYSQLD_DATADIR/test_bug664986_innodb/*; \
1256- chmod 644 $MYSQLD_DATADIR/test_bug664986_myisam/*" INT TERM EXIT
1257-
1258 # Test that wrong directory permissions result in a backup failure
1259 # for both InnoDB and non-InnoDB files
1260 chmod 000 $MYSQLD_DATADIR/test_bug664986_innodb
1261
1262=== modified file 'test/t/bug996493.sh'
1263--- test/t/bug996493.sh 2012-11-29 04:37:56 +0000
1264+++ test/t/bug996493.sh 2013-07-04 07:51:31 +0000
1265@@ -20,9 +20,6 @@
1266 vlog "Remove my.cnf"
1267 mv $topdir/my.cnf $topdir/my.cnf.bak
1268
1269-trap "vlog restoring $topdir/my.cnf ; \
1270-mv $topdir/my.cnf.bak $topdir/my.cnf" INT TERM EXIT
1271-
1272 vlog "Apply log"
1273 # Do not run innobackupex, because it pass option --defaults-file
1274 # which we should avoid. Our goal is to test that backup-my.cnf
1275@@ -31,7 +28,6 @@
1276
1277 vlog "Get my.cnf back"
1278 mv $topdir/my.cnf.bak $topdir/my.cnf
1279-trap "vlog test exit" INT TERM EXIT
1280
1281 vlog "Copy back"
1282 innobackupex --copy-back $backup_dir
1283
1284=== modified file 'test/t/compact_compressed.sh'
1285--- test/t/compact_compressed.sh 2013-01-14 11:02:59 +0000
1286+++ test/t/compact_compressed.sh 2013-07-04 07:51:31 +0000
1287@@ -11,8 +11,13 @@
1288
1289 # Use innodb_strict_mode so that failure to use compression results in an
1290 # error rather than a warning
1291-mysqld_additional_args="--innodb_strict_mode --innodb_file_per_table \
1292---innodb_file_format=Barracuda"
1293+MYSQLD_EXTRA_MY_CNF_OPTS="
1294+innodb_strict_mode
1295+innodb_file_per_table
1296+innodb_file_format=Barracuda
1297+# Set the minimal log file size to minimize xtrabackup_logfile
1298+# Recovering compressed pages is extremely slow with UNIV_ZIP_DEBUG
1299+innodb_log_file_size=1M"
1300
1301 #
1302 # Test compact backup of a compressed tablespace with a specific page size
1303@@ -26,7 +31,7 @@
1304 echo "Testing compact backup with compressed page size=${page_size}KB "
1305 echo "************************************************************************"
1306
1307- start_server ${mysqld_additional_args}
1308+ start_server
1309
1310 load_dbase_schema sakila
1311 load_dbase_data sakila
1312@@ -52,7 +57,7 @@
1313
1314 rm -r $mysql_datadir
1315
1316- innobackupex --apply-log --rebuild-indexes $backup_dir
1317+ innobackupex --apply-log --rebuild-indexes --rebuild-threads=2 $backup_dir
1318
1319 vlog "Restoring MySQL datadir"
1320 mkdir -p $mysql_datadir
1321@@ -63,7 +68,7 @@
1322 verify_db_state sakila
1323
1324 stop_server
1325- clean
1326+ remove_var_dirs
1327 }
1328
1329
1330
1331=== modified file 'test/t/ib_stream_incremental.sh'
1332--- test/t/ib_stream_incremental.sh 2012-06-05 12:35:33 +0000
1333+++ test/t/ib_stream_incremental.sh 2013-07-04 07:51:31 +0000
1334@@ -98,7 +98,7 @@
1335 exit -1
1336 fi
1337
1338- clean
1339+ remove_var_dirs
1340 }
1341
1342 stream_format="xbstream"
1343
1344=== modified file 'test/t/remote_tablespaces.sh'
1345--- test/t/remote_tablespaces.sh 2013-04-23 06:38:06 +0000
1346+++ test/t/remote_tablespaces.sh 2013-07-04 07:51:31 +0000
1347@@ -12,7 +12,7 @@
1348
1349 start_server --innodb_file_per_table
1350
1351-remote_dir=$TEST_BASEDIR/var1/remote_dir
1352+remote_dir=$TEST_VAR_ROOT/var1/remote_dir
1353
1354 $MYSQL $MYSQL_ARGS test <<EOF
1355 CREATE TABLE t(id INT AUTO_INCREMENT PRIMARY KEY, c INT)
1356
1357=== modified file 'test/t/undo_tablespaces.sh'
1358--- test/t/undo_tablespaces.sh 2013-04-27 18:46:54 +0000
1359+++ test/t/undo_tablespaces.sh 2013-07-04 07:51:31 +0000
1360@@ -23,7 +23,7 @@
1361 EOF
1362 }
1363
1364-undo_directory=$TEST_BASEDIR/var1/undo_dir
1365+undo_directory=$TEST_VAR_ROOT/var1/undo_dir
1366
1367 MYSQLD_EXTRA_MY_CNF_OPTS="
1368 innodb_file_per_table=1
1369
1370=== modified file 'test/t/xb_encrypt.sh'
1371--- test/t/xb_encrypt.sh 2013-03-11 19:11:29 +0000
1372+++ test/t/xb_encrypt.sh 2013-07-04 07:51:31 +0000
1373@@ -5,8 +5,6 @@
1374 encrypt_algo="AES256"
1375 encrypt_key="percona_xtrabackup_is_awesome___"
1376
1377-echo -n "Percona XtraBackup is awesome!!!" > $encrypt_key_file
1378-
1379 innobackupex_options="--encrypt=$encrypt_algo --encrypt-key=$encrypt_key --encrypt-threads=4 --encrypt-chunk-size=8K"
1380 data_decrypt_cmd="for i in *.xbcrypt; do \
1381 xbcrypt -d -a $encrypt_algo -k $encrypt_key < \$i > \${i:0:\${#i}-8}; \
1382
1383=== modified file 'test/t/xb_part_range.sh'
1384--- test/t/xb_part_range.sh 2012-07-06 05:13:15 +0000
1385+++ test/t/xb_part_range.sh 2013-07-04 07:51:31 +0000
1386@@ -1,19 +1,9 @@
1387 . inc/common.sh
1388+. inc/ib_part.sh
1389
1390 start_server
1391
1392-function check_partitioning()
1393-{
1394- $MYSQL $MYSQL_ARGS -Ns -e "show variables like 'have_partitioning'"
1395-}
1396-
1397-PARTITION_CHECK=`check_partitioning`
1398-
1399-if [ -z "$PARTITION_CHECK" ]; then
1400- echo "Requires Partitioning." > $SKIPPED_REASON
1401- stop_server
1402- exit $SKIPPED_EXIT_CODE
1403-fi
1404+require_partitioning
1405
1406 run_cmd $MYSQL $MYSQL_ARGS test <<EOF
1407 CREATE TABLE test (
1408
1409=== modified file 'test/t/xb_perm_basic.sh'
1410--- test/t/xb_perm_basic.sh 2012-10-15 16:14:59 +0000
1411+++ test/t/xb_perm_basic.sh 2013-07-04 07:51:31 +0000
1412@@ -4,6 +4,4 @@
1413 start_server
1414
1415 chmod -R 500 $mysql_datadir
1416-trap "vlog restoring permissions on datadir $mysql_datadir ; \
1417-chmod -R 700 $mysql_datadir" INT TERM EXIT
1418 innobackupex --no-timestamp $topdir/backup
1419
1420=== modified file 'test/t/xb_perm_stream.sh'
1421--- test/t/xb_perm_stream.sh 2012-10-15 16:14:59 +0000
1422+++ test/t/xb_perm_stream.sh 2013-07-04 07:51:31 +0000
1423@@ -5,7 +5,5 @@
1424
1425 # Take backup
1426 chmod -R 500 $mysql_datadir
1427-trap "vlog restoring permissions on datadir $mysql_datadir ; \
1428-chmod -R 700 $mysql_datadir" INT TERM EXIT
1429 mkdir -p $topdir/backup
1430 innobackupex --stream=tar $topdir/backup > $topdir/backup/out.tar
1431
1432=== modified file 'test/t/xb_stats_datadir.sh'
1433--- test/t/xb_stats_datadir.sh 2013-05-02 13:05:59 +0000
1434+++ test/t/xb_stats_datadir.sh 2013-07-04 07:51:31 +0000
1435@@ -5,12 +5,11 @@
1436
1437 . inc/common.sh
1438
1439-mkdir ${TEST_BASEDIR}/logs
1440-
1441-trap "rm -rf ${TEST_BASEDIR}/logs" INT TERM EXIT
1442+logdir=${TEST_VAR_ROOT}/logs
1443+mkdir $logdir
1444
1445 MYSQLD_EXTRA_MY_CNF_OPTS="
1446-innodb_log_group_home_dir=${TEST_BASEDIR}/logs
1447+innodb_log_group_home_dir=$logdir
1448 "
1449
1450 start_server
1451@@ -28,6 +27,6 @@
1452 # we pass all necessary options as an arguments, so if someday this
1453 # will be changed, test still will work
1454 xtrabackup --stats --datadir=${MYSQLD_DATADIR} \
1455- --innodb_log_group_home_dir=${TEST_BASEDIR}/logs
1456+ --innodb_log_group_home_dir=$logdir
1457
1458 vlog "stats did not fail"
1459
1460=== removed file 'test/testrun.c'
1461--- test/testrun.c 2013-06-14 00:59:24 +0000
1462+++ test/testrun.c 1970-01-01 00:00:00 +0000
1463@@ -1,405 +0,0 @@
1464-/*
1465- * testrun.c - a parallel test runner for the XtraBackup test suite
1466- */
1467-/* BEGIN LICENSE
1468- * Copyright (C) 2012 Percona Ireland Ltd.
1469- *
1470- * Written by Stewart Smith
1471- *
1472- * This program is free software: you can redistribute it and/or modify it
1473- * under the terms of the GNU General Public License version 2, as published
1474- * by the Free Software Foundation.
1475- *
1476- * This program is distributed in the hope that it will be useful, but
1477- * WITHOUT ANY WARRANTY; without even the implied warranties of
1478- * MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
1479- * PURPOSE. See the GNU General Public License for more details.
1480- *
1481- * You should have received a copy of the GNU General Public License along
1482- * with this program. If not, see <http://www.gnu.org/licenses/>.
1483- * END LICENSE */
1484-
1485-#include <stdio.h>
1486-#include <unistd.h>
1487-#include <dirent.h>
1488-#include <string.h>
1489-#include <limits.h>
1490-#include <stdlib.h>
1491-#include <sys/select.h>
1492-
1493-#include <sys/stat.h>
1494-#include <sys/types.h>
1495-#include <sys/wait.h>
1496-#include <fcntl.h>
1497-#include <assert.h>
1498-
1499-#include <signal.h>
1500-
1501-struct testcase
1502-{
1503- char *name;
1504- char *buf;
1505- size_t end_of_buf;
1506- size_t bufsz;
1507-};
1508-
1509-pid_t *childpid;
1510-int nchildpid=0;
1511-
1512-static void kill_children(int sig)
1513-{
1514- int i;
1515- int status;
1516-
1517- (void)sig;
1518-
1519- fprintf(stderr, "Killing child processes...\n");
1520-
1521- for(i=0; i<nchildpid; i++)
1522- if(childpid[i] > 0)
1523- {
1524- kill(childpid[i], SIGKILL);
1525- wait(&status);
1526- }
1527-
1528- exit(EXIT_SUCCESS);
1529-}
1530-
1531-int collect_testcases_filter(const struct dirent *a)
1532-{
1533- int l;
1534-
1535- if (a->d_name[0] == '.')
1536- return 0;
1537-
1538- l= strlen(a->d_name);
1539-
1540- if (l > 2 && strncmp(a->d_name + l - 3, ".sh", 3)==0)
1541- return 1;
1542-
1543- return 0;
1544-}
1545-
1546-static int collect_testcases(const char* suitedir, struct testcase **cases)
1547-{
1548- struct dirent **namelist;
1549- int n;
1550- int i;
1551-
1552- n= scandir(suitedir, &namelist, collect_testcases_filter, alphasort);
1553-
1554- *cases= (struct testcase*) malloc(sizeof(struct testcase)*n);
1555-
1556- for(i=0; i<n; i++)
1557- {
1558- (*cases)[i].name= strdup(namelist[i]->d_name);
1559- (*cases)[i].buf= NULL;
1560- (*cases)[i].end_of_buf= 0;
1561- (*cases)[i].bufsz= 0;
1562- free(namelist[i]);
1563- }
1564-
1565- free(namelist);
1566-
1567- return n;
1568-}
1569-
1570-static void free_testcases(struct testcase *cases, int n)
1571-{
1572- while (n>0)
1573- {
1574- free(cases[--n].name);
1575- }
1576- free(cases);
1577-}
1578-
1579-static int run_testcase_in_child(int nr, struct testcase *t, pid_t *cpid, const char* xbtarget)
1580-{
1581- int fd[2];
1582-
1583- printf("[%d] LAUNCHING - %s\n", nr, t->name);
1584-
1585- if (pipe(fd) == -1)
1586- {
1587- perror("pipe");
1588- exit(EXIT_FAILURE);
1589- }
1590-
1591- *cpid= fork();
1592- if (*cpid == 0)
1593- {
1594- /* child */
1595- close(fd[0]);
1596-
1597- char tname[500];
1598- snprintf(tname, sizeof(tname), "t/%s",t->name);
1599-
1600- char basedir[PATH_MAX];
1601- char cwd[PATH_MAX];
1602- snprintf(basedir, sizeof(basedir), "%s/var/%d", getcwd(cwd,sizeof(cwd)), nr);
1603-
1604- mkdir("var",0700);
1605- mkdir(basedir,0700);
1606-
1607- char logname[PATH_MAX];
1608- snprintf(logname, sizeof(logname), "%s/var/%d.log", getcwd(cwd,sizeof(cwd)), nr);
1609-
1610- int logfd= open(logname, O_WRONLY|O_APPEND|O_CREAT, 0600);
1611- dup2(logfd, STDOUT_FILENO);
1612- dup2(logfd, STDERR_FILENO);
1613- close(logfd);
1614-
1615- char subunitfd[50];
1616- snprintf(subunitfd, sizeof(subunitfd), "/dev/fd/%d", fd[1]);
1617-
1618- char* xbtarget_param;
1619- if (xbtarget)
1620- xbtarget_param= strdup(xbtarget);
1621- else
1622- xbtarget_param= NULL;
1623-
1624- char *const newargv[] = {"testrun.sh", "-n",
1625- "-t", tname,
1626- "-b", basedir,
1627- "-r", subunitfd,
1628- (xbtarget)? "-c" : NULL, xbtarget_param,
1629- NULL };
1630- char *newenviron[] = { NULL };
1631- execve(newargv[0], newargv, newenviron);
1632- perror("execve");
1633- exit(EXIT_FAILURE);
1634- }
1635- else
1636- {
1637- /* parent */
1638- close(fd[1]);
1639- fcntl(fd[0], F_SETFL, O_NONBLOCK);
1640- return fd[0];
1641- }
1642-}
1643-
1644-static inline void subunit_progress_sign(int fd, int n, char *sign)
1645-{
1646- char *buf;
1647- const char* fmt= "progress: %s%d\n";
1648- size_t sz= 1+strlen(fmt)+100;
1649- size_t l;
1650-
1651- buf= (char*)malloc(sz);
1652-
1653- l= snprintf(buf, sz, fmt, sign, n);
1654- assert(l < sz);
1655-
1656- write(fd, buf, l);
1657-
1658- free(buf);
1659-}
1660-
1661-static void run_testcases(struct testcase *testcases, int nrcases,
1662- int njobs, int timeout, const char* xbtarget)
1663-{
1664- int childfd[njobs];
1665- int nfds= 0;
1666- int retval;
1667- pid_t chpid[njobs];
1668- int status;
1669- int next_testcase= 0;
1670- int i;
1671- fd_set rfds;
1672- fd_set efds;
1673- struct timeval tv;
1674- int nchildren;
1675- int childtest[njobs];
1676-
1677- int subunitfd= open("test_results.subunit", O_TRUNC|O_WRONLY|O_APPEND|O_CREAT, 0600);
1678- subunit_progress_sign(subunitfd, nrcases, "");
1679-
1680- if (nrcases < njobs)
1681- njobs= nrcases;
1682-
1683- childpid= chpid;
1684- nchildpid= njobs;
1685-
1686- for(i=0; i<njobs; i++)
1687- {
1688- childtest[i]=next_testcase++;
1689- childfd[i]= run_testcase_in_child(i, &testcases[childtest[i]], &childpid[i], xbtarget);
1690- }
1691-
1692- fflush(stdout);
1693-
1694-loop:
1695- FD_ZERO(&efds);
1696- FD_ZERO(&rfds);
1697-
1698- nchildren=0;
1699-
1700- for (i=0; i<njobs; i++)
1701- {
1702- if (childfd[i] != -1)
1703- {
1704- FD_SET(childfd[i], &efds);
1705- FD_SET(childfd[i], &rfds);
1706- nfds= (childfd[i] > nfds)? childfd[i] : nfds;
1707- nchildren++;
1708- }
1709- }
1710-
1711- tv.tv_sec= timeout;
1712- tv.tv_usec= 0;
1713-
1714- retval = select(nfds+1, &rfds, NULL, &efds, &tv);
1715-
1716- if (retval == -1)
1717- perror("select()");
1718- else if (retval)
1719- {
1720- int childexited=0;
1721-
1722- for (i=0; i<njobs; i++)
1723- {
1724- if(childfd[i] != -1
1725- && (FD_ISSET(childfd[i], &efds) || FD_ISSET(childfd[i], &rfds)))
1726- {
1727- ssize_t r=0;
1728-
1729- do {
1730- struct testcase *t= &testcases[childtest[i]];
1731-
1732- if(t->bufsz == t->end_of_buf)
1733- {
1734- t->bufsz+=4000;
1735- t->buf= (char*)realloc(t->buf, t->bufsz);
1736- }
1737-
1738- r= read(childfd[i], t->buf+t->end_of_buf, t->bufsz - t->end_of_buf);
1739- if (r>0)
1740- t->end_of_buf+= r;
1741- } while(r>0);
1742-
1743- pid_t waited= waitpid(childpid[i], &status, WNOHANG);
1744- if (!(WIFEXITED(status) || WIFSIGNALED(status)))
1745- continue;
1746-
1747- if (waited != childpid[i])
1748- continue;
1749-
1750- write(subunitfd, testcases[childtest[i]].buf,
1751- testcases[childtest[i]].end_of_buf);
1752-
1753- close(childfd[i]);
1754- printf("[%d] completed %s status %d\n",
1755- i, testcases[childtest[i]].name, WEXITSTATUS(status));
1756- childfd[i]=-1;
1757- nchildren--;
1758-
1759- if (next_testcase < nrcases)
1760- {
1761- childtest[i]=next_testcase++;
1762- childfd[i]= run_testcase_in_child(i, &testcases[childtest[i]], &childpid[i], xbtarget);
1763- nfds= (childfd[i] > nfds)? childfd[i] : nfds;
1764- nchildren++;
1765- }
1766- printf("\nnrchildren= %d, %d tests remaining\n",
1767- nchildren, nrcases-next_testcase);
1768- childexited=1;
1769- fflush(stdout);
1770- }
1771- }
1772- if (childexited)
1773- {
1774- printf ("Running: ");
1775- for(i=0; i<njobs; i++)
1776- if (childfd[i] != -1)
1777- printf("%s ",testcases[childtest[i]].name);
1778- printf("\n");
1779- }
1780- }
1781- else
1782- {
1783- printf("Timeout\n");
1784- kill_children(SIGKILL);
1785- exit(EXIT_FAILURE);
1786- }
1787-
1788- if (nchildren==0)
1789- goto end;
1790-
1791- goto loop;
1792-
1793-end:
1794-
1795- close(subunitfd);
1796- return;
1797-}
1798-
1799-int main(int argc, char* argv[])
1800-{
1801- const char* suitedir= "t/";
1802- int njobs= 4;
1803- int opt;
1804- int nrcases;
1805- struct testcase *testcases;
1806- int timeout= 600;
1807- const char* xbtarget= NULL;
1808-
1809- struct sigaction sa;
1810-
1811- sa.sa_flags= 0;
1812- sigemptyset(&sa.sa_mask);
1813- sa.sa_handler = kill_children;
1814- sigaction(SIGHUP, &sa, NULL);
1815- sigaction(SIGINT, &sa, NULL);
1816- sigaction(SIGQUIT, &sa, NULL);
1817-
1818-#ifdef _SC_NPROCESSORS_ONLN
1819- njobs= sysconf(_SC_NPROCESSORS_ONLN);
1820-#endif
1821-
1822-#ifdef _SC_PHYS_PAGES
1823- long nrpages= sysconf(_SC_PHYS_PAGES);
1824- long pagesize= sysconf(_SC_PAGESIZE);
1825- long pages_per_job= (128*(1 << 20)) / pagesize;
1826- nrpages= nrpages/2;
1827- if ((pages_per_job * njobs) > nrpages)
1828- njobs= nrpages / pages_per_job;
1829-#endif
1830-
1831- if (njobs == 0)
1832- njobs= 1;
1833-
1834- while ((opt = getopt(argc, argv, "j:s:t:c:")) != -1)
1835- {
1836- switch (opt) {
1837- case 'c':
1838- xbtarget= optarg;
1839- break;
1840- case 's':
1841- suitedir= optarg;
1842- break;
1843- case 'j':
1844- njobs= atoi(optarg);
1845- break;
1846- case 't':
1847- timeout= atoi(optarg);
1848- break;
1849- default:
1850- fprintf(stderr, "Usage: %s [-s suite] [-j parallel] [-t timeout]\n",
1851- argv[0]);
1852- exit(EXIT_FAILURE);
1853- }
1854- }
1855-
1856- printf("%s running with: -s %s -j %d -t %d\n\n",
1857- argv[0], suitedir,njobs, timeout);
1858-
1859- nrcases= collect_testcases(suitedir, &testcases);
1860-
1861- printf("Found %d testcases\n", nrcases);
1862-
1863- run_testcases(testcases, nrcases, njobs, timeout, xbtarget);
1864-
1865- free_testcases(testcases, nrcases);
1866-
1867- return 0;
1868-}

Subscribers

People subscribed via source and target branches

to all changes: