Merge lp:~gl-az/percona-xtrabackup/bug1243009-2.2 into lp:percona-xtrabackup/2.2

[George Ormond Lorch III]

Fix for bug 1243009 --debug and --valgrind options needed for XtraBackup's build-binary.sh

util/build-binary.sh:
Added new options --debug, --relwithdebinfo and --valgrind. These will set and export the DEBUG, RELWITHDEBINFO and VALGRIND variables to 'yes' before calling build.sh if specified. Even though they are mutually exclusive, there is no explicit test for this case.

util/build.sh:
Added testing for RELWITHDEBINFO and VALGRIND variables and set up build/make options necessary for each case.

test/run.sh and test/inc/common.sh:
Added -v N option to run.sh to allow enabling of valgrind when using test suite. N specifies a value that will be passed to valgrinds --error-exitcode option. Made changes to accomplish this based on Alexey Kopytovs suggestions.
Moved innobackupex_no_defaults_file into inc/common.sh from t/ib_doublewrite.sh since the concept is used in a few tests and correct fixup of IB_ARGS[@] is necessary in order to preserve args and valgrind options and makes a couple test cases a little simpler.

Fixed up several test cases that use/depend on $IB_ARGS to use "${IB_ARGS[@]}".

jenkins http://jenkins.percona.com/view/XtraBackup/job/percona-xtrabackup-2.2-param/42/
jenkins with '-v 0 -T 1600' options http://jenkins.percona.com/view/XtraBackup/job/percona-xtrabackup-2.2-param/44/

[Alexey Kopytov (akopytov)]

Same question as in the 2.0 MP.

review: Needs Information

[Vlad Lesin (vlad-lesin)]

The same questions as here https://code.launchpad.net/~gl-az/percona-xtrabackup/bug1243009-2.1/+merge/195435/comments/465902 .

review: Needs Fixing (g2)

Unmerged revisions

4893. By George Ormond Lorch III on 2013-11-15

Fix for bug 1243009 --debug and --valgrind options needed for XtraBackup's build-binary.sh

util/build-binary.sh:
  Added new options --debug, --relwithdebinfo and --valgrind. These will set and export the DEBUG, RELWITHDEBINFO and VALGRIND variables to 'yes' before calling build.sh if specified. Even though they are mutually exclusive, there is no explicit test for this case.

util/build.sh:
  Added testing for RELWITHDEBINFO and VALGRIND variables and set up build/make options necessary for each case.

test/run.sh and test/inc/common.sh:
  Added -v N option to run.sh to allow enabling of valgrind when using test suite. N specifies a value that will be bassed to valgrinds --error-exitcode option. Made changes to accomplish this based on Alexey Kopytovs suggestions.
  Moved innobackupex_no_defaults_file into inc/common.sh from t/ib_doublewrite.sh since the concept is used in a few tests and correct fixup of IB_ARGS[@] is necessary in order to preserve args and valgrind options and makes a couple test cases a little simpler.

Fixed up several test cases that use/depend on $IB_ARGS to use "${IB_ARGS[@]}".

=== modified file 'xtrabackup/test/inc/common.sh'
--- xtrabackup/test/inc/common.sh	2013-11-07 12:23:43 +0000
+++ xtrabackup/test/inc/common.sh	2013-11-15 20:35:14 +0000
@@ -2,9 +2,19 @@
 
 function innobackupex()
 {
-    run_cmd $IB_BIN $IB_ARGS $*
+    run_cmd $IB_BIN "${IB_ARGS[@]}" $*
 }
+function innobackupex_no_defaults_file()
+{
+    local IB_ARGS_NO_DEFAULTS_FILE=()
+    for arg in "${IB_ARGS[@]}"; do
+        if [[ $arg != "--defaults-file="* ]]; then
+            IB_ARGS_NO_DEFAULTS_FILE[${#IB_ARGS_NO_DEFAULTS_FILE[@]}]=$arg
+	fi
+    done
 
+    run_cmd $IB_BIN "${IB_ARGS_NO_DEFAULTS_FILE[@]}" $*
+}
 function xtrabackup()
 {
     run_cmd $XB_BIN $XB_ARGS $*
@@ -239,8 +249,8 @@
 	MYSQLD_ARGS="$MYSQLD_ARGS --user=root"
     fi
 
-    IB_ARGS="--defaults-file=$MYSQLD_VARDIR/my.cnf --ibbackup=$XB_BIN \
---no-version-check"
+    IB_ARGS=("--defaults-file=$MYSQLD_VARDIR/my.cnf" "--ibbackup=$XB_BIN"
+             "--no-version-check")
     XB_ARGS="--defaults-file=$MYSQLD_VARDIR/my.cnf"
 
     # Some aliases for compatibility, as tests use the following names

=== modified file 'xtrabackup/test/run.sh'
--- xtrabackup/test/run.sh	2013-09-23 05:21:56 +0000
+++ xtrabackup/test/run.sh	2013-11-15 20:35:14 +0000
@@ -38,10 +38,11 @@
 function usage()
 {
 cat <<EOF
-Usage: $0 [-f] [-g] [-h] [-s suite] [-t test_name] [-d mysql_basedir] [-c build_conf]
+Usage: $0 [-f] [-g] [-v child_error_exit] [-h] [-s suite] [-t test_name] [-d mysql_basedir] [-c build_conf]
 -f          Continue running tests after failures
 -d path     Server installation directory. Default is './server'.
 -g          Debug mode
+-v N        Run XtraBackup binary through "valgrind --error-exitcode=N".
 -t path     Run only a single named test
 -h          Print this help message
 -s suite    Select a test suite to run. Possible values: experimental, t. 
@@ -440,7 +441,12 @@
 	vlog "Cannot find '$XB_BIN' in PATH"
 	return 1
     fi
-    XB_BIN="$XB_PATH"
+    if [ -z "$VALGRIND" ]
+    then
+        XB_BIN="$XB_PATH"
+    else
+        XB_BIN="valgrind --error-exitcode=$VALGRIND $XB_PATH"
+    fi
 
     # Set the correct binary for innobackupex
     IB_BIN="`which innobackupex`"
@@ -771,8 +777,9 @@
 SUBUNIT_OUT=test_results.subunit
 NWORKERS=
 DEBUG_WORKER=""
+VALGRIND=""
 
-while getopts "fgh?:t:s:d:c:j:T:" options; do
+while getopts "fgh?:t:s:d:c:j:v:T:" options; do
         case $options in
             f ) force="yes";;
             t )
@@ -786,6 +793,15 @@
                 ;;
 
             g ) DEBUG=on;;
+            v )
+                if [[ ! $OPTARG =~ ^[0-9]+$ ]]
+                then
+                    echo "Wrong -v argument: $OPTARG"
+                    exit -1
+                fi
+                VALGRIND="$OPTARG"
+                ;;
+
             h ) usage; exit;;
             s ) tname="$OPTARG/*.sh";;
             d ) export MYSQL_BASEDIR="$OPTARG";;
@@ -844,7 +860,7 @@
 echo "Running against $MYSQL_FLAVOR $MYSQL_VERSION ($INNODB_FLAVOR $INNODB_VERSION)" |
   tee -a $OUTFILE
 
-echo "Using '`basename $XB_BIN`' as xtrabackup binary" | tee -a $OUTFILE
+echo "Using '$XB_BIN' as xtrabackup binary" | tee -a $OUTFILE
 
 [ -z "$NWORKERS" ] && autocalc_nworkers
 

=== modified file 'xtrabackup/test/t/bug1037379.sh'
--- xtrabackup/test/t/bug1037379.sh	2012-10-16 15:05:23 +0000
+++ xtrabackup/test/t/bug1037379.sh	2013-11-15 20:35:14 +0000
@@ -40,7 +40,7 @@
 ################################################################################
 
 # The following will fail due to a timeout
-run_cmd_expect_failure $IB_BIN $IB_ARGS --no-timestamp --safe-slave-backup \
+run_cmd_expect_failure $IB_BIN "${IB_ARGS[@]}" --no-timestamp --safe-slave-backup \
     --safe-slave-backup-timeout=3 $topdir/backup1
 
 grep -q "Slave_open_temp_tables did not become zero" $OUTFILE
@@ -57,7 +57,7 @@
 run_cmd $MYSQL $MYSQL_ARGS -e "STOP SLAVE SQL_THREAD"
 
 # The following will fail due to a timeout
-run_cmd_expect_failure $IB_BIN $IB_ARGS --no-timestamp --safe-slave-backup \
+run_cmd_expect_failure $IB_BIN "${IB_ARGS[@]}" --no-timestamp --safe-slave-backup \
     --safe-slave-backup-timeout=3 $topdir/backup2
 
 grep -c "Slave_open_temp_tables did not become zero" $OUTFILE | grep -w 2

=== modified file 'xtrabackup/test/t/bug1112224.sh'
--- xtrabackup/test/t/bug1112224.sh	2013-03-18 14:13:04 +0000
+++ xtrabackup/test/t/bug1112224.sh	2013-11-15 20:35:14 +0000
@@ -35,7 +35,7 @@
 vlog "Log applied to full backup"
 
 # Command should fail and print error message
-run_cmd_expect_failure $IB_BIN $IB_ARGS --apply-log --redo-only --incremental-dir=$topdir/inc \
+run_cmd_expect_failure $IB_BIN "${IB_ARGS[@]}" --apply-log --redo-only --incremental-dir=$topdir/inc \
     $topdir/full
 if ! grep -q "Cannot handle DDL operation" $OUTFILE
 then

=== modified file 'xtrabackup/test/t/bug483827.sh'
--- xtrabackup/test/t/bug483827.sh	2013-04-27 18:46:54 +0000
+++ xtrabackup/test/t/bug483827.sh	2013-11-15 20:35:14 +0000
@@ -2,12 +2,6 @@
 # Bug #483827: support for mysqld_multi
 ########################################################################
 
-function modify_args()
-{
-  XB_ARGS=`echo $XB_ARGS | sed -e 's/my.cnf/my_multi.cnf/'`
-  IB_ARGS=`echo $IB_ARGS | sed -e 's/my.cnf/my_multi.cnf/'`
-}
-
 . inc/common.sh
 
 start_server
@@ -15,14 +9,14 @@
 backup_dir=$topdir/backup
 
 # change defaults file from my.cnf to my_multi.cnf
-modify_args
+XB_ARGS=`echo $XB_ARGS | sed -e 's/my.cnf/my_multi.cnf/'`
 
 # make my_multi.cnf
 sed -e 's/\[mysqld\]/[mysqld1]/' $topdir/my.cnf > $topdir/my_multi.cnf
 
 # Backup
-innobackupex --no-timestamp --defaults-group=mysqld1 $backup_dir
-innobackupex --apply-log $backup_dir
+innobackupex_no_defaults_file --defaults-file=$topdir/my_multi.cnf --no-timestamp --defaults-group=mysqld1 $backup_dir
+innobackupex_no_defaults_file --defaults-file=$topdir/my_multi.cnf --apply-log $backup_dir
 
 stop_server
 
@@ -30,7 +24,7 @@
 rm -rf ${mysql_datadir}/*
 
 # restore backup
-innobackupex --copy-back --defaults-group=mysqld1 $backup_dir
+innobackupex_no_defaults_file --defaults-file=$topdir/my_multi.cnf --copy-back --defaults-group=mysqld1 $backup_dir
 
 # make sure that data are in correct place
 if [ ! -f ${mysql_datadir}/ibdata1 ] ; then

=== modified file 'xtrabackup/test/t/bug664986.sh'
--- xtrabackup/test/t/bug664986.sh	2013-07-03 18:30:28 +0000
+++ xtrabackup/test/t/bug664986.sh	2013-11-15 20:35:14 +0000
@@ -18,23 +18,23 @@
 # Test that wrong directory permissions result in a backup failure
 # for both InnoDB and non-InnoDB files
 chmod 000 $MYSQLD_DATADIR/test_bug664986_innodb
-run_cmd_expect_failure $IB_BIN $IB_ARGS --no-timestamp $topdir/backup
+run_cmd_expect_failure $IB_BIN "${IB_ARGS[@]}" --no-timestamp $topdir/backup
 chmod 777 $MYSQLD_DATADIR/test_bug664986_innodb
 rm -rf $topdir/backup
 
 chmod 000 $MYSQLD_DATADIR/test_bug664986_myisam
-run_cmd_expect_failure $IB_BIN $IB_ARGS --no-timestamp $topdir/backup
+run_cmd_expect_failure $IB_BIN "${IB_ARGS[@]}" --no-timestamp $topdir/backup
 chmod 777 $MYSQLD_DATADIR/test_bug664986_myisam
 rm -rf $topdir/backup
 
 # Test that wrong file permissions result in a backup failure
 # for both InnoDB and non-InnoDB files
 chmod 000 $MYSQLD_DATADIR/test_bug664986_innodb/t.ibd
-run_cmd_expect_failure $IB_BIN $IB_ARGS --no-timestamp $topdir/backup
+run_cmd_expect_failure $IB_BIN "${IB_ARGS[@]}" --no-timestamp $topdir/backup
 chmod 644 $MYSQLD_DATADIR/test_bug664986_innodb/t.ibd
 rm -rf $topdir/backup
 
 chmod 000 $MYSQLD_DATADIR/test_bug664986_myisam/t.MYD
-run_cmd_expect_failure $IB_BIN $IB_ARGS --no-timestamp $topdir/backup
+run_cmd_expect_failure $IB_BIN "${IB_ARGS[@]}" --no-timestamp $topdir/backup
 chmod 644 $MYSQLD_DATADIR/test_bug664986_myisam/t.MYD
 rm -rf $topdir/backup

=== modified file 'xtrabackup/test/t/bug729843.sh'
--- xtrabackup/test/t/bug729843.sh	2013-02-05 11:54:27 +0000
+++ xtrabackup/test/t/bug729843.sh	2013-11-15 20:35:14 +0000
@@ -12,7 +12,7 @@
 # Don't use run_cmd_* or innobackupex functions here to avoid logging
 # the full command line (including the password in plaintext)
 set +e
-$IB_BIN $IB_ARGS --password=secretpassword $topdir/backup 2>&1 | tee $logfile
+$IB_BIN "${IB_ARGS[@]}" --password=secretpassword $topdir/backup 2>&1 | tee $logfile
 set -e
 
 # Check that the password was not logged in plaintext

=== modified file 'xtrabackup/test/t/bug740489.sh'
--- xtrabackup/test/t/bug740489.sh	2013-04-27 18:46:54 +0000
+++ xtrabackup/test/t/bug740489.sh	2013-11-15 20:35:14 +0000
@@ -24,7 +24,7 @@
 backup_dir=$topdir/backup
 run_cmd $IB_BIN \
     --defaults-extra-file=$defaults_extra_file --socket=${MYSQLD_SOCKET} \
-    --ibbackup=$XB_BIN --no-timestamp $backup_dir
+    "--ibbackup=$XB_BIN" --no-timestamp $backup_dir
 vlog "Backup created in directory $backup_dir"
 
 run_cmd ${MYSQL} ${MYSQL_ARGS} --password=password <<EOF

=== modified file 'xtrabackup/test/t/bug996493.sh'
--- xtrabackup/test/t/bug996493.sh	2013-07-03 18:30:28 +0000
+++ xtrabackup/test/t/bug996493.sh	2013-11-15 20:35:14 +0000
@@ -24,7 +24,7 @@
 # Do not run innobackupex, because it pass option --defaults-file
 # which we should avoid. Our goal is to test that backup-my.cnf
 # will be read by default when apply-log is run.
-run_cmd $IB_BIN --ibbackup=$XB_BIN --apply-log $backup_dir
+run_cmd $IB_BIN "--ibbackup=$XB_BIN" --apply-log $backup_dir
 
 vlog "Get my.cnf back"
 mv $topdir/my.cnf.bak $topdir/my.cnf

=== modified file 'xtrabackup/test/t/history_on_server.sh'
--- xtrabackup/test/t/history_on_server.sh	2013-11-08 12:00:43 +0000
+++ xtrabackup/test/t/history_on_server.sh	2013-11-15 20:35:14 +0000
@@ -180,7 +180,7 @@
 "--incremental-history-uuid=$third_uuid --stream=xbstream --compress "\
 "--encrypt=AES256 $backup_dir --encrypt-key=..."
 
-if [ -z "$val" ] || [ "$val" != "$expected_val" ]
+if [ $val != *$expected_val ]
 then
   vlog "Error: $column in history record invalid, got \"$val\" expected \"$expected_val\""
   exit 1

=== modified file 'xtrabackup/test/t/ib_doublewrite.sh'
--- xtrabackup/test/t/ib_doublewrite.sh	2013-07-25 15:04:49 +0000
+++ xtrabackup/test/t/ib_doublewrite.sh	2013-11-15 20:35:14 +0000
@@ -23,13 +23,6 @@
 start_server
 load_dbase_schema incremental_sample
 
-# Workaround for bug #1072695
-IB_ARGS_NO_DEFAULTS_FILE=`echo $IB_ARGS | sed -e 's/--defaults-file=[^ ]* / /'`
-function innobackupex_no_defaults_file ()
-{
-	run_cmd $IB_BIN $IB_ARGS_NO_DEFAULTS_FILE $*
-}
-
 # Adding initial rows
 vlog "Adding initial rows to database..."
 ${MYSQL} ${MYSQL_ARGS} -e "insert into test values (1, 1);" incremental_sample
@@ -75,17 +68,20 @@
 vlog "##############"
 vlog "# PREPARE #1 #"
 vlog "##############"
+# Workaround for bug #1072695
 innobackupex_no_defaults_file --apply-log --redo-only $full_backup_dir
 vlog "Log applied to full backup"
 vlog "##############"
 vlog "# PREPARE #2 #"
 vlog "##############"
+# Workaround for bug #1072695
 innobackupex_no_defaults_file --apply-log --redo-only --incremental-dir=$inc_backup_dir \
     $full_backup_dir
 vlog "Delta applied to full backup"
 vlog "##############"
 vlog "# PREPARE #3 #"
 vlog "##############"
+# Workaround for bug #1072695
 innobackupex_no_defaults_file --apply-log $full_backup_dir
 vlog "Data prepared for restore"
 

=== modified file 'xtrabackup/test/t/ib_empty_dir.sh'
--- xtrabackup/test/t/ib_empty_dir.sh	2012-10-15 16:14:59 +0000
+++ xtrabackup/test/t/ib_empty_dir.sh	2013-11-15 20:35:14 +0000
@@ -22,7 +22,7 @@
 vlog "###########"
 vlog "# RESTORE #"
 vlog "###########"
-run_cmd_expect_failure $IB_BIN $IB_ARGS --copy-back $backup_dir
+run_cmd_expect_failure $IB_BIN "${IB_ARGS[@]}" --copy-back $backup_dir
 
 if grep -q "is not empty!" $OUTFILE
 then

=== modified file 'xtrabackup/test/t/ib_slave_info.sh'
--- xtrabackup/test/t/ib_slave_info.sh	2013-08-18 18:07:09 +0000
+++ xtrabackup/test/t/ib_slave_info.sh	2013-11-15 20:35:14 +0000
@@ -18,7 +18,7 @@
 switch_server $slave_id
 
 vlog "Check that --slave-info with --no-lock and no --safe-slave-backup fails"
-run_cmd_expect_failure $IB_BIN $IB_ARGS --no-timestamp --slave-info --no-lock \
+run_cmd_expect_failure $IB_BIN "${IB_ARGS[@]}" --no-timestamp --slave-info --no-lock \
   $topdir/backup
 
 innobackupex --no-timestamp --slave-info $topdir/backup

=== modified file 'xtrabackup/test/t/kill_long_selects.sh'
--- xtrabackup/test/t/kill_long_selects.sh	2013-10-21 10:32:46 +0000
+++ xtrabackup/test/t/kill_long_selects.sh	2013-11-15 20:35:14 +0000
@@ -141,7 +141,7 @@
 
 sleep 1
 
-run_cmd_expect_failure ${IB_BIN} ${IB_ARGS} $topdir/full \
+run_cmd_expect_failure ${IB_BIN} "${IB_ARGS[@]}" $topdir/full \
                           --lock-wait-timeout=3 \
                           --lock-wait-query-type=all \
                           --lock-wait-threshold=1 \
@@ -160,7 +160,7 @@
 
 sleep 2
 
-run_cmd_expect_failure ${IB_BIN} ${IB_ARGS} $topdir/full \
+run_cmd_expect_failure ${IB_BIN} "${IB_ARGS[@]}" $topdir/full \
                           --lock-wait-timeout=3 \
                           --lock-wait-query-type=update \
                           --lock-wait-threshold=2 \

=== modified file 'xtrabackup/test/t/version_check.sh'
--- xtrabackup/test/t/version_check.sh	2013-11-07 12:23:43 +0000
+++ xtrabackup/test/t/version_check.sh	2013-11-15 20:35:14 +0000
@@ -35,7 +35,13 @@
     die "/tmp/percona-version-check has been created with --no-version-check!"
 fi
 
-IB_ARGS=`echo $IB_ARGS | sed -e 's/--no-version-check//g'`
+for ((i=0; i < ${#IB_ARGS[@]}; i++))
+do
+    if [ "${IB_ARGS[$i]}" = "--no-version-check" ]
+    then
+        IB_ARGS=("${IB_ARGS[@]:0:$i}" "${IB_ARGS[@]:(($i + 1))}")
+    fi
+done
 
 innobackupex --no-timestamp $topdir/backup2
 

=== modified file 'xtrabackup/utils/build-binary.sh'
--- xtrabackup/utils/build-binary.sh	2013-06-05 08:39:54 +0000
+++ xtrabackup/utils/build-binary.sh	2013-11-15 20:35:14 +0000
@@ -22,7 +22,8 @@
 # Check if we have a functional getopt(1)
 if ! getopt --test
 then
-    go_out="$(getopt --options="i" --longoptions=i686 \
+    go_out="$(getopt --options="drvi" \
+        --longoptions="debug, relwithdebinfo, valgrind, i686" \
         --name="$(basename "$0")" -- "$@")"
     test $? -eq 0 || exit 1
     eval set -- $go_out
@@ -32,6 +33,18 @@
 do
     case "$arg" in
     -- ) shift; break;;
+    -d | --debug )
+        shift
+        export DEBUG=yes
+        ;;
+    -r | --relwithdebinfo )
+        shift
+        export RELWITHDEBINFO=yes
+        ;;
+    -v | --valgrind )
+        shift
+        export VALGRIND=yes
+        ;;
     -i | --i686 )
         shift
         TARGET="i686"

=== modified file 'xtrabackup/utils/build.sh'
--- xtrabackup/utils/build.sh	2013-07-20 14:24:26 +0000
+++ xtrabackup/utils/build.sh	2013-11-15 20:35:14 +0000
@@ -35,8 +35,20 @@
         ;;
 esac
 
+if [ -n "$VALGRIND" ]
+then
+    export CFLAGS="$CFLAGS -g -O0"
+    export CXXFLAGS="$CXXFLAGS -g -O0"
+    extra_config_51=
+    extra_config_55plus="-DCMAKE_BUILD_TYPE=RelWithDebInfo -DWITH_VALGRIND=ON"
+elif [ -n "$RELWITHDEBINFO" ]
+then
+    export CFLAGS="$CFLAGS -g -O0"
+    export CXXFLAGS="$CXXFLAGS -g -O0"
+    extra_config_51=
+    extra_config_55plus="-DCMAKE_BUILD_TYPE=RelWithDebInfo"
 # Percona Server 5.5 does not build with -Werror, so ignore DEBUG for now
-if [ -n "$DEBUG" -a "$type" != "xtradb55" -a "$type" != "xtradb51" ]
+elif [ -n "$DEBUG" -a "$type" != "xtradb55" -a "$type" != "xtradb51" ]
 then
     # InnoDB extra debug flags
     innodb_extra_debug="-DUNIV_DEBUG -DUNIV_MEM_DEBUG \