Merge lp:~akopytov/percona-xtrabackup/bug766607 into lp:percona-xtrabackup/2.0

[Alexey Kopytov]

[Alexey Kopytov (akopytov)]

http://jenkins.percona.com/view/Percona%20Xtrabackup/job/percona-xtrabackup-param/27/

[Stewart Smith (stewart)]

review: Approve

=== modified file 'test/inc/common.sh'
--- test/inc/common.sh	2011-06-15 07:04:42 +0000
+++ test/inc/common.sh	2011-06-20 18:06:00 +0000
@@ -4,12 +4,12 @@
 
 function innobackupex()
 {
-    run_cmd `which innobackupex` $IB_ARGS $*
+    run_cmd $IB_BIN $IB_ARGS $*
 }
 
 function xtrabackup()
 {
-    run_cmd `which $XB_BIN` $XB_ARGS $*
+    run_cmd $XB_BIN $XB_ARGS $*
 }
 
 function vlog
@@ -129,7 +129,27 @@
 function run_cmd()
 {
   vlog "===> $@"
-  "$@" || die "===> `basename $1` failed with exit code $?"
+  set +e
+  "$@"
+  local rc=$?
+  set -e
+  if [ $rc -ne 0 ]
+  then
+      die "===> `basename $1` failed with exit code $rc"
+  fi
+}
+
+function run_cmd_expect_failure()
+{
+  vlog "===> $@"
+  set +e
+  "$@"
+  local rc=$?
+  set -e
+  if [ $rc -eq 0 ]
+  then
+      die "===> `basename $1` succeeded when it was expected to fail"
+  fi
 }
 
 function stop_mysqld()

=== modified file 'test/inc/incremental_sample-db/incremental_sample-schema.sql'
--- test/inc/incremental_sample-db/incremental_sample-schema.sql	2010-12-09 09:37:20 +0000
+++ test/inc/incremental_sample-db/incremental_sample-schema.sql	2011-06-20 18:06:00 +0000
@@ -1,40 +1,5 @@
--- MySQL dump 10.13  Distrib 5.1.51, for unknown-linux-gnu (x86_64)
---
--- Host: localhost    Database: test
--- ------------------------------------------------------
--- Server version	5.1.51-rel11.5-log
-
-/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
-/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
-/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
-/*!40101 SET NAMES utf8 */;
-/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
-/*!40103 SET TIME_ZONE='+00:00' */;
-/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
-/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
-/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
-/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
-
---
--- Table structure for table `test`
---
-
 DROP TABLE IF EXISTS `test`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
 CREATE TABLE `test` (
   `a` int(11) DEFAULT NULL,
   `number` int(11) DEFAULT NULL
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
-/*!40101 SET character_set_client = @saved_cs_client */;
-/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
-
-/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
-/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
-/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
-/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
-/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
-/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
-/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-
--- Dump completed on 2010-11-15 15:02:47

=== modified file 'test/run.sh'
--- test/run.sh	2011-06-15 07:04:42 +0000
+++ test/run.sh	2011-06-20 18:06:00 +0000
@@ -94,10 +94,10 @@
 
 function get_version_info()
 {
-    init
+    init >>$OUTFILE 2>&1
     MYSQL_VERSION=""
     INNODB_VERSION=""
-    run_mysqld
+    run_mysqld >>$OUTFILE 2>&1
     # Get MySQL and InnoDB versions
     MYSQL_VERSION=`$MYSQL ${MYSQL_ARGS} -Nsf -e "SHOW VARIABLES LIKE 'version'"`
     MYSQL_VERSION=${MYSQL_VERSION#"version	"}
@@ -105,7 +105,7 @@
     MYSQL_VERSION_COMMENT=${MYSQL_VERSION_COMMENT#"version_comment	"}
     INNODB_VERSION=`$MYSQL ${MYSQL_ARGS} -Nsf -e "SHOW VARIABLES LIKE 'innodb_version'"`
     INNODB_VERSION=${INNODB_VERSION#"innodb_version	"}
-    XTRADB_VERSION=`echo $INNODB_VERSION  | sed 's/[0-9]\.[0-9]\.[0-9][0-9]*\(-[0-9][0-9]*\.[0-9][0-9]*\)*$/\1/'`
+    XTRADB_VERSION="`echo $INNODB_VERSION  | sed 's/[0-9]\.[0-9]\.[0-9][0-9]*\(-[0-9][0-9]*\.[0-9][0-9]*\)*$/\1/'`"
     if [ "${MYSQL_VERSION:0:3}" = "5.1" ]
     then
 	if [ -z "$INNODB_VERSION" ]
@@ -126,11 +126,25 @@
 	vlog "Uknown MySQL/InnoDB version: $MYSQL_VERSION/$INNODB_VERSION"
 	exit -1
     fi
-    vlog "Using '$XB_BIN' as xtrabackup binary"
-    # Set the correct binary for innobackupex
+
+    XB_PATH="`which $XB_BIN`"
+    if [ -z "$XB_PATH" ]
+    then
+	vlog "Cannot find '$XB_BIN' in PATH"
+	return 1
+    fi
+    XB_BIN="$XB_PATH"
+
+    IB_BIN="`which innobackupex`"
+    if [ -z "$IB_BIN" ]
+    then
+	vlog "Cannot find 'innobackupex' in PATH"
+	return 1
+    fi
     IB_ARGS="$IB_ARGS --ibbackup=$XB_BIN"
 
-    export MYSQL_VERSION/ MYSQL_VERSION_COMMENT INNODB_VERSION XTRADB_VERSION XB_BIN
+    export MYSQL_VERSION MYSQL_VERSION_COMMENT INNODB_VERSION XTRADB_VERSION \
+	XB_BIN IB_BIN
 }
 
 tname=""
@@ -163,15 +177,32 @@
 total_count=0
 
 export OUTFILE="$PWD/results/setup"
-get_version_info >$OUTFILE 2>&1
-kill_leftovers >$OUTFILE 2>&1
-clean >$OUTFILE 2>&1
+
+if ! get_version_info
+then
+    echo "get_version_info failed. See $OUTFILE for details."
+    exit -1
+fi
+
+if [ -n "$XTRADB_VERSION" ]
+then
+    echo "Running against Percona Server $MYSQL_VERSION (XtraDB $INNODB_VERSION)" | tee -a $OUTFILE
+else
+    echo "Running against MySQL $MYSQL_VERSION (InnoDB $INNODB_VERSION)" | tee -a $OUTFILE
+fi
+echo "Using '`basename $XB_BIN`' as xtrabackup binary" | tee -a $OUTFILE
+echo | tee -a $OUTFILE
+
+kill_leftovers >>$OUTFILE 2>&1
+clean >>$OUTFILE 2>&1
 
 source subunit.sh
 
 SUBUNIT_OUT=test_results.subunit
 rm -f $SUBUNIT_OUT
 
+echo "========================================================================"
+
 for t in $tests
 do
    printf "%-40s" $t
@@ -212,6 +243,9 @@
    total_count=$((total_count+1))
 done
 
+echo "========================================================================"
+echo
+
 kill_leftovers
 
 if [ $result -eq 1 ]

=== added file 'test/t/bug766607.sh'
--- test/t/bug766607.sh	1970-01-01 00:00:00 +0000
+++ test/t/bug766607.sh	2011-06-20 18:06:00 +0000
@@ -0,0 +1,51 @@
+. inc/common.sh
+
+init
+run_mysqld --innodb_file_per_table
+load_dbase_schema incremental_sample
+
+# Full backup dir
+mkdir -p $topdir/data/full
+# Incremental backup dir
+mkdir -p $topdir/data/delta
+
+xtrabackup --datadir=$mysql_datadir --backup --target-dir=$topdir/data/full
+
+vlog "Creating and filling a new table"
+run_cmd $MYSQL $MYSQL_ARGS test <<EOF
+CREATE TABLE t(a int) ENGINE=InnoDB;
+INSERT INTO t VALUES (1), (2), (3);
+FLUSH LOGS;
+EOF
+
+stop_mysqld
+run_mysqld --innodb_file_per_table
+
+vlog "Making incremental backup"
+xtrabackup --datadir=$mysql_datadir --backup --target-dir=$topdir/data/delta --incremental-basedir=$topdir/data/full
+
+vlog "Preparing full backup"
+xtrabackup --datadir=$mysql_datadir --prepare --apply-log-only \
+    --target-dir=$topdir/data/full
+
+# The following would fail before the bugfix
+vlog "Applying incremental delta"
+xtrabackup --datadir=$mysql_datadir --prepare --apply-log-only \
+    --target-dir=$topdir/data/full --incremental-dir=$topdir/data/delta
+
+vlog "Preparing full backup"
+xtrabackup --datadir=$mysql_datadir --prepare --target-dir=$topdir/data/full
+vlog "Data prepared for restore"
+
+stop_mysqld
+
+vlog "Copying files"
+
+cd $topdir/data/full/
+cp -r * $mysql_datadir
+cd $topdir
+
+vlog "Data restored"
+run_mysqld --innodb_file_per_table
+
+run_cmd $MYSQL $MYSQL_ARGS -e "SELECT * FROM t" test

=== added file 'test/t/xb_defaults_file.sh'
--- test/t/xb_defaults_file.sh	1970-01-01 00:00:00 +0000
+++ test/t/xb_defaults_file.sh	2011-06-20 18:06:00 +0000
@@ -0,0 +1,25 @@
+#
+# Bug #798488: xtrabackup ignores defaults-file in apply-log-only prepare mode 
+#
+# Test xtrabackup fails with an error when --defaults-file is not the first argument
+# on the command line (and thus would be ignored).
+
+. inc/common.sh
+
+init
+run_mysqld
+
+# The following should succeed (can't use xtrabackup directly as it uses
+# --no-defaults)
+run_cmd $XB_BIN --defaults-file=$topdir/my.cnf --backup \
+    --datadir=$mysql_datadir --target-dir=$topdir/backup
+
+rm -rf $topdir/backup
+
+# The following should fail (can't use xtrabackup directly as that would make
+# the test fail)
+
+run_cmd_expect_failure $XB_BIN --backup --defaults-file=$topdir/my.cnf \
+    --datadir=$mysql_datadir --target-dir=$topdir/backup
+
+exit 0

=== modified file 'test/t/xb_stats.sh'
--- test/t/xb_stats.sh	2011-06-10 07:55:26 +0000
+++ test/t/xb_stats.sh	2011-06-20 18:06:00 +0000
@@ -15,11 +15,8 @@
 # when trying to get stats before creating the log files
 
 vlog "===> xtrabackup --stats --datadir=$topdir/backup"
-# Cannot use run_cmd here
-if $XB_BIN $XB_ARGS --stats --datadir=$topdir/backup >$OUTFILE 2>&1
-then
-    die "xtrabackup --stats was expected to fail, but it did not."
-fi
+run_cmd_expect_failure $XB_BIN $XB_ARGS --stats --datadir=$topdir/backup
+
 if ! grep "Cannot find log file ib_logfile0" $OUTFILE
 then
     die "Cannot find the expected error message from xtrabackup --stats"

=== modified file 'xtrabackup.c'
--- xtrabackup.c	2011-06-15 07:04:42 +0000
+++ xtrabackup.c	2011-06-20 18:06:00 +0000
@@ -5058,7 +5058,11 @@
 	return TRUE;
 }
 
-static void
+/************************************************************************
+Applies a given .delta file to the corresponding data file.
+@return TRUE on success */
+static
+ibool
 xtrabackup_apply_delta(
 	const char*	dirname,	/* in: dir name of incremental */
 	const char*	dbname,		/* in: database name (ibdata: NULL) */
@@ -5141,8 +5145,33 @@
 	dst_file = xb_file_create_no_error_handling(dst_path, OS_FILE_OPEN,
 						    OS_FILE_READ_WRITE,
 						    &success);
+again:
 	if (!success) {
-		os_file_get_last_error(TRUE);
+		ulint errcode = os_file_get_last_error(TRUE);
+
+		if (errcode == OS_FILE_NOT_FOUND) {
+			fprintf(stderr, "xtrabackup: target data file %s "
+				"is not found, creating a new one\n", dst_path);
+			/* Create the database directory if it doesn't exist yet
+			*/
+			if (dbname) {
+				char	dst_dir[FN_REFLEN];
+
+				snprintf(dst_dir, sizeof(dst_dir), "%s/%s",
+					 xtrabackup_real_target_dir, dbname);
+				srv_normalize_path_for_win(dst_dir);
+
+				if (!os_file_create_directory(dst_dir, FALSE))
+					goto error;
+			}
+			dst_file =
+				xb_file_create_no_error_handling(dst_path,
+								 OS_FILE_CREATE,
+								 OS_FILE_READ_WRITE,
+								 &success);
+			goto again;
+		}
+
 		fprintf(stderr,
 			"xtrabackup: error: cannot open %s\n",
 			dst_path);
@@ -5155,7 +5184,7 @@
 
 	xb_file_set_nocache(dst_file, dst_path, "OPEN");
 
-	printf("Applying %s ...\n", src_path);
+	fprintf(stderr, "Applying %s ...\n", src_path);
 
 	while (!last_buffer) {
 		ulint cluster_header;
@@ -5241,18 +5270,23 @@
 		os_file_close(src_file);
 	if (dst_file != XB_FILE_UNDEFINED)
 		os_file_close(dst_file);
-	return;
+	return TRUE;
 
 error:
 	if (src_file != XB_FILE_UNDEFINED)
 		os_file_close(src_file);
 	if (dst_file != XB_FILE_UNDEFINED)
 		os_file_close(dst_file);
-	fprintf(stderr, "xtrabackup: Error: xtrabackup_apply_delta() failed.\n");
-	return;
+	fprintf(stderr, "xtrabackup: Error: xtrabackup_apply_delta(): "
+		"failed to apply %s to %s.\n", src_path, dst_path);
+	return FALSE;
 }
 
-static void
+/************************************************************************
+Applies all .delta files from incremental_dir to the full backup.
+@return TRUE on success. */
+static
+ibool
 xtrabackup_apply_deltas(my_bool check_newer)
 {
 	int		ret;
@@ -5283,9 +5317,12 @@
 			    && 0 == strcmp(fileinfo.name + 
 					strlen(fileinfo.name) - 6,
 					".delta")) {
-				xtrabackup_apply_delta(
-					xtrabackup_incremental_dir, NULL,
-					fileinfo.name, check_newer);
+				if (!xtrabackup_apply_delta(
+					    xtrabackup_incremental_dir, NULL,
+					    fileinfo.name, check_newer))
+				{
+					return FALSE;
+				}
 			}
 next_file_item_1:
 			ret = fil_file_readdir_next_file(&err,
@@ -5337,9 +5374,13 @@
 						".delta")) {
 					/* The name ends in .delta; try opening
 					the file */
-					xtrabackup_apply_delta(
-						xtrabackup_incremental_dir, dbinfo.name,
-						fileinfo.name, check_newer);
+					if (!xtrabackup_apply_delta(
+						    xtrabackup_incremental_dir,
+						    dbinfo.name,
+						    fileinfo.name, check_newer))
+					{
+						return FALSE;
+					}
 				}
 next_file_item_2:
 				ret = fil_file_readdir_next_file(&err,
@@ -5357,6 +5398,7 @@
 
 	os_file_closedir(dir);
 
+	return TRUE;
 }
 
 static my_bool
@@ -5508,8 +5550,8 @@
 	if(xtrabackup_init_temp_log())
 		goto error;
 
-	if(xtrabackup_incremental)
-		xtrabackup_apply_deltas(TRUE);
+	if(xtrabackup_incremental && !xtrabackup_apply_deltas(TRUE))
+		goto error;
 
 	sync_close();
 	sync_initialized = FALSE;
@@ -5935,6 +5977,12 @@
 		uint count;
 		struct my_option *opt= (struct my_option *) my_long_options;
 		optend= strcend((argv)[i], '=');
+		if (!strncmp(argv[i], "--defaults-file", optend - argv[i]))
+		{
+			fprintf(stderr, "xtrabackup: Error: --defaults-file "
+				"be specified first on the command line\n");
+			exit(EXIT_FAILURE);
+		}
 		for (count= 0; opt->name; opt++) {
 			if (!getopt_compare_strings(opt->name, (argv)[i] + 2,
 				(uint)(optend - (argv)[i] - 2))) /* match found */
@@ -6201,10 +6249,12 @@
 		print_version();
 		if (xtrabackup_incremental) {
 #ifndef INNODB_VERSION_SHORT
-			printf("incremental backup from %lu:%lu is enabled.\n",
+			fprintf(stderr,
+				"incremental backup from %lu:%lu is enabled.\n",
 				incremental_lsn.high, incremental_lsn.low);
 #else
-			printf("incremental backup from %llu is enabled.\n",
+			fprintf(stderr,
+				"incremental backup from %llu is enabled.\n",
 				incremental_lsn);
 #endif
 		}