Merge lp:~percona-dev/percona-xtrabackup/lp713799 into lp:percona-xtrabackup/2.0

[Alexey Kopytov]

[Valentine Gostev (longbow)]

Passes test case. Current behavior - detetects dropped table and mentions it in the output.

[Alexey Kopytov (akopytov)]

Does the test case fail on trunk?

[Alexey Kopytov (akopytov)]

<Valentine> on 1.5 I got Assertion failure described in bug report

[Yasufumi Kinoshita (yasufumi-kinoshita)]

review: Approve

=== modified file 'fix_innodb_for_backup55.patch'
--- fix_innodb_for_backup55.patch	2010-12-15 21:44:42 +0000
+++ fix_innodb_for_backup55.patch	2011-02-06 13:37:00 +0000
@@ -157,7 +157,41 @@
  
  
  /********************************************************************//**
-@@ -758,8 +760,8 @@
+@@ -308,7 +310,7 @@
+ off the LRU list if it is in the LRU list. The caller must hold the fil_sys
+ mutex. */
+ static
+-void
++ulint
+ fil_node_prepare_for_io(
+ /*====================*/
+ 	fil_node_t*	node,	/*!< in: file node */
+@@ -633,7 +635,7 @@
+ Opens a the file of a node of a tablespace. The caller must own the fil_system
+ mutex. */
+ static
+-void
++ulint
+ fil_node_open_file(
+ /*===============*/
+ 	fil_node_t*	node,	/*!< in: file node */
+@@ -668,7 +670,14 @@
+ 			OS_FILE_READ_ONLY, &success);
+ 		if (!success) {
+ 			/* The following call prints an error message */
+-			os_file_get_last_error(TRUE);
++			if (os_file_get_last_error(TRUE) == OS_FILE_NOT_FOUND)
++			{
++				ut_print_timestamp(stderr);
++				fprintf(stderr,
++					" InnoDB: Warning: cannot open %s\n",
++					node->name);
++				return(OS_FILE_NOT_FOUND);
++			}
+ 
+ 			ut_print_timestamp(stderr);
+ 
+@@ -758,8 +767,8 @@
  		}
  
  		if (size_bytes >= 1024 * 1024) {
@@ -168,7 +202,44 @@
  		}
  
  		if (!(flags & DICT_TF_ZSSIZE_MASK)) {
-@@ -2152,7 +2154,7 @@
+@@ -810,6 +819,8 @@
+ 		/* Put the node to the LRU list */
+ 		UT_LIST_ADD_FIRST(LRU, system->LRU, node);
+ 	}
++
++	return(0);
+ }
+ 
+ /**********************************************************************//**
+@@ -1435,7 +1446,12 @@
+ 		the file yet; the following calls will open it and update the
+ 		size fields */
+ 
+-		fil_node_prepare_for_io(node, fil_system, space);
++		if (fil_node_prepare_for_io(node, fil_system, space))
++		{
++			mutex_exit(&fil_system->mutex);
++
++			return(0);
++		}
+ 		fil_node_complete_io(node, fil_system, OS_FILE_READ);
+ 	}
+ 
+@@ -1487,7 +1503,12 @@
+ 		the file yet; the following calls will open it and update the
+ 		size fields */
+ 
+-		fil_node_prepare_for_io(node, fil_system, space);
++		if (fil_node_prepare_for_io(node, fil_system, space))
++		{
++			mutex_exit(&fil_system->mutex);
++
++			return(ULINT_UNDEFINED);
++		}
+ 		fil_node_complete_io(node, fil_system, OS_FILE_READ);
+ 	}
+ 
+@@ -2152,7 +2173,7 @@
  			if (fil_create_new_single_table_tablespace(
  				    space_id, name, FALSE, flags,
  				    FIL_IBD_FILE_INITIAL_SIZE) != DB_SUCCESS) {
@@ -177,7 +248,7 @@
  			}
  		}
  
-@@ -2315,7 +2317,7 @@
+@@ -2315,7 +2336,7 @@
  	}
  
  	if (success) {
@@ -186,7 +257,7 @@
  		/* Write a log record about the deletion of the .ibd
  		file, so that ibbackup can replay it in the
  		--apply-log phase. We use a dummy mtr and the familiar
-@@ -2616,7 +2618,7 @@
+@@ -2616,7 +2637,7 @@
  
  	mutex_exit(&fil_system->mutex);
  
@@ -195,7 +266,7 @@
  	if (success) {
  		mtr_t		mtr;
  
-@@ -2807,7 +2809,7 @@
+@@ -2807,7 +2828,7 @@
  
  	fil_node_create(path, size, space_id, FALSE);
  
@@ -204,7 +275,7 @@
  	{
  		mtr_t		mtr;
  
-@@ -3062,19 +3064,97 @@
+@@ -3062,19 +3083,97 @@
  		      "InnoDB: open the tablespace file ", stderr);
  		ut_print_filename(stderr, filepath);
  		fputs("!\n"
@@ -312,7 +383,7 @@
  		return(FALSE);
  	}
  
-@@ -3306,7 +3386,7 @@
+@@ -3306,7 +3405,7 @@
  	cannot be ok. */
  
  	size = (((ib_int64_t)size_high) << 32) + (ib_int64_t)size_low;
@@ -321,7 +392,7 @@
  	if (size < FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE) {
  		fprintf(stderr,
  			"InnoDB: Error: the size of single-table tablespace"
-@@ -3447,7 +3527,7 @@
+@@ -3447,7 +3546,7 @@
  idea is to read as much good data as we can and jump over bad data.
  @return 0 if ok, -1 if error even after the retries, 1 if at the end
  of the directory */
@@ -330,7 +401,7 @@
  int
  fil_file_readdir_next_file(
  /*=======================*/
-@@ -3746,15 +3826,97 @@
+@@ -3746,15 +3845,97 @@
  				"InnoDB: in InnoDB data dictionary"
  				" has tablespace id %lu,\n"
  				"InnoDB: but tablespace with that id"
@@ -436,7 +507,40 @@
  		} else {
  			ut_print_timestamp(stderr);
  			fputs("  InnoDB: Error: table ", stderr);
-@@ -4363,6 +4525,16 @@
+@@ -4143,7 +4324,7 @@
+ off the LRU list if it is in the LRU list. The caller must hold the fil_sys
+ mutex. */
+ static
+-void
++ulint
+ fil_node_prepare_for_io(
+ /*====================*/
+ 	fil_node_t*	node,	/*!< in: file node */
+@@ -4163,10 +4344,13 @@
+ 	}
+ 
+ 	if (node->open == FALSE) {
++		ulint	err;
+ 		/* File is closed: open it */
+ 		ut_a(node->n_pending == 0);
+ 
+-		fil_node_open_file(node, system, space);
++		err = fil_node_open_file(node, system, space);
++		if (err)
++			return(err);
+ 	}
+ 
+ 	if (node->n_pending == 0 && space->purpose == FIL_TABLESPACE
+@@ -4179,6 +4363,8 @@
+ 	}
+ 
+ 	node->n_pending++;
++
++	return(0);
+ }
+ 
+ /********************************************************************//**
+@@ -4363,6 +4549,16 @@
  
  	ut_ad((mode != OS_AIO_IBUF) || (space->purpose == FIL_TABLESPACE));
  

=== modified file 'fix_innodb_for_backup_5.1_plugin.patch'
--- fix_innodb_for_backup_5.1_plugin.patch	2010-12-15 21:44:42 +0000
+++ fix_innodb_for_backup_5.1_plugin.patch	2011-02-06 13:37:00 +0000
@@ -156,7 +156,41 @@
  
  
  /********************************************************************//**
-@@ -746,8 +747,8 @@
+@@ -298,7 +299,7 @@
+ off the LRU list if it is in the LRU list. The caller must hold the fil_sys
+ mutex. */
+ static
+-void
++ulint
+ fil_node_prepare_for_io(
+ /*====================*/
+ 	fil_node_t*	node,	/*!< in: file node */
+@@ -622,7 +623,7 @@
+ Opens a the file of a node of a tablespace. The caller must own the fil_system
+ mutex. */
+ static
+-void
++ulint
+ fil_node_open_file(
+ /*===============*/
+ 	fil_node_t*	node,	/*!< in: file node */
+@@ -656,7 +657,14 @@
+ 			node->name, OS_FILE_OPEN, OS_FILE_READ_ONLY, &success);
+ 		if (!success) {
+ 			/* The following call prints an error message */
+-			os_file_get_last_error(TRUE);
++			if (os_file_get_last_error(TRUE) == OS_FILE_NOT_FOUND)
++			{
++				ut_print_timestamp(stderr);
++				fprintf(stderr,
++					" InnoDB: Warning: cannot open %s\n",
++					node->name);
++				return(OS_FILE_NOT_FOUND);
++			}
+ 
+ 			ut_print_timestamp(stderr);
+ 
+@@ -746,8 +754,8 @@
  		}
  
  		if (size_bytes >= 1024 * 1024) {
@@ -167,7 +201,45 @@
  		}
  
  		if (!(flags & DICT_TF_ZSSIZE_MASK)) {
-@@ -2132,7 +2133,7 @@
+@@ -792,6 +800,8 @@
+ 		/* Put the node to the LRU list */
+ 		UT_LIST_ADD_FIRST(LRU, system->LRU, node);
+ 	}
++
++	return(0);
+ }
+ 
+ /**********************************************************************//**
+@@ -1418,7 +1428,13 @@
+ 		the file yet; the following calls will open it and update the
+ 		size fields */
+ 
+-		fil_node_prepare_for_io(node, fil_system, space);
++		if (fil_node_prepare_for_io(node, fil_system, space))
++		{
++			mutex_exit(&fil_system->mutex);
++
++			return(0);
++			
++		}
+ 		fil_node_complete_io(node, fil_system, OS_FILE_READ);
+ 	}
+ 
+@@ -1470,7 +1486,12 @@
+ 		the file yet; the following calls will open it and update the
+ 		size fields */
+ 
+-		fil_node_prepare_for_io(node, fil_system, space);
++		if (fil_node_prepare_for_io(node, fil_system, space))
++		{
++			mutex_exit(&fil_system->mutex);
++
++			return(ULINT_UNDEFINED);
++		}
+ 		fil_node_complete_io(node, fil_system, OS_FILE_READ);
+ 	}
+ 
+@@ -2132,7 +2153,7 @@
  			if (fil_create_new_single_table_tablespace(
  				    space_id, name, FALSE, flags,
  				    FIL_IBD_FILE_INITIAL_SIZE) != DB_SUCCESS) {
@@ -176,7 +248,7 @@
  			}
  		}
  
-@@ -2276,7 +2277,7 @@
+@@ -2276,7 +2297,7 @@
  	}
  
  	if (success) {
@@ -185,7 +257,7 @@
  		/* Write a log record about the deletion of the .ibd
  		file, so that ibbackup can replay it in the
  		--apply-log phase. We use a dummy mtr and the familiar
-@@ -2552,7 +2553,7 @@
+@@ -2552,7 +2573,7 @@
  
  	mutex_exit(&fil_system->mutex);
  
@@ -194,7 +266,7 @@
  	if (success) {
  		mtr_t		mtr;
  
-@@ -2742,7 +2743,7 @@
+@@ -2742,7 +2763,7 @@
  
  	fil_node_create(path, size, space_id, FALSE);
  
@@ -203,7 +275,7 @@
  	{
  		mtr_t		mtr;
  
-@@ -2995,19 +2996,97 @@
+@@ -2995,19 +3016,97 @@
  		      "InnoDB: open the tablespace file ", stderr);
  		ut_print_filename(stderr, filepath);
  		fputs("!\n"
@@ -311,7 +383,7 @@
  		return(FALSE);
  	}
  
-@@ -3238,7 +3317,7 @@
+@@ -3238,7 +3337,7 @@
  	cannot be ok. */
  
  	size = (((ib_int64_t)size_high) << 32) + (ib_int64_t)size_low;
@@ -320,7 +392,7 @@
  	if (size < FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE) {
  		fprintf(stderr,
  			"InnoDB: Error: the size of single-table tablespace"
-@@ -3379,7 +3458,7 @@
+@@ -3379,7 +3478,7 @@
  idea is to read as much good data as we can and jump over bad data.
  @return 0 if ok, -1 if error even after the retries, 1 if at the end
  of the directory */
@@ -329,7 +401,7 @@
  int
  fil_file_readdir_next_file(
  /*=======================*/
-@@ -3678,15 +3757,97 @@
+@@ -3678,15 +3777,97 @@
  				"InnoDB: in InnoDB data dictionary"
  				" has tablespace id %lu,\n"
  				"InnoDB: but tablespace with that id"
@@ -435,7 +507,40 @@
  		} else {
  			ut_print_timestamp(stderr);
  			fputs("  InnoDB: Error: table ", stderr);
-@@ -4295,6 +4456,16 @@
+@@ -4075,7 +4256,7 @@
+ off the LRU list if it is in the LRU list. The caller must hold the fil_sys
+ mutex. */
+ static
+-void
++ulint
+ fil_node_prepare_for_io(
+ /*====================*/
+ 	fil_node_t*	node,	/*!< in: file node */
+@@ -4095,10 +4276,13 @@
+ 	}
+ 
+ 	if (node->open == FALSE) {
++		ulint	err;
+ 		/* File is closed: open it */
+ 		ut_a(node->n_pending == 0);
+ 
+-		fil_node_open_file(node, system, space);
++		err = fil_node_open_file(node, system, space);
++		if (err)
++			return(err);
+ 	}
+ 
+ 	if (node->n_pending == 0 && space->purpose == FIL_TABLESPACE
+@@ -4111,6 +4295,8 @@
+ 	}
+ 
+ 	node->n_pending++;
++
++	return(0);
+ }
+ 
+ /********************************************************************//**
+@@ -4295,6 +4481,16 @@
  
  	ut_ad((mode != OS_AIO_IBUF) || (space->purpose == FIL_TABLESPACE));
  

=== modified file 'fix_innodb_for_backup_percona-server.patch'
--- fix_innodb_for_backup_percona-server.patch	2010-12-01 22:03:06 +0000
+++ fix_innodb_for_backup_percona-server.patch	2011-02-06 13:37:00 +0000
@@ -49,7 +49,41 @@
  
  
  /********************************************************************//**
-@@ -753,8 +753,8 @@
+@@ -305,7 +305,7 @@
+ off the LRU list if it is in the LRU list. The caller must hold the fil_sys
+ mutex. */
+ static
+-void
++ulint
+ fil_node_prepare_for_io(
+ /*====================*/
+ 	fil_node_t*	node,	/*!< in: file node */
+@@ -629,7 +629,7 @@
+ Opens a the file of a node of a tablespace. The caller must own the fil_system
+ mutex. */
+ static
+-void
++ulint
+ fil_node_open_file(
+ /*===============*/
+ 	fil_node_t*	node,	/*!< in: file node */
+@@ -663,7 +663,14 @@
+ 			node->name, OS_FILE_OPEN, OS_FILE_READ_ONLY, &success);
+ 		if (!success) {
+ 			/* The following call prints an error message */
+-			os_file_get_last_error(TRUE);
++			if (os_file_get_last_error(TRUE) == OS_FILE_NOT_FOUND)
++			{
++				ut_print_timestamp(stderr);
++				fprintf(stderr,
++					" InnoDB: Warning: cannot open %s\n",
++					node->name);
++				return(OS_FILE_NOT_FOUND);
++			}
+ 
+ 			ut_print_timestamp(stderr);
+ 
+@@ -753,8 +760,8 @@
  		}
  
  		if (size_bytes >= 1024 * 1024) {
@@ -60,7 +94,44 @@
  		}
  
  		if (!(flags & DICT_TF_ZSSIZE_MASK)) {
-@@ -2143,7 +2143,7 @@
+@@ -799,6 +806,8 @@
+ 		/* Put the node to the LRU list */
+ 		UT_LIST_ADD_FIRST(LRU, system->LRU, node);
+ 	}
++
++	return(0);
+ }
+ 
+ /**********************************************************************//**
+@@ -1428,7 +1437,12 @@
+ 		the file yet; the following calls will open it and update the
+ 		size fields */
+ 
+-		fil_node_prepare_for_io(node, fil_system, space);
++		if (fil_node_prepare_for_io(node, fil_system, space))
++		{
++			mutex_exit(&fil_system->mutex);
++
++			return(0);
++		}
+ 		fil_node_complete_io(node, fil_system, OS_FILE_READ);
+ 	}
+ 
+@@ -1480,7 +1494,12 @@
+ 		the file yet; the following calls will open it and update the
+ 		size fields */
+ 
+-		fil_node_prepare_for_io(node, fil_system, space);
++		if (fil_node_prepare_for_io(node, fil_system, space))
++		{
++			mutex_exit(&fil_system->mutex);
++
++			return(ULINT_UNDEFINED);
++		}
+ 		fil_node_complete_io(node, fil_system, OS_FILE_READ);
+ 	}
+ 
+@@ -2151,7 +2170,7 @@
  			if (fil_create_new_single_table_tablespace(
  				    space_id, name, FALSE, flags,
  				    FIL_IBD_FILE_INITIAL_SIZE) != DB_SUCCESS) {
@@ -69,7 +140,7 @@
  			}
  		}
  
-@@ -2287,7 +2287,7 @@
+@@ -2295,7 +2314,7 @@
  	}
  
  	if (success) {
@@ -78,7 +149,7 @@
  		/* Write a log record about the deletion of the .ibd
  		file, so that ibbackup can replay it in the
  		--apply-log phase. We use a dummy mtr and the familiar
-@@ -2563,7 +2563,7 @@
+@@ -2571,7 +2590,7 @@
  
  	mutex_exit(&fil_system->mutex);
  
@@ -87,7 +158,7 @@
  	if (success) {
  		mtr_t		mtr;
  
-@@ -2753,7 +2753,7 @@
+@@ -2761,7 +2780,7 @@
  
  	fil_node_create(path, size, space_id, FALSE);
  
@@ -96,7 +167,7 @@
  	{
  		mtr_t		mtr;
  
-@@ -3006,19 +3006,97 @@
+@@ -3014,19 +3033,97 @@
  		      "InnoDB: open the tablespace file ", stderr);
  		ut_print_filename(stderr, filepath);
  		fputs("!\n"
@@ -204,7 +275,7 @@
  		return(FALSE);
  	}
  
-@@ -3723,7 +3801,7 @@
+@@ -3731,7 +3828,7 @@
  	cannot be ok. */
  
  	size = (((ib_int64_t)size_high) << 32) + (ib_int64_t)size_low;
@@ -213,7 +284,7 @@
  	if (size < FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE) {
  		fprintf(stderr,
  			"InnoDB: Error: the size of single-table tablespace"
-@@ -3864,7 +3942,7 @@
+@@ -3872,7 +3969,7 @@
  idea is to read as much good data as we can and jump over bad data.
  @return 0 if ok, -1 if error even after the retries, 1 if at the end
  of the directory */
@@ -222,7 +293,7 @@
  int
  fil_file_readdir_next_file(
  /*=======================*/
-@@ -4163,15 +4241,97 @@
+@@ -4171,15 +4268,97 @@
  				"InnoDB: in InnoDB data dictionary"
  				" has tablespace id %lu,\n"
  				"InnoDB: but tablespace with that id"
@@ -328,7 +399,31 @@
  		} else {
  			ut_print_timestamp(stderr);
  			fputs("  InnoDB: Error: table ", stderr);
-@@ -4781,6 +4941,16 @@
+@@ -4568,7 +4747,7 @@
+ off the LRU list if it is in the LRU list. The caller must hold the fil_sys
+ mutex. */
+ static
+-void
++ulint
+ fil_node_prepare_for_io(
+ /*====================*/
+ 	fil_node_t*	node,	/*!< in: file node */
+@@ -4588,10 +4767,13 @@
+ 	}
+ 
+ 	if (node->open == FALSE) {
++		ulint err;
+ 		/* File is closed: open it */
+ 		ut_a(node->n_pending == 0);
+ 
+-		fil_node_open_file(node, system, space);
++		err = fil_node_open_file(node, system, space);
++		if (err)
++			return(err);
+ 	}
+ 
+ 	if (node->n_pending == 0 && space->purpose == FIL_TABLESPACE
+@@ -4789,6 +4971,16 @@
  
  	ut_ad((mode != OS_AIO_IBUF) || (space->purpose == FIL_TABLESPACE));
  

=== modified file 'xtrabackup.c'
--- xtrabackup.c	2010-12-30 16:54:43 +0000
+++ xtrabackup.c	2011-02-06 13:37:00 +0000
@@ -2310,7 +2310,6 @@
 	LSN64		flush_lsn;
 	IB_INT64	file_size;
 	IB_INT64	offset;
-	ibool		src_exist = TRUE;
 	ulint		page_in_buffer;
 	ulint		incremental_buffers = 0;
 	ulint		page_size;
@@ -2423,14 +2422,21 @@
 	page_size_shift = UNIV_PAGE_SIZE_SHIFT;
 #else
 	zip_size = fil_space_get_zip_size(node->space->id);
-	if (zip_size) {
+	if (zip_size == ULINT_UNDEFINED) {
+		fprintf(stderr, "[%02u] xtrabackup: Warning: "
+			"Failed to determine page size for %s.\n"
+			"[%02u] xtrabackup: Warning: We assume the table was "
+			"dropped during xtrabackup execution and ignore the "
+			"file.\n", thread_n, node->name, thread_n);
+		goto skip;
+	} else if (zip_size) {
 		page_size = zip_size;
 		page_size_shift = get_page_size_shift(page_size);
 		fprintf(stderr, "[%02u] %s is compressed with page size = "
 			"%lu bytes\n", thread_n, node->name, page_size);
 		if (page_size_shift < 10 || page_size_shift > 14) {
 			fprintf(stderr, "[%02u] xtrabackup: Error: Invalid "
-				"page size.\n");
+				"page size.\n", thread_n);
 			ut_error;
 		}
 	} else {
@@ -2485,11 +2491,12 @@
 			os_file_get_last_error(TRUE);
 
 			fprintf(stderr,
-				"[%02u] xtrabackup: error: cannot open %s\n"
-				"[%02u] xtrabackup: Have you deleted .ibd"
-				"files under a running mysqld server?\n",
+				"[%02u] xtrabackup: Warning: cannot open %s\n"
+				"[%02u] xtrabackup: Warning: We assume the "
+				"table was dropped during xtrabackup execution "
+				"and ignore the file.\n",
 				thread_n, node->name, thread_n);
-			src_exist = FALSE;
+			goto skip;
 		}
 
 		if (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT) {
@@ -2525,9 +2532,6 @@
 	posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED);
 #endif
 
-	if(!src_exist)
-		goto error;
-
 	/* copy : TODO: tune later */
 	printf("[%02u] Copying %s \n     to %s\n", thread_n,
 	       node->name, dst_path);
@@ -2735,6 +2739,17 @@
 	fprintf(stderr, "[%02u] xtrabackup: Error: "
 		"xtrabackup_copy_datafile() failed.\n", thread_n);
 	return(TRUE); /*ERROR*/
+
+skip:
+	if (src_file != -1 && !node->open)
+		os_file_close(src_file);
+	if (dst_file != -1)
+		os_file_close(dst_file);
+	if (buf2)
+		ut_free(buf2);
+	fprintf(stderr, "[%02u] xtrabackup: Warning: skipping file %s.\n",
+		thread_n, node->name);
+	return(FALSE);
 }
 
 my_bool