Merge lp:~sergei.glushchenko/percona-xtrabackup/transportabe-tablespaces into lp:percona-xtrabackup/2.0

[Sergei Glushchenko]

  Blueprint: Support transportable tablespaces in xtrabackup --export
  https://blueprints.launchpad.net/percona-xtrabackup/+spec/basic-56-support
  MySQL 5.6 has a feature which allows user to export InnoDB tablespaces and
  import them later. This feature is very similar to XtraDB feature.
  Xtrabackup has an option to export single table tablespace from backup.
  It dump table metadata in format which InnoDB can recognize (.exp files).
  But MySQL has it's own format of metadata stored in .cfg files.
  This patch allows Xtrabackup to produce metadata in both formats when
  export is performed. The code to support cfg export is copied from
  MySQL sources with some modifications.
  Existing testcase xb_export.sh has been modified to test export
  on MySQL 5.6 as well. It however works even when .cfg is not generated
  because bug http://bugs.mysql.com/bug.php?id=66715 is fixed in
  MySQL 5.6.8. It fails however in case of incorrect or corrupted .cfg
  file.

Will post Jenkins link later

[Sergei Glushchenko (sergei.glushchenko)]

http://jenkins.percona.com/job/percona-xtrabackup-2.0-param/424/

Not finished yet, but looks promising.

[Alexey Kopytov (akopytov)]

review: Approve

=== modified file 'src/xtrabackup.cc'
--- src/xtrabackup.cc	2013-04-24 14:09:42 +0000
+++ src/xtrabackup.cc	2013-04-26 09:10:34 +0000
@@ -292,6 +292,23 @@
 #define SRV_PATH_SEPARATOR_STR	"/"
 #endif
 
+/** The version number of the export meta-data text file. */
+#define IB_EXPORT_CFG_VERSION_V1	0x1UL
+
+/* Maximum multi-byte character length in bytes, plus 1 */
+#define DATA_MBMAX	5
+
+/* Pack mbminlen, mbmaxlen to mbminmaxlen. */
+#define DATA_MBMINMAXLEN(mbminlen, mbmaxlen)	\
+	((mbmaxlen) * DATA_MBMAX + (mbminlen))
+/* Get mbminlen from mbminmaxlen. Cast the result of UNIV_EXPECT to ulint
+because in GCC it returns a long. */
+#define DATA_MBMINLEN(mbminmaxlen) ((ulint) \
+                                    UNIV_EXPECT(((mbminmaxlen) % DATA_MBMAX), \
+                                                1))
+/* Get mbmaxlen from mbminmaxlen. */
+#define DATA_MBMAXLEN(mbminmaxlen) ((ulint) ((mbminmaxlen) / DATA_MBMAX))
+
 #ifndef UNIV_PAGE_SIZE_MAX
 #define UNIV_PAGE_SIZE_MAX UNIV_PAGE_SIZE
 #endif
@@ -7336,6 +7353,370 @@
 	return(TRUE); /*ERROR*/
 }
 
+
+/*********************************************************************//**
+Write the meta data (index user fields) config file.
+@return true in case of success otherwise false. */
+static
+bool
+xb_export_cfg_write_index_fields(
+/*===========================*/
+	const dict_index_t*	index,	/*!< in: write the meta data for
+					this index */
+	FILE*			file)	/*!< in: file to write to */
+{
+	byte			row[sizeof(ib_uint32_t) * 2];
+
+	for (ulint i = 0; i < index->n_fields; ++i) {
+		byte*			ptr = row;
+		const dict_field_t*	field = &index->fields[i];
+
+		mach_write_to_4(ptr, field->prefix_len);
+		ptr += sizeof(ib_uint32_t);
+
+		mach_write_to_4(ptr, field->fixed_len);
+
+		if (fwrite(row, 1, sizeof(row), file) != sizeof(row)) {
+
+			msg("xtrabackup: Error: writing index fields.");
+
+			return(false);
+		}
+
+		/* Include the NUL byte in the length. */
+		ib_uint32_t	len = strlen(field->name) + 1;
+		ut_a(len > 1);
+
+		mach_write_to_4(row, len);
+
+		if (fwrite(row, 1,  sizeof(len), file) != sizeof(len)
+		    || fwrite(field->name, 1, len, file) != len) {
+
+			msg("xtrabackup: Error: writing index column.");
+
+			return(false);
+		}
+	}
+
+	return(true);
+}
+
+/*********************************************************************//**
+Write the meta data config file index information.
+@return true in case of success otherwise false. */
+static	__attribute__((nonnull, warn_unused_result))
+bool
+xb_export_cfg_write_indexes(
+/*======================*/
+	const dict_table_t*	table,	/*!< in: write the meta data for
+					this table */
+	FILE*			file)	/*!< in: file to write to */
+{
+	{
+		byte		row[sizeof(ib_uint32_t)];
+
+		/* Write the number of indexes in the table. */
+		mach_write_to_4(row, UT_LIST_GET_LEN(table->indexes));
+
+		if (fwrite(row, 1, sizeof(row), file) != sizeof(row)) {
+			msg("xtrabackup: Error: writing index count.");
+
+			return(false);
+		}
+	}
+
+	bool			ret = true;
+
+	/* Write the index meta data. */
+	for (const dict_index_t* index = UT_LIST_GET_FIRST(table->indexes);
+	     index != 0 && ret;
+	     index = UT_LIST_GET_NEXT(indexes, index)) {
+
+		byte*		ptr;
+		byte		row[sizeof(IB_UINT64)
+				    + sizeof(ib_uint32_t) * 8];
+
+		ptr = row;
+
+		ut_ad(sizeof(IB_UINT64) == 8);
+		mach_write_to_8(ptr, index->id);
+		ptr += sizeof(IB_UINT64);
+
+		mach_write_to_4(ptr, index->space);
+		ptr += sizeof(ib_uint32_t);
+
+		mach_write_to_4(ptr, index->page);
+		ptr += sizeof(ib_uint32_t);
+
+		mach_write_to_4(ptr, index->type);
+		ptr += sizeof(ib_uint32_t);
+
+		mach_write_to_4(ptr, index->trx_id_offset);
+		ptr += sizeof(ib_uint32_t);
+
+		mach_write_to_4(ptr, index->n_user_defined_cols);
+		ptr += sizeof(ib_uint32_t);
+
+		mach_write_to_4(ptr, index->n_uniq);
+		ptr += sizeof(ib_uint32_t);
+
+		mach_write_to_4(ptr, index->n_nullable);
+		ptr += sizeof(ib_uint32_t);
+
+		mach_write_to_4(ptr, index->n_fields);
+
+		if (fwrite(row, 1, sizeof(row), file) != sizeof(row)) {
+
+			msg("xtrabackup: Error: writing index meta-data.");
+
+			return(false);
+		}
+
+		/* Write the length of the index name.
+		NUL byte is included in the length. */
+		ib_uint32_t	len = strlen(index->name) + 1;
+		ut_a(len > 1);
+
+		mach_write_to_4(row, len);
+
+		if (fwrite(row, 1, sizeof(len), file) != sizeof(len)
+		    || fwrite(index->name, 1, len, file) != len) {
+
+			msg("xtrabackup: Error: writing index name.");
+
+			return(false);
+		}
+
+		ret = xb_export_cfg_write_index_fields(index, file);
+	}
+
+	return(ret);
+}
+
+/*********************************************************************//**
+Write the meta data (table columns) config file. Serialise the contents of
+dict_col_t structure, along with the column name. All fields are serialized
+as ib_uint32_t.
+@return true in case of success otherwise false. */
+static	__attribute__((nonnull, warn_unused_result))
+bool
+xb_export_cfg_write_table(
+/*====================*/
+	const dict_table_t*	table,	/*!< in: write the meta data for
+					this table */
+	FILE*			file)	/*!< in: file to write to */
+{
+	dict_col_t*		col;
+	byte			row[sizeof(ib_uint32_t) * 7];
+#if (MYSQL_VERSION_ID < 50600)
+	ulint			minlen;
+	ulint			maxlen;
+#endif
+
+	col = table->cols;
+
+	for (ulint i = 0; i < table->n_cols; ++i, ++col) {
+		byte*		ptr = row;
+
+		mach_write_to_4(ptr, col->prtype);
+		ptr += sizeof(ib_uint32_t);
+
+		mach_write_to_4(ptr, col->mtype);
+		ptr += sizeof(ib_uint32_t);
+
+		mach_write_to_4(ptr, col->len);
+		ptr += sizeof(ib_uint32_t);
+
+#if (MYSQL_VERSION_ID >= 50600)
+		mach_write_to_4(ptr, col->mbminmaxlen);
+#else
+		innobase_get_cset_width(dtype_get_charset_coll(col->prtype),
+					&minlen, &maxlen);
+		mach_write_to_4(ptr, DATA_MBMINMAXLEN(minlen, maxlen));
+#endif
+		ptr += sizeof(ib_uint32_t);
+
+		mach_write_to_4(ptr, col->ind);
+		ptr += sizeof(ib_uint32_t);
+
+		mach_write_to_4(ptr, col->ord_part);
+		ptr += sizeof(ib_uint32_t);
+
+#if (MYSQL_VERSION_ID >= 50600)
+		mach_write_to_4(ptr, col->max_prefix);
+#else
+		mach_write_to_4(ptr, 0);
+#endif
+
+		if (fwrite(row, 1, sizeof(row), file) != sizeof(row)) {
+			msg("xtrabackup: Error: writing table column data.");
+
+			return(false);
+		}
+
+		/* Write out the column name as [len, byte array]. The len
+		includes the NUL byte. */
+		ib_uint32_t	len;
+		const char*	col_name;
+
+		col_name = dict_table_get_col_name(table, dict_col_get_no(col));
+
+		/* Include the NUL byte in the length. */
+		len = strlen(col_name) + 1;
+		ut_a(len > 1);
+
+		mach_write_to_4(row, len);
+
+		if (fwrite(row, 1,  sizeof(len), file) != sizeof(len)
+		    || fwrite(col_name, 1, len, file) != len) {
+
+			msg("xtrabackup: Error: writing column name.");
+
+			return(false);
+		}
+	}
+
+	return(true);
+}
+
+/*********************************************************************//**
+Write the meta data config file header.
+@return true in case of success otherwise false. */
+static	__attribute__((nonnull, warn_unused_result))
+bool
+xb_export_cfg_write_header(
+/*=====================*/
+	const dict_table_t*	table,	/*!< in: write the meta data for
+					this table */
+	FILE*			file)	/*!< in: file to write to */
+{
+	byte			value[sizeof(ib_uint32_t)];
+
+	/* Write the meta-data version number. */
+	mach_write_to_4(value, IB_EXPORT_CFG_VERSION_V1);
+
+	if (fwrite(&value, 1, sizeof(value), file) != sizeof(value)) {
+		msg("xtrabackup: Error: writing meta-data version number.");
+
+		return(false);
+	}
+
+	/* Write the server hostname. */
+	ib_uint32_t		len;
+	const char*		hostname = "Hostname unknown";
+
+	/* The server hostname includes the NUL byte. */
+	len = strlen(hostname) + 1;
+	mach_write_to_4(value, len);
+
+	if (fwrite(&value, 1,  sizeof(value), file) != sizeof(value)
+	    || fwrite(hostname, 1,  len, file) != len) {
+
+		msg("xtrabackup: Error: writing hostname.");
+
+		return(false);
+	}
+
+	/* The table name includes the NUL byte. */
+	ut_a(table->name != 0);
+	len = strlen(table->name) + 1;
+
+	/* Write the table name. */
+	mach_write_to_4(value, len);
+
+	if (fwrite(&value, 1,  sizeof(value), file) != sizeof(value)
+	    || fwrite(table->name, 1,  len, file) != len) {
+
+		msg("xtrabackup: Error: writing table name.");
+
+		return(false);
+	}
+
+	byte		row[sizeof(ib_uint32_t) * 3];
+
+	/* Write the next autoinc value. */
+#ifdef INNODB_VERSION_SHORT
+	MACH_WRITE_64(row, table->autoinc);
+#else
+	MACH_WRITE_64(row,
+		      ut_dulint_create((table->autoinc >> 32) & 0xFFFFFFFFUL,
+				       table->autoinc & 0xFFFFFFFFUL));
+#endif
+
+	if (fwrite(row, 1, sizeof(IB_UINT64), file) != sizeof(IB_UINT64)) {
+		msg("xtrabackup: Error: writing table autoinc value.");
+
+		return(false);
+	}
+
+	byte*		ptr = row;
+
+	/* Write the system page size. */
+	mach_write_to_4(ptr, UNIV_PAGE_SIZE);
+	ptr += sizeof(ib_uint32_t);
+
+	/* Write the table->flags. */
+	mach_write_to_4(ptr, table->flags);
+	ptr += sizeof(ib_uint32_t);
+
+	/* Write the number of columns in the table. */
+	mach_write_to_4(ptr, table->n_cols);
+
+	if (fwrite(row, 1,  sizeof(row), file) != sizeof(row)) {
+		msg("xtrabackup: Error: writing table meta-data.");
+
+		return(false);
+	}
+
+	return(true);
+}
+
+/*********************************************************************//**
+Write MySQL 5.6-style meta data config file.
+@return true in case of success otherwise false. */
+static
+bool
+xb_export_cfg_write(
+	const fil_node_t*	node,
+	const dict_table_t*	table)	/*!< in: write the meta data for
+					this table */
+{
+	char	file_path[FN_REFLEN];
+	FILE*	file;
+	bool	success;
+
+	strcpy(file_path, node->name);
+	strcpy(file_path + strlen(file_path) - 4, ".cfg");
+
+	file = fopen(file_path, "w+b");
+
+	if (file == NULL) {
+		msg("xtrabackup: Error: cannot close %s\n", node->name);
+
+		success = false;
+	} else {
+
+		success = xb_export_cfg_write_header(table, file);
+
+		if (success) {
+			success = xb_export_cfg_write_table(table, file);
+		}
+
+		if (success) {
+			success = xb_export_cfg_write_indexes(table, file);
+		}
+
+		if (fclose(file) != 0) {
+			msg("xtrabackup: Error: cannot close %s\n", node->name);
+			success = false;
+		}
+
+	}
+
+	return(success);
+
+}
+
 static void
 xtrabackup_prepare_func(void)
 {
@@ -7581,11 +7962,12 @@
 					ulint		n_index;
 
 					/* node exist == file exist, here */
-					strncpy(info_file_path, node->name, FN_REFLEN);
+					strcpy(info_file_path, node->name);
+					strcpy(info_file_path +
+					       strlen(info_file_path) -
+					       4, ".exp");
+
 					len = strlen(info_file_path);
-					info_file_path[len - 3] = 'e';
-					info_file_path[len - 2] = 'x';
-					info_file_path[len - 1] = 'p';
 
 					p = info_file_path;
 					prev = NULL;
@@ -7619,6 +8001,11 @@
 						goto next_node;
 					}
 
+					/* Write MySQL 5.6 .cfg file */
+					if (!xb_export_cfg_write(node, table)) {
+						goto next_node;
+					}
+
 					/* init exp file */
 					memset(page, 0, UNIV_PAGE_SIZE);
 					mach_write_to_4(page    , 0x78706f72UL);

=== modified file 'test/t/xb_export.sh'
--- test/t/xb_export.sh	2013-03-11 08:22:38 +0000
+++ test/t/xb_export.sh	2013-04-26 09:10:34 +0000
@@ -1,15 +1,21 @@
 . inc/common.sh
 
-if [ -z "$XTRADB_VERSION" ]; then
-    echo "Requires XtraDB" > $SKIPPED_REASON
+if [ -z "$XTRADB_VERSION" -a ${MYSQL_VERSION:0:3} != "5.6" ]; then
+    echo "Requires XtraDB or MySQL 5.6" > $SKIPPED_REASON
     exit $SKIPPED_EXIT_CODE
 fi
 
-if [ ${MYSQL_VERSION:0:3} = "5.5" ]
+import_option=""
+
+if [ ! -z "$XTRADB_VERSION" ]
 then
-    import_option="--innodb_import_table_from_xtrabackup=1"
-else
-    import_option="--innodb_expand_import=1"
+    # additional agrument should be passed to XtraDB
+    if [ ${MYSQL_VERSION:0:3} = "5.5" ]
+    then
+        import_option="--innodb_import_table_from_xtrabackup=1"
+    else
+        import_option="--innodb_expand_import=1"
+    fi
 fi
 
 mysql_extra_args="--innodb_file_per_table $import_option \