Merge lp:~akopytov/percona-xtrabackup/bug759701-1.7 into lp:percona-xtrabackup/2.0

[Alexey Kopytov]

[Alexey Kopytov (akopytov)]

http://jenkins.percona.com/view/Percona%20Xtrabackup/job/percona-xtrabackup-param/101/

[Stewart Smith (stewart)]

review: Approve

=== modified file 'innobackupex'
--- innobackupex	2011-12-17 03:23:36 +0000
+++ innobackupex	2011-12-21 18:48:30 +0000
@@ -15,6 +15,8 @@
 use FileHandle;
 use File::Basename;
 use File::Temp;
+use File::Find;
+use File::Copy;
 use English qw(-no_match_vars);
 
 # version of this script
@@ -208,6 +210,11 @@
 my $xtrabackup_binary_file = 'xtrabackup_binary';
 my %rsync_files_hash;
 
+my $copy_dir_src;
+my $copy_dir_dst;
+my $copy_dir_exclude_regexp;
+my $copy_dir_overwrite;
+
 ######################################################################
 # program execution begins here
 ######################################################################
@@ -519,6 +526,58 @@
 }
 
 #
+# Callback function for copy_dir_recursively that actually does all
+# the copying work
+#
+sub copy_file() {
+    if (/$copy_dir_exclude_regexp/) {
+	return;
+    }
+    (my $dst_path = $File::Find::name) =~ s/^$copy_dir_src/$copy_dir_dst/;
+    if (-d "$File::Find::name") {
+	# Create the directory in the destination if necessary
+	if (! -e "$dst_path") {
+	    print STDERR "$prefix Creating directory '$dst_path'\n";
+	    mkdir "$dst_path" or Die "mkdir failed: $!";
+	} elsif (! -d "$dst_path") {
+	    Die "$dst_path exists, but is not a directory";
+	}
+    } else {
+	# Don't overwrite files unless $copy_dir_overwrite is 1
+	if (!$copy_dir_overwrite && -e "$copy_dir_dst/$_") {
+	    Die "Failed to copy file $File::Find::name: " .
+		"file $copy_dir_dst/$_ exists";
+	}
+
+	print STDERR "$prefix Copying file '$copy_dir_dst/$_'\n";
+	copy($File::Find::name, $dst_path) or Die "copy failed: $!";
+    }
+}
+
+#
+# copy_dir_recursively subroutine does a recursive copy of a specified
+# directory excluding files matching a specifies regexp. If $overwrite
+# is 1, it overwrites the existing files.
+#
+# SYNOPSIS 
+#
+#     copy_dir_recursively($src_dir, $dst_dir, $exclude_regexp,
+#                          $overwrite);
+#
+# TODO
+#
+#     use rsync when --rsync is specified
+#
+sub copy_dir_recursively {
+    $copy_dir_src = shift;
+    $copy_dir_dst = shift;
+    $copy_dir_exclude_regexp = shift;
+    $copy_dir_overwrite = shift;
+
+    find(\&copy_file, $copy_dir_src);
+}
+
+#
 # copy_back subroutine copies data and index files from backup directory 
 # back to their original locations.
 #
@@ -530,10 +589,11 @@
         get_option(\%config, 'mysqld', 'innodb_data_file_path');
     my $orig_iblog_dir =
         get_option(\%config, 'mysqld', 'innodb_log_group_home_dir');
+    my $iblog_files = 'ib_logfile.*';
     my $excluded_files = 
-        '^(\.\.?|backup-my\.cnf|xtrabackup_logfile|mysql-std(err|out)|.*\.ibz)$';
-    my @ibdata_files;
-    my $iblog_files = '^ib_logfile.*$';
+        '\.\.?|backup-my\.cnf|xtrabackup_logfile|' .
+	'xtrabackup_binary|xtrabackup_binlog_info|xtrabackup_checkpoints' .
+	$iblog_files;
     my $compressed_data_file = '.*\.ibz$';
     my $file;
     my $backup_innodb_data_file_path;
@@ -573,59 +633,20 @@
                 Die "Backup data file '$backup_dir/$filename' does not exist.";
             }
         }
-
-        if (!is_in_array($filename, \@ibdata_files)) {
-            push(@ibdata_files, $filename);
-        }
+	
+        $excluded_files .= "|\Q$filename\E";
     }
 
     # copy files from backup dir to their original locations
 
     # copy files to original data directory
-    opendir(DIR, $backup_dir) 
-        || Die "Can't open directory '$backup_dir': $!\n";
-    print STDERR "$prefix Starting to copy MyISAM tables, indexes,\n"; 
-    print STDERR "$prefix .MRG, .TRG, .TRN, .ARM, .ARZ, .CSM, .CSV, .opt, and .frm files\n";
-    print STDERR "$prefix in '$backup_dir'\n";
+    my $excluded_regexp = '^(' . $excluded_files . ')$';
+    print STDERR "$prefix Starting to copy files in '$backup_dir'\n"; 
     print STDERR "$prefix back to original data directory '$orig_datadir'\n";
-    while (defined($file = readdir(DIR))) {
-        if ($file =~ /$excluded_files/) { next; }
-        if (is_in_array($file, \@ibdata_files)) { next; }
-        if ($file =~ /$iblog_files/) { next; }
-        if (-d "$backup_dir/$file") {
-            my $subdir = "$backup_dir/$file";
-            my $file2;
-
-            print STDERR "$prefix Copying directory '$subdir'\n";
-            if (! -x "\"".escape_path("$orig_datadir/$file")."\"") {
-                system("mkdir \"".escape_path("$orig_datadir/$file")."\"") 
-                    and Die "Failed to create directory "
-                            . "'$orig_datadir/$file' : $!";
-            }
-            opendir(SUBDIR, "$subdir") 
-                || Die "Can't open directory '$subdir': $!\n";
-            while (defined($file2 = readdir(SUBDIR))) {
-                if (-d "$subdir/$file2") { next; }
-                if ($file2 =~ /$compressed_data_file/) { next; }
-                $src_name = escape_path("$subdir/$file2");
-                $dst_name = escape_path("$orig_datadir/$file");
-                system("$CP_CMD \"$src_name\" \"$dst_name\"")
-                    and Die "Failed to copy file '$file': $!";
-            }
-            closedir(SUBDIR);
-        } else { 
-            print STDERR "$prefix Copying file " . 
-                "'$backup_dir/$file'\n";
-            $src_name = escape_path("$backup_dir/$file");
-            $dst_name = escape_path("$orig_datadir");
-            system("$CP_CMD \"$src_name\" \"$dst_name\"")
-                and Die "Failed to copy file '$file': $!";
-        }
-    }
-    closedir(DIR);
+    copy_dir_recursively($backup_dir, $orig_datadir, $excluded_regexp, 0);
 
     # copy InnoDB data files to original InnoDB data directory
-    print STDERR "\n$prefix Starting to copy InnoDB tables and indexes\n";
+    print STDERR "\n$prefix Starting to copy InnoDB system tablespace\n";
     print STDERR "$prefix in '$backup_dir'\n";
     print STDERR "$prefix back to original InnoDB data directory '$orig_ibdata_dir'\n";
     foreach my $a (split(/;/, $orig_innodb_data_file_path)) {
@@ -646,7 +667,7 @@
     print STDERR "$prefix in '$backup_dir'\n";
     print STDERR "$prefix back to original InnoDB log directory '$orig_iblog_dir'\n";
     while (defined($file = readdir(DIR))) {
-        if ($file =~ /$iblog_files/ && -f "$backup_dir/$file") {
+        if ($file =~ /'^' . $iblog_files . '$'/ && -f "$backup_dir/$file") {
             print STDERR "$prefix Copying file '$backup_dir/$file'\n";
             $src_name = escape_path("$backup_dir/$file");
             $dst_name = escape_path("$orig_iblog_dir");
@@ -714,20 +735,20 @@
         }
     }
 
-    # If we were applying an incremental change set, we need to make sure the xtrabackup_slave_info is still correct
+    # If we were applying an incremental change set, we need to make
+    # sure non-InnoDB files and xtrabackup_* metainfo files are copied
+    # to the full backup directory.
     if ( $option_incremental_dir ) {
-
-        # If the latest backup has this file, then update the target dir with the new file
-        if ( -e "$option_incremental_dir/xtrabackup_slave_info" ) {
-            print STDERR "\n$now  $prefix Updating xtrabackup_slave_info with new slave position info...";
-            $cmdline = "cp $option_incremental_dir/xtrabackup_slave_info $backup_dir/xtrabackup_slave_info";
-            $rcode = system("$cmdline");
-            if ($rcode) {
-                # failure
-                Die "\n$prefix Failed to update xtrabackup_slave_info in $backup_dir";
-            }
-        # If the latest backup has no file, we need to remove the old xtrabackup_slave_info file, because it is out of date
-        } elsif ( -e "$backup_dir/xtrabackup_slave_info" ) {
+	print STDERR "$prefix Starting to copy non-InnoDB files in '$option_incremental_dir'\n"; 
+	print STDERR "$prefix to the full backup directory '$backup_dir'\n";
+	copy_dir_recursively($option_incremental_dir, $backup_dir,
+			     '^(\.\.?|backup-my\.cnf|xtrabackup_logfile|' .
+			     'xtrabackup_binary|xtrabackup_checkpoints|' .
+			     '.*\.delta|.*\.meta|ib_logfile.*)$', 1);
+        # If the latest backup has no file, we need to remove the old
+        # xtrabackup_slave_info file, because it is out of date
+        # TODO: this will not be needed when bug #856400 is fixed.
+        if ( -e "$backup_dir/xtrabackup_slave_info" ) {
             print STDERR "\n$now  $prefix No updated xtrabackup_slave_info found in incremental dir, removing stale xtrabackup_slave_info file from target dir.";
             $cmdline = "rm $backup_dir/xtrabackup_slave_info";
             $rcode = system("$cmdline");
 
=== modified file 'test/t/bug723097.sh'
--- test/t/bug723097.sh	2011-09-14 17:41:10 +0000
+++ test/t/bug723097.sh	2011-12-21 18:48:30 +0000
@@ -27,7 +27,7 @@
 checksum_b=`checksum_table test messages`
 vlog "Checksum after is $checksum_b"
 
-if [ $checksum_a -eq $checksum_b ]
+if [ "$checksum_a" -eq "$checksum_b" ]
 then
 	vlog "Checksums are Ok"
 else
 
=== added file 'test/t/bug759701.sh'
--- test/t/bug759701.sh	1970-01-01 00:00:00 +0000
+++ test/t/bug759701.sh	2011-12-21 18:48:30 +0000
@@ -0,0 +1,69 @@
+##########################################################################
+# Bug #759701: innobackupex does not copy non-InnoDB files when applying #
+#              incremental backups                                       #
+##########################################################################
+. inc/common.sh
+
+init
+run_mysqld --innodb_file_per_table
+load_sakila
+
+# Full backup
+# backup root directory
+vlog "Starting backup"
+
+full_backup_dir=$topdir/full_backup
+innobackupex  --no-timestamp $full_backup_dir
+
+# Changing data
+
+run_cmd $MYSQL $MYSQL_ARGS -e "CREATE DATABASE newdb"
+run_cmd $MYSQL $MYSQL_ARGS -e \
+    "CREATE TABLE actor_copy ENGINE=MyISAM SELECT * FROM sakila.actor" newdb
+
+# Saving the checksum of original table
+checksum_a=`checksum_table newdb actor_copy`
+test -n "$checksum_a" || die "Failed to checksum table actor_copy"
+
+vlog "Making incremental backup"
+
+# Incremental backup
+inc_backup_dir=$topdir/incremental_backup
+innobackupex --incremental --no-timestamp \
+    --incremental-basedir=$full_backup_dir $inc_backup_dir
+vlog "Incremental backup created in directory $inc_backup_dir"
+
+vlog "Preparing backup"
+innobackupex --apply-log --redo-only $full_backup_dir
+vlog "Log applied to full backup"
+
+innobackupex --apply-log --redo-only --incremental-dir=$inc_backup_dir \
+    $full_backup_dir
+vlog "Delta applied to full backup"
+
+innobackupex --apply-log $full_backup_dir
+vlog "Data prepared for restore"
+
+# Destroying mysql data
+stop_mysqld
+rm -rf $mysql_datadir/*
+vlog "Data destroyed"
+
+# Restore backup
+vlog "Copying files to their original locations"
+innobackupex --copy-back $full_backup_dir
+vlog "Data restored"
+
+run_mysqld --innodb_file_per_table
+
+vlog "Checking checksums"
+checksum_b=`checksum_table newdb actor_copy`
+
+vlog "Old checksum: $checksum_a"
+vlog "New checksum: $checksum_b"
+
+if [ "$checksum_a" != "$checksum_b"  ]
+then 
+    vlog "Checksums do not match"
+    exit -1
+fi
 
=== modified file 'test/t/ib_incremental.sh'
--- test/t/ib_incremental.sh	2011-09-14 17:41:10 +0000
+++ test/t/ib_incremental.sh	2011-12-21 18:48:30 +0000
@@ -89,7 +89,7 @@
 vlog "Checking checksums"
 checksum_b=`checksum_table incremental_sample test`
 
-if [ $checksum_a -ne $checksum_b  ]
+if [ "$checksum_a" != "$checksum_b"  ]
 then 
 	vlog "Checksums are not equal"
 	exit -1
 
=== modified file 'test/t/xb_export.sh'
--- test/t/xb_export.sh	2011-09-14 17:41:10 +0000
+++ test/t/xb_export.sh	2011-12-21 18:48:30 +0000
@@ -123,7 +123,7 @@
 checksum_3=`checksum_table incremental_sample test`
 vlog "checksum_3 is $checksum_3"
 
-if [ "$checksum_3" != $checksum_2  ]
+if [ "$checksum_3" != "$checksum_2"  ]
 then
         vlog "Checksums are not equal"
         exit -1
 
=== modified file 'test/t/xb_incremental.sh'
--- test/t/xb_incremental.sh	2011-09-14 17:41:10 +0000
+++ test/t/xb_incremental.sh	2011-12-21 18:48:30 +0000
@@ -85,7 +85,7 @@
 vlog "Checking checksums"
 checksum_b=`checksum_table incremental_sample test`
 
-if [ $checksum_a -ne $checksum_b  ]
+if [ "$checksum_a" != "$checksum_b"  ]
 then 
 	vlog "Checksums are not equal"
 	exit -1
 
=== modified file 'test/t/xb_incremental_compressed.sh'
--- test/t/xb_incremental_compressed.sh	2011-09-14 17:41:10 +0000
+++ test/t/xb_incremental_compressed.sh	2011-12-21 18:48:30 +0000
@@ -149,7 +149,7 @@
   vlog "Cheking checksums"
   checksum_b=`checksum_table incremental_sample test`
 
-  if [ $checksum_a -ne $checksum_b  ]
+  if [ "$checksum_a" != "$checksum_b"  ]
       then 
       vlog "Checksums are not equal"
       exit -1
 
=== modified file 'test/t/xb_part_range.sh'
--- test/t/xb_part_range.sh	2011-09-14 17:41:10 +0000
+++ test/t/xb_part_range.sh	2011-12-21 18:48:30 +0000
@@ -104,7 +104,7 @@
 vlog "Cheking checksums"
 checksum_b=`checksum_table test test`
 
-if [ $checksum_a -ne $checksum_b  ]
+if [ "$checksum_a" != "$checksum_b"  ]
 then 
 	vlog "Checksums are not equal"
 	exit -1
 
=== modified file 'test/t/xb_partial.sh'
--- test/t/xb_partial.sh	2011-09-14 17:41:10 +0000
+++ test/t/xb_partial.sh	2011-12-21 18:48:30 +0000
@@ -44,7 +44,7 @@
 vlog "Checking checksums"
 checksum_b=`checksum_table incremental_sample test`
 
-if [ $checksum_a -ne $checksum_b  ]
+if [ "$checksum_a" != "$checksum_b"  ]
 then 
 	vlog "Checksums are not equal"
 	exit -1