Merge lp:~akopytov/percona-xtrabackup/bug1164945-2.1 into lp:percona-xtrabackup/2.1

[Alexey Kopytov]

    Bug #1164945: move-back/copy-back should allow non-empty directories in
                  some cases

    The patches introduces a new innobackupex option,
    --force-non-empty-directories. When specified, it makes --copy-back or
    --move-back transfer files to non-empty directories. Note that no
    existing files will be overwritten. If --copy-back or --nove-back has to
    copy a file from the backup directory which already exists in the
    destination directory, it will still fail with an error.

http://jenkins.percona.com/view/XtraBackup/job/percona-xtrabackup-2.1-param/500/

[Alexey Kopytov (akopytov)]

review: Approve

=== modified file 'innobackupex.pl'
--- innobackupex.pl	2013-11-15 11:01:25 +0000
+++ innobackupex.pl	2013-11-20 09:11:18 +0000
@@ -144,6 +144,8 @@
 
 my $option_version_check = '1';
 
+my $option_force_non_empty_dirs = '';
+
 my %mysql;
 my $option_backup = '';
 
@@ -2090,6 +2092,18 @@
 }
 
 #
+# Check if a given directory exists, or fail with an error otherwise
+#
+
+sub if_directory_exists {
+    my $empty_dir = shift;
+    my $is_directory_empty_comment = shift;
+    if (! -d $empty_dir) {
+        die "$is_directory_empty_comment directory '$empty_dir' does not exist!";
+    }
+}
+
+#
 # if_directory_exists_and_empty accepts two arguments:
 # variable with directory name and comment.
 # Sub checks that directory exists and is empty
@@ -2099,14 +2113,30 @@
 sub if_directory_exists_and_empty {
     my $empty_dir = shift;
     my $is_directory_empty_comment = shift;
-    if (! -d $empty_dir) {
-        die "$is_directory_empty_comment directory '$empty_dir' does not exist!";
-    }
-    opendir (my $dh, $empty_dir) or die "$is_directory_empty_comment directory '$empty_dir': Not a directory";
-    if ( ! scalar( grep { $_ ne "." && $_ ne ".." && $_ ne "my.cnf" && $_ ne "master.info"} readdir($dh)) == 0) {
-        die "$is_directory_empty_comment directory '$empty_dir' is not empty!";
-    }
-    closedir($dh);
+
+    if_directory_exists($empty_dir, $is_directory_empty_comment);
+
+    if (!$option_force_non_empty_dirs) {
+        opendir (my $dh, $empty_dir) or
+            die "$is_directory_empty_comment directory '$empty_dir': Not a directory";
+        if ( ! scalar( grep { $_ ne "." && $_ ne ".." && $_ ne "my.cnf" &&
+                                  $_ ne "master.info"} readdir($dh)) == 0) {
+            die "$is_directory_empty_comment directory '$empty_dir' is not empty!";
+        }
+        closedir($dh);
+    }
+}
+
+#
+# Fail with an error if file exists
+#
+
+sub die_if_exists {
+    my $path = shift;
+
+    if (-e $path) {
+        die "Cannot overwrite file: $path";
+    }
 }
 
 #
@@ -2358,10 +2388,12 @@
 
     # check that original data directories exist and they are empty
     if_directory_exists_and_empty($orig_datadir, "Original data");
-    if_directory_exists_and_empty($orig_ibdata_dir, "Original InnoDB data");
-    if_directory_exists_and_empty($orig_iblog_dir, "Original InnoDB log");
+    if ($orig_ibdata_dir) {
+        if_directory_exists($orig_ibdata_dir, "Original InnoDB data");
+    }
+    if_directory_exists($orig_iblog_dir, "Original InnoDB log");
     if ($orig_undo_dir) {
-        if_directory_exists_and_empty($orig_undo_dir,
+        if_directory_exists($orig_undo_dir,
                                       "Original undo directory");
     }
 
@@ -2421,7 +2453,17 @@
     foreach my $c (parse_innodb_data_file_path($orig_innodb_data_file_path)) {
         # get the relative pathname of a data file
         $src_name = escape_path("$backup_dir/$c->{filename}");
-        $dst_name = escape_path("$orig_ibdata_dir/$c->{path}");
+        if ($orig_ibdata_dir) {
+            $dst_name = escape_path("$orig_ibdata_dir/$c->{path}");
+        } else {
+            # If innodb_data_home_dir is empty, but file path(s) in
+            # innodb_data_file_path are relative, InnoDB treats them as if
+            # innodb_data_home_dir was the same as datadir.
+
+            my $dst_root = ($c->{path} =~ /^\//) ? "" : $orig_datadir;
+            $dst_name = escape_path("$dst_root/$c->{path}");
+        }
+        die_if_exists($dst_name);
         &$move_or_copy_file($src_name, $dst_name);
     }
 
@@ -2435,7 +2477,8 @@
     while (defined($file = readdir(DIR))) {
         if ($file =~ /^$ibundo_files$/ && -f "$backup_dir/$file") {
             $src_name = escape_path("$backup_dir/$file");
-            $dst_name = escape_path("$orig_undo_dir");
+            $dst_name = escape_path("$orig_undo_dir/$file");
+            die_if_exists($dst_name);
             &$move_or_copy_file($src_name, $dst_name);
         }
     }
@@ -2450,7 +2493,8 @@
     while (defined($file = readdir(DIR))) {
         if ($file =~ /^$iblog_files$/ && -f "$backup_dir/$file") {
             $src_name = escape_path("$backup_dir/$file");
-            $dst_name = escape_path("$orig_iblog_dir");
+            $dst_name = escape_path("$orig_iblog_dir/$file");
+            die_if_exists($dst_name);
             &$move_or_copy_file($src_name, $dst_name);
         }
     }
@@ -3695,7 +3739,9 @@
                         'lock-wait-threshold=i' => \$option_lock_wait_threshold,
                         'lock-wait-query-type=s' =>
                         \$option_lock_wait_query_type,
-                        'version-check!' => \$option_version_check
+                        'version-check!' => \$option_version_check,
+                        'force-non-empty-directories' =>
+                        \$option_force_non_empty_dirs
     );
 
     if (@ARGV == 0) {
@@ -4950,6 +4996,10 @@
 
 This option specifies the directory in which to save an extra copy of the "xtrabackup_checkpoints" file.  The option accepts a string argument. It is passed directly to xtrabackup's --extra-lsndir option. See the xtrabackup documentation for details.
 
+==item --force-non-empty-directories
+
+This option, when specified, makes --copy-back or --move-back transfer files to non-empty directories. Note that no existing files will be overwritten. If --copy-back or --nove-back has to copy a file from the backup directory which already exists in the destination directory, it will still fail with an error.
+
 =item --galera-info
 
 This options creates the xtrabackup_galera_info file which contains the local node state at the time of the backup. Option should be used when performing the backup of Percona-XtraDB-Cluster.

=== added file 'test/t/bug1049291.sh'
--- test/t/bug1049291.sh	1970-01-01 00:00:00 +0000
+++ test/t/bug1049291.sh	2013-11-20 09:11:18 +0000
@@ -0,0 +1,23 @@
+########################################################################
+# Bug #1049291: innobackupex --copy-back fails with an empty
+#               innodb-data-home-dir
+########################################################################
+
+MYSQLD_EXTRA_MY_CNF_OPTS="
+innodb_data_home_dir=
+"
+start_server
+
+innobackupex --no-timestamp $topdir/backup
+
+stop_server
+
+rm -rf $MYSQLD_DATADIR/*
+
+innobackupex --apply-log $topdir/backup
+
+innobackupex --copy-back $topdir/backup
+
+test -f $MYSQLD_DATADIR/ibdata1
+
+rm -rf $MYSQLD_DATADIR

=== added file 'test/t/bug1164945.sh'
--- test/t/bug1164945.sh	1970-01-01 00:00:00 +0000
+++ test/t/bug1164945.sh	2013-11-20 09:11:18 +0000
@@ -0,0 +1,47 @@
+##############################################################################
+# Bug #1164945: move-back/copy-back should allow non-empty directories in some
+#               cases
+##############################################################################
+
+start_server
+
+innobackupex --no-timestamp $topdir/backup
+
+stop_server
+
+innobackupex --apply-log $topdir/backup
+
+cp -a $topdir/backup $topdir/backup_copy
+
+function test_force_non_empty_dirs()
+{
+    rm -rf $MYSQLD_DATADIR/*
+
+    mkdir $MYSQLD_DATADIR/empty_subdir
+
+    touch $MYSQLD_DATADIR/non_existing_file
+
+    # check that --*-back fails without --force-non-empty-directories
+    run_cmd_expect_failure $IB_BIN $IB_ARGS $1 $topdir/backup
+
+    rm -rf $topdir/backup
+    cp -a $topdir/backup_copy $topdir/backup
+
+    # check that --*-back works with --force-non-empty-directories
+    innobackupex $1 --force-non-empty-directories $topdir/backup
+
+    rm -rf $topdir/backup
+    cp -a $topdir/backup_copy $topdir/backup
+
+    # check that --*-back --force-non-empty-directories does not overwrite files
+    touch $MYSQLD_DATADIR/ibdata1
+    run_cmd_expect_failure $IB_BIN $IB_ARGS $1 --force-non-empty-directories \
+                           $topdir/backup
+
+    rm -rf $topdir/backup
+    cp -a $topdir/backup_copy $topdir/backup
+}
+
+test_force_non_empty_dirs --copy-back
+
+test_force_non_empty_dirs --move-back

=== added file 'test/t/bug382742.sh'
--- test/t/bug382742.sh	1970-01-01 00:00:00 +0000
+++ test/t/bug382742.sh	2013-11-20 09:11:18 +0000
@@ -0,0 +1,41 @@
+########################################################################
+# Bug #382742: Absolute paths in innodb_data_file_path are not supported
+########################################################################
+
+# Use this as an absolute path prefix in innodb_data_file_path
+innodb_data_home_dir=$TEST_VAR_ROOT/innodb_data_home_dir
+
+mkdir $innodb_data_home_dir
+
+MYSQLD_EXTRA_MY_CNF_OPTS="
+innodb_file_per_table=0
+innodb_data_home_dir=/
+innodb_data_file_path=$innodb_data_home_dir/ibdata1:3M;$innodb_data_home_dir/ibdata2:10M:autoextend
+"
+
+start_server
+
+test -f $innodb_data_home_dir/ibdata1
+test -f $innodb_data_home_dir/ibdata2
+
+$MYSQL $MYSQL_ARGS -e "CREATE TABLE test.t(a INT)"
+
+record_db_state test
+
+innobackupex --no-timestamp $topdir/backup
+
+stop_server
+
+rm -rf $MYSQLD_DATADIR/*
+rm -rf $innodb_data_home_dir/*
+
+innobackupex --apply-log $topdir/backup
+
+innobackupex --copy-back $topdir/backup
+
+test -f $innodb_data_home_dir/ibdata1
+test -f $innodb_data_home_dir/ibdata2
+
+start_server
+
+verify_db_state test