Merge lp:~akopytov/percona-xtrabackup/i25625-bug1037379-2.0 into lp:percona-xtrabackup/2.0

[Alexey Kopytov]

  Bug #1037379: SQL_THREAD left in stopped state of
                safe-slave-backup-timeout is reached

  Bug #887803: innobackupex --safe-slave-backup-timeout option doesn't
               work

  The problem in bug #1037379 was that wait_for_safe_slave() in
  innobackupex ran START/STOP SLAVE SQL_THREAD in a loop. If it succeeded
  in getting slave_open_temp_tables to 0 in a specified time out, the
  SQL thread was then started later after copying non-transactional
  data. However, when it reached the timeout it did not restart the SQL
  thread before terminating with an error.

  Another problem was that if wait_for_safe_slave() succeeded,
  innobackupex would start the SQL thread regardless of its initial
  state. Which might be an undesired side effect.

  Fixed by:

  1. Checking the initial SQL thread state and starting it before
  terminating with a timeout error in wait_for_safe_slave().

  2. Starting the SQL thread only if it was running initially in backup()

  Additionally, this revision also fixes bug #887803, because the test
  case for bug #1037379 requires a working --safe-slave-backup-timeout
  option.

[Alexey Kopytov (akopytov)]

Issue #25625.

[Alexey Kopytov (akopytov)]

http://jenkins.percona.com/view/XtraBackup/job/percona-xtrabackup-2.0-param/267/

[Laurynas Biveinis (laurynas-biveinis)]

review: Approve

=== modified file 'innobackupex'
--- innobackupex	2012-09-21 12:32:30 +0000
+++ innobackupex	2012-10-16 15:41:24 +0000
@@ -137,8 +137,8 @@
 # backup my.cnf file
 my $backup_config_file = '';
 
-# whether host is detected as slave in safe slave backup mode
-my $host_is_slave = 0;
+# whether slave SQL thread is running when wait_for_safe_slave() is called
+my $sql_thread_started = 0;
 
 # options from the options file
 my %config;
@@ -431,7 +431,7 @@
     # release read locks on all tables
     mysql_unlockall() if !$option_no_lock;
 
-    if ( $option_safe_slave_backup && $host_is_slave ) {
+    if ( $option_safe_slave_backup && $sql_thread_started) {
       print STDERR "$prefix: Starting slave SQL thread\n";
       mysql_send('START SLAVE SQL_THREAD;');
     }
@@ -1825,7 +1825,7 @@
                         'sshopt=s' => \$option_ssh_opt,
                         'parallel=i' => \$option_parallel,
                         'safe-slave-backup' => \$option_safe_slave_backup,
-                        'safe-slave-backup-timeout' => $option_safe_slave_backup_timeout,
+                        'safe-slave-backup-timeout=i' => \$option_safe_slave_backup_timeout,
     );
                         
     if (!$rcode) {
@@ -2630,22 +2630,28 @@
 # Slave_open_temp_tables has to be zero.  Dies on timeout.
 sub wait_for_safe_slave {
    my @lines;
-
-   $host_is_slave = 0;
+   # whether host is detected as slave in safe slave backup mode
+   my $host_is_slave = 0;
+
+   $sql_thread_started = 0;
+
    mysql_send 'SHOW SLAVE STATUS\G;';
    file_to_array($mysql_stdout, \@lines);
    foreach my $line ( @lines ) {
       if ( $line =~ m/Read_Master_Log_Pos/ ) {
          $host_is_slave = 1;
-         last;
-      }
+     } elsif ( $line =~ m/Slave_SQL_Running:.*Yes/ ) {
+         $sql_thread_started = 1;
+     }
    }
    if ( !$host_is_slave ) {
       print STDERR "$prefix: Not checking slave open temp tables for --safe-slave-backup because host is not a slave\n";
       return;
    }
 
-   mysql_send 'STOP SLAVE SQL_THREAD;';
+   if ($sql_thread_started) {
+       mysql_send 'STOP SLAVE SQL_THREAD;';
+   }
 
    my $open_temp_tables = get_slave_open_temp_tables();
    print STDERR "$prefix: Slave open temp tables: $open_temp_tables\n";
@@ -2669,6 +2675,12 @@
       }
    } 
 
+   # Restart the slave if it was running at start
+   if ($sql_thread_started) {
+       print STDERR "Restarting slave SQL thread.\n";
+       mysql_send 'START SLAVE SQL_THREAD;';
+   }
+
    Die "Slave_open_temp_tables did not become zero after waiting $option_safe_slave_backup_timeout seconds";
 }
 

=== modified file 'test/inc/common.sh'
--- test/inc/common.sh	2012-06-19 06:00:44 +0000
+++ test/inc/common.sh	2012-10-16 15:41:24 +0000
@@ -359,6 +359,47 @@
 }
 
 ########################################################################
+# Wait until slave catches up with master.
+# The timeout is hardcoded to 300 seconds
+#
+# Synopsis:
+#   sync_slave_with_master <slave_id> <master_id>
+#########################################################################
+function sync_slave_with_master()
+{
+    local slave_id=$1
+    local master_id=$2
+    local count
+    local master_file
+    local master_pos
+
+    vlog "Syncing slave (id=#$slave_id) with master (id=#$master_id)"
+
+    # Get master log pos
+    switch_server $master_id
+    count=0
+    while read line; do
+      	if [ $count -eq 1 ] # File:
+      	then
+      	    master_file=`echo "$line" | sed s/File://`
+      	elif [ $count -eq 2 ] # Position:
+      	then
+      	    master_pos=`echo "$line" | sed s/Position://`
+      	fi
+      	count=$((count+1))
+    done <<< "`run_cmd $MYSQL $MYSQL_ARGS -Nse 'SHOW MASTER STATUS\G' mysql`"
+
+    echo "master_file=$master_file, master_pos=$master_pos"
+
+    # Wait for the slave SQL thread to catch up
+    switch_server $slave_id
+
+    run_cmd $MYSQL $MYSQL_ARGS <<EOF
+SELECT MASTER_POS_WAIT('$master_file', $master_pos, 300);
+EOF
+}
+
+########################################################################
 # Prints checksum for a given table.
 # Expects 2 arguments: $1 is the DB name and $2 is the table to checksum
 ########################################################################

=== added file 'test/t/bug1037379.sh'
--- test/t/bug1037379.sh	1970-01-01 00:00:00 +0000
+++ test/t/bug1037379.sh	2012-10-16 15:41:24 +0000
@@ -0,0 +1,69 @@
+################################################################################
+# Bug #1037379: SQL_THREAD left in stopped state of safe-slave-backup-timeout is
+#               reached
+# Bug #887803: innobackupex --safe-slave-backup-timeout option doesn't work
+################################################################################
+
+. inc/common.sh
+
+################################################################################
+# Create a temporary table and pause "indefinitely" to keep the connection open
+################################################################################
+function create_temp_table()
+{
+    switch_server $master_id
+    run_cmd $MYSQL $MYSQL_ARGS test <<EOF
+CREATE TEMPORARY TABLE tmp(a INT);
+INSERT INTO tmp VALUES (1);
+SELECT SLEEP(10000);
+EOF
+}
+
+master_id=1
+slave_id=2
+
+start_server_with_id $master_id
+start_server_with_id $slave_id
+
+setup_slave $slave_id $master_id
+
+create_temp_table &
+job_master=$!
+
+sync_slave_with_master $slave_id $master_id
+
+switch_server $slave_id
+
+################################################################################
+# First check if the SQL thread is left in the running state
+# if it is running when taking a backup
+################################################################################
+
+# The following will fail due to a timeout
+run_cmd_expect_failure $IB_BIN $IB_ARGS --no-timestamp --safe-slave-backup \
+    --safe-slave-backup-timeout=3 $topdir/backup1
+
+grep -q "Slave_open_temp_tables did not become zero" $OUTFILE
+
+# Check that the SQL thread is running
+run_cmd $MYSQL $MYSQL_ARGS -e "SHOW SLAVE STATUS\G" |
+  egrep 'Slave_SQL_Running:[[:space:]]+Yes'
+
+################################################################################
+# Now check if the SQL thread is left in the stopped state
+# if it is stopped when taking a backup
+################################################################################
+
+run_cmd $MYSQL $MYSQL_ARGS -e "STOP SLAVE SQL_THREAD"
+
+# The following will fail due to a timeout
+run_cmd_expect_failure $IB_BIN $IB_ARGS --no-timestamp --safe-slave-backup \
+    --safe-slave-backup-timeout=3 $topdir/backup2
+
+grep -c "Slave_open_temp_tables did not become zero" $OUTFILE | grep -w 2
+
+# Check that the SQL thread is stopped
+run_cmd $MYSQL $MYSQL_ARGS -e "SHOW SLAVE STATUS\G" |
+  egrep 'Slave_SQL_Running:[[:space:]]+No'
+
+kill -SIGKILL $job_master