Merge lp:~percona-toolkit-dev/percona-toolkit/pt-hearbeat-monitor-master-change into lp:percona-toolkit/2.2

[Daniel Nichter]

[lp:~percona-toolkit-dev/percona-toolkit/pt-hearbeat-monitor-master-change]

595. By Daniel Nichter on 2014-02-12

Implement --check-master-server-id.

596. By Daniel Nichter on 2014-02-12

Add more --check-master-server-id docs.

Unmerged revisions

596. By Daniel Nichter on 2014-02-12

Add more --check-master-server-id docs.

595. By Daniel Nichter on 2014-02-12

Implement --check-master-server-id.

594. By Daniel Nichter on 2014-02-12

Add --check-master-server-id and a failing test for it.

=== modified file 'bin/pt-heartbeat'
--- bin/pt-heartbeat	2013-12-18 23:53:01 +0000
+++ bin/pt-heartbeat	2014-02-12 02:05:53 +0000
@@ -4778,6 +4778,7 @@
 use List::Util qw(min max sum);
 use Time::HiRes qw(gettimeofday time sleep usleep);
 use IO::File;
+use Carp qw(confess);
 
 use Percona::Toolkit;
 use constant PTDEBUG => $ENV{PTDEBUG} || 0;
@@ -4840,6 +4841,14 @@
       if ( !$o->get('stop') && !$o->get('database') ) {
          $o->save_error('--database must be specified');
       }
+
+      if ( $o->get('check-master-server-id') > 0 && !$o->get('monitor') ) {
+         $o->save_error('--check-master-server-id requires --monitor');
+      }
+   }
+
+   if ( $o->get('check-master-server-id') > 0 ) {
+         $o->set('print-master-server-id', 1);
    }
 
    eval {
@@ -4958,6 +4967,7 @@
 
    # pk_col and pk_val are used to identify the heartbeat row to update or
    # or monitor.
+   my ($ms, $master_server_id);
    my ($pk_col, $pk_val);
    if ( $id ) {
       # Legacy mode: update heartbeat row WHERE id=1 and monitor heartbeat
@@ -4973,10 +4983,10 @@
          $pk_val = $server_id;
       }
       else {  # monitor or check
-         my $master_server_id = $o->get('master-server-id');
+         $master_server_id = $o->get('master-server-id');
          if ( !$master_server_id ) {
             eval {
-               my $ms = new MasterSlave(
+               $ms = new MasterSlave(
                   OptionParser => $o,
                   DSNParser    => $dp,
                   Quoter       => $q,
@@ -5077,7 +5087,7 @@
 
       $heartbeat_sth = $dbh->prepare($heartbeat_sql);
 
-      my $ro_check = !!$o->get('check-read-only');
+      my $ro_check = $o->get('check-read-only');
       $update_heartbeat = sub {
          my ($sth) = @_;
          my @vals;
@@ -5151,15 +5161,16 @@
          . ($dbi_driver eq 'mysql' ? '/*!50038, @@hostname AS host*/' : '')
          . ($id ? "" : ", server_id")
          . " FROM $db_tbl "
-         . "WHERE $pk_col='$pk_val' "
+         . "WHERE $pk_col=? "
          . "LIMIT 1";
       PTDEBUG && _d("SELECT SQL:", $heartbeat_sql);
 
       $heartbeat_sth = $dbh->prepare($heartbeat_sql);
 
       $get_delay = sub {
-         my ($sth) = @_;
-         $sth->execute();
+         my ($sth, $id) = @_;
+         confess "No master server ID given" unless $id;
+         $sth->execute($id);
          PTDEBUG && _d($sth->{Statement});
          my ($ts, $hostname, $server_id) = $sth->fetchrow_array();
          my $now = time;
@@ -5176,7 +5187,7 @@
          $delay = 0.00 if $delay < 0;
 
          $sth->finish();
-         return ($delay, $hostname, $pk_val);
+         return ($delay, $hostname, $server_id);
       };
 
       # https://bugs.launchpad.net/percona-toolkit/+bug/1163372
@@ -5194,7 +5205,7 @@
       $update_heartbeat->($heartbeat_sth);
    }
    else {
-      $get_delay->($heartbeat_sth);
+      $get_delay->($heartbeat_sth, $pk_val);
    }
    $heartbeat_sth->finish();
 
@@ -5224,6 +5235,7 @@
          sth          => $heartbeat_sth,
          sql          => $heartbeat_sql,
          get_delay    => $get_delay,
+         server_id    => $pk_val,
          interval     => $interval,
          skew         => $skew,
          hires_ts     => $hires_ts,
@@ -5259,6 +5271,9 @@
    # ########################################################################
    # Monitor or update the heartbeat table.
    # ########################################################################
+   my $check_server_id_time = $o->get('check-master-server-id');
+   my $last_server_id_check = int(time);
+
    my $end = $o->get('run-time') ? int(time + $o->get('run-time')) : 0;
    PTDEBUG && _d($end ? ('Will exit at', ts($end)) : 'Running forever');
 
@@ -5290,8 +5305,39 @@
          }
 
          if ( $o->get('monitor') ) {
+            if ( $check_server_id_time ) {
+               # Time to --check-master-server-id?
+               my $now = int(time);
+               if ( $now - $last_server_id_check >= $check_server_id_time ) {
+                  PTDEBUG && _d("Checking master server id");
+                  eval {
+                     my $master_dsn = $ms->get_master_dsn($dbh, $dsn, $dp)
+                        or die "This server is not a slave";
+                     my $master_dbh = $dp->get_dbh(
+                        $dp->get_cxn_params($master_dsn),
+                        { AutoCommit => 1 }
+                     );
+                     my ($new_master_server_id)
+                        = $master_dbh->selectrow_array('SELECT @@server_id');
+                     $master_dbh->disconnect;
+                     if ( $new_master_server_id
+                          && $new_master_server_id != $master_server_id ) {
+                          PTDEBUG && _d("Master server id changed:",
+                             $master_server_id, "to",
+                             $new_master_server_id);
+                          $pk_val = $new_master_server_id;
+                     }
+                  };
+                  if ( $EVAL_ERROR ) {
+                     PTDEBUG && _d("Error checking master id:", $EVAL_ERROR);
+                  }
+                  $last_server_id_check = $now;
+               }
+            }
+
             $heartbeat_sth ||= $dbh->prepare($heartbeat_sql);
-            my ($delay) = $get_delay->($heartbeat_sth);
+            my ($delay, undef, $server_id)
+               = $get_delay->($heartbeat_sth, $pk_val);
 
             unshift @samples, $delay;
             pop @samples if @samples > $limit;
@@ -5302,7 +5348,7 @@
                sum(@samples[0 .. $bound-1]) / $_;
             } @$frames;
 
-            my $output = sprintf $format, $delay, @vals, $pk_val;
+            my $output = sprintf $format, $delay, @vals, $server_id;
             if ( my $file = $o->get('file') ) { 
                open my $file, '>', $file
                   or die "Can't open $file: $OS_ERROR";
@@ -5354,11 +5400,11 @@
 # Check the delay on a single server.  Optionally recurse to all its slaves.
 sub check_delay {
    my ( %args ) = @_;
-   my @required_args = qw(dsn dbh sth sql get_delay interval skew OptionParser DSNParser);
+   my @required_args = qw(dsn dbh sth sql get_delay server_id interval skew OptionParser DSNParser);
    foreach my $arg ( @required_args ) {
       die "I need a $arg argument" unless $args{$arg};
    }
-   my ($dsn, $dbh, $sth, $sql, $get_delay, $interval, $skew, $o, $dp)
+   my ($dsn, $dbh, $sth, $sql, $get_delay, $server_id,  $interval, $skew, $o, $dp)
       = @args{@required_args};
    PTDEBUG && _d('Checking slave delay');
 
@@ -5411,7 +5457,8 @@
       }
       sleep $next_interval - time;
       PTDEBUG && _d('Woke up at', ts(time));
-      my ($delay, $hostname, $master_server_id) = $get_delay->($sth);
+      my ($delay, $hostname, $master_server_id)
+         = $get_delay->($sth, $server_id);
 
       if ( $o->get('recurse') ) {
          # Must print not only the delay, but the server's hostname if
@@ -5642,6 +5689,8 @@
 
 L<"--daemonize"> and L<"--check"> are mutually exclusive.
 
+L<"--master-server-id"> and L<"--check-master-server-id"> are mutually exclusive.
+
 This tool accepts additional command-line arguments.  Refer to the
 L<"SYNOPSIS"> and usage information for details.
 
@@ -5667,6 +5716,20 @@
 their lag, too.  The hostname or IP and port for each slave is printed
 before its delay.  L<"--recurse"> only works with MySQL.
 
+=item --check-master-server-id
+
+type: time; default: 0
+
+Check master server ID periodically and change L<"--monitor"> to match.
+The given time should be longer than L<"--interval">.  For example,
+to L<"--monitor"> every 5 seconds and check the master server ID every
+1 minute:
+
+  --monitor 5s --check-master-server-id 1m
+
+This option implies L<"--print-master-server-id">.  When the master server ID
+changes, the new value is printed.
+
 =item --check-read-only
 
 Check if the server has read_only enabled; If it does, the tool skips doing
 
=== added file 't/pt-heartbeat/check_master.t'
--- t/pt-heartbeat/check_master.t	1970-01-01 00:00:00 +0000
+++ t/pt-heartbeat/check_master.t	2014-02-12 02:05:53 +0000
@@ -0,0 +1,174 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More;
+use Data::Dumper;
+use File::Temp qw(tempfile);
+
+use PerconaTest;
+use Sandbox;
+require "$trunk/bin/pt-heartbeat";
+
+my $dp  = new DSNParser(opts=>$dsn_opts);
+my $sb  = new Sandbox(basedir => '/tmp', DSNParser => $dp);
+my $master1_dbh = $sb->get_dbh_for('master');
+
+# Standard setup is:
+#   12345 -> 12346 -> 12347
+# So we don't mess with standard servers, start also:
+#   12348 -> 12349
+# Then --update 12345 and 12348, start --monitor on 12349, then change
+# its master to 12345.  Disregard the names "master1" and "master2".
+
+#diag("Starting extra sandboxes...");
+
+diag(`$trunk/sandbox/start-sandbox master 12348 >/dev/null`);
+my $master2_dbh = $sb->get_dbh_for('master1');
+
+diag(`$trunk/sandbox/start-sandbox slave 12349 12348 >/dev/null`);
+my $slave_dbh = $sb->get_dbh_for('master2');
+
+if ( !$master1_dbh ) {
+      plan skip_all => 'Cannot connect to sandbox master';
+}
+elsif ( !$master2_dbh ) {
+   plan skip_all => 'Cannot connect to second sandbox master';
+}
+elsif ( !$slave_dbh ) {
+   plan skip_all => 'Cannot connect to sandbox slave1';
+}
+
+$sb->load_file('master', 't/pt-heartbeat/samples/precision-time-table.sql');
+$sb->load_file('master1', 't/pt-heartbeat/samples/precision-time-table.sql');
+
+# Get master relay pos before we create heartbeat table and row,
+# else slave 12349 won't reply these queries and get heartbeat
+# from master 12345.
+my $s = $master1_dbh->selectrow_hashref("SHOW MASTER STATUS");
+
+$master1_dbh->do("INSERT INTO test.heartbeat (ts, server_id) VALUES (UTC_TIMESTAMP(), 12345)");
+$master2_dbh->do("INSERT INTO test.heartbeat (ts, server_id) VALUES (UTC_TIMESTAMP(), 12348)");
+
+# ###########################################################################
+# Helper subs
+# ###########################################################################
+
+my $base_pidfile = (tempfile("/tmp/pt-heartbeat-test.XXXXXXXX", OPEN => 0, UNLINK => 0))[1];
+my $master_port = $sb->port_for('master');
+
+my @exec_pids;
+my @pidfiles;
+
+sub start_update_instance {
+   my ($port) = @_;
+   my $pidfile = "$base_pidfile.$port.pid";
+   push @pidfiles, $pidfile;
+
+   my $pid = fork();
+   if ( $pid == 0 ) {
+      my $cmd = "$trunk/bin/pt-heartbeat";
+      exec { $cmd } $cmd, qw(-h 127.0.0.1 -u msandbox -p msandbox -P), $port,
+                          qw(--database test --table heartbeat --create-table),
+                          qw(--utc --update --interval 0.5 --pid), $pidfile;
+      exit 1;
+   }
+   push @exec_pids, $pid;
+
+   PerconaTest::wait_for_files($pidfile);
+   ok(
+      -f $pidfile,
+      "--update on $port started"
+   );
+}
+
+sub stop_all_instances {
+   my @pids = @exec_pids, map { chomp; $_ } map { slurp_file($_) } @pidfiles;
+   diag(`$trunk/bin/pt-heartbeat --stop >/dev/null`);
+
+   waitpid($_, 0) for @pids;
+   PerconaTest::wait_until(sub{ !-e $_ }) for @pidfiles;
+
+   unlink '/tmp/pt-heartbeat-sentinel';
+}
+
+# ###########################################################################
+# Test --check-master-server-id
+# ###########################################################################
+
+# Start --update on both masters.
+#diag("Starting --update instances...");
+start_update_instance(12345);
+start_update_instance(12348);
+
+# Start --monitor on slave currently attached to master 12348.
+my $output_file = "/tmp/pt-heartbeat-monitor.$PID";
+system("$trunk/bin/pt-heartbeat --monitor h=127.1,P=12349,u=msandbox,p=msandbox -D test --utc --check-master-server-id 1s --interval 0.5 --file $output_file --daemonize --run-time 5");
+#diag("Waiting for slave monitor to start...");
+PerconaTest::wait_for_files($output_file);
+
+# Slave monitor should report master ID is 12348.
+#diag("Waiting for slave monitor output...");
+my $output;
+for (1..3) {
+   $output = `cat $output_file`;
+   if ($output ne "") {
+      last;
+   }
+   sleep(1);
+}
+
+like(
+   $output,
+   qr/12348$/,
+   "Monitor sees master 12348"
+);
+
+# Simulate master VIP change by actually changing slave's master.
+#diag("Changing slave's master...");
+$slave_dbh->do("STOP SLAVE");
+
+$slave_dbh->do("CHANGE MASTER TO master_host='127.0.0.1', master_user='msandbox', master_password='msandbox', master_port=12345, master_log_file='$s->{file}', master_log_pos=$s->{position}");
+
+$slave_dbh->do("START SLAVE");
+
+# Give pt-heartbeat time to detect change.
+#diag("Waiting for pt-heartbeat to check master server ID...");
+sleep(2);
+
+# Slave monitor should report new master ID 12345.
+#diag("Waiting for slave monitor output...");
+for (1..3) {
+   $output = `cat $output_file`;
+   if ($output ne "") {
+      last;
+   }
+   sleep(1);
+}
+
+like(
+   $output,
+   qr/12345$/,
+   "Monitor changed to master 12345"
+);
+
+#diag("Stopping --update instances...");
+stop_all_instances();
+
+# #############################################################################
+# Done.
+# #############################################################################
+#diag("Stopping extra sandboxes...");
+diag(`$trunk/sandbox/stop-sandbox 12349 >/dev/null`);
+diag(`$trunk/sandbox/stop-sandbox 12348 >/dev/null`);
+diag(`rm -rf $output_file >/dev/null`);
+$sb->wipe_clean($master1_dbh);
+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
+done_testing;