Merge lp:~percona-toolkit-dev/percona-toolkit/pt-table-checksum-refuses-to-run-on-PXC-if-server_id-is-the-same-on-all-nodes-1217466 into lp:~percona-toolkit-dev/percona-toolkit/release-2.2.12

[Frank Cizmich]

problem:
Sometimes PXC setups use the same ID for all nodes ( no consensus yet if this is "right" or "wrong" )
This made pt-table-checksum discard nodes with the same ID although they were different nodes.

fix:
Changed remove_duplicate_cxns() function in Cxn and Percona::XtraDB::Cluster to use wsrep_node_incoming_address as a unique identifier if it's a cluster node.
Also changed pt-table-checksum to populate the %seen_ids param accordingly before passing it to the function.

caveat:
the seen_ids parameter has to be populated carefully using this criteria. At this point only pt-table-checksum uses it.
Maybe a cleaner way would be to create a get_unique_id function in Cxn

[Daniel Nichter (daniel-nichter)]

I added on comment. If you agree, then great, feel free to change code. Otherwise, is good.

review: Approve

[lp:~percona-toolkit-dev/percona-toolkit/pt-table-checksum-refuses-to-run-on-PXC-if-server_id-is-the-same-on-all-nodes-1217466]

627. By Frank Cizmich on 2014-11-05

minor style change

=== modified file 'bin/pt-config-diff'
--- bin/pt-config-diff	2014-09-25 13:48:22 +0000
+++ bin/pt-config-diff	2014-11-05 22:22:52 +0000
@@ -2295,7 +2295,7 @@
       set             => $args{set},
       NAME_lc         => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
       dbh_set         => 0,
-      ask_pass        => $o->get('ask-pass'),
+      ask_pass        => $args{ask_pass},
       DSNParser       => $dp,
       is_cluster_node => undef,
       parent          => $args{parent},
@@ -2306,7 +2306,7 @@
 
 sub connect {
    my ( $self, %opts ) = @_;
-   my $dsn = $self->{dsn};
+   my $dsn = $opts{dsn} || $self->{dsn};
    my $dp  = $self->{DSNParser};
 
    my $dbh = $self->{dbh};
@@ -2325,6 +2325,13 @@
    }
 
    $dbh = $self->set_dbh($dbh);
+   if ( $opts{dsn} ) {
+      $self->{dsn}      = $dsn;
+      $self->{dsn_name} = $dp->as_string($dsn, [qw(h P S)])
+                       || $dp->as_string($dsn, [qw(F)])
+                       || '';
+
+   }
    PTDEBUG && _d($dbh, 'Connected dbh to', $self->{hostname},$self->{dsn_name});
    return $dbh;
 }
@@ -2386,6 +2393,17 @@
    return $self->{hostname} || $self->{dsn_name} || 'unknown host';
 }
 
+sub is_cluster_node {
+   my ($self, $cxn) = @_;
+
+   my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'";
+   PTDEBUG && _d($cxn->name, $sql);
+   my $row = $cxn->dbh->selectrow_arrayref($sql);
+   PTDEBUG && _d(Dumper($row));
+   return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0;
+
+}
+
 sub remove_duplicate_cxns {
    my ($self, %args) = @_;
    my @cxns     = @{$args{cxns}};
@@ -2395,7 +2413,8 @@
 
    for my $cxn ( @cxns ) {
       my $dbh  = $cxn->dbh();
-      my $sql  = q{SELECT @@server_id};
+
+      my $sql  = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id};
       PTDEBUG && _d($sql);
       my ($id) = $dbh->selectrow_array($sql);
       PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id);

=== modified file 'bin/pt-deadlock-logger'
--- bin/pt-deadlock-logger	2014-09-25 13:48:22 +0000
+++ bin/pt-deadlock-logger	2014-11-05 22:22:52 +0000
@@ -2639,7 +2639,7 @@
       set             => $args{set},
       NAME_lc         => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
       dbh_set         => 0,
-      ask_pass        => $o->get('ask-pass'),
+      ask_pass        => $args{ask_pass},
       DSNParser       => $dp,
       is_cluster_node => undef,
       parent          => $args{parent},
@@ -2650,7 +2650,7 @@
 
 sub connect {
    my ( $self, %opts ) = @_;
-   my $dsn = $self->{dsn};
+   my $dsn = $opts{dsn} || $self->{dsn};
    my $dp  = $self->{DSNParser};
 
    my $dbh = $self->{dbh};
@@ -2669,6 +2669,13 @@
    }
 
    $dbh = $self->set_dbh($dbh);
+   if ( $opts{dsn} ) {
+      $self->{dsn}      = $dsn;
+      $self->{dsn_name} = $dp->as_string($dsn, [qw(h P S)])
+                       || $dp->as_string($dsn, [qw(F)])
+                       || '';
+
+   }
    PTDEBUG && _d($dbh, 'Connected dbh to', $self->{hostname},$self->{dsn_name});
    return $dbh;
 }
@@ -2730,6 +2737,17 @@
    return $self->{hostname} || $self->{dsn_name} || 'unknown host';
 }
 
+sub is_cluster_node {
+   my ($self, $cxn) = @_;
+
+   my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'";
+   PTDEBUG && _d($cxn->name, $sql);
+   my $row = $cxn->dbh->selectrow_arrayref($sql);
+   PTDEBUG && _d(Dumper($row));
+   return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0;
+
+}
+
 sub remove_duplicate_cxns {
    my ($self, %args) = @_;
    my @cxns     = @{$args{cxns}};
@@ -2739,7 +2757,8 @@
 
    for my $cxn ( @cxns ) {
       my $dbh  = $cxn->dbh();
-      my $sql  = q{SELECT @@server_id};
+
+      my $sql  = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id};
       PTDEBUG && _d($sql);
       my ($id) = $dbh->selectrow_array($sql);
       PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id);

=== modified file 'bin/pt-fk-error-logger'
--- bin/pt-fk-error-logger	2014-09-25 13:48:22 +0000
+++ bin/pt-fk-error-logger	2014-11-05 22:22:52 +0000
@@ -1791,7 +1791,7 @@
       set             => $args{set},
       NAME_lc         => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
       dbh_set         => 0,
-      ask_pass        => $o->get('ask-pass'),
+      ask_pass        => $args{ask_pass},
       DSNParser       => $dp,
       is_cluster_node => undef,
       parent          => $args{parent},
@@ -1802,7 +1802,7 @@
 
 sub connect {
    my ( $self, %opts ) = @_;
-   my $dsn = $self->{dsn};
+   my $dsn = $opts{dsn} || $self->{dsn};
    my $dp  = $self->{DSNParser};
 
    my $dbh = $self->{dbh};
@@ -1821,6 +1821,13 @@
    }
 
    $dbh = $self->set_dbh($dbh);
+   if ( $opts{dsn} ) {
+      $self->{dsn}      = $dsn;
+      $self->{dsn_name} = $dp->as_string($dsn, [qw(h P S)])
+                       || $dp->as_string($dsn, [qw(F)])
+                       || '';
+
+   }
    PTDEBUG && _d($dbh, 'Connected dbh to', $self->{hostname},$self->{dsn_name});
    return $dbh;
 }
@@ -1882,6 +1889,17 @@
    return $self->{hostname} || $self->{dsn_name} || 'unknown host';
 }
 
+sub is_cluster_node {
+   my ($self, $cxn) = @_;
+
+   my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'";
+   PTDEBUG && _d($cxn->name, $sql);
+   my $row = $cxn->dbh->selectrow_arrayref($sql);
+   PTDEBUG && _d(Dumper($row));
+   return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0;
+
+}
+
 sub remove_duplicate_cxns {
    my ($self, %args) = @_;
    my @cxns     = @{$args{cxns}};
@@ -1891,7 +1909,8 @@
 
    for my $cxn ( @cxns ) {
       my $dbh  = $cxn->dbh();
-      my $sql  = q{SELECT @@server_id};
+
+      my $sql  = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id};
       PTDEBUG && _d($sql);
       my ($id) = $dbh->selectrow_array($sql);
       PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id);

=== modified file 'bin/pt-kill'
--- bin/pt-kill	2014-09-25 13:48:22 +0000
+++ bin/pt-kill	2014-11-05 22:22:52 +0000
@@ -5158,7 +5158,7 @@
       set             => $args{set},
       NAME_lc         => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
       dbh_set         => 0,
-      ask_pass        => $o->get('ask-pass'),
+      ask_pass        => $args{ask_pass},
       DSNParser       => $dp,
       is_cluster_node => undef,
       parent          => $args{parent},
@@ -5169,7 +5169,7 @@
 
 sub connect {
    my ( $self, %opts ) = @_;
-   my $dsn = $self->{dsn};
+   my $dsn = $opts{dsn} || $self->{dsn};
    my $dp  = $self->{DSNParser};
 
    my $dbh = $self->{dbh};
@@ -5188,6 +5188,13 @@
    }
 
    $dbh = $self->set_dbh($dbh);
+   if ( $opts{dsn} ) {
+      $self->{dsn}      = $dsn;
+      $self->{dsn_name} = $dp->as_string($dsn, [qw(h P S)])
+                       || $dp->as_string($dsn, [qw(F)])
+                       || '';
+
+   }
    PTDEBUG && _d($dbh, 'Connected dbh to', $self->{hostname},$self->{dsn_name});
    return $dbh;
 }
@@ -5249,6 +5256,17 @@
    return $self->{hostname} || $self->{dsn_name} || 'unknown host';
 }
 
+sub is_cluster_node {
+   my ($self, $cxn) = @_;
+
+   my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'";
+   PTDEBUG && _d($cxn->name, $sql);
+   my $row = $cxn->dbh->selectrow_arrayref($sql);
+   PTDEBUG && _d(Dumper($row));
+   return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0;
+
+}
+
 sub remove_duplicate_cxns {
    my ($self, %args) = @_;
    my @cxns     = @{$args{cxns}};
@@ -5258,7 +5276,8 @@
 
    for my $cxn ( @cxns ) {
       my $dbh  = $cxn->dbh();
-      my $sql  = q{SELECT @@server_id};
+
+      my $sql  = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id};
       PTDEBUG && _d($sql);
       my ($id) = $dbh->selectrow_array($sql);
       PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id);

=== modified file 'bin/pt-online-schema-change'
--- bin/pt-online-schema-change	2014-09-25 13:48:22 +0000
+++ bin/pt-online-schema-change	2014-11-05 22:22:52 +0000
@@ -3755,7 +3755,7 @@
       set             => $args{set},
       NAME_lc         => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
       dbh_set         => 0,
-      ask_pass        => $o->get('ask-pass'),
+      ask_pass        => $args{ask_pass},
       DSNParser       => $dp,
       is_cluster_node => undef,
       parent          => $args{parent},
@@ -3766,12 +3766,12 @@
 
 sub connect {
    my ( $self, %opts ) = @_;
-   my $dsn = $self->{dsn};
+   my $dsn = $opts{dsn} || $self->{dsn};
    my $dp  = $self->{DSNParser};
 
    my $dbh = $self->{dbh};
    if ( !$dbh || !$dbh->ping() ) {
-      if ( $self->{ask_pass} && !$self->{asked_for_pass} && !defined $dsn->{p}) {
+      if ( $self->{ask_pass} && !$self->{asked_for_pass} ) {
          $dsn->{p} = OptionParser::prompt_noecho("Enter MySQL password: ");
          $self->{asked_for_pass} = 1;
       }
@@ -3785,6 +3785,13 @@
    }
 
    $dbh = $self->set_dbh($dbh);
+   if ( $opts{dsn} ) {
+      $self->{dsn}      = $dsn;
+      $self->{dsn_name} = $dp->as_string($dsn, [qw(h P S)])
+                       || $dp->as_string($dsn, [qw(F)])
+                       || '';
+
+   }
    PTDEBUG && _d($dbh, 'Connected dbh to', $self->{hostname},$self->{dsn_name});
    return $dbh;
 }
@@ -3846,6 +3853,17 @@
    return $self->{hostname} || $self->{dsn_name} || 'unknown host';
 }
 
+sub is_cluster_node {
+   my ($self, $cxn) = @_;
+
+   my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'";
+   PTDEBUG && _d($cxn->name, $sql);
+   my $row = $cxn->dbh->selectrow_arrayref($sql);
+   PTDEBUG && _d(Dumper($row));
+   return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0;
+
+}
+
 sub remove_duplicate_cxns {
    my ($self, %args) = @_;
    my @cxns     = @{$args{cxns}};
@@ -3855,7 +3873,8 @@
 
    for my $cxn ( @cxns ) {
       my $dbh  = $cxn->dbh();
-      my $sql  = q{SELECT @@server_id};
+
+      my $sql  = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id};
       PTDEBUG && _d($sql);
       my ($id) = $dbh->selectrow_array($sql);
       PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id);
@@ -7638,8 +7657,26 @@
 sub remove_duplicate_cxns {
    my ($self, %args) = @_;
    my @cxns     = @{$args{cxns}};
-   my $seen_ids = $args{seen_ids};
-   return Cxn->remove_duplicate_cxns(%args);
+   my $seen_ids = $args{seen_ids} || {};
+   PTDEBUG && _d("Removing duplicates nodes from ", join(" ", map { $_->name } @cxns));
+   my @trimmed_cxns;
+
+   for my $cxn ( @cxns ) {
+      my $dbh  = $cxn->dbh();
+      my $sql  = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id};
+      PTDEBUG && _d($sql);
+      my ($id) = $dbh->selectrow_array($sql);
+      PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id);
+
+      if ( ! $seen_ids->{$id}++ ) {
+         push @trimmed_cxns, $cxn
+      }
+      else {
+         PTDEBUG && _d("Removing ", $cxn->name,
+                       ", ID ", $id, ", because we've already seen it");
+      }
+   }
+   return \@trimmed_cxns;
 }
 
 sub same_cluster {

=== modified file 'bin/pt-table-checksum'
--- bin/pt-table-checksum	2014-09-25 13:48:22 +0000
+++ bin/pt-table-checksum	2014-11-05 22:22:52 +0000
@@ -3533,7 +3533,7 @@
       set             => $args{set},
       NAME_lc         => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
       dbh_set         => 0,
-      ask_pass        => $o->get('ask-pass'),
+      ask_pass        => $args{ask_pass},
       DSNParser       => $dp,
       is_cluster_node => undef,
       parent          => $args{parent},
@@ -3544,7 +3544,7 @@
 
 sub connect {
    my ( $self, %opts ) = @_;
-   my $dsn = $self->{dsn};
+   my $dsn = $opts{dsn} || $self->{dsn};
    my $dp  = $self->{DSNParser};
 
    my $dbh = $self->{dbh};
@@ -3563,6 +3563,13 @@
    }
 
    $dbh = $self->set_dbh($dbh);
+   if ( $opts{dsn} ) {
+      $self->{dsn}      = $dsn;
+      $self->{dsn_name} = $dp->as_string($dsn, [qw(h P S)])
+                       || $dp->as_string($dsn, [qw(F)])
+                       || '';
+
+   }
    PTDEBUG && _d($dbh, 'Connected dbh to', $self->{hostname},$self->{dsn_name});
    return $dbh;
 }
@@ -3624,6 +3631,17 @@
    return $self->{hostname} || $self->{dsn_name} || 'unknown host';
 }
 
+sub is_cluster_node {
+   my ($self, $cxn) = @_;
+
+   my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'";
+   PTDEBUG && _d($cxn->name, $sql);
+   my $row = $cxn->dbh->selectrow_arrayref($sql);
+   PTDEBUG && _d(Dumper($row));
+   return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0;
+
+}
+
 sub remove_duplicate_cxns {
    my ($self, %args) = @_;
    my @cxns     = @{$args{cxns}};
@@ -3633,7 +3651,8 @@
 
    for my $cxn ( @cxns ) {
       my $dbh  = $cxn->dbh();
-      my $sql  = q{SELECT @@server_id};
+
+      my $sql  = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id};
       PTDEBUG && _d($sql);
       my ($id) = $dbh->selectrow_array($sql);
       PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id);
@@ -3788,8 +3807,26 @@
 sub remove_duplicate_cxns {
    my ($self, %args) = @_;
    my @cxns     = @{$args{cxns}};
-   my $seen_ids = $args{seen_ids};
-   return Cxn->remove_duplicate_cxns(%args);
+   my $seen_ids = $args{seen_ids} || {};
+   PTDEBUG && _d("Removing duplicates nodes from ", join(" ", map { $_->name } @cxns));
+   my @trimmed_cxns;
+
+   for my $cxn ( @cxns ) {
+      my $dbh  = $cxn->dbh();
+      my $sql  = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id};
+      PTDEBUG && _d($sql);
+      my ($id) = $dbh->selectrow_array($sql);
+      PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id);
+
+      if ( ! $seen_ids->{$id}++ ) {
+         push @trimmed_cxns, $cxn
+      }
+      else {
+         PTDEBUG && _d("Removing ", $cxn->name,
+                       ", ID ", $id, ", because we've already seen it");
+      }
+   }
+   return \@trimmed_cxns;
 }
 
 sub same_cluster {
@@ -9286,7 +9323,8 @@
       my %seen_ids;
       for my $cxn ($master_cxn, @$slaves) {
          my $dbh  = $cxn->dbh();
-         my $sql  = q{SELECT @@server_id};
+         # if it's a cluster node we use its incoming address as id ( see https://bugs.launchpad.net/percona-toolkit/+bug/1217466 )
+         my $sql  = $cluster->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id};
          PTDEBUG && _d($cxn, $dbh, $sql);
          my ($id) = $dbh->selectrow_array($sql);
          $seen_ids{$id}++;

=== modified file 'bin/pt-upgrade'
--- bin/pt-upgrade	2014-09-25 13:48:22 +0000
+++ bin/pt-upgrade	2014-11-05 22:22:52 +0000
@@ -2464,7 +2464,7 @@
       set             => $args{set},
       NAME_lc         => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
       dbh_set         => 0,
-      ask_pass        => $o->get('ask-pass'),
+      ask_pass        => $args{ask_pass},
       DSNParser       => $dp,
       is_cluster_node => undef,
       parent          => $args{parent},
@@ -2475,7 +2475,7 @@
 
 sub connect {
    my ( $self, %opts ) = @_;
-   my $dsn = $self->{dsn};
+   my $dsn = $opts{dsn} || $self->{dsn};
    my $dp  = $self->{DSNParser};
 
    my $dbh = $self->{dbh};
@@ -2494,6 +2494,13 @@
    }
 
    $dbh = $self->set_dbh($dbh);
+   if ( $opts{dsn} ) {
+      $self->{dsn}      = $dsn;
+      $self->{dsn_name} = $dp->as_string($dsn, [qw(h P S)])
+                       || $dp->as_string($dsn, [qw(F)])
+                       || '';
+
+   }
    PTDEBUG && _d($dbh, 'Connected dbh to', $self->{hostname},$self->{dsn_name});
    return $dbh;
 }
@@ -2555,6 +2562,17 @@
    return $self->{hostname} || $self->{dsn_name} || 'unknown host';
 }
 
+sub is_cluster_node {
+   my ($self, $cxn) = @_;
+
+   my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'";
+   PTDEBUG && _d($cxn->name, $sql);
+   my $row = $cxn->dbh->selectrow_arrayref($sql);
+   PTDEBUG && _d(Dumper($row));
+   return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0;
+
+}
+
 sub remove_duplicate_cxns {
    my ($self, %args) = @_;
    my @cxns     = @{$args{cxns}};
@@ -2564,7 +2582,8 @@
 
    for my $cxn ( @cxns ) {
       my $dbh  = $cxn->dbh();
-      my $sql  = q{SELECT @@server_id};
+
+      my $sql  = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id};
       PTDEBUG && _d($sql);
       my ($id) = $dbh->selectrow_array($sql);
       PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id);

=== modified file 'lib/Cxn.pm'
--- lib/Cxn.pm	2013-12-05 00:14:17 +0000
+++ lib/Cxn.pm	2014-11-05 22:22:52 +0000
@@ -226,6 +226,19 @@
    return $self->{hostname} || $self->{dsn_name} || 'unknown host';
 }
 
+# This is used to help remove_duplicate_cxns detect cluster nodes
+# (which often have unreliable server_id's)
+sub is_cluster_node {
+   my ($self, $cxn) = @_;
+
+   my $sql = "SHOW VARIABLES LIKE 'wsrep\_on'";
+   PTDEBUG && _d($cxn->name, $sql);
+   my $row = $cxn->dbh->selectrow_arrayref($sql);
+   PTDEBUG && _d(Dumper($row));
+   return $row && $row->[1] && ($row->[1] eq 'ON' || $row->[1] eq '1') ? 1 : 0;
+
+}
+
 # There's two reasons why there might be dupes:
 # If the "master" is a cluster node, then a DSN table might have been
 # used, and it may have all nodes' DSNs so the user can run the tool
@@ -233,7 +246,7 @@
 # on the command line.
 # On the other hand, maybe find_cluster_nodes worked, in which case
 # we definitely have a dupe for the master cxn, but we may also have a
-# dupe for every other node if this was unsed in conjunction with a
+# dupe for every other node if this was used in conjunction with a
 # DSN table.
 # So try to detect and remove those.
 sub remove_duplicate_cxns {
@@ -245,7 +258,11 @@
 
    for my $cxn ( @cxns ) {
       my $dbh  = $cxn->dbh();
-      my $sql  = q{SELECT @@server_id};
+
+      # Very often cluster nodes are configured with matching server_id's 
+      # So in that case we'll use its incoming address as its unique identifier
+      # Note: this relies on "seen_ids" being populated using the same strategy  
+      my $sql  = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id};
       PTDEBUG && _d($sql);
       my ($id) = $dbh->selectrow_array($sql);
       PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id);

=== modified file 'lib/Percona/XtraDB/Cluster.pm'
--- lib/Percona/XtraDB/Cluster.pm	2013-04-16 20:40:32 +0000
+++ lib/Percona/XtraDB/Cluster.pm	2014-11-05 22:22:52 +0000
@@ -132,8 +132,29 @@
 sub remove_duplicate_cxns {
    my ($self, %args) = @_;
    my @cxns     = @{$args{cxns}};
-   my $seen_ids = $args{seen_ids};
-   return Cxn->remove_duplicate_cxns(%args);
+   my $seen_ids = $args{seen_ids} || {};
+   PTDEBUG && _d("Removing duplicates nodes from ", join(" ", map { $_->name } @cxns));
+   my @trimmed_cxns;
+
+   for my $cxn ( @cxns ) {
+      my $dbh  = $cxn->dbh();
+      # Very often cluster nodes are configured with matching server_id's
+      # So in that case we'll use its incoming address as its unique identifier
+      # Note: This relies on "seen_ids" being populated using the same strategy  
+      my $sql  = $self->is_cluster_node($cxn) ? q{SELECT @@wsrep_node_incoming_address} : q{SELECT @@server_id};
+      PTDEBUG && _d($sql);
+      my ($id) = $dbh->selectrow_array($sql);
+      PTDEBUG && _d('Server ID for ', $cxn->name, ': ', $id);
+
+      if ( ! $seen_ids->{$id}++ ) {
+         push @trimmed_cxns, $cxn
+      }
+      else {
+         PTDEBUG && _d("Removing ", $cxn->name,
+                       ", ID ", $id, ", because we've already seen it");
+      }
+   }
+   return \@trimmed_cxns;
 }
 
 sub same_cluster {

=== modified file 't/pt-table-checksum/pxc.t'
--- t/pt-table-checksum/pxc.t	2013-04-12 15:58:10 +0000
+++ t/pt-table-checksum/pxc.t	2014-11-05 22:22:52 +0000
@@ -144,54 +144,89 @@
    "Node3 not changed"
 );
 
-for my $args (
-      ["using recusion-method=dsn", '--recursion-method', "dsn=$node1_dsn,D=dsns,t=dsns"],
-      ["using recursion-method=cluster", '--recursion-method', 'cluster']
-   )
-{
-   my $test = shift @$args;
-
-   $output = output(
-      sub { pt_table_checksum::main(@args,
-         @$args)
-      },
-      stderr => 1,
-   );
-
-   is(
-      PerconaTest::count_checksum_results($output, 'errors'),
-      0,
-      "1 diff: no errors ($test)"
-   );
-
-   is(
-      PerconaTest::count_checksum_results($output, 'skipped'),
-      0,
-      "1 diff: no skips ($test)"
-   );
-
-   is(
-      PerconaTest::count_checksum_results($output, 'diffs'),
-      1,
-      "1 diff: 1 diff ($test)"
-   ) or diag($output);
-
-   # 11-17T13:02:54      0      1       26       1       0   0.021 test.t
-   like(
-      $output,
-      qr/^\S+\s+  # ts
-         0\s+     # errors
-         1\s+     # diffs
-         26\s+    # rows
-         \d+\s+   # chunks
-         0\s+     # skipped
-         \S+\s+   # time
-         test.t$  # table
-      /xm,
-      "1 diff: it's in test.t ($test)"
-   );
+sub test_recursion_methods {
+   my $same_ids = shift;
+
+   my ($orig_id_1, $orig_id_2, $orig_id_3);
+
+   if ($same_ids) {
+      # save original values
+      my $sql = 'SELECT @@server_id';
+      ($orig_id_1) = $node1->selectrow_array($sql);
+      ($orig_id_2) = $node2->selectrow_array($sql);
+      ($orig_id_3) = $node3->selectrow_array($sql);
+      # set server_id  value to 1 on all nodes
+      $sql = 'SET GLOBAL server_id = 1';
+      $node1->do($sql);
+      $node2->do($sql);
+      $node3->do($sql);
+   }
+
+   for my $args (
+         ["using recusion-method=dsn", '--recursion-method', "dsn=$node1_dsn,D=dsns,t=dsns"],
+         ["using recursion-method=cluster", '--recursion-method', 'cluster']
+      )
+   {
+      my $test = shift @$args;
+      $test = $same_ids ? $test.' - Nodes with different ids' : $test.' - Nodes with same ids';
+
+      $output = output(
+         sub { pt_table_checksum::main(@args,
+            @$args)
+         },
+         stderr => 1,
+      );
+
+      is(
+         PerconaTest::count_checksum_results($output, 'errors'),
+         0,
+         "1 diff: no errors ($test)"
+      );
+
+      is(
+         PerconaTest::count_checksum_results($output, 'skipped'),
+         0,
+         "1 diff: no skips ($test)"
+      );
+
+      is(
+         PerconaTest::count_checksum_results($output, 'diffs'),
+         1,
+         "1 diff: 1 diff ($test)"
+      ) or diag($output);
+
+      # 11-17T13:02:54      0      1       26       1       0   0.021 test.t
+      like(
+         $output,
+         qr/^\S+\s+  # ts
+            0\s+     # errors
+            1\s+     # diffs
+            26\s+    # rows
+            \d+\s+   # chunks
+            0\s+     # skipped
+            \S+\s+   # time
+            test.t$  # table
+         /xm,
+         "1 diff: it's in test.t ($test)"
+      );
+   }
+
+   if ($same_ids) {
+      # reset server_id's to original values
+      $node1->do("SET GLOBAL server_id = $orig_id_1");
+      $node2->do("SET GLOBAL server_id = $orig_id_2");
+      $node3->do("SET GLOBAL server_id = $orig_id_3");
+   }
+
 }
 
+# test recursion methods
+test_recursion_methods(0);
+
+# test recursion methods when all nodes have the same id
+test_recursion_methods(1);
+
+
 # #############################################################################
 # cluster, node1 -> slave, run on node1
 # #############################################################################
@@ -232,7 +267,7 @@
    # Wait for the slave to apply the binlogs from node1 (its master).
    # Then change it so it's not consistent.
    PerconaTest::wait_for_table($slave_dbh, 'test.t');
-   $sb->wait_for_slaves('cslave1');
+   $sb->wait_for_slaves(master=> 'node1', slave => 'cslave1');
    $slave_dbh->do("update test.t set c='zebra' where c='z'");
 
    $output = output(