Merge lp:~percona-toolkit-dev/percona-toolkit/pt-upgrade-2.2 into lp:percona-toolkit/2.2

[Daniel Nichter]

[Daniel Nichter (daniel-nichter)]

All passing: https://refute.testnoir.com/percona-toolkit/jobs/percona-toolkit-full-branch-test/28/results

review: Approve

[lp:~percona-toolkit-dev/percona-toolkit/pt-upgrade-2.2]

542. By Daniel Nichter on 2013-03-12

Remove --read-timeout. Update docs.

543. By Daniel Nichter on 2013-03-12

Fix and test UpgradeResults::format_query_times001().

544. By Daniel Nichter on 2013-03-12

Functional test and fix query time diffs.

[Daniel Nichter (daniel-nichter)]

The diff is worthless: the whole tool was rewritten.

[Daniel Nichter (daniel-nichter)]

All passing again: https://refute.testnoir.com/percona-toolkit/jobs/percona-toolkit-full-branch-test/29/results/todo

review: Approve

=== modified file 'bin/pt-table-usage'
--- bin/pt-table-usage	2013-03-09 16:57:54 +0000
+++ bin/pt-table-usage	2013-03-12 01:25:25 +0000
@@ -6655,9 +6655,6 @@
          # just tweak the code below to be like pt-query-digest.
          my %alias_for = (
             slowlog   => ['SlowLogParser'],
-            # binlog    => ['BinaryLogParser'],
-            # genlog    => ['GeneralLogParser'],
-            # tcpdump   => ['TcpdumpParser','MySQLProtocolParser'],
          );
          my $type = ['slowlog'];
          $type    = $alias_for{$type->[0]} if $alias_for{$type->[0]};

=== modified file 'bin/pt-upgrade'
--- bin/pt-upgrade	2013-03-09 16:57:54 +0000
+++ bin/pt-upgrade	2013-03-12 01:25:25 +0000
@@ -20,35 +20,33 @@
       Lmo::Types
       Lmo
       DSNParser
-      TableParser
       Quoter
       OptionParser
+      Cxn
       Transformers
-      SlowLogParser
-      EventAggregator
-      QueryParser
-      QueryRewriter
       Daemon
-      ChangeHandler
-      RowDiff
-      TableChunker
-      TableNibbler
-      TableChecksum
-      TableSyncer
-      TableSyncChunk
-      TableSyncNibble
-      TableSyncGroupBy
       Outfile
-      MockSyncStream
-      MockSth
-      ReportFormatter
-      UpgradeReportFormatter
-      CompareResults
-      CompareQueryTimes
-      CompareWarnings
       Retry
       HTTPMicro
       VersionCheck
+      QueryRewriter
+      VersionParser
+      FileIterator
+      QueryIterator
+      EventExecutor
+      UpgradeResults
+      ResultWriter
+      ResultIterator
+      FakeSth
+      SlowLogParser
+      GeneralLogParser
+      BinaryLogParser
+      RawLogParser
+      ProtocolParser
+      TcpdumpParser
+      MySQLProtocolParser
+      Runtime
+      Progress
    ));
 }
 
@@ -1138,417 +1136,6 @@
 # ###########################################################################
 
 # ###########################################################################
-# TableParser package
-# This package is a copy without comments from the original.  The original
-# with comments and its test file can be found in the Bazaar repository at,
-#   lib/TableParser.pm
-#   t/lib/TableParser.t
-# See https://launchpad.net/percona-toolkit for more information.
-# ###########################################################################
-{
-package TableParser;
-
-use strict;
-use warnings FATAL => 'all';
-use English qw(-no_match_vars);
-use constant PTDEBUG => $ENV{PTDEBUG} || 0;
-
-use Data::Dumper;
-$Data::Dumper::Indent    = 1;
-$Data::Dumper::Sortkeys  = 1;
-$Data::Dumper::Quotekeys = 0;
-
-local $EVAL_ERROR;
-eval {
-   require Quoter;
-};
-
-sub new {
-   my ( $class, %args ) = @_;
-   my $self = { %args };
-   $self->{Quoter} ||= Quoter->new();
-   return bless $self, $class;
-}
-
-sub Quoter { shift->{Quoter} }
-
-sub get_create_table {
-   my ( $self, $dbh, $db, $tbl ) = @_;
-   die "I need a dbh parameter" unless $dbh;
-   die "I need a db parameter"  unless $db;
-   die "I need a tbl parameter" unless $tbl;
-   my $q = $self->{Quoter};
-
-   my $new_sql_mode
-      = q{/*!40101 SET @OLD_SQL_MODE := @@SQL_MODE, }
-      . q{@@SQL_MODE := '', }
-      . q{@OLD_QUOTE := @@SQL_QUOTE_SHOW_CREATE, }
-      . q{@@SQL_QUOTE_SHOW_CREATE := 1 */};
-
-   my $old_sql_mode
-      = q{/*!40101 SET @@SQL_MODE := @OLD_SQL_MODE, }
-      . q{@@SQL_QUOTE_SHOW_CREATE := @OLD_QUOTE */};
-
-   PTDEBUG && _d($new_sql_mode);
-   eval { $dbh->do($new_sql_mode); };
-   PTDEBUG && $EVAL_ERROR && _d($EVAL_ERROR);
-
-   my $use_sql = 'USE ' . $q->quote($db);
-   PTDEBUG && _d($dbh, $use_sql);
-   $dbh->do($use_sql);
-
-   my $show_sql = "SHOW CREATE TABLE " . $q->quote($db, $tbl);
-   PTDEBUG && _d($show_sql);
-   my $href;
-   eval { $href = $dbh->selectrow_hashref($show_sql); };
-   if ( my $e = $EVAL_ERROR ) {
-      PTDEBUG && _d($old_sql_mode);
-      $dbh->do($old_sql_mode);
-
-      die $e;
-   }
-
-   PTDEBUG && _d($old_sql_mode);
-   $dbh->do($old_sql_mode);
-
-   my ($key) = grep { m/create (?:table|view)/i } keys %$href;
-   if ( !$key ) {
-      die "Error: no 'Create Table' or 'Create View' in result set from "
-         . "$show_sql: " . Dumper($href);
-   }
-
-   return $href->{$key};
-}
-
-sub parse {
-   my ( $self, $ddl, $opts ) = @_;
-   return unless $ddl;
-
-   if ( $ddl =~ m/CREATE (?:TEMPORARY )?TABLE "/ ) {
-      $ddl = $self->ansi_to_legacy($ddl);
-   }
-   elsif ( $ddl !~ m/CREATE (?:TEMPORARY )?TABLE `/ ) {
-      die "TableParser doesn't handle CREATE TABLE without quoting.";
-   }
-
-   my ($name)     = $ddl =~ m/CREATE (?:TEMPORARY )?TABLE\s+(`.+?`)/;
-   (undef, $name) = $self->{Quoter}->split_unquote($name) if $name;
-
-   $ddl =~ s/(`[^`]+`)/\L$1/g;
-
-   my $engine = $self->get_engine($ddl);
-
-   my @defs   = $ddl =~ m/^(\s+`.*?),?$/gm;
-   my @cols   = map { $_ =~ m/`([^`]+)`/ } @defs;
-   PTDEBUG && _d('Table cols:', join(', ', map { "`$_`" } @cols));
-
-   my %def_for;
-   @def_for{@cols} = @defs;
-
-   my (@nums, @null);
-   my (%type_for, %is_nullable, %is_numeric, %is_autoinc);
-   foreach my $col ( @cols ) {
-      my $def = $def_for{$col};
-      my ( $type ) = $def =~ m/`[^`]+`\s([a-z]+)/;
-      die "Can't determine column type for $def" unless $type;
-      $type_for{$col} = $type;
-      if ( $type =~ m/(?:(?:tiny|big|medium|small)?int|float|double|decimal|year)/ ) {
-         push @nums, $col;
-         $is_numeric{$col} = 1;
-      }
-      if ( $def !~ m/NOT NULL/ ) {
-         push @null, $col;
-         $is_nullable{$col} = 1;
-      }
-      $is_autoinc{$col} = $def =~ m/AUTO_INCREMENT/i ? 1 : 0;
-   }
-
-   my ($keys, $clustered_key) = $self->get_keys($ddl, $opts, \%is_nullable);
-
-   my ($charset) = $ddl =~ m/DEFAULT CHARSET=(\w+)/;
-
-   return {
-      name           => $name,
-      cols           => \@cols,
-      col_posn       => { map { $cols[$_] => $_ } 0..$#cols },
-      is_col         => { map { $_ => 1 } @cols },
-      null_cols      => \@null,
-      is_nullable    => \%is_nullable,
-      is_autoinc     => \%is_autoinc,
-      clustered_key  => $clustered_key,
-      keys           => $keys,
-      defs           => \%def_for,
-      numeric_cols   => \@nums,
-      is_numeric     => \%is_numeric,
-      engine         => $engine,
-      type_for       => \%type_for,
-      charset        => $charset,
-   };
-}
-
-sub sort_indexes {
-   my ( $self, $tbl ) = @_;
-
-   my @indexes
-      = sort {
-         (($a ne 'PRIMARY') <=> ($b ne 'PRIMARY'))
-         || ( !$tbl->{keys}->{$a}->{is_unique} <=> !$tbl->{keys}->{$b}->{is_unique} )
-         || ( $tbl->{keys}->{$a}->{is_nullable} <=> $tbl->{keys}->{$b}->{is_nullable} )
-         || ( scalar(@{$tbl->{keys}->{$a}->{cols}}) <=> scalar(@{$tbl->{keys}->{$b}->{cols}}) )
-      }
-      grep {
-         $tbl->{keys}->{$_}->{type} eq 'BTREE'
-      }
-      sort keys %{$tbl->{keys}};
-
-   PTDEBUG && _d('Indexes sorted best-first:', join(', ', @indexes));
-   return @indexes;
-}
-
-sub find_best_index {
-   my ( $self, $tbl, $index ) = @_;
-   my $best;
-   if ( $index ) {
-      ($best) = grep { uc $_ eq uc $index } keys %{$tbl->{keys}};
-   }
-   if ( !$best ) {
-      if ( $index ) {
-         die "Index '$index' does not exist in table";
-      }
-      else {
-         ($best) = $self->sort_indexes($tbl);
-      }
-   }
-   PTDEBUG && _d('Best index found is', $best);
-   return $best;
-}
-
-sub find_possible_keys {
-   my ( $self, $dbh, $database, $table, $quoter, $where ) = @_;
-   return () unless $where;
-   my $sql = 'EXPLAIN SELECT * FROM ' . $quoter->quote($database, $table)
-      . ' WHERE ' . $where;
-   PTDEBUG && _d($sql);
-   my $expl = $dbh->selectrow_hashref($sql);
-   $expl = { map { lc($_) => $expl->{$_} } keys %$expl };
-   if ( $expl->{possible_keys} ) {
-      PTDEBUG && _d('possible_keys =', $expl->{possible_keys});
-      my @candidates = split(',', $expl->{possible_keys});
-      my %possible   = map { $_ => 1 } @candidates;
-      if ( $expl->{key} ) {
-         PTDEBUG && _d('MySQL chose', $expl->{key});
-         unshift @candidates, grep { $possible{$_} } split(',', $expl->{key});
-         PTDEBUG && _d('Before deduping:', join(', ', @candidates));
-         my %seen;
-         @candidates = grep { !$seen{$_}++ } @candidates;
-      }
-      PTDEBUG && _d('Final list:', join(', ', @candidates));
-      return @candidates;
-   }
-   else {
-      PTDEBUG && _d('No keys in possible_keys');
-      return ();
-   }
-}
-
-sub check_table {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(dbh db tbl);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my ($dbh, $db, $tbl) = @args{@required_args};
-   my $q      = $self->{Quoter} || 'Quoter';
-   my $db_tbl = $q->quote($db, $tbl);
-   PTDEBUG && _d('Checking', $db_tbl);
-
-   my $sql = "SHOW TABLES FROM " . $q->quote($db)
-           . ' LIKE ' . $q->literal_like($tbl);
-   PTDEBUG && _d($sql);
-   my $row;
-   eval {
-      $row = $dbh->selectrow_arrayref($sql);
-   };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d($EVAL_ERROR);
-      return 0;
-   }
-   if ( !$row->[0] || $row->[0] ne $tbl ) {
-      PTDEBUG && _d('Table does not exist');
-      return 0;
-   }
-
-   PTDEBUG && _d('Table', $db, $tbl, 'exists');
-   return 1;
-
-}
-
-sub get_engine {
-   my ( $self, $ddl, $opts ) = @_;
-   my ( $engine ) = $ddl =~ m/\).*?(?:ENGINE|TYPE)=(\w+)/;
-   PTDEBUG && _d('Storage engine:', $engine);
-   return $engine || undef;
-}
-
-sub get_keys {
-   my ( $self, $ddl, $opts, $is_nullable ) = @_;
-   my $engine        = $self->get_engine($ddl);
-   my $keys          = {};
-   my $clustered_key = undef;
-
-   KEY:
-   foreach my $key ( $ddl =~ m/^  ((?:[A-Z]+ )?KEY .*)$/gm ) {
-
-      next KEY if $key =~ m/FOREIGN/;
-
-      my $key_ddl = $key;
-      PTDEBUG && _d('Parsed key:', $key_ddl);
-
-      if ( !$engine || $engine !~ m/MEMORY|HEAP/ ) {
-         $key =~ s/USING HASH/USING BTREE/;
-      }
-
-      my ( $type, $cols ) = $key =~ m/(?:USING (\w+))? \((.+)\)/;
-      my ( $special ) = $key =~ m/(FULLTEXT|SPATIAL)/;
-      $type = $type || $special || 'BTREE';
-      my ($name) = $key =~ m/(PRIMARY|`[^`]*`)/;
-      my $unique = $key =~ m/PRIMARY|UNIQUE/ ? 1 : 0;
-      my @cols;
-      my @col_prefixes;
-      foreach my $col_def ( $cols =~ m/`[^`]+`(?:\(\d+\))?/g ) {
-         my ($name, $prefix) = $col_def =~ m/`([^`]+)`(?:\((\d+)\))?/;
-         push @cols, $name;
-         push @col_prefixes, $prefix;
-      }
-      $name =~ s/`//g;
-
-      PTDEBUG && _d( $name, 'key cols:', join(', ', map { "`$_`" } @cols));
-
-      $keys->{$name} = {
-         name         => $name,
-         type         => $type,
-         colnames     => $cols,
-         cols         => \@cols,
-         col_prefixes => \@col_prefixes,
-         is_unique    => $unique,
-         is_nullable  => scalar(grep { $is_nullable->{$_} } @cols),
-         is_col       => { map { $_ => 1 } @cols },
-         ddl          => $key_ddl,
-      };
-
-      if ( ($engine || '') =~ m/InnoDB/i && !$clustered_key ) {
-         my $this_key = $keys->{$name};
-         if ( $this_key->{name} eq 'PRIMARY' ) {
-            $clustered_key = 'PRIMARY';
-         }
-         elsif ( $this_key->{is_unique} && !$this_key->{is_nullable} ) {
-            $clustered_key = $this_key->{name};
-         }
-         PTDEBUG && $clustered_key && _d('This key is the clustered key');
-      }
-   }
-
-   return $keys, $clustered_key;
-}
-
-sub get_fks {
-   my ( $self, $ddl, $opts ) = @_;
-   my $q   = $self->{Quoter};
-   my $fks = {};
-
-   foreach my $fk (
-      $ddl =~ m/CONSTRAINT .* FOREIGN KEY .* REFERENCES [^\)]*\)/mg )
-   {
-      my ( $name ) = $fk =~ m/CONSTRAINT `(.*?)`/;
-      my ( $cols ) = $fk =~ m/FOREIGN KEY \(([^\)]+)\)/;
-      my ( $parent, $parent_cols ) = $fk =~ m/REFERENCES (\S+) \(([^\)]+)\)/;
-
-      my ($db, $tbl) = $q->split_unquote($parent, $opts->{database});
-      my %parent_tbl = (tbl => $tbl);
-      $parent_tbl{db} = $db if $db;
-
-      if ( $parent !~ m/\./ && $opts->{database} ) {
-         $parent = $q->quote($opts->{database}) . ".$parent";
-      }
-
-      $fks->{$name} = {
-         name           => $name,
-         colnames       => $cols,
-         cols           => [ map { s/[ `]+//g; $_; } split(',', $cols) ],
-         parent_tbl     => \%parent_tbl,
-         parent_tblname => $parent,
-         parent_cols    => [ map { s/[ `]+//g; $_; } split(',', $parent_cols) ],
-         parent_colnames=> $parent_cols,
-         ddl            => $fk,
-      };
-   }
-
-   return $fks;
-}
-
-sub remove_auto_increment {
-   my ( $self, $ddl ) = @_;
-   $ddl =~ s/(^\).*?) AUTO_INCREMENT=\d+\b/$1/m;
-   return $ddl;
-}
-
-sub get_table_status {
-   my ( $self, $dbh, $db, $like ) = @_;
-   my $q = $self->{Quoter};
-   my $sql = "SHOW TABLE STATUS FROM " . $q->quote($db);
-   my @params;
-   if ( $like ) {
-      $sql .= ' LIKE ?';
-      push @params, $like;
-   }
-   PTDEBUG && _d($sql, @params);
-   my $sth = $dbh->prepare($sql);
-   eval { $sth->execute(@params); };
-   if ($EVAL_ERROR) {
-      PTDEBUG && _d($EVAL_ERROR);
-      return;
-   }
-   my @tables = @{$sth->fetchall_arrayref({})};
-   @tables = map {
-      my %tbl; # Make a copy with lowercased keys
-      @tbl{ map { lc $_ } keys %$_ } = values %$_;
-      $tbl{engine} ||= $tbl{type} || $tbl{comment};
-      delete $tbl{type};
-      \%tbl;
-   } @tables;
-   return @tables;
-}
-
-my $ansi_quote_re = qr/" [^"]* (?: "" [^"]* )* (?<=.) "/ismx;
-sub ansi_to_legacy {
-   my ($self, $ddl) = @_;
-   $ddl =~ s/($ansi_quote_re)/ansi_quote_replace($1)/ge;
-   return $ddl;
-}
-
-sub ansi_quote_replace {
-   my ($val) = @_;
-   $val =~ s/^"|"$//g;
-   $val =~ s/`/``/g;
-   $val =~ s/""/"/g;
-   return "`$val`";
-}
-
-sub _d {
-   my ($package, undef, $line) = caller 0;
-   @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
-        map { defined $_ ? $_ : 'undef' }
-        @_;
-   print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
-}
-
-1;
-}
-# ###########################################################################
-# End TableParser package
-# ###########################################################################
-
-# ###########################################################################
 # Quoter package
 # This package is a copy without comments from the original.  The original
 # with comments and its test file can be found in the Bazaar repository at,
@@ -2765,6 +2352,155 @@
 # ###########################################################################
 
 # ###########################################################################
+# Cxn package
+# This package is a copy without comments from the original.  The original
+# with comments and its test file can be found in the Bazaar repository at,
+#   lib/Cxn.pm
+#   t/lib/Cxn.t
+# See https://launchpad.net/percona-toolkit for more information.
+# ###########################################################################
+{
+package Cxn;
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Scalar::Util qw(blessed);
+use constant {
+   PTDEBUG => $ENV{PTDEBUG} || 0,
+   PERCONA_TOOLKIT_TEST_USE_DSN_NAMES => $ENV{PERCONA_TOOLKIT_TEST_USE_DSN_NAMES} || 0,
+};
+
+sub new {
+   my ( $class, %args ) = @_;
+   my @required_args = qw(DSNParser OptionParser);
+   foreach my $arg ( @required_args ) {
+      die "I need a $arg argument" unless $args{$arg};
+   };
+   my ($dp, $o) = @args{@required_args};
+
+   my $dsn_defaults = $dp->parse_options($o);
+   my $prev_dsn     = $args{prev_dsn};
+   my $dsn          = $args{dsn};
+   if ( !$dsn ) {
+      $args{dsn_string} ||= 'h=' . ($dsn_defaults->{h} || 'localhost');
+
+      $dsn = $dp->parse(
+         $args{dsn_string}, $prev_dsn, $dsn_defaults);
+   }
+   elsif ( $prev_dsn ) {
+      $dsn = $dp->copy($prev_dsn, $dsn);
+   }
+
+   my $dsn_name = $dp->as_string($dsn, [qw(h P S)])
+               || $dp->as_string($dsn, [qw(F)])
+               || '';
+
+   my $self = {
+      dsn          => $dsn,
+      dbh          => $args{dbh},
+      dsn_name     => $dsn_name,
+      hostname     => '',
+      set          => $args{set},
+      NAME_lc      => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
+      dbh_set      => 0,
+      ask_pass     => $o->get('ask-pass'),
+      DSNParser    => $dp,
+      is_cluster_node => undef,
+   };
+
+   return bless $self, $class;
+}
+
+sub connect {
+   my ( $self ) = @_;
+   my $dsn = $self->{dsn};
+   my $dp  = $self->{DSNParser};
+
+   my $dbh = $self->{dbh};
+   if ( !$dbh || !$dbh->ping() ) {
+      if ( $self->{ask_pass} && !$self->{asked_for_pass} ) {
+         $dsn->{p} = OptionParser::prompt_noecho("Enter MySQL password: ");
+         $self->{asked_for_pass} = 1;
+      }
+      $dbh = $dp->get_dbh($dp->get_cxn_params($dsn),  { AutoCommit => 1 });
+   }
+   PTDEBUG && _d($dbh, 'Connected dbh to', $self->{name});
+
+   return $self->set_dbh($dbh);
+}
+
+sub set_dbh {
+   my ($self, $dbh) = @_;
+
+   if ( $self->{dbh} && $self->{dbh} == $dbh && $self->{dbh_set} ) {
+      PTDEBUG && _d($dbh, 'Already set dbh');
+      return $dbh;
+   }
+
+   PTDEBUG && _d($dbh, 'Setting dbh');
+
+   $dbh->{FetchHashKeyName} = 'NAME_lc' if $self->{NAME_lc};
+
+   my $sql = 'SELECT @@hostname, @@server_id';
+   PTDEBUG && _d($dbh, $sql);
+   my ($hostname, $server_id) = $dbh->selectrow_array($sql);
+   PTDEBUG && _d($dbh, 'hostname:', $hostname, $server_id);
+   if ( $hostname ) {
+      $self->{hostname} = $hostname;
+   }
+
+   if ( my $set = $self->{set}) {
+      $set->($dbh);
+   }
+
+   $self->{dbh}     = $dbh;
+   $self->{dbh_set} = 1;
+   return $dbh;
+}
+
+sub dbh {
+   my ($self) = @_;
+   return $self->{dbh};
+}
+
+sub dsn {
+   my ($self) = @_;
+   return $self->{dsn};
+}
+
+sub name {
+   my ($self) = @_;
+   return $self->{dsn_name} if PERCONA_TOOLKIT_TEST_USE_DSN_NAMES;
+   return $self->{hostname} || $self->{dsn_name} || 'unknown host';
+}
+
+sub DESTROY {
+   my ($self) = @_;
+   if ( $self->{dbh}
+         && blessed($self->{dbh})
+         && $self->{dbh}->can("disconnect") ) {
+      PTDEBUG && _d('Disconnecting dbh', $self->{dbh}, $self->{name});
+      $self->{dbh}->disconnect();
+   }
+   return;
+}
+
+sub _d {
+   my ($package, undef, $line) = caller 0;
+   @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
+        map { defined $_ ? $_ : 'undef' }
+        @_;
+   print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
+}
+
+1;
+}
+# ###########################################################################
+# End Cxn package
+# ###########################################################################
+
+# ###########################################################################
 # Transformers package
 # This package is a copy without comments from the original.  The original
 # with comments and its test file can be found in the Bazaar repository at,
@@ -3118,1909 +2854,6 @@
 # ###########################################################################
 
 # ###########################################################################
-# SlowLogParser package
-# This package is a copy without comments from the original.  The original
-# with comments and its test file can be found in the Bazaar repository at,
-#   lib/SlowLogParser.pm
-#   t/lib/SlowLogParser.t
-# See https://launchpad.net/percona-toolkit for more information.
-# ###########################################################################
-{
-package SlowLogParser;
-
-use strict;
-use warnings FATAL => 'all';
-use English qw(-no_match_vars);
-use constant PTDEBUG => $ENV{PTDEBUG} || 0;
-
-use Data::Dumper;
-$Data::Dumper::Indent    = 1;
-$Data::Dumper::Sortkeys  = 1;
-$Data::Dumper::Quotekeys = 0;
-
-sub new {
-   my ( $class ) = @_;
-   my $self = {
-      pending => [],
-   };
-   return bless $self, $class;
-}
-
-my $slow_log_ts_line = qr/^# Time: ([0-9: ]{15})/;
-my $slow_log_uh_line = qr/# User\@Host: ([^\[]+|\[[^[]+\]).*?@ (\S*) \[(.*)\]/;
-my $slow_log_hd_line = qr{
-      ^(?:
-      T[cC][pP]\s[pP]ort:\s+\d+ # case differs on windows/unix
-      |
-      [/A-Z].*mysqld,\sVersion.*(?:started\swith:|embedded\slibrary)
-      |
-      Time\s+Id\s+Command
-      ).*\n
-   }xm;
-
-sub parse_event {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(next_event tell);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my ($next_event, $tell) = @args{@required_args};
-
-   my $pending = $self->{pending};
-   local $INPUT_RECORD_SEPARATOR = ";\n#";
-   my $trimlen    = length($INPUT_RECORD_SEPARATOR);
-   my $pos_in_log = $tell->();
-   my $stmt;
-
-   EVENT:
-   while (
-         defined($stmt = shift @$pending)
-      or defined($stmt = $next_event->())
-   ) {
-      my @properties = ('cmd', 'Query', 'pos_in_log', $pos_in_log);
-      $pos_in_log = $tell->();
-
-      if ( $stmt =~ s/$slow_log_hd_line//go ){ # Throw away header lines in log
-         my @chunks = split(/$INPUT_RECORD_SEPARATOR/o, $stmt);
-         if ( @chunks > 1 ) {
-            PTDEBUG && _d("Found multiple chunks");
-            $stmt = shift @chunks;
-            unshift @$pending, @chunks;
-         }
-      }
-
-      $stmt = '#' . $stmt unless $stmt =~ m/\A#/;
-      $stmt =~ s/;\n#?\Z//;
-
-
-      my ($got_ts, $got_uh, $got_ac, $got_db, $got_set, $got_embed);
-      my $pos = 0;
-      my $len = length($stmt);
-      my $found_arg = 0;
-      LINE:
-      while ( $stmt =~ m/^(.*)$/mg ) { # /g is important, requires scalar match.
-         $pos     = pos($stmt);  # Be careful not to mess this up!
-         my $line = $1;          # Necessary for /g and pos() to work.
-         PTDEBUG && _d($line);
-
-         if ($line =~ m/^(?:#|use |SET (?:last_insert_id|insert_id|timestamp))/o) {
-
-            if ( !$got_ts && (my ( $time ) = $line =~ m/$slow_log_ts_line/o)) {
-               PTDEBUG && _d("Got ts", $time);
-               push @properties, 'ts', $time;
-               ++$got_ts;
-               if ( !$got_uh
-                  && ( my ( $user, $host, $ip ) = $line =~ m/$slow_log_uh_line/o )
-               ) {
-                  PTDEBUG && _d("Got user, host, ip", $user, $host, $ip);
-                  push @properties, 'user', $user, 'host', $host, 'ip', $ip;
-                  ++$got_uh;
-               }
-            }
-
-            elsif ( !$got_uh
-                  && ( my ( $user, $host, $ip ) = $line =~ m/$slow_log_uh_line/o )
-            ) {
-               PTDEBUG && _d("Got user, host, ip", $user, $host, $ip);
-               push @properties, 'user', $user, 'host', $host, 'ip', $ip;
-               ++$got_uh;
-            }
-
-            elsif (!$got_ac && $line =~ m/^# (?:administrator command:.*)$/) {
-               PTDEBUG && _d("Got admin command");
-               $line =~ s/^#\s+//;  # string leading "# ".
-               push @properties, 'cmd', 'Admin', 'arg', $line;
-               push @properties, 'bytes', length($properties[-1]);
-               ++$found_arg;
-               ++$got_ac;
-            }
-
-            elsif ( $line =~ m/^# +[A-Z][A-Za-z_]+: \S+/ ) { # Make the test cheap!
-               PTDEBUG && _d("Got some line with properties");
-
-               if ( $line =~ m/Schema:\s+\w+: / ) {
-                  PTDEBUG && _d('Removing empty Schema attrib');
-                  $line =~ s/Schema:\s+//;
-                  PTDEBUG && _d($line);
-               }
-
-               my @temp = $line =~ m/(\w+):\s+(\S+|\Z)/g;
-               push @properties, @temp;
-            }
-
-            elsif ( !$got_db && (my ( $db ) = $line =~ m/^use ([^;]+)/ ) ) {
-               PTDEBUG && _d("Got a default database:", $db);
-               push @properties, 'db', $db;
-               ++$got_db;
-            }
-
-            elsif (!$got_set && (my ($setting) = $line =~ m/^SET\s+([^;]*)/)) {
-               PTDEBUG && _d("Got some setting:", $setting);
-               push @properties, split(/,|\s*=\s*/, $setting);
-               ++$got_set;
-            }
-
-            if ( !$found_arg && $pos == $len ) {
-               PTDEBUG && _d("Did not find arg, looking for special cases");
-               local $INPUT_RECORD_SEPARATOR = ";\n";  # get next line
-               if ( defined(my $l = $next_event->()) ) {
-                  if ( $l =~ /^\s*[A-Z][a-z_]+: / ) {
-                     PTDEBUG && _d("Found NULL query before", $l);
-                     local $INPUT_RECORD_SEPARATOR = ";\n#";
-                     my $rest_of_event = $next_event->();
-                     push @{$self->{pending}}, $l . $rest_of_event;
-                     push @properties, 'cmd', 'Query', 'arg', '/* No query */';
-                     push @properties, 'bytes', 0;
-                     $found_arg++;
-                  }
-                  else {
-                     chomp $l;
-                     $l =~ s/^\s+//;
-                     PTDEBUG && _d("Found admin statement", $l);
-                     push @properties, 'cmd', 'Admin', 'arg', $l;
-                     push @properties, 'bytes', length($properties[-1]);
-                     $found_arg++;
-                  }
-               }
-               else {
-                  PTDEBUG && _d("I can't figure out what to do with this line");
-                  next EVENT;
-               }
-            }
-         }
-         else {
-            PTDEBUG && _d("Got the query/arg line");
-            my $arg = substr($stmt, $pos - length($line));
-            push @properties, 'arg', $arg, 'bytes', length($arg);
-            if ( $args{misc} && $args{misc}->{embed}
-               && ( my ($e) = $arg =~ m/($args{misc}->{embed})/)
-            ) {
-               push @properties, $e =~ m/$args{misc}->{capture}/g;
-            }
-            last LINE;
-         }
-      }
-
-      PTDEBUG && _d('Properties of event:', Dumper(\@properties));
-      my $event = { @properties };
-      if ( $args{stats} ) {
-         $args{stats}->{events_read}++;
-         $args{stats}->{events_parsed}++;
-      }
-      return $event;
-   } # EVENT
-
-   @$pending = ();
-   $args{oktorun}->(0) if $args{oktorun};
-   return;
-}
-
-sub _d {
-   my ($package, undef, $line) = caller 0;
-   @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
-        map { defined $_ ? $_ : 'undef' }
-        @_;
-   print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
-}
-
-1;
-}
-# ###########################################################################
-# End SlowLogParser package
-# ###########################################################################
-
-# ###########################################################################
-# EventAggregator package
-# This package is a copy without comments from the original.  The original
-# with comments and its test file can be found in the Bazaar repository at,
-#   lib/EventAggregator.pm
-#   t/lib/EventAggregator.t
-# See https://launchpad.net/percona-toolkit for more information.
-# ###########################################################################
-{
-package EventAggregator;
-
-use strict;
-use warnings FATAL => 'all';
-use English qw(-no_match_vars);
-use constant PTDEBUG => $ENV{PTDEBUG} || 0;
-
-use List::Util qw(min max);
-use Data::Dumper;
-$Data::Dumper::Indent    = 1;
-$Data::Dumper::Sortkeys  = 1;
-$Data::Dumper::Quotekeys = 0;
-
-use constant BUCK_SIZE   => 1.05;
-use constant BASE_LOG    => log(BUCK_SIZE);
-use constant BASE_OFFSET => abs(1 - log(0.000001) / BASE_LOG); # 284.1617969
-use constant NUM_BUCK    => 1000;
-use constant MIN_BUCK    => .000001;
-
-my @buck_vals = map { bucket_value($_); } (0..NUM_BUCK-1);
-
-sub new {
-   my ( $class, %args ) = @_;
-   foreach my $arg ( qw(groupby worst) ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my $attributes = $args{attributes} || {};
-   my $self = {
-      groupby        => $args{groupby},
-      detect_attribs => scalar keys %$attributes == 0 ? 1 : 0,
-      all_attribs    => [ keys %$attributes ],
-      ignore_attribs => {
-         map  { $_ => $args{attributes}->{$_} }
-         grep { $_ ne $args{groupby} }
-         @{$args{ignore_attributes}}
-      },
-      attributes     => {
-         map  { $_ => $args{attributes}->{$_} }
-         grep { $_ ne $args{groupby} }
-         keys %$attributes
-      },
-      alt_attribs    => {
-         map  { $_ => make_alt_attrib(@{$args{attributes}->{$_}}) }
-         grep { $_ ne $args{groupby} }
-         keys %$attributes
-      },
-      worst          => $args{worst},
-      unroll_limit   => $args{unroll_limit} || 1000,
-      attrib_limit   => $args{attrib_limit},
-      result_classes => {},
-      result_globals => {},
-      result_samples => {},
-      class_metrics  => {},
-      global_metrics => {},
-      n_events       => 0,
-      unrolled_loops => undef,
-      type_for       => { %{$args{type_for} || { Query_time => 'num' }} },
-   };
-   return bless $self, $class;
-}
-
-sub reset_aggregated_data {
-   my ( $self ) = @_;
-   foreach my $class ( values %{$self->{result_classes}} ) {
-      foreach my $attrib ( values %$class ) {
-         delete @{$attrib}{keys %$attrib};
-      }
-   }
-   foreach my $class ( values %{$self->{result_globals}} ) {
-      delete @{$class}{keys %$class};
-   }
-   delete @{$self->{result_samples}}{keys %{$self->{result_samples}}};
-   $self->{n_events} = 0;
-}
-
-sub aggregate {
-   my ( $self, $event ) = @_;
-
-   my $group_by = $event->{$self->{groupby}};
-   return unless defined $group_by;
-
-   $self->{n_events}++;
-   PTDEBUG && _d('Event', $self->{n_events});
-
-   return $self->{unrolled_loops}->($self, $event, $group_by)
-      if $self->{unrolled_loops};
-
-   if ( $self->{n_events} <= $self->{unroll_limit} ) {
-
-      $self->add_new_attributes($event) if $self->{detect_attribs};
-
-      ATTRIB:
-      foreach my $attrib ( keys %{$self->{attributes}} ) {
-
-         if ( !exists $event->{$attrib} ) {
-            PTDEBUG && _d("attrib doesn't exist in event:", $attrib);
-            my $alt_attrib = $self->{alt_attribs}->{$attrib}->($event);
-            PTDEBUG && _d('alt attrib:', $alt_attrib);
-            next ATTRIB unless $alt_attrib;
-         }
-
-         GROUPBY:
-         foreach my $val ( ref $group_by ? @$group_by : ($group_by) ) {
-            my $class_attrib  = $self->{result_classes}->{$val}->{$attrib} ||= {};
-            my $global_attrib = $self->{result_globals}->{$attrib} ||= {};
-            my $samples       = $self->{result_samples};
-            my $handler = $self->{handlers}->{ $attrib };
-            if ( !$handler ) {
-               $handler = $self->make_handler(
-                  event      => $event,
-                  attribute  => $attrib,
-                  alternates => $self->{attributes}->{$attrib},
-                  worst      => $self->{worst} eq $attrib,
-               );
-               $self->{handlers}->{$attrib} = $handler;
-            }
-            next GROUPBY unless $handler;
-            $samples->{$val} ||= $event; # Initialize to the first event.
-            $handler->($event, $class_attrib, $global_attrib, $samples, $group_by);
-         }
-      }
-   }
-   else {
-      $self->_make_unrolled_loops($event);
-      $self->{unrolled_loops}->($self, $event, $group_by);
-   }
-
-   return;
-}
-
-sub _make_unrolled_loops {
-   my ( $self, $event ) = @_;
-
-   my $group_by = $event->{$self->{groupby}};
-
-   my @attrs   = grep { $self->{handlers}->{$_} } keys %{$self->{attributes}};
-   my $globs   = $self->{result_globals}; # Global stats for each
-   my $samples = $self->{result_samples};
-
-   my @lines = (
-      'my ( $self, $event, $group_by ) = @_;',
-      'my ($val, $class, $global, $idx);',
-      (ref $group_by ? ('foreach my $group_by ( @$group_by ) {') : ()),
-      'my $temp = $self->{result_classes}->{ $group_by }
-         ||= { map { $_ => { } } @attrs };',
-      '$samples->{$group_by} ||= $event;', # Always start with the first.
-   );
-   foreach my $i ( 0 .. $#attrs ) {
-      push @lines, (
-         '$class  = $temp->{\''  . $attrs[$i] . '\'};',
-         '$global = $globs->{\'' . $attrs[$i] . '\'};',
-         $self->{unrolled_for}->{$attrs[$i]},
-      );
-   }
-   if ( ref $group_by ) {
-      push @lines, '}'; # Close the loop opened above
-   }
-   @lines = map { s/^/   /gm; $_ } @lines; # Indent for debugging
-   unshift @lines, 'sub {';
-   push @lines, '}';
-
-   my $code = join("\n", @lines);
-   PTDEBUG && _d('Unrolled subroutine:', @lines);
-   my $sub = eval $code;
-   die $EVAL_ERROR if $EVAL_ERROR;
-   $self->{unrolled_loops} = $sub;
-
-   return;
-}
-
-sub results {
-   my ( $self ) = @_;
-   return {
-      classes => $self->{result_classes},
-      globals => $self->{result_globals},
-      samples => $self->{result_samples},
-   };
-}
-
-sub set_results {
-   my ( $self, $results ) = @_;
-   $self->{result_classes} = $results->{classes};
-   $self->{result_globals} = $results->{globals};
-   $self->{result_samples} = $results->{samples};
-   return;
-}
-
-sub stats {
-   my ( $self ) = @_;
-   return {
-      classes => $self->{class_metrics},
-      globals => $self->{global_metrics},
-   };
-}
-
-sub attributes {
-   my ( $self ) = @_;
-   return $self->{type_for};
-}
-
-sub set_attribute_types {
-   my ( $self, $attrib_types ) = @_;
-   $self->{type_for} = $attrib_types;
-   return;
-}
-
-sub type_for {
-   my ( $self, $attrib ) = @_;
-   return $self->{type_for}->{$attrib};
-}
-
-sub make_handler {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(event attribute);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my ($event, $attrib) = @args{@required_args};
-
-   my $val;
-   eval { $val= $self->_get_value(%args); };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d("Cannot make", $attrib, "handler:", $EVAL_ERROR);
-      return;
-   }
-   return unless defined $val; # can't determine type if it's undef
-
-   my $float_re = qr{[+-]?(?:(?=\d|[.])\d+(?:[.])\d{0,})(?:E[+-]?\d+)?}i;
-   my $type = $self->type_for($attrib)           ? $self->type_for($attrib)
-            : $attrib =~ m/_crc$/                ? 'string'
-            : $val    =~ m/^(?:\d+|$float_re)$/o ? 'num'
-            : $val    =~ m/^(?:Yes|No)$/         ? 'bool'
-            :                                      'string';
-   PTDEBUG && _d('Type for', $attrib, 'is', $type, '(sample:', $val, ')');
-   $self->{type_for}->{$attrib} = $type;
-
-   my @lines;
-
-   my %track = (
-      sum => $type =~ m/num|bool/    ? 1 : 0,  # sum of values
-      unq => $type =~ m/bool|string/ ? 1 : 0,  # count of unique values seen
-      all => $type eq 'num'          ? 1 : 0,  # all values in bucketed list
-   );
-
-   my $trf = ($type eq 'bool') ? q{(($val || '') eq 'Yes') ? 1 : 0}
-           :                     undef;
-   if ( $trf ) {
-      push @lines, q{$val = } . $trf . ';';
-   }
-
-   if ( $attrib eq 'Query_time' ) {
-      push @lines, (
-         '$val =~ s/^(\d+(?:\.\d+)?).*/$1/;',
-         '$event->{\''.$attrib.'\'} = $val;',
-      );
-   }
-
-   if ( $type eq 'num' && $self->{attrib_limit} ) {
-      push @lines, (
-         "if ( \$val > $self->{attrib_limit} ) {",
-         '   $val = $class->{last} ||= 0;',
-         '}',
-         '$class->{last} = $val;',
-      );
-   }
-
-   my $lt = $type eq 'num' ? '<' : 'lt';
-   my $gt = $type eq 'num' ? '>' : 'gt';
-   foreach my $place ( qw($class $global) ) {
-      my @tmp;  # hold lines until PLACE placeholder is replaced
-
-      push @tmp, '++PLACE->{cnt};';  # count of all values seen
-
-      if ( $attrib =~ m/_crc$/ ) {
-         push @tmp, '$val = $val % 1_000;';
-      }
-
-      push @tmp, (
-         'PLACE->{min} = $val if !defined PLACE->{min} || $val '
-            . $lt . ' PLACE->{min};',
-      );
-      push @tmp, (
-         'PLACE->{max} = $val if !defined PLACE->{max} || $val '
-         . $gt . ' PLACE->{max};',
-      );
-      if ( $track{sum} ) {
-         push @tmp, 'PLACE->{sum} += $val;';
-      }
-
-      if ( $track{all} ) {
-         push @tmp, (
-            'exists PLACE->{all} or PLACE->{all} = {};',
-            '++PLACE->{all}->{ EventAggregator::bucket_idx($val) };',
-         );
-      }
-
-      push @lines, map { s/PLACE/$place/g; $_ } @tmp;
-   }
-
-   if ( $track{unq} ) {
-      push @lines, '++$class->{unq}->{$val}';
-   }
-
-   if ( $args{worst} ) {
-      my $op = $type eq 'num' ? '>=' : 'ge';
-      push @lines, (
-         'if ( $val ' . $op . ' ($class->{max} || 0) ) {',
-         '   $samples->{$group_by} = $event;',
-         '}',
-      );
-   }
-
-   my @unrolled = (
-      "\$val = \$event->{'$attrib'};", 
-      
-      ( map  { "\$val = \$event->{'$_'} unless defined \$val;" }
-        grep { $_ ne $attrib } @{$args{alternates}}
-      ),
-      
-      'defined $val && do {',
-         @lines,
-      '};',
-   );
-   $self->{unrolled_for}->{$attrib} = join("\n", @unrolled);
-
-   my @code = (
-      'sub {',
-         'my ( $event, $class, $global, $samples, $group_by ) = @_;',
-         'my ($val, $idx);',
-
-         $self->{unrolled_for}->{$attrib},
-
-         'return;',
-      '}',
-   );
-   $self->{code_for}->{$attrib} = join("\n", @code);
-   PTDEBUG && _d($attrib, 'handler code:', $self->{code_for}->{$attrib});
-   my $sub = eval $self->{code_for}->{$attrib};
-   if ( $EVAL_ERROR ) {
-      die "Failed to compile $attrib handler code: $EVAL_ERROR";
-   }
-
-   return $sub;
-}
-
-sub bucket_idx {
-   my ( $val ) = @_;
-   return 0 if $val < MIN_BUCK;
-   my $idx = int(BASE_OFFSET + log($val)/BASE_LOG);
-   return $idx > (NUM_BUCK-1) ? (NUM_BUCK-1) : $idx;
-}
-
-sub bucket_value {
-   my ( $bucket ) = @_;
-   return 0 if $bucket == 0;
-   die "Invalid bucket: $bucket" if $bucket < 0 || $bucket > (NUM_BUCK-1);
-   return (BUCK_SIZE**($bucket-1)) * MIN_BUCK;
-}
-
-{
-   my @buck_tens;
-   sub buckets_of {
-      return @buck_tens if @buck_tens;
-
-      my $start_bucket  = 0;
-      my @base10_starts = (0);
-      map { push @base10_starts, (10**$_)*MIN_BUCK } (1..7);
-
-      for my $base10_bucket ( 0..($#base10_starts-1) ) {
-         my $next_bucket = bucket_idx( $base10_starts[$base10_bucket+1] );
-         PTDEBUG && _d('Base 10 bucket', $base10_bucket, 'maps to',
-            'base 1.05 buckets', $start_bucket, '..', $next_bucket-1);
-         for my $base1_05_bucket ($start_bucket..($next_bucket-1)) {
-            $buck_tens[$base1_05_bucket] = $base10_bucket;
-         }
-         $start_bucket = $next_bucket;
-      }
-
-      map { $buck_tens[$_] = 7 } ($start_bucket..(NUM_BUCK-1));
-
-      return @buck_tens;
-   }
-}
-
-sub calculate_statistical_metrics {
-   my ( $self, %args ) = @_;
-   my $classes        = $self->{result_classes};
-   my $globals        = $self->{result_globals};
-   my $class_metrics  = $self->{class_metrics};
-   my $global_metrics = $self->{global_metrics};
-   PTDEBUG && _d('Calculating statistical_metrics');
-   foreach my $attrib ( keys %$globals ) {
-      if ( exists $globals->{$attrib}->{all} ) {
-         $global_metrics->{$attrib}
-            = $self->_calc_metrics(
-               $globals->{$attrib}->{all},
-               $globals->{$attrib},
-            );
-      }
-
-      foreach my $class ( keys %$classes ) {
-         if ( exists $classes->{$class}->{$attrib}->{all} ) {
-            $class_metrics->{$class}->{$attrib}
-               = $self->_calc_metrics(
-                  $classes->{$class}->{$attrib}->{all},
-                  $classes->{$class}->{$attrib}
-               );
-         }
-      }
-   }
-
-   return;
-}
-
-sub _calc_metrics {
-   my ( $self, $vals, $args ) = @_;
-   my $statistical_metrics = {
-      pct_95    => 0,
-      stddev    => 0,
-      median    => 0,
-      cutoff    => undef,
-   };
-
-   return $statistical_metrics
-      unless defined $vals && %$vals && $args->{cnt};
-
-   my $n_vals = $args->{cnt};
-   if ( $n_vals == 1 || $args->{max} == $args->{min} ) {
-      my $v      = $args->{max} || 0;
-      my $bucket = int(6 + ( log($v > 0 ? $v : MIN_BUCK) / log(10)));
-      $bucket    = $bucket > 7 ? 7 : $bucket < 0 ? 0 : $bucket;
-      return {
-         pct_95 => $v,
-         stddev => 0,
-         median => $v,
-         cutoff => $n_vals,
-      };
-   }
-   elsif ( $n_vals == 2 ) {
-      foreach my $v ( $args->{min}, $args->{max} ) {
-         my $bucket = int(6 + ( log($v && $v > 0 ? $v : MIN_BUCK) / log(10)));
-         $bucket = $bucket > 7 ? 7 : $bucket < 0 ? 0 : $bucket;
-      }
-      my $v      = $args->{max} || 0;
-      my $mean = (($args->{min} || 0) + $v) / 2;
-      return {
-         pct_95 => $v,
-         stddev => sqrt((($v - $mean) ** 2) *2),
-         median => $mean,
-         cutoff => $n_vals,
-      };
-   }
-
-   my $cutoff = $n_vals >= 10 ? int ( $n_vals * 0.95 ) : $n_vals;
-   $statistical_metrics->{cutoff} = $cutoff;
-
-   my $total_left = $n_vals;
-   my $top_vals   = $n_vals - $cutoff; # vals > 95th
-   my $sum_excl   = 0;
-   my $sum        = 0;
-   my $sumsq      = 0;
-   my $mid        = int($n_vals / 2);
-   my $median     = 0;
-   my $prev       = NUM_BUCK-1; # Used for getting median when $cutoff is odd
-   my $bucket_95  = 0; # top bucket in 95th
-
-   PTDEBUG && _d('total vals:', $total_left, 'top vals:', $top_vals, 'mid:', $mid);
-
-   my @buckets = map { 0 } (0..NUM_BUCK-1);
-   map { $buckets[$_] = $vals->{$_} } keys %$vals;
-   $vals = \@buckets;  # repoint vals from given hashref to our array
-
-   BUCKET:
-   for my $bucket ( reverse 0..(NUM_BUCK-1) ) {
-      my $val = $vals->[$bucket];
-      next BUCKET unless $val; 
-
-      $total_left -= $val;
-      $sum_excl   += $val;
-      $bucket_95   = $bucket if !$bucket_95 && $sum_excl > $top_vals;
-
-      if ( !$median && $total_left <= $mid ) {
-         $median = (($cutoff % 2) || ($val > 1)) ? $buck_vals[$bucket]
-                 : ($buck_vals[$bucket] + $buck_vals[$prev]) / 2;
-      }
-
-      $sum    += $val * $buck_vals[$bucket];
-      $sumsq  += $val * ($buck_vals[$bucket]**2);
-      $prev   =  $bucket;
-   }
-
-   my $var      = $sumsq/$n_vals - ( ($sum/$n_vals) ** 2 );
-   my $stddev   = $var > 0 ? sqrt($var) : 0;
-   my $maxstdev = (($args->{max} || 0) - ($args->{min} || 0)) / 2;
-   $stddev      = $stddev > $maxstdev ? $maxstdev : $stddev;
-
-   PTDEBUG && _d('sum:', $sum, 'sumsq:', $sumsq, 'stddev:', $stddev,
-      'median:', $median, 'prev bucket:', $prev,
-      'total left:', $total_left, 'sum excl', $sum_excl,
-      'bucket 95:', $bucket_95, $buck_vals[$bucket_95]);
-
-   $statistical_metrics->{stddev} = $stddev;
-   $statistical_metrics->{pct_95} = $buck_vals[$bucket_95];
-   $statistical_metrics->{median} = $median;
-
-   return $statistical_metrics;
-}
-
-sub metrics {
-   my ( $self, %args ) = @_;
-   foreach my $arg ( qw(attrib where) ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   my $attrib = $args{attrib};
-   my $where   = $args{where};
-
-   my $stats      = $self->results();
-   my $metrics    = $self->stats();
-   my $store      = $stats->{classes}->{$where}->{$attrib};
-   my $global_cnt = $stats->{globals}->{$attrib}->{cnt};
-
-   return {
-      cnt    => $store->{cnt},
-      pct    => $global_cnt && $store->{cnt} ? $store->{cnt} / $global_cnt : 0,
-      sum    => $store->{sum},
-      min    => $store->{min},
-      max    => $store->{max},
-      avg    => $store->{sum} && $store->{cnt} ? $store->{sum} / $store->{cnt} : 0,
-      median => $metrics->{classes}->{$where}->{$attrib}->{median} || 0,
-      pct_95 => $metrics->{classes}->{$where}->{$attrib}->{pct_95} || 0,
-      stddev => $metrics->{classes}->{$where}->{$attrib}->{stddev} || 0,
-   };
-}
-
-sub top_events {
-   my ( $self, %args ) = @_;
-   my $classes = $self->{result_classes};
-   my @sorted = reverse sort { # Sorted list of $groupby values
-      $classes->{$a}->{$args{attrib}}->{$args{orderby}}
-         <=> $classes->{$b}->{$args{attrib}}->{$args{orderby}}
-      } grep {
-         defined $classes->{$_}->{$args{attrib}}->{$args{orderby}}
-      } keys %$classes;
-   my @chosen;  # top events
-   my @other;   # other events (< top)
-   my ($total, $count) = (0, 0);
-   foreach my $groupby ( @sorted ) {
-      if ( 
-         (!$args{total} || $total < $args{total} )
-         && ( !$args{count} || $count < $args{count} )
-      ) {
-         push @chosen, [$groupby, 'top', $count+1];
-      }
-
-      elsif ( $args{ol_attrib} && (!$args{ol_freq}
-         || $classes->{$groupby}->{$args{ol_attrib}}->{cnt} >= $args{ol_freq})
-      ) {
-         my $stats = $self->{class_metrics}->{$groupby}->{$args{ol_attrib}};
-         if ( ($stats->{pct_95} || 0) >= $args{ol_limit} ) {
-            push @chosen, [$groupby, 'outlier', $count+1];
-         }
-         else {
-            push @other, [$groupby, 'misc', $count+1];
-         }
-      }
-
-      else {
-         push @other, [$groupby, 'misc', $count+1];
-      }
-
-      $total += $classes->{$groupby}->{$args{attrib}}->{$args{orderby}};
-      $count++;
-   }
-   return \@chosen, \@other;
-}
-
-sub add_new_attributes {
-   my ( $self, $event ) = @_;
-   return unless $event;
-
-   map {
-      my $attrib = $_;
-      $self->{attributes}->{$attrib}  = [$attrib];
-      $self->{alt_attribs}->{$attrib} = make_alt_attrib($attrib);
-      push @{$self->{all_attribs}}, $attrib;
-      PTDEBUG && _d('Added new attribute:', $attrib);
-   }
-   grep {
-      $_ ne $self->{groupby}
-      && !exists $self->{attributes}->{$_}
-      && !exists $self->{ignore_attribs}->{$_}
-   }
-   keys %$event;
-
-   return;
-}
-
-sub get_attributes {
-   my ( $self ) = @_;
-   return $self->{all_attribs};
-}
-
-sub events_processed {
-   my ( $self ) = @_;
-   return $self->{n_events};
-}
-
-sub make_alt_attrib {
-   my ( @attribs ) = @_;
-
-   my $attrib = shift @attribs;  # Primary attribute.
-   return sub {} unless @attribs;  # No alternates.
-
-   my @lines;
-   push @lines, 'sub { my ( $event ) = @_; my $alt_attrib;';
-   push @lines, map  {
-         "\$alt_attrib = '$_' if !defined \$alt_attrib "
-         . "&& exists \$event->{'$_'};"
-      } @attribs;
-   push @lines, 'return $alt_attrib; }';
-   PTDEBUG && _d('alt attrib sub for', $attrib, ':', @lines);
-   my $sub = eval join("\n", @lines);
-   die if $EVAL_ERROR;
-   return $sub;
-}
-
-sub merge {
-   my ( @ea_objs ) = @_;
-   PTDEBUG && _d('Merging', scalar @ea_objs, 'ea');
-   return unless scalar @ea_objs;
-
-   my $ea1   = shift @ea_objs;
-   my $r1    = $ea1->results;
-   my $worst = $ea1->{worst};  # for merging, finding worst sample
-
-   my %attrib_types = %{ $ea1->attributes() };
-
-   foreach my $ea ( @ea_objs ) {
-      die "EventAggregator objects have different groupby: "
-         . "$ea1->{groupby} and $ea->{groupby}"
-         unless $ea1->{groupby} eq $ea->{groupby};
-      die "EventAggregator objects have different worst: "
-         . "$ea1->{worst} and $ea->{worst}"
-         unless $ea1->{worst} eq $ea->{worst};
-      
-      my $attrib_types = $ea->attributes();
-      map {
-         $attrib_types{$_} = $attrib_types->{$_}
-            unless exists $attrib_types{$_};
-      } keys %$attrib_types;
-   }
-
-   my $r_merged = {
-      classes => {},
-      globals => _deep_copy_attribs($r1->{globals}),
-      samples => {},
-   };
-   map {
-      $r_merged->{classes}->{$_}
-         = _deep_copy_attribs($r1->{classes}->{$_});
-
-      @{$r_merged->{samples}->{$_}}{keys %{$r1->{samples}->{$_}}}
-         = values %{$r1->{samples}->{$_}};
-   } keys %{$r1->{classes}};
-
-   for my $i ( 0..$#ea_objs ) {
-      PTDEBUG && _d('Merging ea obj', ($i + 1));
-      my $r2 = $ea_objs[$i]->results;
-
-      eval {
-         CLASS:
-         foreach my $class ( keys %{$r2->{classes}} ) {
-            my $r1_class = $r_merged->{classes}->{$class};
-            my $r2_class = $r2->{classes}->{$class};
-
-            if ( $r1_class && $r2_class ) {
-               CLASS_ATTRIB:
-               foreach my $attrib ( keys %$r2_class ) {
-                  PTDEBUG && _d('merge', $attrib);
-                  if ( $r1_class->{$attrib} && $r2_class->{$attrib} ) {
-                     _add_attrib_vals($r1_class->{$attrib}, $r2_class->{$attrib});
-                  }
-                  elsif ( !$r1_class->{$attrib} ) {
-                  PTDEBUG && _d('copy', $attrib);
-                     $r1_class->{$attrib} =
-                        _deep_copy_attrib_vals($r2_class->{$attrib})
-                  }
-               }
-            }
-            elsif ( !$r1_class ) {
-               PTDEBUG && _d('copy class');
-               $r_merged->{classes}->{$class} = _deep_copy_attribs($r2_class);
-            }
-
-            my $new_worst_sample;
-            if ( $r_merged->{samples}->{$class} && $r2->{samples}->{$class} ) {
-               if (   $r2->{samples}->{$class}->{$worst}
-                    > $r_merged->{samples}->{$class}->{$worst} ) {
-                  $new_worst_sample = $r2->{samples}->{$class}
-               }
-            }
-            elsif ( !$r_merged->{samples}->{$class} ) {
-               $new_worst_sample = $r2->{samples}->{$class};
-            }
-            if ( $new_worst_sample ) {
-               PTDEBUG && _d('New worst sample:', $worst, '=',
-                  $new_worst_sample->{$worst}, 'item:', substr($class, 0, 100));
-               my %new_sample;
-               @new_sample{keys %$new_worst_sample}
-                  = values %$new_worst_sample;
-               $r_merged->{samples}->{$class} = \%new_sample;
-            }
-         }
-      };
-      if ( $EVAL_ERROR ) {
-         warn "Error merging class/sample: $EVAL_ERROR";
-      }
-
-      eval {
-         GLOBAL_ATTRIB:
-         PTDEBUG && _d('Merging global attributes');
-         foreach my $attrib ( keys %{$r2->{globals}} ) {
-            my $r1_global = $r_merged->{globals}->{$attrib};
-            my $r2_global = $r2->{globals}->{$attrib};
-
-            if ( $r1_global && $r2_global ) {
-               PTDEBUG && _d('merge', $attrib);
-               _add_attrib_vals($r1_global, $r2_global);
-            }
-            elsif ( !$r1_global ) {
-               PTDEBUG && _d('copy', $attrib);
-               $r_merged->{globals}->{$attrib}
-                  = _deep_copy_attrib_vals($r2_global);
-            }
-         }
-      };
-      if ( $EVAL_ERROR ) {
-         warn "Error merging globals: $EVAL_ERROR";
-      }
-   }
-
-   my $ea_merged = new EventAggregator(
-      groupby    => $ea1->{groupby},
-      worst      => $ea1->{worst},
-      attributes => { map { $_=>[$_] } keys %attrib_types },
-   );
-   $ea_merged->set_results($r_merged);
-   $ea_merged->set_attribute_types(\%attrib_types);
-   return $ea_merged;
-}
-
-sub _add_attrib_vals {
-   my ( $vals1, $vals2 ) = @_;
-
-   foreach my $val ( keys %$vals1 ) {
-      my $val1 = $vals1->{$val};
-      my $val2 = $vals2->{$val};
-
-      if ( (!ref $val1) && (!ref $val2) ) {
-         die "undefined $val value" unless defined $val1 && defined $val2;
-
-         my $is_num = exists $vals1->{sum} ? 1 : 0;
-         if ( $val eq 'max' ) {
-            if ( $is_num ) {
-               $vals1->{$val} = $val1 > $val2  ? $val1 : $val2;
-            }
-            else {
-               $vals1->{$val} = $val1 gt $val2 ? $val1 : $val2;
-            }
-         }
-         elsif ( $val eq 'min' ) {
-            if ( $is_num ) {
-               $vals1->{$val} = $val1 < $val2  ? $val1 : $val2;
-            }
-            else {
-               $vals1->{$val} = $val1 lt $val2 ? $val1 : $val2;
-            }
-         }
-         else {
-            $vals1->{$val} += $val2;
-         }
-      }
-      elsif ( (ref $val1 eq 'ARRAY') && (ref $val2 eq 'ARRAY') ) {
-         die "Empty $val arrayref" unless @$val1 && @$val2;
-         my $n_buckets = (scalar @$val1) - 1;
-         for my $i ( 0..$n_buckets ) {
-            $vals1->{$val}->[$i] += $val2->[$i];
-         }
-      }
-      elsif ( (ref $val1 eq 'HASH')  && (ref $val2 eq 'HASH')  ) {
-         die "Empty $val hashref" unless %$val1 and %$val2;
-         map { $vals1->{$val}->{$_} += $val2->{$_} } keys %$val2;
-      }
-      else {
-         PTDEBUG && _d('vals1:', Dumper($vals1));
-         PTDEBUG && _d('vals2:', Dumper($vals2));
-         die "$val type mismatch";
-      }
-   }
-
-   return;
-}
-
-sub _deep_copy_attribs {
-   my ( $attribs ) = @_;
-   my $copy = {};
-   foreach my $attrib ( keys %$attribs ) {
-      $copy->{$attrib} = _deep_copy_attrib_vals($attribs->{$attrib});
-   }
-   return $copy;
-}
-
-sub _deep_copy_attrib_vals {
-   my ( $vals ) = @_;
-   my $copy;
-   if ( ref $vals eq 'HASH' ) {
-      $copy = {};
-      foreach my $val ( keys %$vals ) {
-         if ( my $ref_type = ref $val ) {
-            if ( $ref_type eq 'ARRAY' ) {
-               my $n_elems = (scalar @$val) - 1;
-               $copy->{$val} = [ map { undef } ( 0..$n_elems ) ];
-               for my $i ( 0..$n_elems ) {
-                  $copy->{$val}->[$i] = $vals->{$val}->[$i];
-               }
-            }
-            elsif ( $ref_type eq 'HASH' ) {
-               $copy->{$val} = {};
-               map { $copy->{$val}->{$_} += $vals->{$val}->{$_} }
-                  keys %{$vals->{$val}}
-            }
-            else {
-               die "I don't know how to deep copy a $ref_type reference";
-            }
-         }
-         else {
-            $copy->{$val} = $vals->{$val};
-         }
-      }
-   }
-   else {
-      $copy = $vals;
-   }
-   return $copy;
-}
-
-sub _get_value {
-   my ( $self, %args ) = @_;
-   my ($event, $attrib, $alts) = @args{qw(event attribute alternates)};
-   return unless $event && $attrib;
-
-   my $value;
-   if ( exists $event->{$attrib} ) {
-      $value = $event->{$attrib};
-   }
-   elsif ( $alts ) {
-      my $found_value = 0;
-      foreach my $alt_attrib( @$alts ) {
-         if ( exists $event->{$alt_attrib} ) {
-            $value       = $event->{$alt_attrib};
-            $found_value = 1;
-            last;
-         }
-      }
-      die "Event does not have attribute $attrib or any of its alternates"
-         unless $found_value;
-   }
-   else {
-      die "Event does not have attribute $attrib and there are no alterantes";
-   }
-
-   return $value;
-}
-
-sub _d {
-   my ($package, undef, $line) = caller 0;
-   @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
-        map { defined $_ ? $_ : 'undef' }
-        @_;
-   print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
-}
-
-1;
-}
-# ###########################################################################
-# End EventAggregator package
-# ###########################################################################
-
-# ###########################################################################
-# QueryParser package
-# This package is a copy without comments from the original.  The original
-# with comments and its test file can be found in the Bazaar repository at,
-#   lib/QueryParser.pm
-#   t/lib/QueryParser.t
-# See https://launchpad.net/percona-toolkit for more information.
-# ###########################################################################
-{
-package QueryParser;
-
-use strict;
-use warnings FATAL => 'all';
-use English qw(-no_match_vars);
-use constant PTDEBUG => $ENV{PTDEBUG} || 0;
-
-our $tbl_ident = qr/(?:`[^`]+`|\w+)(?:\.(?:`[^`]+`|\w+))?/;
-our $tbl_regex = qr{
-         \b(?:FROM|JOIN|(?<!KEY\s)UPDATE|INTO) # Words that precede table names
-         \b\s*
-         \(?                                   # Optional paren around tables
-         ($tbl_ident
-            (?: (?:\s+ (?:AS\s+)? \w+)?, \s*$tbl_ident )*
-         )
-      }xio;
-our $has_derived = qr{
-      \b(?:FROM|JOIN|,)
-      \s*\(\s*SELECT
-   }xi;
-
-our $data_def_stmts = qr/(?:CREATE|ALTER|TRUNCATE|DROP|RENAME)/i;
-
-our $data_manip_stmts = qr/(?:INSERT|UPDATE|DELETE|REPLACE)/i;
-
-sub new {
-   my ( $class ) = @_;
-   bless {}, $class;
-}
-
-sub get_tables {
-   my ( $self, $query ) = @_;
-   return unless $query;
-   PTDEBUG && _d('Getting tables for', $query);
-
-   my ( $ddl_stmt ) = $query =~ m/^\s*($data_def_stmts)\b/i;
-   if ( $ddl_stmt ) {
-      PTDEBUG && _d('Special table type:', $ddl_stmt);
-      $query =~ s/IF\s+(?:NOT\s+)?EXISTS//i;
-      if ( $query =~ m/$ddl_stmt DATABASE\b/i ) {
-         PTDEBUG && _d('Query alters a database, not a table');
-         return ();
-      }
-      if ( $ddl_stmt =~ m/CREATE/i && $query =~ m/$ddl_stmt\b.+?\bSELECT\b/i ) {
-         my ($select) = $query =~ m/\b(SELECT\b.+)/is;
-         PTDEBUG && _d('CREATE TABLE ... SELECT:', $select);
-         return $self->get_tables($select);
-      }
-      my ($tbl) = $query =~ m/TABLE\s+($tbl_ident)(\s+.*)?/i;
-      PTDEBUG && _d('Matches table:', $tbl);
-      return ($tbl);
-   }
-
-   $query =~ s/ (?:LOW_PRIORITY|IGNORE|STRAIGHT_JOIN)//ig;
-
-   if ( $query =~ s/^\s*LOCK TABLES\s+//i ) {
-      PTDEBUG && _d('Special table type: LOCK TABLES');
-      $query =~ s/\s+(?:READ(?:\s+LOCAL)?|WRITE)\s*//gi;
-      PTDEBUG && _d('Locked tables:', $query);
-      $query = "FROM $query";
-   }
-
-   $query =~ s/\\["']//g;                # quoted strings
-   $query =~ s/".*?"/?/sg;               # quoted strings
-   $query =~ s/'.*?'/?/sg;               # quoted strings
-
-   my @tables;
-   foreach my $tbls ( $query =~ m/$tbl_regex/gio ) {
-      PTDEBUG && _d('Match tables:', $tbls);
-
-      next if $tbls =~ m/\ASELECT\b/i;
-
-      foreach my $tbl ( split(',', $tbls) ) {
-         $tbl =~ s/\s*($tbl_ident)(\s+.*)?/$1/gio;
-
-         if ( $tbl !~ m/[a-zA-Z]/ ) {
-            PTDEBUG && _d('Skipping suspicious table name:', $tbl);
-            next;
-         }
-
-         push @tables, $tbl;
-      }
-   }
-   return @tables;
-}
-
-sub has_derived_table {
-   my ( $self, $query ) = @_;
-   my $match = $query =~ m/$has_derived/;
-   PTDEBUG && _d($query, 'has ' . ($match ? 'a' : 'no') . ' derived table');
-   return $match;
-}
-
-sub get_aliases {
-   my ( $self, $query, $list ) = @_;
-
-   my $result = {
-      DATABASE => {},
-      TABLE    => {},
-   };
-   return $result unless $query;
-
-   $query =~ s/ (?:LOW_PRIORITY|IGNORE|STRAIGHT_JOIN)//ig;
-
-   $query =~ s/ (?:INNER|OUTER|CROSS|LEFT|RIGHT|NATURAL)//ig;
-
-   my @tbl_refs;
-   my ($tbl_refs, $from) = $query =~ m{
-      (
-         (FROM|INTO|UPDATE)\b\s*   # Keyword before table refs
-         .+?                       # Table refs
-      )
-      (?:\s+|\z)                   # If the query does not end with the table
-      (?:WHERE|ORDER|LIMIT|HAVING|SET|VALUES|\z) # Keyword after table refs
-   }ix;
-
-   if ( $tbl_refs ) {
-
-      if ( $query =~ m/^(?:INSERT|REPLACE)/i ) {
-         $tbl_refs =~ s/\([^\)]+\)\s*//;
-      }
-
-      PTDEBUG && _d('tbl refs:', $tbl_refs);
-
-      my $before_tbl = qr/(?:,|JOIN|\s|$from)+/i;
-
-      my $after_tbl  = qr/(?:,|JOIN|ON|USING|\z)/i;
-
-      $tbl_refs =~ s/ = /=/g;
-
-      while (
-         $tbl_refs =~ m{
-            $before_tbl\b\s*
-               ( ($tbl_ident) (?:\s+ (?:AS\s+)? (\w+))? )
-            \s*$after_tbl
-         }xgio )
-      {
-         my ( $tbl_ref, $db_tbl, $alias ) = ($1, $2, $3);
-         PTDEBUG && _d('Match table:', $tbl_ref);
-         push @tbl_refs, $tbl_ref;
-         $alias = $self->trim_identifier($alias);
-
-         if ( $tbl_ref =~ m/^AS\s+\w+/i ) {
-            PTDEBUG && _d('Subquery', $tbl_ref);
-            $result->{TABLE}->{$alias} = undef;
-            next;
-         }
-
-         my ( $db, $tbl ) = $db_tbl =~ m/^(?:(.*?)\.)?(.*)/;
-         $db  = $self->trim_identifier($db);
-         $tbl = $self->trim_identifier($tbl);
-         $result->{TABLE}->{$alias || $tbl} = $tbl;
-         $result->{DATABASE}->{$tbl}        = $db if $db;
-      }
-   }
-   else {
-      PTDEBUG && _d("No tables ref in", $query);
-   }
-
-   if ( $list ) {
-      return \@tbl_refs;
-   }
-   else {
-      return $result;
-   }
-}
-
-sub split {
-   my ( $self, $query ) = @_;
-   return unless $query;
-   $query = $self->clean_query($query);
-   PTDEBUG && _d('Splitting', $query);
-
-   my $verbs = qr{SELECT|INSERT|UPDATE|DELETE|REPLACE|UNION|CREATE}i;
-
-   my @split_statements = grep { $_ } split(m/\b($verbs\b(?!(?:\s*\()))/io, $query);
-
-   my @statements;
-   if ( @split_statements == 1 ) {
-      push @statements, $query;
-   }
-   else {
-      for ( my $i = 0; $i <= $#split_statements; $i += 2 ) {
-         push @statements, $split_statements[$i].$split_statements[$i+1];
-
-         if ( $statements[-2] && $statements[-2] =~ m/on duplicate key\s+$/i ) {
-            $statements[-2] .= pop @statements;
-         }
-      }
-   }
-
-   PTDEBUG && _d('statements:', map { $_ ? "<$_>" : 'none' } @statements);
-   return @statements;
-}
-
-sub clean_query {
-   my ( $self, $query ) = @_;
-   return unless $query;
-   $query =~ s!/\*.*?\*/! !g;  # Remove /* comment blocks */
-   $query =~ s/^\s+//;         # Remove leading spaces
-   $query =~ s/\s+$//;         # Remove trailing spaces
-   $query =~ s/\s{2,}/ /g;     # Remove extra spaces
-   return $query;
-}
-
-sub split_subquery {
-   my ( $self, $query ) = @_;
-   return unless $query;
-   $query = $self->clean_query($query);
-   $query =~ s/;$//;
-
-   my @subqueries;
-   my $sqno = 0;  # subquery number
-   my $pos  = 0;
-   while ( $query =~ m/(\S+)(?:\s+|\Z)/g ) {
-      $pos = pos($query);
-      my $word = $1;
-      PTDEBUG && _d($word, $sqno);
-      if ( $word =~ m/^\(?SELECT\b/i ) {
-         my $start_pos = $pos - length($word) - 1;
-         if ( $start_pos ) {
-            $sqno++;
-            PTDEBUG && _d('Subquery', $sqno, 'starts at', $start_pos);
-            $subqueries[$sqno] = {
-               start_pos => $start_pos,
-               end_pos   => 0,
-               len       => 0,
-               words     => [$word],
-               lp        => 1, # left parentheses
-               rp        => 0, # right parentheses
-               done      => 0,
-            };
-         }
-         else {
-            PTDEBUG && _d('Main SELECT at pos 0');
-         }
-      }
-      else {
-         next unless $sqno;  # next unless we're in a subquery
-         PTDEBUG && _d('In subquery', $sqno);
-         my $sq = $subqueries[$sqno];
-         if ( $sq->{done} ) {
-            PTDEBUG && _d('This subquery is done; SQL is for',
-               ($sqno - 1 ? "subquery $sqno" : "the main SELECT"));
-            next;
-         }
-         push @{$sq->{words}}, $word;
-         my $lp = ($word =~ tr/\(//) || 0;
-         my $rp = ($word =~ tr/\)//) || 0;
-         PTDEBUG && _d('parentheses left', $lp, 'right', $rp);
-         if ( ($sq->{lp} + $lp) - ($sq->{rp} + $rp) == 0 ) {
-            my $end_pos = $pos - 1;
-            PTDEBUG && _d('Subquery', $sqno, 'ends at', $end_pos);
-            $sq->{end_pos} = $end_pos;
-            $sq->{len}     = $end_pos - $sq->{start_pos};
-         }
-      }
-   }
-
-   for my $i ( 1..$#subqueries ) {
-      my $sq = $subqueries[$i];
-      next unless $sq;
-      $sq->{sql} = join(' ', @{$sq->{words}});
-      substr $query,
-         $sq->{start_pos} + 1,  # +1 for (
-         $sq->{len} - 1,        # -1 for )
-         "__subquery_$i";
-   }
-
-   return $query, map { $_->{sql} } grep { defined $_ } @subqueries;
-}
-
-sub query_type {
-   my ( $self, $query, $qr ) = @_;
-   my ($type, undef) = $qr->distill_verbs($query);
-   my $rw;
-   if ( $type =~ m/^SELECT\b/ ) {
-      $rw = 'read';
-   }
-   elsif ( $type =~ m/^$data_manip_stmts\b/
-           || $type =~ m/^$data_def_stmts\b/  ) {
-      $rw = 'write'
-   }
-
-   return {
-      type => $type,
-      rw   => $rw,
-   }
-}
-
-sub get_columns {
-   my ( $self, $query ) = @_;
-   my $cols = [];
-   return $cols unless $query;
-   my $cols_def;
-
-   if ( $query =~ m/^SELECT/i ) {
-      $query =~ s/
-         ^SELECT\s+
-           (?:ALL
-              |DISTINCT
-              |DISTINCTROW
-              |HIGH_PRIORITY
-              |STRAIGHT_JOIN
-              |SQL_SMALL_RESULT
-              |SQL_BIG_RESULT
-              |SQL_BUFFER_RESULT
-              |SQL_CACHE
-              |SQL_NO_CACHE
-              |SQL_CALC_FOUND_ROWS
-           )\s+
-      /SELECT /xgi;
-      ($cols_def) = $query =~ m/^SELECT\s+(.+?)\s+FROM/i;
-   }
-   elsif ( $query =~ m/^(?:INSERT|REPLACE)/i ) {
-      ($cols_def) = $query =~ m/\(([^\)]+)\)\s*VALUE/i;
-   }
-
-   PTDEBUG && _d('Columns:', $cols_def);
-   if ( $cols_def ) {
-      @$cols = split(',', $cols_def);
-      map {
-         my $col = $_;
-         $col = s/^\s+//g;
-         $col = s/\s+$//g;
-         $col;
-      } @$cols;
-   }
-
-   return $cols;
-}
-
-sub parse {
-   my ( $self, $query ) = @_;
-   return unless $query;
-   my $parsed = {};
-
-   $query =~ s/\n/ /g;
-   $query = $self->clean_query($query);
-
-   $parsed->{query}   = $query,
-   $parsed->{tables}  = $self->get_aliases($query, 1);
-   $parsed->{columns} = $self->get_columns($query);
-
-   my ($type) = $query =~ m/^(\w+)/;
-   $parsed->{type} = lc $type;
-
-
-   $parsed->{sub_queries} = [];
-
-   return $parsed;
-}
-
-sub extract_tables {
-   my ( $self, %args ) = @_;
-   my $query      = $args{query};
-   my $default_db = $args{default_db};
-   my $q          = $self->{Quoter} || $args{Quoter};
-   return unless $query;
-   PTDEBUG && _d('Extracting tables');
-   my @tables;
-   my %seen;
-   foreach my $db_tbl ( $self->get_tables($query) ) {
-      next unless $db_tbl;
-      next if $seen{$db_tbl}++; # Unique-ify for issue 337.
-      my ( $db, $tbl ) = $q->split_unquote($db_tbl);
-      push @tables, [ $db || $default_db, $tbl ];
-   }
-   return @tables;
-}
-
-sub trim_identifier {
-   my ($self, $str) = @_;
-   return unless defined $str;
-   $str =~ s/`//g;
-   $str =~ s/^\s+//;
-   $str =~ s/\s+$//;
-   return $str;
-}
-
-sub _d {
-   my ($package, undef, $line) = caller 0;
-   @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
-        map { defined $_ ? $_ : 'undef' }
-        @_;
-   print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
-}
-
-1;
-}
-# ###########################################################################
-# End QueryParser package
-# ###########################################################################
-
-# ###########################################################################
-# QueryRewriter package
-# This package is a copy without comments from the original.  The original
-# with comments and its test file can be found in the Bazaar repository at,
-#   lib/QueryRewriter.pm
-#   t/lib/QueryRewriter.t
-# See https://launchpad.net/percona-toolkit for more information.
-# ###########################################################################
-{
-package QueryRewriter;
-
-use strict;
-use warnings FATAL => 'all';
-use English qw(-no_match_vars);
-use constant PTDEBUG => $ENV{PTDEBUG} || 0;
-
-our $verbs   = qr{^SHOW|^FLUSH|^COMMIT|^ROLLBACK|^BEGIN|SELECT|INSERT
-                  |UPDATE|DELETE|REPLACE|^SET|UNION|^START|^LOCK}xi;
-my $quote_re = qr/"(?:(?!(?<!\\)").)*"|'(?:(?!(?<!\\)').)*'/; # Costly!
-my $bal;
-$bal         = qr/
-                  \(
-                  (?:
-                     (?> [^()]+ )    # Non-parens without backtracking
-                     |
-                     (??{ $bal })    # Group with matching parens
-                  )*
-                  \)
-                 /x;
-
-my $olc_re = qr/(?:--|#)[^'"\r\n]*(?=[\r\n]|\Z)/;  # One-line comments
-my $mlc_re = qr#/\*[^!].*?\*/#sm;                  # But not /*!version */
-my $vlc_re = qr#/\*.*?[0-9+].*?\*/#sm;             # For SHOW + /*!version */
-my $vlc_rf = qr#^(SHOW).*?/\*![0-9+].*?\*/#sm;     # Variation for SHOW
-
-
-sub new {
-   my ( $class, %args ) = @_;
-   my $self = { %args };
-   return bless $self, $class;
-}
-
-sub strip_comments {
-   my ( $self, $query ) = @_;
-   return unless $query;
-   $query =~ s/$olc_re//go;
-   $query =~ s/$mlc_re//go;
-   if ( $query =~ m/$vlc_rf/i ) { # contains show + version
-      $query =~ s/$vlc_re//go;
-   }
-   return $query;
-}
-
-sub shorten {
-   my ( $self, $query, $length ) = @_;
-   $query =~ s{
-      \A(
-         (?:INSERT|REPLACE)
-         (?:\s+LOW_PRIORITY|DELAYED|HIGH_PRIORITY|IGNORE)?
-         (?:\s\w+)*\s+\S+\s+VALUES\s*\(.*?\)
-      )
-      \s*,\s*\(.*?(ON\s+DUPLICATE|\Z)}
-      {$1 /*... omitted ...*/$2}xsi;
-
-   return $query unless $query =~ m/IN\s*\(\s*(?!select)/i;
-
-   my $last_length  = 0;
-   my $query_length = length($query);
-   while (
-      $length          > 0
-      && $query_length > $length
-      && $query_length < ( $last_length || $query_length + 1 )
-   ) {
-      $last_length = $query_length;
-      $query =~ s{
-         (\bIN\s*\()    # The opening of an IN list
-         ([^\)]+)       # Contents of the list, assuming no item contains paren
-         (?=\))           # Close of the list
-      }
-      {
-         $1 . __shorten($2)
-      }gexsi;
-   }
-
-   return $query;
-}
-
-sub __shorten {
-   my ( $snippet ) = @_;
-   my @vals = split(/,/, $snippet);
-   return $snippet unless @vals > 20;
-   my @keep = splice(@vals, 0, 20);  # Remove and save the first 20 items
-   return
-      join(',', @keep)
-      . "/*... omitted "
-      . scalar(@vals)
-      . " items ...*/";
-}
-
-sub fingerprint {
-   my ( $self, $query ) = @_;
-
-   $query =~ m#\ASELECT /\*!40001 SQL_NO_CACHE \*/ \* FROM `# # mysqldump query
-      && return 'mysqldump';
-   $query =~ m#/\*\w+\.\w+:[0-9]/[0-9]\*/#     # pt-table-checksum, etc query
-      && return 'percona-toolkit';
-   $query =~ m/\Aadministrator command: /
-      && return $query;
-   $query =~ m/\A\s*(call\s+\S+)\(/i
-      && return lc($1); # Warning! $1 used, be careful.
-   if ( my ($beginning) = $query =~ m/\A((?:INSERT|REPLACE)(?: IGNORE)?\s+INTO.+?VALUES\s*\(.*?\))\s*,\s*\(/is ) {
-      $query = $beginning; # Shorten multi-value INSERT statements ASAP
-   }
-  
-   $query =~ s/$olc_re//go;
-   $query =~ s/$mlc_re//go;
-   $query =~ s/\Ause \S+\Z/use ?/i       # Abstract the DB in USE
-      && return $query;
-
-   $query =~ s/\\["']//g;                # quoted strings
-   $query =~ s/".*?"/?/sg;               # quoted strings
-   $query =~ s/'.*?'/?/sg;               # quoted strings
-
-   if ( $self->{match_md5_checksums} ) { 
-      $query =~ s/([._-])[a-f0-9]{32}/$1?/g;
-   }
-
-   if ( !$self->{match_embedded_numbers} ) {
-      $query =~ s/[0-9+-][0-9a-f.xb+-]*/?/g;
-   }
-   else {
-      $query =~ s/\b[0-9+-][0-9a-f.xb+-]*/?/g;
-   }
-
-   if ( $self->{match_md5_checksums} ) {
-      $query =~ s/[xb+-]\?/?/g;                
-   }
-   else {
-      $query =~ s/[xb.+-]\?/?/g;
-   }
-
-   $query =~ s/\A\s+//;                  # Chop off leading whitespace
-   chomp $query;                         # Kill trailing whitespace
-   $query =~ tr[ \n\t\r\f][ ]s;          # Collapse whitespace
-   $query = lc $query;
-   $query =~ s/\bnull\b/?/g;             # Get rid of NULLs
-   $query =~ s{                          # Collapse IN and VALUES lists
-               \b(in|values?)(?:[\s,]*\([\s?,]*\))+
-              }
-              {$1(?+)}gx;
-   $query =~ s{                          # Collapse UNION
-               \b(select\s.*?)(?:(\sunion(?:\sall)?)\s\1)+
-              }
-              {$1 /*repeat$2*/}xg;
-   $query =~ s/\blimit \?(?:, ?\?| offset \?)?/limit ?/; # LIMIT
-
-   if ( $query =~ m/\bORDER BY /gi ) {  # Find, anchor on ORDER BY clause
-      1 while $query =~ s/\G(.+?)\s+ASC/$1/gi && pos $query;
-   }
-
-   return $query;
-}
-
-sub distill_verbs {
-   my ( $self, $query ) = @_;
-
-   $query =~ m/\A\s*call\s+(\S+)\(/i && return "CALL $1";
-   $query =~ m/\A\s*use\s+/          && return "USE";
-   $query =~ m/\A\s*UNLOCK TABLES/i  && return "UNLOCK";
-   $query =~ m/\A\s*xa\s+(\S+)/i     && return "XA_$1";
-
-   if ( $query =~ m/\Aadministrator command:/ ) {
-      $query =~ s/administrator command:/ADMIN/;
-      $query = uc $query;
-      return $query;
-   }
-
-   $query = $self->strip_comments($query);
-
-   if ( $query =~ m/\A\s*SHOW\s+/i ) {
-      PTDEBUG && _d($query);
-
-      $query = uc $query;
-      $query =~ s/\s+(?:GLOBAL|SESSION|FULL|STORAGE|ENGINE)\b/ /g;
-      $query =~ s/\s+COUNT[^)]+\)//g;
-
-      $query =~ s/\s+(?:FOR|FROM|LIKE|WHERE|LIMIT|IN)\b.+//ms;
-
-      $query =~ s/\A(SHOW(?:\s+\S+){1,2}).*\Z/$1/s;
-      $query =~ s/\s+/ /g;
-      PTDEBUG && _d($query);
-      return $query;
-   }
-
-   eval $QueryParser::data_def_stmts;
-   eval $QueryParser::tbl_ident;
-   my ( $dds ) = $query =~ /^\s*($QueryParser::data_def_stmts)\b/i;
-   if ( $dds) {
-      my ( $obj ) = $query =~ m/$dds.+(DATABASE|TABLE)\b/i;
-      $obj = uc $obj if $obj;
-      PTDEBUG && _d('Data def statment:', $dds, 'obj:', $obj);
-      my ($db_or_tbl)
-         = $query =~ m/(?:TABLE|DATABASE)\s+($QueryParser::tbl_ident)(\s+.*)?/i;
-      PTDEBUG && _d('Matches db or table:', $db_or_tbl);
-      return uc($dds . ($obj ? " $obj" : '')), $db_or_tbl;
-   }
-
-   my @verbs = $query =~ m/\b($verbs)\b/gio;
-   @verbs    = do {
-      my $last = '';
-      grep { my $pass = $_ ne $last; $last = $_; $pass } map { uc } @verbs;
-   };
-
-   if ( ($verbs[0] || '') eq 'SELECT' && @verbs > 1 ) {
-      PTDEBUG && _d("False-positive verbs after SELECT:", @verbs[1..$#verbs]);
-      my $union = grep { $_ eq 'UNION' } @verbs;
-      @verbs    = $union ? qw(SELECT UNION) : qw(SELECT);
-   }
-
-   my $verb_str = join(q{ }, @verbs);
-   return $verb_str;
-}
-
-sub __distill_tables {
-   my ( $self, $query, $table, %args ) = @_;
-   my $qp = $args{QueryParser} || $self->{QueryParser};
-   die "I need a QueryParser argument" unless $qp;
-
-   my @tables = map {
-      $_ =~ s/`//g;
-      $_ =~ s/(_?)[0-9]+/$1?/g;
-      $_;
-   } grep { defined $_ } $qp->get_tables($query);
-
-   push @tables, $table if $table;
-
-   @tables = do {
-      my $last = '';
-      grep { my $pass = $_ ne $last; $last = $_; $pass } @tables;
-   };
-
-   return @tables;
-}
-
-sub distill {
-   my ( $self, $query, %args ) = @_;
-
-   if ( $args{generic} ) {
-      my ($cmd, $arg) = $query =~ m/^(\S+)\s+(\S+)/;
-      return '' unless $cmd;
-      $query = (uc $cmd) . ($arg ? " $arg" : '');
-   }
-   else {
-      my ($verbs, $table)  = $self->distill_verbs($query, %args);
-
-      if ( $verbs && $verbs =~ m/^SHOW/ ) {
-         my %alias_for = qw(
-            SCHEMA   DATABASE
-            KEYS     INDEX
-            INDEXES  INDEX
-         );
-         map { $verbs =~ s/$_/$alias_for{$_}/ } keys %alias_for;
-         $query = $verbs;
-      }
-      else {
-         my @tables = $self->__distill_tables($query, $table, %args);
-         $query     = join(q{ }, $verbs, @tables); 
-      } 
-   }
-
-   if ( $args{trf} ) {
-      $query = $args{trf}->($query, %args);
-   }
-
-   return $query;
-}
-
-sub convert_to_select {
-   my ( $self, $query ) = @_;
-   return unless $query;
-
-   return if $query =~ m/=\s*\(\s*SELECT /i;
-
-   $query =~ s{
-                 \A.*?
-                 update(?:\s+(?:low_priority|ignore))?\s+(.*?)
-                 \s+set\b(.*?)
-                 (?:\s*where\b(.*?))?
-                 (limit\s*[0-9]+(?:\s*,\s*[0-9]+)?)?
-                 \Z
-              }
-              {__update_to_select($1, $2, $3, $4)}exsi
-      || $query =~ s{
-                    \A.*?
-                    (?:insert(?:\s+ignore)?|replace)\s+
-                    .*?\binto\b(.*?)\(([^\)]+)\)\s*
-                    values?\s*(\(.*?\))\s*
-                    (?:\blimit\b|on\s+duplicate\s+key.*)?\s*
-                    \Z
-                 }
-                 {__insert_to_select($1, $2, $3)}exsi
-      || $query =~ s{
-                    \A.*?
-                    (?:insert(?:\s+ignore)?|replace)\s+
-                    (?:.*?\binto)\b(.*?)\s*
-                    set\s+(.*?)\s*
-                    (?:\blimit\b|on\s+duplicate\s+key.*)?\s*
-                    \Z
-                 }
-                 {__insert_to_select_with_set($1, $2)}exsi
-      || $query =~ s{
-                    \A.*?
-                    delete\s+(.*?)
-                    \bfrom\b(.*)
-                    \Z
-                 }
-                 {__delete_to_select($1, $2)}exsi;
-   $query =~ s/\s*on\s+duplicate\s+key\s+update.*\Z//si;
-   $query =~ s/\A.*?(?=\bSELECT\s*\b)//ism;
-   return $query;
-}
-
-sub convert_select_list {
-   my ( $self, $query ) = @_;
-   $query =~ s{
-               \A\s*select(.*?)\bfrom\b
-              }
-              {$1 =~ m/\*/ ? "select 1 from" : "select isnull(coalesce($1)) from"}exi;
-   return $query;
-}
-
-sub __delete_to_select {
-   my ( $delete, $join ) = @_;
-   if ( $join =~ m/\bjoin\b/ ) {
-      return "select 1 from $join";
-   }
-   return "select * from $join";
-}
-
-sub __insert_to_select {
-   my ( $tbl, $cols, $vals ) = @_;
-   PTDEBUG && _d('Args:', @_);
-   my @cols = split(/,/, $cols);
-   PTDEBUG && _d('Cols:', @cols);
-   $vals =~ s/^\(|\)$//g; # Strip leading/trailing parens
-   my @vals = $vals =~ m/($quote_re|[^,]*${bal}[^,]*|[^,]+)/g;
-   PTDEBUG && _d('Vals:', @vals);
-   if ( @cols == @vals ) {
-      return "select * from $tbl where "
-         . join(' and ', map { "$cols[$_]=$vals[$_]" } (0..$#cols));
-   }
-   else {
-      return "select * from $tbl limit 1";
-   }
-}
-
-sub __insert_to_select_with_set {
-   my ( $from, $set ) = @_;
-   $set =~ s/,/ and /g;
-   return "select * from $from where $set ";
-}
-
-sub __update_to_select {
-   my ( $from, $set, $where, $limit ) = @_;
-   return "select $set from $from "
-      . ( $where ? "where $where" : '' )
-      . ( $limit ? " $limit "      : '' );
-}
-
-sub wrap_in_derived {
-   my ( $self, $query ) = @_;
-   return unless $query;
-   return $query =~ m/\A\s*select/i
-      ? "select 1 from ($query) as x limit 1"
-      : $query;
-}
-
-sub _d {
-   my ($package, undef, $line) = caller 0;
-   @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
-        map { defined $_ ? $_ : 'undef' }
-        @_;
-   print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
-}
-
-1;
-}
-# ###########################################################################
-# End QueryRewriter package
-# ###########################################################################
-
-# ###########################################################################
 # Daemon package
 # This package is a copy without comments from the original.  The original
 # with comments and its test file can be found in the Bazaar repository at,
@@ -5223,3403 +3056,6 @@
 # ###########################################################################
 
 # ###########################################################################
-# ChangeHandler package
-# This package is a copy without comments from the original.  The original
-# with comments and its test file can be found in the Bazaar repository at,
-#   lib/ChangeHandler.pm
-#   t/lib/ChangeHandler.t
-# See https://launchpad.net/percona-toolkit for more information.
-# ###########################################################################
-{
-package ChangeHandler;
-
-use strict;
-use warnings FATAL => 'all';
-use English qw(-no_match_vars);
-use constant PTDEBUG => $ENV{PTDEBUG} || 0;
-
-my $DUPE_KEY  = qr/Duplicate entry/;
-our @ACTIONS  = qw(DELETE REPLACE INSERT UPDATE);
-
-sub new {
-   my ( $class, %args ) = @_;
-   foreach my $arg ( qw(Quoter left_db left_tbl right_db right_tbl
-                        replace queue) ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   my $q = $args{Quoter};
-
-   my $self = {
-      hex_blob     => 1,
-      %args,
-      left_db_tbl  => $q->quote(@args{qw(left_db left_tbl)}),
-      right_db_tbl => $q->quote(@args{qw(right_db right_tbl)}),
-   };
-
-   $self->{src_db_tbl} = $self->{left_db_tbl};
-   $self->{dst_db_tbl} = $self->{right_db_tbl};
-
-   map { $self->{$_} = [] } @ACTIONS;
-   $self->{changes} = { map { $_ => 0 } @ACTIONS };
-
-   return bless $self, $class;
-}
-
-sub fetch_back {
-   my ( $self, $dbh ) = @_;
-   $self->{fetch_back} = $dbh;
-   PTDEBUG && _d('Set fetch back dbh', $dbh);
-   return;
-}
-
-sub set_src {
-   my ( $self, $src, $dbh ) = @_;
-   die "I need a src argument" unless $src;
-   if ( lc $src eq 'left' ) {
-      $self->{src_db_tbl} = $self->{left_db_tbl};
-      $self->{dst_db_tbl} = $self->{right_db_tbl};
-   }
-   elsif ( lc $src eq 'right' ) {
-      $self->{src_db_tbl} = $self->{right_db_tbl};
-      $self->{dst_db_tbl} = $self->{left_db_tbl}; 
-   }
-   else {
-      die "src argument must be either 'left' or 'right'"
-   }
-   PTDEBUG && _d('Set src to', $src);
-   $self->fetch_back($dbh) if $dbh;
-   return;
-}
-
-sub src {
-   my ( $self ) = @_;
-   return $self->{src_db_tbl};
-}
-
-sub dst {
-   my ( $self ) = @_;
-   return $self->{dst_db_tbl};
-}
-
-sub _take_action {
-   my ( $self, $sql, $dbh ) = @_;
-   PTDEBUG && _d('Calling subroutines on', $dbh, $sql);
-   foreach my $action ( @{$self->{actions}} ) {
-      $action->($sql, $dbh);
-   }
-   return;
-}
-
-sub change {
-   my ( $self, $action, $row, $cols, $dbh ) = @_;
-   PTDEBUG && _d($dbh, $action, 'where', $self->make_where_clause($row, $cols));
-
-   return unless $action;
-
-   $self->{changes}->{
-      $self->{replace} && $action ne 'DELETE' ? 'REPLACE' : $action
-   }++;
-   if ( $self->{queue} ) {
-      $self->__queue($action, $row, $cols, $dbh);
-   }
-   else {
-      eval {
-         my $func = "make_$action";
-         $self->_take_action($self->$func($row, $cols), $dbh);
-      };
-      if ( $EVAL_ERROR =~ m/$DUPE_KEY/ ) {
-         PTDEBUG && _d('Duplicate key violation; will queue and rewrite');
-         $self->{queue}++;
-         $self->{replace} = 1;
-         $self->__queue($action, $row, $cols, $dbh);
-      }
-      elsif ( $EVAL_ERROR ) {
-         die $EVAL_ERROR;
-      }
-   }
-   return;
-}
-
-sub __queue {
-   my ( $self, $action, $row, $cols, $dbh ) = @_;
-   PTDEBUG && _d('Queueing change for later');
-   if ( $self->{replace} ) {
-      $action = $action eq 'DELETE' ? $action : 'REPLACE';
-   }
-   push @{$self->{$action}}, [ $row, $cols, $dbh ];
-}
-
-sub process_rows {
-   my ( $self, $queue_level, $trace_msg ) = @_;
-   my $error_count = 0;
-   TRY: {
-      if ( $queue_level && $queue_level < $self->{queue} ) { # see redo below!
-         PTDEBUG && _d('Not processing now', $queue_level, '<', $self->{queue});
-         return;
-      }
-      PTDEBUG && _d('Processing rows:');
-      my ($row, $cur_act);
-      eval {
-         foreach my $action ( @ACTIONS ) {
-            my $func = "make_$action";
-            my $rows = $self->{$action};
-            PTDEBUG && _d(scalar(@$rows), 'to', $action);
-            $cur_act = $action;
-            while ( @$rows ) {
-               $row    = shift @$rows;
-               my $sql = $self->$func(@$row);
-               $sql   .= " /*percona-toolkit $trace_msg*/" if $trace_msg;
-               $self->_take_action($sql, $row->[2]);
-            }
-         }
-         $error_count = 0;
-      };
-      if ( !$error_count++ && $EVAL_ERROR =~ m/$DUPE_KEY/ ) {
-         PTDEBUG && _d('Duplicate key violation; re-queueing and rewriting');
-         $self->{queue}++; # Defer rows to the very end
-         $self->{replace} = 1;
-         $self->__queue($cur_act, @$row);
-         redo TRY;
-      }
-      elsif ( $EVAL_ERROR ) {
-         die $EVAL_ERROR;
-      }
-   }
-}
-
-sub make_DELETE {
-   my ( $self, $row, $cols ) = @_;
-   PTDEBUG && _d('Make DELETE');
-   return "DELETE FROM $self->{dst_db_tbl} WHERE "
-      . $self->make_where_clause($row, $cols)
-      . ' LIMIT 1';
-}
-
-sub make_UPDATE {
-   my ( $self, $row, $cols ) = @_;
-   PTDEBUG && _d('Make UPDATE');
-   if ( $self->{replace} ) {
-      return $self->make_row('REPLACE', $row, $cols);
-   }
-   my %in_where = map { $_ => 1 } @$cols;
-   my $where = $self->make_where_clause($row, $cols);
-   my @cols;
-   if ( my $dbh = $self->{fetch_back} ) {
-      my $sql = $self->make_fetch_back_query($where);
-      PTDEBUG && _d('Fetching data on dbh', $dbh, 'for UPDATE:', $sql);
-      my $res = $dbh->selectrow_hashref($sql);
-      @{$row}{keys %$res} = values %$res;
-      @cols = $self->sort_cols($res);
-   }
-   else {
-      @cols = $self->sort_cols($row);
-   }
-   my $types = $self->{tbl_struct}->{type_for};
-   return "UPDATE $self->{dst_db_tbl} SET "
-      . join(', ', map {
-            my $is_char = ($types->{$_} || '') =~ m/char|text/i;
-            $self->{Quoter}->quote($_)
-            . '=' .  $self->{Quoter}->quote_val($row->{$_},
-                                              is_char => $is_char);
-         } grep { !$in_where{$_} } @cols)
-      . " WHERE $where LIMIT 1";
-}
-
-sub make_INSERT {
-   my ( $self, $row, $cols ) = @_;
-   PTDEBUG && _d('Make INSERT');
-   if ( $self->{replace} ) {
-      return $self->make_row('REPLACE', $row, $cols);
-   }
-   return $self->make_row('INSERT', $row, $cols);
-}
-
-sub make_REPLACE {
-   my ( $self, $row, $cols ) = @_;
-   PTDEBUG && _d('Make REPLACE');
-   return $self->make_row('REPLACE', $row, $cols);
-}
-
-sub make_row {
-   my ( $self, $verb, $row, $cols ) = @_;
-   my @cols; 
-   if ( my $dbh = $self->{fetch_back} ) {
-      my $where = $self->make_where_clause($row, $cols);
-      my $sql   = $self->make_fetch_back_query($where);
-      PTDEBUG && _d('Fetching data on dbh', $dbh, 'for', $verb, ':', $sql);
-      my $res = $dbh->selectrow_hashref($sql);
-      @{$row}{keys %$res} = values %$res;
-      @cols = $self->sort_cols($res);
-   }
-   else {
-      @cols = $self->sort_cols($row);
-   }
-   my $q     = $self->{Quoter};
-   my $type_for = $self->{tbl_struct}->{type_for};
-   return "$verb INTO $self->{dst_db_tbl}("
-      . join(', ', map { $q->quote($_) } @cols)
-      . ') VALUES ('
-      . join(', ', map {
-               my $is_char = ($type_for->{$_} || '') =~ m/char|text/i;
-               $q->quote_val($row->{$_},
-                           is_char => $is_char) } @cols )
-      . ')';
-}
-
-sub make_where_clause {
-   my ( $self, $row, $cols ) = @_;
-   my @clauses = map {
-      my $val = $row->{$_};
-      my $sep = defined $val ? '=' : ' IS ';
-      my $is_char = ($self->{tbl_struct}->{type_for}->{$_} || '') =~ m/char|text/i;
-      $self->{Quoter}->quote($_) . $sep . $self->{Quoter}->quote_val($val,
-                                              is_char => $is_char);
-   } @$cols;
-   return join(' AND ', @clauses);
-}
-
-
-sub get_changes {
-   my ( $self ) = @_;
-   return %{$self->{changes}};
-}
-
-
-sub sort_cols {
-   my ( $self, $row ) = @_;
-   my @cols;
-   if ( $self->{tbl_struct} ) { 
-      my $pos = $self->{tbl_struct}->{col_posn};
-      my @not_in_tbl;
-      @cols = sort {
-            $pos->{$a} <=> $pos->{$b}
-         }
-         grep {
-            if ( !defined $pos->{$_} ) {
-               push @not_in_tbl, $_;
-               0;
-            }
-            else {
-               1;
-            }
-         }
-         keys %$row;
-      push @cols, @not_in_tbl if @not_in_tbl;
-   }
-   else {
-      @cols = sort keys %$row;
-   }
-   return @cols;
-}
-
-sub make_fetch_back_query {
-   my ( $self, $where ) = @_;
-   die "I need a where argument" unless $where;
-   my $cols       = '*';
-   my $tbl_struct = $self->{tbl_struct};
-   if ( $tbl_struct ) {
-      $cols = join(', ',
-         map {
-            my $col = $_;
-            if (    $self->{hex_blob}
-                 && $tbl_struct->{type_for}->{$col} =~ m/b(?:lob|inary)/ ) {
-               $col = "IF(BINARY(`$col`)='', '', CONCAT('0x', HEX(`$col`))) AS `$col`";
-            }
-            else {
-               $col = "`$col`";
-            }
-            $col;
-         } @{ $tbl_struct->{cols} }
-      );
-
-      if ( !$cols ) {
-         PTDEBUG && _d('Failed to make explicit columns list from tbl struct');
-         $cols = '*';
-      }
-   }
-   return "SELECT $cols FROM $self->{src_db_tbl} WHERE $where LIMIT 1";
-}
-
-sub _d {
-   my ($package, undef, $line) = caller 0;
-   @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
-        map { defined $_ ? $_ : 'undef' }
-        @_;
-   print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
-}
-
-1;
-}
-# ###########################################################################
-# End ChangeHandler package
-# ###########################################################################
-
-# ###########################################################################
-# RowDiff package
-# This package is a copy without comments from the original.  The original
-# with comments and its test file can be found in the Bazaar repository at,
-#   lib/RowDiff.pm
-#   t/lib/RowDiff.t
-# See https://launchpad.net/percona-toolkit for more information.
-# ###########################################################################
-{
-package RowDiff;
-
-use strict;
-use warnings FATAL => 'all';
-use English qw(-no_match_vars);
-use constant PTDEBUG => $ENV{PTDEBUG} || 0;
-
-sub new {
-   my ( $class, %args ) = @_;
-   die "I need a dbh" unless $args{dbh};
-   my $self = { %args };
-   return bless $self, $class;
-}
-
-sub compare_sets {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(left_sth right_sth syncer tbl_struct);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   my $left_sth   = $args{left_sth};
-   my $right_sth  = $args{right_sth};
-   my $syncer     = $args{syncer};
-   my $tbl_struct = $args{tbl_struct};
-
-   my ($lr, $rr);    # Current row from the left/right sths.
-   $args{key_cols} = $syncer->key_cols();  # for key_cmp()
-
-   my $left_done  = 0;
-   my $right_done = 0;
-   my $done       = $self->{done};
-
-   do {
-      if ( !$lr && !$left_done ) {
-         PTDEBUG && _d('Fetching row from left');
-         eval { $lr = $left_sth->fetchrow_hashref(); };
-         PTDEBUG && $EVAL_ERROR && _d($EVAL_ERROR);
-         $left_done = !$lr || $EVAL_ERROR ? 1 : 0;
-      }
-      elsif ( PTDEBUG ) {
-         _d('Left still has rows');
-      }
-
-      if ( !$rr && !$right_done ) {
-         PTDEBUG && _d('Fetching row from right');
-         eval { $rr = $right_sth->fetchrow_hashref(); };
-         PTDEBUG && $EVAL_ERROR && _d($EVAL_ERROR);
-         $right_done = !$rr || $EVAL_ERROR ? 1 : 0;
-      }
-      elsif ( PTDEBUG ) {
-         _d('Right still has rows');
-      }
-
-      my $cmp;
-      if ( $lr && $rr ) {
-         $cmp = $self->key_cmp(%args, lr => $lr, rr => $rr);
-         PTDEBUG && _d('Key comparison on left and right:', $cmp);
-      }
-      if ( $lr || $rr ) {
-         if ( $lr && $rr && defined $cmp && $cmp == 0 ) {
-            PTDEBUG && _d('Left and right have the same key');
-            $syncer->same_row(%args, lr => $lr, rr => $rr);
-            $self->{same_row}->(%args, lr => $lr, rr => $rr)
-               if $self->{same_row};
-            $lr = $rr = undef; # Fetch another row from each side.
-         }
-         elsif ( !$rr || ( defined $cmp && $cmp < 0 ) ) {
-            PTDEBUG && _d('Left is not in right');
-            $syncer->not_in_right(%args, lr => $lr, rr => $rr);
-            $self->{not_in_right}->(%args, lr => $lr, rr => $rr)
-               if $self->{not_in_right};
-            $lr = undef;
-         }
-         else {
-            PTDEBUG && _d('Right is not in left');
-            $syncer->not_in_left(%args, lr => $lr, rr => $rr);
-            $self->{not_in_left}->(%args, lr => $lr, rr => $rr)
-               if $self->{not_in_left};
-            $rr = undef;
-         }
-      }
-      $left_done = $right_done = 1 if $done && $done->(%args);
-   } while ( !($left_done && $right_done) );
-   PTDEBUG && _d('No more rows');
-   $syncer->done_with_rows();
-}
-
-sub key_cmp {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(lr rr key_cols tbl_struct);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless exists $args{$arg};
-   }
-   my ($lr, $rr, $key_cols, $tbl_struct) = @args{@required_args};
-   PTDEBUG && _d('Comparing keys using columns:', join(',', @$key_cols));
-
-   my $callback = $self->{key_cmp};
-   my $trf      = $self->{trf};
-
-   foreach my $col ( @$key_cols ) {
-      my $l = $lr->{$col};
-      my $r = $rr->{$col};
-      if ( !defined $l || !defined $r ) {
-         PTDEBUG && _d($col, 'is not defined in both rows');
-         return defined $l ? 1 : defined $r ? -1 : 0;
-      }
-      else {
-         if ( $tbl_struct->{is_numeric}->{$col} ) {   # Numeric column
-            PTDEBUG && _d($col, 'is numeric');
-            ($l, $r) = $trf->($l, $r, $tbl_struct, $col) if $trf;
-            my $cmp = $l <=> $r;
-            if ( $cmp ) {
-               PTDEBUG && _d('Column', $col, 'differs:', $l, '!=', $r);
-               $callback->($col, $l, $r) if $callback;
-               return $cmp;
-            }
-         }
-         elsif ( $l ne $r ) {
-            my $cmp;
-            my $coll = $tbl_struct->{collation_for}->{$col};
-            if ( $coll && ( $coll ne 'latin1_swedish_ci'
-                           || $l =~ m/[^\040-\177]/ || $r =~ m/[^\040-\177]/) )
-            {
-               PTDEBUG && _d('Comparing', $col, 'via MySQL');
-               $cmp = $self->db_cmp($coll, $l, $r);
-            }
-            else {
-               PTDEBUG && _d('Comparing', $col, 'in lowercase');
-               $cmp = lc $l cmp lc $r;
-            }
-            if ( $cmp ) {
-               PTDEBUG && _d('Column', $col, 'differs:', $l, 'ne', $r);
-               $callback->($col, $l, $r) if $callback;
-               return $cmp;
-            }
-         }
-      }
-   }
-   return 0;
-}
-
-sub db_cmp {
-   my ( $self, $collation, $l, $r ) = @_;
-   if ( !$self->{sth}->{$collation} ) {
-      if ( !$self->{charset_for} ) {
-         PTDEBUG && _d('Fetching collations from MySQL');
-         my @collations = @{$self->{dbh}->selectall_arrayref(
-            'SHOW COLLATION', {Slice => { collation => 1, charset => 1 }})};
-         foreach my $collation ( @collations ) {
-            $self->{charset_for}->{$collation->{collation}}
-               = $collation->{charset};
-         }
-      }
-      my $sql = "SELECT STRCMP(_$self->{charset_for}->{$collation}? COLLATE $collation, "
-         . "_$self->{charset_for}->{$collation}? COLLATE $collation) AS res";
-      PTDEBUG && _d($sql);
-      $self->{sth}->{$collation} = $self->{dbh}->prepare($sql);
-   }
-   my $sth = $self->{sth}->{$collation};
-   $sth->execute($l, $r);
-   return $sth->fetchall_arrayref()->[0]->[0];
-}
-
-sub _d {
-   my ($package, undef, $line) = caller 0;
-   @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
-        map { defined $_ ? $_ : 'undef' }
-        @_;
-   print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
-}
-
-1;
-}
-# ###########################################################################
-# End RowDiff package
-# ###########################################################################
-
-# ###########################################################################
-# TableChunker package
-# This package is a copy without comments from the original.  The original
-# with comments and its test file can be found in the Bazaar repository at,
-#   lib/TableChunker.pm
-#   t/lib/TableChunker.t
-# See https://launchpad.net/percona-toolkit for more information.
-# ###########################################################################
-{
-package TableChunker;
-
-use strict;
-use warnings FATAL => 'all';
-use English qw(-no_match_vars);
-use constant PTDEBUG => $ENV{PTDEBUG} || 0;
-
-use POSIX qw(floor ceil);
-use List::Util qw(min max);
-use Data::Dumper;
-$Data::Dumper::Indent    = 1;
-$Data::Dumper::Sortkeys  = 1;
-$Data::Dumper::Quotekeys = 0;
-
-sub new {
-   my ( $class, %args ) = @_;
-   foreach my $arg ( qw(Quoter TableParser) ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-
-   my %int_types  = map { $_ => 1 } qw(bigint date datetime int mediumint smallint time timestamp tinyint year);
-   my %real_types = map { $_ => 1 } qw(decimal double float);
-
-   my $self = {
-      %args,
-      int_types  => \%int_types,
-      real_types => \%real_types,
-      EPOCH      => '1970-01-01',
-   };
-
-   return bless $self, $class;
-}
-
-sub find_chunk_columns {
-   my ( $self, %args ) = @_;
-   foreach my $arg ( qw(tbl_struct) ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my $tbl_struct = $args{tbl_struct};
-
-   my @possible_indexes;
-   foreach my $index ( values %{ $tbl_struct->{keys} } ) {
-
-      next unless $index->{type} eq 'BTREE';
-
-      next if grep { defined } @{$index->{col_prefixes}};
-
-      if ( $args{exact} ) {
-         next unless $index->{is_unique} && @{$index->{cols}} == 1;
-      }
-
-      push @possible_indexes, $index;
-   }
-   PTDEBUG && _d('Possible chunk indexes in order:',
-      join(', ', map { $_->{name} } @possible_indexes));
-
-   my $can_chunk_exact = 0;
-   my @candidate_cols;
-   foreach my $index ( @possible_indexes ) { 
-      my $col = $index->{cols}->[0];
-
-      my $col_type = $tbl_struct->{type_for}->{$col};
-      next unless $self->{int_types}->{$col_type}
-               || $self->{real_types}->{$col_type}
-               || $col_type =~ m/char/;
-
-      push @candidate_cols, { column => $col, index => $index->{name} };
-   }
-
-   $can_chunk_exact = 1 if $args{exact} && scalar @candidate_cols;
-
-   if ( PTDEBUG ) {
-      my $chunk_type = $args{exact} ? 'Exact' : 'Inexact';
-      _d($chunk_type, 'chunkable:',
-         join(', ', map { "$_->{column} on $_->{index}" } @candidate_cols));
-   }
-
-   my @result;
-   PTDEBUG && _d('Ordering columns by order in tbl, PK first');
-   if ( $tbl_struct->{keys}->{PRIMARY} ) {
-      my $pk_first_col = $tbl_struct->{keys}->{PRIMARY}->{cols}->[0];
-      @result          = grep { $_->{column} eq $pk_first_col } @candidate_cols;
-      @candidate_cols  = grep { $_->{column} ne $pk_first_col } @candidate_cols;
-   }
-   my $i = 0;
-   my %col_pos = map { $_ => $i++ } @{$tbl_struct->{cols}};
-   push @result, sort { $col_pos{$a->{column}} <=> $col_pos{$b->{column}} }
-                    @candidate_cols;
-
-   if ( PTDEBUG ) {
-      _d('Chunkable columns:',
-         join(', ', map { "$_->{column} on $_->{index}" } @result));
-      _d('Can chunk exactly:', $can_chunk_exact);
-   }
-
-   return ($can_chunk_exact, @result);
-}
-
-sub calculate_chunks {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(dbh db tbl tbl_struct chunk_col rows_in_range chunk_size);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   PTDEBUG && _d('Calculate chunks for',
-      join(", ", map {"$_=".(defined $args{$_} ? $args{$_} : "undef")}
-         qw(db tbl chunk_col min max rows_in_range chunk_size zero_chunk exact)
-      ));
-
-   if ( !$args{rows_in_range} ) {
-      PTDEBUG && _d("Empty table");
-      return '1=1';
-   }
-
-   if ( $args{rows_in_range} < $args{chunk_size} ) {
-      PTDEBUG && _d("Chunk size larger than rows in range");
-      return '1=1';
-   }
-
-   my $q          = $self->{Quoter};
-   my $dbh        = $args{dbh};
-   my $chunk_col  = $args{chunk_col};
-   my $tbl_struct = $args{tbl_struct};
-   my $col_type   = $tbl_struct->{type_for}->{$chunk_col};
-   PTDEBUG && _d('chunk col type:', $col_type);
-
-   my %chunker;
-   if ( $tbl_struct->{is_numeric}->{$chunk_col} || $col_type =~ /date|time/ ) {
-      %chunker = $self->_chunk_numeric(%args);
-   }
-   elsif ( $col_type =~ m/char/ ) {
-      %chunker = $self->_chunk_char(%args);
-   }
-   else {
-      die "Cannot chunk $col_type columns";
-   }
-   PTDEBUG && _d("Chunker:", Dumper(\%chunker));
-   my ($col, $start_point, $end_point, $interval, $range_func)
-      = @chunker{qw(col start_point end_point interval range_func)};
-
-   my @chunks;
-   if ( $start_point < $end_point ) {
-
-      push @chunks, "$col = 0" if $chunker{have_zero_chunk};
-
-      my ($beg, $end);
-      my $iter = 0;
-      for ( my $i = $start_point; $i < $end_point; $i += $interval ) {
-         ($beg, $end) = $self->$range_func($dbh, $i, $interval, $end_point);
-
-         if ( $iter++ == 0 ) {
-            push @chunks,
-               ($chunker{have_zero_chunk} ? "$col > 0 AND " : "")
-               ."$col < " . $q->quote_val($end);
-         }
-         else {
-            push @chunks, "$col >= " . $q->quote_val($beg) . " AND $col < " . $q->quote_val($end);
-         }
-      }
-
-      my $chunk_range = lc($args{chunk_range} || 'open');
-      my $nullable    = $args{tbl_struct}->{is_nullable}->{$args{chunk_col}};
-      pop @chunks;
-      if ( @chunks ) {
-         push @chunks, "$col >= " . $q->quote_val($beg)
-            . ($chunk_range eq 'openclosed'
-               ? " AND $col <= " . $q->quote_val($args{max}) : "");
-      }
-      else {
-         push @chunks, $nullable ? "$col IS NOT NULL" : '1=1';
-      }
-      if ( $nullable ) {
-         push @chunks, "$col IS NULL";
-      }
-   }
-   else {
-      PTDEBUG && _d('No chunks; using single chunk 1=1');
-      push @chunks, '1=1';
-   }
-
-   return @chunks;
-}
-
-sub _chunk_numeric {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(dbh db tbl tbl_struct chunk_col rows_in_range chunk_size);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   my $q        = $self->{Quoter};
-   my $db_tbl   = $q->quote($args{db}, $args{tbl});
-   my $col_type = $args{tbl_struct}->{type_for}->{$args{chunk_col}};
-
-   my $range_func;
-   if ( $col_type =~ m/(?:int|year|float|double|decimal)$/ ) {
-      $range_func  = 'range_num';
-   }
-   elsif ( $col_type =~ m/^(?:timestamp|date|time)$/ ) {
-      $range_func  = "range_$col_type";
-   }
-   elsif ( $col_type eq 'datetime' ) {
-      $range_func  = 'range_datetime';
-   }
-
-   my ($start_point, $end_point);
-   eval {
-      $start_point = $self->value_to_number(
-         value       => $args{min},
-         column_type => $col_type,
-         dbh         => $args{dbh},
-      );
-      $end_point  = $self->value_to_number(
-         value       => $args{max},
-         column_type => $col_type,
-         dbh         => $args{dbh},
-      );
-   };
-   if ( $EVAL_ERROR ) {
-      if ( $EVAL_ERROR =~ m/don't know how to chunk/ ) {
-         die $EVAL_ERROR;
-      }
-      else {
-         die "Error calculating chunk start and end points for table "
-            . "`$args{tbl_struct}->{name}` on column `$args{chunk_col}` "
-            . "with min/max values "
-            . join('/',
-                  map { defined $args{$_} ? $args{$_} : 'undef' } qw(min max))
-            . ":\n\n"
-            . $EVAL_ERROR
-            . "\nVerify that the min and max values are valid for the column.  "
-            . "If they are valid, this error could be caused by a bug in the "
-            . "tool.";
-      }
-   }
-
-   if ( !defined $start_point ) {
-      PTDEBUG && _d('Start point is undefined');
-      $start_point = 0;
-   }
-   if ( !defined $end_point || $end_point < $start_point ) {
-      PTDEBUG && _d('End point is undefined or before start point');
-      $end_point = 0;
-   }
-   PTDEBUG && _d("Actual chunk range:", $start_point, "to", $end_point);
-
-   my $have_zero_chunk = 0;
-   if ( $args{zero_chunk} ) {
-      if ( $start_point != $end_point && $start_point >= 0 ) {
-         PTDEBUG && _d('Zero chunking');
-         my $nonzero_val = $self->get_nonzero_value(
-            %args,
-            db_tbl   => $db_tbl,
-            col      => $args{chunk_col},
-            col_type => $col_type,
-            val      => $args{min}
-         );
-         $start_point = $self->value_to_number(
-            value       => $nonzero_val,
-            column_type => $col_type,
-            dbh         => $args{dbh},
-         );
-         $have_zero_chunk = 1;
-      }
-      else {
-         PTDEBUG && _d("Cannot zero chunk");
-      }
-   }
-   PTDEBUG && _d("Using chunk range:", $start_point, "to", $end_point);
-
-   my $interval = $args{chunk_size}
-                * ($end_point - $start_point)
-                / $args{rows_in_range};
-   if ( $self->{int_types}->{$col_type} ) {
-      $interval = ceil($interval);
-   }
-   $interval ||= $args{chunk_size};
-   if ( $args{exact} ) {
-      $interval = $args{chunk_size};
-   }
-   PTDEBUG && _d('Chunk interval:', $interval, 'units');
-
-   return (
-      col             => $q->quote($args{chunk_col}),
-      start_point     => $start_point,
-      end_point       => $end_point,
-      interval        => $interval,
-      range_func      => $range_func,
-      have_zero_chunk => $have_zero_chunk,
-   );
-}
-
-sub _chunk_char {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(dbh db tbl tbl_struct chunk_col min max rows_in_range chunk_size);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   my $q          = $self->{Quoter};
-   my $db_tbl     = $q->quote($args{db}, $args{tbl});
-   my $dbh        = $args{dbh};
-   my $chunk_col  = $args{chunk_col};
-   my $qchunk_col = $q->quote($args{chunk_col});
-   my $row;
-   my $sql;
-
-   my ($min_col, $max_col) = @{args}{qw(min max)};
-   $sql = "SELECT ORD(?) AS min_col_ord, ORD(?) AS max_col_ord";
-   PTDEBUG && _d($dbh, $sql);
-   my $ord_sth = $dbh->prepare($sql);  # avoid quoting issues
-   $ord_sth->execute($min_col, $max_col);
-   $row = $ord_sth->fetchrow_arrayref();
-   my ($min_col_ord, $max_col_ord) = ($row->[0], $row->[1]);
-   PTDEBUG && _d("Min/max col char code:", $min_col_ord, $max_col_ord);
-
-   my $base;
-   my @chars;
-   PTDEBUG && _d("Table charset:", $args{tbl_struct}->{charset});
-   if ( ($args{tbl_struct}->{charset} || "") eq "latin1" ) {
-      my @sorted_latin1_chars = (
-          32,  33,  34,  35,  36,  37,  38,  39,  40,  41,  42,  43,  44,  45,
-          46,  47,  48,  49,  50,  51,  52,  53,  54,  55,  56,  57,  58,  59,
-          60,  61,  62,  63,  64,  65,  66,  67,  68,  69,  70,  71,  72,  73,
-          74,  75,  76,  77,  78,  79,  80,  81,  82,  83,  84,  85,  86,  87,
-          88,  89,  90,  91,  92,  93,  94,  95,  96, 123, 124, 125, 126, 161,
-         162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175,
-         176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189,
-         190, 191, 215, 216, 222, 223, 247, 255);
-
-      my ($first_char, $last_char);
-      for my $i ( 0..$#sorted_latin1_chars ) {
-         $first_char = $i and last if $sorted_latin1_chars[$i] >= $min_col_ord;
-      }
-      for my $i ( $first_char..$#sorted_latin1_chars ) {
-         $last_char = $i and last if $sorted_latin1_chars[$i] >= $max_col_ord;
-      };
-
-      @chars = map { chr $_; } @sorted_latin1_chars[$first_char..$last_char];
-      $base  = scalar @chars;
-   }
-   else {
-
-      my $tmp_tbl    = '__maatkit_char_chunking_map';
-      my $tmp_db_tbl = $q->quote($args{db}, $tmp_tbl);
-      $sql = "DROP TABLE IF EXISTS $tmp_db_tbl";
-      PTDEBUG && _d($dbh, $sql);
-      $dbh->do($sql);
-      my $col_def = $args{tbl_struct}->{defs}->{$chunk_col};
-      $sql        = "CREATE TEMPORARY TABLE $tmp_db_tbl ($col_def) "
-                  . "ENGINE=MEMORY";
-      PTDEBUG && _d($dbh, $sql);
-      $dbh->do($sql);
-
-      $sql = "INSERT INTO $tmp_db_tbl VALUES (CHAR(?))";
-      PTDEBUG && _d($dbh, $sql);
-      my $ins_char_sth = $dbh->prepare($sql);  # avoid quoting issues
-      for my $char_code ( $min_col_ord..$max_col_ord ) {
-         $ins_char_sth->execute($char_code);
-      }
-
-      $sql = "SELECT $qchunk_col FROM $tmp_db_tbl "
-           . "WHERE $qchunk_col BETWEEN ? AND ? "
-           . "ORDER BY $qchunk_col";
-      PTDEBUG && _d($dbh, $sql);
-      my $sel_char_sth = $dbh->prepare($sql);
-      $sel_char_sth->execute($min_col, $max_col);
-
-      @chars = map { $_->[0] } @{ $sel_char_sth->fetchall_arrayref() };
-      $base  = scalar @chars;
-
-      $sql = "DROP TABLE $tmp_db_tbl";
-      PTDEBUG && _d($dbh, $sql);
-      $dbh->do($sql);
-   }
-   PTDEBUG && _d("Base", $base, "chars:", @chars);
-
-   die "Cannot chunk table $db_tbl using the character column "
-     . "$chunk_col, most likely because all values start with the "
-     . "same character.  This table must be synced separately by "
-     . "specifying a list of --algorithms without the Chunk algorithm"
-      if $base == 1;
-
-
-   $sql = "SELECT MAX(LENGTH($qchunk_col)) FROM $db_tbl "
-        . ($args{where} ? "WHERE $args{where} " : "") 
-        . "ORDER BY $qchunk_col";
-   PTDEBUG && _d($dbh, $sql);
-   $row = $dbh->selectrow_arrayref($sql);
-   my $max_col_len = $row->[0];
-   PTDEBUG && _d("Max column value:", $max_col, $max_col_len);
-   my $n_values;
-   for my $n_chars ( 1..$max_col_len ) {
-      $n_values = $base**$n_chars;
-      if ( $n_values >= $args{chunk_size} ) {
-         PTDEBUG && _d($n_chars, "chars in base", $base, "expresses",
-            $n_values, "values");
-         last;
-      }
-   }
-
-   my $n_chunks = $args{rows_in_range} / $args{chunk_size};
-   my $interval = floor(($n_values+0.00001) / $n_chunks) || 1;
-
-   my $range_func = sub {
-      my ( $self, $dbh, $start, $interval, $max ) = @_;
-      my $start_char = $self->base_count(
-         count_to => $start,
-         base     => $base,
-         symbols  => \@chars,
-      );
-      my $end_char = $self->base_count(
-         count_to => min($max, $start + $interval),
-         base     => $base,
-         symbols  => \@chars,
-      );
-      return $start_char, $end_char;
-   };
-
-   return (
-      col         => $qchunk_col,
-      start_point => 0,
-      end_point   => $n_values,
-      interval    => $interval,
-      range_func  => $range_func,
-   );
-}
-
-sub get_first_chunkable_column {
-   my ( $self, %args ) = @_;
-   foreach my $arg ( qw(tbl_struct) ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-
-   my ($exact, @cols) = $self->find_chunk_columns(%args);
-   my $col = $cols[0]->{column};
-   my $idx = $cols[0]->{index};
-
-   my $wanted_col = $args{chunk_column};
-   my $wanted_idx = $args{chunk_index};
-   PTDEBUG && _d("Preferred chunk col/idx:", $wanted_col, $wanted_idx);
-
-   if ( $wanted_col && $wanted_idx ) {
-      foreach my $chunkable_col ( @cols ) {
-         if (    $wanted_col eq $chunkable_col->{column}
-              && $wanted_idx eq $chunkable_col->{index} ) {
-            $col = $wanted_col;
-            $idx = $wanted_idx;
-            last;
-         }
-      }
-   }
-   elsif ( $wanted_col ) {
-      foreach my $chunkable_col ( @cols ) {
-         if ( $wanted_col eq $chunkable_col->{column} ) {
-            $col = $wanted_col;
-            $idx = $chunkable_col->{index};
-            last;
-         }
-      }
-   }
-   elsif ( $wanted_idx ) {
-      foreach my $chunkable_col ( @cols ) {
-         if ( $wanted_idx eq $chunkable_col->{index} ) {
-            $col = $chunkable_col->{column};
-            $idx = $wanted_idx;
-            last;
-         }
-      }
-   }
-
-   PTDEBUG && _d('First chunkable col/index:', $col, $idx);
-   return $col, $idx;
-}
-
-sub size_to_rows {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(dbh db tbl chunk_size);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my ($dbh, $db, $tbl, $chunk_size) = @args{@required_args};
-   my $q  = $self->{Quoter};
-   my $tp = $self->{TableParser};
-
-   my ($n_rows, $avg_row_length);
-
-   my ( $num, $suffix ) = $chunk_size =~ m/^(\d+)([MGk])?$/;
-   if ( $suffix ) { # Convert to bytes.
-      $chunk_size = $suffix eq 'k' ? $num * 1_024
-                  : $suffix eq 'M' ? $num * 1_024 * 1_024
-                  :                  $num * 1_024 * 1_024 * 1_024;
-   }
-   elsif ( $num ) {
-      $n_rows = $num;
-   }
-   else {
-      die "Invalid chunk size $chunk_size; must be an integer "
-         . "with optional suffix kMG";
-   }
-
-   if ( $suffix || $args{avg_row_length} ) {
-      my ($status) = $tp->get_table_status($dbh, $db, $tbl);
-      $avg_row_length = $status->{avg_row_length};
-      if ( !defined $n_rows ) {
-         $n_rows = $avg_row_length ? ceil($chunk_size / $avg_row_length) : undef;
-      }
-   }
-
-   return $n_rows, $avg_row_length;
-}
-
-sub get_range_statistics {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(dbh db tbl chunk_col tbl_struct);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my ($dbh, $db, $tbl, $col) = @args{@required_args};
-   my $where = $args{where};
-   my $q     = $self->{Quoter};
-
-   my $col_type       = $args{tbl_struct}->{type_for}->{$col};
-   my $col_is_numeric = $args{tbl_struct}->{is_numeric}->{$col};
-
-   my $db_tbl = $q->quote($db, $tbl);
-   $col       = $q->quote($col);
-
-   my ($min, $max);
-   eval {
-      my $sql = "SELECT MIN($col), MAX($col) FROM $db_tbl"
-              . ($args{index_hint} ? " $args{index_hint}" : "")
-              . ($where ? " WHERE ($where)" : '');
-      PTDEBUG && _d($dbh, $sql);
-      ($min, $max) = $dbh->selectrow_array($sql);
-      PTDEBUG && _d("Actual end points:", $min, $max);
-
-      ($min, $max) = $self->get_valid_end_points(
-         %args,
-         dbh      => $dbh,
-         db_tbl   => $db_tbl,
-         col      => $col,
-         col_type => $col_type,
-         min      => $min,
-         max      => $max,
-      );
-      PTDEBUG && _d("Valid end points:", $min, $max);
-   };
-   if ( $EVAL_ERROR ) {
-      die "Error getting min and max values for table $db_tbl "
-         . "on column $col: $EVAL_ERROR";
-   }
-
-   my $sql = "EXPLAIN SELECT * FROM $db_tbl"
-           . ($args{index_hint} ? " $args{index_hint}" : "")
-           . ($where ? " WHERE $where" : '');
-   PTDEBUG && _d($sql);
-   my $expl = $dbh->selectrow_hashref($sql);
-
-   return (
-      min           => $min,
-      max           => $max,
-      rows_in_range => $expl->{rows},
-   );
-}
-
-sub inject_chunks {
-   my ( $self, %args ) = @_;
-   foreach my $arg ( qw(database table chunks chunk_num query) ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   PTDEBUG && _d('Injecting chunk', $args{chunk_num});
-   my $query   = $args{query};
-   my $comment = sprintf("/*%s.%s:%d/%d*/",
-      $args{database}, $args{table},
-      $args{chunk_num} + 1, scalar @{$args{chunks}});
-   $query =~ s!/\*PROGRESS_COMMENT\*/!$comment!;
-   my $where = "WHERE (" . $args{chunks}->[$args{chunk_num}] . ')';
-   if ( $args{where} && grep { $_ } @{$args{where}} ) {
-      $where .= " AND ("
-         . join(" AND ", map { "($_)" } grep { $_ } @{$args{where}} )
-         . ")";
-   }
-   my $db_tbl     = $self->{Quoter}->quote(@args{qw(database table)});
-   my $index_hint = $args{index_hint} || '';
-
-   PTDEBUG && _d('Parameters:',
-      Dumper({WHERE => $where, DB_TBL => $db_tbl, INDEX_HINT => $index_hint}));
-   $query =~ s!/\*WHERE\*/! $where!;
-   $query =~ s!/\*DB_TBL\*/!$db_tbl!;
-   $query =~ s!/\*INDEX_HINT\*/! $index_hint!;
-   $query =~ s!/\*CHUNK_NUM\*/! $args{chunk_num} AS chunk_num,!;
-
-   return $query;
-}
-
-
-sub value_to_number {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(column_type dbh);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   my $val = $args{value};
-   my ($col_type, $dbh) = @args{@required_args};
-   PTDEBUG && _d('Converting MySQL', $col_type, $val);
-
-   return unless defined $val;  # value is NULL
-
-   my %mysql_conv_func_for = (
-      timestamp => 'UNIX_TIMESTAMP',
-      date      => 'TO_DAYS',
-      time      => 'TIME_TO_SEC',
-      datetime  => 'TO_DAYS',
-   );
-
-   my $num;
-   if ( $col_type =~ m/(?:int|year|float|double|decimal)$/ ) {
-      $num = $val;
-   }
-   elsif ( $col_type =~ m/^(?:timestamp|date|time)$/ ) {
-      my $func = $mysql_conv_func_for{$col_type};
-      my $sql = "SELECT $func(?)";
-      PTDEBUG && _d($dbh, $sql, $val);
-      my $sth = $dbh->prepare($sql);
-      $sth->execute($val);
-      ($num) = $sth->fetchrow_array();
-   }
-   elsif ( $col_type eq 'datetime' ) {
-      $num = $self->timestampdiff($dbh, $val);
-   }
-   else {
-      die "I don't know how to chunk $col_type\n";
-   }
-   PTDEBUG && _d('Converts to', $num);
-   return $num;
-}
-
-sub range_num {
-   my ( $self, $dbh, $start, $interval, $max ) = @_;
-   my $end = min($max, $start + $interval);
-
-
-   $start = sprintf('%.17f', $start) if $start =~ /e/;
-   $end   = sprintf('%.17f', $end)   if $end   =~ /e/;
-
-   $start =~ s/\.(\d{5}).*$/.$1/;
-   $end   =~ s/\.(\d{5}).*$/.$1/;
-
-   if ( $end > $start ) {
-      return ( $start, $end );
-   }
-   else {
-      die "Chunk size is too small: $end !> $start\n";
-   }
-}
-
-sub range_time {
-   my ( $self, $dbh, $start, $interval, $max ) = @_;
-   my $sql = "SELECT SEC_TO_TIME($start), SEC_TO_TIME(LEAST($max, $start + $interval))";
-   PTDEBUG && _d($sql);
-   return $dbh->selectrow_array($sql);
-}
-
-sub range_date {
-   my ( $self, $dbh, $start, $interval, $max ) = @_;
-   my $sql = "SELECT FROM_DAYS($start), FROM_DAYS(LEAST($max, $start + $interval))";
-   PTDEBUG && _d($sql);
-   return $dbh->selectrow_array($sql);
-}
-
-sub range_datetime {
-   my ( $self, $dbh, $start, $interval, $max ) = @_;
-   my $sql = "SELECT DATE_ADD('$self->{EPOCH}', INTERVAL $start SECOND), "
-       . "DATE_ADD('$self->{EPOCH}', INTERVAL LEAST($max, $start + $interval) SECOND)";
-   PTDEBUG && _d($sql);
-   return $dbh->selectrow_array($sql);
-}
-
-sub range_timestamp {
-   my ( $self, $dbh, $start, $interval, $max ) = @_;
-   my $sql = "SELECT FROM_UNIXTIME($start), FROM_UNIXTIME(LEAST($max, $start + $interval))";
-   PTDEBUG && _d($sql);
-   return $dbh->selectrow_array($sql);
-}
-
-sub timestampdiff {
-   my ( $self, $dbh, $time ) = @_;
-   my $sql = "SELECT (COALESCE(TO_DAYS('$time'), 0) * 86400 + TIME_TO_SEC('$time')) "
-      . "- TO_DAYS('$self->{EPOCH} 00:00:00') * 86400";
-   PTDEBUG && _d($sql);
-   my ( $diff ) = $dbh->selectrow_array($sql);
-   $sql = "SELECT DATE_ADD('$self->{EPOCH}', INTERVAL $diff SECOND)";
-   PTDEBUG && _d($sql);
-   my ( $check ) = $dbh->selectrow_array($sql);
-   die <<"   EOF"
-   Incorrect datetime math: given $time, calculated $diff but checked to $check.
-   This could be due to a version of MySQL that overflows on large interval
-   values to DATE_ADD(), or the given datetime is not a valid date.  If not,
-   please report this as a bug.
-   EOF
-      unless $check eq $time;
-   return $diff;
-}
-
-
-
-
-sub get_valid_end_points {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(dbh db_tbl col col_type);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my ($dbh, $db_tbl, $col, $col_type) = @args{@required_args};
-   my ($real_min, $real_max)           = @args{qw(min max)};
-
-   my $err_fmt = "Error finding a valid %s value for table $db_tbl on "
-               . "column $col. The real %s value %s is invalid and "
-               . "no other valid values were found.  Verify that the table "
-               . "has at least one valid value for this column"
-               . ($args{where} ? " where $args{where}." : ".");
-
-   my $valid_min = $real_min;
-   if ( defined $valid_min ) {
-      PTDEBUG && _d("Validating min end point:", $real_min);
-      $valid_min = $self->_get_valid_end_point(
-         %args,
-         val      => $real_min,
-         endpoint => 'min',
-      );
-      die sprintf($err_fmt, 'minimum', 'minimum',
-         (defined $real_min ? $real_min : "NULL"))
-         unless defined $valid_min;
-   }
-
-   my $valid_max = $real_max;
-   if ( defined $valid_max ) {
-      PTDEBUG && _d("Validating max end point:", $real_min);
-      $valid_max = $self->_get_valid_end_point(
-         %args,
-         val      => $real_max,
-         endpoint => 'max',
-      );
-      die sprintf($err_fmt, 'maximum', 'maximum',
-         (defined $real_max ? $real_max : "NULL"))
-         unless defined $valid_max;
-   }
-
-   return $valid_min, $valid_max;
-}
-
-sub _get_valid_end_point {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(dbh db_tbl col col_type);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my ($dbh, $db_tbl, $col, $col_type) = @args{@required_args};
-   my $val = $args{val};
-
-   return $val unless defined $val;
-
-   my $validate = $col_type =~ m/time|date/ ? \&_validate_temporal_value
-                :                             undef;
-
-   if ( !$validate ) {
-      PTDEBUG && _d("No validator for", $col_type, "values");
-      return $val;
-   }
-
-   return $val if defined $validate->($dbh, $val);
-
-   PTDEBUG && _d("Value is invalid, getting first valid value");
-   $val = $self->get_first_valid_value(
-      %args,
-      val      => $val,
-      validate => $validate,
-   );
-
-   return $val;
-}
-
-sub get_first_valid_value {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(dbh db_tbl col validate endpoint);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my ($dbh, $db_tbl, $col, $validate, $endpoint) = @args{@required_args};
-   my $tries = defined $args{tries} ? $args{tries} : 5;
-   my $val   = $args{val};
-
-   return unless defined $val;
-
-   my $cmp = $endpoint =~ m/min/i ? '>'
-           : $endpoint =~ m/max/i ? '<'
-           :                        die "Invalid endpoint arg: $endpoint";
-   my $sql = "SELECT $col FROM $db_tbl "
-           . ($args{index_hint} ? "$args{index_hint} " : "")
-           . "WHERE $col $cmp ? AND $col IS NOT NULL "
-           . ($args{where} ? "AND ($args{where}) " : "")
-           . "ORDER BY $col LIMIT 1";
-   PTDEBUG && _d($dbh, $sql);
-   my $sth = $dbh->prepare($sql);
-
-   my $last_val = $val;
-   while ( $tries-- ) {
-      $sth->execute($last_val);
-      my ($next_val) = $sth->fetchrow_array();
-      PTDEBUG && _d('Next value:', $next_val, '; tries left:', $tries);
-      if ( !defined $next_val ) {
-         PTDEBUG && _d('No more rows in table');
-         last;
-      }
-      if ( defined $validate->($dbh, $next_val) ) {
-         PTDEBUG && _d('First valid value:', $next_val);
-         $sth->finish();
-         return $next_val;
-      }
-      $last_val = $next_val;
-   }
-   $sth->finish();
-   $val = undef;  # no valid value found
-
-   return $val;
-}
-
-sub _validate_temporal_value {
-   my ( $dbh, $val ) = @_;
-   my $sql = "SELECT IF(TIME_FORMAT(?,'%H:%i:%s')=?, TIME_TO_SEC(?), TO_DAYS(?))";
-   my $res;
-   eval {
-      PTDEBUG && _d($dbh, $sql, $val);
-      my $sth = $dbh->prepare($sql);
-      $sth->execute($val, $val, $val, $val);
-      ($res) = $sth->fetchrow_array();
-      $sth->finish();
-   };
-   if ( $EVAL_ERROR ) {
-      PTDEBUG && _d($EVAL_ERROR);
-   }
-   return $res;
-}
-
-sub get_nonzero_value {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(dbh db_tbl col col_type);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my ($dbh, $db_tbl, $col, $col_type) = @args{@required_args};
-   my $tries = defined $args{tries} ? $args{tries} : 5;
-   my $val   = $args{val};
-
-   my $is_nonzero = $col_type =~ m/time|date/ ? \&_validate_temporal_value
-                  :                             sub { return $_[1]; };
-
-   if ( !$is_nonzero->($dbh, $val) ) {  # quasi-double-negative, sorry
-      PTDEBUG && _d('Discarding zero value:', $val);
-      my $sql = "SELECT $col FROM $db_tbl "
-              . ($args{index_hint} ? "$args{index_hint} " : "")
-              . "WHERE $col > ? AND $col IS NOT NULL "
-              . ($args{where} ? "AND ($args{where}) " : '')
-              . "ORDER BY $col LIMIT 1";
-      PTDEBUG && _d($sql);
-      my $sth = $dbh->prepare($sql);
-
-      my $last_val = $val;
-      while ( $tries-- ) {
-         $sth->execute($last_val);
-         my ($next_val) = $sth->fetchrow_array();
-         if ( $is_nonzero->($dbh, $next_val) ) {
-            PTDEBUG && _d('First non-zero value:', $next_val);
-            $sth->finish();
-            return $next_val;
-         }
-         $last_val = $next_val;
-      }
-      $sth->finish();
-      $val = undef;  # no non-zero value found
-   }
-
-   return $val;
-}
-
-sub base_count {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(count_to base symbols);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   my ($n, $base, $symbols) = @args{@required_args};
-
-   return $symbols->[0] if $n == 0;
-
-   my $highest_power = floor(log($n+0.00001)/log($base));
-   if ( $highest_power == 0 ){
-      return $symbols->[$n];
-   }
-
-   my @base_powers;
-   for my $power ( 0..$highest_power ) {
-      push @base_powers, ($base**$power) || 1;  
-   }
-
-   my @base_multiples;
-   foreach my $base_power ( reverse @base_powers ) {
-      my $multiples = floor(($n+0.00001) / $base_power);
-      push @base_multiples, $multiples;
-      $n -= $multiples * $base_power;
-   }
-   return join('', map { $symbols->[$_] } @base_multiples);
-}
-
-sub _d {
-   my ($package, undef, $line) = caller 0;
-   @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
-        map { defined $_ ? $_ : 'undef' }
-        @_;
-   print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
-}
-
-1;
-}
-# ###########################################################################
-# End TableChunker package
-# ###########################################################################
-
-# ###########################################################################
-# TableNibbler package
-# This package is a copy without comments from the original.  The original
-# with comments and its test file can be found in the Bazaar repository at,
-#   lib/TableNibbler.pm
-#   t/lib/TableNibbler.t
-# See https://launchpad.net/percona-toolkit for more information.
-# ###########################################################################
-{
-package TableNibbler;
-
-use strict;
-use warnings FATAL => 'all';
-use English qw(-no_match_vars);
-use constant PTDEBUG => $ENV{PTDEBUG} || 0;
-
-sub new {
-   my ( $class, %args ) = @_;
-   my @required_args = qw(TableParser Quoter);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my $self = { %args };
-   return bless $self, $class;
-}
-
-sub generate_asc_stmt {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(tbl_struct index);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   my ($tbl_struct, $index) = @args{@required_args};
-   my @cols = $args{cols} ? @{$args{cols}} : @{$tbl_struct->{cols}};
-   my $q    = $self->{Quoter};
-
-   die "Index '$index' does not exist in table"
-      unless exists $tbl_struct->{keys}->{$index};
-   PTDEBUG && _d('Will ascend index', $index);  
-
-   my @asc_cols = @{$tbl_struct->{keys}->{$index}->{cols}};
-   if ( $args{asc_first} ) {
-      PTDEBUG && _d('Ascending only first column');
-      @asc_cols = $asc_cols[0];
-   }
-   elsif ( my $n = $args{n_index_cols} ) {
-      $n = scalar @asc_cols if $n > @asc_cols;
-      PTDEBUG && _d('Ascending only first', $n, 'columns');
-      @asc_cols = @asc_cols[0..($n-1)];
-   }
-   PTDEBUG && _d('Will ascend columns', join(', ', @asc_cols));
-
-   my @asc_slice;
-   my %col_posn = do { my $i = 0; map { $_ => $i++ } @cols };
-   foreach my $col ( @asc_cols ) {
-      if ( !exists $col_posn{$col} ) {
-         push @cols, $col;
-         $col_posn{$col} = $#cols;
-      }
-      push @asc_slice, $col_posn{$col};
-   }
-   PTDEBUG && _d('Will ascend, in ordinal position:', join(', ', @asc_slice));
-
-   my $asc_stmt = {
-      cols  => \@cols,
-      index => $index,
-      where => '',
-      slice => [],
-      scols => [],
-   };
-
-   if ( @asc_slice ) {
-      my $cmp_where;
-      foreach my $cmp ( qw(< <= >= >) ) {
-         $cmp_where = $self->generate_cmp_where(
-            type        => $cmp,
-            slice       => \@asc_slice,
-            cols        => \@cols,
-            quoter      => $q,
-            is_nullable => $tbl_struct->{is_nullable},
-         );
-         $asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where};
-      }
-      my $cmp = $args{asc_only} ? '>' : '>=';
-      $asc_stmt->{where} = $asc_stmt->{boundaries}->{$cmp};
-      $asc_stmt->{slice} = $cmp_where->{slice};
-      $asc_stmt->{scols} = $cmp_where->{scols};
-   }
-
-   return $asc_stmt;
-}
-
-sub generate_cmp_where {
-   my ( $self, %args ) = @_;
-   foreach my $arg ( qw(type slice cols is_nullable) ) {
-      die "I need a $arg arg" unless defined $args{$arg};
-   }
-   my @slice       = @{$args{slice}};
-   my @cols        = @{$args{cols}};
-   my $is_nullable = $args{is_nullable};
-   my $type        = $args{type};
-   my $q           = $self->{Quoter};
-
-   (my $cmp = $type) =~ s/=//;
-
-   my @r_slice;    # Resulting slice columns, by ordinal
-   my @r_scols;    # Ditto, by name
-
-   my @clauses;
-   foreach my $i ( 0 .. $#slice ) {
-      my @clause;
-
-      foreach my $j ( 0 .. $i - 1 ) {
-         my $ord = $slice[$j];
-         my $col = $cols[$ord];
-         my $quo = $q->quote($col);
-         if ( $is_nullable->{$col} ) {
-            push @clause, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))";
-            push @r_slice, $ord, $ord;
-            push @r_scols, $col, $col;
-         }
-         else {
-            push @clause, "$quo = ?";
-            push @r_slice, $ord;
-            push @r_scols, $col;
-         }
-      }
-
-      my $ord = $slice[$i];
-      my $col = $cols[$ord];
-      my $quo = $q->quote($col);
-      my $end = $i == $#slice; # Last clause of the whole group.
-      if ( $is_nullable->{$col} ) {
-         if ( $type =~ m/=/ && $end ) {
-            push @clause, "(? IS NULL OR $quo $type ?)";
-         }
-         elsif ( $type =~ m/>/ ) {
-            push @clause, "((? IS NULL AND $quo IS NOT NULL) OR ($quo $cmp ?))";
-         }
-         else { # If $type =~ m/</ ) {
-            push @clause, "((? IS NOT NULL AND $quo IS NULL) OR ($quo $cmp ?))";
-         }
-         push @r_slice, $ord, $ord;
-         push @r_scols, $col, $col;
-      }
-      else {
-         push @r_slice, $ord;
-         push @r_scols, $col;
-         push @clause, ($type =~ m/=/ && $end ? "$quo $type ?" : "$quo $cmp ?");
-      }
-
-      push @clauses, '(' . join(' AND ', @clause) . ')';
-   }
-   my $result = '(' . join(' OR ', @clauses) . ')';
-   my $where = {
-      slice => \@r_slice,
-      scols => \@r_scols,
-      where => $result,
-   };
-   return $where;
-}
-
-sub generate_del_stmt {
-   my ( $self, %args ) = @_;
-
-   my $tbl  = $args{tbl_struct};
-   my @cols = $args{cols} ? @{$args{cols}} : ();
-   my $tp   = $self->{TableParser};
-   my $q    = $self->{Quoter};
-
-   my @del_cols;
-   my @del_slice;
-
-   my $index = $tp->find_best_index($tbl, $args{index});
-   die "Cannot find an ascendable index in table" unless $index;
-
-   if ( $index ) {
-      @del_cols = @{$tbl->{keys}->{$index}->{cols}};
-   }
-   else {
-      @del_cols = @{$tbl->{cols}};
-   }
-   PTDEBUG && _d('Columns needed for DELETE:', join(', ', @del_cols));
-
-   my %col_posn = do { my $i = 0; map { $_ => $i++ } @cols };
-   foreach my $col ( @del_cols ) {
-      if ( !exists $col_posn{$col} ) {
-         push @cols, $col;
-         $col_posn{$col} = $#cols;
-      }
-      push @del_slice, $col_posn{$col};
-   }
-   PTDEBUG && _d('Ordinals needed for DELETE:', join(', ', @del_slice));
-
-   my $del_stmt = {
-      cols  => \@cols,
-      index => $index,
-      where => '',
-      slice => [],
-      scols => [],
-   };
-
-   my @clauses;
-   foreach my $i ( 0 .. $#del_slice ) {
-      my $ord = $del_slice[$i];
-      my $col = $cols[$ord];
-      my $quo = $q->quote($col);
-      if ( $tbl->{is_nullable}->{$col} ) {
-         push @clauses, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))";
-         push @{$del_stmt->{slice}}, $ord, $ord;
-         push @{$del_stmt->{scols}}, $col, $col;
-      }
-      else {
-         push @clauses, "$quo = ?";
-         push @{$del_stmt->{slice}}, $ord;
-         push @{$del_stmt->{scols}}, $col;
-      }
-   }
-
-   $del_stmt->{where} = '(' . join(' AND ', @clauses) . ')';
-
-   return $del_stmt;
-}
-
-sub generate_ins_stmt {
-   my ( $self, %args ) = @_;
-   foreach my $arg ( qw(ins_tbl sel_cols) ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my $ins_tbl  = $args{ins_tbl};
-   my @sel_cols = @{$args{sel_cols}};
-
-   die "You didn't specify any SELECT columns" unless @sel_cols;
-
-   my @ins_cols;
-   my @ins_slice;
-   for my $i ( 0..$#sel_cols ) {
-      next unless $ins_tbl->{is_col}->{$sel_cols[$i]};
-      push @ins_cols, $sel_cols[$i];
-      push @ins_slice, $i;
-   }
-
-   return {
-      cols  => \@ins_cols,
-      slice => \@ins_slice,
-   };
-}
-
-sub _d {
-   my ($package, undef, $line) = caller 0;
-   @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
-        map { defined $_ ? $_ : 'undef' }
-        @_;
-   print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
-}
-
-1;
-}
-# ###########################################################################
-# End TableNibbler package
-# ###########################################################################
-
-# ###########################################################################
-# TableChecksum package
-# This package is a copy without comments from the original.  The original
-# with comments and its test file can be found in the Bazaar repository at,
-#   lib/TableChecksum.pm
-#   t/lib/TableChecksum.t
-# See https://launchpad.net/percona-toolkit for more information.
-# ###########################################################################
-{
-package TableChecksum;
-
-use strict;
-use warnings FATAL => 'all';
-use English qw(-no_match_vars);
-use constant PTDEBUG => $ENV{PTDEBUG} || 0;
-
-use List::Util qw(max);
-
-our %ALGOS = (
-   CHECKSUM => { pref => 0, hash => 0 },
-   BIT_XOR  => { pref => 2, hash => 1 },
-   ACCUM    => { pref => 3, hash => 1 },
-);
-
-sub new {
-   my ( $class, %args ) = @_;
-   foreach my $arg ( qw(Quoter) ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   my $self = { %args };
-   return bless $self, $class;
-}
-
-sub crc32 {
-   my ( $self, $string ) = @_;
-   my $poly = 0xEDB88320;
-   my $crc  = 0xFFFFFFFF;
-   foreach my $char ( split(//, $string) ) {
-      my $comp = ($crc ^ ord($char)) & 0xFF;
-      for ( 1 .. 8 ) {
-         $comp = $comp & 1 ? $poly ^ ($comp >> 1) : $comp >> 1;
-      }
-      $crc = (($crc >> 8) & 0x00FFFFFF) ^ $comp;
-   }
-   return $crc ^ 0xFFFFFFFF;
-}
-
-sub get_crc_wid {
-   my ( $self, $dbh, $func ) = @_;
-   my $crc_wid = 16;
-   if ( uc $func ne 'FNV_64' && uc $func ne 'FNV1A_64' ) {
-      eval {
-         my ($val) = $dbh->selectrow_array("SELECT $func('a')");
-         $crc_wid = max(16, length($val));
-      };
-   }
-   return $crc_wid;
-}
-
-sub get_crc_type {
-   my ( $self, $dbh, $func ) = @_;
-   my $type   = '';
-   my $length = 0;
-   my $sql    = "SELECT $func('a')";
-   my $sth    = $dbh->prepare($sql);
-   eval {
-      $sth->execute();
-      $type   = $sth->{mysql_type_name}->[0];
-      $length = $sth->{mysql_length}->[0];
-      PTDEBUG && _d($sql, $type, $length);
-      if ( $type eq 'bigint' && $length < 20 ) {
-         $type = 'int';
-      }
-   };
-   $sth->finish;
-   PTDEBUG && _d('crc_type:', $type, 'length:', $length);
-   return ($type, $length);
-}
-
-sub best_algorithm {
-   my ( $self, %args ) = @_;
-   my ( $alg, $dbh ) = @args{ qw(algorithm dbh) };
-   my @choices = sort { $ALGOS{$a}->{pref} <=> $ALGOS{$b}->{pref} } keys %ALGOS;
-   die "Invalid checksum algorithm $alg"
-      if $alg && !$ALGOS{$alg};
-
-   if (
-      $args{where} || $args{chunk}        # CHECKSUM does whole table
-      || $args{replicate})                # CHECKSUM can't do INSERT.. SELECT
-   {
-      PTDEBUG && _d('Cannot use CHECKSUM algorithm');
-      @choices = grep { $_ ne 'CHECKSUM' } @choices;
-   }
-
-
-   if ( $alg && grep { $_ eq $alg } @choices ) {
-      PTDEBUG && _d('User requested', $alg, 'algorithm');
-      return $alg;
-   }
-
-   if ( $args{count} && grep { $_ ne 'CHECKSUM' } @choices ) {
-      PTDEBUG && _d('Not using CHECKSUM algorithm because COUNT desired');
-      @choices = grep { $_ ne 'CHECKSUM' } @choices;
-   }
-
-   PTDEBUG && _d('Algorithms, in order:', @choices);
-   return $choices[0];
-}
-
-sub is_hash_algorithm {
-   my ( $self, $algorithm ) = @_;
-   return $ALGOS{$algorithm} && $ALGOS{$algorithm}->{hash};
-}
-
-sub choose_hash_func {
-   my ( $self, %args ) = @_;
-   my @funcs = qw(CRC32 FNV1A_64 FNV_64 MD5 SHA1);
-   if ( $args{function} ) {
-      unshift @funcs, $args{function};
-   }
-   my ($result, $error);
-   do {
-      my $func;
-      eval {
-         $func = shift(@funcs);
-         my $sql = "SELECT $func('test-string')";
-         PTDEBUG && _d($sql);
-         $args{dbh}->do($sql);
-         $result = $func;
-      };
-      if ( $EVAL_ERROR && $EVAL_ERROR =~ m/failed: (.*?) at \S+ line/ ) {
-         $error .= qq{$func cannot be used because "$1"\n};
-         PTDEBUG && _d($func, 'cannot be used because', $1);
-      }
-   } while ( @funcs && !$result );
-
-   die $error unless $result;
-   PTDEBUG && _d('Chosen hash func:', $result);
-   return $result;
-}
-
-sub optimize_xor {
-   my ( $self, %args ) = @_;
-   my ($dbh, $func) = @args{qw(dbh function)};
-
-   die "$func never needs the BIT_XOR optimization"
-      if $func =~ m/^(?:FNV1A_64|FNV_64|CRC32)$/i;
-
-   my $opt_slice = 0;
-   my $unsliced  = uc $dbh->selectall_arrayref("SELECT $func('a')")->[0]->[0];
-   my $sliced    = '';
-   my $start     = 1;
-   my $crc_wid   = length($unsliced) < 16 ? 16 : length($unsliced);
-
-   do { # Try different positions till sliced result equals non-sliced.
-      PTDEBUG && _d('Trying slice', $opt_slice);
-      $dbh->do(q{SET @crc := '', @cnt := 0});
-      my $slices = $self->make_xor_slices(
-         query     => "\@crc := $func('a')",
-         crc_wid   => $crc_wid,
-         opt_slice => $opt_slice,
-      );
-
-      my $sql = "SELECT CONCAT($slices) AS TEST FROM (SELECT NULL) AS x";
-      $sliced = ($dbh->selectrow_array($sql))[0];
-      if ( $sliced ne $unsliced ) {
-         PTDEBUG && _d('Slice', $opt_slice, 'does not work');
-         $start += 16;
-         ++$opt_slice;
-      }
-   } while ( $start < $crc_wid && $sliced ne $unsliced );
-
-   if ( $sliced eq $unsliced ) {
-      PTDEBUG && _d('Slice', $opt_slice, 'works');
-      return $opt_slice;
-   }
-   else {
-      PTDEBUG && _d('No slice works');
-      return undef;
-   }
-}
-
-sub make_xor_slices {
-   my ( $self, %args ) = @_;
-   foreach my $arg ( qw(query crc_wid) ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   my ( $query, $crc_wid, $opt_slice ) = @args{qw(query crc_wid opt_slice)};
-
-   my @slices;
-   for ( my $start = 1; $start <= $crc_wid; $start += 16 ) {
-      my $len = $crc_wid - $start + 1;
-      if ( $len > 16 ) {
-         $len = 16;
-      }
-      push @slices,
-         "LPAD(CONV(BIT_XOR("
-         . "CAST(CONV(SUBSTRING(\@crc, $start, $len), 16, 10) AS UNSIGNED))"
-         . ", 10, 16), $len, '0')";
-   }
-
-   if ( defined $opt_slice && $opt_slice < @slices ) {
-      $slices[$opt_slice] =~ s/\@crc/\@crc := $query/;
-   }
-   else {
-      map { s/\@crc/$query/ } @slices;
-   }
-
-   return join(', ', @slices);
-}
-
-sub make_row_checksum {
-   my ( $self, %args ) = @_;
-   my ( $tbl_struct, $func ) = @args{ qw(tbl_struct function) };
-   my $q = $self->{Quoter};
-
-   my $sep = $args{sep} || '#';
-   $sep =~ s/'//g;
-   $sep ||= '#';
-
-   my $ignorecols = $args{ignorecols} || {};
-
-   my %cols = map { lc($_) => 1 }
-              grep { !exists $ignorecols->{$_} }
-              ($args{cols} ? @{$args{cols}} : @{$tbl_struct->{cols}});
-   my %seen;
-   my @cols =
-      map {
-         my $type = $tbl_struct->{type_for}->{$_};
-         my $result = $q->quote($_);
-         if ( $type eq 'timestamp' ) {
-            $result .= ' + 0';
-         }
-         elsif ( $args{float_precision} && $type =~ m/float|double/ ) {
-            $result = "ROUND($result, $args{float_precision})";
-         }
-         elsif ( $args{trim} && $type =~ m/varchar/ ) {
-            $result = "TRIM($result)";
-         }
-         $result;
-      }
-      grep {
-         $cols{$_} && !$seen{$_}++
-      }
-      @{$tbl_struct->{cols}};
-
-   my $query;
-   if ( !$args{no_cols} ) {
-      $query = join(', ',
-                  map { 
-                     my $col = $_;
-                     if ( $col =~ m/\+ 0/ ) {
-                        my ($real_col) = /^(\S+)/;
-                        $col .= " AS $real_col";
-                     }
-                     elsif ( $col =~ m/TRIM/ ) {
-                        my ($real_col) = m/TRIM\(([^\)]+)\)/;
-                        $col .= " AS $real_col";
-                     }
-                     $col;
-                  } @cols)
-             . ', ';
-   }
-
-   if ( uc $func ne 'FNV_64' && uc $func ne 'FNV1A_64' ) {
-      my @nulls = grep { $cols{$_} } @{$tbl_struct->{null_cols}};
-      if ( @nulls ) {
-         my $bitmap = "CONCAT("
-            . join(', ', map { 'ISNULL(' . $q->quote($_) . ')' } @nulls)
-            . ")";
-         push @cols, $bitmap;
-      }
-
-      $query .= @cols > 1
-              ? "$func(CONCAT_WS('$sep', " . join(', ', @cols) . '))'
-              : "$func($cols[0])";
-   }
-   else {
-      my $fnv_func = uc $func;
-      $query .= "$fnv_func(" . join(', ', @cols) . ')';
-   }
-
-   return $query;
-}
-
-sub make_checksum_query {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(db tbl tbl_struct algorithm crc_wid crc_type);
-   foreach my $arg( @required_args ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my ( $db, $tbl, $tbl_struct, $algorithm,
-        $crc_wid, $crc_type) = @args{@required_args};
-   my $func = $args{function};
-   my $q = $self->{Quoter};
-   my $result;
-
-   die "Invalid or missing checksum algorithm"
-      unless $algorithm && $ALGOS{$algorithm};
-
-   if ( $algorithm eq 'CHECKSUM' ) {
-      return "CHECKSUM TABLE " . $q->quote($db, $tbl);
-   }
-
-   my $expr = $self->make_row_checksum(%args, no_cols=>1);
-
-   if ( $algorithm eq 'BIT_XOR' ) {
-      if ( $crc_type =~ m/int$/ ) {
-         $result = "COALESCE(LOWER(CONV(BIT_XOR(CAST($expr AS UNSIGNED)), 10, 16)), 0) AS crc ";
-      }
-      else {
-         my $slices = $self->make_xor_slices( query => $expr, %args );
-         $result = "COALESCE(LOWER(CONCAT($slices)), 0) AS crc ";
-      }
-   }
-   else {
-      if ( $crc_type =~ m/int$/ ) {
-         $result = "COALESCE(RIGHT(MAX("
-            . "\@crc := CONCAT(LPAD(\@cnt := \@cnt + 1, 16, '0'), "
-            . "CONV(CAST($func(CONCAT(\@crc, $expr)) AS UNSIGNED), 10, 16))"
-            . "), $crc_wid), 0) AS crc ";
-      }
-      else {
-         $result = "COALESCE(RIGHT(MAX("
-            . "\@crc := CONCAT(LPAD(\@cnt := \@cnt + 1, 16, '0'), "
-            . "$func(CONCAT(\@crc, $expr)))"
-            . "), $crc_wid), 0) AS crc ";
-      }
-   }
-   if ( $args{replicate} ) {
-      $result = "REPLACE /*PROGRESS_COMMENT*/ INTO $args{replicate} "
-         . "(db, tbl, chunk, boundaries, this_cnt, this_crc) "
-         . "SELECT ?, ?, /*CHUNK_NUM*/ ?, COUNT(*) AS cnt, $result";
-   }
-   else {
-      $result = "SELECT "
-         . ($args{buffer} ? 'SQL_BUFFER_RESULT ' : '')
-         . "/*PROGRESS_COMMENT*//*CHUNK_NUM*/ COUNT(*) AS cnt, $result";
-   }
-   return $result . "FROM /*DB_TBL*//*INDEX_HINT*//*WHERE*/";
-}
-
-sub find_replication_differences {
-   my ( $self, $dbh, $table ) = @_;
-
-   my $sql
-      = "SELECT db, tbl, CONCAT(db, '.', tbl) AS `table`, "
-      . "chunk, chunk_index, lower_boundary, upper_boundary, "
-      . "COALESCE(this_cnt-master_cnt, 0) AS cnt_diff, "
-      . "COALESCE("
-      .   "this_crc <> master_crc OR ISNULL(master_crc) <> ISNULL(this_crc), 0"
-      . ") AS crc_diff, this_cnt, master_cnt, this_crc, master_crc "
-      . "FROM $table "
-      . "WHERE master_cnt <> this_cnt OR master_crc <> this_crc "
-      . "OR ISNULL(master_crc) <> ISNULL(this_crc)";
-   PTDEBUG && _d($sql);
-   my $diffs = $dbh->selectall_arrayref($sql, { Slice => {} });
-   return $diffs;
-}
-
-sub _d {
-   my ($package, undef, $line) = caller 0;
-   @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
-        map { defined $_ ? $_ : 'undef' }
-        @_;
-   print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
-}
-
-1;
-}
-# ###########################################################################
-# End TableChecksum package
-# ###########################################################################
-
-# ###########################################################################
-# TableSyncer package
-# This package is a copy without comments from the original.  The original
-# with comments and its test file can be found in the Bazaar repository at,
-#   lib/TableSyncer.pm
-#   t/lib/TableSyncer.t
-# See https://launchpad.net/percona-toolkit for more information.
-# ###########################################################################
-{
-package TableSyncer;
-
-use strict;
-use warnings FATAL => 'all';
-use English qw(-no_match_vars);
-use constant PTDEBUG => $ENV{PTDEBUG} || 0;
-
-use Data::Dumper;
-$Data::Dumper::Indent    = 1;
-$Data::Dumper::Sortkeys  = 1;
-$Data::Dumper::Quotekeys = 0;
-
-sub new {
-   my ( $class, %args ) = @_;
-   my @required_args = qw(MasterSlave Quoter TableChecksum Retry);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   my $self = { %args };
-   return bless $self, $class;
-}
-
-sub get_best_plugin {
-   my ( $self, %args ) = @_;
-   foreach my $arg ( qw(plugins tbl_struct) ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   PTDEBUG && _d('Getting best plugin');
-   foreach my $plugin ( @{$args{plugins}} ) {
-      PTDEBUG && _d('Trying plugin', $plugin->name);
-      my ($can_sync, %plugin_args) = $plugin->can_sync(%args);
-      if ( $can_sync ) {
-        PTDEBUG && _d('Can sync with', $plugin->name, Dumper(\%plugin_args));
-        return $plugin, %plugin_args;
-      }
-   }
-   PTDEBUG && _d('No plugin can sync the table');
-   return;
-}
-
-sub sync_table {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(plugins src dst tbl_struct cols chunk_size
-                          RowDiff ChangeHandler);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   PTDEBUG && _d('Syncing table with args:',
-      map { "$_: " . Dumper($args{$_}) }
-      qw(plugins src dst tbl_struct cols chunk_size));
-
-   my ($plugins, $src, $dst, $tbl_struct, $cols, $chunk_size, $rd, $ch)
-      = @args{@required_args};
-   my $dp = $self->{DSNParser};
-   $args{trace} = 1 unless defined $args{trace};
-
-   if ( $args{bidirectional} && $args{ChangeHandler}->{queue} ) {
-      die "Queueing does not work with bidirectional syncing";
-   }
-
-   $args{index_hint}    = 1 unless defined $args{index_hint};
-   $args{lock}        ||= 0;
-   $args{wait}        ||= 0;
-   $args{transaction} ||= 0;
-   $args{timeout_ok}  ||= 0;
-
-   my $q  = $self->{Quoter};
-
-   my ($plugin, %plugin_args) = $self->get_best_plugin(%args);
-   die "No plugin can sync $src->{db}.$src->{tbl}" unless $plugin;
-
-   my $crc_col = '__crc';
-   while ( $tbl_struct->{is_col}->{$crc_col} ) {
-      $crc_col = "_$crc_col"; # Prepend more _ until not a column.
-   }
-   PTDEBUG && _d('CRC column:', $crc_col);
-
-   my $index_hint;
-   if ( $args{chunk_index} ) {
-      PTDEBUG && _d('Using given chunk index for index hint');
-      $index_hint = "FORCE INDEX (" . $q->quote($args{chunk_index}) . ")";
-   }
-   elsif ( $plugin_args{chunk_index} && $args{index_hint} ) {
-      PTDEBUG && _d('Using chunk index chosen by plugin for index hint');
-      $index_hint = "FORCE INDEX (" . $q->quote($plugin_args{chunk_index}) . ")";
-   }
-   PTDEBUG && _d('Index hint:', $index_hint);
-
-   eval {
-      $plugin->prepare_to_sync(
-         %args,
-         %plugin_args,
-         dbh        => $src->{dbh},
-         db         => $src->{db},
-         tbl        => $src->{tbl},
-         crc_col    => $crc_col,
-         index_hint => $index_hint,
-      );
-   };
-   if ( $EVAL_ERROR ) {
-      die 'Failed to prepare TableSync', $plugin->name, ' plugin: ',
-         $EVAL_ERROR;
-   }
-
-   if ( $plugin->uses_checksum() ) {
-      eval {
-         my ($chunk_sql, $row_sql) = $self->make_checksum_queries(%args);
-         $plugin->set_checksum_queries($chunk_sql, $row_sql);
-      };
-      if ( $EVAL_ERROR ) {
-         die "Failed to make checksum queries: $EVAL_ERROR";
-      }
-   } 
-
-   if ( $args{dry_run} ) {
-      return $ch->get_changes(), ALGORITHM => $plugin->name;
-   }
-
-
-   eval {
-      $src->{dbh}->do("USE `$src->{db}`");
-      $dst->{dbh}->do("USE `$dst->{db}`");
-   };
-   if ( $EVAL_ERROR ) {
-      die "Failed to USE database on source or destination: $EVAL_ERROR";
-   }
-
-   PTDEBUG && _d('left dbh', $src->{dbh});
-   PTDEBUG && _d('right dbh', $dst->{dbh});
-
-   chomp(my $hostname = `hostname`);
-   my $trace_msg
-      = $args{trace} ? "src_db:$src->{db} src_tbl:$src->{tbl} "
-         . ($dp && $src->{dsn} ? "src_dsn:".$dp->as_string($src->{dsn}) : "")
-         . " dst_db:$dst->{db} dst_tbl:$dst->{tbl} "
-         . ($dp && $dst->{dsn} ? "dst_dsn:".$dp->as_string($dst->{dsn}) : "")
-         . " " . join(" ", map { "$_:" . ($args{$_} || 0) }
-                     qw(lock transaction changing_src replicate bidirectional))
-         . " pid:$PID "
-         . ($ENV{USER} ? "user:$ENV{USER} " : "")
-         . ($hostname  ? "host:$hostname"   : "")
-      :                "";
-   PTDEBUG && _d("Binlog trace message:", $trace_msg);
-
-   $self->lock_and_wait(%args, lock_level => 2);  # per-table lock
-
-   my $callback = $args{callback};
-   my $cycle    = 0;
-   while ( !$plugin->done() ) {
-
-      PTDEBUG && _d('Beginning sync cycle', $cycle);
-      my $src_sql = $plugin->get_sql(
-         database => $src->{db},
-         table    => $src->{tbl},
-         where    => $args{where},
-      );
-      my $dst_sql = $plugin->get_sql(
-         database => $dst->{db},
-         table    => $dst->{tbl},
-         where    => $args{where},
-      );
-
-      if ( $args{transaction} ) {
-         if ( $args{bidirectional} ) {
-            $src_sql .= ' FOR UPDATE';
-            $dst_sql .= ' FOR UPDATE';
-         }
-         elsif ( $args{changing_src} ) {
-            $src_sql .= ' FOR UPDATE';
-            $dst_sql .= ' LOCK IN SHARE MODE';
-         }
-         else {
-            $src_sql .= ' LOCK IN SHARE MODE';
-            $dst_sql .= ' FOR UPDATE';
-         }
-      }
-      PTDEBUG && _d('src:', $src_sql);
-      PTDEBUG && _d('dst:', $dst_sql);
-
-      $callback->($src_sql, $dst_sql) if $callback;
-
-      $plugin->prepare_sync_cycle($src);
-      $plugin->prepare_sync_cycle($dst);
-
-      my $src_sth = $src->{dbh}->prepare($src_sql);
-      my $dst_sth = $dst->{dbh}->prepare($dst_sql);
-      if ( $args{buffer_to_client} ) {
-         $src_sth->{mysql_use_result} = 1;
-         $dst_sth->{mysql_use_result} = 1;
-      }
-
-      my $executed_src = 0;
-      if ( !$cycle || !$plugin->pending_changes() ) {
-         $executed_src
-            = $self->lock_and_wait(%args, src_sth => $src_sth, lock_level => 1);
-      }
-
-      $src_sth->execute() unless $executed_src;
-      $dst_sth->execute();
-
-      $rd->compare_sets(
-         left_sth   => $src_sth,
-         right_sth  => $dst_sth,
-         left_dbh   => $src->{dbh},
-         right_dbh  => $dst->{dbh},
-         syncer     => $plugin,
-         tbl_struct => $tbl_struct,
-      );
-      $ch->process_rows(1, $trace_msg);
-
-      PTDEBUG && _d('Finished sync cycle', $cycle);
-      $cycle++;
-   }
-
-   $ch->process_rows(0, $trace_msg);
-
-   $self->unlock(%args, lock_level => 2);
-
-   return $ch->get_changes(), ALGORITHM => $plugin->name;
-}
-
-sub make_checksum_queries {
-   my ( $self, %args ) = @_;
-   my @required_args = qw(src dst tbl_struct);
-   foreach my $arg ( @required_args ) {
-      die "I need a $arg argument" unless $args{$arg};
-   }
-   my ($src, $dst, $tbl_struct) = @args{@required_args};
-   my $checksum = $self->{TableChecksum};
-
-   my $src_algo = $checksum->best_algorithm(
-      algorithm => 'BIT_XOR',
-      dbh       => $src->{dbh},
-      where     => 1,
-      chunk     => 1,
-      count     => 1,
-   );
-   my $dst_algo = $checksum->best_algorithm(
-      algorithm => 'BIT_XOR',
-      dbh       => $dst->{dbh},
-      where     => 1,
-      chunk     => 1,
-      count     => 1,
-   );
-   if ( $src_algo ne $dst_algo ) {
-      die "Source and destination checksum algorithms are different: ",
-         "$src_algo on source, $dst_algo on destination"
-   }
-   PTDEBUG && _d('Chosen algo:', $src_algo);
-
-   my $src_func = $checksum->choose_hash_func(dbh => $src->{dbh}, %args);
-   my $dst_func = $checksum->choose_hash_func(dbh => $dst->{dbh}, %args);
-   if ( $src_func ne $dst_func ) {
-      die "Source and destination hash functions are different: ",
-      "$src_func on source, $dst_func on destination";
-   }
-   PTDEBUG && _d('Chosen hash func:', $src_func);
-
-
-   my $crc_wid    = $checksum->get_crc_wid($src->{dbh}, $src_func);
-   my ($crc_type) = $checksum->get_crc_type($src->{dbh}, $src_func);
-   my $opt_slice;
-   if ( $src_algo eq 'BIT_XOR' && $crc_type !~ m/int$/ ) {
-      $opt_slice = $checksum->optimize_xor(
-         dbh      => $src->{dbh},
-         function => $src_func
-      );
-   }
-
-   my $chunk_sql = $checksum->make_checksum_query(
-      %args,
-      db        => $src->{db},
-      tbl       => $src->{tbl},
-      algorithm => $src_algo,
-      function  => $src_func,
-      crc_wid   => $crc_wid,
-      crc_type  => $crc_type,
-      opt_slice => $opt_slice,
-      replicate => undef, # replicate means something different to this sub
-   );                     # than what we use it for; do not pass it!
-   PTDEBUG && _d('Chunk sql:', $chunk_sql);
-   my $row_sql = $checksum->make_row_checksum(
-      %args,
-      function => $src_func,
-   );
-   PTDEBUG && _d('Row sql:', $row_sql);
-   return $chunk_sql, $row_sql;
-}
-
-sub lock_table {
-   my ( $self, $dbh, $where, $db_tbl, $mode ) = @_;
-   my $query = "LOCK TABLES $db_tbl $mode";
-   PTDEBUG && _d($query);
-   $dbh->do($query);
-   PTDEBUG && _d('Acquired table lock on', $where, 'in', $mode, 'mode');
-}
-
-sub unlock {
-   my ( $self, %args ) = @_;
-
-   foreach my $arg ( qw(src dst lock transaction lock_level) ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   my $src = $args{src};
-   my $dst = $args{dst};
-
-   return unless $args{lock} && $args{lock} <= $args{lock_level};
-
-   foreach my $dbh ( $src->{dbh}, $dst->{dbh} ) {
-      if ( $args{transaction} ) {
-         PTDEBUG && _d('Committing', $dbh);
-         $dbh->commit();
-      }
-      else {
-         my $sql = 'UNLOCK TABLES';
-         PTDEBUG && _d($dbh, $sql);
-         $dbh->do($sql);
-      }
-   }
-
-   return;
-}
-
-sub lock_and_wait {
-   my ( $self, %args ) = @_;
-   my $result = 0;
-
-   foreach my $arg ( qw(src dst lock lock_level) ) {
-      die "I need a $arg argument" unless defined $args{$arg};
-   }
-   my $src = $args{src};
-   my $dst = $args{dst};
-
-   return unless $args{lock} && $args{lock} == $args{lock_level};
-   PTDEBUG && _d('lock and wait, lock level', $args{lock});
-
-   foreach my $dbh ( $src->{dbh}, $dst->{dbh} ) {
-      if ( $args{transaction} ) {
-         PTDEBUG && _d('Committing', $dbh);
-         $dbh->commit();
-      }
-      else {