Merge lp:~percona-toolkit-dev/percona-toolkit/ptc-run-time into lp:percona-toolkit/2.1

Proposed by Daniel Nichter
Status: Merged
Merged at revision: 370
Proposed branch: lp:~percona-toolkit-dev/percona-toolkit/ptc-run-time
Merge into: lp:percona-toolkit/2.1
Diff against target: 377 lines (+256/-13) (has conflicts)
4 files modified
bin/pt-kill (+6/-5)
bin/pt-table-checksum (+167/-3)
lib/Cxn.pm (+6/-5)
t/pt-table-checksum/run_time.t (+77/-0)
Text conflict in bin/pt-kill
Text conflict in bin/pt-table-checksum
Text conflict in lib/Cxn.pm
To merge this branch: bzr merge lp:~percona-toolkit-dev/percona-toolkit/ptc-run-time
Reviewer Review Type Date Requested Status
Daniel Nichter Approve
Review via email: mp+121291@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Daniel Nichter (daniel-nichter) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'bin/pt-kill'
2--- bin/pt-kill 2012-08-23 16:22:43 +0000
3+++ bin/pt-kill 2012-08-24 21:18:32 +0000
4@@ -4753,7 +4753,11 @@
5 dsn_name => $dp->as_string($dsn, [qw(h P S)]),
6 hostname => '',
7 set => $args{set},
8+<<<<<<< TREE
9 NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
10+=======
11+ NAME_lc => defined $args{NAME_lc} ? $args{NAME_lc} : 1,
12+>>>>>>> MERGE-SOURCE
13 dbh_set => 0,
14 OptionParser => $o,
15 DSNParser => $dp,
16@@ -4791,11 +4795,8 @@
17
18 PTDEBUG && _d($dbh, 'Setting dbh');
19
20- if ( !exists $self->{NAME_lc}
21- || (defined $self->{NAME_lc} && $self->{NAME_lc}) ) {
22- $dbh->{FetchHashKeyName} = 'NAME_lc';
23- }
24-
25+ $dbh->{FetchHashKeyName} = 'NAME_lc' if $self->{NAME_lc};
26+
27 my $sql = 'SELECT @@hostname, @@server_id';
28 PTDEBUG && _d($dbh, $sql);
29 my ($hostname, $server_id) = $dbh->selectrow_array($sql);
30
31=== modified file 'bin/pt-table-checksum'
32--- bin/pt-table-checksum 2012-08-23 16:22:43 +0000
33+++ bin/pt-table-checksum 2012-08-24 21:18:32 +0000
34@@ -2886,7 +2886,11 @@
35 dsn_name => $dp->as_string($dsn, [qw(h P S)]),
36 hostname => '',
37 set => $args{set},
38+<<<<<<< TREE
39 NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
40+=======
41+ NAME_lc => defined $args{NAME_lc} ? $args{NAME_lc} : 1,
42+>>>>>>> MERGE-SOURCE
43 dbh_set => 0,
44 OptionParser => $o,
45 DSNParser => $dp,
46@@ -2924,11 +2928,16 @@
47
48 PTDEBUG && _d($dbh, 'Setting dbh');
49
50+<<<<<<< TREE
51 if ( !exists $self->{NAME_lc}
52 || (defined $self->{NAME_lc} && $self->{NAME_lc}) ) {
53 $dbh->{FetchHashKeyName} = 'NAME_lc';
54 }
55
56+=======
57+ $dbh->{FetchHashKeyName} = 'NAME_lc' if $self->{NAME_lc};
58+
59+>>>>>>> MERGE-SOURCE
60 my $sql = 'SELECT @@hostname, @@server_id';
61 PTDEBUG && _d($dbh, $sql);
62 my ($hostname, $server_id) = $dbh->selectrow_array($sql);
63@@ -7757,6 +7766,134 @@
64 # ###########################################################################
65
66 # ###########################################################################
67+# Runtime package
68+# This package is a copy without comments from the original. The original
69+# with comments and its test file can be found in the Bazaar repository at,
70+# lib/Runtime.pm
71+# t/lib/Runtime.t
72+# See https://launchpad.net/percona-toolkit for more information.
73+# ###########################################################################
74+{
75+package Runtime;
76+
77+use strict;
78+use warnings FATAL => 'all';
79+use English qw(-no_match_vars);
80+use constant PTDEBUG => $ENV{PTDEBUG} || 0;
81+
82+sub new {
83+ my ( $class, %args ) = @_;
84+ my @required_args = qw(now);
85+ foreach my $arg ( @required_args ) {
86+ die "I need a $arg argument" unless $args{$arg};
87+ }
88+
89+ if ( ($args{runtime} || 0) < 0 ) {
90+ die "runtime argument must be greater than zero"
91+ }
92+
93+ my $self = {
94+ %args,
95+ start_time => undef,
96+ end_time => undef,
97+ time_left => undef,
98+ stop => 0,
99+ };
100+
101+ return bless $self, $class;
102+}
103+
104+sub time_left {
105+ my ( $self, %args ) = @_;
106+
107+ if ( $self->{stop} ) {
108+ PTDEBUG && _d("No time left because stop was called");
109+ return 0;
110+ }
111+
112+ my $now = $self->{now}->(%args);
113+ PTDEBUG && _d("Current time:", $now);
114+
115+ if ( !defined $self->{start_time} ) {
116+ $self->{start_time} = $now;
117+ }
118+
119+ return unless defined $now;
120+
121+ my $runtime = $self->{runtime};
122+ return unless defined $runtime;
123+
124+ if ( !$self->{end_time} ) {
125+ $self->{end_time} = $now + $runtime;
126+ PTDEBUG && _d("End time:", $self->{end_time});
127+ }
128+
129+ $self->{time_left} = $self->{end_time} - $now;
130+ PTDEBUG && _d("Time left:", $self->{time_left});
131+ return $self->{time_left};
132+}
133+
134+sub have_time {
135+ my ( $self, %args ) = @_;
136+ my $time_left = $self->time_left(%args);
137+ return 1 if !defined $time_left; # run forever
138+ return $time_left <= 0 ? 0 : 1; # <=0s means runtime has elapsed
139+}
140+
141+sub time_elapsed {
142+ my ( $self, %args ) = @_;
143+
144+ my $start_time = $self->{start_time};
145+ return 0 unless $start_time;
146+
147+ my $now = $self->{now}->(%args);
148+ PTDEBUG && _d("Current time:", $now);
149+
150+ my $time_elapsed = $now - $start_time;
151+ PTDEBUG && _d("Time elapsed:", $time_elapsed);
152+ if ( $time_elapsed < 0 ) {
153+ warn "Current time $now is earlier than start time $start_time";
154+ }
155+ return $time_elapsed;
156+}
157+
158+sub reset {
159+ my ( $self ) = @_;
160+ $self->{start_time} = undef;
161+ $self->{end_time} = undef;
162+ $self->{time_left} = undef;
163+ $self->{stop} = 0;
164+ PTDEBUG && _d("Reset runtime");
165+ return;
166+}
167+
168+sub stop {
169+ my ( $self ) = @_;
170+ $self->{stop} = 1;
171+ return;
172+}
173+
174+sub start {
175+ my ( $self ) = @_;
176+ $self->{stop} = 0;
177+ return;
178+}
179+
180+sub _d {
181+ my ($package, undef, $line) = caller 0;
182+ @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
183+ map { defined $_ ? $_ : 'undef' }
184+ @_;
185+ print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
186+}
187+
188+1;
189+}
190+# ###########################################################################
191+# End Runtime package
192+# ###########################################################################
193+
194+# ###########################################################################
195 # This is a combination of modules and programs in one -- a runnable module.
196 # http://www.perl.com/pub/a/2006/07/13/lightning-articles.html?page=last
197 # Or, look it up in the Camel book on pages 642 and 643 in the 3rd edition.
198@@ -8315,6 +8452,21 @@
199 }
200
201 # ########################################################################
202+ # Set up the run time, if any.
203+ # ########################################################################
204+ my $have_time;
205+ if ( my $run_time = $o->get('run-time') ) {
206+ my $rt = Runtime->new(
207+ now => sub { return time; },
208+ runtime => $run_time,
209+ );
210+ $have_time = sub { return $rt->have_time(); };
211+ }
212+ else {
213+ $have_time = sub { return 1; };
214+ }
215+
216+ # ########################################################################
217 # Various variables and modules for checksumming the tables.
218 # ########################################################################
219 my $total_rows = 0;
220@@ -8537,7 +8689,7 @@
221 # done in these callbacks. This callback is the only place where we
222 # can prematurely stop nibbling by returning false. This allows
223 # Ctrl-C to stop the tool between nibbles instead of between tables.
224- return $oktorun; # continue nibbling table?
225+ return $oktorun && $have_time->(); # continue nibbling table?
226 },
227 exec_nibble => sub {
228 my (%args) = @_;
229@@ -8782,7 +8934,7 @@
230 # ########################################################################
231
232 TABLE:
233- while ( $oktorun && (my $tbl = $schema_iter->next()) ) {
234+ while ( $oktorun && $have_time->() && (my $tbl = $schema_iter->next()) ) {
235 eval {
236 # Results, stats, and info related to checksuming this table can
237 # be saved here. print_checksum_results() uses this info.
238@@ -8889,7 +9041,9 @@
239 }
240 }
241
242- PTDEBUG && _d('Exit status', $exit_status, 'oktorun', $oktorun);
243+ PTDEBUG && _d('Exit status', $exit_status,
244+ 'oktorun', $oktorun,
245+ 'have time', $have_time->());
246 return $exit_status;
247 }
248
249@@ -10525,6 +10679,16 @@
250 Retry a chunk this many times when there is a nonfatal error. Nonfatal errors
251 are problems such as a lock wait timeout or the query being killed.
252
253+=item --run-time
254+
255+type: time
256+
257+How long to run. Default is to run until all tables have been checksummed.
258+These time value suffixes are allowed: s (seconds), m (minutes), h (hours),
259+and d (days). Combine this option with L<"--resume"> to checksum as many
260+tables within an allotted time, resuming from where the tool left off next
261+time it is ran.
262+
263 =item --separator
264
265 type: string; default: #
266
267=== modified file 'lib/Cxn.pm'
268--- lib/Cxn.pm 2012-07-23 02:37:39 +0000
269+++ lib/Cxn.pm 2012-08-24 21:18:32 +0000
270@@ -103,7 +103,11 @@
271 dsn_name => $dp->as_string($dsn, [qw(h P S)]),
272 hostname => '',
273 set => $args{set},
274+<<<<<<< TREE
275 NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1,
276+=======
277+ NAME_lc => defined $args{NAME_lc} ? $args{NAME_lc} : 1,
278+>>>>>>> MERGE-SOURCE
279 dbh_set => 0,
280 OptionParser => $o,
281 DSNParser => $dp,
282@@ -150,11 +154,8 @@
283 PTDEBUG && _d($dbh, 'Setting dbh');
284
285 # Set stuff for this dbh (i.e. initialize it).
286- if ( !exists $self->{NAME_lc}
287- || (defined $self->{NAME_lc} && $self->{NAME_lc}) ) {
288- $dbh->{FetchHashKeyName} = 'NAME_lc';
289- }
290-
291+ $dbh->{FetchHashKeyName} = 'NAME_lc' if $self->{NAME_lc};
292+
293 # Update the cxn's name. Until we connect, the DSN parts
294 # h and P are used. Once connected, use @@hostname.
295 my $sql = 'SELECT @@hostname, @@server_id';
296
297=== added file 't/pt-table-checksum/run_time.t'
298--- t/pt-table-checksum/run_time.t 1970-01-01 00:00:00 +0000
299+++ t/pt-table-checksum/run_time.t 2012-08-24 21:18:32 +0000
300@@ -0,0 +1,77 @@
301+#!/usr/bin/env perl
302+
303+BEGIN {
304+ die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
305+ unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
306+ unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
307+};
308+
309+use strict;
310+use warnings FATAL => 'all';
311+use English qw(-no_match_vars);
312+use Test::More;
313+use Time::HiRes qw(time);
314+use Data::Dumper;
315+$Data::Dumper::Indent = 1;
316+$Data::Dumper::Sortkeys = 1;
317+$Data::Dumper::Quotekeys = 0;
318+
319+use PerconaTest;
320+use Sandbox;
321+shift @INC; # our unshift (above)
322+shift @INC; # PerconaTest's unshift
323+require "$trunk/bin/pt-table-checksum";
324+
325+my $dp = new DSNParser(opts=>$dsn_opts);
326+my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp);
327+my $master_dbh = $sb->get_dbh_for('master');
328+
329+if ( !$master_dbh ) {
330+ plan skip_all => 'Cannot connect to sandbox master';
331+}
332+
333+# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic
334+# so we need to specify --lock-wait-timeout=3 else the tool will die.
335+# And --max-load "" prevents waiting for status variables.
336+my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox';
337+my @args = ($master_dsn, qw(--lock-wait-timeout 3), '--max-load', '');
338+my $output;
339+my $exit_status;
340+
341+# On my 2.4 GHz with SSD this takes a little more than 3s,
342+# so no test servers should be faster, hopefully.
343+my $t0 = time;
344+$exit_status = pt_table_checksum::main(@args,
345+ qw(--quiet --quiet -d sakila --chunk-size 100 --run-time 1));
346+my $t = time - $t0;
347+
348+ok(
349+ $t >= 1.5 && $t <= 2.0,
350+ "Run in roughly --run-time 1 second"
351+) or diag("Actual run time: $t");
352+
353+my $rows = $master_dbh->selectall_arrayref("SELECT DISTINCT CONCAT(db, '.', tbl) FROM percona.checksums ORDER by db, tbl");
354+my $sakila_finished = grep { $_->[0] eq 'sakila.store' } @$rows;
355+ok(
356+ !$sakila_finished,
357+ "Did not finish checksumming sakila"
358+) or diag(Dumper($rows));
359+
360+# Add --resume to complete the run.
361+$exit_status = pt_table_checksum::main(@args,
362+ qw(--quiet --quiet -d sakila --chunk-size 100));
363+
364+$rows = $master_dbh->selectall_arrayref("SELECT DISTINCT CONCAT(db, '.', tbl) FROM percona.checksums ORDER by db, tbl");
365+$sakila_finished = grep { $_->[0] eq 'sakila.store' } @$rows;
366+ok(
367+ $sakila_finished,
368+ "Resumed and finish checksumming sakila"
369+) or diag(Dumper($rows));
370+
371+# #############################################################################
372+# Done.
373+# #############################################################################
374+$sb->wipe_clean($master_dbh);
375+ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox");
376+done_testing;
377+exit;

Subscribers

People subscribed via source and target branches