Merge lp:~percona-toolkit-dev/percona-toolkit/ptc-run-time into lp:percona-toolkit/2.1
- ptc-run-time
- Merge into 2.1
Proposed by
Daniel Nichter
Status: | Merged |
---|---|
Merged at revision: | 370 |
Proposed branch: | lp:~percona-toolkit-dev/percona-toolkit/ptc-run-time |
Merge into: | lp:percona-toolkit/2.1 |
Diff against target: |
377 lines (+256/-13) (has conflicts) 4 files modified
bin/pt-kill (+6/-5) bin/pt-table-checksum (+167/-3) lib/Cxn.pm (+6/-5) t/pt-table-checksum/run_time.t (+77/-0) Text conflict in bin/pt-kill Text conflict in bin/pt-table-checksum Text conflict in lib/Cxn.pm |
To merge this branch: | bzr merge lp:~percona-toolkit-dev/percona-toolkit/ptc-run-time |
Related bugs: | |
Related blueprints: |
pt-table-checksum --run-time
(Medium)
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Daniel Nichter | Approve | ||
Review via email: mp+121291@code.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
Revision history for this message
Daniel Nichter (daniel-nichter) : | # |
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'bin/pt-kill' |
2 | --- bin/pt-kill 2012-08-23 16:22:43 +0000 |
3 | +++ bin/pt-kill 2012-08-24 21:18:32 +0000 |
4 | @@ -4753,7 +4753,11 @@ |
5 | dsn_name => $dp->as_string($dsn, [qw(h P S)]), |
6 | hostname => '', |
7 | set => $args{set}, |
8 | +<<<<<<< TREE |
9 | NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1, |
10 | +======= |
11 | + NAME_lc => defined $args{NAME_lc} ? $args{NAME_lc} : 1, |
12 | +>>>>>>> MERGE-SOURCE |
13 | dbh_set => 0, |
14 | OptionParser => $o, |
15 | DSNParser => $dp, |
16 | @@ -4791,11 +4795,8 @@ |
17 | |
18 | PTDEBUG && _d($dbh, 'Setting dbh'); |
19 | |
20 | - if ( !exists $self->{NAME_lc} |
21 | - || (defined $self->{NAME_lc} && $self->{NAME_lc}) ) { |
22 | - $dbh->{FetchHashKeyName} = 'NAME_lc'; |
23 | - } |
24 | - |
25 | + $dbh->{FetchHashKeyName} = 'NAME_lc' if $self->{NAME_lc}; |
26 | + |
27 | my $sql = 'SELECT @@hostname, @@server_id'; |
28 | PTDEBUG && _d($dbh, $sql); |
29 | my ($hostname, $server_id) = $dbh->selectrow_array($sql); |
30 | |
31 | === modified file 'bin/pt-table-checksum' |
32 | --- bin/pt-table-checksum 2012-08-23 16:22:43 +0000 |
33 | +++ bin/pt-table-checksum 2012-08-24 21:18:32 +0000 |
34 | @@ -2886,7 +2886,11 @@ |
35 | dsn_name => $dp->as_string($dsn, [qw(h P S)]), |
36 | hostname => '', |
37 | set => $args{set}, |
38 | +<<<<<<< TREE |
39 | NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1, |
40 | +======= |
41 | + NAME_lc => defined $args{NAME_lc} ? $args{NAME_lc} : 1, |
42 | +>>>>>>> MERGE-SOURCE |
43 | dbh_set => 0, |
44 | OptionParser => $o, |
45 | DSNParser => $dp, |
46 | @@ -2924,11 +2928,16 @@ |
47 | |
48 | PTDEBUG && _d($dbh, 'Setting dbh'); |
49 | |
50 | +<<<<<<< TREE |
51 | if ( !exists $self->{NAME_lc} |
52 | || (defined $self->{NAME_lc} && $self->{NAME_lc}) ) { |
53 | $dbh->{FetchHashKeyName} = 'NAME_lc'; |
54 | } |
55 | |
56 | +======= |
57 | + $dbh->{FetchHashKeyName} = 'NAME_lc' if $self->{NAME_lc}; |
58 | + |
59 | +>>>>>>> MERGE-SOURCE |
60 | my $sql = 'SELECT @@hostname, @@server_id'; |
61 | PTDEBUG && _d($dbh, $sql); |
62 | my ($hostname, $server_id) = $dbh->selectrow_array($sql); |
63 | @@ -7757,6 +7766,134 @@ |
64 | # ########################################################################### |
65 | |
66 | # ########################################################################### |
67 | +# Runtime package |
68 | +# This package is a copy without comments from the original. The original |
69 | +# with comments and its test file can be found in the Bazaar repository at, |
70 | +# lib/Runtime.pm |
71 | +# t/lib/Runtime.t |
72 | +# See https://launchpad.net/percona-toolkit for more information. |
73 | +# ########################################################################### |
74 | +{ |
75 | +package Runtime; |
76 | + |
77 | +use strict; |
78 | +use warnings FATAL => 'all'; |
79 | +use English qw(-no_match_vars); |
80 | +use constant PTDEBUG => $ENV{PTDEBUG} || 0; |
81 | + |
82 | +sub new { |
83 | + my ( $class, %args ) = @_; |
84 | + my @required_args = qw(now); |
85 | + foreach my $arg ( @required_args ) { |
86 | + die "I need a $arg argument" unless $args{$arg}; |
87 | + } |
88 | + |
89 | + if ( ($args{runtime} || 0) < 0 ) { |
90 | + die "runtime argument must be greater than zero" |
91 | + } |
92 | + |
93 | + my $self = { |
94 | + %args, |
95 | + start_time => undef, |
96 | + end_time => undef, |
97 | + time_left => undef, |
98 | + stop => 0, |
99 | + }; |
100 | + |
101 | + return bless $self, $class; |
102 | +} |
103 | + |
104 | +sub time_left { |
105 | + my ( $self, %args ) = @_; |
106 | + |
107 | + if ( $self->{stop} ) { |
108 | + PTDEBUG && _d("No time left because stop was called"); |
109 | + return 0; |
110 | + } |
111 | + |
112 | + my $now = $self->{now}->(%args); |
113 | + PTDEBUG && _d("Current time:", $now); |
114 | + |
115 | + if ( !defined $self->{start_time} ) { |
116 | + $self->{start_time} = $now; |
117 | + } |
118 | + |
119 | + return unless defined $now; |
120 | + |
121 | + my $runtime = $self->{runtime}; |
122 | + return unless defined $runtime; |
123 | + |
124 | + if ( !$self->{end_time} ) { |
125 | + $self->{end_time} = $now + $runtime; |
126 | + PTDEBUG && _d("End time:", $self->{end_time}); |
127 | + } |
128 | + |
129 | + $self->{time_left} = $self->{end_time} - $now; |
130 | + PTDEBUG && _d("Time left:", $self->{time_left}); |
131 | + return $self->{time_left}; |
132 | +} |
133 | + |
134 | +sub have_time { |
135 | + my ( $self, %args ) = @_; |
136 | + my $time_left = $self->time_left(%args); |
137 | + return 1 if !defined $time_left; # run forever |
138 | + return $time_left <= 0 ? 0 : 1; # <=0s means runtime has elapsed |
139 | +} |
140 | + |
141 | +sub time_elapsed { |
142 | + my ( $self, %args ) = @_; |
143 | + |
144 | + my $start_time = $self->{start_time}; |
145 | + return 0 unless $start_time; |
146 | + |
147 | + my $now = $self->{now}->(%args); |
148 | + PTDEBUG && _d("Current time:", $now); |
149 | + |
150 | + my $time_elapsed = $now - $start_time; |
151 | + PTDEBUG && _d("Time elapsed:", $time_elapsed); |
152 | + if ( $time_elapsed < 0 ) { |
153 | + warn "Current time $now is earlier than start time $start_time"; |
154 | + } |
155 | + return $time_elapsed; |
156 | +} |
157 | + |
158 | +sub reset { |
159 | + my ( $self ) = @_; |
160 | + $self->{start_time} = undef; |
161 | + $self->{end_time} = undef; |
162 | + $self->{time_left} = undef; |
163 | + $self->{stop} = 0; |
164 | + PTDEBUG && _d("Reset runtime"); |
165 | + return; |
166 | +} |
167 | + |
168 | +sub stop { |
169 | + my ( $self ) = @_; |
170 | + $self->{stop} = 1; |
171 | + return; |
172 | +} |
173 | + |
174 | +sub start { |
175 | + my ( $self ) = @_; |
176 | + $self->{stop} = 0; |
177 | + return; |
178 | +} |
179 | + |
180 | +sub _d { |
181 | + my ($package, undef, $line) = caller 0; |
182 | + @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } |
183 | + map { defined $_ ? $_ : 'undef' } |
184 | + @_; |
185 | + print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; |
186 | +} |
187 | + |
188 | +1; |
189 | +} |
190 | +# ########################################################################### |
191 | +# End Runtime package |
192 | +# ########################################################################### |
193 | + |
194 | +# ########################################################################### |
195 | # This is a combination of modules and programs in one -- a runnable module. |
196 | # http://www.perl.com/pub/a/2006/07/13/lightning-articles.html?page=last |
197 | # Or, look it up in the Camel book on pages 642 and 643 in the 3rd edition. |
198 | @@ -8315,6 +8452,21 @@ |
199 | } |
200 | |
201 | # ######################################################################## |
202 | + # Set up the run time, if any. |
203 | + # ######################################################################## |
204 | + my $have_time; |
205 | + if ( my $run_time = $o->get('run-time') ) { |
206 | + my $rt = Runtime->new( |
207 | + now => sub { return time; }, |
208 | + runtime => $run_time, |
209 | + ); |
210 | + $have_time = sub { return $rt->have_time(); }; |
211 | + } |
212 | + else { |
213 | + $have_time = sub { return 1; }; |
214 | + } |
215 | + |
216 | + # ######################################################################## |
217 | # Various variables and modules for checksumming the tables. |
218 | # ######################################################################## |
219 | my $total_rows = 0; |
220 | @@ -8537,7 +8689,7 @@ |
221 | # done in these callbacks. This callback is the only place where we |
222 | # can prematurely stop nibbling by returning false. This allows |
223 | # Ctrl-C to stop the tool between nibbles instead of between tables. |
224 | - return $oktorun; # continue nibbling table? |
225 | + return $oktorun && $have_time->(); # continue nibbling table? |
226 | }, |
227 | exec_nibble => sub { |
228 | my (%args) = @_; |
229 | @@ -8782,7 +8934,7 @@ |
230 | # ######################################################################## |
231 | |
232 | TABLE: |
233 | - while ( $oktorun && (my $tbl = $schema_iter->next()) ) { |
234 | + while ( $oktorun && $have_time->() && (my $tbl = $schema_iter->next()) ) { |
235 | eval { |
236 | # Results, stats, and info related to checksuming this table can |
237 | # be saved here. print_checksum_results() uses this info. |
238 | @@ -8889,7 +9041,9 @@ |
239 | } |
240 | } |
241 | |
242 | - PTDEBUG && _d('Exit status', $exit_status, 'oktorun', $oktorun); |
243 | + PTDEBUG && _d('Exit status', $exit_status, |
244 | + 'oktorun', $oktorun, |
245 | + 'have time', $have_time->()); |
246 | return $exit_status; |
247 | } |
248 | |
249 | @@ -10525,6 +10679,16 @@ |
250 | Retry a chunk this many times when there is a nonfatal error. Nonfatal errors |
251 | are problems such as a lock wait timeout or the query being killed. |
252 | |
253 | +=item --run-time |
254 | + |
255 | +type: time |
256 | + |
257 | +How long to run. Default is to run until all tables have been checksummed. |
258 | +These time value suffixes are allowed: s (seconds), m (minutes), h (hours), |
259 | +and d (days). Combine this option with L<"--resume"> to checksum as many |
260 | +tables within an allotted time, resuming from where the tool left off next |
261 | +time it is ran. |
262 | + |
263 | =item --separator |
264 | |
265 | type: string; default: # |
266 | |
267 | === modified file 'lib/Cxn.pm' |
268 | --- lib/Cxn.pm 2012-07-23 02:37:39 +0000 |
269 | +++ lib/Cxn.pm 2012-08-24 21:18:32 +0000 |
270 | @@ -103,7 +103,11 @@ |
271 | dsn_name => $dp->as_string($dsn, [qw(h P S)]), |
272 | hostname => '', |
273 | set => $args{set}, |
274 | +<<<<<<< TREE |
275 | NAME_lc => defined($args{NAME_lc}) ? $args{NAME_lc} : 1, |
276 | +======= |
277 | + NAME_lc => defined $args{NAME_lc} ? $args{NAME_lc} : 1, |
278 | +>>>>>>> MERGE-SOURCE |
279 | dbh_set => 0, |
280 | OptionParser => $o, |
281 | DSNParser => $dp, |
282 | @@ -150,11 +154,8 @@ |
283 | PTDEBUG && _d($dbh, 'Setting dbh'); |
284 | |
285 | # Set stuff for this dbh (i.e. initialize it). |
286 | - if ( !exists $self->{NAME_lc} |
287 | - || (defined $self->{NAME_lc} && $self->{NAME_lc}) ) { |
288 | - $dbh->{FetchHashKeyName} = 'NAME_lc'; |
289 | - } |
290 | - |
291 | + $dbh->{FetchHashKeyName} = 'NAME_lc' if $self->{NAME_lc}; |
292 | + |
293 | # Update the cxn's name. Until we connect, the DSN parts |
294 | # h and P are used. Once connected, use @@hostname. |
295 | my $sql = 'SELECT @@hostname, @@server_id'; |
296 | |
297 | === added file 't/pt-table-checksum/run_time.t' |
298 | --- t/pt-table-checksum/run_time.t 1970-01-01 00:00:00 +0000 |
299 | +++ t/pt-table-checksum/run_time.t 2012-08-24 21:18:32 +0000 |
300 | @@ -0,0 +1,77 @@ |
301 | +#!/usr/bin/env perl |
302 | + |
303 | +BEGIN { |
304 | + die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n" |
305 | + unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH}; |
306 | + unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib"; |
307 | +}; |
308 | + |
309 | +use strict; |
310 | +use warnings FATAL => 'all'; |
311 | +use English qw(-no_match_vars); |
312 | +use Test::More; |
313 | +use Time::HiRes qw(time); |
314 | +use Data::Dumper; |
315 | +$Data::Dumper::Indent = 1; |
316 | +$Data::Dumper::Sortkeys = 1; |
317 | +$Data::Dumper::Quotekeys = 0; |
318 | + |
319 | +use PerconaTest; |
320 | +use Sandbox; |
321 | +shift @INC; # our unshift (above) |
322 | +shift @INC; # PerconaTest's unshift |
323 | +require "$trunk/bin/pt-table-checksum"; |
324 | + |
325 | +my $dp = new DSNParser(opts=>$dsn_opts); |
326 | +my $sb = new Sandbox(basedir => '/tmp', DSNParser => $dp); |
327 | +my $master_dbh = $sb->get_dbh_for('master'); |
328 | + |
329 | +if ( !$master_dbh ) { |
330 | + plan skip_all => 'Cannot connect to sandbox master'; |
331 | +} |
332 | + |
333 | +# The sandbox servers run with lock_wait_timeout=3 and it's not dynamic |
334 | +# so we need to specify --lock-wait-timeout=3 else the tool will die. |
335 | +# And --max-load "" prevents waiting for status variables. |
336 | +my $master_dsn = 'h=127.1,P=12345,u=msandbox,p=msandbox'; |
337 | +my @args = ($master_dsn, qw(--lock-wait-timeout 3), '--max-load', ''); |
338 | +my $output; |
339 | +my $exit_status; |
340 | + |
341 | +# On my 2.4 GHz with SSD this takes a little more than 3s, |
342 | +# so no test servers should be faster, hopefully. |
343 | +my $t0 = time; |
344 | +$exit_status = pt_table_checksum::main(@args, |
345 | + qw(--quiet --quiet -d sakila --chunk-size 100 --run-time 1)); |
346 | +my $t = time - $t0; |
347 | + |
348 | +ok( |
349 | + $t >= 1.5 && $t <= 2.0, |
350 | + "Run in roughly --run-time 1 second" |
351 | +) or diag("Actual run time: $t"); |
352 | + |
353 | +my $rows = $master_dbh->selectall_arrayref("SELECT DISTINCT CONCAT(db, '.', tbl) FROM percona.checksums ORDER by db, tbl"); |
354 | +my $sakila_finished = grep { $_->[0] eq 'sakila.store' } @$rows; |
355 | +ok( |
356 | + !$sakila_finished, |
357 | + "Did not finish checksumming sakila" |
358 | +) or diag(Dumper($rows)); |
359 | + |
360 | +# Add --resume to complete the run. |
361 | +$exit_status = pt_table_checksum::main(@args, |
362 | + qw(--quiet --quiet -d sakila --chunk-size 100)); |
363 | + |
364 | +$rows = $master_dbh->selectall_arrayref("SELECT DISTINCT CONCAT(db, '.', tbl) FROM percona.checksums ORDER by db, tbl"); |
365 | +$sakila_finished = grep { $_->[0] eq 'sakila.store' } @$rows; |
366 | +ok( |
367 | + $sakila_finished, |
368 | + "Resumed and finish checksumming sakila" |
369 | +) or diag(Dumper($rows)); |
370 | + |
371 | +# ############################################################################# |
372 | +# Done. |
373 | +# ############################################################################# |
374 | +$sb->wipe_clean($master_dbh); |
375 | +ok($sb->ok(), "Sandbox servers") or BAIL_OUT(__FILE__ . " broke the sandbox"); |
376 | +done_testing; |
377 | +exit; |