Merge lp:~percona-toolkit-dev/percona-toolkit/fix-pqd-distill-bugs into lp:percona-toolkit/2.2

Proposed by Daniel Nichter on 2013-08-03
Status: Merged
Merged at revision: 585
Proposed branch: lp:~percona-toolkit-dev/percona-toolkit/fix-pqd-distill-bugs
Merge into: lp:percona-toolkit/2.2
Diff against target: 1655 lines (+779/-514)
18 files modified
bin/pt-query-digest (+559/-490)
lib/HTTP/Micro.pm (+1/-1)
lib/QueryParser.pm (+16/-4)
lib/QueryRewriter.pm (+10/-0)
lib/VersionCheck.pm (+2/-2)
t/lib/QueryParser.t (+6/-0)
t/lib/QueryRewriter.t (+28/-0)
t/lib/samples/slowlogs/slow051.txt (+2/-2)
t/lib/samples/slowlogs/slow058.txt (+24/-0)
t/pt-query-digest/binlog_analyses.t (+2/-2)
t/pt-query-digest/daemon.t (+2/-2)
t/pt-query-digest/json.t (+1/-1)
t/pt-query-digest/samples/binlog002.txt (+4/-1)
t/pt-query-digest/samples/empty_report.txt (+2/-0)
t/pt-query-digest/samples/slow051.txt (+5/-5)
t/pt-query-digest/samples/slow058.txt (+94/-0)
t/pt-query-digest/slowlog_analyses.t (+20/-3)
util/update-modules (+1/-1)
To merge this branch: bzr merge lp:~percona-toolkit-dev/percona-toolkit/fix-pqd-distill-bugs
Reviewer Review Type Date Requested Status
Daniel Nichter Approve on 2013-08-03
Review via email: mp+178434@code.launchpad.net
To post a comment you must log in.
596. By Daniel Nichter on 2013-08-03

Use new Daemon API.

597. By Daniel Nichter on 2013-08-03

Fix empty json report test.

598. By Daniel Nichter on 2013-08-03

Update t/pt-query-digest/binlog_analyses.t now that INSERT without INTO parses correctly.

599. By Daniel Nichter on 2013-08-03

Fix version check in pqd by calling HTTP::Micro, not old HTTPMicro.

600. By Daniel Nichter on 2013-08-03

Update/fix binlog_analyses.t for real.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'bin/pt-query-digest'
2--- bin/pt-query-digest 2013-07-17 19:41:00 +0000
3+++ bin/pt-query-digest 2013-08-03 19:47:33 +0000
4@@ -48,7 +48,7 @@
5 FileIterator
6 Runtime
7 Pipeline
8- HTTPMicro
9+ HTTP::Micro
10 VersionCheck
11 ));
12 }
13@@ -2928,6 +2928,13 @@
14 $query =~ m/\A\s*UNLOCK TABLES/i && return "UNLOCK";
15 $query =~ m/\A\s*xa\s+(\S+)/i && return "XA_$1";
16
17+ if ( $query =~ m/\A\s*LOAD/i ) {
18+ my ($tbl) = $query =~ m/INTO TABLE\s+(\S+)/i;
19+ $tbl ||= '';
20+ $tbl =~ s/`//g;
21+ return "LOAD DATA $tbl";
22+ }
23+
24 if ( $query =~ m/\Aadministrator command:/ ) {
25 $query =~ s/administrator command:/ADMIN/;
26 $query = uc $query;
27@@ -3021,6 +3028,9 @@
28 map { $verbs =~ s/$_/$alias_for{$_}/ } keys %alias_for;
29 $query = $verbs;
30 }
31+ elsif ( $verbs && $verbs =~ m/^LOAD DATA/ ) {
32+ return $verbs;
33+ }
34 else {
35 my @tables = $self->__distill_tables($query, $table, %args);
36 $query = join(q{ }, $verbs, @tables);
37@@ -8259,7 +8269,7 @@
38 return ($tbl);
39 }
40
41- $query =~ s/ (?:LOW_PRIORITY|IGNORE|STRAIGHT_JOIN)//ig;
42+ $query =~ s/(?:LOW_PRIORITY|IGNORE|STRAIGHT_JOIN|DELAYED)\s+/ /ig;
43
44 if ( $query =~ s/^\s*LOCK TABLES\s+//i ) {
45 PTDEBUG && _d('Special table type: LOCK TABLES');
46@@ -8268,9 +8278,18 @@
47 $query = "FROM $query";
48 }
49
50- $query =~ s/\\["']//g; # quoted strings
51- $query =~ s/".*?"/?/sg; # quoted strings
52- $query =~ s/'.*?'/?/sg; # quoted strings
53+ $query =~ s/\\["']//g; # quoted strings
54+ $query =~ s/".*?"/?/sg; # quoted strings
55+ $query =~ s/'.*?'/?/sg; # quoted strings
56+
57+ if ( $query =~ m/\A\s*(?:INSERT|REPLACE)(?!\s+INTO)/i ) {
58+ $query =~ s/\A\s*((?:INSERT|REPLACE))\s+/$1 INTO /i;
59+ }
60+
61+ if ( $query =~ m/\A\s*LOAD DATA/i ) {
62+ my ($tbl) = $query =~ m/INTO TABLE\s+(\S+)/i;
63+ return $tbl;
64+ }
65
66 my @tables;
67 foreach my $tbls ( $query =~ m/$tbl_regex/gio ) {
68@@ -9253,157 +9272,214 @@
69 use strict;
70 use warnings FATAL => 'all';
71 use English qw(-no_match_vars);
72+
73 use constant PTDEBUG => $ENV{PTDEBUG} || 0;
74
75 use POSIX qw(setsid);
76+use Fcntl qw(:DEFAULT);
77
78 sub new {
79- my ( $class, %args ) = @_;
80- foreach my $arg ( qw(o) ) {
81- die "I need a $arg argument" unless $args{$arg};
82- }
83- my $o = $args{o};
84+ my ($class, %args) = @_;
85 my $self = {
86- o => $o,
87- log_file => $o->has('log') ? $o->get('log') : undef,
88- PID_file => $o->has('pid') ? $o->get('pid') : undef,
89+ log_file => $args{log_file},
90+ pid_file => $args{pid_file},
91+ daemonize => $args{daemonize},
92+ force_log_file => $args{force_log_file},
93+ parent_exit => $args{parent_exit},
94+ pid_file_owner => 0,
95 };
96-
97- check_PID_file(undef, $self->{PID_file});
98-
99- PTDEBUG && _d('Daemonized child will log to', $self->{log_file});
100 return bless $self, $class;
101 }
102
103-sub daemonize {
104- my ( $self ) = @_;
105-
106- PTDEBUG && _d('About to fork and daemonize');
107- defined (my $pid = fork()) or die "Cannot fork: $OS_ERROR";
108- if ( $pid ) {
109- PTDEBUG && _d('Parent PID', $PID, 'exiting after forking child PID',$pid);
110- exit;
111- }
112-
113- PTDEBUG && _d('Daemonizing child PID', $PID);
114- $self->{PID_owner} = $PID;
115- $self->{child} = 1;
116-
117- POSIX::setsid() or die "Cannot start a new session: $OS_ERROR";
118- chdir '/' or die "Cannot chdir to /: $OS_ERROR";
119-
120- $self->_make_PID_file();
121-
122- $OUTPUT_AUTOFLUSH = 1;
123-
124- PTDEBUG && _d('Redirecting STDIN to /dev/null');
125- close STDIN;
126- open STDIN, '/dev/null'
127- or die "Cannot reopen STDIN to /dev/null: $OS_ERROR";
128-
129- if ( $self->{log_file} ) {
130- PTDEBUG && _d('Redirecting STDOUT and STDERR to', $self->{log_file});
131- close STDOUT;
132- open STDOUT, '>>', $self->{log_file}
133- or die "Cannot open log file $self->{log_file}: $OS_ERROR";
134-
135- close STDERR;
136- open STDERR, ">&STDOUT"
137- or die "Cannot dupe STDERR to STDOUT: $OS_ERROR";
138- }
139- else {
140- if ( -t STDOUT ) {
141- PTDEBUG && _d('No log file and STDOUT is a terminal;',
142- 'redirecting to /dev/null');
143+sub run {
144+ my ($self) = @_;
145+
146+ my $daemonize = $self->{daemonize};
147+ my $pid_file = $self->{pid_file};
148+ my $log_file = $self->{log_file};
149+ my $force_log_file = $self->{force_log_file};
150+ my $parent_exit = $self->{parent_exit};
151+
152+ PTDEBUG && _d('Starting daemon');
153+
154+ if ( $pid_file ) {
155+ eval {
156+ $self->_make_pid_file(
157+ pid => $PID, # parent's pid
158+ pid_file => $pid_file,
159+ );
160+ };
161+ die "$EVAL_ERROR\n" if $EVAL_ERROR;
162+ if ( !$daemonize ) {
163+ $self->{pid_file_owner} = $PID; # parent's pid
164+ }
165+ }
166+
167+ if ( $daemonize ) {
168+ defined (my $child_pid = fork()) or die "Cannot fork: $OS_ERROR";
169+ if ( $child_pid ) {
170+ PTDEBUG && _d('Forked child', $child_pid);
171+ $parent_exit->($child_pid) if $parent_exit;
172+ exit 0;
173+ }
174+
175+ POSIX::setsid() or die "Cannot start a new session: $OS_ERROR";
176+ chdir '/' or die "Cannot chdir to /: $OS_ERROR";
177+
178+ if ( $pid_file ) {
179+ $self->_update_pid_file(
180+ pid => $PID, # child's pid
181+ pid_file => $pid_file,
182+ );
183+ $self->{pid_file_owner} = $PID;
184+ }
185+ }
186+
187+ if ( $daemonize || $force_log_file ) {
188+ PTDEBUG && _d('Redirecting STDIN to /dev/null');
189+ close STDIN;
190+ open STDIN, '/dev/null'
191+ or die "Cannot reopen STDIN to /dev/null: $OS_ERROR";
192+ if ( $log_file ) {
193+ PTDEBUG && _d('Redirecting STDOUT and STDERR to', $log_file);
194 close STDOUT;
195- open STDOUT, '>', '/dev/null'
196- or die "Cannot reopen STDOUT to /dev/null: $OS_ERROR";
197- }
198- if ( -t STDERR ) {
199- PTDEBUG && _d('No log file and STDERR is a terminal;',
200- 'redirecting to /dev/null');
201+ open STDOUT, '>>', $log_file
202+ or die "Cannot open log file $log_file: $OS_ERROR";
203+
204 close STDERR;
205- open STDERR, '>', '/dev/null'
206- or die "Cannot reopen STDERR to /dev/null: $OS_ERROR";
207- }
208- }
209-
210- return;
211-}
212-
213-sub check_PID_file {
214- my ( $self, $file ) = @_;
215- my $PID_file = $self ? $self->{PID_file} : $file;
216- PTDEBUG && _d('Checking PID file', $PID_file);
217- if ( $PID_file && -f $PID_file ) {
218- my $pid;
219- eval {
220- chomp($pid = (slurp_file($PID_file) || ''));
221- };
222- if ( $EVAL_ERROR ) {
223- die "The PID file $PID_file already exists but it cannot be read: "
224- . $EVAL_ERROR;
225- }
226- PTDEBUG && _d('PID file exists; it contains PID', $pid);
227- if ( $pid ) {
228- my $pid_is_alive = kill 0, $pid;
229+ open STDERR, ">&STDOUT"
230+ or die "Cannot dupe STDERR to STDOUT: $OS_ERROR";
231+ }
232+ else {
233+ if ( -t STDOUT ) {
234+ PTDEBUG && _d('No log file and STDOUT is a terminal;',
235+ 'redirecting to /dev/null');
236+ close STDOUT;
237+ open STDOUT, '>', '/dev/null'
238+ or die "Cannot reopen STDOUT to /dev/null: $OS_ERROR";
239+ }
240+ if ( -t STDERR ) {
241+ PTDEBUG && _d('No log file and STDERR is a terminal;',
242+ 'redirecting to /dev/null');
243+ close STDERR;
244+ open STDERR, '>', '/dev/null'
245+ or die "Cannot reopen STDERR to /dev/null: $OS_ERROR";
246+ }
247+ }
248+
249+ $OUTPUT_AUTOFLUSH = 1;
250+ }
251+
252+ PTDEBUG && _d('Daemon running');
253+ return;
254+}
255+
256+sub _make_pid_file {
257+ my ($self, %args) = @_;
258+ my @required_args = qw(pid pid_file);
259+ foreach my $arg ( @required_args ) {
260+ die "I need a $arg argument" unless $args{$arg};
261+ };
262+ my $pid = $args{pid};
263+ my $pid_file = $args{pid_file};
264+
265+ eval {
266+ sysopen(PID_FH, $pid_file, O_RDWR|O_CREAT|O_EXCL) or die $OS_ERROR;
267+ print PID_FH $PID, "\n";
268+ close PID_FH;
269+ };
270+ if ( my $e = $EVAL_ERROR ) {
271+ if ( $e =~ m/file exists/i ) {
272+ my $old_pid = $self->_check_pid_file(
273+ pid_file => $pid_file,
274+ pid => $PID,
275+ );
276+ if ( $old_pid ) {
277+ warn "Overwriting PID file $pid_file because PID $old_pid "
278+ . "is not running.\n";
279+ }
280+ $self->_update_pid_file(
281+ pid => $PID,
282+ pid_file => $pid_file
283+ );
284+ }
285+ else {
286+ die "Error creating PID file $pid_file: $e\n";
287+ }
288+ }
289+
290+ return;
291+}
292+
293+sub _check_pid_file {
294+ my ($self, %args) = @_;
295+ my @required_args = qw(pid_file pid);
296+ foreach my $arg ( @required_args ) {
297+ die "I need a $arg argument" unless $args{$arg};
298+ };
299+ my $pid_file = $args{pid_file};
300+ my $pid = $args{pid};
301+
302+ PTDEBUG && _d('Checking if PID in', $pid_file, 'is running');
303+
304+ if ( ! -f $pid_file ) {
305+ PTDEBUG && _d('PID file', $pid_file, 'does not exist');
306+ return;
307+ }
308+
309+ open my $fh, '<', $pid_file
310+ or die "Error opening $pid_file: $OS_ERROR";
311+ my $existing_pid = do { local $/; <$fh> };
312+ chomp($existing_pid) if $existing_pid;
313+ close $fh
314+ or die "Error closing $pid_file: $OS_ERROR";
315+
316+ if ( $existing_pid ) {
317+ if ( $existing_pid == $pid ) {
318+ warn "The current PID $pid already holds the PID file $pid_file\n";
319+ return;
320+ }
321+ else {
322+ PTDEBUG && _d('Checking if PID', $existing_pid, 'is running');
323+ my $pid_is_alive = kill 0, $existing_pid;
324 if ( $pid_is_alive ) {
325- die "The PID file $PID_file already exists "
326- . " and the PID that it contains, $pid, is running";
327- }
328- else {
329- warn "Overwriting PID file $PID_file because the PID that it "
330- . "contains, $pid, is not running";
331- }
332- }
333- else {
334- die "The PID file $PID_file already exists but it does not "
335- . "contain a PID";
336+ die "PID file $pid_file exists and PID $existing_pid is running\n";
337+ }
338 }
339 }
340 else {
341- PTDEBUG && _d('No PID file');
342- }
343- return;
344-}
345-
346-sub make_PID_file {
347- my ( $self ) = @_;
348- if ( exists $self->{child} ) {
349- die "Do not call Daemon::make_PID_file() for daemonized scripts";
350- }
351- $self->_make_PID_file();
352- $self->{PID_owner} = $PID;
353- return;
354-}
355-
356-sub _make_PID_file {
357- my ( $self ) = @_;
358-
359- my $PID_file = $self->{PID_file};
360- if ( !$PID_file ) {
361- PTDEBUG && _d('No PID file to create');
362- return;
363- }
364-
365- $self->check_PID_file();
366-
367- open my $PID_FH, '>', $PID_file
368- or die "Cannot open PID file $PID_file: $OS_ERROR";
369- print $PID_FH $PID
370- or die "Cannot print to PID file $PID_file: $OS_ERROR";
371- close $PID_FH
372- or die "Cannot close PID file $PID_file: $OS_ERROR";
373-
374- PTDEBUG && _d('Created PID file:', $self->{PID_file});
375- return;
376-}
377-
378-sub _remove_PID_file {
379- my ( $self ) = @_;
380- if ( $self->{PID_file} && -f $self->{PID_file} ) {
381- unlink $self->{PID_file}
382- or warn "Cannot remove PID file $self->{PID_file}: $OS_ERROR";
383+ die "PID file $pid_file exists but it is empty. Remove the file "
384+ . "if the process is no longer running.\n";
385+ }
386+
387+ return $existing_pid;
388+}
389+
390+sub _update_pid_file {
391+ my ($self, %args) = @_;
392+ my @required_args = qw(pid pid_file);
393+ foreach my $arg ( @required_args ) {
394+ die "I need a $arg argument" unless $args{$arg};
395+ };
396+ my $pid = $args{pid};
397+ my $pid_file = $args{pid_file};
398+
399+ open my $fh, '>', $pid_file
400+ or die "Cannot open $pid_file: $OS_ERROR";
401+ print { $fh } $pid, "\n"
402+ or die "Cannot print to $pid_file: $OS_ERROR";
403+ close $fh
404+ or warn "Cannot close $pid_file: $OS_ERROR";
405+
406+ return;
407+}
408+
409+sub remove_pid_file {
410+ my ($self, $pid_file) = @_;
411+ $pid_file ||= $self->{pid_file};
412+ if ( $pid_file && -f $pid_file ) {
413+ unlink $self->{pid_file}
414+ or warn "Cannot remove PID file $pid_file: $OS_ERROR";
415 PTDEBUG && _d('Removed PID file');
416 }
417 else {
418@@ -9413,20 +9489,15 @@
419 }
420
421 sub DESTROY {
422- my ( $self ) = @_;
423+ my ($self) = @_;
424
425- $self->_remove_PID_file() if ($self->{PID_owner} || 0) == $PID;
426+ if ( $self->{pid_file_owner} == $PID ) {
427+ $self->remove_pid_file();
428+ }
429
430 return;
431 }
432
433-sub slurp_file {
434- my ($file) = @_;
435- return unless $file;
436- open my $fh, "<", $file or die "Cannot open $file: $OS_ERROR";
437- return do { local $/; <$fh> };
438-}
439-
440 sub _d {
441 my ($package, undef, $line) = caller 0;
442 @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
443@@ -11479,25 +11550,23 @@
444 # ###########################################################################
445
446 # ###########################################################################
447-# HTTPMicro package
448+# HTTP::Micro package
449 # This package is a copy without comments from the original. The original
450 # with comments and its test file can be found in the Bazaar repository at,
451-# lib/HTTPMicro.pm
452-# t/lib/HTTPMicro.t
453+# lib/HTTP/Micro.pm
454+# t/lib/HTTP/Micro.t
455 # See https://launchpad.net/percona-toolkit for more information.
456 # ###########################################################################
457 {
458-
459-package HTTPMicro;
460-BEGIN {
461- $HTTPMicro::VERSION = '0.001';
462-}
463+package HTTP::Micro;
464+
465+our $VERSION = '0.01';
466+
467 use strict;
468-use warnings;
469-
470+use warnings FATAL => 'all';
471+use English qw(-no_match_vars);
472 use Carp ();
473
474-
475 my @attributes;
476 BEGIN {
477 @attributes = qw(agent timeout);
478@@ -11568,7 +11637,7 @@
479 headers => {},
480 };
481
482- my $handle = HTTPMicro::Handle->new(timeout => $self->{timeout});
483+ my $handle = HTTP::Micro::Handle->new(timeout => $self->{timeout});
484
485 $handle->connect($scheme, $host, $port);
486
487@@ -11633,320 +11702,325 @@
488 return ($scheme, $host, $port, $path_query);
489 }
490
491-package
492- HTTPMicro::Handle; # hide from PAUSE/indexers
493-use strict;
494-use warnings;
495-
496-use Carp qw[croak];
497-use Errno qw[EINTR EPIPE];
498-use IO::Socket qw[SOCK_STREAM];
499-
500-sub BUFSIZE () { 32768 }
501-
502-my $Printable = sub {
503- local $_ = shift;
504- s/\r/\\r/g;
505- s/\n/\\n/g;
506- s/\t/\\t/g;
507- s/([^\x20-\x7E])/sprintf('\\x%.2X', ord($1))/ge;
508- $_;
509-};
510-
511-sub new {
512- my ($class, %args) = @_;
513- return bless {
514- rbuf => '',
515- timeout => 60,
516- max_line_size => 16384,
517- %args
518- }, $class;
519-}
520-
521-my $ssl_verify_args = {
522- check_cn => "when_only",
523- wildcards_in_alt => "anywhere",
524- wildcards_in_cn => "anywhere"
525-};
526-
527-sub connect {
528- @_ == 4 || croak(q/Usage: $handle->connect(scheme, host, port)/);
529- my ($self, $scheme, $host, $port) = @_;
530-
531- if ( $scheme eq 'https' ) {
532- eval "require IO::Socket::SSL"
533- unless exists $INC{'IO/Socket/SSL.pm'};
534- croak(qq/IO::Socket::SSL must be installed for https support\n/)
535- unless $INC{'IO/Socket/SSL.pm'};
536- }
537- elsif ( $scheme ne 'http' ) {
538- croak(qq/Unsupported URL scheme '$scheme'\n/);
539- }
540-
541- $self->{fh} = 'IO::Socket::INET'->new(
542- PeerHost => $host,
543- PeerPort => $port,
544- Proto => 'tcp',
545- Type => SOCK_STREAM,
546- Timeout => $self->{timeout}
547- ) or croak(qq/Could not connect to '$host:$port': $@/);
548-
549- binmode($self->{fh})
550- or croak(qq/Could not binmode() socket: '$!'/);
551-
552- if ( $scheme eq 'https') {
553- IO::Socket::SSL->start_SSL($self->{fh});
554- ref($self->{fh}) eq 'IO::Socket::SSL'
555- or die(qq/SSL connection failed for $host\n/);
556- if ( $self->{fh}->can("verify_hostname") ) {
557- $self->{fh}->verify_hostname( $host, $ssl_verify_args );
558- }
559- else {
560- my $fh = $self->{fh};
561- _verify_hostname_of_cert($host, _peer_certificate($fh), $ssl_verify_args)
562- or die(qq/SSL certificate not valid for $host\n/);
563- }
564- }
565-
566- $self->{host} = $host;
567- $self->{port} = $port;
568-
569- return $self;
570-}
571-
572-sub close {
573- @_ == 1 || croak(q/Usage: $handle->close()/);
574- my ($self) = @_;
575- CORE::close($self->{fh})
576- or croak(qq/Could not close socket: '$!'/);
577-}
578-
579-sub write {
580- @_ == 2 || croak(q/Usage: $handle->write(buf)/);
581- my ($self, $buf) = @_;
582-
583- my $len = length $buf;
584- my $off = 0;
585-
586- local $SIG{PIPE} = 'IGNORE';
587-
588- while () {
589- $self->can_write
590- or croak(q/Timed out while waiting for socket to become ready for writing/);
591- my $r = syswrite($self->{fh}, $buf, $len, $off);
592- if (defined $r) {
593- $len -= $r;
594- $off += $r;
595- last unless $len > 0;
596- }
597- elsif ($! == EPIPE) {
598- croak(qq/Socket closed by remote server: $!/);
599- }
600- elsif ($! != EINTR) {
601- croak(qq/Could not write to socket: '$!'/);
602- }
603- }
604- return $off;
605-}
606-
607-sub read {
608- @_ == 2 || @_ == 3 || croak(q/Usage: $handle->read(len)/);
609- my ($self, $len) = @_;
610-
611- my $buf = '';
612- my $got = length $self->{rbuf};
613-
614- if ($got) {
615- my $take = ($got < $len) ? $got : $len;
616- $buf = substr($self->{rbuf}, 0, $take, '');
617- $len -= $take;
618- }
619-
620- while ($len > 0) {
621- $self->can_read
622- or croak(q/Timed out while waiting for socket to become ready for reading/);
623- my $r = sysread($self->{fh}, $buf, $len, length $buf);
624- if (defined $r) {
625- last unless $r;
626- $len -= $r;
627- }
628- elsif ($! != EINTR) {
629- croak(qq/Could not read from socket: '$!'/);
630- }
631- }
632- if ($len) {
633- croak(q/Unexpected end of stream/);
634- }
635- return $buf;
636-}
637-
638-sub readline {
639- @_ == 1 || croak(q/Usage: $handle->readline()/);
640- my ($self) = @_;
641-
642- while () {
643- if ($self->{rbuf} =~ s/\A ([^\x0D\x0A]* \x0D?\x0A)//x) {
644- return $1;
645- }
646- $self->can_read
647- or croak(q/Timed out while waiting for socket to become ready for reading/);
648- my $r = sysread($self->{fh}, $self->{rbuf}, BUFSIZE, length $self->{rbuf});
649- if (defined $r) {
650- last unless $r;
651- }
652- elsif ($! != EINTR) {
653- croak(qq/Could not read from socket: '$!'/);
654- }
655- }
656- croak(q/Unexpected end of stream while looking for line/);
657-}
658-
659-sub read_header_lines {
660- @_ == 1 || @_ == 2 || croak(q/Usage: $handle->read_header_lines([headers])/);
661- my ($self, $headers) = @_;
662- $headers ||= {};
663- my $lines = 0;
664- my $val;
665-
666- while () {
667- my $line = $self->readline;
668-
669- if ($line =~ /\A ([^\x00-\x1F\x7F:]+) : [\x09\x20]* ([^\x0D\x0A]*)/x) {
670- my ($field_name) = lc $1;
671- $val = \($headers->{$field_name} = $2);
672- }
673- elsif ($line =~ /\A [\x09\x20]+ ([^\x0D\x0A]*)/x) {
674- $val
675- or croak(q/Unexpected header continuation line/);
676- next unless length $1;
677- $$val .= ' ' if length $$val;
678- $$val .= $1;
679- }
680- elsif ($line =~ /\A \x0D?\x0A \z/x) {
681- last;
682- }
683- else {
684- croak(q/Malformed header line: / . $Printable->($line));
685- }
686- }
687- return $headers;
688-}
689-
690-sub write_header_lines {
691- (@_ == 2 && ref $_[1] eq 'HASH') || croak(q/Usage: $handle->write_header_lines(headers)/);
692- my($self, $headers) = @_;
693-
694- my $buf = '';
695- while (my ($k, $v) = each %$headers) {
696- my $field_name = lc $k;
697- $field_name =~ /\A [\x21\x23-\x27\x2A\x2B\x2D\x2E\x30-\x39\x41-\x5A\x5E-\x7A\x7C\x7E]+ \z/x
698- or croak(q/Invalid HTTP header field name: / . $Printable->($field_name));
699- $field_name =~ s/\b(\w)/\u$1/g;
700- $buf .= "$field_name: $v\x0D\x0A";
701- }
702- $buf .= "\x0D\x0A";
703- return $self->write($buf);
704-}
705-
706-sub read_content_body {
707- @_ == 3 || @_ == 4 || croak(q/Usage: $handle->read_content_body(callback, response, [read_length])/);
708- my ($self, $cb, $response, $len) = @_;
709- $len ||= $response->{headers}{'content-length'};
710-
711- croak("No content-length in the returned response, and this "
712- . "UA doesn't implement chunking") unless defined $len;
713-
714- while ($len > 0) {
715- my $read = ($len > BUFSIZE) ? BUFSIZE : $len;
716- $cb->($self->read($read), $response);
717- $len -= $read;
718- }
719-
720- return;
721-}
722-
723-sub write_content_body {
724- @_ == 2 || croak(q/Usage: $handle->write_content_body(request)/);
725- my ($self, $request) = @_;
726- my ($len, $content_length) = (0, $request->{headers}{'content-length'});
727-
728- $len += $self->write($request->{content});
729-
730- $len == $content_length
731- or croak(qq/Content-Length missmatch (got: $len expected: $content_length)/);
732-
733- return $len;
734-}
735-
736-sub read_response_header {
737- @_ == 1 || croak(q/Usage: $handle->read_response_header()/);
738- my ($self) = @_;
739-
740- my $line = $self->readline;
741-
742- $line =~ /\A (HTTP\/(0*\d+\.0*\d+)) [\x09\x20]+ ([0-9]{3}) [\x09\x20]+ ([^\x0D\x0A]*) \x0D?\x0A/x
743- or croak(q/Malformed Status-Line: / . $Printable->($line));
744-
745- my ($protocol, $version, $status, $reason) = ($1, $2, $3, $4);
746-
747- return {
748- status => $status,
749- reason => $reason,
750- headers => $self->read_header_lines,
751- protocol => $protocol,
752- };
753-}
754-
755-sub write_request_header {
756- @_ == 4 || croak(q/Usage: $handle->write_request_header(method, request_uri, headers)/);
757- my ($self, $method, $request_uri, $headers) = @_;
758-
759- return $self->write("$method $request_uri HTTP/1.1\x0D\x0A")
760- + $self->write_header_lines($headers);
761-}
762-
763-sub _do_timeout {
764- my ($self, $type, $timeout) = @_;
765- $timeout = $self->{timeout}
766- unless defined $timeout && $timeout >= 0;
767-
768- my $fd = fileno $self->{fh};
769- defined $fd && $fd >= 0
770- or croak(q/select(2): 'Bad file descriptor'/);
771-
772- my $initial = time;
773- my $pending = $timeout;
774- my $nfound;
775-
776- vec(my $fdset = '', $fd, 1) = 1;
777-
778- while () {
779- $nfound = ($type eq 'read')
780- ? select($fdset, undef, undef, $pending)
781- : select(undef, $fdset, undef, $pending) ;
782- if ($nfound == -1) {
783- $! == EINTR
784- or croak(qq/select(2): '$!'/);
785- redo if !$timeout || ($pending = $timeout - (time - $initial)) > 0;
786- $nfound = 0;
787- }
788- last;
789- }
790- $! = 0;
791- return $nfound;
792-}
793-
794-sub can_read {
795- @_ == 1 || @_ == 2 || croak(q/Usage: $handle->can_read([timeout])/);
796- my $self = shift;
797- return $self->_do_timeout('read', @_)
798-}
799-
800-sub can_write {
801- @_ == 1 || @_ == 2 || croak(q/Usage: $handle->can_write([timeout])/);
802- my $self = shift;
803- return $self->_do_timeout('write', @_)
804-}
805+} # HTTP::Micro
806+
807+{
808+ package HTTP::Micro::Handle;
809+
810+ use strict;
811+ use warnings FATAL => 'all';
812+ use English qw(-no_match_vars);
813+
814+ use Carp qw(croak);
815+ use Errno qw(EINTR EPIPE);
816+ use IO::Socket qw(SOCK_STREAM);
817+
818+ sub BUFSIZE () { 32768 }
819+
820+ my $Printable = sub {
821+ local $_ = shift;
822+ s/\r/\\r/g;
823+ s/\n/\\n/g;
824+ s/\t/\\t/g;
825+ s/([^\x20-\x7E])/sprintf('\\x%.2X', ord($1))/ge;
826+ $_;
827+ };
828+
829+ sub new {
830+ my ($class, %args) = @_;
831+ return bless {
832+ rbuf => '',
833+ timeout => 60,
834+ max_line_size => 16384,
835+ %args
836+ }, $class;
837+ }
838+
839+ my $ssl_verify_args = {
840+ check_cn => "when_only",
841+ wildcards_in_alt => "anywhere",
842+ wildcards_in_cn => "anywhere"
843+ };
844+
845+ sub connect {
846+ @_ == 4 || croak(q/Usage: $handle->connect(scheme, host, port)/);
847+ my ($self, $scheme, $host, $port) = @_;
848+
849+ if ( $scheme eq 'https' ) {
850+ eval "require IO::Socket::SSL"
851+ unless exists $INC{'IO/Socket/SSL.pm'};
852+ croak(qq/IO::Socket::SSL must be installed for https support\n/)
853+ unless $INC{'IO/Socket/SSL.pm'};
854+ }
855+ elsif ( $scheme ne 'http' ) {
856+ croak(qq/Unsupported URL scheme '$scheme'\n/);
857+ }
858+
859+ $self->{fh} = IO::Socket::INET->new(
860+ PeerHost => $host,
861+ PeerPort => $port,
862+ Proto => 'tcp',
863+ Type => SOCK_STREAM,
864+ Timeout => $self->{timeout}
865+ ) or croak(qq/Could not connect to '$host:$port': $@/);
866+
867+ binmode($self->{fh})
868+ or croak(qq/Could not binmode() socket: '$!'/);
869+
870+ if ( $scheme eq 'https') {
871+ IO::Socket::SSL->start_SSL($self->{fh});
872+ ref($self->{fh}) eq 'IO::Socket::SSL'
873+ or die(qq/SSL connection failed for $host\n/);
874+ if ( $self->{fh}->can("verify_hostname") ) {
875+ $self->{fh}->verify_hostname( $host, $ssl_verify_args );
876+ }
877+ else {
878+ my $fh = $self->{fh};
879+ _verify_hostname_of_cert($host, _peer_certificate($fh), $ssl_verify_args)
880+ or die(qq/SSL certificate not valid for $host\n/);
881+ }
882+ }
883+
884+ $self->{host} = $host;
885+ $self->{port} = $port;
886+
887+ return $self;
888+ }
889+
890+ sub close {
891+ @_ == 1 || croak(q/Usage: $handle->close()/);
892+ my ($self) = @_;
893+ CORE::close($self->{fh})
894+ or croak(qq/Could not close socket: '$!'/);
895+ }
896+
897+ sub write {
898+ @_ == 2 || croak(q/Usage: $handle->write(buf)/);
899+ my ($self, $buf) = @_;
900+
901+ my $len = length $buf;
902+ my $off = 0;
903+
904+ local $SIG{PIPE} = 'IGNORE';
905+
906+ while () {
907+ $self->can_write
908+ or croak(q/Timed out while waiting for socket to become ready for writing/);
909+ my $r = syswrite($self->{fh}, $buf, $len, $off);
910+ if (defined $r) {
911+ $len -= $r;
912+ $off += $r;
913+ last unless $len > 0;
914+ }
915+ elsif ($! == EPIPE) {
916+ croak(qq/Socket closed by remote server: $!/);
917+ }
918+ elsif ($! != EINTR) {
919+ croak(qq/Could not write to socket: '$!'/);
920+ }
921+ }
922+ return $off;
923+ }
924+
925+ sub read {
926+ @_ == 2 || @_ == 3 || croak(q/Usage: $handle->read(len)/);
927+ my ($self, $len) = @_;
928+
929+ my $buf = '';
930+ my $got = length $self->{rbuf};
931+
932+ if ($got) {
933+ my $take = ($got < $len) ? $got : $len;
934+ $buf = substr($self->{rbuf}, 0, $take, '');
935+ $len -= $take;
936+ }
937+
938+ while ($len > 0) {
939+ $self->can_read
940+ or croak(q/Timed out while waiting for socket to become ready for reading/);
941+ my $r = sysread($self->{fh}, $buf, $len, length $buf);
942+ if (defined $r) {
943+ last unless $r;
944+ $len -= $r;
945+ }
946+ elsif ($! != EINTR) {
947+ croak(qq/Could not read from socket: '$!'/);
948+ }
949+ }
950+ if ($len) {
951+ croak(q/Unexpected end of stream/);
952+ }
953+ return $buf;
954+ }
955+
956+ sub readline {
957+ @_ == 1 || croak(q/Usage: $handle->readline()/);
958+ my ($self) = @_;
959+
960+ while () {
961+ if ($self->{rbuf} =~ s/\A ([^\x0D\x0A]* \x0D?\x0A)//x) {
962+ return $1;
963+ }
964+ $self->can_read
965+ or croak(q/Timed out while waiting for socket to become ready for reading/);
966+ my $r = sysread($self->{fh}, $self->{rbuf}, BUFSIZE, length $self->{rbuf});
967+ if (defined $r) {
968+ last unless $r;
969+ }
970+ elsif ($! != EINTR) {
971+ croak(qq/Could not read from socket: '$!'/);
972+ }
973+ }
974+ croak(q/Unexpected end of stream while looking for line/);
975+ }
976+
977+ sub read_header_lines {
978+ @_ == 1 || @_ == 2 || croak(q/Usage: $handle->read_header_lines([headers])/);
979+ my ($self, $headers) = @_;
980+ $headers ||= {};
981+ my $lines = 0;
982+ my $val;
983+
984+ while () {
985+ my $line = $self->readline;
986+
987+ if ($line =~ /\A ([^\x00-\x1F\x7F:]+) : [\x09\x20]* ([^\x0D\x0A]*)/x) {
988+ my ($field_name) = lc $1;
989+ $val = \($headers->{$field_name} = $2);
990+ }
991+ elsif ($line =~ /\A [\x09\x20]+ ([^\x0D\x0A]*)/x) {
992+ $val
993+ or croak(q/Unexpected header continuation line/);
994+ next unless length $1;
995+ $$val .= ' ' if length $$val;
996+ $$val .= $1;
997+ }
998+ elsif ($line =~ /\A \x0D?\x0A \z/x) {
999+ last;
1000+ }
1001+ else {
1002+ croak(q/Malformed header line: / . $Printable->($line));
1003+ }
1004+ }
1005+ return $headers;
1006+ }
1007+
1008+ sub write_header_lines {
1009+ (@_ == 2 && ref $_[1] eq 'HASH') || croak(q/Usage: $handle->write_header_lines(headers)/);
1010+ my($self, $headers) = @_;
1011+
1012+ my $buf = '';
1013+ while (my ($k, $v) = each %$headers) {
1014+ my $field_name = lc $k;
1015+ $field_name =~ /\A [\x21\x23-\x27\x2A\x2B\x2D\x2E\x30-\x39\x41-\x5A\x5E-\x7A\x7C\x7E]+ \z/x
1016+ or croak(q/Invalid HTTP header field name: / . $Printable->($field_name));
1017+ $field_name =~ s/\b(\w)/\u$1/g;
1018+ $buf .= "$field_name: $v\x0D\x0A";
1019+ }
1020+ $buf .= "\x0D\x0A";
1021+ return $self->write($buf);
1022+ }
1023+
1024+ sub read_content_body {
1025+ @_ == 3 || @_ == 4 || croak(q/Usage: $handle->read_content_body(callback, response, [read_length])/);
1026+ my ($self, $cb, $response, $len) = @_;
1027+ $len ||= $response->{headers}{'content-length'};
1028+
1029+ croak("No content-length in the returned response, and this "
1030+ . "UA doesn't implement chunking") unless defined $len;
1031+
1032+ while ($len > 0) {
1033+ my $read = ($len > BUFSIZE) ? BUFSIZE : $len;
1034+ $cb->($self->read($read), $response);
1035+ $len -= $read;
1036+ }
1037+
1038+ return;
1039+ }
1040+
1041+ sub write_content_body {
1042+ @_ == 2 || croak(q/Usage: $handle->write_content_body(request)/);
1043+ my ($self, $request) = @_;
1044+ my ($len, $content_length) = (0, $request->{headers}{'content-length'});
1045+
1046+ $len += $self->write($request->{content});
1047+
1048+ $len == $content_length
1049+ or croak(qq/Content-Length missmatch (got: $len expected: $content_length)/);
1050+
1051+ return $len;
1052+ }
1053+
1054+ sub read_response_header {
1055+ @_ == 1 || croak(q/Usage: $handle->read_response_header()/);
1056+ my ($self) = @_;
1057+
1058+ my $line = $self->readline;
1059+
1060+ $line =~ /\A (HTTP\/(0*\d+\.0*\d+)) [\x09\x20]+ ([0-9]{3}) [\x09\x20]+ ([^\x0D\x0A]*) \x0D?\x0A/x
1061+ or croak(q/Malformed Status-Line: / . $Printable->($line));
1062+
1063+ my ($protocol, $version, $status, $reason) = ($1, $2, $3, $4);
1064+
1065+ return {
1066+ status => $status,
1067+ reason => $reason,
1068+ headers => $self->read_header_lines,
1069+ protocol => $protocol,
1070+ };
1071+ }
1072+
1073+ sub write_request_header {
1074+ @_ == 4 || croak(q/Usage: $handle->write_request_header(method, request_uri, headers)/);
1075+ my ($self, $method, $request_uri, $headers) = @_;
1076+
1077+ return $self->write("$method $request_uri HTTP/1.1\x0D\x0A")
1078+ + $self->write_header_lines($headers);
1079+ }
1080+
1081+ sub _do_timeout {
1082+ my ($self, $type, $timeout) = @_;
1083+ $timeout = $self->{timeout}
1084+ unless defined $timeout && $timeout >= 0;
1085+
1086+ my $fd = fileno $self->{fh};
1087+ defined $fd && $fd >= 0
1088+ or croak(q/select(2): 'Bad file descriptor'/);
1089+
1090+ my $initial = time;
1091+ my $pending = $timeout;
1092+ my $nfound;
1093+
1094+ vec(my $fdset = '', $fd, 1) = 1;
1095+
1096+ while () {
1097+ $nfound = ($type eq 'read')
1098+ ? select($fdset, undef, undef, $pending)
1099+ : select(undef, $fdset, undef, $pending) ;
1100+ if ($nfound == -1) {
1101+ $! == EINTR
1102+ or croak(qq/select(2): '$!'/);
1103+ redo if !$timeout || ($pending = $timeout - (time - $initial)) > 0;
1104+ $nfound = 0;
1105+ }
1106+ last;
1107+ }
1108+ $! = 0;
1109+ return $nfound;
1110+ }
1111+
1112+ sub can_read {
1113+ @_ == 1 || @_ == 2 || croak(q/Usage: $handle->can_read([timeout])/);
1114+ my $self = shift;
1115+ return $self->_do_timeout('read', @_)
1116+ }
1117+
1118+ sub can_write {
1119+ @_ == 1 || @_ == 2 || croak(q/Usage: $handle->can_write([timeout])/);
1120+ my $self = shift;
1121+ return $self->_do_timeout('write', @_)
1122+ }
1123+} # HTTP::Micro::Handle
1124
1125 my $prog = <<'EOP';
1126 BEGIN {
1127@@ -11967,6 +12041,7 @@
1128 }
1129 }
1130 {
1131+ use Carp qw(croak);
1132 my %dispatcher = (
1133 issuer => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_issuer_name( shift )) },
1134 subject => sub { Net::SSLeay::X509_NAME_oneline( Net::SSLeay::X509_get_subject_name( shift )) },
1135@@ -12122,9 +12197,8 @@
1136 }
1137
1138 1;
1139-}
1140 # ###########################################################################
1141-# End HTTPMicro package
1142+# End HTTP::Micro package
1143 # ###########################################################################
1144
1145 # ###########################################################################
1146@@ -12158,7 +12232,7 @@
1147
1148 eval {
1149 require Percona::Toolkit;
1150- require HTTPMicro;
1151+ require HTTP::Micro;
1152 };
1153
1154 {
1155@@ -12389,7 +12463,7 @@
1156 my $url = $args{url};
1157 my $instances = $args{instances};
1158
1159- my $ua = $args{ua} || HTTPMicro->new( timeout => 3 );
1160+ my $ua = $args{ua} || HTTP::Micro->new( timeout => 3 );
1161
1162 my $response = $ua->request('GET', $url);
1163 PTDEBUG && _d('Server response:', Dumper($response));
1164@@ -13965,17 +14039,12 @@
1165 # ########################################################################
1166 # Daemonize now that everything is setup and ready to work.
1167 # ########################################################################
1168- my $daemon;
1169- if ( $o->get('daemonize') ) {
1170- $daemon = new Daemon(o=>$o);
1171- $daemon->daemonize();
1172- PTDEBUG && _d('I am a daemon now');
1173- }
1174- elsif ( $o->get('pid') ) {
1175- # We're not daemoninzing, it just handles PID stuff.
1176- $daemon = new Daemon(o=>$o);
1177- $daemon->make_PID_file();
1178- }
1179+ my $daemon = Daemon->new(
1180+ daemonize => $o->get('daemonize'),
1181+ pid_file => $o->get('pid'),
1182+ log_file => $o->get('log'),
1183+ );
1184+ $daemon->run();
1185
1186 # ########################################################################
1187 # Do the version-check
1188
1189=== modified file 'lib/HTTP/Micro.pm'
1190--- lib/HTTP/Micro.pm 2013-02-05 17:22:31 +0000
1191+++ lib/HTTP/Micro.pm 2013-08-03 19:47:33 +0000
1192@@ -708,5 +708,5 @@
1193
1194 1;
1195 # ###########################################################################
1196-# End HTTPMicro package
1197+# End HTTP::Micro package
1198 # ###########################################################################
1199
1200=== modified file 'lib/QueryParser.pm'
1201--- lib/QueryParser.pm 2013-01-03 00:19:16 +0000
1202+++ lib/QueryParser.pm 2013-08-03 19:47:33 +0000
1203@@ -98,7 +98,7 @@
1204
1205 # These keywords may appear between UPDATE or SELECT and the table refs.
1206 # They need to be removed so that they are not mistaken for tables.
1207- $query =~ s/ (?:LOW_PRIORITY|IGNORE|STRAIGHT_JOIN)//ig;
1208+ $query =~ s/(?:LOW_PRIORITY|IGNORE|STRAIGHT_JOIN|DELAYED)\s+/ /ig;
1209
1210 # Another special case: LOCK TABLES tbl [[AS] alias] READ|WRITE, etc.
1211 # We strip the LOCK TABLES stuff and append "FROM" to fake a SELECT
1212@@ -110,9 +110,21 @@
1213 $query = "FROM $query";
1214 }
1215
1216- $query =~ s/\\["']//g; # quoted strings
1217- $query =~ s/".*?"/?/sg; # quoted strings
1218- $query =~ s/'.*?'/?/sg; # quoted strings
1219+ $query =~ s/\\["']//g; # quoted strings
1220+ $query =~ s/".*?"/?/sg; # quoted strings
1221+ $query =~ s/'.*?'/?/sg; # quoted strings
1222+
1223+ # INSERT and REPLACE without INTO
1224+ # https://bugs.launchpad.net/percona-toolkit/+bug/984053
1225+ if ( $query =~ m/\A\s*(?:INSERT|REPLACE)(?!\s+INTO)/i ) {
1226+ # Add INTO so the reset of the code work as usual.
1227+ $query =~ s/\A\s*((?:INSERT|REPLACE))\s+/$1 INTO /i;
1228+ }
1229+
1230+ if ( $query =~ m/\A\s*LOAD DATA/i ) {
1231+ my ($tbl) = $query =~ m/INTO TABLE\s+(\S+)/i;
1232+ return $tbl;
1233+ }
1234
1235 my @tables;
1236 foreach my $tbls ( $query =~ m/$tbl_regex/gio ) {
1237
1238=== modified file 'lib/QueryRewriter.pm'
1239--- lib/QueryRewriter.pm 2013-06-27 18:54:53 +0000
1240+++ lib/QueryRewriter.pm 2013-08-03 19:47:33 +0000
1241@@ -246,6 +246,13 @@
1242 $query =~ m/\A\s*UNLOCK TABLES/i && return "UNLOCK";
1243 $query =~ m/\A\s*xa\s+(\S+)/i && return "XA_$1";
1244
1245+ if ( $query =~ m/\A\s*LOAD/i ) {
1246+ my ($tbl) = $query =~ m/INTO TABLE\s+(\S+)/i;
1247+ $tbl ||= '';
1248+ $tbl =~ s/`//g;
1249+ return "LOAD DATA $tbl";
1250+ }
1251+
1252 if ( $query =~ m/\Aadministrator command:/ ) {
1253 $query =~ s/administrator command:/ADMIN/;
1254 $query = uc $query;
1255@@ -386,6 +393,9 @@
1256 map { $verbs =~ s/$_/$alias_for{$_}/ } keys %alias_for;
1257 $query = $verbs;
1258 }
1259+ elsif ( $verbs && $verbs =~ m/^LOAD DATA/ ) {
1260+ return $verbs;
1261+ }
1262 else {
1263 # For everything else, distill the tables.
1264 my @tables = $self->__distill_tables($query, $table, %args);
1265
1266=== modified file 'lib/VersionCheck.pm'
1267--- lib/VersionCheck.pm 2013-06-17 06:23:11 +0000
1268+++ lib/VersionCheck.pm 2013-08-03 19:47:33 +0000
1269@@ -45,7 +45,7 @@
1270
1271 eval {
1272 require Percona::Toolkit;
1273- require HTTPMicro;
1274+ require HTTP::Micro;
1275 };
1276
1277 # Return the version check file used to keep track of
1278@@ -335,7 +335,7 @@
1279 my $instances = $args{instances};
1280
1281 # Optional args
1282- my $ua = $args{ua} || HTTPMicro->new( timeout => 3 );
1283+ my $ua = $args{ua} || HTTP::Micro->new( timeout => 3 );
1284
1285 # GET https://upgrade.percona.com, the server will return
1286 # a plaintext list of items/programs it wants the tool
1287
1288=== modified file 't/lib/QueryParser.t'
1289--- t/lib/QueryParser.t 2012-08-16 22:18:14 +0000
1290+++ t/lib/QueryParser.t 2013-08-03 19:47:33 +0000
1291@@ -828,6 +828,12 @@
1292 [qw(t1 t2)], 'get_tables works for lowercased LOCK TABLES',
1293 );
1294
1295+is_deeply(
1296+ [ $qp->get_tables("LOAD DATA INFILE '/tmp/foo.txt' INTO TABLE db.tbl") ],
1297+ [qw(db.tbl)],
1298+ "LOAD DATA db.tbl"
1299+);
1300+
1301 # #############################################################################
1302 # Done.
1303 # #############################################################################
1304
1305=== modified file 't/lib/QueryRewriter.t'
1306--- t/lib/QueryRewriter.t 2013-06-27 18:53:06 +0000
1307+++ t/lib/QueryRewriter.t 2013-08-03 19:47:33 +0000
1308@@ -1412,6 +1412,34 @@
1309 'distills SELECT with REPLACE function (issue 1176)'
1310 );
1311
1312+# LOAD DATA
1313+# https://bugs.launchpad.net/percona-toolkit/+bug/821692
1314+# INSERT and REPLACE without INTO
1315+# https://bugs.launchpad.net/percona-toolkit/+bug/984053
1316+is(
1317+ $qr->distill("LOAD DATA LOW_PRIORITY LOCAL INFILE 'file' INTO TABLE tbl"),
1318+ "LOAD DATA tbl",
1319+ "distill LOAD DATA (bug 821692)"
1320+);
1321+
1322+is(
1323+ $qr->distill("LOAD DATA LOW_PRIORITY LOCAL INFILE 'file' INTO TABLE `tbl`"),
1324+ "LOAD DATA tbl",
1325+ "distill LOAD DATA (bug 821692)"
1326+);
1327+
1328+is(
1329+ $qr->distill("insert ignore_bar (id) values (4029731)"),
1330+ "INSERT ignore_bar",
1331+ "distill INSERT without INTO (bug 984053)"
1332+);
1333+
1334+is(
1335+ $qr->distill("replace ignore_bar (id) values (4029731)"),
1336+ "REPLACE ignore_bar",
1337+ "distill REPLACE without INTO (bug 984053)"
1338+);
1339+
1340 # #############################################################################
1341 # Done.
1342 # #############################################################################
1343
1344=== modified file 't/lib/samples/slowlogs/slow051.txt'
1345--- t/lib/samples/slowlogs/slow051.txt 2011-06-24 17:22:06 +0000
1346+++ t/lib/samples/slowlogs/slow051.txt 2013-08-03 19:47:33 +0000
1347@@ -1,6 +1,6 @@
1348 # Time: 071218 11:48:27
1349 # Query_time: 0.000012 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
1350-LOAD DATA INFILE '/tmp/foo.txt' INTO db.tbl;
1351+LOAD DATA INFILE '/tmp/foo.txt' INTO TABLE db.tbl;
1352 # Time: 071218 11:48:37
1353 # Query_time: 0.000012 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
1354-LOAD DATA INFILE '/tmp/bar.txt' INTO db.tbl;
1355+LOAD DATA INFILE '/tmp/bar.txt' INTO TABLE db.tbl;
1356
1357=== added file 't/lib/samples/slowlogs/slow058.txt'
1358--- t/lib/samples/slowlogs/slow058.txt 1970-01-01 00:00:00 +0000
1359+++ t/lib/samples/slowlogs/slow058.txt 2013-08-03 19:47:33 +0000
1360@@ -0,0 +1,24 @@
1361+# User@Host: meow[meow] @ [1.2.3.8]
1362+# Thread_id: 5 Schema: db
1363+# Query_time: 0.000002 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
1364+LOAD DATA LOCAL INFILE '/tmp/foo.txt' INTO TABLE `foo`;
1365+# User@Host: meow[meow] @ [1.2.3.8]
1366+# Thread_id: 7 Schema: db
1367+# Query_time: 0.018799 Lock_time: 0.009453 Rows_sent: 0 Rows_examined: 0
1368+INSERT `foo` VALUES("bar");
1369+# User@Host: meow[meow] @ [1.2.3.8]
1370+# Thread_id: 7 Schema: db
1371+# Query_time: 0.018799 Lock_time: 0.009453 Rows_sent: 0 Rows_examined: 0
1372+REPLACE `foo` VALUES("bar");
1373+# User@Host: meow[meow] @ [1.2.3.8]
1374+# Thread_id: 5 Schema: db
1375+# Query_time: 0.000002 Lock_time: 0.000000 Rows_sent: 0 Rows_examined: 0
1376+load data local infile '/tmp/foo.txt' into table `foo`;
1377+# User@Host: meow[meow] @ [1.2.3.8]
1378+# Thread_id: 7 Schema: db
1379+# Query_time: 0.018799 Lock_time: 0.009453 Rows_sent: 0 Rows_examined: 0
1380+insert `foo` values("bar");
1381+# User@Host: meow[meow] @ [1.2.3.8]
1382+# Thread_id: 7 Schema: db
1383+# Query_time: 0.018799 Lock_time: 0.009453 Rows_sent: 0 Rows_examined: 0
1384+replace `foo` values("bar");
1385
1386=== modified file 't/pt-query-digest/binlog_analyses.t'
1387--- t/pt-query-digest/binlog_analyses.t 2012-11-21 16:58:40 +0000
1388+++ t/pt-query-digest/binlog_analyses.t 2013-08-03 19:47:33 +0000
1389@@ -28,7 +28,7 @@
1390 "t/pt-query-digest/samples/binlog001.txt"
1391 ),
1392 'Analysis for binlog001',
1393-);
1394+) or diag($test_diff);
1395
1396 ok(
1397 no_diff(
1398@@ -36,7 +36,7 @@
1399 "t/pt-query-digest/samples/binlog002.txt"
1400 ),
1401 'Analysis for binlog002',
1402-);
1403+) or diag($test_diff);
1404
1405 # #############################################################################
1406 # Done.
1407
1408=== modified file 't/pt-query-digest/daemon.t'
1409--- t/pt-query-digest/daemon.t 2012-06-03 19:14:30 +0000
1410+++ t/pt-query-digest/daemon.t 2013-08-03 19:47:33 +0000
1411@@ -32,8 +32,8 @@
1412 $output = `$trunk/bin/pt-query-digest $trunk/commont/t/samples/slow002.txt --pid $pid_file 2>&1`;
1413 like(
1414 $output,
1415- qr{PID file $pid_file already exists},
1416- 'Dies if PID file already exists (--pid without --daemonize) (issue 391)'
1417+ qr{PID file $pid_file exists},
1418+ 'Dies if PID file exists (--pid without --daemonize) (issue 391)'
1419 );
1420 `rm $pid_file >/dev/null 2>&1`;
1421
1422
1423=== modified file 't/pt-query-digest/json.t'
1424--- t/pt-query-digest/json.t 2013-07-01 20:59:12 +0000
1425+++ t/pt-query-digest/json.t 2013-08-03 19:47:33 +0000
1426@@ -25,7 +25,7 @@
1427 ok(
1428 no_diff(
1429 sub { pt_query_digest::main(@args, "$sample/slowlogs/empty") },
1430- "t/pt-query-digest/samples/empty_report.txt",
1431+ "t/pt-query-digest/samples/empty_json_report.txt",
1432 ),
1433 'json output for empty log'
1434 ) or diag($test_diff);
1435
1436=== modified file 't/pt-query-digest/samples/binlog002.txt'
1437--- t/pt-query-digest/samples/binlog002.txt 2013-01-11 16:45:20 +0000
1438+++ t/pt-query-digest/samples/binlog002.txt 2013-08-03 19:47:33 +0000
1439@@ -89,6 +89,9 @@
1440 # 100ms
1441 # 1s
1442 # 10s+
1443+# Tables
1444+# SHOW TABLE STATUS FROM `d` LIKE 'foo'\G
1445+# SHOW CREATE TABLE `d`.`foo`\G
1446 insert foo values (1) /*... omitted ...*/\G
1447
1448 # Profile
1449@@ -96,4 +99,4 @@
1450 # ==== ================== ============= ===== ====== ===== ===============
1451 # 1 0xF25D6D5AC7C18FF3 0.0000 0.0% 1 0.0000 0.00 CREATE DATABASE d
1452 # 2 0x03409022EB8A4AE7 0.0000 0.0% 1 0.0000 0.00 CREATE TABLE foo
1453-# 3 0xF579EC4A9633EEA0 0.0000 0.0% 1 0.0000 0.00 INSERT
1454+# 3 0xF579EC4A9633EEA0 0.0000 0.0% 1 0.0000 0.00 INSERT foo
1455
1456=== added file 't/pt-query-digest/samples/empty_json_report.txt'
1457=== modified file 't/pt-query-digest/samples/empty_report.txt'
1458--- t/pt-query-digest/samples/empty_report.txt 2013-07-01 20:38:34 +0000
1459+++ t/pt-query-digest/samples/empty_report.txt 2013-08-03 19:47:33 +0000
1460@@ -0,0 +1,2 @@
1461+
1462+# No events processed.
1463
1464=== modified file 't/pt-query-digest/samples/slow051.txt'
1465--- t/pt-query-digest/samples/slow051.txt 2013-01-11 16:45:20 +0000
1466+++ t/pt-query-digest/samples/slow051.txt 2013-08-03 19:47:33 +0000
1467@@ -1,5 +1,5 @@
1468
1469-# Query 1: 0.20 QPS, 0.00x concurrency, ID 0xD989521B246E945B at byte 146
1470+# Query 1: 0.20 QPS, 0.00x concurrency, ID 0x14354E1D979884B4 at byte 152
1471 # This item is included in the report because it matches --limit.
1472 # Scores: V/M = 0.00
1473 # Time range: 2007-12-18 11:48:27 to 11:48:37
1474@@ -10,7 +10,7 @@
1475 # Lock time 0 0 0 0 0 0 0 0
1476 # Rows sent 0 0 0 0 0 0 0 0
1477 # Rows examine 0 0 0 0 0 0 0 0
1478-# Query size 100 86 43 43 43 43 0 43
1479+# Query size 100 98 49 49 49 49 0 49
1480 # Query_time distribution
1481 # 1us
1482 # 10us ################################################################
1483@@ -23,9 +23,9 @@
1484 # Tables
1485 # SHOW TABLE STATUS FROM `db` LIKE 'tbl'\G
1486 # SHOW CREATE TABLE `db`.`tbl`\G
1487-LOAD DATA INFILE '/tmp/bar.txt' INTO db.tbl\G
1488+LOAD DATA INFILE '/tmp/bar.txt' INTO TABLE db.tbl\G
1489
1490 # Profile
1491 # Rank Query ID Response time Calls R/Call V/M Item
1492-# ==== ================== ============= ===== ====== ===== ======
1493-# 1 0xD989521B246E945B 0.0000 100.0% 2 0.0000 0.00 db.tbl
1494+# ==== ================== ============= ===== ====== ===== ===============
1495+# 1 0x14354E1D979884B4 0.0000 100.0% 2 0.0000 0.00 LOAD DATA db.tbl
1496
1497=== added file 't/pt-query-digest/samples/slow058.txt'
1498--- t/pt-query-digest/samples/slow058.txt 1970-01-01 00:00:00 +0000
1499+++ t/pt-query-digest/samples/slow058.txt 2013-08-03 19:47:33 +0000
1500@@ -0,0 +1,94 @@
1501+
1502+# Query 1: 0 QPS, 0x concurrency, ID 0x471A0C4BD7A4EE34 at byte 730 ______
1503+# This item is included in the report because it matches --limit.
1504+# Scores: V/M = 0.00
1505+# Attribute pct total min max avg 95% stddev median
1506+# ============ === ======= ======= ======= ======= ======= ======= =======
1507+# Count 33 2
1508+# Exec time 49 38ms 19ms 19ms 19ms 19ms 0 19ms
1509+# Lock time 50 19ms 9ms 9ms 9ms 9ms 0 9ms
1510+# Rows sent 0 0 0 0 0 0 0 0
1511+# Rows examine 0 0 0 0 0 0 0 0
1512+# Query size 24 52 26 26 26 26 0 26
1513+# String:
1514+# Databases db
1515+# Hosts
1516+# Users meow
1517+# Query_time distribution
1518+# 1us
1519+# 10us
1520+# 100us
1521+# 1ms
1522+# 10ms ################################################################
1523+# 100ms
1524+# 1s
1525+# 10s+
1526+# Tables
1527+# SHOW TABLE STATUS FROM `db` LIKE 'foo'\G
1528+# SHOW CREATE TABLE `db`.`foo`\G
1529+insert `foo` values("bar")\G
1530+
1531+# Query 2: 0 QPS, 0x concurrency, ID 0xF33473286088142B at byte 898 ______
1532+# This item is included in the report because it matches --limit.
1533+# Scores: V/M = 0.00
1534+# Attribute pct total min max avg 95% stddev median
1535+# ============ === ======= ======= ======= ======= ======= ======= =======
1536+# Count 33 2
1537+# Exec time 49 38ms 19ms 19ms 19ms 19ms 0 19ms
1538+# Lock time 50 19ms 9ms 9ms 9ms 9ms 0 9ms
1539+# Rows sent 0 0 0 0 0 0 0 0
1540+# Rows examine 0 0 0 0 0 0 0 0
1541+# Query size 25 54 27 27 27 27 0 27
1542+# String:
1543+# Databases db
1544+# Hosts
1545+# Users meow
1546+# Query_time distribution
1547+# 1us
1548+# 10us
1549+# 100us
1550+# 1ms
1551+# 10ms ################################################################
1552+# 100ms
1553+# 1s
1554+# 10s+
1555+# Tables
1556+# SHOW TABLE STATUS FROM `db` LIKE 'foo'\G
1557+# SHOW CREATE TABLE `db`.`foo`\G
1558+replace `foo` values("bar")\G
1559+
1560+# Query 3: 0 QPS, 0x concurrency, ID 0xEBAC9C76529E62CE at byte 534 ______
1561+# This item is included in the report because it matches --limit.
1562+# Scores: V/M = 0.00
1563+# Attribute pct total min max avg 95% stddev median
1564+# ============ === ======= ======= ======= ======= ======= ======= =======
1565+# Count 33 2
1566+# Exec time 0 4us 2us 2us 2us 2us 0 2us
1567+# Lock time 0 0 0 0 0 0 0 0
1568+# Rows sent 0 0 0 0 0 0 0 0
1569+# Rows examine 0 0 0 0 0 0 0 0
1570+# Query size 50 108 54 54 54 54 0 54
1571+# String:
1572+# Databases db
1573+# Hosts
1574+# Users meow
1575+# Query_time distribution
1576+# 1us ################################################################
1577+# 10us
1578+# 100us
1579+# 1ms
1580+# 10ms
1581+# 100ms
1582+# 1s
1583+# 10s+
1584+# Tables
1585+# SHOW TABLE STATUS FROM `db` LIKE 'foo'\G
1586+# SHOW CREATE TABLE `db`.`foo`\G
1587+load data local infile '/tmp/foo.txt' into table `foo`\G
1588+
1589+# Profile
1590+# Rank Query ID Response time Calls R/Call V/M Item
1591+# ==== ================== ============= ===== ====== ===== =============
1592+# 1 0x471A0C4BD7A4EE34 0.0376 50.0% 2 0.0188 0.00 INSERT foo
1593+# 2 0xF33473286088142B 0.0376 50.0% 2 0.0188 0.00 REPLACE foo
1594+# 3 0xEBAC9C76529E62CE 0.0000 0.0% 2 0.0000 0.00 LOAD DATA foo
1595
1596=== modified file 't/pt-query-digest/slowlog_analyses.t'
1597--- t/pt-query-digest/slowlog_analyses.t 2013-06-26 23:16:15 +0000
1598+++ t/pt-query-digest/slowlog_analyses.t 2013-08-03 19:47:33 +0000
1599@@ -343,7 +343,7 @@
1600 "t/pt-query-digest/samples/slow051.txt",
1601 ),
1602 'Analysis for slow051 (issue 918)',
1603-);
1604+) or diag($test_diff);
1605
1606 # #############################################################################
1607 # Issue 1124: Make mk-query-digest profile include variance-to-mean ratio
1608@@ -394,9 +394,10 @@
1609 );
1610
1611 # #############################################################################
1612-# Bug 1176010: pt-query-digest should know how to group quoted and unquoted database names
1613+# Bug 1176010: pt-query-digest should know how to group quoted and unquoted
1614+# database names
1615 # https://bugs.launchpad.net/percona-toolkit/+bug/1176010
1616-#############################################################################
1617+# #############################################################################
1618 ok(
1619 no_diff(
1620 sub { pt_query_digest::main(@args, $sample.'slow057.txt',
1621@@ -407,6 +408,22 @@
1622 ) or diag($test_diff);
1623
1624 # #############################################################################
1625+# https://bugs.launchpad.net/percona-toolkit/+bug/821692
1626+# pt-query-digest doesn't distill LOAD DATA correctly
1627+# https://bugs.launchpad.net/percona-toolkit/+bug/984053
1628+# pt-query-digest doesn't distill INSERT/REPLACE without INTO correctly
1629+# #############################################################################
1630+ok(
1631+ no_diff(
1632+ sub { pt_query_digest::main($sample.'slow058.txt',
1633+ '--report-format', 'query_report,profile', '--limit', '100%',
1634+ )},
1635+ "t/pt-query-digest/samples/slow058.txt",
1636+ ),
1637+ 'Analysis for slow058 (bug 821692, bug 984053)'
1638+) or diag($test_diff);
1639+
1640+# #############################################################################
1641 # Done.
1642 # #############################################################################
1643 done_testing;
1644
1645=== modified file 'util/update-modules'
1646--- util/update-modules 2013-01-17 22:09:52 +0000
1647+++ util/update-modules 2013-08-03 19:47:33 +0000
1648@@ -91,7 +91,7 @@
1649
1650 $BRANCH/util/extract-package $pkg $pkg_file | grep -v '^ *#' >> $tmp_file
1651
1652- if [ "$tool_lang" = "perl" ]; then
1653+ if [ "$tool_lang" = "perl" -a $pkg != "HTTP::Micro" ]; then
1654 echo "}" >> $tmp_file
1655 fi
1656

Subscribers

People subscribed via source and target branches