Merge lp:~percona-toolkit-dev/percona-toolkit/raw-log-parser-percona-22371 into lp:percona-toolkit/2.1

[Brian Fraser]

[Daniel Nichter (daniel-nichter)]

Create a quick blueprint and target to 2.1.6.

[Daniel Nichter (daniel-nichter)]

review: Approve

[Brian Fraser (fraserbn)]

review: Approve

=== modified file 'bin/pt-query-digest'
--- bin/pt-query-digest	2012-09-24 19:24:36 +0000
+++ bin/pt-query-digest	2012-10-01 14:30:45 +0000
@@ -9757,6 +9757,86 @@
 # ###########################################################################
 
 # ###########################################################################
+# RawLogParser package
+# This package is a copy without comments from the original.  The original
+# with comments and its test file can be found in the Bazaar repository at,
+#   lib/RawLogParser.pm
+#   t/lib/RawLogParser.t
+# See https://launchpad.net/percona-toolkit for more information.
+# ###########################################################################
+{
+package RawLogParser;
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use constant PTDEBUG => $ENV{PTDEBUG} || 0;
+
+use Data::Dumper;
+$Data::Dumper::Indent    = 1;
+$Data::Dumper::Sortkeys  = 1;
+$Data::Dumper::Quotekeys = 0;
+
+sub new {
+   my ( $class ) = @_;
+   my $self = {
+   };
+   return bless $self, $class;
+}
+
+sub parse_event {
+   my ( $self, %args ) = @_;
+   my @required_args = qw(next_event tell);
+   foreach my $arg ( @required_args ) {
+      die "I need a $arg argument" unless $args{$arg};
+   }
+   my ($next_event, $tell) = @args{@required_args};
+
+   my $line;
+   my $pos_in_log = $tell->();
+   LINE:
+   while ( defined($line = $next_event->()) ) {
+      PTDEBUG && _d($line);
+      chomp($line);
+      my @properties = (
+         'pos_in_log', $pos_in_log,
+         'cmd',        'Query',
+         'bytes',      length($line),
+         'Query_time', 0,
+         'arg',        $line,
+      );
+
+      $pos_in_log = $tell->();
+
+      PTDEBUG && _d('Properties of event:', Dumper(\@properties));
+      my $event = { @properties };
+      if ( $args{stats} ) {
+         $args{stats}->{events_read}++;
+         $args{stats}->{events_parsed}++;
+      }
+
+      return $event;
+   }
+
+   $args{oktorun}->(0) if $args{oktorun};
+   return;
+}
+
+sub _d {
+   my ($package, undef, $line) = caller 0;
+   @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
+        map { defined $_ ? $_ : 'undef' }
+        @_;
+   print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
+}
+
+1;
+}
+# ###########################################################################
+# End RawLogParser package
+# ###########################################################################
+
+# ###########################################################################
 # ProtocolParser package
 # This package is a copy without comments from the original.  The original
 # with comments and its test file can be found in the Bazaar repository at,
@@ -13227,7 +13307,7 @@
    my $review_dsn = $o->get('review'); 
    my @groupby    = @{$o->get('group-by')};
    my @orderby;
-   if ( (grep { $_ eq 'genlog' || $_ eq 'GeneralLogParser' } @{$o->get('type')})
+   if ( (grep { $_ =~ m/genlog|GeneralLogParser|rawlog|RawLogParser/ } @{$o->get('type')})
         && !$o->got('order-by') ) {
       @orderby = 'Query_time:cnt';
    }
@@ -13668,6 +13748,7 @@
                           'MemcachedEvent'],
             http      => ['TcpdumpParser','HTTPProtocolParser'],
             pglog     => ['PgLogParser'],
+            rawlog    => ['RawLogParser'],
          );
          my $type = $o->get('type');
          $type    = $alias_for{$type->[0]} if $alias_for{$type->[0]};

=== added file 'lib/RawLogParser.pm'
--- lib/RawLogParser.pm	1970-01-01 00:00:00 +0000
+++ lib/RawLogParser.pm	2012-10-01 14:30:45 +0000
@@ -0,0 +1,95 @@
+# This program is copyright 2012 Percona Inc.
+# Feedback and improvements are welcome.
+#
+# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
+# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation, version 2; OR the Perl Artistic License.  On UNIX and similar
+# systems, you can issue `man perlgpl' or `man perlartistic' to read these
+# licenses.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+# Place, Suite 330, Boston, MA  02111-1307  USA.
+# ###########################################################################
+# RawLogParser package
+# ###########################################################################
+{
+# Package: RawLogParser
+# RawLogParser parses logs with nothing but one query per line.
+package RawLogParser;
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use constant PTDEBUG => $ENV{PTDEBUG} || 0;
+
+use Data::Dumper;
+$Data::Dumper::Indent    = 1;
+$Data::Dumper::Sortkeys  = 1;
+$Data::Dumper::Quotekeys = 0;
+
+sub new {
+   my ( $class ) = @_;
+   my $self = {
+   };
+   return bless $self, $class;
+}
+
+sub parse_event {
+   my ( $self, %args ) = @_;
+   my @required_args = qw(next_event tell);
+   foreach my $arg ( @required_args ) {
+      die "I need a $arg argument" unless $args{$arg};
+   }
+   my ($next_event, $tell) = @args{@required_args};
+
+   my $line;
+   my $pos_in_log = $tell->();
+   LINE:
+   while ( defined($line = $next_event->()) ) {
+      PTDEBUG && _d($line);
+      chomp($line);
+      my @properties = (
+         'pos_in_log', $pos_in_log,
+         'cmd',        'Query',
+         'bytes',      length($line),
+         'Query_time', 0,
+         'arg',        $line,
+      );
+
+      $pos_in_log = $tell->();
+
+      # Don't dump $event; want to see full dump of all properties,
+      # and after it's been cast into a hash, duplicated keys will
+      # be gone.
+      PTDEBUG && _d('Properties of event:', Dumper(\@properties));
+      my $event = { @properties };
+      if ( $args{stats} ) {
+         $args{stats}->{events_read}++;
+         $args{stats}->{events_parsed}++;
+      }
+
+      return $event;
+   }
+
+   $args{oktorun}->(0) if $args{oktorun};
+   return;
+}
+
+sub _d {
+   my ($package, undef, $line) = caller 0;
+   @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
+        map { defined $_ ? $_ : 'undef' }
+        @_;
+   print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
+}
+
+1;
+}
+# ###########################################################################
+# End RawLogParser package
+# ###########################################################################

=== added file 't/lib/RawLogParser.t'
--- t/lib/RawLogParser.t	1970-01-01 00:00:00 +0000
+++ t/lib/RawLogParser.t	2012-10-01 14:30:45 +0000
@@ -0,0 +1,52 @@
+#!/usr/bin/perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More tests => 3;
+
+use RawLogParser;
+use PerconaTest;
+
+my $p = new RawLogParser();
+
+my $oktorun = 1;
+my $sample  = "t/lib/samples/rawlogs/";
+
+test_log_parser(
+   parser  => $p,
+   file    => $sample.'rawlog001.txt',
+   oktorun => sub { $oktorun = $_[0]; },
+   result  => [
+      {  pos_in_log => 0,
+         args       => 'SELECT c FROM t WHERE id=1',
+         bytes      => 26,
+         cmd        => 'Query',
+         Query_time => 0,
+      },
+      {  pos_in_log => 27,
+         args       => '/* Hello, world! */ SELECT * FROM t2 LIMIT 1',
+         bytes      => 44,
+         cmd        => 'Query',
+         Query_time => 0,
+      }
+   ]
+);
+
+is(
+   $oktorun,
+   0,
+   'Sets oktorun'
+);
+$oktorun = 1;
+
+# #############################################################################
+# Done.
+# #############################################################################
+exit;

=== added directory 't/lib/samples/rawlogs'
=== added file 't/lib/samples/rawlogs/rawlog001.txt'
--- t/lib/samples/rawlogs/rawlog001.txt	1970-01-01 00:00:00 +0000
+++ t/lib/samples/rawlogs/rawlog001.txt	2012-10-01 14:30:45 +0000
@@ -0,0 +1,2 @@
+SELECT c FROM t WHERE id=1
+/* Hello, world! */ SELECT * FROM t2 LIMIT 1

=== added file 't/pt-query-digest/rawlog_analyses.t'
--- t/pt-query-digest/rawlog_analyses.t	1970-01-01 00:00:00 +0000
+++ t/pt-query-digest/rawlog_analyses.t	2012-10-01 14:30:45 +0000
@@ -0,0 +1,48 @@
+#!/usr/bin/env perl
+
+BEGIN {
+   die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
+      unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
+   unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
+};
+
+use strict;
+use warnings FATAL => 'all';
+use English qw(-no_match_vars);
+use Test::More tests => 2;
+
+use PerconaTest;
+
+# See 101_slowlog_analyses.t or http://code.google.com/p/maatkit/wiki/Testing
+shift @INC;  # our unshift (above)
+shift @INC;  # PerconaTest's unshift
+
+require "$trunk/bin/pt-query-digest";
+
+# #############################################################################
+# Issue 172: Make mk-query-digest able to read raweral logs
+# #############################################################################
+
+my @args   = ('--report-format', 'header,query_report,profile', '--type', 'rawlog');
+my $sample = "$trunk/t/lib/samples/rawlogs/";
+
+# --help exists so don't run mqd as a module else --help's exit will
+# exit this test script.
+like(
+   `$trunk/bin/pt-query-digest --type rawlog rawlog001.txt --help`,
+   qr/--order-by\s+Query_time:cnt/,
+   '--order-by defaults to Query_time:cnt for --type rawlog',
+);
+
+ok(
+   no_diff(
+      sub { pt_query_digest::main(@args, $sample.'rawlog001.txt') },
+      "t/pt-query-digest/samples/rawlog001.txt"
+   ),
+   'Analysis for rawlog001',
+);
+
+# #############################################################################
+# Done.
+# #############################################################################
+exit;

=== added file 't/pt-query-digest/samples/rawlog001.txt'
--- t/pt-query-digest/samples/rawlog001.txt	1970-01-01 00:00:00 +0000
+++ t/pt-query-digest/samples/rawlog001.txt	2012-10-01 14:30:45 +0000
@@ -0,0 +1,60 @@
+
+# Overall: 2 total, 2 unique, 0 QPS, 0x concurrency ______________________
+# Attribute          total     min     max     avg     95%  stddev  median
+# ============     ======= ======= ======= ======= ======= ======= =======
+# Exec time              0       0       0       0       0       0       0
+# Query size            70      26      44      35      44   12.73      35
+
+# Query 1: 0 QPS, 0x concurrency, ID 0xCB5621E548E5497F at byte 0 ________
+# This item is included in the report because it matches --limit.
+# Scores: Apdex = 1.00 [1.0]*, V/M = 0.00
+# Query_time sparkline: |        |
+# Attribute    pct   total     min     max     avg     95%  stddev  median
+# ============ === ======= ======= ======= ======= ======= ======= =======
+# Count         50       1
+# Exec time      0       0       0       0       0       0       0       0
+# Query size    37      26      26      26      26      26       0      26
+# Query_time distribution
+#   1us
+#  10us
+# 100us
+#   1ms
+#  10ms
+# 100ms
+#    1s
+#  10s+
+# Tables
+#    SHOW TABLE STATUS LIKE 't'\G
+#    SHOW CREATE TABLE `t`\G
+# EXPLAIN /*!50100 PARTITIONS*/
+SELECT c FROM t WHERE id=1\G
+
+# Query 2: 0 QPS, 0x concurrency, ID 0x774B2B0B59EBAC2C at byte 27 _______
+# This item is included in the report because it matches --limit.
+# Scores: Apdex = 1.00 [1.0]*, V/M = 0.00
+# Query_time sparkline: |        |
+# Attribute    pct   total     min     max     avg     95%  stddev  median
+# ============ === ======= ======= ======= ======= ======= ======= =======
+# Count         50       1
+# Exec time      0       0       0       0       0       0       0       0
+# Query size    62      44      44      44      44      44       0      44
+# Query_time distribution
+#   1us
+#  10us
+# 100us
+#   1ms
+#  10ms
+# 100ms
+#    1s
+#  10s+
+# Tables
+#    SHOW TABLE STATUS LIKE 't2'\G
+#    SHOW CREATE TABLE `t2`\G
+# EXPLAIN /*!50100 PARTITIONS*/
+/* Hello, world! */ SELECT * FROM t2 LIMIT 1\G
+
+# Profile
+# Rank Query ID           Response time Calls R/Call Apdx V/M   Item
+# ==== ================== ============= ===== ====== ==== ===== =========
+#    1 0xCB5621E548E5497F  0.0000  0.0%     1 0.0000 1.00  0.00 SELECT t
+#    2 0x774B2B0B59EBAC2C  0.0000  0.0%     1 0.0000 1.00  0.00 SELECT t?