Percona Toolkit moved to https://jira.percona.com/projects/PT

Merge lp:~percona-toolkit-dev/percona-toolkit/raw-log-parser-percona-22371 into lp:percona-toolkit/2.1

  1. raw-log-parser-percona-22371
  2. Merge into 2.1
Proposed by Brian Fraser
Status: Merged
Approved by: Daniel Nichter
Approved revision: 303
Merged at revision: 409
Proposed branch: lp:~percona-toolkit-dev/percona-toolkit/raw-log-parser-percona-22371
Merge into: lp:percona-toolkit/2.1
Diff against target: 390 lines (+339/-1)
6 files modified
bin/pt-query-digest (+82/-1)
lib/RawLogParser.pm (+95/-0)
t/lib/RawLogParser.t (+52/-0)
t/lib/samples/rawlogs/rawlog001.txt (+2/-0)
t/pt-query-digest/rawlog_analyses.t (+48/-0)
t/pt-query-digest/samples/rawlog001.txt (+60/-0)
To merge this branch: bzr merge lp:~percona-toolkit-dev/percona-toolkit/raw-log-parser-percona-22371
Reviewer Review Type Date Requested Status
Brian Fraser (community) Approve
Daniel Nichter Approve
Review via email: mp+127295@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Daniel Nichter (daniel-nichter) wrote :

Create a quick blueprint and target to 2.1.6.

Revision history for this message
Daniel Nichter (daniel-nichter) :
review: Approve
Revision history for this message
Brian Fraser (fraserbn) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'bin/pt-query-digest'
--- bin/pt-query-digest 2012-09-24 19:24:36 +0000
+++ bin/pt-query-digest 2012-10-01 14:30:45 +0000
@@ -9757,6 +9757,86 @@
9757# ###########################################################################9757# ###########################################################################
97589758
9759# ###########################################################################9759# ###########################################################################
9760# RawLogParser package
9761# This package is a copy without comments from the original. The original
9762# with comments and its test file can be found in the Bazaar repository at,
9763# lib/RawLogParser.pm
9764# t/lib/RawLogParser.t
9765# See https://launchpad.net/percona-toolkit for more information.
9766# ###########################################################################
9767{
9768package RawLogParser;
9769
9770use strict;
9771use warnings FATAL => 'all';
9772use English qw(-no_match_vars);
9773use constant PTDEBUG => $ENV{PTDEBUG} || 0;
9774
9775use Data::Dumper;
9776$Data::Dumper::Indent = 1;
9777$Data::Dumper::Sortkeys = 1;
9778$Data::Dumper::Quotekeys = 0;
9779
9780sub new {
9781 my ( $class ) = @_;
9782 my $self = {
9783 };
9784 return bless $self, $class;
9785}
9786
9787sub parse_event {
9788 my ( $self, %args ) = @_;
9789 my @required_args = qw(next_event tell);
9790 foreach my $arg ( @required_args ) {
9791 die "I need a $arg argument" unless $args{$arg};
9792 }
9793 my ($next_event, $tell) = @args{@required_args};
9794
9795 my $line;
9796 my $pos_in_log = $tell->();
9797 LINE:
9798 while ( defined($line = $next_event->()) ) {
9799 PTDEBUG && _d($line);
9800 chomp($line);
9801 my @properties = (
9802 'pos_in_log', $pos_in_log,
9803 'cmd', 'Query',
9804 'bytes', length($line),
9805 'Query_time', 0,
9806 'arg', $line,
9807 );
9808
9809 $pos_in_log = $tell->();
9810
9811 PTDEBUG && _d('Properties of event:', Dumper(\@properties));
9812 my $event = { @properties };
9813 if ( $args{stats} ) {
9814 $args{stats}->{events_read}++;
9815 $args{stats}->{events_parsed}++;
9816 }
9817
9818 return $event;
9819 }
9820
9821 $args{oktorun}->(0) if $args{oktorun};
9822 return;
9823}
9824
9825sub _d {
9826 my ($package, undef, $line) = caller 0;
9827 @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
9828 map { defined $_ ? $_ : 'undef' }
9829 @_;
9830 print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
9831}
9832
98331;
9834}
9835# ###########################################################################
9836# End RawLogParser package
9837# ###########################################################################
9838
9839# ###########################################################################
9760# ProtocolParser package9840# ProtocolParser package
9761# This package is a copy without comments from the original. The original9841# This package is a copy without comments from the original. The original
9762# with comments and its test file can be found in the Bazaar repository at,9842# with comments and its test file can be found in the Bazaar repository at,
@@ -13227,7 +13307,7 @@
13227 my $review_dsn = $o->get('review'); 13307 my $review_dsn = $o->get('review');
13228 my @groupby = @{$o->get('group-by')};13308 my @groupby = @{$o->get('group-by')};
13229 my @orderby;13309 my @orderby;
13230 if ( (grep { $_ eq 'genlog' || $_ eq 'GeneralLogParser' } @{$o->get('type')})13310 if ( (grep { $_ =~ m/genlog|GeneralLogParser|rawlog|RawLogParser/ } @{$o->get('type')})
13231 && !$o->got('order-by') ) {13311 && !$o->got('order-by') ) {
13232 @orderby = 'Query_time:cnt';13312 @orderby = 'Query_time:cnt';
13233 }13313 }
@@ -13668,6 +13748,7 @@
13668 'MemcachedEvent'],13748 'MemcachedEvent'],
13669 http => ['TcpdumpParser','HTTPProtocolParser'],13749 http => ['TcpdumpParser','HTTPProtocolParser'],
13670 pglog => ['PgLogParser'],13750 pglog => ['PgLogParser'],
13751 rawlog => ['RawLogParser'],
13671 );13752 );
13672 my $type = $o->get('type');13753 my $type = $o->get('type');
13673 $type = $alias_for{$type->[0]} if $alias_for{$type->[0]};13754 $type = $alias_for{$type->[0]} if $alias_for{$type->[0]};
1367413755
=== added file 'lib/RawLogParser.pm'
--- lib/RawLogParser.pm 1970-01-01 00:00:00 +0000
+++ lib/RawLogParser.pm 2012-10-01 14:30:45 +0000
@@ -0,0 +1,95 @@
1# This program is copyright 2012 Percona Inc.
2# Feedback and improvements are welcome.
3#
4# THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
5# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
6# MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
7#
8# This program is free software; you can redistribute it and/or modify it under
9# the terms of the GNU General Public License as published by the Free Software
10# Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
11# systems, you can issue `man perlgpl' or `man perlartistic' to read these
12# licenses.
13#
14# You should have received a copy of the GNU General Public License along with
15# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
16# Place, Suite 330, Boston, MA 02111-1307 USA.
17# ###########################################################################
18# RawLogParser package
19# ###########################################################################
20{
21# Package: RawLogParser
22# RawLogParser parses logs with nothing but one query per line.
23package RawLogParser;
24
25use strict;
26use warnings FATAL => 'all';
27use English qw(-no_match_vars);
28use constant PTDEBUG => $ENV{PTDEBUG} || 0;
29
30use Data::Dumper;
31$Data::Dumper::Indent = 1;
32$Data::Dumper::Sortkeys = 1;
33$Data::Dumper::Quotekeys = 0;
34
35sub new {
36 my ( $class ) = @_;
37 my $self = {
38 };
39 return bless $self, $class;
40}
41
42sub parse_event {
43 my ( $self, %args ) = @_;
44 my @required_args = qw(next_event tell);
45 foreach my $arg ( @required_args ) {
46 die "I need a $arg argument" unless $args{$arg};
47 }
48 my ($next_event, $tell) = @args{@required_args};
49
50 my $line;
51 my $pos_in_log = $tell->();
52 LINE:
53 while ( defined($line = $next_event->()) ) {
54 PTDEBUG && _d($line);
55 chomp($line);
56 my @properties = (
57 'pos_in_log', $pos_in_log,
58 'cmd', 'Query',
59 'bytes', length($line),
60 'Query_time', 0,
61 'arg', $line,
62 );
63
64 $pos_in_log = $tell->();
65
66 # Don't dump $event; want to see full dump of all properties,
67 # and after it's been cast into a hash, duplicated keys will
68 # be gone.
69 PTDEBUG && _d('Properties of event:', Dumper(\@properties));
70 my $event = { @properties };
71 if ( $args{stats} ) {
72 $args{stats}->{events_read}++;
73 $args{stats}->{events_parsed}++;
74 }
75
76 return $event;
77 }
78
79 $args{oktorun}->(0) if $args{oktorun};
80 return;
81}
82
83sub _d {
84 my ($package, undef, $line) = caller 0;
85 @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
86 map { defined $_ ? $_ : 'undef' }
87 @_;
88 print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
89}
90
911;
92}
93# ###########################################################################
94# End RawLogParser package
95# ###########################################################################
096
=== added file 't/lib/RawLogParser.t'
--- t/lib/RawLogParser.t 1970-01-01 00:00:00 +0000
+++ t/lib/RawLogParser.t 2012-10-01 14:30:45 +0000
@@ -0,0 +1,52 @@
1#!/usr/bin/perl
2
3BEGIN {
4 die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
5 unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
6 unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
7};
8
9use strict;
10use warnings FATAL => 'all';
11use English qw(-no_match_vars);
12use Test::More tests => 3;
13
14use RawLogParser;
15use PerconaTest;
16
17my $p = new RawLogParser();
18
19my $oktorun = 1;
20my $sample = "t/lib/samples/rawlogs/";
21
22test_log_parser(
23 parser => $p,
24 file => $sample.'rawlog001.txt',
25 oktorun => sub { $oktorun = $_[0]; },
26 result => [
27 { pos_in_log => 0,
28 args => 'SELECT c FROM t WHERE id=1',
29 bytes => 26,
30 cmd => 'Query',
31 Query_time => 0,
32 },
33 { pos_in_log => 27,
34 args => '/* Hello, world! */ SELECT * FROM t2 LIMIT 1',
35 bytes => 44,
36 cmd => 'Query',
37 Query_time => 0,
38 }
39 ]
40);
41
42is(
43 $oktorun,
44 0,
45 'Sets oktorun'
46);
47$oktorun = 1;
48
49# #############################################################################
50# Done.
51# #############################################################################
52exit;
053
=== added directory 't/lib/samples/rawlogs'
=== added file 't/lib/samples/rawlogs/rawlog001.txt'
--- t/lib/samples/rawlogs/rawlog001.txt 1970-01-01 00:00:00 +0000
+++ t/lib/samples/rawlogs/rawlog001.txt 2012-10-01 14:30:45 +0000
@@ -0,0 +1,2 @@
1SELECT c FROM t WHERE id=1
2/* Hello, world! */ SELECT * FROM t2 LIMIT 1
03
=== added file 't/pt-query-digest/rawlog_analyses.t'
--- t/pt-query-digest/rawlog_analyses.t 1970-01-01 00:00:00 +0000
+++ t/pt-query-digest/rawlog_analyses.t 2012-10-01 14:30:45 +0000
@@ -0,0 +1,48 @@
1#!/usr/bin/env perl
2
3BEGIN {
4 die "The PERCONA_TOOLKIT_BRANCH environment variable is not set.\n"
5 unless $ENV{PERCONA_TOOLKIT_BRANCH} && -d $ENV{PERCONA_TOOLKIT_BRANCH};
6 unshift @INC, "$ENV{PERCONA_TOOLKIT_BRANCH}/lib";
7};
8
9use strict;
10use warnings FATAL => 'all';
11use English qw(-no_match_vars);
12use Test::More tests => 2;
13
14use PerconaTest;
15
16# See 101_slowlog_analyses.t or http://code.google.com/p/maatkit/wiki/Testing
17shift @INC; # our unshift (above)
18shift @INC; # PerconaTest's unshift
19
20require "$trunk/bin/pt-query-digest";
21
22# #############################################################################
23# Issue 172: Make mk-query-digest able to read raweral logs
24# #############################################################################
25
26my @args = ('--report-format', 'header,query_report,profile', '--type', 'rawlog');
27my $sample = "$trunk/t/lib/samples/rawlogs/";
28
29# --help exists so don't run mqd as a module else --help's exit will
30# exit this test script.
31like(
32 `$trunk/bin/pt-query-digest --type rawlog rawlog001.txt --help`,
33 qr/--order-by\s+Query_time:cnt/,
34 '--order-by defaults to Query_time:cnt for --type rawlog',
35);
36
37ok(
38 no_diff(
39 sub { pt_query_digest::main(@args, $sample.'rawlog001.txt') },
40 "t/pt-query-digest/samples/rawlog001.txt"
41 ),
42 'Analysis for rawlog001',
43);
44
45# #############################################################################
46# Done.
47# #############################################################################
48exit;
049
=== added file 't/pt-query-digest/samples/rawlog001.txt'
--- t/pt-query-digest/samples/rawlog001.txt 1970-01-01 00:00:00 +0000
+++ t/pt-query-digest/samples/rawlog001.txt 2012-10-01 14:30:45 +0000
@@ -0,0 +1,60 @@
1
2# Overall: 2 total, 2 unique, 0 QPS, 0x concurrency ______________________
3# Attribute total min max avg 95% stddev median
4# ============ ======= ======= ======= ======= ======= ======= =======
5# Exec time 0 0 0 0 0 0 0
6# Query size 70 26 44 35 44 12.73 35
7
8# Query 1: 0 QPS, 0x concurrency, ID 0xCB5621E548E5497F at byte 0 ________
9# This item is included in the report because it matches --limit.
10# Scores: Apdex = 1.00 [1.0]*, V/M = 0.00
11# Query_time sparkline: | |
12# Attribute pct total min max avg 95% stddev median
13# ============ === ======= ======= ======= ======= ======= ======= =======
14# Count 50 1
15# Exec time 0 0 0 0 0 0 0 0
16# Query size 37 26 26 26 26 26 0 26
17# Query_time distribution
18# 1us
19# 10us
20# 100us
21# 1ms
22# 10ms
23# 100ms
24# 1s
25# 10s+
26# Tables
27# SHOW TABLE STATUS LIKE 't'\G
28# SHOW CREATE TABLE `t`\G
29# EXPLAIN /*!50100 PARTITIONS*/
30SELECT c FROM t WHERE id=1\G
31
32# Query 2: 0 QPS, 0x concurrency, ID 0x774B2B0B59EBAC2C at byte 27 _______
33# This item is included in the report because it matches --limit.
34# Scores: Apdex = 1.00 [1.0]*, V/M = 0.00
35# Query_time sparkline: | |
36# Attribute pct total min max avg 95% stddev median
37# ============ === ======= ======= ======= ======= ======= ======= =======
38# Count 50 1
39# Exec time 0 0 0 0 0 0 0 0
40# Query size 62 44 44 44 44 44 0 44
41# Query_time distribution
42# 1us
43# 10us
44# 100us
45# 1ms
46# 10ms
47# 100ms
48# 1s
49# 10s+
50# Tables
51# SHOW TABLE STATUS LIKE 't2'\G
52# SHOW CREATE TABLE `t2`\G
53# EXPLAIN /*!50100 PARTITIONS*/
54/* Hello, world! */ SELECT * FROM t2 LIMIT 1\G
55
56# Profile
57# Rank Query ID Response time Calls R/Call Apdx V/M Item
58# ==== ================== ============= ===== ====== ==== ===== =========
59# 1 0xCB5621E548E5497F 0.0000 0.0% 1 0.0000 1.00 0.00 SELECT t
60# 2 0x774B2B0B59EBAC2C 0.0000 0.0% 1 0.0000 1.00 0.00 SELECT t?

Subscribers

People subscribed via source and target branches