Percona Toolkit moved to https://jira.percona.com/projects/PT

Merge lp:~percona-toolkit-dev/percona-toolkit/pqd-json-anon into lp:~percona-toolkit-dev/percona-toolkit/release-2.2.4

  1. pqd-json-anon
  2. Merge into release-2.2.4
Proposed by Daniel Nichter
Status: Merged
Merged at revision: 600
Proposed branch: lp:~percona-toolkit-dev/percona-toolkit/pqd-json-anon
Merge into: lp:~percona-toolkit-dev/percona-toolkit/release-2.2.4
Diff against target: 504 lines (+328/-51)
7 files modified
bin/pt-query-digest (+27/-21)
lib/JSONReportFormatter.pm (+26/-22)
t/pt-query-digest/json.t (+16/-4)
t/pt-query-digest/samples/empty_report.txt (+0/-2)
t/pt-query-digest/samples/json/slow002-anon.txt (+257/-0)
t/pt-query-digest/samples/json/slow002.txt (+1/-1)
t/pt-query-digest/samples/json/tcpdump021.txt (+1/-1)
To merge this branch: bzr merge lp:~percona-toolkit-dev/percona-toolkit/pqd-json-anon
Reviewer Review Type Date Requested Status
Daniel Nichter Approve
Review via email: mp+172417@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Daniel Nichter (daniel-nichter) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'bin/pt-query-digest'
2--- bin/pt-query-digest 2013-06-27 19:06:24 +0000
3+++ bin/pt-query-digest 2013-07-01 21:05:30 +0000
4@@ -7900,10 +7900,12 @@
5 distillate => $distill,
6 attribute => $groupby,
7 query_count => $times_seen,
8- example => {
9- query => substr($sample->{arg}, 0, $self->max_query_length),
10- ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef,
11- },
12+ $args{anon} ? () : (
13+ example => {
14+ query => substr($sample->{arg}, 0, $self->max_query_length),
15+ ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef,
16+ },
17+ ),
18 };
19
20 my %metrics;
21@@ -7981,19 +7983,21 @@
22 push @tables, { status => $status, create => $create };
23 }
24
25- if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) {
26- if ( $item =~ m/^(?:insert|replace)/ ) {
27+ if ( !$args{anon} ) {
28+ if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) {
29+ if ( $item =~ m/^(?:insert|replace)/ ) {
30+ }
31+ else {
32+
33+ }
34 }
35 else {
36-
37- }
38- }
39- else {
40- my $converted = $qr->convert_to_select(
41- $sample->{arg} || '',
42- );
43- if ( $converted && $converted =~ m/^[\(\s]*select/i ) {
44- $class->{example}->{as_select} = $converted;
45+ my $converted = $qr->convert_to_select(
46+ $sample->{arg} || '',
47+ );
48+ if ( $converted && $converted =~ m/^[\(\s]*select/i ) {
49+ $class->{example}->{as_select} = $converted;
50+ }
51 }
52 }
53 }
54@@ -14155,7 +14159,7 @@
55 $print_header = 1;
56 }
57
58- my $report_class = $o->get('output') =~ m/\Ajson\z/i
59+ my $report_class = $o->get('output') =~ m/^json/i
60 ? 'JSONReportFormatter'
61 : 'QueryReportFormatter';
62 my $qrf = $report_class->new(
63@@ -14182,6 +14186,7 @@
64 variations => $o->get('variations'),
65 group => { map { $_=>1 } qw(rusage date hostname files header) },
66 resume => $resume,
67+ anon => $o->get('output') eq 'json-anon',
68 );
69 }
70
71@@ -15611,11 +15616,12 @@
72
73 How to format and print the query analysis results. Accepted values are:
74
75- VALUE FORMAT
76- ======= ==============================
77- report Standard query analysis report
78- slowlog MySQL slow log
79- json JSON, on array per query class
80+ VALUE FORMAT
81+ ======= ==============================
82+ report Standard query analysis report
83+ slowlog MySQL slow log
84+ json JSON, on array per query class
85+ json-anon JSON without example queries
86
87 The entire C<report> output can be disabled by specifying C<--no-report>
88 (see L<"--[no]report">), and its sections can be disabled or rearranged
89
90=== modified file 'lib/JSONReportFormatter.pm'
91--- lib/JSONReportFormatter.pm 2013-06-15 18:04:25 +0000
92+++ lib/JSONReportFormatter.pm 2013-07-01 21:05:30 +0000
93@@ -228,10 +228,12 @@
94 distillate => $distill,
95 attribute => $groupby,
96 query_count => $times_seen,
97- example => {
98- query => substr($sample->{arg}, 0, $self->max_query_length),
99- ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef,
100- },
101+ $args{anon} ? () : (
102+ example => {
103+ query => substr($sample->{arg}, 0, $self->max_query_length),
104+ ts => $sample->{ts} ? parse_timestamp($sample->{ts}) : undef,
105+ },
106+ ),
107 };
108
109 my %metrics;
110@@ -340,26 +342,28 @@
111 push @tables, { status => $status, create => $create };
112 }
113
114- # Convert possible non-SELECTs for EXPLAIN.
115- if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) {
116- if ( $item =~ m/^(?:insert|replace)/ ) {
117- # Cannot convert or EXPLAIN INSERT or REPLACE queries.
118+ if ( !$args{anon} ) {
119+ # Convert possible non-SELECTs for EXPLAIN.
120+ if ( $item =~ m/^(?:[\(\s]*select|insert|replace)/ ) {
121+ if ( $item =~ m/^(?:insert|replace)/ ) {
122+ # Cannot convert or EXPLAIN INSERT or REPLACE queries.
123+ }
124+ else {
125+ # SELECT queries don't need to converted for EXPLAIN.
126+
127+ # TODO: return the actual EXPLAIN plan
128+ # $self->explain_report($query, $vals->{default_db});
129+ }
130 }
131 else {
132- # SELECT queries don't need to converted for EXPLAIN.
133-
134- # TODO: return the actual EXPLAIN plan
135- # $self->explain_report($query, $vals->{default_db});
136- }
137- }
138- else {
139- # Query is not SELECT, INSERT, or REPLACE, so we can convert
140- # it for EXPLAIN.
141- my $converted = $qr->convert_to_select(
142- $sample->{arg} || '',
143- );
144- if ( $converted && $converted =~ m/^[\(\s]*select/i ) {
145- $class->{example}->{as_select} = $converted;
146+ # Query is not SELECT, INSERT, or REPLACE, so we can convert
147+ # it for EXPLAIN.
148+ my $converted = $qr->convert_to_select(
149+ $sample->{arg} || '',
150+ );
151+ if ( $converted && $converted =~ m/^[\(\s]*select/i ) {
152+ $class->{example}->{as_select} = $converted;
153+ }
154 }
155 }
156 }
157
158=== modified file 't/pt-query-digest/json.t'
159--- t/pt-query-digest/json.t 2013-05-20 23:44:44 +0000
160+++ t/pt-query-digest/json.t 2013-07-01 21:05:30 +0000
161@@ -20,12 +20,12 @@
162
163 my @args = qw(--output json);
164 my $sample = "$trunk/t/lib/samples";
165-my $results = "t/pt-query-digest/samples";
166+my $results = "t/pt-query-digest/samples/json";
167
168 ok(
169 no_diff(
170 sub { pt_query_digest::main(@args, "$sample/slowlogs/empty") },
171- "$results/empty_report.txt",
172+ "t/pt-query-digest/samples/empty_report.txt",
173 ),
174 'json output for empty log'
175 ) or diag($test_diff);
176@@ -33,18 +33,30 @@
177 ok(
178 no_diff(
179 sub { pt_query_digest::main(@args, "$sample/slowlogs/slow002.txt") },
180- "$results/output_json_slow002.txt"
181+ "$results/slow002.txt",
182+ sed => [ qq/'s!$trunk!TRUNK!'/ ],
183 ),
184 'json output for slow002'
185 ) or diag($test_diff);
186
187+ok(
188+ no_diff(
189+ sub { pt_query_digest::main(qw(--output json-anon),
190+ "$sample/slowlogs/slow002.txt") },
191+ "$results/slow002-anon.txt",
192+ sed => [ qq/'s!$trunk!TRUNK!'/ ],
193+ ),
194+ 'json-anon output for slow002'
195+) or diag($test_diff);
196+
197 # --type tcpdump
198
199 ok(
200 no_diff(
201 sub { pt_query_digest::main(qw(--type tcpdump --limit 10 --watch-server 127.0.0.1:12345),
202 @args, "$sample/tcpdump/tcpdump021.txt") },
203- "$results/output_json_tcpdump021.txt",
204+ "$results/tcpdump021.txt",
205+ sed => [ qq/'s!$trunk!TRUNK!'/ ],
206 ),
207 'json output for for tcpdump021',
208 ) or diag($test_diff);
209
210=== modified file 't/pt-query-digest/samples/empty_report.txt'
211--- t/pt-query-digest/samples/empty_report.txt 2013-06-17 06:40:08 +0000
212+++ t/pt-query-digest/samples/empty_report.txt 2013-07-01 21:05:30 +0000
213@@ -1,2 +0,0 @@
214-
215-# No events processed.
216
217=== added directory 't/pt-query-digest/samples/json'
218=== added file 't/pt-query-digest/samples/json/slow002-anon.txt'
219--- t/pt-query-digest/samples/json/slow002-anon.txt 1970-01-01 00:00:00 +0000
220+++ t/pt-query-digest/samples/json/slow002-anon.txt 2013-07-01 21:05:30 +0000
221@@ -0,0 +1,257 @@
222+
223+{
224+ "classes" : [
225+ {
226+ "attribute" : "fingerprint",
227+ "checksum" : "66825DDC008FFA89",
228+ "distillate" : "UPDATE db?.tuningdetail_?_? db?.gonzo",
229+ "fingerprint" : "update d?tuningdetail_?_? n inner join d?gonzo a using(gonzo) set n.column? = a.column?, n.word? = a.word?",
230+ "metrics" : {
231+ "Filesort" : {
232+ "yes" : "0"
233+ },
234+ "Filesort_on_disk" : {
235+ "yes" : "0"
236+ },
237+ "Full_join" : {
238+ "yes" : "0"
239+ },
240+ "Full_scan" : {
241+ "yes" : "1"
242+ },
243+ "Lock_time" : {
244+ "avg" : "0.000091",
245+ "max" : "0.000091",
246+ "median" : "0.000091",
247+ "min" : "0.000091",
248+ "pct" : "0.125000",
249+ "pct_95" : "0.000091",
250+ "stddev" : "0.000000",
251+ "sum" : "0.000091"
252+ },
253+ "Merge_passes" : {
254+ "avg" : "0",
255+ "max" : "0",
256+ "median" : "0",
257+ "min" : "0",
258+ "pct" : "0",
259+ "pct_95" : "0",
260+ "stddev" : "0",
261+ "sum" : "0"
262+ },
263+ "QC_Hit" : {
264+ "yes" : "0"
265+ },
266+ "Query_length" : {
267+ "avg" : "129",
268+ "max" : "129",
269+ "median" : "129",
270+ "min" : "129",
271+ "pct" : "0",
272+ "pct_95" : "129",
273+ "stddev" : "0",
274+ "sum" : "129"
275+ },
276+ "Query_time" : {
277+ "avg" : "0.726052",
278+ "max" : "0.726052",
279+ "median" : "0.726052",
280+ "min" : "0.726052",
281+ "pct" : "0.125000",
282+ "pct_95" : "0.726052",
283+ "stddev" : "0.000000",
284+ "sum" : "0.726052"
285+ },
286+ "Rows_examined" : {
287+ "avg" : "62951",
288+ "max" : "62951",
289+ "median" : "62951",
290+ "min" : "62951",
291+ "pct" : "0",
292+ "pct_95" : "62951",
293+ "stddev" : "0",
294+ "sum" : "62951"
295+ },
296+ "Rows_sent" : {
297+ "avg" : "0",
298+ "max" : "0",
299+ "median" : "0",
300+ "min" : "0",
301+ "pct" : "0",
302+ "pct_95" : "0",
303+ "stddev" : "0",
304+ "sum" : "0"
305+ },
306+ "Tmp_table" : {
307+ "yes" : "0"
308+ },
309+ "Tmp_table_on_disk" : {
310+ "yes" : "0"
311+ },
312+ "db" : {
313+ "value" : "db1"
314+ },
315+ "host" : {
316+ "value" : ""
317+ },
318+ "user" : {
319+ "value" : "[SQL_SLAVE]"
320+ }
321+ },
322+ "query_count" : 1,
323+ "tables" : [
324+ {
325+ "create" : "SHOW CREATE TABLE `db2`.`tuningdetail_21_265507`\\G",
326+ "status" : "SHOW TABLE STATUS FROM `db2` LIKE 'tuningdetail_21_265507'\\G"
327+ },
328+ {
329+ "create" : "SHOW CREATE TABLE `db1`.`gonzo`\\G",
330+ "status" : "SHOW TABLE STATUS FROM `db1` LIKE 'gonzo'\\G"
331+ }
332+ ],
333+ "ts_max" : "2007-12-18 11:48:27",
334+ "ts_min" : "2007-12-18 11:48:27"
335+ }
336+ ],
337+ "global" : {
338+ "files" : [
339+ {
340+ "name" : "TRUNK/t/lib/samples/slowlogs/slow002.txt",
341+ "size" : 3841
342+ }
343+ ],
344+ "metrics" : {
345+ "Filesort" : {
346+ "cnt" : "0"
347+ },
348+ "Filesort_on_disk" : {
349+ "cnt" : "0"
350+ },
351+ "Full_join" : {
352+ "cnt" : "0"
353+ },
354+ "Full_scan" : {
355+ "cnt" : "1"
356+ },
357+ "InnoDB_IO_r_bytes" : {
358+ "avg" : "0",
359+ "max" : "0",
360+ "median" : "0",
361+ "min" : "0",
362+ "pct_95" : "0",
363+ "stddev" : "0",
364+ "sum" : "0"
365+ },
366+ "InnoDB_IO_r_ops" : {
367+ "avg" : "0",
368+ "max" : "0",
369+ "median" : "0",
370+ "min" : "0",
371+ "pct_95" : "0",
372+ "stddev" : "0",
373+ "sum" : "0"
374+ },
375+ "InnoDB_IO_r_wait" : {
376+ "avg" : "0.000000",
377+ "max" : "0.000000",
378+ "median" : "0.000000",
379+ "min" : "0.000000",
380+ "pct_95" : "0.000000",
381+ "stddev" : "0.000000",
382+ "sum" : "0.000000"
383+ },
384+ "InnoDB_pages_distinct" : {
385+ "avg" : "17",
386+ "max" : "24",
387+ "median" : "17",
388+ "min" : "11",
389+ "pct_95" : "23",
390+ "stddev" : "3",
391+ "sum" : "107"
392+ },
393+ "InnoDB_queue_wait" : {
394+ "avg" : "0.000000",
395+ "max" : "0.000000",
396+ "median" : "0.000000",
397+ "min" : "0.000000",
398+ "pct_95" : "0.000000",
399+ "stddev" : "0.000000",
400+ "sum" : "0.000000"
401+ },
402+ "InnoDB_rec_lock_wait" : {
403+ "avg" : "0.000000",
404+ "max" : "0.000000",
405+ "median" : "0.000000",
406+ "min" : "0.000000",
407+ "pct_95" : "0.000000",
408+ "stddev" : "0.000000",
409+ "sum" : "0.000000"
410+ },
411+ "Lock_time" : {
412+ "avg" : "0.000038",
413+ "max" : "0.000091",
414+ "median" : "0.000026",
415+ "min" : "0.000000",
416+ "pct_95" : "0.000089",
417+ "stddev" : "0.000028",
418+ "sum" : "0.000304"
419+ },
420+ "Merge_passes" : {
421+ "avg" : "0",
422+ "max" : "0",
423+ "median" : "0",
424+ "min" : "0",
425+ "pct_95" : "0",
426+ "stddev" : "0",
427+ "sum" : "0"
428+ },
429+ "QC_Hit" : {
430+ "cnt" : "0"
431+ },
432+ "Query_length" : {
433+ "avg" : "62",
434+ "max" : "129",
435+ "median" : "62",
436+ "min" : "5",
437+ "pct_95" : "124",
438+ "stddev" : "34",
439+ "sum" : "502"
440+ },
441+ "Query_time" : {
442+ "avg" : "0.095260",
443+ "max" : "0.726052",
444+ "median" : "0.000516",
445+ "min" : "0.000012",
446+ "pct_95" : "0.705093",
447+ "stddev" : "0.231765",
448+ "sum" : "0.762080"
449+ },
450+ "Rows_examined" : {
451+ "avg" : "7868",
452+ "max" : "62951",
453+ "median" : "0",
454+ "min" : "0",
455+ "pct_95" : "61003",
456+ "stddev" : "20174",
457+ "sum" : "62951"
458+ },
459+ "Rows_sent" : {
460+ "avg" : "0",
461+ "max" : "0",
462+ "median" : "0",
463+ "min" : "0",
464+ "pct_95" : "0",
465+ "stddev" : "0",
466+ "sum" : "0"
467+ },
468+ "Tmp_table" : {
469+ "cnt" : "0"
470+ },
471+ "Tmp_table_on_disk" : {
472+ "cnt" : "0"
473+ }
474+ },
475+ "query_count" : 8,
476+ "unique_query_count" : 7
477+ }
478+}
479
480=== renamed file 't/pt-query-digest/samples/output_json_slow002.txt' => 't/pt-query-digest/samples/json/slow002.txt'
481--- t/pt-query-digest/samples/output_json_slow002.txt 2013-06-17 06:40:08 +0000
482+++ t/pt-query-digest/samples/json/slow002.txt 2013-07-01 21:05:30 +0000
483@@ -121,7 +121,7 @@
484 "global" : {
485 "files" : [
486 {
487- "name" : "/Users/daniel/p/release-2.2.3/t/lib/samples/slowlogs/slow002.txt",
488+ "name" : "TRUNK/t/lib/samples/slowlogs/slow002.txt",
489 "size" : 3841
490 }
491 ],
492
493=== renamed file 't/pt-query-digest/samples/output_json_tcpdump021.txt' => 't/pt-query-digest/samples/json/tcpdump021.txt'
494--- t/pt-query-digest/samples/output_json_tcpdump021.txt 2013-06-17 06:40:08 +0000
495+++ t/pt-query-digest/samples/json/tcpdump021.txt 2013-07-01 21:05:30 +0000
496@@ -187,7 +187,7 @@
497 "global" : {
498 "files" : [
499 {
500- "name" : "/Users/daniel/p/release-2.2.3/t/lib/samples/tcpdump/tcpdump021.txt",
501+ "name" : "TRUNK/t/lib/samples/tcpdump/tcpdump021.txt",
502 "size" : 2827
503 }
504 ],

Subscribers

People subscribed via source and target branches

to all changes: