Merge lp:~percona-toolkit-dev/percona-toolkit/fix-sleep-bug-979092 into lp:percona-toolkit/2.0

Proposed by Daniel Nichter
Status: Merged
Merged at revision: 241
Proposed branch: lp:~percona-toolkit-dev/percona-toolkit/fix-sleep-bug-979092
Merge into: lp:percona-toolkit/2.0
Diff against target: 66121 lines (+52092/-8394) (has conflicts)
341 files modified
Changelog (+21/-0)
MANIFEST (+2/-0)
Makefile.PL (+1/-1)
bin/pt-align (+1/-1)
bin/pt-archiver (+1/-1)
bin/pt-config-diff (+1/-1)
bin/pt-deadlock-logger (+1/-1)
bin/pt-diskstats (+1/-1)
bin/pt-duplicate-key-checker (+171/-92)
bin/pt-fifo-split (+1/-1)
bin/pt-find (+1/-1)
bin/pt-fingerprint (+2143/-0)
bin/pt-fk-error-logger (+1/-1)
bin/pt-heartbeat (+1/-1)
bin/pt-index-usage (+170/-92)
bin/pt-ioprofile (+1/-1)
bin/pt-kill (+1/-1)
bin/pt-log-player (+1/-1)
bin/pt-mext (+1/-1)
bin/pt-mysql-summary (+2079/-679)
bin/pt-online-schema-change (+4617/-1243)
bin/pt-pmp (+1/-1)
bin/pt-query-advisor (+248/-372)
bin/pt-query-digest (+9/-9)
bin/pt-show-grants (+1/-1)
bin/pt-sift (+1/-1)
bin/pt-slave-delay (+1/-1)
bin/pt-slave-find (+1/-1)
bin/pt-slave-restart (+1/-1)
bin/pt-stalk (+1/-1)
bin/pt-summary (+2019/-836)
bin/pt-table-checksum (+191/-81)
bin/pt-table-sync (+554/-405)
bin/pt-table-usage (+7223/-0)
bin/pt-tcp-model (+1/-1)
bin/pt-trend (+1/-1)
bin/pt-upgrade (+1/-1)
bin/pt-variable-advisor (+1/-1)
bin/pt-visual-explain (+1/-1)
config/deb/changelog (+23/-0)
config/sphinx-build/conf.py (+2/-2)
docs/percona-toolkit.pod (+12/-4)
docs/release_notes.rst (+48/-0)
lib/CleanupTask.pm (+7/-2)
lib/Cxn.pm (+1/-1)
lib/MySQLStatusWaiter.pm (+31/-1)
lib/NibbleIterator.pm (+166/-28)
lib/OSCCaptureSync.pm (+0/-142)
lib/OobNibbleIterator.pm (+1/-1)
lib/QueryAdvisorRules.pm (+2/-2)
lib/QueryRewriter.pm (+24/-4)
lib/SQLParser.pm (+103/-8)
lib/Schema.pm (+5/-5)
lib/SchemaIterator.pm (+26/-44)
lib/SlowLogWriter.pm (+2/-2)
lib/TableUsage.pm (+1060/-0)
lib/bash/alt_cmds.sh (+25/-1)
lib/bash/collect_mysql_info.sh (+263/-0)
lib/bash/collect_system_info.sh (+574/-0)
lib/bash/log_warn_die.sh (+6/-0)
lib/bash/report_formatting.sh (+117/-0)
lib/bash/report_mysql_info.sh (+1391/-0)
lib/bash/report_system_info.sh (+1050/-0)
lib/bash/summary_common.sh (+155/-0)
t/lib/CleanupTask.t (+18/-1)
t/lib/MySQLStatusWaiter.t (+41/-0)
t/lib/NibbleIterator.t (+44/-3)
t/lib/OSCCaptureSync.t (+0/-131)
t/lib/OobNibbleIterator.t (+8/-3)
t/lib/QueryRewriter.t (+59/-1)
t/lib/SQLParser.t (+1/-1)
t/lib/SchemaIterator.t (+9/-50)
t/lib/SlowLogParser.t (+63/-63)
t/lib/TableUsage.t (+816/-0)
t/lib/bash/collect_mysql_info.sh (+197/-0)
t/lib/bash/collect_system_info.sh (+307/-0)
t/lib/bash/report_formatting.sh (+109/-0)
t/lib/bash/report_mysql_info.sh (+734/-0)
t/lib/bash/report_system_info.sh (+1568/-0)
t/lib/bash/summary_common.sh (+79/-0)
t/lib/samples/SchemaIterator/all-dbs-tbls-5.0.txt (+421/-0)
t/lib/samples/SchemaIterator/all-dbs-tbls.txt (+486/-0)
t/lib/samples/SchemaIterator/resume-from-ignored-sakila-payment-5.0.txt (+52/-0)
t/lib/samples/SchemaIterator/resume-from-ignored-sakila-payment.txt (+49/-0)
t/lib/samples/SchemaIterator/resume-from-sakila-payment-5.0.txt (+70/-0)
t/lib/samples/SchemaIterator/resume-from-sakila-payment.txt (+66/-0)
t/lib/samples/slowlogs/slow002.txt (+16/-16)
t/lib/samples/slowlogs/slow003.txt (+2/-2)
t/lib/samples/slowlogs/slow005.txt (+2/-2)
t/lib/samples/slowlogs/slow006.txt (+12/-12)
t/lib/samples/slowlogs/slow007.txt (+2/-2)
t/lib/samples/slowlogs/slow022.txt (+12/-12)
t/lib/samples/slowlogs/slow030.txt (+899/-899)
t/lib/samples/slowlogs/slow032-rewritten.txt (+2/-2)
t/lib/samples/slowlogs/slow032.txt (+2/-2)
t/lib/samples/slowlogs/slow034.txt (+18/-18)
t/lib/samples/slowlogs/slow035.txt (+2/-2)
t/pt-deadlock-logger/basics.t (+1/-1)
t/pt-fingerprint/basics.t (+101/-0)
t/pt-fingerprint/samples/query001 (+2/-0)
t/pt-fingerprint/samples/query001.fingerprint (+1/-0)
t/pt-fingerprint/samples/query002 (+2/-0)
t/pt-fingerprint/samples/query002.fingerprint (+1/-0)
t/pt-log-player/samples/log001.txt (+16/-16)
t/pt-mysql-summary/find_my_cnf_file.sh (+0/-20)
t/pt-mysql-summary/format_binlog_filters.sh (+0/-12)
t/pt-mysql-summary/format_innodb_status.sh (+0/-147)
t/pt-mysql-summary/format_overall_db_stats.sh (+0/-61)
t/pt-mysql-summary/format_status_variables.sh (+0/-97)
t/pt-mysql-summary/fuzz.sh (+0/-7)
t/pt-mysql-summary/get_mysql_info.sh (+0/-26)
t/pt-mysql-summary/parse_mysqld_instances.sh (+0/-47)
t/pt-mysql-summary/pretty_print_cnf_file.sh (+0/-40)
t/pt-mysql-summary/pt-mysql-summary.t (+54/-2)
t/pt-mysql-summary/samples/expected_output_temp002.txt (+276/-0)
t/pt-mysql-summary/samples/expected_output_temp003.txt (+219/-0)
t/pt-mysql-summary/samples/expected_output_temp004.txt (+218/-0)
t/pt-mysql-summary/samples/expected_output_temp005.txt (+291/-0)
t/pt-mysql-summary/samples/expected_result_report_summary.txt (+257/-0)
t/pt-mysql-summary/samples/mysql-variables-with-semisync.txt (+326/-0)
t/pt-mysql-summary/samples/temp001/mysql-status (+304/-0)
t/pt-mysql-summary/samples/temp001/mysql-variables (+356/-0)
t/pt-mysql-summary/samples/temp002/innodb-status (+118/-0)
t/pt-mysql-summary/samples/temp002/mysql-config-file (+26/-0)
t/pt-mysql-summary/samples/temp002/mysql-databases (+6/-0)
t/pt-mysql-summary/samples/temp002/mysql-plugins (+35/-0)
t/pt-mysql-summary/samples/temp002/mysql-processlist (+12/-0)
t/pt-mysql-summary/samples/temp002/mysql-status (+370/-0)
t/pt-mysql-summary/samples/temp002/mysql-status-defer (+370/-0)
t/pt-mysql-summary/samples/temp002/mysql-users (+1/-0)
t/pt-mysql-summary/samples/temp002/mysql-variables (+372/-0)
t/pt-mysql-summary/samples/temp002/mysqld-instances (+4/-0)
t/pt-mysql-summary/samples/temp002/mysqldump (+396/-0)
t/pt-mysql-summary/samples/temp003/innodb-status (+77/-0)
t/pt-mysql-summary/samples/temp003/mysql-config-file (+26/-0)
t/pt-mysql-summary/samples/temp003/mysql-databases (+2/-0)
t/pt-mysql-summary/samples/temp003/mysql-master-logs (+1/-0)
t/pt-mysql-summary/samples/temp003/mysql-master-status (+1/-0)
t/pt-mysql-summary/samples/temp003/mysql-plugins (+10/-0)
t/pt-mysql-summary/samples/temp003/mysql-processlist (+9/-0)
t/pt-mysql-summary/samples/temp003/mysql-status (+291/-0)
t/pt-mysql-summary/samples/temp003/mysql-status-defer (+291/-0)
t/pt-mysql-summary/samples/temp003/mysql-users (+1/-0)
t/pt-mysql-summary/samples/temp003/mysql-variables (+285/-0)
t/pt-mysql-summary/samples/temp003/mysqld-instances (+2/-0)
t/pt-mysql-summary/samples/temp004/innodb-status (+77/-0)
t/pt-mysql-summary/samples/temp004/mysql-config-file (+26/-0)
t/pt-mysql-summary/samples/temp004/mysql-databases (+3/-0)
t/pt-mysql-summary/samples/temp004/mysql-master-logs (+2/-0)
t/pt-mysql-summary/samples/temp004/mysql-master-status (+1/-0)
t/pt-mysql-summary/samples/temp004/mysql-plugins (+10/-0)
t/pt-mysql-summary/samples/temp004/mysql-processlist (+9/-0)
t/pt-mysql-summary/samples/temp004/mysql-status (+291/-0)
t/pt-mysql-summary/samples/temp004/mysql-status-defer (+291/-0)
t/pt-mysql-summary/samples/temp004/mysql-users (+1/-0)
t/pt-mysql-summary/samples/temp004/mysql-variables (+285/-0)
t/pt-mysql-summary/samples/temp004/mysqld-instances (+2/-0)
t/pt-mysql-summary/samples/temp005/innodb-status (+108/-0)
t/pt-mysql-summary/samples/temp005/mysql-config-file (+26/-0)
t/pt-mysql-summary/samples/temp005/mysql-databases (+3/-0)
t/pt-mysql-summary/samples/temp005/mysql-master-logs (+1/-0)
t/pt-mysql-summary/samples/temp005/mysql-master-status (+1/-0)
t/pt-mysql-summary/samples/temp005/mysql-plugins (+28/-0)
t/pt-mysql-summary/samples/temp005/mysql-processlist (+18/-0)
t/pt-mysql-summary/samples/temp005/mysql-status (+304/-0)
t/pt-mysql-summary/samples/temp005/mysql-status-defer (+304/-0)
t/pt-mysql-summary/samples/temp005/mysql-users (+1/-0)
t/pt-mysql-summary/samples/temp005/mysql-variables (+363/-0)
t/pt-mysql-summary/samples/temp005/mysqld-executables (+1/-0)
t/pt-mysql-summary/samples/temp005/mysqld-instances (+4/-0)
t/pt-mysql-summary/samples/temp005/mysqldump (+1084/-0)
t/pt-mysql-summary/samples/tempdir/innodb-status (+77/-0)
t/pt-mysql-summary/samples/tempdir/mysql-config-file (+26/-0)
t/pt-mysql-summary/samples/tempdir/mysql-databases (+3/-0)
t/pt-mysql-summary/samples/tempdir/mysql-master-logs (+3/-0)
t/pt-mysql-summary/samples/tempdir/mysql-master-status (+1/-0)
t/pt-mysql-summary/samples/tempdir/mysql-plugins (+10/-0)
t/pt-mysql-summary/samples/tempdir/mysql-processlist (+9/-0)
t/pt-mysql-summary/samples/tempdir/mysql-status (+291/-0)
t/pt-mysql-summary/samples/tempdir/mysql-status-defer (+291/-0)
t/pt-mysql-summary/samples/tempdir/mysql-users (+5/-0)
t/pt-mysql-summary/samples/tempdir/mysql-variables (+283/-0)
t/pt-mysql-summary/samples/tempdir/mysqld-instances (+4/-0)
t/pt-mysql-summary/samples/tempdir/mysqldump (+328/-0)
t/pt-mysql-summary/samples/tempdir/tempfile (+130/-0)
t/pt-mysql-summary/summarize_binlogs.sh (+0/-13)
t/pt-mysql-summary/summarize_processlist.sh (+0/-64)
t/pt-online-schema-change/alter_active_table.t (+56/-46)
t/pt-online-schema-change/basics.t (+541/-218)
t/pt-online-schema-change/check_tables.t (+0/-126)
t/pt-online-schema-change/option_sanity.t (+12/-12)
t/pt-online-schema-change/samples/basic_no_fks.data (+500/-500)
t/pt-online-schema-change/samples/basic_no_fks.sql (+30/-0)
t/pt-online-schema-change/samples/basic_with_fks.sql (+56/-0)
t/pt-online-schema-change/samples/fk_tables_schema.sql (+0/-31)
t/pt-online-schema-change/samples/query_table.pl (+8/-5)
t/pt-online-schema-change/samples/small_table.sql (+0/-27)
t/pt-online-schema-change/sanity_checks.t (+113/-0)
t/pt-query-digest/read_timeout.t (+1/-1)
t/pt-query-digest/review.t (+4/-4)
t/pt-query-digest/samples/save-results/slow002-limit-3.txt (+4/-4)
t/pt-query-digest/samples/save-results/slow002.txt (+2/-2)
t/pt-query-digest/samples/save-results/slow006.txt (+1/-1)
t/pt-query-digest/samples/slow-issue-611.txt (+2/-2)
t/pt-query-digest/samples/slow002-orderbynonexistent.txt (+6/-6)
t/pt-query-digest/samples/slow002_iters_2.txt (+1/-1)
t/pt-query-digest/samples/slow002_orderbyreport.txt (+2/-2)
t/pt-query-digest/samples/slow002_report.txt (+6/-6)
t/pt-query-digest/samples/slow002_report_filtered.txt (+1/-1)
t/pt-query-digest/samples/slow006-first2.txt (+8/-8)
t/pt-query-digest/samples/slow006-order-by-re.txt (+2/-2)
t/pt-query-digest/samples/slow006_AR_1.txt (+2/-2)
t/pt-query-digest/samples/slow006_AR_2.txt (+1/-1)
t/pt-query-digest/samples/slow006_AR_4.txt (+2/-2)
t/pt-query-digest/samples/slow006_AR_5.txt (+1/-1)
t/pt-query-digest/samples/slow006_report.txt (+2/-2)
t/pt-query-digest/samples/slow034-inheritance.txt (+18/-18)
t/pt-query-digest/samples/slow034-no-ts-inheritance.txt (+18/-18)
t/pt-query-digest/samples/slow034-order-by-Locktime-sum-with-Locktime-distro.txt (+8/-8)
t/pt-query-digest/samples/slow034-order-by-Locktime-sum.txt (+8/-8)
t/pt-query-digest/samples/slow035.txt (+1/-1)
t/pt-stalk/pt-stalk.t (+1/-1)
t/pt-summary/format_vmstat.sh (+0/-37)
t/pt-summary/parse_arcconf.sh (+0/-176)
t/pt-summary/parse_dmidecode_mem_devices.sh (+0/-104)
t/pt-summary/parse_ethernet_controller_lspci.sh (+0/-11)
t/pt-summary/parse_fdisk.sh (+0/-16)
t/pt-summary/parse_filesystems.sh (+0/-52)
t/pt-summary/parse_free_minus_b.sh (+0/-67)
t/pt-summary/parse_fusionmpt_lsiutil.sh (+0/-50)
t/pt-summary/parse_hpacucli.sh (+0/-26)
t/pt-summary/parse_ip_s_link.sh (+0/-28)
t/pt-summary/parse_lsi_megaraid.sh (+0/-696)
t/pt-summary/parse_netstat.sh (+0/-49)
t/pt-summary/parse_proc_cpuinfo.sh (+0/-74)
t/pt-summary/parse_raid_controller_dmesg.sh (+0/-32)
t/pt-summary/parse_raid_controller_lspci.sh (+0/-39)
t/pt-summary/parse_virtualization_dmesg.sh (+0/-10)
t/pt-summary/pt-summary.t (+12/-2)
t/pt-summary/samples/BSD/freebsd_001/mounted_fs (+7/-0)
t/pt-summary/samples/BSD/freebsd_001/notable_procs (+2/-0)
t/pt-summary/samples/BSD/freebsd_001/processes (+10/-0)
t/pt-summary/samples/BSD/freebsd_001/summary (+10/-0)
t/pt-summary/samples/BSD/freebsd_001/sysctl (+1481/-0)
t/pt-summary/samples/BSD/freebsd_001/uptime (+1/-0)
t/pt-summary/samples/BSD/freebsd_001/vmstat (+7/-0)
t/pt-summary/samples/BSD/netbsd_001/mounted_fs (+5/-0)
t/pt-summary/samples/BSD/netbsd_001/notable_procs (+2/-0)
t/pt-summary/samples/BSD/netbsd_001/proc_cpuinfo_copy (+14/-0)
t/pt-summary/samples/BSD/netbsd_001/processes (+10/-0)
t/pt-summary/samples/BSD/netbsd_001/summary (+10/-0)
t/pt-summary/samples/BSD/netbsd_001/swapctl (+1/-0)
t/pt-summary/samples/BSD/netbsd_001/sysctl (+511/-0)
t/pt-summary/samples/BSD/netbsd_001/uptime (+1/-0)
t/pt-summary/samples/BSD/netbsd_001/vmstat (+7/-0)
t/pt-summary/samples/BSD/openbsd_001/mounted_fs (+4/-0)
t/pt-summary/samples/BSD/openbsd_001/notable_procs (+2/-0)
t/pt-summary/samples/BSD/openbsd_001/processes (+10/-0)
t/pt-summary/samples/BSD/openbsd_001/summary (+10/-0)
t/pt-summary/samples/BSD/openbsd_001/swapctl (+1/-0)
t/pt-summary/samples/BSD/openbsd_001/sysctl (+423/-0)
t/pt-summary/samples/BSD/openbsd_001/uptime (+1/-0)
t/pt-summary/samples/BSD/openbsd_001/vmstat (+7/-0)
t/pt-summary/samples/Linux/001/dmesg_file (+786/-0)
t/pt-summary/samples/Linux/001/dmidecode (+412/-0)
t/pt-summary/samples/Linux/001/ip (+24/-0)
t/pt-summary/samples/Linux/001/lspci_file (+17/-0)
t/pt-summary/samples/Linux/001/lvs (+1/-0)
t/pt-summary/samples/Linux/001/memory (+50/-0)
t/pt-summary/samples/Linux/001/mounted_fs (+12/-0)
t/pt-summary/samples/Linux/001/netstat (+6/-0)
t/pt-summary/samples/Linux/001/notable_procs (+5/-0)
t/pt-summary/samples/Linux/001/partitioning (+30/-0)
t/pt-summary/samples/Linux/001/proc_cpuinfo_copy (+58/-0)
t/pt-summary/samples/Linux/001/proc_cpuinfo_copy.unq (+1/-0)
t/pt-summary/samples/Linux/001/processes (+10/-0)
t/pt-summary/samples/Linux/001/summary (+23/-0)
t/pt-summary/samples/Linux/001/sysctl (+905/-0)
t/pt-summary/samples/Linux/001/uptime (+1/-0)
t/pt-summary/samples/Linux/001/vmstat (+7/-0)
t/pt-summary/samples/Linux/002/dmesg_file (+283/-0)
t/pt-summary/samples/Linux/002/memory (+34/-0)
t/pt-summary/samples/Linux/002/mounted_fs (+3/-0)
t/pt-summary/samples/Linux/002/netstat (+6/-0)
t/pt-summary/samples/Linux/002/notable_procs (+2/-0)
t/pt-summary/samples/Linux/002/partitioning (+9/-0)
t/pt-summary/samples/Linux/002/proc_cpuinfo_copy (+19/-0)
t/pt-summary/samples/Linux/002/processes (+10/-0)
t/pt-summary/samples/Linux/002/summary (+19/-0)
t/pt-summary/samples/Linux/002/uptime (+1/-0)
t/pt-summary/samples/Linux/002/vmstat (+7/-0)
t/pt-summary/samples/Linux/003/dmesg_file (+283/-0)
t/pt-summary/samples/Linux/003/memory (+34/-0)
t/pt-summary/samples/Linux/003/mounted_fs (+3/-0)
t/pt-summary/samples/Linux/003/netstat (+6/-0)
t/pt-summary/samples/Linux/003/notable_procs (+2/-0)
t/pt-summary/samples/Linux/003/partitioning (+1/-0)
t/pt-summary/samples/Linux/003/proc_cpuinfo_copy (+19/-0)
t/pt-summary/samples/Linux/003/processes (+10/-0)
t/pt-summary/samples/Linux/003/summary (+19/-0)
t/pt-summary/samples/Linux/003/uptime (+1/-0)
t/pt-summary/samples/Linux/003/vmstat (+7/-0)
t/pt-summary/samples/Linux/output_002.txt (+82/-0)
t/pt-summary/samples/Linux/output_003.txt (+79/-0)
t/pt-summary/samples/MegaCli64_AdpAllInfo_aALL001.txt (+227/-0)
t/pt-summary/samples/MegaCli64_LdPdInfo_aALL_886223 (+214/-0)
t/pt-summary/samples/arcconf-001.txt (+133/-0)
t/pt-summary/samples/arcconf-003_900285.txt (+228/-0)
t/pt-summary/samples/arcconf-004_917781.txt (+162/-0)
t/pt-summary/samples/dmesg-005.txt (+787/-0)
t/pt-summary/samples/dmesg-007.txt (+136/-0)
t/pt-summary/samples/hpaculi-001.txt (+11/-0)
t/pt-summary/samples/hpaculi-002.txt (+354/-0)
t/pt-summary/samples/hpaculi-003.txt (+11/-0)
t/pt-summary/samples/ip-s-link-003.txt (+24/-0)
t/pt-summary/samples/lspci-005.txt (+38/-0)
t/pt-summary/samples/netstat-002.txt (+1328/-0)
t/pt-summary/samples/proc_cpuinfo001.txt (+57/-0)
t/pt-summary/samples/proc_cpuinfo001.txt.unq (+1/-0)
t/pt-summary/samples/proc_cpuinfo002.txt (+57/-0)
t/pt-summary/samples/proc_cpuinfo002.txt.unq (+1/-0)
t/pt-table-sync/issue_408.t (+1/-1)
t/pt-table-usage/basics.t (+138/-0)
t/pt-table-usage/create_table_definitions.t (+41/-0)
t/pt-table-usage/explain_extended.t (+79/-0)
t/pt-table-usage/samples/ee.out (+6/-0)
t/pt-table-usage/samples/ee.sql (+26/-0)
t/pt-table-usage/samples/in/slow001.txt (+24/-0)
t/pt-table-usage/samples/in/slow002.txt (+20/-0)
t/pt-table-usage/samples/in/slow003.txt (+3/-0)
t/pt-table-usage/samples/out/create-table-defs-001.txt (+4/-0)
t/pt-table-usage/samples/out/create001.txt (+5/-0)
t/pt-table-usage/samples/out/drop-table-if-exists.txt (+3/-0)
t/pt-table-usage/samples/out/query001.txt (+6/-0)
t/pt-table-usage/samples/out/query002.txt (+5/-0)
t/pt-table-usage/samples/out/slow001.txt (+31/-0)
t/pt-table-usage/samples/out/slow002.txt (+40/-0)
t/pt-table-usage/samples/out/slow003-001.txt (+6/-0)
t/pt-table-usage/samples/out/slow003-002.txt (+8/-0)
t/pt-table-usage/samples/out/slow003-003.txt (+6/-0)
util/test-bash-functions (+48/-0)
Text conflict in bin/pt-online-schema-change
Text conflict in bin/pt-table-checksum
Text conflict in lib/MySQLStatusWaiter.pm
Text conflict in lib/NibbleIterator.pm
Text conflict in t/lib/MySQLStatusWaiter.t
Text conflict in t/lib/NibbleIterator.t
To merge this branch: bzr merge lp:~percona-toolkit-dev/percona-toolkit/fix-sleep-bug-979092
Reviewer Review Type Date Requested Status
Daniel Nichter Approve
Review via email: mp+108057@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Daniel Nichter (daniel-nichter) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'Changelog'
--- Changelog 2012-03-07 23:41:54 +0000
+++ Changelog 2012-05-30 21:00:30 +0000
@@ -1,5 +1,26 @@
1Changelog for Percona Toolkit1Changelog for Percona Toolkit
22
3v2.1.1 released 2012-04-03
4
5 * Completely redesigned pt-online-schema-change
6 * Completely redesigned pt-mysql-summary
7 * Completely redesigned pt-summary
8 * Added new tool: pt-table-usage
9 * Added new tool: pt-fingerprint
10 * Fixed bug 955860: pt-stalk doesn't run vmstat, iostat, and mpstat for --run-time
11 * Fixed bug 960513: SHOW TABLE STATUS is used needlessly
12 * Fixed bug 969726: pt-online-schema-change loses foreign keys
13 * Fixed bug 846028: pt-online-schema-change does not show progress until completed
14 * Fixed bug 898695: pt-online-schema-change add useless ORDER BY
15 * Fixed bug 952727: pt-diskstats shows incorrect wr_mb_s
16 * Fixed bug 963225: pt-query-digest fails to set history columns for disk tmp tables and disk filesort
17 * Fixed bug 967451: Char chunking doesn't quote column name
18 * Fixed bug 972399: pt-table-checksum docs are not rendered right
19 * Fixed bug 896553: Various documentation spelling fixes
20 * Fixed bug 949154: pt-variable-advisor advice for relay-log-space-limit
21 * Fixed bug 953461: pt-upgrade manual broken 'output' section
22 * Fixed bug 949653: pt-table-checksum docs don't mention risks posed by inconsistent schemas
23
3v2.0.4 released 2012-03-0724v2.0.4 released 2012-03-07
425
5 * Added --filter to pt-kill to allow arbitrary --group-by26 * Added --filter to pt-kill to allow arbitrary --group-by
627
=== modified file 'MANIFEST'
--- MANIFEST 2012-02-03 23:25:29 +0000
+++ MANIFEST 2012-05-30 21:00:30 +0000
@@ -12,6 +12,7 @@
12bin/pt-duplicate-key-checker12bin/pt-duplicate-key-checker
13bin/pt-fifo-split13bin/pt-fifo-split
14bin/pt-find14bin/pt-find
15bin/pt-fingerprint
15bin/pt-fk-error-logger16bin/pt-fk-error-logger
16bin/pt-heartbeat17bin/pt-heartbeat
17bin/pt-index-usage18bin/pt-index-usage
@@ -33,6 +34,7 @@
33bin/pt-summary34bin/pt-summary
34bin/pt-table-checksum35bin/pt-table-checksum
35bin/pt-table-sync36bin/pt-table-sync
37bin/pt-table-usage
36bin/pt-tcp-model38bin/pt-tcp-model
37bin/pt-trend39bin/pt-trend
38bin/pt-upgrade40bin/pt-upgrade
3941
=== modified file 'Makefile.PL'
--- Makefile.PL 2012-03-07 23:41:54 +0000
+++ Makefile.PL 2012-05-30 21:00:30 +0000
@@ -2,7 +2,7 @@
22
3WriteMakefile(3WriteMakefile(
4 NAME => 'percona-toolkit',4 NAME => 'percona-toolkit',
5 VERSION => '2.0.4',5 VERSION => '2.1.1',
6 EXE_FILES => [ <bin/*> ],6 EXE_FILES => [ <bin/*> ],
7 MAN1PODS => {7 MAN1PODS => {
8 'docs/percona-toolkit.pod' => 'blib/man1/percona-toolkit.1',8 'docs/percona-toolkit.pod' => 'blib/man1/percona-toolkit.1',
99
=== modified file 'bin/pt-align'
--- bin/pt-align 2012-03-07 23:41:54 +0000
+++ bin/pt-align 2012-05-30 21:00:30 +0000
@@ -218,6 +218,6 @@
218218
219=head1 VERSION219=head1 VERSION
220220
221pt-align 2.0.4221pt-align 2.1.1
222222
223=cut223=cut
224224
=== modified file 'bin/pt-archiver'
--- bin/pt-archiver 2012-05-30 20:59:47 +0000
+++ bin/pt-archiver 2012-05-30 21:00:30 +0000
@@ -5731,6 +5731,6 @@
57315731
5732=head1 VERSION5732=head1 VERSION
57335733
5734pt-archiver 2.0.45734pt-archiver 2.1.1
57355735
5736=cut5736=cut
57375737
=== modified file 'bin/pt-config-diff'
--- bin/pt-config-diff 2012-05-24 17:52:01 +0000
+++ bin/pt-config-diff 2012-05-30 21:00:30 +0000
@@ -3408,6 +3408,6 @@
34083408
3409=head1 VERSION3409=head1 VERSION
34103410
3411pt-config-diff 2.0.43411pt-config-diff 2.1.1
34123412
3413=cut3413=cut
34143414
=== modified file 'bin/pt-deadlock-logger'
--- bin/pt-deadlock-logger 2012-05-30 17:54:33 +0000
+++ bin/pt-deadlock-logger 2012-05-30 21:00:30 +0000
@@ -2748,6 +2748,6 @@
27482748
2749=head1 VERSION2749=head1 VERSION
27502750
2751pt-deadlock-logger 2.0.42751pt-deadlock-logger 2.1.1
27522752
2753=cut2753=cut
27542754
=== modified file 'bin/pt-diskstats'
--- bin/pt-diskstats 2012-05-11 14:28:14 +0000
+++ bin/pt-diskstats 2012-05-30 21:00:30 +0000
@@ -4106,6 +4106,6 @@
41064106
4107=head1 VERSION4107=head1 VERSION
41084108
4109pt-diskstats 2.0.44109pt-diskstats 2.1.1
41104110
4111=cut4111=cut
41124112
=== modified file 'bin/pt-duplicate-key-checker'
--- bin/pt-duplicate-key-checker 2012-05-24 17:52:01 +0000
+++ bin/pt-duplicate-key-checker 2012-05-30 21:00:30 +0000
@@ -199,19 +199,58 @@
199 return bless $self, $class;199 return bless $self, $class;
200}200}
201201
202sub get_create_table {
203 my ( $self, $dbh, $db, $tbl ) = @_;
204 die "I need a dbh parameter" unless $dbh;
205 die "I need a db parameter" unless $db;
206 die "I need a tbl parameter" unless $tbl;
207 my $q = $self->{Quoter};
208
209 my $new_sql_mode
210 = '/*!40101 SET @OLD_SQL_MODE := @@SQL_MODE, '
211 . q{@@SQL_MODE := REPLACE(REPLACE(@@SQL_MODE, 'ANSI_QUOTES', ''), ',,', ','), }
212 . '@OLD_QUOTE := @@SQL_QUOTE_SHOW_CREATE, '
213 . '@@SQL_QUOTE_SHOW_CREATE := 1 */';
214
215 my $old_sql_mode = '/*!40101 SET @@SQL_MODE := @OLD_SQL_MODE, '
216 . '@@SQL_QUOTE_SHOW_CREATE := @OLD_QUOTE */';
217
218 PTDEBUG && _d($new_sql_mode);
219 eval { $dbh->do($new_sql_mode); };
220 PTDEBUG && $EVAL_ERROR && _d($EVAL_ERROR);
221
222 my $use_sql = 'USE ' . $q->quote($db);
223 PTDEBUG && _d($dbh, $use_sql);
224 $dbh->do($use_sql);
225
226 my $show_sql = "SHOW CREATE TABLE " . $q->quote($db, $tbl);
227 PTDEBUG && _d($show_sql);
228 my $href;
229 eval { $href = $dbh->selectrow_hashref($show_sql); };
230 if ( $EVAL_ERROR ) {
231 PTDEBUG && _d($EVAL_ERROR);
232
233 PTDEBUG && _d($old_sql_mode);
234 $dbh->do($old_sql_mode);
235
236 return;
237 }
238
239 PTDEBUG && _d($old_sql_mode);
240 $dbh->do($old_sql_mode);
241
242 my ($key) = grep { m/create (?:table|view)/i } keys %$href;
243 if ( !$key ) {
244 die "Error: no 'Create Table' or 'Create View' in result set from "
245 . "$show_sql: " . Dumper($href);
246 }
247
248 return $href->{$key};
249}
250
202sub parse {251sub parse {
203 my ( $self, $ddl, $opts ) = @_;252 my ( $self, $ddl, $opts ) = @_;
204 return unless $ddl;253 return unless $ddl;
205 if ( ref $ddl eq 'ARRAY' ) {
206 if ( lc $ddl->[0] eq 'table' ) {
207 $ddl = $ddl->[1];
208 }
209 else {
210 return {
211 engine => 'VIEW',
212 };
213 }
214 }
215254
216 if ( $ddl !~ m/CREATE (?:TEMPORARY )?TABLE `/ ) {255 if ( $ddl !~ m/CREATE (?:TEMPORARY )?TABLE `/ ) {
217 die "Cannot parse table definition; is ANSI quoting "256 die "Cannot parse table definition; is ANSI quoting "
@@ -518,41 +557,31 @@
518 return $ddl;557 return $ddl;
519}558}
520559
521sub remove_secondary_indexes {560sub get_table_status {
522 my ( $self, $ddl ) = @_;561 my ( $self, $dbh, $db, $like ) = @_;
523 my $sec_indexes_ddl;562 my $q = $self->{Quoter};
524 my $tbl_struct = $self->parse($ddl);563 my $sql = "SHOW TABLE STATUS FROM " . $q->quote($db);
525564 my @params;
526 if ( ($tbl_struct->{engine} || '') =~ m/InnoDB/i ) {565 if ( $like ) {
527 my $clustered_key = $tbl_struct->{clustered_key};566 $sql .= ' LIKE ?';
528 $clustered_key ||= '';567 push @params, $like;
529568 }
530 my @sec_indexes = map {569 PTDEBUG && _d($sql, @params);
531 my $key_def = $_->{ddl};570 my $sth = $dbh->prepare($sql);
532 $key_def =~ s/([\(\)])/\\$1/g;571 eval { $sth->execute(@params); };
533 $ddl =~ s/\s+$key_def//i;572 if ($EVAL_ERROR) {
534573 PTDEBUG && _d($EVAL_ERROR);
535 my $key_ddl = "ADD $_->{ddl}";574 return;
536 $key_ddl .= ',' unless $key_ddl =~ m/,$/;575 }
537 $key_ddl;576 my @tables = @{$sth->fetchall_arrayref({})};
538 }577 @tables = map {
539 grep { $_->{name} ne $clustered_key }578 my %tbl; # Make a copy with lowercased keys
540 values %{$tbl_struct->{keys}};579 @tbl{ map { lc $_ } keys %$_ } = values %$_;
541 PTDEBUG && _d('Secondary indexes:', Dumper(\@sec_indexes));580 $tbl{engine} ||= $tbl{type} || $tbl{comment};
542581 delete $tbl{type};
543 if ( @sec_indexes ) {582 \%tbl;
544 $sec_indexes_ddl = join(' ', @sec_indexes);583 } @tables;
545 $sec_indexes_ddl =~ s/,$//;584 return @tables;
546 }
547
548 $ddl =~ s/,(\n\) )/$1/s;
549 }
550 else {
551 PTDEBUG && _d('Not removing secondary indexes from',
552 $tbl_struct->{engine}, 'table');
553 }
554
555 return $ddl, $sec_indexes_ddl, $tbl_struct;
556}585}
557586
558sub _d {587sub _d {
@@ -3200,7 +3229,7 @@
32003229
3201sub new {3230sub new {
3202 my ( $class, %args ) = @_;3231 my ( $class, %args ) = @_;
3203 my @required_args = qw(OptionParser Quoter);3232 my @required_args = qw(OptionParser TableParser Quoter);
3204 foreach my $arg ( @required_args ) {3233 foreach my $arg ( @required_args ) {
3205 die "I need a $arg argument" unless $args{$arg};3234 die "I need a $arg argument" unless $args{$arg};
3206 }3235 }
@@ -3209,8 +3238,19 @@
3209 die "I need either a dbh or file_itr argument"3238 die "I need either a dbh or file_itr argument"
3210 if (!$dbh && !$file_itr) || ($dbh && $file_itr);3239 if (!$dbh && !$file_itr) || ($dbh && $file_itr);
32113240
3241 my %resume;
3242 if ( my $table = $args{resume} ) {
3243 PTDEBUG && _d('Will resume from or after', $table);
3244 my ($db, $tbl) = $args{Quoter}->split_unquote($table);
3245 die "Resume table must be database-qualified: $table"
3246 unless $db && $tbl;
3247 $resume{db} = $db;
3248 $resume{tbl} = $tbl;
3249 }
3250
3212 my $self = {3251 my $self = {
3213 %args,3252 %args,
3253 resume => \%resume,
3214 filters => _make_filters(%args),3254 filters => _make_filters(%args),
3215 };3255 };
32163256
@@ -3271,9 +3311,19 @@
3271 return \%filters;3311 return \%filters;
3272}3312}
32733313
3274sub next_schema_object {3314sub next {
3275 my ( $self ) = @_;3315 my ( $self ) = @_;
32763316
3317 if ( !$self->{initialized} ) {
3318 $self->{initialized} = 1;
3319 if ( $self->{resume}->{tbl}
3320 && !$self->table_is_allowed(@{$self->{resume}}{qw(db tbl)}) ) {
3321 PTDEBUG && _d('Will resume after',
3322 join('.', @{$self->{resume}}{qw(db tbl)}));
3323 $self->{resume}->{after} = 1;
3324 }
3325 }
3326
3277 my $schema_obj;3327 my $schema_obj;
3278 if ( $self->{file_itr} ) {3328 if ( $self->{file_itr} ) {
3279 $schema_obj= $self->_iterate_files();3329 $schema_obj= $self->_iterate_files();
@@ -3283,19 +3333,13 @@
3283 }3333 }
32843334
3285 if ( $schema_obj ) {3335 if ( $schema_obj ) {
3286 if ( $schema_obj->{ddl} && $self->{TableParser} ) {
3287 $schema_obj->{tbl_struct}
3288 = $self->{TableParser}->parse($schema_obj->{ddl});
3289 }
3290
3291 delete $schema_obj->{ddl} unless $self->{keep_ddl};
3292
3293 if ( my $schema = $self->{Schema} ) {3336 if ( my $schema = $self->{Schema} ) {
3294 $schema->add_schema_object($schema_obj);3337 $schema->add_schema_object($schema_obj);
3295 }3338 }
3339 PTDEBUG && _d('Next schema object:',
3340 $schema_obj->{db}, $schema_obj->{tbl});
3296 }3341 }
32973342
3298 PTDEBUG && _d('Next schema object:', $schema_obj->{db}, $schema_obj->{tbl});
3299 return $schema_obj;3343 return $schema_obj;
3300}3344}
33013345
@@ -3321,7 +3365,8 @@
3321 my $db = $1; # XXX3365 my $db = $1; # XXX
3322 $db =~ s/^`//; # strip leading `3366 $db =~ s/^`//; # strip leading `
3323 $db =~ s/`$//; # and trailing `3367 $db =~ s/`$//; # and trailing `
3324 if ( $self->database_is_allowed($db) ) {3368 if ( $self->database_is_allowed($db)
3369 && $self->_resume_from_database($db) ) {
3325 $self->{db} = $db;3370 $self->{db} = $db;
3326 }3371 }
3327 }3372 }
@@ -3334,21 +3379,22 @@
3334 my ($tbl) = $chunk =~ m/$tbl_name/;3379 my ($tbl) = $chunk =~ m/$tbl_name/;
3335 $tbl =~ s/^\s*`//;3380 $tbl =~ s/^\s*`//;
3336 $tbl =~ s/`\s*$//;3381 $tbl =~ s/`\s*$//;
3337 if ( $self->table_is_allowed($self->{db}, $tbl) ) {3382 if ( $self->_resume_from_table($tbl)
3383 && $self->table_is_allowed($self->{db}, $tbl) ) {
3338 my ($ddl) = $chunk =~ m/^(?:$open_comment)?(CREATE TABLE.+?;)$/ms;3384 my ($ddl) = $chunk =~ m/^(?:$open_comment)?(CREATE TABLE.+?;)$/ms;
3339 if ( !$ddl ) {3385 if ( !$ddl ) {
3340 warn "Failed to parse CREATE TABLE from\n" . $chunk;3386 warn "Failed to parse CREATE TABLE from\n" . $chunk;
3341 next CHUNK;3387 next CHUNK;
3342 }3388 }
3343 $ddl =~ s/ \*\/;\Z/;/; # remove end of version comment3389 $ddl =~ s/ \*\/;\Z/;/; # remove end of version comment
33443390 my $tbl_struct = $self->{TableParser}->parse($ddl);
3345 my ($engine) = $ddl =~ m/\).*?(?:ENGINE|TYPE)=(\w+)/; 3391 if ( $self->engine_is_allowed($tbl_struct->{engine}) ) {
3346
3347 if ( !$engine || $self->engine_is_allowed($engine) ) {
3348 return {3392 return {
3349 db => $self->{db},3393 db => $self->{db},
3350 tbl => $tbl,3394 tbl => $tbl,
3351 ddl => $ddl,3395 name => $self->{Quoter}->quote($self->{db}, $tbl),
3396 ddl => $ddl,
3397 tbl_struct => $tbl_struct,
3352 };3398 };
3353 }3399 }
3354 }3400 }
@@ -3365,6 +3411,7 @@
3365sub _iterate_dbh {3411sub _iterate_dbh {
3366 my ( $self ) = @_;3412 my ( $self ) = @_;
3367 my $q = $self->{Quoter};3413 my $q = $self->{Quoter};
3414 my $tp = $self->{TableParser};
3368 my $dbh = $self->{dbh};3415 my $dbh = $self->{dbh};
3369 PTDEBUG && _d('Getting next schema object from dbh', $dbh);3416 PTDEBUG && _d('Getting next schema object from dbh', $dbh);
33703417
@@ -3378,7 +3425,9 @@
3378 }3425 }
33793426
3380 if ( !$self->{db} ) {3427 if ( !$self->{db} ) {
3381 $self->{db} = shift @{$self->{dbs}};3428 do {
3429 $self->{db} = shift @{$self->{dbs}};
3430 } until $self->_resume_from_database($self->{db});
3382 PTDEBUG && _d('Next database:', $self->{db});3431 PTDEBUG && _d('Next database:', $self->{db});
3383 return unless $self->{db};3432 return unless $self->{db};
3384 }3433 }
@@ -3391,8 +3440,9 @@
3391 }3440 }
3392 grep {3441 grep {
3393 my ($tbl, $type) = @$_;3442 my ($tbl, $type) = @$_;
3394 $self->table_is_allowed($self->{db}, $tbl)3443 (!$type || ($type ne 'VIEW'))
3395 && (!$type || ($type ne 'VIEW'));3444 && $self->_resume_from_table($tbl)
3445 && $self->table_is_allowed($self->{db}, $tbl);
3396 }3446 }
3397 @{$dbh->selectall_arrayref($sql)};3447 @{$dbh->selectall_arrayref($sql)};
3398 PTDEBUG && _d('Found', scalar @tbls, 'tables in database', $self->{db});3448 PTDEBUG && _d('Found', scalar @tbls, 'tables in database', $self->{db});
@@ -3400,27 +3450,15 @@
3400 }3450 }
34013451
3402 while ( my $tbl = shift @{$self->{tbls}} ) {3452 while ( my $tbl = shift @{$self->{tbls}} ) {
3403 my $engine;3453 my $ddl = $tp->get_create_table($dbh, $self->{db}, $tbl);
3404 if ( $self->{filters}->{'engines'}3454 my $tbl_struct = $tp->parse($ddl);
3405 || $self->{filters}->{'ignore-engines'} ) {3455 if ( $self->engine_is_allowed($tbl_struct->{engine}) ) {
3406 my $sql = "SHOW TABLE STATUS FROM " . $q->quote($self->{db})
3407 . " LIKE \'$tbl\'";
3408 PTDEBUG && _d($sql);
3409 $engine = $dbh->selectrow_hashref($sql)->{engine};
3410 PTDEBUG && _d($tbl, 'uses', $engine, 'engine');
3411 }
3412
3413
3414 if ( !$engine || $self->engine_is_allowed($engine) ) {
3415 my $ddl;
3416 if ( my $du = $self->{MySQLDump} ) {
3417 $ddl = $du->get_create_table($dbh, $q, $self->{db}, $tbl)->[1];
3418 }
3419
3420 return {3456 return {
3421 db => $self->{db},3457 db => $self->{db},
3422 tbl => $tbl,3458 tbl => $tbl,
3423 ddl => $ddl,3459 name => $q->quote($self->{db}, $tbl),
3460 ddl => $ddl,
3461 tbl_struct => $tbl_struct,
3424 };3462 };
3425 }3463 }
3426 }3464 }
@@ -3481,6 +3519,10 @@
34813519
3482 my $filter = $self->{filters};3520 my $filter = $self->{filters};
34833521
3522 if ( $db eq 'mysql' && ($tbl eq 'general_log' || $tbl eq 'slow_log') ) {
3523 return 0;
3524 }
3525
3484 if ( $filter->{'ignore-tables'}->{$tbl}3526 if ( $filter->{'ignore-tables'}->{$tbl}
3485 && ($filter->{'ignore-tables'}->{$tbl} eq '*'3527 && ($filter->{'ignore-tables'}->{$tbl} eq '*'
3486 || $filter->{'ignore-tables'}->{$tbl} eq $db) ) {3528 || $filter->{'ignore-tables'}->{$tbl} eq $db) ) {
@@ -3520,7 +3562,11 @@
35203562
3521sub engine_is_allowed {3563sub engine_is_allowed {
3522 my ( $self, $engine ) = @_;3564 my ( $self, $engine ) = @_;
3523 die "I need an engine argument" unless $engine;3565
3566 if ( !$engine ) {
3567 PTDEBUG && _d('No engine specified; allowing the table');
3568 return 1;
3569 }
35243570
3525 $engine = lc $engine;3571 $engine = lc $engine;
35263572
@@ -3540,6 +3586,40 @@
3540 return 1;3586 return 1;
3541}3587}
35423588
3589sub _resume_from_database {
3590 my ($self, $db) = @_;
3591
3592 return 1 unless $self->{resume}->{db};
3593
3594 if ( $db eq $self->{resume}->{db} ) {
3595 PTDEBUG && _d('At resume db', $db);
3596 delete $self->{resume}->{db};
3597 return 1;
3598 }
3599
3600 return 0;
3601}
3602
3603sub _resume_from_table {
3604 my ($self, $tbl) = @_;
3605
3606 return 1 unless $self->{resume}->{tbl};
3607
3608 if ( $tbl eq $self->{resume}->{tbl} ) {
3609 if ( !$self->{resume}->{after} ) {
3610 PTDEBUG && _d('Resuming from table', $tbl);
3611 delete $self->{resume}->{tbl};
3612 return 1;
3613 }
3614 else {
3615 PTDEBUG && _d('Resuming after table', $tbl);
3616 delete $self->{resume}->{tbl};
3617 }
3618 }
3619
3620 return 0;
3621}
3622
3543sub _d {3623sub _d {
3544 my ($package, undef, $line) = caller 0;3624 my ($package, undef, $line) = caller 0;
3545 @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }3625 @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
@@ -3649,11 +3729,10 @@
3649 MySQLDump => $du,3729 MySQLDump => $du,
3650 TableParser => $tp,3730 TableParser => $tp,
3651 Schema => $schema,3731 Schema => $schema,
3652 keep_ddl => 1,
3653 );3732 );
3654 TABLE:3733 TABLE:
3655 while ( my $tbl = $schema_itr->next_schema_object() ) {3734 while ( my $tbl = $schema_itr->next() ) {
3656 $tbl->{engine} = $tp->get_engine($tbl->{ddl});3735 $tbl->{engine} = $tbl->{tbl_struct}->{engine};
36573736
3658 my ($keys, $clustered_key, $fks);3737 my ($keys, $clustered_key, $fks);
3659 if ( $get_keys ) {3738 if ( $get_keys ) {
@@ -4279,6 +4358,6 @@
42794358
4280=head1 VERSION4359=head1 VERSION
42814360
4282pt-duplicate-key-checker 2.0.44361pt-duplicate-key-checker 2.1.1
42834362
4284=cut4363=cut
42854364
=== modified file 'bin/pt-fifo-split'
--- bin/pt-fifo-split 2012-03-07 23:41:54 +0000
+++ bin/pt-fifo-split 2012-05-30 21:00:30 +0000
@@ -1547,6 +1547,6 @@
15471547
1548=head1 VERSION1548=head1 VERSION
15491549
1550pt-fifo-split 2.0.41550pt-fifo-split 2.1.1
15511551
1552=cut1552=cut
15531553
=== modified file 'bin/pt-find'
--- bin/pt-find 2012-05-24 17:52:01 +0000
+++ bin/pt-find 2012-05-30 21:00:30 +0000
@@ -3827,6 +3827,6 @@
38273827
3828=head1 VERSION3828=head1 VERSION
38293829
3830pt-find 2.0.43830pt-find 2.1.1
38313831
3832=cut3832=cut
38333833
=== added file 'bin/pt-fingerprint'
--- bin/pt-fingerprint 1970-01-01 00:00:00 +0000
+++ bin/pt-fingerprint 2012-05-30 21:00:30 +0000
@@ -0,0 +1,2143 @@
1#!/usr/bin/env perl
2
3# This program is part of Percona Toolkit: http://www.percona.com/software/
4# See "COPYRIGHT, LICENSE, AND WARRANTY" at the end of this file for legal
5# notices and disclaimers.
6
7use strict;
8use warnings FATAL => 'all';
9use constant MKDEBUG => $ENV{MKDEBUG} || 0;
10
11# ###########################################################################
12# OptionParser package
13# This package is a copy without comments from the original. The original
14# with comments and its test file can be found in the Bazaar repository at,
15# lib/OptionParser.pm
16# t/lib/OptionParser.t
17# See https://launchpad.net/percona-toolkit for more information.
18# ###########################################################################
19{
20package OptionParser;
21
22use strict;
23use warnings FATAL => 'all';
24use English qw(-no_match_vars);
25use constant PTDEBUG => $ENV{PTDEBUG} || 0;
26
27use List::Util qw(max);
28use Getopt::Long;
29
30my $POD_link_re = '[LC]<"?([^">]+)"?>';
31
32sub new {
33 my ( $class, %args ) = @_;
34 my @required_args = qw();
35 foreach my $arg ( @required_args ) {
36 die "I need a $arg argument" unless $args{$arg};
37 }
38
39 my ($program_name) = $PROGRAM_NAME =~ m/([.A-Za-z-]+)$/;
40 $program_name ||= $PROGRAM_NAME;
41 my $home = $ENV{HOME} || $ENV{HOMEPATH} || $ENV{USERPROFILE} || '.';
42
43 my %attributes = (
44 'type' => 1,
45 'short form' => 1,
46 'group' => 1,
47 'default' => 1,
48 'cumulative' => 1,
49 'negatable' => 1,
50 );
51
52 my $self = {
53 head1 => 'OPTIONS', # These args are used internally
54 skip_rules => 0, # to instantiate another Option-
55 item => '--(.*)', # Parser obj that parses the
56 attributes => \%attributes, # DSN OPTIONS section. Tools
57 parse_attributes => \&_parse_attribs, # don't tinker with these args.
58
59 %args,
60
61 strict => 1, # disabled by a special rule
62 program_name => $program_name,
63 opts => {},
64 got_opts => 0,
65 short_opts => {},
66 defaults => {},
67 groups => {},
68 allowed_groups => {},
69 errors => [],
70 rules => [], # desc of rules for --help
71 mutex => [], # rule: opts are mutually exclusive
72 atleast1 => [], # rule: at least one opt is required
73 disables => {}, # rule: opt disables other opts
74 defaults_to => {}, # rule: opt defaults to value of other opt
75 DSNParser => undef,
76 default_files => [
77 "/etc/percona-toolkit/percona-toolkit.conf",
78 "/etc/percona-toolkit/$program_name.conf",
79 "$home/.percona-toolkit.conf",
80 "$home/.$program_name.conf",
81 ],
82 types => {
83 string => 's', # standard Getopt type
84 int => 'i', # standard Getopt type
85 float => 'f', # standard Getopt type
86 Hash => 'H', # hash, formed from a comma-separated list
87 hash => 'h', # hash as above, but only if a value is given
88 Array => 'A', # array, similar to Hash
89 array => 'a', # array, similar to hash
90 DSN => 'd', # DSN
91 size => 'z', # size with kMG suffix (powers of 2^10)
92 time => 'm', # time, with an optional suffix of s/h/m/d
93 },
94 };
95
96 return bless $self, $class;
97}
98
99sub get_specs {
100 my ( $self, $file ) = @_;
101 $file ||= $self->{file} || __FILE__;
102 my @specs = $self->_pod_to_specs($file);
103 $self->_parse_specs(@specs);
104
105 open my $fh, "<", $file or die "Cannot open $file: $OS_ERROR";
106 my $contents = do { local $/ = undef; <$fh> };
107 close $fh;
108 if ( $contents =~ m/^=head1 DSN OPTIONS/m ) {
109 PTDEBUG && _d('Parsing DSN OPTIONS');
110 my $dsn_attribs = {
111 dsn => 1,
112 copy => 1,
113 };
114 my $parse_dsn_attribs = sub {
115 my ( $self, $option, $attribs ) = @_;
116 map {
117 my $val = $attribs->{$_};
118 if ( $val ) {
119 $val = $val eq 'yes' ? 1
120 : $val eq 'no' ? 0
121 : $val;
122 $attribs->{$_} = $val;
123 }
124 } keys %$attribs;
125 return {
126 key => $option,
127 %$attribs,
128 };
129 };
130 my $dsn_o = new OptionParser(
131 description => 'DSN OPTIONS',
132 head1 => 'DSN OPTIONS',
133 dsn => 0, # XXX don't infinitely recurse!
134 item => '\* (.)', # key opts are a single character
135 skip_rules => 1, # no rules before opts
136 attributes => $dsn_attribs,
137 parse_attributes => $parse_dsn_attribs,
138 );
139 my @dsn_opts = map {
140 my $opts = {
141 key => $_->{spec}->{key},
142 dsn => $_->{spec}->{dsn},
143 copy => $_->{spec}->{copy},
144 desc => $_->{desc},
145 };
146 $opts;
147 } $dsn_o->_pod_to_specs($file);
148 $self->{DSNParser} = DSNParser->new(opts => \@dsn_opts);
149 }
150
151 if ( $contents =~ m/^=head1 VERSION\n\n^(.+)$/m ) {
152 $self->{version} = $1;
153 PTDEBUG && _d($self->{version});
154 }
155
156 return;
157}
158
159sub DSNParser {
160 my ( $self ) = @_;
161 return $self->{DSNParser};
162};
163
164sub get_defaults_files {
165 my ( $self ) = @_;
166 return @{$self->{default_files}};
167}
168
169sub _pod_to_specs {
170 my ( $self, $file ) = @_;
171 $file ||= $self->{file} || __FILE__;
172 open my $fh, '<', $file or die "Cannot open $file: $OS_ERROR";
173
174 my @specs = ();
175 my @rules = ();
176 my $para;
177
178 local $INPUT_RECORD_SEPARATOR = '';
179 while ( $para = <$fh> ) {
180 next unless $para =~ m/^=head1 $self->{head1}/;
181 last;
182 }
183
184 while ( $para = <$fh> ) {
185 last if $para =~ m/^=over/;
186 next if $self->{skip_rules};
187 chomp $para;
188 $para =~ s/\s+/ /g;
189 $para =~ s/$POD_link_re/$1/go;
190 PTDEBUG && _d('Option rule:', $para);
191 push @rules, $para;
192 }
193
194 die "POD has no $self->{head1} section" unless $para;
195
196 do {
197 if ( my ($option) = $para =~ m/^=item $self->{item}/ ) {
198 chomp $para;
199 PTDEBUG && _d($para);
200 my %attribs;
201
202 $para = <$fh>; # read next paragraph, possibly attributes
203
204 if ( $para =~ m/: / ) { # attributes
205 $para =~ s/\s+\Z//g;
206 %attribs = map {
207 my ( $attrib, $val) = split(/: /, $_);
208 die "Unrecognized attribute for --$option: $attrib"
209 unless $self->{attributes}->{$attrib};
210 ($attrib, $val);
211 } split(/; /, $para);
212 if ( $attribs{'short form'} ) {
213 $attribs{'short form'} =~ s/-//;
214 }
215 $para = <$fh>; # read next paragraph, probably short help desc
216 }
217 else {
218 PTDEBUG && _d('Option has no attributes');
219 }
220
221 $para =~ s/\s+\Z//g;
222 $para =~ s/\s+/ /g;
223 $para =~ s/$POD_link_re/$1/go;
224
225 $para =~ s/\.(?:\n.*| [A-Z].*|\Z)//s;
226 PTDEBUG && _d('Short help:', $para);
227
228 die "No description after option spec $option" if $para =~ m/^=item/;
229
230 if ( my ($base_option) = $option =~ m/^\[no\](.*)/ ) {
231 $option = $base_option;
232 $attribs{'negatable'} = 1;
233 }
234
235 push @specs, {
236 spec => $self->{parse_attributes}->($self, $option, \%attribs),
237 desc => $para
238 . (defined $attribs{default} ? " (default $attribs{default})" : ''),
239 group => ($attribs{'group'} ? $attribs{'group'} : 'default'),
240 };
241 }
242 while ( $para = <$fh> ) {
243 last unless $para;
244 if ( $para =~ m/^=head1/ ) {
245 $para = undef; # Can't 'last' out of a do {} block.
246 last;
247 }
248 last if $para =~ m/^=item /;
249 }
250 } while ( $para );
251
252 die "No valid specs in $self->{head1}" unless @specs;
253
254 close $fh;
255 return @specs, @rules;
256}
257
258sub _parse_specs {
259 my ( $self, @specs ) = @_;
260 my %disables; # special rule that requires deferred checking
261
262 foreach my $opt ( @specs ) {
263 if ( ref $opt ) { # It's an option spec, not a rule.
264 PTDEBUG && _d('Parsing opt spec:',
265 map { ($_, '=>', $opt->{$_}) } keys %$opt);
266
267 my ( $long, $short ) = $opt->{spec} =~ m/^([\w-]+)(?:\|([^!+=]*))?/;
268 if ( !$long ) {
269 die "Cannot parse long option from spec $opt->{spec}";
270 }
271 $opt->{long} = $long;
272
273 die "Duplicate long option --$long" if exists $self->{opts}->{$long};
274 $self->{opts}->{$long} = $opt;
275
276 if ( length $long == 1 ) {
277 PTDEBUG && _d('Long opt', $long, 'looks like short opt');
278 $self->{short_opts}->{$long} = $long;
279 }
280
281 if ( $short ) {
282 die "Duplicate short option -$short"
283 if exists $self->{short_opts}->{$short};
284 $self->{short_opts}->{$short} = $long;
285 $opt->{short} = $short;
286 }
287 else {
288 $opt->{short} = undef;
289 }
290
291 $opt->{is_negatable} = $opt->{spec} =~ m/!/ ? 1 : 0;
292 $opt->{is_cumulative} = $opt->{spec} =~ m/\+/ ? 1 : 0;
293 $opt->{is_required} = $opt->{desc} =~ m/required/ ? 1 : 0;
294
295 $opt->{group} ||= 'default';
296 $self->{groups}->{ $opt->{group} }->{$long} = 1;
297
298 $opt->{value} = undef;
299 $opt->{got} = 0;
300
301 my ( $type ) = $opt->{spec} =~ m/=(.)/;
302 $opt->{type} = $type;
303 PTDEBUG && _d($long, 'type:', $type);
304
305
306 $opt->{spec} =~ s/=./=s/ if ( $type && $type =~ m/[HhAadzm]/ );
307
308 if ( (my ($def) = $opt->{desc} =~ m/default\b(?: ([^)]+))?/) ) {
309 $self->{defaults}->{$long} = defined $def ? $def : 1;
310 PTDEBUG && _d($long, 'default:', $def);
311 }
312
313 if ( $long eq 'config' ) {
314 $self->{defaults}->{$long} = join(',', $self->get_defaults_files());
315 }
316
317 if ( (my ($dis) = $opt->{desc} =~ m/(disables .*)/) ) {
318 $disables{$long} = $dis;
319 PTDEBUG && _d('Deferring check of disables rule for', $opt, $dis);
320 }
321
322 $self->{opts}->{$long} = $opt;
323 }
324 else { # It's an option rule, not a spec.
325 PTDEBUG && _d('Parsing rule:', $opt);
326 push @{$self->{rules}}, $opt;
327 my @participants = $self->_get_participants($opt);
328 my $rule_ok = 0;
329
330 if ( $opt =~ m/mutually exclusive|one and only one/ ) {
331 $rule_ok = 1;
332 push @{$self->{mutex}}, \@participants;
333 PTDEBUG && _d(@participants, 'are mutually exclusive');
334 }
335 if ( $opt =~ m/at least one|one and only one/ ) {
336 $rule_ok = 1;
337 push @{$self->{atleast1}}, \@participants;
338 PTDEBUG && _d(@participants, 'require at least one');
339 }
340 if ( $opt =~ m/default to/ ) {
341 $rule_ok = 1;
342 $self->{defaults_to}->{$participants[0]} = $participants[1];
343 PTDEBUG && _d($participants[0], 'defaults to', $participants[1]);
344 }
345 if ( $opt =~ m/restricted to option groups/ ) {
346 $rule_ok = 1;
347 my ($groups) = $opt =~ m/groups ([\w\s\,]+)/;
348 my @groups = split(',', $groups);
349 %{$self->{allowed_groups}->{$participants[0]}} = map {
350 s/\s+//;
351 $_ => 1;
352 } @groups;
353 }
354 if( $opt =~ m/accepts additional command-line arguments/ ) {
355 $rule_ok = 1;
356 $self->{strict} = 0;
357 PTDEBUG && _d("Strict mode disabled by rule");
358 }
359
360 die "Unrecognized option rule: $opt" unless $rule_ok;
361 }
362 }
363
364 foreach my $long ( keys %disables ) {
365 my @participants = $self->_get_participants($disables{$long});
366 $self->{disables}->{$long} = \@participants;
367 PTDEBUG && _d('Option', $long, 'disables', @participants);
368 }
369
370 return;
371}
372
373sub _get_participants {
374 my ( $self, $str ) = @_;
375 my @participants;
376 foreach my $long ( $str =~ m/--(?:\[no\])?([\w-]+)/g ) {
377 die "Option --$long does not exist while processing rule $str"
378 unless exists $self->{opts}->{$long};
379 push @participants, $long;
380 }
381 PTDEBUG && _d('Participants for', $str, ':', @participants);
382 return @participants;
383}
384
385sub opts {
386 my ( $self ) = @_;
387 my %opts = %{$self->{opts}};
388 return %opts;
389}
390
391sub short_opts {
392 my ( $self ) = @_;
393 my %short_opts = %{$self->{short_opts}};
394 return %short_opts;
395}
396
397sub set_defaults {
398 my ( $self, %defaults ) = @_;
399 $self->{defaults} = {};
400 foreach my $long ( keys %defaults ) {
401 die "Cannot set default for nonexistent option $long"
402 unless exists $self->{opts}->{$long};
403 $self->{defaults}->{$long} = $defaults{$long};
404 PTDEBUG && _d('Default val for', $long, ':', $defaults{$long});
405 }
406 return;
407}
408
409sub get_defaults {
410 my ( $self ) = @_;
411 return $self->{defaults};
412}
413
414sub get_groups {
415 my ( $self ) = @_;
416 return $self->{groups};
417}
418
419sub _set_option {
420 my ( $self, $opt, $val ) = @_;
421 my $long = exists $self->{opts}->{$opt} ? $opt
422 : exists $self->{short_opts}->{$opt} ? $self->{short_opts}->{$opt}
423 : die "Getopt::Long gave a nonexistent option: $opt";
424
425 $opt = $self->{opts}->{$long};
426 if ( $opt->{is_cumulative} ) {
427 $opt->{value}++;
428 }
429 else {
430 $opt->{value} = $val;
431 }
432 $opt->{got} = 1;
433 PTDEBUG && _d('Got option', $long, '=', $val);
434}
435
436sub get_opts {
437 my ( $self ) = @_;
438
439 foreach my $long ( keys %{$self->{opts}} ) {
440 $self->{opts}->{$long}->{got} = 0;
441 $self->{opts}->{$long}->{value}
442 = exists $self->{defaults}->{$long} ? $self->{defaults}->{$long}
443 : $self->{opts}->{$long}->{is_cumulative} ? 0
444 : undef;
445 }
446 $self->{got_opts} = 0;
447
448 $self->{errors} = [];
449
450 if ( @ARGV && $ARGV[0] eq "--config" ) {
451 shift @ARGV;
452 $self->_set_option('config', shift @ARGV);
453 }
454 if ( $self->has('config') ) {
455 my @extra_args;
456 foreach my $filename ( split(',', $self->get('config')) ) {
457 eval {
458 push @extra_args, $self->_read_config_file($filename);
459 };
460 if ( $EVAL_ERROR ) {
461 if ( $self->got('config') ) {
462 die $EVAL_ERROR;
463 }
464 elsif ( PTDEBUG ) {
465 _d($EVAL_ERROR);
466 }
467 }
468 }
469 unshift @ARGV, @extra_args;
470 }
471
472 Getopt::Long::Configure('no_ignore_case', 'bundling');
473 GetOptions(
474 map { $_->{spec} => sub { $self->_set_option(@_); } }
475 grep { $_->{long} ne 'config' } # --config is handled specially above.
476 values %{$self->{opts}}
477 ) or $self->save_error('Error parsing options');
478
479 if ( exists $self->{opts}->{version} && $self->{opts}->{version}->{got} ) {
480 if ( $self->{version} ) {
481 print $self->{version}, "\n";
482 }
483 else {
484 print "Error parsing version. See the VERSION section of the tool's documentation.\n";
485 }
486 exit 0;
487 }
488
489 if ( @ARGV && $self->{strict} ) {
490 $self->save_error("Unrecognized command-line options @ARGV");
491 }
492
493 foreach my $mutex ( @{$self->{mutex}} ) {
494 my @set = grep { $self->{opts}->{$_}->{got} } @$mutex;
495 if ( @set > 1 ) {
496 my $err = join(', ', map { "--$self->{opts}->{$_}->{long}" }
497 @{$mutex}[ 0 .. scalar(@$mutex) - 2] )
498 . ' and --'.$self->{opts}->{$mutex->[-1]}->{long}
499 . ' are mutually exclusive.';
500 $self->save_error($err);
501 }
502 }
503
504 foreach my $required ( @{$self->{atleast1}} ) {
505 my @set = grep { $self->{opts}->{$_}->{got} } @$required;
506 if ( @set == 0 ) {
507 my $err = join(', ', map { "--$self->{opts}->{$_}->{long}" }
508 @{$required}[ 0 .. scalar(@$required) - 2] )
509 .' or --'.$self->{opts}->{$required->[-1]}->{long};
510 $self->save_error("Specify at least one of $err");
511 }
512 }
513
514 $self->_check_opts( keys %{$self->{opts}} );
515 $self->{got_opts} = 1;
516 return;
517}
518
519sub _check_opts {
520 my ( $self, @long ) = @_;
521 my $long_last = scalar @long;
522 while ( @long ) {
523 foreach my $i ( 0..$#long ) {
524 my $long = $long[$i];
525 next unless $long;
526 my $opt = $self->{opts}->{$long};
527 if ( $opt->{got} ) {
528 if ( exists $self->{disables}->{$long} ) {
529 my @disable_opts = @{$self->{disables}->{$long}};
530 map { $self->{opts}->{$_}->{value} = undef; } @disable_opts;
531 PTDEBUG && _d('Unset options', @disable_opts,
532 'because', $long,'disables them');
533 }
534
535 if ( exists $self->{allowed_groups}->{$long} ) {
536
537 my @restricted_groups = grep {
538 !exists $self->{allowed_groups}->{$long}->{$_}
539 } keys %{$self->{groups}};
540
541 my @restricted_opts;
542 foreach my $restricted_group ( @restricted_groups ) {
543 RESTRICTED_OPT:
544 foreach my $restricted_opt (
545 keys %{$self->{groups}->{$restricted_group}} )
546 {
547 next RESTRICTED_OPT if $restricted_opt eq $long;
548 push @restricted_opts, $restricted_opt
549 if $self->{opts}->{$restricted_opt}->{got};
550 }
551 }
552
553 if ( @restricted_opts ) {
554 my $err;
555 if ( @restricted_opts == 1 ) {
556 $err = "--$restricted_opts[0]";
557 }
558 else {
559 $err = join(', ',
560 map { "--$self->{opts}->{$_}->{long}" }
561 grep { $_ }
562 @restricted_opts[0..scalar(@restricted_opts) - 2]
563 )
564 . ' or --'.$self->{opts}->{$restricted_opts[-1]}->{long};
565 }
566 $self->save_error("--$long is not allowed with $err");
567 }
568 }
569
570 }
571 elsif ( $opt->{is_required} ) {
572 $self->save_error("Required option --$long must be specified");
573 }
574
575 $self->_validate_type($opt);
576 if ( $opt->{parsed} ) {
577 delete $long[$i];
578 }
579 else {
580 PTDEBUG && _d('Temporarily failed to parse', $long);
581 }
582 }
583
584 die "Failed to parse options, possibly due to circular dependencies"
585 if @long == $long_last;
586 $long_last = @long;
587 }
588
589 return;
590}
591
592sub _validate_type {
593 my ( $self, $opt ) = @_;
594 return unless $opt;
595
596 if ( !$opt->{type} ) {
597 $opt->{parsed} = 1;
598 return;
599 }
600
601 my $val = $opt->{value};
602
603 if ( $val && $opt->{type} eq 'm' ) { # type time
604 PTDEBUG && _d('Parsing option', $opt->{long}, 'as a time value');
605 my ( $prefix, $num, $suffix ) = $val =~ m/([+-]?)(\d+)([a-z])?$/;
606 if ( !$suffix ) {
607 my ( $s ) = $opt->{desc} =~ m/\(suffix (.)\)/;
608 $suffix = $s || 's';
609 PTDEBUG && _d('No suffix given; using', $suffix, 'for',
610 $opt->{long}, '(value:', $val, ')');
611 }
612 if ( $suffix =~ m/[smhd]/ ) {
613 $val = $suffix eq 's' ? $num # Seconds
614 : $suffix eq 'm' ? $num * 60 # Minutes
615 : $suffix eq 'h' ? $num * 3600 # Hours
616 : $num * 86400; # Days
617 $opt->{value} = ($prefix || '') . $val;
618 PTDEBUG && _d('Setting option', $opt->{long}, 'to', $val);
619 }
620 else {
621 $self->save_error("Invalid time suffix for --$opt->{long}");
622 }
623 }
624 elsif ( $val && $opt->{type} eq 'd' ) { # type DSN
625 PTDEBUG && _d('Parsing option', $opt->{long}, 'as a DSN');
626 my $prev = {};
627 my $from_key = $self->{defaults_to}->{ $opt->{long} };
628 if ( $from_key ) {
629 PTDEBUG && _d($opt->{long}, 'DSN copies from', $from_key, 'DSN');
630 if ( $self->{opts}->{$from_key}->{parsed} ) {
631 $prev = $self->{opts}->{$from_key}->{value};
632 }
633 else {
634 PTDEBUG && _d('Cannot parse', $opt->{long}, 'until',
635 $from_key, 'parsed');
636 return;
637 }
638 }
639 my $defaults = $self->{DSNParser}->parse_options($self);
640 $opt->{value} = $self->{DSNParser}->parse($val, $prev, $defaults);
641 }
642 elsif ( $val && $opt->{type} eq 'z' ) { # type size
643 PTDEBUG && _d('Parsing option', $opt->{long}, 'as a size value');
644 $self->_parse_size($opt, $val);
645 }
646 elsif ( $opt->{type} eq 'H' || (defined $val && $opt->{type} eq 'h') ) {
647 $opt->{value} = { map { $_ => 1 } split(/(?<!\\),\s*/, ($val || '')) };
648 }
649 elsif ( $opt->{type} eq 'A' || (defined $val && $opt->{type} eq 'a') ) {
650 $opt->{value} = [ split(/(?<!\\),\s*/, ($val || '')) ];
651 }
652 else {
653 PTDEBUG && _d('Nothing to validate for option',
654 $opt->{long}, 'type', $opt->{type}, 'value', $val);
655 }
656
657 $opt->{parsed} = 1;
658 return;
659}
660
661sub get {
662 my ( $self, $opt ) = @_;
663 my $long = (length $opt == 1 ? $self->{short_opts}->{$opt} : $opt);
664 die "Option $opt does not exist"
665 unless $long && exists $self->{opts}->{$long};
666 return $self->{opts}->{$long}->{value};
667}
668
669sub got {
670 my ( $self, $opt ) = @_;
671 my $long = (length $opt == 1 ? $self->{short_opts}->{$opt} : $opt);
672 die "Option $opt does not exist"
673 unless $long && exists $self->{opts}->{$long};
674 return $self->{opts}->{$long}->{got};
675}
676
677sub has {
678 my ( $self, $opt ) = @_;
679 my $long = (length $opt == 1 ? $self->{short_opts}->{$opt} : $opt);
680 return defined $long ? exists $self->{opts}->{$long} : 0;
681}
682
683sub set {
684 my ( $self, $opt, $val ) = @_;
685 my $long = (length $opt == 1 ? $self->{short_opts}->{$opt} : $opt);
686 die "Option $opt does not exist"
687 unless $long && exists $self->{opts}->{$long};
688 $self->{opts}->{$long}->{value} = $val;
689 return;
690}
691
692sub save_error {
693 my ( $self, $error ) = @_;
694 push @{$self->{errors}}, $error;
695 return;
696}
697
698sub errors {
699 my ( $self ) = @_;
700 return $self->{errors};
701}
702
703sub usage {
704 my ( $self ) = @_;
705 warn "No usage string is set" unless $self->{usage}; # XXX
706 return "Usage: " . ($self->{usage} || '') . "\n";
707}
708
709sub descr {
710 my ( $self ) = @_;
711 warn "No description string is set" unless $self->{description}; # XXX
712 my $descr = ($self->{description} || $self->{program_name} || '')
713 . " For more details, please use the --help option, "
714 . "or try 'perldoc $PROGRAM_NAME' "
715 . "for complete documentation.";
716 $descr = join("\n", $descr =~ m/(.{0,80})(?:\s+|$)/g)
717 unless $ENV{DONT_BREAK_LINES};
718 $descr =~ s/ +$//mg;
719 return $descr;
720}
721
722sub usage_or_errors {
723 my ( $self, $file, $return ) = @_;
724 $file ||= $self->{file} || __FILE__;
725
726 if ( !$self->{description} || !$self->{usage} ) {
727 PTDEBUG && _d("Getting description and usage from SYNOPSIS in", $file);
728 my %synop = $self->_parse_synopsis($file);
729 $self->{description} ||= $synop{description};
730 $self->{usage} ||= $synop{usage};
731 PTDEBUG && _d("Description:", $self->{description},
732 "\nUsage:", $self->{usage});
733 }
734
735 if ( $self->{opts}->{help}->{got} ) {
736 print $self->print_usage() or die "Cannot print usage: $OS_ERROR";
737 exit 0 unless $return;
738 }
739 elsif ( scalar @{$self->{errors}} ) {
740 print $self->print_errors() or die "Cannot print errors: $OS_ERROR";
741 exit 0 unless $return;
742 }
743
744 return;
745}
746
747sub print_errors {
748 my ( $self ) = @_;
749 my $usage = $self->usage() . "\n";
750 if ( (my @errors = @{$self->{errors}}) ) {
751 $usage .= join("\n * ", 'Errors in command-line arguments:', @errors)
752 . "\n";
753 }
754 return $usage . "\n" . $self->descr();
755}
756
757sub print_usage {
758 my ( $self ) = @_;
759 die "Run get_opts() before print_usage()" unless $self->{got_opts};
760 my @opts = values %{$self->{opts}};
761
762 my $maxl = max(
763 map {
764 length($_->{long}) # option long name
765 + ($_->{is_negatable} ? 4 : 0) # "[no]" if opt is negatable
766 + ($_->{type} ? 2 : 0) # "=x" where x is the opt type
767 }
768 @opts);
769
770 my $maxs = max(0,
771 map {
772 length($_)
773 + ($self->{opts}->{$_}->{is_negatable} ? 4 : 0)
774 + ($self->{opts}->{$_}->{type} ? 2 : 0)
775 }
776 values %{$self->{short_opts}});
777
778 my $lcol = max($maxl, ($maxs + 3));
779 my $rcol = 80 - $lcol - 6;
780 my $rpad = ' ' x ( 80 - $rcol );
781
782 $maxs = max($lcol - 3, $maxs);
783
784 my $usage = $self->descr() . "\n" . $self->usage();
785
786 my @groups = reverse sort grep { $_ ne 'default'; } keys %{$self->{groups}};
787 push @groups, 'default';
788
789 foreach my $group ( reverse @groups ) {
790 $usage .= "\n".($group eq 'default' ? 'Options' : $group).":\n\n";
791 foreach my $opt (
792 sort { $a->{long} cmp $b->{long} }
793 grep { $_->{group} eq $group }
794 @opts )
795 {
796 my $long = $opt->{is_negatable} ? "[no]$opt->{long}" : $opt->{long};
797 my $short = $opt->{short};
798 my $desc = $opt->{desc};
799
800 $long .= $opt->{type} ? "=$opt->{type}" : "";
801
802 if ( $opt->{type} && $opt->{type} eq 'm' ) {
803 my ($s) = $desc =~ m/\(suffix (.)\)/;
804 $s ||= 's';
805 $desc =~ s/\s+\(suffix .\)//;
806 $desc .= ". Optional suffix s=seconds, m=minutes, h=hours, "
807 . "d=days; if no suffix, $s is used.";
808 }
809 $desc = join("\n$rpad", grep { $_ } $desc =~ m/(.{0,$rcol})(?:\s+|$)/g);
810 $desc =~ s/ +$//mg;
811 if ( $short ) {
812 $usage .= sprintf(" --%-${maxs}s -%s %s\n", $long, $short, $desc);
813 }
814 else {
815 $usage .= sprintf(" --%-${lcol}s %s\n", $long, $desc);
816 }
817 }
818 }
819
820 $usage .= "\nOption types: s=string, i=integer, f=float, h/H/a/A=comma-separated list, d=DSN, z=size, m=time\n";
821
822 if ( (my @rules = @{$self->{rules}}) ) {
823 $usage .= "\nRules:\n\n";
824 $usage .= join("\n", map { " $_" } @rules) . "\n";
825 }
826 if ( $self->{DSNParser} ) {
827 $usage .= "\n" . $self->{DSNParser}->usage();
828 }
829 $usage .= "\nOptions and values after processing arguments:\n\n";
830 foreach my $opt ( sort { $a->{long} cmp $b->{long} } @opts ) {
831 my $val = $opt->{value};
832 my $type = $opt->{type} || '';
833 my $bool = $opt->{spec} =~ m/^[\w-]+(?:\|[\w-])?!?$/;
834 $val = $bool ? ( $val ? 'TRUE' : 'FALSE' )
835 : !defined $val ? '(No value)'
836 : $type eq 'd' ? $self->{DSNParser}->as_string($val)
837 : $type =~ m/H|h/ ? join(',', sort keys %$val)
838 : $type =~ m/A|a/ ? join(',', @$val)
839 : $val;
840 $usage .= sprintf(" --%-${lcol}s %s\n", $opt->{long}, $val);
841 }
842 return $usage;
843}
844
845sub prompt_noecho {
846 shift @_ if ref $_[0] eq __PACKAGE__;
847 my ( $prompt ) = @_;
848 local $OUTPUT_AUTOFLUSH = 1;
849 print $prompt
850 or die "Cannot print: $OS_ERROR";
851 my $response;
852 eval {
853 require Term::ReadKey;
854 Term::ReadKey::ReadMode('noecho');
855 chomp($response = <STDIN>);
856 Term::ReadKey::ReadMode('normal');
857 print "\n"
858 or die "Cannot print: $OS_ERROR";
859 };
860 if ( $EVAL_ERROR ) {
861 die "Cannot read response; is Term::ReadKey installed? $EVAL_ERROR";
862 }
863 return $response;
864}
865
866sub _read_config_file {
867 my ( $self, $filename ) = @_;
868 open my $fh, "<", $filename or die "Cannot open $filename: $OS_ERROR\n";
869 my @args;
870 my $prefix = '--';
871 my $parse = 1;
872
873 LINE:
874 while ( my $line = <$fh> ) {
875 chomp $line;
876 next LINE if $line =~ m/^\s*(?:\#|\;|$)/;
877 $line =~ s/\s+#.*$//g;
878 $line =~ s/^\s+|\s+$//g;
879 if ( $line eq '--' ) {
880 $prefix = '';
881 $parse = 0;
882 next LINE;
883 }
884 if ( $parse
885 && (my($opt, $arg) = $line =~ m/^\s*([^=\s]+?)(?:\s*=\s*(.*?)\s*)?$/)
886 ) {
887 push @args, grep { defined $_ } ("$prefix$opt", $arg);
888 }
889 elsif ( $line =~ m/./ ) {
890 push @args, $line;
891 }
892 else {
893 die "Syntax error in file $filename at line $INPUT_LINE_NUMBER";
894 }
895 }
896 close $fh;
897 return @args;
898}
899
900sub read_para_after {
901 my ( $self, $file, $regex ) = @_;
902 open my $fh, "<", $file or die "Can't open $file: $OS_ERROR";
903 local $INPUT_RECORD_SEPARATOR = '';
904 my $para;
905 while ( $para = <$fh> ) {
906 next unless $para =~ m/^=pod$/m;
907 last;
908 }
909 while ( $para = <$fh> ) {
910 next unless $para =~ m/$regex/;
911 last;
912 }
913 $para = <$fh>;
914 chomp($para);
915 close $fh or die "Can't close $file: $OS_ERROR";
916 return $para;
917}
918
919sub clone {
920 my ( $self ) = @_;
921
922 my %clone = map {
923 my $hashref = $self->{$_};
924 my $val_copy = {};
925 foreach my $key ( keys %$hashref ) {
926 my $ref = ref $hashref->{$key};
927 $val_copy->{$key} = !$ref ? $hashref->{$key}
928 : $ref eq 'HASH' ? { %{$hashref->{$key}} }
929 : $ref eq 'ARRAY' ? [ @{$hashref->{$key}} ]
930 : $hashref->{$key};
931 }
932 $_ => $val_copy;
933 } qw(opts short_opts defaults);
934
935 foreach my $scalar ( qw(got_opts) ) {
936 $clone{$scalar} = $self->{$scalar};
937 }
938
939 return bless \%clone;
940}
941
942sub _parse_size {
943 my ( $self, $opt, $val ) = @_;
944
945 if ( lc($val || '') eq 'null' ) {
946 PTDEBUG && _d('NULL size for', $opt->{long});
947 $opt->{value} = 'null';
948 return;
949 }
950
951 my %factor_for = (k => 1_024, M => 1_048_576, G => 1_073_741_824);
952 my ($pre, $num, $factor) = $val =~ m/^([+-])?(\d+)([kMG])?$/;
953 if ( defined $num ) {
954 if ( $factor ) {
955 $num *= $factor_for{$factor};
956 PTDEBUG && _d('Setting option', $opt->{y},
957 'to num', $num, '* factor', $factor);
958 }
959 $opt->{value} = ($pre || '') . $num;
960 }
961 else {
962 $self->save_error("Invalid size for --$opt->{long}: $val");
963 }
964 return;
965}
966
967sub _parse_attribs {
968 my ( $self, $option, $attribs ) = @_;
969 my $types = $self->{types};
970 return $option
971 . ($attribs->{'short form'} ? '|' . $attribs->{'short form'} : '' )
972 . ($attribs->{'negatable'} ? '!' : '' )
973 . ($attribs->{'cumulative'} ? '+' : '' )
974 . ($attribs->{'type'} ? '=' . $types->{$attribs->{type}} : '' );
975}
976
977sub _parse_synopsis {
978 my ( $self, $file ) = @_;
979 $file ||= $self->{file} || __FILE__;
980 PTDEBUG && _d("Parsing SYNOPSIS in", $file);
981
982 local $INPUT_RECORD_SEPARATOR = ''; # read paragraphs
983 open my $fh, "<", $file or die "Cannot open $file: $OS_ERROR";
984 my $para;
985 1 while defined($para = <$fh>) && $para !~ m/^=head1 SYNOPSIS/;
986 die "$file does not contain a SYNOPSIS section" unless $para;
987 my @synop;
988 for ( 1..2 ) { # 1 for the usage, 2 for the description
989 my $para = <$fh>;
990 push @synop, $para;
991 }
992 close $fh;
993 PTDEBUG && _d("Raw SYNOPSIS text:", @synop);
994 my ($usage, $desc) = @synop;
995 die "The SYNOPSIS section in $file is not formatted properly"
996 unless $usage && $desc;
997
998 $usage =~ s/^\s*Usage:\s+(.+)/$1/;
999 chomp $usage;
1000
1001 $desc =~ s/\n/ /g;
1002 $desc =~ s/\s{2,}/ /g;
1003 $desc =~ s/\. ([A-Z][a-z])/. $1/g;
1004 $desc =~ s/\s+$//;
1005
1006 return (
1007 description => $desc,
1008 usage => $usage,
1009 );
1010};
1011
1012sub _d {
1013 my ($package, undef, $line) = caller 0;
1014 @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
1015 map { defined $_ ? $_ : 'undef' }
1016 @_;
1017 print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
1018}
1019
1020if ( PTDEBUG ) {
1021 print '# ', $^X, ' ', $], "\n";
1022 if ( my $uname = `uname -a` ) {
1023 $uname =~ s/\s+/ /g;
1024 print "# $uname\n";
1025 }
1026 print '# Arguments: ',
1027 join(' ', map { my $a = "_[$_]_"; $a =~ s/\n/\n# /g; $a; } @ARGV), "\n";
1028}
1029
10301;
1031}
1032# ###########################################################################
1033# End OptionParser package
1034# ###########################################################################
1035
1036# ###########################################################################
1037# QueryParser package
1038# This package is a copy without comments from the original. The original
1039# with comments and its test file can be found in the Bazaar repository at,
1040# lib/QueryParser.pm
1041# t/lib/QueryParser.t
1042# See https://launchpad.net/percona-toolkit for more information.
1043# ###########################################################################
1044{
1045package QueryParser;
1046
1047use strict;
1048use warnings FATAL => 'all';
1049use English qw(-no_match_vars);
1050use constant PTDEBUG => $ENV{PTDEBUG} || 0;
1051
1052our $tbl_ident = qr/(?:`[^`]+`|\w+)(?:\.(?:`[^`]+`|\w+))?/;
1053our $tbl_regex = qr{
1054 \b(?:FROM|JOIN|(?<!KEY\s)UPDATE|INTO) # Words that precede table names
1055 \b\s*
1056 \(? # Optional paren around tables
1057 ($tbl_ident
1058 (?: (?:\s+ (?:AS\s+)? \w+)?, \s*$tbl_ident )*
1059 )
1060 }xio;
1061our $has_derived = qr{
1062 \b(?:FROM|JOIN|,)
1063 \s*\(\s*SELECT
1064 }xi;
1065
1066our $data_def_stmts = qr/(?:CREATE|ALTER|TRUNCATE|DROP|RENAME)/i;
1067
1068our $data_manip_stmts = qr/(?:INSERT|UPDATE|DELETE|REPLACE)/i;
1069
1070sub new {
1071 my ( $class ) = @_;
1072 bless {}, $class;
1073}
1074
1075sub get_tables {
1076 my ( $self, $query ) = @_;
1077 return unless $query;
1078 PTDEBUG && _d('Getting tables for', $query);
1079
1080 my ( $ddl_stmt ) = $query =~ m/^\s*($data_def_stmts)\b/i;
1081 if ( $ddl_stmt ) {
1082 PTDEBUG && _d('Special table type:', $ddl_stmt);
1083 $query =~ s/IF\s+(?:NOT\s+)?EXISTS//i;
1084 if ( $query =~ m/$ddl_stmt DATABASE\b/i ) {
1085 PTDEBUG && _d('Query alters a database, not a table');
1086 return ();
1087 }
1088 if ( $ddl_stmt =~ m/CREATE/i && $query =~ m/$ddl_stmt\b.+?\bSELECT\b/i ) {
1089 my ($select) = $query =~ m/\b(SELECT\b.+)/is;
1090 PTDEBUG && _d('CREATE TABLE ... SELECT:', $select);
1091 return $self->get_tables($select);
1092 }
1093 my ($tbl) = $query =~ m/TABLE\s+($tbl_ident)(\s+.*)?/i;
1094 PTDEBUG && _d('Matches table:', $tbl);
1095 return ($tbl);
1096 }
1097
1098 $query =~ s/ (?:LOW_PRIORITY|IGNORE|STRAIGHT_JOIN)//ig;
1099
1100 if ( $query =~ /^\s*LOCK TABLES/i ) {
1101 PTDEBUG && _d('Special table type: LOCK TABLES');
1102 $query =~ s/^(\s*LOCK TABLES\s+)//;
1103 $query =~ s/\s+(?:READ|WRITE|LOCAL)+\s*//g;
1104 PTDEBUG && _d('Locked tables:', $query);
1105 $query = "FROM $query";
1106 }
1107
1108 $query =~ s/\\["']//g; # quoted strings
1109 $query =~ s/".*?"/?/sg; # quoted strings
1110 $query =~ s/'.*?'/?/sg; # quoted strings
1111
1112 my @tables;
1113 foreach my $tbls ( $query =~ m/$tbl_regex/gio ) {
1114 PTDEBUG && _d('Match tables:', $tbls);
1115
1116 next if $tbls =~ m/\ASELECT\b/i;
1117
1118 foreach my $tbl ( split(',', $tbls) ) {
1119 $tbl =~ s/\s*($tbl_ident)(\s+.*)?/$1/gio;
1120
1121 if ( $tbl !~ m/[a-zA-Z]/ ) {
1122 PTDEBUG && _d('Skipping suspicious table name:', $tbl);
1123 next;
1124 }
1125
1126 push @tables, $tbl;
1127 }
1128 }
1129 return @tables;
1130}
1131
1132sub has_derived_table {
1133 my ( $self, $query ) = @_;
1134 my $match = $query =~ m/$has_derived/;
1135 PTDEBUG && _d($query, 'has ' . ($match ? 'a' : 'no') . ' derived table');
1136 return $match;
1137}
1138
1139sub get_aliases {
1140 my ( $self, $query, $list ) = @_;
1141
1142 my $result = {
1143 DATABASE => {},
1144 TABLE => {},
1145 };
1146 return $result unless $query;
1147
1148 $query =~ s/ (?:LOW_PRIORITY|IGNORE|STRAIGHT_JOIN)//ig;
1149
1150 $query =~ s/ (?:INNER|OUTER|CROSS|LEFT|RIGHT|NATURAL)//ig;
1151
1152 my @tbl_refs;
1153 my ($tbl_refs, $from) = $query =~ m{
1154 (
1155 (FROM|INTO|UPDATE)\b\s* # Keyword before table refs
1156 .+? # Table refs
1157 )
1158 (?:\s+|\z) # If the query does not end with the table
1159 (?:WHERE|ORDER|LIMIT|HAVING|SET|VALUES|\z) # Keyword after table refs
1160 }ix;
1161
1162 if ( $tbl_refs ) {
1163
1164 if ( $query =~ m/^(?:INSERT|REPLACE)/i ) {
1165 $tbl_refs =~ s/\([^\)]+\)\s*//;
1166 }
1167
1168 PTDEBUG && _d('tbl refs:', $tbl_refs);
1169
1170 my $before_tbl = qr/(?:,|JOIN|\s|$from)+/i;
1171
1172 my $after_tbl = qr/(?:,|JOIN|ON|USING|\z)/i;
1173
1174 $tbl_refs =~ s/ = /=/g;
1175
1176 while (
1177 $tbl_refs =~ m{
1178 $before_tbl\b\s*
1179 ( ($tbl_ident) (?:\s+ (?:AS\s+)? (\w+))? )
1180 \s*$after_tbl
1181 }xgio )
1182 {
1183 my ( $tbl_ref, $db_tbl, $alias ) = ($1, $2, $3);
1184 PTDEBUG && _d('Match table:', $tbl_ref);
1185 push @tbl_refs, $tbl_ref;
1186 $alias = $self->trim_identifier($alias);
1187
1188 if ( $tbl_ref =~ m/^AS\s+\w+/i ) {
1189 PTDEBUG && _d('Subquery', $tbl_ref);
1190 $result->{TABLE}->{$alias} = undef;
1191 next;
1192 }
1193
1194 my ( $db, $tbl ) = $db_tbl =~ m/^(?:(.*?)\.)?(.*)/;
1195 $db = $self->trim_identifier($db);
1196 $tbl = $self->trim_identifier($tbl);
1197 $result->{TABLE}->{$alias || $tbl} = $tbl;
1198 $result->{DATABASE}->{$tbl} = $db if $db;
1199 }
1200 }
1201 else {
1202 PTDEBUG && _d("No tables ref in", $query);
1203 }
1204
1205 if ( $list ) {
1206 return \@tbl_refs;
1207 }
1208 else {
1209 return $result;
1210 }
1211}
1212
1213sub split {
1214 my ( $self, $query ) = @_;
1215 return unless $query;
1216 $query = $self->clean_query($query);
1217 PTDEBUG && _d('Splitting', $query);
1218
1219 my $verbs = qr{SELECT|INSERT|UPDATE|DELETE|REPLACE|UNION|CREATE}i;
1220
1221 my @split_statements = grep { $_ } split(m/\b($verbs\b(?!(?:\s*\()))/io, $query);
1222
1223 my @statements;
1224 if ( @split_statements == 1 ) {
1225 push @statements, $query;
1226 }
1227 else {
1228 for ( my $i = 0; $i <= $#split_statements; $i += 2 ) {
1229 push @statements, $split_statements[$i].$split_statements[$i+1];
1230
1231 if ( $statements[-2] && $statements[-2] =~ m/on duplicate key\s+$/i ) {
1232 $statements[-2] .= pop @statements;
1233 }
1234 }
1235 }
1236
1237 PTDEBUG && _d('statements:', map { $_ ? "<$_>" : 'none' } @statements);
1238 return @statements;
1239}
1240
1241sub clean_query {
1242 my ( $self, $query ) = @_;
1243 return unless $query;
1244 $query =~ s!/\*.*?\*/! !g; # Remove /* comment blocks */
1245 $query =~ s/^\s+//; # Remove leading spaces
1246 $query =~ s/\s+$//; # Remove trailing spaces
1247 $query =~ s/\s{2,}/ /g; # Remove extra spaces
1248 return $query;
1249}
1250
1251sub split_subquery {
1252 my ( $self, $query ) = @_;
1253 return unless $query;
1254 $query = $self->clean_query($query);
1255 $query =~ s/;$//;
1256
1257 my @subqueries;
1258 my $sqno = 0; # subquery number
1259 my $pos = 0;
1260 while ( $query =~ m/(\S+)(?:\s+|\Z)/g ) {
1261 $pos = pos($query);
1262 my $word = $1;
1263 PTDEBUG && _d($word, $sqno);
1264 if ( $word =~ m/^\(?SELECT\b/i ) {
1265 my $start_pos = $pos - length($word) - 1;
1266 if ( $start_pos ) {
1267 $sqno++;
1268 PTDEBUG && _d('Subquery', $sqno, 'starts at', $start_pos);
1269 $subqueries[$sqno] = {
1270 start_pos => $start_pos,
1271 end_pos => 0,
1272 len => 0,
1273 words => [$word],
1274 lp => 1, # left parentheses
1275 rp => 0, # right parentheses
1276 done => 0,
1277 };
1278 }
1279 else {
1280 PTDEBUG && _d('Main SELECT at pos 0');
1281 }
1282 }
1283 else {
1284 next unless $sqno; # next unless we're in a subquery
1285 PTDEBUG && _d('In subquery', $sqno);
1286 my $sq = $subqueries[$sqno];
1287 if ( $sq->{done} ) {
1288 PTDEBUG && _d('This subquery is done; SQL is for',
1289 ($sqno - 1 ? "subquery $sqno" : "the main SELECT"));
1290 next;
1291 }
1292 push @{$sq->{words}}, $word;
1293 my $lp = ($word =~ tr/\(//) || 0;
1294 my $rp = ($word =~ tr/\)//) || 0;
1295 PTDEBUG && _d('parentheses left', $lp, 'right', $rp);
1296 if ( ($sq->{lp} + $lp) - ($sq->{rp} + $rp) == 0 ) {
1297 my $end_pos = $pos - 1;
1298 PTDEBUG && _d('Subquery', $sqno, 'ends at', $end_pos);
1299 $sq->{end_pos} = $end_pos;
1300 $sq->{len} = $end_pos - $sq->{start_pos};
1301 }
1302 }
1303 }
1304
1305 for my $i ( 1..$#subqueries ) {
1306 my $sq = $subqueries[$i];
1307 next unless $sq;
1308 $sq->{sql} = join(' ', @{$sq->{words}});
1309 substr $query,
1310 $sq->{start_pos} + 1, # +1 for (
1311 $sq->{len} - 1, # -1 for )
1312 "__subquery_$i";
1313 }
1314
1315 return $query, map { $_->{sql} } grep { defined $_ } @subqueries;
1316}
1317
1318sub query_type {
1319 my ( $self, $query, $qr ) = @_;
1320 my ($type, undef) = $qr->distill_verbs($query);
1321 my $rw;
1322 if ( $type =~ m/^SELECT\b/ ) {
1323 $rw = 'read';
1324 }
1325 elsif ( $type =~ m/^$data_manip_stmts\b/
1326 || $type =~ m/^$data_def_stmts\b/ ) {
1327 $rw = 'write'
1328 }
1329
1330 return {
1331 type => $type,
1332 rw => $rw,
1333 }
1334}
1335
1336sub get_columns {
1337 my ( $self, $query ) = @_;
1338 my $cols = [];
1339 return $cols unless $query;
1340 my $cols_def;
1341
1342 if ( $query =~ m/^SELECT/i ) {
1343 $query =~ s/
1344 ^SELECT\s+
1345 (?:ALL
1346 |DISTINCT
1347 |DISTINCTROW
1348 |HIGH_PRIORITY
1349 |STRAIGHT_JOIN
1350 |SQL_SMALL_RESULT
1351 |SQL_BIG_RESULT
1352 |SQL_BUFFER_RESULT
1353 |SQL_CACHE
1354 |SQL_NO_CACHE
1355 |SQL_CALC_FOUND_ROWS
1356 )\s+
1357 /SELECT /xgi;
1358 ($cols_def) = $query =~ m/^SELECT\s+(.+?)\s+FROM/i;
1359 }
1360 elsif ( $query =~ m/^(?:INSERT|REPLACE)/i ) {
1361 ($cols_def) = $query =~ m/\(([^\)]+)\)\s*VALUE/i;
1362 }
1363
1364 PTDEBUG && _d('Columns:', $cols_def);
1365 if ( $cols_def ) {
1366 @$cols = split(',', $cols_def);
1367 map {
1368 my $col = $_;
1369 $col = s/^\s+//g;
1370 $col = s/\s+$//g;
1371 $col;
1372 } @$cols;
1373 }
1374
1375 return $cols;
1376}
1377
1378sub parse {
1379 my ( $self, $query ) = @_;
1380 return unless $query;
1381 my $parsed = {};
1382
1383 $query =~ s/\n/ /g;
1384 $query = $self->clean_query($query);
1385
1386 $parsed->{query} = $query,
1387 $parsed->{tables} = $self->get_aliases($query, 1);
1388 $parsed->{columns} = $self->get_columns($query);
1389
1390 my ($type) = $query =~ m/^(\w+)/;
1391 $parsed->{type} = lc $type;
1392
1393
1394 $parsed->{sub_queries} = [];
1395
1396 return $parsed;
1397}
1398
1399sub extract_tables {
1400 my ( $self, %args ) = @_;
1401 my $query = $args{query};
1402 my $default_db = $args{default_db};
1403 my $q = $self->{Quoter} || $args{Quoter};
1404 return unless $query;
1405 PTDEBUG && _d('Extracting tables');
1406 my @tables;
1407 my %seen;
1408 foreach my $db_tbl ( $self->get_tables($query) ) {
1409 next unless $db_tbl;
1410 next if $seen{$db_tbl}++; # Unique-ify for issue 337.
1411 my ( $db, $tbl ) = $q->split_unquote($db_tbl);
1412 push @tables, [ $db || $default_db, $tbl ];
1413 }
1414 return @tables;
1415}
1416
1417sub trim_identifier {
1418 my ($self, $str) = @_;
1419 return unless defined $str;
1420 $str =~ s/`//g;
1421 $str =~ s/^\s+//;
1422 $str =~ s/\s+$//;
1423 return $str;
1424}
1425
1426sub _d {
1427 my ($package, undef, $line) = caller 0;
1428 @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
1429 map { defined $_ ? $_ : 'undef' }
1430 @_;
1431 print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
1432}
1433
14341;
1435}
1436# ###########################################################################
1437# End QueryParser package
1438# ###########################################################################
1439
1440# ###########################################################################
1441# QueryRewriter package
1442# This package is a copy without comments from the original. The original
1443# with comments and its test file can be found in the Bazaar repository at,
1444# lib/QueryRewriter.pm
1445# t/lib/QueryRewriter.t
1446# See https://launchpad.net/percona-toolkit for more information.
1447# ###########################################################################
1448{
1449package QueryRewriter;
1450
1451use strict;
1452use warnings FATAL => 'all';
1453use English qw(-no_match_vars);
1454use constant PTDEBUG => $ENV{PTDEBUG} || 0;
1455
1456our $verbs = qr{^SHOW|^FLUSH|^COMMIT|^ROLLBACK|^BEGIN|SELECT|INSERT
1457 |UPDATE|DELETE|REPLACE|^SET|UNION|^START|^LOCK}xi;
1458my $quote_re = qr/"(?:(?!(?<!\\)").)*"|'(?:(?!(?<!\\)').)*'/; # Costly!
1459my $bal;
1460$bal = qr/
1461 \(
1462 (?:
1463 (?> [^()]+ ) # Non-parens without backtracking
1464 |
1465 (??{ $bal }) # Group with matching parens
1466 )*
1467 \)
1468 /x;
1469
1470my $olc_re = qr/(?:--|#)[^'"\r\n]*(?=[\r\n]|\Z)/; # One-line comments
1471my $mlc_re = qr#/\*[^!].*?\*/#sm; # But not /*!version */
1472my $vlc_re = qr#/\*.*?[0-9+].*?\*/#sm; # For SHOW + /*!version */
1473my $vlc_rf = qr#^(SHOW).*?/\*![0-9+].*?\*/#sm; # Variation for SHOW
1474
1475
1476sub new {
1477 my ( $class, %args ) = @_;
1478 my $self = { %args };
1479 return bless $self, $class;
1480}
1481
1482sub strip_comments {
1483 my ( $self, $query ) = @_;
1484 return unless $query;
1485 $query =~ s/$olc_re//go;
1486 $query =~ s/$mlc_re//go;
1487 if ( $query =~ m/$vlc_rf/i ) { # contains show + version
1488 $query =~ s/$vlc_re//go;
1489 }
1490 return $query;
1491}
1492
1493sub shorten {
1494 my ( $self, $query, $length ) = @_;
1495 $query =~ s{
1496 \A(
1497 (?:INSERT|REPLACE)
1498 (?:\s+LOW_PRIORITY|DELAYED|HIGH_PRIORITY|IGNORE)?
1499 (?:\s\w+)*\s+\S+\s+VALUES\s*\(.*?\)
1500 )
1501 \s*,\s*\(.*?(ON\s+DUPLICATE|\Z)}
1502 {$1 /*... omitted ...*/$2}xsi;
1503
1504 return $query unless $query =~ m/IN\s*\(\s*(?!select)/i;
1505
1506 my $last_length = 0;
1507 my $query_length = length($query);
1508 while (
1509 $length > 0
1510 && $query_length > $length
1511 && $query_length < ( $last_length || $query_length + 1 )
1512 ) {
1513 $last_length = $query_length;
1514 $query =~ s{
1515 (\bIN\s*\() # The opening of an IN list
1516 ([^\)]+) # Contents of the list, assuming no item contains paren
1517 (?=\)) # Close of the list
1518 }
1519 {
1520 $1 . __shorten($2)
1521 }gexsi;
1522 }
1523
1524 return $query;
1525}
1526
1527sub __shorten {
1528 my ( $snippet ) = @_;
1529 my @vals = split(/,/, $snippet);
1530 return $snippet unless @vals > 20;
1531 my @keep = splice(@vals, 0, 20); # Remove and save the first 20 items
1532 return
1533 join(',', @keep)
1534 . "/*... omitted "
1535 . scalar(@vals)
1536 . " items ...*/";
1537}
1538
1539sub fingerprint {
1540 my ( $self, $query ) = @_;
1541
1542 $query =~ m#\ASELECT /\*!40001 SQL_NO_CACHE \*/ \* FROM `# # mysqldump query
1543 && return 'mysqldump';
1544 $query =~ m#/\*\w+\.\w+:[0-9]/[0-9]\*/# # pt-table-checksum, etc query
1545 && return 'percona-toolkit';
1546 $query =~ m/\Aadministrator command: /
1547 && return $query;
1548 $query =~ m/\A\s*(call\s+\S+)\(/i
1549 && return lc($1); # Warning! $1 used, be careful.
1550 if ( my ($beginning) = $query =~ m/\A((?:INSERT|REPLACE)(?: IGNORE)?\s+INTO.+?VALUES\s*\(.*?\))\s*,\s*\(/is ) {
1551 $query = $beginning; # Shorten multi-value INSERT statements ASAP
1552 }
1553
1554 $query =~ s/$olc_re//go;
1555 $query =~ s/$mlc_re//go;
1556 $query =~ s/\Ause \S+\Z/use ?/i # Abstract the DB in USE
1557 && return $query;
1558
1559 $query =~ s/\\["']//g; # quoted strings
1560 $query =~ s/".*?"/?/sg; # quoted strings
1561 $query =~ s/'.*?'/?/sg; # quoted strings
1562
1563 if ( $self->{match_md5_checksums} ) {
1564 $query =~ s/([._-])[a-f0-9]{32}/$1?/g;
1565 }
1566
1567 if ( !$self->{match_embedded_numbers} ) {
1568 $query =~ s/[0-9+-][0-9a-f.xb+-]*/?/g;
1569 }
1570 else {
1571 $query =~ s/\b[0-9+-][0-9a-f.xb+-]*/?/g;
1572 }
1573
1574 if ( $self->{match_md5_checksums} ) {
1575 $query =~ s/[xb+-]\?/?/g;
1576 }
1577 else {
1578 $query =~ s/[xb.+-]\?/?/g;
1579 }
1580
1581 $query =~ s/\A\s+//; # Chop off leading whitespace
1582 chomp $query; # Kill trailing whitespace
1583 $query =~ tr[ \n\t\r\f][ ]s; # Collapse whitespace
1584 $query = lc $query;
1585 $query =~ s/\bnull\b/?/g; # Get rid of NULLs
1586 $query =~ s{ # Collapse IN and VALUES lists
1587 \b(in|values?)(?:[\s,]*\([\s?,]*\))+
1588 }
1589 {$1(?+)}gx;
1590 $query =~ s{ # Collapse UNION
1591 \b(select\s.*?)(?:(\sunion(?:\sall)?)\s\1)+
1592 }
1593 {$1 /*repeat$2*/}xg;
1594 $query =~ s/\blimit \?(?:, ?\?| offset \?)?/limit ?/; # LIMIT
1595
1596 if ( $query =~ m/\bORDER BY /gi ) { # Find, anchor on ORDER BY clause
1597 1 while $query =~ s/\G(.+?)\s+ASC/$1/gi && pos $query;
1598 }
1599
1600 return $query;
1601}
1602
1603sub distill_verbs {
1604 my ( $self, $query ) = @_;
1605
1606 $query =~ m/\A\s*call\s+(\S+)\(/i && return "CALL $1";
1607 $query =~ m/\A\s*use\s+/ && return "USE";
1608 $query =~ m/\A\s*UNLOCK TABLES/i && return "UNLOCK";
1609 $query =~ m/\A\s*xa\s+(\S+)/i && return "XA_$1";
1610
1611 if ( $query =~ m/\Aadministrator command:/ ) {
1612 $query =~ s/administrator command:/ADMIN/;
1613 $query = uc $query;
1614 return $query;
1615 }
1616
1617 $query = $self->strip_comments($query);
1618
1619 if ( $query =~ m/\A\s*SHOW\s+/i ) {
1620 PTDEBUG && _d($query);
1621
1622 $query = uc $query;
1623 $query =~ s/\s+(?:GLOBAL|SESSION|FULL|STORAGE|ENGINE)\b/ /g;
1624 $query =~ s/\s+COUNT[^)]+\)//g;
1625
1626 $query =~ s/\s+(?:FOR|FROM|LIKE|WHERE|LIMIT|IN)\b.+//ms;
1627
1628 $query =~ s/\A(SHOW(?:\s+\S+){1,2}).*\Z/$1/s;
1629 $query =~ s/\s+/ /g;
1630 PTDEBUG && _d($query);
1631 return $query;
1632 }
1633
1634 eval $QueryParser::data_def_stmts;
1635 eval $QueryParser::tbl_ident;
1636 my ( $dds ) = $query =~ /^\s*($QueryParser::data_def_stmts)\b/i;
1637 if ( $dds) {
1638 my ( $obj ) = $query =~ m/$dds.+(DATABASE|TABLE)\b/i;
1639 $obj = uc $obj if $obj;
1640 PTDEBUG && _d('Data def statment:', $dds, 'obj:', $obj);
1641 my ($db_or_tbl)
1642 = $query =~ m/(?:TABLE|DATABASE)\s+($QueryParser::tbl_ident)(\s+.*)?/i;
1643 PTDEBUG && _d('Matches db or table:', $db_or_tbl);
1644 return uc($dds . ($obj ? " $obj" : '')), $db_or_tbl;
1645 }
1646
1647 my @verbs = $query =~ m/\b($verbs)\b/gio;
1648 @verbs = do {
1649 my $last = '';
1650 grep { my $pass = $_ ne $last; $last = $_; $pass } map { uc } @verbs;
1651 };
1652
1653 if ( ($verbs[0] || '') eq 'SELECT' && @verbs > 1 ) {
1654 PTDEBUG && _d("False-positive verbs after SELECT:", @verbs[1..$#verbs]);
1655 my $union = grep { $_ eq 'UNION' } @verbs;
1656 @verbs = $union ? qw(SELECT UNION) : qw(SELECT);
1657 }
1658
1659 my $verb_str = join(q{ }, @verbs);
1660 return $verb_str;
1661}
1662
1663sub __distill_tables {
1664 my ( $self, $query, $table, %args ) = @_;
1665 my $qp = $args{QueryParser} || $self->{QueryParser};
1666 die "I need a QueryParser argument" unless $qp;
1667
1668 my @tables = map {
1669 $_ =~ s/`//g;
1670 $_ =~ s/(_?)[0-9]+/$1?/g;
1671 $_;
1672 } grep { defined $_ } $qp->get_tables($query);
1673
1674 push @tables, $table if $table;
1675
1676 @tables = do {
1677 my $last = '';
1678 grep { my $pass = $_ ne $last; $last = $_; $pass } @tables;
1679 };
1680
1681 return @tables;
1682}
1683
1684sub distill {
1685 my ( $self, $query, %args ) = @_;
1686
1687 if ( $args{generic} ) {
1688 my ($cmd, $arg) = $query =~ m/^(\S+)\s+(\S+)/;
1689 return '' unless $cmd;
1690 $query = (uc $cmd) . ($arg ? " $arg" : '');
1691 }
1692 else {
1693 my ($verbs, $table) = $self->distill_verbs($query, %args);
1694
1695 if ( $verbs && $verbs =~ m/^SHOW/ ) {
1696 my %alias_for = qw(
1697 SCHEMA DATABASE
1698 KEYS INDEX
1699 INDEXES INDEX
1700 );
1701 map { $verbs =~ s/$_/$alias_for{$_}/ } keys %alias_for;
1702 $query = $verbs;
1703 }
1704 else {
1705 my @tables = $self->__distill_tables($query, $table, %args);
1706 $query = join(q{ }, $verbs, @tables);
1707 }
1708 }
1709
1710 if ( $args{trf} ) {
1711 $query = $args{trf}->($query, %args);
1712 }
1713
1714 return $query;
1715}
1716
1717sub convert_to_select {
1718 my ( $self, $query ) = @_;
1719 return unless $query;
1720
1721 return if $query =~ m/=\s*\(\s*SELECT /i;
1722
1723 $query =~ s{
1724 \A.*?
1725 update(?:\s+(?:low_priority|ignore))?\s+(.*?)
1726 \s+set\b(.*?)
1727 (?:\s*where\b(.*?))?
1728 (limit\s*[0-9]+(?:\s*,\s*[0-9]+)?)?
1729 \Z
1730 }
1731 {__update_to_select($1, $2, $3, $4)}exsi
1732 || $query =~ s{
1733 \A.*?
1734 (?:insert(?:\s+ignore)?|replace)\s+
1735 .*?\binto\b(.*?)\(([^\)]+)\)\s*
1736 values?\s*(\(.*?\))\s*
1737 (?:\blimit\b|on\s+duplicate\s+key.*)?\s*
1738 \Z
1739 }
1740 {__insert_to_select($1, $2, $3)}exsi
1741 || $query =~ s{
1742 \A.*?
1743 (?:insert(?:\s+ignore)?|replace)\s+
1744 (?:.*?\binto)\b(.*?)\s*
1745 set\s+(.*?)\s*
1746 (?:\blimit\b|on\s+duplicate\s+key.*)?\s*
1747 \Z
1748 }
1749 {__insert_to_select_with_set($1, $2)}exsi
1750 || $query =~ s{
1751 \A.*?
1752 delete\s+(.*?)
1753 \bfrom\b(.*)
1754 \Z
1755 }
1756 {__delete_to_select($1, $2)}exsi;
1757 $query =~ s/\s*on\s+duplicate\s+key\s+update.*\Z//si;
1758 $query =~ s/\A.*?(?=\bSELECT\s*\b)//ism;
1759 return $query;
1760}
1761
1762sub convert_select_list {
1763 my ( $self, $query ) = @_;
1764 $query =~ s{
1765 \A\s*select(.*?)\bfrom\b
1766 }
1767 {$1 =~ m/\*/ ? "select 1 from" : "select isnull(coalesce($1)) from"}exi;
1768 return $query;
1769}
1770
1771sub __delete_to_select {
1772 my ( $delete, $join ) = @_;
1773 if ( $join =~ m/\bjoin\b/ ) {
1774 return "select 1 from $join";
1775 }
1776 return "select * from $join";
1777}
1778
1779sub __insert_to_select {
1780 my ( $tbl, $cols, $vals ) = @_;
1781 PTDEBUG && _d('Args:', @_);
1782 my @cols = split(/,/, $cols);
1783 PTDEBUG && _d('Cols:', @cols);
1784 $vals =~ s/^\(|\)$//g; # Strip leading/trailing parens
1785 my @vals = $vals =~ m/($quote_re|[^,]*${bal}[^,]*|[^,]+)/g;
1786 PTDEBUG && _d('Vals:', @vals);
1787 if ( @cols == @vals ) {
1788 return "select * from $tbl where "
1789 . join(' and ', map { "$cols[$_]=$vals[$_]" } (0..$#cols));
1790 }
1791 else {
1792 return "select * from $tbl limit 1";
1793 }
1794}
1795
1796sub __insert_to_select_with_set {
1797 my ( $from, $set ) = @_;
1798 $set =~ s/,/ and /g;
1799 return "select * from $from where $set ";
1800}
1801
1802sub __update_to_select {
1803 my ( $from, $set, $where, $limit ) = @_;
1804 return "select $set from $from "
1805 . ( $where ? "where $where" : '' )
1806 . ( $limit ? " $limit " : '' );
1807}
1808
1809sub wrap_in_derived {
1810 my ( $self, $query ) = @_;
1811 return unless $query;
1812 return $query =~ m/\A\s*select/i
1813 ? "select 1 from ($query) as x limit 1"
1814 : $query;
1815}
1816
1817sub _d {
1818 my ($package, undef, $line) = caller 0;
1819 @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
1820 map { defined $_ ? $_ : 'undef' }
1821 @_;
1822 print STDERR "# $package:$line $PID ", join(' ', @_), "\n";
1823}
1824
18251;
1826}
1827# ###########################################################################
1828# End QueryRewriter package
1829# ###########################################################################
1830
1831# ###########################################################################
1832# This is a combination of modules and programs in one -- a runnable module.
1833# http://www.perl.com/pub/a/2006/07/13/lightning-articles.html?page=last
1834# Or, look it up in the Camel book on pages 642 and 643 in the 3rd edition.
1835#
1836# Check at the end of this package for the call to main() which actually runs
1837# the program.
1838# ###########################################################################
1839package pt_fingerprint;
1840
1841use English qw(-no_match_vars);
1842use Data::Dumper;
1843$Data::Dumper::Indent = 1;
1844$OUTPUT_AUTOFLUSH = 1;
1845
1846use constant MKDEBUG => $ENV{MKDEBUG} || 0;
1847
1848sub main {
1849 @ARGV = @_; # set global ARGV for this package
1850
1851 # ##########################################################################
1852 # Get configuration information.
1853 # ##########################################################################
1854 my $o = new OptionParser();
1855 $o->get_specs();
1856 $o->get_opts();
1857 $o->usage_or_errors();
1858
1859 my $qp = new QueryParser();
1860 my $qr = new QueryRewriter(
1861 QueryParser => $qp,
1862 match_md5_checksums => $o->get('match-md5-checksums'),
1863 match_embedded_numbers => $o->get('match-embedded-numbers'),
1864 );
1865
1866 if ( $o->got('query') ) {
1867 print $qr->fingerprint($o->get('query')), "\n";
1868 }
1869 else {
1870 local $INPUT_RECORD_SEPARATOR = ";\n";
1871 while ( <> ) {
1872 my $query = $_;
1873 chomp $query;
1874 $query =~ s/^#.+$//mg;
1875 $query =~ s/^\s+//;
1876 next unless $query =~ m/^\w/;
1877 print $qr->fingerprint($query), "\n";
1878 }
1879 }
1880}
1881
1882# ############################################################################
1883# Run the program.
1884# ############################################################################
1885if ( !caller ) { exit main(@ARGV); }
1886
18871; # Because this is a module as well as a script.
1888
1889# #############################################################################
1890# Documentation.
1891# #############################################################################
1892
1893=pod
1894
1895=head1 NAME
1896
1897pt-fingerprint - Convert queries into fingerprints.
1898
1899=head1 SYNOPSIS
1900
1901Usage: pt-fingerprint [OPTIONS] [FILES]
1902
1903pt-fingerprint converts queries into fingerprints. With the --query
1904option, converts the option's value into a fingerprint. With no options, treats
1905command-line arguments as FILEs and reads and converts semicolon-separated
1906queries from the FILEs. When FILE is -, it read standard input.
1907
1908Convert a single query:
1909
1910 pt-fingerprint --query "select a, b, c from users where id = 500"
1911
1912Convert a file full of queries:
1913
1914 pt-fingerprint /path/to/file.txt
1915
1916=head1 RISKS
1917
1918The following section is included to inform users about the potential risks,
1919whether known or unknown, of using this tool. The two main categories of risks
1920are those created by the nature of the tool (e.g. read-only tools vs. read-write
1921tools) and those created by bugs.
1922
1923The pt-fingerprint tool simply reads data and transforms it, so risks are
1924minimal.
1925
1926See also L<"BUGS"> for more information on filing bugs and getting help.
1927
1928=head1 DESCRIPTION
1929
1930A query fingerprint is the abstracted form of a query, which makes it possible
1931to group similar queries together. Abstracting a query removes literal values,
1932normalizes whitespace, and so on. For example, consider these two queries:
1933
1934 SELECT name, password FROM user WHERE id='12823';
1935 select name, password from user
1936 where id=5;
1937
1938Both of those queries will fingerprint to
1939
1940 select name, password from user where id=?
1941
1942Once the query's fingerprint is known, we can then talk about a query as though
1943it represents all similar queries.
1944
1945Query fingerprinting accommodates a great many special cases, which have proven
1946necessary in the real world. For example, an IN list with 5 literals is really
1947equivalent to one with 4 literals, so lists of literals are collapsed to a
1948single one. If you want to understand more about how and why all of these cases
1949are handled, please review the test cases in the Subversion repository. If you
1950find something that is not fingerprinted properly, please submit a bug report
1951with a reproducible test case. Here is a list of transformations during
1952fingerprinting, which might not be exhaustive:
1953
1954=over
1955
1956=item *
1957
1958Group all SELECT queries from mysqldump together, even if they are against
1959different tables. Ditto for all of pt-table-checksum's checksum queries.
1960
1961=item *
1962
1963Shorten multi-value INSERT statements to a single VALUES() list.
1964
1965=item *
1966
1967Strip comments.
1968
1969=item *
1970
1971Abstract the databases in USE statements, so all USE statements are grouped
1972together.
1973
1974=item *
1975
1976Replace all literals, such as quoted strings. For efficiency, the code that
1977replaces literal numbers is somewhat non-selective, and might replace some
1978things as numbers when they really are not. Hexadecimal literals are also
1979replaced. NULL is treated as a literal. Numbers embedded in identifiers are
1980also replaced, so tables named similarly will be fingerprinted to the same
1981values (e.g. users_2009 and users_2010 will fingerprint identically).
1982
1983=item *
1984
1985Collapse all whitespace into a single space.
1986
1987=item *
1988
1989Lowercase the entire query.
1990
1991=item *
1992
1993Replace all literals inside of IN() and VALUES() lists with a single
1994placeholder, regardless of cardinality.
1995
1996=item *
1997
1998Collapse multiple identical UNION queries into a single one.
1999
2000=back
2001
2002=head1 OPTIONS
2003
2004This tool accepts additional command-line arguments. Refer to the
2005L<"SYNOPSIS"> and usage information for details.
2006
2007=over
2008
2009=item --config
2010
2011type: Array
2012
2013Read this comma-separated list of config files; if specified, this must be the
2014first option on the command line.
2015
2016=item --help
2017
2018Show help and exit.
2019
2020=item --match-embedded-numbers
2021
2022Match numbers embedded in words and replace as single values. This option
2023causes the tool to be more careful about matching numbers so that words
2024with numbers, like C<catch22> are matched and replaced as a single C<?>
2025placeholder. Otherwise the default number matching pattern will replace
2026C<catch22> as C<catch?>.
2027
2028This is helpful if database or table names contain numbers.
2029
2030=item --match-md5-checksums
2031
2032Match MD5 checksums and replace as single values. This option causes
2033the tool to be more careful about matching numbers so that MD5 checksums
2034like C<fbc5e685a5d3d45aa1d0347fdb7c4d35> are matched and replaced as a
2035single C<?> placeholder. Otherwise, the default number matching pattern will
2036replace C<fbc5e685a5d3d45aa1d0347fdb7c4d35> as C<fbc?>.
2037
2038=item --query
2039
2040type: string
2041
2042The query to convert into a fingerprint.
2043
2044=item --version
2045
2046Show version and exit.
2047
2048=back
2049
2050=head1 ENVIRONMENT
2051
2052The environment variable C<PTDEBUG> enables verbose debugging output to STDERR.
2053To enable debugging and capture all output to a file, run the tool like:
2054
2055 PTDEBUG=1 pt-fingerprint ... > FILE 2>&1
2056
2057Be careful: debugging output is voluminous and can generate several megabytes
2058of output.
2059
2060=head1 SYSTEM REQUIREMENTS
2061
2062You need Perl, DBI, DBD::mysql, and some core packages that ought to be
2063installed in any reasonably new version of Perl.
2064
2065=head1 BUGS
2066
2067For a list of known bugs, see L<http://www.percona.com/bugs/pt-fingerprint>.
2068
2069Please report bugs at L<https://bugs.launchpad.net/percona-toolkit>.
2070Include the following information in your bug report:
2071
2072=over
2073
2074=item * Complete command-line used to run the tool
2075
2076=item * Tool L<"--version">
2077
2078=item * MySQL version of all servers involved
2079
2080=item * Output from the tool including STDERR
2081
2082=item * Input files (log/dump/config files, etc.)
2083
2084=back
2085
2086If possible, include debugging output by running the tool with C<PTDEBUG>;
2087see L<"ENVIRONMENT">.
2088
2089=head1 DOWNLOADING
2090
2091Visit L<http://www.percona.com/software/percona-toolkit/> to download the
2092latest release of Percona Toolkit. Or, get the latest release from the
2093command line:
2094
2095 wget percona.com/get/percona-toolkit.tar.gz
2096
2097 wget percona.com/get/percona-toolkit.rpm
2098
2099 wget percona.com/get/percona-toolkit.deb
2100
2101You can also get individual tools from the latest release:
2102
2103 wget percona.com/get/TOOL
2104
2105Replace C<TOOL> with the name of any tool.
2106
2107=head1 AUTHORS
2108
2109Baron Schwartz and Daniel Nichter
2110
2111=head1 ABOUT PERCONA TOOLKIT
2112
2113This tool is part of Percona Toolkit, a collection of advanced command-line
2114tools developed by Percona for MySQL support and consulting. Percona Toolkit
2115was forked from two projects in June, 2011: Maatkit and Aspersa. Those
2116projects were created by Baron Schwartz and developed primarily by him and
2117Daniel Nichter, both of whom are employed by Percona. Visit
2118L<http://www.percona.com/software/> for more software developed by Percona.
2119
2120=head1 COPYRIGHT, LICENSE, AND WARRANTY
2121
2122This program is copyright 2011-2012 Percona Inc.
2123Feedback and improvements are welcome.
2124
2125THIS PROGRAM IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
2126WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
2127MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
2128
2129This program is free software; you can redistribute it and/or modify it under
2130the terms of the GNU General Public License as published by the Free Software
2131Foundation, version 2; OR the Perl Artistic License. On UNIX and similar
2132systems, you can issue `man perlgpl' or `man perlartistic' to read these
2133licenses.
2134
2135You should have received a copy of the GNU General Public License along with
2136this program; if not, write to the Free Software Foundation, Inc., 59 Temple
2137Place, Suite 330, Boston, MA 02111-1307 USA.
2138
2139=head1 VERSION
2140
2141pt-fingerprint 2.1.1
2142
2143=cut
02144
=== modified file 'bin/pt-fk-error-logger'
--- bin/pt-fk-error-logger 2012-05-24 17:52:01 +0000
+++ bin/pt-fk-error-logger 2012-05-30 21:00:30 +0000
@@ -2465,6 +2465,6 @@
24652465
2466=head1 VERSION2466=head1 VERSION
24672467
2468pt-fk-error-logger 2.0.42468pt-fk-error-logger 2.1.1
24692469
2470=cut2470=cut
24712471
=== modified file 'bin/pt-heartbeat'
--- bin/pt-heartbeat 2012-05-24 17:52:01 +0000
+++ bin/pt-heartbeat 2012-05-30 21:00:30 +0000
@@ -4400,6 +4400,6 @@
44004400
4401=head1 VERSION4401=head1 VERSION
44024402
4403pt-heartbeat 2.0.44403pt-heartbeat 2.1.1
44044404
4405=cut4405=cut
44064406
=== modified file 'bin/pt-index-usage'
--- bin/pt-index-usage 2012-05-24 17:52:01 +0000
+++ bin/pt-index-usage 2012-05-30 21:00:30 +0000
@@ -2674,19 +2674,58 @@
2674 return bless $self, $class;2674 return bless $self, $class;
2675}2675}
26762676
2677sub get_create_table {
2678 my ( $self, $dbh, $db, $tbl ) = @_;
2679 die "I need a dbh parameter" unless $dbh;
2680 die "I need a db parameter" unless $db;
2681 die "I need a tbl parameter" unless $tbl;
2682 my $q = $self->{Quoter};
2683
2684 my $new_sql_mode
2685 = '/*!40101 SET @OLD_SQL_MODE := @@SQL_MODE, '
2686 . q{@@SQL_MODE := REPLACE(REPLACE(@@SQL_MODE, 'ANSI_QUOTES', ''), ',,', ','), }
2687 . '@OLD_QUOTE := @@SQL_QUOTE_SHOW_CREATE, '
2688 . '@@SQL_QUOTE_SHOW_CREATE := 1 */';
2689
2690 my $old_sql_mode = '/*!40101 SET @@SQL_MODE := @OLD_SQL_MODE, '
2691 . '@@SQL_QUOTE_SHOW_CREATE := @OLD_QUOTE */';
2692
2693 PTDEBUG && _d($new_sql_mode);
2694 eval { $dbh->do($new_sql_mode); };
2695 PTDEBUG && $EVAL_ERROR && _d($EVAL_ERROR);
2696
2697 my $use_sql = 'USE ' . $q->quote($db);
2698 PTDEBUG && _d($dbh, $use_sql);
2699 $dbh->do($use_sql);
2700
2701 my $show_sql = "SHOW CREATE TABLE " . $q->quote($db, $tbl);
2702 PTDEBUG && _d($show_sql);
2703 my $href;
2704 eval { $href = $dbh->selectrow_hashref($show_sql); };
2705 if ( $EVAL_ERROR ) {
2706 PTDEBUG && _d($EVAL_ERROR);
2707
2708 PTDEBUG && _d($old_sql_mode);
2709 $dbh->do($old_sql_mode);
2710
2711 return;
2712 }
2713
2714 PTDEBUG && _d($old_sql_mode);
2715 $dbh->do($old_sql_mode);
2716
2717 my ($key) = grep { m/create (?:table|view)/i } keys %$href;
2718 if ( !$key ) {
2719 die "Error: no 'Create Table' or 'Create View' in result set from "
2720 . "$show_sql: " . Dumper($href);
2721 }
2722
2723 return $href->{$key};
2724}
2725
2677sub parse {2726sub parse {
2678 my ( $self, $ddl, $opts ) = @_;2727 my ( $self, $ddl, $opts ) = @_;
2679 return unless $ddl;2728 return unless $ddl;
2680 if ( ref $ddl eq 'ARRAY' ) {
2681 if ( lc $ddl->[0] eq 'table' ) {
2682 $ddl = $ddl->[1];
2683 }
2684 else {
2685 return {
2686 engine => 'VIEW',
2687 };
2688 }
2689 }
26902729
2691 if ( $ddl !~ m/CREATE (?:TEMPORARY )?TABLE `/ ) {2730 if ( $ddl !~ m/CREATE (?:TEMPORARY )?TABLE `/ ) {
2692 die "Cannot parse table definition; is ANSI quoting "2731 die "Cannot parse table definition; is ANSI quoting "
@@ -2993,41 +3032,31 @@
2993 return $ddl;3032 return $ddl;
2994}3033}
29953034
2996sub remove_secondary_indexes {3035sub get_table_status {
2997 my ( $self, $ddl ) = @_;3036 my ( $self, $dbh, $db, $like ) = @_;
2998 my $sec_indexes_ddl;3037 my $q = $self->{Quoter};
2999 my $tbl_struct = $self->parse($ddl);3038 my $sql = "SHOW TABLE STATUS FROM " . $q->quote($db);
30003039 my @params;
3001 if ( ($tbl_struct->{engine} || '') =~ m/InnoDB/i ) {3040 if ( $like ) {
3002 my $clustered_key = $tbl_struct->{clustered_key};3041 $sql .= ' LIKE ?';
3003 $clustered_key ||= '';3042 push @params, $like;
30043043 }
3005 my @sec_indexes = map {3044 PTDEBUG && _d($sql, @params);
3006 my $key_def = $_->{ddl};3045 my $sth = $dbh->prepare($sql);
3007 $key_def =~ s/([\(\)])/\\$1/g;3046 eval { $sth->execute(@params); };
3008 $ddl =~ s/\s+$key_def//i;3047 if ($EVAL_ERROR) {
30093048 PTDEBUG && _d($EVAL_ERROR);
3010 my $key_ddl = "ADD $_->{ddl}";3049 return;
3011 $key_ddl .= ',' unless $key_ddl =~ m/,$/;3050 }
3012 $key_ddl;3051 my @tables = @{$sth->fetchall_arrayref({})};
3013 }3052 @tables = map {
3014 grep { $_->{name} ne $clustered_key }3053 my %tbl; # Make a copy with lowercased keys
3015 values %{$tbl_struct->{keys}};3054 @tbl{ map { lc $_ } keys %$_ } = values %$_;
3016 PTDEBUG && _d('Secondary indexes:', Dumper(\@sec_indexes));3055 $tbl{engine} ||= $tbl{type} || $tbl{comment};
30173056 delete $tbl{type};
3018 if ( @sec_indexes ) {3057 \%tbl;
3019 $sec_indexes_ddl = join(' ', @sec_indexes);3058 } @tables;
3020 $sec_indexes_ddl =~ s/,$//;3059 return @tables;
3021 }
3022
3023 $ddl =~ s/,(\n\) )/$1/s;
3024 }
3025 else {
3026 PTDEBUG && _d('Not removing secondary indexes from',
3027 $tbl_struct->{engine}, 'table');
3028 }
3029
3030 return $ddl, $sec_indexes_ddl, $tbl_struct;
3031}3060}
30323061
3033sub _d {3062sub _d {
@@ -3917,7 +3946,7 @@
39173946
3918sub new {3947sub new {
3919 my ( $class, %args ) = @_;3948 my ( $class, %args ) = @_;
3920 my @required_args = qw(OptionParser Quoter);3949 my @required_args = qw(OptionParser TableParser Quoter);
3921 foreach my $arg ( @required_args ) {3950 foreach my $arg ( @required_args ) {
3922 die "I need a $arg argument" unless $args{$arg};3951 die "I need a $arg argument" unless $args{$arg};
3923 }3952 }
@@ -3926,8 +3955,19 @@
3926 die "I need either a dbh or file_itr argument"3955 die "I need either a dbh or file_itr argument"
3927 if (!$dbh && !$file_itr) || ($dbh && $file_itr);3956 if (!$dbh && !$file_itr) || ($dbh && $file_itr);
39283957
3958 my %resume;
3959 if ( my $table = $args{resume} ) {
3960 PTDEBUG && _d('Will resume from or after', $table);
3961 my ($db, $tbl) = $args{Quoter}->split_unquote($table);
3962 die "Resume table must be database-qualified: $table"
3963 unless $db && $tbl;
3964 $resume{db} = $db;
3965 $resume{tbl} = $tbl;
3966 }
3967
3929 my $self = {3968 my $self = {
3930 %args,3969 %args,
3970 resume => \%resume,
3931 filters => _make_filters(%args),3971 filters => _make_filters(%args),
3932 };3972 };
39333973
@@ -3988,9 +4028,19 @@
3988 return \%filters;4028 return \%filters;
3989}4029}
39904030
3991sub next_schema_object {4031sub next {
3992 my ( $self ) = @_;4032 my ( $self ) = @_;
39934033
4034 if ( !$self->{initialized} ) {
4035 $self->{initialized} = 1;
4036 if ( $self->{resume}->{tbl}
4037 && !$self->table_is_allowed(@{$self->{resume}}{qw(db tbl)}) ) {
4038 PTDEBUG && _d('Will resume after',
4039 join('.', @{$self->{resume}}{qw(db tbl)}));
4040 $self->{resume}->{after} = 1;
4041 }
4042 }
4043
3994 my $schema_obj;4044 my $schema_obj;
3995 if ( $self->{file_itr} ) {4045 if ( $self->{file_itr} ) {
3996 $schema_obj= $self->_iterate_files();4046 $schema_obj= $self->_iterate_files();
@@ -4000,19 +4050,13 @@
4000 }4050 }
40014051
4002 if ( $schema_obj ) {4052 if ( $schema_obj ) {
4003 if ( $schema_obj->{ddl} && $self->{TableParser} ) {
4004 $schema_obj->{tbl_struct}
4005 = $self->{TableParser}->parse($schema_obj->{ddl});
4006 }
4007
4008 delete $schema_obj->{ddl} unless $self->{keep_ddl};
4009
4010 if ( my $schema = $self->{Schema} ) {4053 if ( my $schema = $self->{Schema} ) {
4011 $schema->add_schema_object($schema_obj);4054 $schema->add_schema_object($schema_obj);
4012 }4055 }
4056 PTDEBUG && _d('Next schema object:',
4057 $schema_obj->{db}, $schema_obj->{tbl});
4013 }4058 }
40144059
4015 PTDEBUG && _d('Next schema object:', $schema_obj->{db}, $schema_obj->{tbl});
4016 return $schema_obj;4060 return $schema_obj;
4017}4061}
40184062
@@ -4038,7 +4082,8 @@
4038 my $db = $1; # XXX4082 my $db = $1; # XXX
4039 $db =~ s/^`//; # strip leading `4083 $db =~ s/^`//; # strip leading `
4040 $db =~ s/`$//; # and trailing `4084 $db =~ s/`$//; # and trailing `
4041 if ( $self->database_is_allowed($db) ) {4085 if ( $self->database_is_allowed($db)
4086 && $self->_resume_from_database($db) ) {
4042 $self->{db} = $db;4087 $self->{db} = $db;
4043 }4088 }
4044 }4089 }
@@ -4051,21 +4096,22 @@
4051 my ($tbl) = $chunk =~ m/$tbl_name/;4096 my ($tbl) = $chunk =~ m/$tbl_name/;
4052 $tbl =~ s/^\s*`//;4097 $tbl =~ s/^\s*`//;
4053 $tbl =~ s/`\s*$//;4098 $tbl =~ s/`\s*$//;
4054 if ( $self->table_is_allowed($self->{db}, $tbl) ) {4099 if ( $self->_resume_from_table($tbl)
4100 && $self->table_is_allowed($self->{db}, $tbl) ) {
4055 my ($ddl) = $chunk =~ m/^(?:$open_comment)?(CREATE TABLE.+?;)$/ms;4101 my ($ddl) = $chunk =~ m/^(?:$open_comment)?(CREATE TABLE.+?;)$/ms;
4056 if ( !$ddl ) {4102 if ( !$ddl ) {
4057 warn "Failed to parse CREATE TABLE from\n" . $chunk;4103 warn "Failed to parse CREATE TABLE from\n" . $chunk;
4058 next CHUNK;4104 next CHUNK;
4059 }4105 }
4060 $ddl =~ s/ \*\/;\Z/;/; # remove end of version comment4106 $ddl =~ s/ \*\/;\Z/;/; # remove end of version comment
40614107 my $tbl_struct = $self->{TableParser}->parse($ddl);
4062 my ($engine) = $ddl =~ m/\).*?(?:ENGINE|TYPE)=(\w+)/; 4108 if ( $self->engine_is_allowed($tbl_struct->{engine}) ) {
4063
4064 if ( !$engine || $self->engine_is_allowed($engine) ) {
4065 return {4109 return {
4066 db => $self->{db},4110 db => $self->{db},
4067 tbl => $tbl,4111 tbl => $tbl,
4068 ddl => $ddl,4112 name => $self->{Quoter}->quote($self->{db}, $tbl),
4113 ddl => $ddl,
4114 tbl_struct => $tbl_struct,
4069 };4115 };
4070 }4116 }
4071 }4117 }
@@ -4082,6 +4128,7 @@
4082sub _iterate_dbh {4128sub _iterate_dbh {
4083 my ( $self ) = @_;4129 my ( $self ) = @_;
4084 my $q = $self->{Quoter};4130 my $q = $self->{Quoter};
4131 my $tp = $self->{TableParser};
4085 my $dbh = $self->{dbh};4132 my $dbh = $self->{dbh};
4086 PTDEBUG && _d('Getting next schema object from dbh', $dbh);4133 PTDEBUG && _d('Getting next schema object from dbh', $dbh);
40874134
@@ -4095,7 +4142,9 @@
4095 }4142 }
40964143
4097 if ( !$self->{db} ) {4144 if ( !$self->{db} ) {
4098 $self->{db} = shift @{$self->{dbs}};4145 do {
4146 $self->{db} = shift @{$self->{dbs}};
4147 } until $self->_resume_from_database($self->{db});
4099 PTDEBUG && _d('Next database:', $self->{db});4148 PTDEBUG && _d('Next database:', $self->{db});
4100 return unless $self->{db};4149 return unless $self->{db};
4101 }4150 }
@@ -4108,8 +4157,9 @@
4108 }4157 }
4109 grep {4158 grep {
4110 my ($tbl, $type) = @$_;4159 my ($tbl, $type) = @$_;
4111 $self->table_is_allowed($self->{db}, $tbl)4160 (!$type || ($type ne 'VIEW'))
4112 && (!$type || ($type ne 'VIEW'));4161 && $self->_resume_from_table($tbl)
4162 && $self->table_is_allowed($self->{db}, $tbl);
4113 }4163 }
4114 @{$dbh->selectall_arrayref($sql)};4164 @{$dbh->selectall_arrayref($sql)};
4115 PTDEBUG && _d('Found', scalar @tbls, 'tables in database', $self->{db});4165 PTDEBUG && _d('Found', scalar @tbls, 'tables in database', $self->{db});
@@ -4117,27 +4167,15 @@
4117 }4167 }
41184168
4119 while ( my $tbl = shift @{$self->{tbls}} ) {4169 while ( my $tbl = shift @{$self->{tbls}} ) {
4120 my $engine;4170 my $ddl = $tp->get_create_table($dbh, $self->{db}, $tbl);
4121 if ( $self->{filters}->{'engines'}4171 my $tbl_struct = $tp->parse($ddl);
4122 || $self->{filters}->{'ignore-engines'} ) {4172 if ( $self->engine_is_allowed($tbl_struct->{engine}) ) {
4123 my $sql = "SHOW TABLE STATUS FROM " . $q->quote($self->{db})
4124 . " LIKE \'$tbl\'";
4125 PTDEBUG && _d($sql);
4126 $engine = $dbh->selectrow_hashref($sql)->{engine};
4127 PTDEBUG && _d($tbl, 'uses', $engine, 'engine');
4128 }
4129
4130
4131 if ( !$engine || $self->engine_is_allowed($engine) ) {
4132 my $ddl;
4133 if ( my $du = $self->{MySQLDump} ) {
4134 $ddl = $du->get_create_table($dbh, $q, $self->{db}, $tbl)->[1];
4135 }
4136
4137 return {4173 return {
4138 db => $self->{db},4174 db => $self->{db},
4139 tbl => $tbl,4175 tbl => $tbl,
4140 ddl => $ddl,4176 name => $q->quote($self->{db}, $tbl),
4177 ddl => $ddl,
4178 tbl_struct => $tbl_struct,
4141 };4179 };
4142 }4180 }
4143 }4181 }
@@ -4198,6 +4236,10 @@
41984236
4199 my $filter = $self->{filters};4237 my $filter = $self->{filters};
42004238
4239 if ( $db eq 'mysql' && ($tbl eq 'general_log' || $tbl eq 'slow_log') ) {
4240 return 0;
4241 }
4242
4201 if ( $filter->{'ignore-tables'}->{$tbl}4243 if ( $filter->{'ignore-tables'}->{$tbl}
4202 && ($filter->{'ignore-tables'}->{$tbl} eq '*'4244 && ($filter->{'ignore-tables'}->{$tbl} eq '*'
4203 || $filter->{'ignore-tables'}->{$tbl} eq $db) ) {4245 || $filter->{'ignore-tables'}->{$tbl} eq $db) ) {
@@ -4237,7 +4279,11 @@
42374279
4238sub engine_is_allowed {4280sub engine_is_allowed {
4239 my ( $self, $engine ) = @_;4281 my ( $self, $engine ) = @_;
4240 die "I need an engine argument" unless $engine;4282
4283 if ( !$engine ) {
4284 PTDEBUG && _d('No engine specified; allowing the table');
4285 return 1;
4286 }
42414287
4242 $engine = lc $engine;4288 $engine = lc $engine;
42434289
@@ -4257,6 +4303,40 @@
4257 return 1;4303 return 1;
4258}4304}
42594305
4306sub _resume_from_database {
4307 my ($self, $db) = @_;
4308
4309 return 1 unless $self->{resume}->{db};
4310
4311 if ( $db eq $self->{resume}->{db} ) {
4312 PTDEBUG && _d('At resume db', $db);
4313 delete $self->{resume}->{db};
4314 return 1;
4315 }
4316
4317 return 0;
4318}
4319
4320sub _resume_from_table {
4321 my ($self, $tbl) = @_;
4322
4323 return 1 unless $self->{resume}->{tbl};
4324
4325 if ( $tbl eq $self->{resume}->{tbl} ) {
4326 if ( !$self->{resume}->{after} ) {
4327 PTDEBUG && _d('Resuming from table', $tbl);
4328 delete $self->{resume}->{tbl};
4329 return 1;
4330 }
4331 else {
4332 PTDEBUG && _d('Resuming after table', $tbl);
4333 delete $self->{resume}->{tbl};
4334 }
4335 }
4336
4337 return 0;
4338}
4339
4260sub _d {4340sub _d {
4261 my ($package, undef, $line) = caller 0;4341 my ($package, undef, $line) = caller 0;
4262 @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }4342 @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; }
@@ -5149,13 +5229,11 @@
5149 dbh => $dbh,5229 dbh => $dbh,
5150 OptionParser => $o,5230 OptionParser => $o,
5151 Quoter => $q,5231 Quoter => $q,
5152 MySQLDump => $du,
5153 TableParser => $tp,5232 TableParser => $tp,
5154 Schema => $schema,5233 Schema => $schema,
5155 keep_ddl => 1,
5156 );5234 );
5157 TALBE:5235 TALBE:
5158 while ( my $tbl = $schema_itr->next_schema_object() ) {5236 while ( my $tbl = $schema_itr->next() ) {
5159 eval {5237 eval {
5160 my ($indexes) = $tp->get_keys($tbl->{ddl}, {version => $version});5238 my ($indexes) = $tp->get_keys($tbl->{ddl}, {version => $version});
5161 $iu->add_indexes(%$tbl, indexes=>$indexes);5239 $iu->add_indexes(%$tbl, indexes=>$indexes);
@@ -6191,6 +6269,6 @@
61916269
6192=head1 VERSION6270=head1 VERSION
61936271
6194pt-index-usage 2.0.46272pt-index-usage 2.1.1
61956273
6196=cut6274=cut
61976275
=== modified file 'bin/pt-ioprofile'
--- bin/pt-ioprofile 2012-05-16 17:31:10 +0000
+++ bin/pt-ioprofile 2012-05-30 21:00:30 +0000
@@ -1037,7 +1037,7 @@
10371037
1038=head1 VERSION1038=head1 VERSION
10391039
1040pt-ioprofile 2.0.41040pt-ioprofile 2.1.1
10411041
1042=cut1042=cut
10431043
10441044
=== modified file 'bin/pt-kill'
--- bin/pt-kill 2012-05-24 17:52:01 +0000
+++ bin/pt-kill 2012-05-30 21:00:30 +0000
@@ -4860,6 +4860,6 @@
48604860
4861=head1 VERSION4861=head1 VERSION
48624862
4863pt-kill 2.0.44863pt-kill 2.1.1
48644864
4865=cut4865=cut
48664866
=== modified file 'bin/pt-log-player'
--- bin/pt-log-player 2012-05-24 17:52:01 +0000
+++ bin/pt-log-player 2012-05-30 21:00:30 +0000
@@ -3609,6 +3609,6 @@
36093609
3610=head1 VERSION3610=head1 VERSION
36113611
3612pt-log-player 2.0.43612pt-log-player 2.1.1
36133613
3614=cut3614=cut
36153615
=== modified file 'bin/pt-mext'
--- bin/pt-mext 2012-05-16 17:31:10 +0000
+++ bin/pt-mext 2012-05-30 21:00:30 +0000
@@ -279,7 +279,7 @@
279279
280=head1 VERSION280=head1 VERSION
281281
282pt-mext 2.0.4282pt-mext 2.1.1
283283
284=cut284=cut
285285
286286
=== modified file 'bin/pt-mysql-summary'
--- bin/pt-mysql-summary 2012-05-16 17:31:10 +0000
+++ bin/pt-mysql-summary 2012-05-30 21:00:30 +0000
@@ -4,14 +4,421 @@
4# See "COPYRIGHT, LICENSE, AND WARRANTY" at the end of this file for legal4# See "COPYRIGHT, LICENSE, AND WARRANTY" at the end of this file for legal
5# notices and disclaimers.5# notices and disclaimers.
66
7set -u
8
9# ###########################################################################
10# log_warn_die package
11# This package is a copy without comments from the original. The original
12# with comments and its test file can be found in the Bazaar repository at,
13# lib/bash/log_warn_die.sh
14# t/lib/bash/log_warn_die.sh
15# See https://launchpad.net/percona-toolkit for more information.
16# ###########################################################################
17
18
19set -u
20
21PTFUNCNAME=""
22PTDEBUG="${PTDEBUG:-""}"
23EXIT_STATUS=0
24
25log() {
26 TS=$(date +%F-%T | tr :- _);
27 echo "$TS $*"
28}
29
30warn() {
31 log "$*" >&2
32 EXIT_STATUS=1
33}
34
35die() {
36 warn "$*"
37 exit 1
38}
39
40_d () {
41 [ "$PTDEBUG" ] && echo "# $PTFUNCNAME: $(log "$*")" >&2
42}
43
44# ###########################################################################
45# End log_warn_die package
46# ###########################################################################
47
48# ###########################################################################
49# parse_options package
50# This package is a copy without comments from the original. The original
51# with comments and its test file can be found in the Bazaar repository at,
52# lib/bash/parse_options.sh
53# t/lib/bash/parse_options.sh
54# See https://launchpad.net/percona-toolkit for more information.
55# ###########################################################################
56
57
58
59
60
61set -u
62
63ARGV="" # Non-option args (probably input files)
64EXT_ARGV="" # Everything after -- (args for an external command)
65HAVE_EXT_ARGV="" # Got --, everything else is put into EXT_ARGV
66OPT_ERRS=0 # How many command line option errors
67OPT_VERSION="" # If --version was specified
68OPT_HELP="" # If --help was specified
69PO_DIR="" # Directory with program option spec files
70
7usage() {71usage() {
8 if [ "${OPT_ERR}" ]; then72 local file="$1"
9 echo "${OPT_ERR}" >&273
10 fi74 local usage=$(grep '^Usage: ' "$file")
11 echo "Usage: pt-mysql-summary [MYSQL-OPTIONS]" >&275 echo $usage
12 echo "For more information, 'man pt-mysql-summary' or 'perldoc $0'" >&276 echo
13 exit 177 echo "For more information, 'man $TOOL' or 'perldoc $file'."
14}78}
79
80usage_or_errors() {
81 local file="$1"
82
83 if [ "$OPT_VERSION" ]; then
84 local version=$(grep '^pt-[^ ]\+ [0-9]' "$file")
85 echo "$version"
86 return 1
87 fi
88
89 if [ "$OPT_HELP" ]; then
90 usage "$file"
91 echo
92 echo "Command line options:"
93 echo
94 perl -e '
95 use strict;
96 use warnings FATAL => qw(all);
97 my $lcol = 20; # Allow this much space for option names.
98 my $rcol = 80 - $lcol; # The terminal is assumed to be 80 chars wide.
99 my $name;
100 while ( <> ) {
101 my $line = $_;
102 chomp $line;
103 if ( $line =~ s/^long:/ --/ ) {
104 $name = $line;
105 }
106 elsif ( $line =~ s/^desc:// ) {
107 $line =~ s/ +$//mg;
108 my @lines = grep { $_ }
109 $line =~ m/(.{0,$rcol})(?:\s+|\Z)/g;
110 if ( length($name) >= $lcol ) {
111 print $name, "\n", (q{ } x $lcol);
112 }
113 else {
114 printf "%-${lcol}s", $name;
115 }
116 print join("\n" . (q{ } x $lcol), @lines);
117 print "\n";
118 }
119 }
120 ' "$PO_DIR"/*
121 echo
122 echo "Options and values after processing arguments:"
123 echo
124 for opt in $(ls "$PO_DIR"); do
125 local varname="OPT_$(echo "$opt" | tr a-z- A-Z_)"
126 local varvalue="${!varname}"
127 printf -- " --%-30s %s" "$opt" "${varvalue:-(No value)}"
128 echo
129 done
130 return 1
131 fi
132
133 if [ $OPT_ERRS -gt 0 ]; then
134 echo
135 usage "$file"
136 return 1
137 fi
138
139 return 0
140}
141
142option_error() {
143 local err="$1"
144 OPT_ERRS=$(($OPT_ERRS + 1))
145 echo "$err" >&2
146}
147
148parse_options() {
149 local file="$1"
150 shift
151
152 ARGV=""
153 EXT_ARGV=""
154 HAVE_EXT_ARGV=""
155 OPT_ERRS=0
156 OPT_VERSION=""
157 OPT_HELP=""
158 PO_DIR="$TMPDIR/po"
159
160 if [ ! -d "$PO_DIR" ]; then
161 mkdir "$PO_DIR"
162 if [ $? -ne 0 ]; then
163 echo "Cannot mkdir $PO_DIR" >&2
164 exit 1
165 fi
166 fi
167
168 rm -rf "$PO_DIR"/*
169 if [ $? -ne 0 ]; then
170 echo "Cannot rm -rf $PO_DIR/*" >&2
171 exit 1
172 fi
173
174 _parse_pod "$file" # Parse POD into program option (po) spec files
175 _eval_po # Eval po into existence with default values
176
177 if [ $# -ge 2 ] && [ "$1" = "--config" ]; then
178 shift # --config
179 local user_config_files="$1"
180 shift # that ^
181 local IFS=","
182 for user_config_file in $user_config_files; do
183 _parse_config_files "$user_config_file"
184 done
185 else
186 _parse_config_files "/etc/percona-toolkit/percona-toolkit.conf" "/etc/percona-toolkit/$TOOL.conf" "$HOME/.percona-toolkit.conf" "$HOME/.$TOOL.conf"
187 fi
188
189 _parse_command_line "$@"
190}
191
192_parse_pod() {
193 local file="$1"
194
195 cat "$file" | PO_DIR="$PO_DIR" perl -ne '
196 BEGIN { $/ = ""; }
197 next unless $_ =~ m/^=head1 OPTIONS/;
198 while ( defined(my $para = <>) ) {
199 last if $para =~ m/^=head1/;
200 chomp;
201 if ( $para =~ m/^=item --(\S+)/ ) {
202 my $opt = $1;
203 my $file = "$ENV{PO_DIR}/$opt";
204 open my $opt_fh, ">", $file or die "Cannot open $file: $!";
205 print $opt_fh "long:$opt\n";
206 $para = <>;
207 chomp;
208 if ( $para =~ m/^[a-z ]+:/ ) {
209 map {
210 chomp;
211 my ($attrib, $val) = split(/: /, $_);
212 print $opt_fh "$attrib:$val\n";
213 } split(/; /, $para);
214 $para = <>;
215 chomp;
216 }
217 my ($desc) = $para =~ m/^([^?.]+)/;
218 print $opt_fh "desc:$desc.\n";
219 close $opt_fh;
220 }
221 }
222 last;
223 '
224}
225
226_eval_po() {
227 local IFS=":"
228 for opt_spec in "$PO_DIR"/*; do
229 local opt=""
230 local default_val=""
231 local neg=0
232 local size=0
233 while read key val; do
234 case "$key" in
235 long)
236 opt=$(echo $val | sed 's/-/_/g' | tr [:lower:] [:upper:])
237 ;;
238 default)
239 default_val="$val"
240 ;;
241 "short form")
242 ;;
243 type)
244 [ "$val" = "size" ] && size=1
245 ;;
246 desc)
247 ;;
248 negatable)
249 if [ "$val" = "yes" ]; then
250 neg=1
251 fi
252 ;;
253 *)
254 echo "Invalid attribute in $opt_spec: $line" >&2
255 exit 1
256 esac
257 done < "$opt_spec"
258
259 if [ -z "$opt" ]; then
260 echo "No long attribute in option spec $opt_spec" >&2
261 exit 1
262 fi
263
264 if [ $neg -eq 1 ]; then
265 if [ -z "$default_val" ] || [ "$default_val" != "yes" ]; then
266 echo "Option $opt_spec is negatable but not default: yes" >&2
267 exit 1
268 fi
269 fi
270
271 if [ $size -eq 1 -a -n "$default_val" ]; then
272 default_val=$(size_to_bytes $default_val)
273 fi
274
275 eval "OPT_${opt}"="$default_val"
276 done
277}
278
279_parse_config_files() {
280
281 for config_file in "$@"; do
282 test -f "$config_file" || continue
283
284 while read config_opt; do
285
286 echo "$config_opt" | grep '^[ ]*[^#]' >/dev/null 2>&1 || continue
287
288 config_opt="$(echo "$config_opt" | sed -e 's/^ *//g' -e 's/ *$//g' -e 's/[ ]*=[ ]*/=/' -e 's/[ ]*#.*$//')"
289
290 [ "$config_opt" = "" ] && continue
291
292 if ! [ "$HAVE_EXT_ARGV" ]; then
293 config_opt="--$config_opt"
294 fi
295
296 _parse_command_line "$config_opt"
297
298 done < "$config_file"
299
300 HAVE_EXT_ARGV="" # reset for each file
301
302 done
303}
304
305_parse_command_line() {
306 local opt=""
307 local val=""
308 local next_opt_is_val=""
309 local opt_is_ok=""
310 local opt_is_negated=""
311 local real_opt=""
312 local required_arg=""
313 local spec=""
314
315 for opt in "$@"; do
316 if [ "$opt" = "--" -o "$opt" = "----" ]; then
317 HAVE_EXT_ARGV=1
318 continue
319 fi
320 if [ "$HAVE_EXT_ARGV" ]; then
321 if [ "$EXT_ARGV" ]; then
322 EXT_ARGV="$EXT_ARGV $opt"
323 else
324 EXT_ARGV="$opt"
325 fi
326 continue
327 fi
328
329 if [ "$next_opt_is_val" ]; then
330 next_opt_is_val=""
331 if [ $# -eq 0 ] || [ $(expr "$opt" : "-") -eq 1 ]; then
332 option_error "$real_opt requires a $required_arg argument"
333 continue
334 fi
335 val="$opt"
336 opt_is_ok=1
337 else
338 if [ $(expr "$opt" : "-") -eq 0 ]; then
339 if [ -z "$ARGV" ]; then
340 ARGV="$opt"
341 else
342 ARGV="$ARGV $opt"
343 fi
344 continue
345 fi
346
347 real_opt="$opt"
348
349 if $(echo $opt | grep '^--no-' >/dev/null); then
350 opt_is_negated=1
351 opt=$(echo $opt | sed 's/^--no-//')
352 else
353 opt_is_negated=""
354 opt=$(echo $opt | sed 's/^-*//')
355 fi
356
357 if $(echo $opt | grep '^[a-z-][a-z-]*=' >/dev/null 2>&1); then
358 val="$(echo $opt | awk -F= '{print $2}')"
359 opt="$(echo $opt | awk -F= '{print $1}')"
360 fi
361
362 if [ -f "$TMPDIR/po/$opt" ]; then
363 spec="$TMPDIR/po/$opt"
364 else
365 spec=$(grep "^short form:-$opt\$" "$TMPDIR"/po/* | cut -d ':' -f 1)
366 if [ -z "$spec" ]; then
367 option_error "Unknown option: $real_opt"
368 continue
369 fi
370 fi
371
372 required_arg=$(cat "$spec" | awk -F: '/^type:/{print $2}')
373 if [ "$required_arg" ]; then
374 if [ "$val" ]; then
375 opt_is_ok=1
376 else
377 next_opt_is_val=1
378 fi
379 else
380 if [ "$val" ]; then
381 option_error "Option $real_opt does not take a value"
382 continue
383 fi
384 if [ "$opt_is_negated" ]; then
385 val=""
386 else
387 val="yes"
388 fi
389 opt_is_ok=1
390 fi
391 fi
392
393 if [ "$opt_is_ok" ]; then
394 opt=$(cat "$spec" | grep '^long:' | cut -d':' -f2 | sed 's/-/_/g' | tr [:lower:] [:upper:])
395
396 if grep "^type:size" "$spec" >/dev/null; then
397 val=$(size_to_bytes $val)
398 fi
399
400 eval "OPT_$opt"="'$val'"
401
402 opt=""
403 val=""
404 next_opt_is_val=""
405 opt_is_ok=""
406 opt_is_negated=""
407 real_opt=""
408 required_arg=""
409 spec=""
410 fi
411 done
412}
413
414size_to_bytes() {
415 local size="$1"
416 echo $size | perl -ne '%f=(B=>1, K=>1_024, M=>1_048_576, G=>1_073_741_824, T=>1_099_511_627_776); m/^(\d+)([kMGT])?/i; print $1 * $f{uc($2 || "B")};'
417}
418
419# ###########################################################################
420# End parse_options package
421# ###########################################################################
15422
16# ###########################################################################423# ###########################################################################
17# tmpdir package424# tmpdir package
@@ -22,6 +429,9 @@
22# See https://launchpad.net/percona-toolkit for more information.429# See https://launchpad.net/percona-toolkit for more information.
23# ###########################################################################430# ###########################################################################
24431
432
433set -u
434
25TMPDIR=""435TMPDIR=""
26436
27mk_tmpdir() {437mk_tmpdir() {
@@ -51,27 +461,68 @@
51# End tmpdir package461# End tmpdir package
52# ###########################################################################462# ###########################################################################
53463
54# ########################################################################464# ###########################################################################
55# Some global setup is necessary for cross-platform compatibility, even465# alt_cmds package
56# when sourcing this script for testing purposes.466# This package is a copy without comments from the original. The original
57# ########################################################################467# with comments and its test file can be found in the Bazaar repository at,
58AP_AWK="$(which awk)"468# lib/bash/alt_cmds.sh
59which gawk >/dev/null 2>&1 && AP_AWK="$(which gawk)"469# t/lib/bash/alt_cmds.sh
60AP_SED="$(which sed)"470# See https://launchpad.net/percona-toolkit for more information.
61which gsed >/dev/null 2>&1 && AP_SED="$(which gsed)"471# ###########################################################################
62AP_GREP="$(which grep)"472
63which ggrep >/dev/null 2>&1 && AP_GREP="$(which ggrep)"473
64474set -u
65# ########################################################################475
66# Globals, helper functions476_seq() {
67# ########################################################################477 local i="$1"
68478 awk "BEGIN { for(i=1; i<=$i; i++) print i; }"
69# The awk code for fuzzy rounding. (It's used in a few places, so makes sense479}
70# not to duplicate). It fuzzy-rounds the variable named fuzzy_var. It goes in480
71# steps of 5, 10, 25, then repeats by a factor of 10 larger (50, 100, 250), and481_pidof() {
72# so on, until it finds a number that's large enough. The pattern is slightly482 local cmd="$1"
73# broken between the initial 1 and 50, because rounding to the nearest 2.5483 if ! pidof "$cmd" 2>/dev/null; then
74# doesn't seem right to me.484 ps -eo pid,ucomm | awk -v comm="$cmd" '$2 == comm { print $1 }'
485 fi
486}
487
488_lsof() {
489 local pid="$1"
490 if ! lsof -p $pid 2>/dev/null; then
491 /bin/ls -l /proc/$pid/fd 2>/dev/null
492 fi
493}
494
495
496
497_which() {
498 if [ -x /usr/bin/which ]; then
499 /usr/bin/which "$1" 2>/dev/null | awk '{print $1}'
500 elif which which 1>/dev/null 2>&1; then
501 which "$1" 2>/dev/null | awk '{print $1}'
502 else
503 echo "$1"
504 fi
505}
506
507# ###########################################################################
508# End alt_cmds package
509# ###########################################################################
510
511# ###########################################################################
512# report_formatting package
513# This package is a copy without comments from the original. The original
514# with comments and its test file can be found in the Bazaar repository at,
515# lib/bash/report_formatting.sh
516# t/lib/bash/report_formatting.sh
517# See https://launchpad.net/percona-toolkit for more information.
518# ###########################################################################
519
520
521set -u
522
523POSIXLY_CORRECT=1
524export POSIXLY_CORRECT
525
75fuzzy_formula='526fuzzy_formula='
76 rounded = 0;527 rounded = 0;
77 if (fuzzy_var <= 10 ) {528 if (fuzzy_var <= 10 ) {
@@ -94,108 +545,461 @@
94 factor = factor * 10;545 factor = factor * 10;
95 }'546 }'
96547
97# The temp files are for storing working results so we don't call commands many548fuzz () {
98# times (gives inconsistent results, maybe adds load on things I don't want to549 awk -v fuzzy_var="$1" "BEGIN { ${fuzzy_formula} print fuzzy_var;}"
99# such as RAID controllers). They must not exist -- if they did, someone would550}
100# symlink them to /etc/passwd and then run this program as root. Call this551
101# function with "rm" or "touch" as an argument.552fuzzy_pct () {
102temp_files() {553 local pct="$(awk -v one="$1" -v two="$2" 'BEGIN{ if (two > 0) { printf "%d", one/two*100; } else {print 0} }')";
103 for file in $TMPDIR/percona-toolkit{,-mysql-variables,-mysql-status,-innodb-status} \554 echo "$(fuzz "${pct}")%"
104 $TMPDIR/percona-toolkit{2,-mysql-databases,-mysql-processlist,-noncounters} \555}
105 $TMPDIR/percona-toolkit-mysql{dump,-slave};556
106 do
107 case "$1" in
108 touch)
109 if ! touch "${file}"; then
110 echo "I can't make my temp file ${file}";
111 exit 1;
112 fi
113 ;;
114 rm)
115 rm -f "${file}"
116 ;;
117 esac
118 done
119}
120
121# Print a space-padded string into $line. Then translate spaces to hashes, and
122# underscores to spaces. End result is a line of hashes with words at the
123# start.
124section () {557section () {
125 line="$(printf '#_%-60s' "$1_")"558 local str="$1"
126 line="${line// /#}"559 awk -v var="${str} _" 'BEGIN {
127 printf "%s\n" "${line//_/ }"560 line = sprintf("# %-60s", var);
128}561 i = index(line, "_");
129562 x = substr(line, i);
130# Print a "name | value" line.563 gsub(/[_ \t]/, "#", x);
131name_val() {564 printf("%s%s\n", substr(line, 1, i-1), x);
132 printf "%20s | %s\n" "$1" "$2"565 }'
133}566}
134567
135# Converts a value to units of power of 2. Optional precision is $2.568NAME_VAL_LEN=12
569name_val () {
570 printf "%+*s | %s\n" "${NAME_VAL_LEN}" "$1" "$2"
571}
572
136shorten() {573shorten() {
137 unit=k574 local num="$1"
138 size=1024575 local prec="${2:-2}"
139 if [ $1 -ge 1099511627776 ] ; then576 local div="${3:-1024}"
140 size=1099511627776577
141 unit=T578 echo "$num" | awk -v prec="$prec" -v div="$div" '
142 elif [ $1 -ge 1073741824 ] ; then579 {
143 size=1073741824580 size = 4;
144 unit=G581 val = $1;
145 elif [ $1 -ge 1048576 ] ; then582
146 size=1048576583 unit = val >= 1099511627776 ? "T" : val >= 1073741824 ? "G" : val >= 1048576 ? "M" : val >= 1024 ? "k" : "";
147 unit=M584
148 fi585 while ( int(val) && !(val % 1024) ) {
149 result=$(echo "$1 $size ${2:-0}" | $AP_AWK '{printf "%." $3 "f", $1 / $2}')586 val /= 1024;
150 echo "${result}${unit}"587 }
588
589 while ( val > 1000 ) {
590 val /= div;
591 }
592
593 printf "%.*f%s", prec, val, unit;
594 }
595 '
151}596}
152597
153# Collapse a file into an aggregated list; file must be created with 'sort |
154# uniq -c'. This function is copy-pasted from 'summary' so see there for full
155# docs and tests.
156# ##############################################################################
157group_concat () {598group_concat () {
158 sed -e '{H; $!d}' -e 'x' -e 's/\n[[:space:]]*\([[:digit:]]*\)[[:space:]]*/, \1x/g' -e 's/[[:space:]][[:space:]]*/ /g' -e 's/, //' ${1}599 sed -e '{H; $!d;}' -e 'x' -e 's/\n[[:space:]]*\([[:digit:]]*\)[[:space:]]*/, \1x/g' -e 's/[[:space:]][[:space:]]*/ /g' -e 's/, //' "${1}"
159}600}
160601
161# Accepts a number of seconds, and outputs a d+h:m:s formatted string602# ###########################################################################
603# End report_formatting package
604# ###########################################################################
605
606# ###########################################################################
607# summary_common package
608# This package is a copy without comments from the original. The original
609# with comments and its test file can be found in the Bazaar repository at,
610# lib/bash/summary_common.sh
611# t/lib/bash/summary_common.sh
612# See https://launchpad.net/percona-toolkit for more information.
613# ###########################################################################
614
615
616set -u
617
618CMD_FILE="$( _which file 2>/dev/null )"
619CMD_NM="$( _which nm 2>/dev/null )"
620CMD_OBJDUMP="$( _which objdump 2>/dev/null )"
621
622get_nice_of_pid () {
623 local pid="$1"
624 local niceness="$(ps -p $pid -o nice | awk '$1 !~ /[^0-9]/ {print $1; exit}')"