Merge lp:~percona-toolkit-dev/percona-toolkit/pt-osc-2.1.1 into lp:percona-toolkit/2.1
- pt-osc-2.1.1
- Merge into 2.1
Proposed by
Daniel Nichter
Status: | Merged | ||||||||
---|---|---|---|---|---|---|---|---|---|
Merged at revision: | 228 | ||||||||
Proposed branch: | lp:~percona-toolkit-dev/percona-toolkit/pt-osc-2.1.1 | ||||||||
Merge into: | lp:percona-toolkit/2.1 | ||||||||
Diff against target: |
9987 lines (+6370/-2594) (has conflicts) 24 files modified
bin/pt-online-schema-change (+4615/-1243) bin/pt-table-checksum (+114/-37) lib/CleanupTask.pm (+7/-2) lib/MySQLStatusWaiter.pm (+34/-12) lib/NibbleIterator.pm (+165/-49) lib/OSCCaptureSync.pm (+0/-142) lib/OobNibbleIterator.pm (+1/-1) lib/SchemaIterator.pm (+12/-0) t/lib/CleanupTask.t (+18/-1) t/lib/MySQLStatusWaiter.t (+38/-4) t/lib/NibbleIterator.t (+42/-4) t/lib/OSCCaptureSync.t (+0/-131) t/lib/OobNibbleIterator.t (+8/-3) t/pt-online-schema-change/alter_active_table.t (+56/-46) t/pt-online-schema-change/basics.t (+541/-218) t/pt-online-schema-change/check_tables.t (+0/-126) t/pt-online-schema-change/option_sanity.t (+12/-12) t/pt-online-schema-change/samples/basic_no_fks.data (+500/-500) t/pt-online-schema-change/samples/basic_no_fks.sql (+30/-0) t/pt-online-schema-change/samples/basic_with_fks.sql (+56/-0) t/pt-online-schema-change/samples/fk_tables_schema.sql (+0/-31) t/pt-online-schema-change/samples/query_table.pl (+8/-5) t/pt-online-schema-change/samples/small_table.sql (+0/-27) t/pt-online-schema-change/sanity_checks.t (+113/-0) Text conflict in bin/pt-online-schema-change Text conflict in bin/pt-table-checksum Text conflict in lib/NibbleIterator.pm Text conflict in lib/SchemaIterator.pm |
||||||||
To merge this branch: | bzr merge lp:~percona-toolkit-dev/percona-toolkit/pt-osc-2.1.1 | ||||||||
Related bugs: |
|
||||||||
Related blueprints: |
Redesign pt-online-schema-change
(Essential)
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Baron Schwartz (community) | Approve | ||
Daniel Nichter | Approve | ||
Brian Fraser | Pending | ||
Review via email: mp+100540@code.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
Revision history for this message
Daniel Nichter (daniel-nichter) : | # |
review:
Approve
Revision history for this message
Baron Schwartz (baron-xaprb) : | # |
review:
Approve
- 251. By Daniel Nichter
-
Add comment about code that will fail with new Cxn due to NAME_lc.
- 252. By Daniel Nichter
-
Catch lost cleanup task (should rarely happen).
- 253. By Daniel Nichter
-
Handle Key_name or key_name in NibbleIterator:
:_get_index_ cardinality( ). - 254. By Daniel Nichter
-
Add XXX comment about improperly copying .
- 255. By Daniel Nichter
-
Uncomment cleanup lines in alter_active_
table.t. - 256. By Daniel Nichter
-
Update modules in pt-online-
schema- change. - 257. By Daniel Nichter
-
Fix typo.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'bin/pt-online-schema-change' | |||
2 | --- bin/pt-online-schema-change 2012-03-30 16:37:16 +0000 | |||
3 | +++ bin/pt-online-schema-change 2012-04-03 02:13:24 +0000 | |||
4 | @@ -1784,245 +1784,246 @@ | |||
5 | 1784 | # ########################################################################### | 1784 | # ########################################################################### |
6 | 1785 | 1785 | ||
7 | 1786 | # ########################################################################### | 1786 | # ########################################################################### |
9 | 1787 | # Transformers package | 1787 | # TableNibbler package |
10 | 1788 | # This package is a copy without comments from the original. The original | 1788 | # This package is a copy without comments from the original. The original |
11 | 1789 | # with comments and its test file can be found in the Bazaar repository at, | 1789 | # with comments and its test file can be found in the Bazaar repository at, |
14 | 1790 | # lib/Transformers.pm | 1790 | # lib/TableNibbler.pm |
15 | 1791 | # t/lib/Transformers.t | 1791 | # t/lib/TableNibbler.t |
16 | 1792 | # See https://launchpad.net/percona-toolkit for more information. | 1792 | # See https://launchpad.net/percona-toolkit for more information. |
17 | 1793 | # ########################################################################### | 1793 | # ########################################################################### |
18 | 1794 | { | 1794 | { |
20 | 1795 | package Transformers; | 1795 | package TableNibbler; |
21 | 1796 | 1796 | ||
22 | 1797 | use strict; | 1797 | use strict; |
23 | 1798 | use warnings FATAL => 'all'; | 1798 | use warnings FATAL => 'all'; |
24 | 1799 | use English qw(-no_match_vars); | 1799 | use English qw(-no_match_vars); |
25 | 1800 | use constant PTDEBUG => $ENV{PTDEBUG} || 0; | 1800 | use constant PTDEBUG => $ENV{PTDEBUG} || 0; |
26 | 1801 | 1801 | ||
251 | 1802 | use Time::Local qw(timegm timelocal); | 1802 | sub new { |
252 | 1803 | use Digest::MD5 qw(md5_hex); | 1803 | my ( $class, %args ) = @_; |
253 | 1804 | 1804 | my @required_args = qw(TableParser Quoter); | |
254 | 1805 | require Exporter; | 1805 | foreach my $arg ( @required_args ) { |
255 | 1806 | our @ISA = qw(Exporter); | 1806 | die "I need a $arg argument" unless $args{$arg}; |
256 | 1807 | our %EXPORT_TAGS = (); | 1807 | } |
257 | 1808 | our @EXPORT = (); | 1808 | my $self = { %args }; |
258 | 1809 | our @EXPORT_OK = qw( | 1809 | return bless $self, $class; |
259 | 1810 | micro_t | 1810 | } |
260 | 1811 | percentage_of | 1811 | |
261 | 1812 | secs_to_time | 1812 | sub generate_asc_stmt { |
262 | 1813 | time_to_secs | 1813 | my ( $self, %args ) = @_; |
263 | 1814 | shorten | 1814 | my @required_args = qw(tbl_struct index); |
264 | 1815 | ts | 1815 | foreach my $arg ( @required_args ) { |
265 | 1816 | parse_timestamp | 1816 | die "I need a $arg argument" unless defined $args{$arg}; |
266 | 1817 | unix_timestamp | 1817 | } |
267 | 1818 | any_unix_timestamp | 1818 | my ($tbl_struct, $index) = @args{@required_args}; |
268 | 1819 | make_checksum | 1819 | my @cols = $args{cols} ? @{$args{cols}} : @{$tbl_struct->{cols}}; |
269 | 1820 | crc32 | 1820 | my $q = $self->{Quoter}; |
270 | 1821 | ); | 1821 | |
271 | 1822 | 1822 | die "Index '$index' does not exist in table" | |
272 | 1823 | our $mysql_ts = qr/(\d\d)(\d\d)(\d\d) +(\d+):(\d+):(\d+)(\.\d+)?/; | 1823 | unless exists $tbl_struct->{keys}->{$index}; |
273 | 1824 | our $proper_ts = qr/(\d\d\d\d)-(\d\d)-(\d\d)[T ](\d\d):(\d\d):(\d\d)(\.\d+)?/; | 1824 | PTDEBUG && _d('Will ascend index', $index); |
274 | 1825 | our $n_ts = qr/(\d{1,5})([shmd]?)/; # Limit \d{1,5} because \d{6} looks | 1825 | |
275 | 1826 | 1826 | my @asc_cols = @{$tbl_struct->{keys}->{$index}->{cols}}; | |
276 | 1827 | sub micro_t { | 1827 | if ( $args{asc_first} ) { |
277 | 1828 | my ( $t, %args ) = @_; | 1828 | @asc_cols = $asc_cols[0]; |
278 | 1829 | my $p_ms = defined $args{p_ms} ? $args{p_ms} : 0; # precision for ms vals | 1829 | PTDEBUG && _d('Ascending only first column'); |
279 | 1830 | my $p_s = defined $args{p_s} ? $args{p_s} : 0; # precision for s vals | 1830 | } |
280 | 1831 | my $f; | 1831 | PTDEBUG && _d('Will ascend columns', join(', ', @asc_cols)); |
281 | 1832 | 1832 | ||
282 | 1833 | $t = 0 if $t < 0; | 1833 | my @asc_slice; |
283 | 1834 | 1834 | my %col_posn = do { my $i = 0; map { $_ => $i++ } @cols }; | |
284 | 1835 | $t = sprintf('%.17f', $t) if $t =~ /e/; | 1835 | foreach my $col ( @asc_cols ) { |
285 | 1836 | 1836 | if ( !exists $col_posn{$col} ) { | |
286 | 1837 | $t =~ s/\.(\d{1,6})\d*/\.$1/; | 1837 | push @cols, $col; |
287 | 1838 | 1838 | $col_posn{$col} = $#cols; | |
288 | 1839 | if ($t > 0 && $t <= 0.000999) { | 1839 | } |
289 | 1840 | $f = ($t * 1000000) . 'us'; | 1840 | push @asc_slice, $col_posn{$col}; |
290 | 1841 | } | 1841 | } |
291 | 1842 | elsif ($t >= 0.001000 && $t <= 0.999999) { | 1842 | PTDEBUG && _d('Will ascend, in ordinal position:', join(', ', @asc_slice)); |
292 | 1843 | $f = sprintf("%.${p_ms}f", $t * 1000); | 1843 | |
293 | 1844 | $f = ($f * 1) . 'ms'; # * 1 to remove insignificant zeros | 1844 | my $asc_stmt = { |
294 | 1845 | } | 1845 | cols => \@cols, |
295 | 1846 | elsif ($t >= 1) { | 1846 | index => $index, |
296 | 1847 | $f = sprintf("%.${p_s}f", $t); | 1847 | where => '', |
297 | 1848 | $f = ($f * 1) . 's'; # * 1 to remove insignificant zeros | 1848 | slice => [], |
298 | 1849 | } | 1849 | scols => [], |
299 | 1850 | else { | 1850 | }; |
300 | 1851 | $f = 0; # $t should = 0 at this point | 1851 | |
301 | 1852 | } | 1852 | if ( @asc_slice ) { |
302 | 1853 | 1853 | my $cmp_where; | |
303 | 1854 | return $f; | 1854 | foreach my $cmp ( qw(< <= >= >) ) { |
304 | 1855 | } | 1855 | $cmp_where = $self->generate_cmp_where( |
305 | 1856 | 1856 | type => $cmp, | |
306 | 1857 | sub percentage_of { | 1857 | slice => \@asc_slice, |
307 | 1858 | my ( $is, $of, %args ) = @_; | 1858 | cols => \@cols, |
308 | 1859 | my $p = $args{p} || 0; # float precision | 1859 | quoter => $q, |
309 | 1860 | my $fmt = $p ? "%.${p}f" : "%d"; | 1860 | is_nullable => $tbl_struct->{is_nullable}, |
310 | 1861 | return sprintf $fmt, ($is * 100) / ($of ||= 1); | 1861 | ); |
311 | 1862 | } | 1862 | $asc_stmt->{boundaries}->{$cmp} = $cmp_where->{where}; |
312 | 1863 | 1863 | } | |
313 | 1864 | sub secs_to_time { | 1864 | my $cmp = $args{asc_only} ? '>' : '>='; |
314 | 1865 | my ( $secs, $fmt ) = @_; | 1865 | $asc_stmt->{where} = $asc_stmt->{boundaries}->{$cmp}; |
315 | 1866 | $secs ||= 0; | 1866 | $asc_stmt->{slice} = $cmp_where->{slice}; |
316 | 1867 | return '00:00' unless $secs; | 1867 | $asc_stmt->{scols} = $cmp_where->{scols}; |
317 | 1868 | 1868 | } | |
318 | 1869 | $fmt ||= $secs >= 86_400 ? 'd' | 1869 | |
319 | 1870 | : $secs >= 3_600 ? 'h' | 1870 | return $asc_stmt; |
320 | 1871 | : 'm'; | 1871 | } |
321 | 1872 | 1872 | ||
322 | 1873 | return | 1873 | sub generate_cmp_where { |
323 | 1874 | $fmt eq 'd' ? sprintf( | 1874 | my ( $self, %args ) = @_; |
324 | 1875 | "%d+%02d:%02d:%02d", | 1875 | foreach my $arg ( qw(type slice cols is_nullable) ) { |
325 | 1876 | int($secs / 86_400), | 1876 | die "I need a $arg arg" unless defined $args{$arg}; |
326 | 1877 | int(($secs % 86_400) / 3_600), | 1877 | } |
327 | 1878 | int(($secs % 3_600) / 60), | 1878 | my @slice = @{$args{slice}}; |
328 | 1879 | $secs % 60) | 1879 | my @cols = @{$args{cols}}; |
329 | 1880 | : $fmt eq 'h' ? sprintf( | 1880 | my $is_nullable = $args{is_nullable}; |
330 | 1881 | "%02d:%02d:%02d", | 1881 | my $type = $args{type}; |
331 | 1882 | int(($secs % 86_400) / 3_600), | 1882 | my $q = $self->{Quoter}; |
332 | 1883 | int(($secs % 3_600) / 60), | 1883 | |
333 | 1884 | $secs % 60) | 1884 | (my $cmp = $type) =~ s/=//; |
334 | 1885 | : sprintf( | 1885 | |
335 | 1886 | "%02d:%02d", | 1886 | my @r_slice; # Resulting slice columns, by ordinal |
336 | 1887 | int(($secs % 3_600) / 60), | 1887 | my @r_scols; # Ditto, by name |
337 | 1888 | $secs % 60); | 1888 | |
338 | 1889 | } | 1889 | my @clauses; |
339 | 1890 | 1890 | foreach my $i ( 0 .. $#slice ) { | |
340 | 1891 | sub time_to_secs { | 1891 | my @clause; |
341 | 1892 | my ( $val, $default_suffix ) = @_; | 1892 | |
342 | 1893 | die "I need a val argument" unless defined $val; | 1893 | foreach my $j ( 0 .. $i - 1 ) { |
343 | 1894 | my $t = 0; | 1894 | my $ord = $slice[$j]; |
344 | 1895 | my ( $prefix, $num, $suffix ) = $val =~ m/([+-]?)(\d+)([a-z])?$/; | 1895 | my $col = $cols[$ord]; |
345 | 1896 | $suffix = $suffix || $default_suffix || 's'; | 1896 | my $quo = $q->quote($col); |
346 | 1897 | if ( $suffix =~ m/[smhd]/ ) { | 1897 | if ( $is_nullable->{$col} ) { |
347 | 1898 | $t = $suffix eq 's' ? $num * 1 # Seconds | 1898 | push @clause, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))"; |
348 | 1899 | : $suffix eq 'm' ? $num * 60 # Minutes | 1899 | push @r_slice, $ord, $ord; |
349 | 1900 | : $suffix eq 'h' ? $num * 3600 # Hours | 1900 | push @r_scols, $col, $col; |
350 | 1901 | : $num * 86400; # Days | 1901 | } |
351 | 1902 | 1902 | else { | |
352 | 1903 | $t *= -1 if $prefix && $prefix eq '-'; | 1903 | push @clause, "$quo = ?"; |
353 | 1904 | } | 1904 | push @r_slice, $ord; |
354 | 1905 | else { | 1905 | push @r_scols, $col; |
355 | 1906 | die "Invalid suffix for $val: $suffix"; | 1906 | } |
356 | 1907 | } | 1907 | } |
357 | 1908 | return $t; | 1908 | |
358 | 1909 | } | 1909 | my $ord = $slice[$i]; |
359 | 1910 | 1910 | my $col = $cols[$ord]; | |
360 | 1911 | sub shorten { | 1911 | my $quo = $q->quote($col); |
361 | 1912 | my ( $num, %args ) = @_; | 1912 | my $end = $i == $#slice; # Last clause of the whole group. |
362 | 1913 | my $p = defined $args{p} ? $args{p} : 2; # float precision | 1913 | if ( $is_nullable->{$col} ) { |
363 | 1914 | my $d = defined $args{d} ? $args{d} : 1_024; # divisor | 1914 | if ( $type =~ m/=/ && $end ) { |
364 | 1915 | my $n = 0; | 1915 | push @clause, "(? IS NULL OR $quo $type ?)"; |
365 | 1916 | my @units = ('', qw(k M G T P E Z Y)); | 1916 | } |
366 | 1917 | while ( $num >= $d && $n < @units - 1 ) { | 1917 | elsif ( $type =~ m/>/ ) { |
367 | 1918 | $num /= $d; | 1918 | push @clause, "((? IS NULL AND $quo IS NOT NULL) OR ($quo $cmp ?))"; |
368 | 1919 | ++$n; | 1919 | } |
369 | 1920 | } | 1920 | else { # If $type =~ m/</ ) { |
370 | 1921 | return sprintf( | 1921 | push @clause, "((? IS NOT NULL AND $quo IS NULL) OR ($quo $cmp ?))"; |
371 | 1922 | $num =~ m/\./ || $n | 1922 | } |
372 | 1923 | ? "%.${p}f%s" | 1923 | push @r_slice, $ord, $ord; |
373 | 1924 | : '%d', | 1924 | push @r_scols, $col, $col; |
374 | 1925 | $num, $units[$n]); | 1925 | } |
375 | 1926 | } | 1926 | else { |
376 | 1927 | 1927 | push @r_slice, $ord; | |
377 | 1928 | sub ts { | 1928 | push @r_scols, $col; |
378 | 1929 | my ( $time, $gmt ) = @_; | 1929 | push @clause, ($type =~ m/=/ && $end ? "$quo $type ?" : "$quo $cmp ?"); |
379 | 1930 | my ( $sec, $min, $hour, $mday, $mon, $year ) | 1930 | } |
380 | 1931 | = $gmt ? gmtime($time) : localtime($time); | 1931 | |
381 | 1932 | $mon += 1; | 1932 | push @clauses, '(' . join(' AND ', @clause) . ')'; |
382 | 1933 | $year += 1900; | 1933 | } |
383 | 1934 | my $val = sprintf("%d-%02d-%02dT%02d:%02d:%02d", | 1934 | my $result = '(' . join(' OR ', @clauses) . ')'; |
384 | 1935 | $year, $mon, $mday, $hour, $min, $sec); | 1935 | my $where = { |
385 | 1936 | if ( my ($us) = $time =~ m/(\.\d+)$/ ) { | 1936 | slice => \@r_slice, |
386 | 1937 | $us = sprintf("%.6f", $us); | 1937 | scols => \@r_scols, |
387 | 1938 | $us =~ s/^0\././; | 1938 | where => $result, |
388 | 1939 | $val .= $us; | 1939 | }; |
389 | 1940 | } | 1940 | return $where; |
390 | 1941 | return $val; | 1941 | } |
391 | 1942 | } | 1942 | |
392 | 1943 | 1943 | sub generate_del_stmt { | |
393 | 1944 | sub parse_timestamp { | 1944 | my ( $self, %args ) = @_; |
394 | 1945 | my ( $val ) = @_; | 1945 | |
395 | 1946 | if ( my($y, $m, $d, $h, $i, $s, $f) | 1946 | my $tbl = $args{tbl_struct}; |
396 | 1947 | = $val =~ m/^$mysql_ts$/ ) | 1947 | my @cols = $args{cols} ? @{$args{cols}} : (); |
397 | 1948 | { | 1948 | my $tp = $self->{TableParser}; |
398 | 1949 | return sprintf "%d-%02d-%02d %02d:%02d:" | 1949 | my $q = $self->{Quoter}; |
399 | 1950 | . (defined $f ? '%09.6f' : '%02d'), | 1950 | |
400 | 1951 | $y + 2000, $m, $d, $h, $i, (defined $f ? $s + $f : $s); | 1951 | my @del_cols; |
401 | 1952 | } | 1952 | my @del_slice; |
402 | 1953 | return $val; | 1953 | |
403 | 1954 | } | 1954 | my $index = $tp->find_best_index($tbl, $args{index}); |
404 | 1955 | 1955 | die "Cannot find an ascendable index in table" unless $index; | |
405 | 1956 | sub unix_timestamp { | 1956 | |
406 | 1957 | my ( $val, $gmt ) = @_; | 1957 | if ( $index ) { |
407 | 1958 | if ( my($y, $m, $d, $h, $i, $s, $us) = $val =~ m/^$proper_ts$/ ) { | 1958 | @del_cols = @{$tbl->{keys}->{$index}->{cols}}; |
408 | 1959 | $val = $gmt | 1959 | } |
409 | 1960 | ? timegm($s, $i, $h, $d, $m - 1, $y) | 1960 | else { |
410 | 1961 | : timelocal($s, $i, $h, $d, $m - 1, $y); | 1961 | @del_cols = @{$tbl->{cols}}; |
411 | 1962 | if ( defined $us ) { | 1962 | } |
412 | 1963 | $us = sprintf('%.6f', $us); | 1963 | PTDEBUG && _d('Columns needed for DELETE:', join(', ', @del_cols)); |
413 | 1964 | $us =~ s/^0\././; | 1964 | |
414 | 1965 | $val .= $us; | 1965 | my %col_posn = do { my $i = 0; map { $_ => $i++ } @cols }; |
415 | 1966 | } | 1966 | foreach my $col ( @del_cols ) { |
416 | 1967 | } | 1967 | if ( !exists $col_posn{$col} ) { |
417 | 1968 | return $val; | 1968 | push @cols, $col; |
418 | 1969 | } | 1969 | $col_posn{$col} = $#cols; |
419 | 1970 | 1970 | } | |
420 | 1971 | sub any_unix_timestamp { | 1971 | push @del_slice, $col_posn{$col}; |
421 | 1972 | my ( $val, $callback ) = @_; | 1972 | } |
422 | 1973 | 1973 | PTDEBUG && _d('Ordinals needed for DELETE:', join(', ', @del_slice)); | |
423 | 1974 | if ( my ($n, $suffix) = $val =~ m/^$n_ts$/ ) { | 1974 | |
424 | 1975 | $n = $suffix eq 's' ? $n # Seconds | 1975 | my $del_stmt = { |
425 | 1976 | : $suffix eq 'm' ? $n * 60 # Minutes | 1976 | cols => \@cols, |
426 | 1977 | : $suffix eq 'h' ? $n * 3600 # Hours | 1977 | index => $index, |
427 | 1978 | : $suffix eq 'd' ? $n * 86400 # Days | 1978 | where => '', |
428 | 1979 | : $n; # default: Seconds | 1979 | slice => [], |
429 | 1980 | PTDEBUG && _d('ts is now - N[shmd]:', $n); | 1980 | scols => [], |
430 | 1981 | return time - $n; | 1981 | }; |
431 | 1982 | } | 1982 | |
432 | 1983 | elsif ( $val =~ m/^\d{9,}/ ) { | 1983 | my @clauses; |
433 | 1984 | PTDEBUG && _d('ts is already a unix timestamp'); | 1984 | foreach my $i ( 0 .. $#del_slice ) { |
434 | 1985 | return $val; | 1985 | my $ord = $del_slice[$i]; |
435 | 1986 | } | 1986 | my $col = $cols[$ord]; |
436 | 1987 | elsif ( my ($ymd, $hms) = $val =~ m/^(\d{6})(?:\s+(\d+:\d+:\d+))?/ ) { | 1987 | my $quo = $q->quote($col); |
437 | 1988 | PTDEBUG && _d('ts is MySQL slow log timestamp'); | 1988 | if ( $tbl->{is_nullable}->{$col} ) { |
438 | 1989 | $val .= ' 00:00:00' unless $hms; | 1989 | push @clauses, "((? IS NULL AND $quo IS NULL) OR ($quo = ?))"; |
439 | 1990 | return unix_timestamp(parse_timestamp($val)); | 1990 | push @{$del_stmt->{slice}}, $ord, $ord; |
440 | 1991 | } | 1991 | push @{$del_stmt->{scols}}, $col, $col; |
441 | 1992 | elsif ( ($ymd, $hms) = $val =~ m/^(\d{4}-\d\d-\d\d)(?:[T ](\d+:\d+:\d+))?/) { | 1992 | } |
442 | 1993 | PTDEBUG && _d('ts is properly formatted timestamp'); | 1993 | else { |
443 | 1994 | $val .= ' 00:00:00' unless $hms; | 1994 | push @clauses, "$quo = ?"; |
444 | 1995 | return unix_timestamp($val); | 1995 | push @{$del_stmt->{slice}}, $ord; |
445 | 1996 | } | 1996 | push @{$del_stmt->{scols}}, $col; |
446 | 1997 | else { | 1997 | } |
447 | 1998 | PTDEBUG && _d('ts is MySQL expression'); | 1998 | } |
448 | 1999 | return $callback->($val) if $callback && ref $callback eq 'CODE'; | 1999 | |
449 | 2000 | } | 2000 | $del_stmt->{where} = '(' . join(' AND ', @clauses) . ')'; |
450 | 2001 | 2001 | ||
451 | 2002 | PTDEBUG && _d('Unknown ts type:', $val); | 2002 | return $del_stmt; |
452 | 2003 | return; | 2003 | } |
453 | 2004 | } | 2004 | |
454 | 2005 | 2005 | sub generate_ins_stmt { | |
455 | 2006 | sub make_checksum { | 2006 | my ( $self, %args ) = @_; |
456 | 2007 | my ( $val ) = @_; | 2007 | foreach my $arg ( qw(ins_tbl sel_cols) ) { |
457 | 2008 | my $checksum = uc substr(md5_hex($val), -16); | 2008 | die "I need a $arg argument" unless $args{$arg}; |
458 | 2009 | PTDEBUG && _d($checksum, 'checksum for', $val); | 2009 | } |
459 | 2010 | return $checksum; | 2010 | my $ins_tbl = $args{ins_tbl}; |
460 | 2011 | } | 2011 | my @sel_cols = @{$args{sel_cols}}; |
461 | 2012 | 2012 | ||
462 | 2013 | sub crc32 { | 2013 | die "You didn't specify any SELECT columns" unless @sel_cols; |
463 | 2014 | my ( $string ) = @_; | 2014 | |
464 | 2015 | return unless $string; | 2015 | my @ins_cols; |
465 | 2016 | my $poly = 0xEDB88320; | 2016 | my @ins_slice; |
466 | 2017 | my $crc = 0xFFFFFFFF; | 2017 | for my $i ( 0..$#sel_cols ) { |
467 | 2018 | foreach my $char ( split(//, $string) ) { | 2018 | next unless $ins_tbl->{is_col}->{$sel_cols[$i]}; |
468 | 2019 | my $comp = ($crc ^ ord($char)) & 0xFF; | 2019 | push @ins_cols, $sel_cols[$i]; |
469 | 2020 | for ( 1 .. 8 ) { | 2020 | push @ins_slice, $i; |
470 | 2021 | $comp = $comp & 1 ? $poly ^ ($comp >> 1) : $comp >> 1; | 2021 | } |
471 | 2022 | } | 2022 | |
472 | 2023 | $crc = (($crc >> 8) & 0x00FFFFFF) ^ $comp; | 2023 | return { |
473 | 2024 | } | 2024 | cols => \@ins_cols, |
474 | 2025 | return $crc ^ 0xFFFFFFFF; | 2025 | slice => \@ins_slice, |
475 | 2026 | }; | ||
476 | 2026 | } | 2027 | } |
477 | 2027 | 2028 | ||
478 | 2028 | sub _d { | 2029 | sub _d { |
479 | @@ -2036,7 +2037,7 @@ | |||
480 | 2036 | 1; | 2037 | 1; |
481 | 2037 | } | 2038 | } |
482 | 2038 | # ########################################################################### | 2039 | # ########################################################################### |
484 | 2039 | # End Transformers package | 2040 | # End TableNibbler package |
485 | 2040 | # ########################################################################### | 2041 | # ########################################################################### |
486 | 2041 | 2042 | ||
487 | 2042 | # ########################################################################### | 2043 | # ########################################################################### |
488 | @@ -2077,41 +2078,43 @@ | |||
489 | 2077 | die "I need a tbl parameter" unless $tbl; | 2078 | die "I need a tbl parameter" unless $tbl; |
490 | 2078 | my $q = $self->{Quoter}; | 2079 | my $q = $self->{Quoter}; |
491 | 2079 | 2080 | ||
498 | 2080 | my $sql = '/*!40101 SET @OLD_SQL_MODE := @@SQL_MODE, ' | 2081 | my $new_sql_mode |
499 | 2081 | . q{@@SQL_MODE := REPLACE(REPLACE(@@SQL_MODE, 'ANSI_QUOTES', ''), ',,', ','), } | 2082 | = '/*!40101 SET @OLD_SQL_MODE := @@SQL_MODE, ' |
500 | 2082 | . '@OLD_QUOTE := @@SQL_QUOTE_SHOW_CREATE, ' | 2083 | . q{@@SQL_MODE := REPLACE(REPLACE(@@SQL_MODE, 'ANSI_QUOTES', ''), ',,', ','), } |
501 | 2083 | . '@@SQL_QUOTE_SHOW_CREATE := 1 */'; | 2084 | . '@OLD_QUOTE := @@SQL_QUOTE_SHOW_CREATE, ' |
502 | 2084 | PTDEBUG && _d($sql); | 2085 | . '@@SQL_QUOTE_SHOW_CREATE := 1 */'; |
503 | 2085 | eval { $dbh->do($sql); }; | 2086 | |
504 | 2087 | my $old_sql_mode = '/*!40101 SET @@SQL_MODE := @OLD_SQL_MODE, ' | ||
505 | 2088 | . '@@SQL_QUOTE_SHOW_CREATE := @OLD_QUOTE */'; | ||
506 | 2089 | |||
507 | 2090 | PTDEBUG && _d($new_sql_mode); | ||
508 | 2091 | eval { $dbh->do($new_sql_mode); }; | ||
509 | 2086 | PTDEBUG && $EVAL_ERROR && _d($EVAL_ERROR); | 2092 | PTDEBUG && $EVAL_ERROR && _d($EVAL_ERROR); |
510 | 2087 | 2093 | ||
514 | 2088 | $sql = 'USE ' . $q->quote($db); | 2094 | my $use_sql = 'USE ' . $q->quote($db); |
515 | 2089 | PTDEBUG && _d($dbh, $sql); | 2095 | PTDEBUG && _d($dbh, $use_sql); |
516 | 2090 | $dbh->do($sql); | 2096 | $dbh->do($use_sql); |
517 | 2091 | 2097 | ||
520 | 2092 | $sql = "SHOW CREATE TABLE " . $q->quote($db, $tbl); | 2098 | my $show_sql = "SHOW CREATE TABLE " . $q->quote($db, $tbl); |
521 | 2093 | PTDEBUG && _d($sql); | 2099 | PTDEBUG && _d($show_sql); |
522 | 2094 | my $href; | 2100 | my $href; |
524 | 2095 | eval { $href = $dbh->selectrow_hashref($sql); }; | 2101 | eval { $href = $dbh->selectrow_hashref($show_sql); }; |
525 | 2096 | if ( $EVAL_ERROR ) { | 2102 | if ( $EVAL_ERROR ) { |
526 | 2097 | PTDEBUG && _d($EVAL_ERROR); | 2103 | PTDEBUG && _d($EVAL_ERROR); |
527 | 2104 | |||
528 | 2105 | PTDEBUG && _d($old_sql_mode); | ||
529 | 2106 | $dbh->do($old_sql_mode); | ||
530 | 2107 | |||
531 | 2098 | return; | 2108 | return; |
532 | 2099 | } | 2109 | } |
533 | 2100 | 2110 | ||
538 | 2101 | $sql = '/*!40101 SET @@SQL_MODE := @OLD_SQL_MODE, ' | 2111 | PTDEBUG && _d($old_sql_mode); |
539 | 2102 | . '@@SQL_QUOTE_SHOW_CREATE := @OLD_QUOTE */'; | 2112 | $dbh->do($old_sql_mode); |
536 | 2103 | PTDEBUG && _d($sql); | ||
537 | 2104 | $dbh->do($sql); | ||
540 | 2105 | 2113 | ||
550 | 2106 | my ($key) = grep { m/create table/i } keys %$href; | 2114 | my ($key) = grep { m/create (?:table|view)/i } keys %$href; |
551 | 2107 | if ( $key ) { | 2115 | if ( !$key ) { |
552 | 2108 | PTDEBUG && _d('This table is a base table'); | 2116 | die "Error: no 'Create Table' or 'Create View' in result set from " |
553 | 2109 | $href->{$key} =~ s/\b[ ]{2,}/ /g; | 2117 | . "$show_sql: " . Dumper($href); |
545 | 2110 | $href->{$key} .= "\n"; | ||
546 | 2111 | } | ||
547 | 2112 | else { | ||
548 | 2113 | PTDEBUG && _d('This table is a view'); | ||
549 | 2114 | ($key) = grep { m/create view/i } keys %$href; | ||
554 | 2115 | } | 2118 | } |
555 | 2116 | 2119 | ||
556 | 2117 | return $href->{$key}; | 2120 | return $href->{$key}; |
557 | @@ -2468,6 +2471,7 @@ | |||
558 | 2468 | # ########################################################################### | 2471 | # ########################################################################### |
559 | 2469 | 2472 | ||
560 | 2470 | # ########################################################################### | 2473 | # ########################################################################### |
561 | 2474 | <<<<<<< TREE | ||
562 | 2471 | # TableChunker package | 2475 | # TableChunker package |
563 | 2472 | # This package is a copy without comments from the original. The original | 2476 | # This package is a copy without comments from the original. The original |
564 | 2473 | # with comments and its test file can be found in the Bazaar repository at, | 2477 | # with comments and its test file can be found in the Bazaar repository at, |
565 | @@ -3399,6 +3403,8 @@ | |||
566 | 3399 | # ########################################################################### | 3403 | # ########################################################################### |
567 | 3400 | 3404 | ||
568 | 3401 | # ########################################################################### | 3405 | # ########################################################################### |
569 | 3406 | ======= | ||
570 | 3407 | >>>>>>> MERGE-SOURCE | ||
571 | 3402 | # Progress package | 3408 | # Progress package |
572 | 3403 | # This package is a copy without comments from the original. The original | 3409 | # This package is a copy without comments from the original. The original |
573 | 3404 | # with comments and its test file can be found in the Bazaar repository at, | 3410 | # with comments and its test file can be found in the Bazaar repository at, |
574 | @@ -3547,241 +3553,6 @@ | |||
575 | 3547 | # ########################################################################### | 3553 | # ########################################################################### |
576 | 3548 | 3554 | ||
577 | 3549 | # ########################################################################### | 3555 | # ########################################################################### |
578 | 3550 | # OSCCaptureSync package | ||
579 | 3551 | # This package is a copy without comments from the original. The original | ||
580 | 3552 | # with comments and its test file can be found in the Bazaar repository at, | ||
581 | 3553 | # lib/OSCCaptureSync.pm | ||
582 | 3554 | # t/lib/OSCCaptureSync.t | ||
583 | 3555 | # See https://launchpad.net/percona-toolkit for more information. | ||
584 | 3556 | # ########################################################################### | ||
585 | 3557 | { | ||
586 | 3558 | package OSCCaptureSync; | ||
587 | 3559 | |||
588 | 3560 | use strict; | ||
589 | 3561 | use warnings FATAL => 'all'; | ||
590 | 3562 | use English qw(-no_match_vars); | ||
591 | 3563 | use constant PTDEBUG => $ENV{PTDEBUG} || 0; | ||
592 | 3564 | |||
593 | 3565 | sub new { | ||
594 | 3566 | my ( $class, %args ) = @_; | ||
595 | 3567 | my @required_args = qw(Quoter); | ||
596 | 3568 | foreach my $arg ( @required_args ) { | ||
597 | 3569 | die "I need a $arg argument" unless $args{$arg}; | ||
598 | 3570 | } | ||
599 | 3571 | |||
600 | 3572 | my $self = { | ||
601 | 3573 | Quoter => $args{Quoter}, | ||
602 | 3574 | }; | ||
603 | 3575 | |||
604 | 3576 | return bless $self, $class; | ||
605 | 3577 | } | ||
606 | 3578 | |||
607 | 3579 | sub capture { | ||
608 | 3580 | my ( $self, %args ) = @_; | ||
609 | 3581 | my @required_args = qw(msg dbh db tbl tmp_tbl columns chunk_column); | ||
610 | 3582 | foreach my $arg ( @required_args ) { | ||
611 | 3583 | die "I need a $arg argument" unless $args{$arg}; | ||
612 | 3584 | } | ||
613 | 3585 | my ($msg, $dbh) = @args{@required_args}; | ||
614 | 3586 | |||
615 | 3587 | my @triggers = $self->_make_triggers(%args); | ||
616 | 3588 | foreach my $sql ( @triggers ) { | ||
617 | 3589 | $msg->($sql); | ||
618 | 3590 | $dbh->do($sql) unless $args{print}; | ||
619 | 3591 | } | ||
620 | 3592 | |||
621 | 3593 | return; | ||
622 | 3594 | } | ||
623 | 3595 | |||
624 | 3596 | sub _make_triggers { | ||
625 | 3597 | my ( $self, %args ) = @_; | ||
626 | 3598 | my @required_args = qw(db tbl tmp_tbl chunk_column columns); | ||
627 | 3599 | foreach my $arg ( @required_args ) { | ||
628 | 3600 | die "I need a $arg argument" unless $args{$arg}; | ||
629 | 3601 | } | ||
630 | 3602 | my ($db, $tbl, $tmp_tbl, $chunk_column) = @args{@required_args}; | ||
631 | 3603 | my $q = $self->{Quoter}; | ||
632 | 3604 | |||
633 | 3605 | $chunk_column = $q->quote($chunk_column); | ||
634 | 3606 | |||
635 | 3607 | my $old_table = $q->quote($db, $tbl); | ||
636 | 3608 | my $new_table = $q->quote($db, $tmp_tbl); | ||
637 | 3609 | my $new_values = join(', ', map { "NEW.".$q->quote($_) } @{$args{columns}}); | ||
638 | 3610 | my $columns = join(', ', map { $q->quote($_) } @{$args{columns}}); | ||
639 | 3611 | |||
640 | 3612 | my $delete_trigger = "CREATE TRIGGER mk_osc_del AFTER DELETE ON $old_table " | ||
641 | 3613 | . "FOR EACH ROW " | ||
642 | 3614 | . "DELETE IGNORE FROM $new_table " | ||
643 | 3615 | . "WHERE $new_table.$chunk_column = OLD.$chunk_column"; | ||
644 | 3616 | |||
645 | 3617 | my $insert_trigger = "CREATE TRIGGER mk_osc_ins AFTER INSERT ON $old_table " | ||
646 | 3618 | . "FOR EACH ROW " | ||
647 | 3619 | . "REPLACE INTO $new_table ($columns) " | ||
648 | 3620 | . "VALUES($new_values)"; | ||
649 | 3621 | |||
650 | 3622 | my $update_trigger = "CREATE TRIGGER mk_osc_upd AFTER UPDATE ON $old_table " | ||
651 | 3623 | . "FOR EACH ROW " | ||
652 | 3624 | . "REPLACE INTO $new_table ($columns) " | ||
653 | 3625 | . "VALUES ($new_values)"; | ||
654 | 3626 | |||
655 | 3627 | return $delete_trigger, $update_trigger, $insert_trigger; | ||
656 | 3628 | } | ||
657 | 3629 | |||
658 | 3630 | sub sync { | ||
659 | 3631 | my ( $self, %args ) = @_; | ||
660 | 3632 | my @required_args = qw(); | ||
661 | 3633 | foreach my $arg ( @required_args ) { | ||
662 | 3634 | die "I need a $arg argument" unless $args{$arg}; | ||
663 | 3635 | } | ||
664 | 3636 | return; | ||
665 | 3637 | } | ||
666 | 3638 | |||
667 | 3639 | sub cleanup { | ||
668 | 3640 | my ( $self, %args ) = @_; | ||
669 | 3641 | my @required_args = qw(dbh db msg); | ||
670 | 3642 | foreach my $arg ( @required_args ) { | ||
671 | 3643 | die "I need a $arg argument" unless $args{$arg}; | ||
672 | 3644 | } | ||
673 | 3645 | my ($dbh, $db, $msg) = @args{@required_args}; | ||
674 | 3646 | my $q = $self->{Quoter}; | ||
675 | 3647 | |||
676 | 3648 | foreach my $trigger ( qw(del ins upd) ) { | ||
677 | 3649 | my $sql = "DROP TRIGGER IF EXISTS " . $q->quote($db, "mk_osc_$trigger"); | ||
678 | 3650 | $msg->($sql); | ||
679 | 3651 | $dbh->do($sql) unless $args{print}; | ||
680 | 3652 | } | ||
681 | 3653 | |||
682 | 3654 | return; | ||
683 | 3655 | } | ||
684 | 3656 | |||
685 | 3657 | sub _d { | ||
686 | 3658 | my ($package, undef, $line) = caller 0; | ||
687 | 3659 | @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } | ||
688 | 3660 | map { defined $_ ? $_ : 'undef' } | ||
689 | 3661 | @_; | ||
690 | 3662 | print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; | ||
691 | 3663 | } | ||
692 | 3664 | |||
693 | 3665 | 1; | ||
694 | 3666 | } | ||
695 | 3667 | # ########################################################################### | ||
696 | 3668 | # End OSCCaptureSync package | ||
697 | 3669 | # ########################################################################### | ||
698 | 3670 | |||
699 | 3671 | # ########################################################################### | ||
700 | 3672 | # CopyRowsInsertSelect package | ||
701 | 3673 | # This package is a copy without comments from the original. The original | ||
702 | 3674 | # with comments and its test file can be found in the Bazaar repository at, | ||
703 | 3675 | # lib/CopyRowsInsertSelect.pm | ||
704 | 3676 | # t/lib/CopyRowsInsertSelect.t | ||
705 | 3677 | # See https://launchpad.net/percona-toolkit for more information. | ||
706 | 3678 | # ########################################################################### | ||
707 | 3679 | { | ||
708 | 3680 | package CopyRowsInsertSelect; | ||
709 | 3681 | |||
710 | 3682 | use strict; | ||
711 | 3683 | use warnings FATAL => 'all'; | ||
712 | 3684 | use English qw(-no_match_vars); | ||
713 | 3685 | use constant PTDEBUG => $ENV{PTDEBUG} || 0; | ||
714 | 3686 | |||
715 | 3687 | sub new { | ||
716 | 3688 | my ( $class, %args ) = @_; | ||
717 | 3689 | my @required_args = qw(Retry Quoter); | ||
718 | 3690 | foreach my $arg ( @required_args ) { | ||
719 | 3691 | die "I need a $arg argument" unless $args{$arg}; | ||
720 | 3692 | } | ||
721 | 3693 | |||
722 | 3694 | my $self = { | ||
723 | 3695 | Retry => $args{Retry}, | ||
724 | 3696 | Quoter => $args{Quoter}, | ||
725 | 3697 | }; | ||
726 | 3698 | |||
727 | 3699 | return bless $self, $class; | ||
728 | 3700 | } | ||
729 | 3701 | |||
730 | 3702 | sub copy { | ||
731 | 3703 | my ( $self, %args ) = @_; | ||
732 | 3704 | my @required_args = qw(dbh msg from_table to_table chunks columns); | ||
733 | 3705 | foreach my $arg ( @required_args ) { | ||
734 | 3706 | die "I need a $arg argument" unless $args{$arg}; | ||
735 | 3707 | } | ||
736 | 3708 | my ($dbh, $msg, $from_table, $to_table, $chunks) = @args{@required_args}; | ||
737 | 3709 | my $q = $self->{Quoter}; | ||
738 | 3710 | my $pr = $args{Progress}; | ||
739 | 3711 | my $sleep = $args{sleep}; | ||
740 | 3712 | my $columns = join(', ', map { $q->quote($_) } @{$args{columns}}); | ||
741 | 3713 | my $n_chunks = @$chunks - 1; | ||
742 | 3714 | |||
743 | 3715 | for my $chunkno ( 0..$n_chunks ) { | ||
744 | 3716 | if ( !$chunks->[$chunkno] ) { | ||
745 | 3717 | warn "Chunk number ", ($chunkno + 1), "is undefined"; | ||
746 | 3718 | next; | ||
747 | 3719 | } | ||
748 | 3720 | |||
749 | 3721 | my $sql = "INSERT IGNORE INTO $to_table ($columns) " | ||
750 | 3722 | . "SELECT $columns FROM $from_table " | ||
751 | 3723 | . "WHERE ($chunks->[$chunkno])" | ||
752 | 3724 | . ($args{where} ? " AND ($args{where})" : "") | ||
753 | 3725 | . ($args{engine_flags} ? " $args{engine_flags}" : ""); | ||
754 | 3726 | |||
755 | 3727 | if ( $args{print} ) { | ||
756 | 3728 | $msg->($sql); | ||
757 | 3729 | } | ||
758 | 3730 | else { | ||
759 | 3731 | PTDEBUG && _d($dbh, $sql); | ||
760 | 3732 | my $error; | ||
761 | 3733 | $self->{Retry}->retry( | ||
762 | 3734 | wait => sub { sleep 1; }, | ||
763 | 3735 | tries => 3, | ||
764 | 3736 | try => sub { | ||
765 | 3737 | $dbh->do($sql); | ||
766 | 3738 | return; | ||
767 | 3739 | }, | ||
768 | 3740 | fail => sub { | ||
769 | 3741 | my (%args) = @_; | ||
770 | 3742 | my $error = $args{error}; | ||
771 | 3743 | PTDEBUG && _d($error); | ||
772 | 3744 | if ( $error =~ m/Lock wait timeout exceeded/ ) { | ||
773 | 3745 | $msg->("Lock wait timeout exceeded; retrying $sql"); | ||
774 | 3746 | return 1; # call wait, call try | ||
775 | 3747 | } | ||
776 | 3748 | return 0; # call final_fail | ||
777 | 3749 | }, | ||
778 | 3750 | final_fail => sub { | ||
779 | 3751 | my (%args) = @_; | ||
780 | 3752 | die $args{error}; | ||
781 | 3753 | }, | ||
782 | 3754 | ); | ||
783 | 3755 | } | ||
784 | 3756 | |||
785 | 3757 | $pr->update(sub { return $chunkno + 1; }) if $pr; | ||
786 | 3758 | |||
787 | 3759 | $sleep->($chunkno + 1) if $sleep && $chunkno < $n_chunks; | ||
788 | 3760 | } | ||
789 | 3761 | |||
790 | 3762 | return; | ||
791 | 3763 | } | ||
792 | 3764 | |||
793 | 3765 | sub cleanup { | ||
794 | 3766 | my ( $self, %args ) = @_; | ||
795 | 3767 | return; | ||
796 | 3768 | } | ||
797 | 3769 | |||
798 | 3770 | sub _d { | ||
799 | 3771 | my ($package, undef, $line) = caller 0; | ||
800 | 3772 | @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } | ||
801 | 3773 | map { defined $_ ? $_ : 'undef' } | ||
802 | 3774 | @_; | ||
803 | 3775 | print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; | ||
804 | 3776 | } | ||
805 | 3777 | |||
806 | 3778 | 1; | ||
807 | 3779 | } | ||
808 | 3780 | # ########################################################################### | ||
809 | 3781 | # End CopyRowsInsertSelect package | ||
810 | 3782 | # ########################################################################### | ||
811 | 3783 | |||
812 | 3784 | # ########################################################################### | ||
813 | 3785 | # Retry package | 3556 | # Retry package |
814 | 3786 | # This package is a copy without comments from the original. The original | 3557 | # This package is a copy without comments from the original. The original |
815 | 3787 | # with comments and its test file can be found in the Bazaar repository at, | 3558 | # with comments and its test file can be found in the Bazaar repository at, |
816 | @@ -3860,6 +3631,2187 @@ | |||
817 | 3860 | # ########################################################################### | 3631 | # ########################################################################### |
818 | 3861 | 3632 | ||
819 | 3862 | # ########################################################################### | 3633 | # ########################################################################### |
820 | 3634 | # Cxn package | ||
821 | 3635 | # This package is a copy without comments from the original. The original | ||
822 | 3636 | # with comments and its test file can be found in the Bazaar repository at, | ||
823 | 3637 | # lib/Cxn.pm | ||
824 | 3638 | # t/lib/Cxn.t | ||
825 | 3639 | # See https://launchpad.net/percona-toolkit for more information. | ||
826 | 3640 | # ########################################################################### | ||
827 | 3641 | { | ||
828 | 3642 | package Cxn; | ||
829 | 3643 | |||
830 | 3644 | use strict; | ||
831 | 3645 | use warnings FATAL => 'all'; | ||
832 | 3646 | use English qw(-no_match_vars); | ||
833 | 3647 | use constant PTDEBUG => $ENV{PTDEBUG} || 0; | ||
834 | 3648 | |||
835 | 3649 | use constant PERCONA_TOOLKIT_TEST_USE_DSN_NAMES => $ENV{PERCONA_TOOLKIT_TEST_USE_DSN_NAMES} || 0; | ||
836 | 3650 | |||
837 | 3651 | sub new { | ||
838 | 3652 | my ( $class, %args ) = @_; | ||
839 | 3653 | my @required_args = qw(DSNParser OptionParser); | ||
840 | 3654 | foreach my $arg ( @required_args ) { | ||
841 | 3655 | die "I need a $arg argument" unless $args{$arg}; | ||
842 | 3656 | }; | ||
843 | 3657 | my ($dp, $o) = @args{@required_args}; | ||
844 | 3658 | |||
845 | 3659 | my $dsn_defaults = $dp->parse_options($o); | ||
846 | 3660 | my $prev_dsn = $args{prev_dsn}; | ||
847 | 3661 | my $dsn = $args{dsn}; | ||
848 | 3662 | if ( !$dsn ) { | ||
849 | 3663 | $args{dsn_string} ||= 'h=' . ($dsn_defaults->{h} || 'localhost'); | ||
850 | 3664 | |||
851 | 3665 | $dsn = $dp->parse( | ||
852 | 3666 | $args{dsn_string}, $prev_dsn, $dsn_defaults); | ||
853 | 3667 | } | ||
854 | 3668 | elsif ( $prev_dsn ) { | ||
855 | 3669 | $dsn = $dp->copy($prev_dsn, $dsn); | ||
856 | 3670 | } | ||
857 | 3671 | |||
858 | 3672 | my $self = { | ||
859 | 3673 | dsn => $dsn, | ||
860 | 3674 | dbh => $args{dbh}, | ||
861 | 3675 | dsn_name => $dp->as_string($dsn, [qw(h P S)]), | ||
862 | 3676 | hostname => '', | ||
863 | 3677 | set => $args{set}, | ||
864 | 3678 | dbh_set => 0, | ||
865 | 3679 | OptionParser => $o, | ||
866 | 3680 | DSNParser => $dp, | ||
867 | 3681 | }; | ||
868 | 3682 | |||
869 | 3683 | return bless $self, $class; | ||
870 | 3684 | } | ||
871 | 3685 | |||
872 | 3686 | sub connect { | ||
873 | 3687 | my ( $self ) = @_; | ||
874 | 3688 | my $dsn = $self->{dsn}; | ||
875 | 3689 | my $dp = $self->{DSNParser}; | ||
876 | 3690 | my $o = $self->{OptionParser}; | ||
877 | 3691 | |||
878 | 3692 | my $dbh = $self->{dbh}; | ||
879 | 3693 | if ( !$dbh || !$dbh->ping() ) { | ||
880 | 3694 | if ( $o->get('ask-pass') && !$self->{asked_for_pass} ) { | ||
881 | 3695 | $dsn->{p} = OptionParser::prompt_noecho("Enter MySQL password: "); | ||
882 | 3696 | $self->{asked_for_pass} = 1; | ||
883 | 3697 | } | ||
884 | 3698 | $dbh = $dp->get_dbh($dp->get_cxn_params($dsn), { AutoCommit => 1 }); | ||
885 | 3699 | } | ||
886 | 3700 | PTDEBUG && _d($dbh, 'Connected dbh to', $self->{name}); | ||
887 | 3701 | |||
888 | 3702 | return $self->set_dbh($dbh); | ||
889 | 3703 | } | ||
890 | 3704 | |||
891 | 3705 | sub set_dbh { | ||
892 | 3706 | my ($self, $dbh) = @_; | ||
893 | 3707 | |||
894 | 3708 | if ( $self->{dbh} && $self->{dbh} == $dbh && $self->{dbh_set} ) { | ||
895 | 3709 | PTDEBUG && _d($dbh, 'Already set dbh'); | ||
896 | 3710 | return $dbh; | ||
897 | 3711 | } | ||
898 | 3712 | |||
899 | 3713 | PTDEBUG && _d($dbh, 'Setting dbh'); | ||
900 | 3714 | |||
901 | 3715 | $dbh->{FetchHashKeyName} = 'NAME_lc'; | ||
902 | 3716 | |||
903 | 3717 | my $sql = 'SELECT @@hostname, @@server_id'; | ||
904 | 3718 | PTDEBUG && _d($dbh, $sql); | ||
905 | 3719 | my ($hostname, $server_id) = $dbh->selectrow_array($sql); | ||
906 | 3720 | PTDEBUG && _d($dbh, 'hostname:', $hostname, $server_id); | ||
907 | 3721 | if ( $hostname ) { | ||
908 | 3722 | $self->{hostname} = $hostname; | ||
909 | 3723 | } | ||
910 | 3724 | |||
911 | 3725 | if ( my $set = $self->{set}) { | ||
912 | 3726 | $set->($dbh); | ||
913 | 3727 | } | ||
914 | 3728 | |||
915 | 3729 | $self->{dbh} = $dbh; | ||
916 | 3730 | $self->{dbh_set} = 1; | ||
917 | 3731 | return $dbh; | ||
918 | 3732 | } | ||
919 | 3733 | |||
920 | 3734 | sub dbh { | ||
921 | 3735 | my ($self) = @_; | ||
922 | 3736 | return $self->{dbh}; | ||
923 | 3737 | } | ||
924 | 3738 | |||
925 | 3739 | sub dsn { | ||
926 | 3740 | my ($self) = @_; | ||
927 | 3741 | return $self->{dsn}; | ||
928 | 3742 | } | ||
929 | 3743 | |||
930 | 3744 | sub name { | ||
931 | 3745 | my ($self) = @_; | ||
932 | 3746 | return $self->{dsn_name} if PERCONA_TOOLKIT_TEST_USE_DSN_NAMES; | ||
933 | 3747 | return $self->{hostname} || $self->{dsn_name} || 'unknown host'; | ||
934 | 3748 | } | ||
935 | 3749 | |||
936 | 3750 | sub DESTROY { | ||
937 | 3751 | my ($self) = @_; | ||
938 | 3752 | if ( $self->{dbh} ) { | ||
939 | 3753 | PTDEBUG && _d('Disconnecting dbh', $self->{dbh}, $self->{name}); | ||
940 | 3754 | $self->{dbh}->disconnect(); | ||
941 | 3755 | } | ||
942 | 3756 | return; | ||
943 | 3757 | } | ||
944 | 3758 | |||
945 | 3759 | sub _d { | ||
946 | 3760 | my ($package, undef, $line) = caller 0; | ||
947 | 3761 | @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } | ||
948 | 3762 | map { defined $_ ? $_ : 'undef' } | ||
949 | 3763 | @_; | ||
950 | 3764 | print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; | ||
951 | 3765 | } | ||
952 | 3766 | |||
953 | 3767 | 1; | ||
954 | 3768 | } | ||
955 | 3769 | # ########################################################################### | ||
956 | 3770 | # End Cxn package | ||
957 | 3771 | # ########################################################################### | ||
958 | 3772 | |||
959 | 3773 | # ########################################################################### | ||
960 | 3774 | # MasterSlave package | ||
961 | 3775 | # This package is a copy without comments from the original. The original | ||
962 | 3776 | # with comments and its test file can be found in the Bazaar repository at, | ||
963 | 3777 | # lib/MasterSlave.pm | ||
964 | 3778 | # t/lib/MasterSlave.t | ||
965 | 3779 | # See https://launchpad.net/percona-toolkit for more information. | ||
966 | 3780 | # ########################################################################### | ||
967 | 3781 | { | ||
968 | 3782 | package MasterSlave; | ||
969 | 3783 | |||
970 | 3784 | use strict; | ||
971 | 3785 | use warnings FATAL => 'all'; | ||
972 | 3786 | use English qw(-no_match_vars); | ||
973 | 3787 | use constant PTDEBUG => $ENV{PTDEBUG} || 0; | ||
974 | 3788 | |||
975 | 3789 | sub new { | ||
976 | 3790 | my ( $class, %args ) = @_; | ||
977 | 3791 | my $self = { | ||
978 | 3792 | %args, | ||
979 | 3793 | replication_thread => {}, | ||
980 | 3794 | }; | ||
981 | 3795 | return bless $self, $class; | ||
982 | 3796 | } | ||
983 | 3797 | |||
984 | 3798 | sub get_slaves { | ||
985 | 3799 | my ($self, %args) = @_; | ||
986 | 3800 | my @required_args = qw(make_cxn OptionParser DSNParser Quoter); | ||
987 | 3801 | foreach my $arg ( @required_args ) { | ||
988 | 3802 | die "I need a $arg argument" unless $args{$arg}; | ||
989 | 3803 | } | ||
990 | 3804 | my ($make_cxn, $o, $dp) = @args{@required_args}; | ||
991 | 3805 | |||
992 | 3806 | my $slaves = []; | ||
993 | 3807 | my $method = $o->get('recursion-method'); | ||
994 | 3808 | PTDEBUG && _d('Slave recursion method:', $method); | ||
995 | 3809 | if ( !$method || $method =~ m/processlist|hosts/i ) { | ||
996 | 3810 | my @required_args = qw(dbh dsn); | ||
997 | 3811 | foreach my $arg ( @required_args ) { | ||
998 | 3812 | die "I need a $arg argument" unless $args{$arg}; | ||
999 | 3813 | } | ||
1000 | 3814 | my ($dbh, $dsn) = @args{@required_args}; | ||
1001 | 3815 | $self->recurse_to_slaves( | ||
1002 | 3816 | { dbh => $dbh, | ||
1003 | 3817 | dsn => $dsn, | ||
1004 | 3818 | dsn_parser => $dp, | ||
1005 | 3819 | recurse => $o->get('recurse'), | ||
1006 | 3820 | method => $o->get('recursion-method'), | ||
1007 | 3821 | callback => sub { | ||
1008 | 3822 | my ( $dsn, $dbh, $level, $parent ) = @_; | ||
1009 | 3823 | return unless $level; | ||
1010 | 3824 | PTDEBUG && _d('Found slave:', $dp->as_string($dsn)); | ||
1011 | 3825 | push @$slaves, $make_cxn->(dsn => $dsn, dbh => $dbh); | ||
1012 | 3826 | return; | ||
1013 | 3827 | }, | ||
1014 | 3828 | } | ||
1015 | 3829 | ); | ||
1016 | 3830 | } | ||
1017 | 3831 | elsif ( $method =~ m/^dsn=/i ) { | ||
1018 | 3832 | my ($dsn_table_dsn) = $method =~ m/^dsn=(.+)/i; | ||
1019 | 3833 | $slaves = $self->get_cxn_from_dsn_table( | ||
1020 | 3834 | %args, | ||
1021 | 3835 | dsn_table_dsn => $dsn_table_dsn, | ||
1022 | 3836 | ); | ||
1023 | 3837 | } | ||
1024 | 3838 | else { | ||
1025 | 3839 | die "Invalid --recursion-method: $method. Valid values are: " | ||
1026 | 3840 | . "dsn=DSN, hosts, or processlist.\n"; | ||
1027 | 3841 | } | ||
1028 | 3842 | |||
1029 | 3843 | return $slaves; | ||
1030 | 3844 | } | ||
1031 | 3845 | |||
1032 | 3846 | sub recurse_to_slaves { | ||
1033 | 3847 | my ( $self, $args, $level ) = @_; | ||
1034 | 3848 | $level ||= 0; | ||
1035 | 3849 | my $dp = $args->{dsn_parser}; | ||
1036 | 3850 | my $dsn = $args->{dsn}; | ||
1037 | 3851 | |||
1038 | 3852 | my $dbh; | ||
1039 | 3853 | eval { | ||
1040 | 3854 | $dbh = $args->{dbh} || $dp->get_dbh( | ||
1041 | 3855 | $dp->get_cxn_params($dsn), { AutoCommit => 1 }); | ||
1042 | 3856 | PTDEBUG && _d('Connected to', $dp->as_string($dsn)); | ||
1043 | 3857 | }; | ||
1044 | 3858 | if ( $EVAL_ERROR ) { | ||
1045 | 3859 | print STDERR "Cannot connect to ", $dp->as_string($dsn), "\n" | ||
1046 | 3860 | or die "Cannot print: $OS_ERROR"; | ||
1047 | 3861 | return; | ||
1048 | 3862 | } | ||
1049 | 3863 | |||
1050 | 3864 | my $sql = 'SELECT @@SERVER_ID'; | ||
1051 | 3865 | PTDEBUG && _d($sql); | ||
1052 | 3866 | my ($id) = $dbh->selectrow_array($sql); | ||
1053 | 3867 | PTDEBUG && _d('Working on server ID', $id); | ||
1054 | 3868 | my $master_thinks_i_am = $dsn->{server_id}; | ||
1055 | 3869 | if ( !defined $id | ||
1056 | 3870 | || ( defined $master_thinks_i_am && $master_thinks_i_am != $id ) | ||
1057 | 3871 | || $args->{server_ids_seen}->{$id}++ | ||
1058 | 3872 | ) { | ||
1059 | 3873 | PTDEBUG && _d('Server ID seen, or not what master said'); | ||
1060 | 3874 | if ( $args->{skip_callback} ) { | ||
1061 | 3875 | $args->{skip_callback}->($dsn, $dbh, $level, $args->{parent}); | ||
1062 | 3876 | } | ||
1063 | 3877 | return; | ||
1064 | 3878 | } | ||
1065 | 3879 | |||
1066 | 3880 | $args->{callback}->($dsn, $dbh, $level, $args->{parent}); | ||
1067 | 3881 | |||
1068 | 3882 | if ( !defined $args->{recurse} || $level < $args->{recurse} ) { | ||
1069 | 3883 | |||
1070 | 3884 | my @slaves = | ||
1071 | 3885 | grep { !$_->{master_id} || $_->{master_id} == $id } # Only my slaves. | ||
1072 | 3886 | $self->find_slave_hosts($dp, $dbh, $dsn, $args->{method}); | ||
1073 | 3887 | |||
1074 | 3888 | foreach my $slave ( @slaves ) { | ||
1075 | 3889 | PTDEBUG && _d('Recursing from', | ||
1076 | 3890 | $dp->as_string($dsn), 'to', $dp->as_string($slave)); | ||
1077 | 3891 | $self->recurse_to_slaves( | ||
1078 | 3892 | { %$args, dsn => $slave, dbh => undef, parent => $dsn }, $level + 1 ); | ||
1079 | 3893 | } | ||
1080 | 3894 | } | ||
1081 | 3895 | } | ||
1082 | 3896 | |||
1083 | 3897 | sub find_slave_hosts { | ||
1084 | 3898 | my ( $self, $dsn_parser, $dbh, $dsn, $method ) = @_; | ||
1085 | 3899 | |||
1086 | 3900 | my @methods = qw(processlist hosts); | ||
1087 | 3901 | if ( $method ) { | ||
1088 | 3902 | @methods = grep { $_ ne $method } @methods; | ||
1089 | 3903 | unshift @methods, $method; | ||
1090 | 3904 | } | ||
1091 | 3905 | else { | ||
1092 | 3906 | if ( ($dsn->{P} || 3306) != 3306 ) { | ||
1093 | 3907 | PTDEBUG && _d('Port number is non-standard; using only hosts method'); | ||
1094 | 3908 | @methods = qw(hosts); | ||
1095 | 3909 | } | ||
1096 | 3910 | } | ||
1097 | 3911 | PTDEBUG && _d('Looking for slaves on', $dsn_parser->as_string($dsn), | ||
1098 | 3912 | 'using methods', @methods); | ||
1099 | 3913 | |||
1100 | 3914 | my @slaves; | ||
1101 | 3915 | METHOD: | ||
1102 | 3916 | foreach my $method ( @methods ) { | ||
1103 | 3917 | my $find_slaves = "_find_slaves_by_$method"; | ||
1104 | 3918 | PTDEBUG && _d('Finding slaves with', $find_slaves); | ||
1105 | 3919 | @slaves = $self->$find_slaves($dsn_parser, $dbh, $dsn); | ||
1106 | 3920 | last METHOD if @slaves; | ||
1107 | 3921 | } | ||
1108 | 3922 | |||
1109 | 3923 | PTDEBUG && _d('Found', scalar(@slaves), 'slaves'); | ||
1110 | 3924 | return @slaves; | ||
1111 | 3925 | } | ||
1112 | 3926 | |||
1113 | 3927 | sub _find_slaves_by_processlist { | ||
1114 | 3928 | my ( $self, $dsn_parser, $dbh, $dsn ) = @_; | ||
1115 | 3929 | |||
1116 | 3930 | my @slaves = map { | ||
1117 | 3931 | my $slave = $dsn_parser->parse("h=$_", $dsn); | ||
1118 | 3932 | $slave->{source} = 'processlist'; | ||
1119 | 3933 | $slave; | ||
1120 | 3934 | } | ||
1121 | 3935 | grep { $_ } | ||
1122 | 3936 | map { | ||
1123 | 3937 | my ( $host ) = $_->{host} =~ m/^([^:]+):/; | ||
1124 | 3938 | if ( $host eq 'localhost' ) { | ||
1125 | 3939 | $host = '127.0.0.1'; # Replication never uses sockets. | ||
1126 | 3940 | } | ||
1127 | 3941 | $host; | ||
1128 | 3942 | } $self->get_connected_slaves($dbh); | ||
1129 | 3943 | |||
1130 | 3944 | return @slaves; | ||
1131 | 3945 | } | ||
1132 | 3946 | |||
1133 | 3947 | sub _find_slaves_by_hosts { | ||
1134 | 3948 | my ( $self, $dsn_parser, $dbh, $dsn ) = @_; | ||
1135 | 3949 | |||
1136 | 3950 | my @slaves; | ||
1137 | 3951 | my $sql = 'SHOW SLAVE HOSTS'; | ||
1138 | 3952 | PTDEBUG && _d($dbh, $sql); | ||
1139 | 3953 | @slaves = @{$dbh->selectall_arrayref($sql, { Slice => {} })}; | ||
1140 | 3954 | |||
1141 | 3955 | if ( @slaves ) { | ||
1142 | 3956 | PTDEBUG && _d('Found some SHOW SLAVE HOSTS info'); | ||
1143 | 3957 | @slaves = map { | ||
1144 | 3958 | my %hash; | ||
1145 | 3959 | @hash{ map { lc $_ } keys %$_ } = values %$_; | ||
1146 | 3960 | my $spec = "h=$hash{host},P=$hash{port}" | ||
1147 | 3961 | . ( $hash{user} ? ",u=$hash{user}" : '') | ||
1148 | 3962 | . ( $hash{password} ? ",p=$hash{password}" : ''); | ||
1149 | 3963 | my $dsn = $dsn_parser->parse($spec, $dsn); | ||
1150 | 3964 | $dsn->{server_id} = $hash{server_id}; | ||
1151 | 3965 | $dsn->{master_id} = $hash{master_id}; | ||
1152 | 3966 | $dsn->{source} = 'hosts'; | ||
1153 | 3967 | $dsn; | ||
1154 | 3968 | } @slaves; | ||
1155 | 3969 | } | ||
1156 | 3970 | |||
1157 | 3971 | return @slaves; | ||
1158 | 3972 | } | ||
1159 | 3973 | |||
1160 | 3974 | sub get_connected_slaves { | ||
1161 | 3975 | my ( $self, $dbh ) = @_; | ||
1162 | 3976 | |||
1163 | 3977 | my $show = "SHOW GRANTS FOR "; | ||
1164 | 3978 | my $user = 'CURRENT_USER()'; | ||
1165 | 3979 | my $vp = $self->{VersionParser}; | ||
1166 | 3980 | if ( $vp && !$vp->version_ge($dbh, '4.1.2') ) { | ||
1167 | 3981 | $user = $dbh->selectrow_arrayref('SELECT USER()')->[0]; | ||
1168 | 3982 | $user =~ s/([^@]+)@(.+)/'$1'\@'$2'/; | ||
1169 | 3983 | } | ||
1170 | 3984 | my $sql = $show . $user; | ||
1171 | 3985 | PTDEBUG && _d($dbh, $sql); | ||
1172 | 3986 | |||
1173 | 3987 | my $proc; | ||
1174 | 3988 | eval { | ||
1175 | 3989 | $proc = grep { | ||
1176 | 3990 | m/ALL PRIVILEGES.*?\*\.\*|PROCESS/ | ||
1177 | 3991 | } @{$dbh->selectcol_arrayref($sql)}; | ||
1178 | 3992 | }; | ||
1179 | 3993 | if ( $EVAL_ERROR ) { | ||
1180 | 3994 | |||
1181 | 3995 | if ( $EVAL_ERROR =~ m/no such grant defined for user/ ) { | ||
1182 | 3996 | PTDEBUG && _d('Retrying SHOW GRANTS without host; error:', | ||
1183 | 3997 | $EVAL_ERROR); | ||
1184 | 3998 | ($user) = split('@', $user); | ||
1185 | 3999 | $sql = $show . $user; | ||
1186 | 4000 | PTDEBUG && _d($sql); | ||
1187 | 4001 | eval { | ||
1188 | 4002 | $proc = grep { | ||
1189 | 4003 | m/ALL PRIVILEGES.*?\*\.\*|PROCESS/ | ||
1190 | 4004 | } @{$dbh->selectcol_arrayref($sql)}; | ||
1191 | 4005 | }; | ||
1192 | 4006 | } | ||
1193 | 4007 | |||
1194 | 4008 | die "Failed to $sql: $EVAL_ERROR" if $EVAL_ERROR; | ||
1195 | 4009 | } | ||
1196 | 4010 | if ( !$proc ) { | ||
1197 | 4011 | die "You do not have the PROCESS privilege"; | ||
1198 | 4012 | } | ||
1199 | 4013 | |||
1200 | 4014 | $sql = 'SHOW PROCESSLIST'; | ||
1201 | 4015 | PTDEBUG && _d($dbh, $sql); | ||
1202 | 4016 | grep { $_->{command} =~ m/Binlog Dump/i } | ||
1203 | 4017 | map { # Lowercase the column names | ||
1204 | 4018 | my %hash; | ||
1205 | 4019 | @hash{ map { lc $_ } keys %$_ } = values %$_; | ||
1206 | 4020 | \%hash; | ||
1207 | 4021 | } | ||
1208 | 4022 | @{$dbh->selectall_arrayref($sql, { Slice => {} })}; | ||
1209 | 4023 | } | ||
1210 | 4024 | |||
1211 | 4025 | sub is_master_of { | ||
1212 | 4026 | my ( $self, $master, $slave ) = @_; | ||
1213 | 4027 | my $master_status = $self->get_master_status($master) | ||
1214 | 4028 | or die "The server specified as a master is not a master"; | ||
1215 | 4029 | my $slave_status = $self->get_slave_status($slave) | ||
1216 | 4030 | or die "The server specified as a slave is not a slave"; | ||
1217 | 4031 | my @connected = $self->get_connected_slaves($master) | ||
1218 | 4032 | or die "The server specified as a master has no connected slaves"; | ||
1219 | 4033 | my (undef, $port) = $master->selectrow_array('SHOW VARIABLES LIKE "port"'); | ||
1220 | 4034 | |||
1221 | 4035 | if ( $port != $slave_status->{master_port} ) { | ||
1222 | 4036 | die "The slave is connected to $slave_status->{master_port} " | ||
1223 | 4037 | . "but the master's port is $port"; | ||
1224 | 4038 | } | ||
1225 | 4039 | |||
1226 | 4040 | if ( !grep { $slave_status->{master_user} eq $_->{user} } @connected ) { | ||
1227 | 4041 | die "I don't see any slave I/O thread connected with user " | ||
1228 | 4042 | . $slave_status->{master_user}; | ||
1229 | 4043 | } | ||
1230 | 4044 | |||
1231 | 4045 | if ( ($slave_status->{slave_io_state} || '') | ||
1232 | 4046 | eq 'Waiting for master to send event' ) | ||
1233 | 4047 | { | ||
1234 | 4048 | my ( $master_log_name, $master_log_num ) | ||
1235 | 4049 | = $master_status->{file} =~ m/^(.*?)\.0*([1-9][0-9]*)$/; | ||
1236 | 4050 | my ( $slave_log_name, $slave_log_num ) | ||
1237 | 4051 | = $slave_status->{master_log_file} =~ m/^(.*?)\.0*([1-9][0-9]*)$/; | ||
1238 | 4052 | if ( $master_log_name ne $slave_log_name | ||
1239 | 4053 | || abs($master_log_num - $slave_log_num) > 1 ) | ||
1240 | 4054 | { | ||
1241 | 4055 | die "The slave thinks it is reading from " | ||
1242 | 4056 | . "$slave_status->{master_log_file}, but the " | ||
1243 | 4057 | . "master is writing to $master_status->{file}"; | ||
1244 | 4058 | } | ||
1245 | 4059 | } | ||
1246 | 4060 | return 1; | ||
1247 | 4061 | } | ||
1248 | 4062 | |||
1249 | 4063 | sub get_master_dsn { | ||
1250 | 4064 | my ( $self, $dbh, $dsn, $dsn_parser ) = @_; | ||
1251 | 4065 | my $master = $self->get_slave_status($dbh) or return undef; | ||
1252 | 4066 | my $spec = "h=$master->{master_host},P=$master->{master_port}"; | ||
1253 | 4067 | return $dsn_parser->parse($spec, $dsn); | ||
1254 | 4068 | } | ||
1255 | 4069 | |||
1256 | 4070 | sub get_slave_status { | ||
1257 | 4071 | my ( $self, $dbh ) = @_; | ||
1258 | 4072 | if ( !$self->{not_a_slave}->{$dbh} ) { | ||
1259 | 4073 | my $sth = $self->{sths}->{$dbh}->{SLAVE_STATUS} | ||
1260 | 4074 | ||= $dbh->prepare('SHOW SLAVE STATUS'); | ||
1261 | 4075 | PTDEBUG && _d($dbh, 'SHOW SLAVE STATUS'); | ||
1262 | 4076 | $sth->execute(); | ||
1263 | 4077 | my ($ss) = @{$sth->fetchall_arrayref({})}; | ||
1264 | 4078 | |||
1265 | 4079 | if ( $ss && %$ss ) { | ||
1266 | 4080 | $ss = { map { lc($_) => $ss->{$_} } keys %$ss }; # lowercase the keys | ||
1267 | 4081 | return $ss; | ||
1268 | 4082 | } | ||
1269 | 4083 | |||
1270 | 4084 | PTDEBUG && _d('This server returns nothing for SHOW SLAVE STATUS'); | ||
1271 | 4085 | $self->{not_a_slave}->{$dbh}++; | ||
1272 | 4086 | } | ||
1273 | 4087 | } | ||
1274 | 4088 | |||
1275 | 4089 | sub get_master_status { | ||
1276 | 4090 | my ( $self, $dbh ) = @_; | ||
1277 | 4091 | |||
1278 | 4092 | if ( $self->{not_a_master}->{$dbh} ) { | ||
1279 | 4093 | PTDEBUG && _d('Server on dbh', $dbh, 'is not a master'); | ||
1280 | 4094 | return; | ||
1281 | 4095 | } | ||
1282 | 4096 | |||
1283 | 4097 | my $sth = $self->{sths}->{$dbh}->{MASTER_STATUS} | ||
1284 | 4098 | ||= $dbh->prepare('SHOW MASTER STATUS'); | ||
1285 | 4099 | PTDEBUG && _d($dbh, 'SHOW MASTER STATUS'); | ||
1286 | 4100 | $sth->execute(); | ||
1287 | 4101 | my ($ms) = @{$sth->fetchall_arrayref({})}; | ||
1288 | 4102 | PTDEBUG && _d( | ||
1289 | 4103 | $ms ? map { "$_=" . (defined $ms->{$_} ? $ms->{$_} : '') } keys %$ms | ||
1290 | 4104 | : ''); | ||
1291 | 4105 | |||
1292 | 4106 | if ( !$ms || scalar keys %$ms < 2 ) { | ||
1293 | 4107 | PTDEBUG && _d('Server on dbh', $dbh, 'does not seem to be a master'); | ||
1294 | 4108 | $self->{not_a_master}->{$dbh}++; | ||
1295 | 4109 | } | ||
1296 | 4110 | |||
1297 | 4111 | return { map { lc($_) => $ms->{$_} } keys %$ms }; # lowercase the keys | ||
1298 | 4112 | } | ||
1299 | 4113 | |||
1300 | 4114 | sub wait_for_master { | ||
1301 | 4115 | my ( $self, %args ) = @_; | ||
1302 | 4116 | my @required_args = qw(master_status slave_dbh); | ||
1303 | 4117 | foreach my $arg ( @required_args ) { | ||
1304 | 4118 | die "I need a $arg argument" unless $args{$arg}; | ||
1305 | 4119 | } | ||
1306 | 4120 | my ($master_status, $slave_dbh) = @args{@required_args}; | ||
1307 | 4121 | my $timeout = $args{timeout} || 60; | ||
1308 | 4122 | |||
1309 | 4123 | my $result; | ||
1310 | 4124 | my $waited; | ||
1311 | 4125 | if ( $master_status ) { | ||
1312 | 4126 | my $sql = "SELECT MASTER_POS_WAIT('$master_status->{file}', " | ||
1313 | 4127 | . "$master_status->{position}, $timeout)"; | ||
1314 | 4128 | PTDEBUG && _d($slave_dbh, $sql); | ||
1315 | 4129 | my $start = time; | ||
1316 | 4130 | ($result) = $slave_dbh->selectrow_array($sql); | ||
1317 | 4131 | |||
1318 | 4132 | $waited = time - $start; | ||
1319 | 4133 | |||
1320 | 4134 | PTDEBUG && _d('Result of waiting:', $result); | ||
1321 | 4135 | PTDEBUG && _d("Waited", $waited, "seconds"); | ||
1322 | 4136 | } | ||
1323 | 4137 | else { | ||
1324 | 4138 | PTDEBUG && _d('Not waiting: this server is not a master'); | ||
1325 | 4139 | } | ||
1326 | 4140 | |||
1327 | 4141 | return { | ||
1328 | 4142 | result => $result, | ||
1329 | 4143 | waited => $waited, | ||
1330 | 4144 | }; | ||
1331 | 4145 | } | ||
1332 | 4146 | |||
1333 | 4147 | sub stop_slave { | ||
1334 | 4148 | my ( $self, $dbh ) = @_; | ||
1335 | 4149 | my $sth = $self->{sths}->{$dbh}->{STOP_SLAVE} | ||
1336 | 4150 | ||= $dbh->prepare('STOP SLAVE'); | ||
1337 | 4151 | PTDEBUG && _d($dbh, $sth->{Statement}); | ||
1338 | 4152 | $sth->execute(); | ||
1339 | 4153 | } | ||
1340 | 4154 | |||
1341 | 4155 | sub start_slave { | ||
1342 | 4156 | my ( $self, $dbh, $pos ) = @_; | ||
1343 | 4157 | if ( $pos ) { | ||
1344 | 4158 | my $sql = "START SLAVE UNTIL MASTER_LOG_FILE='$pos->{file}', " | ||
1345 | 4159 | . "MASTER_LOG_POS=$pos->{position}"; | ||
1346 | 4160 | PTDEBUG && _d($dbh, $sql); | ||
1347 | 4161 | $dbh->do($sql); | ||
1348 | 4162 | } | ||
1349 | 4163 | else { | ||
1350 | 4164 | my $sth = $self->{sths}->{$dbh}->{START_SLAVE} | ||
1351 | 4165 | ||= $dbh->prepare('START SLAVE'); | ||
1352 | 4166 | PTDEBUG && _d($dbh, $sth->{Statement}); | ||
1353 | 4167 | $sth->execute(); | ||
1354 | 4168 | } | ||
1355 | 4169 | } | ||
1356 | 4170 | |||
1357 | 4171 | sub catchup_to_master { | ||
1358 | 4172 | my ( $self, $slave, $master, $timeout ) = @_; | ||
1359 | 4173 | $self->stop_slave($master); | ||
1360 | 4174 | $self->stop_slave($slave); | ||
1361 | 4175 | my $slave_status = $self->get_slave_status($slave); | ||
1362 | 4176 | my $slave_pos = $self->repl_posn($slave_status); | ||
1363 | 4177 | my $master_status = $self->get_master_status($master); | ||
1364 | 4178 | my $master_pos = $self->repl_posn($master_status); | ||
1365 | 4179 | PTDEBUG && _d('Master position:', $self->pos_to_string($master_pos), | ||
1366 | 4180 | 'Slave position:', $self->pos_to_string($slave_pos)); | ||
1367 | 4181 | |||
1368 | 4182 | my $result; | ||
1369 | 4183 | if ( $self->pos_cmp($slave_pos, $master_pos) < 0 ) { | ||
1370 | 4184 | PTDEBUG && _d('Waiting for slave to catch up to master'); | ||
1371 | 4185 | $self->start_slave($slave, $master_pos); | ||
1372 | 4186 | |||
1373 | 4187 | $result = $self->wait_for_master( | ||
1374 | 4188 | master_status => $master_status, | ||
1375 | 4189 | slave_dbh => $slave, | ||
1376 | 4190 | timeout => $timeout, | ||
1377 | 4191 | master_status => $master_status | ||
1378 | 4192 | ); | ||
1379 | 4193 | if ( !defined $result->{result} ) { | ||
1380 | 4194 | $slave_status = $self->get_slave_status($slave); | ||
1381 | 4195 | if ( !$self->slave_is_running($slave_status) ) { | ||
1382 | 4196 | PTDEBUG && _d('Master position:', | ||
1383 | 4197 | $self->pos_to_string($master_pos), | ||
1384 | 4198 | 'Slave position:', $self->pos_to_string($slave_pos)); | ||
1385 | 4199 | $slave_pos = $self->repl_posn($slave_status); | ||
1386 | 4200 | if ( $self->pos_cmp($slave_pos, $master_pos) != 0 ) { | ||
1387 | 4201 | die "MASTER_POS_WAIT() returned NULL but slave has not " | ||
1388 | 4202 | . "caught up to master"; | ||
1389 | 4203 | } | ||
1390 | 4204 | PTDEBUG && _d('Slave is caught up to master and stopped'); | ||
1391 | 4205 | } | ||
1392 | 4206 | else { | ||
1393 | 4207 | die "Slave has not caught up to master and it is still running"; | ||
1394 | 4208 | } | ||
1395 | 4209 | } | ||
1396 | 4210 | } | ||
1397 | 4211 | else { | ||
1398 | 4212 | PTDEBUG && _d("Slave is already caught up to master"); | ||
1399 | 4213 | } | ||
1400 | 4214 | |||
1401 | 4215 | return $result; | ||
1402 | 4216 | } | ||
1403 | 4217 | |||
1404 | 4218 | sub catchup_to_same_pos { | ||
1405 | 4219 | my ( $self, $s1_dbh, $s2_dbh ) = @_; | ||
1406 | 4220 | $self->stop_slave($s1_dbh); | ||
1407 | 4221 | $self->stop_slave($s2_dbh); | ||
1408 | 4222 | my $s1_status = $self->get_slave_status($s1_dbh); | ||
1409 | 4223 | my $s2_status = $self->get_slave_status($s2_dbh); | ||
1410 | 4224 | my $s1_pos = $self->repl_posn($s1_status); | ||
1411 | 4225 | my $s2_pos = $self->repl_posn($s2_status); | ||
1412 | 4226 | if ( $self->pos_cmp($s1_pos, $s2_pos) < 0 ) { | ||
1413 | 4227 | $self->start_slave($s1_dbh, $s2_pos); | ||
1414 | 4228 | } | ||
1415 | 4229 | elsif ( $self->pos_cmp($s2_pos, $s1_pos) < 0 ) { | ||
1416 | 4230 | $self->start_slave($s2_dbh, $s1_pos); | ||
1417 | 4231 | } | ||
1418 | 4232 | |||
1419 | 4233 | $s1_status = $self->get_slave_status($s1_dbh); | ||
1420 | 4234 | $s2_status = $self->get_slave_status($s2_dbh); | ||
1421 | 4235 | $s1_pos = $self->repl_posn($s1_status); | ||
1422 | 4236 | $s2_pos = $self->repl_posn($s2_status); | ||
1423 | 4237 | |||
1424 | 4238 | if ( $self->slave_is_running($s1_status) | ||
1425 | 4239 | || $self->slave_is_running($s2_status) | ||
1426 | 4240 | || $self->pos_cmp($s1_pos, $s2_pos) != 0) | ||
1427 | 4241 | { | ||
1428 | 4242 | die "The servers aren't both stopped at the same position"; | ||
1429 | 4243 | } | ||
1430 | 4244 | |||
1431 | 4245 | } | ||
1432 | 4246 | |||
1433 | 4247 | sub slave_is_running { | ||
1434 | 4248 | my ( $self, $slave_status ) = @_; | ||
1435 | 4249 | return ($slave_status->{slave_sql_running} || 'No') eq 'Yes'; | ||
1436 | 4250 | } | ||
1437 | 4251 | |||
1438 | 4252 | sub has_slave_updates { | ||
1439 | 4253 | my ( $self, $dbh ) = @_; | ||
1440 | 4254 | my $sql = q{SHOW VARIABLES LIKE 'log_slave_updates'}; | ||
1441 | 4255 | PTDEBUG && _d($dbh, $sql); | ||
1442 | 4256 | my ($name, $value) = $dbh->selectrow_array($sql); | ||
1443 | 4257 | return $value && $value =~ m/^(1|ON)$/; | ||
1444 | 4258 | } | ||
1445 | 4259 | |||
1446 | 4260 | sub repl_posn { | ||
1447 | 4261 | my ( $self, $status ) = @_; | ||
1448 | 4262 | if ( exists $status->{file} && exists $status->{position} ) { | ||
1449 | 4263 | return { | ||
1450 | 4264 | file => $status->{file}, | ||
1451 | 4265 | position => $status->{position}, | ||
1452 | 4266 | }; | ||
1453 | 4267 | } | ||
1454 | 4268 | else { | ||
1455 | 4269 | return { | ||
1456 | 4270 | file => $status->{relay_master_log_file}, | ||
1457 | 4271 | position => $status->{exec_master_log_pos}, | ||
1458 | 4272 | }; | ||
1459 | 4273 | } | ||
1460 | 4274 | } | ||
1461 | 4275 | |||
1462 | 4276 | sub get_slave_lag { | ||
1463 | 4277 | my ( $self, $dbh ) = @_; | ||
1464 | 4278 | my $stat = $self->get_slave_status($dbh); | ||
1465 | 4279 | return unless $stat; # server is not a slave | ||
1466 | 4280 | return $stat->{seconds_behind_master}; | ||
1467 | 4281 | } | ||
1468 | 4282 | |||
1469 | 4283 | sub pos_cmp { | ||
1470 | 4284 | my ( $self, $a, $b ) = @_; | ||
1471 | 4285 | return $self->pos_to_string($a) cmp $self->pos_to_string($b); | ||
1472 | 4286 | } | ||
1473 | 4287 | |||
1474 | 4288 | sub short_host { | ||
1475 | 4289 | my ( $self, $dsn ) = @_; | ||
1476 | 4290 | my ($host, $port); | ||
1477 | 4291 | if ( $dsn->{master_host} ) { | ||
1478 | 4292 | $host = $dsn->{master_host}; | ||
1479 | 4293 | $port = $dsn->{master_port}; | ||
1480 | 4294 | } | ||
1481 | 4295 | else { | ||
1482 | 4296 | $host = $dsn->{h}; | ||
1483 | 4297 | $port = $dsn->{P}; | ||
1484 | 4298 | } | ||
1485 | 4299 | return ($host || '[default]') . ( ($port || 3306) == 3306 ? '' : ":$port" ); | ||
1486 | 4300 | } | ||
1487 | 4301 | |||
1488 | 4302 | sub is_replication_thread { | ||
1489 | 4303 | my ( $self, $query, %args ) = @_; | ||
1490 | 4304 | return unless $query; | ||
1491 | 4305 | |||
1492 | 4306 | my $type = lc($args{type} || 'all'); | ||
1493 | 4307 | die "Invalid type: $type" | ||
1494 | 4308 | unless $type =~ m/^binlog_dump|slave_io|slave_sql|all$/i; | ||
1495 | 4309 | |||
1496 | 4310 | my $match = 0; | ||
1497 | 4311 | if ( $type =~ m/binlog_dump|all/i ) { | ||
1498 | 4312 | $match = 1 | ||
1499 | 4313 | if ($query->{Command} || $query->{command} || '') eq "Binlog Dump"; | ||
1500 | 4314 | } | ||
1501 | 4315 | if ( !$match ) { | ||
1502 | 4316 | if ( ($query->{User} || $query->{user} || '') eq "system user" ) { | ||
1503 | 4317 | PTDEBUG && _d("Slave replication thread"); | ||
1504 | 4318 | if ( $type ne 'all' ) { | ||
1505 | 4319 | my $state = $query->{State} || $query->{state} || ''; | ||
1506 | 4320 | |||
1507 | 4321 | if ( $state =~ m/^init|end$/ ) { | ||
1508 | 4322 | PTDEBUG && _d("Special state:", $state); | ||
1509 | 4323 | $match = 1; | ||
1510 | 4324 | } | ||
1511 | 4325 | else { | ||
1512 | 4326 | my ($slave_sql) = $state =~ m/ | ||
1513 | 4327 | ^(Waiting\sfor\sthe\snext\sevent | ||
1514 | 4328 | |Reading\sevent\sfrom\sthe\srelay\slog | ||
1515 | 4329 | |Has\sread\sall\srelay\slog;\swaiting | ||
1516 | 4330 | |Making\stemp\sfile | ||
1517 | 4331 | |Waiting\sfor\sslave\smutex\son\sexit)/xi; | ||
1518 | 4332 | |||
1519 | 4333 | $match = $type eq 'slave_sql' && $slave_sql ? 1 | ||
1520 | 4334 | : $type eq 'slave_io' && !$slave_sql ? 1 | ||
1521 | 4335 | : 0; | ||
1522 | 4336 | } | ||
1523 | 4337 | } | ||
1524 | 4338 | else { | ||
1525 | 4339 | $match = 1; | ||
1526 | 4340 | } | ||
1527 | 4341 | } | ||
1528 | 4342 | else { | ||
1529 | 4343 | PTDEBUG && _d('Not system user'); | ||
1530 | 4344 | } | ||
1531 | 4345 | |||
1532 | 4346 | if ( !defined $args{check_known_ids} || $args{check_known_ids} ) { | ||
1533 | 4347 | my $id = $query->{Id} || $query->{id}; | ||
1534 | 4348 | if ( $match ) { | ||
1535 | 4349 | $self->{replication_thread}->{$id} = 1; | ||
1536 | 4350 | } | ||
1537 | 4351 | else { | ||
1538 | 4352 | if ( $self->{replication_thread}->{$id} ) { | ||
1539 | 4353 | PTDEBUG && _d("Thread ID is a known replication thread ID"); | ||
1540 | 4354 | $match = 1; | ||
1541 | 4355 | } | ||
1542 | 4356 | } | ||
1543 | 4357 | } | ||
1544 | 4358 | } | ||
1545 | 4359 | |||
1546 | 4360 | PTDEBUG && _d('Matches', $type, 'replication thread:', | ||
1547 | 4361 | ($match ? 'yes' : 'no'), '; match:', $match); | ||
1548 | 4362 | |||
1549 | 4363 | return $match; | ||
1550 | 4364 | } | ||
1551 | 4365 | |||
1552 | 4366 | |||
1553 | 4367 | sub get_replication_filters { | ||
1554 | 4368 | my ( $self, %args ) = @_; | ||
1555 | 4369 | my @required_args = qw(dbh); | ||
1556 | 4370 | foreach my $arg ( @required_args ) { | ||
1557 | 4371 | die "I need a $arg argument" unless $args{$arg}; | ||
1558 | 4372 | } | ||
1559 | 4373 | my ($dbh) = @args{@required_args}; | ||
1560 | 4374 | |||
1561 | 4375 | my %filters = (); | ||
1562 | 4376 | |||
1563 | 4377 | my $status = $self->get_master_status($dbh); | ||
1564 | 4378 | if ( $status ) { | ||
1565 | 4379 | map { $filters{$_} = $status->{$_} } | ||
1566 | 4380 | grep { defined $status->{$_} && $status->{$_} ne '' } | ||
1567 | 4381 | qw( | ||
1568 | 4382 | binlog_do_db | ||
1569 | 4383 | binlog_ignore_db | ||
1570 | 4384 | ); | ||
1571 | 4385 | } | ||
1572 | 4386 | |||
1573 | 4387 | $status = $self->get_slave_status($dbh); | ||
1574 | 4388 | if ( $status ) { | ||
1575 | 4389 | map { $filters{$_} = $status->{$_} } | ||
1576 | 4390 | grep { defined $status->{$_} && $status->{$_} ne '' } | ||
1577 | 4391 | qw( | ||
1578 | 4392 | replicate_do_db | ||
1579 | 4393 | replicate_ignore_db | ||
1580 | 4394 | replicate_do_table | ||
1581 | 4395 | replicate_ignore_table | ||
1582 | 4396 | replicate_wild_do_table | ||
1583 | 4397 | replicate_wild_ignore_table | ||
1584 | 4398 | ); | ||
1585 | 4399 | |||
1586 | 4400 | my $sql = "SHOW VARIABLES LIKE 'slave_skip_errors'"; | ||
1587 | 4401 | PTDEBUG && _d($dbh, $sql); | ||
1588 | 4402 | my $row = $dbh->selectrow_arrayref($sql); | ||
1589 | 4403 | $filters{slave_skip_errors} = $row->[1] if $row->[1] && $row->[1] ne 'OFF'; | ||
1590 | 4404 | } | ||
1591 | 4405 | |||
1592 | 4406 | return \%filters; | ||
1593 | 4407 | } | ||
1594 | 4408 | |||
1595 | 4409 | |||
1596 | 4410 | sub pos_to_string { | ||
1597 | 4411 | my ( $self, $pos ) = @_; | ||
1598 | 4412 | my $fmt = '%s/%020d'; | ||
1599 | 4413 | return sprintf($fmt, @{$pos}{qw(file position)}); | ||
1600 | 4414 | } | ||
1601 | 4415 | |||
1602 | 4416 | sub reset_known_replication_threads { | ||
1603 | 4417 | my ( $self ) = @_; | ||
1604 | 4418 | $self->{replication_thread} = {}; | ||
1605 | 4419 | return; | ||
1606 | 4420 | } | ||
1607 | 4421 | |||
1608 | 4422 | sub get_cxn_from_dsn_table { | ||
1609 | 4423 | my ($self, %args) = @_; | ||
1610 | 4424 | my @required_args = qw(dsn_table_dsn make_cxn DSNParser Quoter); | ||
1611 | 4425 | foreach my $arg ( @required_args ) { | ||
1612 | 4426 | die "I need a $arg argument" unless $args{$arg}; | ||
1613 | 4427 | } | ||
1614 | 4428 | my ($dsn_table_dsn, $make_cxn, $dp, $q) = @args{@required_args}; | ||
1615 | 4429 | PTDEBUG && _d('DSN table DSN:', $dsn_table_dsn); | ||
1616 | 4430 | |||
1617 | 4431 | my $dsn = $dp->parse($dsn_table_dsn); | ||
1618 | 4432 | my $dsn_table; | ||
1619 | 4433 | if ( $dsn->{D} && $dsn->{t} ) { | ||
1620 | 4434 | $dsn_table = $q->quote($dsn->{D}, $dsn->{t}); | ||
1621 | 4435 | } | ||
1622 | 4436 | elsif ( $dsn->{t} && $dsn->{t} =~ m/\./ ) { | ||
1623 | 4437 | $dsn_table = $q->quote($q->split_unquote($dsn->{t})); | ||
1624 | 4438 | } | ||
1625 | 4439 | else { | ||
1626 | 4440 | die "DSN table DSN does not specify a database (D) " | ||
1627 | 4441 | . "or a database-qualified table (t)"; | ||
1628 | 4442 | } | ||
1629 | 4443 | |||
1630 | 4444 | my $dsn_tbl_cxn = $make_cxn->(dsn => $dsn); | ||
1631 | 4445 | my $dbh = $dsn_tbl_cxn->connect(); | ||
1632 | 4446 | my $sql = "SELECT dsn FROM $dsn_table ORDER BY id"; | ||
1633 | 4447 | PTDEBUG && _d($sql); | ||
1634 | 4448 | my $dsn_strings = $dbh->selectcol_arrayref($sql); | ||
1635 | 4449 | my @cxn; | ||
1636 | 4450 | if ( $dsn_strings ) { | ||
1637 | 4451 | foreach my $dsn_string ( @$dsn_strings ) { | ||
1638 | 4452 | PTDEBUG && _d('DSN from DSN table:', $dsn_string); | ||
1639 | 4453 | push @cxn, $make_cxn->(dsn_string => $dsn_string); | ||
1640 | 4454 | } | ||
1641 | 4455 | } | ||
1642 | 4456 | return \@cxn; | ||
1643 | 4457 | } | ||
1644 | 4458 | |||
1645 | 4459 | sub _d { | ||
1646 | 4460 | my ($package, undef, $line) = caller 0; | ||
1647 | 4461 | @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } | ||
1648 | 4462 | map { defined $_ ? $_ : 'undef' } | ||
1649 | 4463 | @_; | ||
1650 | 4464 | print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; | ||
1651 | 4465 | } | ||
1652 | 4466 | |||
1653 | 4467 | 1; | ||
1654 | 4468 | } | ||
1655 | 4469 | # ########################################################################### | ||
1656 | 4470 | # End MasterSlave package | ||
1657 | 4471 | # ########################################################################### | ||
1658 | 4472 | |||
1659 | 4473 | # ########################################################################### | ||
1660 | 4474 | # ReplicaLagWaiter package | ||
1661 | 4475 | # This package is a copy without comments from the original. The original | ||
1662 | 4476 | # with comments and its test file can be found in the Bazaar repository at, | ||
1663 | 4477 | # lib/ReplicaLagWaiter.pm | ||
1664 | 4478 | # t/lib/ReplicaLagWaiter.t | ||
1665 | 4479 | # See https://launchpad.net/percona-toolkit for more information. | ||
1666 | 4480 | # ########################################################################### | ||
1667 | 4481 | { | ||
1668 | 4482 | package ReplicaLagWaiter; | ||
1669 | 4483 | |||
1670 | 4484 | use strict; | ||
1671 | 4485 | use warnings FATAL => 'all'; | ||
1672 | 4486 | use English qw(-no_match_vars); | ||
1673 | 4487 | use constant PTDEBUG => $ENV{PTDEBUG} || 0; | ||
1674 | 4488 | |||
1675 | 4489 | use Time::HiRes qw(sleep time); | ||
1676 | 4490 | use Data::Dumper; | ||
1677 | 4491 | |||
1678 | 4492 | sub new { | ||
1679 | 4493 | my ( $class, %args ) = @_; | ||
1680 | 4494 | my @required_args = qw(oktorun get_lag sleep max_lag slaves); | ||
1681 | 4495 | foreach my $arg ( @required_args ) { | ||
1682 | 4496 | die "I need a $arg argument" unless defined $args{$arg}; | ||
1683 | 4497 | } | ||
1684 | 4498 | |||
1685 | 4499 | my $self = { | ||
1686 | 4500 | %args, | ||
1687 | 4501 | }; | ||
1688 | 4502 | |||
1689 | 4503 | return bless $self, $class; | ||
1690 | 4504 | } | ||
1691 | 4505 | |||
1692 | 4506 | sub wait { | ||
1693 | 4507 | my ( $self, %args ) = @_; | ||
1694 | 4508 | my @required_args = qw(); | ||
1695 | 4509 | foreach my $arg ( @required_args ) { | ||
1696 | 4510 | die "I need a $arg argument" unless $args{$arg}; | ||
1697 | 4511 | } | ||
1698 | 4512 | my $pr = $args{Progress}; | ||
1699 | 4513 | |||
1700 | 4514 | my $oktorun = $self->{oktorun}; | ||
1701 | 4515 | my $get_lag = $self->{get_lag}; | ||
1702 | 4516 | my $sleep = $self->{sleep}; | ||
1703 | 4517 | my $slaves = $self->{slaves}; | ||
1704 | 4518 | my $max_lag = $self->{max_lag}; | ||
1705 | 4519 | |||
1706 | 4520 | my $worst; # most lagging slave | ||
1707 | 4521 | my $pr_callback; | ||
1708 | 4522 | my $pr_first_report; | ||
1709 | 4523 | if ( $pr ) { | ||
1710 | 4524 | $pr_callback = sub { | ||
1711 | 4525 | my ($fraction, $elapsed, $remaining, $eta, $completed) = @_; | ||
1712 | 4526 | my $dsn_name = $worst->{cxn}->name(); | ||
1713 | 4527 | if ( defined $worst->{lag} ) { | ||
1714 | 4528 | print STDERR "Replica lag is " . ($worst->{lag} || '?') | ||
1715 | 4529 | . " seconds on $dsn_name. Waiting.\n"; | ||
1716 | 4530 | } | ||
1717 | 4531 | else { | ||
1718 | 4532 | print STDERR "Replica $dsn_name is stopped. Waiting.\n"; | ||
1719 | 4533 | } | ||
1720 | 4534 | return; | ||
1721 | 4535 | }; | ||
1722 | 4536 | $pr->set_callback($pr_callback); | ||
1723 | 4537 | |||
1724 | 4538 | $pr_first_report = sub { | ||
1725 | 4539 | my $dsn_name = $worst->{cxn}->name(); | ||
1726 | 4540 | if ( !defined $worst->{lag} ) { | ||
1727 | 4541 | print STDERR "Replica $dsn_name is stopped. Waiting.\n"; | ||
1728 | 4542 | } | ||
1729 | 4543 | return; | ||
1730 | 4544 | }; | ||
1731 | 4545 | } | ||
1732 | 4546 | |||
1733 | 4547 | my @lagged_slaves = map { {cxn=>$_, lag=>undef} } @$slaves; | ||
1734 | 4548 | while ( $oktorun->() && @lagged_slaves ) { | ||
1735 | 4549 | PTDEBUG && _d('Checking slave lag'); | ||
1736 | 4550 | for my $i ( 0..$#lagged_slaves ) { | ||
1737 | 4551 | my $lag = $get_lag->($lagged_slaves[$i]->{cxn}); | ||
1738 | 4552 | PTDEBUG && _d($lagged_slaves[$i]->{cxn}->name(), | ||
1739 | 4553 | 'slave lag:', $lag); | ||
1740 | 4554 | if ( !defined $lag || $lag > $max_lag ) { | ||
1741 | 4555 | $lagged_slaves[$i]->{lag} = $lag; | ||
1742 | 4556 | } | ||
1743 | 4557 | else { | ||
1744 | 4558 | delete $lagged_slaves[$i]; | ||
1745 | 4559 | } | ||
1746 | 4560 | } | ||
1747 | 4561 | |||
1748 | 4562 | @lagged_slaves = grep { defined $_ } @lagged_slaves; | ||
1749 | 4563 | if ( @lagged_slaves ) { | ||
1750 | 4564 | @lagged_slaves = reverse sort { | ||
1751 | 4565 | defined $a->{lag} && defined $b->{lag} ? $a->{lag} <=> $b->{lag} | ||
1752 | 4566 | : defined $a->{lag} ? -1 | ||
1753 | 4567 | : 1; | ||
1754 | 4568 | } @lagged_slaves; | ||
1755 | 4569 | $worst = $lagged_slaves[0]; | ||
1756 | 4570 | PTDEBUG && _d(scalar @lagged_slaves, 'slaves are lagging, worst:', | ||
1757 | 4571 | $worst->{lag}, 'on', Dumper($worst->{cxn}->dsn())); | ||
1758 | 4572 | |||
1759 | 4573 | if ( $pr ) { | ||
1760 | 4574 | $pr->update( | ||
1761 | 4575 | sub { return 0; }, | ||
1762 | 4576 | first_report => $pr_first_report, | ||
1763 | 4577 | ); | ||
1764 | 4578 | } | ||
1765 | 4579 | |||
1766 | 4580 | PTDEBUG && _d('Calling sleep callback'); | ||
1767 | 4581 | $sleep->($worst->{cxn}, $worst->{lag}); | ||
1768 | 4582 | } | ||
1769 | 4583 | } | ||
1770 | 4584 | |||
1771 | 4585 | PTDEBUG && _d('All slaves caught up'); | ||
1772 | 4586 | return; | ||
1773 | 4587 | } | ||
1774 | 4588 | |||
1775 | 4589 | sub _d { | ||
1776 | 4590 | my ($package, undef, $line) = caller 0; | ||
1777 | 4591 | @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } | ||
1778 | 4592 | map { defined $_ ? $_ : 'undef' } | ||
1779 | 4593 | @_; | ||
1780 | 4594 | print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; | ||
1781 | 4595 | } | ||
1782 | 4596 | |||
1783 | 4597 | 1; | ||
1784 | 4598 | } | ||
1785 | 4599 | # ########################################################################### | ||
1786 | 4600 | # End ReplicaLagWaiter package | ||
1787 | 4601 | # ########################################################################### | ||
1788 | 4602 | |||
1789 | 4603 | # ########################################################################### | ||
1790 | 4604 | # MySQLStatusWaiter package | ||
1791 | 4605 | # This package is a copy without comments from the original. The original | ||
1792 | 4606 | # with comments and its test file can be found in the Bazaar repository at, | ||
1793 | 4607 | # lib/MySQLStatusWaiter.pm | ||
1794 | 4608 | # t/lib/MySQLStatusWaiter.t | ||
1795 | 4609 | # See https://launchpad.net/percona-toolkit for more information. | ||
1796 | 4610 | # ########################################################################### | ||
1797 | 4611 | { | ||
1798 | 4612 | package MySQLStatusWaiter; | ||
1799 | 4613 | |||
1800 | 4614 | use strict; | ||
1801 | 4615 | use warnings FATAL => 'all'; | ||
1802 | 4616 | use English qw(-no_match_vars); | ||
1803 | 4617 | use constant PTDEBUG => $ENV{PTDEBUG} || 0; | ||
1804 | 4618 | |||
1805 | 4619 | sub new { | ||
1806 | 4620 | my ( $class, %args ) = @_; | ||
1807 | 4621 | my @required_args = qw(max_spec get_status sleep oktorun); | ||
1808 | 4622 | foreach my $arg ( @required_args ) { | ||
1809 | 4623 | die "I need a $arg argument" unless defined $args{$arg}; | ||
1810 | 4624 | } | ||
1811 | 4625 | |||
1812 | 4626 | PTDEBUG && _d('Parsing spec for max thresholds'); | ||
1813 | 4627 | my $max_val_for = _parse_spec( | ||
1814 | 4628 | spec => $args{max_spec}, | ||
1815 | 4629 | get_status => $args{get_status}, | ||
1816 | 4630 | threshold_factor => 0.2, # +20% | ||
1817 | 4631 | ); | ||
1818 | 4632 | |||
1819 | 4633 | PTDEBUG && _d('Parsing spec for critical thresholds'); | ||
1820 | 4634 | my $critical_val_for = _parse_spec( | ||
1821 | 4635 | spec => $args{critical_spec} || [], | ||
1822 | 4636 | get_status => $args{get_status}, | ||
1823 | 4637 | threshold_factor => 1.0, # double (x2; +100%) | ||
1824 | 4638 | ); | ||
1825 | 4639 | |||
1826 | 4640 | my $self = { | ||
1827 | 4641 | get_status => $args{get_status}, | ||
1828 | 4642 | sleep => $args{sleep}, | ||
1829 | 4643 | oktorun => $args{oktorun}, | ||
1830 | 4644 | max_val_for => $max_val_for, | ||
1831 | 4645 | critical_val_for => $critical_val_for, | ||
1832 | 4646 | }; | ||
1833 | 4647 | |||
1834 | 4648 | return bless $self, $class; | ||
1835 | 4649 | } | ||
1836 | 4650 | |||
1837 | 4651 | sub _parse_spec { | ||
1838 | 4652 | my ( %args ) = @_; | ||
1839 | 4653 | my @required_args = qw(spec get_status); | ||
1840 | 4654 | foreach my $arg ( @required_args ) { | ||
1841 | 4655 | die "I need a $arg argument" unless defined $args{$arg}; | ||
1842 | 4656 | } | ||
1843 | 4657 | my ($spec, $get_status) = @args{@required_args}; | ||
1844 | 4658 | |||
1845 | 4659 | return unless $spec && scalar @$spec; | ||
1846 | 4660 | my $threshold_factor = $args{threshold_factor} || 0.20; | ||
1847 | 4661 | |||
1848 | 4662 | my %max_val_for; | ||
1849 | 4663 | foreach my $var_val ( @$spec ) { | ||
1850 | 4664 | my ($var, $val) = split /[:=]/, $var_val; | ||
1851 | 4665 | die "Invalid spec: $var_val" unless $var; | ||
1852 | 4666 | if ( !$val ) { | ||
1853 | 4667 | my $init_val = $get_status->($var); | ||
1854 | 4668 | PTDEBUG && _d('Initial', $var, 'value:', $init_val); | ||
1855 | 4669 | $val = int(($init_val * $threshold_factor) + $init_val); | ||
1856 | 4670 | } | ||
1857 | 4671 | PTDEBUG && _d('Wait if', $var, '>=', $val); | ||
1858 | 4672 | $max_val_for{$var} = $val; | ||
1859 | 4673 | } | ||
1860 | 4674 | |||
1861 | 4675 | return \%max_val_for; | ||
1862 | 4676 | } | ||
1863 | 4677 | |||
1864 | 4678 | sub max_values { | ||
1865 | 4679 | my ($self) = @_; | ||
1866 | 4680 | return $self->{max_val_for}; | ||
1867 | 4681 | } | ||
1868 | 4682 | |||
1869 | 4683 | sub critical_values { | ||
1870 | 4684 | my ($self) = @_; | ||
1871 | 4685 | return $self->{critical_val_for}; | ||
1872 | 4686 | } | ||
1873 | 4687 | |||
1874 | 4688 | sub wait { | ||
1875 | 4689 | my ( $self, %args ) = @_; | ||
1876 | 4690 | |||
1877 | 4691 | return unless $self->{max_val_for}; | ||
1878 | 4692 | |||
1879 | 4693 | my $pr = $args{Progress}; # optional | ||
1880 | 4694 | |||
1881 | 4695 | my $oktorun = $self->{oktorun}; | ||
1882 | 4696 | my $get_status = $self->{get_status}; | ||
1883 | 4697 | my $sleep = $self->{sleep}; | ||
1884 | 4698 | |||
1885 | 4699 | my %vals_too_high = %{$self->{max_val_for}}; | ||
1886 | 4700 | my $pr_callback; | ||
1887 | 4701 | if ( $pr ) { | ||
1888 | 4702 | $pr_callback = sub { | ||
1889 | 4703 | print STDERR "Pausing because " | ||
1890 | 4704 | . join(', ', | ||
1891 | 4705 | map { | ||
1892 | 4706 | "$_=" | ||
1893 | 4707 | . (defined $vals_too_high{$_} ? $vals_too_high{$_} | ||
1894 | 4708 | : 'unknown') | ||
1895 | 4709 | } sort keys %vals_too_high | ||
1896 | 4710 | ) | ||
1897 | 4711 | . ".\n"; | ||
1898 | 4712 | return; | ||
1899 | 4713 | }; | ||
1900 | 4714 | $pr->set_callback($pr_callback); | ||
1901 | 4715 | } | ||
1902 | 4716 | |||
1903 | 4717 | while ( $oktorun->() ) { | ||
1904 | 4718 | PTDEBUG && _d('Checking status variables'); | ||
1905 | 4719 | foreach my $var ( sort keys %vals_too_high ) { | ||
1906 | 4720 | my $val = $get_status->($var); | ||
1907 | 4721 | PTDEBUG && _d($var, '=', $val); | ||
1908 | 4722 | if ( $val | ||
1909 | 4723 | && exists $self->{critical_val_for}->{$var} | ||
1910 | 4724 | && $val >= $self->{critical_val_for}->{$var} ) { | ||
1911 | 4725 | die "$var=$val exceeds its critical threshold " | ||
1912 | 4726 | . "$self->{critical_val_for}->{$var}\n"; | ||
1913 | 4727 | } | ||
1914 | 4728 | if ( !$val || $val >= $self->{max_val_for}->{$var} ) { | ||
1915 | 4729 | $vals_too_high{$var} = $val; | ||
1916 | 4730 | } | ||
1917 | 4731 | else { | ||
1918 | 4732 | delete $vals_too_high{$var}; | ||
1919 | 4733 | } | ||
1920 | 4734 | } | ||
1921 | 4735 | |||
1922 | 4736 | last unless scalar keys %vals_too_high; | ||
1923 | 4737 | |||
1924 | 4738 | PTDEBUG && _d(scalar keys %vals_too_high, 'values are too high:', | ||
1925 | 4739 | %vals_too_high); | ||
1926 | 4740 | if ( $pr ) { | ||
1927 | 4741 | $pr->update(sub { return 0; }); | ||
1928 | 4742 | } | ||
1929 | 4743 | PTDEBUG && _d('Calling sleep callback'); | ||
1930 | 4744 | $sleep->(); | ||
1931 | 4745 | %vals_too_high = %{$self->{max_val_for}}; # recheck all vars | ||
1932 | 4746 | } | ||
1933 | 4747 | |||
1934 | 4748 | PTDEBUG && _d('All var vals are low enough'); | ||
1935 | 4749 | return; | ||
1936 | 4750 | } | ||
1937 | 4751 | |||
1938 | 4752 | sub _d { | ||
1939 | 4753 | my ($package, undef, $line) = caller 0; | ||
1940 | 4754 | @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } | ||
1941 | 4755 | map { defined $_ ? $_ : 'undef' } | ||
1942 | 4756 | @_; | ||
1943 | 4757 | print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; | ||
1944 | 4758 | } | ||
1945 | 4759 | |||
1946 | 4760 | 1; | ||
1947 | 4761 | } | ||
1948 | 4762 | # ########################################################################### | ||
1949 | 4763 | # End MySQLStatusWaiter package | ||
1950 | 4764 | # ########################################################################### | ||
1951 | 4765 | |||
1952 | 4766 | # ########################################################################### | ||
1953 | 4767 | # WeightedAvgRate package | ||
1954 | 4768 | # This package is a copy without comments from the original. The original | ||
1955 | 4769 | # with comments and its test file can be found in the Bazaar repository at, | ||
1956 | 4770 | # lib/WeightedAvgRate.pm | ||
1957 | 4771 | # t/lib/WeightedAvgRate.t | ||
1958 | 4772 | # See https://launchpad.net/percona-toolkit for more information. | ||
1959 | 4773 | # ########################################################################### | ||
1960 | 4774 | { | ||
1961 | 4775 | package WeightedAvgRate; | ||
1962 | 4776 | |||
1963 | 4777 | use strict; | ||
1964 | 4778 | use warnings FATAL => 'all'; | ||
1965 | 4779 | use English qw(-no_match_vars); | ||
1966 | 4780 | use constant PTDEBUG => $ENV{PTDEBUG} || 0; | ||
1967 | 4781 | |||
1968 | 4782 | sub new { | ||
1969 | 4783 | my ( $class, %args ) = @_; | ||
1970 | 4784 | my @required_args = qw(target_t); | ||
1971 | 4785 | foreach my $arg ( @required_args ) { | ||
1972 | 4786 | die "I need a $arg argument" unless defined $args{$arg}; | ||
1973 | 4787 | } | ||
1974 | 4788 | |||
1975 | 4789 | my $self = { | ||
1976 | 4790 | %args, | ||
1977 | 4791 | avg_n => 0, | ||
1978 | 4792 | avg_t => 0, | ||
1979 | 4793 | weight => $args{weight} || 0.75, | ||
1980 | 4794 | }; | ||
1981 | 4795 | |||
1982 | 4796 | return bless $self, $class; | ||
1983 | 4797 | } | ||
1984 | 4798 | |||
1985 | 4799 | sub update { | ||
1986 | 4800 | my ($self, $n, $t) = @_; | ||
1987 | 4801 | PTDEBUG && _d('Master op time:', $n, 'n /', $t, 's'); | ||
1988 | 4802 | |||
1989 | 4803 | if ( $self->{avg_n} && $self->{avg_t} ) { | ||
1990 | 4804 | $self->{avg_n} = ($self->{avg_n} * $self->{weight}) + $n; | ||
1991 | 4805 | $self->{avg_t} = ($self->{avg_t} * $self->{weight}) + $t; | ||
1992 | 4806 | $self->{avg_rate} = $self->{avg_n} / $self->{avg_t}; | ||
1993 | 4807 | PTDEBUG && _d('Weighted avg rate:', $self->{avg_rate}, 'n/s'); | ||
1994 | 4808 | } | ||
1995 | 4809 | else { | ||
1996 | 4810 | $self->{avg_n} = $n; | ||
1997 | 4811 | $self->{avg_t} = $t; | ||
1998 | 4812 | $self->{avg_rate} = $self->{avg_n} / $self->{avg_t}; | ||
1999 | 4813 | PTDEBUG && _d('Initial avg rate:', $self->{avg_rate}, 'n/s'); | ||
2000 | 4814 | } | ||
2001 | 4815 | |||
2002 | 4816 | my $new_n = int($self->{avg_rate} * $self->{target_t}); | ||
2003 | 4817 | PTDEBUG && _d('Adjust n to', $new_n); | ||
2004 | 4818 | return $new_n; | ||
2005 | 4819 | } | ||
2006 | 4820 | |||
2007 | 4821 | sub _d { | ||
2008 | 4822 | my ($package, undef, $line) = caller 0; | ||
2009 | 4823 | @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } | ||
2010 | 4824 | map { defined $_ ? $_ : 'undef' } | ||
2011 | 4825 | @_; | ||
2012 | 4826 | print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; | ||
2013 | 4827 | } | ||
2014 | 4828 | |||
2015 | 4829 | 1; | ||
2016 | 4830 | } | ||
2017 | 4831 | # ########################################################################### | ||
2018 | 4832 | # End WeightedAvgRate package | ||
2019 | 4833 | # ########################################################################### | ||
2020 | 4834 | |||
2021 | 4835 | # ########################################################################### | ||
2022 | 4836 | # NibbleIterator package | ||
2023 | 4837 | # This package is a copy without comments from the original. The original | ||
2024 | 4838 | # with comments and its test file can be found in the Bazaar repository at, | ||
2025 | 4839 | # lib/NibbleIterator.pm | ||
2026 | 4840 | # t/lib/NibbleIterator.t | ||
2027 | 4841 | # See https://launchpad.net/percona-toolkit for more information. | ||
2028 | 4842 | # ########################################################################### | ||
2029 | 4843 | { | ||
2030 | 4844 | package NibbleIterator; | ||
2031 | 4845 | |||
2032 | 4846 | use strict; | ||
2033 | 4847 | use warnings FATAL => 'all'; | ||
2034 | 4848 | use English qw(-no_match_vars); | ||
2035 | 4849 | use constant PTDEBUG => $ENV{PTDEBUG} || 0; | ||
2036 | 4850 | |||
2037 | 4851 | use Data::Dumper; | ||
2038 | 4852 | $Data::Dumper::Indent = 1; | ||
2039 | 4853 | $Data::Dumper::Sortkeys = 1; | ||
2040 | 4854 | $Data::Dumper::Quotekeys = 0; | ||
2041 | 4855 | |||
2042 | 4856 | sub new { | ||
2043 | 4857 | my ( $class, %args ) = @_; | ||
2044 | 4858 | my @required_args = qw(Cxn tbl chunk_size OptionParser Quoter TableNibbler TableParser); | ||
2045 | 4859 | foreach my $arg ( @required_args ) { | ||
2046 | 4860 | die "I need a $arg argument" unless $args{$arg}; | ||
2047 | 4861 | } | ||
2048 | 4862 | my ($cxn, $tbl, $chunk_size, $o, $q) = @args{@required_args}; | ||
2049 | 4863 | |||
2050 | 4864 | my $nibble_params = can_nibble(%args); | ||
2051 | 4865 | |||
2052 | 4866 | my %comments = ( | ||
2053 | 4867 | bite => "bite table", | ||
2054 | 4868 | nibble => "nibble table", | ||
2055 | 4869 | ); | ||
2056 | 4870 | if ( $args{comments} ) { | ||
2057 | 4871 | map { $comments{$_} = $args{comments}->{$_} } | ||
2058 | 4872 | grep { defined $args{comments}->{$_} } | ||
2059 | 4873 | keys %{$args{comments}}; | ||
2060 | 4874 | } | ||
2061 | 4875 | |||
2062 | 4876 | my $where = $o->has('where') ? $o->get('where') : ''; | ||
2063 | 4877 | my $tbl_struct = $tbl->{tbl_struct}; | ||
2064 | 4878 | my $ignore_col = $o->has('ignore-columns') | ||
2065 | 4879 | ? ($o->get('ignore-columns') || {}) | ||
2066 | 4880 | : {}; | ||
2067 | 4881 | my $all_cols = $o->has('columns') | ||
2068 | 4882 | ? ($o->get('columns') || $tbl_struct->{cols}) | ||
2069 | 4883 | : $tbl_struct->{cols}; | ||
2070 | 4884 | my @cols = grep { !$ignore_col->{$_} } @$all_cols; | ||
2071 | 4885 | my $self; | ||
2072 | 4886 | if ( $nibble_params->{one_nibble} ) { | ||
2073 | 4887 | my $nibble_sql | ||
2074 | 4888 | = ($args{dml} ? "$args{dml} " : "SELECT ") | ||
2075 | 4889 | . ($args{select} ? $args{select} | ||
2076 | 4890 | : join(', ', map { $q->quote($_) } @cols)) | ||
2077 | 4891 | . " FROM $tbl->{name}" | ||
2078 | 4892 | . ($where ? " WHERE $where" : '') | ||
2079 | 4893 | . " /*$comments{bite}*/"; | ||
2080 | 4894 | PTDEBUG && _d('One nibble statement:', $nibble_sql); | ||
2081 | 4895 | |||
2082 | 4896 | my $explain_nibble_sql | ||
2083 | 4897 | = "EXPLAIN SELECT " | ||
2084 | 4898 | . ($args{select} ? $args{select} | ||
2085 | 4899 | : join(', ', map { $q->quote($_) } @cols)) | ||
2086 | 4900 | . " FROM $tbl->{name}" | ||
2087 | 4901 | . ($where ? " WHERE $where" : '') | ||
2088 | 4902 | . " /*explain $comments{bite}*/"; | ||
2089 | 4903 | PTDEBUG && _d('Explain one nibble statement:', $explain_nibble_sql); | ||
2090 | 4904 | |||
2091 | 4905 | $self = { | ||
2092 | 4906 | %args, | ||
2093 | 4907 | one_nibble => 1, | ||
2094 | 4908 | limit => 0, | ||
2095 | 4909 | nibble_sql => $nibble_sql, | ||
2096 | 4910 | explain_nibble_sql => $explain_nibble_sql, | ||
2097 | 4911 | }; | ||
2098 | 4912 | } | ||
2099 | 4913 | else { | ||
2100 | 4914 | my $index = $nibble_params->{index}; # brevity | ||
2101 | 4915 | my $index_cols = $tbl->{tbl_struct}->{keys}->{$index}->{cols}; | ||
2102 | 4916 | |||
2103 | 4917 | my $asc = $args{TableNibbler}->generate_asc_stmt( | ||
2104 | 4918 | %args, | ||
2105 | 4919 | tbl_struct => $tbl->{tbl_struct}, | ||
2106 | 4920 | index => $index, | ||
2107 | 4921 | cols => \@cols, | ||
2108 | 4922 | asc_only => 1, | ||
2109 | 4923 | ); | ||
2110 | 4924 | PTDEBUG && _d('Ascend params:', Dumper($asc)); | ||
2111 | 4925 | |||
2112 | 4926 | my $from = "$tbl->{name} FORCE INDEX(`$index`)"; | ||
2113 | 4927 | my $order_by = join(', ', map {$q->quote($_)} @{$index_cols}); | ||
2114 | 4928 | |||
2115 | 4929 | my $first_lb_sql | ||
2116 | 4930 | = "SELECT /*!40001 SQL_NO_CACHE */ " | ||
2117 | 4931 | . join(', ', map { $q->quote($_) } @{$asc->{scols}}) | ||
2118 | 4932 | . " FROM $from" | ||
2119 | 4933 | . ($where ? " WHERE $where" : '') | ||
2120 | 4934 | . " ORDER BY $order_by" | ||
2121 | 4935 | . " LIMIT 1" | ||
2122 | 4936 | . " /*first lower boundary*/"; | ||
2123 | 4937 | PTDEBUG && _d('First lower boundary statement:', $first_lb_sql); | ||
2124 | 4938 | |||
2125 | 4939 | my $resume_lb_sql; | ||
2126 | 4940 | if ( $args{resume} ) { | ||
2127 | 4941 | $resume_lb_sql | ||
2128 | 4942 | = "SELECT /*!40001 SQL_NO_CACHE */ " | ||
2129 | 4943 | . join(', ', map { $q->quote($_) } @{$asc->{scols}}) | ||
2130 | 4944 | . " FROM $from" | ||
2131 | 4945 | . " WHERE " . $asc->{boundaries}->{'>'} | ||
2132 | 4946 | . ($where ? " AND ($where)" : '') | ||
2133 | 4947 | . " ORDER BY $order_by" | ||
2134 | 4948 | . " LIMIT 1" | ||
2135 | 4949 | . " /*resume lower boundary*/"; | ||
2136 | 4950 | PTDEBUG && _d('Resume lower boundary statement:', $resume_lb_sql); | ||
2137 | 4951 | } | ||
2138 | 4952 | |||
2139 | 4953 | my $last_ub_sql | ||
2140 | 4954 | = "SELECT /*!40001 SQL_NO_CACHE */ " | ||
2141 | 4955 | . join(', ', map { $q->quote($_) } @{$asc->{scols}}) | ||
2142 | 4956 | . " FROM $from" | ||
2143 | 4957 | . ($where ? " WHERE $where" : '') | ||
2144 | 4958 | . " ORDER BY " | ||
2145 | 4959 | . join(' DESC, ', map {$q->quote($_)} @{$index_cols}) . ' DESC' | ||
2146 | 4960 | . " LIMIT 1" | ||
2147 | 4961 | . " /*last upper boundary*/"; | ||
2148 | 4962 | PTDEBUG && _d('Last upper boundary statement:', $last_ub_sql); | ||
2149 | 4963 | |||
2150 | 4964 | my $ub_sql | ||
2151 | 4965 | = "SELECT /*!40001 SQL_NO_CACHE */ " | ||
2152 | 4966 | . join(', ', map { $q->quote($_) } @{$asc->{scols}}) | ||
2153 | 4967 | . " FROM $from" | ||
2154 | 4968 | . " WHERE " . $asc->{boundaries}->{'>='} | ||
2155 | 4969 | . ($where ? " AND ($where)" : '') | ||
2156 | 4970 | . " ORDER BY $order_by" | ||
2157 | 4971 | . " LIMIT ?, 2" | ||
2158 | 4972 | . " /*next chunk boundary*/"; | ||
2159 | 4973 | PTDEBUG && _d('Upper boundary statement:', $ub_sql); | ||
2160 | 4974 | |||
2161 | 4975 | my $nibble_sql | ||
2162 | 4976 | = ($args{dml} ? "$args{dml} " : "SELECT ") | ||
2163 | 4977 | . ($args{select} ? $args{select} | ||
2164 | 4978 | : join(', ', map { $q->quote($_) } @{$asc->{cols}})) | ||
2165 | 4979 | . " FROM $from" | ||
2166 | 4980 | . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary | ||
2167 | 4981 | . " AND " . $asc->{boundaries}->{'<='} # upper boundary | ||
2168 | 4982 | . ($where ? " AND ($where)" : '') | ||
2169 | 4983 | . ($args{order_by} ? " ORDER BY $order_by" : "") | ||
2170 | 4984 | . " /*$comments{nibble}*/"; | ||
2171 | 4985 | PTDEBUG && _d('Nibble statement:', $nibble_sql); | ||
2172 | 4986 | |||
2173 | 4987 | my $explain_nibble_sql | ||
2174 | 4988 | = "EXPLAIN SELECT " | ||
2175 | 4989 | . ($args{select} ? $args{select} | ||
2176 | 4990 | : join(', ', map { $q->quote($_) } @{$asc->{cols}})) | ||
2177 | 4991 | . " FROM $from" | ||
2178 | 4992 | . " WHERE " . $asc->{boundaries}->{'>='} # lower boundary | ||
2179 | 4993 | . " AND " . $asc->{boundaries}->{'<='} # upper boundary | ||
2180 | 4994 | . ($where ? " AND ($where)" : '') | ||
2181 | 4995 | . ($args{order_by} ? " ORDER BY $order_by" : "") | ||
2182 | 4996 | . " /*explain $comments{nibble}*/"; | ||
2183 | 4997 | PTDEBUG && _d('Explain nibble statement:', $explain_nibble_sql); | ||
2184 | 4998 | |||
2185 | 4999 | my $limit = $chunk_size - 1; | ||
2186 | 5000 | PTDEBUG && _d('Initial chunk size (LIMIT):', $limit); | ||
2187 | 5001 | |||
2188 | 5002 | $self = { | ||
2189 | 5003 | %args, | ||
2190 | 5004 | index => $index, | ||
2191 | 5005 | limit => $limit, | ||
2192 | 5006 | first_lb_sql => $first_lb_sql, | ||
2193 | 5007 | last_ub_sql => $last_ub_sql, | ||
2194 | 5008 | ub_sql => $ub_sql, | ||
2195 | 5009 | nibble_sql => $nibble_sql, | ||
2196 | 5010 | explain_ub_sql => "EXPLAIN $ub_sql", | ||
2197 | 5011 | explain_nibble_sql => $explain_nibble_sql, | ||
2198 | 5012 | resume_lb_sql => $resume_lb_sql, | ||
2199 | 5013 | sql => { | ||
2200 | 5014 | columns => $asc->{scols}, | ||
2201 | 5015 | from => $from, | ||
2202 | 5016 | where => $where, | ||
2203 | 5017 | boundaries => $asc->{boundaries}, | ||
2204 | 5018 | order_by => $order_by, | ||
2205 | 5019 | }, | ||
2206 | 5020 | }; | ||
2207 | 5021 | } | ||
2208 | 5022 | |||
2209 | 5023 | $self->{row_est} = $nibble_params->{row_est}, | ||
2210 | 5024 | $self->{nibbleno} = 0; | ||
2211 | 5025 | $self->{have_rows} = 0; | ||
2212 | 5026 | $self->{rowno} = 0; | ||
2213 | 5027 | $self->{oktonibble} = 1; | ||
2214 | 5028 | |||
2215 | 5029 | return bless $self, $class; | ||
2216 | 5030 | } | ||
2217 | 5031 | |||
2218 | 5032 | sub next { | ||
2219 | 5033 | my ($self) = @_; | ||
2220 | 5034 | |||
2221 | 5035 | if ( !$self->{oktonibble} ) { | ||
2222 | 5036 | PTDEBUG && _d('Not ok to nibble'); | ||
2223 | 5037 | return; | ||
2224 | 5038 | } | ||
2225 | 5039 | |||
2226 | 5040 | my %callback_args = ( | ||
2227 | 5041 | Cxn => $self->{Cxn}, | ||
2228 | 5042 | tbl => $self->{tbl}, | ||
2229 | 5043 | NibbleIterator => $self, | ||
2230 | 5044 | ); | ||
2231 | 5045 | |||
2232 | 5046 | if ($self->{nibbleno} == 0) { | ||
2233 | 5047 | $self->_prepare_sths(); | ||
2234 | 5048 | $self->_get_bounds(); | ||
2235 | 5049 | if ( my $callback = $self->{callbacks}->{init} ) { | ||
2236 | 5050 | $self->{oktonibble} = $callback->(%callback_args); | ||
2237 | 5051 | PTDEBUG && _d('init callback returned', $self->{oktonibble}); | ||
2238 | 5052 | if ( !$self->{oktonibble} ) { | ||
2239 | 5053 | $self->{no_more_boundaries} = 1; | ||
2240 | 5054 | return; | ||
2241 | 5055 | } | ||
2242 | 5056 | } | ||
2243 | 5057 | } | ||
2244 | 5058 | |||
2245 | 5059 | NIBBLE: | ||
2246 | 5060 | while ( $self->{have_rows} || $self->_next_boundaries() ) { | ||
2247 | 5061 | if ( !$self->{have_rows} ) { | ||
2248 | 5062 | $self->{nibbleno}++; | ||
2249 | 5063 | PTDEBUG && _d($self->{nibble_sth}->{Statement}, 'params:', | ||
2250 | 5064 | join(', ', (@{$self->{lower}}, @{$self->{upper}}))); | ||
2251 | 5065 | if ( my $callback = $self->{callbacks}->{exec_nibble} ) { | ||
2252 | 5066 | $self->{have_rows} = $callback->(%callback_args); | ||
2253 | 5067 | } | ||
2254 | 5068 | else { | ||
2255 | 5069 | $self->{nibble_sth}->execute(@{$self->{lower}}, @{$self->{upper}}); | ||
2256 | 5070 | $self->{have_rows} = $self->{nibble_sth}->rows(); | ||
2257 | 5071 | } | ||
2258 | 5072 | PTDEBUG && _d($self->{have_rows}, 'rows in nibble', $self->{nibbleno}); | ||
2259 | 5073 | } | ||
2260 | 5074 | |||
2261 | 5075 | if ( $self->{have_rows} ) { | ||
2262 | 5076 | my $row = $self->{nibble_sth}->fetchrow_arrayref(); | ||
2263 | 5077 | if ( $row ) { | ||
2264 | 5078 | $self->{rowno}++; | ||
2265 | 5079 | PTDEBUG && _d('Row', $self->{rowno}, 'in nibble',$self->{nibbleno}); | ||
2266 | 5080 | return [ @$row ]; | ||
2267 | 5081 | } | ||
2268 | 5082 | } | ||
2269 | 5083 | |||
2270 | 5084 | PTDEBUG && _d('No rows in nibble or nibble skipped'); | ||
2271 | 5085 | if ( my $callback = $self->{callbacks}->{after_nibble} ) { | ||
2272 | 5086 | $callback->(%callback_args); | ||
2273 | 5087 | } | ||
2274 | 5088 | $self->{rowno} = 0; | ||
2275 | 5089 | $self->{have_rows} = 0; | ||
2276 | 5090 | } | ||
2277 | 5091 | |||
2278 | 5092 | PTDEBUG && _d('Done nibbling'); | ||
2279 | 5093 | if ( my $callback = $self->{callbacks}->{done} ) { | ||
2280 | 5094 | $callback->(%callback_args); | ||
2281 | 5095 | } | ||
2282 | 5096 | |||
2283 | 5097 | return; | ||
2284 | 5098 | } | ||
2285 | 5099 | |||
2286 | 5100 | sub nibble_number { | ||
2287 | 5101 | my ($self) = @_; | ||
2288 | 5102 | return $self->{nibbleno}; | ||
2289 | 5103 | } | ||
2290 | 5104 | |||
2291 | 5105 | sub set_nibble_number { | ||
2292 | 5106 | my ($self, $n) = @_; | ||
2293 | 5107 | die "I need a number" unless $n; | ||
2294 | 5108 | $self->{nibbleno} = $n; | ||
2295 | 5109 | PTDEBUG && _d('Set new nibble number:', $n); | ||
2296 | 5110 | return; | ||
2297 | 5111 | } | ||
2298 | 5112 | |||
2299 | 5113 | sub nibble_index { | ||
2300 | 5114 | my ($self) = @_; | ||
2301 | 5115 | return $self->{index}; | ||
2302 | 5116 | } | ||
2303 | 5117 | |||
2304 | 5118 | sub statements { | ||
2305 | 5119 | my ($self) = @_; | ||
2306 | 5120 | return { | ||
2307 | 5121 | nibble => $self->{nibble_sth}, | ||
2308 | 5122 | explain_nibble => $self->{explain_nibble_sth}, | ||
2309 | 5123 | upper_boundary => $self->{ub_sth}, | ||
2310 | 5124 | explain_upper_boundary => $self->{explain_ub_sth}, | ||
2311 | 5125 | } | ||
2312 | 5126 | } | ||
2313 | 5127 | |||
2314 | 5128 | sub boundaries { | ||
2315 | 5129 | my ($self) = @_; | ||
2316 | 5130 | return { | ||
2317 | 5131 | first_lower => $self->{first_lower}, | ||
2318 | 5132 | lower => $self->{lower}, | ||
2319 | 5133 | upper => $self->{upper}, | ||
2320 | 5134 | next_lower => $self->{next_lower}, | ||
2321 | 5135 | last_upper => $self->{last_upper}, | ||
2322 | 5136 | }; | ||
2323 | 5137 | } | ||
2324 | 5138 | |||
2325 | 5139 | sub set_boundary { | ||
2326 | 5140 | my ($self, $boundary, $values) = @_; | ||
2327 | 5141 | die "I need a boundary parameter" | ||
2328 | 5142 | unless $boundary; | ||
2329 | 5143 | die "Invalid boundary: $boundary" | ||
2330 | 5144 | unless $boundary =~ m/^(?:lower|upper|next_lower|last_upper)$/; | ||
2331 | 5145 | die "I need a values arrayref parameter" | ||
2332 | 5146 | unless $values && ref $values eq 'ARRAY'; | ||
2333 | 5147 | $self->{$boundary} = $values; | ||
2334 | 5148 | PTDEBUG && _d('Set new', $boundary, 'boundary:', Dumper($values)); | ||
2335 | 5149 | return; | ||
2336 | 5150 | } | ||
2337 | 5151 | |||
2338 | 5152 | sub one_nibble { | ||
2339 | 5153 | my ($self) = @_; | ||
2340 | 5154 | return $self->{one_nibble}; | ||
2341 | 5155 | } | ||
2342 | 5156 | |||
2343 | 5157 | sub chunk_size { | ||
2344 | 5158 | my ($self) = @_; | ||
2345 | 5159 | return $self->{limit} + 1; | ||
2346 | 5160 | } | ||
2347 | 5161 | |||
2348 | 5162 | sub set_chunk_size { | ||
2349 | 5163 | my ($self, $limit) = @_; | ||
2350 | 5164 | return if $self->{one_nibble}; | ||
2351 | 5165 | die "Chunk size must be > 0" unless $limit; | ||
2352 | 5166 | $self->{limit} = $limit - 1; | ||
2353 | 5167 | PTDEBUG && _d('Set new chunk size (LIMIT):', $limit); | ||
2354 | 5168 | return; | ||
2355 | 5169 | } | ||
2356 | 5170 | |||
2357 | 5171 | sub sql { | ||
2358 | 5172 | my ($self) = @_; | ||
2359 | 5173 | return $self->{sql}; | ||
2360 | 5174 | } | ||
2361 | 5175 | |||
2362 | 5176 | sub more_boundaries { | ||
2363 | 5177 | my ($self) = @_; | ||
2364 | 5178 | return !$self->{no_more_boundaries}; | ||
2365 | 5179 | } | ||
2366 | 5180 | |||
2367 | 5181 | sub row_estimate { | ||
2368 | 5182 | my ($self) = @_; | ||
2369 | 5183 | return $self->{row_est}; | ||
2370 | 5184 | } | ||
2371 | 5185 | |||
2372 | 5186 | sub can_nibble { | ||
2373 | 5187 | my (%args) = @_; | ||
2374 | 5188 | my @required_args = qw(Cxn tbl chunk_size OptionParser TableParser); | ||
2375 | 5189 | foreach my $arg ( @required_args ) { | ||
2376 | 5190 | die "I need a $arg argument" unless $args{$arg}; | ||
2377 | 5191 | } | ||
2378 | 5192 | my ($cxn, $tbl, $chunk_size, $o) = @args{@required_args}; | ||
2379 | 5193 | |||
2380 | 5194 | my ($row_est, $mysql_index) = get_row_estimate( | ||
2381 | 5195 | Cxn => $cxn, | ||
2382 | 5196 | tbl => $tbl, | ||
2383 | 5197 | where => $o->has('where') ? $o->get('where') : '', | ||
2384 | 5198 | ); | ||
2385 | 5199 | |||
2386 | 5200 | my $one_nibble = !defined $args{one_nibble} || $args{one_nibble} | ||
2387 | 5201 | ? $row_est <= $chunk_size * $o->get('chunk-size-limit') | ||
2388 | 5202 | : 0; | ||
2389 | 5203 | PTDEBUG && _d('One nibble:', $one_nibble ? 'yes' : 'no'); | ||
2390 | 5204 | |||
2391 | 5205 | if ( $args{resume} | ||
2392 | 5206 | && !defined $args{resume}->{lower_boundary} | ||
2393 | 5207 | && !defined $args{resume}->{upper_boundary} ) { | ||
2394 | 5208 | PTDEBUG && _d('Resuming from one nibble table'); | ||
2395 | 5209 | $one_nibble = 1; | ||
2396 | 5210 | } | ||
2397 | 5211 | |||
2398 | 5212 | my $index = _find_best_index(%args, mysql_index => $mysql_index); | ||
2399 | 5213 | if ( !$index && !$one_nibble ) { | ||
2400 | 5214 | die "There is no good index and the table is oversized."; | ||
2401 | 5215 | } | ||
2402 | 5216 | |||
2403 | 5217 | return { | ||
2404 | 5218 | row_est => $row_est, # nibble about this many rows | ||
2405 | 5219 | index => $index, # using this index | ||
2406 | 5220 | one_nibble => $one_nibble, # if the table fits in one nibble/chunk | ||
2407 | 5221 | }; | ||
2408 | 5222 | } | ||
2409 | 5223 | |||
2410 | 5224 | sub _find_best_index { | ||
2411 | 5225 | my (%args) = @_; | ||
2412 | 5226 | my @required_args = qw(Cxn tbl TableParser); | ||
2413 | 5227 | my ($cxn, $tbl, $tp) = @args{@required_args}; | ||
2414 | 5228 | my $tbl_struct = $tbl->{tbl_struct}; | ||
2415 | 5229 | my $indexes = $tbl_struct->{keys}; | ||
2416 | 5230 | |||
2417 | 5231 | my $want_index = $args{chunk_index}; | ||
2418 | 5232 | if ( $want_index ) { | ||
2419 | 5233 | PTDEBUG && _d('User wants to use index', $want_index); | ||
2420 | 5234 | if ( !exists $indexes->{$want_index} ) { | ||
2421 | 5235 | PTDEBUG && _d('Cannot use user index because it does not exist'); | ||
2422 | 5236 | $want_index = undef; | ||
2423 | 5237 | } | ||
2424 | 5238 | } | ||
2425 | 5239 | |||
2426 | 5240 | if ( !$want_index && $args{mysql_index} ) { | ||
2427 | 5241 | PTDEBUG && _d('MySQL wants to use index', $args{mysql_index}); | ||
2428 | 5242 | $want_index = $args{mysql_index}; | ||
2429 | 5243 | } | ||
2430 | 5244 | |||
2431 | 5245 | my $best_index; | ||
2432 | 5246 | my @possible_indexes; | ||
2433 | 5247 | if ( $want_index ) { | ||
2434 | 5248 | if ( $indexes->{$want_index}->{is_unique} ) { | ||
2435 | 5249 | PTDEBUG && _d('Will use wanted index'); | ||
2436 | 5250 | $best_index = $want_index; | ||
2437 | 5251 | } | ||
2438 | 5252 | else { | ||
2439 | 5253 | PTDEBUG && _d('Wanted index is a possible index'); | ||
2440 | 5254 | push @possible_indexes, $want_index; | ||
2441 | 5255 | } | ||
2442 | 5256 | } | ||
2443 | 5257 | else { | ||
2444 | 5258 | PTDEBUG && _d('Auto-selecting best index'); | ||
2445 | 5259 | foreach my $index ( $tp->sort_indexes($tbl_struct) ) { | ||
2446 | 5260 | if ( $index eq 'PRIMARY' || $indexes->{$index}->{is_unique} ) { | ||
2447 | 5261 | $best_index = $index; | ||
2448 | 5262 | last; | ||
2449 | 5263 | } | ||
2450 | 5264 | else { | ||
2451 | 5265 | push @possible_indexes, $index; | ||
2452 | 5266 | } | ||
2453 | 5267 | } | ||
2454 | 5268 | } | ||
2455 | 5269 | |||
2456 | 5270 | if ( !$best_index && @possible_indexes ) { | ||
2457 | 5271 | PTDEBUG && _d('No PRIMARY or unique indexes;', | ||
2458 | 5272 | 'will use index with highest cardinality'); | ||
2459 | 5273 | foreach my $index ( @possible_indexes ) { | ||
2460 | 5274 | $indexes->{$index}->{cardinality} = _get_index_cardinality( | ||
2461 | 5275 | %args, | ||
2462 | 5276 | index => $index, | ||
2463 | 5277 | ); | ||
2464 | 5278 | } | ||
2465 | 5279 | @possible_indexes = sort { | ||
2466 | 5280 | my $cmp | ||
2467 | 5281 | = $indexes->{$b}->{cardinality} <=> $indexes->{$b}->{cardinality}; | ||
2468 | 5282 | if ( $cmp == 0 ) { | ||
2469 | 5283 | $cmp = scalar @{$indexes->{$b}->{cols}} | ||
2470 | 5284 | <=> scalar @{$indexes->{$a}->{cols}}; | ||
2471 | 5285 | } | ||
2472 | 5286 | $cmp; | ||
2473 | 5287 | } @possible_indexes; | ||
2474 | 5288 | $best_index = $possible_indexes[0]; | ||
2475 | 5289 | } | ||
2476 | 5290 | |||
2477 | 5291 | PTDEBUG && _d('Best index:', $best_index); | ||
2478 | 5292 | return $best_index; | ||
2479 | 5293 | } | ||
2480 | 5294 | |||
2481 | 5295 | sub _get_index_cardinality { | ||
2482 | 5296 | my (%args) = @_; | ||
2483 | 5297 | my @required_args = qw(Cxn tbl index); | ||
2484 | 5298 | my ($cxn, $tbl, $index) = @args{@required_args}; | ||
2485 | 5299 | |||
2486 | 5300 | my $sql = "SHOW INDEXES FROM $tbl->{name} " | ||
2487 | 5301 | . "WHERE Key_name = '$index'"; | ||
2488 | 5302 | PTDEBUG && _d($sql); | ||
2489 | 5303 | my $cardinality = 1; | ||
2490 | 5304 | my $rows = $cxn->dbh()->selectall_hashref($sql, 'key_name'); | ||
2491 | 5305 | foreach my $row ( values %$rows ) { | ||
2492 | 5306 | $cardinality *= $row->{cardinality} if $row->{cardinality}; | ||
2493 | 5307 | } | ||
2494 | 5308 | PTDEBUG && _d('Index', $index, 'cardinality:', $cardinality); | ||
2495 | 5309 | return $cardinality; | ||
2496 | 5310 | } | ||
2497 | 5311 | |||
2498 | 5312 | sub get_row_estimate { | ||
2499 | 5313 | my (%args) = @_; | ||
2500 | 5314 | my @required_args = qw(Cxn tbl); | ||
2501 | 5315 | foreach my $arg ( @required_args ) { | ||
2502 | 5316 | die "I need a $arg argument" unless $args{$arg}; | ||
2503 | 5317 | } | ||
2504 | 5318 | my ($cxn, $tbl) = @args{@required_args}; | ||
2505 | 5319 | |||
2506 | 5320 | if ( !$args{where} && exists $tbl->{tbl_status} ) { | ||
2507 | 5321 | PTDEBUG && _d('Using table status for row estimate'); | ||
2508 | 5322 | return $tbl->{tbl_status}->{rows} || 0; | ||
2509 | 5323 | } | ||
2510 | 5324 | else { | ||
2511 | 5325 | PTDEBUG && _d('Use EXPLAIN for row estimate'); | ||
2512 | 5326 | my $sql = "EXPLAIN SELECT * FROM $tbl->{name} " | ||
2513 | 5327 | . "WHERE " . ($args{where} || '1=1'); | ||
2514 | 5328 | PTDEBUG && _d($sql); | ||
2515 | 5329 | my $expl = $cxn->dbh()->selectrow_hashref($sql); | ||
2516 | 5330 | PTDEBUG && _d(Dumper($expl)); | ||
2517 | 5331 | return ($expl->{rows} || 0), $expl->{key}; | ||
2518 | 5332 | } | ||
2519 | 5333 | } | ||
2520 | 5334 | |||
2521 | 5335 | sub _prepare_sths { | ||
2522 | 5336 | my ($self) = @_; | ||
2523 | 5337 | PTDEBUG && _d('Preparing statement handles'); | ||
2524 | 5338 | |||
2525 | 5339 | my $dbh = $self->{Cxn}->dbh(); | ||
2526 | 5340 | |||
2527 | 5341 | $self->{nibble_sth} = $dbh->prepare($self->{nibble_sql}); | ||
2528 | 5342 | $self->{explain_nibble_sth} = $dbh->prepare($self->{explain_nibble_sql}); | ||
2529 | 5343 | |||
2530 | 5344 | if ( !$self->{one_nibble} ) { | ||
2531 | 5345 | $self->{ub_sth} = $dbh->prepare($self->{ub_sql}); | ||
2532 | 5346 | $self->{explain_ub_sth} = $dbh->prepare($self->{explain_ub_sql}); | ||
2533 | 5347 | } | ||
2534 | 5348 | |||
2535 | 5349 | return; | ||
2536 | 5350 | } | ||
2537 | 5351 | |||
2538 | 5352 | sub _get_bounds { | ||
2539 | 5353 | my ($self) = @_; | ||
2540 | 5354 | |||
2541 | 5355 | if ( $self->{one_nibble} ) { | ||
2542 | 5356 | if ( $self->{resume} ) { | ||
2543 | 5357 | $self->{no_more_boundaries} = 1; | ||
2544 | 5358 | } | ||
2545 | 5359 | return; | ||
2546 | 5360 | } | ||
2547 | 5361 | |||
2548 | 5362 | my $dbh = $self->{Cxn}->dbh(); | ||
2549 | 5363 | |||
2550 | 5364 | $self->{first_lower} = $dbh->selectrow_arrayref($self->{first_lb_sql}); | ||
2551 | 5365 | PTDEBUG && _d('First lower boundary:', Dumper($self->{first_lower})); | ||
2552 | 5366 | |||
2553 | 5367 | if ( my $nibble = $self->{resume} ) { | ||
2554 | 5368 | if ( defined $nibble->{lower_boundary} | ||
2555 | 5369 | && defined $nibble->{upper_boundary} ) { | ||
2556 | 5370 | my $sth = $dbh->prepare($self->{resume_lb_sql}); | ||
2557 | 5371 | my @ub = split ',', $nibble->{upper_boundary}; | ||
2558 | 5372 | PTDEBUG && _d($sth->{Statement}, 'params:', @ub); | ||
2559 | 5373 | $sth->execute(@ub); | ||
2560 | 5374 | $self->{next_lower} = $sth->fetchrow_arrayref(); | ||
2561 | 5375 | $sth->finish(); | ||
2562 | 5376 | } | ||
2563 | 5377 | } | ||
2564 | 5378 | else { | ||
2565 | 5379 | $self->{next_lower} = $self->{first_lower}; | ||
2566 | 5380 | } | ||
2567 | 5381 | PTDEBUG && _d('Next lower boundary:', Dumper($self->{next_lower})); | ||
2568 | 5382 | |||
2569 | 5383 | if ( !$self->{next_lower} ) { | ||
2570 | 5384 | PTDEBUG && _d('At end of table, or no more boundaries to resume'); | ||
2571 | 5385 | $self->{no_more_boundaries} = 1; | ||
2572 | 5386 | } | ||
2573 | 5387 | |||
2574 | 5388 | return; | ||
2575 | 5389 | } | ||
2576 | 5390 | |||
2577 | 5391 | sub _next_boundaries { | ||
2578 | 5392 | my ($self) = @_; | ||
2579 | 5393 | |||
2580 | 5394 | if ( $self->{no_more_boundaries} ) { | ||
2581 | 5395 | PTDEBUG && _d('No more boundaries'); | ||
2582 | 5396 | return; # stop nibbling | ||
2583 | 5397 | } | ||
2584 | 5398 | |||
2585 | 5399 | if ( $self->{one_nibble} ) { | ||
2586 | 5400 | $self->{lower} = $self->{upper} = []; | ||
2587 | 5401 | $self->{no_more_boundaries} = 1; # for next call | ||
2588 | 5402 | return 1; # continue nibbling | ||
2589 | 5403 | } | ||
2590 | 5404 | |||
2591 | 5405 | if ( $self->identical_boundaries($self->{lower}, $self->{next_lower}) ) { | ||
2592 | 5406 | PTDEBUG && _d('Infinite loop detected'); | ||
2593 | 5407 | my $tbl = $self->{tbl}; | ||
2594 | 5408 | my $index = $tbl->{tbl_struct}->{keys}->{$self->{index}}; | ||
2595 | 5409 | my $n_cols = scalar @{$index->{cols}}; | ||
2596 | 5410 | my $chunkno = $self->{nibbleno}; | ||
2597 | 5411 | die "Possible infinite loop detected! " | ||
2598 | 5412 | . "The lower boundary for chunk $chunkno is " | ||
2599 | 5413 | . "<" . join(', ', @{$self->{lower}}) . "> and the lower " | ||
2600 | 5414 | . "boundary for chunk " . ($chunkno + 1) . " is also " | ||
2601 | 5415 | . "<" . join(', ', @{$self->{next_lower}}) . ">. " | ||
2602 | 5416 | . "This usually happens when using a non-unique single " | ||
2603 | 5417 | . "column index. The current chunk index for table " | ||
2604 | 5418 | . "$tbl->{db}.$tbl->{tbl} is $self->{index} which is" | ||
2605 | 5419 | . ($index->{is_unique} ? '' : ' not') . " unique and covers " | ||
2606 | 5420 | . ($n_cols > 1 ? "$n_cols columns" : "1 column") . ".\n"; | ||
2607 | 5421 | } | ||
2608 | 5422 | $self->{lower} = $self->{next_lower}; | ||
2609 | 5423 | |||
2610 | 5424 | if ( my $callback = $self->{callbacks}->{next_boundaries} ) { | ||
2611 | 5425 | my $oktonibble = $callback->( | ||
2612 | 5426 | Cxn => $self->{Cxn}, | ||
2613 | 5427 | tbl => $self->{tbl}, | ||
2614 | 5428 | NibbleIterator => $self, | ||
2615 | 5429 | ); | ||
2616 | 5430 | PTDEBUG && _d('next_boundaries callback returned', $oktonibble); | ||
2617 | 5431 | if ( !$oktonibble ) { | ||
2618 | 5432 | $self->{no_more_boundaries} = 1; | ||
2619 | 5433 | return; # stop nibbling | ||
2620 | 5434 | } | ||
2621 | 5435 | } | ||
2622 | 5436 | |||
2623 | 5437 | PTDEBUG && _d($self->{ub_sth}->{Statement}, 'params:', | ||
2624 | 5438 | join(', ', @{$self->{lower}}), $self->{limit}); | ||
2625 | 5439 | $self->{ub_sth}->execute(@{$self->{lower}}, $self->{limit}); | ||
2626 | 5440 | my $boundary = $self->{ub_sth}->fetchall_arrayref(); | ||
2627 | 5441 | PTDEBUG && _d('Next boundary:', Dumper($boundary)); | ||
2628 | 5442 | if ( $boundary && @$boundary ) { | ||
2629 | 5443 | $self->{upper} = $boundary->[0]; | ||
2630 | 5444 | |||
2631 | 5445 | if ( $boundary->[1] ) { | ||
2632 | 5446 | $self->{next_lower} = $boundary->[1]; | ||
2633 | 5447 | } | ||
2634 | 5448 | else { | ||
2635 | 5449 | PTDEBUG && _d('End of table boundary:', Dumper($boundary->[0])); | ||
2636 | 5450 | $self->{no_more_boundaries} = 1; # for next call | ||
2637 | 5451 | |||
2638 | 5452 | $self->{last_upper} = $boundary->[0]; | ||
2639 | 5453 | } | ||
2640 | 5454 | } | ||
2641 | 5455 | else { | ||
2642 | 5456 | my $dbh = $self->{Cxn}->dbh(); | ||
2643 | 5457 | $self->{upper} = $dbh->selectrow_arrayref($self->{last_ub_sql}); | ||
2644 | 5458 | PTDEBUG && _d('Last upper boundary:', Dumper($self->{upper})); | ||
2645 | 5459 | $self->{no_more_boundaries} = 1; # for next call | ||
2646 | 5460 | |||
2647 | 5461 | $self->{last_upper} = $self->{upper}; | ||
2648 | 5462 | } | ||
2649 | 5463 | $self->{ub_sth}->finish(); | ||
2650 | 5464 | |||
2651 | 5465 | return 1; # continue nibbling | ||
2652 | 5466 | } | ||
2653 | 5467 | |||
2654 | 5468 | sub identical_boundaries { | ||
2655 | 5469 | my ($self, $b1, $b2) = @_; | ||
2656 | 5470 | |||
2657 | 5471 | return 0 if ($b1 && !$b2) || (!$b1 && $b2); | ||
2658 | 5472 | |||
2659 | 5473 | return 1 if !$b1 && !$b2; | ||
2660 | 5474 | |||
2661 | 5475 | die "Boundaries have different numbers of values" | ||
2662 | 5476 | if scalar @$b1 != scalar @$b2; # shouldn't happen | ||
2663 | 5477 | my $n_vals = scalar @$b1; | ||
2664 | 5478 | for my $i ( 0..($n_vals-1) ) { | ||
2665 | 5479 | return 0 if $b1->[$i] ne $b2->[$i]; # diff | ||
2666 | 5480 | } | ||
2667 | 5481 | return 1; | ||
2668 | 5482 | } | ||
2669 | 5483 | |||
2670 | 5484 | sub DESTROY { | ||
2671 | 5485 | my ( $self ) = @_; | ||
2672 | 5486 | foreach my $key ( keys %$self ) { | ||
2673 | 5487 | if ( $key =~ m/_sth$/ ) { | ||
2674 | 5488 | PTDEBUG && _d('Finish', $key); | ||
2675 | 5489 | $self->{$key}->finish(); | ||
2676 | 5490 | } | ||
2677 | 5491 | } | ||
2678 | 5492 | return; | ||
2679 | 5493 | } | ||
2680 | 5494 | |||
2681 | 5495 | sub _d { | ||
2682 | 5496 | my ($package, undef, $line) = caller 0; | ||
2683 | 5497 | @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } | ||
2684 | 5498 | map { defined $_ ? $_ : 'undef' } | ||
2685 | 5499 | @_; | ||
2686 | 5500 | print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; | ||
2687 | 5501 | } | ||
2688 | 5502 | |||
2689 | 5503 | 1; | ||
2690 | 5504 | } | ||
2691 | 5505 | # ########################################################################### | ||
2692 | 5506 | # End NibbleIterator package | ||
2693 | 5507 | # ########################################################################### | ||
2694 | 5508 | |||
2695 | 5509 | # ########################################################################### | ||
2696 | 5510 | # Transformers package | ||
2697 | 5511 | # This package is a copy without comments from the original. The original | ||
2698 | 5512 | # with comments and its test file can be found in the Bazaar repository at, | ||
2699 | 5513 | # lib/Transformers.pm | ||
2700 | 5514 | # t/lib/Transformers.t | ||
2701 | 5515 | # See https://launchpad.net/percona-toolkit for more information. | ||
2702 | 5516 | # ########################################################################### | ||
2703 | 5517 | { | ||
2704 | 5518 | package Transformers; | ||
2705 | 5519 | |||
2706 | 5520 | use strict; | ||
2707 | 5521 | use warnings FATAL => 'all'; | ||
2708 | 5522 | use English qw(-no_match_vars); | ||
2709 | 5523 | use constant PTDEBUG => $ENV{PTDEBUG} || 0; | ||
2710 | 5524 | |||
2711 | 5525 | use Time::Local qw(timegm timelocal); | ||
2712 | 5526 | use Digest::MD5 qw(md5_hex); | ||
2713 | 5527 | |||
2714 | 5528 | require Exporter; | ||
2715 | 5529 | our @ISA = qw(Exporter); | ||
2716 | 5530 | our %EXPORT_TAGS = (); | ||
2717 | 5531 | our @EXPORT = (); | ||
2718 | 5532 | our @EXPORT_OK = qw( | ||
2719 | 5533 | micro_t | ||
2720 | 5534 | percentage_of | ||
2721 | 5535 | secs_to_time | ||
2722 | 5536 | time_to_secs | ||
2723 | 5537 | shorten | ||
2724 | 5538 | ts | ||
2725 | 5539 | parse_timestamp | ||
2726 | 5540 | unix_timestamp | ||
2727 | 5541 | any_unix_timestamp | ||
2728 | 5542 | make_checksum | ||
2729 | 5543 | crc32 | ||
2730 | 5544 | ); | ||
2731 | 5545 | |||
2732 | 5546 | our $mysql_ts = qr/(\d\d)(\d\d)(\d\d) +(\d+):(\d+):(\d+)(\.\d+)?/; | ||
2733 | 5547 | our $proper_ts = qr/(\d\d\d\d)-(\d\d)-(\d\d)[T ](\d\d):(\d\d):(\d\d)(\.\d+)?/; | ||
2734 | 5548 | our $n_ts = qr/(\d{1,5})([shmd]?)/; # Limit \d{1,5} because \d{6} looks | ||
2735 | 5549 | |||
2736 | 5550 | sub micro_t { | ||
2737 | 5551 | my ( $t, %args ) = @_; | ||
2738 | 5552 | my $p_ms = defined $args{p_ms} ? $args{p_ms} : 0; # precision for ms vals | ||
2739 | 5553 | my $p_s = defined $args{p_s} ? $args{p_s} : 0; # precision for s vals | ||
2740 | 5554 | my $f; | ||
2741 | 5555 | |||
2742 | 5556 | $t = 0 if $t < 0; | ||
2743 | 5557 | |||
2744 | 5558 | $t = sprintf('%.17f', $t) if $t =~ /e/; | ||
2745 | 5559 | |||
2746 | 5560 | $t =~ s/\.(\d{1,6})\d*/\.$1/; | ||
2747 | 5561 | |||
2748 | 5562 | if ($t > 0 && $t <= 0.000999) { | ||
2749 | 5563 | $f = ($t * 1000000) . 'us'; | ||
2750 | 5564 | } | ||
2751 | 5565 | elsif ($t >= 0.001000 && $t <= 0.999999) { | ||
2752 | 5566 | $f = sprintf("%.${p_ms}f", $t * 1000); | ||
2753 | 5567 | $f = ($f * 1) . 'ms'; # * 1 to remove insignificant zeros | ||
2754 | 5568 | } | ||
2755 | 5569 | elsif ($t >= 1) { | ||
2756 | 5570 | $f = sprintf("%.${p_s}f", $t); | ||
2757 | 5571 | $f = ($f * 1) . 's'; # * 1 to remove insignificant zeros | ||
2758 | 5572 | } | ||
2759 | 5573 | else { | ||
2760 | 5574 | $f = 0; # $t should = 0 at this point | ||
2761 | 5575 | } | ||
2762 | 5576 | |||
2763 | 5577 | return $f; | ||
2764 | 5578 | } | ||
2765 | 5579 | |||
2766 | 5580 | sub percentage_of { | ||
2767 | 5581 | my ( $is, $of, %args ) = @_; | ||
2768 | 5582 | my $p = $args{p} || 0; # float precision | ||
2769 | 5583 | my $fmt = $p ? "%.${p}f" : "%d"; | ||
2770 | 5584 | return sprintf $fmt, ($is * 100) / ($of ||= 1); | ||
2771 | 5585 | } | ||
2772 | 5586 | |||
2773 | 5587 | sub secs_to_time { | ||
2774 | 5588 | my ( $secs, $fmt ) = @_; | ||
2775 | 5589 | $secs ||= 0; | ||
2776 | 5590 | return '00:00' unless $secs; | ||
2777 | 5591 | |||
2778 | 5592 | $fmt ||= $secs >= 86_400 ? 'd' | ||
2779 | 5593 | : $secs >= 3_600 ? 'h' | ||
2780 | 5594 | : 'm'; | ||
2781 | 5595 | |||
2782 | 5596 | return | ||
2783 | 5597 | $fmt eq 'd' ? sprintf( | ||
2784 | 5598 | "%d+%02d:%02d:%02d", | ||
2785 | 5599 | int($secs / 86_400), | ||
2786 | 5600 | int(($secs % 86_400) / 3_600), | ||
2787 | 5601 | int(($secs % 3_600) / 60), | ||
2788 | 5602 | $secs % 60) | ||
2789 | 5603 | : $fmt eq 'h' ? sprintf( | ||
2790 | 5604 | "%02d:%02d:%02d", | ||
2791 | 5605 | int(($secs % 86_400) / 3_600), | ||
2792 | 5606 | int(($secs % 3_600) / 60), | ||
2793 | 5607 | $secs % 60) | ||
2794 | 5608 | : sprintf( | ||
2795 | 5609 | "%02d:%02d", | ||
2796 | 5610 | int(($secs % 3_600) / 60), | ||
2797 | 5611 | $secs % 60); | ||
2798 | 5612 | } | ||
2799 | 5613 | |||
2800 | 5614 | sub time_to_secs { | ||
2801 | 5615 | my ( $val, $default_suffix ) = @_; | ||
2802 | 5616 | die "I need a val argument" unless defined $val; | ||
2803 | 5617 | my $t = 0; | ||
2804 | 5618 | my ( $prefix, $num, $suffix ) = $val =~ m/([+-]?)(\d+)([a-z])?$/; | ||
2805 | 5619 | $suffix = $suffix || $default_suffix || 's'; | ||
2806 | 5620 | if ( $suffix =~ m/[smhd]/ ) { | ||
2807 | 5621 | $t = $suffix eq 's' ? $num * 1 # Seconds | ||
2808 | 5622 | : $suffix eq 'm' ? $num * 60 # Minutes | ||
2809 | 5623 | : $suffix eq 'h' ? $num * 3600 # Hours | ||
2810 | 5624 | : $num * 86400; # Days | ||
2811 | 5625 | |||
2812 | 5626 | $t *= -1 if $prefix && $prefix eq '-'; | ||
2813 | 5627 | } | ||
2814 | 5628 | else { | ||
2815 | 5629 | die "Invalid suffix for $val: $suffix"; | ||
2816 | 5630 | } | ||
2817 | 5631 | return $t; | ||
2818 | 5632 | } | ||
2819 | 5633 | |||
2820 | 5634 | sub shorten { | ||
2821 | 5635 | my ( $num, %args ) = @_; | ||
2822 | 5636 | my $p = defined $args{p} ? $args{p} : 2; # float precision | ||
2823 | 5637 | my $d = defined $args{d} ? $args{d} : 1_024; # divisor | ||
2824 | 5638 | my $n = 0; | ||
2825 | 5639 | my @units = ('', qw(k M G T P E Z Y)); | ||
2826 | 5640 | while ( $num >= $d && $n < @units - 1 ) { | ||
2827 | 5641 | $num /= $d; | ||
2828 | 5642 | ++$n; | ||
2829 | 5643 | } | ||
2830 | 5644 | return sprintf( | ||
2831 | 5645 | $num =~ m/\./ || $n | ||
2832 | 5646 | ? "%.${p}f%s" | ||
2833 | 5647 | : '%d', | ||
2834 | 5648 | $num, $units[$n]); | ||
2835 | 5649 | } | ||
2836 | 5650 | |||
2837 | 5651 | sub ts { | ||
2838 | 5652 | my ( $time, $gmt ) = @_; | ||
2839 | 5653 | my ( $sec, $min, $hour, $mday, $mon, $year ) | ||
2840 | 5654 | = $gmt ? gmtime($time) : localtime($time); | ||
2841 | 5655 | $mon += 1; | ||
2842 | 5656 | $year += 1900; | ||
2843 | 5657 | my $val = sprintf("%d-%02d-%02dT%02d:%02d:%02d", | ||
2844 | 5658 | $year, $mon, $mday, $hour, $min, $sec); | ||
2845 | 5659 | if ( my ($us) = $time =~ m/(\.\d+)$/ ) { | ||
2846 | 5660 | $us = sprintf("%.6f", $us); | ||
2847 | 5661 | $us =~ s/^0\././; | ||
2848 | 5662 | $val .= $us; | ||
2849 | 5663 | } | ||
2850 | 5664 | return $val; | ||
2851 | 5665 | } | ||
2852 | 5666 | |||
2853 | 5667 | sub parse_timestamp { | ||
2854 | 5668 | my ( $val ) = @_; | ||
2855 | 5669 | if ( my($y, $m, $d, $h, $i, $s, $f) | ||
2856 | 5670 | = $val =~ m/^$mysql_ts$/ ) | ||
2857 | 5671 | { | ||
2858 | 5672 | return sprintf "%d-%02d-%02d %02d:%02d:" | ||
2859 | 5673 | . (defined $f ? '%09.6f' : '%02d'), | ||
2860 | 5674 | $y + 2000, $m, $d, $h, $i, (defined $f ? $s + $f : $s); | ||
2861 | 5675 | } | ||
2862 | 5676 | return $val; | ||
2863 | 5677 | } | ||
2864 | 5678 | |||
2865 | 5679 | sub unix_timestamp { | ||
2866 | 5680 | my ( $val, $gmt ) = @_; | ||
2867 | 5681 | if ( my($y, $m, $d, $h, $i, $s, $us) = $val =~ m/^$proper_ts$/ ) { | ||
2868 | 5682 | $val = $gmt | ||
2869 | 5683 | ? timegm($s, $i, $h, $d, $m - 1, $y) | ||
2870 | 5684 | : timelocal($s, $i, $h, $d, $m - 1, $y); | ||
2871 | 5685 | if ( defined $us ) { | ||
2872 | 5686 | $us = sprintf('%.6f', $us); | ||
2873 | 5687 | $us =~ s/^0\././; | ||
2874 | 5688 | $val .= $us; | ||
2875 | 5689 | } | ||
2876 | 5690 | } | ||
2877 | 5691 | return $val; | ||
2878 | 5692 | } | ||
2879 | 5693 | |||
2880 | 5694 | sub any_unix_timestamp { | ||
2881 | 5695 | my ( $val, $callback ) = @_; | ||
2882 | 5696 | |||
2883 | 5697 | if ( my ($n, $suffix) = $val =~ m/^$n_ts$/ ) { | ||
2884 | 5698 | $n = $suffix eq 's' ? $n # Seconds | ||
2885 | 5699 | : $suffix eq 'm' ? $n * 60 # Minutes | ||
2886 | 5700 | : $suffix eq 'h' ? $n * 3600 # Hours | ||
2887 | 5701 | : $suffix eq 'd' ? $n * 86400 # Days | ||
2888 | 5702 | : $n; # default: Seconds | ||
2889 | 5703 | PTDEBUG && _d('ts is now - N[shmd]:', $n); | ||
2890 | 5704 | return time - $n; | ||
2891 | 5705 | } | ||
2892 | 5706 | elsif ( $val =~ m/^\d{9,}/ ) { | ||
2893 | 5707 | PTDEBUG && _d('ts is already a unix timestamp'); | ||
2894 | 5708 | return $val; | ||
2895 | 5709 | } | ||
2896 | 5710 | elsif ( my ($ymd, $hms) = $val =~ m/^(\d{6})(?:\s+(\d+:\d+:\d+))?/ ) { | ||
2897 | 5711 | PTDEBUG && _d('ts is MySQL slow log timestamp'); | ||
2898 | 5712 | $val .= ' 00:00:00' unless $hms; | ||
2899 | 5713 | return unix_timestamp(parse_timestamp($val)); | ||
2900 | 5714 | } | ||
2901 | 5715 | elsif ( ($ymd, $hms) = $val =~ m/^(\d{4}-\d\d-\d\d)(?:[T ](\d+:\d+:\d+))?/) { | ||
2902 | 5716 | PTDEBUG && _d('ts is properly formatted timestamp'); | ||
2903 | 5717 | $val .= ' 00:00:00' unless $hms; | ||
2904 | 5718 | return unix_timestamp($val); | ||
2905 | 5719 | } | ||
2906 | 5720 | else { | ||
2907 | 5721 | PTDEBUG && _d('ts is MySQL expression'); | ||
2908 | 5722 | return $callback->($val) if $callback && ref $callback eq 'CODE'; | ||
2909 | 5723 | } | ||
2910 | 5724 | |||
2911 | 5725 | PTDEBUG && _d('Unknown ts type:', $val); | ||
2912 | 5726 | return; | ||
2913 | 5727 | } | ||
2914 | 5728 | |||
2915 | 5729 | sub make_checksum { | ||
2916 | 5730 | my ( $val ) = @_; | ||
2917 | 5731 | my $checksum = uc substr(md5_hex($val), -16); | ||
2918 | 5732 | PTDEBUG && _d($checksum, 'checksum for', $val); | ||
2919 | 5733 | return $checksum; | ||
2920 | 5734 | } | ||
2921 | 5735 | |||
2922 | 5736 | sub crc32 { | ||
2923 | 5737 | my ( $string ) = @_; | ||
2924 | 5738 | return unless $string; | ||
2925 | 5739 | my $poly = 0xEDB88320; | ||
2926 | 5740 | my $crc = 0xFFFFFFFF; | ||
2927 | 5741 | foreach my $char ( split(//, $string) ) { | ||
2928 | 5742 | my $comp = ($crc ^ ord($char)) & 0xFF; | ||
2929 | 5743 | for ( 1 .. 8 ) { | ||
2930 | 5744 | $comp = $comp & 1 ? $poly ^ ($comp >> 1) : $comp >> 1; | ||
2931 | 5745 | } | ||
2932 | 5746 | $crc = (($crc >> 8) & 0x00FFFFFF) ^ $comp; | ||
2933 | 5747 | } | ||
2934 | 5748 | return $crc ^ 0xFFFFFFFF; | ||
2935 | 5749 | } | ||
2936 | 5750 | |||
2937 | 5751 | sub _d { | ||
2938 | 5752 | my ($package, undef, $line) = caller 0; | ||
2939 | 5753 | @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } | ||
2940 | 5754 | map { defined $_ ? $_ : 'undef' } | ||
2941 | 5755 | @_; | ||
2942 | 5756 | print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; | ||
2943 | 5757 | } | ||
2944 | 5758 | |||
2945 | 5759 | 1; | ||
2946 | 5760 | } | ||
2947 | 5761 | # ########################################################################### | ||
2948 | 5762 | # End Transformers package | ||
2949 | 5763 | # ########################################################################### | ||
2950 | 5764 | |||
2951 | 5765 | # ########################################################################### | ||
2952 | 5766 | # CleanupTask package | ||
2953 | 5767 | # This package is a copy without comments from the original. The original | ||
2954 | 5768 | # with comments and its test file can be found in the Bazaar repository at, | ||
2955 | 5769 | # lib/CleanupTask.pm | ||
2956 | 5770 | # t/lib/CleanupTask.t | ||
2957 | 5771 | # See https://launchpad.net/percona-toolkit for more information. | ||
2958 | 5772 | # ########################################################################### | ||
2959 | 5773 | { | ||
2960 | 5774 | package CleanupTask; | ||
2961 | 5775 | |||
2962 | 5776 | use strict; | ||
2963 | 5777 | use warnings FATAL => 'all'; | ||
2964 | 5778 | use English qw(-no_match_vars); | ||
2965 | 5779 | use constant PTDEBUG => $ENV{PTDEBUG} || 0; | ||
2966 | 5780 | |||
2967 | 5781 | sub new { | ||
2968 | 5782 | my ( $class, $task ) = @_; | ||
2969 | 5783 | die "I need a task parameter" unless $task; | ||
2970 | 5784 | die "The task parameter must be a coderef" unless ref $task eq 'CODE'; | ||
2971 | 5785 | my $self = { | ||
2972 | 5786 | task => $task, | ||
2973 | 5787 | }; | ||
2974 | 5788 | PTDEBUG && _d('Created cleanup task', $task); | ||
2975 | 5789 | return bless $self, $class; | ||
2976 | 5790 | } | ||
2977 | 5791 | |||
2978 | 5792 | sub DESTROY { | ||
2979 | 5793 | my ($self) = @_; | ||
2980 | 5794 | my $task = $self->{task}; | ||
2981 | 5795 | PTDEBUG && _d('Calling cleanup task', $task); | ||
2982 | 5796 | $task->(); | ||
2983 | 5797 | return; | ||
2984 | 5798 | } | ||
2985 | 5799 | |||
2986 | 5800 | sub _d { | ||
2987 | 5801 | my ($package, undef, $line) = caller 0; | ||
2988 | 5802 | @_ = map { (my $temp = $_) =~ s/\n/\n# /g; $temp; } | ||
2989 | 5803 | map { defined $_ ? $_ : 'undef' } | ||
2990 | 5804 | @_; | ||
2991 | 5805 | print STDERR "# $package:$line $PID ", join(' ', @_), "\n"; | ||
2992 | 5806 | } | ||
2993 | 5807 | |||
2994 | 5808 | 1; | ||
2995 | 5809 | } | ||
2996 | 5810 | # ########################################################################### | ||
2997 | 5811 | # End CleanupTask package | ||
2998 | 5812 | # ########################################################################### | ||
2999 | 5813 | |||
3000 | 5814 | # ########################################################################### | ||
3001 | 3863 | # This is a combination of modules and programs in one -- a runnable module. | 5815 | # This is a combination of modules and programs in one -- a runnable module. |
3002 | 3864 | # http://www.perl.com/pub/a/2006/07/13/lightning-articles.html?page=last | 5816 | # http://www.perl.com/pub/a/2006/07/13/lightning-articles.html?page=last |
3003 | 3865 | # Or, look it up in the Camel book on pages 642 and 643 in the 3rd edition. | 5817 | # Or, look it up in the Camel book on pages 642 and 643 in the 3rd edition. |
3004 | @@ -3869,29 +5821,37 @@ | |||
3005 | 3869 | # ########################################################################### | 5821 | # ########################################################################### |
3006 | 3870 | package pt_online_schema_change; | 5822 | package pt_online_schema_change; |
3007 | 3871 | 5823 | ||
3008 | 5824 | use strict; | ||
3009 | 5825 | use warnings FATAL => 'all'; | ||
3010 | 3872 | use English qw(-no_match_vars); | 5826 | use English qw(-no_match_vars); |
3012 | 3873 | use Time::HiRes qw(sleep); | 5827 | use constant PTDEBUG => $ENV{PTDEBUG} || 0; |
3013 | 5828 | |||
3014 | 5829 | use Time::HiRes qw(time); | ||
3015 | 3874 | use Data::Dumper; | 5830 | use Data::Dumper; |
3016 | 3875 | $Data::Dumper::Indent = 1; | 5831 | $Data::Dumper::Indent = 1; |
3017 | 3876 | $Data::Dumper::Sortkeys = 1; | 5832 | $Data::Dumper::Sortkeys = 1; |
3018 | 3877 | $Data::Dumper::Quotekeys = 0; | 5833 | $Data::Dumper::Quotekeys = 0; |
3019 | 3878 | 5834 | ||
3025 | 3879 | Transformers->import(qw(ts)); | 5835 | use sigtrap 'handler', \&sig_int, 'normal-signals'; |
3026 | 3880 | 5836 | ||
3027 | 3881 | use constant PTDEBUG => $ENV{PTDEBUG} || 0; | 5837 | my $exit_status = 0; |
3028 | 3882 | 5838 | my $oktorun = 1; | |
3029 | 3883 | my $quiet = 0; # for msg() | 5839 | my @drop_trigger_sqls; |
3030 | 5840 | |||
3031 | 5841 | $OUTPUT_AUTOFLUSH = 1; | ||
3032 | 3884 | 5842 | ||
3033 | 3885 | sub main { | 5843 | sub main { |
3039 | 3886 | @ARGV = @_; # set global ARGV for this package | 5844 | # Reset global vars else tests will fail. |
3040 | 3887 | my $vp = new VersionParser(); | 5845 | @ARGV = @_; |
3041 | 3888 | my $q = new Quoter(); | 5846 | $oktorun = 1; |
3042 | 3889 | my $tp = new TableParser(Quoter => $q); | 5847 | @drop_trigger_sqls = (); |
3043 | 3890 | my $chunker = new TableChunker(Quoter => $q, TableParser => $tp); | 5848 | |
3044 | 5849 | $exit_status = 0; | ||
3045 | 3891 | 5850 | ||
3046 | 3892 | # ######################################################################## | 5851 | # ######################################################################## |
3047 | 3893 | # Get configuration information. | 5852 | # Get configuration information. |
3048 | 3894 | # ######################################################################## | 5853 | # ######################################################################## |
3049 | 5854 | my $q = new Quoter(); | ||
3050 | 3895 | my $o = new OptionParser(); | 5855 | my $o = new OptionParser(); |
3051 | 3896 | $o->get_specs(); | 5856 | $o->get_specs(); |
3052 | 3897 | $o->get_opts(); | 5857 | $o->get_opts(); |
3053 | @@ -3899,41 +5859,36 @@ | |||
3054 | 3899 | my $dp = $o->DSNParser(); | 5859 | my $dp = $o->DSNParser(); |
3055 | 3900 | $dp->prop('set-vars', $o->get('set-vars')); | 5860 | $dp->prop('set-vars', $o->get('set-vars')); |
3056 | 3901 | 5861 | ||
3061 | 3902 | $quiet = $o->get('quiet'); # for msg() | 5862 | # The original table, i.e. the one being altered, must be specified |
3062 | 3903 | 5863 | # on the command line via the DSN. | |
3063 | 3904 | my ($dsn, $db, $tbl); | 5864 | my ($db, $tbl); |
3064 | 3905 | $dsn = shift @ARGV; | 5865 | my $dsn = shift @ARGV; |
3065 | 3906 | if ( !$dsn ) { | 5866 | if ( !$dsn ) { |
3067 | 3907 | $o->save_error('A DSN with a t part must be specified'); | 5867 | $o->save_error('A DSN must be specified'); |
3068 | 3908 | } | 5868 | } |
3069 | 3909 | else { | 5869 | else { |
3070 | 5870 | # Parse DSN string and convert it to a DSN data struct. | ||
3071 | 3910 | $dsn = $dp->parse($dsn, $dp->parse_options($o)); | 5871 | $dsn = $dp->parse($dsn, $dp->parse_options($o)); |
3085 | 3911 | if ( !$dsn->{t} ) { | 5872 | $db = $dsn->{D}; |
3086 | 3912 | $o->save_error('The DSN must specify a t (table) part'); | 5873 | $tbl = $dsn->{t}; |
3087 | 3913 | } | 5874 | } |
3088 | 3914 | else { | 5875 | |
3089 | 3915 | ($db, $tbl) = $q->split_unquote($dsn->{t} || "", $dsn->{D} || ""); | 5876 | my $alter_fk_method = $o->get('alter-foreign-keys-method') || ''; |
3090 | 3916 | } | 5877 | if ( $alter_fk_method eq 'drop_swap' ) { |
3091 | 3917 | } | 5878 | $o->set('swap-tables', 0); |
3092 | 3918 | 5879 | $o->set('drop-old-table', 0); | |
3093 | 3919 | my $rename_fk_method = lc($o->get('update-foreign-keys-method') || ''); | 5880 | } |
3094 | 3920 | if ( ($rename_fk_method || '') eq 'drop_old_table' ) { | 5881 | |
3095 | 3921 | $o->set('rename-tables', 0); | 5882 | # Explicit --chunk-size disable auto chunk sizing. |
3096 | 3922 | $o->set('drop-old-table', 0), | 5883 | $o->set('chunk-time', 0) if $o->got('chunk-size'); |
3084 | 3923 | } | ||
3097 | 3924 | 5884 | ||
3098 | 3925 | if ( !$o->get('help') ) { | 5885 | if ( !$o->get('help') ) { |
3099 | 3926 | if ( @ARGV ) { | 5886 | if ( @ARGV ) { |
3100 | 3927 | $o->save_error('Specify only one DSN on the command line'); | 5887 | $o->save_error('Specify only one DSN on the command line'); |
3101 | 3928 | } | 5888 | } |
3102 | 3929 | 5889 | ||
3110 | 3930 | if ( !$db ) { | 5890 | if ( !$db || !$tbl ) { |
3111 | 3931 | $o->save_error("No database was specified in the DSN or by " | 5891 | $o->save_error("The DSN must specify a database (D) and a table (t)"); |
3105 | 3932 | . "--database (-D)"); | ||
3106 | 3933 | } | ||
3107 | 3934 | |||
3108 | 3935 | if ( $tbl && $tbl eq ($o->get('tmp-table') || "") ) { | ||
3109 | 3936 | $o->save_error("--tmp-table cannot be the same as the table"); | ||
3112 | 3937 | } | 5892 | } |
3113 | 3938 | 5893 | ||
3114 | 3939 | if ( $o->get('progress') ) { | 5894 | if ( $o->get('progress') ) { |
3115 | @@ -3944,523 +5899,1755 @@ | |||
3116 | 3944 | } | 5899 | } |
3117 | 3945 | } | 5900 | } |
3118 | 3946 | 5901 | ||
3134 | 3947 | if ( $o->get('child-tables') && !$o->get('update-foreign-keys-method') ) { | 5902 | # See the "pod-based-option-value-validation" spec for how this may |
3135 | 3948 | $o->save_error("--child-tables requires --update-foreign-keys-method"); | 5903 | # be automagically validated. |
3136 | 3949 | } | 5904 | if ( $alter_fk_method |
3137 | 3950 | 5905 | && $alter_fk_method ne 'auto' | |
3138 | 3951 | if ( $rename_fk_method | 5906 | && $alter_fk_method ne 'rebuild_constraints' |
3139 | 3952 | && $rename_fk_method ne 'rebuild_constraints' | 5907 | && $alter_fk_method ne 'drop_swap' |
3140 | 3953 | && $rename_fk_method ne 'drop_old_table' ) { | 5908 | && $alter_fk_method ne 'none' ) |
3141 | 3954 | $o->save_error("Invalid --update-foreign-keys-method value"); | 5909 | { |
3142 | 3955 | } | 5910 | $o->save_error("Invalid --alter-foreign-keys-method value: $alter_fk_method"); |
3143 | 3956 | } | 5911 | } |
3144 | 3957 | 5912 | } | |
3145 | 3958 | $o->usage_or_errors(); | 5913 | |
3146 | 3959 | 5914 | $o->usage_or_errors(); | |
3147 | 3960 | msg("$PROGRAM_NAME started"); | 5915 | |
3148 | 3961 | my $exit_status = 0; | 5916 | if ( $o->get('quiet') ) { |
3149 | 5917 | # BARON: this will fail on Windows, where there is no /dev/null. I feel | ||
3150 | 5918 | # it's a hack, like ignoring a problem instead of fixing it somehow. We | ||
3151 | 5919 | # should take a look at the things that get printed in a "normal" | ||
3152 | 5920 | # non-quiet run, and "if !quiet" them, and then do some kind of Logger.pm | ||
3153 | 5921 | # or Messager.pm module for a future release. | ||
3154 | 5922 | close STDOUT; | ||
3155 | 5923 | open STDOUT, '>', '/dev/null' | ||
3156 | 5924 | or warn "Cannot reopen STDOUT to /dev/null: $OS_ERROR"; | ||
3157 | 5925 | } | ||
3158 | 3962 | 5926 | ||
3159 | 3963 | # ######################################################################## | 5927 | # ######################################################################## |
3160 | 3964 | # Connect to MySQL. | 5928 | # Connect to MySQL. |
3161 | 3965 | # ######################################################################## | 5929 | # ######################################################################## |
3165 | 3966 | my $dbh = get_cxn( | 5930 | my $set_on_connect = sub { |
3166 | 3967 | dsn => $dsn, | 5931 | my ($dbh) = @_; |
3167 | 3968 | DSNParser => $dp, | 5932 | |
3168 | 5933 | # See the same code in pt-table-checksum. | ||
3169 | 5934 | my $lock_wait_timeout = $o->get('lock-wait-timeout'); | ||
3170 | 5935 | my $set_lwt = "SET SESSION innodb_lock_wait_timeout=$lock_wait_timeout"; | ||
3171 | 5936 | PTDEBUG && _d($set_lwt); | ||
3172 | 5937 | eval { | ||
3173 | 5938 | $dbh->do($set_lwt); | ||
3174 | 5939 | }; | ||
3175 | 5940 | if ( $EVAL_ERROR ) { | ||
3176 | 5941 | PTDEBUG && _d($EVAL_ERROR); | ||
3177 | 5942 | # Get the server's current value. | ||
3178 | 5943 | my $sql = "SHOW SESSION VARIABLES LIKE 'innodb_lock_wait_timeout'"; | ||
3179 | 5944 | PTDEBUG && _d($dbh, $sql); | ||
3180 | 5945 | my (undef, $curr_lwt) = $dbh->selectrow_array($sql); | ||
3181 | 5946 | PTDEBUG && _d('innodb_lock_wait_timeout on server:', $curr_lwt); | ||
3182 | 5947 | if ( $curr_lwt > $lock_wait_timeout ) { | ||
3183 | 5948 | warn "Failed to $set_lwt: $EVAL_ERROR\n" | ||
3184 | 5949 | . "The current innodb_lock_wait_timeout value " | ||
3185 | 5950 | . "$curr_lwt is greater than the --lock-wait-timeout " | ||
3186 | 5951 | . "value $lock_wait_timeout and the variable cannot be " | ||
3187 | 5952 | . "changed. innodb_lock_wait_timeout is only dynamic when " | ||
3188 | 5953 | . "using the InnoDB plugin. To prevent this warning, either " | ||
3189 | 5954 | . "specify --lock-wait-time=$curr_lwt, or manually set " | ||
3190 | 5955 | . "innodb_lock_wait_timeout to a value less than or equal " | ||
3191 | 5956 | . "to $lock_wait_timeout and restart MySQL.\n"; | ||
3192 | 5957 | } | ||
3193 | 5958 | } | ||
3194 | 5959 | }; | ||
3195 | 5960 | |||
3196 | 5961 | # Do not call "new Cxn(" directly; use this sub so that set_on_connect | ||
3197 | 5962 | # is applied to every cxn. | ||
3198 | 5963 | # BARON: why not make this a subroutine instead of a subroutine variable? I | ||
3199 | 5964 | # think that can be less confusing. Also, the $set_on_connect variable can be | ||
3200 | 5965 | # inlined into this subroutine. Many of our tools have a get_dbh() subroutine | ||
3201 | 5966 | # and it might be good to just make a convention of it. | ||
3202 | 5967 | my $make_cxn = sub { | ||
3203 | 5968 | my (%args) = @_; | ||
3204 | 5969 | my $cxn = new Cxn( | ||
3205 | 5970 | %args, | ||
3206 | 5971 | DSNParser => $dp, | ||
3207 | 5972 | OptionParser => $o, | ||
3208 | 5973 | set => $set_on_connect, | ||
3209 | 5974 | ); | ||
3210 | 5975 | eval { $cxn->connect() }; # connect or die trying | ||
3211 | 5976 | if ( $EVAL_ERROR ) { | ||
3212 | 5977 | die "Cannot connect to MySQL: $EVAL_ERROR\n"; | ||
3213 | 5978 | } | ||
3214 | 5979 | return $cxn; | ||
3215 | 5980 | }; | ||
3216 | 5981 | |||
3217 | 5982 | my $cxn = $make_cxn->(dsn => $dsn); | ||
3218 | 5983 | |||
3219 | 5984 | # ######################################################################## | ||
3220 | 5985 | # Check if MySQL is new enough to have the triggers we need. | ||
3221 | 5986 | # Although triggers were introduced in 5.0.2, "Prior to MySQL 5.0.10, | ||
3222 | 5987 | # triggers cannot contain direct references to tables by name." | ||
3223 | 5988 | # ######################################################################## | ||
3224 | 5989 | my $vp = new VersionParser(); | ||
3225 | 5990 | if ( !$vp->version_ge($cxn->dbh(), '5.0.10') ) { | ||
3226 | 5991 | die "This tool requires MySQL 5.0.10 or newer.\n"; | ||
3227 | 5992 | } | ||
3228 | 5993 | |||
3229 | 5994 | # ######################################################################## | ||
3230 | 5995 | # Setup lag and load monitors. | ||
3231 | 5996 | # ######################################################################## | ||
3232 | 5997 | my $slaves; # all slaves that are found or specified | ||
3233 | 5998 | my $slave_lag_cxns; # slaves whose lag we'll check | ||
3234 | 5999 | my $replica_lag; # ReplicaLagWaiter object | ||
3235 | 6000 | my $replica_lag_pr; # Progress for ReplicaLagWaiter | ||
3236 | 6001 | my $sys_load; # MySQLStatusWaiter object | ||
3237 | 6002 | my $sys_load_pr; # Progress for MySQLStatusWaiter object | ||
3238 | 6003 | |||
3239 | 6004 | if ( $o->get('execute') ) { | ||
3240 | 6005 | # ##################################################################### | ||
3241 | 6006 | # Find and connect to slaves. | ||
3242 | 6007 | # ##################################################################### | ||
3243 | 6008 | my $ms = new MasterSlave(); | ||
3244 | 6009 | $slaves = $ms->get_slaves( | ||
3245 | 6010 | dbh => $cxn->dbh(), | ||
3246 | 6011 | dsn => $cxn->dsn(), | ||
3247 | 6012 | OptionParser => $o, | ||
3248 | 6013 | DSNParser => $dp, | ||
3249 | 6014 | Quoter => $q, | ||
3250 | 6015 | make_cxn => sub { | ||
3251 | 6016 | return $make_cxn->(@_, prev_dsn => $cxn->dsn()); | ||
3252 | 6017 | }, | ||
3253 | 6018 | ); | ||
3254 | 6019 | PTDEBUG && _d(scalar @$slaves, 'slaves found'); | ||
3255 | 6020 | |||
3256 | 6021 | if ( $o->get('check-slave-lag') ) { | ||
3257 | 6022 | PTDEBUG && _d('Will use --check-slave-lag to check for slave lag'); | ||
3258 | 6023 | my $cxn = $make_cxn->( | ||
3259 | 6024 | dsn_string => $o->get('check-slave-lag'), | ||
3260 | 6025 | prev_dsn => $cxn->dsn(), | ||
3261 | 6026 | ); | ||
3262 | 6027 | $slave_lag_cxns = [ $cxn ]; | ||
3263 | 6028 | } | ||
3264 | 6029 | else { | ||
3265 | 6030 | PTDEBUG && _d('Will check slave lag on all slaves'); | ||
3266 | 6031 | $slave_lag_cxns = $slaves; | ||
3267 | 6032 | } | ||
3268 | 6033 | |||
3269 | 6034 | # ##################################################################### | ||
3270 | 6035 | # Check for replication filters. | ||
3271 | 6036 | # ##################################################################### | ||
3272 | 6037 | if ( $o->get('check-replication-filters') ) { | ||
3273 | 6038 | PTDEBUG && _d("Checking slave replication filters"); | ||
3274 | 6039 | my @all_repl_filters; | ||
3275 | 6040 | foreach my $slave ( @$slaves ) { | ||
3276 | 6041 | my $repl_filters = $ms->get_replication_filters( | ||
3277 | 6042 | dbh => $slave->dbh(), | ||
3278 | 6043 | ); | ||
3279 | 6044 | if ( keys %$repl_filters ) { | ||
3280 | 6045 | push @all_repl_filters, | ||
3281 | 6046 | { name => $slave->name(), | ||
3282 | 6047 | filters => $repl_filters, | ||
3283 | 6048 | }; | ||
3284 | 6049 | } | ||
3285 | 6050 | } | ||
3286 | 6051 | if ( @all_repl_filters ) { | ||
3287 | 6052 | my $msg = "Replication filters are set on these hosts:\n"; | ||
3288 | 6053 | foreach my $host ( @all_repl_filters ) { | ||
3289 | 6054 | my $filters = $host->{filters}; | ||
3290 | 6055 | $msg .= " $host->{name}\n" | ||
3291 | 6056 | . join("\n", map { " $_ = $host->{filters}->{$_}" } | ||
3292 | 6057 | keys %{$host->{filters}}) | ||
3293 | 6058 | . "\n"; | ||
3294 | 6059 | } | ||
3295 | 6060 | $msg .= "Please read the --check-replication-filters documentation " | ||
3296 | 6061 | . "to learn how to solve this problem."; | ||
3297 | 6062 | die $msg; | ||
3298 | 6063 | } | ||
3299 | 6064 | } | ||
3300 | 6065 | |||
3301 | 6066 | # ##################################################################### | ||
3302 | 6067 | # Make a ReplicaLagWaiter to help wait for slaves after each chunk. | ||
3303 | 6068 | # ##################################################################### | ||
3304 | 6069 | my $sleep = sub { | ||
3305 | 6070 | # Don't let the master dbh die while waiting for slaves because we | ||
3306 | 6071 | # may wait a very long time for slaves. | ||
3307 | 6072 | my $dbh = $cxn->dbh(); | ||
3308 | 6073 | if ( !$dbh || !$dbh->ping() ) { | ||
3309 | 6074 | eval { $dbh = $cxn->connect() }; # connect or die trying | ||
3310 | 6075 | if ( $EVAL_ERROR ) { | ||
3311 | 6076 | $oktorun = 0; # flag for cleanup tasks | ||
3312 | 6077 | chomp $EVAL_ERROR; | ||
3313 | 6078 | die "Lost connection to " . $cxn->name() . " while waiting for " | ||
3314 | 6079 | . "replica lag ($EVAL_ERROR)\n"; | ||
3315 | 6080 | } | ||
3316 | 6081 | } | ||
3317 | 6082 | $dbh->do("SELECT 'pt-online-schema-change keepalive'"); | ||
3318 | 6083 | sleep $o->get('check-interval'); | ||
3319 | 6084 | return; | ||
3320 | 6085 | }; | ||
3321 | 6086 | |||
3322 | 6087 | my $get_lag = sub { | ||
3323 | 6088 | my ($cxn) = @_; | ||
3324 | 6089 | my $dbh = $cxn->dbh(); | ||
3325 | 6090 | if ( !$dbh || !$dbh->ping() ) { | ||
3326 | 6091 | eval { $dbh = $cxn->connect() }; # connect or die trying | ||
3327 | 6092 | if ( $EVAL_ERROR ) { | ||
3328 | 6093 | $oktorun = 0; # flag for cleanup tasks | ||
3329 | 6094 | chomp $EVAL_ERROR; | ||
3330 | 6095 | die "Lost connection to replica " . $cxn->name() | ||
3331 | 6096 | . " while attempting to get its lag ($EVAL_ERROR)\n"; | ||
3332 | 6097 | } | ||
3333 | 6098 | } | ||
3334 | 6099 | return $ms->get_slave_lag($dbh); | ||
3335 | 6100 | }; | ||
3336 | 6101 | |||
3337 | 6102 | $replica_lag = new ReplicaLagWaiter( | ||
3338 | 6103 | slaves => $slave_lag_cxns, | ||
3339 | 6104 | max_lag => $o->get('max-lag'), | ||
3340 | 6105 | oktorun => sub { return $oktorun }, | ||
3341 | 6106 | get_lag => $get_lag, | ||
3342 | 6107 | sleep => $sleep, | ||
3343 | 6108 | ); | ||
3344 | 6109 | |||
3345 | 6110 | my $get_status; | ||
3346 | 6111 | { | ||
3347 | 6112 | my $sql = "SHOW GLOBAL STATUS LIKE ?"; | ||
3348 | 6113 | my $sth = $cxn->dbh()->prepare($sql); | ||
3349 | 6114 | |||
3350 | 6115 | $get_status = sub { | ||
3351 | 6116 | my ($var) = @_; | ||
3352 | 6117 | PTDEBUG && _d($sth->{Statement}, $var); | ||
3353 | 6118 | $sth->execute($var); | ||
3354 | 6119 | my (undef, $val) = $sth->fetchrow_array(); | ||
3355 | 6120 | return $val; | ||
3356 | 6121 | }; | ||
3357 | 6122 | } | ||
3358 | 6123 | |||
3359 | 6124 | $sys_load = new MySQLStatusWaiter( | ||
3360 | 6125 | max_spec => $o->get('max-load'), | ||
3361 | 6126 | critical_spec => $o->get('critical-load'), | ||
3362 | 6127 | get_status => $get_status, | ||
3363 | 6128 | oktorun => sub { return $oktorun }, | ||
3364 | 6129 | sleep => $sleep, | ||
3365 | 6130 | ); | ||
3366 | 6131 | |||
3367 | 6132 | if ( $o->get('progress') ) { | ||
3368 | 6133 | $replica_lag_pr = new Progress( | ||
3369 | 6134 | jobsize => scalar @$slaves, | ||
3370 | 6135 | spec => $o->get('progress'), | ||
3371 | 6136 | name => "Waiting for replicas to catch up", # not used | ||
3372 | 6137 | ); | ||
3373 | 6138 | |||
3374 | 6139 | $sys_load_pr = new Progress( | ||
3375 | 6140 | jobsize => scalar @{$o->get('max-load')}, | ||
3376 | 6141 | spec => $o->get('progress'), | ||
3377 | 6142 | name => "Waiting for --max-load", # not used | ||
3378 | 6143 | ); | ||
3379 | 6144 | } | ||
3380 | 6145 | } | ||
3381 | 6146 | |||
3382 | 6147 | # ######################################################################## | ||
3383 | 6148 | # Setup and check the original table. | ||
3384 | 6149 | # ######################################################################## | ||
3385 | 6150 | my $tp = new TableParser(Quoter => $q); | ||
3386 | 6151 | |||
3387 | 6152 | # Common table data struct (that modules like NibbleIterator expect). | ||
3388 | 6153 | my $orig_tbl = { | ||
3389 | 6154 | db => $db, | ||
3390 | 6155 | tbl => $tbl, | ||
3391 | 6156 | name => $q->quote($db, $tbl), | ||
3392 | 6157 | }; | ||
3393 | 6158 | |||
3394 | 6159 | check_orig_table( | ||
3395 | 6160 | orig_tbl => $orig_tbl, | ||
3396 | 6161 | Cxn => $cxn, | ||
3397 | 3969 | OptionParser => $o, | 6162 | OptionParser => $o, |
3405 | 3970 | AutoCommit => 1, | 6163 | TableParser => $tp, |
3406 | 3971 | ); | 6164 | Quoter => $q, |
3407 | 3972 | msg("USE `$db`"); | 6165 | ); |
3408 | 3973 | $dbh->do("USE `$db`"); | 6166 | |
3409 | 3974 | 6167 | # ######################################################################## | |
3410 | 3975 | # ######################################################################## | 6168 | # Get child tables of the original table, if necessary. |
3411 | 3976 | # Daemonize only after (potentially) asking for passwords for --ask-pass. | 6169 | # ######################################################################## |
3412 | 6170 | my $child_tables = find_child_tables( | ||
3413 | 6171 | tbl => $orig_tbl, | ||
3414 | 6172 | Cxn => $cxn, | ||
3415 | 6173 | Quoter => $q, | ||
3416 | 6174 | ); | ||
3417 | 6175 | if ( !$child_tables ) { | ||
3418 | 6176 | if ( $alter_fk_method ) { | ||
3419 | 6177 | warn "No foreign keys reference $orig_tbl->{name}; ignoring " | ||
3420 | 6178 | . "--alter-foreign-keys-method.\n"; | ||
3421 | 6179 | |||
3422 | 6180 | if ( $alter_fk_method eq 'drop_swap' ) { | ||
3423 | 6181 | # These opts are disabled at the start if the user specifies | ||
3424 | 6182 | # the drop_swap method, but now that we know there are no | ||
3425 | 6183 | # child tables, we must re-enable these to make the alter work. | ||
3426 | 6184 | $o->set('swap-tables', 1); | ||
3427 | 6185 | $o->set('drop-old-table', 1); | ||
3428 | 6186 | } | ||
3429 | 6187 | |||
3430 | 6188 | $alter_fk_method = ''; | ||
3431 | 6189 | } | ||
3432 | 6190 | # No child tables and --alter-fk-method wasn't specified, | ||
3433 | 6191 | # so nothing to do. | ||
3434 | 6192 | } | ||
3435 | 6193 | else { | ||
3436 | 6194 | print "Child tables:\n"; | ||
3437 | 6195 | foreach my $child_table ( @$child_tables ) { | ||
3438 | 6196 | printf " %s (approx. %s rows)\n", | ||
3439 | 6197 | $child_table->{name}, | ||
3440 | 6198 | $child_table->{row_est} || '?'; | ||
3441 | 6199 | } | ||
3442 | 6200 | |||
3443 | 6201 | if ( $alter_fk_method ) { | ||
3444 | 6202 | # Let the user know how we're going to update the child table fk refs. | ||
3445 | 6203 | my $choice | ||
3446 | 6204 | = $alter_fk_method eq 'none' ? "not" | ||
3447 | 6205 | : $alter_fk_method eq 'auto' ? "automatically choose the method to" | ||
3448 | 6206 | : "use the $alter_fk_method method to"; | ||
3449 | 6207 | print "Will $choice update foreign keys.\n"; | ||
3450 | 6208 | } | ||
3451 | 6209 | else { | ||
3452 | 6210 | print "You did not specify --alter-foreign-keys-method, but there " | ||
3453 | 6211 | . "are foreign keys that reference the table. " | ||
3454 | 6212 | . "Please read the tool's documentation carefully.\n"; | ||
3455 | 6213 | return 1; | ||
3456 | 6214 | } | ||
3457 | 6215 | } | ||
3458 | 6216 | |||
3459 | 6217 | # ######################################################################## | ||
3460 | 6218 | # XXX | ||
3461 | 6219 | # Ready to begin the alter! Nothing has been changed on the server at | ||
3462 | 6220 | # this point; we've just checked and looked for things. Past this point, | ||
3463 | 6221 | # the code is live if --execute, else it's doing a --dry-run. Or, if | ||
3464 | 6222 | # the user didn't read the docs, we may bail out here. | ||
3465 | 6223 | # XXX | ||
3466 | 6224 | # ######################################################################## | ||
3467 | 6225 | if ( $o->get('dry-run') ) { | ||
3468 | 6226 | print "Starting a dry run. $orig_tbl->{name} will not be altered. " | ||
3469 | 6227 | . "Specify --execute instead of --dry-run to alter the table.\n"; | ||
3470 | 6228 | } | ||
3471 | 6229 | elsif ( $o->get('execute') ) { | ||
3472 | 6230 | print "Altering $orig_tbl->{name}...\n"; | ||
3473 | 6231 | } | ||
3474 | 6232 | else { | ||
3475 | 6233 | print "Exiting without altering $orig_tbl->{name} because neither " | ||
3476 | 6234 | . "--dry-run nor --execute was specified. Please read the tool's " | ||
3477 | 6235 | . "documentation carefully before using this tool.\n"; | ||
3478 | 6236 | return 1; | ||
3479 | 6237 | } | ||
3480 | 6238 | |||
3481 | 6239 | # ######################################################################## | ||
3482 | 6240 | # Create a cleanup task object to undo changes (i.e. clean up) if the | ||
3483 | 6241 | # code dies, or we may call this explicitly at the end if all goes well. | ||
3484 | 6242 | # ######################################################################## | ||
3485 | 6243 | my @cleanup_tasks; | ||
3486 | 6244 | my $cleanup = new CleanupTask( | ||
3487 | 6245 | sub { | ||
3488 | 6246 | # XXX We shouldn't copy $EVAL_ERROR here, but I found that | ||
3489 | 6247 | # errors are not re-thrown in tests. If you comment out this | ||
3490 | 6248 | # line and the die below, an error fails: | ||
3491 | 6249 | # not ok 5 - Doesn't try forever to find a new table name | ||
3492 | 6250 | # Failed test 'Doesn't try forever to find a new table name' | ||
3493 | 6251 | # at /Users/daniel/p/pt-osc-2.1.1/lib/PerconaTest.pm line 559. | ||
3494 | 6252 | # '' | ||
3495 | 6253 | # doesn't match '(?-xism:Failed to find a unique new table name)' | ||
3496 | 6254 | my $original_error = $EVAL_ERROR; | ||
3497 | 6255 | foreach my $task ( reverse @cleanup_tasks ) { | ||
3498 | 6256 | eval { | ||
3499 | 6257 | $task->(); | ||
3500 | 6258 | }; | ||
3501 | 6259 | if ( $EVAL_ERROR ) { | ||
3502 | 6260 | warn "Error cleaning up: $EVAL_ERROR\n"; | ||
3503 | 6261 | } | ||
3504 | 6262 | } | ||
3505 | 6263 | die $original_error if $original_error; # rethrow original error | ||
3506 | 6264 | return; | ||
3507 | 6265 | } | ||
3508 | 6266 | ); | ||
3509 | 6267 | |||
3510 | 6268 | # The last cleanup task is to report whether or not the orig table | ||
3511 | 6269 | # was altered. | ||
3512 | 6270 | push @cleanup_tasks, sub { | ||
3513 | 6271 | PTDEBUG && _d('Clean up done, report if orig table was altered'); | ||
3514 | 6272 | if ( $o->get('dry-run') ) { | ||
3515 | 6273 | print "Dry run complete. $orig_tbl->{name} was not altered.\n"; | ||
3516 | 6274 | } | ||
3517 | 6275 | else { | ||
3518 | 6276 | if ( $orig_tbl->{swapped} ) { | ||
3519 | 6277 | if ( $orig_tbl->{success} ) { | ||
3520 | 6278 | print "Successfully altered $orig_tbl->{name}.\n"; | ||
3521 | 6279 | } | ||
3522 | 6280 | else { | ||
3523 | 6281 | print "Altered $orig_tbl->{name} but there were errors " | ||
3524 | 6282 | . "or warnings.\n"; | ||
3525 | 6283 | } | ||
3526 | 6284 | } | ||
3527 | 6285 | else { | ||
3528 | 6286 | print "$orig_tbl->{name} was not altered.\n"; | ||
3529 | 6287 | } | ||
3530 | 6288 | } | ||
3531 | 6289 | return; | ||
3532 | 6290 | }; | ||
3533 | 6291 | |||
3534 | 6292 | # ######################################################################## | ||
3535 | 6293 | # Check and create PID file if user specified --pid. | ||
3536 | 3977 | # ######################################################################## | 6294 | # ######################################################################## |
3537 | 3978 | my $daemon; | 6295 | my $daemon; |
3539 | 3979 | if ( $o->get('pid') ) { | 6296 | if ( $o->get('execute') && $o->get('pid') ) { |
3540 | 3980 | # We're not daemoninzing, it just handles PID stuff. | 6297 | # We're not daemoninzing, it just handles PID stuff. |
3541 | 3981 | $daemon = new Daemon(o=>$o); | 6298 | $daemon = new Daemon(o=>$o); |
3542 | 3982 | $daemon->make_PID_file(); | 6299 | $daemon->make_PID_file(); |
3543 | 3983 | } | 6300 | } |
3544 | 3984 | 6301 | ||
3592 | 3985 | # ######################################################################## | 6302 | # ##################################################################### |
3593 | 3986 | # Setup/init some vars. | 6303 | # Step 1: Create the new table. |
3594 | 3987 | # ######################################################################## | 6304 | # ##################################################################### |
3595 | 3988 | my $tmp_tbl = $o->get('tmp-table') || "__tmp_$tbl"; | 6305 | my $new_tbl; |
3596 | 3989 | my $old_tbl = "__old_$tbl"; # what tbl becomes after swapped with tmp tbl | 6306 | eval { |
3597 | 3990 | my %tables = ( | 6307 | $new_tbl = create_new_table( |
3598 | 3991 | db => $db, | 6308 | orig_tbl => $orig_tbl, |
3599 | 3992 | tbl => $tbl, | 6309 | suffix => '_new', |
3600 | 3993 | tmp_tbl => $tmp_tbl, | 6310 | Cxn => $cxn, |
3601 | 3994 | old_tbl => $old_tbl, | 6311 | Quoter => $q, |
3602 | 3995 | ); | 6312 | OptionParser => $o, |
3603 | 3996 | msg("Alter table $tbl using temporary table $tmp_tbl"); | 6313 | TableParser => $tp, |
3604 | 3997 | 6314 | ); | |
3605 | 3998 | my %common_modules = ( | 6315 | }; |
3606 | 3999 | OptionParser => $o, | 6316 | if ( $EVAL_ERROR ) { |
3607 | 4000 | DSNParser => $dp, | 6317 | die "Error creating new table: $EVAL_ERROR\n"; |
3608 | 4001 | Quoter => $q, | 6318 | } |
3609 | 4002 | TableParser => $tp, | 6319 | |
3610 | 4003 | TableChunker => $chunker, | 6320 | # If the new table still exists, drop it unless the tool was interrupted. |
3611 | 4004 | VersionParser => $vp, | 6321 | push @cleanup_tasks, sub { |
3612 | 4005 | ); | 6322 | PTDEBUG && _('Clean up new table'); |
3613 | 4006 | 6323 | my $new_tbl_exists = $tp->check_table( | |
3614 | 4007 | # ######################################################################## | 6324 | dbh => $cxn->dbh(), |
3615 | 4008 | # Create the capture-sync and copy-rows plugins. Currently, we just have | 6325 | db => $new_tbl->{db}, |
3616 | 4009 | # one method for each. | 6326 | tbl => $new_tbl->{tbl}, |
3617 | 4010 | # ######################################################################## | 6327 | ); |
3618 | 4011 | my $capture_sync = new OSCCaptureSync(Quoter => $q); | 6328 | PTDEBUG && _d('New table exists:', $new_tbl_exists ? 'yes' : 'no'); |
3619 | 4012 | my $copy_rows = new CopyRowsInsertSelect( | 6329 | return unless $new_tbl_exists; |
3620 | 4013 | Retry => new Retry(), | 6330 | |
3621 | 4014 | Quoter => $q, | 6331 | my $sql = "DROP TABLE IF EXISTS $new_tbl->{name};"; |
3622 | 4015 | ); | 6332 | if ( !$oktorun ) { |
3623 | 4016 | 6333 | # The tool was interrupted, so do not drop the new table | |
3624 | 4017 | # More values are added later. These are the minimum need to do --cleanup. | 6334 | # in case the user wants to resume (once resume capability |
3625 | 4018 | my %plugin_args = ( | 6335 | # is implemented). |
3626 | 4019 | dbh => $dbh, | 6336 | print "Not dropping the new table $new_tbl->{name} because " |
3627 | 4020 | msg => \&msg, # so plugin can talk back to user | 6337 | . "the tool was interrupted. To drop the new table, " |
3628 | 4021 | print => $o->get('print'), | 6338 | . "execute:\n$sql\n"; |
3629 | 4022 | %tables, | 6339 | } |
3630 | 4023 | %common_modules, | 6340 | elsif ( $orig_tbl->{copied} && !$orig_tbl->{swapped} ) { |
3631 | 4024 | ); | 6341 | print "Not dropping the new table $new_tbl->{name} because " |
3632 | 4025 | 6342 | . "--swap-tables failed. To drop the new table, " | |
3633 | 4026 | if ( my $sleep_time = $o->get('sleep') ) { | 6343 | . "execute:\n$sql\n"; |
3634 | 4027 | PTDEBUG && _d("Sleep time:", $sleep_time); | 6344 | } |
3635 | 4028 | $plugin_args{sleep} = sub { | 6345 | else { |
3636 | 4029 | my ( $chunkno ) = @_; | 6346 | print "Dropping new table...\n"; |
3637 | 4030 | PTDEBUG && _d("Sleeping after chunk", $chunkno); | 6347 | print $sql, "\n" if $o->get('print'); |
3638 | 4031 | sleep($sleep_time); | 6348 | PTDEBUG && _d($sql); |
3639 | 6349 | eval { | ||
3640 | 6350 | $cxn->dbh()->do($sql); | ||
3641 | 6351 | }; | ||
3642 | 6352 | if ( $EVAL_ERROR ) { | ||
3643 | 6353 | warn "Error dropping new table $new_tbl->{name}: $EVAL_ERROR\n" | ||
3644 | 6354 | . "To try dropping the new table again, execute:\n$sql\n"; | ||
3645 | 6355 | } | ||
3646 | 6356 | print "Dropped new table OK.\n"; | ||
3647 | 6357 | } | ||
3648 | 6358 | }; | ||
3649 | 6359 | |||
3650 | 6360 | # ##################################################################### | ||
3651 | 6361 | # Step 2: Alter the new, empty table. This should be very quick, | ||
3652 | 6362 | # or die if the user specified a bad alter statement. | ||
3653 | 6363 | # ##################################################################### | ||
3654 | 6364 | if ( my $alter = $o->get('alter') ) { | ||
3655 | 6365 | print "Altering new table...\n"; | ||
3656 | 6366 | my $sql = "ALTER TABLE $new_tbl->{name} $alter"; | ||
3657 | 6367 | print $sql, "\n" if $o->get('print'); | ||
3658 | 6368 | PTDEBUG && _d($sql); | ||
3659 | 6369 | eval { | ||
3660 | 6370 | $cxn->dbh()->do($sql); | ||
3661 | 4032 | }; | 6371 | }; |
3682 | 4033 | } | 6372 | if ( $EVAL_ERROR ) { |
3683 | 4034 | 6373 | die "Error altering new table $new_tbl->{name}: $EVAL_ERROR\n" | |
3684 | 4035 | # ######################################################################## | 6374 | } |
3685 | 4036 | # Just cleanup and exit. | 6375 | print "Altered $new_tbl->{name} OK.\n" |
3686 | 4037 | # ######################################################################## | 6376 | } |
3687 | 4038 | if ( $o->get('cleanup-and-exit') ) { | 6377 | |
3688 | 4039 | msg("Calling " . (ref $copy_rows). "::cleanup()"); | 6378 | # Get the new table struct. This shouldn't die because |
3689 | 4040 | $copy_rows->cleanup(%plugin_args); | 6379 | # we just created the table successfully so we know it's |
3690 | 4041 | 6380 | # there. But the ghost of Ryan is everywhere. | |
3691 | 4042 | msg("Calling " . (ref $capture_sync) . "::cleanup()"); | 6381 | my $ddl = $tp->get_create_table( |
3692 | 4043 | $capture_sync->cleanup(%plugin_args); | 6382 | $cxn->dbh(), |
3693 | 4044 | 6383 | $new_tbl->{db}, | |
3694 | 4045 | msg("$PROGRAM_NAME ending for --cleanup-and-exit"); | 6384 | $new_tbl->{tbl}, |
3695 | 4046 | return 0; | 6385 | ); |
3696 | 4047 | } | 6386 | $new_tbl->{tbl_struct} = $tp->parse($ddl); |
3697 | 4048 | 6387 | ||
3698 | 4049 | # ######################################################################## | 6388 | # Determine what columns the original and new table share. |
3699 | 4050 | # Check that table can be altered. | 6389 | # If the user drops a col, that's easy: just don't copy it. If they |
3700 | 4051 | # ######################################################################## | 6390 | # add a column, it must have a default value. Other alterations |
3701 | 4052 | my %tbl_info; | 6391 | # may or may not affect the copy process--we'll know when we try! |
3702 | 6392 | # Note: we don't want to examine the --alter statement to see if the | ||
3703 | 6393 | # cols have changed because that's messy and prone to parsing errors. | ||
3704 | 6394 | # Col posn (position) is just for looks because user's like | ||
3705 | 6395 | # to see columns listed in their original order, not Perl's | ||
3706 | 6396 | # random hash key sorting. | ||
3707 | 6397 | my $col_posn = $orig_tbl->{tbl_struct}->{col_posn}; | ||
3708 | 6398 | my $orig_cols = $orig_tbl->{tbl_struct}->{is_col}; | ||
3709 | 6399 | my $new_cols = $new_tbl->{tbl_struct}->{is_col}; | ||
3710 | 6400 | my @common_cols = sort { $col_posn->{$a} <=> $col_posn->{$b} } | ||
3711 | 6401 | grep { $new_cols->{$_} } | ||
3712 | 6402 | keys %$orig_cols; | ||
3713 | 6403 | PTDEBUG && _d('Common columns', @common_cols); | ||
3714 | 6404 | |||
3715 | 6405 | # ######################################################################## | ||
3716 | 6406 | # Step 3: Create the triggers to capture changes on the original table and | ||
3717 | 6407 | # apply them to the new table. | ||
3718 | 6408 | # ######################################################################## | ||
3719 | 6409 | |||
3720 | 6410 | # Drop the triggers. We can save this cleanup task before | ||
3721 | 6411 | # adding the triggers because if adding them fails, this will be | ||
3722 | 6412 | # called which will drop whichever triggers were created. | ||
3723 | 6413 | push @cleanup_tasks, sub { | ||
3724 | 6414 | PTDEBUG && _d('Clean up triggers'); | ||
3725 | 6415 | if ( $oktorun ) { | ||
3726 | 6416 | drop_triggers( | ||
3727 | 6417 | tbl => $orig_tbl, | ||
3728 | 6418 | Cxn => $cxn, | ||
3729 | 6419 | Quoter => $q, | ||
3730 | 6420 | OptionParser => $o, | ||
3731 | 6421 | ); | ||
3732 | 6422 | } | ||
3733 | 6423 | else { | ||
3734 | 6424 | print "Not dropping triggers because the tool was interrupted. " | ||
3735 | 6425 | . "To drop the triggers, execute:\n" | ||
3736 | 6426 | . join("\n", @drop_trigger_sqls) . "\n"; | ||
3737 | 6427 | } | ||
3738 | 6428 | }; | ||
3739 | 6429 | |||
3740 | 4053 | eval { | 6430 | eval { |
3742 | 4054 | %tbl_info = check_tables(%plugin_args); | 6431 | create_triggers( |
3743 | 6432 | orig_tbl => $orig_tbl, | ||
3744 | 6433 | new_tbl => $new_tbl, | ||
3745 | 6434 | columns => \@common_cols, | ||
3746 | 6435 | Cxn => $cxn, | ||
3747 | 6436 | Quoter => $q, | ||
3748 | 6437 | OptionParser => $o, | ||
3749 | 6438 | ); | ||
3750 | 4055 | }; | 6439 | }; |
3751 | 4056 | if ( $EVAL_ERROR ) { | 6440 | if ( $EVAL_ERROR ) { |
3840 | 4057 | chomp $EVAL_ERROR; | 6441 | die "Error creating triggers: $EVAL_ERROR\n"; |
3841 | 4058 | msg("Table $tbl cannot be altered: $EVAL_ERROR"); | 6442 | }; |
3842 | 4059 | return 1; | 6443 | |
3843 | 4060 | } | 6444 | # ##################################################################### |
3844 | 4061 | 6445 | # Step 4: Copy rows. | |
3845 | 4062 | @plugin_args{keys %tbl_info} = values %tbl_info; | 6446 | # ##################################################################### |
3846 | 4063 | msg("Table $tbl can be altered"); | 6447 | |
3847 | 4064 | msg("Chunk column $plugin_args{chunk_column}, index $plugin_args{chunk_index}"); | 6448 | # The hashref of callbacks below is what NibbleIterator calls internally |
3848 | 4065 | 6449 | # to do all the copy work. The callbacks do not need to eval their work | |
3849 | 4066 | if ( $o->get('check-tables-and-exit') ) { | 6450 | # because the higher call to $nibble_iter->next() is eval'ed which will |
3850 | 4067 | msg("$PROGRAM_NAME ending for --check-tables-and-exit"); | 6451 | # catch any errors in the callbacks. |
3851 | 4068 | return 0; | 6452 | my $total_rows = 0; |
3852 | 4069 | } | 6453 | my $total_time = 0; |
3853 | 4070 | 6454 | my $avg_rate = 0; # rows/second | |
3854 | 4071 | # ##################################################################### | 6455 | my $retry = new Retry(); # for retrying to exec the copy statement |
3855 | 4072 | # Chunk the table. If the checks pass, then this shouldn't fail. | 6456 | my $limit = $o->get('chunk-size-limit'); # brevity |
3856 | 4073 | # ##################################################################### | 6457 | my $chunk_time = $o->get('chunk-time'); # brevity |
3857 | 4074 | my %range_stats = $chunker->get_range_statistics( | 6458 | |
3858 | 4075 | dbh => $dbh, | 6459 | my $callbacks = { |
3859 | 4076 | db => $db, | 6460 | init => sub { |
3860 | 4077 | tbl => $tbl, | 6461 | my (%args) = @_; |
3861 | 4078 | chunk_col => $plugin_args{chunk_column}, | 6462 | my $tbl = $args{tbl}; |
3862 | 4079 | tbl_struct => $plugin_args{tbl_struct}, | 6463 | my $nibble_iter = $args{NibbleIterator}; |
3863 | 4080 | ); | 6464 | |
3864 | 4081 | my @chunks = $chunker->calculate_chunks( | 6465 | if ( $o->get('dry-run') ) { |
3865 | 4082 | dbh => $dbh, | 6466 | print "Not copying rows because this is a dry run.\n"; |
3778 | 4083 | db => $db, | ||
3779 | 4084 | tbl => $tbl, | ||
3780 | 4085 | chunk_col => $plugin_args{chunk_column}, | ||
3781 | 4086 | tbl_struct => $plugin_args{tbl_struct}, | ||
3782 | 4087 | chunk_size => $o->get('chunk-size'), | ||
3783 | 4088 | %range_stats, | ||
3784 | 4089 | ); | ||
3785 | 4090 | $plugin_args{chunks} = \@chunks; | ||
3786 | 4091 | $plugin_args{Progress} = new Progress( | ||
3787 | 4092 | jobsize => scalar @chunks, | ||
3788 | 4093 | spec => $o->get('progress'), | ||
3789 | 4094 | name => "Copying rows", | ||
3790 | 4095 | ); | ||
3791 | 4096 | msg("Chunked table $tbl into " . scalar @chunks . " chunks"); | ||
3792 | 4097 | |||
3793 | 4098 | # ##################################################################### | ||
3794 | 4099 | # Get child tables if necessary. | ||
3795 | 4100 | # ##################################################################### | ||
3796 | 4101 | my @child_tables; | ||
3797 | 4102 | if ( my $child_tables = $o->get('child-tables') ) { | ||
3798 | 4103 | if ( lc $child_tables eq 'auto_detect' ) { | ||
3799 | 4104 | msg("Auto-detecting child tables of $tbl"); | ||
3800 | 4105 | @child_tables = get_child_tables(%plugin_args); | ||
3801 | 4106 | msg("Child tables of $tables{old_tbl}: " | ||
3802 | 4107 | . (@child_tables ? join(', ', @child_tables) : "(none)")); | ||
3803 | 4108 | } | ||
3804 | 4109 | else { | ||
3805 | 4110 | @child_tables = split(',', $child_tables); | ||
3806 | 4111 | msg("User-specified child tables: " . join(', ', @child_tables)); | ||
3807 | 4112 | } | ||
3808 | 4113 | } | ||
3809 | 4114 | |||
3810 | 4115 | # ##################################################################### | ||
3811 | 4116 | # Do the online alter. | ||
3812 | 4117 | # ##################################################################### | ||
3813 | 4118 | if ( !$o->get('execute') ) { | ||
3814 | 4119 | msg("Exiting without altering $db.$tbl because you did not " | ||
3815 | 4120 | . "specify --execute. Please read the tool's documentation " | ||
3816 | 4121 | . "carefully before using this tool."); | ||
3817 | 4122 | return $exit_status; | ||
3818 | 4123 | } | ||
3819 | 4124 | |||
3820 | 4125 | msg("Starting online schema change"); | ||
3821 | 4126 | eval { | ||
3822 | 4127 | my $sql = ""; | ||
3823 | 4128 | |||
3824 | 4129 | # ##################################################################### | ||
3825 | 4130 | # Create and alter the new table. | ||
3826 | 4131 | # ##################################################################### | ||
3827 | 4132 | if ( $o->get('create-tmp-table') ) { | ||
3828 | 4133 | $sql = "CREATE TABLE `$db`.`$tmp_tbl` LIKE `$db`.`$tbl`"; | ||
3829 | 4134 | msg($sql); | ||
3830 | 4135 | $dbh->do($sql) unless $o->get('print'); | ||
3831 | 4136 | } | ||
3832 | 4137 | |||
3833 | 4138 | if ( my $alter = $o->get('alter') ) { | ||
3834 | 4139 | my @stmts; | ||
3835 | 4140 | if ( -f $alter && -r $alter ) { | ||
3836 | 4141 | msg("Reading ALTER TABLE statements from file $alter"); | ||
3837 | 4142 | open my $fh, '<', $alter or die "Cannot open $alter: $OS_ERROR"; | ||
3838 | 4143 | @stmts = <$fh>; | ||
3839 | 4144 | close $fh; | ||
3866 | 4145 | } | 6467 | } |
3867 | 4146 | else { | 6468 | else { |
3875 | 4147 | @stmts = split(';', $alter); | 6469 | print "Copying approximately ", $nibble_iter->row_estimate(), |
3876 | 4148 | } | 6470 | " rows...\n"; |
3877 | 4149 | 6471 | } | |
3878 | 4150 | foreach my $stmt ( @stmts ) { | 6472 | |
3879 | 4151 | $sql = "ALTER TABLE `$db`.`$tmp_tbl` $stmt"; | 6473 | if ( $o->get('print') ) { |
3880 | 4152 | msg($sql); | 6474 | # Print the checksum and next boundary statements. |
3881 | 4153 | $dbh->do($sql) unless $o->get('print'); | 6475 | my $statements = $nibble_iter->statements(); |
3882 | 6476 | foreach my $sth ( sort keys %$statements ) { | ||
3883 | 6477 | next if $sth =~ m/^explain/; | ||
3884 | 6478 | if ( $statements->{$sth} ) { | ||
3885 | 6479 | print $statements->{$sth}->{Statement}, "\n"; | ||
3886 | 6480 | } | ||
3887 | 6481 | } | ||
3888 | 6482 | } | ||
3889 | 6483 | |||
3890 | 6484 | return unless $o->get('execute'); | ||
3891 | 6485 | |||
3892 | 6486 | # If table is a single chunk on the master, make sure it's also | ||
3893 | 6487 | # a single chunk on all slaves. E.g. if a slave is out of sync | ||
3894 | 6488 | # and has a lot more rows than the master, single chunking on the | ||
3895 | 6489 | # master could cause the slave to choke. | ||
3896 | 6490 | if ( $nibble_iter->one_nibble() ) { | ||
3897 | 6491 | PTDEBUG && _d('Getting table row estimate on replicas'); | ||
3898 | 6492 | my @too_large; | ||
3899 | 6493 | foreach my $slave ( @$slaves ) { | ||
3900 | 6494 | my ($n_rows) = NibbleIterator::get_row_estimate( | ||
3901 | 6495 | Cxn => $slave, | ||
3902 | 6496 | tbl => $tbl, | ||
3903 | 6497 | ); | ||
3904 | 6498 | PTDEBUG && _d('Table on',$slave->name(),'has', $n_rows, 'rows'); | ||
3905 | 6499 | if ( $n_rows && $n_rows > ($tbl->{chunk_size} * $limit) ) { | ||
3906 | 6500 | PTDEBUG && _d('Table too large on', $slave->name()); | ||
3907 | 6501 | push @too_large, [$slave->name(), $n_rows || 0]; | ||
3908 | 6502 | } | ||
3909 | 6503 | } | ||
3910 | 6504 | if ( @too_large ) { | ||
3911 | 6505 | my $msg | ||
3912 | 6506 | = "Cannot copy table $tbl->{name} because" | ||
3913 | 6507 | . " on the master it would be checksummed in one chunk" | ||
3914 | 6508 | . " but on these replicas it has too many rows:\n"; | ||
3915 | 6509 | foreach my $info ( @too_large ) { | ||
3916 | 6510 | $msg .= " $info->[1] rows on $info->[0]\n"; | ||
3917 | 6511 | } | ||
3918 | 6512 | $msg .= "The current chunk size limit is " | ||
3919 | 6513 | . ($tbl->{chunk_size} * $limit) | ||
3920 | 6514 | . " rows (chunk size=$tbl->{chunk_size}" | ||
3921 | 6515 | . " * chunk size limit=$limit).\n"; | ||
3922 | 6516 | die $msg; | ||
3923 | 6517 | } | ||
3924 | 6518 | } | ||
3925 | 6519 | |||
3926 | 6520 | return 1; # continue nibbling table | ||
3927 | 6521 | }, | ||
3928 | 6522 | next_boundaries => sub { | ||
3929 | 6523 | my (%args) = @_; | ||
3930 | 6524 | my $tbl = $args{tbl}; | ||
3931 | 6525 | my $nibble_iter = $args{NibbleIterator}; | ||
3932 | 6526 | my $sth = $nibble_iter->statements(); | ||
3933 | 6527 | my $boundary = $nibble_iter->boundaries(); | ||
3934 | 6528 | |||
3935 | 6529 | return 0 if $o->get('dry-run'); | ||
3936 | 6530 | return 1 if $nibble_iter->one_nibble(); | ||
3937 | 6531 | |||
3938 | 6532 | # Check that MySQL will use the nibble index for the next upper | ||
3939 | 6533 | # boundary sql. This check applies to the next nibble. So if | ||
3940 | 6534 | # the current nibble number is 5, then nibble 5 is already done | ||
3941 | 6535 | # and we're checking nibble number 6. | ||
3942 | 6536 | my $expl = explain_statement( | ||
3943 | 6537 | tbl => $tbl, | ||
3944 | 6538 | sth => $sth->{explain_upper_boundary}, | ||
3945 | 6539 | vals => [ @{$boundary->{lower}}, $nibble_iter->chunk_size() ], | ||
3946 | 6540 | ); | ||
3947 | 6541 | if (lc($expl->{key} || '') ne lc($nibble_iter->nibble_index() || '')) { | ||
3948 | 6542 | my $msg | ||
3949 | 6543 | = "Aborting copying table $tbl->{name} at chunk " | ||
3950 | 6544 | . ($nibble_iter->nibble_number() + 1) | ||
3951 | 6545 | . " because it is not safe to ascend. Chunking should " | ||
3952 | 6546 | . "use the " | ||
3953 | 6547 | . ($nibble_iter->nibble_index() || '?') | ||
3954 | 6548 | . " index, but MySQL EXPLAIN reports that " | ||
3955 | 6549 | . ($expl->{key} ? "the $expl->{key}" : "no") | ||
3956 | 6550 | . " index will be used for " | ||
3957 | 6551 | . $sth->{upper_boundary}->{Statement} | ||
3958 | 6552 | . " with values " | ||
3959 | 6553 | . join(", ", map { defined $_ ? $_ : "NULL" } | ||
3960 | 6554 | (@{$boundary->{lower}}, $nibble_iter->chunk_size())) | ||
3961 | 6555 | . "\n"; | ||
3962 | 6556 | die $msg; | ||
3963 | 4154 | } | 6557 | } |
4049 | 4155 | } | 6558 | |
4050 | 4156 | 6559 | # Once nibbling begins for a table, control does not return to this | |
4051 | 4157 | # ##################################################################### | 6560 | # tool until nibbling is done because, as noted above, all work is |
4052 | 4158 | # Determine what columns the two tables have in common. | 6561 | # done in these callbacks. This callback is the only place where we |
4053 | 4159 | # ##################################################################### | 6562 | # can prematurely stop nibbling by returning false. This allows |
4054 | 4160 | my @columns; | 6563 | # Ctrl-C to stop the tool between nibbles instead of between tables. |
4055 | 4161 | # If --print is in effect, then chances are the new table wasn't | 6564 | return $oktorun; # continue nibbling table? |
4056 | 4162 | # created above, so we can't get it's struct. | 6565 | }, |
4057 | 4163 | # TODO: check if the new table exists because user might have created | 6566 | exec_nibble => sub { |
4058 | 4164 | # it manually. | 6567 | my (%args) = @_; |
4059 | 4165 | if ( !$o->get('print') ) { | 6568 | my $tbl = $args{tbl}; |
4060 | 4166 | my $tmp_tbl_struct = $tp->parse( | 6569 | my $nibble_iter = $args{NibbleIterator}; |
4061 | 4167 | $tp->get_create_table($dbh, $db, $tmp_tbl)); | 6570 | my $sth = $nibble_iter->statements(); |
4062 | 4168 | 6571 | my $boundary = $nibble_iter->boundaries(); | |
4063 | 4169 | @columns = intersection([ | 6572 | |
4064 | 4170 | $plugin_args{tbl_struct}->{is_col}, | 6573 | return if $o->get('dry-run'); |
4065 | 4171 | $tmp_tbl_struct->{is_col}, | 6574 | |
4066 | 4172 | ]); | 6575 | # Count every chunk, even if it's ultimately skipped, etc. |
4067 | 4173 | 6576 | $tbl->{results}->{n_chunks}++; | |
4068 | 4174 | # Order columns according to new table because people like/expect | 6577 | |
4069 | 4175 | # to see things in a certain order (this has been an issue before). | 6578 | # If the table is being chunk (i.e., it's not small enough to be |
4070 | 4176 | # This just matters to us; does't make a difference to MySQL. | 6579 | # consumed by one nibble), then check index usage and chunk size. |
4071 | 4177 | my $col_posn = $plugin_args{tbl_struct}->{col_posn}; | 6580 | if ( !$nibble_iter->one_nibble() ) { |
4072 | 4178 | @columns = sort { $col_posn->{$a} <=> $col_posn->{$b} } @columns; | 6581 | my $expl = explain_statement( |
4073 | 4179 | msg("Shared columns: " . join(', ', @columns)); | 6582 | tbl => $tbl, |
4074 | 4180 | } | 6583 | sth => $sth->{explain_nibble}, |
4075 | 4181 | $plugin_args{columns} = \@columns; | 6584 | vals => [ @{$boundary->{lower}}, @{$boundary->{upper}} ], |
4076 | 4182 | 6585 | ); | |
4077 | 4183 | # ##################################################################### | 6586 | |
4078 | 4184 | # Start capturing changes to the new table. | 6587 | # Ensure that MySQL is using the chunk index. |
4079 | 4185 | # ##################################################################### | 6588 | if ( lc($expl->{key} || '') |
4080 | 4186 | msg("Calling " . (ref $capture_sync) . "::capture()"); | 6589 | ne lc($nibble_iter->nibble_index() || '') ) { |
4081 | 4187 | $capture_sync->capture(%plugin_args); | 6590 | my $msg |
4082 | 4188 | 6591 | = "Aborting copying table $tbl->{name} at chunk " | |
4083 | 4189 | # ##################################################################### | 6592 | . $nibble_iter->nibble_number() |
4084 | 4190 | # Copy rows from new table to old table. | 6593 | . " because it is not safe to chunk. Chunking should " |
4085 | 4191 | # ##################################################################### | 6594 | . "use the " |
4086 | 4192 | msg("Calling " . (ref $copy_rows) . "::copy()"); | 6595 | . ($nibble_iter->nibble_index() || '?') |
4087 | 4193 | $copy_rows->copy( | 6596 | . " index, but MySQL EXPLAIN reports that " |
4088 | 4194 | from_table => $q->quote($db, $tbl), | 6597 | . ($expl->{key} ? "the $expl->{key}" : "no") |
4089 | 4195 | to_table => $q->quote($db, $tmp_tbl), | 6598 | . " index will be used for " |
4090 | 4196 | %plugin_args | 6599 | . $sth->{explain_nibble}->{Statement} |
4091 | 4197 | ); | 6600 | . " with values " |
4092 | 4198 | 6601 | . join(", ", map { defined $_ ? $_ : "NULL" } | |
4093 | 4199 | # ##################################################################### | 6602 | (@{$boundary->{lower}}, @{$boundary->{upper}})) |
4094 | 4200 | # Sync tables. | 6603 | . "\n"; |
4095 | 4201 | # ##################################################################### | 6604 | die $msg; |
4096 | 4202 | msg("Calling " . (ref $capture_sync) . "::sync()"); | 6605 | } |
4097 | 4203 | $capture_sync->sync(%plugin_args); | 6606 | |
4098 | 4204 | 6607 | # Check chunk size limit if the upper boundary and next lower | |
4099 | 4205 | # ##################################################################### | 6608 | # boundary are identical. |
4100 | 4206 | # Rename tables. | 6609 | if ( $limit ) { |
4101 | 4207 | # ##################################################################### | 6610 | my $boundary = $nibble_iter->boundaries(); |
4102 | 4208 | if ( $o->get('rename-tables') ) { | 6611 | my $oversize_chunk |
4103 | 4209 | msg("Renaming tables"); | 6612 | = $limit ? ($expl->{rows} || 0) >= $tbl->{chunk_size} * $limit |
4104 | 4210 | $sql = "RENAME TABLE `$db`.`$tbl` TO `$db`.`$old_tbl`," | 6613 | : 0; |
4105 | 4211 | . " `$db`.`$tmp_tbl` TO `$db`.`$tbl`"; | 6614 | if ( $oversize_chunk |
4106 | 4212 | msg($sql); | 6615 | && $nibble_iter->identical_boundaries( |
4107 | 4213 | $dbh->do($sql) unless $o->get('print'); | 6616 | $boundary->{upper}, $boundary->{next_lower}) ) |
4108 | 4214 | msg("Original table $tbl renamed to $old_tbl"); | 6617 | { |
4109 | 4215 | } | 6618 | my $msg |
4110 | 4216 | 6619 | = "Aborting copying table $tbl->{name} at chunk " | |
4111 | 4217 | # ##################################################################### | 6620 | . $nibble_iter->nibble_number() |
4112 | 4218 | # Update foreign key constraints if there are child tables. | 6621 | . " because the chunk is too large: MySQL estimates " |
4113 | 4219 | # ##################################################################### | 6622 | . ($expl->{rows} || 0) . "rows. The current chunk " |
4114 | 4220 | if ( @child_tables ) { | 6623 | . "size limit is " . ($tbl->{chunk_size} * $limit) |
4115 | 4221 | msg("Renaming foreign key constraints in child table"); | 6624 | . " rows (chunk size=$tbl->{chunk_size}" |
4116 | 4222 | if ( $rename_fk_method eq 'rebuild_constraints' ) { | 6625 | . " * chunk size limit=$limit).\n"; |
4117 | 4223 | update_foreign_key_constraints( | 6626 | die $msg; |
4118 | 4224 | child_tables => \@child_tables, | 6627 | } |
4119 | 4225 | %plugin_args, | 6628 | } |
4120 | 4226 | ); | 6629 | } |
4121 | 4227 | } | 6630 | |
4122 | 4228 | elsif ( $rename_fk_method eq 'drop_old_table' ) { | 6631 | # Exec and time the chunk checksum query. |
4123 | 4229 | $sql = "SET foreign_key_checks=0"; | 6632 | $tbl->{nibble_time} = exec_nibble( |
4124 | 4230 | msg($sql); | 6633 | %args, |
4125 | 4231 | $dbh->do($sql) unless $o->get('print'); | 6634 | Retry => $retry, |
4126 | 4232 | 6635 | Quoter => $q, | |
4127 | 4233 | $sql = "DROP TABLE IF EXISTS `$db`.`$tbl`"; | 6636 | OptionParser => $o, |
4128 | 4234 | msg($sql); | 6637 | ); |
4129 | 4235 | $dbh->do($sql) unless $o->get('print'); | 6638 | PTDEBUG && _d('Nibble time:', $tbl->{nibble_time}); |
4130 | 4236 | 6639 | ||
4131 | 4237 | $sql = "RENAME TABLE `$db`.`$tmp_tbl` TO `$db`.`$tbl`"; | 6640 | # We're executing REPLACE queries which don't return rows. |
4132 | 4238 | msg($sql); | 6641 | # Returning 0 from this callback causes the nibble iter to |
4133 | 4239 | $dbh->do($sql) unless $o->get('print'); | 6642 | # get the next boundaries/nibble. |
4134 | 6643 | return 0; | ||
4135 | 6644 | }, | ||
4136 | 6645 | after_nibble => sub { | ||
4137 | 6646 | my (%args) = @_; | ||
4138 | 6647 | my $tbl = $args{tbl}; | ||
4139 | 6648 | my $nibble_iter = $args{NibbleIterator}; | ||
4140 | 6649 | |||
4141 | 6650 | return unless $o->get('execute'); | ||
4142 | 6651 | |||
4143 | 6652 | # Update rate, chunk size, and progress if the nibble actually | ||
4144 | 6653 | # selected some rows. | ||
4145 | 6654 | my $cnt = $tbl->{row_cnt}; | ||
4146 | 6655 | if ( ($cnt || 0) > 0 ) { | ||
4147 | 6656 | # Update the rate of rows per second for the entire server. | ||
4148 | 6657 | # This is used for the initial chunk size of the next table. | ||
4149 | 6658 | $total_rows += $cnt; | ||
4150 | 6659 | $total_time += $tbl->{nibble_time}; | ||
4151 | 6660 | $avg_rate = int($total_rows / $total_time); | ||
4152 | 6661 | PTDEBUG && _d('Average copy rate (rows/s):', $avg_rate); | ||
4153 | 6662 | |||
4154 | 6663 | # Adjust chunk size. This affects the next chunk. | ||
4155 | 6664 | if ( $chunk_time ) { | ||
4156 | 6665 | # Calcuate a new chunk-size based on the rate of rows/s. | ||
4157 | 6666 | $tbl->{chunk_size} = $tbl->{rate}->update( | ||
4158 | 6667 | $cnt, # processed this many rows | ||
4159 | 6668 | $tbl->{nibble_time}, # is this amount of time | ||
4160 | 6669 | ); | ||
4161 | 6670 | |||
4162 | 6671 | if ( $tbl->{chunk_size} < 1 ) { | ||
4163 | 6672 | # This shouldn't happen. WeightedAvgRate::update() may | ||
4164 | 6673 | # return a value < 1, but minimum chunk size is 1. | ||
4165 | 6674 | $tbl->{chunk_size} = 1; | ||
4166 | 6675 | |||
4167 | 6676 | # This warning is printed once per table. | ||
4168 | 6677 | if ( !$tbl->{warned_slow} ) { | ||
4169 | 6678 | warn "Rows are copying very slowly. " | ||
4170 | 6679 | . "--chunk-size has been automatically reduced to 1. " | ||
4171 | 6680 | . "Check that the server is not being overloaded, " | ||
4172 | 6681 | . "or increase --chunk-time. The last chunk " | ||
4173 | 6682 | . "selected $cnt rows and took " | ||
4174 | 6683 | . sprintf('%.3f', $tbl->{nibble_time}) | ||
4175 | 6684 | . " seconds to execute.\n"; | ||
4176 | 6685 | $tbl->{warned_slow} = 1; | ||
4177 | 6686 | } | ||
4178 | 6687 | } | ||
4179 | 6688 | |||
4180 | 6689 | # Update chunk-size based on the rate of rows/s. | ||
4181 | 6690 | $nibble_iter->set_chunk_size($tbl->{chunk_size}); | ||
4182 | 6691 | } | ||
4183 | 6692 | |||
4184 | 6693 | # Every table should have a Progress obj; update it. | ||
4185 | 6694 | if ( my $tbl_pr = $tbl->{progress} ) { | ||
4186 | 6695 | $tbl_pr->update( sub { return $total_rows } ); | ||
4187 | 6696 | } | ||
4188 | 6697 | } | ||
4189 | 6698 | |||
4190 | 6699 | # Wait forever for slaves to catch up. | ||
4191 | 6700 | $replica_lag_pr->start() if $replica_lag_pr; | ||
4192 | 6701 | $replica_lag->wait(Progress => $replica_lag_pr); | ||
4193 | 6702 | |||
4194 | 6703 | # Wait forever for system load to abate. wait() will die if | ||
4195 | 6704 | # --critical load is reached. | ||
4196 | 6705 | $sys_load_pr->start() if $sys_load_pr; | ||
4197 | 6706 | $sys_load->wait(Progress => $sys_load_pr); | ||
4198 | 6707 | |||
4199 | 6708 | return; | ||
4200 | 6709 | }, | ||
4201 | 6710 | done => sub { | ||
4202 | 6711 | if ( $o->get('execute') ) { | ||
4203 | 6712 | print "Copied rows OK.\n"; | ||
4204 | 6713 | } | ||
4205 | 6714 | }, | ||
4206 | 6715 | }; | ||
4207 | 6716 | |||
4208 | 6717 | # NibbleIterator combines these two statements and adds | ||
4209 | 6718 | # "FROM $orig_table->{name} WHERE <nibble stuff>". | ||
4210 | 6719 | my $dml = "INSERT LOW_PRIORITY IGNORE INTO $new_tbl->{name} " | ||
4211 | 6720 | . "(" . join(', ', map { $q->quote($_) } @common_cols) . ") " | ||
4212 | 6721 | . "SELECT"; | ||
4213 | 6722 | my $select = join(', ', map { $q->quote($_) } @common_cols); | ||
4214 | 6723 | |||
4215 | 6724 | # The chunk size is auto-adjusted, so use --chunk-size as | ||
4216 | 6725 | # the initial value, but then save and update the adjusted | ||
4217 | 6726 | # chunk size in the table data struct. | ||
4218 | 6727 | $orig_tbl->{chunk_size} = $o->get('chunk-size'); | ||
4219 | 6728 | |||
4220 | 6729 | # This won't (shouldn't) fail because we already verified in | ||
4221 | 6730 | # check_orig_table() table we can NibbleIterator::can_nibble(). | ||
4222 | 6731 | my $nibble_iter = new NibbleIterator( | ||
4223 | 6732 | Cxn => $cxn, | ||
4224 | 6733 | tbl => $orig_tbl, | ||
4225 | 6734 | chunk_size => $orig_tbl->{chunk_size}, | ||
4226 | 6735 | chunk_index => $o->get('chunk-index'), | ||
4227 | 6736 | dml => $dml, | ||
4228 | 6737 | select => $select, | ||
4229 | 6738 | callbacks => $callbacks, | ||
4230 | 6739 | OptionParser => $o, | ||
4231 | 6740 | Quoter => $q, | ||
4232 | 6741 | TableParser => $tp, | ||
4233 | 6742 | TableNibbler => new TableNibbler(TableParser => $tp, Quoter => $q), | ||
4234 | 6743 | comments => { | ||
4235 | 6744 | bite => "pt-online-schema-change $PID copy table", | ||
4236 | 6745 | nibble => "pt-online-schema-change $PID copy nibble", | ||
4237 | 6746 | }, | ||
4238 | 6747 | ); | ||
4239 | 6748 | |||
4240 | 6749 | # Init a new weighted avg rate calculator for the table. | ||
4241 | 6750 | $orig_tbl->{rate} = new WeightedAvgRate(target_t => $chunk_time); | ||
4242 | 6751 | |||
4243 | 6752 | # Make a Progress obj for this table. It may not be used; | ||
4244 | 6753 | # depends on how many rows, chunk size, how fast the server | ||
4245 | 6754 | # is, etc. But just in case, all tables have a Progress obj. | ||
4246 | 6755 | if ( $o->get('progress') | ||
4247 | 6756 | && !$nibble_iter->one_nibble() | ||
4248 | 6757 | && $nibble_iter->row_estimate() ) | ||
4249 | 6758 | { | ||
4250 | 6759 | $orig_tbl->{progress} = new Progress( | ||
4251 | 6760 | jobsize => $nibble_iter->row_estimate(), | ||
4252 | 6761 | spec => $o->get('progress'), | ||
4253 | 6762 | name => "Copying $orig_tbl->{name}", | ||
4254 | 6763 | ); | ||
4255 | 6764 | } | ||
4256 | 6765 | |||
4257 | 6766 | # Start copying rows. This may take awhile, but --progress is on | ||
4258 | 6767 | # by default so there will be progress updates to stderr. | ||
4259 | 6768 | eval { | ||
4260 | 6769 | 1 while $nibble_iter->next(); | ||
4261 | 6770 | }; | ||
4262 | 6771 | if ( $EVAL_ERROR ) { | ||
4263 | 6772 | die "Error copying rows from $orig_tbl->{name} to " | ||
4264 | 6773 | . "$new_tbl->{name}: $EVAL_ERROR\n"; | ||
4265 | 6774 | } | ||
4266 | 6775 | $orig_tbl->{copied} = 1; # flag for cleanup tasks | ||
4267 | 6776 | |||
4268 | 6777 | |||
4269 | 6778 | # XXX Auto-choose the alter fk method BEFORE swapping/renaming tables | ||
4270 | 6779 | # else everything will break because if drop_swap is chosen, then we | ||
4271 | 6780 | # most NOT rename tables or drop the old table. | ||
4272 | 6781 | if ( $alter_fk_method eq 'auto' ) { | ||
4273 | 6782 | # If chunk time is set, then use the average rate of rows/s | ||
4274 | 6783 | # from copying the orig table to determine the max size of | ||
4275 | 6784 | # a child table that can be altered within one chunk time. | ||
4276 | 6785 | # The limit is a fudge factor. Chunk time won't be set if | ||
4277 | 6786 | # the user specified --chunk-size=N on the cmd line, in which | ||
4278 | 6787 | # case the max child table size is their specified chunk size | ||
4279 | 6788 | # times the fudge factor. | ||
4280 | 6789 | my $max_rows | ||
4281 | 6790 | = $o->get('dry-run') ? $o->get('chunk-size') * $limit | ||
4282 | 6791 | : $chunk_time ? $avg_rate * $chunk_time * $limit | ||
4283 | 6792 | : $o->get('chunk-size') * $limit; | ||
4284 | 6793 | PTDEBUG && _d('Max allowed child table size:', $max_rows); | ||
4285 | 6794 | |||
4286 | 6795 | $alter_fk_method = determine_alter_fk_method( | ||
4287 | 6796 | child_tables => $child_tables, | ||
4288 | 6797 | max_rows => $max_rows, | ||
4289 | 6798 | Cxn => $cxn, | ||
4290 | 6799 | OptionParser => $o, | ||
4291 | 6800 | ); | ||
4292 | 6801 | |||
4293 | 6802 | if ( $alter_fk_method eq 'drop_swap' ) { | ||
4294 | 6803 | $o->set('swap-tables', 0); | ||
4295 | 6804 | $o->set('drop-old-table', 0); | ||
4296 | 6805 | } | ||
4297 | 6806 | } | ||
4298 | 6807 | |||
4299 | 6808 | # ##################################################################### | ||
4300 | 6809 | # XXX | ||
4301 | 6810 | # Step 5: Rename tables: orig -> old, new -> orig | ||
4302 | 6811 | # Past this step, the original table has been altered. This shouldn't | ||
4303 | 6812 | # fail, but if it does, the failure could be serious depending on what | ||
4304 | 6813 | # state the tables are left in. | ||
4305 | 6814 | # XXX | ||
4306 | 6815 | # ##################################################################### | ||
4307 | 6816 | my $old_tbl; | ||
4308 | 6817 | if ( $o->get('swap-tables') ) { | ||
4309 | 6818 | eval { | ||
4310 | 6819 | $old_tbl = swap_tables( | ||
4311 | 6820 | orig_tbl => $orig_tbl, | ||
4312 | 6821 | new_tbl => $new_tbl, | ||
4313 | 6822 | suffix => '_old', | ||
4314 | 6823 | Cxn => $cxn, | ||
4315 | 6824 | Quoter => $q, | ||
4316 | 6825 | OptionParser => $o, | ||
4317 | 6826 | ); | ||
4318 | 6827 | }; | ||
4319 | 6828 | if ( $EVAL_ERROR ) { | ||
4320 | 6829 | die "Error swapping the tables: $EVAL_ERROR\n" | ||
4321 | 6830 | . "Verify that the original table $orig_tbl->{name} has not " | ||
4322 | 6831 | . "been modified or renamed to the old table $old_tbl->{name}. " | ||
4323 | 6832 | . "Then drop the new table $new_tbl->{name} if it exists.\n"; | ||
4324 | 6833 | } | ||
4325 | 6834 | } | ||
4326 | 6835 | $orig_tbl->{swapped} = 1; # flag for cleanup tasks | ||
4327 | 6836 | PTDEBUG && _d('Old table:', Dumper($old_tbl)); | ||
4328 | 6837 | |||
4329 | 6838 | # ##################################################################### | ||
4330 | 6839 | # Step 6: Update foreign key constraints if there are child tables. | ||
4331 | 6840 | # ##################################################################### | ||
4332 | 6841 | if ( $child_tables ) { | ||
4333 | 6842 | eval { | ||
4334 | 6843 | if ( $alter_fk_method eq 'none' ) { | ||
4335 | 6844 | print "Not updating foreign keys because " | ||
4336 | 6845 | . "--alter-foreign-keys-method=none. Foreign keys " | ||
4337 | 6846 | . "that reference the table will no longer work.\n"; | ||
4338 | 6847 | } | ||
4339 | 6848 | elsif ( $alter_fk_method eq 'rebuild_constraints' ) { | ||
4340 | 6849 | rebuild_constraints( | ||
4341 | 6850 | orig_tbl => $orig_tbl, | ||
4342 | 6851 | old_tbl => $old_tbl, | ||
4343 | 6852 | child_tables => $child_tables, | ||
4344 | 6853 | OptionParser => $o, | ||
4345 | 6854 | Quoter => $q, | ||
4346 | 6855 | Cxn => $cxn, | ||
4347 | 6856 | TableParser => $tp, | ||
4348 | 6857 | ); | ||
4349 | 6858 | } | ||
4350 | 6859 | elsif ( $alter_fk_method eq 'drop_swap' ) { | ||
4351 | 6860 | drop_swap( | ||
4352 | 6861 | orig_tbl => $orig_tbl, | ||
4353 | 6862 | new_tbl => $new_tbl, | ||
4354 | 6863 | Cxn => $cxn, | ||
4355 | 6864 | OptionParser => $o, | ||
4356 | 6865 | ); | ||
4357 | 4240 | } | 6866 | } |
4358 | 4241 | else { | 6867 | else { |
4386 | 4242 | die "Invalid --update-foreign-keys-method value: $rename_fk_method"; | 6868 | # This should "never" happen because we check this var earlier. |
4387 | 4243 | } | 6869 | die "Invalid --alter-foreign-keys-method: $alter_fk_method\n"; |
4388 | 4244 | } | 6870 | } |
4389 | 4245 | 6871 | }; | |
4390 | 4246 | # ##################################################################### | 6872 | if ( $EVAL_ERROR ) { |
4391 | 4247 | # Cleanup. | 6873 | # TODO: improve error message and handling. |
4392 | 4248 | # ##################################################################### | 6874 | die "Error updating foreign key constraints: $EVAL_ERROR\n"; |
4393 | 4249 | msg("Calling " . (ref $copy_rows). "::cleanup()"); | 6875 | } |
4394 | 4250 | $copy_rows->cleanup(%plugin_args); | 6876 | } |
4395 | 4251 | 6877 | ||
4396 | 4252 | msg("Calling " . (ref $capture_sync) . "::cleanup()"); | 6878 | # ######################################################################## |
4397 | 4253 | $capture_sync->cleanup(%plugin_args); | 6879 | # Step 7: Drop the old table. |
4398 | 4254 | 6880 | # ######################################################################## | |
4399 | 4255 | if ( $o->get('rename-tables') && $o->get('drop-old-table') ) { | 6881 | if ( $o->get('drop-old-table') ) { |
4400 | 4256 | $sql = "DROP TABLE IF EXISTS `$db`.`$old_tbl`"; | 6882 | if ( $o->get('dry-run') ) { |
4401 | 4257 | msg($sql); | 6883 | print "Not dropping old table because this is a dry run.\n"; |
4402 | 4258 | $dbh->do($sql) unless $o->get('print'); | 6884 | } |
4403 | 4259 | } | 6885 | else { |
4404 | 4260 | }; | 6886 | print "Dropping old table...\n"; |
4405 | 4261 | if ( $EVAL_ERROR ) { | 6887 | |
4406 | 4262 | warn "An error occurred:\n\n$EVAL_ERROR\n" | 6888 | if ( $alter_fk_method eq 'none' ) { |
4407 | 4263 | . "Some triggers, temp tables, etc. may not have been removed. " | 6889 | # Child tables still reference the old table, but the user |
4408 | 4264 | . "Run with --cleanup-and-exit to remove these items.\n"; | 6890 | # has chosen to break fks, so we need to disable fk checks |
4409 | 4265 | $exit_status = 1; | 6891 | # in order to drop the old table. |
4410 | 4266 | } | 6892 | my $sql = "SET foreign_key_checks=0"; |
4411 | 4267 | 6893 | PTDEBUG && _d($sql); | |
4412 | 4268 | msg("$PROGRAM_NAME ended, exit status $exit_status"); | 6894 | print $sql, "\n" if $o->get('print'); |
4413 | 6895 | $cxn->dbh()->do($sql); | ||
4414 | 6896 | } | ||
4415 | 6897 | |||
4416 | 6898 | my $sql = "DROP TABLE IF EXISTS $old_tbl->{name}"; | ||
4417 | 6899 | print $sql, "\n" if $o->get('print'); | ||
4418 | 6900 | PTDEBUG && _d($sql); | ||
4419 | 6901 | eval { | ||
4420 | 6902 | $cxn->dbh()->do($sql); | ||
4421 | 6903 | }; | ||
4422 | 6904 | if ( $EVAL_ERROR ) { | ||
4423 | 6905 | die "Error dropping the old table: $EVAL_ERROR\n"; | ||
4424 | 6906 | } | ||
4425 | 6907 | print "Dropped old table $old_tbl->{name} OK.\n"; | ||
4426 | 6908 | } | ||
4427 | 6909 | } | ||
4428 | 6910 | |||
4429 | 6911 | # ######################################################################## | ||
4430 | 6912 | # Done. | ||
4431 | 6913 | # ######################################################################## | ||
4432 | 6914 | $orig_tbl->{success} = 1; # flag for cleanup tasks | ||
4433 | 6915 | $cleanup = undef; # exec cleanup tasks | ||
4434 | 6916 | |||
4435 | 4269 | return $exit_status; | 6917 | return $exit_status; |
4436 | 4270 | } | 6918 | } |
4437 | 4271 | 6919 | ||
4438 | 4272 | # ############################################################################ | 6920 | # ############################################################################ |
4439 | 4273 | # Subroutines. | 6921 | # Subroutines. |
4440 | 4274 | # ############################################################################ | 6922 | # ############################################################################ |
4442 | 4275 | sub check_tables { | 6923 | sub create_new_table{ |
4443 | 6924 | my (%args) = @_; | ||
4444 | 6925 | my @required_args = qw(orig_tbl Cxn Quoter OptionParser TableParser); | ||
4445 | 6926 | foreach my $arg ( @required_args ) { | ||
4446 | 6927 | die "I need a $arg argument" unless $args{$arg}; | ||
4447 | 6928 | } | ||
4448 | 6929 | my ($orig_tbl, $cxn, $q, $o, $tp) = @args{@required_args}; | ||
4449 | 6930 | |||
4450 | 6931 | # Get the original table struct. | ||
4451 | 6932 | my $ddl = $tp->get_create_table( | ||
4452 | 6933 | $cxn->dbh(), | ||
4453 | 6934 | $orig_tbl->{db}, | ||
4454 | 6935 | $orig_tbl->{tbl}, | ||
4455 | 6936 | ); | ||
4456 | 6937 | |||
4457 | 6938 | my $tries = $args{tries} || 10; # don't try forever | ||
4458 | 6939 | my $prefix = $args{prefix} || '_'; | ||
4459 | 6940 | my $suffix = $args{suffix} || '_new'; | ||
4460 | 6941 | my $table_name = $orig_tbl->{tbl} . $suffix; | ||
4461 | 6942 | |||
4462 | 6943 | print "Creating new table...\n"; | ||
4463 | 6944 | my $tryno = 1; | ||
4464 | 6945 | my @old_tables; | ||
4465 | 6946 | while ( $tryno++ < $tries ) { | ||
4466 | 6947 | $table_name = $prefix . $table_name; | ||
4467 | 6948 | my $quoted = $q->quote($orig_tbl->{db}, $table_name); | ||
4468 | 6949 | |||
4469 | 6950 | # Generate SQL to create the new table. We do not use CREATE TABLE LIKE | ||
4470 | 6951 | # because it doesn't preserve foreign key constraints. Here we need to | ||
4471 | 6952 | # rename the FK constraints, too. This is because FK constraints are | ||
4472 | 6953 | # internally stored as <database>.<constraint> and there cannot be | ||
4473 | 6954 | # duplicates. If we don't rename the constraints, then InnoDB will throw | ||
4474 | 6955 | # error 121 (duplicate key violation) when we try to execute the CREATE | ||
4475 | 6956 | # TABLE. TODO: this code isn't perfect. If we rename a constraint from | ||
4476 | 6957 | # foo to _foo and there is already a constraint with that name in this | ||
4477 | 6958 | # or another table, we can still have a collision. But if there are | ||
4478 | 6959 | # multiple FKs on this table, it's hard to know which one is causing the | ||
4479 | 6960 | # trouble. Should we generate random/UUID FK names or something instead? | ||
4480 | 6961 | my $sql = $ddl; | ||
4481 | 6962 | $sql =~ s/\ACREATE TABLE .*?\($/CREATE TABLE $quoted (/m; | ||
4482 | 6963 | $sql =~ s/^ CONSTRAINT `/ CONSTRAINT `_/gm; | ||
4483 | 6964 | PTDEBUG && _d($sql); | ||
4484 | 6965 | eval { | ||
4485 | 6966 | $cxn->dbh()->do($sql); | ||
4486 | 6967 | }; | ||
4487 | 6968 | if ( $EVAL_ERROR ) { | ||
4488 | 6969 | # Ignore this error because if multiple instances of the tool | ||
4489 | 6970 | # are running, or previous runs failed and weren't cleaned up, | ||
4490 | 6971 | # then there will be other similarly named tables with fewer | ||
4491 | 6972 | # leading prefix chars. Or, in rarer cases, the db just happens | ||
4492 | 6973 | # to have a similarly named table created by the user for other | ||
4493 | 6974 | # purposes. | ||
4494 | 6975 | if ( $EVAL_ERROR =~ m/table.+?already exists/i ) { | ||
4495 | 6976 | push @old_tables, $q->quote($orig_tbl->{db}, $table_name); | ||
4496 | 6977 | next; | ||
4497 | 6978 | } | ||
4498 | 6979 | |||
4499 | 6980 | # Some other error happened. Let the caller catch it. | ||
4500 | 6981 | die $EVAL_ERROR; | ||
4501 | 6982 | } | ||
4502 | 6983 | print $sql, "\n" if $o->get('print'); # the sql that work | ||
4503 | 6984 | print "Created new table $orig_tbl->{db}.$table_name OK.\n"; | ||
4504 | 6985 | return { # success | ||
4505 | 6986 | db => $orig_tbl->{db}, | ||
4506 | 6987 | tbl => $table_name, | ||
4507 | 6988 | name => $q->quote($orig_tbl->{db}, $table_name), | ||
4508 | 6989 | }; | ||
4509 | 6990 | } | ||
4510 | 6991 | |||
4511 | 6992 | die "Failed to find a unique new table name after $tries attemps. " | ||
4512 | 6993 | . "The following tables exist which may be left over from previous " | ||
4513 | 6994 | . "failed runs of the tool:\n" | ||
4514 | 6995 | . join("\n", map { " $_" } @old_tables) | ||
4515 | 6996 | . "\nExamine these tables and drop some or all of them if they are " | ||
4516 | 6997 | . "no longer need, then re-run the tool.\n"; | ||
4517 | 6998 | } | ||
4518 | 6999 | |||
4519 | 7000 | sub swap_tables { | ||
4520 | 7001 | my (%args) = @_; | ||
4521 | 7002 | my @required_args = qw(orig_tbl new_tbl Cxn Quoter OptionParser); | ||
4522 | 7003 | foreach my $arg ( @required_args ) { | ||
4523 | 7004 | die "I need a $arg argument" unless $args{$arg}; | ||
4524 | 7005 | } | ||
4525 | 7006 | my ($orig_tbl, $new_tbl, $cxn, $q, $o) = @args{@required_args}; | ||
4526 | 7007 | |||
4527 | 7008 | my $prefix = '_'; | ||
4528 | 7009 | my $table_name = $orig_tbl->{tbl} . ($args{suffix} || ''); | ||
4529 | 7010 | my $tries = 10; # don't try forever | ||
4530 | 7011 | |||
4531 | 7012 | # This sub only works for --execute. Since the options are | ||
4532 | 7013 | # mutually exclusive and we return in the if case, the elsif | ||
4533 | 7014 | # is just a paranoid check because swapping the tables is one | ||
4534 | 7015 | # of the most sensitive/dangerous operations. | ||
4535 | 7016 | if ( $o->get('dry-run') ) { | ||
4536 | 7017 | print "Not swapping tables because this is a dry run.\n"; | ||
4537 | 7018 | |||
4538 | 7019 | # A return value really isn't needed, but this trick allows | ||
4539 | 7020 | # rebuild_constraints() to parse and show the sql statements | ||
4540 | 7021 | # it would used. Otherwise, this has no effect. | ||
4541 | 7022 | return $orig_tbl; | ||
4542 | 7023 | } | ||
4543 | 7024 | elsif ( $o->get('execute') ) { | ||
4544 | 7025 | print "Swapping tables...\n"; | ||
4545 | 7026 | |||
4546 | 7027 | while ( $tries-- ) { | ||
4547 | 7028 | $table_name = $prefix . $table_name; | ||
4548 | 7029 | my $sql = "RENAME TABLE $orig_tbl->{name} " | ||
4549 | 7030 | . "TO " . $q->quote($orig_tbl->{db}, $table_name) | ||
4550 | 7031 | . ", $new_tbl->{name} TO $orig_tbl->{name}"; | ||
4551 | 7032 | PTDEBUG && _d($sql); | ||
4552 | 7033 | eval { | ||
4553 | 7034 | $cxn->dbh()->do($sql); | ||
4554 | 7035 | }; | ||
4555 | 7036 | if ( $EVAL_ERROR ) { | ||
4556 | 7037 | # Ignore this error because if multiple instances of the tool | ||
4557 | 7038 | # are running, or previous runs failed and weren't cleaned up, | ||
4558 | 7039 | # then there will be other similarly named tables with fewer | ||
4559 | 7040 | # leading prefix chars. Or, in rarer cases, the db just happens | ||
4560 | 7041 | # to have a similarly named table created by the user for other | ||
4561 | 7042 | # purposes. | ||
4562 | 7043 | next if $EVAL_ERROR =~ m/table.+?already exists/i; | ||
4563 | 7044 | |||
4564 | 7045 | # Some other error happened. Let caller catch it. | ||
4565 | 7046 | die $EVAL_ERROR; | ||
4566 | 7047 | } | ||
4567 | 7048 | print $sql, "\n" if $o->get('print'); | ||
4568 | 7049 | print "Swapped original and new tables OK.\n"; | ||
4569 | 7050 | return { # success | ||
4570 | 7051 | db => $orig_tbl->{db}, | ||
4571 | 7052 | tbl => $table_name, | ||
4572 | 7053 | name => $q->quote($orig_tbl->{db}, $table_name), | ||
4573 | 7054 | }; | ||
4574 | 7055 | } | ||
4575 | 7056 | |||
4576 | 7057 | # This shouldn't happen. | ||
4577 | 7058 | # Here and in the attempt to find a new table name we probably ought to | ||
4578 | 7059 | # use --retries (and maybe a Retry object?) | ||
4579 | 7060 | die "Failed to find a unique old table name after serveral attempts.\n"; | ||
4580 | 7061 | } | ||
4581 | 7062 | } | ||
4582 | 7063 | |||
4583 | 7064 | sub check_orig_table { | ||
4584 | 4276 | my ( %args ) = @_; | 7065 | my ( %args ) = @_; |
4586 | 4277 | my @required_args = qw(dbh db tbl tmp_tbl old_tbl VersionParser Quoter TableParser OptionParser TableChunker); | 7066 | my @required_args = qw(orig_tbl Cxn TableParser OptionParser Quoter); |
4587 | 4278 | foreach my $arg ( @required_args ) { | 7067 | foreach my $arg ( @required_args ) { |
4588 | 4279 | die "I need a $arg argument" unless $args{$arg}; | 7068 | die "I need a $arg argument" unless $args{$arg}; |
4589 | 4280 | } | 7069 | } |
4617 | 4281 | my ($dbh, $db, $tbl, $tmp_tbl, $old_tbl, $o, $tp) | 7070 | my ($orig_tbl, $cxn, $tp, $o, $q) = @args{@required_args}; |
4618 | 4282 | = @args{qw(dbh db tbl tmp_tbl old_tbl OptionParser TableParser)}; | 7071 | |
4619 | 4283 | 7072 | my $dbh = $cxn->dbh(); | |
4620 | 4284 | msg("Checking if table $tbl can be altered"); | 7073 | |
4621 | 4285 | my %tbl_info; | 7074 | # The original table must exist, of course. |
4622 | 4286 | my $sql = ""; | 7075 | if (!$tp->check_table(dbh=>$dbh,db=>$orig_tbl->{db},tbl=>$orig_tbl->{tbl})) { |
4623 | 4287 | 7076 | die "The original table $orig_tbl->{name} does not exist.\n"; | |
4624 | 4288 | # ######################################################################## | 7077 | } |
4625 | 4289 | # Check MySQL. | 7078 | |
4626 | 4290 | # ######################################################################## | 7079 | # There cannot be any triggers on the original table. |
4627 | 4291 | # Although triggers were introduced in 5.0.2, "Prior to MySQL 5.0.10, | 7080 | my $sql = 'SHOW TRIGGERS FROM ' . $q->quote($orig_tbl->{db}) |
4628 | 4292 | # triggers cannot contain direct references to tables by name." | 7081 | . ' LIKE ' . $q->literal_like($orig_tbl->{tbl}); |
4629 | 4293 | if ( !$args{VersionParser}->version_ge($dbh, '5.0.10') ) { | 7082 | PTDEBUG && _d($sql); |
4603 | 4294 | die "This tool requires MySQL 5.0.10 or newer\n"; | ||
4604 | 4295 | } | ||
4605 | 4296 | |||
4606 | 4297 | # ######################################################################## | ||
4607 | 4298 | # Check the (original) table. | ||
4608 | 4299 | # ######################################################################## | ||
4609 | 4300 | # The table must exist of course. | ||
4610 | 4301 | if ( !$tp->check_table(dbh=>$dbh, db=>$db, tbl=>$tbl) ) { | ||
4611 | 4302 | die "Table $db.$tbl does not exist\n"; | ||
4612 | 4303 | } | ||
4613 | 4304 | |||
4614 | 4305 | # There cannot be any triggers on the table. | ||
4615 | 4306 | $sql = "SHOW TRIGGERS FROM `$db` LIKE '$tbl'"; | ||
4616 | 4307 | msg($sql); | ||
4630 | 4308 | my $triggers = $dbh->selectall_arrayref($sql); | 7083 | my $triggers = $dbh->selectall_arrayref($sql); |
4631 | 4309 | if ( $triggers && @$triggers ) { | 7084 | if ( $triggers && @$triggers ) { |
4634 | 4310 | die "Table $db.$tbl has triggers. This tool needs to create " | 7085 | die "The table $orig_tbl->{name} has triggers. This tool " |
4635 | 4311 | . "its own triggers, so the table cannot already have triggers.\n"; | 7086 | . "needs to create its own triggers, so the table cannot " |
4636 | 7087 | . "already have triggers.\n"; | ||
4637 | 4312 | } | 7088 | } |
4638 | 4313 | 7089 | ||
4645 | 4314 | # For now, we require that the old table has an exact-chunkable | 7090 | # Get the table struct. NibbleIterator needs this, and so do we. |
4646 | 4315 | # column (i.e. unique single-column). | 7091 | my $ddl = $tp->get_create_table( |
4647 | 4316 | $tbl_info{tbl_struct} = $tp->parse($tp->get_create_table($dbh, $db, $tbl)); | 7092 | $cxn->dbh(), |
4648 | 4317 | my ($exact, @chunkable_cols) = $args{TableChunker}->find_chunk_columns( | 7093 | $orig_tbl->{db}, |
4649 | 4318 | tbl_struct => $tbl_info{tbl_struct}, | 7094 | $orig_tbl->{tbl}, |
4644 | 4319 | exact => 1, | ||
4650 | 4320 | ); | 7095 | ); |
4684 | 4321 | if ( !$exact || !@chunkable_cols ) { | 7096 | $orig_tbl->{tbl_struct} = $tp->parse($ddl); |
4685 | 4322 | die "Table $db.$tbl cannot be chunked because it does not have " | 7097 | |
4686 | 4323 | . "a unique, single-column index\n"; | 7098 | # Must be able to nibble the original table (to copy rows to the new table). |
4687 | 4324 | } | 7099 | eval { |
4688 | 4325 | $tbl_info{chunk_column} = $chunkable_cols[0]->{column}; | 7100 | NibbleIterator::can_nibble( |
4689 | 4326 | $tbl_info{chunk_index} = $chunkable_cols[0]->{index}; | 7101 | Cxn => $cxn, |
4690 | 4327 | 7102 | tbl => $orig_tbl, | |
4691 | 4328 | # ######################################################################## | 7103 | chunk_size => $o->get('chunk-size'), |
4692 | 4329 | # Check the tmp table. | 7104 | chunk_indx => $o->get('chunk-index'), |
4693 | 4330 | # ######################################################################## | 7105 | OptionParser => $o, |
4694 | 4331 | # The tmp table should not exist if we're supposed to create it. | 7106 | TableParser => $tp, |
4695 | 4332 | # Else, if user specifies --no-create-tmp-table, they should ensure | 7107 | ); |
4696 | 4333 | # that it exists. | 7108 | }; |
4697 | 4334 | if ( $o->get('create-tmp-table') | 7109 | if ( $EVAL_ERROR ) { |
4698 | 4335 | && $tp->check_table(dbh=>$dbh, db=>$db, tbl=>$tmp_tbl) ) { | 7110 | die "Cannot chunk the original table $orig_tbl->{name}: $EVAL_ERROR\n"; |
4699 | 4336 | die "Temporary table $db.$tmp_tbl exists which will prevent " | 7111 | } |
4700 | 4337 | . "--create-tmp-table from creating the temporary table.\n"; | 7112 | |
4701 | 4338 | } | 7113 | # Find a pk or unique index to use for the delete trigger. can_nibble() |
4702 | 4339 | 7114 | # above returns an index, but NibbleIterator will use non-unique indexes, | |
4703 | 4340 | # ######################################################################## | 7115 | # so we have to do this again here. |
4704 | 4341 | # Check the old table. | 7116 | my $indexes = $orig_tbl->{tbl_struct}->{indexes}; # brevity |
4705 | 4342 | # ######################################################################## | 7117 | foreach my $index ( $tp->sort_indexes($orig_tbl->{tbl_struct}) ) { |
4706 | 4343 | # If we're going to rename the tables, which we do by default, then | 7118 | if ( $index eq 'PRIMARY' || $indexes->{$index}->{is_unique} ) { |
4707 | 4344 | # the old table cannot already exist. | 7119 | PTDEBUG && _d('Delete trigger index:', Dumper($index)); |
4708 | 4345 | if ( $o->get('rename-tables') | 7120 | $orig_tbl->{del_index} = $index; |
4709 | 4346 | && $tp->check_table(dbh=>$dbh, db=>$db, tbl=>$old_tbl) ) { | 7121 | last; |
4710 | 4347 | die "Table $db.$old_tbl exists which will prevent $db.$tbl " | 7122 | } |
4711 | 4348 | . "from being renamed to it. Table $db.$old_tbl could be from " | 7123 | } |
4712 | 4349 | . "a previous run that failed. See --drop-old-table for more " | 7124 | if ( !$orig_tbl->{del_index} ) { |
4713 | 4350 | . "information.\n"; | 7125 | die "The original table $orig_tbl->{name} does not have a PRIMARY KEY " |
4714 | 4351 | } | 7126 | . "or a unique index which is required for the DELETE trigger.\n"; |
4715 | 4352 | 7127 | } | |
4716 | 4353 | return %tbl_info; | 7128 | |
4717 | 7129 | return; # success | ||
4718 | 4354 | } | 7130 | } |
4719 | 4355 | 7131 | ||
4721 | 4356 | sub get_child_tables { | 7132 | sub find_child_tables { |
4722 | 4357 | my ( %args ) = @_; | 7133 | my ( %args ) = @_; |
4724 | 4358 | my @required_args = qw(dbh db tbl Quoter); | 7134 | my @required_args = qw(tbl Cxn Quoter); |
4725 | 4359 | foreach my $arg ( @required_args ) { | 7135 | foreach my $arg ( @required_args ) { |
4726 | 4360 | die "I need a $arg argument" unless $args{$arg}; | 7136 | die "I need a $arg argument" unless $args{$arg}; |
4727 | 4361 | } | 7137 | } |
4731 | 4362 | my ($dbh, $db, $tbl, $q) = @args{@required_args}; | 7138 | my ($tbl, $cxn, $q) = @args{@required_args}; |
4732 | 4363 | 7139 | ||
4733 | 4364 | my $sql = "SELECT table_name " | 7140 | PTDEBUG && _d('Finding child tables'); |
4734 | 7141 | |||
4735 | 7142 | my $sql = "SELECT table_schema, table_name " | ||
4736 | 4365 | . "FROM information_schema.key_column_usage " | 7143 | . "FROM information_schema.key_column_usage " |
4760 | 4366 | . "WHERE constraint_schema='$db' AND referenced_table_name='$tbl'"; | 7144 | . "WHERE constraint_schema='$tbl->{db}' " |
4761 | 4367 | PTDEBUG && _d($dbh, $sql); | 7145 | . "AND referenced_table_name='$tbl->{tbl}'"; |
4762 | 4368 | my $child_tables; | 7146 | PTDEBUG && _d($sql); |
4763 | 4369 | eval { | 7147 | my $rows = $cxn->dbh()->selectall_arrayref($sql); |
4764 | 4370 | $child_tables = $dbh->selectall_arrayref($sql); | 7148 | if ( !$rows || !@$rows ) { |
4765 | 4371 | }; | 7149 | PTDEBUG && _d('No child tables found'); |
4766 | 4372 | if ( $EVAL_ERROR ) { | 7150 | return; |
4767 | 4373 | die "Error executing query to check $tbl for child tables.\n\n" | 7151 | } |
4768 | 4374 | . "Query: $sql\n\n" | 7152 | |
4769 | 4375 | . "Error: $EVAL_ERROR" | 7153 | my @child_tables; |
4770 | 4376 | } | 7154 | foreach my $row ( @$rows ) { |
4771 | 4377 | 7155 | my $tbl = { | |
4772 | 4378 | PTDEBUG && _d("Child tables:", join(', ', map { $_->[0] } @$child_tables)); | 7156 | db => $row->[0], |
4773 | 4379 | return map { $_->[0] } @$child_tables; | 7157 | tbl => $row->[1], |
4774 | 4380 | } | 7158 | name => $q->quote(@$row), |
4775 | 4381 | 7159 | }; | |
4776 | 4382 | sub update_foreign_key_constraints { | 7160 | |
4777 | 4383 | my ( %args ) = @_; | 7161 | # Get row estimates for each child table so we can give the user |
4778 | 4384 | my @required_args = qw(msg dbh db tbl old_tbl child_tables Quoter); | 7162 | # some input on choosing an --alter-foreign-keys-method if they |
4779 | 4385 | foreach my $arg ( @required_args ) { | 7163 | # don't use "auto". |
4780 | 4386 | die "I need a $arg argument" unless $args{$arg}; | 7164 | my ($n_rows) = NibbleIterator::get_row_estimate( |
4781 | 4387 | } | 7165 | Cxn => $cxn, |
4782 | 4388 | my ($msg, $dbh, $db, $tbl, $old_tbl, $child_tables, $q) | 7166 | tbl => $tbl, |
4783 | 7167 | ); | ||
4784 | 7168 | $tbl->{row_est} = $n_rows; | ||
4785 | 7169 | |||
4786 | 7170 | push @child_tables, $tbl; | ||
4787 | 7171 | } | ||
4788 | 7172 | |||
4789 | 7173 | PTDEBUG && _d('Child tables:', Dumper(\@child_tables)); | ||
4790 | 7174 | return \@child_tables; | ||
4791 | 7175 | } | ||
4792 | 7176 | |||
4793 | 7177 | sub determine_alter_fk_method { | ||
4794 | 7178 | my ( %args ) = @_; | ||
4795 | 7179 | my @required_args = qw(child_tables max_rows Cxn OptionParser); | ||
4796 | 7180 | foreach my $arg ( @required_args ) { | ||
4797 | 7181 | die "I need a $arg argument" unless $args{$arg}; | ||
4798 | 7182 | } | ||
4799 | 7183 | my ($child_tables, $max_rows, $cxn, $o) = @args{@required_args}; | ||
4800 | 7184 | |||
4801 | 7185 | if ( $o->get('dry-run') ) { | ||
4802 | 7186 | print "Not determining the method to update foreign keys " | ||
4803 | 7187 | . "because this is a dry run.\n"; | ||
4804 | 7188 | return ''; # $alter_fk_method can't be undef | ||
4805 | 7189 | } | ||
4806 | 7190 | |||
4807 | 7191 | # The rebuild_constraints method is the default becuase it's safer | ||
4808 | 7192 | # and doesn't cause the orig table to go missing for a moment. | ||
4809 | 7193 | my $method = 'rebuild_constraints'; | ||
4810 | 7194 | |||
4811 | 7195 | print "Max rows for the rebuild_constraints method: $max_rows\n", | ||
4812 | 7196 | "Determining the method to update foreign keys...\n"; | ||
4813 | 7197 | foreach my $child_tbl ( @$child_tables ) { | ||
4814 | 7198 | print " $child_tbl->{name}: "; | ||
4815 | 7199 | my ($n_rows) = NibbleIterator::get_row_estimate( | ||
4816 | 7200 | Cxn => $cxn, | ||
4817 | 7201 | tbl => $child_tbl, | ||
4818 | 7202 | ); | ||
4819 | 7203 | if ( $n_rows > $max_rows ) { | ||
4820 | 7204 | print "too many rows: $n_rows; must use drop_swap\n"; | ||
4821 | 7205 | $method = 'drop_swap'; | ||
4822 | 7206 | last; | ||
4823 | 7207 | } | ||
4824 | 7208 | else { | ||
4825 | 7209 | print "$n_rows rows; can use rebuild_constraints\n"; | ||
4826 | 7210 | } | ||
4827 | 7211 | } | ||
4828 | 7212 | |||
4829 | 7213 | return $method || ''; # $alter_fk_method can't be undef | ||
4830 | 7214 | } | ||
4831 | 7215 | |||
4832 | 7216 | sub rebuild_constraints { | ||
4833 | 7217 | my ( %args ) = @_; | ||
4834 | 7218 | my @required_args = qw(orig_tbl old_tbl child_tables | ||
4835 | 7219 | Cxn Quoter OptionParser TableParser); | ||
4836 | 7220 | foreach my $arg ( @required_args ) { | ||
4837 | 7221 | die "I need a $arg argument" unless $args{$arg}; | ||
4838 | 7222 | } | ||
4839 | 7223 | my ($orig_tbl, $old_tbl, $child_tables, $cxn, $q, $o, $tp) | ||
4840 | 4389 | = @args{@required_args}; | 7224 | = @args{@required_args}; |
4841 | 4390 | 7225 | ||
4843 | 4391 | my $constraint = qr/^\s+(CONSTRAINT.+?REFERENCES `$old_tbl`.+)$/mo; | 7226 | # MySQL has a "feature" where if the parent tbl is in the same db, |
4844 | 7227 | # then the child tbl ref is simply `parent_tbl`, but if the parent tbl | ||
4845 | 7228 | # is in another db, then the child tbl ref is `other_db`.`parent_tbl`. | ||
4846 | 7229 | # When we recreate the ref below, we use the db-qualified form, and | ||
4847 | 7230 | # MySQL will automatically trim the db if the tables are in the same db. | ||
4848 | 7231 | my $quoted_old_table = $q->quote($old_tbl->{tbl}); | ||
4849 | 7232 | my $constraint = qr/ | ||
4850 | 7233 | ^\s+ | ||
4851 | 7234 | ( | ||
4852 | 7235 | CONSTRAINT.+? | ||
4853 | 7236 | REFERENCES\s(?:$quoted_old_table|$old_tbl->{name}) | ||
4854 | 7237 | .+ | ||
4855 | 7238 | )$ | ||
4856 | 7239 | /xm; | ||
4857 | 7240 | PTDEBUG && _d('Rebuilding fk constraint matching', $constraint); | ||
4858 | 7241 | |||
4859 | 7242 | if ( $o->get('dry-run') ) { | ||
4860 | 7243 | print "Not rebuilding foreign key constraints because this is a dry run.\n"; | ||
4861 | 7244 | } | ||
4862 | 7245 | else { | ||
4863 | 7246 | print "Rebuilding foreign key constraints...\n"; | ||
4864 | 7247 | } | ||
4865 | 4392 | 7248 | ||
4866 | 4393 | CHILD_TABLE: | 7249 | CHILD_TABLE: |
4880 | 4394 | foreach my $child_table ( @$child_tables ) { | 7250 | foreach my $child_tbl ( @$child_tables ) { |
4881 | 4395 | my $sql = "SHOW CREATE TABLE `$db`.`$child_table`"; | 7251 | my $table_def = $tp->get_create_table( |
4882 | 4396 | PTDEBUG && _d($dbh, $sql); | 7252 | $cxn->dbh(), |
4883 | 4397 | $msg->($sql); | 7253 | $child_tbl->{db}, |
4884 | 4398 | my $table_def; | 7254 | $child_tbl->{tbl}, |
4885 | 4399 | eval { | 7255 | ); |
4873 | 4400 | $table_def = $dbh->selectrow_arrayref($sql)->[1]; | ||
4874 | 4401 | }; | ||
4875 | 4402 | if ( $EVAL_ERROR ) { | ||
4876 | 4403 | warn "Skipping child table $child_table: $EVAL_ERROR"; | ||
4877 | 4404 | next CHILD_TABLE; | ||
4878 | 4405 | } | ||
4879 | 4406 | |||
4886 | 4407 | my @constraints = $table_def =~ m/$constraint/g; | 7256 | my @constraints = $table_def =~ m/$constraint/g; |
4887 | 4408 | if ( !@constraints ) { | 7257 | if ( !@constraints ) { |
4890 | 4409 | warn "Child table `$child_table` does not have any foreign key " | 7258 | warn "$child_tbl->{name} has no foreign key " |
4891 | 4410 | . "constraints referencing $old_tbl"; | 7259 | . "constraints referencing $old_tbl->{name}.\n"; |
4892 | 4411 | next CHILD_TABLE; | 7260 | next CHILD_TABLE; |
4893 | 4412 | } | 7261 | } |
4894 | 4413 | 7262 | ||
4895 | 7263 | my @rebuilt_constraints; | ||
4896 | 4414 | foreach my $constraint ( @constraints ) { | 7264 | foreach my $constraint ( @constraints ) { |
4946 | 4415 | my ($fk_symbol) = $constraint =~ m/CONSTRAINT\s+(\S+)/; | 7265 | PTDEBUG && _d('Rebuilding fk constraint:', $constraint); |
4947 | 4416 | $sql = "ALTER TABLE `$db`.`$child_table` DROP FOREIGN KEY $fk_symbol"; | 7266 | |
4948 | 4417 | $msg->($sql); | 7267 | # Remove trailing commas in case there are multiple constraints on the |
4949 | 4418 | $dbh->do($sql) unless $args{print}; | 7268 | # table. |
4950 | 4419 | 7269 | $constraint =~ s/,$//; | |
4951 | 4420 | $constraint =~ s/REFERENCES `$old_tbl`/REFERENCES `$tbl`/o; | 7270 | |
4952 | 4421 | $sql = "ALTER TABLE `$db`.`$child_table` ADD $constraint"; | 7271 | # Find the constraint name. It will be quoted already. |
4953 | 4422 | $msg->($sql); | 7272 | my ($fk) = $constraint =~ m/CONSTRAINT\s+`([^`]+)`/; |
4954 | 4423 | $dbh->do($sql) unless $args{print}; | 7273 | |
4955 | 4424 | } | 7274 | # Drop the reference to the old table/renamed orig table, and add a new |
4956 | 4425 | } | 7275 | # reference to the new table. InnoDB will throw an error if the new |
4957 | 4426 | 7276 | # constraint has the same name as the old one, so we must rename it. | |
4958 | 4427 | return; | 7277 | # Example: after renaming sakila.actor to sakila.actor_old (for |
4959 | 4428 | } | 7278 | # example), the foreign key on film_actor looks like this: |
4960 | 4429 | 7279 | # CONSTRAINT `fk_film_actor_actor` FOREIGN KEY (`actor_id`) REFERENCES | |
4961 | 4430 | sub intersection { | 7280 | # `actor_old` (`actor_id`) ON UPDATE CASCADE |
4962 | 4431 | my ( $hashes ) = @_; | 7281 | # We need it to look like this instead: |
4963 | 4432 | my %keys = map { $_ => 1 } keys %{$hashes->[0]}; | 7282 | # CONSTRAINT `_fk_film_actor_actor` FOREIGN KEY (`actor_id`) REFERENCES |
4964 | 4433 | my $n_hashes = (scalar @$hashes) - 1; | 7283 | # `actor` (`actor_id`) ON UPDATE CASCADE |
4965 | 4434 | my @isect = grep { $keys{$_} } map { keys %{$hashes->[$_]} } 1..$n_hashes; | 7284 | # Reference the correct table name... |
4966 | 4435 | return @isect; | 7285 | $constraint =~ s/REFERENCES[^\(]+/REFERENCES $orig_tbl->{name} /; |
4967 | 4436 | } | 7286 | # And rename the constraint to avoid conflict |
4968 | 4437 | 7287 | $constraint =~ s/CONSTRAINT `$fk`/CONSTRAINT `_$fk`/; | |
4969 | 4438 | sub get_cxn { | 7288 | |
4970 | 4439 | my ( %args ) = @_; | 7289 | my $sql = "DROP FOREIGN KEY `$fk`, " |
4971 | 4440 | my ($dsn, $ac, $dp, $o) = @args{qw(dsn AutoCommit DSNParser OptionParser)}; | 7290 | . "ADD $constraint"; |
4972 | 4441 | 7291 | push @rebuilt_constraints, $sql; | |
4973 | 4442 | if ( $o->get('ask-pass') ) { | 7292 | } |
4974 | 4443 | $dsn->{p} = OptionParser::prompt_noecho("Enter password: "); | 7293 | |
4975 | 4444 | } | 7294 | my $sql = "ALTER TABLE $child_tbl->{name} " |
4976 | 4445 | my $dbh = $dp->get_dbh($dp->get_cxn_params($dsn), {AutoCommit => $ac}); | 7295 | . join(', ', @rebuilt_constraints); |
4977 | 4446 | 7296 | print $sql, "\n" if $o->get('print'); | |
4978 | 4447 | $dbh->do('SET SQL_LOG_BIN=0') unless $o->get('bin-log'); | 7297 | if ( $o->get('execute') ) { |
4979 | 4448 | $dbh->do('SET FOREIGN_KEY_CHECKS=0') unless $o->get('foreign-key-checks'); | 7298 | PTDEBUG && _d($sql); |
4980 | 4449 | 7299 | $cxn->dbh()->do($sql); | |
4981 | 4450 | return $dbh; | 7300 | } |
4982 | 4451 | } | 7301 | } |
4983 | 4452 | 7302 | ||
4984 | 4453 | sub msg { | 7303 | if ( $o->get('execute') ) { |
4985 | 4454 | my ( $msg ) = @_; | 7304 | print "Rebuilt foreign key constraints OK.\n"; |
4986 | 4455 | chomp $msg; | 7305 | } |
4987 | 4456 | print '# ', ts(time), " $msg\n" unless $quiet; | 7306 | |
4988 | 4457 | PTDEBUG && _d($msg); | 7307 | return; |
4989 | 4458 | return; | 7308 | } |
4990 | 4459 | } | 7309 | |
4991 | 4460 | 7310 | sub drop_swap { | |
4992 | 4461 | # Only for tests which may not call main(). | 7311 | my ( %args ) = @_; |
4993 | 4462 | sub __set_quiet { | 7312 | my @required_args = qw(orig_tbl new_tbl Cxn OptionParser); |
4994 | 4463 | $quiet = $_[0]; | 7313 | foreach my $arg ( @required_args ) { |
4995 | 7314 | die "I need a $arg argument" unless $args{$arg}; | ||
4996 | 7315 | } | ||
4997 | 7316 | my ($orig_tbl, $new_tbl, $cxn, $o) = @args{@required_args}; | ||
4998 | 7317 | |||
4999 | 7318 | if ( $o->get('dry-run') ) { | ||
5000 | 7319 | print "Not drop-swapping tables because this is a dry run.\n"; |
The diff has been truncated for viewing.