========================= Error log:
2014-07-13 12:54:33 7f6f2637c700 InnoDB: Assertion failure in thread 140115359287040 in file log0online.cc line 1461
InnoDB: Failing assertion: bitmap_files->files[i].start_lsn >= bitmap_files->files[i - 1].start_lsn
InnoDB: We intentionally generate a memory trap.
InnoDB: Submit a detailed bug report to http://bugs.mysql.com.
InnoDB: If you get repeated assertion failures or crashes, even
InnoDB: immediately after the mysqld startup, there may be
InnoDB: corruption in the InnoDB tablespace. Please refer to
InnoDB: http://dev.mysql.com/doc/refman/5.6/en/forcing-innodb-recovery.html
InnoDB: about forcing recovery.
09:54:33 UTC - mysqld got signal 6 ;
--
--
Trying to get some variables.
Some pointers may be invalid and cause the dump to abort.
Query (7f6f18828010): SELECT start_lsn, end_lsn, space_id, page_id FROM INFORMATION_SCHEMA.INNODB_CHANGED_PAGES LIMIT 5
Connection ID (thread ID): 12
Status: NOT_KILLED
========================= gdb :
+bt
#0 0x00007f6fbe28f69c in pthread_kill () from /lib64/libpthread.so.0
#1 0x0000000000aab116 in my_write_core (sig=6) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/mysys/stacktrace.c:422
#2 0x0000000000727fc6 in handle_fatal_signal (sig=6) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/signal_handler.cc:254
#3 <signal handler called>
#4 0x00007f6fbcf3e8e5 in raise () from /lib64/libc.so.6
#5 0x00007f6fbcf400c5 in abort () from /lib64/libc.so.6
#6 0x0000000000b81410 in log_online_setup_bitmap_file_range (bitmap_files=0x7f6f263792c8, range_start=0, range_end=18446744073709551615) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/storage/innobase/log/log0online.cc:1460
#7 0x0000000000b816c4 in log_online_bitmap_iterator_init (i=0x7f6f263792c0, min_lsn=0, max_lsn=18446744073709551615) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/storage/innobase/log/log0online.cc:1588
#8 0x0000000000b46b5e in i_s_innodb_changed_pages_fill (thd=0x7f6f883f8000, tables=0x7f6f18828650, cond=0x0) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/storage/innobase/handler/i_s.cc:8439
#9 0x0000000000838477 in do_fill_table (thd=0x7f6f883f8000, table_list=0x7f6f18828650, join_table=0x7f6f28511190) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/sql_show.cc:7799
#10 0x0000000000838846 in get_schema_tables_result (join=0x7f6f18829200, executed_place=PROCESSED_BY_JOIN_EXEC) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/sql_show.cc:7900
#11 0x000000000080b0c2 in JOIN::prepare_result (this=0x7f6f18829200, columns_list=0x7f6f26379820) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/sql_select.cc:824
#12 0x00000000007a8557 in JOIN::exec (this=0x7f6f18829200) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/sql_executor.cc:116
#13 0x000000000080bbbb in mysql_execute_select (thd=0x7f6f883f8000, select_lex=0x7f6f883fac58, free_join=true) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/sql_select.cc:1101
#14 0x000000000080beb7 in mysql_select (thd=0x7f6f883f8000, tables=0x7f6f18828650, wild_num=0, fields=..., conds=0x0, order=0x7f6f883fae20, group=0x7f6f883fad58, having=0x0, select_options=2684619520, result=0x7f6f188291d8, unit=0x7f6f883fa610, select_lex=0x7f6f883fac58) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/sql_select.cc:1222
#15 0x0000000000809f46 in handle_select (thd=0x7f6f883f8000, result=0x7f6f188291d8, setup_tables_done_option=0) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/sql_select.cc:111
#16 0x00000000007e26ea in execute_sqlcom_select (thd=0x7f6f883f8000, all_tables=0x7f6f18828650) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/sql_parse.cc:5570
#17 0x00000000007dad14 in mysql_execute_command (thd=0x7f6f883f8000) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/sql_parse.cc:2955
#18 0x00000000007e4feb in mysql_parse (thd=0x7f6f883f8000, rawbuf=0x7f6f18828010 "SELECT start_lsn, end_lsn, space_id, page_id FROM INFORMATION_SCHEMA.INNODB_CHANGED_PAGES LIMIT 5", length=97, parser_state=0x7f6f2637ac90) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/sql_parse.cc:6746
#19 0x00000000007d7586 in dispatch_command (command=COM_QUERY, thd=0x7f6f883f8000, packet=0x7f6f87e17001 "SELECT start_lsn, end_lsn, space_id, page_id FROM INFORMATION_SCHEMA.INNODB_CHANGED_PAGES LIMIT 5", packet_length=97) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/sql_parse.cc:1434
#20 0x00000000007d64b2 in do_command (thd=0x7f6f883f8000) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/sql_parse.cc:1051
#21 0x00000000008ba17f in threadpool_process_request (thd=0x7f6f883f8000) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/threadpool_common.cc:319
#22 0x00000000008bca61 in handle_event (connection=0x7f6f887c6d30) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/threadpool_unix.cc:1550
#23 0x00000000008bcc91 in worker_main (param=0x1881c00) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/sql/threadpool_unix.cc:1603
#24 0x0000000000af83b4 in pfs_spawn_thread (arg=0x7f6fa07be9c0) at /mnt/workspace/percona-server-5.6-binaries-debug-yassl/label_exp/centos6-64/percona-server-5.6.19-67.0/storage/perfschema/pfs.cc:1860
#25 0x00007f6fbe28a851 in start_thread () from /lib64/libpthread.so.0
#26 0x00007f6fbcff494d in clone () from /lib64/libc.so.6
(gdb) +set logging off
========================= cmd details:
if [ -r /usr/lib64/libjemalloc.so.1 ]; then export LD_PRELOAD=/usr/lib64/libjemalloc.so.1
elif [ -r /usr/lib/x86_64-linux-gnu/libjemalloc.so.1 ]; then export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
elif [ -r /ssd/qa56dbg/45/lib/mysql/libjemalloc.so.1 ]; then export LD_PRELOAD=/ssd/qa56dbg/45/lib/mysql/libjemalloc.so.1
else echo 'Error: jemalloc not found, please install it first'; exit 1; fi
ps -ef | grep 'rundir1_587' | grep -v grep | awk '{print $2}' | xargs sudo kill -9 > /dev/null 2>&1
rm -Rf /ssd/qa56dbg/45/rundir1_587
rm -Rf /ssd/qa56dbg/45/rundir1_587_epoch
mkdir /ssd/qa56dbg/45/rundir1_587
mkdir /ssd/qa56dbg/45/rundir1_587_epoch
cd /data/bench/qa56dbg/randgen
bash -c "set -o pipefail; perl /data/bench/qa56dbg/randgen/runall.pl --queries=100000000 --seed=8854 --duration=300 --querytimeout=60 --short_column_names --reporter=Shutdown,Backtrace,QueryTimeout,ErrorLog,ErrorLogAlarm --mysqld=--log-output=none --mysqld=--sql_mode=ONLY_FULL_GROUP_BY --mysqld=--plugin-load=tokudb=ha_tokudb.so --mysqld=--init-file=/data/bench/qa56dbg/randgen/conf/percona_qa/5.6/TokuDB.sql --mysqld=--utility-user-password=test --grammar=conf/percona_qa/5.6/5.6.yy --gendata=conf/percona_qa/5.6/5.6.zz1 --threads=17 --mask-level=1 --validator=Transformer --basedir=/ssd/qa56dbg/Percona-Server-5.6.19-rel67.0-619.Linux.x86_64-debug --mysqld=--enforce-storage-engine=InnoDB --notnull --mysqld=--innodb_changed_pages=FORCE --mysqld=--innodb_fast_shutdown=0 --mysqld=--innodb_flush_log_at_trx_commit=2 --mysqld=--innodb_flush_method=O_DIRECT --mysqld=--innodb_log_arch_dir=/ssd/qa56dbg/45/rundir1_587_epoch --mysqld=--innodb_log_arch_expire_sec=120 --mysqld=--innodb_log_archive=1 --mysqld=--innodb_log_block_size=512 --mysqld=--innodb_log_buffer_size=1048577 --mysqld=--innodb_log_file_size=10485761 --mysqld=--innodb_log_files_in_group=3 --mysqld=--innodb_max_bitmap_file_size=4097 --mysqld=--innodb_max_changed_pages=0 --mysqld=--innodb_track_changed_pages=1 --mysqld=--innodb_use_global_flush_log_at_trx_commit=0 --mysqld=--innodb-buffer-pool-populate --mysqld=--loose-readonly-key-cache-division-limit=1 --mysqld=--thread_handling=pool-of-threads --mysqld=--transaction-isolation=READ-UNCOMMITTED --mysqld=--utility-user-password=test --mysqld=--innodb_purge_threads=1 --mysqld=--innodb_adaptive_hash_index_partitions=8 --mysqld=--innodb_buffer_pool_instances=4 --mtr-build-thread=716 --mask=44218 --vardir1=/ssd/qa56dbg/45/rundir1_587 > /ssd/qa56dbg/45/cmd587.log 2>&1"
tail -n1 /ssd/qa56dbg/45/cmd587.log
========================= version:
Server : 619 <email address hidden>
RQG: 973 <email address hidden>
I need a core dump and a listing of directory with xdb files (should be the datadir).