Merge lp:~akopytov/percona-xtrabackup/bug1183793-2.1 into lp:percona-xtrabackup/2.1

[Alexey Kopytov]

    Bug #1183794: XtraBackup should raise the open files limit if possible

    The xtrabackup binaries now recognizes a new my.cnf option,
    open_files_limit. The effect is the same as for the server: it changes
    the maximum number of file descriptors available to the xtrabackup
    process. The actual limit depends on the platform and ulimit settings.

http://jenkins.percona.com/view/PXB%202.1/job/percona-xtrabackup-2.1-param/559/

[Sergei Glushchenko (sergei.glushchenko)]

Are lines

88 + rlimit.rlim_cur = 0; /* Safety if next call fails */
89 +
90 + (void) getrlimit(RLIMIT_NOFILE, &rlimit);
91 +
92 + if (rlimit.rlim_cur) {

work different from

nn + if (!getrlimit(RLIMIT_NOFILE, &rlimit)) {

man page suggest to use return value for error handling and says nothing about using rlim_cur for this purpose.

review: Needs Information (g2)

[Alexey Kopytov (akopytov)]

I think the logic there is: "if setrlimit() succeeds, check what limit has actually been set". That code doesn't expect getrlimit() call to fail (since the same call succeeded previously), but assumes the actually set limit may be different from what was requested. In which case it returns the actual limit instead.

[Sergei Glushchenko (sergei.glushchenko)]

Yes, I understand that logic and it normally can be achieved without zeroing rlimit.rlim_cur.
But since it work for server code should also work for XtraBackup.

review: Approve (g2)

=== modified file 'src/xtrabackup.cc'
--- src/xtrabackup.cc	2014-02-24 20:21:05 +0000
+++ src/xtrabackup.cc	2014-04-28 14:43:27 +0000
@@ -63,6 +63,8 @@
 # include <sys/prctl.h>
 #endif
 
+#include <sys/resource.h>
+
 #define G_PTR uchar*
 
 #include "common.h"
@@ -292,6 +294,8 @@
 /* The maximum LSN of found archived log files */
 lsn_t xtrabackup_arch_last_file_lsn = 0ULL;
 
+ulong xb_open_files_limit= 0;
+
 /* Datasinks */
 ds_ctxt_t       *ds_data     = NULL;
 ds_ctxt_t       *ds_meta     = NULL;
@@ -478,7 +482,8 @@
   OPT_INNODB_LOG_CHECKSUM_ALGORITHM,
 #endif
   OPT_XTRA_INCREMENTAL_FORCE_SCAN,
-  OPT_DEFAULTS_GROUP
+  OPT_DEFAULTS_GROUP,
+  OPT_OPEN_FILES_LIMIT
 };
 
 #if MYSQL_VERSION_ID >= 50600
@@ -896,6 +901,12 @@
   {"defaults_group", OPT_DEFAULTS_GROUP, "defaults group in config file (default \"mysqld\").",
    (G_PTR*) &defaults_group, (G_PTR*) &defaults_group,
    0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+  {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file "
+   "descriptors to reserve with setrlimit().",
+   (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG,
+   REQUIRED_ARG, 0, 0, UINT_MAX, 0, 1, 0},
+
   { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
 
@@ -2949,6 +2960,63 @@
 	}
 }
 
+/***********************************************************************
+Set the open files limit. Based on set_max_open_files().
+
+@return the resulting open files limit. May be less or more than the requested
+value.  */
+static uint
+xb_set_max_open_files(
+/*==================*/
+	uint max_file_limit)	/*!<in: open files limit */
+{
+#if defined(RLIMIT_NOFILE)
+	struct rlimit rlimit;
+	uint old_cur;
+
+	if (getrlimit(RLIMIT_NOFILE, &rlimit)) {
+
+		goto end;
+	}
+
+	old_cur = (uint) rlimit.rlim_cur;
+
+	if (rlimit.rlim_cur == RLIM_INFINITY) {
+
+		rlimit.rlim_cur = max_file_limit;
+	}
+
+	if (rlimit.rlim_cur >= max_file_limit) {
+
+		max_file_limit = rlimit.rlim_cur;
+		goto end;
+	}
+
+	rlimit.rlim_cur = rlimit.rlim_max = max_file_limit;
+
+	if (setrlimit(RLIMIT_NOFILE, &rlimit)) {
+
+		max_file_limit = old_cur;	/* Use original value */
+	} else {
+
+		rlimit.rlim_cur = 0;	/* Safety if next call fails */
+
+		(void) getrlimit(RLIMIT_NOFILE, &rlimit);
+
+		if (rlimit.rlim_cur) {
+
+			/* If call didn't fail */
+			max_file_limit = (uint) rlimit.rlim_cur;
+		}
+	}
+
+end:
+	return(max_file_limit);
+#else
+	return(0);
+#endif
+}
+
 static void
 xtrabackup_backup_func(void)
 {
@@ -2972,6 +3040,10 @@
 	}
 	msg("xtrabackup: cd to %s\n", mysql_real_data_home);
 
+	msg("xtrabackup: open files limit requested %u, set to %u\n",
+	    (uint) xb_open_files_limit,
+	    xb_set_max_open_files(xb_open_files_limit));
+
 	mysql_data_home= mysql_data_home_buff;
 	mysql_data_home[0]=FN_CURLIB;		// all paths are relative from here
 	mysql_data_home[1]=0;

=== added file 'test/t/bug1183793.sh'
--- test/t/bug1183793.sh	1970-01-01 00:00:00 +0000
+++ test/t/bug1183793.sh	2014-04-28 14:43:27 +0000
@@ -0,0 +1,14 @@
+########################################################################
+# Bug #1183793: XtraBackup should raise the open files limit if possible
+########################################################################
+
+MYSQLD_EXTRA_MY_CNF_OPTS="
+open_files_limit=1234
+"
+
+start_server
+
+innobackupex --no-timestamp $topdir/backup
+
+grep -q 'open files limit requested 1234' $OUTFILE \
+    || die "Could not set the open files limit"