Merge lp:~gl-az/percona-xtrabackup/BT23557-bug1185343-2.2 into lp:percona-xtrabackup/2.2

[George Ormond Lorch III]

Merged from 2.1:

Fix for https://bugs.launchpad.net/percona-xtrabackup/+bug/1185343 - Fixed IV used in Xtrabackup encryption

In order to fix this in the safest possible way, followed general advice from various sources but summed up nicely here:
    "An initialization vector has different security requirements than a key, so the IV usually does not need to be secret. However, in most cases, it is important that an initialization vector is never reused under the same key. For CBC and CFB, reusing an IV leaks some information about the first block of plaintext, and about any common prefix shared by the two messages. For OFB and CTR, reusing an IV completely destroys security. This can be seen because both modes effectively create a bitstream that is XORed with the plaintext, and this bitstream is dependent on the password and IV only. Reusing a bitstream destroys security. In CBC mode, the IV must, in addition, be unpredictable at encryption time; in particular, the (previously) common practice of re-using the last ciphertext block of a message as the IV for the next message is insecure (for example, this method was used by SSL 2.0). If an attacker knows the IV (or the previous block of ciphertext) before he specifies the next plaintext, he can check his guess about plaintext of some block that was encrypted with the same key before (this is known as the TLS CBC IV attack)." - http://en.wikipedia.org/wiki/Block_cipher_modes_of_operation

This fix advances the XBCRYPT chunk header version and adds an iv field to store the iv that was used to encrypt that block.

On decryption, if the XBCRYPT version is 1 then the original fixed iv is used to decrypt, otherwise the iv in the block header is used.

On encryption, for each block being encrypted, an iv of the correct length is generated by using values returned from random() until the iv has been fully populated. This iv will be written into the XBCRYPT chunk header along with its size.

One new test case added, xbcrypt.sh. Added two files in inc: decrypt_v1_test_file.txt is a simple text file that was then encrypted @ v1 levels and put in decrypt_v1_test_file.xbcrypt. The new test case tests two simple scenarios:
      1 - Tests that a file can go through an encryption/decryption cycle and be 100% identical to the original file.
      2 - Tests that a file previously encrypted with and earlier version can be decrypted with the current version.

http://jenkins.percona.com/view/XtraBackup/job/percona-xtrabackup-2.2-param/20/

[Alexey Kopytov (akopytov)]

review: Approve

=== modified file 'xtrabackup/src/ds_encrypt.c'
--- xtrabackup/src/ds_encrypt.c	2013-08-18 06:54:14 +0000
+++ xtrabackup/src/ds_encrypt.c	2013-10-17 16:35:32 +0000
@@ -42,6 +42,7 @@
 	const char 		*from;
 	size_t			from_len;
 	char			*to;
+	char			*iv;
 	size_t			to_len;
 	gcry_cipher_hd_t	cipher_handle;
 } crypt_thread_ctxt_t;
@@ -171,6 +172,8 @@
 	xb_a(encrypt_iv_len > 0);
 	xb_a(encrypt_iv_len <= sizeof(encrypt_iv));
 
+	xb_crypt_init_iv();
+
 	/* Now set up the key */
 	if (xtrabackup_encrypt_key == NULL &&
 	    xtrabackup_encrypt_key_file == NULL) {
@@ -334,7 +337,9 @@
 			if (xb_crypt_write_chunk(crypt_file->xbcrypt_file,
 						 threads[i].to,
 						 threads[i].from_len,
-						 threads[i].to_len)) {
+						 threads[i].to_len,
+						 threads[i].iv,
+						 encrypt_iv_len)) {
 				msg("encrypt: write to the destination file "
 				    "failed.\n");
 				return 1;
@@ -411,6 +416,9 @@
 		thd->to = (char *) my_malloc(XB_CRYPT_CHUNK_SIZE,
 						   MYF(MY_FAE));
 
+		thd->iv = (char *) my_malloc(encrypt_iv_len,
+						   MYF(MY_FAE));
+
 		/* Initialize the control mutex and condition var */
 		if (pthread_mutex_init(&thd->ctrl_mutex, NULL) ||
 		    pthread_cond_init(&thd->ctrl_cond, NULL)) {
@@ -501,6 +509,7 @@
 			gcry_cipher_close(thd->cipher_handle);
 
 		MY_FREE(thd->to);
+		MY_FREE(thd->iv);
 	}
 
 	MY_FREE(threads);
@@ -521,7 +530,6 @@
 
 	pthread_mutex_unlock(&thd->ctrl_mutex);
 
-
 	while (1) {
 		thd->data_avail = FALSE;
 		pthread_cond_signal(&thd->data_cond);
@@ -548,8 +556,9 @@
 				continue;
 			}
 
+			xb_crypt_create_iv(thd->iv, encrypt_iv_len);
 			gcry_error = gcry_cipher_setiv(thd->cipher_handle,
-							encrypt_iv,
+							thd->iv,
 							encrypt_iv_len);
 			if (gcry_error) {
 				msg("encrypt: unable to set cipher ctr - "

=== modified file 'xtrabackup/src/xbcrypt.c'
--- xtrabackup/src/xbcrypt.c	2013-08-18 06:54:14 +0000
+++ xtrabackup/src/xbcrypt.c	2013-10-17 16:35:32 +0000
@@ -56,7 +56,7 @@
 static int		encrypt_algo = 0;
 static int		encrypt_mode = GCRY_CIPHER_MODE_CTR;
 static uint 		encrypt_key_len = 0;
-static const unsigned char encrypt_iv[] =
+static const unsigned char v1_encrypt_iv[] =
 			    "Percona Xtrabackup is Awesome!!!";
 static size_t 		encrypt_iv_len = 0;
 
@@ -294,6 +294,8 @@
 	void			*chunkbuf = NULL;
 	size_t			chunksize;
 	size_t			originalsize;
+	void			*ivbuf = NULL;
+	size_t			ivsize;
 	void			*decryptbuf = NULL;
 	size_t			decryptbufsize = 0;
 	ulonglong		ttlchunksread = 0;
@@ -335,7 +337,8 @@
 
 	/* Walk the encrypted chunks, decrypting them and writing out */
 	while ((result = xb_crypt_read_chunk(xbcrypt_file, &chunkbuf,
-					     &originalsize, &chunksize))
+					     &originalsize, &chunksize,
+					     &ivbuf, &ivsize))
 		== XB_CRYPT_READ_CHUNK) {
 
 		if (encrypt_algo != GCRY_CIPHER_NONE) {
@@ -348,9 +351,15 @@
 				goto err;
 			}
 
-			gcry_error = gcry_cipher_setiv(cipher_handle,
-							encrypt_iv,
-							encrypt_iv_len);
+			if (ivsize) {
+				gcry_error = gcry_cipher_setiv(cipher_handle,
+							       ivbuf,
+							       ivsize);
+			} else {
+				gcry_error = gcry_cipher_setiv(cipher_handle,
+							       v1_encrypt_iv,
+							       encrypt_iv_len);
+			}
 			if (gcry_error) {
 				msg("%s:decrypt: unable to set cipher iv - "
 				    "%s : %s\n", my_progname,
@@ -448,6 +457,7 @@
 	size_t			bytesread;
 	size_t			chunkbuflen;
 	void			*chunkbuf = NULL;
+	void			*ivbuf = NULL;
 	size_t			encryptbuflen = 0;
 	size_t			encryptedlen = 0;
 	void			*encryptbuf = NULL;
@@ -491,7 +501,11 @@
 		goto err;
 	}
 
-	/* now read in data in chunk size, encryptand write out */
+	ivbuf = my_malloc(encrypt_iv_len, MYF(MY_FAE));
+
+	xb_crypt_init_iv();
+
+	/* now read in data in chunk size, encrypt and write out */
 	chunkbuflen = opt_encrypt_chunk_size;
 	chunkbuf = my_malloc(chunkbuflen, MYF(MY_FAE));
 	while ((bytesread = my_read(filein, chunkbuf, chunkbuflen,
@@ -507,8 +521,10 @@
 				    gcry_strerror(gcry_error));
 				goto err;
 			}
+
+			xb_crypt_create_iv(ivbuf, encrypt_iv_len);
 			gcry_error = gcry_cipher_setiv(cipher_handle,
-							encrypt_iv,
+							ivbuf,
 							encrypt_iv_len);
 
 			if (gcry_error) {
@@ -553,7 +569,8 @@
 			encryptbuf = chunkbuf;
 		}
 
-		if (xb_crypt_write_chunk(xbcrypt_file, encryptbuf, bytesread, encryptedlen)) {
+		if (xb_crypt_write_chunk(xbcrypt_file, encryptbuf, bytesread,
+					 encryptedlen, ivbuf, encrypt_iv_len)) {
 			msg("%s:encrypt: abcrypt_write_chunk() failed.\n",
 			    my_progname);
 			goto err;
@@ -568,6 +585,7 @@
 			    ttlbyteswritten);
 	}
 
+	MY_FREE(ivbuf);
 	MY_FREE(chunkbuf);
 
 	if (encryptbuf && encryptbuflen)

=== modified file 'xtrabackup/src/xbcrypt.h'
--- xtrabackup/src/xbcrypt.h	2013-08-18 06:54:14 +0000
+++ xtrabackup/src/xbcrypt.h	2013-10-17 16:35:32 +0000
@@ -24,8 +24,9 @@
 #include <my_base.h>
 #include "common.h"
 
-#define XB_CRYPT_CHUNK_MAGIC "XBCRYP01"
-#define XB_CRYPT_CHUNK_MAGIC_SIZE (sizeof(XB_CRYPT_CHUNK_MAGIC)-1)
+#define XB_CRYPT_CHUNK_MAGIC1 "XBCRYP01"
+#define XB_CRYPT_CHUNK_MAGIC2 "XBCRYP02" /* must be same size as ^^ */
+#define XB_CRYPT_CHUNK_MAGIC_SIZE (sizeof(XB_CRYPT_CHUNK_MAGIC1)-1)
 
 /******************************************************************************
 Write interface */
@@ -38,11 +39,11 @@
 xb_wcrypt_t *xb_crypt_write_open(void *userdata,
 				 xb_crypt_write_callback *onwrite);
 
-/* Takes buffer, original length and encrypted length, formats output buffer
-   and calls write callback.
+/* Takes buffer, original length, encrypted length iv and iv length, formats
+   output buffer and calls write callback.
    Returns 0 on success, 1 on error */
 int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen,
-			 size_t elen);
+			 size_t elen, const void *iv, size_t ivlen);
 
 /* Returns 0 on success, 1 on error */
 int xb_crypt_write_close(xb_wcrypt_t *crypt);
@@ -51,7 +52,7 @@
 Read interface */
 typedef struct xb_rcrypt_struct xb_rcrypt_t;
 
-/* Callback on write for i/o, must return # of bytes read or -1 on error */
+/* Callback on read for i/o, must return # of bytes read or -1 on error */
 typedef ssize_t xb_crypt_read_callback(void *userdata,
 				       void *buf, size_t len, int flags);
 
@@ -65,7 +66,8 @@
 } xb_rcrypt_result_t;
 
 xb_rcrypt_result_t xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf,
-				       size_t *olen, size_t *elen);
+				       size_t *olen, size_t *elen, void **iv,
+				       size_t *ivlen);
 
 int xb_crypt_read_close(xb_rcrypt_t *crypt);
 
@@ -74,4 +76,7 @@
 my_bool xb_crypt_read_key_file(const char *filename,
 			       void** key, uint *keylength);
 
+void xb_crypt_init_iv();
+
+void xb_crypt_create_iv(void* ivbuf, size_t ivlen);
 #endif

=== modified file 'xtrabackup/src/xbcrypt_common.c'
--- xtrabackup/src/xbcrypt_common.c	2013-08-18 06:54:14 +0000
+++ xtrabackup/src/xbcrypt_common.c	2013-10-17 16:35:32 +0000
@@ -23,7 +23,6 @@
 #include "xbcrypt.h"
 #include <gcrypt.h>
 
-
 my_bool
 xb_crypt_read_key_file(const char *filename, void** key, uint *keylength)
 {
@@ -43,3 +42,25 @@
 	my_fclose(fp, MYF(0));
 	return TRUE;
 }
+
+
+void 
+xb_crypt_init_iv()
+{
+	uint seed = time(NULL);
+	srandom(seed);
+}
+
+void
+xb_crypt_create_iv(void* ivbuf, size_t ivlen)
+{
+	size_t i;
+	ulong rndval;
+
+	for (i = 0; i < ivlen; i++) {
+		if (i % 4 == 0) {
+			rndval = (ulong) random();
+		}
+		((uchar*)ivbuf)[i] = ((uchar*)&rndval)[i % 4];
+	}
+}

=== modified file 'xtrabackup/src/xbcrypt_read.c'
--- xtrabackup/src/xbcrypt_read.c	2013-08-18 06:54:14 +0000
+++ xtrabackup/src/xbcrypt_read.c	2013-10-17 16:35:32 +0000
@@ -25,6 +25,8 @@
 	xb_crypt_read_callback		*read;
 	void				*buffer;
 	size_t				bufsize;
+	void				*ivbuffer;
+	size_t				ivbufsize;
 	ulonglong			offset;
 };
 
@@ -42,17 +44,20 @@
 	crypt->buffer = NULL;
 	crypt->bufsize = 0;
 	crypt->offset = 0;
-
+	crypt->ivbuffer = NULL;
+	crypt->ivbufsize = 0;
 	return crypt;
 }
 
 xb_rcrypt_result_t
-xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, size_t *olen, size_t *elen)
+xb_crypt_read_chunk(xb_rcrypt_t *crypt, void **buf, size_t *olen, size_t *elen,
+		    void **iv, size_t *ivlen)
+
 {
 	uchar		tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4];
 	uchar		*ptr;
 	ulonglong	tmp;
-	ulong		checksum, checksum_exp;
+	ulong		checksum, checksum_exp, version;
 	ssize_t		bytesread;
 	xb_rcrypt_result_t result = XB_CRYPT_READ_CHUNK;
 
@@ -72,8 +77,11 @@
 
 	ptr = tmpbuf;
 
-	if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC, XB_CRYPT_CHUNK_MAGIC_SIZE)
-	    != 0) {
+	if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC1, XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
+		version = 1;
+	} else if (memcmp(ptr, XB_CRYPT_CHUNK_MAGIC2, XB_CRYPT_CHUNK_MAGIC_SIZE) == 0) {
+		version = 2;
+	} else {
 		msg("%s:%s: wrong chunk magic at offset 0x%llx.\n",
 		    my_progname, __FUNCTION__, crypt->offset);
 		result = XB_CRYPT_READ_ERROR;
@@ -113,6 +121,72 @@
 	ptr += 4;
 	crypt->offset += 4;
 
+	/* iv size */
+	if (version == 1) {
+		*ivlen = 0;
+		*iv = 0;
+	} else {
+		if ((bytesread = crypt->read(crypt->userdata, tmpbuf, 8,
+					     MYF(MY_WME))) != 8) {
+			if (bytesread == 0) {
+				result = XB_CRYPT_READ_EOF;
+				goto err;
+			} else {
+				msg("%s:%s: unable to read chunk iv size at "
+				    "offset 0x%llx.\n",
+				    my_progname, __FUNCTION__, crypt->offset);
+				result =  XB_CRYPT_READ_ERROR;
+				goto err;
+			}
+		}
+
+		tmp = uint8korr(tmpbuf);
+		if (tmp > INT_MAX) {
+			msg("%s:%s: invalid iv size at offset 0x%llx.\n",
+			    my_progname, __FUNCTION__, crypt->offset);
+			result = XB_CRYPT_READ_ERROR;
+			goto err;
+		}
+		crypt->offset += 8;
+		*ivlen = (size_t)tmp;
+	}
+
+	if (*ivlen > crypt->ivbufsize) {
+		if (crypt->ivbuffer) {
+			crypt->ivbuffer = my_realloc(crypt->ivbuffer, *ivlen,
+						   MYF(MY_WME));
+			if (crypt->ivbuffer == NULL) {
+				msg("%s:%s: failed to increase iv buffer to "
+				    "%llu bytes.\n", my_progname, __FUNCTION__,
+				    (ulonglong)*ivlen);
+				result = XB_CRYPT_READ_ERROR;
+				goto err;
+			}
+		} else {
+			crypt->ivbuffer = my_malloc(*ivlen, MYF(MY_WME));
+			if (crypt->ivbuffer == NULL) {
+				msg("%s:%s: failed to allocate iv buffer of "
+				    "%llu bytes.\n", my_progname, __FUNCTION__,
+				    (ulonglong)*ivlen);
+				result = XB_CRYPT_READ_ERROR;
+				goto err;
+			}
+		}
+		crypt->ivbufsize = *ivlen;
+	}
+
+	if (*ivlen > 0) {
+		if (crypt->read(crypt->userdata, crypt->ivbuffer, *ivlen, MYF(MY_WME|MY_FULL_IO))
+		    != (ssize_t)*ivlen) {
+			msg("%s:%s: failed to read %lld bytes for chunk iv "
+			    "at offset 0x%llx.\n", my_progname, __FUNCTION__,
+			    (ulonglong)*ivlen, crypt->offset);
+			result = XB_CRYPT_READ_ERROR;
+			goto err;
+		}
+		*iv = crypt->ivbuffer;
+	}
+
 	if (*olen > crypt->bufsize) {
 		if (crypt->buffer) {
 			crypt->buffer = my_realloc(crypt->buffer, *olen,
@@ -165,6 +239,8 @@
 	*buf = NULL;
 	*olen = 0;
 	*elen = 0;
+	*ivlen = 0;
+	*iv = 0;
 exit:
 	return result;
 }

=== modified file 'xtrabackup/src/xbcrypt_write.c'
--- xtrabackup/src/xbcrypt_write.c	2013-08-18 06:54:14 +0000
+++ xtrabackup/src/xbcrypt_write.c	2013-10-17 16:35:32 +0000
@@ -41,9 +41,9 @@
 }
 
 int xb_crypt_write_chunk(xb_wcrypt_t *crypt, const void *buf, size_t olen,
-			 size_t elen)
+			 size_t elen, const void *iv, size_t ivlen)
 {
-	uchar		tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4];
+	uchar		tmpbuf[XB_CRYPT_CHUNK_MAGIC_SIZE + 8 + 8 + 8 + 4 + 8];
 	uchar		*ptr;
 	ulong		checksum;
 
@@ -55,9 +55,13 @@
 	if (elen > INT_MAX)
 		return 0;
 
+	xb_ad(ivlen <= INT_MAX);
+	if (ivlen > INT_MAX)
+		return 0;
+
 	ptr = tmpbuf;
 
-	memcpy(ptr, XB_CRYPT_CHUNK_MAGIC, XB_CRYPT_CHUNK_MAGIC_SIZE);
+	memcpy(ptr, XB_CRYPT_CHUNK_MAGIC2, XB_CRYPT_CHUNK_MAGIC_SIZE);
 	ptr += XB_CRYPT_CHUNK_MAGIC_SIZE;
 
 	int8store(ptr, (ulonglong)0);	/* reserved */
@@ -73,11 +77,17 @@
 	int4store(ptr, checksum);	/* checksum */
 	ptr += 4;
 
+	int8store(ptr, (ulonglong)ivlen); /* iv size */
+	ptr += 8;
+
 	xb_ad(ptr <= tmpbuf + sizeof(tmpbuf));
 
 	if (crypt->write(crypt->userdata, tmpbuf, ptr-tmpbuf) == -1)
 		return 1;
 
+	if (crypt->write(crypt->userdata, iv, ivlen) == -1)
+		return 1;
+
 	if (crypt->write(crypt->userdata, buf, elen) == -1)
 		return 1;
 

=== added file 'xtrabackup/test/inc/decrypt_v1_test_file.txt'
--- xtrabackup/test/inc/decrypt_v1_test_file.txt	1970-01-01 00:00:00 +0000
+++ xtrabackup/test/inc/decrypt_v1_test_file.txt	2013-10-17 16:35:32 +0000
@@ -0,0 +1,9 @@
+*******************************************************************************
+*    This is just a simple text file with enough data to make sure to fill a  *
+*     full encryption window. It will be stored in the source tree in both    *
+*  encrypted in v1 format and decrypted for comparison after decrypting with  *
+* the xbcrypt utility to validate that newer xbcrypt decryptions can properly *
+*     read and decrypt files encrypted with older versions of xtrabackup.     *
+*   The encrypted version of this file will be encrypted with AES256 and the  *
+*                      key percona_xtrabackup_is_awesome___                   *
+*******************************************************************************

=== added file 'xtrabackup/test/inc/decrypt_v1_test_file.xbcrypt'
Binary files xtrabackup/test/inc/decrypt_v1_test_file.xbcrypt	1970-01-01 00:00:00 +0000 and xtrabackup/test/inc/decrypt_v1_test_file.xbcrypt	2013-10-17 16:35:32 +0000 differ
=== added file 'xtrabackup/test/t/xbcrypt.sh'
--- xtrabackup/test/t/xbcrypt.sh	1970-01-01 00:00:00 +0000
+++ xtrabackup/test/t/xbcrypt.sh	2013-10-17 16:35:32 +0000
@@ -0,0 +1,24 @@
+############################################################################
+# A few tests for encryption:
+#  1 - Test that a file that goes through the encrypt/decrypt cycle is
+#      exactly the same after decryption as the original.
+#  2 - Test that files encrypted with prior versions of xbcrypt can be 
+#      correctly decrypted. Introduced when fixing bug 1185343 - Fixed IV 
+#      used in Xtrabackup encryption
+############################################################################
+
+encrypt_algo="AES256"
+encrypt_key="percona_xtrabackup_is_awesome___"
+
+# test that decrypted file is identical to original
+run_cmd xbcrypt -i inc/decrypt_v1_test_file.txt -o ${topdir}/decrypt_v1_test_file.xbcrypt -a ${encrypt_algo} -k ${encrypt_key}
+run_cmd xbcrypt -d -i ${topdir}/decrypt_v1_test_file.xbcrypt -o ${topdir}/decrypt_v1_test_file.txt -a ${encrypt_algo} -k ${encrypt_key}
+run_cmd cmp inc/decrypt_v1_test_file.txt ${topdir}/decrypt_v1_test_file.txt
+rm ${topdir}/decrypt_v1_test_file.xbcrypt
+rm ${topdir}/decrypt_v1_test_file.txt
+
+
+# test that file encrypted w/ v1 can be decrypted perfectly w/ v2
+run_cmd xbcrypt -d -i inc/decrypt_v1_test_file.xbcrypt -o ${topdir}/decrypt_v1_test_file.txt -a ${encrypt_algo} -k ${encrypt_key}
+run_cmd cmp inc/decrypt_v1_test_file.txt ${topdir}/decrypt_v1_test_file.txt
+rm ${topdir}/decrypt_v1_test_file.txt