Merge lp:~laurynas-biveinis/percona-server/csv-engine-ietf-quotes-5.6 into lp:percona-server/5.6

[Laurynas Biveinis]

Merge csv-engine-ietf-quotes blueprint implementation / bug 1316042
fix from 5.5, implementing
https://blueprints.launchpad.net/percona-server/+spec/csv-engine-ietf-quotes-5.6.

An automerge, with an exception of percona_server_variables_debug and
percona_server_variables_release null merge.

    http://jenkins.percona.com/job/percona-server-5.6-param/725/

[Sergei Glushchenko (sergei.glushchenko)]

Approve

review: Approve (g2)

=== modified file 'mysql-test/r/csv.result'
--- mysql-test/r/csv.result	2012-08-22 01:40:20 +0000
+++ mysql-test/r/csv.result	2014-10-02 09:29:18 +0000
@@ -5470,3 +5470,47 @@
 ERROR HY000: Table 't1' is marked as crashed and should be repaired
 DROP TABLE t1;
 End of 5.1 tests
+SET @old_csv_mode = @@SESSION.csv_mode;
+SET @@SESSION.csv_mode='ietf_quotes';
+CREATE TABLE t1(c1 TEXT NOT NULL, c2 TEXT NOT NULL) ENGINE=CSV;
+INSERT INTO t1 VALUES("a\"b,c","d");
+INSERT INTO t1 VALUES("d","a\"b,c");
+INSERT INTO t1 VALUES(",\"a","e");
+INSERT INTO t1 VALUES("e",",\"a");
+INSERT INTO t1 VALUES("\"","f");
+INSERT INTO t1 VALUES("f","\"");
+INSERT INTO t1 VALUES(",","g");
+INSERT INTO t1 VALUES("g",",");
+SELECT * FROM t1;
+c1	c2
+a"b,c	d
+d	a"b,c
+,"a	e
+e	,"a
+"	f
+f	"
+,	g
+g	,
+CSV file contents:
+"a""b,c","d"
+"d","a""b,c"
+",""a","e"
+"e",",""a"
+"""","f"
+"f",""""
+",","g"
+"g",","
+DROP TABLE t1;
+CREATE TABLE t1(c1 TEXT NOT NULL, c2 TEXT NOT NULL) ENGINE=CSV;
+Replacing t1.CSV
+SELECT * FROM t1;
+c1	c2
+a	b
+a	b
+a"b,c	d
+d	,"a
+a",	e
+e	"
+,	f
+DROP TABLE t1;
+SET @@SESSION.csv_mode=@old_csv_mode;

=== modified file 'mysql-test/r/mysqld--help-notwin.result'
--- mysql-test/r/mysqld--help-notwin.result	2014-09-26 04:43:24 +0000
+++ mysql-test/r/mysqld--help-notwin.result	2014-10-02 09:29:18 +0000
@@ -137,6 +137,8 @@
  --console           Write error output on screen; don't remove the console
  window on windows.
  --core-file         Write core on errors.
+ --csv-mode=name     Control CSV parser mode: []: default, ietf_quotes:
+ standards-compatible embedded quote and comma parsing
  -h, --datadir=name  Path to the database root directory
  --date-format=name  The DATE format (ignored)
  --datetime-format=name 
@@ -1139,6 +1141,7 @@
 concurrent-insert AUTO
 connect-timeout 10
 console FALSE
+csv-mode 
 date-format %Y-%m-%d
 datetime-format %Y-%m-%d %H:%i:%s
 default-storage-engine InnoDB

=== added file 'mysql-test/suite/sys_vars/r/csv_mode_basic.result'
--- mysql-test/suite/sys_vars/r/csv_mode_basic.result	1970-01-01 00:00:00 +0000
+++ mysql-test/suite/sys_vars/r/csv_mode_basic.result	2014-10-02 09:29:18 +0000
@@ -0,0 +1,55 @@
+SET @saved_csv_mode = @@SESSION.csv_mode;
+SET @@SESSION.csv_mode = 'ietf_quotes';
+SELECT @@SESSION.csv_mode;
+@@SESSION.csv_mode
+IETF_QUOTES
+SET @@SESSION.csv_mode = 'IETF_QUOTES';
+SELECT @@SESSION.csv_mode;
+@@SESSION.csv_mode
+IETF_QUOTES
+SET @@SESSION.csv_mode = IETF_QUOTES;
+SELECT @@SESSION.csv_mode;
+@@SESSION.csv_mode
+IETF_QUOTES
+SET @@SESSION.csv_mode = '';
+SELECT @@SESSION.csv_mode;
+@@SESSION.csv_mode
+
+SET @@SESSION.csv_mode = 0;
+SELECT @@SESSION.csv_mode;
+@@SESSION.csv_mode
+
+SET @@SESSION.csv_mode = 1;
+SELECT @@SESSION.csv_mode;
+@@SESSION.csv_mode
+IETF_QUOTES
+SET @@SESSION.csv_mode = NULL;
+ERROR 42000: Variable 'csv_mode' can't be set to the value of 'NULL'
+SET @@SESSION.csv_mode = FOO;
+ERROR 42000: Variable 'csv_mode' can't be set to the value of 'FOO'
+SET @@SESSION.csv_mode = 'foo';
+ERROR 42000: Variable 'csv_mode' can't be set to the value of 'foo'
+SELECT @@SESSION.csv_mode;
+@@SESSION.csv_mode
+IETF_QUOTES
+SET @@SESSION.csv_mode = 0;
+SELECT @@SESSION.csv_mode;
+@@SESSION.csv_mode
+
+SET @@SESSION.csv_mode = -1;
+ERROR 42000: Variable 'csv_mode' can't be set to the value of '-1'
+SET @@SESSION.csv_mode = 2;
+ERROR 42000: Variable 'csv_mode' can't be set to the value of '2'
+SET @@SESSION.csv_mode = @saved_csv_mode;
+SET @saved_csv_mode = @@GLOBAL.csv_mode;
+SET @@GLOBAL.csv_mode = '';
+SELECT @@GLOBAL.csv_mode;
+@@GLOBAL.csv_mode
+
+SET @@GLOBAL.csv_mode = IETF_QUOTES;
+SELECT @@GLOBAL.csv_mode;
+@@GLOBAL.csv_mode
+IETF_QUOTES
+SET @@GLOBAL.csv_mode = FOO;
+ERROR 42000: Variable 'csv_mode' can't be set to the value of 'FOO'
+SET @@GLOBAL.csv_mode = @saved_csv_mode;

=== added file 'mysql-test/suite/sys_vars/t/csv_mode_basic.test'
--- mysql-test/suite/sys_vars/t/csv_mode_basic.test	1970-01-01 00:00:00 +0000
+++ mysql-test/suite/sys_vars/t/csv_mode_basic.test	2014-10-02 09:29:18 +0000
@@ -0,0 +1,47 @@
+# Basic sys_vars test for csv_mode variable
+
+SET @saved_csv_mode = @@SESSION.csv_mode;
+
+# Valid values
+SET @@SESSION.csv_mode = 'ietf_quotes';
+SELECT @@SESSION.csv_mode;
+SET @@SESSION.csv_mode = 'IETF_QUOTES';
+SELECT @@SESSION.csv_mode;
+SET @@SESSION.csv_mode = IETF_QUOTES;
+SELECT @@SESSION.csv_mode;
+SET @@SESSION.csv_mode = '';
+SELECT @@SESSION.csv_mode;
+SET @@SESSION.csv_mode = 0;
+SELECT @@SESSION.csv_mode;
+SET @@SESSION.csv_mode = 1;
+SELECT @@SESSION.csv_mode;
+
+# Invalid values
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@SESSION.csv_mode = NULL;
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@SESSION.csv_mode = FOO;
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@SESSION.csv_mode = 'foo';
+SELECT @@SESSION.csv_mode;
+SET @@SESSION.csv_mode = 0;
+SELECT @@SESSION.csv_mode;
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@SESSION.csv_mode = -1;
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@SESSION.csv_mode = 2;
+
+SET @@SESSION.csv_mode = @saved_csv_mode;
+
+# Test GLOBAL variable
+
+SET @saved_csv_mode = @@GLOBAL.csv_mode;
+
+SET @@GLOBAL.csv_mode = '';
+SELECT @@GLOBAL.csv_mode;
+SET @@GLOBAL.csv_mode = IETF_QUOTES;
+SELECT @@GLOBAL.csv_mode;
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@GLOBAL.csv_mode = FOO;
+
+SET @@GLOBAL.csv_mode = @saved_csv_mode;

=== modified file 'mysql-test/t/csv.test'
--- mysql-test/t/csv.test	2011-12-04 19:11:16 +0000
+++ mysql-test/t/csv.test	2014-10-02 09:29:18 +0000
@@ -1895,3 +1895,47 @@
 
 DROP TABLE t1;
 --echo End of 5.1 tests
+
+#
+# Bug #71091 CSV engine does not properly process "", in quotes
+#
+
+SET @old_csv_mode = @@SESSION.csv_mode;
+SET @@SESSION.csv_mode='ietf_quotes';
+CREATE TABLE t1(c1 TEXT NOT NULL, c2 TEXT NOT NULL) ENGINE=CSV;
+
+INSERT INTO t1 VALUES("a\"b,c","d");
+INSERT INTO t1 VALUES("d","a\"b,c");
+INSERT INTO t1 VALUES(",\"a","e");
+INSERT INTO t1 VALUES("e",",\"a");
+INSERT INTO t1 VALUES("\"","f");
+INSERT INTO t1 VALUES("f","\"");
+INSERT INTO t1 VALUES(",","g");
+INSERT INTO t1 VALUES("g",",");
+
+SELECT * FROM t1;
+
+--echo CSV file contents:
+--cat_file $MYSQLD_DATADIR/test/t1.CSV
+
+DROP TABLE t1;
+
+CREATE TABLE t1(c1 TEXT NOT NULL, c2 TEXT NOT NULL) ENGINE=CSV;
+
+--echo Replacing t1.CSV
+--remove_file $MYSQLD_DATADIR/test/t1.CSV
+--write_file $MYSQLD_DATADIR/test/t1.CSV
+a,b
+"a","b"
+"a""b,c","d"
+"d",",""a"
+"a"",",e
+e,""""
+",",f
+EOF
+
+SELECT * FROM t1;
+
+DROP TABLE t1;
+
+SET @@SESSION.csv_mode=@old_csv_mode;

=== modified file 'storage/csv/ha_tina.cc'
--- storage/csv/ha_tina.cc	2012-08-22 01:40:20 +0000
+++ storage/csv/ha_tina.cc	2014-10-02 09:29:18 +0000
@@ -67,6 +67,23 @@
 #define CSN_EXT ".CSN"               // Files used during repair and update
 #define CSM_EXT ".CSM"               // Meta file
 
+enum csv_mode_enum { csv_mode_none= 0, csv_mode_ietf_quotes= 1 };
+
+static const char *csv_mode_names[]=
+{
+  "IETF_QUOTES", NullS
+};
+
+static TYPELIB csv_mode_typelib=
+{
+  array_elements(csv_mode_names) - 1, "",
+  csv_mode_names, NULL
+};
+
+static MYSQL_THDVAR_SET(mode, PLUGIN_VAR_RQCMDARG,
+  "Control CSV parser mode: []: default, ietf_quotes: "
+  "standards-compatible embedded quote and comma parsing",
+  NULL, NULL, 0, &csv_mode_typelib);
 
 static TINA_SHARE *get_share(const char *table_name, TABLE *table);
 static int free_share(TINA_SHARE *share);
@@ -507,7 +524,7 @@
   char attribute_buffer[1024];
   String attribute(attribute_buffer, sizeof(attribute_buffer),
                    &my_charset_bin);
-
+  bool ietf_quotes= THDVAR(current_thd, mode) & csv_mode_ietf_quotes;
   my_bitmap_map *org_bitmap= dbug_tmp_use_all_columns(table, table->read_set);
   buffer.length(0);
 
@@ -543,7 +560,7 @@
       {
         if (*ptr == '"')
         {
-          buffer.append('\\');
+          buffer.append(ietf_quotes ? '"' : '\\');
           buffer.append('"');
         }
         else if (*ptr == '\r')
@@ -635,6 +652,7 @@
   my_bitmap_map *org_bitmap;
   int error;
   bool read_all;
+  bool ietf_quotes= THDVAR(current_thd, mode) & csv_mode_ietf_quotes;
   DBUG_ENTER("ha_tina::find_current_row");
 
   free_root(&blobroot, MYF(0));
@@ -668,8 +686,10 @@
                      a) If end of current field is reached, move
                         to next field and jump to step 2.3
                      b) If current character is a \\ handle
-                        \\n, \\r, \\, \\"
-                     c) else append the current character into the buffer
+                        \\n, \\r, \\, and \\" if not in ietf_quotes mode
+                     c) if in ietf_quotes mode and the current character is
+                        a ", handle ""
+                     d) else append the current character into the buffer
                         before checking that EOL has not been reached.
           2.2) If the current character does not begin with a quote
                2.2.1) Until EOL has not been reached
@@ -710,15 +730,25 @@
           curr_offset+= 2;
           break;
         }
-        if (curr_char == '\\' && curr_offset != (end_offset - 1))
-        {
+        if (ietf_quotes && curr_char == '"'
+            && file_buff->get_value(curr_offset + 1) == '"')
+        {
+          /* Embedded IETF quote */
+          curr_offset++;
+          buffer.append('"');
+        }
+        else if (curr_char == '\\' && curr_offset != (end_offset - 1))
+        {
+          /* A quote followed by something else than a comma, end of line, or
+          (in IETF mode) another quote will be handled as a regular
+          character. */
           curr_offset++;
           curr_char= file_buff->get_value(curr_offset);
           if (curr_char == 'r')
             buffer.append('\r');
           else if (curr_char == 'n' )
             buffer.append('\n');
-          else if (curr_char == '\\' || curr_char == '"')
+          else if (curr_char == '\\' || (!ietf_quotes && curr_char == '"'))
             buffer.append(curr_char);
           else  /* This could only happed with an externally created file */
           {
@@ -1740,6 +1770,11 @@
   return COMPATIBLE_DATA_YES;
 }
 
+static struct st_mysql_sys_var* csv_system_variables[]= {
+  MYSQL_SYSVAR(mode),
+  NULL
+};
+
 struct st_mysql_storage_engine csv_storage_engine=
 { MYSQL_HANDLERTON_INTERFACE_VERSION };
 
@@ -1755,7 +1790,7 @@
   tina_done_func, /* Plugin Deinit */
   0x0100 /* 1.0 */,
   NULL,                       /* status variables                */
-  NULL,                       /* system variables                */
+  csv_system_variables,
   NULL,                       /* config options                  */
   0,                          /* flags                           */
 }