Merge lp:~zorba-coders/zorba/bug-878508 into lp:zorba

[Paul J. Lucas]

Now properly serializing JSON for JsonML.

Now properly serializing JSON for JsonML.

[Paul J. Lucas (paul-lucas)]

review: Approve

[Zorba Build Bot (zorba-buildbot)]

Validation queue starting for merge proposal.
Log at: http://zorbatest.lambda.nu:8080/remotequeue/bug-878508-2012-07-25T23-46-57.424Z/log.html

[Zorba Build Bot (zorba-buildbot)]

Validation queue job bug-878508-2012-07-25T23-46-57.424Z is finished. The final status was:

All tests succeeded!

[Zorba Build Bot (zorba-buildbot)]

Voting does not meet specified criteria. Required: Approve > 1, Disapprove < 1, Needs Fixing < 1, Pending < 1. Got: 1 Approve.

[lp:~zorba-coders/zorba/bug-878508]

10956. By Paul J. Lucas on 2012-07-26

Now emitting JSON supplemental plane characters as ordinary UTF-8.

10957. By Paul J. Lucas on 2012-07-26

Added ZORBA_JSON_EMIT_SURROGATES.

[Chris Hillery (ceejatec)]

review: Approve

[Zorba Build Bot (zorba-buildbot)]

Validation queue starting for merge proposal.
Log at: http://zorbatest.lambda.nu:8080/remotequeue/bug-878508-2012-07-26T07-37-01.736Z/log.html

[Zorba Build Bot (zorba-buildbot)]

Validation queue job bug-878508-2012-07-26T07-37-01.736Z is finished. The final status was:

All tests succeeded!

=== modified file 'ChangeLog'
--- ChangeLog	2012-07-24 08:48:48 +0000
+++ ChangeLog	2012-07-26 04:33:22 +0000
@@ -50,6 +50,7 @@
     var folding into if-then-else)
   * Fixed bug #1006166 (disabling 2 functions with the same qname)
   * Fixed bug #960083 (improper error handling of NaN comparisons)
+  * Fixed bug #878508 (JsonML serialization not escaping characters)
   * Xml-format plan serialization not supported anymore.
 
 

=== modified file 'src/api/serialization/serializer.cpp'
--- src/api/serialization/serializer.cpp	2012-07-25 00:15:29 +0000
+++ src/api/serialization/serializer.cpp	2012-07-26 04:33:22 +0000
@@ -31,9 +31,9 @@
 #include "api/unmarshaller.h"
 
 #include "util/ascii_util.h"
+#include "util/json_util.h"
 #include "util/string_util.h"
 #include "util/unicode_util.h"
-#include "util/utf8_string.h"
 #include "util/utf8_util.h"
 #include "util/xml_util.h"
 
@@ -1208,47 +1208,7 @@
 ********************************************************************************/
 void serializer::json_emitter::emit_json_string(zstring const &string)
 {
-  tr << '"';
-  utf8_string<zstring const> const u( string );
-  FOR_EACH( utf8_string<zstring const>, i, u ) {
-    unicode::code_point const cp = *i;
-    if ( ascii::is_cntrl( cp ) ) {
-      switch ( cp ) {
-        case '\b': tr << "\\b"; break;
-        case '\f': tr << "\\f"; break;
-        case '\n': tr << "\\n"; break;
-        case '\r': tr << "\\r"; break;
-        case '\t': tr << "\\t"; break;
-        default: {
-          std::ostringstream oss;
-          oss << std::hex << std::setfill('0') << "\\u" << std::setw(4) << cp;
-          tr << oss.str();
-        }
-      }
-      continue;
-    }
-    if ( unicode::is_supplementary_plane( cp ) ) {
-      unsigned high, low;
-      unicode::convert_surrogate( cp, &high, &low );
-      std::ostringstream oss;
-      oss << std::hex << std::setfill('0')
-          << "\\u" << std::setw(4) << high
-          << "\\u" << std::setw(4) << low;
-      tr << oss.str();
-      continue;
-    }
-    switch ( cp ) {
-      case '\\':
-      case '"':
-        tr << '\\';
-        // no break;
-      default: {
-        utf8::encoded_char_type ec;
-        tr.write( ec, utf8::encode( cp, ec ) );
-      }
-    }
-  }
-  tr << '"';
+  tr << '"' << json::serialize( string ) << '"';
 }
 
 

=== modified file 'src/runtime/json/jsonml_array.cpp'
--- src/runtime/json/jsonml_array.cpp	2012-07-24 08:48:48 +0000
+++ src/runtime/json/jsonml_array.cpp	2012-07-26 04:33:22 +0000
@@ -26,6 +26,7 @@
 #include "util/ascii_util.h"
 #include "util/cxx_util.h"
 #include "util/json_parser.h"
+#include "util/json_util.h"
 #include "util/mem_streambuf.h"
 #include "util/omanip.h"
 #include "util/oseparator.h"
@@ -199,7 +200,7 @@
     
     o << '"' << att_name << '"'
       << if_emit( ws, ' ' ) << ':' << if_emit( ws, ' ' )
-      << '"' << att_item->getStringValue() << '"';
+      << '"' << json::serialize( att_item->getStringValue() ) << '"';
   }
   i->close();
   if ( emitted_attributes )
@@ -242,7 +243,7 @@
         o << sep << serialize_element( child, sep, ws );
         break;
       case store::StoreConsts::textNode:
-        o << sep << '"' << child->getStringValue() << '"';
+        o << sep << '"' << json::serialize( child->getStringValue() ) << '"';
         break;
       default:
         break;

=== modified file 'src/runtime/json/snelson.cpp'
--- src/runtime/json/snelson.cpp	2012-02-14 03:41:58 +0000
+++ src/runtime/json/snelson.cpp	2012-07-26 04:33:22 +0000
@@ -16,6 +16,7 @@
 #include "stdafx.h"
 
 #include <sstream>
+#include <string>
 
 #include <zorba/diagnostic_list.h>
 
@@ -27,6 +28,7 @@
 #include "util/cxx_util.h"
 #include "util/indent.h"
 #include "util/json_parser.h"
+#include "util/json_util.h"
 #include "util/mem_streambuf.h"
 #include "util/omanip.h"
 #include "util/oseparator.h"
@@ -85,16 +87,6 @@
 #define POP_ITEM_ELEMENT()  \
   if ( !IN_STATE( in_array ) ) ; else POP_ITEM()
 
-static void escape_json_chars( zstring *s ) {
-  ascii::replace_all( *s, "\"", 1, "\\\"", 2 );
-  ascii::replace_all( *s, "\\", 1, "\\\\", 2 );
-  ascii::replace_all( *s, "\b", 1, "\\b", 2 );
-  ascii::replace_all( *s, "\f", 1, "\\f", 2 );
-  ascii::replace_all( *s, "\n", 1, "\\n", 2 );
-  ascii::replace_all( *s, "\r", 1, "\\r", 2 );
-  ascii::replace_all( *s, "\t", 1, "\\t", 2 );
-}
-
 ///////////////////////////////////////////////////////////////////////////////
 
 namespace snelson {
@@ -172,10 +164,6 @@
       case json::token::string:
         ADD_TYPE_ATTRIBUTE( "string" );
         value = token.get_value();
-#if 0
-        escape_json_chars( &value );
-#endif
-
         if ( next_string_is_key ) {
           // <pair name="..." ...>
           GENV_ITEMFACTORY->createQName( element_name, SNELSON_NS, "", "pair" );
@@ -344,10 +332,9 @@
 DEF_OMANIP1( serialize_number, zstring const& )
 
 static ostream& serialize_string( ostream &o, zstring const &s ) {
-  zstring temp( s );
-  escape_json_chars( &temp );
-  temp.insert( (zstring::size_type)0, 1, '"' );
-  temp.append( 1, '"' );
+  ostringstream oss;
+  oss << '"' << json::serialize( s ) << '"';
+  string const temp( oss.str() );
   assert_json_type( json::string, temp );
   return o << temp;
 }

=== modified file 'src/util/CMakeLists.txt'
--- src/util/CMakeLists.txt	2012-07-24 08:48:48 +0000
+++ src/util/CMakeLists.txt	2012-07-26 04:33:22 +0000
@@ -21,6 +21,7 @@
   fs_util.cpp
   indent.cpp
   json_parser.cpp
+  json_util.cpp
   mem_streambuf.cpp
   regex.cpp
   string_util.cpp

=== added file 'src/util/json_util.cpp'
--- src/util/json_util.cpp	1970-01-01 00:00:00 +0000
+++ src/util/json_util.cpp	2012-07-26 04:33:22 +0000
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2006-2008 The FLWOR Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <iomanip>
+#include <iostream>
+
+//#define ZORBA_JSON_EMIT_SURROGATES
+
+#include "util/stl_util.h"
+#ifdef ZORBA_JSON_EMIT_SURROGATES
+#include "util/unicode_util.h"
+#endif /* ZORBA_JSON_EMIT_SURROGATES */
+#include "util/utf8_util.h"
+
+using namespace std;
+
+namespace zorba {
+namespace json {
+
+///////////////////////////////////////////////////////////////////////////////
+
+ostream& serialize( ostream &os, char const *s ) {
+  while ( true ) {
+    unicode::code_point const cp = utf8::next_char( s );
+    if ( !cp )
+      break;
+    if ( ascii::is_cntrl( cp ) ) {
+      switch ( cp ) {
+        case '\b': os << "\\b"; break;
+        case '\f': os << "\\f"; break;
+        case '\n': os << "\\n"; break;
+        case '\r': os << "\\r"; break;
+        case '\t': os << "\\t"; break;
+        default: {
+          std::ostringstream oss;
+          oss << std::hex << std::setfill('0') << "\\u" << std::setw(4) << cp;
+          os << oss.str();
+        }
+      }
+      continue;
+    }
+#ifdef ZORBA_JSON_EMIT_SURROGATES
+    if ( unicode::is_supplementary_plane( cp ) ) {
+      unsigned high, low;
+      unicode::convert_surrogate( cp, &high, &low );
+      std::ostringstream oss;
+      oss << std::hex << std::setfill('0')
+          << "\\u" << std::setw(4) << high
+          << "\\u" << std::setw(4) << low;
+      os << oss.str();
+      continue;
+    }
+#endif /* ZORBA_JSON_EMIT_SURROGATES */
+    switch ( cp ) {
+      case '\\':
+      case '"':
+        os << '\\';
+        // no break;
+      default: {
+        utf8::encoded_char_type ec;
+        os.write( ec, utf8::encode( cp, ec ) );
+      }
+    }
+  }
+  return os;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+} // namespace json
+} // namespace zorba
+/* vim:set et sw=2 ts=2: */

=== added file 'src/util/json_util.h'
--- src/util/json_util.h	1970-01-01 00:00:00 +0000
+++ src/util/json_util.h	2012-07-26 04:33:22 +0000
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2006-2008 The FLWOR Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ZORBA_JSON_UTIL_H
+#define ZORBA_JSON_UTIL_H
+
+#include <iostream>
+
+#include <zorba/config.h>
+
+#include "string_util.h"
+#include "omanip.h"
+
+namespace zorba {
+namespace json {
+
+///////////////////////////////////////////////////////////////////////////////
+
+/**
+ * Serializes the given string as a valid JSON string: any characters that must
+ * be escaped are escaped.
+ *
+ * @param os The ostream to serialize to.
+ * @param s The C string to serialize as JSON.
+ * @return Returns \a os.
+ */
+std::ostream& serialize( std::ostream &os, char const *s );
+
+// An ostream manipulator version of the above.
+DEF_OMANIP1( serialize, char const* )
+
+/**
+ * Serializes the given string as a valid JSON string: any characters that must
+ * be escaped are escaped.
+ *
+ * @tparam The string type.
+ * @param os The ostream to serialize to.
+ * @param s The string to serialize as JSON.
+ * @return Returns \a os.
+ */
+template<class StringType> inline
+typename std::enable_if<ztd::has_c_str<StringType,
+                          char const* (StringType::*)() const>::value,
+                        std::ostream&>::type
+serialize( std::ostream &os, StringType const &s ) {
+  return serialize( os, s.c_str() );
+}
+
+// An ostream manipulator version of the above.
+template<class StringType>
+DEF_OMANIP1( serialize, StringType const& )
+
+///////////////////////////////////////////////////////////////////////////////
+
+} // namespace json
+} // namespace zorba
+
+#endif /* ZORBA_JSON_UTIL_H */
+/* vim:set et sw=2 ts=2: */

=== removed file 'test/rbkt/ExpQueryResults/zorba/jsoniq/supplemental_plane.xml.res'
--- test/rbkt/ExpQueryResults/zorba/jsoniq/supplemental_plane.xml.res	2012-07-19 00:13:14 +0000
+++ test/rbkt/ExpQueryResults/zorba/jsoniq/supplemental_plane.xml.res	1970-01-01 00:00:00 +0000
@@ -1,1 +0,0 @@
-{ "string" : "\ud83d\udc4a" }

=== removed file 'test/rbkt/Queries/zorba/jsoniq/supplemental_plane.xq'
--- test/rbkt/Queries/zorba/jsoniq/supplemental_plane.xq	2012-07-19 00:13:14 +0000
+++ test/rbkt/Queries/zorba/jsoniq/supplemental_plane.xq	1970-01-01 00:00:00 +0000
@@ -1,2 +0,0 @@
-let $s := "&#128074;"
-return { "string": $s }