Merge lp:~jderose/microfiber/ensure_ascii into lp:microfiber

[Jason Gerard DeRose]

* Adds non-ascii values in _json_body() test

* Fixes _json_body() so it uses ensure_ascii=False

* Adds non-ascii values in _queryiter() test

* Fixes _queryiter() so it uses ensure_ascii=False

* Fixes Database.dump() so it uses ensure_ascii=False, even though it's not an official part of the API yet, doesn't have tests yet

[David Jordan (dmj726)]

Good catch!

review: Approve

=== modified file 'microfiber.py'
--- microfiber.py	2012-07-26 09:23:56 +0000
+++ microfiber.py	2012-07-31 06:38:23 +0000
@@ -133,7 +133,11 @@
         return None
     if isinstance(obj, (bytes, BufferedReader)):
         return obj
-    return json.dumps(obj, sort_keys=True, separators=(',',':')).encode('utf-8')
+    return json.dumps(obj,
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(',',':'),
+    ).encode('utf-8')
 
 
 def _queryiter(options):
@@ -146,7 +150,7 @@
     for key in sorted(options):
         value = options[key]
         if key in ('key', 'startkey', 'endkey') or not isinstance(value, str):
-            value = json.dumps(value)
+            value = json.dumps(value, ensure_ascii=False)
         yield (key, value)
 
 
@@ -834,7 +838,7 @@
         rows = self.get('_all_docs')['rows']
         iterable = iter_all_docs(rows, self, attachments)
         docs = FakeList(len(rows), iterable)
-        json.dump({'docs': docs}, fp, sort_keys=True, indent=4, separators=(',', ': '))
+        json.dump({'docs': docs}, fp, ensure_ascii=False, sort_keys=True, indent=4, separators=(',', ': '))
         
     def load(self, fp):
         return self.post(fp, '_bulk_docs')

=== modified file 'test_microfiber.py'
--- test_microfiber.py	2012-07-26 09:23:56 +0000
+++ test_microfiber.py	2012-07-31 06:38:23 +0000
@@ -120,14 +120,31 @@
         self.assertIsInstance(b, bytes)
         self.assertEqual(len(b) * 8, 80)
 
+    def test_json(self):
+        """
+        Test our assumptions about json.dumps().
+        """
+        tm = '™'
+        self.assertEqual(json.dumps(tm), '"\\u2122"')
+        self.assertEqual(json.dumps(tm, ensure_ascii=False), '"™"')
+
     def test_json_body(self):
         doc = {
             '_id': 'foo',
             'bar': 'baz',
             'hello': 'world',
+            'name': 'Jon Åslund',
         }
-        json_str = json.dumps(doc, sort_keys=True, separators=(',',':'))
-        json_str2 = json.dumps(json_str, sort_keys=True, separators=(',',':'))
+        json_str = json.dumps(doc,
+            ensure_ascii=False,
+            sort_keys=True,
+            separators=(',',':'),
+        )
+        json_str2 = json.dumps(json_str,
+            ensure_ascii=False,
+            sort_keys=True,
+            separators=(',',':'),
+        )
         json_bytes = json_str.encode('utf-8')
 
         # Test with obj=None
@@ -141,11 +158,15 @@
             microfiber._json_body(json_str),
             json_str2.encode('utf-8')
         )
-        
+
         # Test other stuff that should get JSON encoded:
         self.assertEqual(microfiber._json_body(True), b'true')
         self.assertEqual(microfiber._json_body(False), b'false')
         self.assertEqual(microfiber._json_body('hello'), b'"hello"')
+        self.assertEqual(
+            microfiber._json_body('*safe solvent™'),
+            b'"*safe solvent\xe2\x84\xa2"'
+        )   
         self.assertEqual(microfiber._json_body(18), b'18')
         self.assertEqual(microfiber._json_body(17.9), b'17.9')
         self.assertEqual(microfiber._json_body({}), b'{}')
@@ -153,7 +174,7 @@
             microfiber._json_body(['one', True, 3]),
             b'["one",true,3]'
         )
-        
+
         # Test when obj in an open file
         d = tempfile.mkdtemp()
         try:
@@ -219,6 +240,21 @@
             ]
         )
 
+        # Test key, startkey, endkey with non-ascii values
+        options = dict(
+            key='Hanna Sköld',
+            startkey='Matias Särs',
+            endkey='Paweł Moll',
+        )
+        self.assertEqual(
+            list(microfiber._queryiter(options)),
+            [
+                ('endkey', '"Paweł Moll"'),
+                ('key', '"Hanna Sköld"'),
+                ('startkey', '"Matias Särs"'),
+            ]
+        )
+
     def test_oauth_base_string(self):
         f = microfiber._oauth_base_string
 
@@ -1216,6 +1252,15 @@
         self.assertEqual(inst.delete(), {'ok': True})
         self.assertRaises(NotFound, inst.get)
 
+    def test_non_ascii(self):
+        inst = self.klass(self.db, self.env)
+        self.assertTrue(inst.ensure())
+        _id = test_id()
+        name = '*safe solvent™'
+        doc = {'_id': _id, 'name': name}
+        inst.save(doc)
+        self.assertEqual(inst.get(_id)['name'], name)
+
     def test_save(self):
         inst = self.klass(self.db, self.env)