Merge lp:~cjwatson/lazr.restful/py3-unicode into lp:lazr.restful

[Colin Watson]

Port simple uses of str/unicode/basestring to Python 3.

Most cases are fairly obvious, but we need to pay some attention to whether str is intended to be a byte string or a native string. basestring normally means "native string or text" (i.e. six.string_types), but in a few places it means "bytes or text" instead: in particular, the return value from EntryResource.do_GET and CollectionResource.do_GET may be either.

[Ioana Lasc (ilasc)]

review: Approve

=== modified file 'src/lazr/restful/_operation.py'
--- src/lazr/restful/_operation.py	2020-02-04 11:52:59 +0000
+++ src/lazr/restful/_operation.py	2020-07-22 22:45:07 +0000
@@ -5,6 +5,7 @@
 from __future__ import absolute_import, print_function
 
 import simplejson
+import six
 
 from zope.component import getMultiAdapter, getUtility, queryMultiAdapter
 from zope.event import notify
@@ -145,7 +146,8 @@
             # has its response batched.
             return True
 
-        if zope_isinstance(result, (basestring, dict, set, list, tuple)):
+        if zope_isinstance(
+                result, (bytes, six.text_type, dict, set, list, tuple)):
             # Ordinary Python data structures generally are not
             # batched.
             return False
 
=== modified file 'src/lazr/restful/_resource.py'
--- src/lazr/restful/_resource.py	2020-06-30 15:52:03 +0000
+++ src/lazr/restful/_resource.py	2020-07-22 22:45:07 +0000
@@ -151,12 +151,12 @@
 
 def decode_value(value):
     """Return a unicode value curresponding to `value`."""
-    if isinstance(value, unicode):
+    if isinstance(value, six.text_type):
         return value
-    elif isinstance(value, str):
+    elif isinstance(value, bytes):
         return value.decode("utf-8")
     else:
-        return unicode(value)
+        return six.text_type(value)
 
 
 def encode_value(value):
@@ -164,17 +164,17 @@
 
     Non-unicode strings are assumed to be UTF-8 already.
     """
-    if isinstance(value, unicode):
+    if isinstance(value, six.text_type):
         return value.encode("utf-8")
-    elif isinstance(value, str):
+    elif isinstance(value, bytes):
         return value
     else:
-        return str(value)
+        return bytes(value)
 
 
 def _default_html_renderer(value):
     """The default technique for rendering a value as an HTML snippet."""
-    return cgi.escape(unicode(value))
+    return cgi.escape(six.text_type(value))
 
 
 @adapter(Interface, IField, IWebServiceClientRequest)
@@ -701,7 +701,7 @@
         :return: The result of the operation: either a string or an
         object that needs to be serialized to JSON.
         """
-        if not isinstance(operation_name, basestring):
+        if not isinstance(operation_name, six.string_types):
             self.request.response.setStatus(400)
             return "Expected a single operation: %r" % (operation_name,)
 
@@ -727,7 +727,7 @@
         :return: The result of the operation: either a string or an
         object that needs to be serialized to JSON.
         """
-        if not isinstance(operation_name, basestring):
+        if not isinstance(operation_name, six.string_types):
             self.request.response.setStatus(400)
             return "Expected a single operation: %r" % (operation_name,)
 
@@ -1163,7 +1163,7 @@
                     # instead of whatever object the raise site
                     # thought would be a good idea.
                     if (len(e.args) > 0 and
-                        isinstance(e.args[0], basestring)):
+                        isinstance(e.args[0], six.string_types)):
                         error = e.args[0]
                     else:
                         error = "Constraint not satisfied."
@@ -1311,7 +1311,7 @@
         # generating the representation might itself change across
         # versions.
         revno = getUtility(IWebServiceConfiguration).code_revision
-        return [core.encode('utf-8') for core in [revno, unicode(value)]]
+        return [core.encode('utf-8') for core in [revno, six.text_type(value)]]
 
     def _representation(self, media_type):
         """Create a representation of the field value."""
@@ -1547,7 +1547,7 @@
         operation_name = self.request.form.pop('ws.op', None)
         if operation_name is not None:
             result = self.handleCustomGET(operation_name)
-            if isinstance(result, basestring):
+            if isinstance(result, (bytes, six.text_type)):
                 # The custom operation took care of everything and
                 # just needs this string served to the client.
                 return result
@@ -1788,7 +1788,7 @@
         operation_name = self.request.form.pop('ws.op', None)
         if operation_name is not None:
             result = self.handleCustomGET(operation_name)
-            if isinstance(result, str) or isinstance(result, unicode):
+            if isinstance(result, (bytes, six.text_type)):
                 # The custom operation took care of everything and
                 # just needs this string served to the client.
                 return result
 
=== modified file 'src/lazr/restful/declarations.py'
--- src/lazr/restful/declarations.py	2020-02-05 10:46:46 +0000
+++ src/lazr/restful/declarations.py	2020-07-22 22:45:07 +0000
@@ -47,6 +47,7 @@
 import itertools
 import sys
 
+import six
 from zope.component import getUtility, getGlobalSiteManager
 from zope.interface import classImplements
 from zope.interface.advice import addClassAdvisor
@@ -738,7 +739,8 @@
             # zope schema are expected to be unicode, whereas it's
             # really possible that the method's default is a simple
             # string.
-            if isinstance(default, str) and IText.providedBy(param_def):
+            if (six.PY2 and
+                    isinstance(default, str) and IText.providedBy(param_def)):
                 default = unicode(default)
             param_def.default = default
             param_def.required = False
@@ -1805,5 +1807,5 @@
         # runtime. Use a generic string that won't conflict with a
         # real version string.
         version = "__Earliest"
-    name = "%s_%s" % (base_name, version.encode('utf8'))
+    name = "%s_%s" % (base_name, six.ensure_str(version))
     return make_identifier_safe(name)
 
=== modified file 'src/lazr/restful/docs/webservice.rst'
--- src/lazr/restful/docs/webservice.rst	2020-02-04 13:17:32 +0000
+++ src/lazr/restful/docs/webservice.rst	2020-07-22 22:45:07 +0000
@@ -1092,8 +1092,9 @@
 'home page' for the web service.
 
     >>> import simplejson
+    >>> import six
     >>> response = app(request)
-    >>> representation = simplejson.loads(unicode(response))
+    >>> representation = simplejson.loads(six.text_type(response))
 
     >>> representation["authors_collection_link"]
     u'http://api.cookbooks.dev/beta/authors'
@@ -1168,7 +1169,7 @@
 
     >>> def load_json(s):
     ...     """Convert a JSON string to Unicode and then load it."""
-    ...     return simplejson.loads(unicode(s))
+    ...     return simplejson.loads(six.text_type(s))
 
     >>> representation = load_json(collection())
     >>> representation['resource_type_link']
 
=== modified file 'src/lazr/restful/marshallers.py'
--- src/lazr/restful/marshallers.py	2020-06-30 15:52:03 +0000
+++ src/lazr/restful/marshallers.py	2020-07-22 22:45:07 +0000
@@ -29,6 +29,7 @@
 from StringIO import StringIO
 
 import simplejson
+import six
 from six.moves.urllib.parse import unquote
 
 from zope.datetime import (
@@ -89,7 +90,7 @@
             request_host = full_request_host
             request_port = default_port
 
-        if not isinstance(url, basestring):
+        if not isinstance(url, six.string_types):
             raise ValueError(u"got '%s', expected string: %r" % (
                 type(url).__name__, url))
         if url.startswith('/'):
@@ -178,10 +179,10 @@
         if value != '':
             try:
                 v = value
-                if isinstance(v, str):
+                if isinstance(v, bytes):
                     v = v.decode('utf8') # assume utf8
-                elif not isinstance(v, unicode):
-                    v = unicode(v)
+                elif not isinstance(v, six.text_type):
+                    v = six.text_type(v)
                 value = simplejson.loads(v)
             except (ValueError, TypeError):
                 # Pass the value as is. This saves client from having to encode
@@ -267,7 +268,7 @@
 class BytesFieldMarshaller(SimpleFieldMarshaller):
     """FieldMarshaller for IBytes field."""
 
-    _type = str
+    _type = bytes
     _type_error_message = 'not a string: %r'
 
     @property
@@ -295,8 +296,8 @@
         if safe_hasattr(value, 'seek'):
             value.seek(0)
             value = value.read()
-        elif not isinstance(value, basestring):
-            value = str(value)
+        elif not isinstance(value, (bytes, six.text_type)):
+            value = bytes(value)
         else:
             # Leave string conversion to _marshall_from_json_data.
             pass
@@ -307,7 +308,7 @@
 
         Convert all strings to byte strings.
         """
-        if isinstance(value, unicode):
+        if isinstance(value, six.text_type):
             value = value.encode('utf-8')
         return super(
             BytesFieldMarshaller, self)._marshall_from_json_data(value)
@@ -316,7 +317,7 @@
 class TextFieldMarshaller(SimpleFieldMarshaller):
     """FieldMarshaller for IText fields."""
 
-    _type = unicode
+    _type = six.text_type
     _type_error_message = 'not a unicode string: %r'
 
     def _marshall_from_request(self, value):
@@ -324,7 +325,7 @@
 
         Converts the value to unicode.
         """
-        value = unicode(value)
+        value = six.text_type(value)
         return super(TextFieldMarshaller, self)._marshall_from_request(value)
 
 
@@ -366,7 +367,8 @@
         Looks up the value as a token in the vocabulary.
         """
         try:
-            return self.field.vocabulary.getTermByToken(str(value)).value
+            return self.field.vocabulary.getTermByToken(
+                six.text_type(value)).value
         except LookupError:
             raise ValueError(u"%r isn't a valid token" % value)
 
 
=== modified file 'src/lazr/restful/testing/helpers.py'
--- src/lazr/restful/testing/helpers.py	2020-02-04 11:52:59 +0000
+++ src/lazr/restful/testing/helpers.py	2020-07-22 22:45:07 +0000
@@ -50,7 +50,9 @@
 
     :param response: an httplib HTTPResponse object.
     """
-    response_unicode = str(response).decode("utf-8")
+    response_unicode = str(response)
+    if isinstance(response_unicode, bytes):  # Python 2
+        response_unicode = response_unicode.decode("utf-8")
     return encode_unicode(response_unicode)
 
 
 
=== modified file 'src/lazr/restful/testing/webservice.py'
--- src/lazr/restful/testing/webservice.py	2020-02-04 21:53:16 +0000
+++ src/lazr/restful/testing/webservice.py	2020-07-22 22:45:07 +0000
@@ -28,6 +28,7 @@
 
 import wsgi_intercept
 
+import six
 from six.moves.urllib.parse import (
     quote,
     urlencode,
@@ -122,7 +123,7 @@
         if result is None:
             result = ''
 
-        if not isinstance(result, basestring):
+        if not isinstance(result, six.string_types):
             raise ValueError('only strings and None results are handled')
 
         self.result = result
@@ -209,7 +210,7 @@
     """
 
     def __init__(self, global_config, publication, **options):
-        if isinstance(publication, basestring):
+        if isinstance(publication, six.string_types):
             Application.__init__(self, global_config, publication, **options)
         else:
             self.publication = publication(global_config, **options)
@@ -334,7 +335,7 @@
         # To be properly marshalled all values must be strings or converted to
         # JSON.
         for key, value in args.items():
-            if not isinstance(value, basestring):
+            if not isinstance(value, six.string_types):
                 args[key] = simplejson.dumps(value)
         return urlencode(args)
 
@@ -363,7 +364,7 @@
 
         This may mean turning the value into a JSON string.
         """
-        if not isinstance(value, basestring):
+        if not isinstance(value, six.string_types):
             value = simplejson.dumps(value)
         return quote(value)
 
@@ -408,7 +409,7 @@
     def jsonBody(self):
         """Return the body of the web service request as a JSON document."""
         try:
-            json = simplejson.loads(unicode(self.body))
+            json = simplejson.loads(six.ensure_text(self.body))
             if isinstance(json, list):
                 json = sorted(json)
             return json
 
=== modified file 'src/lazr/restful/tests/test_webservice.py'
--- src/lazr/restful/tests/test_webservice.py	2020-02-19 16:09:10 +0000
+++ src/lazr/restful/tests/test_webservice.py	2020-07-22 22:45:07 +0000
@@ -18,6 +18,7 @@
 import simplejson
 import unittest
 
+import six
 from zope.component import (
     eventtesting,
     getGlobalSiteManager,
@@ -536,8 +537,8 @@
     def resource(self, value_1="value 1", value_2="value 2"):
         """Simplify the entry_resource call."""
         with self.entry_resource(
-            IHasTwoFields, HasTwoFields,
-            unicode(value_1), unicode(value_2)) as resource:
+                IHasTwoFields, HasTwoFields,
+                six.text_type(value_1), six.text_type(value_2)) as resource:
             yield resource
 
     def test_web_layer_json_representation_omits_lp_html(self):
@@ -598,7 +599,7 @@
         json = None
         with self.resource() as resource:
             json_plus_xhtml = resource.JSON_PLUS_XHTML_TYPE
-            json = simplejson.loads(unicode(
+            json = simplejson.loads(six.text_type(
                     resource._representation(json_plus_xhtml)))
             resource.applyChanges(json, json_plus_xhtml)
             self.assertEqual(resource.request.response.getStatus(), 209)
@@ -607,7 +608,7 @@
         self.register_html_field_renderer()
         json = None
         with self.resource() as resource:
-            json = simplejson.loads(unicode(
+            json = simplejson.loads(six.text_type(
                     resource._representation(resource.JSON_PLUS_XHTML_TYPE)))
             resource.applyChanges(json, resource.JSON_TYPE)
             self.assertEqual(resource.request.response.getStatus(), 400)
 
=== modified file 'src/lazr/restful/utils.py'
--- src/lazr/restful/utils.py	2020-02-04 11:52:59 +0000
+++ src/lazr/restful/utils.py	2020-07-22 22:45:07 +0000
@@ -31,6 +31,7 @@
 import subprocess
 
 from simplejson import encoder
+import six
 
 from zope.component import getUtility
 from zope.schema import getFieldsInOrder
@@ -359,7 +360,7 @@
 
 def smartquote(str):
     """Return a copy of the string, with typographical quote marks applied."""
-    str = unicode(str)
+    str = six.text_type(str)
     str = re.compile(u'(^| )(")([^" ])').sub(u'\\1\u201c\\3', str)
     str = re.compile(u'([^ "])(")($|[\s.,;:!?])').sub(u'\\1\u201d\\3', str)
     return str