Merge lp:~leonardr/lazr.restful/benji-updated-size-link into lp:lazr.restful

[Leonard Richardson]

This branch makes lazr.restful send a 'total_size_link' in lieu of a 'total_size', to spare the underlying data model the pain of calculating the total size of a collection all the time.

The branch has already been reviewed (https://code.edge.launchpad.net/~benji/lazr.restful/size-link/+merge/31971). I'm asking for a review of the changes benji and I made in response to the review. A diff of just these changes is here: http://paste.ubuntu.com/475892/

Summary of changes:

1. Rather than passing total_size_only as an argument into batch(), create a new method get_total_size and call it when the total size is desired, instead of calling batch().

2. When generating the WADL, use a new TALES method to determine whether or not total_size_link is active, rather than putting that boolean in the ZPT namespace. I had to rename namespace['context'] to namespace['service'] to get this to work, because there's code in the ZPT template that assigns local variables to 'context', clobbering namespace['context'].

3. Removed a no-longer-necessary named operation and field from webservice-declarations.txt. (This operation was introduced earlier in the branch, at a time when you had to explicitly tag a named operation to make use of total_size_only. Now, all named operations implicitly have total_size_only.)

[Graham Binns (gmb)]

The changes in http://paste.ubuntu.com/475892/ look good; r=me.

review: Approve (code)

=== modified file 'setup.py'
--- setup.py	2010-03-04 14:27:58 +0000
+++ setup.py	2010-08-10 12:24:46 +0000
@@ -57,7 +57,7 @@
         'docutils',
         'epydoc', # used by wadl generation
         'grokcore.component==1.6',
-        'lazr.batchnavigator',
+        'lazr.batchnavigator>=1.2.0-dev',
         'lazr.delegates',
         'lazr.enum',
         'lazr.lifecycle',

=== modified file 'src/lazr/restful/NEWS.txt'
--- src/lazr/restful/NEWS.txt	2010-08-05 14:01:44 +0000
+++ src/lazr/restful/NEWS.txt	2010-08-10 12:24:46 +0000
@@ -2,6 +2,15 @@
 NEWS for lazr.restful
 =====================
 
+0.11.0 (unreleased)
+===================
+
+Added an optimization to total_size so that it is fetched via a link when
+possible.  The new configuration option first_version_with_total_size_link
+specifies what version should be the first to expose the behavior.  The default
+is for it to be enabled for all versions so set this option to preserve the
+earlier behavior for previously released web services.
+
 0.10.0 (2010-08-05)
 ===================
 

=== modified file 'src/lazr/restful/_operation.py'
--- src/lazr/restful/_operation.py	2010-04-20 18:26:49 +0000
+++ src/lazr/restful/_operation.py	2010-08-10 12:24:46 +0000
@@ -44,6 +44,23 @@
     def __init__(self, context, request):
         self.context = context
         self.request = request
+        self.total_size_only = False
+
+    def total_size_link(self, navigator):
+        """Return a link to the total size of a collection."""
+        if getattr(self, 'include_total_size', True):
+            # This is a named operation that includes the total size
+            # inline rather than with a link.
+            return None
+        if not IResourceGETOperation.providedBy(self):
+            # Only GET operations can have their total size split out into
+            # a link, because only GET operations are safe.
+            return None
+        base = str(self.request.URL)
+        query = navigator.getCleanQueryString()
+        if query != '':
+            query += '&'
+        return base + '?' + query + "ws.show=total_size"
 
     def __call__(self):
         values, errors = self.validate()
@@ -80,13 +97,22 @@
             # this object served to the client.
             return result
 
+        # The similar patterns in the two branches below suggest some deeper
+        # symmetry that should be extracted.
         if queryMultiAdapter((result, self.request), ICollection):
             # If the result is a web service collection, serve only one
             # batch of the collection.
             collection = getMultiAdapter((result, self.request), ICollection)
-            result = CollectionResource(collection, self.request).batch() + '}'
+            resource = CollectionResource(collection, self.request)
+            if self.total_size_only:
+                result = resource.get_total_size(collection)
+            else:
+                result = resource.batch() + '}'
         elif self.should_batch(result):
-            result = self.batch(result, self.request) + '}'
+            if self.total_size_only:
+                result = self.get_total_size(result)
+            else:
+                result = self.batch(result, self.request) + '}'
         else:
             # Serialize the result to JSON. Any embedded entries will be
             # automatically serialized.

=== modified file 'src/lazr/restful/_resource.py'
--- src/lazr/restful/_resource.py	2010-08-04 18:25:21 +0000
+++ src/lazr/restful/_resource.py	2010-08-10 12:24:46 +0000
@@ -110,6 +110,7 @@
         status_reasons[code] = reason
 init_status_codes()
 
+
 def decode_value(value):
     """Return a unicode value curresponding to `value`."""
     if isinstance(value, unicode):
@@ -589,27 +590,45 @@
 
     """A mixin for resources that need to batch lists of entries."""
 
-    def __init__(self, context, request):
-        """A basic constructor."""
-        # Like all mixin classes, this class is designed to be used
-        # with multiple inheritance. That requires defining __init__
-        # to call the next constructor in the chain, which means using
-        # super() even though this class itself has no superclass.
-        super(BatchingResourceMixin, self).__init__(context, request)
+    # TODO: determine real need for __init__ and super() call
+
+    def total_size_link(self, navigator):
+        """Return the URL to fetch to find out the collection's total size.
+
+        If this is None, the total size will be included inline.
+
+        :param navigator: A BatchNavigator object for the current batch.
+        """
+        return None
+
+    def get_total_size(self, entries):
+        """Get the number of items in entries.
+
+        :return: a JSON string representing the number of objects in the list
+        """
+        if not hasattr(entries, '__len__'):
+            entries = IFiniteSequence(entries)
+
+        return simplejson.dumps(len(entries))
+
 
     def batch(self, entries, request):
         """Prepare a batch from a (possibly huge) list of entries.
 
-        :return: A JSON string representing a hash:
+        :return: a JSON string representing a hash:
+
         'entries' contains a list of EntryResource objects for the
           entries that actually made it into this batch
         'total_size' contains the total size of the list.
+        'total_size_link' contains a link to the total size of the list.
         'next_url', if present, contains a URL to get the next batch
          in the list.
         'prev_url', if present, contains a URL to get the previous batch
          in the list.
         'start' contains the starting index of this batch
 
+        Only one of 'total_size' or 'total_size_link' will be present.
+
         Note that the JSON string will be missing its final curly
         brace. This is in case the caller wants to add some additional
         keys to the JSON hash. It's the caller's responsibility to add
@@ -620,11 +639,12 @@
         navigator = WebServiceBatchNavigator(entries, request)
 
         view_permission = getUtility(IWebServiceConfiguration).view_permission
-        resources = [EntryResource(entry, request)
-                     for entry in navigator.batch
-                     if checkPermission(view_permission, entry)]
-        batch = { 'total_size' : navigator.batch.listlength,
-                  'start' : navigator.batch.start }
+        batch = { 'start' : navigator.batch.start }
+        total_size_link = self.total_size_link(navigator)
+        if total_size_link is None:
+            batch['total_size'] = navigator.batch.listlength
+        else:
+            batch['total_size_link'] = total_size_link
         if navigator.batch.start < 0:
             batch['start'] = None
         next_url = navigator.nextBatchURL()
@@ -637,6 +657,9 @@
 
         # String together a bunch of entry representations, possibly
         # obtained from a representation cache.
+        resources = [EntryResource(entry, request)
+                     for entry in navigator.batch
+                     if checkPermission(view_permission, entry)]
         entry_strings = [
             resource._representation(HTTPResource.JSON_TYPE)
             for resource in resources]
@@ -674,6 +697,10 @@
         except ComponentLookupError:
             self.request.response.setStatus(400)
             return "No such operation: " + operation_name
+
+        show = self.request.form.get('ws.show')
+        if show == 'total_size':
+            operation.total_size_only = True
         return operation()
 
     def handleCustomPOST(self, operation_name):
@@ -1664,14 +1691,17 @@
         self.request.response.setHeader('Content-type', self.JSON_TYPE)
         return result
 
-    def batch(self, entries=None):
+    def batch(self, entries=None, request=None):
         """Return a JSON representation of a batch of entries.
 
         :param entries: (Optional) A precomputed list of entries to batch.
+        :param request: (Optional) The current request.
         """
         if entries is None:
             entries = self.collection.find()
-        result = super(CollectionResource, self).batch(entries, self.request)
+        if request is None:
+            request = self.request
+        result = super(CollectionResource, self).batch(entries, request)
         result += (
             ', "resource_type_link" : ' + simplejson.dumps(self.type_url)
             + '}')
@@ -1862,7 +1892,7 @@
                     # by the entry classes.
                     collection_classes.append(registration.factory)
         namespace = self.WADL_TEMPLATE.pt_getContext()
-        namespace['context'] = self
+        namespace['service'] = self
         namespace['request'] = self.request
         namespace['entries'] = entry_classes
         namespace['collections'] = collection_classes
@@ -2061,13 +2091,13 @@
     def singular_type(self):
         """Return the singular name for this object type."""
         interface = self.entry_interface
-        return interface.queryTaggedValue(LAZR_WEBSERVICE_NAME)['singular']
+        return interface.getTaggedValue(LAZR_WEBSERVICE_NAME)['singular']
 
     @property
     def plural_type(self):
         """Return the plural name for this object type."""
         interface = self.entry_interface
-        return interface.queryTaggedValue(LAZR_WEBSERVICE_NAME)['plural']
+        return interface.getTaggedValue(LAZR_WEBSERVICE_NAME)['plural']
 
     @property
     def type_link(self):

=== modified file 'src/lazr/restful/declarations.py'
--- src/lazr/restful/declarations.py	2010-08-04 18:25:21 +0000
+++ src/lazr/restful/declarations.py	2010-08-10 12:24:46 +0000
@@ -63,7 +63,7 @@
 from lazr.restful.security import protect_schema
 from lazr.restful.utils import (
     camelcase_to_underscore_separated, get_current_web_service_request,
-    make_identifier_safe, VersionedDict)
+    make_identifier_safe, VersionedDict, is_total_size_link_active)
 
 LAZR_WEBSERVICE_EXPORTED = '%s.exported' % LAZR_WEBSERVICE_NS
 LAZR_WEBSERVICE_MUTATORS = '%s.exported.mutators' % LAZR_WEBSERVICE_NS
@@ -712,10 +712,9 @@
 class operation_returns_collection_of(_method_annotator):
     """Specify that the exported operation returns a collection.
 
-    The decorator takes a single argument: an interface that's been
-    exported as an entry.
+    The decorator takes one required argument, "schema", an interface that's
+    been exported as an entry.
     """
-
     def __init__(self, schema):
         _check_called_from_interface_def('%s()' % self.__class__.__name__)
         if not IInterface.providedBy(schema):
@@ -1209,23 +1208,22 @@
         version = getUtility(IWebServiceConfiguration).active_versions[0]
 
     bases = (BaseResourceOperationAdapter, )
-    if tag['type'] == 'read_operation':
+    operation_type = tag['type']
+    if operation_type == 'read_operation':
         prefix = 'GET'
         provides = IResourceGETOperation
-    elif tag['type'] in ('factory', 'write_operation'):
+    elif operation_type in ('factory', 'write_operation'):
         provides = IResourcePOSTOperation
         prefix = 'POST'
-        if tag['type'] == 'factory':
+        if operation_type == 'factory':
             bases = (BaseFactoryResourceOperationAdapter,)
-    elif tag['type'] == 'destructor':
+    elif operation_type == 'destructor':
         provides = IResourceDELETEOperation
         prefix = 'DELETE'
     else:
-        raise AssertionError('Unknown method export type: %s' % tag['type'])
+        raise AssertionError('Unknown method export type: %s' % operation_type)
 
     return_type = tag['return_type']
-    if return_type is None:
-        return_type = None
 
     name = _versioned_class_name(
         '%s_%s_%s' % (prefix, method.interface.__name__, tag['as']),
@@ -1238,7 +1236,16 @@
         '_method_name': method.__name__,
         '__doc__': method.__doc__}
 
-    if tag['type'] == 'write_operation':
+    if isinstance(return_type, CollectionField):
+        # If the version we're being asked for is equal to or later than the
+        # version in which we started exposing total_size_link and this is a
+        # read operation, then include it, otherwise include total_size.
+        config = getUtility(IWebServiceConfiguration)
+        class_dict['include_total_size'] = not (
+            is_total_size_link_active(version, config) and
+            operation_type == 'read_operation')
+
+    if operation_type == 'write_operation':
         class_dict['send_modification_event'] = True
     factory = type(name, bases, class_dict)
     classImplements(factory, provides)

=== modified file 'src/lazr/restful/docs/webservice-declarations.txt'
--- src/lazr/restful/docs/webservice-declarations.txt	2010-08-02 20:10:33 +0000
+++ src/lazr/restful/docs/webservice-declarations.txt	2010-08-10 12:24:46 +0000
@@ -32,7 +32,7 @@
 field, but not the inventory_number field.
 
     >>> from zope.interface import Interface
-    >>> from zope.schema import TextLine, Float, List
+    >>> from zope.schema import Text, TextLine, Float, List
     >>> from lazr.restful.declarations import (
     ...     export_as_webservice_entry, exported)
     >>> class IBook(Interface):
@@ -213,7 +213,7 @@
     TypeError: export_as_webservice_collection() is missing a method
     tagged with @collection_default_content.
 
-As it is an error, to mark more than one methods:
+As it is an error, to mark more than one method:
 
     >>> class TwoDefaultContent(Interface):
     ...     export_as_webservice_collection(IDummyInterface)
@@ -330,8 +330,8 @@
     ...         text=copy_field(IBook['title'], title=u'Text to search for.'))
     ...     @operation_returns_collection_of(IBook)
     ...     @export_read_operation()
-    ...     def searchBooks(text):
-    ...         """Return list of books containing 'text'."""
+    ...     def searchBookTitles(text):
+    ...         """Return list of books whose titles contain 'text'."""
     ...
     ...     @operation_parameters(
     ...         text=copy_field(IBook['title'], title=u'Text to search for.'))
@@ -393,11 +393,11 @@
     return_type: <lazr.restful._operation.ObjectLink object...>
     type: 'factory'
 
-We did specify the return type for the 'searchBooks' method: it
+We did specify the return type for the 'searchBookTitles' method: it
 returns a collection.
 
-    >>> print_export_tag(IBookSetOnSteroids['searchBooks'])
-    as: 'searchBooks'
+    >>> print_export_tag(IBookSetOnSteroids['searchBookTitles'])
+    as: 'searchBookTitles'
     call_with: {}
     params: {'text': <...TextLine...>}
     return_type: <lazr.restful.fields.CollectionField object...>
@@ -920,7 +920,8 @@
     >>> class Book(object):
     ...     """Simple IBook implementation."""
     ...     implements(IBook)
-    ...     def __init__(self, author, title, base_price, inventory_number):
+    ...     def __init__(self, author, title, base_price,
+    ...                  inventory_number):
     ...         self.author = author
     ...         self.title = title
     ...         self.base_price = base_price
@@ -937,6 +938,7 @@
     >>> class MyWebServiceConfiguration(TestWebServiceConfiguration):
     ...     active_versions = ["beta", "1.0", "2.0", "3.0"]
     ...     last_version_with_mutator_named_operations = "1.0"
+    ...     first_version_with_total_size_link = "2.0"
     ...     code_revision = "1.0b"
     ...     default_batch_size = 50
     >>> provideUtility(MyWebServiceConfiguration(), IWebServiceConfiguration)
@@ -1087,7 +1089,7 @@
     ...     generate_operation_adapter)
 
     >>> read_method_adapter_factory = generate_operation_adapter(
-    ...     IBookSetOnSteroids['searchBooks'])
+    ...     IBookSetOnSteroids['searchBookTitles'])
     >>> IResourceGETOperation.implementedBy(read_method_adapter_factory)
     True
 
@@ -1099,14 +1101,14 @@
 
     >>> from lazr.restful import ResourceOperation
     >>> read_method_adapter_factory.__name__
-    'GET_IBookSetOnSteroids_searchBooks_beta'
+    'GET_IBookSetOnSteroids_searchBookTitles_beta'
     >>> issubclass(read_method_adapter_factory, ResourceOperation)
     True
 
 The adapter's docstring is taken from the decorated method docstring.
 
     >>> read_method_adapter_factory.__doc__
-    "Return list of books containing 'text'."
+    "Return list of books whose titles contain 'text'."
 
 The adapter's params attribute contains the specification of the
 parameters accepted by the operation.
@@ -1126,7 +1128,7 @@
     ...
     ...     result = None
     ...
-    ...     def searchBooks(self, text):
+    ...     def searchBookTitles(self, text):
     ...         return self.result
     ...
     ...     def new(self, author, base_price, title):
@@ -1148,7 +1150,7 @@
 return value is a dictionary containing a batched list.
 
     >>> print read_method_adapter.call(text='')
-    {"total_size": 0, "start": null, "entries": []}
+    {"total_size": 0, "start": 0, "entries": []}
 
 Methods exported as a write operations generates an adapter providing
 IResourcePOSTOperation.
@@ -1649,6 +1651,51 @@
     AssertionError: 'IMultiVersionCollection' isn't tagged for export
     to web service version 'NoSuchVersion'.
 
+total_size_link
+~~~~~~~~~~~~~~~
+
+Collections previously exposed their total size via a `total_size` attribute.
+However, newer versions of lazr.restful expose a `total_size_link` intead.  To
+facilitate transitioning from one approach to the other the configuration
+option `first_version_with_total_size_link` has been added to
+IWebServiceConfiguration.
+
+By default the first_version_with_total_size_link is set to the earliest
+available web service version, but if you have stable versions of your web
+service you wish to maintain compatability with you can specify the version in
+which you want the new behavior to take effect in the web service
+configuration.
+
+    >>> from zope.component import getUtility
+    >>> config = getUtility(IWebServiceConfiguration)
+    >>> config.last_version_with_mutator_named_operations = '1.0'
+
+    >>> from lazr.restful.declarations import (
+    ...     export_read_operation, operation_returns_collection_of,
+    ...     operation_for_version)
+    >>> class IWithMultiVersionCollection(Interface):
+    ...     export_as_webservice_entry()
+    ...
+    ...     @operation_for_version('2.0')
+    ...     @operation_for_version('1.0')
+    ...     @operation_returns_collection_of(Interface)
+    ...     @export_read_operation()
+    ...     def method():
+    ...         """A method that returns a collection."""
+
+    >>> method = IWithMultiVersionCollection['method']
+    >>> dummy_data = None # this would be an intance that has the method
+    >>> v10 = generate_operation_adapter(method, '1.0')(dummy_data, request)
+    >>> v20 = generate_operation_adapter(method, '2.0')(dummy_data, request)
+
+We can see that version 1.0 includes the total size for backward compatability
+while version 2.0 includes a link to fetch the total size.
+
+    >>> v10.include_total_size
+    True
+    >>> v20.include_total_size
+    False
+
 Entries
 -------
 
@@ -2541,8 +2588,8 @@
     >>> request_interface = IWebServiceClientRequest
     >>> adapter_registry.lookup(
     ...     (IBookSetOnSteroids, request_interface),
-    ...     IResourceGETOperation, 'searchBooks')
-    <class '...GET_IBookSetOnSteroids_searchBooks_beta'>
+    ...     IResourceGETOperation, 'searchBookTitles')
+    <class '...GET_IBookSetOnSteroids_searchBookTitles_beta'>
     >>> adapter_registry.lookup(
     ...     (IBookSetOnSteroids, request_interface),
     ...     IResourcePOSTOperation, 'create_book')

=== modified file 'src/lazr/restful/docs/webservice.txt'
--- src/lazr/restful/docs/webservice.txt	2010-08-04 18:25:21 +0000
+++ src/lazr/restful/docs/webservice.txt	2010-08-10 12:24:46 +0000
@@ -503,6 +503,7 @@
     ...     code_revision = 'test'
     ...     max_batch_size = 100
     ...     directives.publication_class(WebServiceTestPublication)
+    ...     first_version_with_total_size_link = 'devel'
 
     >>> from grokcore.component.testing import grok_component
     >>> ignore = grok_component(

=== modified file 'src/lazr/restful/example/base/root.py'
--- src/lazr/restful/example/base/root.py	2010-06-14 14:09:31 +0000
+++ src/lazr/restful/example/base/root.py	2010-08-10 12:24:46 +0000
@@ -398,6 +398,7 @@
     <p>Don't use this unless you like changing things.</p>"""
                              }
     last_version_with_mutator_named_operations = None
+    first_version_with_total_size_link = None
     use_https = False
     view_permission = 'lazr.restful.example.base.View'
 

=== modified file 'src/lazr/restful/example/base/tests/collection.txt'
--- src/lazr/restful/example/base/tests/collection.txt	2009-04-24 15:14:07 +0000
+++ src/lazr/restful/example/base/tests/collection.txt	2010-08-10 12:24:46 +0000
@@ -173,14 +173,12 @@
     ...         "/cookbooks?ws.op=find_recipes&%s" % args).jsonBody()
 
     >>> s_recipes = search_recipes("chicken")
-    >>> s_recipes['total_size']
-    3
     >>> sorted(r['instructions'] for r in s_recipes['entries'])
     [u'Draw, singe, stuff, and truss...', u'You can always judge...']
 
     >>> veg_recipes = search_recipes("chicken", True)
-    >>> veg_recipes['total_size']
-    0
+    >>> veg_recipes['entries']
+    []
 
 A custom operation that returns a list of objects is paginated, just
 like a collection.
@@ -196,11 +194,21 @@
 empty list of results:
 
     >>> empty_collection = search_recipes("nosuchrecipe")
-    >>> empty_collection['total_size']
-    0
     >>> [r['instructions'] for r in empty_collection['entries']]
     []
 
+When an operation yields a collection of objects, the representation
+includes a link that yields the total size of the collection.
+
+    >>> print s_recipes['total_size_link']
+    http://.../cookbooks?search=chicken&vegetarian=false&ws.op=find_recipes&ws.show=total_size
+
+Sending a GET request to that link yields a JSON representation of the
+total size.
+
+   >>> print webservice.get(s_recipes['total_size_link']).jsonBody()
+   3
+
 Custom operations may have error handling. In this case, the error
 handling is in the validate() method of the 'search' field.
 
@@ -216,8 +224,9 @@
 
     >>> general_cookbooks = webservice.get(
     ...     "/cookbooks?ws.op=find_for_cuisine&cuisine=General")
-    >>> general_cookbooks.jsonBody()['total_size']
-    3
+    >>> print general_cookbooks.jsonBody()['total_size_link']
+    http://cookbooks.dev/devel/cookbooks?cuisine=General&ws.op=find_for_cuisine&ws.show=total_size
+
 
 POST operations
 ===============

=== modified file 'src/lazr/restful/example/base/tests/wadl.txt'
--- src/lazr/restful/example/base/tests/wadl.txt	2010-03-10 18:45:04 +0000
+++ src/lazr/restful/example/base/tests/wadl.txt	2010-08-10 12:24:46 +0000
@@ -537,8 +537,9 @@
 'entry_resource_descriptions'.
 
     >>> entry_resource_descriptions = []
-    >>> entry_resource_types = other_children[first_entry_type_index:-2]
-    >>> hosted_binary_resource_type, simple_binary_type = other_children[-2:]
+    >>> entry_resource_types = other_children[first_entry_type_index:-3]
+    >>> (hosted_binary_resource_type, scalar_type, simple_binary_type
+    ...     ) = other_children[-3:]
     >>> for index in range(0, len(entry_resource_types), 5):
     ...     entry_resource_descriptions.append(
     ...         (tuple(entry_resource_types[index:index + 5])))
@@ -1155,9 +1156,9 @@
 All collection representations have the same five <param> tags.
 
     >>> [param.attrib['name'] for param in collection_rep]
-    ['resource_type_link', 'total_size', 'start', 'next_collection_link',
-     'prev_collection_link', 'entries', 'entry_links']
-    >>> (type_link, size, start, next, prev, entries,
+    ['resource_type_link', 'total_size', 'total_size_link', 'start',
+     'next_collection_link', 'prev_collection_link', 'entries', 'entry_links']
+    >>> (type_link, size, size_link, start, next, prev, entries,
     ...  entry_links) = collection_rep
 
 So what's the difference between a collection of people and a

=== modified file 'src/lazr/restful/example/multiversion/tests/wadl.txt'
--- src/lazr/restful/example/multiversion/tests/wadl.txt	2010-03-10 20:44:03 +0000
+++ src/lazr/restful/example/multiversion/tests/wadl.txt	2010-08-10 12:24:46 +0000
@@ -92,3 +92,32 @@
     >>> pair_collection = contents['pair_collection']
     >>> sorted([method.attrib['id'] for method in pair_collection])
     ['key_value_pairs-get']
+
+total_size_link
+===============
+
+The version in which total_size_link is introduced is controlled by the
+first_version_with_total_size_link attribute of the web service configuration
+(IWebServiceConfiguration) utility.
+
+We'll configure the web service to begin including `total_size_link` values
+in version 3.0:
+
+    >>> from zope.component import getUtility
+    >>> from lazr.restful.interfaces import IWebServiceConfiguration
+    >>> config = getUtility(IWebServiceConfiguration)
+    >>> config.first_version_with_total_size_link = '3.0'
+
+Now if we request the WADL for 3.0 it will include a description of
+total_size_link.
+
+    >>> webservice.get('/', media_type='application/vnd.sun.wadl+xml',
+    ...     api_version='3.0').body
+    '...<wadl:param style="plain" name="total_size_link"...'
+
+If we request an earlier version, total_size_link is not described.
+
+    >>> wadl = webservice.get('/', media_type='application/vnd.sun.wadl+xml',
+    ...     api_version='2.0').body
+    >>> 'total_size_link' in wadl
+    False

=== modified file 'src/lazr/restful/interfaces/_rest.py'
--- src/lazr/restful/interfaces/_rest.py	2010-06-03 14:42:44 +0000
+++ src/lazr/restful/interfaces/_rest.py	2010-08-10 12:24:46 +0000
@@ -507,6 +507,13 @@
         all subsequent versions, they will not be published as named
         operations.""")
 
+    first_version_with_total_size_link = TextLine(
+        default=None,
+        description=u"""In earlier versions of lazr.restful collections
+        included a total_size field, now they include a total_size_link
+        instead. Setting this value determines in which version the new
+        behavior takes effect.""")
+
     code_revision = TextLine(
         default=u"",
         description=u"""A string designating the current revision

=== modified file 'src/lazr/restful/simple.py'
--- src/lazr/restful/simple.py	2010-06-14 18:47:39 +0000
+++ src/lazr/restful/simple.py	2010-08-10 12:24:46 +0000
@@ -477,5 +477,6 @@
 
 
 BaseWebServiceConfiguration = implement_from_dict(
-    "BaseWebServiceConfiguration", IWebServiceConfiguration, {}, object)
+    "BaseWebServiceConfiguration", IWebServiceConfiguration,
+    {'first_version_with_total_size_link': None}, object)
 

=== modified file 'src/lazr/restful/tales.py'
--- src/lazr/restful/tales.py	2010-03-10 20:44:03 +0000
+++ src/lazr/restful/tales.py	2010-08-10 12:24:46 +0000
@@ -34,7 +34,8 @@
     IResourceOperation, IResourcePOSTOperation, IScopedCollection,
     ITopLevelEntryLink, IWebServiceClientRequest, IWebServiceConfiguration,
     IWebServiceVersion, LAZR_WEBSERVICE_NAME)
-from lazr.restful.utils import get_current_web_service_request
+from lazr.restful.utils import (get_current_web_service_request,
+    is_total_size_link_active)
 
 
 class WadlDocstringLinker(DocstringLinker):
@@ -234,6 +235,11 @@
         return config.version_descriptions.get(self.service_version, None)
 
     @property
+    def is_total_size_link_active(self):
+        config = getUtility(IWebServiceConfiguration)
+        return is_total_size_link_active(self.resource.request.version, config)
+
+    @property
     def top_level_resources(self):
         """Return a list of dicts describing the top-level resources."""
         resource_dicts = []

=== modified file 'src/lazr/restful/templates/wadl-root.pt'
--- src/lazr/restful/templates/wadl-root.pt	2010-03-10 18:50:27 +0000
+++ src/lazr/restful/templates/wadl-root.pt	2010-08-10 12:24:46 +0000
@@ -13,21 +13,21 @@
 
   <wadl:doc xmlns="http://www.w3.org/1999/xhtml"
        title="About this service"
-       tal:content="structure context/wadl:description">
+       tal:content="structure service/wadl:description">
     Version-independent description of the web service.
   </wadl:doc>
 
   <wadl:doc xmlns="http://www.w3.org/1999/xhtml"
-       tal:define="version context/wadl:service_version"
+       tal:define="version service/wadl:service_version"
        tal:attributes="title string:About version ${version}"
-       tal:content="structure context/wadl:version_description">
+       tal:content="structure service/wadl:version_description">
     Description of this version of the web service.
   </wadl:doc>
 
   <!--There is one "service root" resource, located (as you'd expect)
       at the service root. This very document is the WADL
       representation of the "service root" resource.-->
-  <resources tal:attributes="base context/wadl:url">
+  <resources tal:attributes="base service/wadl:url">
     <resource path="" type="#service-root" />
   </resources>
 
@@ -46,7 +46,7 @@
   <!--The JSON representation of a "service root" resource contains a
       number of links to collection-type resources.-->
   <representation mediaType="application/json" id="service-root-json">
-    <tal:root_params tal:repeat="param context/wadl:top_level_resources">
+    <tal:root_params tal:repeat="param service/wadl:top_level_resources">
       <param style="plain" tal:attributes="name param/name;
                                            path param/path">
         <link tal:attributes="resource_type param/resource/wadl:type_link" />
@@ -284,15 +284,29 @@
 
     <representation mediaType="application/json"
        tal:attributes="id
-         string:${context/wadl_entry:entry_page_representation_id}">
+         string:${context/wadl_entry:entry_page_representation_id}"
+       tal:define="is_total_size_link_active
+                    service/wadl:is_total_size_link_active">
 
       <param style="plain" name="resource_type_link"
              path="$['resource_type_link']">
         <link />
       </param>
 
+      <tal:comment condition="nothing">
+        If we are not using total_size_link, continue to signal that
+        total_size is required as it has been in the past.
+      </tal:comment>
+
       <param style="plain" name="total_size" path="$['total_size']"
-             required="true" />
+          tal:attributes="
+            required python:is_total_size_link_active and 'false' or 'true'"/>
+
+      <param tal:condition="is_total_size_link_active"
+             style="plain" name="total_size_link" path="$['total_size_link']"
+             required="false">
+        <link resource_type="#ScalarValue" />
+      </param>
 
       <param style="plain" name="start" path="$['start']" required="true" />
 
@@ -321,7 +335,7 @@
   <!--End representation and resource_type definitions for entry
       resources. -->
 
-  <!--Finally, describe the 'hosted binary file' type.-->
+  <!--Finally, describe the 'hosted binary file' type...-->
   <resource_type id="HostedFile">
     <method name="GET" id="HostedFile-get">
       <response>
@@ -334,6 +348,15 @@
     <method name="DELETE" id="HostedFile-delete" />
   </resource_type>
 
+  <!--...and the simple 'scalar value' type.-->
+  <resource_type id="ScalarValue">
+   <method name="GET" id="ScalarValue-get">
+    <response>
+     <representation mediaType="application/json" />
+    </response>
+   </method>
+  </resource_type>
+
   <!--Define a data type for binary data.-->
   <xsd:simpleType name="binary">
    <xsd:list itemType="byte" />

=== modified file 'src/lazr/restful/testing/webservice.py'
--- src/lazr/restful/testing/webservice.py	2010-03-03 16:40:14 +0000
+++ src/lazr/restful/testing/webservice.py	2010-08-10 12:24:46 +0000
@@ -135,6 +135,7 @@
         self.traversed_objects = []
         self.query_string_params = {}
         self.method = 'GET'
+        self.URL = 'http://api.example.org/'
 
 
     def getTraversalStack(self):
@@ -173,6 +174,8 @@
 def pprint_collection(json_body):
     """Pretty-print a webservice collection JSON representation."""
     for key, value in sorted(json_body.items()):
+        if key == 'total_size_link':
+            continue
         if key != 'entries':
             print '%s: %r' % (key, value)
     print '---'

=== modified file 'src/lazr/restful/tests/test_utils.py'
--- src/lazr/restful/tests/test_utils.py	2010-03-03 13:08:12 +0000
+++ src/lazr/restful/tests/test_utils.py	2010-08-10 12:24:46 +0000
@@ -10,7 +10,8 @@
 from zope.security.management import (
     endInteraction, newInteraction, queryInteraction)
 
-from lazr.restful.utils import get_current_browser_request
+from lazr.restful.utils import (get_current_browser_request,
+    is_total_size_link_active)
 
 
 class TestUtils(unittest.TestCase):
@@ -28,6 +29,27 @@
         self.assertEquals(request, get_current_browser_request())
         endInteraction()
 
+    def test_is_total_size_link_active(self):
+        # Parts of the code want to know if the sizes of collections should be
+        # reported in an attribute or via a link back to the service.  The
+        # is_total_size_link_active function takes the version of the API in
+        # question and a web service configuration object and returns a
+        # boolean that is true if a link should be used, false otherwise.
+
+        # Here's the fake web service config we'll be using.
+        class FakeConfig:
+            active_versions = ['1.0', '2.0', '3.0']
+            first_version_with_total_size_link = '2.0'
+
+        # First, if the version is lower than the threshold for using links,
+        # the result is false (i.e., links should not be used).
+        self.assertEqual(is_total_size_link_active('1.0', FakeConfig), False)
+
+        # However, if the requested version is equal to, or higher than the
+        # threshold, the result is true (i.e., links should be used).
+        self.assertEqual(is_total_size_link_active('2.0', FakeConfig), True)
+        self.assertEqual(is_total_size_link_active('3.0', FakeConfig), True)
+
     # For the sake of convenience, test_get_current_web_service_request()
     # and tag_request_with_version_name() are tested in test_webservice.py.
 

=== modified file 'src/lazr/restful/tests/test_webservice.py'
--- src/lazr/restful/tests/test_webservice.py	2010-03-03 16:33:21 +0000
+++ src/lazr/restful/tests/test_webservice.py	2010-08-10 12:24:46 +0000
@@ -9,20 +9,22 @@
 import unittest
 
 from zope.component import getGlobalSiteManager, getUtility
-from zope.interface import implements, Interface
+from zope.interface import implements, Interface, directlyProvides
 from zope.publisher.browser import TestRequest
 from zope.schema import Date, Datetime, TextLine
 from zope.security.management import (
     endInteraction, newInteraction, queryInteraction)
 from zope.traversing.browser.interfaces import IAbsoluteURL
 
+from lazr.restful import ResourceOperation
 from lazr.restful.fields import Reference
 from lazr.restful.interfaces import (
     ICollection, IEntry, IEntryResource, IResourceGETOperation,
     IServiceRootResource, IWebServiceConfiguration,
     IWebServiceClientRequest, IWebServiceVersion)
 from lazr.restful import EntryResource, ResourceGETOperation
-from lazr.restful.declarations import exported, export_as_webservice_entry
+from lazr.restful.declarations import (exported, export_as_webservice_entry,
+    LAZR_WEBSERVICE_NAME)
 from lazr.restful.testing.webservice import (
     create_web_service_request, IGenericCollection, IGenericEntry,
     WebServiceTestCase, WebServiceTestPublication)
@@ -30,6 +32,7 @@
 from lazr.restful.utils import (
     get_current_browser_request, get_current_web_service_request,
     tag_request_with_version_name)
+from lazr.restful._resource import CollectionResource, BatchingResourceMixin
 
 
 def get_resource_factory(model_interface, resource_interface):
@@ -350,5 +353,78 @@
         self.assertEquals("2.0", webservice_request.version)
         self.assertTrue(marker_20.providedBy(webservice_request))
 
+
 def additional_tests():
     return unittest.TestLoader().loadTestsFromName(__name__)
+
+
+class ITestEntry(IEntry):
+    """Interface for a test entry."""
+    export_as_webservice_entry()
+
+
+class TestEntry:
+    implements(ITestEntry)
+    def __init__(self, context, request):
+        pass
+
+
+class BaseBatchingTest:
+    """A base class which tests BatchingResourceMixin and subclasses."""
+
+    testmodule_objects = [HasRestrictedField, IHasRestrictedField]
+
+    def setUp(self):
+        super(BaseBatchingTest, self).setUp()
+        # Register TestEntry as the IEntry implementation for ITestEntry.
+        getGlobalSiteManager().registerAdapter(
+            TestEntry, [ITestEntry, IWebServiceClientRequest], provided=IEntry)
+        # Is doing this by hand the right way?
+        ITestEntry.setTaggedValue(
+            LAZR_WEBSERVICE_NAME,
+            dict(singular='test_entity', plural='test_entities'))
+
+
+    def make_instance(self, entries, request):
+        raise NotImplementedError('You have to make your own instances.')
+
+    def test_getting_a_batch(self):
+        entries = [1, 2, 3]
+        request = create_web_service_request('/devel')
+        instance = self.make_instance(entries, request)
+        total_size = instance.get_total_size(entries)
+        self.assertEquals(total_size, '3')
+
+
+class TestBatchingResourceMixin(BaseBatchingTest, WebServiceTestCase):
+    """Test that BatchingResourceMixin does batching correctly."""
+
+    def make_instance(self, entries, request):
+        return BatchingResourceMixin()
+
+
+class TestCollectionResourceBatching(BaseBatchingTest, WebServiceTestCase):
+    """Test that CollectionResource does batching correctly."""
+
+    def make_instance(self, entries, request):
+        class Collection:
+            implements(ICollection)
+            entry_schema = ITestEntry
+
+            def __init__(self, entries):
+                self.entries = entries
+
+            def find(self):
+                return self.entries
+
+        return CollectionResource(Collection(entries), request)
+
+
+class TestResourceOperationBatching(BaseBatchingTest, WebServiceTestCase):
+    """Test that ResourceOperation does batching correctly."""
+
+    def make_instance(self, entries, request):
+        # constructor parameters are ignored
+        return ResourceOperation(None, request)
+
+

=== modified file 'src/lazr/restful/utils.py'
--- src/lazr/restful/utils.py	2010-03-10 20:44:03 +0000
+++ src/lazr/restful/utils.py	2010-08-10 12:24:46 +0000
@@ -37,6 +37,18 @@
 missing = object()
 
 
+def is_total_size_link_active(version, config):
+    versions = config.active_versions
+    total_size_link_version = config.first_version_with_total_size_link
+    # The version "None" is a special marker for the earliest version.
+    if total_size_link_version is None:
+        total_size_link_version = versions[0]
+    # If the version we're being asked about is equal to or later than the
+    # version in which we started exposing total_size_link, then we should
+    # return True, False otherwise.
+    return versions.index(total_size_link_version) <= versions.index(version)
+
+
 class VersionedDict(object):
     """A stack of named dictionaries.
 

=== modified file 'src/lazr/restful/version.txt'
--- src/lazr/restful/version.txt	2010-08-05 14:01:44 +0000
+++ src/lazr/restful/version.txt	2010-08-10 12:24:46 +0000
@@ -1,1 +1,1 @@
-0.10.0
+0.11.0-dev

=== modified file 'versions.cfg'
--- versions.cfg	2009-04-15 14:43:40 +0000
+++ versions.cfg	2010-08-10 12:24:46 +0000
@@ -1,172 +1,69 @@
+[buildout]
+versions = versions
+allow-picked-versions = false
+use-dependency-links = false
+
 [versions]
-ClientForm = 0.2.9
-RestrictedPython = 3.4.2
-ZConfig = 2.5.1
-ZODB3 = 3.8.1
-docutils = 0.4
-jquery.javascript = 1.0.0
-jquery.layer = 1.0.0
-lxml = 1.3.6
-mechanize = 0.1.7b
-pytz = 2007k
-setuptools = 0.6c9
-z3c.coverage = 1.1.2
-z3c.csvvocabulary = 1.0.0
-z3c.etestbrowser = 1.0.4
-z3c.form = 1.9.0
-z3c.formdemo = 1.5.3
-z3c.formjs = 0.4.0
-z3c.formjsdemo = 0.3.1
-z3c.formui = 1.4.2
-z3c.i18n = 0.1.1
-z3c.layer = 0.2.3
-z3c.macro = 1.1.0
-z3c.macroviewlet = 1.0.0
-z3c.menu = 0.2.0
-z3c.optionstorage = 1.0.4
-z3c.pagelet = 1.0.2
-z3c.rml = 0.7.3
-z3c.skin.pagelet = 1.0.2
-z3c.template = 1.1.0
-z3c.testing = 0.2.0
-z3c.traverser = 0.2.3
-z3c.viewlet = 1.0.0
-z3c.viewtemplate = 0.3.2
-z3c.zrtresource = 1.0.1
-zc.buildout = 1.1.1
-zc.catalog = 1.2.0
-zc.datetimewidget = 0.5.2
-zc.i18n = 0.5.2
-zc.recipe.egg = 1.0.0
-zc.recipe.filestorage = 1.0.0
-zc.recipe.testrunner = 1.0.0
-zc.resourcelibrary = 1.0.1
-zc.table = 0.6
-zc.zope3recipes = 0.6.2
-zdaemon = 2.0.2
-zodbcode = 3.4.0
-zope.annotation = 3.4.1
-zope.app.annotation = 3.4.0
-zope.app.apidoc = 3.4.3
-zope.app.applicationcontrol = 3.4.3
-zope.app.appsetup = 3.4.1
-zope.app.authentication = 3.4.4
-zope.app.basicskin = 3.4.0
-zope.app.boston = 3.4.0
-zope.app.broken = 3.4.0
-zope.app.cache = 3.4.1
-zope.app.catalog = 3.5.1
-zope.app.component = 3.4.1
-zope.app.container = 3.5.6
-zope.app.content = 3.4.0
-zope.app.dav = 3.4.1
-zope.app.debug = 3.4.1
-zope.app.debugskin = 3.4.0
-zope.app.dependable = 3.4.0
-zope.app.dtmlpage = 3.4.1
-zope.app.error = 3.5.1
-zope.app.exception = 3.4.1
-zope.app.externaleditor = 3.4.0
-zope.app.file = 3.4.4
-zope.app.folder = 3.4.0
-zope.app.form = 3.4.1
-zope.app.ftp = 3.4.0
-zope.app.generations = 3.4.1
-zope.app.homefolder = 3.4.0
-zope.app.http = 3.4.1
-zope.app.i18n = 3.4.4
-zope.app.i18nfile = 3.4.1
-zope.app.interface = 3.4.0
-zope.app.interpreter = 3.4.0
-zope.app.intid = 3.4.1
-zope.app.keyreference = 3.4.1
-zope.app.layers = 3.4.0
-zope.app.locales = 3.4.5
-zope.app.locking = 3.4.0
-zope.app.module = 3.4.0
-zope.app.onlinehelp = 3.4.1
-zope.app.pagetemplate = 3.4.1
-zope.app.pluggableauth = 3.4.0
-zope.app.preference = 3.4.1
-zope.app.preview = 3.4.0
-zope.app.principalannotation = 3.4.0
-zope.app.publication = 3.4.3
-zope.app.publisher = 3.4.1
-zope.app.pythonpage = 3.4.1
-zope.app.renderer = 3.4.0
-zope.app.rotterdam = 3.4.1
-zope.app.schema = 3.4.0
-zope.app.security = 3.5.2
-zope.app.securitypolicy = 3.4.6
-zope.app.server = 3.4.2
-zope.app.session = 3.5.1
-zope.app.skins = 3.4.0
-zope.app.sqlscript = 3.4.1
-zope.app.testing = 3.4.3
-zope.app.traversing = 3.4.0
-zope.app.tree = 3.4.0
-zope.app.twisted = 3.4.1
-zope.app.undo = 3.4.0
-zope.app.wfmc = 0.1.2
-zope.app.workflow = 3.4.1
-zope.app.wsgi = 3.4.1
-zope.app.xmlrpcintrospection = 3.4.0
-zope.app.zapi = 3.4.0
-zope.app.zcmlfiles = 3.4.3
-zope.app.zopeappgenerations = 3.4.0
-zope.app.zptpage = 3.4.1
-zope.cachedescriptors = 3.4.1
-zope.component = 3.4.0
-zope.configuration = 3.4.0
-zope.contentprovider = 3.4.0
-zope.contenttype = 3.4.0
-zope.copypastemove = 3.4.0
+# Alphabetical, case-SENSITIVE, blank line after this comment
+
+Jinja2 = 2.5
+Pygments = 1.3.1
+RestrictedPython = 3.5.1
+Sphinx = 1.0.1
+ZConfig = 2.7.1
+ZODB3 = 3.9.2
+docutils = 0.5
+epydoc = 3.0.1
+grokcore.component = 1.6
+lazr.batchnavigator = 1.2.0
+lazr.delegates = 1.2.0
+lazr.enum = 1.1.2
+lazr.lifecycle = 1.0
+lazr.uri = 1.0.2
+lxml = 2.2.7
+martian = 0.11
+pytz = 2010h
+setuptools = 0.6c11
+simplejson = 2.0.9
+transaction = 1.0.0
+van.testing = 2.0.1
+wsgi-intercept = 0.4
+wsgiref = 0.1.2
+z3c.recipe.sphinxdoc = 0.0.8
+z3c.recipe.staticlxml = 0.7.1
+z3c.recipe.tag = 0.2.0
+zc.buildout = 1.4.3
+zc.lockfile = 1.0.0
+zc.recipe.cmmi = 1.3.1
+zc.recipe.egg = 1.2.3b2
+zc.recipe.testrunner = 1.3.0
+zdaemon = 2.0.4
+zope.annotation = 3.5.0
+zope.app.pagetemplate = 3.7.1
+zope.browser = 1.2
+zope.cachedescriptors = 3.5.0
+zope.component = 3.9.3
+zope.configuration = 3.6.0
+zope.contenttype = 3.5.0
+zope.copy = 3.5.0
 zope.datetime = 3.4.0
-zope.decorator = 3.4.0
-zope.deferredimport = 3.4.0
-zope.deprecation = 3.4.0
-zope.documenttemplate = 3.4.0
-zope.dottedname = 3.4.2
-zope.dublincore = 3.4.0
-zope.error = 3.5.1
-zope.event = 3.4.0
-zope.exceptions = 3.4.0
-zope.file = 0.3.0
-zope.filerepresentation = 3.4.0
-zope.formlib = 3.4.0
-zope.hookable = 3.4.0
-zope.html = 1.0.1
-zope.i18n = 3.4.0
-zope.i18nmessageid = 3.4.3
-zope.index = 3.4.1
-zope.interface = 3.4.1
-zope.lifecycleevent = 3.4.0
-zope.location = 3.4.0
-zope.mimetype = 0.3.0
-zope.minmax = 1.1.0
-zope.modulealias = 3.4.0
-zope.pagetemplate = 3.4.0
-zope.proxy = 3.4.2
-zope.publisher = 3.4.6
-zope.rdb = 3.4.0
-zope.schema = 3.4.0
-zope.security = 3.4.1
-zope.securitypolicy = 3.4.1
-zope.sendmail = 3.4.0
-zope.sequencesort = 3.4.0
-zope.server = 3.4.3
-zope.session = 3.4.1
-zope.size = 3.4.0
-zope.structuredtext = 3.4.0
-zope.tal = 3.4.1
+zope.dublincore = 3.5.0
+zope.event = 3.4.1
+zope.exceptions = 3.5.2
+zope.hookable = 3.4.1
+zope.i18n = 3.7.1
+zope.i18nmessageid = 3.5.0
+zope.interface = 3.5.2
+zope.lifecycleevent = 3.5.2
+zope.location = 3.7.0
+zope.pagetemplate = 3.5.0
+zope.proxy = 3.5.0
+zope.publisher = 3.12.0
+zope.schema = 3.5.4
+zope.security = 3.7.1
+zope.size = 3.4.1
+zope.tal = 3.5.1
 zope.tales = 3.4.0
-zope.testbrowser = 3.4.2
-zope.testing = 3.5.6
-zope.testrecorder = 0.3.0
-zope.thread = 3.4
-zope.traversing = 3.6.0
-zope.ucol = 1.0.2
-zope.viewlet = 3.4.2
-zope.wfmc = 3.4.0
-zope.xmlpickle = 3.4.0
-van.testing = 2.0.1
+zope.testing = 3.9.4
+zope.testrunner = 4.0.0b5
+zope.traversing = 3.8.0