Merge lp:~txconvore-maint/txconvore/initial-tests into lp:txconvore

[Jonathan Jacobs]

=== added file '.bzrignore'
--- .bzrignore	1970-01-01 00:00:00 +0000
+++ .bzrignore	2011-04-13 16:28:00 +0000
@@ -0,0 +1,1 @@
+_trial_temp

=== modified file 'txconvore/client.py'
--- txconvore/client.py	2011-03-25 16:49:16 +0000
+++ txconvore/client.py	2011-04-13 16:28:00 +0000
@@ -1,12 +1,13 @@
+import urllib
 from base64 import b64encode
-from urllib import quote, urlencode
 
 try:
     import json
+    json # For Pyflakes.
 except:
     import simplejson as json
 
-from twisted.web.client import getPage
+from txconvore.util import PerseverantDownloader
 
 
 
@@ -37,6 +38,93 @@
 
 
 
+class Endpoint(object):
+    """
+    A Convore endpoint.
+
+    Used to submit requests to Convore.
+
+    @type debug: C{bool}
+    @cvar debug: Enable debug logging.
+
+    @type baseURL: C{str}
+    @ivar baseURL: Base URL of this endpoint, including the trailing slash.
+
+    @type version: C{str} or C{None}
+    @ivar version: API version to use, or C{None} for the latest.
+
+    @ivar creds: A credentials object, or C{None} to authenticate anonymously.
+    """
+    debug = False
+
+
+    def __init__(self, baseURL='https://convore.com/api/', version=None,
+                 creds=None):
+        if creds is not None and not isinstance(creds, BasicCreds):
+            raise ValueError('Only basic authentication is supported')
+        self.baseURL = baseURL
+        self.version = version
+        self.creds = creds
+
+
+    def getRootURL(self):
+        """
+        Get the root endpoint URL, including trailing slash.
+        """
+        url = self.baseURL
+        if self.version is not None:
+            url += self.version + '/'
+        return url
+
+
+    def getPage(self, *a, **kw):
+        downloader = PerseverantDownloader(*a, **kw)
+        downloader.debug = self.debug
+        return downloader.download()
+
+
+    def submit(self, url, method, data=None, headers=None):
+        """
+        Submit a request to a Convore endpoint.
+
+        @type  url: C{str}
+        @param url: URL to submit to.
+
+        @type  method: C{str}
+        @param method: HTTP method to use when submitting a request.
+
+        @type  data: C{dict}
+        @param data: HTTP POST data, or C{None}.
+
+        @type  headers: C{dict}
+        @param headers: HTTP headers to include when submitting a request.
+
+        @rtype: C{Deferred}
+        """
+        if headers is None:
+            headers = {}
+
+        headers['Accept'] = 'application/json'
+        if data is not None and not isinstance(data, str):
+            data = urllib.urlencode(data)
+            headers['Content-Type'] = 'application/x-www-form-urlencoded'
+
+        if self.creds is not None:
+            headers['Authorization'] = self.creds.encode()
+
+        def _parse((res, headers)):
+            if res:
+                return json.loads(unicode(res, 'utf-8'))
+
+        d = self.getPage(url=url,
+                         method=method,
+                         postdata=data,
+                         headers=headers)
+        d.addCallback(_parse)
+        return d
+
+
+
 class _ConvoreAPI(object):
     collectionName = None
 
@@ -45,41 +133,57 @@
         self.components = []
 
 
+    def _translateValue(self, value):
+        if isinstance(value, bool):
+            return [u'false', u'true'][value]
+        elif not isinstance(value, unicode):
+            return unicode(value, 'ascii')
+        return value
+
+
     def getURL(self, endpoint, prefix=None, suffix=[], params={}):
         """
         Get the URL for this path as accessed through the given endpoint.
 
         @type  endpoint: L{Endpoint}
-        @param endpoint: The endpoint to operate through.
+        @param endpoint: Endpoint to operate through.
 
         @type  prefix: C{list} of C{unicode}
-        @param prefix: A list of components to prepend before this path.
+        @param prefix: List of components to prepend before this path, or
+            C{None}.
 
         @type  suffix: C{list} of C{unicode}
-        @param suffix: A list of components to append after this path.
+        @param suffix: List of components to append after this path, or
+            C{None}.
+
+        @type  params: C{dict} mapping C{unicode} to C{unicode}
+        @param params: Mapping of URL query parameter names and values, or
+            C{None}.
 
         @rtype: C{str}
         """
-        if self.collectionName is None:
-            raise NotImplementedError(
-                'Must override collectionName on _ConvoreAPI subclasses')
-
         components = []
         if prefix is None:
+            if self.collectionName is None:
+                raise NotImplementedError(
+                    'Must override collectionName on _ConvoreAPI subclasses')
             components.append(self.collectionName)
         else:
             components.extend(prefix)
         components.extend(self.components)
         components.extend(suffix)
 
-        def _quote(xs):
-            return [quote(x.encode('utf-8'), safe='')
-                    for x in xs]
+        def _quote(v):
+            return urllib.quote(
+                self._translateValue(v).encode('utf-8'), safe='')
 
-        components = [quote(component.encode('utf-8'), safe='')
-                      for component in components]
-        params = [(k, v) for k, v in params.iteritems() if v is not None]
-        return endpoint.getRootURL() + '/'.join(components) + '?' + urlencode(params)
+        components = [_quote(component) for component in components]
+        url = endpoint.getRootURL() + '/'.join(components)
+        params = [(_quote(k), _quote(v))
+                  for k, v in params.iteritems() if v is not None]
+        if params:
+            url = url + '?' + urllib.urlencode(params)
+        return url
 
 
 
@@ -143,6 +247,24 @@
 
 
 
+class User(_ConvoreAPI):
+    collectionName = 'users'
+
+
+    def __init__(self, userID):
+        super(User, self).__init__()
+        self.components = [userID]
+
+
+    def info(self, endpoint):
+        """
+        Get detailed information about the user.
+        """
+        url = self.getURL(endpoint) + '.json'
+        return endpoint.submit(url, 'GET')
+
+
+
 class Group(_ConvoreAPI):
     collectionName = 'groups'
 
@@ -191,8 +313,8 @@
         """
         Mark all messages in the group as read.
         """
-        url = self.geturl(endpoint, suffix=['mark_read.json'])
-        return endpoint.submit(url, 'post')
+        url = self.getURL(endpoint, suffix=['mark_read.json'])
+        return endpoint.submit(url, 'POST')
 
 
     def members(self, endpoint, filter=None):
@@ -250,15 +372,19 @@
         self.components = [topicID]
 
 
-    def createMessage(self, endpoint, message):
+    def createMessage(self, endpoint, message, pasted=False):
         """
         Post a new message.
+
+        @type  pasted: C{bool}
+        @param pasted: Is the message preformatted?
         """
         url = self.getURL(endpoint, suffix=['messages', 'create.json'])
         return endpoint.submit(
             url,
             'POST',
-            data=dict(message=message))
+            data=dict(message=message,
+                      pasted=pasted))
 
 
     def delete(self, endpoint):
@@ -270,6 +396,15 @@
         return endpoint.submit(url, 'POST')
 
 
+    def edit(self, endpoint, name):
+        """
+        Edit a topic. You must be the creator of the topic or a group admin in
+        order to edit the topic.
+        """
+        url = self.getURL(endpoint, suffix=['edit.json'])
+        return endpoint.submit(url, 'POST', data=dict(name=name))
+
+
     def info(self, endpoint):
         """
         Get detailed information about the topic.
@@ -282,8 +417,8 @@
         """
         Mark all messages in a topic as read.
         """
-        url = self.geturl(endpoint, suffix=['mark_read.json'])
-        return endpoint.submit(url, 'post')
+        url = self.getURL(endpoint, suffix=['mark_read.json'])
+        return endpoint.submit(url, 'POST')
 
 
     def messages(self, endpoint, untilMessageID=None, markRead=False):
@@ -331,54 +466,3 @@
         """
         url = self.getURL(endpoint, suffix=['delete.json'])
         return endpoint.submit(url, 'POST')
-
-
-
-class Endpoint(object):
-    def __init__(self, baseURL='https://convore.com/api/', version=None,
-                 creds=None):
-        if creds is not None and not isinstance(creds, BasicCreds):
-            raise ValueError('Only basic authentication is supported')
-
-        self.baseURL = baseURL
-        self.version = version
-        self.creds = creds
-
-
-    def getRootURL(self):
-        """
-        Get the root endpoint URL, including trailing slash.
-        """
-        url = self.baseURL
-        if self.version is not None:
-            url += self.version + '/'
-        return url
-
-
-    def getPage(self, *a, **kw):
-        return getPage(*a, **kw)
-
-
-    def submit(self, url, method, data=None, headers=None):
-        print 'Submitting %s (%s: %r)...' % (url, method, data)
-        if headers is None:
-            headers = {}
-
-        headers['Accept'] = 'application/json'
-        if data is not None and not isinstance(data, str):
-            data = urlencode(data)
-            headers['Content-Type'] = 'application/x-www-form-urlencoded'
-
-        if self.creds is not None:
-            headers['Authorization'] = self.creds.encode()
-
-        def _parse(res):
-            if res:
-                return json.loads(unicode(res, 'utf-8'))
-
-        d = self.getPage(url=url,
-                         method=method,
-                         postdata=data,
-                         headers=headers,
-                         agent='txConvore')
-        return d.addCallback(_parse)

=== added directory 'txconvore/test'
=== added file 'txconvore/test/__init__.py'
=== added directory 'txconvore/test/data'
=== added file 'txconvore/test/data/account_groups.json'
--- txconvore/test/data/account_groups.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/account_groups.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,64 @@
+{
+    "groups": [
+        {
+            "creator": {
+                "id": "XXX", 
+                "img": "XXX", 
+                "url": "XXX", 
+                "username": "fdrake"
+            }, 
+            "date_created": 1297277511.4270909, 
+            "date_latest_message": 1301140974.9002681, 
+            "id": "XXX", 
+            "is_admin": false, 
+            "is_member": true, 
+            "kind": "public", 
+            "members_count": 2051, 
+            "name": "Python", 
+            "slug": "python", 
+            "topics_count": 103, 
+            "unread": 1, 
+            "url": "XXX"
+        }, 
+        {
+            "creator": {
+                "id": "XXX", 
+                "img": "XXX", 
+                "url": "XXX", 
+                "username": "djchall"
+            }, 
+            "date_created": 1296454650.842808, 
+            "date_latest_message": 1301122342.4191129, 
+            "id": "XXX", 
+            "is_admin": false, 
+            "is_member": true, 
+            "kind": "public", 
+            "members_count": 378, 
+            "name": "Feedback", 
+            "slug": "feedback", 
+            "topics_count": 446, 
+            "unread": 0, 
+            "url": "XXX"
+        }, 
+        {
+            "creator": {
+                "id": "XXX", 
+                "img": "XXX", 
+                "url": "XXX", 
+                "username": "mithrandi"
+            }, 
+            "date_created": 1300960771.1402569, 
+            "date_latest_message": 1301099446.2922111, 
+            "id": "XXX", 
+            "is_admin": false, 
+            "is_member": true, 
+            "kind": "public", 
+            "members_count": 13, 
+            "name": "Twisted", 
+            "slug": "twisted", 
+            "topics_count": 1, 
+            "unread": 0, 
+            "url": "XXX"
+        }
+    ]
+}

=== added file 'txconvore/test/data/account_groups_create.json'
--- txconvore/test/data/account_groups_create.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/account_groups_create.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,18 @@
+{
+    "group": {
+        "creator": {
+            "id": "XXX", 
+            "img": "XXX", 
+            "url": "XXX", 
+            "username": "XXX"
+        }, 
+        "date_created": 1302707186.200645, 
+        "date_latest_message": 1302707186.2006581, 
+        "id": "XXX", 
+        "kind": "private", 
+        "members_count": 1, 
+        "name": "xxx", 
+        "slug": "xxx", 
+        "url": "XXX"
+    }
+}

=== added file 'txconvore/test/data/account_mentions.json'
--- txconvore/test/data/account_mentions.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/account_mentions.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,51 @@
+{
+    "mentions": [
+        {
+            "message": {
+                "date_created": 1301096699.259259, 
+                "embeds": [], 
+                "id": "XXX", 
+                "message": "@jonathanj Does it do static inspection though", 
+                "stars": [], 
+                "user": {
+                    "id": "XXX", 
+                    "img": "XXX", 
+                    "url": "XXX", 
+                    "username": "lvh"
+                }
+            }, 
+            "topic": {
+                "date_created": 1301007329.5264051, 
+                "date_latest_message": 1301109138.3067639, 
+                "id": "XXX", 
+                "name": "Programmatically building __all__ with a decorator", 
+                "slug": "programmatically-building-__all__-with-a-decorator", 
+                "url": "XXX"
+            }
+        }, 
+        {
+            "message": {
+                "date_created": 1301072572.6535151, 
+                "embeds": [], 
+                "id": "XXX", 
+                "message": "(also @jonathanj is awesome)", 
+                "stars": [], 
+                "user": {
+                    "id": "XXX", 
+                    "img": "XXX", 
+                    "url": "XXX", 
+                    "username": "mithrandi"
+                }
+            }, 
+            "topic": {
+                "date_created": 1301072549.0416789, 
+                "date_latest_message": 1301099446.2922111, 
+                "id": "XXX", 
+                "name": "txConvore", 
+                "slug": "txconvore", 
+                "url": "XXX"
+            }
+        }
+    ], 
+    "unread": 0
+}

=== added file 'txconvore/test/data/account_online.json'
--- txconvore/test/data/account_online.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/account_online.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,15 @@
+{
+    "count": 1, 
+    "online": [
+        {
+            "bio": "Foo", 
+            "id": "XXX", 
+            "img": "XXX", 
+            "location": "Earth", 
+            "name": "Chuck Whatshisface", 
+            "url": "XXX", 
+            "username": "XXX", 
+            "web": ""
+        }
+    ]
+}

=== added file 'txconvore/test/data/account_verify.json'
--- txconvore/test/data/account_verify.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/account_verify.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,6 @@
+{
+    "id": "XXX", 
+    "img": "XXX", 
+    "url": "XXX", 
+    "username": "jonathanj"
+}

=== added file 'txconvore/test/data/groups.json'
--- txconvore/test/data/groups.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/groups.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,18 @@
+{
+    "group": {
+        "creator": {
+            "id": "XXX", 
+            "img": "XXX", 
+            "url": "XXX", 
+            "username": "x"
+        }, 
+        "date_created": 1300949568.68558, 
+        "date_latest_message": 1301574176.861922, 
+        "id": "4321", 
+        "kind": "public", 
+        "members_count": 42, 
+        "name": "agroup", 
+        "slug": "agroup", 
+        "url": "XXX"
+    }
+}

=== added file 'txconvore/test/data/groups_members.json'
--- txconvore/test/data/groups_members.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/groups_members.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,22 @@
+{
+    "members": [
+        {
+            "admin": false, 
+            "user": {
+                "id": "1", 
+                "img": "XXX", 
+                "url": "XXX", 
+                "username": "aye"
+            }
+        }, 
+        {
+            "admin": true, 
+            "user": {
+                "id": "2", 
+                "img": "XXX", 
+                "url": "XXX", 
+                "username": "bee"
+            }
+        }
+    ]
+}

=== added file 'txconvore/test/data/groups_online.json'
--- txconvore/test/data/groups_online.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/groups_online.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,25 @@
+{
+    "count": 2, 
+    "online": [
+        {
+            "bio": "", 
+            "id": "1", 
+            "img": "", 
+            "location": "", 
+            "name": "Aye", 
+            "url": "XXX", 
+            "username": "aye", 
+            "web": ""
+        }, 
+        {
+            "bio": "", 
+            "id": "2", 
+            "img": "", 
+            "location": "", 
+            "name": "Bee", 
+            "url": "XXX", 
+            "username": "bee", 
+            "web": ""
+        }
+    ]
+}

=== added file 'txconvore/test/data/groups_topics.json'
--- txconvore/test/data/groups_topics.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/groups_topics.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,37 @@
+{
+    "topics": [
+        {
+            "creator": {
+                "id": "XXX", 
+                "img": "XXX", 
+                "url": "XXX", 
+                "username": "XXX"
+            }, 
+            "date_created": 1300959156.58742, 
+            "date_latest_message": 1301585498.7861631, 
+            "id": "1", 
+            "message_count": 935, 
+            "name": "General Chat", 
+            "slug": "general-chat", 
+            "unread": 0, 
+            "url": "XXX"
+        }, 
+        {
+            "creator": {
+                "id": "XXX", 
+                "img": "XXX", 
+                "url": "XXX", 
+                "username": "XXX"
+            }, 
+            "date_created": 1301574442.4583809, 
+            "date_latest_message": 1301574442.4583919, 
+            "id": "2", 
+            "message_count": 1, 
+            "name": "txConvore Testing", 
+            "slug": "txconvore-testing", 
+            "unread": 1, 
+            "url": "XXX"
+        }
+    ], 
+    "until_id": null
+}

=== added file 'txconvore/test/data/groups_topics_create.json'
--- txconvore/test/data/groups_topics_create.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/groups_topics_create.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,16 @@
+{
+    "topic": {
+        "creator": {
+            "id": "XXX", 
+            "img": "XXX", 
+            "url": "XXX", 
+            "username": "XXX"
+        }, 
+        "date_created": 1301574442.4583809, 
+        "date_latest_message": 1301574442.4583919, 
+        "id": "1234", 
+        "name": "txConvore Testing", 
+        "slug": "txconvore-testing", 
+        "url": "XXX"
+    }
+}

=== added file 'txconvore/test/data/messages_star.json'
--- txconvore/test/data/messages_star.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/messages_star.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,11 @@
+{
+    "star": {
+        "date_created": 1302597756.098841, 
+        "user": {
+            "id": "1", 
+            "img": "XXX", 
+            "url": "XXX", 
+            "username": "XXX"
+        }
+    }
+}

=== added file 'txconvore/test/data/messages_unstar.json'
--- txconvore/test/data/messages_unstar.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/messages_unstar.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,11 @@
+{
+    "unstar": {
+        "date_created": 1302597756.098841, 
+        "user": {
+            "id": "1", 
+            "img": "XXX", 
+            "url": "XXX", 
+            "username": "XXX"
+        }
+    }
+}

=== added file 'txconvore/test/data/topics_edit.json'
--- txconvore/test/data/topics_edit.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/topics_edit.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,16 @@
+{
+    "topic": {
+        "creator": {
+            "id": "XXX", 
+            "img": "XXX", 
+            "url": "XXX", 
+            "username": "XXX"
+        }, 
+        "date_created": 1301574442.4583809, 
+        "date_latest_message": 1301574442.4583919, 
+        "id": "XXX", 
+        "name": "New Name", 
+        "slug": "test", 
+        "url": "XXX"
+    }
+}

=== added file 'txconvore/test/data/topics_info.json'
--- txconvore/test/data/topics_info.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/topics_info.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,16 @@
+{
+    "topic": {
+        "creator": {
+            "id": "XXX", 
+            "img": "XXX", 
+            "url": "XXX", 
+            "username": "XXX"
+        }, 
+        "date_created": 1300959156.58742, 
+        "date_latest_message": 1302596569.9464591, 
+        "id": "1", 
+        "name": "General Chat", 
+        "slug": "general-chat", 
+        "url": "XXX"
+    }
+}

=== added file 'txconvore/test/data/topics_messages.json'
--- txconvore/test/data/topics_messages.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/topics_messages.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,45 @@
+
+{
+    "messages": [
+        {
+            "date_created": 1302596453.564373, 
+            "embeds": [], 
+            "id": "XXX", 
+            "message": "Test message", 
+            "stars": [], 
+            "user": {
+                "id": "XXX", 
+                "img": "XXX", 
+                "url": "XXX", 
+                "username": "XXX"
+            }
+        }, 
+        {
+            "date_created": 1302596569.9341509, 
+            "embeds": [], 
+            "id": "XXX", 
+            "message": "Test response", 
+            "stars": [], 
+            "user": {
+                "id": "XXX", 
+                "img": "XXX", 
+                "url": "XXX", 
+                "username": "XXX"
+            }
+        }, 
+        {
+            "date_created": 1302596724.3432169, 
+            "embeds": [], 
+            "id": "XXX", 
+            "message": "oops, got the wrong ID", 
+            "stars": [], 
+            "user": {
+                "id": "XXX", 
+                "img": "XXX", 
+                "url": "XXX", 
+                "username": "XXX"
+            }
+        }
+    ], 
+    "until_id": "639139"
+}

=== added file 'txconvore/test/data/topics_messages_create.json'
--- txconvore/test/data/topics_messages_create.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/topics_messages_create.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,15 @@
+{
+    "message": {
+        "date_created": 1302596453.564373, 
+        "embeds": [], 
+        "id": "XXX", 
+        "message": "Test message", 
+        "stars": [], 
+        "user": {
+            "id": "XXX", 
+            "img": "XXX", 
+            "url": "XXX", 
+            "username": "XXX"
+        }
+    }
+}

=== added file 'txconvore/test/data/users_info.json'
--- txconvore/test/data/users_info.json	1970-01-01 00:00:00 +0000
+++ txconvore/test/data/users_info.json	2011-04-13 16:28:00 +0000
@@ -0,0 +1,12 @@
+{
+    "user": {
+        "bio": "", 
+        "id": "1", 
+        "img": "XXX", 
+        "location": "", 
+        "name": "Chuck Finley", 
+        "url": "XXX", 
+        "username": "Y", 
+        "web": "http://web.com/"
+    }
+}

=== added file 'txconvore/test/test_client.py'
--- txconvore/test/test_client.py	1970-01-01 00:00:00 +0000
+++ txconvore/test/test_client.py	2011-04-13 16:28:00 +0000
@@ -0,0 +1,697 @@
+from twisted.internet.defer import succeed
+from twisted.python.filepath import FilePath
+from twisted.trial.unittest import TestCase
+
+from txconvore.client import (
+    Endpoint, BasicCreds, Account, _ConvoreAPI, Group, Topic, Message, User)
+
+
+
+class MockEndpoint(Endpoint):
+    """
+    Endpoint with submit mocked out.
+    """
+    def getPage(self, url, method, postdata=None, headers=None, *a, **kw):
+        """
+        Store our parameters instead of submitting a request.
+        """
+        self.url = url
+        self.method = method
+        self.data = postdata
+        self.headers = headers
+        return succeed((self.response, {}))
+
+
+
+class EndpointTests(TestCase):
+    """
+    Tests for L{txconvore.client.Endpoint}.
+    """
+    def test_defaultEndpoint(self):
+        """
+        L{Endpoint} defaults to the real global Convore endpoint.
+        """
+        endpoint = Endpoint()
+        self.assertEqual(endpoint.baseURL, 'https://convore.com/api/')
+
+
+    def test_getDefaultRoot(self):
+        """
+        The default root URL has no API version.
+        """
+        endpoint = Endpoint()
+        self.assertEqual(endpoint.getRootURL(), endpoint.baseURL)
+
+
+    def test_versionedRoot(self):
+        """
+        The root URL must include the API version if specified.
+        """
+        endpoint = Endpoint('http://convore.test.url/', '20090817')
+        self.assertEqual(endpoint.getRootURL(),
+                         'http://convore.test.url/20090817/')
+
+
+    def test_basicAuth(self):
+        """
+        An endpoint constructed with basic auth credentials sends them along
+        with every request.
+        """
+        creds = BasicCreds('testuser', 'password')
+        endpoint = MockEndpoint('http://convore.test.url/', creds=creds)
+        endpoint.response = ''
+        endpoint.submit('http://convore.test.url/blah', 'GET')
+        self.assertEqual(endpoint.headers['Authorization'],
+                         'Basic dGVzdHVzZXI6cGFzc3dvcmQ=')
+
+
+    def test_unsuppportedAuth(self):
+        """
+        Constructing an endpoint with an unsupported credentials object raises
+        an exception.
+        """
+        self.assertRaises(ValueError, Endpoint, creds=42)
+
+
+
+class ConvoreAPITests(TestCase):
+    """
+    Tests for L{txconvore.client._ConvoreAPI}.
+    """
+    def setUp(self):
+        creds = BasicCreds('testuser', 'password')
+        self.endpoint = MockEndpoint('http://convore.test.url/api/',
+                                     creds=creds)
+
+
+    def test_getURL(self):
+        """
+        Constructing an endpoint URL uses the I{collectionName} attribute as a
+        prefix unless the I{prefix} parameter is given. If the I{suffix}
+        parameter is given those paths are appended to the URL. The I{params}
+        parameter specifies URL query parameters.
+        """
+        c = _ConvoreAPI()
+        c.collectionName = 'a'
+        self.assertEqual(
+            c.getURL(self.endpoint),
+            'http://convore.test.url/api/a')
+        self.assertEqual(
+            c.getURL(self.endpoint,
+                     prefix=['b'],
+                     suffix=['c'],
+                     params=dict(quux='baz')),
+            'http://convore.test.url/api/b/c?quux=baz')
+
+
+    def test_getURLWithPrefix(self):
+        """
+        Construct an endpoint URL overriding the URL prefix.
+        """
+        c = _ConvoreAPI()
+        c.collectionName = 'a'
+        self.assertEqual(
+            c.getURL(self.endpoint, prefix=['b']),
+            'http://convore.test.url/api/b')
+        self.assertEqual(
+            c.getURL(self.endpoint, prefix=['c', 'd']),
+            'http://convore.test.url/api/c/d')
+
+
+    def test_getURLWithSuffix(self):
+        """
+        Construct an endpoint URL specifying the URL suffix.
+        """
+        c = _ConvoreAPI()
+        c.collectionName = 'a'
+        self.assertEqual(
+            c.getURL(self.endpoint, suffix=['b']),
+            'http://convore.test.url/api/a/b')
+        self.assertEqual(
+            c.getURL(self.endpoint, suffix=['b', 'c']),
+            'http://convore.test.url/api/a/b/c')
+
+
+    def test_getURLWithPrefixAndSuffix(self):
+        """
+        Construct an endpoint URL overriding the URL prefix and specifying the
+        URL suffix.
+        """
+        c = _ConvoreAPI()
+        c.collectionName = 'a'
+        self.assertEqual(
+            c.getURL(self.endpoint, prefix=['b', 'c'], suffix=['d']),
+            'http://convore.test.url/api/b/c/d')
+        self.assertEqual(
+            c.getURL(self.endpoint, prefix=['b'], suffix=['c', 'd']),
+            'http://convore.test.url/api/b/c/d')
+        self.assertEqual(
+            c.getURL(self.endpoint,
+                     prefix=[u'\N{DAGGER}'],
+                     suffix=[u'\N{BULLET}']),
+            'http://convore.test.url/api/%E2%80%A0/%E2%80%A2')
+
+
+    def test_getURLWithoutCollectionName(self):
+        """
+        Constructing a URL without specifying I{collectionName} or I{prefix}
+        raises C{NotImplementedError}.
+        """
+        c = _ConvoreAPI()
+        self.assertRaises(
+            NotImplementedError, c.getURL, self.endpoint)
+        self.assertEqual(
+            c.getURL(self.endpoint, prefix=['a']),
+            'http://convore.test.url/api/a')
+
+
+    def test_getURLParams(self):
+        """
+        Construct an endpoint URL specifying URL query parameters.
+        """
+        c = _ConvoreAPI()
+        c.collectionName = 'a'
+        self.assertEqual(
+            c.getURL(
+                self.endpoint,
+                params=dict(
+                    foo='bar')),
+            'http://convore.test.url/api/a?foo=bar')
+        self.assertEqual(
+            c.getURL(
+                self.endpoint,
+                params={
+                    u'k': u'\N{DAGGER}',
+                    u'\N{BULLET}': u'v'}),
+            'http://convore.test.url/api/a?k=%25E2%2580%25A0&%25E2%2580%25A2=v')
+
+
+
+class ConvoreAPITestCase(TestCase):
+    def setUp(self):
+        creds = BasicCreds('testuser', 'password')
+        self.endpoint = MockEndpoint(
+            'http://convore.test.url/api/', creds=creds)
+
+
+    def getTestData(self, filename):
+        """
+        Open a test data file, in the C{test/data} directory, by name and
+        return its contents.
+        """
+        dataPath = FilePath(__file__).sibling('data').child(filename)
+        return dataPath.open('rb').read()
+
+
+    def callAPI(self, method, url, func, *a, **kw):
+        """
+        Call C{func}, a txConvore API function, verifying the HTTP method on
+        the endpoint and the URL the endpoint contacted match C{method} and
+        C{url} respectively.
+        """
+        d = func(self.endpoint, *a, **kw)
+        self.assertEqual(self.endpoint.method, method)
+        self.assertEqual(self.endpoint.url, url)
+        return d
+
+
+
+class AccountTests(ConvoreAPITestCase):
+    """
+    Tests for L{txconvore.client.Account}.
+    """
+    def setUp(self):
+        super(AccountTests, self).setUp()
+        self.account = Account()
+
+
+    def test_createGroup(self):
+        """
+        Create a new group.
+        """
+        self.endpoint.response = self.getTestData('account_groups_create.json')
+        d = self.callAPI(
+            'POST',
+            'http://convore.test.url/api/groups/create.json',
+            self.account.createGroup, u'xxx', u'private')
+
+        @d.addCallback
+        def gotData(data):
+            self.assertEqual(data[u'group'][u'name'], u'xxx')
+            self.assertEqual(data[u'group'][u'kind'], u'private')
+
+        return d
+
+
+    def test_groups(self):
+        """
+        Get a list of the current user's groups.
+        """
+        self.endpoint.response = self.getTestData('account_groups.json')
+        d = self.callAPI(
+            'GET',
+            'http://convore.test.url/api/groups.json',
+            self.account.groups)
+
+        @d.addCallback
+        def gotData(data):
+            self.assertEqual(len(data['groups']), 3)
+
+        return d
+
+
+    def test_markRead(self):
+        """
+        Mark all messages as read.
+        """
+        self.endpoint.response = ''
+        return self.callAPI(
+            'GET',
+            'http://convore.test.url/api/account/mark_read.json',
+            self.account.markRead)
+
+
+    def test_mentions(self):
+        """
+        Get the user's mentions.
+        """
+        self.endpoint.response = self.getTestData('account_mentions.json')
+        d = self.callAPI(
+            'GET',
+            'http://convore.test.url/api/account/mentions.json',
+            self.account.mentions)
+
+        @d.addCallback
+        def gotData(data):
+            self.assertEqual(len(data['mentions']), 2)
+
+        return d
+
+
+    def test_online(self):
+        """
+        Get members online now.
+        """
+        self.endpoint.response = self.getTestData('account_online.json')
+        d = self.callAPI(
+            'GET',
+            'http://convore.test.url/api/account/online.json',
+            self.account.online)
+
+        @d.addCallback
+        def gotData(data):
+            self.assertEqual(len(data['online']), 1)
+            self.assertEqual(data['count'], 1)
+
+        return d
+
+
+    def test_verify(self):
+        """
+        Determine whether the user is properly logged in.
+        """
+        self.endpoint.response = self.getTestData('account_verify.json')
+        d = self.callAPI(
+            'GET',
+            'http://convore.test.url/api/account/verify.json',
+            self.account.verify)
+
+        @d.addCallback
+        def gotData(data):
+            self.assertEqual(data['username'], u'jonathanj')
+
+        return d
+
+
+
+class UserTests(ConvoreAPITestCase):
+    """
+    Tests for L{txconvore.client.User}.
+    """
+    def setUp(self):
+        super(UserTests, self).setUp()
+        self.user = User(u'1')
+
+
+    def test_info(self):
+        """
+        Get detailed information about the user.
+        """
+        self.endpoint.response = self.getTestData('users_info.json')
+        d = self.callAPI(
+            'GET',
+            'http://convore.test.url/api/users/1.json',
+            self.user.info)
+
+        @d.addCallback
+        def gotData(data):
+            self.assertEqual(data['user']['id'], u'1')
+            self.assertEqual(data['user']['name'], u'Chuck Finley')
+            self.assertEqual(data['user']['username'], u'Y')
+
+        return d
+
+
+
+class GroupTests(ConvoreAPITestCase):
+    def setUp(self):
+        super(GroupTests, self).setUp()
+        self.group = Group(u'4321')
+
+
+    def test_createTopic(self):
+        """
+        Create a new topic.
+        """
+        self.endpoint.response = self.getTestData('groups_topics_create.json')
+        d = self.callAPI(
+            'POST',
+            'http://convore.test.url/api/groups/4321/topics/create.json',
+            self.group.createTopic, u'txConvore Testing')
+
+        @d.addCallback
+        def gotData(data):
+            self.assertEqual(data['topic']['id'], u'1234')
+            self.assertEqual(data['topic']['name'], u'txConvore Testing')
+
+        return d
+
+
+    def test_info(self):
+        """
+        Get detailed information about the group.
+        """
+        self.endpoint.response = self.getTestData('groups.json')
+        d = self.callAPI(
+            'GET',
+            'http://convore.test.url/api/groups/4321.json',
+            self.group.info)
+
+        @d.addCallback
+        def gotData(data):
+            self.assertEqual(data['group']['creator']['username'], u'x')
+            self.assertEqual(data['group']['id'], u'4321')
+            self.assertEqual(data['group']['name'], u'agroup')
+            self.assertEqual(data['group']['slug'], u'agroup')
+
+        return d
+
+
+    def test_join(self):
+        """
+        Join a public group.
+        """
+        self.endpoint.response = '{"message": "Group joined."}'
+        d = self.callAPI(
+            'POST',
+            'http://convore.test.url/api/groups/4321/join.json',
+            self.group.join)
+        return d
+
+
+    def test_leave(self):
+        """
+        Leave a group.
+        """
+        self.endpoint.response = '{"message": "Group left."}'
+        d = self.callAPI(
+            'POST',
+            'http://convore.test.url/api/groups/4321/leave.json',
+            self.group.leave)
+        return d
+
+
+    def test_markRead(self):
+        """
+        Mark all messages in the group as read.
+        """
+        self.endpoint.response = '{"message": "Group marked as read."}'
+        d = self.callAPI(
+            'POST',
+            'http://convore.test.url/api/groups/4321/mark_read.json',
+            self.group.markRead)
+        return d
+
+
+    def test_members(self):
+        """
+        Get all the members of a group.
+        """
+        self.endpoint.response = self.getTestData('groups_members.json')
+        d = self.callAPI(
+            'GET',
+            'http://convore.test.url/api/groups/4321/members.json',
+            self.group.members)
+
+        @d.addCallback
+        def gotData(data):
+            members = data['members']
+            self.assertEqual(len(members), 2)
+            self.assertEqual(members[0]['admin'], False)
+            self.assertEqual(members[0]['user']['id'], u'1')
+            self.assertEqual(members[1]['admin'], True)
+            self.assertEqual(members[1]['user']['id'], u'2')
+
+        return d
+
+
+    def test_membersFilter(self):
+        """
+        Get all the members of a group, requesting only certain member types.
+        """
+        self.endpoint.response = ''
+        d = self.callAPI(
+            'GET',
+            'http://convore.test.url/api/groups/4321/members.json?filter=admin',
+            self.group.members, filter=u'admin')
+        return d
+
+
+    def test_onlineMembers(self):
+        """
+        Get group members online now.
+        """
+        self.endpoint.response = self.getTestData('groups_online.json')
+        d = self.callAPI(
+            'GET',
+            'http://convore.test.url/api/groups/4321/online.json',
+            self.group.onlineMembers)
+
+        @d.addCallback
+        def gotData(data):
+            online = data['online']
+            self.assertEqual(len(online), 2)
+            self.assertEqual(data['count'], len(online))
+            self.assertEqual(online[0]['id'], u'1')
+            self.assertEqual(online[1]['id'], u'2')
+
+        return d
+
+
+    def test_request(self):
+        """
+        Request to join a private group.
+        """
+        self.endpoint.response = '{"message": "Something something."}'
+        d = self.callAPI(
+            'POST',
+            'http://convore.test.url/api/groups/4321/request.json',
+            self.group.request)
+        return d
+
+
+    def test_topics(self):
+        """
+        Get the latest topics in a group.
+        """
+        self.endpoint.response = self.getTestData('groups_topics.json')
+        d = self.callAPI(
+            'GET',
+            'http://convore.test.url/api/groups/4321/topics.json',
+            self.group.topics)
+
+        @d.addCallback
+        def gotData(data):
+            topics = data['topics']
+            self.assertEqual(len(topics), 2)
+            self.assertEqual(topics[0]['id'], u'1')
+            self.assertEqual(topics[1]['id'], u'2')
+
+        return d
+
+
+
+class TopicTests(ConvoreAPITestCase):
+    def setUp(self):
+        super(TopicTests, self).setUp()
+        self.topic = Topic(u'1')
+
+
+    def test_createMessage(self):
+        """
+        Post a new message.
+        """
+        self.endpoint.response = self.getTestData('topics_messages_create.json')
+        d = self.callAPI(
+            'POST',
+            'http://convore.test.url/api/topics/1/messages/create.json',
+            self.topic.createMessage, u'Test message')
+
+        @d.addCallback
+        def gotData(data):
+            message = data[u'message']
+            self.assertEqual(message[u'message'], u'Test message')
+
+        return d
+
+
+    def test_delete(self):
+        """
+        Delete a topic.
+        """
+        self.endpoint.response = '{"message": "Something something."}'
+        d = self.callAPI(
+            'POST',
+            'http://convore.test.url/api/topics/1/delete.json',
+            self.topic.delete)
+        return d
+
+
+    def test_edit(self):
+        """
+        Delete a topic.
+        """
+        self.endpoint.response = self.getTestData('topics_edit.json')
+        d = self.callAPI(
+            'POST',
+            'http://convore.test.url/api/topics/1/edit.json',
+            self.topic.edit, name=u'New Name')
+        return d
+
+
+    def test_info(self):
+        """
+        Get detailed information about the topic.
+        """
+        self.endpoint.response = self.getTestData('topics_info.json')
+        d = self.callAPI(
+            'GET',
+            'http://convore.test.url/api/topics/1.json',
+            self.topic.info)
+
+        @d.addCallback
+        def gotData(data):
+            self.assertEqual(data[u'topic'][u'id'], u'1')
+
+        return d
+
+
+    def test_markRead(self):
+        """
+        Mark all messages in a topic as read.
+        """
+        self.endpoint.response = '{"message": "Something something."}'
+        d = self.callAPI(
+            'POST',
+            'http://convore.test.url/api/topics/1/mark_read.json',
+            self.topic.markRead)
+        return d
+
+
+    def test_messages(self):
+        """
+        Get the latest messages in a topic.
+        """
+        self.endpoint.response = self.getTestData('topics_messages.json')
+        d = self.callAPI(
+            'GET',
+            'http://convore.test.url/api/topics/1/messages.json?mark_read=false',
+            self.topic.messages)
+
+        @d.addCallback
+        def gotData(data):
+            self.assertEqual(len(data[u'messages']), 3)
+            self.assertEqual(data[u'until_id'], u'639139')
+
+        return d
+
+
+    def test_messagesMarkRead(self):
+        """
+        Get the latest messages in a topic.
+        """
+        self.endpoint.response = '{"message": "Something something."}'
+        d = self.callAPI(
+            'GET',
+            'http://convore.test.url/api/topics/1/messages.json?mark_read=true',
+            self.topic.messages, markRead=True)
+        return d
+
+
+    def test_messagesUntilMessageID(self):
+        """
+        Get the latest messages in a topic.
+        """
+        self.endpoint.response = '{"message": "Something something."}'
+        d = self.callAPI(
+            'GET',
+            'http://convore.test.url/api/topics/1/messages.json?until_id=42&mark_read=false',
+            self.topic.messages, untilMessageID=u'42')
+        return d
+
+
+
+class MessageTests(ConvoreAPITestCase):
+    def setUp(self):
+        super(MessageTests, self).setUp()
+        self.message = Message(u'21')
+
+
+    def test_star(self):
+        """
+        Star a message. If the message has already been starred by this user,
+        this endpoint will then unstar the message.
+        """
+        self.endpoint.response = self.getTestData('messages_star.json')
+        d = self.callAPI(
+            'POST',
+            'http://convore.test.url/api/messages/21/star.json',
+            self.message.star)
+
+        @d.addCallback
+        def gotData(data):
+            self.assertTrue(data.get(u'star') is not None)
+            self.assertEqual(data[u'star'][u'user'][u'id'], u'1')
+
+        return d
+
+
+    def test_unstar(self):
+        """
+        Star a message. If the message has already been starred by this user,
+        this endpoint will then unstar the message.
+        """
+        self.endpoint.response = self.getTestData('messages_unstar.json')
+        d = self.callAPI(
+            'POST',
+            'http://convore.test.url/api/messages/21/star.json',
+            self.message.star)
+
+        @d.addCallback
+        def gotData(data):
+            self.assertTrue(data.get(u'unstar') is not None)
+            self.assertEqual(data['unstar'][u'user'][u'id'], u'1')
+
+        return d
+
+
+    def test_delete(self):
+        """
+        Delete a topic.
+        """
+        self.endpoint.response = '{"message": "Message deleted."}'
+        d = self.callAPI(
+            'POST',
+            'http://convore.test.url/api/messages/21/delete.json',
+            self.message.delete)
+        return d

=== added file 'txconvore/test/test_util.py'
--- txconvore/test/test_util.py	1970-01-01 00:00:00 +0000
+++ txconvore/test/test_util.py	2011-04-13 16:28:00 +0000
@@ -0,0 +1,148 @@
+from twisted.internet.defer import fail, succeed
+from twisted.trial.unittest import TestCase
+from twisted.web import client, http, error as weberror
+
+from txconvore.util import PerseverantDownloader
+
+
+
+class PerseverantDownloaderTests(TestCase):
+    """
+    Tests for L{txconvore.util.PerseverantDownloader}.
+    """
+    def wrapPageGetter(self, func, response_headers=None):
+        """
+        Wrap a function returning a C{Deferred} with something suitable to
+        replace L{PerseverantDownloader._getPage}.
+        """
+        def _getPage(*a, **kw):
+            fact = client.HTTPClientFactory('a_url')
+            fact.response_headers = response_headers
+            return func(), fact
+        return _getPage
+
+
+    def test_partialContent(self):
+        """
+        Return any partial content when HTTP 206 is returned.
+        """
+        def _partialContent():
+            return fail(weberror.Error(
+                http.PARTIAL_CONTENT,
+                response='partial content'))
+
+        downloader = PerseverantDownloader('a_url')
+        downloader._getPage = self.wrapPageGetter(_partialContent)
+        d = downloader.download()
+
+        @d.addCallback
+        def gotData((data, headers)):
+            self.assertEqual(data, 'partial content')
+
+        return d
+
+
+    def test_brokenError(self):
+        """
+        If a C{twisted.web.error.Error} with a non-stringified integer as its
+        C{status} occurs then no retrying or partial content handling occurs,
+        the Failure is simply returned along the Deferred chain.
+        """
+        def _brokenThing():
+            self.called += 1
+            return fail(weberror.Error(''))
+
+        self.called = 0
+        downloader = PerseverantDownloader('a_url')
+        downloader._getPage = self.wrapPageGetter(_brokenThing)
+        d = downloader.download()
+
+        @d.addCallback
+        def gotData((data, headers)):
+            self.fail('Expected errback')
+
+        @d.addErrback
+        def gotFailure(f):
+            f.trap(weberror.Error)
+            self.assertEqual(f.value.status, '')
+            self.assertEqual(self.called, 1)
+
+        return d
+
+
+    def test_retryableHTTPError(self):
+        """
+        If a fetch fails with an error code present in
+        L{PerseverantDownloader.retryableHTTPCodes} then the fetch is retried
+        L{PerseverantDownloader.tries} times, eventually failing.
+        """
+        def _retryableWebError():
+            self.called += 1
+            return fail(weberror.Error(errorCode))
+
+        self.called = 0
+        tries = 3
+        downloader = PerseverantDownloader('a_url', tries=tries)
+        errorCode = http.INTERNAL_SERVER_ERROR
+        self.assertIn(errorCode, downloader.retryableHTTPCodes)
+        # Let's not make the tests take longer than they have to.
+        downloader.factor = 0
+        downloader._getPage = self.wrapPageGetter(_retryableWebError)
+        d = self.assertFailure(downloader.download(), weberror.Error)
+
+        @d.addCallback
+        def checkError(e):
+            failures = self.flushLoggedErrors(weberror.Error)
+            self.assertEqual(len(failures), tries)
+            self.assertEqual(self.called, tries)
+            self.assertEqual(e.status, errorCode)
+
+        return d
+
+
+    def test_notRetryableHTTPError(self):
+        """
+        If a fetch fails with an error code B{not} present in
+        L{PerseverantDownloader.retryableHTTPCodes} then no retry attempt is
+        made.
+        """
+        def _notRetryableWeberror():
+            self.called += 1
+            return fail(weberror.Error(errorCode))
+
+        self.called = 0
+        tries = 3
+        downloader = PerseverantDownloader('a_url', tries=tries)
+        errorCode = http.NOT_FOUND
+        self.assertNotIn(errorCode, downloader.retryableHTTPCodes)
+        downloader._getPage = self.wrapPageGetter(_notRetryableWeberror)
+        d = self.assertFailure(downloader.download(), weberror.Error)
+
+        @d.addCallback
+        def checkError(e):
+            self.assertEqual(self.called, 1)
+            self.assertEqual(e.status, errorCode)
+
+        return d
+
+
+    def test_download(self):
+        """
+        L{PerseverantDownloader.download} fetches the data at the given URL and
+        returns it along with the response headers.
+        """
+        def _getPage():
+            return succeed('some data')
+
+        downloader = PerseverantDownloader('a_url')
+        downloader._getPage = self.wrapPageGetter(
+            _getPage,
+            response_headers=dict(foo=['bar']))
+        d = downloader.download()
+
+        @d.addCallback
+        def checkData((data, headers)):
+            self.assertEqual(data, 'some data')
+            self.assertEqual(headers.get('foo'), ['bar'])
+
+        return d

=== added file 'txconvore/util.py'
--- txconvore/util.py	1970-01-01 00:00:00 +0000
+++ txconvore/util.py	2011-04-13 16:28:00 +0000
@@ -0,0 +1,190 @@
+import time
+import urllib
+
+from twisted.internet import reactor, task, error as ineterror
+from twisted.python import log
+from twisted.web import client, http, error as weberror
+
+
+
+class PerseverantDownloader(object):
+    """
+    Perseverantly attempt to download a URL.
+
+    Each retry attempt is delayed by L{factor} up to a maximum of L{maxDelay},
+    starting at L{initialDelay}.
+
+    @type url: C{str}
+    @ivar url: HTTP URL to download.
+
+    @type maxDelay: C{float}
+    @cvar maxDelay: Maximum delay, in seconds, between retry attempts.
+
+    @type initialDelay: C{float}
+    @cvar initialDelay: Delay before the first retry attempt.
+
+    @type factor: C{float}
+    @cvar factor: Factor to increase the delay by after each attempt.
+
+    @type retryableHTTPCodes: C{list}
+    @cvar retryableHTTPCodes: HTTP error codes that suggest the error is
+        intermittent and that a retry should be attempted.
+
+    @type defaultTimeout: C{float}
+    @cvar defaultTimeout: Default fetch timeout value.
+    """
+    maxDelay = 3600
+    initialDelay = 1.0
+    factor = 1.6180339887498948
+
+    retryableHTTPCodes = [408, 500, 502, 503, 504]
+
+    defaultTimeout = 300.0
+
+    debug = False
+
+
+    def __init__(self, url, tries=10, timeout=defaultTimeout, *args, **kwargs):
+        """
+        Prepare the download information.
+
+        Any additional positional or keyword arguments are passed on to
+        C{twisted.web.client.HTTPPageGetter}.
+
+        @type url: C{nevow.url.URL} or C{unicode} or C{str}
+        @param url: The HTTP URL to attempt to download
+
+        @type tries: C{int}
+        @param tries: The maximum number of retry attempts before giving up
+
+        @type timeout: C{float}
+        @param timeout: Timeout value, in seconds, for the page fetch;
+            defaults to L{defaultTimeout}
+
+        @param *args: Positional arguments to pass to the page getter, see
+        L{twisted.web.client.getPage}.
+
+        @param **kwargs: Keyword arguments to pass to the page getter, see
+        L{twisted.web.client.getPage}.
+        """
+        self.url = url
+        self.delay = self.initialDelay
+        self.tries = tries
+        self.timeout = timeout
+        self.args = args
+        self.kwargs = kwargs
+
+
+    def _writeLog(self, result, factory):
+        """
+        Log an HTTP request or response.
+        """
+        if factory.response_headers is None:
+            suffix = 'req'
+            headers = factory.headers
+            body = factory.postdata
+        else:
+            suffix = 'res'
+            headers = factory.response_headers
+            body = result
+
+        path = urllib.splitquery(factory.path)[0]
+        path = '%s_%s_%s.log' % (
+            time.time(),
+            path[1:].replace('/', '_').replace('.', '_'),
+            suffix)
+        fd = file(path, 'wb')
+        fd.write('%s %s\n' % (factory.method, factory.url))
+
+        if headers:
+            fd.write('\n')
+            for k, v in headers.iteritems():
+                if not isinstance(v, list):
+                    v = [v]
+                v = map(str, v)
+                fd.write('%s: %s\n' % (k, ';'.join(v)))
+
+        fd.write('\n')
+        if body:
+            fd.write(body)
+        fd.close()
+        return result
+
+
+    def _handle206(self, f):
+        """
+        Return any partial content when HTTP 206 is returned.
+        """
+        f.trap(weberror.Error)
+        err = f.value
+        try:
+            if int(err.status) == http.PARTIAL_CONTENT:
+                return err.response
+        except (TypeError, ValueError):
+            pass
+        return f
+
+
+    def _getPage(self, *args, **kwargs):
+        """
+        Fetch data from a URL over HTTP.
+        """
+        kwargs.setdefault('agent', 'txConvore')
+        factory = client._makeGetterFactory(
+            self.url,
+            client.HTTPClientFactory,
+            timeout=self.timeout,
+            *args, **kwargs)
+        return factory.deferred, factory
+
+
+    def download(self):
+        """
+        Download L{self.url}, retrying if necessary.
+
+        @rtype: C{Deferred<response, headers>}
+        """
+        d, f = self._getPage(*self.args, **self.kwargs)
+        if self.debug:
+            self._writeLog(None, f)
+            d.addCallback(self._writeLog, f)
+        d.addErrback(self._handle206)
+        d.addErrback(self.retryWeb)
+        d.addCallback(lambda data: (data, f.response_headers))
+        return d
+
+
+    def retryWeb(self, f):
+        """
+        Retry failed downloads in the case of "web errors."
+
+        Only errors that are web related are considered for a retry attempt
+        and then only when the HTTP status code is one of those in
+        L{self.retryableHTTPCodes}.
+
+        Other errors are not trapped.
+        """
+        f.trap(weberror.Error, ineterror.ConnectionDone)
+        err = f.value
+        try:
+            if int(err.status) in self.retryableHTTPCodes:
+                return self.retry(f)
+        except (TypeError, ValueError):
+            pass
+        return f
+
+
+    def retry(self, f):
+        """
+        The retry machinery.
+
+        If C{self.tries} is greater than zero, a retry is attempted for
+        C{self.delay} seconds in the future.
+        """
+        self.tries -= 1
+        log.err(f, 'PerseverantDownloader is retrying, %d attempts left.' % (
+            self.tries,))
+        self.delay = min(self.delay * self.factor, self.maxDelay)
+        if self.tries == 0:
+            return f
+        return task.deferLater(reactor, self.delay, self.download)