Merge lp:~eday/burrow/backend-unittests into lp:burrow

[Eric Day]

Backend unittest and cleanup party!

=== modified file 'burrow/backend/__init__.py'
--- burrow/backend/__init__.py	2011-08-04 06:57:15 +0000
+++ burrow/backend/__init__.py	2011-08-09 23:35:51 +0000
@@ -14,6 +14,8 @@
 
 '''Backends for burrow.'''
 
+import time
+
 import eventlet
 
 import burrow.common
@@ -49,7 +51,7 @@
     def get_messages(self, account, queue, filters={}):
         return []
 
-    def update_messages(self, account, queue, attributes={}, filters={}):
+    def update_messages(self, account, queue, attributes, filters={}):
         return []
 
     def create_message(self, account, queue, message, body, attributes={}):
@@ -61,8 +63,7 @@
     def get_message(self, account, queue, message, filters={}):
         return None
 
-    def update_message(self, account, queue, message, attributes={},
-        filters={}):
+    def update_message(self, account, queue, message, attributes, filters={}):
         return None
 
     def notify(self, account, queue):
@@ -97,6 +98,32 @@
             self.clean()
             eventlet.sleep(1)
 
+    def _get_attributes(self, attributes, ttl=None, hide=None):
+        ttl = attributes.get('ttl', ttl)
+        if ttl is not None and ttl > 0:
+            ttl += int(time.time())
+        hide = attributes.get('hide', hide)
+        if hide is not None and hide > 0:
+            hide += int(time.time())
+        return ttl, hide
+
+    def _get_detail(self, filters, default=None):
+        detail = filters.get('detail', default)
+        if detail == 'none':
+            detail = None
+        elif detail is not None and detail not in ['id', 'all']:
+            raise burrow.backend.BadDetail(detail)
+        return detail
+
+    def _get_message_detail(self, filters, default=None):
+        detail = filters.get('detail', default)
+        options = ['id', 'attributes', 'body', 'all']
+        if detail == 'none':
+            detail = None
+        elif detail is not None and detail not in options:
+            raise burrow.backend.BadDetail(detail)
+        return detail
+
 
 class NotFound(Exception):
     pass

=== modified file 'burrow/backend/memory.py'
--- burrow/backend/memory.py	2011-08-09 08:05:39 +0000
+++ burrow/backend/memory.py	2011-08-09 23:35:51 +0000
@@ -22,8 +22,8 @@
 class Backend(burrow.backend.Backend):
     '''This backend stores all data using native Python data
     structures. It uses a linked list of objects to store data
-    (accounts, queues, and messages) with a dict as a secondary index
-    into this list. This is required so we can have O(1) appends,
+    (accounts, queues, and messages) with a dictionary as a secondary
+    index into this list. This is required so we can have O(1) appends,
     deletes, and lookups by id, along with easy traversal starting
     anywhere in the list.'''
 
@@ -49,8 +49,6 @@
 
     def delete_queues(self, account, filters={}):
         account = self.accounts.get(account)
-        if account is None:
-            raise burrow.backend.NotFound()
         if len(filters) == 0:
             account.queues.reset()
         else:
@@ -64,8 +62,6 @@
 
     def get_queues(self, account, filters={}):
         account = self.accounts.get(account)
-        if account is None:
-            raise burrow.backend.NotFound()
         detail = self._get_detail(filters, 'id')
         for queue in account.queues.iter(filters):
             if detail is not None:
@@ -73,8 +69,6 @@
 
     def delete_messages(self, account, queue, filters={}):
         account, queue = self.accounts.get_queue(account, queue)
-        if queue is None:
-            raise burrow.backend.NotFound()
         detail = self._get_message_detail(filters)
         for message in queue.messages.iter(filters):
             queue.messages.delete(message.id)
@@ -85,17 +79,13 @@
 
     def get_messages(self, account, queue, filters={}):
         account, queue = self.accounts.get_queue(account, queue)
-        if queue is None:
-            raise burrow.backend.NotFound()
         detail = self._get_message_detail(filters, 'all')
         for message in queue.messages.iter(filters):
             if detail is not None:
                 yield message.detail(detail)
 
-    def update_messages(self, account, queue, attributes={}, filters={}):
+    def update_messages(self, account, queue, attributes, filters={}):
         account, queue = self.accounts.get_queue(account, queue)
-        if queue is None:
-            raise burrow.backend.NotFound()
         notify = False
         ttl, hide = self._get_attributes(attributes)
         detail = self._get_message_detail(filters)
@@ -113,13 +103,13 @@
 
     def create_message(self, account, queue, message, body, attributes={}):
         account, queue = self.accounts.get_queue(account, queue, True)
-        ttl, hide = self._get_attributes(attributes, default_ttl=0,
-            default_hide=0)
-        if queue.messages.get(message) is None:
+        ttl, hide = self._get_attributes(attributes, ttl=0, hide=0)
+        try:
+            message = queue.messages.get(message)
+            created = False
+        except burrow.backend.NotFound:
+            message = queue.messages.get(message, True)
             created = True
-        else:
-            created = False
-        message = queue.messages.get(message, True)
         message.ttl = ttl
         message.hide = hide
         message.body = body
@@ -129,41 +119,31 @@
 
     def delete_message(self, account, queue, message, filters={}):
         account, queue = self.accounts.get_queue(account, queue)
-        if queue is None:
-            raise burrow.backend.NotFound()
         message = queue.messages.get(message)
-        if message is None:
-            raise burrow.backend.NotFound()
+        detail = self._get_message_detail(filters)
         queue.messages.delete(message.id)
         if queue.messages.count() == 0:
             self.accounts.delete_queue(account.id, queue.id)
-        return message.detail()
+        return message.detail(detail)
 
     def get_message(self, account, queue, message, filters={}):
         account, queue = self.accounts.get_queue(account, queue)
-        if queue is None:
-            raise burrow.backend.NotFound()
         message = queue.messages.get(message)
-        if message is None:
-            raise burrow.backend.NotFound()
-        return message.detail()
+        detail = self._get_message_detail(filters, 'all')
+        return message.detail(detail)
 
-    def update_message(self, account, queue, message, attributes={},
-        filters={}):
+    def update_message(self, account, queue, message, attributes, filters={}):
         account, queue = self.accounts.get_queue(account, queue)
-        if queue is None:
-            raise burrow.backend.NotFound()
+        message = queue.messages.get(message)
         ttl, hide = self._get_attributes(attributes)
-        message = queue.messages.get(message)
-        if message is None:
-            raise burrow.backend.NotFound()
+        detail = self._get_message_detail(filters)
         if ttl is not None:
             message.ttl = ttl
         if hide is not None:
             message.hide = hide
             if hide == 0:
                 self.notify(account.id, queue.id)
-        return message.detail()
+        return message.detail(detail)
 
     def clean(self):
         now = int(time.time())
@@ -181,32 +161,6 @@
                 if queue.messages.count() == 0:
                     self.accounts.delete_queue(account.id, queue.id)
 
-    def _get_attributes(self, attributes, default_ttl=None, default_hide=None):
-        ttl = attributes.get('ttl', default_ttl)
-        if ttl is not None and ttl > 0:
-            ttl += int(time.time())
-        hide = attributes.get('hide', default_hide)
-        if hide is not None and hide > 0:
-            hide += int(time.time())
-        return ttl, hide
-
-    def _get_detail(self, filters, default=None):
-        detail = filters.get('detail', default)
-        if detail == 'none':
-            detail = None
-        elif detail is not None and detail not in ['id', 'all']:
-            raise burrow.backend.BadDetail(detail)
-        return detail
-
-    def _get_message_detail(self, filters, default=None):
-        detail = filters.get('detail', default)
-        options = ['id', 'attributes', 'body', 'all']
-        if detail == 'none':
-            detail = None
-        elif detail is not None and detail not in options:
-            raise burrow.backend.BadDetail(detail)
-        return detail
-
 
 class Item(object):
     '''Object to represent elements in a indexed linked list.'''
@@ -217,9 +171,11 @@
         self.prev = None
 
     def detail(self, detail):
-        if detail == 'all':
+        if detail == 'id':
+            return self.id
+        elif detail == 'all':
             return dict(id=self.id)
-        return self.id
+        return None
 
 
 class IndexedList(object):
@@ -259,7 +215,7 @@
             return self.index[id]
         elif create:
             return self.add(self.item_class(id))
-        return None
+        raise burrow.backend.NotFound()
 
     def iter(self, filters={}):
         marker = filters.get('marker', None)
@@ -310,7 +266,7 @@
         elif create:
             account = self.add(Account(account))
         else:
-            return None, None
+            raise burrow.backend.NotFound()
         return account, account.queues.get(queue, create)
 
 
@@ -347,7 +303,9 @@
             hide -= int(time.time())
         if detail == 'attributes':
             return dict(id=self.id, ttl=ttl, hide=hide)
-        return dict(id=self.id, ttl=ttl, hide=hide, body=self.body)
+        elif detail == 'all':
+            return dict(id=self.id, ttl=ttl, hide=hide, body=self.body)
+        return None
 
 
 class Messages(IndexedList):

=== modified file 'burrow/backend/sqlite.py'
--- burrow/backend/sqlite.py	2011-08-09 08:05:39 +0000
+++ burrow/backend/sqlite.py	2011-08-09 23:35:51 +0000
@@ -22,6 +22,7 @@
 
 # Default configuration values for this module.
 DEFAULT_DATABASE = ':memory:'
+DEFAULT_SYNCHRONOUS = 'FULL'
 
 # Maximum number of parameters to pass to execute. Testing shows a max of
 # 999, so leave a few extra for parameters not added by a list of IDs.
@@ -38,6 +39,8 @@
             self.config.set('database', url.netloc)
         database = self.config.get('database', DEFAULT_DATABASE)
         self.db = sqlite3.connect(database)
+        synchronous = self.config.get('synchronous', DEFAULT_SYNCHRONOUS)
+        self.db.execute('PRAGMA synchronous=' + synchronous)
         self.db.isolation_level = None
         queries = [
             'CREATE TABLE IF NOT EXISTS accounts ('
@@ -66,20 +69,16 @@
             self.db.execute('DELETE FROM queues')
             self.db.execute('DELETE FROM messages')
             return
-        count = 0
         detail = self._get_detail(filters)
         ids = []
         query = 'SELECT rowid,account FROM accounts'
         for row in self._get_accounts(query, filters):
-            count += 1
             if detail is not None:
                 yield self._detail(row[1:], detail)
             ids.append(row[0])
             if len(ids) == MAXIMUM_PARAMETERS:
                 self._delete_accounts(ids)
                 ids = []
-        if count == 0:
-            raise burrow.backend.NotFound()
         if len(ids) > 0:
             self._delete_accounts(ids)
 
@@ -103,29 +102,19 @@
         query = 'DELETE FROM accounts WHERE rowid IN '
         self.db.execute(query + query_values, ids)
 
-    def _get_detail(self, filters, default=None):
-        detail = filters.get('detail', default)
-        if detail == 'none':
-            detail = None
-        elif detail is not None and detail not in ['id', 'all']:
-            raise burrow.backend.BadDetail(detail)
-        return detail
-
     def _detail(self, row, detail):
         if detail == 'id':
             return row[0]
-        return dict(id=row[0])
+        elif detail == 'all':
+            return dict(id=row[0])
+        return None
 
     def get_accounts(self, filters={}):
-        count = 0
         detail = self._get_detail(filters, 'id')
         query = 'SELECT account FROM accounts'
         for row in self._get_accounts(query, filters):
-            count += 1
             if detail is not None:
                 yield self._detail(row, detail)
-        if count == 0:
-            raise burrow.backend.NotFound()
 
     def _get_accounts(self, query, filters):
         values = tuple()
@@ -141,31 +130,32 @@
         if limit is not None:
             query += ' LIMIT ?'
             values += (limit,)
-        return self.db.execute(query, values)
+        count = 0
+        for row in self.db.execute(query, values):
+            count += 1
+            yield row
+        if count == 0:
+            raise burrow.backend.NotFound()
 
     def _get_account(self, account):
         query = 'SELECT rowid FROM accounts WHERE account=?'
-        account_rowid = self.db.execute(query, (account,)).fetchall()
-        if len(account_rowid) == 0:
+        rows = self.db.execute(query, (account,)).fetchall()
+        if len(rows) == 0:
             raise burrow.backend.NotFound()
-        return account_rowid[0][0]
+        return rows[0][0]
 
     def delete_queues(self, account, filters={}):
         account_rowid = self._get_account(account)
-        count = 0
         detail = self._get_detail(filters)
         ids = []
         query = 'SELECT rowid,queue FROM queues'
         for row in self._get_queues(query, account_rowid, filters):
-            count += 1
             if detail is not None:
                 yield self._detail(row[1:], detail)
             ids.append(row[0])
             if len(ids) == MAXIMUM_PARAMETERS:
                 self._delete_queues(ids)
                 ids = []
-        if count == 0:
-            raise burrow.backend.NotFound()
         if len(ids) > 0:
             self._delete_queues(ids)
         self._check_empty_account(account_rowid)
@@ -186,15 +176,11 @@
 
     def get_queues(self, account, filters={}):
         account_rowid = self._get_account(account)
-        count = 0
         detail = self._get_detail(filters, 'id')
         query = 'SELECT queue FROM queues'
         for row in self._get_queues(query, account_rowid, filters):
-            count += 1
             if detail is not None:
                 yield self._detail(row, detail)
-        if count == 0:
-            raise burrow.backend.NotFound()
 
     def _get_queues(self, query, account_rowid, filters):
         query += ' WHERE account=?'
@@ -211,32 +197,33 @@
         if limit is not None:
             query += ' LIMIT ?'
             values += (limit,)
-        return self.db.execute(query, values)
+        count = 0
+        for row in self.db.execute(query, values):
+            count += 1
+            yield row
+        if count == 0:
+            raise burrow.backend.NotFound()
 
     def _get_queue(self, account_rowid, queue):
         query = 'SELECT rowid FROM queues WHERE account=? AND queue=?'
-        queue_rowid = self.db.execute(query, (account_rowid, queue)).fetchall()
-        if len(queue_rowid) == 0:
+        rows = self.db.execute(query, (account_rowid, queue)).fetchall()
+        if len(rows) == 0:
             raise burrow.backend.NotFound()
-        return queue_rowid[0][0]
+        return rows[0][0]
 
     def delete_messages(self, account, queue, filters={}):
         account_rowid = self._get_account(account)
         queue_rowid = self._get_queue(account_rowid, queue)
-        count = 0
         detail = self._get_message_detail(filters)
         ids = []
         query = 'SELECT rowid,message,ttl,hide,body FROM messages'
         for row in self._get_messages(query, queue_rowid, filters):
-            count += 1
             if detail is not None:
                 yield self._message_detail(row[1:], detail)
             ids.append(row[0])
             if len(ids) == MAXIMUM_PARAMETERS:
                 self._delete_messages(ids)
                 ids = []
-        if count == 0:
-            raise burrow.backend.NotFound()
         if len(ids) > 0:
             self._delete_messages(ids)
         self._check_empty_queue(account_rowid, queue_rowid)
@@ -253,15 +240,6 @@
             self.db.execute('DELETE FROM queues WHERE rowid=?', (queue_rowid,))
             self._check_empty_account(account_rowid)
 
-    def _get_message_detail(self, filters, default=None):
-        detail = filters.get('detail', default)
-        options = ['id', 'attributes', 'body', 'all']
-        if detail == 'none':
-            detail = None
-        elif detail is not None and detail not in options:
-            raise burrow.backend.BadDetail(detail)
-        return detail
-
     def _message_detail(self, row, detail):
         if detail == 'id':
             return row[0]
@@ -275,20 +253,18 @@
             hide -= int(time.time())
         if detail == 'attributes':
             return dict(id=row[0], ttl=ttl, hide=hide)
-        return dict(id=row[0], ttl=ttl, hide=hide, body=str(row[3]))
+        elif detail == 'all':
+            return dict(id=row[0], ttl=ttl, hide=hide, body=str(row[3]))
+        return None
 
     def get_messages(self, account, queue, filters={}):
         account_rowid = self._get_account(account)
         queue_rowid = self._get_queue(account_rowid, queue)
-        count = 0
         detail = self._get_message_detail(filters, 'all')
         query = 'SELECT message,ttl,hide,body FROM messages'
         for row in self._get_messages(query, queue_rowid, filters):
-            count += 1
             if detail is not None:
                 yield self._message_detail(row, detail)
-        if count == 0:
-            raise burrow.backend.NotFound()
 
     def _get_messages(self, query, queue_rowid, filters):
         query += ' WHERE queue=?'
@@ -308,32 +284,35 @@
         if limit is not None:
             query += ' LIMIT ?'
             values += (limit,)
-        return self.db.execute(query, values)
-
-    def _get_message(self, queue_rowid, message):
-        query = 'SELECT rowid FROM messages WHERE queue=? AND message=?'
-        values = (queue_rowid, message)
-        message_rowid = self.db.execute(query, values).fetchall()
-        if len(message_rowid) == 0:
-            raise burrow.backend.NotFound()
-        return message_rowid[0][0]
-
-    def update_messages(self, account, queue, attributes={}, filters={}):
+        count = 0
+        for row in self.db.execute(query, values):
+            count += 1
+            yield row
+        if count == 0:
+            raise burrow.backend.NotFound()
+
+    def _get_message(self, queue_rowid, message, full=False):
+        if full:
+            query = 'SELECT rowid,message,ttl,hide,body'
+        else:
+            query = 'SELECT rowid'
+        query += ' FROM messages WHERE queue=? AND message=?'
+        rows = self.db.execute(query, (queue_rowid, message)).fetchall()
+        if len(rows) == 0:
+            raise burrow.backend.NotFound()
+        if full:
+            return rows[0]
+        return rows[0][0]
+
+    def update_messages(self, account, queue, attributes, filters={}):
         account_rowid = self._get_account(account)
         queue_rowid = self._get_queue(account_rowid, queue)
-        count = 0
         detail = self._get_message_detail(filters)
         ids = []
         notify = False
-        ttl = attributes.get('ttl', None)
-        if ttl is not None and ttl > 0:
-            ttl += int(time.time())
-        hide = attributes.get('hide', None)
-        if hide is not None and hide > 0:
-            hide += int(time.time())
+        ttl, hide = self._get_attributes(attributes)
         query = 'SELECT rowid,message,ttl,hide,body FROM messages'
         for row in self._get_messages(query, queue_rowid, filters):
-            count += 1
             if detail is not None:
                 row = list(row)
                 if ttl is not None:
@@ -346,8 +325,6 @@
                 if self._update_messages(ttl, hide, ids):
                     notify = True
                 ids = []
-        if count == 0:
-            raise burrow.backend.NotFound()
         if len(ids) > 0:
             if self._update_messages(ttl, hide, ids):
                 notify = True
@@ -373,6 +350,7 @@
         return True
 
     def create_message(self, account, queue, message, body, attributes={}):
+        ttl, hide = self._get_attributes(attributes, ttl=0, hide=0)
         try:
             account_rowid = self._get_account(account)
         except burrow.backend.NotFound:
@@ -381,122 +359,89 @@
         try:
             queue_rowid = self._get_queue(account_rowid, queue)
         except burrow.backend.NotFound:
-            query = 'INSERT INTO queues VALUES (?, ?)'
+            query = 'INSERT INTO queues VALUES (?,?)'
             values = (account_rowid, queue)
             queue_rowid = self.db.execute(query, values).lastrowid
-        ttl = attributes.get('ttl', 0)
-        if ttl > 0:
-            ttl += int(time.time())
-        hide = attributes.get('hide', 0)
-        if hide > 0:
-            hide += int(time.time())
-        query = 'SELECT rowid FROM messages WHERE queue=? AND message=?'
-        values = (queue_rowid, message)
-        message_rowid = self.db.execute(query, values).fetchall()
-        if len(message_rowid) == 0:
+        try:
+            message_rowid = self._get_message(queue_rowid, message)
+            query = 'UPDATE messages SET ttl=?,hide=?,body=? WHERE rowid=?'
+            self.db.execute(query, (ttl, hide, body, message_rowid))
+            created = False
+        except burrow.backend.NotFound:
             query = 'INSERT INTO messages VALUES (?,?,?,?,?)'
             self.db.execute(query, (queue_rowid, message, ttl, hide, body))
-            self.notify(account, queue)
-            return True
-        query = 'UPDATE messages SET ttl=?,hide=?,body=? WHERE rowid=?'
-        self.db.execute(query, (ttl, hide, body, message_rowid[0][0]))
-        if hide == 0:
-            self.notify(account, queue)
-        return False
+            created = True
+        if created or hide == 0:
+            self.notify(account, queue)
+        return created
 
-    def delete_message(self, account, queue, message):
+    def delete_message(self, account, queue, message, filters={}):
         account_rowid = self._get_account(account)
         queue_rowid = self._get_queue(account_rowid, queue)
-        message_rowid = self._get_message(queue_rowid, message)
-        message = self.get_message(account, queue, message)
-        self.db.execute('DELETE FROM messages WHERE rowid=?', (message_rowid,))
+        row = self._get_message(queue_rowid, message, True)
+        detail = self._get_message_detail(filters)
+        self.db.execute('DELETE FROM messages WHERE rowid=?', (row[0],))
         self._check_empty_queue(account_rowid, queue_rowid)
-        return message
-
-    def get_message(self, account, queue, message):
-        queue_rowid = self._get_queue(self._get_account(account), queue)
-        query = 'SELECT message,ttl,hide,body FROM messages ' \
-            'WHERE queue=? AND message=?'
-        result = self.db.execute(query, (queue_rowid, message)).fetchall()
-        if len(result) == 0:
-            raise burrow.backend.NotFound()
-        row = result[0]
-        ttl = row[1]
-        if ttl > 0:
-            ttl -= int(time.time())
-        hide = row[2]
-        if hide > 0:
-            hide -= int(time.time())
-        return dict(id=row[0], ttl=ttl, hide=hide, body=str(row[3]))
-
-    def update_message(self, account, queue, message, attributes):
-        queue_rowid = self._get_queue(self._get_account(account), queue)
-        message = self.get_message(account, queue, message)
-        if message is None:
-            raise burrow.backend.NotFound()
-        query = 'UPDATE messages SET'
-        values = tuple()
-        comma = ''
-        ttl = attributes.get('ttl', None)
+        return self._message_detail(row[1:], detail)
+
+    def get_message(self, account, queue, message, filters={}):
+        queue_rowid = self._get_queue(self._get_account(account), queue)
+        row = self._get_message(queue_rowid, message, True)
+        detail = self._get_message_detail(filters, 'all')
+        return self._message_detail(row[1:], detail)
+
+    def update_message(self, account, queue, message, attributes, filters={}):
+        queue_rowid = self._get_queue(self._get_account(account), queue)
+        row = self._get_message(queue_rowid, message, True)
+        detail = self._get_message_detail(filters)
+        ttl, hide = self._get_attributes(attributes)
+        if self._update_messages(ttl, hide, [row[0]]):
+            self.notify(account, queue)
+        row = list(row)
         if ttl is not None:
-            message['ttl'] = ttl
-            if ttl > 0:
-                ttl += int(time.time())
-            query += comma + ' ttl=?'
-            values += (ttl,)
-            comma = ','
-        hide = attributes.get('hide', None)
+            row[2] = ttl
         if hide is not None:
-            message['hide'] = hide
-            if hide > 0:
-                hide += int(time.time())
-            query += comma + ' hide=?'
-            values += (hide,)
-            comma = ','
-        if comma == '':
-            return message
-        query += ' WHERE queue=? AND message=?'
-        values += (queue_rowid, message['id'])
-        self.db.execute(query, values)
-        if hide == 0:
-            self.notify(account, queue)
-        return message
+            row[3] = hide
+        return self._message_detail(row[1:], detail)
 
     def clean(self):
         now = int(time.time())
-        query = 'SELECT rowid,queue FROM messages ' \
-            'WHERE ttl > 0 AND ttl <= ?'
-        result = self.db.execute(query, (now,)).fetchall()
-        if len(result) > 0:
-            messages = []
-            queues = []
-            for row in result:
-                messages.append(str(row[0]))
-                queues.append(row[1])
-            query = 'DELETE FROM messages WHERE rowid in (%s)' % \
-                ','.join(messages)
-            self.db.execute(query)
-            for queue_rowid in queues:
-                query = 'SELECT account FROM queues WHERE rowid=?'
-                account_rowid = self.db.execute(query, (queue_rowid,))
-                account_rowid = account_rowid.fetchall()[0][0]
-                self._check_empty_queue(account_rowid, queue_rowid)
-        query = "SELECT rowid,queue FROM messages WHERE " \
-            "hide > 0 AND hide <= %d" % now
-        result = self.db.execute(query).fetchall()
-        if len(result) > 0:
-            messages = []
-            queues = []
-            for row in result:
-                messages.append(str(row[0]))
-                queues.append(row[1])
-            query = 'UPDATE messages SET hide=0 WHERE rowid in (%s)' % \
-                ','.join(messages)
-            self.db.execute(query)
-            for queue in queues:
-                query = 'SELECT accounts.account,queues.queue ' \
-                    'FROM queues JOIN accounts ' \
-                    'ON queues.account=accounts.rowid ' \
-                    'WHERE queues.rowid=?'
-                result = self.db.execute(query, (queue,)).fetchall()[0]
-                self.notify(result[0], result[1])
+        query = 'SELECT rowid,queue FROM messages WHERE ttl > 0 AND ttl <= ?'
+        messages = []
+        queues = set()
+        message_query = 'DELETE FROM messages WHERE rowid IN '
+        message_query_values = '(?' + (',?' * (MAXIMUM_PARAMETERS - 1)) + ')'
+        for row in self.db.execute(query, (now,)):
+            messages.append(row[0])
+            if len(messages) == MAXIMUM_PARAMETERS:
+                self.db.execute(message_query + message_query_values, messages)
+                messages = []
+            queues.add(row[1])
+        if len(messages) > 0:
+            message_query_values = '(?' + (',?' * (len(messages) - 1)) + ')'
+            self.db.execute(message_query + message_query_values, messages)
+        for queue in queues:
+            query = 'SELECT account FROM queues WHERE rowid=?'
+            account = self.db.execute(query, (queue,)).fetchall()[0][0]
+            self._check_empty_queue(account, queue)
+        query = 'SELECT rowid,queue FROM messages WHERE hide > 0 AND hide <= ?'
+        messages = []
+        queues = set()
+        message_query = 'UPDATE messages SET hide=0 WHERE rowid IN '
+        message_query_values = '(?' + (',?' * (MAXIMUM_PARAMETERS - 1)) + ')'
+        for row in self.db.execute(query, (now,)):
+            messages.append(row[0])
+            if len(messages) == MAXIMUM_PARAMETERS:
+                self.db.execute(message_query + message_query_values, messages)
+                messages = []
+            queues.add(row[1])
+        if len(messages) > 0:
+            message_query_values = '(?' + (',?' * (len(messages) - 1)) + ')'
+            self.db.execute(message_query + message_query_values, messages)
+        for queue in queues:
+            query = 'SELECT accounts.account,queues.queue ' \
+                'FROM queues JOIN accounts ' \
+                'ON queues.account=accounts.rowid ' \
+                'WHERE queues.rowid=?'
+            result = self.db.execute(query, (queue,)).fetchall()[0]
+            self.notify(result[0], result[1])

=== modified file 'burrow/frontend/wsgi.py'
--- burrow/frontend/wsgi.py	2011-08-09 08:05:39 +0000
+++ burrow/frontend/wsgi.py	2011-08-09 23:35:51 +0000
@@ -172,29 +172,34 @@
 
     @webob.dec.wsgify
     def _delete_message(self, req, account, queue, message):
+        filters = self._parse_filters(req)
         try:
-            message = self.backend.delete_message(account, queue, message)
+            message = self.backend.delete_message(account, queue, message,
+                filters)
         except burrow.backend.NotFound:
             return self._response(status=404)
-        return self._return_message(req, account, queue, message, 'none')
+        return self._response(body=message)
 
     @webob.dec.wsgify
     def _get_message(self, req, account, queue, message):
+        filters = self._parse_filters(req)
         try:
-            message = self.backend.get_message(account, queue, message)
+            message = self.backend.get_message(account, queue, message,
+                filters)
         except burrow.backend.NotFound:
             return self._response(status=404)
-        return self._return_message(req, account, queue, message, 'all')
+        return self._response(body=message)
 
     @webob.dec.wsgify
     def _post_message(self, req, account, queue, message):
         attributes = self._parse_attributes(req)
+        filters = self._parse_filters(req)
         try:
             message = self.backend.update_message(account, queue, message,
-                attributes)
+                attributes, filters)
         except burrow.backend.NotFound:
             return self._response(status=404)
-        return self._return_message(req, account, queue, message, 'id')
+        return self._response(body=message)
 
     @webob.dec.wsgify
     def _put_message(self, req, account, queue, message):
@@ -208,28 +213,6 @@
             return self._response(status=201)
         return self._response()
 
-    def _filter_message(self, detail, message):
-        if detail == 'id':
-            return dict(id=message['id'])
-        elif detail == 'attributes':
-            message = message.copy()
-            del message['body']
-            return message
-        elif detail == 'all':
-            return message
-        return None
-
-    def _return_message(self, req, account, queue, message, detail):
-        if 'detail' in req.params:
-            detail = req.params['detail']
-        if detail == 'body':
-            return self._response(body=message['body'],
-                content_type="application/octet-stream")
-        message = self._filter_message(detail, message)
-        if message is not None:
-            return self._response(body=message)
-        return self._response()
-
     def _parse_filters(self, req):
         filters = {}
         if 'limit' in req.params:
@@ -272,7 +255,10 @@
                 status = 204
         else:
             if content_type is None:
-                content_type = 'application/json'
+                if isinstance(body, list) or isinstance(body, dict):
+                    content_type = 'application/json'
+                else:
+                    content_type = "application/octet-stream"
             if content_type == 'application/json':
                 body = json.dumps(body, indent=2)
         return webob.Response(status=status, body=body,

=== modified file 'etc/burrowd.conf'
--- etc/burrowd.conf	2011-04-22 02:18:26 +0000
+++ etc/burrowd.conf	2011-08-09 23:35:51 +0000
@@ -29,6 +29,10 @@
 # Database file to use, passed to sqlite3.connect.
 database = :memory:
 
+# Synchronous mode to set for SQLite. Current options are: 'OFF', 'ON', 'FULL'
+# See the SQLite PRAGMA documentation for more information on this setting.
+synchronous = FULL
+
 
 [burrow.backend.http]
 

=== modified file 'test/backend/test_memory.py'
--- test/backend/test_memory.py	2011-08-08 20:04:40 +0000
+++ test/backend/test_memory.py	2011-08-09 23:35:51 +0000
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import ConfigParser
+import time
 import unittest
 
 import burrow.backend
@@ -21,27 +22,23 @@
 
 class TestMemory(unittest.TestCase):
     '''Unittests for the memory backend.'''
-    backend_class = burrow.backend.memory.Backend
 
     def setUp(self):
         config = (ConfigParser.ConfigParser(), 'test')
-        self.backend = self.backend_class(config)
-        accounts = self.backend.get_accounts()
-        self.assertRaises(burrow.backend.NotFound, list, accounts)
-        queues = self.backend.get_queues('a')
-        self.assertRaises(burrow.backend.NotFound, list, queues)
-        filters = dict(match_hidden=True)
-        messages = self.backend.get_messages('a', 'q', filters)
-        self.assertRaises(burrow.backend.NotFound, list, messages)
+        self.backend = burrow.backend.memory.Backend(config)
+        self.check_empty()
 
     def tearDown(self):
+        self.check_empty()
+
+    def check_empty(self):
+        accounts = self.backend.get_accounts()
+        self.assertRaises(burrow.backend.NotFound, list, accounts)
+        queues = self.backend.get_queues('a')
+        self.assertRaises(burrow.backend.NotFound, list, queues)
         filters = dict(match_hidden=True)
         messages = self.backend.get_messages('a', 'q', filters)
         self.assertRaises(burrow.backend.NotFound, list, messages)
-        queues = self.backend.get_queues('a')
-        self.assertRaises(burrow.backend.NotFound, list, queues)
-        accounts = self.backend.get_accounts()
-        self.assertRaises(burrow.backend.NotFound, list, accounts)
 
     def test_accounts(self):
         self.backend.create_message('a', 'q', 'm', 'test')
@@ -50,6 +47,12 @@
         accounts = self.backend.delete_accounts()
         self.assertRaises(burrow.backend.NotFound, list, accounts)
 
+    def test_accounts_large(self):
+        for x in xrange(0, 1000):
+            self.backend.create_message(str(x), str(x), str(x), str(x))
+        filters = dict(marker='unknown')
+        self.assertEquals([], list(self.backend.delete_accounts(filters)))
+
     def test_accounts_delete_detail_all(self):
         self.backend.create_message('a', 'q', 'm', 'test')
         filters = dict(detail='all')
@@ -206,6 +209,12 @@
         queues = self.backend.delete_queues('a')
         self.assertRaises(burrow.backend.NotFound, list, queues)
 
+    def test_queues_large(self):
+        for x in xrange(0, 1000):
+            self.backend.create_message('a', str(x), str(x), str(x))
+        filters = dict(marker='unknown')
+        self.assertEquals([], list(self.backend.delete_queues('a', filters)))
+
     def test_queues_delete_detail_all(self):
         self.backend.create_message('a', 'q', 'm', 'test')
         filters = dict(detail='all')
@@ -363,12 +372,26 @@
         attributes = dict(ttl=100, hide=200)
         messages = list(self.backend.update_messages('a', 'q', attributes))
         self.assertEquals([], messages)
+        attributes = dict(ttl=0, hide=0)
+        filters = dict(match_hidden=True)
+        messages = self.backend.update_messages('a', 'q', attributes, filters)
+        self.assertEquals([], list(messages))
+        messages = self.backend.update_messages('a', 'q', dict(), filters)
+        self.assertEquals([], list(messages))
         self.delete_messages()
         messages = self.backend.delete_messages('a', 'q')
         self.assertRaises(burrow.backend.NotFound, list, messages)
         messages = self.backend.update_messages('a', 'q', attributes)
         self.assertRaises(burrow.backend.NotFound, list, messages)
 
+    def test_messages_large(self):
+        for x in xrange(0, 1000):
+            self.backend.create_message('a', 'q', str(x), str(x))
+        attributes = dict(ttl=100, hide=200)
+        messages = self.backend.update_messages('a', 'q', attributes)
+        self.assertEquals([], list(messages))
+        self.delete_messages()
+
     def test_messages_delete_detail_all(self):
         self.backend.create_message('a', 'q', 'm', 'test')
         message = dict(id='m', ttl=0, hide=0, body='test')
@@ -661,6 +684,206 @@
         self.assertEquals(messages[1:3], list(messages2))
         self.delete_messages()
 
+    def test_message(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        message = self.backend.get_message('a', 'q', 'm')
+        self.assertEquals(dict(id='m', ttl=0, hide=0, body='test'), message)
+        attributes = dict(ttl=100, hide=200)
+        message = self.backend.update_message('a', 'q', 'm', attributes)
+        attributes = dict(ttl=0, hide=0)
+        message = self.backend.update_message('a', 'q', 'm', attributes)
+        self.assertEquals(None, message)
+        message = self.backend.update_message('a', 'q', 'm', dict())
+        self.assertEquals(None, message)
+        message = self.backend.delete_message('a', 'q', 'm')
+        self.assertEquals(None, message)
+
+    def test_message_create(self):
+        created = self.backend.create_message('a', 'q', 'm', 'test1')
+        self.assertEquals(created, True)
+        message = self.backend.get_message('a', 'q', 'm')
+        self.assertEquals(dict(id='m', ttl=0, hide=0, body='test1'), message)
+        attributes = dict(ttl=100, hide=200)
+        created = self.backend.create_message('a', 'q', 'm', 'test2',
+            attributes)
+        self.assertEquals(created, False)
+        message = self.backend.get_message('a', 'q', 'm')
+        self.assertEquals(dict(id='m', ttl=100, hide=200, body='test2'),
+            message)
+        attributes = dict(ttl=0, hide=0)
+        created = self.backend.create_message('a', 'q', 'm', 'test3',
+            attributes)
+        self.assertEquals(created, False)
+        message = self.backend.get_message('a', 'q', 'm')
+        self.assertEquals(dict(id='m', ttl=0, hide=0, body='test3'), message)
+
+        self.delete_messages()
+
+    def test_message_delete_detail_all(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        filters = dict(detail='all')
+        message = self.backend.delete_message('a', 'q', 'm', filters)
+        self.assertEquals(dict(id='m', ttl=0, hide=0, body='test'), message)
+
+    def test_message_delete_detail_attributes(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        filters = dict(detail='attributes')
+        message = self.backend.delete_message('a', 'q', 'm', filters)
+        self.assertEquals(dict(id='m', ttl=0, hide=0), message)
+
+    def test_message_delete_detail_body(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        filters = dict(detail='body')
+        message = self.backend.delete_message('a', 'q', 'm', filters)
+        self.assertEquals('test', message)
+
+    def test_message_delete_detail_id(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        filters = dict(detail='id')
+        message = self.backend.delete_message('a', 'q', 'm', filters)
+        self.assertEquals('m', message)
+
+    def test_message_delete_detail_none(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        filters = dict(detail='none')
+        message = self.backend.delete_message('a', 'q', 'm', filters)
+        self.assertEquals(None, message)
+
+    def test_message_delete_detail_bad(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        filters = dict(detail='bad')
+        self.assertRaises(burrow.backend.BadDetail,
+            self.backend.delete_message, 'a', 'q', 'm', filters)
+        self.delete_messages()
+
+    def test_message_get_detail_all(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        filters = dict(detail='all')
+        message = self.backend.get_message('a', 'q', 'm', filters)
+        self.assertEquals(dict(id='m', ttl=0, hide=0, body='test'), message)
+        self.delete_messages()
+
+    def test_message_get_detail_attributes(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        filters = dict(detail='attributes')
+        message = self.backend.get_message('a', 'q', 'm', filters)
+        self.assertEquals(dict(id='m', ttl=0, hide=0), message)
+        self.delete_messages()
+
+    def test_message_get_detail_body(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        filters = dict(detail='body')
+        message = self.backend.get_message('a', 'q', 'm', filters)
+        self.assertEquals('test', message)
+        self.delete_messages()
+
+    def test_message_get_detail_id(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        filters = dict(detail='id')
+        message = self.backend.get_message('a', 'q', 'm', filters)
+        self.assertEquals('m', message)
+        self.delete_messages()
+
+    def test_message_get_detail_none(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        filters = dict(detail='none')
+        message = self.backend.get_message('a', 'q', 'm', filters)
+        self.assertEquals(None, message)
+        self.delete_messages()
+
+    def test_message_get_detail_bad(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        filters = dict(detail='bad')
+        self.assertRaises(burrow.backend.BadDetail,
+            self.backend.get_message, 'a', 'q', 'm', filters)
+        self.delete_messages()
+
+    def test_message_update_detail_all(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        attributes = dict(ttl=100, hide=200)
+        filters = dict(detail='all')
+        message = self.backend.update_message('a', 'q', 'm', attributes,
+            filters)
+        self.assertEquals(dict(id='m', ttl=100, hide=200, body='test'),
+            message)
+        self.delete_messages()
+
+    def test_message_update_detail_attributes(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        attributes = dict(ttl=100, hide=200)
+        filters = dict(detail='attributes')
+        message = self.backend.update_message('a', 'q', 'm', attributes,
+            filters)
+        self.assertEquals(dict(id='m', ttl=100, hide=200), message)
+        self.delete_messages()
+
+    def test_message_update_detail_body(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        attributes = dict(ttl=100, hide=200)
+        filters = dict(detail='body')
+        message = self.backend.update_message('a', 'q', 'm', attributes,
+            filters)
+        self.assertEquals('test', message)
+        self.delete_messages()
+
+    def test_message_update_detail_id(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        attributes = dict(ttl=100, hide=200)
+        filters = dict(detail='id')
+        message = self.backend.update_message('a', 'q', 'm', attributes,
+            filters)
+        self.assertEquals('m', message)
+        self.delete_messages()
+
+    def test_message_update_detail_none(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        attributes = dict(ttl=100, hide=200)
+        filters = dict(detail='none')
+        message = self.backend.update_message('a', 'q', 'm', attributes,
+            filters)
+        self.assertEquals(None, message)
+        self.delete_messages()
+
+    def test_message_update_detail_bad(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        attributes = dict(ttl=100, hide=200)
+        filters = dict(detail='bad')
+        self.assertRaises(burrow.backend.BadDetail,
+            self.backend.update_message, 'a', 'q', 'm', attributes, filters)
+        self.delete_messages()
+
+    def test_message_ttl(self):
+        attributes = dict(ttl=1)
+        self.backend.create_message('a', 'q', 'm', 'test', attributes)
+        time.sleep(1)
+        self.backend.clean()
+
+    def test_message_ttl_large(self):
+        attributes = dict(ttl=1)
+        for x in xrange(0, 1000):
+            self.backend.create_message('a', 'q', str(x), str(x), attributes)
+        time.sleep(1)
+        self.backend.clean()
+
+    def test_message_hide(self):
+        attributes = dict(hide=1)
+        self.backend.create_message('a', 'q', 'm', 'test', attributes)
+        time.sleep(1)
+        self.backend.clean()
+        message = self.backend.get_message('a', 'q', 'm')
+        self.assertEquals(dict(id='m', ttl=0, hide=0, body='test'), message)
+        self.delete_messages()
+
+    def test_message_hide_large(self):
+        attributes = dict(hide=1)
+        for x in xrange(0, 1000):
+            self.backend.create_message('a', 'q', str(x), str(x), attributes)
+        time.sleep(1)
+        self.backend.clean()
+        message = self.backend.get_message('a', 'q', '0')
+        self.assertEquals(dict(id='0', ttl=0, hide=0, body='0'), message)
+        self.delete_messages()
+
     def delete_messages(self):
         filters = dict(match_hidden=True)
         messages = list(self.backend.delete_messages('a', 'q', filters))

=== modified file 'test/backend/test_sqlite.py'
--- test/backend/test_sqlite.py	2011-07-22 18:24:23 +0000
+++ test/backend/test_sqlite.py	2011-08-09 23:35:51 +0000
@@ -12,10 +12,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import ConfigParser
+import os
+
 import burrow.backend.sqlite
 import test.backend.test_memory
 
 
 class TestSQLite(test.backend.test_memory.TestMemory):
-    '''Unittests for the SQLite backend.'''
-    backend_class = burrow.backend.sqlite.Backend
+    '''Unittests for the memory-based SQLite backend.'''
+
+    def setUp(self):
+        config = (ConfigParser.ConfigParser(), 'test')
+        self.backend = burrow.backend.sqlite.Backend(config)
+        self.check_empty()
+
+
+class TestSQLiteFile(test.backend.test_memory.TestMemory):
+    '''Unittests for the file-based SQLite backend.'''
+
+    def setUp(self):
+        try:
+            os.unlink('TestSQLiteFile.db')
+        except OSError:
+            pass
+        config = ConfigParser.ConfigParser()
+        config.add_section('test')
+        config.set('test', 'url', 'sqlite://TestSQLiteFile.db')
+        config.set('test', 'synchronous', 'OFF')
+        config = (config, 'test')
+        self.backend = burrow.backend.sqlite.Backend(config)
+        self.check_empty()
+
+    def tearDown(self):
+        self.check_empty()
+        os.unlink('TestSQLiteFile.db')

=== modified file 'test/frontend/test_wsgi.py'
--- test/frontend/test_wsgi.py	2011-08-04 23:43:09 +0000
+++ test/frontend/test_wsgi.py	2011-08-09 23:35:51 +0000
@@ -64,8 +64,7 @@
     def test_message_post(self):
         self._put_url('/a/q/1', body='b')
         for x in range(0, 3):
-            result = self._post_url('/a/q/1?ttl=%d&hide=%d' % (x, x))
-            self.assertEquals(result, {'id': '1'})
+            self._post_url('/a/q/1?ttl=%d&hide=%d' % (x, x), status=204)
             result = self._get_url('/a/q?match_hidden=true')
             message = self.message('1', x, x, body='b')
             self.assertMessages(result, [message])
@@ -186,7 +185,6 @@
     def test_message_ttl(self):
         self._put_url('/a/q/1?ttl=1')
         result = self._get_url('/a/q/1')
-        message = self.message('1', 1)
         self.assertMessages([result], [self.message('1', 1)])
         time.sleep(1)
         self.backend.clean()
@@ -194,7 +192,7 @@
         self._put_url('/a/q/1')
         result = self._get_url('/a/q/1')
         self.assertMessages([result], [self.message('1')])
-        self._post_url('/a/q/1?ttl=1')
+        self._post_url('/a/q/1?ttl=1', status=204)
         result = self._get_url('/a/q/1')
         self.assertMessages([result], [self.message('1', 1)])
         time.sleep(1)
@@ -209,7 +207,7 @@
         self.backend.clean()
         result = self._get_url('/a/q/1')
         self.assertMessages([result], [self.message('1')])
-        self._post_url('/a/q/1?hide=1')
+        self._post_url('/a/q/1?hide=1', status=204)
         result = self._get_url('/a/q/1')
         self.assertMessages([result], [self.message('1', hide=1)])
         time.sleep(1)
@@ -253,7 +251,7 @@
         self.success = False
         self._put_url('/a/q/1?hide=10')
         thread = eventlet.spawn(self._message_wait)
-        eventlet.spawn_after(0.2, self._post_url, '/a/q/1?hide=0')
+        eventlet.spawn_after(0.2, self._post_url, '/a/q/1?hide=0', status=204)
         thread.wait()
         self.assertTrue(self.success)
         self._delete_url('/a/q/1')