Merge lp:~eday/burrow/backend-unittests into lp:burrow

[Eric Day]

Added more backend unit tests, reworked memory backend to use mutable objects instead of tuples, added account filter options.

[OpenStack Infra (hudson-openstack)]

The attempt to merge lp:~eday/burrow/backend-unittests into lp:burrow failed. Below is the output from the failed tests.

running test
running egg_info
creating burrow.egg-info
writing requirements to burrow.egg-info/requires.txt
writing burrow.egg-info/PKG-INFO
writing top-level names to burrow.egg-info/top_level.txt
writing dependency_links to burrow.egg-info/dependency_links.txt
writing manifest file 'burrow.egg-info/SOURCES.txt'
reading manifest file 'burrow.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
warning: no files found matching 'ChangeLog'
writing manifest file 'burrow.egg-info/SOURCES.txt'
running build_ext

test_account (test.backend.test_memory.TestMemory) ... ok
test_account_delete_detail_all (test.backend.test_memory.TestMemory) ... ok
test_account_delete_detail_bad (test.backend.test_memory.TestMemory) ... ERROR
ERROR
test_account_delete_detail_id (test.backend.test_memory.TestMemory) ... ok
test_account_delete_detail_none (test.backend.test_memory.TestMemory) ... ok
test_account_get_detail_all (test.backend.test_memory.TestMemory) ... ok
test_account_get_detail_bad (test.backend.test_memory.TestMemory) ... ERROR
ERROR
test_account_get_detail_id (test.backend.test_memory.TestMemory) ... ok
test_account_get_detail_none (test.backend.test_memory.TestMemory) ... ok
test_account_get_marker (test.backend.test_memory.TestMemory) ... ok
test_account (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_delete_limit (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_delete_limit_marker (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_delete_marker (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_get_limit (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_get_limit_marker (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_get_marker (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_hide (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_post (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_post_limit (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_post_limit_marker (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_post_marker (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_post_wait (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_post_wait_queue (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_put (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_put_wait (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_put_wait_cleanup (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_put_wait_overwrite (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_message_ttl (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_queue (test.frontend.test_wsgi.TestWSGIMemory) ... ok
test_account (test.frontend.test_wsgi.TestWSGISQLite) ... ok
test_message (test.frontend.test_wsgi.TestWSGISQLite) ... ok
test_message_delete_limit (test.frontend.test_wsgi.TestWSGISQLite) ... ok
test_message_delete_limit_marker (test.frontend.test_wsgi.TestWSGISQLite) ... ok
test_message_delete_marker (test.frontend.t...

[lp:~eday/burrow/backend-unittests]

18. By Eric Day on 2011-06-13

Changed to older style assertRaises for compatibility.

=== modified file 'burrow/backend/__init__.py'
--- burrow/backend/__init__.py	2011-04-22 02:18:26 +0000
+++ burrow/backend/__init__.py	2011-06-13 16:44:08 +0000
@@ -32,13 +32,13 @@
         thread_pool.spawn_n(self._clean)
 
     def delete_accounts(self, filters={}):
-        pass
+        return []
 
     def get_accounts(self, filters={}):
         return []
 
     def delete_queues(self, account, filters={}):
-        pass
+        return []
 
     def get_queues(self, account, filters={}):
         return []
@@ -55,13 +55,14 @@
     def create_message(self, account, queue, message, body, attributes={}):
         return True
 
-    def delete_message(self, account, queue, message):
-        return None
-
-    def get_message(self, account, queue, message):
-        return None
-
-    def update_message(self, account, queue, message, attributes={}):
+    def delete_message(self, account, queue, message, filters={}):
+        return None
+
+    def get_message(self, account, queue, message, filters={}):
+        return None
+
+    def update_message(self, account, queue, message, attributes={},
+        filters={}):
         return None
 
     def notify(self, account, queue):
@@ -95,3 +96,6 @@
         while True:
             self.clean()
             eventlet.sleep(1)
+
+class BadDetail(Exception):
+    pass

=== modified file 'burrow/backend/memory.py'
--- burrow/backend/memory.py	2011-04-24 22:47:02 +0000
+++ burrow/backend/memory.py	2011-06-13 16:44:08 +0000
@@ -32,18 +32,24 @@
         self.accounts = Accounts()
 
     def delete_accounts(self, filters={}):
-        self.accounts.delete(filters)
+        return self.accounts.delete(filters)
 
     def get_accounts(self, filters={}):
+        detail = filters.get('detail', 'id')
+        if detail is 'none':
+            detail = None
+        elif detail is not None and detail not in ['id', 'all']:
+            raise burrow.backend.BadDetail(detail)
         for account in self.accounts.iter(filters):
-            yield account[0]
+            if detail is not None:
+                yield account.name
 
     def delete_queues(self, account, filters={}):
-        self.accounts.delete_queues(account, filters)
+        return self.accounts.delete_queues(account, filters)
 
     def get_queues(self, account, filters={}):
-        for queue in self.accounts.iter_queues(account, filters):
-            yield queue[0]
+        for queue in self.accounts.get_queues(account, filters):
+            yield queue.name
 
     def delete_messages(self, account, queue, filters={}):
         return self._scan_queue(account, queue, filters, delete=True)
@@ -62,30 +68,30 @@
         hide = attributes.get('hide', 0)
         if hide > 0:
             hide += int(time.time())
-        for index in xrange(0, len(queue[3])):
-            if queue[3][index]['id'] == message:
-                message = queue[3][index]
+        for index in xrange(0, len(queue.messages)):
+            if queue.messages[index]['id'] == message:
+                message = queue.messages[index]
                 message['ttl'] = ttl
                 message['hide'] = hide
                 message['body'] = body
                 if hide == 0:
-                    self.notify(account[0], queue[0])
+                    self.notify(account.name, queue.name)
                 return False
         message = dict(id=message, ttl=ttl, hide=hide, body=body)
-        queue[3].append(message)
-        self.notify(account[0], queue[0])
+        queue.messages.append(message)
+        self.notify(account.name, queue.name)
         return True
 
-    def delete_message(self, account, queue, message):
+    def delete_message(self, account, queue, message, filters={}):
         account, queue = self.accounts.get_queue(account, queue)
         if queue is None:
             return None
-        for index in xrange(0, len(queue[3])):
-            if queue[3][index]['id'] == message:
-                message = queue[3][index]
-                del queue[3][index]
-                if len(queue[3]) == 0:
-                    self.accounts.delete_queue(account[0], queue[0])
+        for index in xrange(0, len(queue.messages)):
+            if queue.messages[index]['id'] == message:
+                message = queue.messages[index]
+                del queue.messages[index]
+                if len(queue.messages) == 0:
+                    self.accounts.delete_queue(account.name, queue.name)
                 if message['ttl'] > 0:
                     message['ttl'] -= int(time.time())
                 if message['hide'] > 0:
@@ -93,23 +99,24 @@
                 return message
         return None
 
-    def get_message(self, account, queue, message):
+    def get_message(self, account, queue, message, filters={}):
         account, queue = self.accounts.get_queue(account, queue)
         if queue is None:
             return None
-        for index in xrange(0, len(queue[3])):
-            if queue[3][index]['id'] == message:
-                ttl = queue[3][index]['ttl']
+        for index in xrange(0, len(queue.messages)):
+            if queue.messages[index]['id'] == message:
+                ttl = queue.messages[index]['ttl']
                 if ttl > 0:
                     ttl -= int(time.time())
-                hide = queue[3][index]['hide']
+                hide = queue.messages[index]['hide']
                 if hide > 0:
                     hide -= int(time.time())
                 return dict(id=message, ttl=ttl, hide=hide,
-                    body=queue[3][index]['body'])
+                    body=queue.messages[index]['body'])
         return None
 
-    def update_message(self, account, queue, message, attributes={}):
+    def update_message(self, account, queue, message, attributes={},
+        filters={}):
         account, queue = self.accounts.get_queue(account, queue)
         if queue is None:
             return None
@@ -119,29 +126,29 @@
         hide = attributes.get('hide', None)
         if hide is not None and hide > 0:
             hide += int(time.time())
-        for index in xrange(0, len(queue[3])):
-            if queue[3][index]['id'] == message:
-                message = queue[3][index]
+        for index in xrange(0, len(queue.messages)):
+            if queue.messages[index]['id'] == message:
+                message = queue.messages[index]
                 if ttl is not None:
                     message['ttl'] = ttl
                 if hide is not None:
                     message['hide'] = hide
                     if hide == 0:
-                        self.notify(account[0], queue[0])
+                        self.notify(account.name, queue.name)
                 return message
         return None
 
     def clean(self):
         now = int(time.time())
         for account in self.accounts.iter():
-            for queue in account[3].iter():
+            for queue in account.queues.iter():
                 index = 0
                 notify = False
-                total = len(queue[3])
+                total = len(queue.messages)
                 while index < total:
-                    message = queue[3][index]
+                    message = queue.messages[index]
                     if 0 < message['ttl'] <= now:
-                        del queue[3][index]
+                        del queue.messages[index]
                         total -= 1
                     else:
                         if 0 < message['hide'] <= now:
@@ -149,9 +156,9 @@
                             notify = True
                         index += 1
                 if notify:
-                    self.notify(account[0], queue[0])
-                if len(queue[3]) == 0:
-                    self.accounts.delete_queue(account[0], queue[0])
+                    self.notify(account.name, queue.name)
+                if len(queue.messages) == 0:
+                    self.accounts.delete_queue(account.name, queue.name)
 
     def _scan_queue(self, account, queue, filters, attributes={},
         delete=False):
@@ -162,8 +169,8 @@
         notify = False
         if 'marker' in filters and filters['marker'] is not None:
             found = False
-            for index in xrange(0, len(queue[3])):
-                message = queue[3][index]
+            for index in xrange(0, len(queue.messages)):
+                message = queue.messages[index]
                 if message['id'] == filters['marker']:
                     index += 1
                     found = True
@@ -171,7 +178,7 @@
             if not found:
                 index = 0
         messages = []
-        total = len(queue[3])
+        total = len(queue.messages)
         limit = filters.get('limit', None)
         match_hidden = filters.get('match_hidden', False)
         ttl = attributes.get('ttl', None)
@@ -181,7 +188,7 @@
         if hide is not None and hide > 0:
             hide += int(time.time())
         while index < total:
-            message = queue[3][index]
+            message = queue.messages[index]
             if not match_hidden and message['hide'] != 0:
                 index += 1
                 continue
@@ -192,7 +199,7 @@
                 if hide == 0:
                     notify = True
             if delete:
-                del queue[3][index]
+                del queue.messages[index]
                 total -= 1
             else:
                 index += 1
@@ -209,15 +216,22 @@
                 if limit == 0:
                     break
         if notify:
-            self.notify(account[0], queue[0])
-        if len(queue[3]) == 0:
-            self.accounts.delete_queue(account[0], queue[0])
-
-
-class IndexedTupleList(object):
-    '''Class for managing an indexed tuple list. The tuple must be at
-    least three elements and must reserve the first three for (name,
-    next, previous).'''
+            self.notify(account.name, queue.name)
+        if len(queue.messages) == 0:
+            self.accounts.delete_queue(account.name, queue.name)
+
+
+class Item(object):
+    '''Object to represent elements in a indexed linked list.'''
+
+    def __init__(self, name=None):
+        self.name = name
+        self.next = None
+        self.prev = None
+
+
+class IndexedList(object):
+    '''Class for managing an indexed linked list.'''
 
     def __init__(self):
         self.first = None
@@ -225,11 +239,13 @@
         self.index = {}
 
     def add(self, item):
-        item = (item[0], None, self.last) + item[3:]
         if self.first is None:
             self.first = item
+        if self.last is not None:
+            item.prev = self.last
+            self.last.next = item
         self.last = item
-        self.index[item[0]] = item
+        self.index[item.name] = item
         return item
 
     def count(self):
@@ -241,25 +257,28 @@
             self.last = None
             self.index.clear()
             return
+        detail = filters.get('detail', None)
+        if detail is 'none':
+            detail = None
+        elif detail is not None and detail not in ['id', 'all']:
+            raise burrow.backend.BadDetail(detail)
         for item in self.iter(filters):
-            self.delete_item(item[0])
+            self.delete_item(item.name)
+            if detail is not None:
+                yield item.name
 
     def delete_item(self, name):
         if name not in self.index:
             return
-        item = self.index[name][1]
-        if item is not None:
-            prev_item = self.index[name][2]
-            self.index[item[0]] = (item[0], item[1], prev_item) + item[3:]
-        item = self.index[name][2]
-        if item is not None:
-            next_item = self.index[name][1]
-            self.index[item[0]] = (item[0], next_item, item[2]) + item[3:]
-        if self.first == self.index[name]:
-            self.first = self.index[name][1]
-        if self.last == self.index[name]:
-            self.last = self.index[name][2]
-        del self.index[name]
+        item = self.index.pop(name)
+        if item.next is not None:
+            item.next.prev = item.prev
+        if item.prev is not None:
+            item.prev.next = item.next
+        if self.first == item:
+            self.first = item.next
+        if self.last == item:
+            self.last = item.prev
 
     def get(self, name):
         if name in self.index:
@@ -269,7 +288,7 @@
     def iter(self, filters={}):
         marker = filters.get('marker', None)
         if marker is not None and marker in self.index:
-            item = self.index[marker]
+            item = self.index[marker].next
         else:
             item = self.first
         limit = filters.get('limit', None)
@@ -279,46 +298,61 @@
                 limit -= 1
                 if limit == 0:
                     break
-            item = item[1]
-
-
-class Accounts(IndexedTupleList):
+            item = item.next
+
+
+class Account(Item):
+
+    def __init__(self, name=None):
+        super(Account, self).__init__(name)
+        self.queues = Queues()
+
+
+class Accounts(IndexedList):
 
     def delete_queue(self, account, queue):
         account = self.get(account)
         if account is not None:
-            account[3].delete_item(queue)
-            if account[3].count() == 0:
-                self.delete_item(account[0])
+            account.queues.delete_item(queue)
+            if account.queues.count() == 0:
+                self.delete_item(account.name)
 
     def delete_queues(self, account, filters):
         account = self.get(account)
         if account is not None:
-            account[3].delete(filters)
-            if account[3].count() == 0:
-                self.delete_item(account[0])
+            for queue in account.queues.delete(filters):
+                yield queue
+            if account.queues.count() == 0:
+                self.delete_item(account.name)
 
     def get_queue(self, account, queue, create=False):
         if account in self.index:
             account = self.index[account]
         elif create:
-            account = self.add((account, None, None, Queues()))
+            account = self.add(Account(account))
         else:
             return None, None
-        return account, account[3].get(queue, create)
+        return account, account.queues.get(queue, create)
 
-    def iter_queues(self, account, filters={}):
+    def get_queues(self, account, filters={}):
         account = self.get(account)
         if account is not None:
-            for queue in account[3].iter(filters):
+            for queue in account.queues.iter(filters):
                 yield queue
 
 
-class Queues(IndexedTupleList):
+class Queue(Item):
+
+    def __init__(self, name=None):
+        super(Queue, self).__init__(name)
+        self.messages = []
+
+
+class Queues(IndexedList):
 
     def get(self, queue, create=False):
         if queue in self.index:
             return self.index[queue]
         elif create:
-            return self.add((queue, None, None, []))
+            return self.add(Queue(queue))
         return None

=== modified file 'burrow/backend/sqlite.py'
--- burrow/backend/sqlite.py	2011-04-24 22:47:02 +0000
+++ burrow/backend/sqlite.py	2011-06-13 16:44:08 +0000
@@ -54,7 +54,8 @@
         if len(filters) == 0:
             self.db.execute('DELETE FROM queues')
             self.db.execute('DELETE FROM messages')
-            return
+            return []
+        return []
 
     def get_accounts(self, filters={}):
         query = 'SELECT DISTINCT account FROM queues'
@@ -76,10 +77,11 @@
         for row in self.db.execute(query, (account,)):
             ids.append(str(row[0]))
         if len(ids) == 0:
-            return
+            return []
         query = 'DELETE FROM messages WHERE queue IN (%s)'
         self.db.execute(query % ','.join(ids))
         self.db.execute('DELETE FROM queues WHERE account=?', (account,))
+        return []
 
     def get_queues(self, account, filters={}):
         query = 'SELECT queue FROM queues WHERE account=?'

=== modified file 'burrow/frontend/wsgi.py'
--- burrow/frontend/wsgi.py	2011-05-24 03:07:25 +0000
+++ burrow/frontend/wsgi.py	2011-06-13 16:44:08 +0000
@@ -128,7 +128,7 @@
     @webob.dec.wsgify
     def _delete_version(self, req):
         filters = self._parse_filters(req)
-        self.backend.delete_accounts(filters)
+        [account for account in self.backend.delete_accounts(filters)]
         return webob.exc.HTTPNoContent()
 
     @webob.dec.wsgify
@@ -142,7 +142,7 @@
     @webob.dec.wsgify
     def _delete_account(self, req, account):
         filters = self._parse_filters(req)
-        self.backend.delete_queues(account, filters)
+        [queue for queue in self.backend.delete_queues(account, filters)]
         return webob.exc.HTTPNoContent()
 
     @webob.dec.wsgify

=== added directory 'test/backend'
=== added file 'test/backend/__init__.py'
=== added file 'test/backend/test_memory.py'
--- test/backend/test_memory.py	1970-01-01 00:00:00 +0000
+++ test/backend/test_memory.py	2011-06-13 16:44:08 +0000
@@ -0,0 +1,121 @@
+# Copyright (C) 2011 OpenStack LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import ConfigParser
+import time
+import unittest
+
+import eventlet
+
+import burrow.backend.memory
+
+
+class TestMemory(unittest.TestCase):
+    '''Unittests for the memory backend.'''
+    backend_class = burrow.backend.memory.Backend
+
+    def setUp(self):
+        config = (ConfigParser.ConfigParser(), 'test')
+        self.backend = self.backend_class(config)
+        self.assertEquals([], list(self.backend.get_accounts()))
+        self.assertEquals([], list(self.backend.get_queues('a')))
+        self.assertEquals([], list(self.backend.get_messages('a', 'q')))
+
+    def tearDown(self):
+        self.assertEquals([], list(self.backend.get_messages('a', 'q')))
+        self.assertEquals([], list(self.backend.get_queues('a')))
+        self.assertEquals([], list(self.backend.get_accounts()))
+
+    def test_account(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        self.assertEquals(['a'], list(self.backend.get_accounts()))
+        self.assertEquals([], list(self.backend.delete_accounts()))
+
+    def test_account_delete_detail_all(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        self.assertEquals(['a'], list(self.backend.get_accounts()))
+        filters = dict(detail='all')
+        accounts = list(self.backend.delete_accounts(filters))
+        self.assertEquals(['a'], accounts)
+
+    def test_account_delete_detail_id(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        self.assertEquals(['a'], list(self.backend.get_accounts()))
+        filters = dict(detail='id')
+        accounts = list(self.backend.delete_accounts(filters))
+        self.assertEquals(['a'], accounts)
+
+    def test_account_delete_detail_none(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        self.assertEquals(['a'], list(self.backend.get_accounts()))
+        filters = dict(detail='none')
+        accounts = list(self.backend.delete_accounts(filters))
+        self.assertEquals([], accounts)
+
+    def test_account_delete_detail_bad(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        self.assertEquals(['a'], list(self.backend.get_accounts()))
+        filters = dict(detail='bad')
+        accounts = self.backend.delete_accounts(filters)
+        self.assertRaises(burrow.backend.BadDetail, list, accounts)
+        self.assertEquals([], list(self.backend.delete_accounts()))
+
+    def test_account_get_detail_all(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        self.assertEquals(['a'], list(self.backend.get_accounts()))
+        filters = dict(detail='all')
+        accounts = list(self.backend.get_accounts(filters))
+        self.assertEquals(['a'], accounts)
+        self.assertEquals([], list(self.backend.delete_accounts()))
+
+    def test_account_get_detail_id(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        self.assertEquals(['a'], list(self.backend.get_accounts()))
+        filters = dict(detail='id')
+        accounts = list(self.backend.get_accounts(filters))
+        self.assertEquals(['a'], accounts)
+        self.assertEquals([], list(self.backend.delete_accounts()))
+
+    def test_account_get_detail_none(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        self.assertEquals(['a'], list(self.backend.get_accounts()))
+        filters = dict(detail='none')
+        accounts = list(self.backend.get_accounts(filters))
+        self.assertEquals([], accounts)
+        self.assertEquals([], list(self.backend.delete_accounts()))
+
+    def test_account_get_detail_bad(self):
+        self.backend.create_message('a', 'q', 'm', 'test')
+        self.assertEquals(['a'], list(self.backend.get_accounts()))
+        filters = dict(detail='bad')
+        accounts = self.backend.get_accounts(filters)
+        self.assertRaises(burrow.backend.BadDetail, list, accounts)
+        self.assertEquals([], list(self.backend.delete_accounts()))
+
+    def test_account_get_marker(self):
+        self.backend.create_message('a1', 'q', 'm', 'test')
+        self.backend.create_message('a2', 'q', 'm', 'test')
+        self.backend.create_message('a3', 'q', 'm', 'test')
+        accounts = list(self.backend.get_accounts())
+        self.assertEquals(3, len(accounts))
+        filters = dict(marker=accounts[0])
+        accounts2 = list(self.backend.get_accounts(filters))
+        self.assertEquals(accounts[1:], accounts2)
+        filters = dict(marker=accounts[1])
+        accounts3 = list(self.backend.get_accounts(filters))
+        self.assertEquals(accounts[2:], accounts3)
+        filters = dict(marker=accounts[2])
+        accounts4 = list(self.backend.get_accounts(filters))
+        self.assertEquals([], accounts4)
+        self.assertEquals([], list(self.backend.delete_accounts()))