Merge lp:~jderose/microfiber/views into lp:microfiber

[Jason Gerard DeRose]

Some misc API changes I grouped up here:

* Database.view() now includes reduce=False by default, which is really how the CouchDB API should work in the first place. For code that only uses a non-reduced view, it sucks for this code to have to know whether there is a reduce function. James and I discussed this a while back, so I'm trying it first in Microfiber, and then will add it to couch.js if we like it.

* Renamed new "non-atomic" Database.bulksave() to Database.save_many() because combined with Database.get_many(), it makes for a clearer, more consistent API.

* Renamed original "all-or-nothing" Database.bulksave2() back to Database.bulksave(). This way there is no API breakage, just in case anyone out there has been using this and wants the "all-or-nothing" behavior, understanding the implications. But I think we should deprecate this method soon. My idea with the convenience methods isn't to capture every scenario, but instead to just capture a few important patterns that might be tricky to get right using post(), get(), etc.

* Adds experimental id_slice_iter() function to give you all the ids from the rows is a view result, but chunked into groups of 25 (override with size=10 or whatever). This is for certain operations in Dmedia where we need to update a large number of docs, like the new Core.purge_store() method. So the idea is as you go through the view results, you use Database.get_many() to get 25 docs, update them all, and then save them back with Database.save_many(). This provides a huge performance improvement over get()/save() for each doc... 10x easily.

[Jason Gerard DeRose (jderose)]

I'm gonna self-approve this as it's blocking other work.

review: Approve

=== modified file 'microfiber.py'
--- microfiber.py	2012-07-06 06:59:53 +0000
+++ microfiber.py	2012-07-07 12:53:19 +0000
@@ -49,6 +49,7 @@
 from urllib.parse import urlparse, urlencode, quote_plus
 from http.client import HTTPConnection, HTTPSConnection, BadStatusLine
 import threading
+import math
 
 
 __all__ = (
@@ -234,6 +235,11 @@
     return replication_body(name, peer, **kw)
 
 
+def id_slice_iter(rows, size=25):
+    for i in range(math.ceil(len(rows) / size)):
+        yield [row['id'] for row in rows[i*size : (i+1)*size]]
+
+
 class HTTPError(Exception):
     """
     Base class for custom `microfiber` exceptions.
@@ -682,7 +688,8 @@
         * `Database.server()` - return a `Server` pointing at same URL
         * `Database.ensure()` - ensure the database exists
         * `Database.save(doc)` - save to CouchDB, update doc _id & _rev in place
-        * `Database.bulksave(docs)` - as above, but with a list of docs
+        * `Database.save_many(docs)` - as above, but with a list of docs
+        * `Database.get_many(doc_ids)` - retrieve many docs at once
         * `Datebase.view(design, view, **options)` - shortcut method, that's all
     """
     def __init__(self, name, env=SERVER):
@@ -749,7 +756,7 @@
         doc['_rev'] = r['rev']
         return r
 
-    def bulksave(self, docs):
+    def save_many(self, docs):
         """
         Bulk-save using non-atomic semantics, updates all _rev in-place.
 
@@ -778,7 +785,7 @@
             raise BulkConflict(conflicts, rows)
         return rows
 
-    def bulksave2(self, docs):
+    def bulksave(self, docs):
         """
         Bulk-save using all-or-nothing semantics, updates all _rev in-place.
 
@@ -819,6 +826,8 @@
 
             ``Database.get('_design', design, '_view', view, **options)``
         """
+        if 'reduce' not in options:
+            options['reduce'] = False
         return self.get('_design', design, '_view', view, **options)
 
     def dump(self, fp, attachments=True):

=== modified file 'test_microfiber.py'
--- test_microfiber.py	2012-07-06 06:59:53 +0000
+++ test_microfiber.py	2012-07-07 12:53:19 +0000
@@ -46,6 +46,7 @@
     usercouch = None
 
 import microfiber
+from microfiber import random_id
 from microfiber import NotFound, MethodNotAllowed, Conflict, PreconditionFailed
 
 
@@ -65,10 +66,6 @@
     )
 
 
-def random_id():
-    return b32encode(os.urandom(10)).decode('ascii')
-
-
 def random_oauth():
     return dict(
         (k, random_id())
@@ -92,7 +89,6 @@
 
 
 assert is_microfiber_id(microfiber.random_id())
-assert not is_microfiber_id(random_id())
 assert not is_microfiber_id(test_id())
 
 
@@ -560,6 +556,44 @@
             }
         )
 
+    def test_id_slice_iter(self):
+        ids = [random_id() for i in range(74)]
+        rows = [{'id': _id} for _id in ids]
+        chunks = list(microfiber.id_slice_iter(rows))
+        self.assertEqual(len(chunks), 3)
+        self.assertEqual(len(chunks[0]), 25)
+        self.assertEqual(len(chunks[1]), 25)
+        self.assertEqual(len(chunks[2]), 24)
+        accum = []
+        for chunk in chunks:
+            accum.extend(chunk)
+        self.assertEqual(accum, ids)
+
+        ids = [random_id() for i in range(75)]
+        rows = [{'id': _id} for _id in ids]
+        chunks = list(microfiber.id_slice_iter(rows))
+        self.assertEqual(len(chunks), 3)
+        self.assertEqual(len(chunks[0]), 25)
+        self.assertEqual(len(chunks[1]), 25)
+        self.assertEqual(len(chunks[2]), 25)
+        accum = []
+        for chunk in chunks:
+            accum.extend(chunk)
+        self.assertEqual(accum, ids)
+
+        ids = [random_id() for i in range(76)]
+        rows = [{'id': _id} for _id in ids]
+        chunks = list(microfiber.id_slice_iter(rows))
+        self.assertEqual(len(chunks), 4)
+        self.assertEqual(len(chunks[0]), 25)
+        self.assertEqual(len(chunks[1]), 25)
+        self.assertEqual(len(chunks[2]), 25)
+        self.assertEqual(len(chunks[3]), 1)
+        accum = []
+        for chunk in chunks:
+            accum.extend(chunk)
+        self.assertEqual(accum, ids)
+
 
 class TestErrors(TestCase):
     def test_errors(self):
@@ -825,6 +859,38 @@
         self.assertEqual(s._basic, 'foo')
         self.assertEqual(s._oauth, 'bar')
 
+    def test_view(self):
+        class Mock(microfiber.Database):
+            def get(self, *parts, **options):
+                self._parts = parts
+                self._options = options
+                assert not hasattr(self, '_return')
+                self._return = random_id()
+                return self._return
+
+        db = Mock('mydb')
+        self.assertEqual(db.view('foo', 'bar'), db._return)
+        self.assertEqual(db._parts, ('_design', 'foo', '_view', 'bar'))
+        self.assertEqual(db._options, {'reduce': False})
+
+        db = Mock('mydb')
+        self.assertEqual(db.view('foo', 'bar', reduce=True), db._return)
+        self.assertEqual(db._parts, ('_design', 'foo', '_view', 'bar'))
+        self.assertEqual(db._options, {'reduce': True})
+
+        db = Mock('mydb')
+        self.assertEqual(db.view('foo', 'bar', include_docs=True), db._return)
+        self.assertEqual(db._parts, ('_design', 'foo', '_view', 'bar'))
+        self.assertEqual(db._options, {'reduce': False, 'include_docs': True})
+
+        db = Mock('mydb')
+        self.assertEqual(
+            db.view('foo', 'bar', include_docs=True, reduce=True),
+            db._return
+        )
+        self.assertEqual(db._parts, ('_design', 'foo', '_view', 'bar'))
+        self.assertEqual(db._options, {'reduce': True, 'include_docs': True})
+
 
 class ReplicationTestCase(TestCase):
     def setUp(self):
@@ -1348,14 +1414,14 @@
             }
         )
 
-    def test_bulksave(self):
+    def test_save_many(self):
         db = microfiber.Database(self.db, self.env)
         self.assertTrue(db.ensure())
 
         # Test that doc['_id'] gets set automatically
         markers = tuple(test_id() for i in range(10))
         docs = [{'marker': m} for m in markers]
-        rows = db.bulksave(docs)
+        rows = db.save_many(docs)
         for (marker, doc, row) in zip(markers, docs, rows):
             self.assertEqual(doc['marker'], marker)
             self.assertEqual(doc['_id'], row['id'])
@@ -1366,7 +1432,7 @@
         # Test when doc['_id'] is already present
         ids = tuple(test_id() for i in range(10))
         docs = [{'_id': _id} for _id in ids]
-        rows = db.bulksave(docs)
+        rows = db.save_many(docs)
         for (_id, doc, row) in zip(ids, docs, rows):
             self.assertEqual(doc['_id'], _id)
             self.assertEqual(row['id'], _id)
@@ -1377,7 +1443,7 @@
         # Let's update all the docs
         for doc in docs:
             doc['x'] = 'foo'    
-        rows = db.bulksave(docs)
+        rows = db.save_many(docs)
         for (_id, doc, row) in zip(ids, docs, rows):
             self.assertEqual(doc['_id'], _id)
             self.assertEqual(row['id'], _id)
@@ -1404,7 +1470,7 @@
                 good.append(doc)
 
         with self.assertRaises(microfiber.BulkConflict) as cm:
-            rows = db.bulksave(docs)
+            rows = db.save_many(docs)
         self.assertEqual(str(cm.exception), 'conflict on 5 docs')
         self.assertEqual(cm.exception.conflicts, bad)
         self.assertEqual(len(cm.exception.rows), 10)
@@ -1424,14 +1490,14 @@
                 self.assertEqual(row['rev'], doc['_rev'])
                 self.assertEqual(real, doc)
 
-    def test_bulksave2(self):
+    def test_bulksave(self):
         db = microfiber.Database(self.db, self.env)
         self.assertTrue(db.ensure())
 
         # Test that doc['_id'] gets set automatically
         markers = tuple(test_id() for i in range(10))
         docs = [{'marker': m} for m in markers]
-        rows = db.bulksave2(docs)
+        rows = db.bulksave(docs)
         for (marker, doc, row) in zip(markers, docs, rows):
             self.assertEqual(doc['marker'], marker)
             self.assertEqual(doc['_id'], row['id'])
@@ -1442,7 +1508,7 @@
         # Test when doc['_id'] is already present
         ids = tuple(test_id() for i in range(10))
         docs = [{'_id': _id} for _id in ids]
-        rows = db.bulksave2(docs)
+        rows = db.bulksave(docs)
         for (_id, doc, row) in zip(ids, docs, rows):
             self.assertEqual(doc['_id'], _id)
             self.assertEqual(row['id'], _id)
@@ -1453,7 +1519,7 @@
         # Let's update all the docs
         for doc in docs:
             doc['x'] = 'foo'    
-        rows = db.bulksave2(docs)
+        rows = db.bulksave(docs)
         for (_id, doc, row) in zip(ids, docs, rows):
             self.assertEqual(doc['_id'], _id)
             self.assertEqual(row['id'], _id)
@@ -1472,7 +1538,7 @@
         # Now let's update all the docs, test all-or-nothing behavior
         for doc in docs:
             doc['x'] = 'bar'    
-        rows = db.bulksave2(docs)
+        rows = db.bulksave(docs)
         for (_id, doc, row) in zip(ids, docs, rows):
             self.assertEqual(doc['_id'], _id)
             self.assertEqual(row['id'], _id)
@@ -1494,7 +1560,7 @@
         # Now update all the docs again, realize all-or-nothing is a bad idea:
         for doc in docs:
             doc['x'] = 'baz'    
-        rows = db.bulksave2(docs)
+        rows = db.bulksave(docs)
         for (i, row) in enumerate(rows):
             _id = ids[i]
             doc = docs[i]
@@ -1516,7 +1582,7 @@
 
         ids = tuple(test_id() for i in range(50))
         docs = [{'_id': _id} for _id in ids]
-        db.bulksave(docs)
+        db.save_many(docs)
 
         # Test an empty doc_ids list
         self.assertEqual(db.get_many([]), [])