Merge lp:~thisfred/u1db/get_all_the_docs into lp:u1db

[Eric Casteleijn]

added db.get_all_docs() method to all backends that returns all (non-deleted) documents.

added db.get_all_docs() method to all backends that returns all (non-deleted) documents.

[John Lenton (chipaca)]

Two questions: 1. don't we want to return a generation number, so you can call get_all_docs and then move to whats_changed from there on out? 2. isn't this a candidate for include_deleted?

[Eric Casteleijn (thisfred)]

thanks,

1. is a fair point, though I'm not sure what the best way to return the generation is
2. I had thought about but didn't add yet, should not be hard, I'll add it tomorrow

[John A Meinel (jameinel)]

On 5/28/2012 4:09 PM, Eric Casteleijn wrote:
> thanks,
>
> 1. is a fair point, though I'm not sure what the best way to return the generation is
> 2. I had thought about but didn't add yet, should not be hard, I'll add it tomorrow
>

It needs to be a return parameter so that it it accurate to the point of
the documents we return.

The cheap way to do it, is to call db._get_generation() at the
beginning, then just iterate whatever documents, and then return the
value we saw at the beginning.

Concurrent updates then will have you see the same value twice, but you
won't ever miss content.

John
=:->

[lp:~thisfred/u1db/get_all_the_docs]

313. By Eric Casteleijn on 2012-05-29

changed api definition

314. By Eric Casteleijn on 2012-05-29

merged trunk

315. By Eric Casteleijn on 2012-05-29

added include_deleted parameter, and included db generation in results

[Samuele Pedroni (pedronis)]

review: Approve

=== modified file 'include/u1db/u1db.h'
--- include/u1db/u1db.h	2012-05-24 21:09:21 +0000
+++ include/u1db/u1db.h	2012-05-29 14:35:21 +0000
@@ -143,6 +143,7 @@
  * Get the document defined by the given document id.
  *
  * @param doc_id The document we are looking for
+ * @param include_deleted If true, return the document even if it was deleted.
  * @param doc (OUT) a document (or NULL) matching the request
  * @return status, will be U1DB_OK if there is no error, even if there is no
  *      document matching that doc_id.
@@ -158,6 +159,7 @@
  * @param doc_ids An array of document ids to retrieve.
  * @param check_for_conflicts If true, check if each document has any
  *          conflicts, if false, the conflict checking will be skipped.
+ * @param include_deleted If true, return documents even if they were deleted.
  * @param context A void* that is returned via the callback function.
  * @param cb This will be called with each document requested. The api is
  *           cb(void* context, u1db_document *doc). The returned documents are
@@ -169,6 +171,20 @@
                   u1db_doc_callback cb);
 
 /**
+ * Retrieve all documents from the database.
+ *
+ * @param include_deleted If true, return documents even if they were deleted.
+ * @param generation (OUT) the generation the database is at
+ * @param context A void* that is returned via the callback function.
+ * @param cb This will be called with each document requested. The api is
+ *           cb(void* context, u1db_document *doc). The returned documents are
+ *           allocated on the heap, and must be freed by the caller via
+ *           u1db_free_doc.
+ */
+int u1db_get_all_docs(u1database *db, int include_deleted, int *generation,
+                      void *context, u1db_doc_callback cb);
+
+/**
  * Get all of the contents associated with a conflicted document.
  *
  * If a document is not conflicted, then this will not invoke the callback

=== modified file 'src/u1db.c'
--- src/u1db.c	2012-05-25 20:33:57 +0000
+++ src/u1db.c	2012-05-29 14:35:21 +0000
@@ -1037,6 +1037,45 @@
     return status;
 }
 
+int
+u1db_get_all_docs(u1database *db, int include_deleted, int *generation,
+                  void *context, u1db_doc_callback cb)
+{
+    int status;
+    sqlite3_stmt *statement;
+
+    if (db == NULL || cb == NULL) {
+        return U1DB_INVALID_PARAMETER;
+    }
+    status = u1db__get_generation(db, generation);
+    if (status != U1DB_OK)
+        return status;
+    status = sqlite3_prepare_v2(db->sql_handle,
+        "SELECT doc_id, doc_rev, content FROM document", -1, &statement, NULL);
+    if (status != SQLITE_OK) { goto finish; }
+    status = sqlite3_step(statement);
+    while (status == SQLITE_ROW) {
+        const char *doc_id;
+        const char *revision;
+        const char *content;
+        u1db_document *doc;
+        doc_id = (char *)sqlite3_column_text(statement, 0);
+        revision = (char *)sqlite3_column_text(statement, 1);
+        content = (char *)sqlite3_column_text(statement, 2);
+        if (content != NULL || include_deleted) {
+            doc = u1db__allocate_document(doc_id, revision, content, 0);
+            cb(context, doc);
+        }
+        status = sqlite3_step(statement);
+    }
+    if (status == SQLITE_DONE) {
+        status = SQLITE_OK;
+    }
+finish:
+    sqlite3_finalize(statement);
+    return status;
+}
+
 // Take cur_rev, and update it to have a version incremented based on the
 // database replica uid
 static int

=== modified file 'u1db/__init__.py'
--- u1db/__init__.py	2012-05-24 21:09:21 +0000
+++ u1db/__init__.py	2012-05-29 14:35:21 +0000
@@ -93,6 +93,18 @@
         """
         raise NotImplementedError(self.get_docs)
 
+    def get_all_docs(self, include_deleted=False):
+        """Get the JSON content for all documents in the database.
+
+        :param include_deleted: If set to True, deleted documents will be
+            returned with empty content. Otherwise deleted documents will not
+            be included in the results.
+        :return: (generation, [Document])
+            The current generation of the database, followed by a list of all
+            the documents in the database.
+        """
+        raise NotImplementedError(self.get_all_docs)
+
     def create_doc(self, content, doc_id=None):
         """Create a new document.
 

=== modified file 'u1db/backends/inmemory.py'
--- u1db/backends/inmemory.py	2012-05-25 20:33:57 +0000
+++ u1db/backends/inmemory.py	2012-05-29 14:35:21 +0000
@@ -117,6 +117,16 @@
         doc.has_conflicts = (doc.doc_id in self._conflicts)
         return doc
 
+    def get_all_docs(self, include_deleted=False):
+        """Return all documents in the database."""
+        generation = self._get_generation()
+        results = []
+        for doc_id, (doc_rev, content) in self._docs.items():
+            if content is None and not include_deleted:
+                continue
+            results.append(Document(doc_id, doc_rev, content))
+        return (generation, results)
+
     def get_doc_conflicts(self, doc_id):
         if doc_id not in self._conflicts:
             return []

=== modified file 'u1db/backends/sqlite_backend.py'
--- u1db/backends/sqlite_backend.py	2012-05-25 20:33:57 +0000
+++ u1db/backends/sqlite_backend.py	2012-05-29 14:35:21 +0000
@@ -301,6 +301,19 @@
         doc.has_conflicts = self._has_conflicts(doc.doc_id)
         return doc
 
+    def get_all_docs(self, include_deleted=False):
+        """Get all documents from the database."""
+        generation = self._get_generation()
+        results = []
+        c = self._db_handle.cursor()
+        c.execute("SELECT doc_id, doc_rev, content FROM document;")
+        rows = c.fetchall()
+        for doc_id, doc_rev, content in rows:
+            if content is None and not include_deleted:
+                continue
+            results.append(Document(doc_id, doc_rev, content))
+        return (generation, results)
+
     def put_doc(self, doc):
         if doc.doc_id is None:
             raise errors.InvalidDocId()

=== modified file 'u1db/tests/c_backend_wrapper.pyx'
--- u1db/tests/c_backend_wrapper.pyx	2012-05-24 21:09:21 +0000
+++ u1db/tests/c_backend_wrapper.pyx	2012-05-29 14:35:21 +0000
@@ -76,6 +76,8 @@
     int u1db_get_docs(u1database *db, int n_doc_ids, const_char_ptr *doc_ids,
                       int check_for_conflicts, int include_deleted,
                       void *context, u1db_doc_callback cb)
+    int u1db_get_all_docs(u1database *db, int include_deleted, int *generation,
+                          void *context, u1db_doc_callback cb)
     int u1db_put_doc(u1database *db, u1db_document *doc)
     int u1db__put_doc_if_newer(u1database *db, u1db_document *doc,
                                int save_conflict, char *replica_uid,
@@ -934,6 +936,19 @@
         free(<void*>c_doc_ids)
         return a_list
 
+    def get_all_docs(self, include_deleted=False):
+        cdef int c_generation
+
+        a_list = []
+        deleted = 1 if include_deleted else 0
+        generation = 0
+        c_generation = generation
+        handle_status(
+            "get_all_docs", u1db_get_all_docs(
+                self._db, deleted, &c_generation, <void*>a_list,
+                _append_doc_to_list))
+        return (c_generation, a_list)
+
     def resolve_doc(self, CDocument doc, conflicted_doc_revs):
         cdef const_char_ptr *revs
         cdef int n_revs

=== modified file 'u1db/tests/test_backends.py'
--- u1db/tests/test_backends.py	2012-05-25 20:33:57 +0000
+++ u1db/tests/test_backends.py	2012-05-29 14:35:21 +0000
@@ -281,6 +281,34 @@
     def test_get_docs_empty_list(self):
         self.assertEqual([], self.db.get_docs([]))
 
+    def test_get_all_docs_empty(self):
+        self.assertEqual([], self.db.get_all_docs()[1])
+
+    def test_get_all_docs(self):
+        doc1 = self.db.create_doc(simple_doc)
+        doc2 = self.db.create_doc(nested_doc)
+        self.assertEqual(
+            sorted([doc1, doc2]), sorted(self.db.get_all_docs()[1]))
+
+    def test_get_all_docs_exclude_deleted(self):
+        doc1 = self.db.create_doc(simple_doc)
+        doc2 = self.db.create_doc(nested_doc)
+        self.db.delete_doc(doc2)
+        self.assertEqual([doc1], self.db.get_all_docs()[1])
+
+    def test_get_all_docs_include_deleted(self):
+        doc1 = self.db.create_doc(simple_doc)
+        doc2 = self.db.create_doc(nested_doc)
+        self.db.delete_doc(doc2)
+        self.assertEqual(
+            sorted([doc1, doc2]),
+            sorted(self.db.get_all_docs(include_deleted=True)[1]))
+
+    def test_get_all_docs_generation(self):
+        self.db.create_doc(simple_doc)
+        self.db.create_doc(nested_doc)
+        self.assertEqual(2, self.db.get_all_docs()[0])
+
     def test_simple_put_doc_if_newer(self):
         doc = self.make_document('my-doc-id', 'test:1', simple_doc)
         state_at_gen = self.db._put_doc_if_newer(doc, save_conflict=False)