Merge lp:~jameinel/u1db/put_doc_if_newer_private_999574 into lp:u1db

[John A Meinel]

This exposes the transaction_id stuff in put_doc_if_newer. This still is not a complete implementation, as it gets None as the trans_id from the sync code.

However, it is a step further, and should avoid more enormous diffs to review.

[lp:~jameinel/u1db/put_doc_if_newer_private_999574]

310. By John A Meinel on 2012-05-18

Do the right thing with a const issue for the tmp variable.

311. By John A Meinel on 2012-05-18

Catch the other https tests.

[John O'Brien (jdobrien)]

+1 tastes great, less filling.

review: Approve

=== modified file 'include/u1db/u1db_internal.h'
--- include/u1db/u1db_internal.h	2012-05-18 14:08:24 +0000
+++ include/u1db/u1db_internal.h	2012-05-18 14:08:24 +0000
@@ -182,6 +182,7 @@
  *                    this document came from. (Can be NULL)
  * @param replica_gen Generation of the replica. Only meaningful if
  *                    replica_uid is set.
+ * @param replica_trans_id Transaction id associated with generation.
  * @param state (OUT) Return one of:
  *  U1DB_INSERTED   The document is newer than what we have
  *  U1DB_SUPERSEDED We already have a newer document than what was passed
@@ -193,7 +194,8 @@
  */
 int u1db__put_doc_if_newer(u1database *db, u1db_document *doc,
                            int save_conflict, const char *replica_uid,
-                           int replica_gen, int *state, int *at_gen);
+                           int replica_gen, const char *replica_trans_id,
+                           int *state, int *at_gen);
 
 /**
  * Internal API, Get the global database rev.

=== modified file 'src/u1db.c'
--- src/u1db.c	2012-05-18 14:08:24 +0000
+++ src/u1db.c	2012-05-18 14:08:24 +0000
@@ -723,8 +723,8 @@
 
 int
 u1db__put_doc_if_newer(u1database *db, u1db_document *doc, int save_conflict,
-                       const char *replica_uid, int replica_gen, int *state,
-                       int *at_gen)
+                       const char *replica_uid, int replica_gen,
+                       const char *replica_trans_id, int *state, int *at_gen)
 {
     const char *stored_content = NULL, *stored_doc_rev = NULL;
     int status = U1DB_INVALID_PARAMETER, store = 0;
@@ -810,7 +810,8 @@
                            (stored_doc_rev != NULL));
     }
     if (status == U1DB_OK && replica_uid != NULL) {
-        status = u1db__set_sync_info(db, replica_uid, replica_gen, "T-sid");
+        status = u1db__set_sync_info(db, replica_uid, replica_gen,
+                                     replica_trans_id);
     }
     if (status == U1DB_OK && at_gen != NULL) {
         status = u1db__get_generation(db, at_gen);
@@ -1341,16 +1342,23 @@
         status = SQLITE_OK;
         *generation = 0;
         *trans_id = strdup("");
+        if (*trans_id == NULL) {
+            status = U1DB_NOMEM;
+        }
     } else if (status == SQLITE_ROW) {
         *generation = sqlite3_column_int(statement, 0);
         // Note: We may want to handle the column containing NULL
         tmp = (const char *)sqlite3_column_text(statement, 1);
-        *trans_id = strdup(tmp);
+        if (tmp == NULL) {
+            *trans_id = NULL;
+        } else {
+            *trans_id = strdup(tmp);
+            if (*trans_id == NULL) {
+                status = U1DB_NOMEM;
+            }
+        }
         status = SQLITE_OK;
     }
-    if (*trans_id == NULL) {
-        status = U1DB_NOMEM;
-    }
 finish:
     sqlite3_finalize(statement);
     return status;

=== modified file 'src/u1db_http_sync_target.c'
--- src/u1db_http_sync_target.c	2012-05-18 14:08:24 +0000
+++ src/u1db_http_sync_target.c	2012-05-18 14:08:24 +0000
@@ -362,6 +362,7 @@
     struct _http_state *state;
     struct _http_request req = {0};
     char *url = NULL;
+    const char *tmp = NULL;
     int status;
     long http_code;
     struct curl_slist *headers = NULL;
@@ -455,12 +456,17 @@
     *source_gen = json_object_get_int(obj);
     obj = json_object_object_get(json, "source_transaction_id");
     if (obj == NULL) {
-        status = U1DB_INVALID_HTTP_RESPONSE;
-        goto finish;
-    }
-    *trans_id = strdup(json_object_get_string(obj));
-    if (*trans_id == NULL) {
-        status = U1DB_NOMEM;
+        *trans_id = NULL;
+    } else {
+        tmp = json_object_get_string(obj);
+        if (tmp == NULL) {
+            *trans_id = NULL;
+        } else {
+            *trans_id = strdup(tmp);
+            if (*trans_id == NULL) {
+                status = U1DB_NOMEM;
+            }
+        }
     }
 finish:
     if (req.header_buffer != NULL) {

=== modified file 'src/u1db_sync_target.c'
--- src/u1db_sync_target.c	2012-05-18 14:08:24 +0000
+++ src/u1db_sync_target.c	2012-05-18 14:08:24 +0000
@@ -280,7 +280,7 @@
     }
     // fprintf(stderr, "Inserting %s from source\n", doc->doc_id);
     status = u1db__put_doc_if_newer(se->db, doc, 0, se->source_replica_uid,
-                                    source_gen, &insert_state, &at_gen);
+                                    source_gen, NULL, &insert_state, &at_gen);
     if (insert_state == U1DB_INSERTED || insert_state == U1DB_CONVERGED) {
 	lh_table_insert(se->seen_ids, strdup(doc->doc_id), (void *)(intptr_t)at_gen);
     } else {
@@ -437,7 +437,7 @@
     state = (struct _return_doc_state *)context;
 
     status = u1db__put_doc_if_newer(state->db, doc, 1, state->target_uid, gen,
-                                    &insert_state, NULL);
+                                    NULL, &insert_state, NULL);
     u1db_free_doc(&doc);
     if (status == U1DB_OK) {
         if (insert_state == U1DB_INSERTED || insert_state == U1DB_CONFLICTED) {

=== modified file 'u1db/__init__.py'
--- u1db/__init__.py	2012-05-18 14:08:24 +0000
+++ u1db/__init__.py	2012-05-18 14:08:24 +0000
@@ -248,7 +248,7 @@
         raise NotImplementedError(self._set_sync_info)
 
     def _put_doc_if_newer(self, doc, save_conflict, replica_uid=None,
-                          replica_gen=None):
+                          replica_gen=None, replica_trans_id=None):
         """Insert/update document into the database with a given revision.
 
         This api is used during synchronization operations.
@@ -273,6 +273,8 @@
         :param replica_gen: The generation of the replica corresponding to the
             this document. The replica arguments are optional, but are used
             during synchronization.
+        :param replica_trans_id: The transaction_id associated with the
+            generation.
         :return: (state, at_gen) -  If we don't have doc_id already,
             or if doc_rev supersedes the existing document revision,
             then the content will be inserted, and state is 'inserted'.

=== modified file 'u1db/backends/__init__.py'
--- u1db/backends/__init__.py	2012-05-18 14:08:24 +0000
+++ u1db/backends/__init__.py	2012-05-18 14:08:24 +0000
@@ -91,7 +91,7 @@
         return result
 
     def _put_doc_if_newer(self, doc, save_conflict, replica_uid=None,
-                          replica_gen=None):
+                          replica_gen=None, replica_trans_id=None):
         cur_doc = self._get_doc(doc.doc_id)
         doc_vcr = VectorClockRev(doc.rev)
         if cur_doc is None:
@@ -115,7 +115,7 @@
             if save_conflict:
                 self._force_doc_sync_conflict(doc)
         if replica_uid is not None and replica_gen is not None:
-            self._do_set_sync_info(replica_uid, replica_gen, 'T-sid')
+            self._do_set_sync_info(replica_uid, replica_gen, replica_trans_id)
         return state, self._get_generation()
 
     def _ensure_maximal_rev(self, cur_rev, extra_revs):

=== modified file 'u1db/backends/sqlite_backend.py'
--- u1db/backends/sqlite_backend.py	2012-05-18 14:08:24 +0000
+++ u1db/backends/sqlite_backend.py	2012-05-18 14:08:24 +0000
@@ -438,11 +438,12 @@
                        other_transaction_id))
 
     def _put_doc_if_newer(self, doc, save_conflict, replica_uid=None,
-                          replica_gen=None):
+                          replica_gen=None, replica_trans_id=None):
         with self._db_handle:
             return super(SQLiteDatabase, self)._put_doc_if_newer(doc,
                 save_conflict=save_conflict,
-                replica_uid=replica_uid, replica_gen=replica_gen)
+                replica_uid=replica_uid, replica_gen=replica_gen,
+                replica_trans_id=replica_trans_id)
 
     def _add_conflict(self, c, doc_id, my_doc_rev, my_content):
         c.execute("INSERT INTO conflicts VALUES (?, ?, ?)",

=== modified file 'u1db/sync.py'
--- u1db/sync.py	2012-05-18 14:08:24 +0000
+++ u1db/sync.py	2012-05-18 14:08:24 +0000
@@ -53,7 +53,8 @@
         # Increases self.num_inserted depending whether the document
         # was effectively inserted.
         state, _ = self.source._put_doc_if_newer(doc, save_conflict=True,
-            replica_uid=self.target_replica_uid, replica_gen=replica_gen)
+            replica_uid=self.target_replica_uid, replica_gen=replica_gen,
+            replica_trans_id=None)
         if state == 'inserted':
             self.num_inserted += 1
         elif state == 'converged':
@@ -166,7 +167,8 @@
         :return: None
         """
         state, at_gen = self._db._put_doc_if_newer(doc, save_conflict=False,
-            replica_uid=self.source_replica_uid, replica_gen=source_gen)
+            replica_uid=self.source_replica_uid, replica_gen=source_gen,
+            replica_trans_id=None)
         if state == 'inserted':
             self.seen_ids[doc.doc_id] = at_gen
         elif state == 'converged':

=== modified file 'u1db/tests/c_backend_wrapper.pyx'
--- u1db/tests/c_backend_wrapper.pyx	2012-05-18 14:08:24 +0000
+++ u1db/tests/c_backend_wrapper.pyx	2012-05-18 14:08:24 +0000
@@ -78,7 +78,8 @@
     int u1db_put_doc(u1database *db, u1db_document *doc)
     int u1db__put_doc_if_newer(u1database *db, u1db_document *doc,
                                int save_conflict, char *replica_uid,
-                               int replica_gen, int *state, int *at_gen)
+                               int replica_gen, char *replica_trans_id,
+                               int *state, int *at_gen)
     int u1db_resolve_doc(u1database *db, u1db_document *doc,
                          int n_revs, const_char_ptr *revs)
     int u1db_delete_doc(u1database *db, u1db_document *doc)
@@ -868,21 +869,25 @@
         return doc.rev
 
     def _put_doc_if_newer(self, CDocument doc, save_conflict, replica_uid=None,
-                          replica_gen=None):
-        cdef char *c_uid
+                          replica_gen=None, replica_trans_id=None):
+        cdef char *c_uid, *c_trans_id
         cdef int gen, state = 0, at_gen = -1
 
         if replica_uid is None:
             c_uid = NULL
         else:
             c_uid = replica_uid
+        if replica_trans_id is None:
+            c_trans_id = NULL
+        else:
+            c_trans_id = replica_trans_id
         if replica_gen is None:
             gen = 0
         else:
             gen = replica_gen
         handle_status("Failed to _put_doc_if_newer",
             u1db__put_doc_if_newer(self._db, doc._doc, save_conflict,
-                c_uid, gen, &state, &at_gen))
+                c_uid, gen, c_trans_id, &state, &at_gen))
         if state == U1DB_INSERTED:
             return 'inserted', at_gen
         elif state == U1DB_SUPERSEDED:

=== modified file 'u1db/tests/test_backends.py'
--- u1db/tests/test_backends.py	2012-05-18 14:08:24 +0000
+++ u1db/tests/test_backends.py	2012-05-18 14:08:24 +0000
@@ -301,22 +301,25 @@
                                   nested_doc)
         self.assertEqual('inserted',
             self.db._put_doc_if_newer(doc2, save_conflict=False,
-                                      replica_uid='other', replica_gen=2)[0])
-        self.assertEqual(2, self.db._get_sync_gen_info('other')[0])
+                                      replica_uid='other', replica_gen=2,
+                                      replica_trans_id='T-id2')[0])
+        self.assertEqual((2, 'T-id2'), self.db._get_sync_gen_info('other'))
         # Compare to the old rev, should be superseded
         doc2 = self.make_document(doc1.doc_id, doc1.rev, nested_doc)
         self.assertEqual('superseded',
             self.db._put_doc_if_newer(doc2, save_conflict=False,
-                                      replica_uid='other', replica_gen=3)[0])
-        self.assertEqual(3, self.db._get_sync_gen_info('other')[0])
+                                      replica_uid='other', replica_gen=3,
+                                      replica_trans_id='T-id3')[0])
+        self.assertEqual((3, 'T-id3'), self.db._get_sync_gen_info('other'))
         # A conflict that isn't saved still records the sync gen, because we
         # don't need to see it again
         doc2 = self.make_document(doc1.doc_id, doc1.rev + '|fourth:1',
                                   nested_doc)
         self.assertEqual('conflicted',
             self.db._put_doc_if_newer(doc2, save_conflict=False,
-                                      replica_uid='other', replica_gen=4)[0])
-        self.assertEqual(4, self.db._get_sync_gen_info('other')[0])
+                                      replica_uid='other', replica_gen=4,
+                                      replica_trans_id='T-id4')[0])
+        self.assertEqual((4, 'T-id4'), self.db._get_sync_gen_info('other'))
 
     def test__get_sync_gen_info(self):
         self.assertEqual((0, ''), self.db._get_sync_gen_info('other-db'))
@@ -579,18 +582,20 @@
 
     def test_put_doc_if_newer_replica_uid(self):
         doc1 = self.db.create_doc(simple_doc)
-        self.db._set_sync_info('other', 1, 'T-sid')
+        self.db._set_sync_info('other', 1, 'T-id')
         doc2 = self.make_document(doc1.doc_id, doc1.rev + '|other:1',
                                   nested_doc)
         self.db._put_doc_if_newer(doc2, save_conflict=True,
-                                  replica_uid='other', replica_gen=2)
+                                  replica_uid='other', replica_gen=2,
+                                  replica_trans_id='T-id2')
         # Conflict vs the current update
         doc2 = self.make_document(doc1.doc_id, doc1.rev + '|third:3',
                                   nested_doc)
         self.assertEqual('conflicted',
             self.db._put_doc_if_newer(doc2, save_conflict=True,
-                replica_uid='other', replica_gen=3)[0])
-        self.assertEqual(3, self.db._get_sync_gen_info('other')[0])
+                replica_uid='other', replica_gen=3,
+                replica_trans_id='T-id3')[0])
+        self.assertEqual((3, 'T-id3'), self.db._get_sync_gen_info('other'))
 
     def test_put_refuses_to_update_conflicted(self):
         doc1 = self.db.create_doc(simple_doc)

=== modified file 'u1db/tests/test_c_backend.py'
--- u1db/tests/test_c_backend.py	2012-05-18 14:08:24 +0000
+++ u1db/tests/test_c_backend.py	2012-05-18 14:08:24 +0000
@@ -201,7 +201,7 @@
         exc.insert_doc_from_source(doc, 10)
         self.assertGetDoc(self.db, 'doc-id', 'replica:1', tests.simple_doc,
                           False)
-        self.assertEqual((10, 'T-sid'),
+        self.assertEqual((10, None),
                          self.db._get_sync_gen_info('source-uid'))
         self.assertEqual(['doc-id'], exc.get_seen_ids())
 

=== modified file 'u1db/tests/test_https.py'
--- u1db/tests/test_https.py	2012-05-18 14:08:24 +0000
+++ u1db/tests/test_https.py	2012-05-18 14:08:24 +0000
@@ -96,7 +96,7 @@
         db = self.request_state._create_database('test')
         remote_target = self.getSyncTarget('localhost', 'test')
         try:
-            remote_target.record_sync_info('other-id', 2)
+            remote_target.record_sync_info('other-id', 2, 'T-id')
         except ssl.SSLError, e:
             self.assertIn("certificate verify failed", str(e))
         else:
@@ -111,7 +111,7 @@
         self.patch(http_client, 'CA_CERTS', self.cacert_pem)
         remote_target = self.getSyncTarget('127.0.0.1', 'test')
         self.assertRaises(http_client.CertificateError,
-                          remote_target.record_sync_info, 'other-id', 2)
+                          remote_target.record_sync_info, 'other-id', 2, 'T-id')
 
 
 load_tests = tests.load_with_scenarios

=== modified file 'u1db/tests/test_remote_sync_target.py'
--- u1db/tests/test_remote_sync_target.py	2012-05-18 14:08:24 +0000
+++ u1db/tests/test_remote_sync_target.py	2012-05-18 14:08:24 +0000
@@ -202,11 +202,13 @@
         _put_doc_if_newer = db._put_doc_if_newer
         trigger_ids = ['doc-here2']
         def bomb_put_doc_if_newer(doc, save_conflict,
-                                  replica_uid=None, replica_gen=None):
+                                  replica_uid=None, replica_gen=None,
+                                  replica_trans_id=None):
             if doc.doc_id in trigger_ids:
                 raise Exception
             return _put_doc_if_newer(doc, save_conflict=save_conflict,
-                replica_uid=replica_uid, replica_gen=replica_gen)
+                replica_uid=replica_uid, replica_gen=replica_gen,
+                replica_trans_id=replica_trans_id)
         self.patch(db, '_put_doc_if_newer', bomb_put_doc_if_newer)
         remote_target = self.getSyncTarget('test')
         other_changes = []
@@ -222,7 +224,7 @@
                 return_doc_cb=receive_doc)
         self.assertGetDoc(db, 'doc-here', 'replica:1', '{"value": "here"}',
                           False)
-        self.assertEqual((10, 'T-sid'), db._get_sync_gen_info('replica'))
+        self.assertEqual((10, None), db._get_sync_gen_info('replica'))
         self.assertEqual([], other_changes)
         # retry
         trigger_ids = []
@@ -232,7 +234,7 @@
                 return_doc_cb=receive_doc)
         self.assertGetDoc(db, 'doc-here2', 'replica:1', '{"value": "here2"}',
                           False)
-        self.assertEqual((11, 'T-sid'), db._get_sync_gen_info('replica'))
+        self.assertEqual((11, None), db._get_sync_gen_info('replica'))
         self.assertEqual(2, new_gen)
         # bounced back to us
         self.assertEqual([('doc-here', 'replica:1', '{"value": "here"}', 1)],