Merge lp:~thisfred/u1db/put_after_delete into lp:u1db

[Eric Casteleijn]

Changed conflict detection so that creating a new document with the id of a deleted one does not trigger a conflict.

Changed conflict detection so that creating a new document with the id of a deleted one does not trigger a conflict.

[lp:~thisfred/u1db/put_after_delete]

313. By Eric Casteleijn on 2012-05-25

check vectorclocks and add test for put

314. By Eric Casteleijn on 2012-05-25

merged lp:~thisfred/u1db/exclude_deleted_documents

[Eric Casteleijn (thisfred)]

added vector clock checks in tests and fixed bugs thus exposed

[lp:~thisfred/u1db/put_after_delete]

315. By Eric Casteleijn on 2012-05-29

expanded comment

316. By Eric Casteleijn on 2012-05-29

remerged trunk

Unmerged revisions

=== modified file 'include/u1db/u1db.h'
--- include/u1db/u1db.h	2012-05-23 17:33:57 +0000
+++ include/u1db/u1db.h	2012-05-25 19:26:18 +0000
@@ -147,7 +147,8 @@
  * @return status, will be U1DB_OK if there is no error, even if there is no
  *      document matching that doc_id.
  */
-int u1db_get_doc(u1database *db, const char *doc_id, u1db_document **doc);
+int u1db_get_doc(u1database *db, const char *doc_id, int include_deleted,
+                 u1db_document **doc);
 
 
 /**
@@ -164,7 +165,7 @@
  *           u1db_free_doc.
  */
 int u1db_get_docs(u1database *db, int n_doc_ids, const char **doc_ids,
-                  int check_for_conflicts, void *context,
+                  int check_for_conflicts, int include_deleted, void *context,
                   u1db_doc_callback cb);
 
 /**

=== modified file 'src/u1db.c'
--- src/u1db.c	2012-05-25 17:48:41 +0000
+++ src/u1db.c	2012-05-25 19:26:18 +0000
@@ -513,7 +513,7 @@
                         &old_content_len, &statement);
     if (status != SQLITE_OK) { goto finish; }
     if (doc->doc_rev == NULL) {
-        if (old_doc_rev == NULL) {
+        if (old_doc_rev == NULL || old_content == NULL) {
             // We are creating a new document from scratch. No problem.
             status = 0;
         } else {
@@ -941,7 +941,8 @@
 
 
 int
-u1db_get_doc(u1database *db, const char *doc_id, u1db_document **doc)
+u1db_get_doc(u1database *db, const char *doc_id, int include_deleted,
+             u1db_document **doc)
 {
     int status = 0, content_len = 0;
     sqlite3_stmt *statement;
@@ -959,12 +960,16 @@
             *doc = NULL;
             goto finish;
         }
-        *doc = u1db__allocate_document(doc_id, (const char*)doc_rev,
-                                       (const char*)content, 0);
+        if (content != NULL || include_deleted) {
+            *doc = u1db__allocate_document(doc_id, (const char*)doc_rev,
+                                        (const char*)content, 0);
 
-        if (*doc != NULL) {
-            status = lookup_conflict(db, (*doc)->doc_id,
-                                     &((*doc)->has_conflicts));
+            if (*doc != NULL) {
+                status = lookup_conflict(db, (*doc)->doc_id,
+                                        &((*doc)->has_conflicts));
+            }
+        } else {
+            *doc = NULL;
         }
     } else {
         *doc = NULL;
@@ -976,7 +981,8 @@
 
 int
 u1db_get_docs(u1database *db, int n_doc_ids, const char **doc_ids,
-              int check_for_conflicts, void *context, u1db_doc_callback cb)
+              int check_for_conflicts, int include_deleted,
+              void *context, u1db_doc_callback cb)
 {
     int status, i;
     sqlite3_stmt *statement;
@@ -1005,11 +1011,13 @@
             u1db_document *doc;
             revision = (char *)sqlite3_column_text(statement, 0);
             content = (char *)sqlite3_column_text(statement, 1);
-            doc = u1db__allocate_document(doc_ids[i], revision, content, 0);
-            if (check_for_conflicts) {
-                status = lookup_conflict(db, doc_ids[i], &(doc->has_conflicts));
+            if (content != NULL || include_deleted) {
+                doc = u1db__allocate_document(doc_ids[i], revision, content, 0);
+                if (check_for_conflicts) {
+                    status = lookup_conflict(db, doc_ids[i], &(doc->has_conflicts));
+                }
+                cb(context, doc);
             }
-            cb(context, doc);
         } else if (status == SQLITE_DONE) {
             // This document doesn't exist
             // TODO: I believe the python implementation returns the Null

=== modified file 'src/u1db_http_sync_target.c'
--- src/u1db_http_sync_target.c	2012-05-22 18:58:04 +0000
+++ src/u1db_http_sync_target.c	2012-05-25 19:26:18 +0000
@@ -921,7 +921,7 @@
 
 
 int u1db_get_docs(u1database *db, int n_doc_ids, const char **doc_ids,
-                  int check_for_conflicts, void *context,
+                  int check_for_conflicts, int include_deleted, void *context,
                   u1db_doc_callback cb);
 
 static int
@@ -951,7 +951,7 @@
     state.num = n_doc_ids;
     state.generations = generations;
     state.temp_fd = temp_fd;
-    status = u1db_get_docs(source_db, n_doc_ids, doc_ids, 0,
+    status = u1db_get_docs(source_db, n_doc_ids, doc_ids, 0, 1,
             &state, get_docs_to_tempfile);
     if (status != U1DB_OK) { goto finish; }
     status = finalize_and_send_temp_file(st, temp_fd, source_replica_uid, &req);

=== modified file 'src/u1db_query.c'
--- src/u1db_query.c	2012-05-23 10:57:02 +0000
+++ src/u1db_query.c	2012-05-25 19:26:18 +0000
@@ -591,7 +591,8 @@
         doc_id = (char*)sqlite3_column_text(statement, 0);
         // We use u1db_get_docs so we can pass check_for_conflicts=0, which is
         // currently expected by the test suite.
-        status = u1db_get_docs(db, 1, (const char**)&doc_id, 0, context, cb);
+        status = u1db_get_docs(
+            db, 1, (const char**)&doc_id, 0, 0, context, cb);
         if (status != U1DB_OK) { goto finish; }
         status = sqlite3_step(statement);
     }

=== modified file 'src/u1db_sync_target.c'
--- src/u1db_sync_target.c	2012-05-21 18:28:05 +0000
+++ src/u1db_sync_target.c	2012-05-25 19:26:18 +0000
@@ -422,7 +422,7 @@
     if (se->num_doc_ids > 0) {
         status = u1db_get_docs(se->db, se->num_doc_ids,
                 (const char **)se->doc_ids_to_return,
-                0, &state, get_docs_to_gen_docs);
+                0, 1, &state, get_docs_to_gen_docs);
     }
 finish:
     return status;
@@ -469,7 +469,7 @@
         (u1db_doc_gen_callback)u1db__sync_exchange_insert_doc_from_source;
     get_doc_state.gen_for_doc_ids = generations;
     return u1db_get_docs(source_db, n_doc_ids, doc_ids,
-            0, &get_doc_state, get_docs_to_gen_docs);
+            0, 1, &get_doc_state, get_docs_to_gen_docs);
 }
 
 

=== modified file 'u1db/__init__.py'
--- u1db/__init__.py	2012-05-23 13:11:46 +0000
+++ u1db/__init__.py	2012-05-25 19:26:18 +0000
@@ -68,20 +68,27 @@
         """
         raise NotImplementedError(self.whats_changed)
 
-    def get_doc(self, doc_id):
+    def get_doc(self, doc_id, include_deleted=False):
         """Get the JSON string for the given document.
 
         :param doc_id: The unique document identifier
+        :param include_deleted: If set to True, deleted documents will be
+            returned with empty content. Otherwise asking for a deleted
+            document will return None.
         :return: a Document object.
         """
         raise NotImplementedError(self.get_doc)
 
-    def get_docs(self, doc_ids, check_for_conflicts=True):
+    def get_docs(self, doc_ids, check_for_conflicts=True,
+                 include_deleted=False):
         """Get the JSON content for many documents.
 
         :param doc_ids: A list of document identifiers.
         :param check_for_conflicts: If set to False, then the conflict check
             will be skipped, and 'None' will be returned instead of True/False.
+        :param include_deleted: If set to True, deleted documents will be
+            returned with empty content. Otherwise deleted documents will not
+            be included in the results.
         :return: [Document] for each document id and matching doc_ids order.
         """
         raise NotImplementedError(self.get_docs)

=== modified file 'u1db/backends/__init__.py'
--- u1db/backends/__init__.py	2012-05-22 20:38:49 +0000
+++ u1db/backends/__init__.py	2012-05-25 19:26:18 +0000
@@ -81,10 +81,13 @@
     def _put_and_update_indexes(self, doc_id, old_doc, new_rev, content):
         raise NotImplementedError(self._put_and_update_indexes)
 
-    def get_docs(self, doc_ids, check_for_conflicts=True):
+    def get_docs(self, doc_ids, check_for_conflicts=True,
+                 include_deleted=False):
         result = []
         for doc_id in doc_ids:
             doc = self._get_doc(doc_id)
+            if doc.is_tombstone() and not include_deleted:
+                continue
             if check_for_conflicts:
                 doc.has_conflicts = self._has_conflicts(doc_id)
             result.append(doc)

=== modified file 'u1db/backends/inmemory.py'
--- u1db/backends/inmemory.py	2012-05-25 16:19:49 +0000
+++ u1db/backends/inmemory.py	2012-05-25 19:26:18 +0000
@@ -77,13 +77,16 @@
         if self._has_conflicts(doc.doc_id):
             raise errors.ConflictedDoc()
         old_doc = self._get_doc(doc.doc_id)
-        if old_doc is not None:
-            if old_doc.rev != doc.rev:
-                raise errors.RevisionConflict()
+        if old_doc and doc.rev is None and old_doc.is_tombstone():
+            new_rev = self._allocate_doc_rev(old_doc.rev)
         else:
-            if doc.rev is not None:
-                raise errors.RevisionConflict()
-        new_rev = self._allocate_doc_rev(doc.rev)
+            if old_doc is not None:
+                if old_doc.rev != doc.rev:
+                    raise errors.RevisionConflict()
+            else:
+                if doc.rev is not None:
+                    raise errors.RevisionConflict()
+            new_rev = self._allocate_doc_rev(doc.rev)
         doc.rev = new_rev
         self._put_and_update_indexes(old_doc, doc)
         return new_rev
@@ -108,10 +111,12 @@
     def _has_conflicts(self, doc_id):
         return doc_id in self._conflicts
 
-    def get_doc(self, doc_id):
+    def get_doc(self, doc_id, include_deleted=False):
         doc = self._get_doc(doc_id)
         if doc is None:
             return None
+        if doc.is_tombstone() and not include_deleted:
+            return None
         doc.has_conflicts = (doc.doc_id in self._conflicts)
         return doc
 

=== modified file 'u1db/backends/sqlite_backend.py'
--- u1db/backends/sqlite_backend.py	2012-05-25 16:19:49 +0000
+++ u1db/backends/sqlite_backend.py	2012-05-25 19:26:18 +0000
@@ -291,10 +291,12 @@
         else:
             return True
 
-    def get_doc(self, doc_id):
+    def get_doc(self, doc_id, include_deleted=False):
         doc = self._get_doc(doc_id)
         if doc is None:
             return None
+        if doc.is_tombstone() and not include_deleted:
+            return None
         # TODO: A doc which appears deleted could still have conflicts...
         doc.has_conflicts = self._has_conflicts(doc.doc_id)
         return doc
@@ -307,13 +309,16 @@
             if self._has_conflicts(doc.doc_id):
                 raise errors.ConflictedDoc()
             old_doc = self._get_doc(doc.doc_id)
-            if old_doc is not None:
-                if old_doc.rev != doc.rev:
-                    raise errors.RevisionConflict()
+            if old_doc and doc.rev is None and old_doc.is_tombstone():
+                new_rev = self._allocate_doc_rev(old_doc.rev)
             else:
-                if doc.rev is not None:
-                    raise errors.RevisionConflict()
-            new_rev = self._allocate_doc_rev(doc.rev)
+                if old_doc is not None:
+                        if old_doc.rev != doc.rev:
+                            raise errors.RevisionConflict()
+                else:
+                    if doc.rev is not None:
+                        raise errors.RevisionConflict()
+                new_rev = self._allocate_doc_rev(doc.rev)
             doc.rev = new_rev
             self._put_and_update_indexes(old_doc, doc)
         return new_rev

=== modified file 'u1db/remote/http_app.py'
--- u1db/remote/http_app.py	2012-05-23 13:11:46 +0000
+++ u1db/remote/http_app.py	2012-05-25 19:26:18 +0000
@@ -38,6 +38,13 @@
     )
 
 
+def parse_bool(expression):
+    """Parse boolean querystring parameter."""
+    if expression == 'true':
+        return True
+    return False
+
+
 class BadRequest(Exception):
     """Bad request."""
 
@@ -89,7 +96,8 @@
 
 
 def http_method(**control):
-    """Decoration for handling of query arguments and content for a HTTP method.
+    """Decoration for handling of query arguments and content for a HTTP
+       method.
 
        args and content here are the query arguments and body of the incoming
        HTTP requests.
@@ -118,6 +126,7 @@
     content_as_args = control.pop('content_as_args', False)
     no_query = control.pop('no_query', False)
     conversions = control.items()
+
     def wrap(f):
         argspec = inspect.getargspec(f)
         assert argspec.args[0] == "self"
@@ -125,6 +134,7 @@
         ndefaults = len(argspec.defaults or ())
         required_args = set(argspec.args[1:nargs - ndefaults])
         all_args = set(argspec.args)
+
         @functools.wraps(f)
         def wrapper(self, args, content):
             if no_query and args:
@@ -147,7 +157,9 @@
                 except ValueError:
                     raise BadRequest()
             return f(self, **args)
+
         return wrapper
+
     return wrap
 
 
@@ -243,9 +255,9 @@
         self.db.delete_doc(doc)
         self.responder.send_response_json(200, rev=doc.rev)
 
-    @http_method()
-    def get(self):
-        doc = self.db.get_doc(self.id)
+    @http_method(include_deleted=parse_bool)
+    def get(self, include_deleted=False):
+        doc = self.db.get_doc(self.id, include_deleted=include_deleted)
         if doc is None:
             wire_descr = errors.DocumentDoesNotExist.wire_description
             self.responder.send_response_json(
@@ -262,7 +274,8 @@
             }
         if doc.is_tombstone():
             self.responder.send_response_json(
-               http_errors.wire_description_to_status[errors.DOCUMENT_DELETED],
+               http_errors.wire_description_to_status[
+                   errors.DOCUMENT_DELETED],
                error=errors.DOCUMENT_DELETED,
                headers=headers)
         else:
@@ -288,11 +301,11 @@
     @http_method()
     def get(self):
         result = self.target.get_sync_info(self.source_replica_uid)
-        self.responder.send_response_json(target_replica_uid=result[0],
-                                     target_replica_generation=result[1],
-                                     source_replica_uid=self.source_replica_uid,
-                                     source_replica_generation=result[2],
-                                     source_transaction_id=result[3])
+        self.responder.send_response_json(
+            target_replica_uid=result[0], target_replica_generation=result[1],
+            source_replica_uid=self.source_replica_uid,
+            source_replica_generation=result[2],
+            source_transaction_id=result[3])
 
     @http_method(generation=int,
                  content_as_args=True, no_query=True)
@@ -421,7 +434,8 @@
         args = urlparse.parse_qsl(self.environ['QUERY_STRING'],
                                   strict_parsing=False)
         try:
-            args = dict((k.decode('utf-8'), v.decode('utf-8')) for k, v in args)
+            args = dict(
+                (k.decode('utf-8'), v.decode('utf-8')) for k, v in args)
         except ValueError:
             raise BadRequest()
         method = self.environ['REQUEST_METHOD'].lower()
@@ -433,7 +447,7 @@
             # to support chunked enconding
             try:
                 content_length = int(self.environ['CONTENT_LENGTH'])
-            except (ValueError, KeyError), e:
+            except (ValueError, KeyError):
                 raise BadRequest
             if content_length <= 0:
                 raise BadRequest
@@ -486,7 +500,8 @@
         resource_cls, params = url_to_resource.match(environ['PATH_INFO'])
         if resource_cls is None:
             raise BadRequest  # 404 instead?
-        resource = resource_cls(state=self.state, responder=responder, **params)
+        resource = resource_cls(
+            state=self.state, responder=responder, **params)
         return resource
 
     def __call__(self, environ, start_response):

=== modified file 'u1db/remote/http_client.py'
--- u1db/remote/http_client.py	2012-05-11 13:40:45 +0000
+++ u1db/remote/http_client.py	2012-05-25 19:26:18 +0000
@@ -42,6 +42,16 @@
 CA_CERTS = "/etc/ssl/certs/ca-certificates.crt"
 
 
+def _encode_query_parameter(value):
+    """Encode query parameter."""
+    if isinstance(value, bool):
+        if value:
+            value = 'true'
+        else:
+            value = 'false'
+    return unicode(value).encode('utf-8')
+
+
 class _VerifiedHTTPSConnection(httplib.HTTPSConnection):
     """HTTPSConnection verifying server side certificates."""
     # derived from httplib.py
@@ -87,8 +97,9 @@
 
     def set_oauth_credentials(self, consumer_key, consumer_secret,
                               token_key, token_secret):
-        self._oauth_creds = (oauth.OAuthConsumer(consumer_key, consumer_secret),
-                             oauth.OAuthToken(token_key, token_secret))
+        self._oauth_creds = (
+            oauth.OAuthConsumer(consumer_key, consumer_secret),
+            oauth.OAuthToken(token_key, token_secret))
 
     def _ensure_connection(self):
         if self._conn is not None:
@@ -142,7 +153,8 @@
                 parameters=params,
                 http_url=full_url
                 )
-            oauth_req.sign_request(self.oauth_signature_method, consumer, token)
+            oauth_req.sign_request(
+                self.oauth_signature_method, consumer, token)
             # Authorization: OAuth ...
             return oauth_req.to_header().items()
         else:
@@ -160,18 +172,20 @@
                                   for part in url_parts)
             # oauth performs its own quoting
             unquoted_url += '/'.join(url_parts)
+        encoded_params = {}
         if params:
-            params = dict((unicode(v).encode('utf-8'),
-                           unicode(k).encode('utf-8'))
-                                                   for v, k in params.items())
-            url_query += ('?' + urllib.urlencode(params))
+            for key, value in params.items():
+                key = unicode(key).encode('utf-8')
+                encoded_params[key] = _encode_query_parameter(value)
+            url_query += ('?' + urllib.urlencode(encoded_params))
         if body is not None and not isinstance(body, basestring):
             body = simplejson.dumps(body)
             content_type = 'application/json'
         headers = {}
         if content_type:
             headers['content-type'] = content_type
-        headers.update(self._sign_request(method, unquoted_url, params))
+        headers.update(
+            self._sign_request(method, unquoted_url, encoded_params))
         self._conn.request(method, url_query, body, headers)
         return self._response()
 

=== modified file 'u1db/remote/http_database.py'
--- u1db/remote/http_database.py	2012-05-23 13:11:46 +0000
+++ u1db/remote/http_database.py	2012-05-25 19:26:18 +0000
@@ -76,9 +76,10 @@
         doc.rev = res['rev']
         return res['rev']
 
-    def get_doc(self, doc_id):
+    def get_doc(self, doc_id, include_deleted=False):
         try:
-            res, headers = self._request('GET', ['doc', doc_id])
+            res, headers = self._request(
+                'GET', ['doc', doc_id], {"include_deleted": include_deleted})
         except errors.DocumentDoesNotExist:
             return None
         except errors.HTTPError, e:

=== modified file 'u1db/sync.py'
--- u1db/sync.py	2012-05-18 15:12:42 +0000
+++ u1db/sync.py	2012-05-25 19:26:18 +0000
@@ -95,19 +95,21 @@
         # get target identifier, its current generation,
         # and its last-seen database generation for this source
         (self.target_replica_uid, target_gen, target_my_gen,
-         target_my_trans_id) = sync_target.get_sync_info(self.source._replica_uid)
+         target_my_trans_id) = sync_target.get_sync_info(
+             self.source._replica_uid)
         # what's changed since that generation and this current gen
         my_gen, _, changes = self.source.whats_changed(target_my_gen)
 
         # this source last-seen database generation for the target
-        target_last_known_gen, target_trans_id = self.source._get_sync_gen_info(
-            self.target_replica_uid)
+        (target_last_known_gen,
+         target_trans_id) = self.source._get_sync_gen_info(
+             self.target_replica_uid)
         if not changes and target_last_known_gen == target_gen:
             return my_gen
         changed_doc_ids = [doc_id for doc_id, _, _ in changes]
         # prepare to send all the changed docs
         docs_to_send = self.source.get_docs(changed_doc_ids,
-            check_for_conflicts=False)
+            check_for_conflicts=False, include_deleted=True)
         docs_by_generation = zip(docs_to_send, (gen for _, gen, _ in changes))
 
         # exchange documents and try to insert the returned ones with
@@ -229,7 +231,8 @@
         # return docs, including conflicts
         changed_doc_ids = [doc_id for doc_id, _ in changes_to_return]
         self._trace('before get_docs')
-        docs = self._db.get_docs(changed_doc_ids, check_for_conflicts=False)
+        docs = self._db.get_docs(
+            changed_doc_ids, check_for_conflicts=False, include_deleted=True)
 
         docs_by_gen = izip(docs, (gen for _, gen in changes_to_return))
         for doc, gen in docs_by_gen:

=== modified file 'u1db/tests/__init__.py'
--- u1db/tests/__init__.py	2012-05-22 18:58:04 +0000
+++ u1db/tests/__init__.py	2012-05-25 19:26:18 +0000
@@ -71,6 +71,13 @@
                                      has_conflicts=has_conflicts)
         self.assertEqual(exp_doc, db.get_doc(doc_id))
 
+    def assertGetDocIncludeDeleted(self, db, doc_id, doc_rev, content,
+                                   has_conflicts):
+        """Assert that the document in the database looks correct."""
+        exp_doc = self.make_document(doc_id, doc_rev, content,
+                                     has_conflicts=has_conflicts)
+        self.assertEqual(exp_doc, db.get_doc(doc_id, include_deleted=True))
+
     def assertGetDocConflicts(self, db, doc_id, conflicts):
         """Assert what conflicts are stored for a given doc_id.
 

=== modified file 'u1db/tests/c_backend_wrapper.pyx'
--- u1db/tests/c_backend_wrapper.pyx	2012-05-23 17:33:57 +0000
+++ u1db/tests/c_backend_wrapper.pyx	2012-05-25 19:26:18 +0000
@@ -71,10 +71,11 @@
     int u1db_create_doc(u1database *db, char *json, char *doc_id,
                         u1db_document **doc)
     int u1db_delete_doc(u1database *db, u1db_document *doc)
-    int u1db_get_doc(u1database *db, char *doc_id, u1db_document **doc)
+    int u1db_get_doc(u1database *db, char *doc_id, int include_deleted,
+                     u1db_document **doc)
     int u1db_get_docs(u1database *db, int n_doc_ids, const_char_ptr *doc_ids,
-                      int check_for_conflicts, void *context,
-                      u1db_doc_callback cb)
+                      int check_for_conflicts, int include_deleted,
+                      void *context, u1db_doc_callback cb)
     int u1db_put_doc(u1database *db, u1db_document *doc)
     int u1db__put_doc_if_newer(u1database *db, u1db_document *doc,
                                int save_conflict, char *replica_uid,
@@ -907,30 +908,29 @@
         else:
             raise RuntimeError("Unknown _put_doc_if_newer state: %d" % (state,))
 
-    def get_doc(self, doc_id):
+    def get_doc(self, doc_id, include_deleted=False):
         cdef u1db_document *doc = NULL
-
+        deleted = 1 if include_deleted else 0
         handle_status("get_doc failed",
-            u1db_get_doc(self._db, doc_id, &doc))
+            u1db_get_doc(self._db, doc_id, deleted, &doc))
         if doc == NULL:
             return None
         pydoc = CDocument()
         pydoc._doc = doc
         return pydoc
 
-    def get_docs(self, doc_ids, check_for_conflicts=True):
+    def get_docs(self, doc_ids, check_for_conflicts=True,
+                 include_deleted=False):
         cdef int n_doc_ids, conflicts
         cdef const_char_ptr *c_doc_ids
 
         _list_to_array(doc_ids, &c_doc_ids, &n_doc_ids)
-        if check_for_conflicts:
-            conflicts = 1
-        else:
-            conflicts = 0
+        deleted = 1 if include_deleted else 0
+        conflicts = 1 if check_for_conflicts else 0
         a_list = []
         handle_status("get_docs",
             u1db_get_docs(self._db, n_doc_ids, c_doc_ids,
-                conflicts, <void*>a_list, _append_doc_to_list))
+                conflicts, deleted, <void*>a_list, _append_doc_to_list))
         free(<void*>c_doc_ids)
         return a_list
 

=== modified file 'u1db/tests/commandline/test_client.py'
--- u1db/tests/commandline/test_client.py	2012-05-24 14:50:19 +0000
+++ u1db/tests/commandline/test_client.py	2012-05-25 19:26:18 +0000
@@ -50,7 +50,7 @@
         # parsing.
         try:
             return self.parser.parse_args(args)
-        except SystemExit, e:
+        except SystemExit:
             raise AssertionError('got SystemExit')
 
     def test_create(self):
@@ -203,7 +203,7 @@
         doc = self.db.create_doc(tests.simple_doc)
         cmd = self.make_command(client.CmdDelete)
         cmd.run(self.db_path, doc.doc_id, doc.rev)
-        doc2 = self.db.get_doc(doc.doc_id)
+        doc2 = self.db.get_doc(doc.doc_id, include_deleted=True)
         self.assertEqual(doc.doc_id, doc2.doc_id)
         self.assertNotEqual(doc.rev, doc2.rev)
         self.assertIs(None, doc2.get_json())
@@ -390,7 +390,7 @@
         cmd = self.make_command(client.CmdCreateIndex)
         retval = cmd.run(self.db_path, "foo", ["bar", "baz"])
         self.assertEqual(self.db.list_indexes(), [('foo', ['bar', "baz"])])
-        self.assertEqual(retval, None) # conveniently mapped to 0
+        self.assertEqual(retval, None)  # conveniently mapped to 0
         self.assertEqual(cmd.stdout.getvalue(), '')
         self.assertEqual(cmd.stderr.getvalue(), '')
 
@@ -688,7 +688,7 @@
         doc = self.db.create_doc(tests.simple_doc, doc_id='test-id')
         ret, stdout, stderr = self.run_main(
             ['delete', self.db_path, 'test-id', doc.rev])
-        doc = self.db.get_doc('test-id')
+        doc = self.db.get_doc('test-id', include_deleted=True)
         self.assertEqual(0, ret)
         self.assertEqual('', stdout)
         self.assertEqual('rev: %s\n' % (doc.rev,), stderr)
@@ -696,7 +696,7 @@
     def test_init_db(self):
         path = self.working_dir + '/test2.db'
         ret, stdout, stderr = self.run_main(['init-db', path])
-        db2 = u1db_open(path, create=False)
+        u1db_open(path, create=False)
 
     def test_put(self):
         doc = self.db.create_doc(tests.simple_doc, doc_id='test-id')
@@ -720,7 +720,8 @@
         self.assertEqual(0, ret)
         self.assertEqual('', stdout)
         self.assertEqual('', stderr)
-        self.assertGetDoc(self.db2, 'test-id', doc.rev, tests.simple_doc, False)
+        self.assertGetDoc(
+            self.db2, 'test-id', doc.rev, tests.simple_doc, False)
 
 
 class TestHTTPIntegration(tests.TestCaseWithServer, RunMainHelper):
@@ -743,7 +744,7 @@
     def test_init_db(self):
         url = self.getURL('new.db')
         ret, stdout, stderr = self.run_main(['init-db', url])
-        db2 = u1db_open(self.getPath('new.db'), create=False)
+        u1db_open(self.getPath('new.db'), create=False)
 
     def test_create_get_put_delete(self):
         db = u1db_open(self.getPath('test.db'), create=True)
@@ -766,4 +767,4 @@
         self.assertEqual(0, ret)
         self.assertTrue(stderr.startswith('rev: '))
         doc_rev2 = stderr[len('rev: '):].rstrip()
-        self.assertGetDoc(db, doc_id, doc_rev2, None, False)
+        self.assertGetDocIncludeDeleted(db, doc_id, doc_rev2, None, False)

=== modified file 'u1db/tests/test_backends.py'
--- u1db/tests/test_backends.py	2012-05-25 16:19:49 +0000
+++ u1db/tests/test_backends.py	2012-05-25 19:26:18 +0000
@@ -152,6 +152,31 @@
         self.assertRaises(errors.RevisionConflict, self.db.put_doc, bad_doc)
         self.assertGetDoc(self.db, 'my_doc_id', old_rev, simple_doc, False)
 
+    def test_create_succeeds_after_delete(self):
+        doc = self.db.create_doc(simple_doc, doc_id='my_doc_id')
+        self.db.delete_doc(doc)
+        deleted_doc = self.db.get_doc('my_doc_id', include_deleted=True)
+        deleted_vc = vectorclock.VectorClockRev(deleted_doc.rev)
+        new_doc = self.db.create_doc(simple_doc, doc_id='my_doc_id')
+        self.assertGetDoc(self.db, 'my_doc_id', new_doc.rev, simple_doc, False)
+        new_vc = vectorclock.VectorClockRev(new_doc.rev)
+        self.assertTrue(
+            new_vc.is_newer(deleted_vc),
+            "%s does not supersede %s" % (new_doc.rev, deleted_doc.rev))
+
+    def test_put_succeeds_after_delete(self):
+        doc = self.db.create_doc(simple_doc, doc_id='my_doc_id')
+        self.db.delete_doc(doc)
+        deleted_doc = self.db.get_doc('my_doc_id', include_deleted=True)
+        deleted_vc = vectorclock.VectorClockRev(deleted_doc.rev)
+        doc2 = self.make_document('my_doc_id', None, simple_doc)
+        self.db.put_doc(doc2)
+        self.assertGetDoc(self.db, 'my_doc_id', doc2.rev, simple_doc, False)
+        new_vc = vectorclock.VectorClockRev(doc2.rev)
+        self.assertTrue(
+            new_vc.is_newer(deleted_vc),
+            "%s does not supersede %s" % (doc2.rev, deleted_doc.rev))
+
     def test_get_doc_after_put(self):
         doc = self.db.create_doc(simple_doc, doc_id='my_doc_id')
         self.assertGetDoc(self.db, 'my_doc_id', doc.rev, simple_doc, False)
@@ -159,6 +184,17 @@
     def test_get_doc_nonexisting(self):
         self.assertIs(None, self.db.get_doc('non-existing'))
 
+    def test_get_doc_deleted(self):
+        doc = self.db.create_doc(simple_doc, doc_id='my_doc_id')
+        self.db.delete_doc(doc)
+        self.assertIs(None, self.db.get_doc('my_doc_id'))
+
+    def test_get_doc_include_deleted(self):
+        doc = self.db.create_doc(simple_doc, doc_id='my_doc_id')
+        self.db.delete_doc(doc)
+        self.assertGetDocIncludeDeleted(
+            self.db, doc.doc_id, doc.rev, None, False)
+
     def test_handles_nested_content(self):
         doc = self.db.create_doc(nested_doc)
         self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False)
@@ -173,8 +209,9 @@
         orig_rev = doc.rev
         self.db.delete_doc(doc)
         self.assertNotEqual(orig_rev, doc.rev)
-        self.assertGetDoc(self.db, doc.doc_id, doc.rev, None, False)
-        self.assertIsNot(None, self.db.get_doc(doc.doc_id))
+        self.assertGetDocIncludeDeleted(
+            self.db, doc.doc_id, doc.rev, None, False)
+        self.assertIs(None, self.db.get_doc(doc.doc_id))
 
     def test_delete_doc_non_existant(self):
         doc = self.make_document('non-existing', 'other:1', simple_doc)
@@ -186,7 +223,8 @@
         self.db.delete_doc(doc)
         self.assertRaises(errors.DocumentAlreadyDeleted,
                           self.db.delete_doc, doc)
-        self.assertGetDoc(self.db, doc.doc_id, doc.rev, None, False)
+        self.assertGetDocIncludeDeleted(
+            self.db, doc.doc_id, doc.rev, None, False)
 
     def test_delete_doc_bad_rev(self):
         doc1 = self.db.create_doc(simple_doc)
@@ -216,7 +254,8 @@
     def test_delete_then_put(self):
         doc = self.db.create_doc(simple_doc)
         self.db.delete_doc(doc)
-        self.assertGetDoc(self.db, doc.doc_id, doc.rev, None, False)
+        self.assertGetDocIncludeDeleted(
+            self.db, doc.doc_id, doc.rev, None, False)
         doc.set_json(nested_doc)
         self.db.put_doc(doc)
         self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False)
@@ -242,6 +281,20 @@
         self.assertEqual([doc1, doc2],
                          self.db.get_docs([doc1.doc_id, doc2.doc_id]))
 
+    def test_get_docs_deleted(self):
+        doc1 = self.db.create_doc(simple_doc)
+        doc2 = self.db.create_doc(nested_doc)
+        self.db.delete_doc(doc1)
+        self.assertEqual([doc2], self.db.get_docs([doc1.doc_id, doc2.doc_id]))
+
+    def test_get_docs_include_deleted(self):
+        doc1 = self.db.create_doc(simple_doc)
+        doc2 = self.db.create_doc(nested_doc)
+        self.db.delete_doc(doc1)
+        self.assertEqual(
+            [doc1, doc2],
+            self.db.get_docs([doc1.doc_id, doc2.doc_id], include_deleted=True))
+
     def test_get_docs_request_ordered(self):
         doc1 = self.db.create_doc(simple_doc)
         doc2 = self.db.create_doc(nested_doc)
@@ -264,7 +317,8 @@
         doc = self.make_document('my-doc-id', 'test:2', None)
         state_at_gen = self.db._put_doc_if_newer(doc, save_conflict=False)
         self.assertEqual(('inserted', 2), state_at_gen)
-        self.assertGetDoc(self.db, 'my-doc-id', 'test:2', None, False)
+        self.assertGetDocIncludeDeleted(
+            self.db, 'my-doc-id', 'test:2', None, False)
 
     def test_put_doc_if_newer_already_superseded(self):
         orig_doc = '{"new": "doc"}'
@@ -518,7 +572,8 @@
                                     (doc1.rev, None)])
         self.db.resolve_doc(doc1, [doc1.rev, doc2.rev])
         self.assertGetDocConflicts(self.db, doc1.doc_id, [])
-        self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, None, False)
+        self.assertGetDocIncludeDeleted(
+            self.db, doc1.doc_id, doc1.rev, None, False)
 
     def test_put_doc_if_newer_save_conflicted(self):
         doc1 = self.db.create_doc(simple_doc)

=== modified file 'u1db/tests/test_http_app.py'
--- u1db/tests/test_http_app.py	2012-05-22 18:58:04 +0000
+++ u1db/tests/test_http_app.py	2012-05-25 19:26:18 +0000
@@ -292,7 +292,8 @@
         res = invoke()
         self.assertEqual('Put/end', res)
         self.assertEqual({'a': '1', 'b': 2}, resource.args)
-        self.assertEqual(['{"entry": "x"}', '{"entry": "y"}'], resource.entries)
+        self.assertEqual(
+            ['{"entry": "x"}', '{"entry": "y"}'], resource.entries)
         self.assertEqual(['a', 's', 's', 'e'], resource.order)
 
     def _put_sync_stream(self, body):
@@ -303,7 +304,7 @@
                    'CONTENT_TYPE': 'application/x-u1db-sync-stream'}
         invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ,
                                                          parameters)
-        res = invoke()
+        invoke()
 
     def test_put_sync_stream_wrong_start(self):
         self.assertRaises(http_app.BadRequest,
@@ -362,9 +363,11 @@
                    'wsgi.input': StringIO.StringIO('{}'),
                    'CONTENT_LENGTH': '10000',
                    'CONTENT_TYPE': 'text/plain'}
+
         class params:
             max_request_size = 5000
             max_entry_size = sys.maxint  # we don't get to use this
+
         invoke = http_app.HTTPInvocationByMethodWithBody(resource, environ,
                                                          params)
         self.assertRaises(http_app.BadRequest, invoke)
@@ -430,8 +433,10 @@
         self.status = status
         self.headers = dict(headers)
         self.response_body = []
+
         def write(data):
             self.response_body.append(data)
+
         return write
 
     def test_send_response_content_w_headers(self):
@@ -621,7 +626,7 @@
     def test_delete_doc(self):
         doc = self.db0.create_doc('{"x": 1}', doc_id='doc1')
         resp = self.app.delete('/db0/doc/doc1?old_rev=%s' % doc.rev)
-        doc = self.db0.get_doc('doc1')
+        doc = self.db0.get_doc('doc1', include_deleted=True)
         self.assertEqual(None, doc.content)
         self.assertEqual(200, resp.status)
         self.assertEqual('application/json', resp.header('content-type'))
@@ -651,8 +656,30 @@
         resp = self.app.get('/db0/doc/doc1', expect_errors=True)
         self.assertEqual(404, resp.status)
         self.assertEqual('application/json', resp.header('content-type'))
-        self.assertEqual({"error": errors.DOCUMENT_DELETED},
-                         simplejson.loads(resp.body))
+        self.assertEqual(
+            {"error": errors.DocumentDoesNotExist.wire_description},
+            simplejson.loads(resp.body))
+
+    def test_get_doc_deleted_explicit_exclude(self):
+        doc = self.db0.create_doc('{"x": 1}', doc_id='doc1')
+        self.db0.delete_doc(doc)
+        resp = self.app.get(
+            '/db0/doc/doc1?include_deleted=false', expect_errors=True)
+        self.assertEqual(404, resp.status)
+        self.assertEqual('application/json', resp.header('content-type'))
+        self.assertEqual(
+            {"error": errors.DocumentDoesNotExist.wire_description},
+            simplejson.loads(resp.body))
+
+    def test_get_deleted_doc(self):
+        doc = self.db0.create_doc('{"x": 1}', doc_id='doc1')
+        self.db0.delete_doc(doc)
+        resp = self.app.get(
+            '/db0/doc/doc1?include_deleted=true', expect_errors=True)
+        self.assertEqual(404, resp.status)
+        self.assertEqual('application/json', resp.header('content-type'))
+        self.assertEqual(
+            {"error": errors.DOCUMENT_DELETED}, simplejson.loads(resp.body))
         self.assertEqual(doc.rev, resp.header('x-u1db-rev'))
         self.assertEqual('false', resp.header('x-u1db-has-conflicts'))
 
@@ -695,6 +722,7 @@
 
         gens = []
         _do_set_sync_info = self.db0._do_set_sync_info
+
         def set_sync_generation_witness(other_uid, other_gen, other_trans_id):
             gens.append((other_uid, other_gen))
             _do_set_sync_info(other_uid, other_gen, other_trans_id)
@@ -767,8 +795,10 @@
     def test_sync_exchange_error_in_stream(self):
         args = dict(last_known_generation=0)
         body = "[\r\n%s\r\n]" % simplejson.dumps(args)
+
         def boom(self, return_doc_cb):
             raise errors.Unavailable
+
         self.patch(sync.SyncExchange, 'return_docs',
                    boom)
         resp = self.app.post('/db0/sync-from/replica',
@@ -797,32 +827,38 @@
 
     def test_begin_and_done(self):
         calls = []
+
         def begin(environ):
             self.assertTrue('PATH_INFO' in environ)
             calls.append('begin')
+
         def done(environ):
             self.assertTrue('PATH_INFO' in environ)
             calls.append('done')
+
         self.http_app.request_begin = begin
         self.http_app.request_done = done
 
         doc = self.db0.create_doc('{"x": 1}', doc_id='doc1')
-        resp = self.app.get('/db0/doc/%s' % doc.doc_id)
+        self.app.get('/db0/doc/%s' % doc.doc_id)
 
         self.assertEqual(['begin', 'done'], calls)
 
     def test_bad_request(self):
         calls = []
+
         def begin(environ):
             self.assertTrue('PATH_INFO' in environ)
             calls.append('begin')
+
         def bad_request(environ):
             self.assertTrue('PATH_INFO' in environ)
             calls.append('bad-request')
+
         self.http_app.request_begin = begin
         self.http_app.request_bad_request = bad_request
         # shouldn't be called
-        self.http_app.request_done = lambda env: borken
+        self.http_app.request_done = lambda env: 1 / 0
 
         resp = self.app.put('/db0/foo/doc1', params='{"x": 1}',
                             headers={'content-type': 'application/json'},
@@ -843,6 +879,7 @@
         super(TestHTTPAppErrorHandling, self).setUp()
         self.exc = None
         self.state = tests.ServerStateForTests()
+
         class ErroringResource(object):
 
             def post(_, args, content):
@@ -887,16 +924,19 @@
 
     def test_generic_u1db_errors_hooks(self):
         calls = []
+
         def begin(environ):
             self.assertTrue('PATH_INFO' in environ)
             calls.append('begin')
+
         def u1db_error(environ, exc):
             self.assertTrue('PATH_INFO' in environ)
             calls.append(('error', exc))
+
         self.http_app.request_begin = begin
         self.http_app.request_u1db_error = u1db_error
         # shouldn't be called
-        self.http_app.request_done = lambda env: borken
+        self.http_app.request_done = lambda env: 1 / 0
 
         self.exc = errors.U1DBError()
         resp = self.app.post('/req', params='{}',
@@ -916,15 +956,18 @@
         class Failure(Exception):
             pass
         calls = []
+
         def begin(environ):
             calls.append('begin')
+
         def failed(environ):
             self.assertTrue('PATH_INFO' in environ)
             calls.append(('failed', sys.exc_info()))
+
         self.http_app.request_begin = begin
         self.http_app.request_failed = failed
         # shouldn't be called
-        self.http_app.request_done = lambda env: borken
+        self.http_app.request_done = lambda env: 1 / 0
 
         self.exc = Failure()
         self.assertRaises(Failure, self.app.post, '/req', params='{}',
@@ -945,9 +988,11 @@
         self.state.ensure_database('foo')
 
     def test_plugging(self):
+
         class MySyncExchange(object):
             def __init__(self, db, source_replica_uid, last_known_generation):
                 pass
+
         class MySyncResource(http_app.SyncResource):
             sync_exchange_class = MySyncExchange
 

=== modified file 'u1db/tests/test_http_client.py'
--- u1db/tests/test_http_client.py	2012-04-25 12:14:42 +0000
+++ u1db/tests/test_http_client.py	2012-05-25 19:26:18 +0000
@@ -29,6 +29,18 @@
     )
 
 
+class TestEncoder(tests.TestCase):
+
+    def test_encode_string(self):
+        self.assertEqual("foo", http_client._encode_query_parameter("foo"))
+
+    def test_encode_true(self):
+        self.assertEqual("true", http_client._encode_query_parameter(True))
+
+    def test_encode_false(self):
+        self.assertEqual("false", http_client._encode_query_parameter(False))
+
+
 class TestHTTPClientBase(tests.TestCaseWithServer):
 
     def app(self, environ, start_response):
@@ -44,7 +56,8 @@
             return [simplejson.dumps(ret)]
         elif environ['PATH_INFO'].endswith('error'):
             content_length = int(environ['CONTENT_LENGTH'])
-            error = simplejson.loads(environ['wsgi.input'].read(content_length))
+            error = simplejson.loads(
+                environ['wsgi.input'].read(content_length))
             response = error['response']
             # In debug mode, wsgiref has an assertion that the status parameter
             # is a 'str' object. However error['status'] returns a unicode
@@ -69,7 +82,8 @@
             oauth_server = oauth.OAuthServer(tests.testingOAuthStore)
             oauth_server.add_signature_method(tests.sign_meth_HMAC_SHA1)
             try:
-                consumer, token, params = oauth_server.verify_request(oauth_req)
+                consumer, token, params = oauth_server.verify_request(
+                    oauth_req)
             except oauth.OAuthError, e:
                 start_response("401 Unauthorized",
                                [('Content-Type', 'application/json')])
@@ -85,9 +99,11 @@
             srv = simple_server.WSGIServer(host_port, handler)
             srv.set_app(self.app)
             return srv
+
         class req_handler(simple_server.WSGIRequestHandler):
             def log_request(*args):
                 pass  # suppress
+
         return make_server, req_handler, "shutdown", "http"
 
     def getClient(self):
@@ -153,7 +169,8 @@
 
     def test__request_json(self):
         cli = self.getClient()
-        res, headers = cli._request_json('POST', ['echo'], {'b': 2}, {'a': 'x'})
+        res, headers = cli._request_json(
+            'POST', ['echo'], {'b': 2}, {'a': 'x'})
         self.assertEqual('application/json', headers['content-type'])
         self.assertEqual({'CONTENT_TYPE': 'application/json',
                           'PATH_INFO': '/dbase/echo',
@@ -248,8 +265,9 @@
         # oauth does its own internal quoting
         params = {'x': u'\xf0', 'y': "foo"}
         res, headers = cli._request('GET', ['doc', 'oauth', 'foo bar'], params)
-        self.assertEqual(['/dbase/doc/oauth/foo bar', tests.token1.key, params],
-                         simplejson.loads(res))
+        self.assertEqual(
+            ['/dbase/doc/oauth/foo bar', tests.token1.key, params],
+            simplejson.loads(res))
 
     def test_oauth_Unauthorized(self):
         cli = self.getClient()

=== modified file 'u1db/tests/test_http_database.py'
--- u1db/tests/test_http_database.py	2012-05-22 18:58:04 +0000
+++ u1db/tests/test_http_database.py	2012-05-25 19:26:18 +0000
@@ -107,26 +107,38 @@
         self.response_val = '{"v": 2}', {'x-u1db-rev': 'doc-rev',
                                          'x-u1db-has-conflicts': 'false'}
         self.assertGetDoc(self.db, 'doc-id', 'doc-rev', '{"v": 2}', False)
-        self.assertEqual(('GET', ['doc', 'doc-id'], None, None, None),
-                         self.got)
+        self.assertEqual(
+            ('GET', ['doc', 'doc-id'], {'include_deleted': False}, None, None),
+            self.got)
 
     def test_get_doc_non_existing(self):
         self.response_val = errors.DocumentDoesNotExist()
         self.assertIs(None, self.db.get_doc('not-there'))
-        self.assertEqual(('GET', ['doc', 'not-there'], None, None, None),
-                         self.got)
+        self.assertEqual(
+            ('GET', ['doc', 'not-there'], {'include_deleted': False}, None,
+             None), self.got)
 
     def test_get_doc_deleted(self):
+        self.response_val = errors.DocumentDoesNotExist()
+        self.assertIs(None, self.db.get_doc('deleted'))
+        self.assertEqual(
+            ('GET', ['doc', 'deleted'], {'include_deleted': False}, None,
+             None), self.got)
+
+    def test_get_doc_deleted_include_deleted(self):
         self.response_val = errors.HTTPError(404,
                                              simplejson.dumps(
                                              {"error": errors.DOCUMENT_DELETED}
                                              ),
                                              {'x-u1db-rev': 'doc-rev-gone',
                                               'x-u1db-has-conflicts': 'false'})
-        doc = self.db.get_doc('deleted')
+        doc = self.db.get_doc('deleted', include_deleted=True)
         self.assertEqual('deleted', doc.doc_id)
         self.assertEqual('doc-rev-gone', doc.rev)
         self.assertIs(None, doc.content)
+        self.assertEqual(
+            ('GET', ['doc', 'deleted'], {'include_deleted': True}, None, None),
+            self.got)
 
     def test_get_doc_pass_through_errors(self):
         self.response_val = errors.HTTPError(500, 'Crash.')

=== modified file 'u1db/tests/test_remote_sync_target.py'
--- u1db/tests/test_remote_sync_target.py	2012-05-22 18:58:04 +0000
+++ u1db/tests/test_remote_sync_target.py	2012-05-25 19:26:18 +0000
@@ -242,21 +242,28 @@
 
     def test_sync_exchange_in_stream_error(self):
         self.startServer()
+
         def blackhole_getstderr(inst):
             return cStringIO.StringIO()
+
         self.patch(self.server.RequestHandlerClass, 'get_stderr',
                    blackhole_getstderr)
         db = self.request_state._create_database('test')
         doc = db.create_doc('{"value": "there"}')
-        def bomb_get_docs(doc_ids, check_for_conflicts=None):
+
+        def bomb_get_docs(doc_ids, check_for_conflicts=None,
+                          include_deleted=False):
             yield doc
             # delayed failure case
             raise errors.Unavailable
+
         self.patch(db, 'get_docs', bomb_get_docs)
         remote_target = self.getSyncTarget('test')
         other_changes = []
+
         def receive_doc(doc, gen):
             other_changes.append((doc.doc_id, doc.rev, doc.get_json(), gen))
+
         self.assertRaises(errors.Unavailable, remote_target.sync_exchange,
                           [], 'replica', last_known_generation=0,
                           return_doc_cb=receive_doc)

=== modified file 'u1db/tests/test_sync.py'
--- u1db/tests/test_sync.py	2012-05-22 18:58:04 +0000
+++ u1db/tests/test_sync.py	2012-05-25 19:26:18 +0000
@@ -134,7 +134,7 @@
     def set_trace_hook(self, callback):
         try:
             self.st._set_trace_hook(callback)
-        except NotImplementedError, e:
+        except NotImplementedError:
             self.skipTest("%s does not implement _set_trace_hook"
                           % (self.st.__class__.__name__,))
 
@@ -146,7 +146,7 @@
 
     def test_create_doc_updates_sync_info(self):
         self.assertEqual(('test', 0, 0, ''), self.st.get_sync_info('other'))
-        doc = self.db.create_doc(simple_doc)
+        self.db.create_doc(simple_doc)
         self.assertEqual(('test', 1, 0, ''), self.st.get_sync_info('other'))
 
     def test_record_sync_info(self):
@@ -174,7 +174,8 @@
         new_gen = self.st.sync_exchange(docs_by_gen, 'replica',
                                         last_known_generation=0,
                                         return_doc_cb=self.receive_doc)
-        self.assertGetDoc(self.db, doc.doc_id, edit_rev, None, False)
+        self.assertGetDocIncludeDeleted(
+            self.db, doc.doc_id, edit_rev, None, False)
         self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db)
         self.assertEqual(([], 2), (self.other_changes, new_gen))
         self.assertEqual(10, self.st.get_sync_info('replica')[2])
@@ -258,9 +259,10 @@
                            (doc2.doc_id, doc2.rev, nested_doc, 2)], 2),
                          (self.other_changes, new_gen))
         if self.whitebox:
-            self.assertEqual(self.db._last_exchange_log['return'],
-                             {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev),
-                                                      (doc2.doc_id, doc2.rev)]})
+            self.assertEqual(
+                self.db._last_exchange_log['return'],
+                {'last_gen': 2, 'docs':
+                 [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]})
 
     def test_sync_exchange_getting_newer_docs(self):
         doc = self.db.create_doc(simple_doc)
@@ -276,6 +278,7 @@
 
     def test_sync_exchange_with_concurrent_updates_of_synced_doc(self):
         expected = []
+
         def before_whatschanged_cb(state):
             if state != 'before whats_changed':
                 return
@@ -283,6 +286,7 @@
             conc_rev = self.db.put_doc(
                 self.make_document(doc.doc_id, 'test:1|z:2', cont))
             expected.append((doc.doc_id, conc_rev, cont, 3))
+
         self.set_trace_hook(before_whatschanged_cb)
         doc = self.db.create_doc(simple_doc)
         self.assertTransactionLog([doc.doc_id], self.db)
@@ -351,8 +355,10 @@
 
     def test__set_trace_hook(self):
         called = []
+
         def cb(state):
             called.append(state)
+
         self.set_trace_hook(cb)
         self.st.sync_exchange([], 'replica', 0, self.receive_doc)
         self.st.record_sync_info('replica', 0, 'T-sid')
@@ -465,10 +471,12 @@
         # new record.
         # When we finish synchronizing, we can notice that something locally
         # was updated, and we cannot tell c2 our new updated generation
+
         def before_get_docs(state):
             if state != 'before get_docs':
                 return
             self.db1.create_doc(simple_doc)
+
         self.assertEqual(0, self.sync(self.db1, self.db2,
                                       trace_hook=before_get_docs))
         self.assertLastExchangeLog(self.db2,
@@ -482,7 +490,8 @@
         self.assertEqual((0, ''), self.db2._get_sync_gen_info('test1'))
 
     def test_sync_doesnt_update_other_if_nothing_pulled(self):
-        doc = self.db1.create_doc(simple_doc)
+        self.db1.create_doc(simple_doc)
+
         def no_record_sync_info(state):
             if state != 'record_sync_info':
                 return
@@ -565,8 +574,9 @@
              'return': {'docs': [(doc_id, doc2.rev)],
                         'last_gen': 2}})
         self.assertTransactionLog([doc_id, doc_id, doc_id], self.db1)
-        self.assertGetDoc(self.db1, doc_id, doc2.rev, None, True)
-        self.assertGetDoc(self.db2, doc_id, doc2.rev, None, False)
+        self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True)
+        self.assertGetDocIncludeDeleted(
+            self.db2, doc_id, doc2.rev, None, False)
         self.assertEqual([], self.db1.get_from_index('test-idx', [('value',)]))
 
     def test_sync_local_race_conflicted(self):
@@ -581,12 +591,14 @@
         self.db2.put_doc(doc)
         doc2_rev2 = doc.rev
         triggered = []
+
         def after_whatschanged(state):
             if state != 'after whats_changed':
                 return
             triggered.append(True)
             doc = self.make_document(doc_id, doc1_rev, content1)
             self.db1.put_doc(doc)
+
         self.sync(self.db1, self.db2, trace_hook=after_whatschanged)
         self.assertEqual([True], triggered)
         self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True)
@@ -599,7 +611,6 @@
     def test_sync_propagates_deletes(self):
         doc1 = self.db1.create_doc(simple_doc)
         doc_id = doc1.doc_id
-        doc1_rev = doc1.rev
         self.db1.create_index('test-idx', ['key'])
         self.sync(self.db1, self.db2)
         self.db2.create_index('test-idx', ['key'])
@@ -613,8 +624,10 @@
                          'source_uid': 'test1',
                          'source_gen': 2, 'last_known_gen': 1},
              'return': {'docs': [], 'last_gen': 2}})
-        self.assertGetDoc(self.db1, doc_id, deleted_rev, None, False)
-        self.assertGetDoc(self.db2, doc_id, deleted_rev, None, False)
+        self.assertGetDocIncludeDeleted(
+            self.db1, doc_id, deleted_rev, None, False)
+        self.assertGetDocIncludeDeleted(
+            self.db2, doc_id, deleted_rev, None, False)
         self.assertEqual([], self.db1.get_from_index('test-idx', [('value',)]))
         self.assertEqual([], self.db2.get_from_index('test-idx', [('value',)]))
         self.sync(self.db2, self.db3)
@@ -623,7 +636,8 @@
                          'source_uid': 'test2',
                          'source_gen': 2, 'last_known_gen': 0},
              'return': {'docs': [], 'last_gen': 2}})
-        self.assertGetDoc(self.db3, doc_id, deleted_rev, None, False)
+        self.assertGetDocIncludeDeleted(
+            self.db3, doc_id, deleted_rev, None, False)
 
     def test_sync_propagates_resolution(self):
         doc1 = self.db1.create_doc('{"a": 1}', doc_id='the-doc')
@@ -659,8 +673,8 @@
     def test_sync_supersedes_conflicts(self):
         db3 = self.create_database('test3')
         doc1 = self.db1.create_doc('{"a": 1}', doc_id='the-doc')
-        doc2 = self.db2.create_doc('{"b": 1}', doc_id='the-doc')
-        doc3 = db3.create_doc('{"c": 1}', doc_id='the-doc')
+        self.db2.create_doc('{"b": 1}', doc_id='the-doc')
+        db3.create_doc('{"c": 1}', doc_id='the-doc')
         self.sync(db3, self.db1)
         self.sync(db3, self.db2)
         self.assertEqual(3, len(db3.get_doc_conflicts('the-doc')))
@@ -724,6 +738,7 @@
         progress1 = []
         progress2 = []
         _do_set_sync_info = self.db1._do_set_sync_info
+
         def set_sync_generation_witness1(other_uid, other_gen, trans_id):
             progress1.append((other_uid, other_gen,
                 [d for d, t in self.db1._get_transaction_log()[2:]]))
@@ -732,6 +747,7 @@
                    set_sync_generation_witness1)
 
         _do_set_sync_info2 = self.db2._do_set_sync_info
+
         def set_sync_generation_witness2(other_uid, other_gen, trans_id):
             progress2.append((other_uid, other_gen,
                 [d for d, t in self.db2._get_transaction_log()[2:]]))