Merge lp:~jamesh/bindwood/sync-pull-changes into lp:bindwood

[James Henstridge]

Add a Synchroniser.pullChanges() method to read changed records from CouchDB.

Code to retrieve changes from CouchDB. There are two modes of operation:

1. on initial sync, or if the last sync point has been compacted out of the DB, return all rows. A view is used to filter to just the given profile.

2. on subsequent pulls, the changes feed is used to retrieve just the changed documents. A filter is used to remove changes to other profiles.

The changed records are fed to the Synchroniser.processRecord() method, which at this point is a no-op but will create/update local bookmarks. Also, I am not currently persisting the last_seq value, but plan to handle it similar to the existing Bindwood code.

Some of the complexity in this code comes from trying to handle records marked as deleted by desktopcouch. I've tried to make the handling of these as close to actual deleted records as I can.

[Eric Casteleijn (thisfred)]

Oh, you won't have to deal with documents marked deleted anymore (well as long as nothing breaks very badly when encountering them, as people may or may not have upgraded their desktopcouch software):

Documents that are deleted, should now *really* be deleted, and moved to the /dc_trash database, under an autogenerated new id, and with an added annotation, like:

        new_record.application_annotations['desktopcouch'] = {
            'private_application_annotations':
            {'original_database_name': self._database_name,
             'original_id': record_id}}

[Eric Casteleijn (thisfred)]

Approving though, as you may want to fix that on a different branch.

review: Approve

[lp:~jamesh/bindwood/sync-pull-changes]

37. By James Henstridge on 2011-02-14

Don't bother supporting the old desktopcouch form of marking records as
deleted.

[James Henstridge (jamesh)]

I've updated the branch to remove the special handling of old-style deleted records.

[Zachery Bir (urbanape)]

Just an FYI: Line 73: "var new_last_seq;" is unused.

Other than that, looks good.

review: Approve

=== modified file 'modules/sync.jsm'
--- modules/sync.jsm	2011-02-14 03:17:47 +0000
+++ modules/sync.jsm	2011-02-14 03:17:47 +0000
@@ -14,7 +14,7 @@
  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-const EXPORTED_SYMBOLS = ["BookmarksObserver"];
+const EXPORTED_SYMBOLS = ["Synchroniser", "BookmarksObserver"];
 
 const Cc = Components.classes;
 const Ci = Components.interfaces;
@@ -28,6 +28,105 @@
     .getService(Ci.nsIAnnotationService);
 
 
+function Synchroniser(couch, profile) {
+    this.couch = couch;
+    this.profile = profile;
+    this.last_seq = 0;
+}
+
+Synchroniser.prototype = {
+    ensureCouchViews: function() {
+        var design_doc_id = "_design/bindwood", doc;
+
+        try {
+            doc = this.couch.open(design_doc_id);
+        } catch (e) {
+            dump("Problem retrieving view view:" + e + "\n");
+            return;
+        }
+        if (!doc) {
+            doc = { _id: design_doc_id };
+        }
+        if (!doc.views)
+            doc.views = {};
+        doc.views.bookmarks = {
+            map: ("function(doc) {\n" +
+                  "  try {\n" +
+                  "    var annot = doc.application_annotations.Firefox;\n" +
+                  "    if (annot.profile)\n" +
+                  "      emit(annot.profile, doc._id);\n" +
+                  "  } catch (e) { /* ignore error */ }\n" +
+                  "}")
+        };
+        if (!doc.filters)
+            doc.filters = {};
+        doc.filters.by_profile = (
+            "function (doc, req) {\n" +
+            "  if (doc._deleted)\n" +
+            "    return true;\n" +
+            "  try {\n" +
+            "    return (doc.application_annotations.Firefox.profile == \n" +
+            "            req.query.profile);\n" +
+            "  } catch (e) { /* ignore error */ }\n" +
+            "}")
+        try {
+            this.couch.save(doc);
+        } catch (e) {
+            dump("Problem saving view:" + e + "\n");
+        }
+    },
+
+    sync: function() {
+        //this.recordChangedIds();
+        this.pullChanges();
+        //this.pushChanges();
+        //this.cleanup();
+    },
+
+    pullChanges: function() {
+        var new_last_seq;
+        var info = this.couch.info();
+        if (this.last_seq <= info.purge_seq) {
+            // We haven't pulled any records before, or the database
+            // has been compacted past the recorded last_seq.
+            this.last_seq = info.update_seq;
+            var result = this.couch.view("bindwood/bookmarks", {
+                    key: this.profile,
+                    include_docs: true,
+                });
+            for each (var row in result.rows) {
+                this.processRecord(row.doc);
+            }
+        } else {
+            var changes = this.couch.changes({
+                    since: this.last_seq,
+                    filter: "bindwood/by_profile",
+                    profile: this.profile,
+                    include_docs: true,
+                });
+            this.last_seq = changes.last_seq;
+            // Iterate backwards through the changes list, keeping
+            // only the last change for each record ID.
+            var seen_ids = {};
+            for (var i = changes.results.length-1; i >= 0; i--) {
+                var rev_id = changes.results[i].changes[0].rev;
+                if (seen_ids[rev_id]) {
+                    changes.results.splice(i, 1);
+                }
+                seen_ids[rev_id] = true;
+            }
+            for each (var row in changes.results) {
+                this.processRecord(row.doc);
+            }
+        }
+    },
+
+    processRecord: function (doc) {
+        /* Process incoming document */
+    }
+};
+
+
 function BookmarksObserver(sync) {
     this.sync = sync;
     this.changed_guids = {}

=== added file 'mozmill/tests/test_sync_pullchanges.js'
--- mozmill/tests/test_sync_pullchanges.js	1970-01-01 00:00:00 +0000
+++ mozmill/tests/test_sync_pullchanges.js	2011-02-14 03:17:47 +0000
@@ -0,0 +1,140 @@
+var bm = require("../shared-modules/bookmarks");
+
+const TIMEOUT = 5000;
+
+const LOCAL_TEST_FOLDER = collector.addHttpResource('../test-files/');
+const LOCAL_TEST_PAGE = LOCAL_TEST_FOLDER + 'test.html';
+const LOCAL_TEST_FEED = LOCAL_TEST_FOLDER + 'feed.atom';
+
+var setupModule = function(module) {
+    module.controller = mozmill.getBrowserController();
+    module.jum = {};
+    module.desktopcouch = {};
+    module.sync = {};
+    Cu.import("resource://mozmill/modules/jum.js", module.jum);
+    Cu.import("resource://bindwood/desktopcouch.jsm", module.desktopcouch);
+    Cu.import("resource://bindwood/sync.jsm", module.sync);
+    bm.clearBookmarks();
+    module.couch = null;
+};
+
+
+var setupTest = function(test) {
+    var done = false;
+    desktopcouch.connect_desktopcouch("test_bookmarks", function(db) {
+            couch = db;
+            done = true;
+        }, function (message) {});
+    controller.waitFor(
+        function() { return done; }, "Could not connect to CouchDB", TIMEOUT);
+    jum.assertNotEquals(couch, null);
+
+    try {
+        couch.createDb();
+    } catch (e) {
+        if (e.error != 'file_exists')
+            throw(e);
+    }
+};
+
+
+var teardownTest = function(test) {
+    bm.clearBookmarks();
+    couch.deleteDb();
+};
+
+var compareIds = function(doc1, doc2) {
+    if (doc1._id < doc2._id)
+        return -1;
+    if (doc1._id > doc2._id)
+        return 1;
+    return 0;
+}
+
+var test_ensureViews = function() {
+    var synchroniser = new sync.Synchroniser(couch, "profile_name");
+    synchroniser.ensureCouchViews();
+
+    var doc = couch.open("_design/bindwood");
+    jum.assertNotEquals(doc, null);
+    jum.assert(
+        doc.views.bookmarks, "Design doc should have bookmarks view");
+    jum.assert(
+        doc.filters.by_profile, "Design doc should have by_profile filter");
+};
+
+var test_pullChanges = function() {
+    var synchroniser = new sync.Synchroniser(couch, "profile_name");
+    synchroniser.ensureCouchViews();
+
+    var documents = [];
+    synchroniser.processRecord = function(doc) {
+        documents.push(doc);
+    };
+
+    couch.save({
+            _id: 'guid-1',
+            application_annotations: {
+                Firefox: {
+                    profile: 'profile_name',
+                },
+            },
+        });
+    couch.save({
+            _id: 'guid-2',
+            application_annotations: {
+                Firefox: {
+                    profile: 'some_other_profile',
+                },
+            },
+        });
+    couch.save({
+            _id: 'guid-3',
+            application_annotations: {
+                Firefox: {
+                    profile: 'profile_name',
+                },
+            },
+        });
+    synchroniser.pullChanges();
+    jum.assertEquals(documents.length, 2);
+    documents.sort(compareIds);
+    jum.assertEquals(documents[0]._id, 'guid-1');
+    jum.assertEquals(documents[1]._id, 'guid-3');
+
+    // Delete one of the documents.
+    couch.deleteDoc(couch.open('guid-1'));
+
+    // Create a new document, and then modify it.
+    couch.save({
+            _id: 'guid-4',
+            application_annotations: {
+                Firefox: {
+                    profile: 'profile_name',
+                },
+            },
+        })
+    doc = couch.open('guid-4');
+    doc.attr = 'value';
+    couch.save(doc);
+
+    // Create another record belonging to a different profile:
+    couch.save({
+            _id: 'guid-5',
+            application_annotations: {
+                Firefox: {
+                    profile: 'some_other_profile',
+                },
+            },
+        });
+
+
+    // Calling pullChanges() again gives us incremental changes.
+    documents = []
+    synchroniser.pullChanges();
+    jum.assertEquals(documents.length, 2);
+    documents.sort(compareIds);
+    jum.assertEquals(documents[0]._id, 'guid-1');
+    jum.assertEquals(documents[0]._deleted, true);
+    jum.assertEquals(documents[1]._id, 'guid-4');
+};