Merge lp:~jamesh/bindwood/sync-from-couch into lp:bindwood

[James Henstridge]

Add code to the new synchroniser to create local bookmarks from CouchDB documents.

Create local bookmarks based on incoming documents from CouchDB.

Unlike the existing Bindwood schema, I am relying on a reference to the parent item in each bookmark, and treat the children list in folder items as an ordering hint.

If there is no local bookmark for a given bookmark, we create them under unfiledBookmarksFolder and reparent them when processing the parent record. This way we should get the same result no matter which order we receive the records.

Since we won't necessarily have imported all the children when processing a folder record, we don't bother trying to place the bookmarks in order. Instead, we make a note of the requested child ordering so that it can be applied after all the bookmarks have been imported.

[Stuart Langridge (sil)]

Will this affect the website bookmark rendering code?

sil

[James Henstridge (jamesh)]

Looking at the bookmarks web code, it ignores folders entirely, and only deals with the "uri" and "title" attributes on leaf bookmarks, so that part doesn't matter.

One place where my changes may be an issue is the detection of available profiles. Currently it uses a view that searches for documents with an ID of the form "root_*". In my new code, I haven't been generating that document, but it would certainly be possible to add that back in.

[Stuart Langridge (sil)]

All tests now pass (natty, FF4). Looking good.

review: Approve

[Manuel de la Peña (mandel)]

Looks good when testing on M.

review: Approve

[Ubuntu One Auto Pilot (otto-pilot)]

The prerequisite lp:~jamesh/bindwood/sync-guid-map has not yet been merged into lp:bindwood.

=== modified file 'modules/sync.jsm'
--- modules/sync.jsm	2011-02-22 11:07:07 +0000
+++ modules/sync.jsm	2011-02-22 11:07:07 +0000
@@ -19,6 +19,13 @@
 const GUID_ANNOTATION = "bindwood/uuid";
 const PARENT_ANNOTATION = "bindwood/parent";
 
+const RECORD_PREFIX = (
+    "http://www.freedesktop.org/wiki/Specifications/desktopcouch/");
+const TYPE_BOOKMARK = RECORD_PREFIX + "bookmark";
+const TYPE_FOLDER = RECORD_PREFIX + "folder";
+const TYPE_FEED = RECORD_PREFIX + "feed";
+const TYPE_SEPARATOR = RECORD_PREFIX + "separator";
+
 const Cc = Components.classes;
 const Ci = Components.interfaces;
 const Cr = Components.results;
@@ -29,8 +36,12 @@
     .getService(Ci.nsILivemarkService);
 var annotationService = Cc["@mozilla.org/browser/annotation-service;1"]
     .getService(Ci.nsIAnnotationService);
+var historyService = Cc["@mozilla.org/browser/nav-history-service;1"]
+    .getService(Ci.nsINavHistoryService);
 var uuidService = Cc["@mozilla.org/uuid-generator;1"]
     .getService(Ci.nsIUUIDGenerator);
+var ioService = Cc["@mozilla.org/network/io-service;1"]
+    .getService(Ci.nsIIOService);
 
 
 function Synchroniser(couch, profile) {
@@ -38,6 +49,7 @@
     this.profile = profile;
     this.last_seq = 0;
     this.guid_item_map = {};
+    this.folders_to_reorder = {};
 }
 
 Synchroniser.prototype = {
@@ -193,8 +205,175 @@
     },
 
     processRecord: function (doc) {
-        /* Process incoming document */
-    }
+        var item_id = this.guid_to_id(doc._id);
+        // XXX: We should perform duplicate detection here if item_id
+        // is null.
+        if (item_id != null) {
+            this.updateItem(item_id, doc);
+        } else {
+            this.createItem(doc);
+        }
+    },
+
+    createItem: function(doc) {
+        if (doc._deleted) {
+            /* Bookmark must have been created and deleted within the
+             * period of one sync cycle, so ignore. */
+            return;
+        }
+        // Determine the parent folder.  If it doesn't exist,
+        // temporarily add it as an unfiled bookmark to handle later.
+        var parent_id = this.guid_to_id(doc.parent_guid);
+        var has_parent = (parent_id != null);
+        if (!has_parent) {
+            parent_id = bookmarksService.unfiledBookmarksFolder;
+        }
+
+        var item_id;
+        switch (doc.record_type) {
+        case TYPE_BOOKMARK:
+            var uri = ioService.newURI(doc.uri, null, null);
+            item_id = bookmarksService.insertBookmark(
+                parent_id, uri, bookmarksService.DEFAULT_INDEX, doc.title);
+            break;
+        case TYPE_FOLDER:
+            item_id = bookmarksService.createFolder(
+                parent_id, doc.title, bookmarksService.DEFAULT_INDEX);
+            this.reparent_orphans(item_id, doc._id);
+            this.folders_to_reorder[doc._id] = doc.children;
+            break;
+        case TYPE_FEED:
+            var site_uri = ioService.newURI(doc.site_uri, null, null);
+            var feed_uri = ioService.newURI(doc.feed_uri, null, null);
+            item_id = livemarkService.createLivemark(
+                parent_id, doc.title, site_uri, feed_uri,
+                bookmarksService.DEFAULT_INDEX);
+            break;
+        case TYPE_SEPARATOR:
+            item_id = bookmarksService.insertSeparator(
+                parent_id, bookmarksService.DEFAULT_INDEX);
+            break;
+        default:
+            item_id = null;
+        }
+        if (item_id) {
+            this.set_guid(item_id, doc._id);
+            if (!has_parent) {
+                this.set_orphan(item_id, doc.parent_guid);
+            }
+        }
+    },
+
+    updateItem: function(item_id, doc) {
+        if (doc._deleted) {
+            // Document has been deleted in Couch: propagate to local.
+            bookmarksService.removeItem(item_id);
+            return;
+        }
+
+        var parent_id = this.guid_to_id(doc.parent_guid);
+        if (parent_id == null) {
+            this.set_orphan(item_id, doc.parent_guid);
+        } else {
+            if (bookmarksService.getFolderIdForItem(item_id) != parent_id) {
+                bookmarksService.moveItem(
+                    item_id, parent_id, bookmarksService.DEFAULT_INDEX);
+            }
+            annotationService.removeItemAnnotation(
+                item_id, PARENT_ANNOTATION);
+        }
+
+        switch (doc.record_type) {
+        case TYPE_BOOKMARK:
+            var uri = ioService.newURI(doc.uri, null, null);
+            bookmarksService.setItemTitle(item_id, doc.title);
+            bookmarksService.changeBookmarkURI(item_id, uri);
+            break;
+        case TYPE_FOLDER:
+            bookmarksService.setItemTitle(item_id, doc.title);
+            this.folders_to_reorder[doc._id] = doc.children;
+            break;
+        case TYPE_FEED:
+            var site_uri = ioService.newURI(doc.site_uri, null, null);
+            var feed_uri = ioService.newURI(doc.feed_uri, null, null);
+            bookmarksService.setItemTitle(item_id, doc.title);
+            livemarkService.setSiteURI(item_id, site_uri);
+            livemarkService.setFeedURI(item_id, feed_uri);
+            break;
+        case TYPE_SEPARATOR:
+            item_id = bookmarksService.insertSeparator(
+                parent_id, bookmarksService.DEFAULT_INDEX);
+            break;
+        default:
+            item_id = null;
+        }
+    },
+
+    set_orphan: function (item_id, parent_guid) {
+        annotationService.setItemAnnotation(
+            item_id, PARENT_ANNOTATION, parent_guid,
+            0, annotationService.EXPIRE_NEVER);
+    },
+
+    reparent_orphans: function (parent_id, parent_guid) {
+        var items = annotationService.getItemsWithAnnotation(
+                PARENT_ANNOTATION, {});
+        for each (orphan_id in items) {
+            if (annotationService.getItemAnnotation(
+                orphan_id, PARENT_ANNOTATION) != parent_guid) {
+                continue;
+            }
+            // XXX: check for errors
+            bookmarksService.moveItem(
+                orphan_id, parent_id, bookmarksService.DEFAULT_INDEX)
+            annotationService.removeItemAnnotation(
+                orphan_id, PARENT_ANNOTATION);
+        }
+    },
+
+    get_folder_children: function(parent_id) {
+        var query = historyService.getNewQuery();
+        var options = historyService.getNewQueryOptions();
+        query.setFolders([parent_id], 1);
+        var root = historyService.executeQuery(query, options).root;
+
+        var children = [];
+        root.containerOpen = true;
+        for (var i = 0; i < root.childCount; i++) {
+            var child = root.getChild(i);
+            children.push(child.itemId);
+        }
+        return children;
+    },
+
+    reorder_children: function() {
+        for (var parent_guid in this.folders_to_reorder) {
+            var parent_id = this.guid_to_id(parent_guid);
+            if (!parent_id)
+                continue;
+
+            var children = this.get_folder_children(parent_id);
+            var idx = 0;
+            for each (child_guid in this.folders_to_reorder[parent_guid]) {
+                var child_id = this.guid_to_id(child_guid);
+                if (!child_id)
+                    continue;
+
+                // Set the item's index.
+                bookmarksService.setItemIndex(child_id, idx++);
+
+                // Remove from the list of existing children, if they exist.
+                var existing_idx = children.indexOf(child_id);
+                if (existing_idx >= 0)
+                    children.splice(existing_idx, 1);
+            }
+            // Stick the remaining items at the end of the folder.
+            for each (var child_id in children) {
+                bookmarksService.setItemIndex(child_id, idx++);
+            }
+        }
+        this.folders_to_reorder = {}
+    },
 };
 
 

=== added file 'mozmill/tests/test_sync_from_couch.js'
--- mozmill/tests/test_sync_from_couch.js	1970-01-01 00:00:00 +0000
+++ mozmill/tests/test_sync_from_couch.js	2011-02-22 11:07:07 +0000
@@ -0,0 +1,304 @@
+var bm = require("../shared-modules/bookmarks");
+
+const TIMEOUT = 5000;
+
+const LOCAL_TEST_FOLDER = collector.addHttpResource('../test-files/');
+const LOCAL_TEST_PAGE = LOCAL_TEST_FOLDER + 'test.html';
+const LOCAL_TEST_FEED = LOCAL_TEST_FOLDER + 'feed.atom';
+
+var setupModule = function(module) {
+    module.controller = mozmill.getBrowserController();
+    module.jum = {}
+    module.sync = {}
+    Cu.import("resource://mozmill/modules/jum.js", module.jum);
+    Cu.import("resource://bindwood/sync.jsm", module.sync);
+    module.synchroniser = null;
+    bm.clearBookmarks();
+};
+
+
+var setupTest = function() {
+    synchroniser = new sync.Synchroniser(null, "profile_name");
+};
+
+var teardownTest = function() {
+    bm.clearBookmarks();
+};
+
+var test_create_bookmark = function() {
+    synchroniser.processRecord({
+        _id: '12345',
+        record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        parent_guid: 'toolbar_profile_name',
+        title: 'Bookmark title',
+        uri: LOCAL_TEST_PAGE,
+    });
+    var bookmark_id = synchroniser.guid_to_id('12345');
+    jum.assertEquals(bm.bookmarksService.getItemType(bookmark_id),
+                     bm.bookmarksService.TYPE_BOOKMARK);
+    jum.assertEquals(bm.bookmarksService.getFolderIdForItem(bookmark_id),
+                     bm.bookmarksService.toolbarFolder);
+    jum.assertEquals(bm.bookmarksService.getItemTitle(bookmark_id),
+                     'Bookmark title');
+    jum.assertEquals(bm.bookmarksService.getBookmarkURI(bookmark_id).spec,
+                     LOCAL_TEST_PAGE);
+};
+
+var test_update_bookmark = function() {
+    doc = {
+        _id: '12345',
+        record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        parent_guid: 'toolbar_profile_name',
+        title: 'Bookmark title',
+        uri: LOCAL_TEST_PAGE,
+    };
+    synchroniser.processRecord(doc);
+    doc.title = 'New title';
+    doc.uri = LOCAL_TEST_PAGE + "#updated";
+    synchroniser.processRecord(doc);
+
+    var bookmark_id = synchroniser.guid_to_id('12345');
+    jum.assertEquals(bm.bookmarksService.getItemTitle(bookmark_id),
+                     'New title');
+    jum.assertEquals(bm.bookmarksService.getBookmarkURI(bookmark_id).spec,
+                     LOCAL_TEST_PAGE + "#updated");
+};
+
+var test_create_folder = function() {
+    synchroniser.processRecord({
+        _id: '12345',
+        record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/folder',
+        parent_guid: 'toolbar_profile_name',
+        title: 'Folder',
+        children: [],
+    });
+    var bookmark_id = synchroniser.guid_to_id('12345');
+    jum.assertEquals(bm.bookmarksService.getItemType(bookmark_id),
+                     bm.bookmarksService.TYPE_FOLDER);
+    jum.assertEquals(bm.bookmarksService.getFolderIdForItem(bookmark_id),
+                     bm.bookmarksService.toolbarFolder);
+    jum.assertEquals(
+        bm.bookmarksService.getItemTitle(bookmark_id), 'Folder');
+    // Reordering of folder's children has been scheduled.
+    jum.assertNotEquals(synchroniser.folders_to_reorder['12345'], null);
+};
+
+var test_update_folder = function() {
+    doc = {
+        _id: '12345',
+        record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/folder',
+        parent_guid: 'toolbar_profile_name',
+        title: 'Folder',
+        children: [],
+    };
+    synchroniser.processRecord(doc);
+    doc.title = 'Updated folder';
+    synchroniser.processRecord(doc);
+
+    var bookmark_id = synchroniser.guid_to_id('12345');
+    jum.assertEquals(bm.bookmarksService.getItemTitle(bookmark_id),
+                     'Updated folder');
+    // Reordering of folder's children has been scheduled.
+    jum.assertNotEquals(synchroniser.folders_to_reorder['12345'], null);
+};
+
+var test_create_livemark = function() {
+    synchroniser.processRecord({
+        _id: '12345',
+        record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/feed',
+        parent_guid: 'toolbar_profile_name',
+        title: 'Livemark',
+        site_uri: "http://www.example.com/",
+        feed_uri: LOCAL_TEST_FEED,
+    });
+    var bookmark_id = synchroniser.guid_to_id('12345');
+    jum.assertTrue(bm.livemarkService.isLivemark(bookmark_id))
+    jum.assertEquals(bm.bookmarksService.getFolderIdForItem(bookmark_id),
+                     bm.bookmarksService.toolbarFolder);
+    jum.assertEquals(
+        bm.bookmarksService.getItemTitle(bookmark_id), 'Livemark');
+    jum.assertEquals(bm.livemarkService.getSiteURI(bookmark_id).spec,
+                     "http://www.example.com/");
+    jum.assertEquals(bm.livemarkService.getFeedURI(bookmark_id).spec,
+                     LOCAL_TEST_FEED);
+};
+
+var test_update_livemark = function() {
+    doc = {
+        _id: '12345',
+        record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/feed',
+        parent_guid: 'toolbar_profile_name',
+        title: 'Livemark',
+        site_uri: "http://www.example.com/",
+        feed_uri: LOCAL_TEST_FEED,
+    };
+    synchroniser.processRecord(doc);
+    doc.title = 'New title';
+    doc.feed_uri = LOCAL_TEST_FEED + "#updated";
+    synchroniser.processRecord(doc);
+
+    var bookmark_id = synchroniser.guid_to_id('12345');
+    jum.assertEquals(bm.bookmarksService.getItemTitle(bookmark_id),
+                     'New title');
+    jum.assertEquals(bm.livemarkService.getFeedURI(bookmark_id).spec,
+                     LOCAL_TEST_FEED + "#updated");
+};
+
+var test_create_separator = function() {
+    synchroniser.processRecord({
+        _id: '12345',
+        record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/separator',
+        parent_guid: 'toolbar_profile_name',
+    });
+    var bookmark_id = synchroniser.guid_to_id('12345');
+    jum.assertEquals(bm.bookmarksService.getItemType(bookmark_id),
+                     bm.bookmarksService.TYPE_SEPARATOR);
+    jum.assertEquals(bm.bookmarksService.getFolderIdForItem(bookmark_id),
+                     bm.bookmarksService.toolbarFolder);
+};
+
+var test_update_separator = function() {
+    doc = {
+        _id: '12345',
+        record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/separator',
+        parent_guid: 'toolbar_profile_name',
+    };
+    synchroniser.processRecord(doc);
+    // No properties to change on the separator, but verify that the
+    // update code path runs anyway.
+    synchroniser.processRecord(doc);
+    var bookmark_id = synchroniser.guid_to_id('12345');
+    jum.assertEquals(bm.bookmarksService.getItemType(bookmark_id),
+                     bm.bookmarksService.TYPE_SEPARATOR);
+};
+
+var test_update_item_reparent = function() {
+    doc = {
+        _id: '12345',
+        record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        parent_guid: 'toolbar_profile_name',
+        title: 'Bookmark title',
+        uri: LOCAL_TEST_PAGE,
+    };
+    synchroniser.processRecord(doc);
+    var bookmark_id = synchroniser.guid_to_id('12345');
+    jum.assertEquals(bm.bookmarksService.getFolderIdForItem(bookmark_id),
+                     bm.bookmarksService.toolbarFolder);
+
+    // Move to a new parent folder.
+    doc.parent_guid = "menu_profile_name";
+    synchroniser.processRecord(doc);
+    jum.assertEquals(bm.bookmarksService.getFolderIdForItem(bookmark_id),
+                     bm.bookmarksService.bookmarksMenuFolder);
+};
+
+var test_create_parent_then_child = function() {
+    synchroniser.processRecord({
+        _id: '12345',
+        record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/folder',
+        parent_guid: 'toolbar_profile_name',
+        title: 'Folder',
+        children: ['67890'],
+    });
+    var parent_id = synchroniser.guid_to_id('12345');
+    jum.assertEquals(bm.bookmarksService.getFolderIdForItem(parent_id),
+                     bm.bookmarksService.toolbarFolder);
+    // Reordering of folder's children has been scheduled.
+    jum.assertNotEquals(synchroniser.folders_to_reorder['12345'], null);
+
+    synchroniser.processRecord({
+        _id: '67890',
+        record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        parent_guid: '12345',
+        title: 'Bookmark title',
+        uri: LOCAL_TEST_PAGE,
+    });
+    var child_id = synchroniser.guid_to_id('67890');
+    jum.assertEquals(bm.bookmarksService.getFolderIdForItem(child_id),
+                     parent_id);
+};
+
+var test_create_child_then_parent = function() {
+    synchroniser.processRecord({
+        _id: '67890',
+        record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        parent_guid: '12345',
+        title: 'Bookmark title',
+        uri: LOCAL_TEST_PAGE,
+    });
+    var child_id = synchroniser.guid_to_id('67890');
+    // The bookmark is temporarily stashed in unfiled bookmarks.
+    jum.assertEquals(bm.bookmarksService.getFolderIdForItem(child_id),
+                     bm.bookmarksService.unfiledBookmarksFolder);
+
+    synchroniser.processRecord({
+        _id: '12345',
+        record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/folder',
+        parent_guid: 'toolbar_profile_name',
+        title: 'Folder',
+        children: ['67890'],
+    });
+    var parent_id = synchroniser.guid_to_id('12345');
+    jum.assertEquals(bm.bookmarksService.getFolderIdForItem(parent_id),
+                     bm.bookmarksService.toolbarFolder);
+    // The child has been reparented.
+    jum.assertEquals(bm.bookmarksService.getFolderIdForItem(child_id),
+                     parent_id);
+    // Reordering of folder's children has been scheduled.
+    jum.assertNotEquals(synchroniser.folders_to_reorder['12345'], null);
+};
+
+var test_reorder_children = function() {
+    synchroniser.processRecord({
+            _id: 'parent',
+            record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/folder',
+            parent_guid: 'toolbar_profile_name',
+            title: 'Folder',
+            children: ['child1', 'child2', 'child3', 'missing_child'],
+        });
+    var parent_id = synchroniser.guid_to_id('parent');
+    // Add an untracked bookmark to the folder.
+    var untracked_child = bm.bookmarksService.insertBookmark(
+        parent_id, bm.createURI(LOCAL_TEST_PAGE),
+        bm.bookmarksService.DEFAULT_INDEX,
+        "Untracked bookmark");
+
+    // Add expected children.
+    synchroniser.processRecord({
+            _id: 'child1',
+            record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+            parent_guid: 'parent',
+            title: 'Child 1',
+            uri: LOCAL_TEST_PAGE + "#child1",
+        });
+    synchroniser.processRecord({
+            _id: 'child3',
+            record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+            parent_guid: 'parent',
+            title: 'Child 3',
+            uri: LOCAL_TEST_PAGE + "#child1",
+        });
+    synchroniser.processRecord({
+            _id: 'child2',
+            record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+            parent_guid: 'parent',
+            title: 'Child 2',
+            uri: LOCAL_TEST_PAGE + "#child1",
+        });
+
+    // The parent has been scheduled for reordering, so trigger
+    // reordering.
+    jum.assertNotEquals(synchroniser.folders_to_reorder['parent'], null);
+    synchroniser.reorder_children();
+
+    // Children should now be in the requested order, with the
+    // untracked child at the end.
+    jum.assertEquals(bm.bookmarksService.getIdForItemAt(parent_id, 0),
+                     synchroniser.guid_to_id('child1'));
+    jum.assertEquals(bm.bookmarksService.getIdForItemAt(parent_id, 1),
+                     synchroniser.guid_to_id('child2'));
+    jum.assertEquals(bm.bookmarksService.getIdForItemAt(parent_id, 2),
+                     synchroniser.guid_to_id('child3'));
+    jum.assertEquals(bm.bookmarksService.getIdForItemAt(parent_id, 3),
+                     untracked_child);
+};