Merge lp:~jamesh/bindwood/migration into lp:bindwood

[James Henstridge]

Migration code to upgrade databases from Bindwood 0.4.x and 1.0.x schemas.

Add code to migrate old Bindwood bookmarks over to the new schema.

We determine the schema version like so:
 * If there is no root_$profile document, assume we're using the
   0.4.x schema (which I call version 0).
 * If there is a root_$profile document but has no record_type_version,
   assume we're using the 1.0.x schema (which I call version 1).
 * If there is a root_$profile document and it has record_type_version,
   assume it is equal to the schema version.

This gives us a way to detect future formats without much trouble.

I don't have code to convert from version 0, but instead rely on the ability to rebuild the database from Firefox's local places database.

I've also added a few attributes that will make it easier to implement a duplicate detection algorithm:
 * parent_title: the title of the parent folder (combine this with
   other bookmark data to detect duplicates).
 * position: the numeric index of a separator (provides a way to
   distinguish different separators in a folder).

[Manuel de la Peña (mandel)]

review: Approve

[Eric Casteleijn (thisfred)]

Looks good, tests pass.

review: Approve

[Ubuntu One Auto Pilot (otto-pilot)]

The prerequisite lp:~jamesh/bindwood/switch-to-new-synchroniser has not yet been merged into lp:bindwood.

=== modified file 'modules/bindwood.jsm'
--- modules/bindwood.jsm	2011-03-04 13:02:54 +0000
+++ modules/bindwood.jsm	2011-03-04 13:02:54 +0000
@@ -169,115 +169,4 @@
             Log.exception("Problem setting up repeater.", e);
         }
     },
-
-    migrateOlderBookmarkRecords: function() {
-        Log.debug(
-            "We're an older client. " +
-            "Let's migrate the remote records and re-sync.");
-
-        var additional = [];
-        var all_docs = Bindwood.couch.view("bookmarks/profile",
-            {
-                startkey: Bindwood.currentProfile,
-                endkey: Bindwood.currentProfile
-            });
-        var rows = all_docs.rows;
-
-        //   Pull all records from Couch, and for each:
-
-        for (var i = 0; i < rows.length; i++) {
-            var row = rows[i];
-            Log.debug("Got a row: " + row);
-            var id = row.id;
-            Log.debug("Got an id: " + id);
-            var doc = Bindwood.couch.open(id);
-            Log.debug("Got a doc: " + doc);
-
-            if (doc.record_type_version >= 1) {
-                Log.debug(
-                    "Record is already migrated Skipping...");
-                continue;
-            }
-
-            // get the uuid off the bookmark.
-            var old_uuid = doc.application_annotations.Firefox.uuid;
-            Log.debug(
-                "Got the old uuid from the record: " + old_uuid);
-            // look up itemId by uuid
-            // XXX: This probably needs to be made more robust
-            var itemId = Bindwood.itemIdForUUID(old_uuid);
-            Log.debug(
-                "Found its local itemID from the map: " + itemId);
-            // annotate the itemId with the document's _id
-            Bindwood.annotateItemWithUUID(itemId, id);
-            Log.debug(
-                "Annotating the item's uuid with the document's actual id");
-            // delete the uuid field off the record
-            delete doc.application_annotations.Firefox.uuid;
-            Log.debug("Deleted the document's uuid annotation");
-            // delete the folder field off the record
-            delete doc.application_annotations.Firefox.folder;
-            Log.debug("Deleted the document's folder annotation");
-
-            if (doc.deleted) {
-                Log.debug(
-                    "The document was flagged as deleted, cleaning up.");
-                // swap .deleted for conventional .deleted
-                if (!doc.application_annotations) {
-                    doc.application_annotations = {};
-                }
-                if (!doc.application_annotations['Ubuntu One']) {
-                    doc.application_annotations['Ubuntu One'] = {};
-                }
-                if (!doc.application_annotations['Ubuntu One'].private_application_annotations) {
-                    doc.application_annotations['Ubuntu One'].private_application_annotations = {};
-                }
-                doc.application_annotations['Ubuntu One'].private_application_annotations.deleted = true;
-                Log.debug(
-                    "Moved deleted flag into private application annotations.");
-                delete doc.deleted;
-                Log.debug("Deleted the top-level .deleted flag.");
-            }
-
-            // update the document's record_type_version
-            doc.record_type_version = 1;
-            Log.debug("Set the schema version to 1");
-
-            // Ensure we're dealing with the proper record type on Migrate.
-            doc = Bindwood.decorateRecordByType(doc, itemId);
-
-            // add to additional
-            additional.push(doc);
-            Log.debug(
-                "Adding this doc to the stack of addition docs to push back.");
-        }
-
-        for (var i = 0; i < additional.length; i++) {
-            var doc = additional[i];
-            Log.debug(
-                "Preparing to push back " + doc.title || doc.record_type);
-            // Ensure that any remote folders have their .children
-            // populated, and in particular make sure that we've already
-            // modified their children's annotations/uuids
-            if (doc.record_type == Bindwood.TYPE_FOLDER) {
-                // XXX: This probably needs to be made more robust
-                var itemId = Bindwood.itemIdForUUID(doc._id);
-                doc.children = Bindwood.getUUIDsFromFolder(itemId);
-                Log.debug(
-                    "Folder needed updating, calculated children: " +
-                    doc.children);
-            }
-
-            try {
-                var response = Bindwood.couch.save(doc);
-                // We can avoid having to process this revision when we
-                // pull it later
-                Bindwood.seen_revisions[response.rev] = true;
-            } catch(e) {
-                Log.exception(
-                    "Problem saving record to CouchDB; record is " +
-                        JSON.stringify(doc + ": ", e));
-            }
-        }
-    },
 };
 
=== modified file 'modules/couch.jsm'
--- modules/couch.jsm	2011-02-04 06:03:15 +0000
+++ modules/couch.jsm	2011-03-04 13:02:54 +0000
@@ -100,7 +100,7 @@
       if (docs[i]._id == undefined)
         newCount++;
     }
-    var newUuids = CouchDB.newUuids(docs.length);
+    var newUuids = CouchDB.newUuids(newCount);
     var newCount = 0;
     for (var i=0; i<docs.length; i++) {
       if (docs[i]._id == undefined)
@@ -112,6 +112,7 @@
       json[option] = options[option];
     }
     this.last_req = this.request("POST", this.uri + "_bulk_docs", {
+      headers: {"Content-Type": "application/json"},
       body: JSON.stringify(json)
     });
     if (this.last_req.status == 417) {
 
=== added file 'modules/migration.jsm'
--- modules/migration.jsm	1970-01-01 00:00:00 +0000
+++ modules/migration.jsm	2011-03-04 13:02:54 +0000
@@ -0,0 +1,248 @@
+/*
+ * Copyright 2009-2011 Canonical Ltd.
+ *
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 3, as published
+ * by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranties of
+ * MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+ * PURPOSE.  See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+const EXPORTED_SYMBOLS = ['SchemaMigration'];
+
+const Cu = Components.utils;
+
+Cu.import("resource://bindwood/logging.jsm");
+
+
+function SchemaMigration(couch, profile) {
+    this.couch = couch;
+    this.profile = profile;
+}
+
+SchemaMigration.prototype = {
+
+    get_schema_version: function() {
+        var doc = this.couch.open('root_' + this.profile);
+
+        if (!doc) {
+            // If there is no root document, we're dealing with the
+            // original Bindwood datbase schema.
+            return 0;
+        }
+        // If the root document does not have a record_type_version,
+        // assume it is the second schema version.
+        if (!doc.record_type_version) {
+            return 1;
+        }
+        // Otherwise, use the value of record_type_version.
+        return doc.record_type_version;
+    },
+
+    upgrade: function() {
+        var schema_version = this.get_schema_version();
+        Log.debug("Current schema version is " + schema_version);
+        switch (schema_version) {
+        case 0:
+            // Haven't written migration code for this format, so just
+            // wipe out the bookmarks and assume that we can rebuild
+            // from the local places database.
+            this.wipe_bookmarks();
+            return true;
+        case 1:
+            this.upgrade_1_to_2();
+            return true;
+        case 2:
+            // Nothing to be done: we're at the most recent version.
+            return false;
+        default:
+            Log.error("Unknown schema version " + schema_version);
+            return false;
+        }
+    },
+
+    wipe_bookmarks: function() {
+        Log.debug("Wiping all bookmarks for profile " + this.profile);
+        var result = this.couch.view("bindwood/bookmarks", {
+            key: this.profile,
+            include_docs: true,
+        });
+        var changes = [];
+        for each (var row in result.rows) {
+            var doc = row.doc;
+            changes.push({
+                _id: doc._id,
+                _rev: doc._rev,
+                _deleted: true
+            });
+        }
+        // Delete all the bookmarks.
+        this.couch.bulkSave(changes);
+    },
+
+    /* The format used by Bindwood 0.4.2.  Bookmarks were represented as:
+     *
+     * {
+     *   _id: '...',
+     *   record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+     *   title: 'Bookmark title',
+     *   uri: 'http://www.example.com/',
+     *   application_annotations: {
+     *     Firefox: {
+     *       uuid: '...', // This is the UUID associated with the local item.
+     *       folder: 'Title of parent folder',
+     *       profile: 'profile name'
+     *     }
+     *   }
+     * }
+     *
+     * Folders and separators had the appropriate record types, and
+     * had no URI property.  Folder child ordering is not stored
+     * anywhere.
+     *
+     * There is no special handling of livemarks.  They are stored as
+     * folders, and their children are also stored.
+     *
+     * The special parent names "toolbarFolder", "bookmarksMenuFolder"
+     * and "unfiledBookmarksFolder" were used to represent items at
+     * the top levels of each folder hierarchy.
+     */
+
+    /* The format used by Bindwood 1.0.x.  Bookmarks look like this:
+     *
+     * {
+     *   _id: '...', // This is the UUID associated with the local item.
+     *   record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+     *   record_type_version: 1,
+     *   title: 'Bookmark title',
+     *   uri: 'http://www.example.com/',
+     *   application_annotations: {
+     *     Firefox: {
+     *       profile: 'profile name'
+     *       last_modified: ..., // time in microseconds since epoch
+     *     }
+     *   }
+     * }
+     *
+     * Folders contain a list of the IDs of their children (the
+     * opposite of the 0.4.2 format), which also acts as a way to
+     * store the child ordering.
+     *
+     * Livemarks are stored with their own record type.  They look
+     * similar to normal bookmarks, but store site_uri and feed_uri
+     * instead of a single bookmark URI.
+     */
+    upgrade_1_to_2: function() {
+        Log.debug("Upgrading bookmarks for profile " + this.profile +
+                  " from schema v1 to v2");
+        var result = this.couch.view("bindwood/bookmarks", {
+            key: this.profile,
+            include_docs: true,
+        });
+
+        var changes = [];
+
+        // Build a map of document IDs to the documents they reflect.
+        var docs_by_id = {};
+        for each (var row in result.rows) {
+            docs_by_id[row.id] = row.doc;
+        }
+
+        // Fix up children of the root document.  The children (which
+        // represent the toolbar, bookmarks menu and unfiled bookmarks
+        // folders) should be renamed to fixed identifiers.
+        var root_doc = docs_by_id['root_' + this.profile];
+        root_doc.record_type = 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/folder';
+        var expected_children = [
+            'toolbar_' + this.profile,
+            'menu_' + this.profile,
+            'unfiled_' + this.profile];
+        for (var i = 0; i < expected_children.length; i++) {
+            var old_id = root_doc.children[i];
+            var new_id = expected_children[i];
+            // Existing document has expected name.
+            if (old_id == new_id) {
+                continue;
+            }
+            var doc = docs_by_id[old_id];
+            delete docs_by_id[old_id];
+
+            // Schedule deletion of old document
+            changes.push({
+                _id: doc._id,
+                _rev: doc._rev,
+                _deleted: true
+            });
+            doc._id = new_id;
+            delete doc._rev;
+            docs_by_id[new_id] = doc;
+            root_doc.children[i] = new_id;
+        }
+
+        // Walk the bookmarks tree from the root node to determine
+        // which documents are reachable, and build up a parent map.
+        var reachable = {};
+        var parents_by_id = {};
+
+        var remaining = [root_doc];
+        reachable[root_doc._id] = true;
+        while (remaining.length > 0) {
+            var doc = remaining.pop();
+            for each (var child_id in doc.children) {
+                parents_by_id[child_id] = doc;
+                reachable[child_id] = true;
+
+                var child_doc = docs_by_id[child_id];
+                if (child_doc && child_doc.record_type == 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/folder') {
+                    remaining.push(child_doc);
+                }
+            }
+        }
+
+        // Now update the documents.
+        for each (var doc in docs_by_id) {
+            // If the document can not be reached from the root, then
+            // it does not belong in the document tree.
+            if (!reachable[doc._id]) {
+                Log.debug("Deleting bookmark " + doc._id +
+                          " is not reachable from root.");
+                changes.push({
+                    _id: doc._id,
+                    _rev: doc._rev,
+                    _deleted: true
+                });
+                continue;
+            }
+
+            doc.record_type_version = 2;
+            var parent_doc = parents_by_id[doc._id]
+            if (parent_doc) {
+                doc.parent_guid = parent_doc._id;
+                doc.parent_title = parent_doc.title;
+            }
+
+            // Bindwood 1.0.x would store a lot of useless data in the
+            // Firefox section of the document (basically anything
+            // that triggered an onItemChanged callback).  Keep only
+            // the information we care about.
+            var old_annotations = doc.application_annotations.Firefox;
+            var new_annotations = {
+                profile: old_annotations.profile,
+                last_modified: old_annotations.last_modified
+            };
+            doc.application_annotations.Firefox = new_annotations;
+
+            changes.push(doc);
+        }
+        // Now push the updates back to CouchDB.
+        Log.debug("Saving modifications to database.");
+        this.couch.bulkSave(changes);
+        Log.debug("Done.");
+    }
+};
 
=== modified file 'modules/sync.jsm'
--- modules/sync.jsm	2011-03-04 13:02:54 +0000
+++ modules/sync.jsm	2011-03-04 13:02:54 +0000
@@ -22,6 +22,7 @@
 const Cu = Components.utils;
 
 Cu.import("resource://bindwood/logging.jsm");
+Cu.import("resource://bindwood/migration.jsm");
 
 const GUID_ANNOTATION = "bindwood/uuid";
 const PARENT_ANNOTATION = "bindwood/parent";
@@ -33,6 +34,8 @@
 const TYPE_FEED = RECORD_PREFIX + "feed";
 const TYPE_SEPARATOR = RECORD_PREFIX + "separator";
 
+const RECORD_TYPE_VERSION = 2;
+
 var bookmarksService = Cc["@mozilla.org/browser/nav-bookmarks-service;1"]
     .getService(Ci.nsINavBookmarksService);
 var livemarkService = Cc["@mozilla.org/browser/livemark-service;2"]
@@ -62,6 +65,7 @@
         Log.debug("Initialising synchroniser.");
         this.ensureDatabase();
         this.ensureCouchViews();
+        this.maybeMigrateData();
         this.first_push = true;
         this.observer = new BookmarksObserver(this);
         bookmarksService.addObserver(this.observer, false);
@@ -132,6 +136,16 @@
         }
     },
 
+    maybeMigrateData: function() {
+        migration = new SchemaMigration(this.couch, this.profile);
+
+        if (migration.upgrade()) {
+            Log.debug("Schema was upgraded, so resetting last known " +
+                      "sequence number.");
+            this.last_seq = 0;
+        }
+    },
+
     set_guid: function(item_id, guid) {
         // If the item has already been annotated with a GUID, clear
         // the old GUID mapping from our cache.
@@ -543,6 +557,7 @@
             }
         }
 
+        _setattr(doc, "record_type_version", RECORD_TYPE_VERSION);
         if (!doc.application_annotations)
             doc.application_annotations = {};
         if (!doc.application_annotations.Firefox)
@@ -553,6 +568,8 @@
         if (parent_id > 0) {
             var parent_guid = this.guid_from_id(parent_id);
             _setattr(doc, "parent_guid", parent_guid);
+            _setattr(
+                doc, "parent_title", bookmarksService.getItemTitle(parent_id));
         }
         switch (item_type) {
         case bookmarksService.TYPE_BOOKMARK:
@@ -579,6 +596,7 @@
             break;
         case bookmarksService.TYPE_SEPARATOR:
             _setattr(doc, "record_type", TYPE_SEPARATOR);
+            _setattr(doc, "position", bookmarksService.getItemIndex(item_id));
             break;
         default:
             Log.error("Can not handle item " + item_id + " of type " +
 
=== modified file 'mozmill/shared-modules/bookmarks.js'
--- mozmill/shared-modules/bookmarks.js	2011-02-15 09:33:35 +0000
+++ mozmill/shared-modules/bookmarks.js	2011-03-04 13:02:54 +0000
@@ -62,8 +62,15 @@
     }
 }
 
+function setGuid(item_id, guid) {
+    annotationService.setItemAnnotation(
+        item_id, GUID_ANNOTATION, guid,
+        0, annotationService.EXPIRE_NEVER);
+}
+
 exports.bookmarksService = bookmarksService;
 exports.livemarkService = livemarkService;
 exports.annotationService = annotationService;
 exports.createURI = createURI;
 exports.clearBookmarks = clearBookmarks;
+exports.setGuid = setGuid;
 
=== added file 'mozmill/tests/test_migration.js'
--- mozmill/tests/test_migration.js	1970-01-01 00:00:00 +0000
+++ mozmill/tests/test_migration.js	2011-03-04 13:02:54 +0000
@@ -0,0 +1,425 @@
+var bm = require("../shared-modules/bookmarks");
+
+const TIMEOUT = 5000;
+
+const RECORD_PREFIX = (
+    "http://www.freedesktop.org/wiki/Specifications/desktopcouch/");
+const TYPE_BOOKMARK = RECORD_PREFIX + "bookmark";
+const TYPE_FOLDER = RECORD_PREFIX + "folder";
+const TYPE_FEED = RECORD_PREFIX + "feed";
+const TYPE_SEPARATOR = RECORD_PREFIX + "separator";
+
+
+var setupModule = function(module) {
+    module.controller = mozmill.getBrowserController();
+    module.jum = {};
+    module.desktopcouch = {};
+    module.sync = {};
+    module.migration = {};
+    Cu.import("resource://mozmill/modules/jum.js", module.jum);
+    Cu.import("resource://bindwood/desktopcouch.jsm", module.desktopcouch);
+    Cu.import("resource://bindwood/sync.jsm", module.sync);
+    Cu.import("resource://bindwood/migration.jsm", module.migration);
+    module.couch = null;
+    module.synchroniser = null;
+    bm.clearBookmarks();
+};
+
+
+var setupTest = function(test) {
+    var done = false;
+    desktopcouch.connect_desktopcouch("test_bookmarks", function(db) {
+            couch = db;
+            done = true;
+        }, function (message) {});
+    controller.waitFor(
+        function() { return done; }, "Could not connect to CouchDB", TIMEOUT);
+    jum.assertNotEquals(couch, null);
+
+    try {
+        couch.createDb();
+    } catch (e) {
+        if (e.error != 'file_exists')
+            throw(e);
+    }
+    synchroniser = new sync.Synchroniser(couch, "profile_name");
+    synchroniser.ensureCouchViews();
+};
+
+
+var teardownTest = function(test) {
+    synchroniser.uninit();
+    couch.deleteDb();
+    bm.clearBookmarks();
+};
+
+
+var test_get_schema_version = function() {
+    var sm = new migration.SchemaMigration(couch, 'profile_name');
+    // With no root document, the database must be from 0.4.2.
+    jum.assertEquals(sm.get_schema_version(), 0);
+
+    // Create a 1.0.x style root document.
+    var root_doc = {
+        _id: 'root_profile_name',
+        children: ['old_toolbar', 'old_menu', 'old_unfiled'],
+        application_annotations: {
+            Firefox: {
+                profile: 'profile_name',
+                last_modified: 1
+            }
+        }
+    };
+    couch.save(root_doc);
+    jum.assertEquals(sm.get_schema_version(), 1);
+
+    // Now modify it to match this version.
+    root_doc.record_type = TYPE_FOLDER;
+    root_doc.record_type_version = 2;
+    root_doc.children = [
+        'toolbar_profile_name', 'menu_profile_name', 'unfiled_profile_name'];
+    couch.save(root_doc);
+    jum.assertEquals(sm.get_schema_version(), 2);
+
+    // Future schema versions will update record_type_version.
+    root_doc.record_type_version = 42;
+    couch.save(root_doc);
+    jum.assertEquals(sm.get_schema_version(), 42);
+};
+
+
+// Build up bookmarks that match the sample data below.
+var make_sample_bookmarks = function() {
+    var folder1_id = bm.bookmarksService.createFolder(
+        bm.bookmarksService.toolbarFolder, 'Folder 1',
+        bm.bookmarksService.DEFAULT_INDEX);
+    bm.setGuid(folder1_id, 'folder1');
+
+    var livemark1_id = bm.livemarkService.createLivemark(
+        folder1_id, 'Livemark 1',
+        bm.createURI('http://www.example.com/'),
+        bm.createURI('http://www.example.com/feed'),
+        bm.bookmarksService.DEFAULT_INDEX);
+    bm.setGuid(livemark1_id, 'livemark1');
+
+    var bookmark1_id = bm.bookmarksService.insertBookmark(
+        bm.bookmarksService.toolbarFolder,
+        bm.createURI('http://www.example.com/bookmark1'),
+        bm.bookmarksService.DEFAULT_INDEX, 'Bookmark 1');
+    bm.setGuid(bookmark1_id, 'bookmark1');
+
+    var bookmark2_id = bm.bookmarksService.insertBookmark(
+        bm.bookmarksService.bookmarksMenuFolder,
+        bm.createURI('http://www.example.com/bookmark2'),
+        bm.bookmarksService.DEFAULT_INDEX, 'Bookmark 2');
+    bm.setGuid(bookmark2_id, 'bookmark2');
+
+    var separator1_id = bm.bookmarksService.insertSeparator(
+        bm.bookmarksService.bookmarksMenuFolder,
+        bm.bookmarksService.DEFAULT_INDEX);
+    bm.setGuid(separator1_id, 'separator1');
+
+    var bookmark3_id = bm.bookmarksService.insertBookmark(
+        bm.bookmarksService.unfiledBookmarksFolder,
+        bm.createURI('http://www.example.com/bookmark3'),
+        bm.bookmarksService.DEFAULT_INDEX, 'Bookmark 3');
+    bm.setGuid(bookmark3_id, 'bookmark3');
+};
+
+
+// Make Version 0 (Bindwood 0.4.2) sample data
+var make_sample_data_v0 = function() {
+    couch.bulkSave([
+        {
+            _id: 'couchdb_folder1',
+            record_type: TYPE_FOLDER,
+            title: 'Folder 1',
+            application_annotations: {
+                uuid: 'folder1',
+                folder: 'toolbarFolder',
+                profile: 'profile_name'
+            }
+        },
+        {
+            _id: 'couchdb_livemark1',
+            record_type: TYPE_FOLDER,
+            title: 'Livemark 1',
+            application_annotations: {
+                uuid: 'livemark1',
+                folder: 'Folder 1',
+                profile: 'profile_name'
+            }
+        },
+        {
+            _id: 'couchdb_bookmark1',
+            record_type: TYPE_BOOKMARK,
+            title: 'Bookmark 1',
+            uri: 'http://www.example.com/bookmark1',
+            application_annotations: {
+                uuid: 'bookmark1',
+                folder: 'toolbarFolder',
+                profile: 'profile_name'
+            }
+        },
+        {
+            _id: 'couchdb_bookmark2',
+            record_type: TYPE_BOOKMARK,
+            title: 'Bookmark 2',
+            uri: 'http://www.example.com/bookmark2',
+            application_annotations: {
+                uuid: 'bookmark2',
+                folder: 'bookmarksMenuFolder',
+                profile: 'profile_name'
+            }
+        },
+        {
+            _id: 'couchdb_separator1',
+            record_type: TYPE_SEPARATOR,
+            title: null,
+            application_annotations: {
+                uuid: 'separator1',
+                folder: 'bookmarksMenuFolder',
+                profile: 'profile_name'
+            }
+        },
+        {
+            _id: 'couchdb_bookmark3',
+            record_type: TYPE_BOOKMARK,
+            title: 'Bookmark 3',
+            uri: 'http://www.example.com/bookmark3',
+            application_annotations: {
+                uuid: 'bookmark3',
+                folder: 'unfiledBookmarksFolder',
+                profile: 'profile_name'
+            }
+        }
+    ]);
+};
+
+
+// Make Version 1 (Bindwood 1.0.x) sample data
+var make_sample_data_v1 = function() {
+    var common_annotations = {
+        Firefox: {
+            profile: 'profile_name',
+            last_modified: 1,
+            garbage: ''
+        }
+    };
+    couch.bulkSave([
+        {
+            _id: 'root_profile_name',
+            children: ['obsolete_toolbar', 'obsolete_menu', 'obsolete_unfiled'],
+            application_annotations: common_annotations
+        },
+        {
+            _id: 'obsolete_toolbar',
+            record_type: TYPE_FOLDER,
+            record_type_version: 1,
+            title: 'Bookmarks Toolbar',
+            children: ['folder1', 'bookmark1'],
+            application_annotations: common_annotations
+        },
+        {
+            _id: 'folder1',
+            record_type: TYPE_FOLDER,
+            record_type_version: 1,
+            title: 'Folder 1',
+            children: ['livemark1'],
+            application_annotations: common_annotations
+        },
+        {
+            _id: 'livemark1',
+            record_type: TYPE_FEED,
+            record_type_version: 1,
+            title: 'Livemark 1',
+            feed_uri: 'http://www.example.com/feed',
+            site_uri: 'http://www.example.com/',
+            application_annotations: common_annotations
+        },
+        {
+            _id: 'bookmark1',
+            record_type: TYPE_BOOKMARK,
+            record_type_version: 1,
+            title: 'Bookmark 1',
+            uri: 'http://www.example.com/bookmark1',
+            application_annotations: common_annotations
+        },
+        {
+            _id: 'obsolete_menu',
+            record_type: TYPE_FOLDER,
+            record_type_version: 1,
+            title: 'Bookmarks Menu',
+            children: ['bookmark2', 'separator1'],
+            application_annotations: common_annotations
+        },
+        {
+            _id: 'bookmark2',
+            record_type: TYPE_BOOKMARK,
+            record_type_version: 1,
+            title: 'Bookmark 2',
+            uri: 'http://www.example.com/bookmark2',
+            application_annotations: common_annotations
+        },
+        {
+            _id: 'separator1',
+            record_type: TYPE_SEPARATOR,
+            record_type_version: 1,
+            application_annotations: common_annotations
+        },
+        {
+            _id: 'obsolete_unfiled',
+            record_type: TYPE_FOLDER,
+            record_type_version: 1,
+            title: 'Unfiled Bookmarks',
+            children: ['bookmark3'],
+            application_annotations: common_annotations
+        },
+        {
+            _id: 'bookmark3',
+            record_type: TYPE_BOOKMARK,
+            record_type_version: 1,
+            title: 'Bookmark 3',
+            uri: 'http://www.example.com/bookmark3',
+            application_annotations: common_annotations
+        },
+        {
+            _id: 'orphaned_bookmark',
+            record_type: TYPE_BOOKMARK,
+            record_type_version: 1,
+            title: 'Orphaned Bookmark',
+            uri: 'http://www.example.com/orphan',
+            application_annotations: common_annotations
+        }
+    ]);
+};
+
+// Verify that the database contains expected v2 sample data.
+var verify_sample_data_v2 = function() {
+    var root = couch.open('root_profile_name');
+    jum.assertNotNull(root);
+    jum.assertEquals(root.record_type, TYPE_FOLDER);
+    jum.assertEquals(root.record_type_version, 2);
+    jum.assertEquals(root.children.length, 3);
+    jum.assertEquals(root.children[0], 'toolbar_profile_name');
+    jum.assertEquals(root.children[1], 'menu_profile_name');
+    jum.assertEquals(root.children[2], 'unfiled_profile_name');
+
+    var toolbar = couch.open('toolbar_profile_name');
+    jum.assertNotNull(toolbar);
+    jum.assertEquals(toolbar.record_type, TYPE_FOLDER);
+    jum.assertEquals(toolbar.record_type_version, 2);
+    jum.assertEquals(toolbar.parent_guid, 'root_profile_name');
+    jum.assertEquals(toolbar.children.length, 2);
+    jum.assertEquals(toolbar.children[0], 'folder1');
+    jum.assertEquals(toolbar.children[1], 'bookmark1');
+
+    var folder1 = couch.open('folder1');
+    jum.assertNotNull(folder1);
+    jum.assertEquals(folder1.record_type, TYPE_FOLDER);
+    jum.assertEquals(folder1.record_type_version, 2);
+    jum.assertEquals(folder1.parent_guid, 'toolbar_profile_name');
+    jum.assertEquals(folder1.title, 'Folder 1');
+    jum.assertEquals(folder1.children.length, 1);
+    jum.assertEquals(folder1.children[0], 'livemark1');
+
+    var livemark1 = couch.open('livemark1');
+    jum.assertNotNull(livemark1);
+    jum.assertEquals(livemark1.record_type, TYPE_FEED);
+    jum.assertEquals(livemark1.record_type_version, 2);
+    jum.assertEquals(livemark1.parent_guid, 'folder1');
+    jum.assertEquals(livemark1.title, 'Livemark 1');
+    jum.assertEquals(livemark1.site_uri, 'http://www.example.com/')
+    jum.assertEquals(livemark1.feed_uri, 'http://www.example.com/feed')
+
+    var bookmark1 = couch.open('bookmark1');
+    jum.assertNotNull(bookmark1);
+    jum.assertEquals(bookmark1.record_type, TYPE_BOOKMARK);
+    jum.assertEquals(bookmark1.record_type_version, 2);
+    jum.assertEquals(bookmark1.parent_guid, 'toolbar_profile_name');
+    jum.assertEquals(bookmark1.title, 'Bookmark 1');
+    jum.assertEquals(bookmark1.uri, 'http://www.example.com/bookmark1');
+
+    var menu = couch.open('menu_profile_name');
+    jum.assertNotNull(menu);
+    jum.assertEquals(menu.record_type, TYPE_FOLDER);
+    jum.assertEquals(menu.record_type_version, 2);
+    jum.assertEquals(menu.parent_guid, 'root_profile_name');
+    jum.assertEquals(menu.children.length, 2);
+    jum.assertEquals(menu.children[0], 'bookmark2');
+    jum.assertEquals(menu.children[1], 'separator1');
+
+    var bookmark2 = couch.open('bookmark2');
+    jum.assertNotNull(bookmark2);
+    jum.assertEquals(bookmark2.record_type, TYPE_BOOKMARK);
+    jum.assertEquals(bookmark2.record_type_version, 2);
+    jum.assertEquals(bookmark2.parent_guid, 'menu_profile_name');
+    jum.assertEquals(bookmark2.title, 'Bookmark 2');
+    jum.assertEquals(bookmark2.uri, 'http://www.example.com/bookmark2');
+
+    var separator1 = couch.open('separator1');
+    jum.assertNotNull(separator1);
+    jum.assertEquals(separator1.record_type, TYPE_SEPARATOR);
+    jum.assertEquals(separator1.record_type_version, 2);
+    jum.assertEquals(separator1.parent_guid, 'menu_profile_name');
+
+    var unfiled = couch.open('unfiled_profile_name');
+    jum.assertNotNull(unfiled);
+    jum.assertEquals(unfiled.record_type, TYPE_FOLDER);
+    jum.assertEquals(unfiled.record_type_version, 2);
+    jum.assertEquals(unfiled.parent_guid, 'root_profile_name');
+    jum.assertEquals(unfiled.children.length, 1);
+    jum.assertEquals(unfiled.children[0], 'bookmark3');
+
+    var bookmark3 = couch.open('bookmark3');
+    jum.assertNotNull(bookmark3);
+    jum.assertEquals(bookmark3.record_type, TYPE_BOOKMARK);
+    jum.assertEquals(bookmark3.record_type_version, 2);
+    jum.assertEquals(bookmark3.parent_guid, 'unfiled_profile_name');
+    jum.assertEquals(bookmark3.title, 'Bookmark 3');
+    jum.assertEquals(bookmark3.uri, 'http://www.example.com/bookmark3');
+};
+
+
+var test_upgrade_1_to_2 = function() {
+    make_sample_data_v1();
+    var sm = new migration.SchemaMigration(couch, 'profile_name');
+    sm.upgrade_1_to_2();
+
+    verify_sample_data_v2();
+
+    // The old documents for the special folders have been removed.
+    jum.assertNull(couch.open('obsolete_toolbar'));
+    jum.assertNull(couch.open('obsolete_menu'));
+    jum.assertNull(couch.open('obsolete_unfiled'));
+
+    // Unneeded attributes removed from annotation.
+    var bookmark1_doc = couch.open('bookmark1');
+    jum.assertUndefined(bookmark1_doc.application_annotations.Firefox.garbage);
+
+    // Orphaned items have been deleted.
+    jum.assertNull(couch.open('orphaned_bookmark'));
+};
+
+
+var test_migrate_v0 = function() {
+    make_sample_bookmarks();
+    make_sample_data_v0();
+
+    synchroniser.last_seq = couch.info().last_seq;
+    synchroniser.init();
+    synchroniser.sync();
+
+    verify_sample_data_v2();
+};
+
+var test_migrate_v1 = function() {
+    make_sample_bookmarks();
+    make_sample_data_v1();
+
+    synchroniser.last_seq = couch.info().last_seq;
+    synchroniser.init();
+    synchroniser.sync();
+
+    verify_sample_data_v2();
+};
 
=== modified file 'mozmill/tests/test_sync_all.js'
--- mozmill/tests/test_sync_all.js	2011-03-04 13:02:54 +0000
+++ mozmill/tests/test_sync_all.js	2011-03-04 13:02:54 +0000
@@ -100,10 +100,23 @@
 };
 
 var test_sync_remote = function() {
+    couch.save({
+        _id: 'root_profile_name',
+        record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/folder',
+        record_type_version: 2,
+        children: [
+            'toolbar_profile_name',
+            'menu_profile_name',
+            'unfiled_profile_name'
+        ]
+    });
+
     var item_guid = '12345';
     var doc = {
         _id: item_guid,
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        record_type_version: 2,
+        parent_title: 'Bookmarks Toolbar',
         parent_guid: 'toolbar_profile_name',
         title: 'Bookmark title',
         uri: LOCAL_TEST_PAGE,
@@ -173,6 +186,7 @@
     var doc = {
         _id: item_guid,
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        record_type_version: 2,
         parent_guid: 'toolbar_profile_name',
         title: 'Bookmark title',
         uri: LOCAL_TEST_PAGE,
 
=== modified file 'mozmill/tests/test_sync_from_couch.js'
--- mozmill/tests/test_sync_from_couch.js	2011-03-04 13:02:54 +0000
+++ mozmill/tests/test_sync_from_couch.js	2011-03-04 13:02:54 +0000
@@ -29,6 +29,7 @@
     synchroniser.processRecord({
         _id: '12345',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        record_type_version: 2,
         parent_guid: 'toolbar_profile_name',
         title: 'Bookmark title',
         uri: LOCAL_TEST_PAGE,
@@ -54,6 +55,7 @@
     doc = {
         _id: '12345',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        record_type_version: 2,
         parent_guid: 'toolbar_profile_name',
         title: 'Bookmark title',
         uri: LOCAL_TEST_PAGE,
@@ -82,6 +84,7 @@
     synchroniser.processRecord({
         _id: '12345',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/folder',
+        record_type_version: 2,
         parent_guid: 'toolbar_profile_name',
         title: 'Folder',
         children: [],
@@ -107,6 +110,7 @@
     doc = {
         _id: '12345',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/folder',
+        record_type_version: 2,
         parent_guid: 'toolbar_profile_name',
         title: 'Folder',
         children: [],
@@ -134,6 +138,7 @@
     synchroniser.processRecord({
         _id: '12345',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/feed',
+        record_type_version: 2,
         parent_guid: 'toolbar_profile_name',
         title: 'Livemark',
         site_uri: "http://www.example.com/",
@@ -161,6 +166,7 @@
     doc = {
         _id: '12345',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/feed',
+        record_type_version: 2,
         parent_guid: 'toolbar_profile_name',
         title: 'Livemark',
         site_uri: "http://www.example.com/",
@@ -190,6 +196,7 @@
     synchroniser.processRecord({
         _id: '12345',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/separator',
+        record_type_version: 2,
         parent_guid: 'toolbar_profile_name',
         application_annotations: {
             Firefox: {
@@ -209,6 +216,7 @@
     doc = {
         _id: '12345',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/separator',
+        record_type_version: 2,
         parent_guid: 'toolbar_profile_name',
         application_annotations: {
             Firefox: {
@@ -233,6 +241,7 @@
     doc = {
         _id: '12345',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        record_type_version: 2,
         parent_guid: 'toolbar_profile_name',
         title: 'Bookmark title',
         uri: LOCAL_TEST_PAGE,
@@ -262,6 +271,7 @@
     synchroniser.processRecord({
         _id: '12345',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/folder',
+        record_type_version: 2,
         parent_guid: 'toolbar_profile_name',
         title: 'Folder',
         children: ['67890'],
@@ -281,6 +291,7 @@
     synchroniser.processRecord({
         _id: '67890',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        record_type_version: 2,
         parent_guid: '12345',
         title: 'Bookmark title',
         uri: LOCAL_TEST_PAGE,
@@ -300,6 +311,7 @@
     synchroniser.processRecord({
         _id: '67890',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        record_type_version: 2,
         parent_guid: '12345',
         title: 'Bookmark title',
         uri: LOCAL_TEST_PAGE,
@@ -318,6 +330,7 @@
     synchroniser.processRecord({
         _id: '12345',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/folder',
+        record_type_version: 2,
         parent_guid: 'toolbar_profile_name',
         title: 'Folder',
         children: ['67890'],
@@ -342,6 +355,7 @@
     synchroniser.processRecord({
         _id: 'parent',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/folder',
+        record_type_version: 2,
         parent_guid: 'toolbar_profile_name',
         title: 'Folder',
         children: ['child1', 'child2', 'child3', 'missing_child'],
@@ -363,6 +377,7 @@
     synchroniser.processRecord({
         _id: 'child1',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        record_type_version: 2,
         parent_guid: 'parent',
         title: 'Child 1',
         uri: LOCAL_TEST_PAGE + "#child1",
@@ -376,6 +391,7 @@
     synchroniser.processRecord({
         _id: 'child3',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        record_type_version: 2,
         parent_guid: 'parent',
         title: 'Child 3',
         uri: LOCAL_TEST_PAGE + "#child1",
@@ -389,6 +405,7 @@
     synchroniser.processRecord({
         _id: 'child2',
         record_type: 'http://www.freedesktop.org/wiki/Specifications/desktopcouch/bookmark',
+        record_type_version: 2,
         parent_guid: 'parent',
         title: 'Child 2',
         uri: LOCAL_TEST_PAGE + "#child1",
 
=== modified file 'mozmill/tests/test_sync_to_couch.js'
--- mozmill/tests/test_sync_to_couch.js	2011-03-04 13:02:54 +0000
+++ mozmill/tests/test_sync_to_couch.js	2011-03-04 13:02:54 +0000
@@ -56,6 +56,7 @@
     var doc = couch.open(item_guid);
     jum.assertEquals(doc.record_type, "http://www.freedesktop.org/wiki/" +
                      "Specifications/desktopcouch/bookmark");
+    jum.assertEquals(doc.record_type_version, 2);
     jum.assertEquals(doc.application_annotations.Firefox.profile,
                      "profile_name");
     jum.assertEquals(doc.parent_guid, "toolbar_profile_name");
@@ -91,6 +92,7 @@
     var doc = couch.open(folder_guid);
     jum.assertEquals(doc.record_type, "http://www.freedesktop.org/wiki/" +
                      "Specifications/desktopcouch/folder");
+    jum.assertEquals(doc.record_type_version, 2);
     jum.assertEquals(doc.application_annotations.Firefox.profile,
                      "profile_name");
     jum.assertEquals(doc.parent_guid, "toolbar_profile_name");
@@ -128,6 +130,7 @@
     var doc = couch.open(item_guid);
     jum.assertEquals(doc.record_type, "http://www.freedesktop.org/wiki/" +
                      "Specifications/desktopcouch/feed");
+    jum.assertEquals(doc.record_type_version, 2);
     jum.assertEquals(doc.application_annotations.Firefox.profile,
                      "profile_name");
     jum.assertEquals(doc.parent_guid, "toolbar_profile_name");
@@ -157,6 +160,7 @@
     var doc = couch.open(item_guid);
     jum.assertEquals(doc.record_type, "http://www.freedesktop.org/wiki/" +
                      "Specifications/desktopcouch/separator");
+    jum.assertEquals(doc.record_type_version, 2);
     jum.assertEquals(doc.application_annotations.Firefox.profile,
                      "profile_name");
     jum.assertEquals(doc.parent_guid, "toolbar_profile_name");
@@ -191,6 +195,7 @@
     var doc = couch.open(root_guid);
     jum.assertEquals(doc.record_type, "http://www.freedesktop.org/wiki/" +
                      "Specifications/desktopcouch/folder");
+    jum.assertEquals(doc.record_type_version, 2);
     jum.assertUndefined(doc.parent_guid);
     jum.assertEquals(doc.children.length, 3);
     jum.assertEquals(doc.children[0], synchroniser.guid_from_id(