Merge lp:~verterok/ubuntuone-client/vm-refactor into lp:ubuntuone-client

[Guillermo Gonzalez]

VolumeManager Volume hierarchy refactor and metadata migration to new version

[Guillermo Gonzalez (verterok)]

This branch is the last of the VM refactor series:
 - implement new Volume hierarchy in VolumeManager
 - implenment VMFileShelf to pickle VM metadata as dict
 - add metadata migration to new version
 - refactor md migration tests:
  - fix a few bugs found (thanks to better tests) in previous metadata migration functions
 - update tools.py and dbus_interface.py
 - update tools, dbus and vm tests

[lp:~verterok/ubuntuone-client/vm-refactor]

328. By Guillermo Gonzalez on 2010-01-26

merge with trunk

329. By Guillermo Gonzalez on 2010-01-26

merge lp:~facundo/ubuntuone-client/second-inotify-processor

330. By Guillermo Gonzalez on 2010-01-26

fix tools.show_folders function to use volume_id instead of id

[Facundo Batista (facundo)]

===============================================================================
[ERROR]: tests.syncdaemon.test_u1sdtool.U1SDToolTests.test_show_folders

Traceback (most recent call last):
  File "/usr/lib/python2.6/dist-packages/twisted/internet/defer.py", line 749, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/usr/lib/python2.6/dist-packages/twisted/python/failure.py", line 338, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/facundo/canonical/u1-client/review_vm-refactor/tests/syncdaemon/test_u1sdtool.py", line 291, in test_show_folders
    yield d
  File "/usr/lib/python2.6/dist-packages/twisted/internet/defer.py", line 328, in _runCallbacks
    self.result = callback(self.result, *args, **kw)
  File "/home/facundo/canonical/u1-client/review_vm-refactor/tests/syncdaemon/test_u1sdtool.py", line 286, in <lambda>
    d.addCallback(lambda result: show_folders(result, out))
  File "/home/facundo/canonical/u1-client/review_vm-refactor/ubuntuone/syncdaemon/tools.py", line 618, in show_folders
    out.write(msg_template % (folder['id'], folder['subscribed'],
exceptions.KeyError: 'id'
-------------------------------------------------------------------------------

review: Needs Fixing

[Guillermo Gonzalez (verterok)]

Already pushed and fixed in revno: 330, thanks!

[Facundo Batista (facundo)]

Ok, that is already fixed, and all the tests went ok!

review: Approve

[Zachery Bir (urbanape)]

All tests pass. Gonna say go.

review: Approve

=== modified file 'tests/syncdaemon/test_dbus.py'
--- tests/syncdaemon/test_dbus.py	2010-01-25 22:28:46 +0000
+++ tests/syncdaemon/test_dbus.py	2010-01-26 20:35:29 +0000
@@ -36,7 +36,7 @@
     DBUS_IFACE_FOLDERS_NAME,
     EventListener,
 )
-from ubuntuone.syncdaemon.volume_manager import Share, UDF
+from ubuntuone.syncdaemon.volume_manager import Share, Shared, UDF
 from ubuntuone.syncdaemon.tools import DBusClient
 from ubuntuone.syncdaemon import event_queue, states, main, config
 from contrib.testing.testcase import (
@@ -500,21 +500,23 @@
                                      access_level='Read', accepted=False))
         client = DBusClient(self.bus, '/shares', DBUS_IFACE_SHARES_NAME)
         d = defer.Deferred()
-        def shares_handler(shares):
+        def check(shares):
             """ handle get_shares reply """
             self.assertEquals(1, len(shares))
             for share in shares:
-                if share['id'] == '':
-                    self.assertEquals('', str(share['id']))
+                if share['volume_id'] == '':
+                    self.assertEquals('', str(share['volume_id']))
                     self.assertEquals(self.root_dir, str(share['path']))
                     self.assertEquals('Modify', str(share['access_level']))
                     self.assertEquals('False', str(share['accepted']))
                 else:
-                    self.assertEquals('share_id', str(share['id']))
+                    self.assertEquals('share_id', str(share['volume_id']))
                     self.assertEquals(share_path, str(share['path']))
                     self.assertEquals('Read', str(share['access_level']))
                     self.assertEquals('False', str(share['accepted']))
-            d.callback(True)
+
+        def shares_handler(shares):
+            d.callback(shares)
 
         client.call_method('get_shares', reply_handler=shares_handler,
                            error_handler=self.error_handler)
@@ -539,7 +541,7 @@
         def check(result):
             """the async checker"""
             self.assertEquals('Yes', result['answer'])
-            self.assertEquals('share_id', result['share_id'])
+            self.assertEquals('share_id', result['volume_id'])
             self.assertEquals(True, self.main.vm.shares['share_id'].accepted)
 
         d.addCallback(check)
@@ -563,7 +565,7 @@
         def check(result):
             """the async checker"""
             self.assertEquals('No', result['answer'])
-            self.assertEquals('share_id', result['share_id'])
+            self.assertEquals('share_id', result['volume_id'])
             self.assertEquals(False, self.main.vm.shares['share_id'].accepted)
 
         d.addCallback(check)
@@ -648,8 +650,8 @@
             self.assertEquals(1, len(results))
             shared = results[0]
             self.assertEquals(a_dir, str(shared['path']))
-            self.assertEquals('node_id', str(shared['subtree']))
-            self.assertEquals('share_id', str(shared['id']))
+            self.assertEquals('node_id', str(shared['node_id']))
+            self.assertEquals('share_id', str(shared['volume_id']))
             self.assertEquals('View', str(shared['access_level']))
             d.callback(True)
         client.call_method('get_shared',
@@ -680,14 +682,15 @@
             self.assertEquals(1, len(results))
             shared = results[0]
             self.assertEquals('', str(shared['path']))
-            self.assertEquals('node_id', str(shared['subtree']))
-            self.assertEquals('share_id', str(shared['id']))
+            self.assertEquals('node_id', str(shared['node_id']))
+            self.assertEquals('share_id', str(shared['volume_id']))
             self.assertEquals('View', str(shared['access_level']))
             d.callback(True)
         client.call_method('get_shared',
                            reply_handler=reply_handler,
                            error_handler=self.error_handler)
         return d
+
     def test_refresh_shares(self):
         """ Just check that refresh_shares method API works. """
         client = DBusClient(self.bus, '/shares', DBUS_IFACE_SHARES_NAME)
@@ -800,7 +803,7 @@
         a_dir = os.path.join(self.root_dir, u'ñoño'.encode('utf-8'))
         self.fs_manager.create(a_dir, "", is_dir=True)
         self.fs_manager.set_node_id(a_dir, "node_id")
-        share = Share(path=a_dir, volume_id='shared_id', name=u'ñoño_shared',
+        share = Shared(path=a_dir, volume_id='shared_id', name=u'ñoño_shared',
                       access_level='View', other_username=u'test_username',
                       node_id='node_id')
         self.main.vm.add_shared(share)
@@ -812,8 +815,8 @@
             self.assertEquals(1, len(results))
             shared = results[0]
             self.assertEquals(a_dir, shared['path'].encode('utf-8'))
-            self.assertEquals('node_id', str(shared['subtree']))
-            self.assertEquals('shared_id', str(shared['id']))
+            self.assertEquals('node_id', str(shared['node_id']))
+            self.assertEquals('shared_id', str(shared['volume_id']))
             self.assertEquals('View', str(shared['access_level']))
 
         d.addCallback(check)
@@ -1072,14 +1075,15 @@
                                                      u'visible_name', 'Write')
 
         d = defer.Deferred()
-        def share_handler(share):
+        def check(share):
             """ handler for ShareChanged signal. """
-            self.assertEquals('a_share_id', str(share['id']))
+            self.assertEquals('a_share_id', str(share['volume_id']))
             self.assertEquals(share_path, str(share['path']))
             self.assertEquals('Write', str(share['access_level']))
-            self.assertEquals('False', str(share['accepted']))
-            d.callback(True)
-
+            self.assertEquals('', str(share['accepted']))
+        d.addCallback(check)
+        def share_handler(result):
+            d.callback(result)
         match = self.bus.add_signal_receiver(share_handler,
                                      signal_name='ShareChanged')
         self.signal_receivers.add(match)
@@ -1089,25 +1093,36 @@
     def test_share_deleted(self):
         """ Test the ShareDeleted signal. """
         share_path = os.path.join(self.main.shares_dir, 'share')
-        self.main.vm.add_share(Share(path=share_path, volume_id='a_share_id',
-                                     access_level='Read', accepted=False))
         share_holder = NotifyShareHolder.from_params('a_share_id', 'subtree',
                                                      u'fake_share',
                                                      u'test_username',
                                                      u'visible_name', 'Read')
 
+        self.main.vm.add_share(Share.from_notify_holder(share_holder, share_path))
         d = defer.Deferred()
         def share_handler(share_dict):
             """ handler for ShareDeletedsignal. """
-            expected_dict = dict(share_id='a_share_id',
-                                 subtree='subtree',
-                                 share_name=u'fake_share',
-                                 from_username=u'test_username',
-                                 from_visible_name=u'visible_name',
+            d.callback(share_dict)
+
+        match = self.bus.add_signal_receiver(share_handler,
+                                     signal_name='ShareDeleted')
+        self.signal_receivers.add(match)
+
+        def check(share_dict):
+            """Check the result."""
+            expected_dict = dict(volume_id='a_share_id',
+                                 node_id='subtree',
+                                 name=u'fake_share',
+                                 other_username=u'test_username',
+                                 other_visible_name=u'visible_name',
+                                 free_bytes='',
+                                 path=share_path,
+                                 accepted='',
                                  access_level='Read')
+            expected_dict['type'] = 'Share'
             for k, v in share_dict.items():
                 self.assertEquals(expected_dict[str(k)], str(v))
-            d.callback(True)
+        d.addCallback(check)
 
         match = self.bus.add_signal_receiver(share_handler,
                                      signal_name='ShareDeleted')
@@ -1495,7 +1510,7 @@
         udf_dict = self.dbus_iface.folders._get_udf_dict(udf)
         # check the path it's unicode
         self.assertEquals(udf_dict['path'], udf.path.decode('utf-8'))
-        self.assertEquals(udf_dict['id'], udf.volume_id)
+        self.assertEquals(udf_dict['volume_id'], udf.id)
         self.assertEquals(udf_dict['suggested_path'], udf.suggested_path)
         self.assertEquals(udf_dict['node_id'], udf.node_id)
         self.assertFalse(udf_dict['subscribed'])
@@ -1684,13 +1699,13 @@
         d = defer.Deferred()
         def delete_volume(path):
             """Fake delete_volume"""
-            self.main.event_q.push("AQ_DELETE_VOLUME_OK", volume_id=udf.volume_id)
+            self.main.event_q.push("AQ_DELETE_VOLUME_OK", volume_id=udf.id)
         self.main.action_q.delete_volume = delete_volume
         def deleted_handler(info):
             """FolderDeleted handler."""
             self.assertRaises(KeyError, self.main.fs.get_by_path,
                               info['path'].decode('utf-8'))
-            self.assertRaises(KeyError, self.main.vm.get_volume, info['id'])
+            self.assertRaises(KeyError, self.main.vm.get_volume, info['volume_id'])
             d.callback(True)
         match = self.bus.add_signal_receiver(deleted_handler,
                                              signal_name='FolderDeleted')
@@ -1719,7 +1734,7 @@
         self.main.action_q.delete_volume = delete_volume
         def deleted_error_handler(info, error):
             """FolderDeleteError handler"""
-            self.assertEquals(info['id'], udf.volume_id)
+            self.assertEquals(info['volume_id'], udf.volume_id)
             self.assertEquals(error, "I'm broken")
             d.callback(True)
         match = self.bus.add_signal_receiver(deleted_error_handler,
@@ -1821,7 +1836,7 @@
             """FolderDeleted handler."""
             self.assertRaises(KeyError, self.main.fs.get_by_path,
                               info['path'].decode('utf-8'))
-            self.assertRaises(KeyError, self.main.vm.get_volume, info['id'])
+            self.assertRaises(KeyError, self.main.vm.get_volume, info['volume_id'])
             d.callback(True)
         match = self.bus.add_signal_receiver(deleted_handler,
                                              signal_name='ShareDeleted')
@@ -1852,7 +1867,7 @@
         self.main.action_q.delete_volume = delete_volume
         def deleted_error_handler(info, error):
             """FolderDeleteError handler"""
-            self.assertEquals(info['id'], share.volume_id)
+            self.assertEquals(info['volume_id'], share.volume_id)
             self.assertEquals(error, "I'm broken")
             d.callback(True)
         match = self.bus.add_signal_receiver(deleted_error_handler,

=== modified file 'tests/syncdaemon/test_eq_inotify.py'
--- tests/syncdaemon/test_eq_inotify.py	2010-01-26 19:15:10 +0000
+++ tests/syncdaemon/test_eq_inotify.py	2010-01-26 20:35:29 +0000
@@ -33,31 +33,104 @@
 class WatchTests(BaseEQTestCase):
     """Test the EQ API to add and remove watchs."""
 
-    def test_add_watch(self):
-        """Test that watchs can be added."""
+    def _create_udf(self, path):
+        """Create an UDF and returns it and the volume"""
+        os.makedirs(path)
+        udf = volume_manager.UDF("vol_id", "node_id", path, path, True)
+        self.vm.add_udf(udf)
+
+    def test_add_general_watch(self):
+        """Test that general watchs can be added."""
         # we should have what we asked for
         self.eq.inotify_add_watch(self.root_dir)
-        # pylint: disable-msg=W0212
-        self.assertTrue(self.root_dir in self.eq._watchs)
-
-        # we shouldn't have other stuff
-        self.assertTrue("not-added-dir" not in self.eq._watchs)
-
-    def test_rm_watch(self):
-        """Test that watchs can be removed."""
-        # remove what we added
+
+        # check only added dir in watchs
+        # pylint: disable-msg=W0212
+        self.assertTrue(self.root_dir in self.eq._general_watchs)
+        self.assertTrue("not-added-dir" not in self.eq._general_watchs)
+
+        # nothing in the udf ancestors watch
+        self.assertEqual(self.eq._ancestors_watchs, {})
+
+    def test_add_watch_on_udf_ancestor(self):
+        """Test that ancestors watchs can be added."""
+        # create the udf and add the watch
+        path_udf = os.path.join(self.home_dir, "path/to/UDF")
+        self._create_udf(path_udf)
+        path_ancestor = os.path.join(self.home_dir, "path")
+        self.eq.inotify_add_watch(path_ancestor)
+
+        # check only added dir in watchs
+        # pylint: disable-msg=W0212
+        self.assertTrue(path_ancestor in self.eq._ancestors_watchs)
+        self.assertTrue("not-added-dir" not in self.eq._ancestors_watchs)
+
+        # nothing in the general watch
+        self.assertEqual(self.eq._general_watchs, {})
+
+    def test_add_watch_on_udf_exact(self):
+        """Test adding the watch exactly on UDF."""
+        # create the udf and add the watch
+        path_udf = os.path.join(self.home_dir, "path/to/UDF")
+        self._create_udf(path_udf)
+        self.eq.inotify_add_watch(path_udf)
+
+        # pylint: disable-msg=W0212
+        self.assertTrue(path_udf in self.eq._general_watchs)
+        self.assertEqual(self.eq._ancestors_watchs, {})
+
+    def test_add_watch_on_udf_child(self):
+        """Test adding the watch inside UDF."""
+        # create the udf and add the watch
+        path_udf = os.path.join(self.home_dir, "path/to/UDF")
+        self._create_udf(path_udf)
+        path_ancestor = os.path.join(self.home_dir, "path/to/UDF/inside")
+        os.mkdir(path_ancestor)
+        self.eq.inotify_add_watch(path_ancestor)
+
+        # pylint: disable-msg=W0212
+        self.assertTrue(path_ancestor in self.eq._general_watchs)
+        self.assertEqual(self.eq._ancestors_watchs, {})
+
+    def test_rm_watch_wrong(self):
+        """Test that general watchs can be removed."""
+        # add two types of watchs
         self.eq.inotify_add_watch(self.root_dir)
-        self.eq.inotify_rm_watch(self.root_dir)
-        # pylint: disable-msg=W0212
-        self.assertTrue(self.root_dir not in self.eq._watchs)
+        path_udf = os.path.join(self.home_dir, "path/to/UDF")
+        self._create_udf(path_udf)
+        path_ancestor = os.path.join(self.home_dir, "path")
+        self.eq.inotify_add_watch(path_ancestor)
 
         # remove different stuff
-        self.eq.inotify_add_watch(self.root_dir)
         self.assertRaises(ValueError,
                           self.eq.inotify_rm_watch, "not-added-dir")
 
-    def test_has_watch(self):
-        """Test that a path is watched."""
+    def test_rm_watch_general(self):
+        """Test that general watchs can be removed."""
+        # remove what we added
+        self.eq.inotify_add_watch(self.root_dir)
+        self.eq.inotify_rm_watch(self.root_dir)
+
+        # pylint: disable-msg=W0212
+        self.assertEqual(self.eq._general_watchs, {})
+        self.assertEqual(self.eq._ancestors_watchs, {})
+
+    def test_rm_watch_ancestor(self):
+        """Test that ancestor watchs can be removed."""
+        # create the udf and add the watch
+        path_udf = os.path.join(self.home_dir, "path/to/UDF")
+        self._create_udf(path_udf)
+        path_ancestor = os.path.join(self.home_dir, "path")
+        self.eq.inotify_add_watch(path_ancestor)
+
+        # remove what we added
+        self.eq.inotify_rm_watch(path_ancestor)
+        # pylint: disable-msg=W0212
+        self.assertEqual(self.eq._general_watchs, {})
+        self.assertEqual(self.eq._ancestors_watchs, {})
+
+    def test_has_watch_general(self):
+        """Test that a general path is watched."""
         self.assertFalse(self.eq.inotify_has_watch(self.root_dir))
 
         # add
@@ -68,6 +141,24 @@
         self.eq.inotify_rm_watch(self.root_dir)
         self.assertFalse(self.eq.inotify_has_watch(self.root_dir))
 
+    def test_has_watch_ancestor(self):
+        """Test that an ancestor path is watched."""
+        path_udf = os.path.join(self.home_dir, "path/to/UDF")
+        self._create_udf(path_udf)
+        path_ancestor = os.path.join(self.home_dir, "path")
+
+        self.assertFalse(self.eq.inotify_has_watch(path_ancestor))
+
+        # add
+        # create the udf and add the watch
+        self.eq.inotify_add_watch(path_ancestor)
+        self.assertTrue(self.eq.inotify_has_watch(path_ancestor))
+
+        # remove
+        self.eq.inotify_rm_watch(path_ancestor)
+        self.assertFalse(self.eq.inotify_has_watch(path_ancestor))
+
+
 class DynamicHitMe(object):
     """Helper class to test a sequence of signals."""
 
@@ -1157,7 +1248,6 @@
 
         @param msg: A string describing the failure that's included in the
             exception.
-
         """
         if not first == second:
             if msg is None:
@@ -1246,9 +1336,6 @@
                                        suggested_path, path, True)
         other_ancestors = other_udf.ancestors
 
-        # pylint: disable-msg=W0212
-        assert not self.eq._processor._is_udf_ancestor(path)
-
         os.makedirs(path)
         # every ancestor has a watch already, added by LocalRescan. Copy that.
         self.eq.inotify_add_watch(other_udf.path)

=== modified file 'tests/syncdaemon/test_eventqueue.py'
--- tests/syncdaemon/test_eventqueue.py	2009-11-20 22:00:25 +0000
+++ tests/syncdaemon/test_eventqueue.py	2010-01-26 20:35:29 +0000
@@ -39,9 +39,11 @@
         self.fsmdir = self.mktemp('fsmdir')
         self.partials_dir = self.mktemp('partials_dir')
         self.root_dir = self.mktemp('root_dir')
+        self.home_dir = self.mktemp('home_dir')
+        self.vm = testcase.FakeVolumeManager(self.root_dir)
         self.fs = filesystem_manager.FileSystemManager(self.fsmdir,
                                                        self.partials_dir,
-                                   testcase.FakeVolumeManager(self.root_dir))
+                                                       self.vm)
         self.fs.create(path=self.root_dir,
                        share_id='', is_dir=True)
         self.fs.set_by_path(path=self.root_dir,

=== modified file 'tests/syncdaemon/test_tools.py'
--- tests/syncdaemon/test_tools.py	2010-01-26 19:11:29 +0000
+++ tests/syncdaemon/test_tools.py	2010-01-26 20:35:29 +0000
@@ -182,7 +182,7 @@
         def check(result):
             """do the asserts"""
             self.assertEquals('Yes', result['answer'])
-            self.assertEquals('share_id', result['share_id'])
+            self.assertEquals('share_id', result['volume_id'])
             self.assertEquals(True, self.main.vm.shares['share_id'].accepted)
 
         d.addCallback(check)
@@ -199,7 +199,7 @@
         def check(result):
             """do the asserts"""
             self.assertEquals('No', result['answer'])
-            self.assertEquals('share_id', result['share_id'])
+            self.assertEquals('share_id', result['volume_id'])
             self.assertEquals(False, self.main.vm.shares['share_id'].accepted)
 
         d.addCallback(check)

=== modified file 'tests/syncdaemon/test_vm.py'
--- tests/syncdaemon/test_vm.py	2010-01-26 19:40:05 +0000
+++ tests/syncdaemon/test_vm.py	2010-01-26 20:35:29 +0000
@@ -21,7 +21,6 @@
 import logging
 import os
 import uuid
-import sys
 
 from ubuntuone.storageprotocol.client import ListShares, ListVolumes
 from ubuntuone.storageprotocol.sharersp import (
@@ -36,10 +35,16 @@
 )
 from ubuntuone.syncdaemon.volume_manager import (
     Share,
+    Shared,
+    UDF,
+    Root,
+    _Share,
+    _UDF,
     allow_writes,
-    UDF,
     VolumeManager,
     LegacyShareFileShelf,
+    MetadataUpgrader,
+    VMFileShelf,
 )
 from twisted.internet import defer, reactor
 
@@ -1373,64 +1378,118 @@
         self.assertTrue(isinstance(share.node_id, basestring))
 
 
-class ShareShelfUpgradeTests(BaseTwistedTestCase):
-    """ Tests for shares shelf upgrades"""
+class MetadataTestCase(BaseTwistedTestCase):
+    md_version_None = False
+    main = None
+    data_dir = None
+    share_md_dir = None
+    shared_md_dir = None
+    partials_dir = None
+    u1_dir = None
+    root_dir = None
+    shares_dir = None
+    shares_dir_link = None
 
     def setUp(self):
-        """ setup the test """
+        """Create some directories."""
         BaseTwistedTestCase.setUp(self)
-        self.root_dir = self.mktemp('Ubuntu One')
-        self.data_dir = self.mktemp('data_dir')
-        self.partials_dir = self.mktemp('partials_dir')
-        self.shares_dir = self.mktemp(os.path.join('Ubuntu One',
-                                                   'Shared with Me'))
+        self.data_dir = os.path.join(self.tmpdir, 'data_dir')
+        self.vm_data_dir = os.path.join(self.tmpdir, 'data_dir', 'vm')
+        self.partials_dir = self.mktemp('partials')
+        self.u1_dir = os.path.join(self.tmpdir, 'Ubuntu One')
+        self.version_file = os.path.join(self.vm_data_dir, '.version')
 
     def tearDown(self):
-        """Cleanup main and remove the temp dir."""
-        main = getattr(self, 'main', None)
-        if main:
-            main.shutdown()
-        if os.path.exists(self.root_dir):
-            self.rmtree(self.root_dir)
-        if os.path.exists(self.data_dir):
-            self.rmtree(self.data_dir)
-        if os.path.exists(self.shares_dir):
-            self.rmtree(self.shares_dir)
+        """Cleanup all the cruft."""
+        for path in [self.data_dir, self.partials_dir, self.root_dir,
+                     self.shares_dir]:
+            if path and os.path.exists(path):
+                self.rmtree(path)
+        if self.main:
+            self.main.shutdown()
         VolumeManager.METADATA_VERSION = CURRENT_METADATA_VERSION
-        return BaseTwistedTestCase.tearDown(self)
+        BaseTwistedTestCase.tearDown(self)
 
     def check_version(self):
-        """ check if the current version in the .version file is the lastone.
-        """
-        with open(os.path.join(self.data_dir, 'vm', '.version'), 'r') as fd:
+        """Check if the current version in the .version file is the last one."""
+        with open(self.version_file, 'r') as fd:
             self.assertEquals(CURRENT_METADATA_VERSION, fd.read().strip())
 
-    def test_0_to_1(self):
-        """ Test the upgrade from the first shelf layout version to v. 1"""
-        # ensure a clean data_dir
-        self.rmtree(self.data_dir)
-        vm_data_dir = os.path.join(self.data_dir, 'vm')
-        old_shelf = LegacyShareFileShelf(vm_data_dir)
+    def set_md_version(self, md_version):
+        """Write md_version to the .version file."""
+        if not os.path.exists(self.vm_data_dir):
+            os.makedirs(self.vm_data_dir)
+        with open(self.version_file, 'w') as fd:
+            fd.write(md_version)
+
+
+class MetadataOldLayoutTests(MetadataTestCase):
+    """Tests for 'old' layouts and metadata upgrade"""
+
+    def setUp(self):
+        MetadataTestCase.setUp(self)
+        self.root_dir = os.path.join(self.u1_dir, 'My Files')
+        self.shares_dir = os.path.join(self.u1_dir, 'Shared With Me')
+        self.new_root_dir = self.u1_dir
+        self.new_shares_dir = self.mktemp('shares_dir')
+
+    def tearDown(self):
+        """Cleanup all the cruft."""
+        for path in [self.u1_dir, self.new_shares_dir]:
+            if path and os.path.exists(path):
+                self.rmtree(path)
+        MetadataTestCase.tearDown(self)
+
+    def _build_layout_version_0(self):
+        """Build the dir structure to mimic md v.0/None."""
+        self.share_md_dir = os.path.join(self.tmpdir, 'data_dir', 'vm')
+        os.makedirs(self.share_md_dir)
+        os.makedirs(self.root_dir)
+        os.makedirs(self.shares_dir)
+
+    def _build_layout_version_1(self):
+        """Build the dir structure to mimic md v.1"""
+        self.share_md_dir = os.path.join(self.vm_data_dir, 'shares')
+        self.shared_md_dir = os.path.join(self.vm_data_dir, 'shared')
+        os.makedirs(self.share_md_dir)
+        os.makedirs(self.shared_md_dir)
+        os.makedirs(self.root_dir)
+        os.makedirs(self.shares_dir)
+
+    def _set_permissions(self):
+        """Set the RO perms in the root and the shares directory."""
+        os.chmod(self.shares_dir, 0500)
+        os.chmod(self.u1_dir, 0500)
+
+    def test_upgrade_0(self):
+        """Test the upgrade from the first shelf layout version."""
+        self._build_layout_version_0()
+        old_shelf = LegacyShareFileShelf(self.share_md_dir)
         # add the root_uuid key
-        root_share = Share(path=self.root_dir)
+        root_share = _Share(path=self.root_dir)
         root_share.access_level = 'Modify'
         old_shelf[''] = root_share
         for idx in range(1, 10):
-            old_shelf[str(uuid.uuid4())] = \
-                    Share(path=os.path.join(self.shares_dir, str(idx)))
-        # LegacyShareFileShelf.keys returns a generator
+            sid = str(uuid.uuid4())
+            old_shelf[sid] =  _Share(path=os.path.join(self.shares_dir, str(idx)),
+                                     share_id=sid)
+        # ShareFileShelf.keys returns a generator
         old_keys = [key for key in old_shelf.keys()]
         self.assertEquals(10, len(old_keys))
+        if self.md_version_None:
+            self.set_md_version('')
+        # set the ro permissions
+        self._set_permissions()
         # we want to keep a refernece to main in order to shutdown
         # pylint: disable-msg=W0201
-        self.main = FakeMain(self.root_dir, self.shares_dir,
+        self.main = FakeMain(self.new_root_dir, self.new_shares_dir,
                              self.data_dir, self.partials_dir)
         new_keys = [new_key for new_key in self.main.vm.shares.keys()]
         self.assertEquals(10, len(new_keys))
         for new_key in new_keys:
             self.assertIn(new_key, old_keys)
         # check the old data is still there (in the backup)
-        backup_shelf = LegacyShareFileShelf(os.path.join(vm_data_dir, '0.bkp'))
+        backup_shelf = LegacyShareFileShelf(os.path.join(self.vm_data_dir, '0.bkp'))
         backup_keys = [key for key in backup_shelf.keys()]
         for old_key in old_keys:
             self.assertIn(old_key, backup_keys)
@@ -1438,18 +1497,13 @@
             self.assertIn(new_key, backup_keys)
         self.check_version()
 
-    def test_1_to_2(self):
-        """ Test the upgrade from v.1 of the metadata to v.2"""
-        # ensure a clean data_dir
-        self.rmtree(self.data_dir)
-        vm_data_dir = os.path.join(self.data_dir, 'vm')
-        vm_shares_dir = os.path.join(vm_data_dir, 'shares')
-        os.makedirs(vm_data_dir)
+    def test_upgrade_1(self):
+        """ Test the upgrade from v.1"""
+        self._build_layout_version_1()
         # write the .version file with v.1
-        with open(os.path.join(vm_data_dir, '.version'), 'w') as fd:
-            fd.write('1')
+        self.set_md_version('1')
 
-        share_file = os.path.join(vm_shares_dir,
+        share_file = os.path.join(self.share_md_dir,
                                   '0/6/6/0664f050-9254-45c5-9f31-3482858709e4')
         os.makedirs(os.path.dirname(share_file))
         # this is the str of a version 2 pickle
@@ -1465,222 +1519,228 @@
         with open(share_file, 'w') as fd:
             fd.write(share_value)
 
-        # fake the old namespace
-        sys.modules['canonical.ubuntuone.storage.syncdaemon.volume_manager'] = \
-                sys.modules['ubuntuone.syncdaemon.volume_manager']
         # try to load the shelf
-        old_shelf = LegacyShareFileShelf(vm_shares_dir)
+        old_shelf = LegacyShareFileShelf(self.share_md_dir)
         share = old_shelf['0664f050-9254-45c5-9f31-3482858709e4']
         self.assertTrue(share is not None)
-        del sys.modules['canonical.ubuntuone.storage.syncdaemon.volume_manager']
+        if self.md_version_None:
+            self.set_md_version('')
+
+        self._set_permissions()
         # now use the real VolumeManager
         # we want to keep a refernece to main in order to shutdown
         # pylint: disable-msg=W0201
-        self.main = FakeMain(self.root_dir, self.shares_dir,
+        self.main = FakeMain(self.new_root_dir, self.new_shares_dir,
                              self.data_dir, self.partials_dir)
         new_keys = [new_key for new_key in self.main.vm.shares.keys()]
         self.assertEquals(2, len(new_keys)) # the fake share plus root
-        for key in ['', share.volume_id]:
+        for key in ['', share.id]:
             self.assertIn(key, new_keys)
         self.check_version()
 
-    def test_2_to_3(self):
-        """ Test the upgrade from v.2 of the metadata to v.3"""
-        vm_data_dir = os.path.join(self.data_dir, 'vm')
-        os.makedirs(vm_data_dir)
-        with open(os.path.join(vm_data_dir, '.version'), 'w') as fd:
-            fd.write('2')
+    def test_upgrade_2(self):
+        """Test the upgrade from v.2."""
+        self._build_layout_version_1()
+        self.set_md_version('2')
         open(self.root_dir + '/foo.conflict', 'w').close()
         open(self.root_dir + '/foo.conflict.23', 'w').close()
         open(self.shares_dir + '/bar.partial', 'w').close()
         os.mkdir(self.shares_dir + '/baz/')
         open(self.shares_dir + '/baz/baz.conflict', 'w').close()
         os.chmod(self.shares_dir + '/baz/', 0500)
-        self.main = FakeMain(self.root_dir, self.shares_dir,
+        if self.md_version_None:
+            self.set_md_version('')
+        self._set_permissions()
+        self.main = FakeMain(self.new_root_dir, self.new_shares_dir,
                              self.data_dir, self.partials_dir)
-        self.assertTrue(os.path.exists(self.root_dir + '/foo.u1conflict'))
-        self.assertTrue(os.path.exists(self.root_dir + '/foo.u1conflict.23'))
-        self.assertTrue(os.path.exists(self.shares_dir + '/.u1partial.bar'))
-        self.assertTrue(os.path.exists(self.shares_dir + '/baz/baz.u1conflict'))
+        self.assertTrue(os.path.exists(self.new_root_dir + '/foo.u1conflict'))
+        self.assertTrue(os.path.exists(self.new_root_dir + '/foo.u1conflict.23'))
+        self.assertTrue(os.path.exists(self.new_shares_dir + '/.u1partial.bar'))
+        self.assertTrue(os.path.exists(self.new_shares_dir + '/baz/baz.u1conflict'))
+        self.check_version()
 
-    def test_2_to_3_more(self):
-        """ Test the upgrade from v.2 of the metadata to v.3 some more"""
-        vm_data_dir = os.path.join(self.data_dir, 'vm')
-        os.makedirs(vm_data_dir)
-        with open(os.path.join(vm_data_dir, '.version'), 'w') as fd:
-            fd.write('2')
+    def test_upgrade_2_more(self):
+        """Test the upgrade from v.2 some more."""
+        self._build_layout_version_1()
+        self.set_md_version('2')
 
         expected = []
 
-        for dirname in self.root_dir, self.shares_dir:
+        for dirname, new_dirname in [(self.root_dir, self.new_root_dir),
+                                          (self.shares_dir, self.new_shares_dir)]:
             # a plain .conflict...
             # ...on a file
             open(dirname + '/1a.conflict', 'w').close()
-            expected.append(dirname + '/1a.u1conflict')
+            expected.append(new_dirname + '/1a.u1conflict')
             # ...on an empty directory
             os.mkdir(dirname + '/1b.conflict')
-            expected.append(dirname + '/1b.u1conflict')
+            expected.append(new_dirname + '/1b.u1conflict')
             # ...on a directory with content
             os.mkdir(dirname + '/1c.conflict')
             os.mkdir(dirname + '/1c.conflict/1c')
-            expected.append(dirname + '/1c.u1conflict/1c')
+            expected.append(new_dirname + '/1c.u1conflict/1c')
             # ...in a readonly directory
             os.mkdir(dirname + '/1d')
             os.mkdir(dirname + '/1d/1d.conflict')
             os.chmod(dirname + '/1d', 0500)
-            expected.append(dirname + '/1d/1d.u1conflict')
+            expected.append(new_dirname + '/1d/1d.u1conflict')
             # ...in a directory that is also a .conflict
             os.mkdir(dirname + '/1e.conflict')
             os.mkdir(dirname + '/1e.conflict/1e.conflict')
-            expected.append(dirname + '/1e.u1conflict/1e.u1conflict')
+            expected.append(new_dirname + '/1e.u1conflict/1e.u1conflict')
 
             # a numbered .conflict...
             # ...on a file
             open(dirname + '/2a.conflict.2', 'w').close()
-            expected.append(dirname + '/2a.u1conflict.2')
+            expected.append(new_dirname + '/2a.u1conflict.2')
             # ...on an empty directory
             os.mkdir(dirname + '/2b.conflict.3')
-            expected.append(dirname + '/2b.u1conflict.3')
+            expected.append(new_dirname + '/2b.u1conflict.3')
             # ...on a directory with content
             os.mkdir(dirname + '/2c.conflict.4')
             os.mkdir(dirname + '/2c.conflict.4/2c')
-            expected.append(dirname + '/2c.u1conflict.4/2c')
+            expected.append(new_dirname + '/2c.u1conflict.4/2c')
             # ...in a readonly directory
             os.mkdir(dirname + '/2d')
             os.mkdir(dirname + '/2d/2d.conflict.5')
             os.chmod(dirname + '/2d', 0500)
-            expected.append(dirname + '/2d/2d.u1conflict.5')
+            expected.append(new_dirname + '/2d/2d.u1conflict.5')
             # ...in a directory that is also a .conflict
             os.mkdir(dirname + '/2e.conflict')
             os.mkdir(dirname + '/2e.conflict/2e.conflict.6')
-            expected.append(dirname + '/2e.u1conflict/2e.u1conflict.6')
+            expected.append(new_dirname + '/2e.u1conflict/2e.u1conflict.6')
 
             # a plain .conflict of which there already exists a .u1conflict...
             # ...on a file
             open(dirname + '/3a.conflict', 'w').close()
             open(dirname + '/3a.u1conflict', 'w').close()
-            expected.append(dirname + '/3a.u1conflict')
-            expected.append(dirname + '/3a.u1conflict.1')
+            expected.append(new_dirname + '/3a.u1conflict')
+            expected.append(new_dirname + '/3a.u1conflict.1')
             # ...on an empty directory
             os.mkdir(dirname + '/3b.conflict')
             os.mkdir(dirname + '/3b.u1conflict')
-            expected.append(dirname + '/3b.u1conflict')
-            expected.append(dirname + '/3b.u1conflict.1')
+            expected.append(new_dirname + '/3b.u1conflict')
+            expected.append(new_dirname + '/3b.u1conflict.1')
             # ...on a directory with content
             os.mkdir(dirname + '/3c.conflict')
             os.mkdir(dirname + '/3c.conflict/3c')
             os.mkdir(dirname + '/3c.u1conflict')
             os.mkdir(dirname + '/3c.u1conflict/3c2')
-            expected.append(dirname + '/3c.u1conflict.1/3c')
-            expected.append(dirname + '/3c.u1conflict/3c2')
+            expected.append(new_dirname + '/3c.u1conflict.1/3c')
+            expected.append(new_dirname + '/3c.u1conflict/3c2')
             # ...in a readonly directory
             os.mkdir(dirname + '/3d')
             os.mkdir(dirname + '/3d/3d.conflict')
             os.mkdir(dirname + '/3d/3d.u1conflict')
             os.mkdir(dirname + '/3d/3d.u1conflict/3d')
             os.chmod(dirname + '/3d', 0500)
-            expected.append(dirname + '/3d/3d.u1conflict/3d')
-            expected.append(dirname + '/3d/3d.u1conflict.1')
+            expected.append(new_dirname + '/3d/3d.u1conflict/3d')
+            expected.append(new_dirname + '/3d/3d.u1conflict.1')
             # ...in a directory that is also a .conflict
             os.mkdir(dirname + '/3e.conflict')
             os.mkdir(dirname + '/3e.conflict/3e.conflict')
             os.mkdir(dirname + '/3e.conflict/3e.u1conflict')
             os.mkdir(dirname + '/3e.conflict/3e.u1conflict/3e')
-            expected.append(dirname + '/3e.u1conflict/3e.u1conflict/3e')
-            expected.append(dirname + '/3e.u1conflict/3e.u1conflict.1')
+            expected.append(new_dirname + '/3e.u1conflict/3e.u1conflict/3e')
+            expected.append(new_dirname + '/3e.u1conflict/3e.u1conflict.1')
 
             # a numbered .conflict of which there already exists a .u1conflict...
             # ...on a file
             open(dirname + '/4a.conflict.1', 'w').close()
             open(dirname + '/4a.u1conflict.1', 'w').close()
-            expected.append(dirname + '/4a.u1conflict.1')
-            expected.append(dirname + '/4a.u1conflict.2')
+            expected.append(new_dirname + '/4a.u1conflict.1')
+            expected.append(new_dirname + '/4a.u1conflict.2')
             # ...on an empty directory
             os.mkdir(dirname + '/4b.conflict.2')
             os.mkdir(dirname + '/4b.u1conflict.2')
-            expected.append(dirname + '/4b.u1conflict.2')
-            expected.append(dirname + '/4b.u1conflict.3')
+            expected.append(new_dirname + '/4b.u1conflict.2')
+            expected.append(new_dirname + '/4b.u1conflict.3')
             # ...on a directory with content
             os.mkdir(dirname + '/4c.conflict.3')
             os.mkdir(dirname + '/4c.conflict.3/4c')
             os.mkdir(dirname + '/4c.u1conflict.3')
-            expected.append(dirname + '/4c.u1conflict.4/4c')
-            expected.append(dirname + '/4c.u1conflict.3')
+            expected.append(new_dirname + '/4c.u1conflict.4/4c')
+            expected.append(new_dirname + '/4c.u1conflict.3')
             # ...in a readonly directory
             os.mkdir(dirname + '/4d')
             os.mkdir(dirname + '/4d/4d.conflict.4')
             os.mkdir(dirname + '/4d/4d.u1conflict.4')
             os.chmod(dirname + '/4d', 0500)
-            expected.append(dirname + '/4d/4d.u1conflict.4')
-            expected.append(dirname + '/4d/4d.u1conflict.5')
+            expected.append(new_dirname + '/4d/4d.u1conflict.4')
+            expected.append(new_dirname + '/4d/4d.u1conflict.5')
             # ...in a directory that is also a .conflict
             os.mkdir(dirname + '/4e.conflict')
             os.mkdir(dirname + '/4e.conflict/4e.conflict.5')
             os.mkdir(dirname + '/4e.conflict/4e.u1conflict.5')
-            expected.append(dirname + '/4e.u1conflict/4e.u1conflict.5')
-            expected.append(dirname + '/4e.u1conflict/4e.u1conflict.6')
+            expected.append(new_dirname + '/4e.u1conflict/4e.u1conflict.5')
+            expected.append(new_dirname + '/4e.u1conflict/4e.u1conflict.6')
 
             # a plain .partial...
             # ...of a file
             open(dirname + '/5a.partial', 'w').close()
-            expected.append(dirname + '/.u1partial.5a')
+            expected.append(new_dirname + '/.u1partial.5a')
             # ...of a directory
             os.mkdir(dirname + '/5b')
             open(dirname + '/5b/.partial', 'w').close()
-            expected.append(dirname + '/5b/.u1partial')
+            expected.append(new_dirname + '/5b/.u1partial')
             # ...of a readonly directory
             os.mkdir(dirname + '/5c')
             open(dirname + '/5c/.partial', 'w').close()
             os.chmod(dirname + '/5c', 0500)
-            expected.append(dirname + '/5c/.u1partial')
+            expected.append(new_dirname + '/5c/.u1partial')
 
             # a plain .partial of which there already exists a .u1partial...
             # ...of a file
             open(dirname + '/6a.partial', 'w').close()
             open(dirname + '/.u1partial.6a', 'w').close()
-            expected.append(dirname + '/.u1partial.6a')
-            expected.append(dirname + '/.u1partial.6a.1')
+            expected.append(new_dirname + '/.u1partial.6a')
+            expected.append(new_dirname + '/.u1partial.6a.1')
             # ...of a directory
             os.mkdir(dirname + '/6b')
             open(dirname + '/6b/.partial', 'w').close()
             open(dirname + '/6b/.u1partial', 'w').close()
-            expected.append(dirname + '/6b/.u1partial')
-            expected.append(dirname + '/6b/.u1partial.1')
+            expected.append(new_dirname + '/6b/.u1partial')
+            expected.append(new_dirname + '/6b/.u1partial.1')
             # ...of a readonly directory
             os.mkdir(dirname + '/6c')
             open(dirname + '/6c/.partial', 'w').close()
             open(dirname + '/6c/.u1partial', 'w').close()
             os.chmod(dirname + '/6c', 0500)
-            expected.append(dirname + '/6c/.u1partial')
-            expected.append(dirname + '/6c/.u1partial.1')
+            expected.append(new_dirname + '/6c/.u1partial')
+            expected.append(new_dirname + '/6c/.u1partial.1')
 
-        self.main = FakeMain(self.root_dir, self.shares_dir,
+        self._set_permissions()
+        self.main = FakeMain(self.new_root_dir, self.new_shares_dir,
                              self.data_dir, self.partials_dir)
 
         for path in expected:
             self.assertTrue(os.path.exists(path), 'missing ' + path)
+        self.check_version()
 
     def test_missing_version_file_with_version_non_0(self):
-        """ Test the upgrade from the first shelf layout version to v3
-        while the metadata sould be in v3 format
+        """Test the upgrade from the first shelf layout version
+        while the metadata sould be in v3 or greater format.
+
         """
-        # ensure a clean data_dir
-        self.rmtree(self.data_dir)
-        vm_data_dir = os.path.join(self.data_dir, 'vm', 'shares')
-        maybe_old_shelf = LegacyShareFileShelf(vm_data_dir)
+        self._build_layout_version_1()
+        maybe_old_shelf = LegacyShareFileShelf(self.share_md_dir)
         # add the root_uuid key
-        root_share = Share(path=self.root_dir)
+        root_share = _Share(self.root_dir)
         root_share.access_level = 'Modify'
         maybe_old_shelf[''] = root_share
         for idx in range(1, 10):
-            maybe_old_shelf[str(uuid.uuid4())] = \
-                    Share(path=os.path.join(self.shares_dir, str(idx)))
+            share_id = str(uuid.uuid4())
+            maybe_old_shelf[share_id] = \
+                    _Share(share_id=share_id,
+                           path=os.path.join(self.shares_dir, str(idx)))
         # ShareFileShelf.keys returns a generator
         maybe_old_keys = [key for key in maybe_old_shelf.keys()]
         self.assertEquals(10, len(maybe_old_keys))
+        if self.md_version_None:
+            self.set_md_version('')
         # we want to keep a refernece to main in order to shutdown
         # pylint: disable-msg=W0201
-        self.main = FakeMain(self.root_dir, self.shares_dir,
+        self.main = FakeMain(self.new_root_dir, self.new_shares_dir,
                              self.data_dir, self.partials_dir)
         new_keys = [new_key for new_key in self.main.vm.shares.keys()]
         self.assertEquals(10, len(new_keys))
@@ -1689,160 +1749,444 @@
         # as we didn't actually upgrade the shelf, just the .version file
         # check the empty 0.bkp
         # check the old data is still there (in the backup)
-        backup_shelf = LegacyShareFileShelf(os.path.join(vm_data_dir, '0.bkp'))
+        backup_shelf = LegacyShareFileShelf(os.path.join(self.vm_data_dir, '0.bkp'))
         backup_keys = [key for key in backup_shelf.keys()]
         self.assertEquals(0, len(backup_keys))
         self.check_version()
 
-    def test_3_to_4(self):
-        """upgrade from version 3 to 4"""
-        vm_data_dir = os.path.join(self.data_dir, 'vm')
-        os.makedirs(vm_data_dir)
-        with open(os.path.join(vm_data_dir, '.version'), 'w') as fd:
-            fd.write('3')
-        os.rmdir(self.shares_dir)
-        # build the old layout
-        old_root = os.path.join(self.root_dir, 'My Files')
-        old_shares = os.path.join(self.root_dir, 'Shared With Me')
-        os.makedirs(os.path.join(old_root, 'test_dir'))
-        open(os.path.join(old_root, 'test_file'), 'w').close()
+    def test_upgrade_3(self):
+        """Test upgrade from version 3."""
+        self._build_layout_version_1()
+        self.set_md_version('3')
+        # create a dir in the root
+        os.makedirs(os.path.join(self.root_dir, 'test_dir'))
         # create a file in the root
         open(os.path.join(self.root_dir, 'test_file'), 'w').close()
-        share_path = os.path.join(old_shares, 'Bla from Foo')
+        # create a file in the new root
+        open(os.path.join(self.new_root_dir, 'test_file'), 'w').close()
+        share_path = os.path.join(self.shares_dir, 'Bla from Foo')
         os.makedirs(share_path)
         os.makedirs(os.path.join(share_path, 'test_dir'))
         open(os.path.join(share_path, 'test_file'), 'w').close()
         # fix permissions
-        os.chmod(self.root_dir, 0555)
-        os.chmod(old_shares, 0555)
+        self._set_permissions()
+        if self.md_version_None:
+            self.set_md_version('')
         # migrate the data
-        self.main = FakeMain(self.root_dir, self.shares_dir,
+        self.main = FakeMain(self.new_root_dir, self.new_shares_dir,
                              self.data_dir, self.partials_dir)
-        self.assertFalse(os.path.exists(old_root))
-        self.assertTrue(os.path.exists(old_shares))
-        self.assertTrue(os.path.islink(old_shares))
-        self.assertEquals(old_shares, self.main.shares_dir_link)
-        self.assertTrue(os.path.exists(os.path.join(self.root_dir,
+        self.assertFalse(os.path.exists(self.root_dir))
+        self.assertTrue(os.path.exists(self.shares_dir))
+        self.assertTrue(os.path.islink(self.shares_dir), self.shares_dir)
+        self.assertEquals(self.shares_dir, self.main.shares_dir_link)
+        self.assertTrue(os.path.exists(os.path.join(self.new_root_dir,
                                                     'test_dir')))
-        self.assertTrue(os.path.exists(os.path.join(self.root_dir,
+        self.assertTrue(os.path.exists(os.path.join(self.new_root_dir,
                                                     'test_file')))
-        self.assertTrue(os.path.exists(os.path.join(self.root_dir,
+        self.assertTrue(os.path.exists(os.path.join(self.new_root_dir,
                                                     'test_file.u1conflict')))
         self.assertTrue(os.path.exists(share_path))
         self.assertTrue(os.path.exists(os.path.join(share_path, 'test_dir')))
         self.assertTrue(os.path.exists(os.path.join(share_path, 'test_file')))
+        self.check_version()
 
-    def test_3_to_4_with_symlink_in_myfiles(self):
-        """upgrade from version 3 to 4"""
-        vm_data_dir = os.path.join(self.data_dir, 'vm')
-        os.makedirs(vm_data_dir)
-        with open(os.path.join(vm_data_dir, '.version'), 'w') as fd:
-            fd.write('3')
-        os.rmdir(self.shares_dir)
+    def test_upgrade_3_with_symlink_in_myfiles(self):
+        """Test upgrade from version 3 with symlink in 'My Files'."""
+        self._build_layout_version_1()
+        self.set_md_version('3')
         # build the old layout
-        old_root = os.path.join(self.root_dir, 'My Files')
-        old_shares = os.path.join(self.root_dir, 'Shared With Me')
-        os.makedirs(os.path.join(old_root, 'test_dir'))
-        open(os.path.join(old_root, 'test_file'), 'w').close()
+        os.makedirs(os.path.join(self.root_dir, 'test_dir'))
+        open(os.path.join(self.root_dir, 'test_file'), 'w').close()
         # create a file in the root
-        open(os.path.join(self.root_dir, 'test_file'), 'w').close()
-        share_path = os.path.join(old_shares, 'Bla from Foo')
+        open(os.path.join(self.new_root_dir, 'test_file'), 'w').close()
+        share_path = os.path.join(self.shares_dir, 'Bla from Foo')
         os.makedirs(share_path)
         os.makedirs(os.path.join(share_path, 'test_dir'))
         open(os.path.join(share_path, 'test_file'), 'w').close()
         # create the Shared with Me symlink in My Files
-        os.symlink(old_shares, os.path.join(old_root, 'Shared With Me'))
+        os.symlink(self.shares_dir, os.path.join(self.root_dir,
+                                                 "Shared With Me"))
         # fix permissions
-        os.chmod(self.root_dir, 0555)
-        os.chmod(old_shares, 0555)
+        self._set_permissions()
+        if self.md_version_None:
+            self.set_md_version('')
         # migrate the data
-        self.shares_dir = os.path.join(self.tmpdir, 'shares')
-        self.main = FakeMain(self.root_dir, self.shares_dir,
+        self.main = FakeMain(self.new_root_dir, self.new_shares_dir,
                              self.data_dir, self.partials_dir)
-        self.assertFalse(os.path.exists(old_root))
-        self.assertTrue(os.path.exists(old_shares))
-        self.assertTrue(os.path.islink(old_shares))
-        self.assertEquals(old_shares, self.main.shares_dir_link)
-        self.assertTrue(os.path.exists(os.path.join(self.root_dir,
+        self.assertFalse(os.path.exists(self.root_dir))
+        self.assertTrue(os.path.exists(self.shares_dir))
+        self.assertTrue(os.path.islink(self.shares_dir))
+        self.assertEquals(self.shares_dir, self.main.shares_dir_link)
+        self.assertTrue(os.path.exists(os.path.join(self.new_root_dir,
                                                     'test_dir')))
-        self.assertTrue(os.path.exists(os.path.join(self.root_dir,
+        self.assertTrue(os.path.exists(os.path.join(self.new_root_dir,
                                                     'test_file')))
-        self.assertTrue(os.path.exists(os.path.join(self.root_dir,
+        self.assertTrue(os.path.exists(os.path.join(self.new_root_dir,
                                                     'test_file.u1conflict')))
         self.assertTrue(os.path.exists(share_path))
         self.assertTrue(os.path.exists(os.path.join(share_path, 'test_dir')))
         self.assertTrue(os.path.exists(os.path.join(share_path, 'test_file')))
         self.assertEquals(self.main.shares_dir,
                           os.readlink(self.main.shares_dir_link))
-
-    def test_None_to_4(self):
-        """upgrade from version None to 4 (possibly a clean start)"""
-        VolumeManager.METADATA_VERSION = '4'
-        vm_data_dir = os.path.join(self.data_dir, 'vm')
-        version_file = os.path.join(vm_data_dir, '.version')
-        if os.path.exists(version_file):
-            os.remove(version_file)
-        os.rmdir(self.shares_dir)
-        os.rmdir(self.root_dir)
+        self.check_version()
+
+
+class MetadataNewLayoutTests(MetadataTestCase):
+    """Test for 'new' layout and metadata upgrade."""
+
+    def setUp(self):
+        MetadataTestCase.setUp(self)
+        self.share_md_dir = os.path.join(self.vm_data_dir, 'shares')
+        self.shared_md_dir = os.path.join(self.vm_data_dir, 'shared')
+        self.home_dir = os.path.join(self.tmpdir, 'home', 'ubuntuonehacker')
+        self.u1_dir = os.path.join(self.home_dir, os.path.split(self.u1_dir)[1])
+        self.root_dir = self.u1_dir
+        self.shares_dir = os.path.join(self.tmpdir, 'shares')
+        self.shares_dir_link = os.path.join(self.u1_dir, 'Shared With Me')
+
+    def _build_layout_version_4(self):
+        """Build the directory structure to mimic md v.4/5."""
+        os.makedirs(self.share_md_dir)
+        os.makedirs(self.shared_md_dir)
+        os.makedirs(self.root_dir)
+        os.makedirs(self.shares_dir)
+        os.symlink(self.shares_dir, self.shares_dir_link)
+
+    def _fix_permissions(self):
+        """Fix shares dir permissions, making it read-only."""
+        os.chmod(self.shares_dir, 0500)
+
+    def test_upgrade_None_to_last(self):
+        """Upgrade from version 'None' (possibly a clean start)."""
         old_root = os.path.join(self.root_dir, 'My Files')
         old_shares = os.path.join(self.root_dir, 'Shared With Me')
         # start and check that everything is ok
         self.main = FakeMain(self.root_dir, self.shares_dir,
                              self.data_dir, self.partials_dir)
         self.assertFalse(os.path.exists(old_root))
+        self.assertTrue(os.path.exists(self.root_dir))
         self.assertTrue(os.path.exists(old_shares))
         self.assertTrue(os.path.islink(old_shares))
         self.assertEquals(old_shares, self.main.shares_dir_link)
-        if os.path.exists(version_file):
-            with open(os.path.join(vm_data_dir, '.version'), 'r') as fd:
-                self.assertEquals('4', fd.read())
-        else:
-            self.fail('missing .version file')
-
-    def test_None_to_4_phantom_share_path(self):
-        """upgrade from version None to 4 (possibly a clean start)"""
-        VolumeManager.METADATA_VERSION = '4'
-        vm_data_dir = os.path.join(self.data_dir, 'vm')
-        version_file = os.path.join(vm_data_dir, '.version')
-        if os.path.exists(version_file):
-            os.remove(version_file)
-        os.rmdir(self.shares_dir)
-        os.rmdir(self.root_dir)
+        self.check_version()
+
+    def test_upgrade_None_to_last_phantom_share_path(self):
+        """Upgrade from version 'None' (possibly a clean start) with a root
+        with missing path.
+
+        """
         old_root = os.path.join(self.root_dir, 'My Files')
         old_shares = os.path.join(self.root_dir, 'Shared With Me')
-        # start and check that everything is ok
         self.main = FakeMain(self.root_dir, self.shares_dir,
                              self.data_dir, self.partials_dir)
-        root = self.main.vm.shares['']
+        self.main.shutdown()
+        self.rmtree(self.vm_data_dir)
+        os.makedirs(self.vm_data_dir)
+        self.set_md_version('')
+        shares = LegacyShareFileShelf(self.share_md_dir)
+        root_share = _Share(self.root_dir)
+        root_share.access_level = 'Modify'
         # set None to the share path
-        root.path = None
-        self.main.vm.shares['test'] = root
-        if os.path.exists(version_file):
-            os.remove(version_file)
-        self.main.shutdown()
+        root_share.path = None
+        shares[''] = root_share
+
+        if self.md_version_None:
+            self.set_md_version('')
         # check that it's all OK
         self.main = FakeMain(self.root_dir, self.shares_dir,
                              self.data_dir, self.partials_dir)
         self.assertFalse(os.path.exists(old_root))
-        self.assertTrue(os.path.exists(old_shares))
+        self.assertTrue(os.path.exists(self.root_dir))
+        self.assertTrue(os.path.exists(self.shares_dir))
         self.assertTrue(os.path.islink(old_shares))
         self.assertEquals(old_shares, self.main.shares_dir_link)
-        if os.path.exists(version_file):
-            with open(os.path.join(vm_data_dir, '.version'), 'r') as fd:
-                self.assertEquals('4', fd.read())
-        else:
-            self.fail('missing .version file')
-
-    def test_4_to_5(self):
-        """test migration from 4 to 5 (broken symlink in the root)"""
-        vm_data_dir = os.path.join(self.data_dir, 'vm')
-        os.makedirs(vm_data_dir)
-        with open(os.path.join(vm_data_dir, '.version'), 'w') as fd:
-            fd.write('4')
-        # build the new layout with a broken symlink
-        shares_link = os.path.join(self.root_dir, 'Shared With Me')
-        os.symlink(shares_link, shares_link)
+        self.check_version()
+
+    def test_upgrade_4(self):
+        """Test migration from 4 to 5 (broken symlink in the root)."""
+        self._build_layout_version_4()
+        self.set_md_version('4')
+        # break the symlink
+        if os.path.exists(self.shares_dir_link):
+            os.unlink(self.shares_dir_link)
+        os.symlink(self.shares_dir_link, self.shares_dir_link)
+
+        if self.md_version_None:
+            self.set_md_version('')
         self.main = FakeMain(self.root_dir, self.shares_dir,
                              self.data_dir, self.partials_dir)
         self.assertEquals(self.main.shares_dir,
                           os.readlink(self.main.shares_dir_link))
+        self.check_version()
+
+    def test_upgrade_5(self):
+        """Test the migration from version 5."""
+        # build a fake version 5 state
+        self._build_layout_version_4()
+        self.set_md_version('5')
+        # create some old shares and shared metadata
+        legacy_shares = LegacyShareFileShelf(self.share_md_dir)
+        root_share = _Share(path=self.root_dir, share_id='',
+                            access_level='Modify')
+        legacy_shares[''] = root_share
+        for idx, name in enumerate(['share'] * 1000):
+            sid = str(uuid.uuid4())
+            share_name = name + '_' + str(idx)
+            share = _Share(path=os.path.join(self.shares_dir, share_name),
+                           share_id=sid, name=share_name,
+                           node_id=str(uuid.uuid4()),
+                           other_username='username'+str(idx),
+                           other_visible_name='visible name ' + str(idx))
+            if idx % 2:
+                share.access_level = 'Modify'
+            else:
+                share.access_level = 'View'
+            legacy_shares[sid] = share
+
+        # create shared shares
+        legacy_shared = LegacyShareFileShelf(self.shared_md_dir)
+        for idx, name in enumerate(['dir'] * 5):
+            sid = str(uuid.uuid4())
+            share_name = name + '_' + str(idx)
+            share = _Share(path=os.path.join(self.root_dir, share_name),
+                           share_id=sid, node_id=str(uuid.uuid4()),
+                           name=share_name, other_username='hola',
+                           other_visible_name='hola')
+            if idx % 2:
+                share.access_level = 'Modify'
+            else:
+                share.access_level = 'View'
+            legacy_shared[sid] = share
+
+        # keep a copy of the current shares and shared metadata to check
+        # the upgrade went ok
+        legacy_shares = dict(legacy_shares.items())
+        legacy_shared = dict(legacy_shared.items())
+
+        if self.md_version_None:
+            self.set_md_version('')
+        # upgrade it!
+        self.main = FakeMain(self.root_dir, self.shares_dir,
+                             self.data_dir, self.partials_dir)
+        vm = self.main.vm
+        def compare_share(share, old_share):
+            """Compare two shares, new and old"""
+            self.assertEquals(share.volume_id, old_share.id)
+            self.assertEquals(share.path, old_share.path)
+            self.assertEquals(share.node_id, old_share.subtree)
+            if not isinstance(share, Root):
+                self.assertEquals(share.name, old_share.name)
+                self.assertEquals(share.other_username, old_share.other_username)
+                self.assertEquals(share.other_visible_name, old_share.other_visible_name)
+                self.assertEquals(share.access_level, old_share.access_level)
+
+        for sid in vm.shares:
+            old_share = legacy_shares[sid]
+            share = vm.shares[sid]
+            self.assertTrue(isinstance(share, Share) or isinstance(share, Root))
+            compare_share(share, old_share)
+
+        for sid in vm.shared:
+            old_share = legacy_shared[sid]
+            share = vm.shared[sid]
+            self.assertTrue(isinstance(share, Shared))
+            compare_share(share, old_share)
+
+    def test_upgrade_5_with_udfs(self):
+        """Test the migration from version 5 with old UDFs."""
+        # build a fake version 5 state
+        self._build_layout_version_4()
+        self.set_md_version('5')
+        self.udfs_md_dir = os.path.join(self.vm_data_dir, 'udfs')
+        # create some old shares and shared metadata
+        legacy_shares = LegacyShareFileShelf(self.share_md_dir)
+        root_share = _Share(path=self.root_dir, share_id='',
+                            access_level='Modify')
+        legacy_shares[''] = root_share
+        for idx, name in enumerate(['share'] * 1000):
+            sid = str(uuid.uuid4())
+            share_name = name + '_' + str(idx)
+            share = _Share(path=os.path.join(self.shares_dir, share_name),
+                           share_id=sid, name=share_name,
+                           node_id=str(uuid.uuid4()),
+                           other_username='username'+str(idx),
+                           other_visible_name='visible name ' + str(idx))
+            if idx % 2:
+                share.access_level = 'Modify'
+            else:
+                share.access_level = 'View'
+            legacy_shares[sid] = share
+
+        # create shared shares
+        legacy_shared = LegacyShareFileShelf(self.shared_md_dir)
+        for idx, name in enumerate(['dir'] * 5):
+            sid = str(uuid.uuid4())
+            share_name = name + '_' + str(idx)
+            share = _Share(path=os.path.join(self.root_dir, share_name),
+                           share_id=sid, node_id=str(uuid.uuid4()),
+                           name=share_name, other_username='hola',
+                           other_visible_name='hola')
+            if idx % 2:
+                share.access_level = 'Modify'
+            else:
+                share.access_level = 'View'
+            legacy_shared[sid] = share
+
+        # create some udfs
+        legacy_udfs = LegacyShareFileShelf(self.udfs_md_dir)
+        for idx, name in enumerate(['dir'] * 5):
+            udf_id = str(uuid.uuid4())
+            udf_name = name + '_' + str(idx)
+            udf = _UDF(udf_id, str(uuid.uuid4()), '~/' + udf_name,
+                         os.path.join(self.home_dir, udf_name))
+            if idx % 2:
+                udf.subscribed = True
+            else:
+                udf.subscribed = False
+            legacy_udfs[sid] = udf
+
+        # keep a copy of the current shares and shared metadata to check
+        # the upgrade went ok
+        legacy_shares = dict(legacy_shares.items())
+        legacy_shared = dict(legacy_shared.items())
+        legacy_udfs = dict(legacy_udfs.items())
+
+        if self.md_version_None:
+            self.set_md_version('')
+        # upgrade it!
+        self.main = FakeMain(self.root_dir, self.shares_dir,
+                             self.data_dir, self.partials_dir)
+        vm = self.main.vm
+        def compare_share(share, old_share):
+            """Compare two shares, new and old"""
+            self.assertEquals(share.volume_id, old_share.id)
+            self.assertEquals(share.path, old_share.path)
+            self.assertEquals(share.node_id, old_share.subtree)
+            if not isinstance(share, Root):
+                self.assertEquals(share.name, old_share.name)
+                self.assertEquals(share.other_username, old_share.other_username)
+                self.assertEquals(share.other_visible_name, old_share.other_visible_name)
+                self.assertEquals(share.access_level, old_share.access_level)
+
+        for sid in vm.shares:
+            old_share = legacy_shares[sid]
+            share = vm.shares[sid]
+            self.assertTrue(isinstance(share, Share) or isinstance(share, Root))
+            compare_share(share, old_share)
+
+        for sid in vm.shared:
+            old_share = legacy_shared[sid]
+            share = vm.shared[sid]
+            self.assertTrue(isinstance(share, Shared))
+            compare_share(share, old_share)
+
+        for udf_id in vm.udfs:
+            old_udf = legacy_udfs[udf_id]
+            udf = vm.udfs[udf_id]
+            self.assertTrue(isinstance(udf, UDF))
+            self.assertEquals(udf.volume_id, old_udf.id)
+            self.assertEquals(udf.path, old_udf.path)
+            self.assertEquals(udf.node_id, old_udf.node_id)
+            self.assertEquals(udf.suggested_path, old_udf.suggested_path)
+            self.assertEquals(udf.subscribed, old_udf.subscribed)
+
+
+class BrokenOldMDVersionUpgradeTests(MetadataOldLayoutTests):
+    """MetadataOldLayoutTests with broken .version file."""
+    md_version_None = True
+
+
+class BrokenNewMDVersionUpgradeTests(MetadataNewLayoutTests):
+    """MetadataNewLayoutTests with broken .version file."""
+    md_version_None = True
+
+
+class MetadataUpgraderTests(MetadataTestCase):
+    """MetadataUpgrader tests."""
+
+    def setUp(self):
+        """Create the MetadataUpgrader instance."""
+        MetadataTestCase.setUp(self)
+        self.share_md_dir = os.path.join(self.vm_data_dir, 'shares')
+        self.shared_md_dir = os.path.join(self.vm_data_dir, 'shared')
+        self.udfs_md_dir = os.path.join(self.vm_data_dir, 'udfs')
+        self.home_dir = os.path.join(self.tmpdir, 'home', 'ubuntuonehacker')
+        self.u1_dir = os.path.join(self.home_dir, os.path.split(self.u1_dir)[1])
+        self.root_dir = self.u1_dir
+        self.shares_dir = os.path.join(self.tmpdir, 'shares')
+        self.shares_dir_link = os.path.join(self.u1_dir, 'Shared With Me')
+        for path in [self.share_md_dir, self.shared_md_dir,
+                     self.root_dir, self.shares_dir]:
+            if not os.path.exists(path):
+                os.makedirs(path)
+        os.symlink(self.shares_dir, self.shares_dir_link)
+        self.old_get_md_version = MetadataUpgrader._get_md_version
+        MetadataUpgrader._get_md_version = lambda _: None
+        self.md_upgrader = MetadataUpgrader(self.vm_data_dir, self.share_md_dir,
+                                            self.shared_md_dir,
+                                            self.udfs_md_dir, self.root_dir,
+                                            self.shares_dir,
+                                            self.shares_dir_link)
+    def tearDown(self):
+        """Restorre _get_md_version"""
+        MetadataUpgrader._get_md_version = self.old_get_md_version
+        MetadataTestCase.tearDown(self)
+
+    def test_guess_metadata_version_None(self):
+        """Test _guess_metadata_version method for pre-version."""
+        # fake a version None layout
+        if os.path.exists(self.version_file):
+            os.unlink(self.version_file)
+        for path in [self.share_md_dir, self.shared_md_dir,
+                     self.root_dir, self.shares_dir]:
+            if os.path.exists(path):
+                self.rmtree(path)
+        os.makedirs(os.path.join(self.root_dir, 'My Files'))
+        shares_dir = os.path.join(self.root_dir, 'Shared With Me')
+        os.makedirs(shares_dir)
+        os.chmod(self.root_dir, 0500)
+        os.chmod(shares_dir, 0500)
+        version = self.md_upgrader._guess_metadata_version()
+        self.assertEquals(None, version)
+
+    def test_guess_metadata_version_1_or_2(self):
+        """Test _guess_metadata_version method for version 1 or 2."""
+        # fake a version 1 layout
+        if os.path.exists(self.version_file):
+            os.unlink(self.version_file)
+        self.rmtree(self.root_dir)
+        os.makedirs(os.path.join(self.root_dir, 'My Files'))
+        shares_dir = os.path.join(self.root_dir, 'Shared With Me')
+        os.makedirs(shares_dir)
+        os.chmod(self.root_dir, 0500)
+        os.chmod(shares_dir, 0500)
+        self.rmtree(self.shares_dir)
+        version = self.md_upgrader._guess_metadata_version()
+        self.assertIn(version, ['1', '2'])
+
+    def test_guess_metadata_version_4(self):
+        """Test _guess_metadata_version method for version 4."""
+        # fake a version 4 layout
+        if os.path.exists(self.version_file):
+            os.unlink(self.version_file)
+        os.unlink(self.shares_dir_link)
+        os.symlink(self.shares_dir_link, self.shares_dir_link)
+        version = self.md_upgrader._guess_metadata_version()
+        self.assertEquals(version, '4')
+
+    def test_guess_metadata_version_5(self):
+        """Test _guess_metadata_version method for version 5."""
+        # fake a version 5 layout and metadata
+        shelf = LegacyShareFileShelf(self.share_md_dir)
+        shelf['foobar'] = _Share(path='/foo/bar', share_id='foobar')
+        version = self.md_upgrader._guess_metadata_version()
+        self.assertEquals(version, '5')
+
+    def test_guess_metadata_version_6(self):
+        """Test _guess_metadata_version method for version 6."""
+        # fake a version 6 layout and metadata
+        shelf = VMFileShelf(self.share_md_dir)
+        shelf['foobar'] = Share(path='/foo/bar', volume_id='foobar')
+        version = self.md_upgrader._guess_metadata_version()
+        self.assertEquals(version, '6')
+

=== modified file 'ubuntuone/syncdaemon/dbus_interface.py'
--- ubuntuone/syncdaemon/dbus_interface.py	2010-01-26 14:32:54 +0000
+++ ubuntuone/syncdaemon/dbus_interface.py	2010-01-26 20:35:29 +0000
@@ -544,7 +544,9 @@
     def handle_SV_SHARE_CHANGED(self, message, share):
         """ handle SV_SHARE_CHANGED event, emit's ShareChanged signal. """
         self.handle_default('SV_SHARE_CHANGED', message, share)
-        self.dbus_iface.shares.emit_share_changed(message, share)
+        if message != 'deleted':
+            # deleted shares are handled in VM
+            self.dbus_iface.shares.emit_share_changed(message, share)
 
     def handle_SV_FREE_SPACE(self, share_id, free_bytes):
         """ handle SV_FREE_SPACE event, emit ShareChanged signal. """
@@ -871,6 +873,8 @@
                 share_dict[unicode(k)] = ''
             elif k == 'path':
                 share_dict[unicode(k)] = v.decode('utf-8')
+            elif k == 'accepted':
+                share_dict[unicode(k)] = self.bool_str(v)
             else:
                 share_dict[unicode(k)] = unicode(v)
         return share_dict
@@ -949,7 +953,7 @@
 
     def emit_share_answer_response(self, share_id, answer, error=None):
         """ emits ShareCreated signal """
-        answer_info = dict(share_id=share_id, answer=answer)
+        answer_info = dict(volume_id=share_id, answer=answer)
         if error:
             answer_info['error'] = error
         self.ShareAnswerResponse(answer_info)

=== modified file 'ubuntuone/syncdaemon/event_queue.py'
--- ubuntuone/syncdaemon/event_queue.py	2010-01-26 15:54:29 +0000
+++ ubuntuone/syncdaemon/event_queue.py	2010-01-26 20:35:29 +0000
@@ -159,7 +159,7 @@
 }
 
 # these are the events that will listen from inotify
-INOTIFY_EVENTS = (
+INOTIFY_EVENTS_GENERAL = (
     evtcodes.IN_OPEN |
     evtcodes.IN_CLOSE_NOWRITE |
     evtcodes.IN_CLOSE_WRITE |
@@ -169,6 +169,12 @@
     evtcodes.IN_MOVED_TO |
     evtcodes.IN_MOVE_SELF
 )
+INOTIFY_EVENTS_ANCESTORS = (
+    evtcodes.IN_DELETE |
+    evtcodes.IN_MOVED_FROM |
+    evtcodes.IN_MOVED_TO |
+    evtcodes.IN_MOVE_SELF
+)
 
 DEFAULT_HANDLER = "handle_default" # receives (event_name, *args, **kwargs)
 
@@ -201,14 +207,58 @@
         return True
 
 
-class _INotifyProcessor(pyinotify.ProcessEvent):
-    """Helper class that is called from inpotify when an event happens.
+class _AncestorsINotifyProcessor(pyinotify.ProcessEvent):
+    """inotify's processor when an event happens on an UDFs ancestor."""
+    def __init__(self, eq):
+        self.log = logging.getLogger('ubuntuone.SyncDaemon.AncestorsINotProc')
+        self.eq = eq
+
+    def _get_udf(self, path):
+        """Get the udf for a specific path.
+
+        It can return None in case the UDF was deleted in the meantime.
+        """
+        for udf in self.eq.fs.vm.udfs.itervalues():
+            parent = os.path.dirname(udf.path) + os.path.sep
+            if parent.startswith(path + os.path.sep):
+                return udf
+        return None
+
+    def process_IN_MOVE_SELF(self, event):
+        """Don't do anything here.
+
+        We just turned this event on because pyinotify does some
+        path-fixing in its internal processing when this happens.
+        """
+    process_IN_MOVED_TO = process_IN_MOVE_SELF
+
+    def process_IN_MOVED_FROM(self, event):
+        """Getting it out or renaming means unsuscribe."""
+        if event.mask & evtcodes.IN_ISDIR:
+            udf = self._get_udf(event.path)
+            if udf is not None:
+                self.log.info("Got MOVED_FROM on path %r, unsubscribing "
+                               "udf %s", event.path, udf)
+                self.eq.fs.vm.unsubscribe_udf(udf.volume_id)
+
+    def process_IN_DELETE(self, event):
+        """Check to see if the UDF was deleted."""
+        if event.mask & evtcodes.IN_ISDIR:
+            udf = self._get_udf(event.path)
+            if udf is not None and udf.path == event.pathname:
+                self.log.info("Got DELETE on path %r, deleting udf %s",
+                               event.path, udf)
+                self.eq.fs.vm.delete_volume(udf.volume_id)
+
+
+class _GeneralINotifyProcessor(pyinotify.ProcessEvent):
+    """inotify's processor when a general event happens.
 
     This class also catchs the MOVEs events, and synthetises a new
     FS_(DIR|FILE)_MOVE event when possible.
     """
     def __init__(self, eq):
-        self.log = logging.getLogger('ubuntuone.SyncDaemon.INotifyProcessor')
+        self.log = logging.getLogger('ubuntuone.SyncDaemon.GeneralINotProc')
         self.eq = eq
         self.held_event = None
         self.timer = None
@@ -216,16 +266,6 @@
         self.frozen_evts = False
         self._to_mute = MuteFilter()
 
-    def _is_udf_ancestor(self, path):
-        """Decide if path is an UDF ancestor or not."""
-        result = None
-        for udf in self.eq.fs.vm.udfs.itervalues():
-            parent = os.path.dirname(udf.path) + os.path.sep
-            if parent.startswith(path + os.path.sep):
-                return udf
-
-        return result
-
     def add_to_mute_filter(self, event, *paths):
         """Add an event and path(s) to the mute filter."""
         # all events have one path except the MOVEs
@@ -261,14 +301,12 @@
 
     def process_IN_OPEN(self, event):
         """Filter IN_OPEN to make it happen only in files."""
-        if not (event.mask & evtcodes.IN_ISDIR) and \
-           not self._is_udf_ancestor(event.path):
+        if not (event.mask & evtcodes.IN_ISDIR):
             self.push_event(event)
 
     def process_IN_CLOSE_NOWRITE(self, event):
         """Filter IN_CLOSE_NOWRITE to make it happen only in files."""
-        if not (event.mask & evtcodes.IN_ISDIR) and \
-           not self._is_udf_ancestor(event.path):
+        if not (event.mask & evtcodes.IN_ISDIR):
             self.push_event(event)
 
     def process_IN_MOVE_SELF(self, event):
@@ -281,11 +319,6 @@
 
     def process_IN_MOVED_FROM(self, event):
         """Capture the MOVED_FROM to maybe syntethize FILE_MOVED."""
-        udf = self._is_udf_ancestor(event.path)
-        if udf is not None:
-            self.eq.fs.vm.unsubscribe_udf(udf.volume_id)
-            return
-
         if self.held_event is not None:
             self.release_held_event()
 
@@ -317,9 +350,6 @@
 
     def process_IN_MOVED_TO(self, event):
         """Capture the MOVED_TO to maybe syntethize FILE_MOVED."""
-        if self._is_udf_ancestor(event.path):
-            return
-
         if self.held_event is not None:
             if event.cookie == self.held_event.cookie:
                 try:
@@ -381,16 +411,6 @@
 
     def process_default(self, event):
         """Push the event into the EventQueue."""
-        udf = self._is_udf_ancestor(event.path)
-        if udf is not None:
-            # if event is the deletion of the UDF per se,
-            # call delete_volume on VolumeManager for that UDF.
-            ename = NAME_TRANSLATIONS.get(event.mask, None)
-            is_dir_delete = ename is not None and ename == 'FS_DIR_DELETE'
-            if udf.path == event.pathname and is_dir_delete:
-                self.eq.fs.vm.delete_volume(udf.volume_id)
-            return
-
         if self.held_event is not None:
             self.release_held_event()
         self.push_event(event)
@@ -482,13 +502,23 @@
 
         self.log = logging.getLogger('ubuntuone.SyncDaemon.EQ')
         self.fs = fs
-        # hook inotify
-        self._inotify_reader = None
-        self._inotify_wm = wm = pyinotify.WatchManager()
-        self._processor = _INotifyProcessor(self)
-        self._inotify_notifier = pyinotify.Notifier(wm, self._processor)
-        self._hook_inotify_to_twisted(wm, self._inotify_notifier)
-        self._watchs = {}
+
+        # general inotify
+        self._inotify_general_wm = wm = pyinotify.WatchManager()
+        self._processor = _GeneralINotifyProcessor(self)
+        self._inotify_notifier_gral = pyinotify.Notifier(wm, self._processor)
+        self._inotify_reader_gral = self._hook_inotify_to_twisted(
+                                            wm, self._inotify_notifier_gral)
+        self._general_watchs = {}
+
+        # ancestors inotify
+        self._inotify_ancestors_wm = wm = pyinotify.WatchManager()
+        antr_processor = _AncestorsINotifyProcessor(self)
+        self._inotify_notifier_antr = pyinotify.Notifier(wm, antr_processor)
+        self._inotify_reader_antr = self._hook_inotify_to_twisted(
+                                            wm, self._inotify_notifier_antr)
+        self._ancestors_watchs = {}
+
         self.dispatching = False
         self.dispatch_queue = Queue()
         self.empty_event_queue_callbacks = set()
@@ -526,34 +556,64 @@
                 notifier.process_events()
 
         reader = MyReader()
-        self._inotify_reader = reader
         reactor.addReader(reader)
+        return reader
 
     def shutdown(self):
         """Prepares the EQ to be closed."""
-        self._inotify_notifier.stop()
-        reactor.removeReader(self._inotify_reader)
+        self._inotify_notifier_gral.stop()
+        self._inotify_notifier_antr.stop()
+        reactor.removeReader(self._inotify_reader_gral)
+        reactor.removeReader(self._inotify_reader_antr)
 
     def inotify_rm_watch(self, dirpath):
         """Remove watch from a dir."""
-        try:
-            wd = self._watchs[dirpath]
-        except KeyError:
+        if dirpath in self._general_watchs:
+            w_dict = self._general_watchs
+            w_manager = self._inotify_general_wm
+        elif dirpath in self._ancestors_watchs:
+            w_dict = self._ancestors_watchs
+            w_manager = self._inotify_ancestors_wm
+        else:
             raise ValueError("The path %r is not watched right now!" % dirpath)
-        result = self._inotify_wm.rm_watch(wd)
+
+        wd = w_dict[dirpath]
+        result = w_manager.rm_watch(wd)
         if not result[wd]:
             raise RuntimeError("The path %r couldn't be removed!" % dirpath)
-        del self._watchs[dirpath]
+        del w_dict[dirpath]
 
     def inotify_add_watch(self, dirpath):
         """Add watch to a dir."""
-        self.log.debug("Adding inotify watch to %r", dirpath)
-        result = self._inotify_wm.add_watch(dirpath, INOTIFY_EVENTS)
-        self._watchs[dirpath] = result[dirpath]
+        # see where to add it
+        if self._is_udf_ancestor(dirpath):
+            w_type = "ancestors"
+            w_manager = self._inotify_ancestors_wm
+            w_dict = self._ancestors_watchs
+            events = INOTIFY_EVENTS_ANCESTORS
+        else:
+            w_type = "general"
+            w_manager = self._inotify_general_wm
+            w_dict = self._general_watchs
+            events = INOTIFY_EVENTS_GENERAL
+
+        # add the watch!
+        self.log.debug("Adding %s inotify watch to %r", w_type, dirpath)
+        result = w_manager.add_watch(dirpath, events)
+        w_dict[dirpath] = result[dirpath]
 
     def inotify_has_watch(self, dirpath):
         """Check if a dirpath is watched."""
-        return dirpath in self._watchs
+        return (dirpath in self._general_watchs or
+                dirpath in self._ancestors_watchs)
+
+    def _is_udf_ancestor(self, path):
+        """Decide if path is an UDF ancestor or not."""
+        for udf in self.fs.vm.udfs.itervalues():
+            parent = os.path.dirname(udf.path) + os.path.sep
+            if parent.startswith(path + os.path.sep):
+                return True
+        return False
 
     def unsubscribe(self, obj):
         """Removes the callback object from the listener queue.

=== modified file 'ubuntuone/syncdaemon/tools.py'
--- ubuntuone/syncdaemon/tools.py	2010-01-25 14:58:45 +0000
+++ ubuntuone/syncdaemon/tools.py	2010-01-26 20:35:29 +0000
@@ -298,7 +298,7 @@
         self.log.debug('accept_share(%s)', share_id)
         shares_client = DBusClient(self.bus, '/shares', DBUS_IFACE_SHARES_NAME)
         d = self.wait_for_signal('ShareAnswerResponse',
-                                 lambda info: info['share_id']==share_id)
+                                 lambda info: info['volume_id']==share_id)
         shares_client.call_method('accept_share', share_id,
                                   reply_handler=lambda _: None,
                                   error_handler=d.errback)
@@ -309,7 +309,7 @@
         self.log.debug('reject_share(%s)', share_id)
         shares_client = DBusClient(self.bus, '/shares', DBUS_IFACE_SHARES_NAME)
         d = self.wait_for_signal('ShareAnswerResponse',
-                                    lambda info: info['share_id']==share_id)
+                                    lambda info: info['volume_id']==share_id)
         shares_client.call_method('reject_share', share_id,
                                   reply_handler=lambda _: None,
                                   error_handler=d.errback)
@@ -601,8 +601,8 @@
     for share in shares:
         msg_template = '  id=%s name=%s accepted=%s ' + \
                 'access_level=%s to=%s path=%s\n'
-        out.write(msg_template % (share['id'], share['name'],
-                                  share['accepted'], share['access_level'],
+        out.write(msg_template % (share['volume_id'], share['name'],
+                                  bool(share['accepted']), share['access_level'],
                                   share['other_username'],
                                   share['path']))
 
@@ -615,7 +615,7 @@
         out.write("Folder list:\n")
     for folder in folders:
         msg_template = '  id=%s subscribed=%s path=%s\n'
-        out.write(msg_template % (folder['id'], folder['subscribed'],
+        out.write(msg_template % (folder['volume_id'], folder['subscribed'],
                                   folder['path']))
 
 
@@ -636,7 +636,7 @@
         out.write("Shares list:\n")
     for share in shares:
         out.write(' id=%s name=%s accepted=%s access_level=%s from=%s\n' % \
-                  (share['id'], share['name'], share['accepted'],
+                  (share['volume_id'], share['name'], bool(share['accepted']),
                    share['access_level'], share['other_username']))
 
 

=== modified file 'ubuntuone/syncdaemon/volume_manager.py'
--- ubuntuone/syncdaemon/volume_manager.py	2010-01-26 19:40:05 +0000
+++ ubuntuone/syncdaemon/volume_manager.py	2010-01-26 20:35:29 +0000
@@ -41,10 +41,10 @@
 from twisted.internet import defer
 
 
-class Share(object):
+class _Share(object):
     """Represents a share or mount point"""
 
-    def __init__(self, volume_id=request.ROOT, node_id=None, path=None,
+    def __init__(self, share_id=request.ROOT, node_id=None, path=None,
                  name=None, access_level='View', accepted=False,
                  other_username=None, other_visible_name=None):
         """ Creates the instance.
@@ -55,7 +55,7 @@
             self.path = None
         else:
             self.path = os.path.normpath(path)
-        self.id = str(volume_id)
+        self.id = str(share_id)
         self.access_level = access_level
         self.accepted = accepted
         self.name = name
@@ -64,16 +64,77 @@
         self.subtree = node_id
         self.free_bytes = None
 
+
+class _UDF(object):
+    """A representation of a User Defined Folder."""
+
+    def __init__(self, udf_id, node_id, suggested_path,
+                 path, subscribed=True):
+        """Create the UDF, subscribed by default"""
+        # id and node_id should be str or None
+        assert isinstance(udf_id, basestring) or udf_id is None
+        assert isinstance(node_id, basestring) or node_id is None
+        self.id = udf_id
+        self.node_id = node_id
+        self.suggested_path = suggested_path
+        self.path = path
+        self.subscribed = subscribed
+
+
+class Volume(object):
+    """A generic volume."""
+
+    def __init__(self, volume_id, node_id):
+        """Create the volume."""
+        # id and node_id should be str or None
+        assert isinstance(volume_id, basestring) or volume_id is None
+        assert isinstance(node_id, basestring) or node_id is None
+        self.volume_id = volume_id
+        self.node_id = node_id
+
+    @property
+    def id(self):
+        return self.volume_id
+
+    def can_write(self):
+        raise NotImplementedError('Subclass responsability')
+
+    def __eq__(self, other):
+        result = (self.id == other.id and
+                  self.node_id == other.node_id)
+        return result
+
+
+class Share(Volume):
+    """A volume representing a Share."""
+
+    def __init__(self, volume_id=None, node_id=None, path=None, name=None,
+                 other_username=None, other_visible_name=None, accepted=False,
+                 access_level='View', free_bytes=None):
+        """Create the share."""
+        super(Share, self).__init__(volume_id, node_id)
+        self.__dict__['type'] = 'Share'
+        if path is None:
+            self.path = None
+        else:
+            self.path = os.path.normpath(path)
+        self.name = name
+        self.other_username = other_username
+        self.other_visible_name = other_visible_name
+        self.accepted = accepted
+        self.access_level = access_level
+        self.free_bytes = free_bytes
+
     @classmethod
     def from_response(cls, share_response, path):
         """ Creates a Share instance from a ShareResponse.
 
         The received path should be 'bytes'
         """
-        share = cls(str(share_response.id), share_response.subtree, path,
-                    share_response.name, share_response.access_level,
-                    share_response.accepted, share_response.other_username,
-                    share_response.other_visible_name)
+        share = cls(str(share_response.id), str(share_response.subtree),
+                    path, share_response.name, share_response.other_username,
+                    share_response.other_visible_name,
+                    share_response.accepted, share_response.access_level)
         return share
 
     @classmethod
@@ -82,12 +143,12 @@
 
         The received path should be 'bytes'
         """
-        share = cls(path=path, volume_id=str(share_notify.share_id),
-                    name=share_notify.share_name,
-                    access_level=share_notify.access_level,
+        share = cls(volume_id=str(share_notify.share_id),
+                    node_id=str(share_notify.subtree),
+                    path=path, name=share_notify.share_name,
                     other_username=share_notify.from_username,
                     other_visible_name=share_notify.from_visible_name,
-                    node_id=share_notify.subtree)
+                    access_level=share_notify.access_level)
         return share
 
     @classmethod
@@ -97,13 +158,11 @@
         The received path should be 'bytes'
 
         """
-        share = cls(volume_id=str(share_volume.volume_id), path=path,
-                    name=share_volume.share_name,
-                    access_level=share_volume.access_level,
-                    other_username=share_volume.other_username,
-                    other_visible_name=share_volume.other_visible_name,
-                    node_id=str(share_volume.node_id),
-                    accepted=share_volume.accepted)
+        share = cls(str(share_volume.volume_id), str(share_volume.node_id),
+                    path, share_volume.share_name,
+                    share_volume.other_username,
+                    share_volume.other_visible_name, share_volume.accepted,
+                    share_volume.access_level)
         return share
 
     def can_write(self):
@@ -114,35 +173,64 @@
 
     @property
     def active(self):
-        """Returns True if the Share is accepted."""
+        """Return True if this Share is accepted."""
         return self.accepted
 
-    # node_id property
-    def _set_node_id(self, node_id):
-        self.subtree = node_id
-    node_id = property(lambda self: self.subtree, _set_node_id)
-
-    # volume_id property
-    def _set_volume_id(self, volume_id):
-        self.id = volume_id
-    volume_id = property(lambda self: self.id, _set_volume_id)
-
-
-class UDF(object):
-    """A representation of a User Defined Folder."""
-
-    def __init__(self, volume_id, node_id, suggested_path,
-                 path, subscribed=True):
+    def __eq__(self, other):
+        result = (super(Share, self).__eq__(other) and
+                  self.path == other.path and
+                  self.name == other.name and
+                  self.other_username == other.other_username and
+                  self.other_visible_name == other.other_visible_name and
+                  self.accepted == other.accepted and
+                  self.access_level == other.access_level)
+        return result
+
+
+class Shared(Share):
+
+    def __init__(self, *args, **kwargs):
+        super(Shared, self).__init__(*args, **kwargs)
+        self.__dict__['type'] = 'Shared'
+
+
+class Root(Volume):
+    """A volume representing the root."""
+
+    def __init__(self, volume_id=None, node_id=None, path=None):
+        """Create the Root."""
+        super(Root, self).__init__(volume_id, node_id)
+        self.__dict__['type'] = 'Root'
+        self.path = path
+
+    def __eq__(self, other):
+        result = (super(Root, self).__eq__(other) and
+                  self.path == other.path)
+        return result
+
+    def can_write(self):
+        return True
+
+    def is_active(self):
+        return True
+
+
+class UDF(Volume):
+    """A volume representing a User Defined Folder."""
+
+    def __init__(self, volume_id=None, node_id=None,
+                 suggested_path=None, path=None, subscribed=True):
         """Create the UDF, subscribed by default"""
-        # id and node_id should be str or None
-        assert isinstance(volume_id, basestring) or volume_id is None
-        assert isinstance(node_id, basestring) or node_id is None
-        self.id = volume_id
+        super(UDF, self).__init__(volume_id, node_id)
+        self.__dict__['type'] = 'UDF'
         self.node_id = node_id
         self.suggested_path = suggested_path
         self.path = path
         self.subscribed = subscribed
 
+    def __repr__(self):
+        return "<UDF id %r, real path %r>" % (self.id, self.path)
+
     @property
     def ancestors(self):
         """Calculate all the ancestors for this UDF's path."""
@@ -161,6 +249,11 @@
         """We always can write in a UDF."""
         return True
 
+    @property
+    def active(self):
+        """Returns True if the UDF is subscribed."""
+        return self.subscribed
+
     @classmethod
     def from_udf_volume(cls, udf_volume, path):
         """Creates a UDF instance from a volumes.UDFVolume.
@@ -171,21 +264,18 @@
         return cls(str(udf_volume.volume_id), str(udf_volume.node_id),
                    udf_volume.suggested_path, path)
 
-    @property
-    def active(self):
-        """Returns True if the UDF is subscribed."""
-        return self.subscribed
-
-    # volume_id property
-    def _set_volume_id(self, volume_id):
-        self.id = volume_id
-    volume_id = property(lambda self: self.id, _set_volume_id)
+    def __eq__(self, other):
+        result = (super(UDF, self).__eq__(other) and
+                  self.suggested_path == other.suggested_path and
+                  self.path == other.path and
+                  self.subscribed == other.subscribed)
+        return result
 
 
 class VolumeManager(object):
     """Manages shares and mount points."""
 
-    METADATA_VERSION = '5'
+    METADATA_VERSION = '6'
 
     def __init__(self, main):
         """Create the instance and populate the shares/d attributes
@@ -199,8 +289,9 @@
         self._udfs_dir = os.path.join(self._data_dir, 'udfs')
 
         md_upgrader = MetadataUpgrader(self._data_dir, self._shares_dir,
-                                   self._shared_dir, self.m.root_dir,
-                                   self.m.shares_dir, self.m.shares_dir_link)
+                                       self._shared_dir, self._udfs_dir,
+                                       self.m.root_dir, self.m.shares_dir,
+                                       self.m.shares_dir_link)
         md_upgrader.upgrade_metadata()
 
         # build the dir layout
@@ -233,7 +324,7 @@
         self.shared = VMFileShelf(self._shared_dir)
         self.udfs = VMFileShelf(self._udfs_dir)
         if self.shares.get(request.ROOT) is None:
-            self.root = Share(self.m.root_dir)
+            self.root = Root(path=self.m.root_dir)
         else:
             self.root = self.shares[request.ROOT]
         self.root.access_level = 'Modify'
@@ -356,7 +447,7 @@
                     self.log.warning("we got a share with 'from_me' direction,"
                             " but don't have the node_id in the metadata yet")
                     path = None
-                share = Share.from_response(a_share, path)
+                share = Shared.from_response(a_share, path)
                 shared.append(share.volume_id)
                 self.add_shared(share)
         self._cleanup_volumes(shares, shared)
@@ -583,7 +674,7 @@
         mdobj = self.m.fs.get_by_path(path)
         mdid = mdobj.mdid
         marker = MDMarker(mdid)
-        share = Share(path=self.m.fs.get_abspath("", mdobj.path),
+        share = Shared(path=self.m.fs.get_abspath("", mdobj.path),
                       volume_id=marker,
                       name=name, access_level=access_level,
                       other_username=username, other_visible_name=None,
@@ -748,17 +839,18 @@
             udf = self.udfs[udf_id]
         except KeyError:
             push_error("DOES_NOT_EXIST")
-        udf.subscribed = True
-        self.udfs[udf_id] = udf
-        try:
-            d = self._scan_udf(udf)
-        except KeyError, e:
-            push_error("METADATA_DOES_NOT_EXIST")
         else:
-            d.addCallbacks(
-                lambda _: self.m.event_q.push('VM_UDF_SUBSCRIBED', udf),
-                lambda f: push_error(f.getErrorMessage()))
-        return d
+            udf.subscribed = True
+            self.udfs[udf_id] = udf
+            try:
+                d = self._scan_udf(udf)
+            except KeyError, e:
+                push_error("METADATA_DOES_NOT_EXIST")
+            else:
+                d.addCallbacks(
+                    lambda _: self.m.event_q.push('VM_UDF_SUBSCRIBED', udf),
+                    lambda f: push_error(f.getErrorMessage()))
+            return d
 
     def _scan_udf(self, udf):
         """Local and server rescan of a UDF."""
@@ -844,7 +936,7 @@
 class MetadataUpgrader(object):
     """A class that loads old metadata and migrate it."""
 
-    def __init__(self, data_dir, shares_md_dir, shared_md_dir,
+    def __init__(self, data_dir, shares_md_dir, shared_md_dir, udfs_md_dir,
                  root_dir, shares_dir, shares_dir_link):
         """Creates the instance"""
         self.log = logging.getLogger('ubuntuone.SyncDaemon.VM.MD')
@@ -852,6 +944,7 @@
         self._shares_dir = shares_dir
         self._shares_md_dir = shares_md_dir
         self._shared_md_dir = shared_md_dir
+        self._udfs_md_dir = udfs_md_dir
         self._root_dir = root_dir
         self._shares_dir_link = shares_dir_link
         self._version_file = os.path.join(self._data_dir, '.version')
@@ -878,8 +971,58 @@
             if not md_version:
                 # we don't have a version of the metadata but a .version file?
                 # assume it's None and do an upgrade from version 0
-                md_version = None
-        else:
+                md_version = self._guess_metadata_version()
+        else:
+            md_version = self._guess_metadata_version()
+        self.log.debug('metadata version: %s', md_version)
+        return md_version
+
+    def _guess_metadata_version(self):
+        """Try to guess the metadata version based on current metadata
+        and layout, fallbacks to md_version = None if can't guess it.
+
+        """
+        #md_version = None
+        if os.path.exists(self._shares_md_dir) \
+           and os.path.exists(self._shared_md_dir):
+            # we have shares and shared dirs
+            # md_version >= 1
+            old_root_dir = os.path.join(self._root_dir, 'My Files')
+            old_share_dir = os.path.join(self._root_dir, 'Shared With Me')
+            if os.path.exists(old_share_dir) and os.path.exists(old_root_dir) \
+               and not os.path.islink(old_share_dir):
+                # md >= 1 and <= 3
+                # we have a My Files dir, 'Shared With Me' isn't a
+                # symlink and ~/.local/share/ubuntuone/shares doesn't
+                # exists.
+                # md_version <= 3, set it to 2 as it will migrate
+                # .conflict to .u1conflict, and we don't need to upgrade
+                # from version 1 any more as the LegacyShareFileShelf
+                # takes care of that.
+                md_version = '2'
+            else:
+                try:
+                    target = os.readlink(self._shares_dir_link)
+                except OSError:
+                    target = None
+                if os.path.islink(self._shares_dir_link) \
+                   and os.path.normpath(target) == self._shares_dir_link:
+                    # broken symlink, md_version = 4
+                    md_version = '4'
+                else:
+                    # md_version >= 5
+                    shelf = LegacyShareFileShelf(self._shares_md_dir)
+                    # check a pickled value to check if it's in version
+                    # 5 or 6
+                    for key in shelf:
+                        share = shelf[key]
+                        if isinstance(share, _Share):
+                            md_version = '5'
+                        else:
+                            md_version = '6'
+                        break
+        else:
+            # this is metadata 'None'
             md_version = None
         return md_version
 
@@ -902,9 +1045,6 @@
                     if dir != os.path.basename(backup):
                         shutil.move(os.path.join(dirname, dir),
                                     os.path.join(backup, dir))
-        # add the old module FQN to sys.modules in order to load the metadata
-        sys.modules['canonical.ubuntuone.storage.syncdaemon.volume_manager'] = \
-                sys.modules['ubuntuone.syncdaemon.volume_manager']
         # regenerate the shelf using the new layout using the backup as src
         old_shelf = LegacyShareFileShelf(backup)
         if not os.path.exists(self._shares_dir):
@@ -912,9 +1052,7 @@
         new_shelf = LegacyShareFileShelf(self._shares_md_dir)
         for key in old_shelf.keys():
             new_shelf[key] = old_shelf[key]
-        # undo the change to sys.modules
-        del sys.modules['canonical.ubuntuone.storage.syncdaemon.volume_manager']
-        # now upgrade to metadata 3
+        # now upgrade to metadata 2
         self._upgrade_metadata_2(md_version)
 
     def _upgrade_metadata_1(self, md_version):
@@ -946,7 +1084,6 @@
                     for names in filenames, dirnames:
                         self._upgrade_names(dirpath, names)
         self._upgrade_metadata_3(md_version)
-        self.update_metadata_version()
 
     def _upgrade_names(self, dirpath, names):
         """
@@ -997,6 +1134,14 @@
         old_root_dir = os.path.join(self._root_dir, 'My Files')
         # change permissions
         os.chmod(self._root_dir, 0775)
+
+        def move(src, dst):
+            """Move a file/dir taking care if it's read-only."""
+            prev_mode = stat.S_IMODE(os.stat(src).st_mode)
+            os.chmod(src, 0755)
+            shutil.move(src, dst)
+            os.chmod(dst, prev_mode)
+
         # update the path's in metadata and move the folder
         if os.path.exists(old_share_dir) and not os.path.islink(old_share_dir):
             os.chmod(old_share_dir, 0775)
@@ -1004,14 +1149,23 @@
                 os.makedirs(os.path.dirname(self._shares_dir))
             self.log.debug('moving shares dir from: %r to %r',
                            old_share_dir, self._shares_dir)
-            shutil.move(old_share_dir, self._shares_dir)
+            for path in os.listdir(old_share_dir):
+                src = os.path.join(old_share_dir, path)
+                dst = os.path.join(self._shares_dir, path)
+                move(src, dst)
+            os.rmdir(old_share_dir)
+
         # update the shares metadata
         shares = LegacyShareFileShelf(self._shares_md_dir)
         for key in shares.keys():
             share = shares[key]
             if share.path is not None:
-                share.path = share.path.replace(old_share_dir,
-                                                self._shares_dir)
+                if share.path == old_root_dir:
+                    share.path = share.path.replace(old_root_dir,
+                                                    self._root_dir)
+                else:
+                    share.path = share.path.replace(old_share_dir,
+                                                    self._shares_dir)
                 shares[key] = share
 
         shared = LegacyShareFileShelf(self._shared_md_dir)
@@ -1021,7 +1175,7 @@
                 share.path = share.path.replace(old_root_dir, self._root_dir)
             shared[key] = share
         # move the My Files contents, taking care of dir/files with the same
-        # in the new root
+        # name in the new root
         if os.path.exists(old_root_dir):
             self.log.debug('moving My Files contents to the root')
             # make My Files rw
@@ -1038,10 +1192,11 @@
                     os.remove(old_path)
                 else:
                     self.log.debug('moving %r to %r', old_path, new_path)
-                    shutil.move(old_path, new_path)
+                    move(old_path, new_path)
             self.log.debug('removing old root: %r', old_root_dir)
             os.rmdir(old_root_dir)
 
+        # fix broken symlink (md_version 4)
         self._upgrade_metadata_4(md_version)
 
     def _upgrade_metadata_4(self, md_version):
@@ -1056,9 +1211,61 @@
                 self.log.debug('removing broken shares symlink: %r -> %r',
                                self._shares_dir_link, target)
                 os.remove(self._shares_dir_link)
+        self._upgrade_metadata_5(md_version)
 
+    def _upgrade_metadata_5(self, md_version):
+        """
+        Upgrade to version 6 (plain dict storage)
+        """
+        self.log.debug('upgrading from metadata 5')
+        # upgrade shares
+        old_shares = LegacyShareFileShelf(self._shares_md_dir)
+        shares = VMFileShelf(self._shares_md_dir)
+        for key in old_shares.keys():
+            share = old_shares[key]
+            shares[key] = self._upgrade_share_to_volume(share)
+        # upgrade shared folders
+        old_shared = LegacyShareFileShelf(self._shared_md_dir)
+        shared = VMFileShelf(self._shared_md_dir)
+        for key in shared.keys():
+            share = old_shared[key]
+            shared[key] = self._upgrade_share_to_volume(share, shared=True)
+        # upgrade the udfs
+        old_udfs = LegacyShareFileShelf(self._udfs_md_dir)
+        udfs = VMFileShelf(self._udfs_md_dir)
+        for key in old_udfs.keys():
+            udf = old_udfs[key]
+            udfs[key] = UDF(udf.id, udf.node_id, udf.suggested_path,
+                            udf.path, udf.subscribed)
         self.update_metadata_version()
 
+    def _upgrade_share_to_volume(self, share, shared=False):
+        """Upgrade from _Share to new Volume hierarchy."""
+        def upgrade_share_dict(share):
+            """Upgrade share __dict__ to be compatible with the
+            new Share.__init__.
+
+            """
+            if 'subtree' in share.__dict__:
+                share.node_id = share.__dict__.pop('subtree')
+            if 'id' in share.__dict__:
+                share.volume_id = share.__dict__.pop('id')
+            if 'free_bytes' in share.__dict__:
+                free_bytes = share.__dict__.pop('free_bytes')
+            else:
+                free_bytes = None
+            return share
+        # handle the root special case
+        if share.path == self._root_dir or share.id == '':
+            r = Root(share.id, share.subtree, share.path)
+            return r
+        else:
+            share = upgrade_share_dict(share)
+            if shared:
+                return Shared(**share.__dict__)
+            else:
+                return Share(**share.__dict__)
+
     def update_metadata_version(self):
         """write the version of the metadata"""
         if not os.path.exists(os.path.dirname(self._version_file)):
@@ -1082,8 +1289,13 @@
 class VMFileShelf(file_shelf.FileShelf):
     """ Custom file shelf that allow request.ROOT as key, it's replaced
     by the string: root_node_id.
+
     """
 
+    TYPE = 'type'
+    classes = dict((sub.__name__, sub) for sub in \
+                   Volume.__subclasses__() + Share.__subclasses__())
+
     def __init__(self, *args, **kwargs):
         """ Create the instance. """
         super(VMFileShelf, self).__init__(*args, **kwargs)
@@ -1103,6 +1315,22 @@
             else:
                 yield key
 
+    def _unpickle(self, fd):
+        """Unpickle a dict and build the class instance specified in
+        value['type'].
+        """
+        value = cPickle.load(fd)
+        class_name = value[self.TYPE]
+        clazz = self.classes[class_name]
+        obj = clazz.__new__(clazz)
+        obj.__dict__.update(value)
+        return obj
+
+    def _pickle(self, value, fd, protocol):
+        """Pickle value in fd using protocol."""
+        cPickle.dump(value.__dict__, fd, protocol=protocol)
+
+
 class LegacyShareFileShelf(VMFileShelf):
     """A FileShelf capable of replacing pickled classes
     with a different class.
@@ -1112,11 +1340,12 @@
     """
 
     upgrade_map = {
+        ('ubuntuone.syncdaemon.volume_manager', 'UDF'):_UDF,
+        ('ubuntuone.syncdaemon.volume_manager', 'Share'):_Share,
         ('canonical.ubuntuone.storage.syncdaemon.volume_manager',
-         'Share'):Share
+         'Share'):_Share
     }
 
-
     def _find_global(self, module, name):
         """Returns the class object for (module, name) or None."""
         # handle our 'migration types'