Merge lp:~jderose/dmedia/icore into lp:dmedia

[Jason Gerard DeRose]

For background, see: https://bugs.launchpad.net/dmedia/+bug/1216662

Changes include:

* Core.__init__() now has two new required arguments, *machine* and *user*, the machine and user docs, thereby making the identity parts of the Core API non-optional

* Removed Core.load_identity() method as Core.__init__() now does the same

* Local file-stores and peers are now only saved in the dmedia/machine doc, are no longer saved in the _local/dmedia doc

* LocalStores.local_stores() now returns a dict whose keys are the FileStore.id rather than the FileStore.parentdir, which makes it easy to use set intersection to determine whether a given peer has a specific file, like this:

    set(peer['stores']).intersection(file['stored'])

* As many different bits of code relied on the stores being in _local/dmedia, the rest of the change is basically porting to the new API and updating tests

[James Raymond (jamesmr)]

review: Approve

=== modified file 'dmedia-gtk'
--- dmedia-gtk	2013-08-16 19:44:40 +0000
+++ dmedia-gtk	2013-08-25 20:53:26 +0000
@@ -102,7 +102,7 @@
     def on_batch_finished(self, hub, batch_id, stats, copies, msg):
         log.info('batch_finished: %s', batch_id)
         log.info('Calling Dmedia.SnapshotProject(%r)...', self.project_id)
-        #self.proxy.SnapshotProject(self.project_id)
+        self.proxy.SnapshotProject(self.project_id)
         self.blocking = True
         self.batch_id = batch_id
         self.copies = copies

=== modified file 'dmedia-service'
--- dmedia-service	2013-07-29 12:43:06 +0000
+++ dmedia-service	2013-08-25 20:53:26 +0000
@@ -172,9 +172,11 @@
         env = self.couch.auto_bootstrap()
         log.info('%r', self.couch._welcome)
         log.info('Starting CouchDB took %.3f', time.monotonic() - start)
-        self.core = Core(env, self.couch.get_ssl_config())
-        self.core.reset_local()
-        self.core.load_identity(self.couch.machine, self.couch.user)
+        self.core = Core(env,
+            self.couch.machine,
+            self.couch.user,
+            self.couch.get_ssl_config()
+        )
         self.env_s = dumps(self.core.env, pretty=True)
         self.snapshots = Snapshots(
                 self.core.env,
@@ -233,8 +235,8 @@
         port = env['port']
         self.avahi = Avahi(self.core, port)
         self.avahi.run()
-        GLib.timeout_add(5000, self.on_idle4)
-        GLib.timeout_add(90 * 1000, self.core.reclaim_if_possible)
+        GLib.timeout_add(9000, self.on_idle4)
+        GLib.timeout_add(45 * 1000, self.core.reclaim_if_possible)
 
     def on_idle4(self):
         """
@@ -458,14 +460,14 @@
         """
         Return currently connected filestores
         """
-        return dumps(self.core.local['stores'], pretty=True)
+        return dumps(self.core.machine['stores'], pretty=True)
 
     @dbus.service.method(IFACE, in_signature='', out_signature='s')
     def Peers(self):
         """
         Return peers currently known on local network.
         """
-        return dumps(self.core.local['peers'], pretty=True)
+        return dumps(self.core.machine['peers'], pretty=True)
 
     @dbus.service.method(IFACE, in_signature='s', out_signature='s')
     def CreateFileStore(self, parentdir):
@@ -550,7 +552,7 @@
         if not self.UpdateProject(project_id):
             self.pending_update = project_id
         self.Snapshot(schema.project_db_name(project_id))
-        self.Snapshot(schema.DB_NAME)
+        #self.Snapshot(schema.DB_NAME)
 
     @dbus.service.method(IFACE, in_signature='s', out_signature='s')
     def AutoFormat(self, value):

=== modified file 'dmedia/core.py'
--- dmedia/core.py	2013-07-28 20:31:16 +0000
+++ dmedia/core.py	2013-08-25 20:53:26 +0000
@@ -467,6 +467,11 @@
     stores.pop(fs_id, None)
 
 
+def mark_connected_stores(doc, atime, stores):
+    assert isinstance(stores, dict)
+    doc['atime'] = atime
+    doc['stores'] = stores
+
 def mark_add_peer(doc, atime, peer_id, info):
     assert isinstance(info, dict)
     doc['atime'] = atime
@@ -481,9 +486,12 @@
 
 
 class Core:
-    def __init__(self, env, ssl_config=None):
+    def __init__(self, env, machine, user, ssl_config=None):
+        env.update({
+            'machine_id': machine['_id'],
+            'user_id': user['_id'],
+        })
         self.env = env
-        self.ssl_config = ssl_config
         self.db = util.get_db(env, init=True)
         self.log_db = self.db.database(schema.LOG_DB_NAME)
         self.log_db.ensure()
@@ -491,26 +499,28 @@
         self.ms = MetaStore(self.db)
         self.stores = LocalStores()
         self.task_manager = TaskManager(env, ssl_config)
+        self.ssl_config = ssl_config
         try:
             self.local = self.db.get(LOCAL_ID)
         except NotFound:
-            self.local = {
-                '_id': LOCAL_ID,
-                'stores': {},
-                'peers': {},
-            }
-        self.__local = deepcopy(self.local)
-        self.machine = None
-        self.user = None
+            self.local = {'_id': LOCAL_ID}
+        self.local.update({
+            'machine_id': machine['_id'],
+            'user_id': user['_id'],
+        })
+        self.local.pop('stores', None)
+        self.local.pop('peers', None)
+        (self.machine, self.user) = self.db.get_defaults([machine, user])
+        self.machine.update({
+            'stores': {},
+            'peers': {},
+        })
+        self.db.save_many([self.local, self.machine, self.user])
+        log.info('machine_id = %s', machine['_id'])
+        log.info('user_id = %s', user['_id'])
 
     def save_local(self):
-        if self.local != self.__local:
-            self.db.save(self.local)
-            self.__local = deepcopy(self.local)
-
-    def reset_local(self):
-        self.local['stores'] = {}
-        self.local['peers'] = {}
+        self.db.save(self.local)
 
     def start_background_tasks(self):
         self.task_manager.start_tasks()
@@ -537,53 +547,29 @@
         self.local['skip_internal'] = flag
         self.save_local()
 
-    def load_identity(self, machine, user, timestamp=None):
-        if timestamp is None:
-            timestamp = int(time.time())
-        assert isinstance(timestamp, int) and timestamp > 0
-        try:
-            self.db.save_many([machine, user])
-        except BulkConflict:
-            pass
-        log.info('machine_id = %s', machine['_id'])
-        log.info('user_id = %s', user['_id'])
-        self.env['machine_id'] = machine['_id']
-        self.env['user_id'] = user['_id']
-        self.local['machine_id'] = machine['_id']
-        self.local['user_id'] = user['_id']
-        self.save_local()
-        self.machine = self.db.update(mark_machine_start, machine, timestamp)
-
     def add_peer(self, peer_id, info):
         assert isdb32(peer_id) and len(peer_id) == 48
         assert isinstance(info, dict)
         assert isinstance(info['url'], str)
-        self.local['peers'][peer_id] = info
-        self.save_local()
-        if self.machine:
-            atime = int(time.time())
-            self.machine = self.db.update(
-                mark_add_peer, self.machine, atime, peer_id, info
-            )
+        self.machine = self.db.update(
+            mark_add_peer, self.machine, int(time.time()), peer_id, info
+        )
         self.restart_vigilance()
 
     def remove_peer(self, peer_id):
-        if self.machine:
-            atime = int(time.time())
-            self.machine = self.db.update(
-                mark_remove_peer, self.machine, atime, peer_id
-            )
-        try:
-            del self.local['peers'][peer_id]
-            self.save_local()
-            self.restart_vigilance()
-            return True
-        except KeyError:
+        if peer_id not in self.machine['peers']:
             return False
+        self.machine = self.db.update(
+            mark_remove_peer, self.machine, int(time.time()), peer_id
+        )
+        self.restart_vigilance()
+        return True
 
     def _sync_stores(self):
-        self.local['stores'] = self.stores.local_stores()
-        self.save_local()
+        stores = self.stores.local_stores()
+        self.machine = self.db.update(
+            mark_connected_stores, self.machine, int(time.time()), stores
+        )
         self.restart_vigilance()
 
     def _add_filestore(self, fs):
@@ -601,23 +587,12 @@
             pass
         self.task_manager.queue_filestore_tasks(fs)
         self._sync_stores()
-        if self.machine:
-            atime = int(time.time())
-            info = {'parentdir': fs.parentdir}
-            self.machine = self.db.update(
-                mark_add_filestore, self.machine, atime, fs.id, info
-            )
 
     def _remove_filestore(self, fs):
         log.info('Removing %r', fs)
         self.stores.remove(fs)
         self.task_manager.stop_filestore_tasks(fs)
         self._sync_stores()
-        if self.machine:
-            atime = int(time.time())
-            self.machine = self.db.update(
-                mark_remove_filestore, self.machine, atime, fs.id
-            )
 
     def _iter_project_dbs(self):
         for (name, _id) in projects_iter(self.server):

=== modified file 'dmedia/importer.py'
--- dmedia/importer.py	2013-08-16 19:44:40 +0000
+++ dmedia/importer.py	2013-08-25 20:53:26 +0000
@@ -43,7 +43,7 @@
 from dmedia.util import get_project_db
 from dmedia.units import bytes10
 from dmedia import workers, schema
-from dmedia.metastore import create_stored, merge_stored, TimeDelta
+from dmedia.metastore import MetaStore, create_stored, merge_stored, TimeDelta
 from dmedia.extractor import extract, merge_thumbnail
 
 
@@ -216,9 +216,9 @@
         # FIXME: Should pick up to 2 filestores based size of import and
         # available space on the filestores.
         stores = []
-        for parentdir in sorted(self.env['stores']):
-            info = self.env['stores'][parentdir]
-            fs = FileStore(parentdir, info['id'])
+        for _id in sorted(self.env['stores']):
+            info = self.env['stores'][_id]
+            fs = FileStore(info['parentdir'], _id)
             stores.append(fs)
         return stores
 
@@ -325,6 +325,7 @@
         self._reset()
         if not workers.isregistered(ImportWorker):
             workers.register(ImportWorker)
+        self.ms = MetaStore(self.db)
 
     def _reset(self):
         self._error = None
@@ -340,7 +341,8 @@
         assert self.doc is None
         assert self._workers == {}
         self._reset()
-        stores = self.db.get('_local/dmedia')['stores']
+        self.machine = self.ms.get_machine()
+        stores = self.machine['stores']
         assert isinstance(stores, dict)
         if not stores:
             raise ValueError('No FileStores to import into!')

=== modified file 'dmedia/local.py'
--- dmedia/local.py	2013-05-14 21:28:00 +0000
+++ dmedia/local.py	2013-08-25 20:53:26 +0000
@@ -191,25 +191,25 @@
             return fs
 
     def local_stores(self):
-        stores = {}
-        for fs in self.ids.values():
-            stores[fs.parentdir] = {'id': fs.id, 'copies': fs.copies}
-        return stores
-        
+        return dict(
+            (fs.id, {'parentdir': fs.parentdir, 'copies': fs.copies})
+            for fs in self.ids.values()
+        )
 
 
 class LocalSlave:
     def __init__(self, env):
         self.db = get_db(env)
+        self.machine_id = env['machine_id']
         self.last_rev = None
 
     def update_stores(self):
-        local = self.db.get('_local/dmedia')
-        if local['_rev'] != self.last_rev:
-            self.last_rev = local['_rev']
+        machine = self.db.get(self.machine_id)
+        if machine['_rev'] != self.last_rev:
+            self.last_rev = machine['_rev']
             self.stores = LocalStores()
-            for (parentdir, info) in local['stores'].items():
-                fs = FileStore(parentdir, info['id'])
+            for (_id, info) in machine['stores'].items():
+                fs = FileStore(info['parentdir'], _id)
                 self.stores.add(fs)
 
     def get_doc(self, _id):

=== modified file 'dmedia/metastore.py'
--- dmedia/metastore.py	2013-07-24 04:56:45 +0000
+++ dmedia/metastore.py	2013-08-25 20:53:26 +0000
@@ -354,17 +354,26 @@
         except NotFound:
             return {}
 
+    def get_machine(self):
+        machine_id = self.get_local_dmedia().get('machine_id')
+        if machine_id is None:
+            return {}
+        try:
+            return self.db.get(machine_id)
+        except NotFound:
+            return {}
+
     def get_local_stores(self):
-        doc = self.get_local_dmedia()
+        doc = self.get_machine()
         stores = get_dict(doc, 'stores')
         local_stores = LocalStores()
-        for (parentdir, info) in stores.items():
-            fs = FileStore(parentdir, info['id'])
+        for (_id, info) in stores.items():
+            fs = FileStore(info['parentdir'], _id)
             local_stores.add(fs)
         return local_stores
 
     def get_local_peers(self):
-        doc = self.get_local_dmedia()
+        doc = self.get_machine()
         self._peers = get_dict(doc, 'peers')
         return self._peers
 

=== modified file 'dmedia/service/tests/test_avahi.py'
--- dmedia/service/tests/test_avahi.py	2013-05-25 00:57:20 +0000
+++ dmedia/service/tests/test_avahi.py	2013-08-25 20:53:26 +0000
@@ -47,13 +47,15 @@
     def test_init(self):
         pki = TempPKI(client_pki=True)
         ssl_config = pki.get_client_config()
-        core = Core(self.env, ssl_config)
+        machine = {'_id': random_id(30)}
+        user = {'_id': random_id(30)}
+        core = Core(self.env, machine, user, ssl_config)
         port = random_port()
         inst = avahi.Avahi(core, port)
         self.assertIs(inst.core, core)
         self.assertEqual(inst.port, port)
-        self.assertEqual(inst.machine_id, self.machine_id)
-        self.assertEqual(inst.user_id, self.user_id)
+        self.assertEqual(inst.machine_id, machine['_id'])
+        self.assertEqual(inst.user_id, user['_id'])
         self.assertIs(inst.server, core.server)
         self.assertIsInstance(inst.ssl_context, ssl.SSLContext)
         self.assertEqual(inst.replications, {})

=== modified file 'dmedia/tests/test_core.py'
--- dmedia/tests/test_core.py	2013-07-28 01:11:13 +0000
+++ dmedia/tests/test_core.py	2013-08-25 20:53:26 +0000
@@ -40,6 +40,7 @@
 from filestore import FileStore
 from filestore.misc import TempFileStore
 from filestore.migration import Migration, b32_to_db32
+from usercouch.misc import CouchTestCase
 
 from dmedia.local import LocalStores
 from dmedia.metastore import MetaStore, get_mtime
@@ -148,6 +149,39 @@
             'stores': {},
         })
 
+    def test_mark_connected_stores(self):
+        atime = int(time.time())
+        fs1 = TempFileStore()
+        fs2 = TempFileStore()
+
+        doc = {}
+        stores = {
+            fs1.id: {'parentdir': fs1.parentdir}
+        }
+        self.assertIsNone(core.mark_connected_stores(doc, atime, stores))
+        self.assertEqual(doc, {
+            'atime': atime,
+            'stores': {
+                fs1.id: {'parentdir': fs1.parentdir}
+            },
+        })
+        self.assertIs(doc['stores'], stores)
+
+        doc = {
+            'atime': atime - 123456,
+            'stores': {
+                fs1.id: {'parentdir': fs1.parentdir},
+                fs2.id: {'parentdir': fs2.parentdir},
+            },
+        }
+        stores = {}
+        self.assertIsNone(core.mark_connected_stores(doc, atime, stores))
+        self.assertEqual(doc, {
+            'atime': atime,
+            'stores': {},
+        })
+        self.assertIs(doc['stores'], stores)
+
     def test_mark_add_peer(self):
         doc = {}
         atime = int(time.time())
@@ -265,100 +299,81 @@
         self.assertEqual(tq.popitem(), ('a', ('aye', 2)))
 
 
-class TestCore(CouchCase):
+class TestCore(CouchTestCase):
+    def create(self):
+        self.machine_id = random_id(30)
+        self.user_id = random_id(30)
+        self.machine = {'_id': self.machine_id}
+        self.user = {'_id': self.user_id}
+        return core.Core(self.env, self.machine, self.user)
+
     def test_init(self):
-        inst = core.Core(self.env)
-        self.assertIs(inst.env, self.env)
-        self.assertIsInstance(inst.db, microfiber.Database)
-        self.assertEqual(inst.db.name, DB_NAME)
-        self.assertIsInstance(inst.server, microfiber.Server)
-        self.assertIs(inst.db.ctx, inst.server.ctx)
-        self.assertIsInstance(inst.stores, LocalStores)
-        self.assertEqual(inst.local,
-            {
-                '_id': '_local/dmedia',
-                'stores': {},
-                'peers': {},
-            }
-        )
-        self.assertIsNone(inst.machine)
-        self.assertIsNone(inst.user)
-
-    def test_load_identity(self):
-        timestamp = int(time.time())
         machine_id = random_id(30)
         user_id = random_id(30)
-        inst = core.Core(self.env)
-        self.assertIsNone(
-            inst.load_identity({'_id': machine_id}, {'_id': user_id}, timestamp)
-        )
-
-        machine = inst.db.get(machine_id)
-        self.assertTrue(machine['_rev'].startswith('2-'))
-        self.assertEqual(machine, {
-            '_id': machine_id,
-            '_rev': machine['_rev'],
-            'atime': timestamp,
-            'stores': {},
-            'peers': {},
-        })
-
-        user = inst.db.get(user_id)
-        self.assertEqual(set(user), set(['_id', '_rev']))
-        self.assertTrue(user['_rev'].startswith('1-'))
-
-        self.assertEqual(
-            inst.db.get('_local/dmedia'),
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-1',
-                'stores': {},
-                'peers': {},
-                'machine_id': machine_id,
-                'user_id': user_id,
-            }
-        )
-        self.assertEqual(inst.local, inst.db.get('_local/dmedia'))
-        self.assertEqual(self.env['machine_id'], machine_id)
-        self.assertEqual(self.env['user_id'], user_id)
-
-        # Now try when machine and user docs already exist:
-        machine['atime'] = timestamp - 12345
-        machine['stores'] = 'foo'
-        machine['peers'] = 'bar'
-        inst.db.save(machine)
-        inst = core.Core(self.env)
-        self.assertIsNone(
-            inst.load_identity({'_id': machine_id}, {'_id': user_id}, timestamp)
-        )
-
-        machine = inst.db.get(machine_id)
-        self.assertTrue(machine['_rev'].startswith('4-'))
-        self.assertEqual(machine, {
-            '_id': machine_id,
-            '_rev': machine['_rev'],
-            'atime': timestamp,
-            'stores': {},
-            'peers': {},
-        })
-
-        user = inst.db.get(user_id)
-        self.assertEqual(set(user), set(['_id', '_rev']))
-        self.assertTrue(user['_rev'].startswith('1-'))
-
-        self.assertEqual(inst.db.get('_local/dmedia'),
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-1',
-                'stores': {},
-                'peers': {},
-                'machine_id': machine_id,
-                'user_id': user_id,
-            }
-        )
+        machine = {'_id': machine_id}
+        user = {'_id': user_id}
+
+        inst = core.Core(self.env, machine, user)
+        self.assertIs(inst.env, self.env)
+        self.assertEqual(inst.env['machine_id'], machine_id)
+        self.assertEqual(inst.env['user_id'], user_id)
+        self.assertIsInstance(inst.db, microfiber.Database)
+        self.assertEqual(inst.db.name, 'dmedia-1')
+        self.assertIsInstance(inst.log_db, microfiber.Database)
+        self.assertEqual(inst.log_db.name, 'log-1')
+        self.assertIsInstance(inst.server, microfiber.Server)
+        self.assertIsInstance(inst.ms, MetaStore)
+        self.assertIs(inst.ms.db, inst.db)
+        self.assertIsInstance(inst.stores, LocalStores)
+        self.assertIsInstance(inst.task_manager, core.TaskManager)
+        self.assertIsNone(inst.ssl_config)
+        self.assertEqual(inst.db.get('_local/dmedia'), {
+            '_id': '_local/dmedia',
+            '_rev': '0-1',
+            'machine_id': machine_id,
+            'user_id': user_id,
+        })
+        self.assertIs(inst.machine, machine)
+        self.assertEqual(inst.db.get(machine_id), machine)
+        self.assertEqual(inst.machine['_rev'][:2], '1-')
+        self.assertEqual(inst.machine['stores'], {})
+        self.assertEqual(inst.machine['peers'], {})
+        self.assertIs(inst.user, user)
+        self.assertEqual(inst.db.get(user_id), user)
+        self.assertEqual(inst.user['_rev'][:2], '1-')
+
+        ssl_config = random_id()
+        inst = core.Core(self.env, machine, user, ssl_config)
+        self.assertIs(inst.env, self.env)
+        self.assertEqual(inst.env['machine_id'], machine_id)
+        self.assertEqual(inst.env['user_id'], user_id)
+        self.assertIsInstance(inst.db, microfiber.Database)
+        self.assertEqual(inst.db.name, 'dmedia-1')
+        self.assertIsInstance(inst.log_db, microfiber.Database)
+        self.assertEqual(inst.log_db.name, 'log-1')
+        self.assertIsInstance(inst.server, microfiber.Server)
+        self.assertIsInstance(inst.ms, MetaStore)
+        self.assertIs(inst.ms.db, inst.db)
+        self.assertIsInstance(inst.stores, LocalStores)
+        self.assertIsInstance(inst.task_manager, core.TaskManager)
+        self.assertIs(inst.ssl_config, ssl_config)
+        self.assertEqual(inst.db.get('_local/dmedia'), {
+            '_id': '_local/dmedia',
+            '_rev': '0-2',
+            'machine_id': machine_id,
+            'user_id': user_id,
+        })
+        self.assertIsNot(inst.machine, machine)
+        self.assertEqual(inst.db.get(machine_id), inst.machine)
+        self.assertEqual(inst.machine['_rev'][:2], '2-')
+        self.assertEqual(inst.machine['stores'], {})
+        self.assertEqual(inst.machine['peers'], {})
+        self.assertIsNot(inst.user, user)
+        self.assertEqual(inst.db.get(user_id), inst.user)
+        self.assertEqual(inst.user['_rev'][:2], '2-')
 
     def test_add_peer(self):
-        inst = core.Core(self.env)
+        inst = self.create()
         id1 = random_id(30)
         info1 = {
             'host': 'jderose-Gazelle-Professional',
@@ -371,109 +386,63 @@
 
         # id1 is not yet a peer:
         self.assertIsNone(inst.add_peer(id1, info1))
-        self.assertEqual(inst.db.get('_local/dmedia'),
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-1',
-                'stores': {},
-                'peers': {
-                    id1: info1,
-                },
-            }
-        )
+        self.assertEqual(inst.machine['peers'], {id1: info1})
+        self.assertEqual(inst.db.get(self.machine_id), inst.machine)
 
         # id2 is not yet a peer:
         self.assertIsNone(inst.add_peer(id2, info2))
-        self.assertEqual(inst.db.get('_local/dmedia'),
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-2',
-                'stores': {},
-                'peers': {
-                    id1: info1,
-                    id2: info2,
-                },
-            }
-        )
+        self.assertEqual(inst.machine['peers'], {id1: info1, id2: info2})
+        self.assertEqual(inst.db.get(self.machine_id), inst.machine)
 
         # id1 is already a peer, make sure info is replaced
         new1 = {'url': random_id()}
         self.assertIsNone(inst.add_peer(id1, new1))
-        self.assertEqual(inst.db.get('_local/dmedia'),
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-3',
-                'stores': {},
-                'peers': {
-                    id1: new1,
-                    id2: info2,
-                },
-            }
-        )
+        self.assertEqual(inst.machine['peers'], {id1: new1, id2: info2})
+        self.assertEqual(inst.db.get(self.machine_id), inst.machine)
 
     def test_remove_peer(self):
+        inst = self.create()
         id1 = random_id(30)
         id2 = random_id(30)
         info1 = {'url': random_id()}
         info2 = {'url': random_id()}
-
-        db = microfiber.Database('dmedia-1', self.env)
-        db.ensure()
-        local = {
-            '_id': '_local/dmedia',
-            'stores': {},
-            'peers': {
-                id1: info1,
-                id2: info2,
-            },
-        }
-        db.save(local)
-        inst = core.Core(self.env)
+        inst.machine['peers'] = {id1: info1, id2: info2}
+        inst.db.save(inst.machine)
+        self.assertEqual(inst.machine['_rev'][:2], '2-')
 
         # Test with a peer_id that doesn't exist:
         nope = random_id(30)
         self.assertIs(inst.remove_peer(nope), False)
-        self.assertEqual(db.get('_local/dmedia'), local)
+        self.assertEqual(inst.db.get(self.machine_id), inst.machine)
+        self.assertEqual(inst.machine['peers'], {id1: info1, id2: info2})
+        self.assertEqual(inst.machine['_rev'][:2], '2-')
 
         # id1 is present
         self.assertIs(inst.remove_peer(id1), True)
-        self.assertEqual(db.get('_local/dmedia'),
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-2',
-                'stores': {},
-                'peers': {
-                    id2: info2,
-                },
-            }
-        )
+        self.assertEqual(inst.db.get(self.machine_id), inst.machine)
+        self.assertEqual(inst.machine['peers'], {id2: info2})
+        self.assertEqual(inst.machine['_rev'][:2], '3-')
 
         # id1 is missing
         self.assertIs(inst.remove_peer(id1), False)
-        self.assertEqual(db.get('_local/dmedia'),
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-2',
-                'stores': {},
-                'peers': {
-                    id2: info2,
-                },
-            }
-        )
+        self.assertEqual(inst.db.get(self.machine_id), inst.machine)
+        self.assertEqual(inst.machine['peers'], {id2: info2})
+        self.assertEqual(inst.machine['_rev'][:2], '3-')
 
         # id2 is present
         self.assertIs(inst.remove_peer(id2), True)
-        self.assertEqual(db.get('_local/dmedia'),
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-3',
-                'stores': {},
-                'peers': {},
-            }
-        )
+        self.assertEqual(inst.db.get(self.machine_id), inst.machine)
+        self.assertEqual(inst.machine['peers'], {})
+        self.assertEqual(inst.machine['_rev'][:2], '4-')
+
+        # id2 is missing
+        self.assertIs(inst.remove_peer(id2), False)
+        self.assertEqual(inst.db.get(self.machine_id), inst.machine)
+        self.assertEqual(inst.machine['peers'], {})
+        self.assertEqual(inst.machine['_rev'][:2], '4-')
 
     def test_create_filestore(self):
-        inst = core.Core(self.env)
+        inst = self.create()
 
         # Test when a FileStore already exists
         tmp = TempDir()
@@ -493,34 +462,22 @@
         self.assertEqual(fs.copies, 1)
         self.assertIs(inst.stores.by_id(fs.id), fs)
         self.assertIs(inst.stores.by_parentdir(fs.parentdir), fs)
-        self.assertEqual(
-            inst.db.get('_local/dmedia'),
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-1',
-                'stores': {
-                    fs.parentdir: {'id': fs.id, 'copies': fs.copies},
-                },
-                'peers': {},
-            }
-        )
+        self.assertEqual(inst.db.get(self.machine_id), inst.machine)
+        self.assertEqual(inst.machine['stores'], {
+            fs.id: {'parentdir': fs.parentdir, 'copies': 1},
+        })
+        self.assertEqual(inst.machine['_rev'][:2], '2-')
 
         # Make sure we can disconnect a store that was just created
         inst.disconnect_filestore(fs.parentdir)
-        self.assertEqual(
-            inst.db.get('_local/dmedia'),
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-2',
-                'stores': {},
-                'peers': {},
-            }
-        )
+        self.assertEqual(inst.db.get(self.machine_id), inst.machine)
+        self.assertEqual(inst.machine['stores'], {})
+        self.assertEqual(inst.machine['_rev'][:2], '3-')
 
     def test_connect_filestore(self):
         tmp = TempDir()
         basedir = tmp.join(filestore.DOTNAME)
-        inst = core.Core(self.env)
+        inst = self.create()
 
         # Test when .dmedia/ doesn't exist
         with self.assertRaises(FileNotFoundError) as cm:
@@ -554,7 +511,7 @@
         )
 
         # Test when expected_id is provided and matches:
-        inst = core.Core(self.env)
+        inst = self.create()
         fs_b = inst.connect_filestore(tmp.dir, expected_id=fs.id)
         self.assertIsInstance(fs_b, FileStore)
         self.assertEqual(fs_b.parentdir, tmp.dir)
@@ -571,18 +528,15 @@
         self.assertEqual(fs2_a.copies, 1)
         self.assertIs(inst.stores.by_id(fs2.id), fs2_a)
         self.assertIs(inst.stores.by_parentdir(fs2.parentdir), fs2_a)
-        self.assertEqual(
-            inst.db.get('_local/dmedia'),
+
+        self.assertEqual(inst.machine, inst.db.get(self.machine_id))
+        self.assertEqual(inst.machine['stores'], 
             {
-                '_id': '_local/dmedia',
-                '_rev': '0-2',
-                'stores': {
-                    fs.parentdir: {'id': fs.id, 'copies': 1},
-                    fs2.parentdir: {'id': fs2.id, 'copies': 1},
-                },
-                'peers': {},
-            }
+                fs.id: {'parentdir': fs.parentdir, 'copies': 1},
+                fs2.id: {'parentdir': fs2.parentdir, 'copies': 1},
+            },
         )
+        self.assertEqual(inst.machine['_rev'][:2], '3-')
 
         # Test when migration is needed
         tmp = TempDir()
@@ -592,12 +546,9 @@
         self.assertEqual(b32_to_db32(old['_id']), fs.id)
 
     def test_disconnect_filestore(self):
-        inst = core.Core(self.env)
-
-        tmp1 = TempDir()
-        fs1 = FileStore.create(tmp1.dir)
-        tmp2 = TempDir()
-        fs2 = FileStore.create(tmp2.dir)
+        inst = self.create()
+        fs1 = TempFileStore()
+        fs2 = TempFileStore()
 
         # Test when not connected:
         with self.assertRaises(KeyError) as cm:
@@ -607,44 +558,30 @@
         # Connect both, then disconnect one by one
         inst.connect_filestore(fs1.parentdir, fs1.id)
         inst.connect_filestore(fs2.parentdir, fs2.id)
-        self.assertEqual(
-            inst.db.get('_local/dmedia'),
+        self.assertEqual(inst.machine, inst.db.get(self.machine_id))
+        self.assertEqual(inst.machine['stores'], 
             {
-                '_id': '_local/dmedia',
-                '_rev': '0-2',
-                'stores': {
-                    fs1.parentdir: {'id': fs1.id, 'copies': 1},
-                    fs2.parentdir: {'id': fs2.id, 'copies': 1},
-                },
-                'peers': {},
-            }
+                fs1.id: {'parentdir': fs1.parentdir, 'copies': 1},
+                fs2.id: {'parentdir': fs2.parentdir, 'copies': 1},
+            },
         )
+        self.assertEqual(inst.machine['_rev'][:2], '3-')
 
         # Disconnect fs1
         inst.disconnect_filestore(fs1.parentdir)
-        self.assertEqual(
-            inst.db.get('_local/dmedia'),
+        self.assertEqual(inst.machine, inst.db.get(self.machine_id))
+        self.assertEqual(inst.machine['stores'], 
             {
-                '_id': '_local/dmedia',
-                '_rev': '0-3',
-                'stores': {
-                    fs2.parentdir: {'id': fs2.id, 'copies': 1},
-                },
-                'peers': {},
-            }
+                fs2.id: {'parentdir': fs2.parentdir, 'copies': 1},
+            },
         )
+        self.assertEqual(inst.machine['_rev'][:2], '4-')
 
         # Disconnect fs2
         inst.disconnect_filestore(fs2.parentdir)
-        self.assertEqual(
-            inst.db.get('_local/dmedia'),
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-4',
-                'stores': {},
-                'peers': {},
-            }
-        )
+        self.assertEqual(inst.machine, inst.db.get(self.machine_id))
+        self.assertEqual(inst.machine['stores'], {})
+        self.assertEqual(inst.machine['_rev'][:2], '5-')
 
         # Again test when not connected:
         with self.assertRaises(KeyError) as cm:
@@ -655,7 +592,7 @@
         self.assertEqual(str(cm.exception), repr(fs1.parentdir))
 
     def test_resolve(self):
-        inst = core.Core(self.env)
+        inst = self.create()
 
         bad_id1 = random_id(25)  # Wrong length
         self.assertEqual(inst.resolve(bad_id1),
@@ -705,7 +642,7 @@
         )
 
     def test_resolve_many(self):
-        inst = core.Core(self.env)
+        inst = self.create()
         tmp = TempDir()
         fs = inst.create_filestore(tmp.dir)
 
@@ -765,7 +702,7 @@
         )
 
     def test_allocate_tmp(self):
-        inst = core.Core(self.env)
+        inst = self.create()
 
         with self.assertRaises(Exception) as cm:        
             inst.allocate_tmp()
@@ -778,7 +715,7 @@
         self.assertEqual(path.getsize(name), 0)
 
     def test_hash_and_move(self):
-        inst = core.Core(self.env)
+        inst = self.create()
         tmp = TempDir()
         fs = inst.create_filestore(tmp.dir)
         tmp_fp = fs.allocate_tmp()

=== modified file 'dmedia/tests/test_importer.py'
--- dmedia/tests/test_importer.py	2013-05-15 19:43:08 +0000
+++ dmedia/tests/test_importer.py	2013-08-25 20:53:26 +0000
@@ -32,13 +32,14 @@
 from os import path
 
 import filestore
+from filestore.misc import TempFileStore
+from usercouch.misc import CouchTestCase
 from microfiber import random_id, Database
 
-from .couch import CouchCase
 from .base import TempDir, DummyQueue, MagicLanternTestCase2
 
 from dmedia.util import get_db
-from dmedia.metastore import get_mtime
+from dmedia.metastore import MetaStore, get_mtime
 from dmedia import importer, schema
 
 
@@ -202,42 +203,40 @@
         )
 
 
-class ImportCase(CouchCase):
-
+class ImportCase(CouchTestCase):
     def setUp(self):
         super().setUp()
         self.q = DummyQueue()
-
         self.src = TempDir()
 
-        temps = [TempDir() for i in range(2)]
-        (self.dst1, self.dst2) = sorted(temps, key=lambda t: t.dir)
-
-        fs1 = filestore.FileStore.create(self.dst1.dir, copies=1)
-        fs2 = filestore.FileStore.create(self.dst2.dir, copies=2)
-
-        self.store1_id = fs1.id
-        self.store2_id = fs2.id
+        filestores = [TempFileStore(copies=1), TempFileStore(copies=2)]
+        (self.fs1, self.fs2) = sorted(filestores, key=lambda fs: fs.id)
         self.stores = {
-            self.dst1.dir: {'id': self.store1_id, 'copies': 1},
-            self.dst2.dir: {'id': self.store2_id, 'copies': 2},
-        }
-        self.db = get_db(self.env)
-        self.db.ensure()
+            self.fs1.id: {'parentdir': self.fs1.parentdir, 'copies': 1},
+            self.fs2.id: {'parentdir': self.fs2.parentdir, 'copies': 2},
+        }
+
+        self.machine_id = random_id(30)
+        self.env['machine_id'] = self.machine_id
+        machine = {
+            '_id': self.machine_id,
+            'stores': self.stores,
+        }
+        self.db = get_db(self.env, True)
+        self.db.save(machine)
+
         self.project_id = random_id()
-        self.env['extract'] = False
         self.env['project_id'] = self.project_id
 
     def tearDown(self):
         super().tearDown()
-        self.q = None
-        self.src = None
-        self.dst1 = None
-        self.dst2 = None
+        del self.q
+        del self.src
+        del self.fs1
+        del self.fs2
 
 
 class TestImportWorker(ImportCase):
-
     def setUp(self):
         super().setUp()
         self.batch_id = random_id()
@@ -301,15 +300,15 @@
         self.assertEqual(len(stores), 2)
         fs1 = stores[0]
         self.assertIsInstance(fs1, filestore.FileStore)
-        self.assertEquals(fs1.parentdir, self.dst1.dir)
-        self.assertEquals(fs1.id, self.store1_id)
-        self.assertEquals(fs1.copies, 1)
+        self.assertEquals(fs1.parentdir, self.fs1.parentdir)
+        self.assertEquals(fs1.id, self.fs1.id)
+        self.assertEquals(fs1.copies, self.fs1.copies)
 
         fs2 = stores[1]
         self.assertIsInstance(fs2, filestore.FileStore)
-        self.assertEquals(fs2.parentdir, self.dst2.dir)
-        self.assertEquals(fs2.id, self.store2_id)
-        self.assertEquals(fs2.copies, 2)
+        self.assertEquals(fs2.parentdir, self.fs2.parentdir)
+        self.assertEquals(fs2.id, self.fs2.id)
+        self.assertEquals(fs2.copies, self.fs2.copies)
 
         # import_all()
         for (file, ch) in result:
@@ -355,13 +354,13 @@
             self.assertEqual(leaf_hashes, ch.leaf_hashes)
             self.assertEqual(doc['stored'],
                 {
-                    self.store1_id: {
-                        'copies': 1,
-                        'mtime': get_mtime(fs1, ch.id),
+                    self.fs1.id: {
+                        'copies': self.fs1.copies,
+                        'mtime': get_mtime(self.fs1, ch.id),
                     },
-                    self.store2_id: {
-                        'copies': 2,
-                        'mtime': get_mtime(fs2, ch.id),
+                    self.fs2.id: {
+                        'copies': self.fs2.copies,
+                        'mtime': get_mtime(self.fs2, ch.id),
                     }
                 
                 }
@@ -395,13 +394,23 @@
         super().setUp()
         local = {
             '_id': '_local/dmedia',
-            'stores': self.stores,
+            'machine_id': self.machine_id,
         }
         self.db.save(local)
 
     def new(self, callback=None):
         return self.klass(self.env, callback)
 
+    def test_init(self):
+        callback = DummyCallback()
+        inst = importer.ImportManager(self.env, callback)
+        self.assertIsNone(inst.doc)
+        self.assertIsNone(inst._error)
+        self.assertEqual(inst._progress, {})
+        self.assertIsInstance(inst.ms, MetaStore)
+        self.assertIs(inst.ms.db, inst.db)
+        self.assertEqual(inst.db.name, 'dmedia-1')
+
     def test_first_worker_starting(self):
         callback = DummyCallback()
         inst = self.new(callback)
@@ -821,10 +830,8 @@
             ('batch_finished', (batch_id, stats, 3, importer.notify_stats2(stats)))
         )
 
-        fs1 = filestore.FileStore(self.dst1.dir, self.store1_id)
-        fs2 = filestore.FileStore(self.dst2.dir, self.store2_id)
-        self.assertEqual(set(st.id for st in fs1), ids)
-        self.assertEqual(set(st.id for st in fs2), ids)
+        self.assertEqual(set(st.id for st in self.fs1), ids)
+        self.assertEqual(set(st.id for st in self.fs2), ids)
 
         # Check all the dmedia/file docs:
         for (file, ch) in result:
@@ -842,13 +849,13 @@
             self.assertEqual(leaf_hashes, ch.leaf_hashes)
             self.assertEqual(doc['stored'],
                 {
-                    self.store1_id: {
-                        'copies': 1,
-                        'mtime': get_mtime(fs1, ch.id),
+                    self.fs1.id: {
+                        'copies': self.fs1.copies,
+                        'mtime': get_mtime(self.fs1, ch.id),
                     },
-                    self.store2_id: {
-                        'copies': 2,
-                        'mtime': get_mtime(fs2, ch.id),
+                    self.fs2.id: {
+                        'copies': self.fs2.copies,
+                        'mtime': get_mtime(self.fs2, ch.id),
                     }
                 
                 }
@@ -858,8 +865,8 @@
         for (file, ch) in result:
             if ch is None:
                 continue
-            self.assertEqual(fs1.verify(ch.id), ch)
-            self.assertEqual(fs2.verify(ch.id), ch)
+            self.assertEqual(self.fs1.verify(ch.id), ch)
+            self.assertEqual(self.fs2.verify(ch.id), ch)
 
         ##################################################################
         # Okay, now run the whole thing again when they're all duplicates:
@@ -910,10 +917,8 @@
             ('batch_finished', (batch_id, stats, 3, importer.notify_stats2(stats)))
         )
 
-        fs1 = filestore.FileStore(self.dst1.dir)
-        fs2 = filestore.FileStore(self.dst2.dir)
-        self.assertEqual(set(st.id for st in fs1), ids)
-        self.assertEqual(set(st.id for st in fs2), ids)
+        self.assertEqual(set(st.id for st in self.fs1), ids)
+        self.assertEqual(set(st.id for st in self.fs2), ids)
 
         # Check all the dmedia/file docs:
         for (file, ch) in result:
@@ -931,13 +936,13 @@
             self.assertEqual(leaf_hashes, ch.leaf_hashes)
             self.assertEqual(doc['stored'],
                 {
-                    self.store1_id: {
-                        'copies': 1,
-                        'mtime': get_mtime(fs1, ch.id),
+                    self.fs1.id: {
+                        'copies': self.fs1.copies,
+                        'mtime': get_mtime(self.fs1, ch.id),
                     },
-                    self.store2_id: {
-                        'copies': 2,
-                        'mtime': get_mtime(fs2, ch.id),
+                    self.fs2.id: {
+                        'copies': self.fs2.copies,
+                        'mtime': get_mtime(self.fs2, ch.id),
                     }
                 
                 }
@@ -947,8 +952,8 @@
         for (file, ch) in result:
             if ch is None:
                 continue
-            self.assertEqual(fs1.verify(ch.id), ch)
-            self.assertEqual(fs2.verify(ch.id), ch)
+            self.assertEqual(self.fs1.verify(ch.id), ch)
+            self.assertEqual(self.fs2.verify(ch.id), ch)
             
             
 MAGIC_LANTERN = (

=== modified file 'dmedia/tests/test_local.py'
--- dmedia/tests/test_local.py	2013-05-14 21:28:00 +0000
+++ dmedia/tests/test_local.py	2013-08-25 20:53:26 +0000
@@ -27,6 +27,7 @@
 from random import Random
 import time
 
+import microfiber
 import filestore
 from filestore import DIGEST_BYTES
 from filestore.misc import TempFileStore
@@ -190,18 +191,17 @@
         inst.add(fs1)
         self.assertEqual(inst.local_stores(),
             {
-                fs1.parentdir: {'id': fs1.id, 'copies': 1},
+                fs1.id: {'parentdir': fs1.parentdir, 'copies': 1},
             }
         )
-        
+
         inst.add(fs2)
         self.assertEqual(inst.local_stores(),
             {
-                fs1.parentdir: {'id': fs1.id, 'copies': 1},
-                fs2.parentdir: {'id': fs2.id, 'copies': 0},  
+                fs1.id: {'parentdir': fs1.parentdir, 'copies': 1},
+                fs2.id: {'parentdir': fs2.parentdir, 'copies': 0},
             }
         )
-        
 
 
 class TestLocalSlave(CouchCase):
@@ -209,6 +209,57 @@
         super().setUp()
         util.get_db(self.env, True)
 
+    def test_init(self):
+        inst = local.LocalSlave(self.env)
+        self.assertIsInstance(inst.db, microfiber.Database)
+        self.assertEqual(inst.machine_id, self.machine_id)
+        self.assertIsNone(inst.last_rev)
+
+    def test_update_stores(self):
+        inst = local.LocalSlave(self.env)
+        machine = {
+            '_id': self.machine_id,
+            'stores': {},
+        }
+        inst.db.save(machine)
+
+        # No stores
+        self.assertIsNone(inst.update_stores())
+        self.assertEqual(inst.last_rev, machine['_rev'])
+        self.assertIsInstance(inst.stores, local.LocalStores)
+        self.assertEqual(inst.stores.local_stores(), {})
+
+        # One store
+        fs1 = TempFileStore()
+        machine['stores'] = {
+            fs1.id: {'parentdir': fs1.parentdir, 'copies': fs1.copies},
+        }
+        inst.db.save(machine)
+        self.assertIsNone(inst.update_stores())
+        self.assertEqual(inst.last_rev, machine['_rev'])
+        self.assertIsInstance(inst.stores, local.LocalStores)
+        self.assertEqual(inst.stores.local_stores(), machine['stores'])
+
+        # Two stores
+        fs2 = TempFileStore()
+        machine['stores'] = {
+            fs1.id: {'parentdir': fs1.parentdir, 'copies': fs1.copies},
+            fs2.id: {'parentdir': fs2.parentdir, 'copies': fs2.copies},
+        }
+        inst.db.save(machine)
+        self.assertIsNone(inst.update_stores())
+        self.assertEqual(inst.last_rev, machine['_rev'])
+        self.assertIsInstance(inst.stores, local.LocalStores)
+        self.assertEqual(inst.stores.local_stores(), machine['stores'])
+
+        # Make sure LocalStores doesn't needlessly get rebuilt
+        old = inst.stores
+        rev = inst.last_rev
+        self.assertIsNone(inst.update_stores())
+        self.assertIs(inst.stores, old)
+        self.assertIs(inst.last_rev, rev)
+        self.assertEqual(inst.stores.local_stores(), machine['stores'])
+
     def test_get_doc(self):
         inst = local.LocalSlave(self.env)
 

=== modified file 'dmedia/tests/test_metastore.py'
--- dmedia/tests/test_metastore.py	2013-07-02 08:24:57 +0000
+++ dmedia/tests/test_metastore.py	2013-08-25 20:53:26 +0000
@@ -1432,6 +1432,7 @@
         db = util.get_db(self.env, True)
         ms = metastore.MetaStore(db)
         local_id = '_local/dmedia'
+        machine_id = random_id()
 
         # _local/dmedia NotFound:
         self.assertEqual(ms.get_local_peers(), {})
@@ -1439,18 +1440,28 @@
         with self.assertRaises(microfiber.NotFound) as cm:
             db.get(local_id)
 
-        # _local/dmedia exists, but is missing doc['peers']:
-        doc = {'_id': local_id}
-        db.save(doc)
-        self.assertEqual(ms.get_local_peers(), {})
-
-        # has doc['peers']:
+        # _local/dmedia exists, but is missing 'machine_id':
+        local = {'_id': local_id}
+        db.save(local)
+        self.assertEqual(ms.get_local_peers(), {})
+
+        # _local/dmedia has 'machine_id', but machine doc is missing:
+        local['machine_id'] = machine_id
+        db.save(local)
+        self.assertEqual(ms.get_local_peers(), {})
+
+        # machine exists, but is missing 'peers':
+        machine = {'_id': machine_id}
+        db.save(machine)
+        self.assertEqual(ms.get_local_peers(), {})
+
+        # machine has 'peers':
         peers = {
             random_id(30): {'url': random_id()},
             random_id(30): {'url': random_id()},
         }
-        doc['peers'] = peers
-        db.save(doc)
+        machine['peers'] = peers
+        db.save(machine)
         self.assertEqual(ms.get_local_peers(), peers)
 
     def test_schema_check(self):