Merge lp:~jderose/dmedia/simple-default into lp:dmedia

[Jason Gerard DeRose]

For background, see this bug:
https://bugs.launchpad.net/dmedia/+bug/1082864

Changes include:

 * Removed debian/dmedia.postinst script so /home/.dmedia FileStore is no longer created at package install

 * Reworked Core so it always has a default file store, doesn't use the shared store in /home/.dmedia anymore

 * Moved the private store from ~/.dmedia to ~/.local/share/dmedia/.dmedia

 * Removed SetDefaultStore() DBus method

 * Added Core.purge_store() implementation and test

 * Added PurgeStore() DBus method

 * Added quick and dirty migration functionality for the default store changes, which:

1) If ~/.dmedia exists, it is move to ~/.local/share/dmedia/.dmedia

2) If /home/.dmedia exists, all files there are moved (or copied if needed) into ~/.local/share/dmedia/.dmedia

3) Finally, calls Core.purge_store() to remove any references to copies in /home/.dmedia

[James Raymond (jamesmr)]

review: Approve

=== removed file 'debian/dmedia.postinst'
--- debian/dmedia.postinst	2012-07-26 21:05:25 +0000
+++ debian/dmedia.postinst	1970-01-01 00:00:00 +0000
@@ -1,10 +0,0 @@
-#!/bin/sh -e
-
-case $1 in
-    configure)
-        # Initalize shared file-store in /home
-        /usr/lib/dmedia/init-filestore /home
-esac
-
-
-#DEBHELPER#

=== modified file 'dmedia-cli'
--- dmedia-cli	2012-11-01 11:06:53 +0000
+++ dmedia-cli	2012-11-26 00:24:26 +0000
@@ -134,10 +134,10 @@
         return [path.abspath(directory)]
 
 
-class SetDefaultStore(_Method):
-    "Set default store to 'private', 'shared', or 'none'"
+class PurgeStore(_Method):
+    'Purge references to a store'
 
-    args = ['value']
+    args = ['store_id']
 
 
 class Resolve(_Method):

=== modified file 'dmedia-service'
--- dmedia-service	2012-11-25 07:25:50 +0000
+++ dmedia-service	2012-11-26 00:24:26 +0000
@@ -181,11 +181,6 @@
         self.snapshot_queue_in.put(dbname)
         return True
 
-    @dbus.service.method(IFACE, in_signature='as', out_signature='')
-    def SnapshotMany(self, names):
-        for dbname in names:
-            self.Snapshot(dbname)
-
     @dbus.service.method(IFACE, in_signature='', out_signature='')
     def SnapshotAll(self):
         log.info('Dmedia.SnapshotAll()')
@@ -209,9 +204,7 @@
         log.info('Starting CouchDB took %.3f', time.time() - start)
         self.core = Core(env)
         self.core.load_identity(self.couch.machine, self.couch.user)
-        self.core.init_default_store()
-        if self.core.local.get('default_store') is None:
-            self.core.set_default_store('shared')
+        self.core.load_default_filestore(self.couch.basedir)
         self.env_s = dumps(self.core.env, pretty=True)
         log.info('Finished core startup in %.3f', time.time() - start)
         GObject.timeout_add(250, self.on_idle1)
@@ -468,12 +461,11 @@
         self.core.create_filestore(parentdir)
         return self.LocalDmedia()
 
-    @dbus.service.method(IFACE, in_signature='s', out_signature='s')
-    def SetDefaultStore(self, value):
-        value = str(value)
-        log.info('Dmedia.SetDefaultStore(%r)', value)
-        self.core.set_default_store(value)
-        return self.LocalDmedia()
+    @dbus.service.method(IFACE, in_signature='s', out_signature='')
+    def PurgeStore(self, store_id):
+        store_id = str(store_id)
+        log.info('Dmedia.PurgeStore(%r)', store_id)
+        start_thread(self.core.purge_store, store_id)
 
     @dbus.service.method(IFACE, in_signature='s', out_signature='s')
     def Resolve(self, _id):

=== modified file 'dmedia/core.py'
--- dmedia/core.py	2012-11-18 11:57:51 +0000
+++ dmedia/core.py	2012-11-26 00:24:26 +0000
@@ -42,7 +42,7 @@
 from base64 import b64encode
 
 from microfiber import Server, Database, NotFound, Conflict, BulkConflict
-from filestore import FileStore, check_root_hash, check_id
+from filestore import FileStore, check_root_hash, check_id, DOTNAME
 
 import dmedia
 from dmedia.parallel import start_thread, start_process
@@ -54,8 +54,10 @@
 
 log = logging.getLogger()
 LOCAL_ID = '_local/dmedia'
+HOME = path.abspath(os.environ['HOME'])
+if not path.isdir(HOME):
+    raise Exception('$HOME is not a directory: {!r}'.format(HOME))
 SHARED = '/home'
-PRIVATE = path.abspath(os.environ['HOME'])
 
 
 def start_httpd(couch_env, ssl_config):
@@ -185,18 +187,37 @@
         log.exception('Error updating project stats for %r', project_id)
 
 
+def migrate_shared(srcdir, dstdir):
+    try:
+        count = 0
+        src = FileStore(srcdir)
+        dst = FileStore(dstdir)
+        log.info('Migrating files from %r to %r', src, dst)
+        for st in src:
+            if dst.exists(st.id):
+                continue
+            log.info('Migrating %s %s', st.id, st.size)
+            try:
+                os.rename(st.name, dst.path(st.id))
+            except OSError:
+                src.copy(st.id, dst)
+                src.remove(st.id)
+            count += 1
+        log.info('Migrating %d files from %r to %r', count, src, dst)
+    except Exception:
+        log.exception('Error migrating files from shared FileStore')
+    return count
+
+
 class Core:
-    def __init__(self, env, private=None, shared=None):
+    def __init__(self, env):
         self.env = env
-        self._private = (PRIVATE if private is None else private)
-        self._shared = (SHARED if shared is None else shared)
         self.db = util.get_db(env, init=True)
         self.server = self.db.server()
         self.ms = MetaStore(self.db)
         self.stores = LocalStores()
         self.queue = Queue()
         self.thread = None
-        self.default = None
         try:
             self.local = self.db.get(LOCAL_ID)
         except NotFound:
@@ -230,23 +251,18 @@
         self.local['user_id'] = user['_id']
         self.save_local()
 
-    def init_default_store(self):
-        value = self.local.get('default_store')
-        if value not in ('private', 'shared'):
-            log.info('no default FileStore')
-            self.default = None
-            self._sync_stores()
-            return
-        if value == 'shared' and not util.isfilestore(self._shared):
-            log.warning('Switching to private, no shared FileStore at %r', self._shared)
-            value = 'private'
-            self.local['default_store'] = value
-            self.db.save(self.local)
-        parentdir = (self._private if value == 'private' else self._shared)
+    def load_default_filestore(self, parentdir):
+        if util.isfilestore(HOME) and not util.isfilestore(parentdir):
+            src = FileStore(HOME)
+            src.init_dirs()
+            dstdir = path.join(parentdir, DOTNAME)
+            log.info('Moving %r to %r', src.basedir, dstdir)
+            os.rename(src.basedir, dstdir)
+        self.parentdir = parentdir
         (fs, doc) = util.init_filestore(parentdir)
-        self.default = fs
-        log.info('Connecting default FileStore %r at %r', fs.id, fs.parentdir)
+        log.info('Default FileStore %s at %r', doc['_id'], parentdir)
         self._add_filestore(fs, doc)
+        return fs
 
     def _sync_stores(self):
         self.local['stores'] = self.stores.local_stores()
@@ -270,6 +286,13 @@
         self._sync_stores()
 
     def _background_worker(self):
+        if util.isfilestore(SHARED):
+            log.info('Running migration...')
+            process = start_process(migrate_shared, SHARED, self.parentdir)
+            process.join()
+            store_id = util.get_filestore_id(SHARED)
+            if store_id is not None:
+                self.purge_store(store_id)
         log.info('Background worker listing to queue...')
         while True:
             try:
@@ -327,19 +350,6 @@
     def update_project(self, project_id):
         update_project(self.db, project_id)     
 
-    def set_default_store(self, value):
-        if value not in ('private', 'shared', 'none'):
-            raise ValueError(
-                "need 'private', 'shared', or 'none'; got {!r}".format(value)
-            )
-        if self.local.get('default_store') != value:
-            self.local['default_store'] = value
-            self.db.save(self.local)
-        if self.default is not None:
-            self.disconnect_filestore(self.default.parentdir, self.default.id)
-            self.default = None
-        self.init_default_store()
-
     def create_filestore(self, parentdir):
         """
         Create a new file-store in *parentdir*.
@@ -412,16 +422,36 @@
             * ``doc['partial'][store_id]``
 
         Some scenarios in which you might want to do this:
-        
+
             1. The HDD was run over by a bus, the data is gone.  We need to
                embrace reality, the sooner the better.
 
             2. We're going to format or otherwise repurpose an HDD.  Ideally, we
                would have called `Core2.downgrade_store()` first.
- 
+
         Note that this method makes sense for remote cloud stores as well as for
         local file-stores
         """
+        log.info('Purging store %s', store_id)
+        ids = []
+        while True:
+            rows = self.db.view('file', 'stored',
+                key=store_id,
+                include_docs=True,
+                limit=25,
+            )['rows']
+            if not rows:
+                break
+            ids.extend(r['id'] for r in rows)
+            docs = [r['doc'] for r in rows]
+            for doc in docs:
+                del doc['stored'][store_id]
+            try:
+                self.db.save_many(docs)
+            except BulkConflict:
+                log.exception('Conflict purging %s', store_id)
+        log.info('Purged %d references to %s', len(ids), store_id)
+        return ids
 
     def stat(self, _id):
         doc = self.db.get(_id)

=== modified file 'dmedia/tests/base.py'
--- dmedia/tests/base.py	2012-05-06 22:10:00 +0000
+++ dmedia/tests/base.py	2012-11-26 00:24:26 +0000
@@ -33,6 +33,7 @@
 from random import SystemRandom
 from zipfile import ZipFile
 
+import filestore
 from filestore import File, Leaf, ContentHash, Batch, Hasher, LEAF_SIZE
 from filestore import scandir
 from microfiber import random_id
@@ -42,6 +43,10 @@
 random = SystemRandom()
 
 
+def random_file_id():
+    return random_id(filestore.DIGEST_BYTES)
+
+
 class DummyQueue(object):
     def __init__(self):
         self.items = []

=== modified file 'dmedia/tests/test_core.py'
--- dmedia/tests/test_core.py	2012-10-31 20:42:27 +0000
+++ dmedia/tests/test_core.py	2012-11-26 00:24:26 +0000
@@ -41,7 +41,7 @@
 from dmedia import util, core
 
 from .couch import CouchCase
-from .base import TempDir
+from .base import TempDir, random_file_id
 
 
 class TestFunctions(TestCase):
@@ -64,6 +64,32 @@
             }
         )
 
+    def test_migrate_shared(self):
+        tmp1 = TempDir()
+        tmp2 = TempDir()
+        src = filestore.FileStore(tmp1.dir)
+        dst = filestore.FileStore(tmp2.dir)
+        st_list = []
+        for i in range(10):
+            (file, ch) = tmp1.random_file()
+            os.rename(file.name, src.path(ch.id))
+            st = src.stat(ch.id)
+            assert st.size == ch.file_size
+            st_list.append(st)
+        st_list.sort(key=lambda st: st.id)
+        self.assertEqual(list(src), st_list)
+        self.assertEqual(list(dst), [])
+        self.assertEqual(core.migrate_shared(tmp1.dir, tmp2.dir), 10)
+        self.assertEqual(list(src), [])
+        self.assertEqual(
+            [st.id for st in dst],
+            [st.id for st in st_list]
+        )
+        for st in st_list:
+            ch = dst.verify(st.id)
+            self.assertEqual(st.size, ch.file_size)
+            self.assertEqual(dst.stat(st.id).mtime, st.mtime)
+
 
 class TestCouchFunctions(CouchCase):
     def test_projects_iter(self):
@@ -135,191 +161,6 @@
             }
         )
 
-    def test_init_default_store(self):
-        private = TempDir()
-        shared = TempDir()
-        machine_id = random_id()
-
-        # Test when default_store is missing
-        inst = core.Core(self.env, private.dir, shared.dir)
-        self.assertEqual(inst._private, private.dir)
-        self.assertEqual(inst._shared, shared.dir)
-        inst.init_default_store()
-        self.assertIsNone(inst.default)
-        self.assertEqual(inst.local,
-            {
-                '_id': '_local/dmedia',
-                'stores': {},
-            }
-        )
-
-        # Test when default_store is 'private'
-        inst = core.Core(self.env, private.dir, shared.dir)
-        self.assertEqual(inst._private, private.dir)
-        self.assertEqual(inst._shared, shared.dir)
-        inst.local['default_store'] = 'private'
-        self.assertFalse(util.isfilestore(private.dir))
-        inst.init_default_store()
-        self.assertEqual(
-            set(inst.local['stores']),
-            set([private.dir])
-        )
-        store_id = inst.local['stores'][private.dir]['id']
-        (fs1, doc) = util.get_filestore(private.dir, store_id)
-        self.assertEqual(inst.local,
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-1',
-                'default_store': 'private',
-                'stores': {
-                    fs1.parentdir: {
-                        'id': fs1.id,
-                        'copies': fs1.copies,
-                    }
-                }
-            }
-        )
-
-        # Again test when default_store is 'private' to make sure local isn't
-        # updated needlessly
-        inst = core.Core(self.env, private.dir, shared.dir)
-        self.assertEqual(inst._private, private.dir)
-        self.assertEqual(inst._shared, shared.dir)
-        inst.init_default_store()
-        self.assertEqual(inst.local,
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-1',
-                'default_store': 'private',
-                'stores': {
-                    fs1.parentdir: {
-                        'id': fs1.id,
-                        'copies': fs1.copies,
-                    }
-                }
-            }
-        )
-
-        # Test when default_store is 'shared' (which we're assuming exists)
-        self.assertFalse(util.isfilestore(shared.dir))
-        (fs2, doc) = util.init_filestore(shared.dir)
-        inst = core.Core(self.env, private.dir, shared.dir)
-        self.assertEqual(inst._private, private.dir)
-        self.assertEqual(inst._shared, shared.dir)
-        inst.local['default_store'] = 'shared'
-        inst.init_default_store()
-        self.assertEqual(inst.local,
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-2',
-                'default_store': 'shared',
-                'stores': {
-                    fs2.parentdir: {
-                        'id': fs2.id,
-                        'copies': fs2.copies,
-                    }
-                }
-            }
-        )
-
-        # Test when default_store is 'shared' and it doesn't exist, in which
-        # case it should automatically switch to 'private' instead
-        nope = TempDir()
-        inst = core.Core(self.env, private.dir, nope.dir)
-        self.assertEqual(inst._private, private.dir)
-        self.assertEqual(inst._shared, nope.dir)
-        inst.local['default_store'] = 'shared'
-        inst.init_default_store()
-        self.assertEqual(inst.local,
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-4',
-                'default_store': 'private',
-                'stores': {
-                    fs1.parentdir: {
-                        'id': fs1.id,
-                        'copies': fs1.copies,
-                    }
-                }
-            }
-        )
-
-    def test_set_default_store(self):
-        private = TempDir()
-        shared = TempDir()
-        (fs1, doc1) = util.init_filestore(private.dir)
-        (fs2, doc2) = util.init_filestore(shared.dir)
-        inst = core.Core(self.env, private.dir, shared.dir)
-        self.assertEqual(inst._private, private.dir)
-        self.assertEqual(inst._shared, shared.dir)
-
-        with self.assertRaises(ValueError) as cm:
-            inst.set_default_store('foobar')
-        self.assertEqual(
-            str(cm.exception),
-            "need 'private', 'shared', or 'none'; got 'foobar'"
-        )
-        self.assertEqual(inst.local,
-            {
-                '_id': '_local/dmedia',
-                'stores': {},
-            }
-        )
-
-        # Test with 'private'
-        inst.set_default_store('private')
-        self.assertEqual(inst.local,
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-2',
-                'default_store': 'private',
-                'stores': {
-                    fs1.parentdir: {'id': fs1.id, 'copies': 1},
-                },
-            }
-        )
-
-        # Test with 'shared'
-        inst.set_default_store('shared')
-        self.assertEqual(inst.local,
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-5',
-                'default_store': 'shared',
-                'stores': {
-                    fs2.parentdir: {'id': fs2.id, 'copies': 1},
-                },
-            }
-        )
-
-        # Test with 'none'
-        inst.set_default_store('none')
-        self.assertEqual(inst.local,
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-7',
-                'default_store': 'none',
-                'stores': {},
-            }
-        )
-
-        # Test with 'shared' when it doesn't exist
-        nope = TempDir()
-        inst = core.Core(self.env, private.dir, nope.dir)
-        self.assertEqual(inst._private, private.dir)
-        self.assertEqual(inst._shared, nope.dir)
-        inst.set_default_store('shared')
-        self.assertEqual(inst.local,
-            {
-                '_id': '_local/dmedia',
-                '_rev': '0-10',
-                'default_store': 'private',
-                'stores': {
-                    fs1.parentdir: {'id': fs1.id, 'copies': 1},
-                },
-            }
-        )
-
     def test_create_filestore(self):
         inst = core.Core(self.env)
 
@@ -502,6 +343,97 @@
             inst.disconnect_filestore(fs1.parentdir, fs1.id)
         self.assertEqual(str(cm.exception), repr(fs1.parentdir))
 
+    def test_purge_store(self):
+        store_id1 = random_id()
+        store_id2 = random_id()
+        store_id3 = random_id()
+        inst = core.Core(self.env)
+        db = inst.db
+
+        # Test when empty
+        self.assertEqual(inst.purge_store(store_id1), [])
+
+        docs = [
+            {
+                '_id': random_file_id(),
+                'type': 'dmedia/file',
+                'bytes': 1776,
+                'stored': {
+                    store_id1: {
+                        'copies': 1,
+                        'mtime': 1234567890,
+                    },
+                    store_id2: {
+                        'copies': 2,
+                        'mtime': 1234567891,
+                    },
+                },
+            }
+            for i in range(533)
+        ]
+        ids = [doc['_id'] for doc in docs]
+        ids.sort()
+        db.save_many(docs)
+
+        # Test when store isn't present
+        self.assertEqual(inst.purge_store(store_id3), [])
+        for doc in docs:
+            self.assertEqual(db.get(doc['_id']), doc)
+
+        # Purge one of the stores, make sure the other remains
+        self.assertEqual(inst.purge_store(store_id1), ids)
+        for doc in db.get_many(ids):
+            _id = doc['_id']
+            rev = doc.pop('_rev')
+            self.assertTrue(rev.startswith('2-'))
+            self.assertEqual(
+                doc,
+                {
+                    '_id': _id,
+                    'type': 'dmedia/file',
+                    'bytes': 1776,
+                    'stored': {
+                        store_id2: {
+                            'copies': 2,
+                            'mtime': 1234567891,
+                        },
+                    },
+                }
+            )
+
+        # Purge the other store
+        self.assertEqual(inst.purge_store(store_id2), ids)
+        for doc in db.get_many(ids):
+            _id = doc['_id']
+            rev = doc.pop('_rev')
+            self.assertTrue(rev.startswith('3-'))
+            self.assertEqual(
+                doc,
+                {
+                    '_id': _id,
+                    'type': 'dmedia/file',
+                    'bytes': 1776,
+                    'stored': {},
+                }
+            )
+
+        # Purge both again, make sure no doc changes result:
+        self.assertEqual(inst.purge_store(store_id1), [])
+        self.assertEqual(inst.purge_store(store_id2), [])
+        for doc in db.get_many(ids):
+            _id = doc['_id']
+            rev = doc.pop('_rev')
+            self.assertTrue(rev.startswith('3-'))
+            self.assertEqual(
+                doc,
+                {
+                    '_id': _id,
+                    'type': 'dmedia/file',
+                    'bytes': 1776,
+                    'stored': {},
+                }
+            )
+
     def test_update_atime(self):
         inst = core.Core(self.env)
         _id = random_id()

=== modified file 'dmedia/tests/test_util.py'
--- dmedia/tests/test_util.py	2012-07-09 14:45:38 +0000
+++ dmedia/tests/test_util.py	2012-11-26 00:24:26 +0000
@@ -51,7 +51,32 @@
         tmp.makedirs('.dmedia')
         self.assertTrue(util.isfilestore(tmp.dir))
 
-    def test_getfilestore(self):
+    def test_get_filestore_id(self):
+        tmp = TempDir()
+        doc = schema.create_filestore(1)
+        _id = doc['_id']
+
+        # Test when .dmedia/ doesn't exist
+        self.assertIsNone(util.get_filestore_id(tmp.dir))
+
+        # Test when .dmedia/ exists, but store.json doesn't:
+        tmp.makedirs('.dmedia')
+        self.assertIsNone(util.get_filestore_id(tmp.dir))
+
+        # Test when .dmedia/store.json exists
+        store = tmp.join('.dmedia', 'store.json')
+        json.dump(doc, open(store, 'w'))
+        self.assertEqual(util.get_filestore_id(tmp.dir), _id)
+
+        # Test with bad JSON
+        open(store, 'wb').write(b'bad JSON, no cookie for you')
+        self.assertIsNone(util.get_filestore_id(tmp.dir))
+
+        # Test with non-dict
+        json.dump('hello', open(store, 'w'))
+        self.assertIsNone(util.get_filestore_id(tmp.dir))
+
+    def test_get_filestore(self):
         tmp = TempDir()
         doc = schema.create_filestore(1)
 

=== modified file 'dmedia/util.py'
--- dmedia/util.py	2012-10-19 00:11:46 +0000
+++ dmedia/util.py	2012-11-26 00:24:26 +0000
@@ -43,6 +43,15 @@
     return path.isdir(path.join(parentdir, DOTNAME))
 
 
+def get_filestore_id(parentdir):
+    store = path.join(parentdir, DOTNAME, 'store.json')
+    try:
+        doc = json.load(open(store, 'r'))
+        return doc['_id']
+    except Exception:
+        pass
+
+
 def get_filestore(parentdir, store_id, copies=None):
     store = path.join(parentdir, DOTNAME, 'store.json')
     doc = json.load(open(store, 'r'))