Merge lp:~jderose/filestore/v1-on into lp:filestore

[Jason Gerard DeRose]

For details see this bug:

  https://bugs.launchpad.net/filestore/+bug/1174022

Changes:

  * Makes V1 the active protocol and updates all the related unit tests

  * Adds misc.write_files() helper that writes the test-vector files out into a directory, handy for unit testing things like the Hasher class against a specific protocol version

  * So it can support V0 and V1, Hasher.__init__() now takes optional "protocol" and "enc" kwargs (which default to `VERSION1` and `db32enc` respectively)

  * When FileStore.__init__() sees that ".dmedia/files/" contains a V0, Base32 layout, this directory is moved to ".dmedia/files0/" and the correct V1, Dbase32 layout is created in its place

  * And when above happens, if ".dmedia/store.json" exists it will be moved to ".dmedia/store0.json" and a new store.json file will written with the doc['_id'] re-encoded to Dbase32

  * Adds doodle `Migration` class that verifies files in "files0" according to their V0 ID, while re-hashing with V1 to calculate their new V1 ID

[lp:~jderose/filestore/v1-on]

327. By Jason Gerard DeRose on 2013-04-28

Small tweak to Migration

[xzcvczx (xzcvczx)]

review: Approve

=== modified file 'filestore/__init__.py'
--- filestore/__init__.py	2013-04-22 06:23:41 +0000
+++ filestore/__init__.py	2013-04-28 20:23:24 +0000
@@ -35,7 +35,7 @@
 22
 >>> ch = fs.hash_and_move(tmp_fp)
 >>> ch.id
-'MV2DIDJV66B7LCAXIAZRPSMN7I3LZJC6ANTODLJGZOZ3ZGTA'
+'Y685HWMJEE5J39SLBEP4Y3WY7D8Y9JCIQYAFUFQ39MMV84EW'
 >>> ch.file_size
 22
 
@@ -77,16 +77,17 @@
 import io
 import stat
 from base64 import b64encode
+import json
 from threading import Thread
 from queue import Queue
 from collections import namedtuple
 import logging
 
-from dbase32 import random_id, DB32ALPHABET
-from dbase32.rfc3548 import b32enc, isb32
+from dbase32 import db32enc, isdb32, random_id, DB32ALPHABET
+from dbase32.rfc3548 import isb32, b32enc
 
-from .protocols import TYPE_ERROR
-from .protocols import VERSION0 as PROTOCOL
+from .protocols import TYPE_ERROR, VERSION0
+from .protocols import VERSION1 as PROTOCOL
 
 try:
     from _filestore import fallocate, fastread
@@ -157,8 +158,8 @@
 
     For example:
 
-    >>> b32enc(hash_leaf(2, b'XYZ'))
-    'D7GIW5I5NB6SLJC5ALAX4WU7S7CNYUB3ULMECPY67FFQG4F7'
+    >>> db32enc(hash_leaf(2, b'XYZ'))
+    'YLP5A3SDEEP69SSTFRK7J98AUBS7SW4E75RQN9D4HKRCUXWS'
 
     :param leaf_index: an ``int`` >= 0
     :param leaf_data: optional ``bytes`` instance with contents of this leaf
@@ -173,13 +174,13 @@
     For example:
 
     >>> hash_root(31415, b'NNNNNNNNNNNNNNNNNNNNNNNNNNNNNN')
-    'YFZNR7K6DENL77BAMXIYXJZ4VDRPQEYGKGIZT74M4UPLBBAO'
+    'YFXKI7P3BFRMDBBLL4UDVHK3QTHGSXKU4VQ3SPJNKF96TBPV'
 
     :param file_size: an ``int`` >= 1
     :param leaf_hashes: a ``bytes`` instance that is the concatenated leaf
         hashes produced by `hash_leaf()`
     """
-    return b32enc(PROTOCOL.hash_root(file_size, leaf_hashes))
+    return db32enc(PROTOCOL.hash_root(file_size, leaf_hashes))
 
 
 class Hasher:
@@ -187,9 +188,11 @@
     A helper to keep track of state as you hash leaf after leaf.
     """
 
-    __slots__ = ('file_size', 'leaf_index', 'array', 'closed')
+    __slots__ = ('protocol', 'enc', 'file_size', 'leaf_index', 'array', 'closed')
 
-    def __init__(self):
+    def __init__(self, protocol=PROTOCOL, enc=db32enc):
+        self.protocol = protocol
+        self.enc = enc
         self.file_size = 0
         self.leaf_index = 0
         self.array = bytearray()
@@ -206,7 +209,7 @@
             )
         if len(leaf.data) < LEAF_SIZE:
             self.closed = True
-        leaf_hash = PROTOCOL.hash_leaf(leaf.index, leaf.data)
+        leaf_hash = self.protocol.hash_leaf(leaf.index, leaf.data)
         self.array.extend(leaf_hash)
         self.file_size += len(leaf.data)
         self.leaf_index += 1
@@ -216,7 +219,7 @@
         self.closed = True
         leaf_hashes = bytes(self.array)
         return ContentHash(
-            b32enc(PROTOCOL.hash_root(self.file_size, leaf_hashes)),
+            self.enc(self.protocol.hash_root(self.file_size, leaf_hashes)),
             self.file_size,
             leaf_hashes
         )
@@ -319,26 +322,33 @@
 
 def check_id(_id):
     """
-    Verify that *_id* is a valid base32-encoded ID of the correct length.
+    Verify that *_id* is a valid Dbase32 encoded ID of the correct length.
 
     A malicious *_id* could cause path traversal or other security gotchas,
     thus this sanity check.  When *_id* is valid, it is returned unchanged:
 
-    >>> check_id('OMPLTTYVTIJINDZWIS2PBZ4THWA6CTGCGT27RFIDKV7FSTCA')
-    'OMPLTTYVTIJINDZWIS2PBZ4THWA6CTGCGT27RFIDKV7FSTCA'
-
-    However, when *_id* does not conform, an `IDError` is raised.
-    raised:
-
-    >>> check_id('NWBNVXVK5DQGIOW7MYR4K3KA')
-    Traceback (most recent call last):
-      ...
-    filestore.IDError: invalid file ID: 'NWBNVXVK5DQGIOW7MYR4K3KA'
+    >>> check_id('39AY39AY39AY39AY39AY39AY39AY39AY39AY39AY39AY39AY')
+    '39AY39AY39AY39AY39AY39AY39AY39AY39AY39AY39AY39AY'
+
+    However, if *_id* is the incorrect length, an `IDError` is raised:
+
+    >>> check_id('39AY39AY39AY39AY39AY39AY')
+    Traceback (most recent call last):
+      ...
+    filestore.IDError: invalid file ID: '39AY39AY39AY39AY39AY39AY'
+
+    Likewise when *_id* is the correct length but contains symbols not included
+    in the Dbase32 alphabet:
+
+    >>> check_id('29AZ29AZ29AZ29AZ29AZ29AZ29AZ29AZ29AZ29AZ29AZ29AZ')
+    Traceback (most recent call last):
+      ...
+    filestore.IDError: invalid file ID: '29AZ29AZ29AZ29AZ29AZ29AZ29AZ29AZ29AZ29AZ29AZ29AZ'
 
     """
     if not isinstance(_id, str):
         raise TypeError(TYPE_ERROR.format('_id', str, type(_id), _id))
-    if not (len(_id) == DIGEST_B32LEN and isb32(_id)):
+    if not (len(_id) == DIGEST_B32LEN and isdb32(_id)):
         raise IDError(_id)
     return _id
 
@@ -381,14 +391,14 @@
     >>> check_root_hash(_id, 21, leaf_hash)  # 21 instead of 22 bytes
     Traceback (most recent call last):
       ...
-    filestore.RootHashError: 'MV2DIDJV66B7LCAXIAZRPSMN7I3LZJC6ANTODLJGZOZ3ZGTA'
+    filestore.RootHashError: 'Y685HWMJEE5J39SLBEP4Y3WY7D8Y9JCIQYAFUFQ39MMV84EW'
 
     If the claimed *file_size* and *leaf_hashes* are correct, the 3 values are
     returned in a `ContentHash` named tuple:
 
     >>> ch = check_root_hash(_id, 22, leaf_hash)
     >>> ch.id
-    'MV2DIDJV66B7LCAXIAZRPSMN7I3LZJC6ANTODLJGZOZ3ZGTA'
+    'Y685HWMJEE5J39SLBEP4Y3WY7D8Y9JCIQYAFUFQ39MMV84EW'
     >>> ch.file_size
     22
 
@@ -820,11 +830,103 @@
     return StatVFS(size, used, avail, readonly, st.f_frsize)
 
 
+def is_v0_files(files):
+    for name in NAMES_DIFF:
+        if path.isdir(path.join(files, name)):
+            return True
+    return False
+
+
+def dumps(obj):
+    return json.dumps(obj,
+        ensure_ascii=False,
+        sort_keys=True,
+        separators=(',',': '),
+        indent=4,
+    )
+
+
+def migrate_store_doc(basedir):
+    store = path.join(basedir, 'store.json')
+    store0 = path.join(basedir, 'store0.json')
+    try:
+        doc = json.load(open(store, 'r'))
+    except FileNotFoundError:
+        log.error("'store.json' does not exist in %r", basedir)
+        return False
+
+    if path.exists(store0):
+        raise Exception("'store0.json' already exists in {!r}".format(basedir))
+
+    log.warning("Moving V0 'store.json' to 'store0.json' in %r", basedir)
+    os.rename(store, store0)
+
+    from dbase32.rfc3548 import b32dec
+    assert doc.get('migrated') is None
+    old_id = doc['_id']
+    new_id = db32enc(b32dec(old_id))
+    log.warning('Migrating FileStore ID from %r to %r in %r',
+            old_id, new_id, basedir)
+    doc['_id'] = new_id
+    doc['migrated'] = True
+    text = dumps(doc)
+    tmp = path.join(basedir, 'store.json.' + random_id())
+    fp = open(tmp, 'x')
+    fp.write(text)
+    fp.flush()
+    os.fsync(fp.fileno())
+    os.chmod(fp.fileno(), 0o444)
+    fp.close()
+    os.rename(tmp, store)
+    return True
+
+
+###################################################
+# The `Migration` class use for V0 => V1 migration:
+
+class Migration:
+    def __init__(self, fs):
+        assert isinstance(fs, FileStore)
+        self.fs = fs
+        self.files0 = fs.join('files0')
+        assert path.isdir(self.files0)
+
+    def __iter__(self):
+        for prefix in B32NAMES:
+            subdir = path.join(self.files0, prefix)
+            for name in sorted(os.listdir(subdir)):
+                src = path.join(subdir, name)     
+                v0_id = prefix + name       
+                assert path.isfile(src) or path.islink(src)
+                assert isb32(v0_id) and len(v0_id) == 48
+
+                if path.islink(src):
+                    log.info('Reading symlink %r', src)
+                    yield (v0_id, os.readlink(src), None)
+                else:
+                    src_fp = open(src, 'rb')
+                    h0 = Hasher(protocol=VERSION0, enc=b32enc)
+                    h1 = Hasher()
+                    for leaf in reader_iter(src_fp):
+                        h0.hash_leaf(leaf)
+                        h1.hash_leaf(leaf)
+                    ch0 = h0.content_hash()
+                    ch1 = h1.content_hash()
+                    assert isdb32(ch1.id)
+                    if ch0.id != v0_id:
+                        yield (v0_id, None, None)
+                    else:
+                        dst = self.fs.path(ch1.id)
+                        log.info('Moving %r to %r', src, dst)
+                        os.rename(src, dst)
+                        os.symlink(ch1.id, src)
+                        yield (v0_id, ch1.id, ch1)
+
 
 ########################
 # The `FileStore` class:
 
-class FileStore(object):
+class FileStore:
     """
     Arranges files in a special layout according to their content-hash.
 
@@ -868,10 +970,24 @@
                 self.__class__.__name__, self.parentdir)
             )
 
+        files = self.join('files')
+        files0 = self.join('files0')
+
         # If basedir doesn't exist, create it and initialize all dirs in layout:
         if ensuredir(self.basedir):
             log.info('Initalizing FileStore in %r', self.basedir)
             self.init_dirs()
+        elif is_v0_files(files):
+            if path.exists(files0):
+                raise Exception(
+                    "'files' is V0 layout but 'files0' exists in {!r}".format(self.basedir)
+                )
+            log.warning("Moving V0 'files' to 'files0' in %r", self.basedir)
+            os.rename(files, files0)
+            self.init_dirs()
+            migrate_store_doc(self.basedir)
+
+        self.needs_migration = path.isdir(files0)
 
     def __repr__(self):
         return '{}({!r})'.format(self.__class__.__name__, self.parentdir)
@@ -894,11 +1010,11 @@
             * the entry is a symlink (even if to a valid file)
             * the file is zero bytes in size
         """
-        for prefix in B32NAMES:
+        for prefix in DB32NAMES:
             subdir = path.join(self.basedir, 'files', prefix)
             for name in sorted(os.listdir(subdir)):
                 _id = prefix + name
-                if len(_id) != DIGEST_B32LEN or not isb32(_id):
+                if len(_id) != DIGEST_B32LEN or not isdb32(_id):
                     continue
                 fullname = path.join(subdir, name)            
                 st = os.lstat(fullname)
@@ -917,7 +1033,7 @@
             d = path.join(self.basedir, name)
             ensuredir(d)
             os.chmod(d, 0o777)
-        for name in B32NAMES:
+        for name in DB32NAMES:
             d = path.join(self.basedir, 'files', name)
             ensuredir(d)
             os.chmod(d, 0o777)

=== modified file 'filestore/misc.py'
--- filestore/misc.py	2013-02-28 23:50:17 +0000
+++ filestore/misc.py	2013-04-28 20:23:24 +0000
@@ -65,6 +65,16 @@
     }
 
 
+def write_files(tmpdir, protocol=VERSION1):
+    leaves = build_leaves(protocol.leaf_size)
+    for (key, data) in leaves.items():
+        open(path.join(tmpdir, key), 'xb').write(data)
+        fp = open(path.join(tmpdir, 'C' + key), 'xb')
+        fp.write(leaves['C'])
+        fp.write(data)
+        fp.close()
+
+
 def build_vectors(protocol, encoder=db32enc):
     leaves = build_leaves(protocol.leaf_size)
 

=== modified file 'filestore/tests/__init__.py'
--- filestore/tests/__init__.py	2013-02-21 15:43:24 +0000
+++ filestore/tests/__init__.py	2013-04-28 20:23:24 +0000
@@ -28,15 +28,15 @@
 from os import path
 import io
 import stat
-from base64 import b32encode, b32decode
 from subprocess import check_call
 import tempfile
 import shutil
+import json
 from random import SystemRandom
 
-from skein import skein512
-from dbase32.rfc3548 import b32enc, b32dec
-from dbase32 import isdb32
+from _skein import skein512
+from dbase32 import isdb32, db32enc, random_id
+from dbase32.rfc3548 import b32enc
 
 from filestore import protocols, misc
 import filestore
@@ -82,8 +82,8 @@
     return str(n).encode('utf-8')
 
 
-def random_id(id_bytes=filestore.DIGEST_BYTES):
-    return b32encode(os.urandom(id_bytes)).decode('ascii')
+def random_file_id(numbytes=filestore.DIGEST_BYTES):
+    return random_id(numbytes)
 
 
 class TempDir(object):
@@ -99,6 +99,11 @@
             shutil.rmtree(self.dir)
             self.dir = None
 
+    def mkdir(self, *parts):
+        d = self.join(*parts)
+        os.mkdir(d)
+        return d
+
     def makedirs(self, *parts):
         d = self.join(*parts)
         if not path.exists(d):
@@ -410,23 +415,15 @@
 
         # Test with good values:
         content = b'N'
-        digest = f(2, content)
         self.assertEqual(
-            digest,
-            skein512(protocols.VERSION0._hash_leaf_index(b'2') + content,
-                digest_bits=240,
-                pers=protocols.PERS_LEAF,
-            ).digest()
+            filestore.hash_leaf(2, content),
+            protocols.VERSION1.hash_leaf(2, content)
         )
 
         content = b'N' * filestore.LEAF_SIZE
-        digest = f(2, content)
         self.assertEqual(
-            digest,
-            skein512(protocols.VERSION0._hash_leaf_index(b'2') + content,
-                digest_bits=240,
-                pers=protocols.PERS_LEAF,
-            ).digest()
+            filestore.hash_leaf(2, content),
+            protocols.VERSION1.hash_leaf(2, content)
         )
 
         # A 25k value sanity check on our crytographic claim that the
@@ -524,11 +521,11 @@
         leaf_hashes = b'D' * filestore.DIGEST_BYTES
         self.assertEqual(
             filestore.hash_root(1, leaf_hashes),
-            '4AZOU4R7M6JKJJRQMVX42YB7ULRUCS6FZGJNZCDVOATXYPML'
+            '39QLJTDIFYBSMR8A9IHAIGWMDCOX3TLWVKIAY9KSHGDGHCEL'
         )
         self.assertEqual(
             filestore.hash_root(filestore.LEAF_SIZE, leaf_hashes),
-            'RXBEFCKXWKYNPXZOR234QHYI475L2AF7C4AOQUG7EG7UJBDJ'
+            'JL57GWEV6OC4DGGE5UV4YRJ3J3ARVU8GDPO6TNEJAK9ULNJW'
         )
 
         # A 25k value sanity check on our crytographic claim that the
@@ -608,7 +605,7 @@
         self.assertEqual(cm.exception.file_size, FILE_SIZE + 1)
         self.assertEqual(cm.exception.leaf_hashes, LEAF_HASHES)
         self.assertEqual(cm.exception.bad_id,
-            'U5FK5XRT33ZJTCYIO3WJB7YTTGESXCBKZEW35J7FIQI7UN7S'
+            'K7BCQ9MWWKIEPQ9FNSWUATRW8UC845H7HDESWTRIK8NPMIRT'
         )
 
         with self.assertRaises(filestore.RootHashError) as cm:
@@ -617,7 +614,7 @@
         self.assertEqual(cm.exception.file_size, FILE_SIZE - 1)
         self.assertEqual(cm.exception.leaf_hashes, LEAF_HASHES)
         self.assertEqual(cm.exception.bad_id,
-            'OYESBWEZ4Y2AGSLMNZB4ZF75A2VG7NXVB4R25SSMRGXLN4CR'
+            'EYYA6GQD64CR94H3OQEVX88S7R83CWBY8ECM4KRNCN4G7DC4'
         )
 
     def test_enumerate_leaf_hashes(self):
@@ -738,7 +735,7 @@
 
     def test_check_id(self):
         # Test with wrong type
-        bad = random_id(30).encode('utf-8')
+        bad = random_file_id(30).encode('utf-8')
         with self.assertRaises(TypeError) as cm:
             filestore.check_id(bad)
         self.assertEqual(
@@ -757,7 +754,7 @@
             self.assertIs(cm.exception.id, value)
 
         # Test with 48 and 56 character:
-        id48 = random_id(30)
+        id48 = random_file_id(30)
         self.assertIs(filestore.check_id(id48), id48)
 
         # Test case sensitivity:
@@ -1200,7 +1197,7 @@
         ch = filestore.hash_fp(src_fp)
         self.assertIsInstance(ch, filestore.ContentHash)
         self.assertEqual(ch.id,
-            'DMJGE4OTWZVKSX426GHE46GZCGICMQOVWNGRVB7E665Y2RAM'
+            'CLSFQE444IGFX49Y9M6R9PLTF97HG7OQXAIEYYP54HNBN559'
         )
         self.assertEqual(ch.file_size, 20971520)
         self.assertFalse(path.exists(dst))
@@ -1210,7 +1207,7 @@
         ch = filestore.hash_fp(src_fp, dst_fp)
         self.assertIsInstance(ch, filestore.ContentHash)
         self.assertEqual(ch.id,
-            'DMJGE4OTWZVKSX426GHE46GZCGICMQOVWNGRVB7E665Y2RAM'
+            'CLSFQE444IGFX49Y9M6R9PLTF97HG7OQXAIEYYP54HNBN559'
         )
         self.assertEqual(ch.file_size, 20971520)
         self.assertTrue(path.isfile(dst))
@@ -1224,7 +1221,7 @@
             tmp.write(data, L)
             tmp.write(C + data, 'C' + L)
 
-        vectors = misc.load_data('V0')
+        vectors = misc.load_data('V1')
         for name in ['A', 'B', 'C', 'CA', 'CB', 'CC']:
             src_fp = open(tmp.join(name), 'rb')
             ch = filestore.hash_fp(src_fp)
@@ -1290,10 +1287,105 @@
             filestore.StatVFS(size, size - free, avail, False, s1.f_frsize)
         )
 
+    def test_is_v0_files(self):
+        v0 = TempDir()
+        for name in filestore.B32NAMES:
+            v0.mkdir(name)
+        self.assertIs(filestore.is_v0_files(v0.dir), True)
+        v1 = TempDir()
+        for name in filestore.DB32NAMES:
+            v1.mkdir(name)
+        self.assertIs(filestore.is_v0_files(v1.dir), False)
+        tmp = TempDir()
+        fs = filestore.FileStore(tmp.dir)
+        files = tmp.join('.dmedia', 'files')
+        self.assertTrue(path.isdir(files))
+        self.assertIs(filestore.is_v0_files(files), False)
+
+    def test_migrate_store_doc(self):
+        tmp = TempDir()
+        store = tmp.join('store.json')
+        store0 = tmp.join('store0.json')
+        self.assertIs(filestore.migrate_store_doc(tmp.dir), False)
+
+        # A V0 doc:
+        doc = {
+            '_id': 'DLA4NDZRW2LXEPF3RV7YHMON',
+            'copies': 1,
+            'plugin': 'filestore',
+            'time': 1320063400.353743,
+            'type': 'dmedia/store',
+        }
+        json.dump(doc, open(store, 'x'))
+        self.assertIs(filestore.migrate_store_doc(tmp.dir), True)
+        self.assertEqual(json.load(open(store, 'r')),
+            {
+                '_id': '6E3VG6SKPTEQ7I8UKOYRAFHG',
+                'copies': 1,
+                'plugin': 'filestore',
+                'time': 1320063400.353743,
+                'type': 'dmedia/store',
+                'migrated': True,
+            }
+        )
+        self.assertEqual(stat.S_IMODE(os.stat(store).st_mode), 0o444)
+        self.assertEqual(json.load(open(store0, 'r')), doc)
+        self.assertEqual(
+            set(os.listdir(tmp.dir)),
+            {'store.json', 'store0.json'}
+        )
+        with self.assertRaises(Exception) as cm:
+            filestore.migrate_store_doc(tmp.dir)
+        self.assertEqual(
+            str(cm.exception),
+            "'store0.json' already exists in {!r}".format(tmp.dir)
+        )
+
+        # Try with some random ID values:
+        for i in range(25):
+            tmp = TempDir()
+            store = tmp.join('store.json')
+            store0 = tmp.join('store0.json')
+            data = os.urandom(15)
+            b32_id = b32enc(data)
+            db32_id = db32enc(data)
+            self.assertNotEqual(b32_id, db32_id)
+            json.dump({'_id': b32_id}, open(store, 'x'))
+            self.assertIs(filestore.migrate_store_doc(tmp.dir), True)
+            self.assertEqual(json.load(open(store, 'r')),
+                {
+                    '_id': db32_id,
+                    'migrated': True,
+                }
+            )
+            self.assertEqual(stat.S_IMODE(os.stat(store).st_mode), 0o444)
+            self.assertEqual(json.load(open(store0, 'r')), {'_id': b32_id})
+            self.assertEqual(
+                set(os.listdir(tmp.dir)),
+                {'store.json', 'store0.json'}
+            )
+            with self.assertRaises(Exception) as cm:
+                filestore.migrate_store_doc(tmp.dir)
+            self.assertEqual(
+                str(cm.exception),
+                "'store0.json' already exists in {!r}".format(tmp.dir)
+            )
+
 
 class TestHasher(TestCase):
     def test_init(self):
         h = filestore.Hasher()
+        self.assertIs(h.protocol, protocols.VERSION1)
+        self.assertIs(h.enc, db32enc)
+        self.assertEqual(h.file_size, 0)
+        self.assertEqual(h.leaf_index, 0)
+        self.assertIsInstance(h.array, bytearray)
+        self.assertEqual(h.array, b'')
+        self.assertFalse(h.closed)
+
+        h = filestore.Hasher(protocols.VERSION0, b32enc)
+        self.assertIs(h.protocol, protocols.VERSION0)
+        self.assertIs(h.enc, b32enc)
         self.assertEqual(h.file_size, 0)
         self.assertEqual(h.leaf_index, 0)
         self.assertIsInstance(h.array, bytearray)
@@ -1334,7 +1426,7 @@
             str(cm.exception),
             'Expected leaf.index 1, got 0'
         )
- 
+
         # Test when it's all good
         h = filestore.Hasher()
 
@@ -1370,6 +1462,28 @@
         h.content_hash()
         self.assertTrue(h.closed)
 
+    def test_V1(self):
+        tmp = TempDir()
+        root_hashes = misc.load_data('V1')['root_hashes']
+        misc.write_files(tmp.dir, protocols.VERSION1)
+        for (name, _id) in root_hashes.items():
+            fp = open(tmp.join(name), 'rb')
+            h = filestore.Hasher()
+            for leaf in filestore.reader_iter(fp):
+                h.hash_leaf(leaf)
+            self.assertEqual(h.content_hash().id, _id)
+
+    def test_V0(self):
+        tmp = TempDir()
+        root_hashes = misc.load_data('V0')['root_hashes']
+        misc.write_files(tmp.dir, protocols.VERSION0)
+        for (name, _id) in root_hashes.items():
+            fp = open(tmp.join(name), 'rb')
+            h = filestore.Hasher(protocols.VERSION0, b32enc)
+            for leaf in filestore.reader_iter(fp):
+                h.hash_leaf(leaf)
+            self.assertEqual(h.content_hash().id, _id)
+
 
 class TestFileStore(TestCase):
     def test_init(self):
@@ -1438,12 +1552,14 @@
         self.assertTrue(path.isdir(fs.tmp))
         self.assertIsNone(fs.id)
         self.assertEqual(fs.copies, 0)
+        self.assertFalse(fs.needs_migration)
 
         # Test when _id and copies are supplied
         tmp = TempDir()
         fs = filestore.FileStore(tmp.dir, 'foo', 1)
         self.assertEqual(fs.id, 'foo')
         self.assertEqual(fs.copies, 1)
+        self.assertFalse(fs.needs_migration)
 
         # Test when basedir exists and is a directory
         tmp = TempDir()
@@ -1454,6 +1570,7 @@
         self.assertTrue(path.isdir(basedir))
         self.assertEqual(fs.tmp, path.join(basedir, 'tmp'))
         self.assertFalse(path.isdir(fs.tmp))
+        self.assertFalse(fs.needs_migration)
 
         # Test when basedir exists and is a file
         tmp = TempDir()
@@ -1477,6 +1594,89 @@
             '{!r} is symlink to {!r}'.format(basedir, d)
         )
 
+        # Test when .dmedia/files/ contains a V0, Base32 layout:
+        tmp = TempDir()
+        files = tmp.join('.dmedia', 'files')
+        files0 = tmp.join('.dmedia', 'files0')
+        fs = filestore.FileStore(tmp.dir)
+        shutil.rmtree(files)
+        os.mkdir(files)
+        for name in filestore.B32NAMES:
+            os.mkdir(path.join(files, name))
+        fs = filestore.FileStore(tmp.dir)
+        self.assertTrue(fs.needs_migration)
+        self.assertTrue(path.isdir(files))
+        for name in filestore.DB32NAMES:
+            self.assertTrue(path.isdir(path.join(files, name)))
+        self.assertEqual(sorted(os.listdir(files)), list(filestore.DB32NAMES))
+        self.assertTrue(path.isdir(files0))
+        for name in filestore.B32NAMES:
+            self.assertTrue(path.isdir(path.join(files0, name)))
+        self.assertEqual(sorted(os.listdir(files0)), list(filestore.B32NAMES))
+
+        # Test that no futher change is done:
+        fs = filestore.FileStore(tmp.dir)
+        self.assertTrue(fs.needs_migration)
+        self.assertTrue(path.isdir(files))
+        for name in filestore.DB32NAMES:
+            self.assertTrue(path.isdir(path.join(files, name)))
+        self.assertEqual(sorted(os.listdir(files)), list(filestore.DB32NAMES))
+        self.assertTrue(path.isdir(files0))
+        for name in filestore.B32NAMES:
+            self.assertTrue(path.isdir(path.join(files0, name)))
+        self.assertEqual(sorted(os.listdir(files0)), list(filestore.B32NAMES))
+
+        # Test when files contains V0/Base32 layout but files0 already exists:
+        shutil.rmtree(files)
+        os.mkdir(files)
+        for name in filestore.B32NAMES:
+            os.mkdir(path.join(files, name))
+        with self.assertRaises(Exception) as cm:
+            fs = filestore.FileStore(tmp.dir)
+        self.assertEqual(
+            str(cm.exception),
+            "'files' is V0 layout but 'files0' exists in {!r}".format(tmp.join('.dmedia'))   
+        )
+
+        # Test that store.json gets properly migrated:
+        tmp = TempDir()
+        files = tmp.join('.dmedia', 'files')
+        files0 = tmp.join('.dmedia', 'files0')
+        store = tmp.join('.dmedia', 'store.json')
+        store0 = tmp.join('.dmedia', 'store0.json')
+
+        # Setup:
+        fs = filestore.FileStore(tmp.dir)
+        shutil.rmtree(files)
+        os.mkdir(files)
+        for name in filestore.B32NAMES:
+            os.mkdir(path.join(files, name))
+        data = os.urandom(15)
+        b32_id = b32enc(data)
+        db32_id = db32enc(data)
+        self.assertNotEqual(b32_id, db32_id)
+        json.dump({'_id': b32_id}, open(store, 'x'))
+
+        # And test:
+        fs = filestore.FileStore(tmp.dir)
+        self.assertTrue(fs.needs_migration)
+        self.assertTrue(path.isdir(files))
+        for name in filestore.DB32NAMES:
+            self.assertTrue(path.isdir(path.join(files, name)))
+        self.assertEqual(sorted(os.listdir(files)), list(filestore.DB32NAMES))
+        self.assertTrue(path.isdir(files0))
+        for name in filestore.B32NAMES:
+            self.assertTrue(path.isdir(path.join(files0, name)))
+        self.assertEqual(sorted(os.listdir(files0)), list(filestore.B32NAMES))
+        self.assertEqual(json.load(open(store, 'r')),
+            {
+                '_id': db32_id,
+                'migrated': True,
+            }
+        )
+        self.assertEqual(stat.S_IMODE(os.stat(store).st_mode), 0o444)
+        self.assertEqual(json.load(open(store0, 'r')), {'_id': b32_id})
+
     def test_repr(self):
         tmp = TempDir()
         fs = filestore.FileStore(tmp.dir)
@@ -1487,7 +1687,7 @@
         fs = filestore.FileStore(tmp.dir)
 
         # Should ignore files with wrong ID length:
-        short = tuple(random_id(25) for i in range(50))
+        short = tuple(random_file_id(25) for i in range(50))
         for _id in short:
             f = fs.join('files', _id[:2], _id[2:])
             assert not path.exists(f)
@@ -1495,7 +1695,7 @@
             os.chmod(f, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
             self.assertEqual(path.getsize(f), 7)
         self.assertEqual(list(fs), [])
-        long = tuple(random_id(40) for i in range(50))
+        long = tuple(random_file_id(40) for i in range(50))
         for _id in long:
             f = fs.join('files', _id[:2], _id[2:])
             assert not path.exists(f)
@@ -1506,7 +1706,7 @@
 
         # Should ignore files invalid b32 letters:
         for i in range(50):
-            _id = random_id(25) + '1ABCDEFG'  # 1 is not in B32ALPHABET
+            _id = random_file_id(25) + '1ABCDEFG'  # 1 is not in B32ALPHABET
             assert len(_id) == filestore.DIGEST_B32LEN
             f = fs.join('files', _id[:2], _id[2:])
             assert not path.exists(f)
@@ -1517,7 +1717,7 @@
 
         # Should ignore empty files:
         for i in range(50):
-            _id = random_id()
+            _id = random_file_id()
             f = fs.join('files', _id[:2], _id[2:])
             assert not path.exists(f)
             open(f, 'wb').close()
@@ -1528,7 +1728,7 @@
 
         # Should ignore directories
         for i in range(50):
-            _id = random_id()
+            _id = random_file_id()
             d = fs.join('files', _id[:2], _id[2:])
             assert not path.exists(d)
             os.mkdir(d)
@@ -1539,7 +1739,7 @@
         # Now add valid files in (48 character IDs)
         stats = []
         for i in range(2000):
-            _id = random_id(30)
+            _id = random_file_id(30)
             size = i + 1
             f = fs.path(_id)
             assert not path.exists(f)
@@ -1556,7 +1756,7 @@
         # Should ignore symlinks, even if to valid files
         # This makes sure os.lstat() is being used rather than os.stat()  
         for i in range(50):
-            _id = random_id()
+            _id = random_file_id()
             link = fs.path(_id)
             assert not path.exists(link)
             file = fs.path(stats[i].id)
@@ -1578,7 +1778,7 @@
 
         self.assertEqual(
             sorted(os.listdir(path.join(basedir, 'files'))),
-            list(B32NAMES)
+            list(filestore.DB32NAMES)
         )
         for name in ['corrupt', 'partial', 'tmp']:
             d = path.join(basedir, name)
@@ -1588,7 +1788,7 @@
             d = path.join(basedir, name)
             self.assertTrue(path.isdir(d))
             self.assertFalse(path.islink(d))
-        for name in B32NAMES:
+        for name in filestore.DB32NAMES:
             d = path.join(basedir, 'files', name)
             self.assertTrue(path.isdir(d))
             self.assertFalse(path.islink(d))
@@ -1612,7 +1812,7 @@
 
         # Test when some subdirs exist:
         os.rmdir(path.join(basedir, 'tmp'))
-        for (i, name) in enumerate(B32NAMES):
+        for (i, name) in enumerate(filestore.DB32NAMES):
             if i % 3 == 0:
                 d = path.join(basedir, 'files', name)
         self.assertIsNone(fs.init_dirs())
@@ -1702,7 +1902,7 @@
         parentdir = tmp.makedirs('foo')
         fs = filestore.FileStore(parentdir)
 
-        _id = random_id()
+        _id = random_file_id()
         self.assertEqual(
             fs.path(_id),
             tmp.join('foo', '.dmedia', 'files', _id[:2], _id[2:])
@@ -1719,7 +1919,7 @@
         parentdir = tmp.makedirs('foo')
         fs = filestore.FileStore(parentdir)
 
-        _id = random_id()
+        _id = random_file_id()
         self.assertEqual(
             fs.partial_path(_id),
             tmp.join('foo', '.dmedia', 'partial', _id)
@@ -1736,7 +1936,7 @@
         parentdir = tmp.makedirs('foo')
         fs = filestore.FileStore(parentdir)
 
-        _id = random_id()
+        _id = random_file_id()
         self.assertEqual(
             fs.corrupt_path(_id),
             tmp.join('foo', '.dmedia', 'corrupt', _id)
@@ -1762,7 +1962,7 @@
         tmp = TempDir()
         fs = filestore.FileStore(tmp.dir)
 
-        id1 = random_id()
+        id1 = random_file_id()
 
         # File doesn't exist
         self.assertFalse(fs.exists(id1))
@@ -1772,13 +1972,13 @@
         self.assertTrue(fs.exists(id1))
 
         # Not file:
-        id2 = random_id()
+        id2 = random_file_id()
         tmp.makedirs('.dmedia', 'files', id2[:2], id2[2:])
         self.assertTrue(path.isdir(fs.path(id2)))
         self.assertFalse(fs.exists(id2))
 
         # Empty file
-        id3 = random_id()
+        id3 = random_file_id()
         f = fs.path(id3)
         assert not path.exists(f)
         open(f, 'wb').close()
@@ -1789,7 +1989,7 @@
         # File doesn't exist
         tmp = TempDir()
         fs = filestore.FileStore(tmp.dir)
-        _id = random_id()
+        _id = random_file_id()
         with self.assertRaises(filestore.FileNotFound) as cm:
             st = fs.stat(_id)
         self.assertEqual(cm.exception.id, _id)
@@ -1802,9 +2002,9 @@
         # File is a symlink:
         tmp = TempDir()
         fs = filestore.FileStore(tmp.dir)
-        file = random_id()
-        link = random_id()
-        open(fs.path(file), 'wb').write(b'Novacut')
+        file = random_file_id()
+        link = random_file_id()
+        open(fs.path(file), 'xb').write(b'Novacut')
         os.symlink(fs.path(file), fs.path(link))
         assert path.isfile(fs.path(link))
         assert path.islink(fs.path(link))
@@ -1820,7 +2020,7 @@
         # File is a directory
         tmp = TempDir()
         fs = filestore.FileStore(tmp.dir)
-        _id = random_id()
+        _id = random_file_id()
         os.mkdir(fs.path(_id))
         assert path.isdir(fs.path(_id))
         with self.assertRaises(filestore.FileNotFound) as cm:
@@ -1835,7 +2035,7 @@
         # Empty file
         tmp = TempDir()
         fs = filestore.FileStore(tmp.dir)
-        _id = random_id()
+        _id = random_file_id()
         open(fs.path(_id), 'wb').close()
         assert path.isfile(fs.path(_id))
         assert not path.islink(fs.path(_id))
@@ -1852,7 +2052,7 @@
         # Valid file
         tmp = TempDir()
         fs = filestore.FileStore(tmp.dir)
-        _id = random_id()
+        _id = random_file_id()
         open(fs.path(_id), 'wb').write(b'Novacut')
         st = fs.stat(_id)
         self.assertIsInstance(st, filestore.Stat)
@@ -1865,7 +2065,7 @@
         tmp = TempDir()
         fs = filestore.FileStore(tmp.dir)
 
-        _id = random_id()
+        _id = random_file_id()
 
         # File doesn't exist
         with self.assertRaises(filestore.FileNotFound) as cm:
@@ -2123,7 +2323,7 @@
     def test_remove(self):
         tmp = TempDir()
         fs = filestore.FileStore(tmp.dir)
-        _id = random_id()
+        _id = random_file_id()
         canonical = fs.path(_id)
 
         # File doesn't exist
@@ -2259,7 +2459,7 @@
     def test_allocate_partial(self):
         tmp = TempDir()
         fs = filestore.FileStore(tmp.dir)
-        _id = random_id()
+        _id = random_file_id()
         filename = tmp.join('.dmedia', 'partial', _id)
 
         # Test when file dosen't yet exist
@@ -2294,7 +2494,7 @@
         self.assertEqual(os.fstat(fp.fileno()).st_size, 2311)
         self.assertEqual(stat.S_IMODE(os.fstat(fp.fileno()).st_mode), 0o666)
 
-        _id = random_id()  # We'll use a new ID for below
+        _id = random_file_id()  # We'll use a new ID for below
         filename = tmp.join('.dmedia', 'partial', _id)
 
         # Test with bad size type:
@@ -2331,7 +2531,7 @@
     def test_move_to_canonical(self):
         tmp = TempDir()
         fs = filestore.FileStore(tmp.dir)
-        _id = random_id()
+        _id = random_file_id()
         dst = fs.path(_id)
 
         # Test with wrong src_fp type
@@ -2402,7 +2602,7 @@
     def test_move_to_corrupt(self):
         tmp = TempDir()
         fs = filestore.FileStore(tmp.dir)
-        _id = random_id()
+        _id = random_file_id()
         corrupt = fs.corrupt_path(_id)
         canonical = fs.path(_id)
 

=== modified file 'filestore/tests/test_misc.py'
--- filestore/tests/test_misc.py	2013-03-01 00:02:29 +0000
+++ filestore/tests/test_misc.py	2013-04-28 20:23:24 +0000
@@ -33,6 +33,8 @@
 from dbase32 import db32enc, db32dec
 from dbase32.rfc3548 import b32enc, b32dec
 
+from . import TempDir
+
 import filestore
 from filestore.protocols import MIN_LEAF_SIZE
 from filestore import misc, protocols
@@ -79,6 +81,24 @@
         self.assertEqual(len(obj['B']), 2 * MIN_LEAF_SIZE - 1)
         self.assertEqual(len(obj['C']), 2 * MIN_LEAF_SIZE)
 
+    def test_build_leaves(self):
+        tmp = TempDir()
+        leaves = misc.build_leaves(protocols.VERSION1.leaf_size)
+        self.assertIsNone(misc.write_files(tmp.dir))
+        self.assertEqual(
+            sorted(os.listdir(tmp.dir)),
+            ['A', 'B', 'C', 'CA', 'CB', 'CC']
+        )
+        for (key, data) in leaves.items():
+            self.assertEqual(
+                open(tmp.join(key), 'rb').read(),
+                data
+            )
+            self.assertEqual(
+                open(tmp.join('C' + key), 'rb').read(),
+                leaves['C'] + data
+            )
+
     def test_tohex(self):
         for i in range(10):
             h = md5(os.urandom(16))