Merge lp:~jderose/dmedia/schema-ver0 into lp:dmedia

[Jason Gerard DeRose]

There is still a bit of work to do finalizing how tags work, but the important part is already enforced in the schema: 'tags' is a dictionary. I'll open another bug for finishing the tags schema, but this is already a fairly large change, so I'm proposing this for merge before it gets out of hand.

Oh, and this includes an important unrelated bugfix: I goofed in the 0.5 transition from PyGTK to PyGI and am no longer calling GObject.threads_init() when ./dmedia-service runs... oops. This was causing a lot of weird problems with GUI hangs and the logging order (in conjunction with multiprocessing) getting wacky. Anyway, I sliped that little one-line fix in here too (in dmedia/gtkui/__init__.py).

[Jason Gerard DeRose (jderose)]

Okay, I'm gonna self-approve as I want to get the important PyGI fix into trunk. This month needs to be an especially high-velocity month for dmedia, so I might break the rules often. To make up for it, I will write an unusually high number of unit tests, even by my standards.

"""
Jason, looks great! Amazingly there isn't even a single typo, I'm sure!

--Jason
"""

review: Approve

=== modified file 'dmedia/extractor.py'
--- dmedia/extractor.py	2011-01-28 14:07:33 +0000
+++ dmedia/extractor.py	2011-04-07 03:13:28 +0000
@@ -223,13 +223,18 @@
 def merge_metadata(src, doc):
     ext = doc['ext']
     attachments = doc.get('_attachments', {})
+    meta = doc.get('meta', {})
     if ext in _extractors:
         callback = _extractors[ext]
         for (key, value) in callback(src, attachments):
-            if key not in doc or key == 'mtime':
+            if key == 'mtime':
                 doc[key] = value
+            elif key not in meta:
+                meta[key] = value
     if attachments and '_attachments' not in doc:
         doc['_attachments'] = attachments
+    if meta and 'meta' not in doc:
+        doc['meta'] = meta
 
 
 def merge_exif(src, attachments):

=== modified file 'dmedia/filestore.py'
--- dmedia/filestore.py	2011-02-27 03:47:55 +0000
+++ dmedia/filestore.py	2011-04-07 03:13:28 +0000
@@ -613,6 +613,9 @@
             raise ValueError('%s.base not a directory: %r' %
                 (self.__class__.__name__, self.base)
             )
+
+        # FIXME: This is too high-level for FileStore, should instead be deault
+        # with by the core API entry point as FileStore are first initialized
         self.record = path.join(self.base, 'store.json')
         try:
             fp = open(self.record, 'rb')

=== modified file 'dmedia/gtkui/__init__.py'
--- dmedia/gtkui/__init__.py	2011-03-27 08:01:30 +0000
+++ dmedia/gtkui/__init__.py	2011-04-07 03:13:28 +0000
@@ -26,4 +26,6 @@
 import gi
 gi.require_version('Gtk', '2.0')
 gi.require_version('WebKit', '1.0')
-from gi.repository import Gtk, WebKit
+from gi.repository import GObject, Gtk, WebKit
+
+GObject.threads_init()

=== modified file 'dmedia/importer.py'
--- dmedia/importer.py	2011-03-28 12:38:29 +0000
+++ dmedia/importer.py	2011-04-07 03:13:28 +0000
@@ -33,7 +33,7 @@
 
 import couchdb
 
-from .schema import random_id
+from .schema import random_id, create_file, create_batch, create_import
 from .errors import DuplicateFile
 from .workers import (
     CouchWorker, CouchManager, register, isregistered, exception_name
@@ -48,6 +48,24 @@
 log = logging.getLogger()
 
 
+# FIXME: This needs to be done with some real inspection of the file contents,
+# but this is just a stopgap for the sake of getting the schema stable:
+MEDIA_MAP = {
+    'ogv': 'video',
+    'mov': 'video',
+    'avi': 'video',
+
+    'oga': 'audio',
+    'flac': 'audio',
+    'wav': 'audio',
+    'mp3': 'audio',
+
+    'jpg': 'image',
+    'cr2': 'image',
+    'png': 'image',
+}
+
+
 def normalize_ext(name):
     """
     Return (root, ext) from *name* where extension is normalized to lower-case.
@@ -143,53 +161,6 @@
             yield tup
 
 
-def create_batch(machine_id=None):
-    """
-    Create initial 'dmedia/batch' accounting document.
-    """
-    return {
-        '_id': random_id(),
-        'type': 'dmedia/batch',
-        'time': time.time(),
-        'machine_id': machine_id,
-        'imports': [],
-        'errors': [],
-        'stats': {
-            'considered': {'count': 0, 'bytes': 0},
-            'imported': {'count': 0, 'bytes': 0},
-            'skipped': {'count': 0, 'bytes': 0},
-            'empty': {'count': 0, 'bytes': 0},
-            'error': {'count': 0, 'bytes': 0},
-        }
-    }
-
-
-def create_import(base, batch_id=None, machine_id=None):
-    """
-    Create initial 'dmedia/import' accounting document.
-    """
-    return {
-        '_id': random_id(),
-        'type': 'dmedia/import',
-        'time': time.time(),
-        'batch_id': batch_id,
-        'machine_id': machine_id,
-        'base': base,
-        'log': {
-            'imported': [],
-            'skipped': [],
-            'empty': [],
-            'error': [],
-        },
-        'stats': {
-            'imported': {'count': 0, 'bytes': 0},
-            'skipped': {'count': 0, 'bytes': 0},
-            'empty': {'count': 0, 'bytes': 0},
-            'error': {'count': 0, 'bytes': 0},
-        }
-    }
-
-
 class ImportWorker(CouchWorker):
     def __init__(self, env, q, key, args):
         super(ImportWorker, self).__init__(env, q, key, args)
@@ -308,34 +279,19 @@
         except couchdb.ResourceNotFound as e:
             pass
 
-        ts = time.time()
-        doc = {
-            '_id': chash,
-            '_attachments': {
-                'leaves': {
-                    'data': b64encode(pack_leaves(leaves)),
-                    'content_type': 'application/octet-stream',
-                }
-            },
-            'type': 'dmedia/file',
-            'time': ts,
-            'bytes': stat.st_size,
-            'ext': ext,
-            'origin': 'user',
-            'stored': {
-                self.filestore._id: {
-                    'copies': 1,
-                    'time': ts,
-                },
-            },
-
-            'import_id': self._id,
-            'mtime': stat.st_mtime,
-            'name': name,
-            'dir': path.relpath(path.dirname(src), self.base),
-        }
+        doc = create_file(stat.st_size, leaves, self.filestore._id,
+            copies=1, ext=ext
+        )
+        assert doc['_id'] == chash
+        doc.update(
+            import_id=self._id,
+            mtime=stat.st_mtime,
+            name=name,
+            dir=path.relpath(path.dirname(src), self.base),
+        )
         if ext:
-            doc['content_type'] = mimetypes.types_map.get('.' + ext)
+            doc['mime'] = mimetypes.types_map.get('.' + ext)
+            doc['media'] = MEDIA_MAP.get(ext)
         if self.extract:
             merge_metadata(src, doc)
         (_id, _rev) = self.db.save(doc)

=== modified file 'dmedia/metastore.py'
--- dmedia/metastore.py	2011-03-27 09:05:32 +0000
+++ dmedia/metastore.py	2011-04-07 03:13:28 +0000
@@ -62,6 +62,19 @@
 }
 """
 
+# views in the 'file' design only index docs for which doc.type == 'dmedia/file'
+file_stored = """
+// Get list of all files on a given store, total bytes on that store
+function(doc) {
+    if (doc.type == 'dmedia/file') {
+        var key;
+        for (key in doc.stored) {
+            emit(key, doc.bytes);
+        }
+    }
+}
+"""
+
 file_bytes = """
 function(doc) {
     if (doc.type == 'dmedia/file' && typeof(doc.bytes) == 'number') {
@@ -78,10 +91,10 @@
 }
 """
 
-file_content_type = """
+file_mime = """
 function(doc) {
     if (doc.type == 'dmedia/file') {
-        emit(doc.content_type, null);
+        emit(doc.mime, null);
     }
 }
 """
@@ -94,16 +107,6 @@
 }
 """
 
-file_tags = """
-function(doc) {
-    if (doc.type == 'dmedia/file' && doc.tags) {
-        doc.tags.forEach(function(tag) {
-            emit(tag, null);
-        });
-    }
-}
-"""
-
 file_import_id = """
 function(doc) {
     if (doc.type == 'dmedia/file' && doc.import_id) {
@@ -112,13 +115,87 @@
 }
 """
 
+# views in the 'user' design only index docs for which doc.type == 'dmedia/file'
+# and doc.origin == 'user'
+user_copies = """
+// Durability of user's personal files
+function(doc) {
+    if (doc.type == 'dmedia/file' && doc.origin == 'user') {
+        var copies = 0;
+        var key;
+        for (key in doc.stored) {
+            copies += doc.stored[key].copies;
+        }
+        emit(copies, null);
+    }
+}
+"""
+
+user_media = """
+function(doc) {
+    if (doc.type == 'dmedia/file' && doc.origin == 'user') {
+        emit(doc.media, null);
+    }
+}
+"""
+
+user_tags = """
+function(doc) {
+    if (doc.type == 'dmedia/file' && doc.origin == 'user' && doc.tags) {
+        var key;
+        for (key in doc.tags) {
+            emit(key, doc.tags[key]);
+        }
+    }
+}
+"""
+
+user_all = """
+function(doc) {
+    if (doc.type == 'dmedia/file' && doc.origin == 'user') {
+        emit(doc.mtime, null);
+    }
+}
+"""
+
+user_video = """
+function(doc) {
+    if (doc.type == 'dmedia/file' && doc.origin == 'user') {
+        if (doc.media == 'video') {
+            emit(doc.mtime, null);
+        }
+    }
+}
+"""
+
+user_image = """
+function(doc) {
+    if (doc.type == 'dmedia/file' && doc.origin == 'user') {
+        if (doc.media == 'image') {
+            emit(doc.mtime, null);
+        }
+    }
+}
+"""
+
+user_audio = """
+function(doc) {
+    if (doc.type == 'dmedia/file' && doc.origin == 'user') {
+        if (doc.media == 'audio') {
+            emit(doc.mtime, null);
+        }
+    }
+}
+"""
+
+
 def build_design_doc(design, views):
     _id = '_design/' + design
     d = {}
     for (view, map_, reduce_) in views:
-        d[view] = {'map': map_}
+        d[view] = {'map': map_.strip()}
         if reduce_ is not None:
-            d[view]['reduce'] = reduce_
+            d[view]['reduce'] = reduce_.strip()
     doc = {
         '_id': _id,
         'language': 'javascript',
@@ -168,12 +245,22 @@
         )),
 
         ('file', (
+            ('stored', file_stored, _sum),
             ('import_id', file_import_id, None),
             ('bytes', file_bytes, _sum),
             ('ext', file_ext, _count),
-            ('content_type', file_content_type, _count),
+            ('mime', file_mime, _count),
             ('mtime', file_mtime, None),
-            ('tags', file_tags, _count),
+        )),
+
+        ('user', (
+            ('copies', user_copies, None),
+            ('media', user_media, _count),
+            ('tags', user_tags, _count),
+            ('all', user_all, None),
+            ('video', user_video, None),
+            ('image', user_image, None),
+            ('audio', user_audio, None),
         )),
     )
 

=== modified file 'dmedia/schema.py'
--- dmedia/schema.py	2011-03-27 09:05:32 +0000
+++ dmedia/schema.py	2011-04-07 03:13:28 +0000
@@ -33,6 +33,7 @@
 
 >>> good = {
 ...     '_id': 'NZXXMYLDOV2F6ZTUO5PWM5DX',
+...     'ver': 0,
 ...     'type': 'dmedia/foo',
 ...     'time': 1234567890,
 ... }
@@ -40,6 +41,7 @@
 >>> check_dmedia(good)  # Returns None
 >>> bad = {
 ...     '_id': 'NZXXMYLDOV2F6ZTUO5PWM5DX',
+...     'ver': 0,
 ...     'kind': 'dmedia/foo',
 ...     'timestamp': 1234567890,
 ... }
@@ -222,6 +224,7 @@
 
 >>> doc = {
 ...     '_id': 'MZZG2ZDSOQVSW2TEMVZG643F',
+...     'ver': 0,
 ...     'type': 'dmedia/batch',
 ...     'time': 1234567890,
 ... }
@@ -308,9 +311,11 @@
 from __future__ import print_function
 
 import os
-from base64 import b32encode, b32decode
+from hashlib import sha1
+from base64 import b32encode, b32decode, b64encode
 import re
 import time
+
 from .constants import TYPE_ERROR, EXT_PAT
 
 # Some private helper functions that don't directly define any schema.
@@ -325,6 +330,201 @@
 #
 # That is all.
 
+
+# FIXME: These functions are a step toward making the checks more concise and
+# the error messages consistent and even more helpful.  However, these functions
+# aren't used much yet... but all the schema checks should be ported to these
+# functions eventually.
+def _label(path):
+    """
+    Create a helpful debugging label to indicate the attribute in question.
+
+    For example:
+
+    >>> _label([])
+    'doc'
+    >>> _label(['log'])
+    "doc['log']"
+    >>> _label(['log', 'considered', 2, 'src'])
+    "doc['log']['considered'][2]['src']"
+
+
+    See also `_value()`.
+    """
+    return 'doc' + ''.join('[{!r}]'.format(key) for key in path)
+
+
+def _value(doc, path):
+    """
+    Retrieve value from *doc* by traversing *path*.
+
+    For example:
+
+    >>> doc = {'log': {'considered': [None, None, {'src': 'hello'}, None]}}
+    >>> _value(doc, [])
+    {'log': {'considered': [None, None, {'src': 'hello'}, None]}}
+    >>> _value(doc, ['log'])
+    {'considered': [None, None, {'src': 'hello'}, None]}
+    >>> _value(doc, ['log', 'considered', 2, 'src'])
+    'hello'
+
+
+    Or if you try to retrieve something that doesn't exist:
+
+    >>> _value(doc, ['log', 'considered', 7])
+    Traceback (most recent call last):
+      ...
+    ValueError: doc['log']['considered'][7] does not exists
+
+
+    Or if a key/index is missing higher up in the path:
+
+    >>> _value(doc, ['dog', 'considered', 7])
+    Traceback (most recent call last):
+      ...
+    ValueError: doc['dog'] does not exists
+
+
+    See also `_label()`.
+    """
+    value = doc
+    p = []
+    for key in path:
+        p.append(key)
+        try:
+            value = value[key]
+        except (KeyError, IndexError):
+            raise ValueError(
+                '{} does not exists'.format(_label(p))
+            )
+    return value
+
+
+def _exists(doc, path):
+    """
+    Return ``True`` if the end of *path* exists.
+
+    For example:
+
+    >>> doc = {'foo': {'hello': 'world'}, 'bar': ['hello', 'naughty', 'nurse']}
+    >>> _exists(doc, ['foo', 'hello'])
+    True
+    >>> _exists(doc, ['foo', 'sup'])
+    False
+    >>> _exists(doc, ['bar', 2])
+    True
+    >>> _exists(doc, ['bar', 3])
+    False
+
+
+    Or if a key/index is missing higher up the path:
+
+    >>> _exists(doc, ['stuff', 'junk'])
+    Traceback (most recent call last):
+      ...
+    ValueError: doc['stuff'] does not exists
+
+
+    See also `_check_if_exists()`.
+    """
+    if len(path) == 0:
+        return True
+    base = _value(doc, path[:-1])
+    key = path[-1]
+    try:
+        value = base[key]
+        return True
+    except (KeyError, IndexError):
+        return False
+
+
+def _check(doc, path, *checks):
+    """
+    Run a series of *checks* on the value in *doc* addressed by *path*.
+
+    For example:
+
+    >>> doc = {'foo': [None, {'bar': 'aye'}, None]}
+    >>> _check(doc, ['foo', 1, 'bar'],
+    ...     _check_str,
+    ...     (_check_in, 'bee', 'sea'),
+    ... )
+    ...
+    Traceback (most recent call last):
+      ...
+    ValueError: doc['foo'][1]['bar'] value 'aye' not in ('bee', 'sea')
+
+
+    Or if a value is missing:
+
+    >>> _check(doc, ['foo', 3],
+    ...     _can_be_none,
+    ... )
+    ...
+    Traceback (most recent call last):
+      ...
+    ValueError: doc['foo'][3] does not exists
+
+
+    See also `_check_if_exists()`.
+    """
+    value = _value(doc, path)
+    label = _label(path)
+    for c in checks:
+        if isinstance(c, tuple):
+            (c, args) = (c[0], c[1:])
+        else:
+            args = tuple()
+        if c(value, label, *args) is True:
+            break
+
+
+def _check_if_exists(doc, path, *checks):
+    """
+    Run *checks* only if value at *path* exists.
+
+    For example:
+
+    >>> doc = {'name': 17}
+    >>> _check_if_exists(doc, ['dir'], _check_str)
+    >>> _check_if_exists(doc, ['name'], _check_str)
+    Traceback (most recent call last):
+      ...
+    TypeError: doc['name']: need a <type 'basestring'>; got a <type 'int'>: 17
+
+
+    See also `_check()` and `_exists()`.
+    """
+    if _exists(doc, path):
+        _check(doc, path, *checks)
+
+
+def _can_be_none(value, label):
+    """
+    Stop execution of check if *value* is ``None``.
+
+    `_check()` will abort upon a check function returning ``True``.
+
+    For example, here a ``TypeError`` is raised:
+
+    >>> doc = {'ext': None}
+    >>> _check(doc, ['ext'], _check_str)
+    Traceback (most recent call last):
+      ...
+    TypeError: doc['ext']: need a <type 'basestring'>; got a <type 'NoneType'>: None
+
+
+    But here it is not:
+
+    >>> _check(doc, ['ext'], _can_be_none, _check_str)
+
+    """
+    if value is None:
+        return True
+
+# /FIXME new helper functions
+
+
 def _check_dict(value, label):
     """
     Verify that *value* is a ``dict`` instance.
@@ -387,13 +587,13 @@
     if not isinstance(value, (int, float)):
         raise TypeError(TYPE_ERROR % (label, (int, float), type(value), value))
 
-def _check_at_least(value, minvalue, label):
+def _check_at_least(value, label, minvalue=0):
     """
     Verify that *value* is greater than or equal to *minvalue*.
 
     For example:
 
-    >>> _check_at_least(0, 1, 'bytes')
+    >>> _check_at_least(0, 'bytes', 1)
     Traceback (most recent call last):
       ...
     ValueError: bytes must be >= 1; got 0
@@ -477,6 +677,25 @@
         )
 
 
+def _check_in(value, label, *possible):
+    """
+    Check that *value* is one of *possible*.
+
+    For example:
+
+    >>> _check_in('foo', "doc['media']", 'video', 'audio', 'image')
+    Traceback (most recent call last):
+      ...
+    ValueError: doc['media'] value 'foo' not in ('video', 'audio', 'image')
+
+    """
+    if value not in possible:
+        raise ValueError(
+            '{} value {!r} not in {!r}'.format(label, value, possible)
+        )
+
+
+
 # The schema defining functions:
 
 def check_base32(value, label='_id'):
@@ -579,7 +798,7 @@
 
     """
     _check_int_float(value, label)
-    _check_at_least(value, 0, label)
+    _check_at_least(value, label, 0)
 
 
 def check_dmedia(doc):
@@ -591,14 +810,17 @@
 
         1. have '_id' that passes `check_base32()`
 
-        2. have 'type' that passes `check_type()`
-
-        3. have 'time' that passes `check_time()`
+        2. have a 'ver' equal to ``0``
+
+        3. have 'type' that passes `check_type()`
+
+        4. have 'time' that passes `check_time()`
 
     For example, a conforming value:
 
     >>> doc = {
     ...     '_id': 'NZXXMYLDOV2F6ZTUO5PWM5DX',
+    ...     'ver': 0,
     ...     'type': 'dmedia/file',
     ...     'time': 1234567890,
     ... }
@@ -610,6 +832,7 @@
 
     >>> doc = {
     ...     '_id': 'NZXXMYLDOV2F6ZTUO5PWM5DX',
+    ...     'ver': 0,
     ...     'kind': 'dmedia/file',
     ...     'timestamp': 1234567890,
     ... }
@@ -620,8 +843,13 @@
     ValueError: doc missing keys: ['time', 'type']
 
     """
-    _check_required(doc, ['_id', 'type', 'time'])
+    _check_required(doc, ['_id', 'ver', 'type', 'time'])
     check_base32(doc['_id'])
+    _check_int(doc['ver'], 'ver')
+    if doc['ver'] != 0:
+        raise ValueError(
+            "doc['ver'] must be 0; got {!r}".format(doc['ver'])
+        )
     check_type(doc['type'])
     check_time(doc['time'])
 
@@ -638,7 +866,7 @@
 
         3. have values that are themselves ``dict`` instances
 
-        4. values must have 'copies' that is an ``int`` >= 1
+        4. values must have 'copies' that is an ``int`` >= 0
 
         5. values must have 'time' that conforms with `check_time()`
 
@@ -686,7 +914,7 @@
         copies = value['copies']
         l3 = l2 + "['copies']"
         _check_int(copies, l3)
-        _check_at_least(copies, 1, l3)
+        _check_at_least(copies, l3, 0)
 
         # Check 'time':
         check_time(value['time'], l2 + "['time']")
@@ -780,7 +1008,6 @@
         raise ValueError('%s: %r not in %r' % (label, value, allowed))
 
 
-
 def check_dmedia_file(doc):
     """
     Verify that *doc* is a valid 'dmedia/file' record type.
@@ -803,6 +1030,7 @@
 
     >>> doc = {
     ...     '_id': 'ZR765XWSF6S7JQHLUI4GCG5BHGPE252O',
+    ...     'ver': 0,
     ...     'type': 'dmedia/file',
     ...     'time': 1234567890,
     ...     'bytes': 20202333,
@@ -823,6 +1051,7 @@
 
     >>> doc = {
     ...     '_id': 'ZR765XWSF6S7JQHLUI4GCG5BHGPE252O',
+    ...     'ver': 0,
     ...     'type': 'dmedia/file',
     ...     'time': 1234567890,
     ...     'bytes': 20202333,
@@ -854,7 +1083,7 @@
     # Check 'bytes':
     b = doc['bytes']
     _check_int(b, 'bytes')
-    _check_at_least(b, 1, 'bytes')
+    _check_at_least(b, 'bytes', 1)
 
     # Check 'ext':
     check_ext(doc['ext'])
@@ -865,6 +1094,63 @@
     # Check 'stored'
     check_stored(doc['stored'])
 
+    check_dmedia_file_optional(doc)
+
+
+def check_dmedia_file_optional(doc):
+    """
+    Check the optional attributes in a 'dmedia/file' document.
+    """
+    _check_dict(doc, 'doc')
+
+    # 'mime' like 'video/quicktime'
+    _check_if_exists(doc, ['mime'],
+        _can_be_none,
+        _check_str,
+    )
+
+    # 'media' like 'video'
+    _check_if_exists(doc, ['media'],
+        _can_be_none,
+        _check_str,
+        (_check_in, 'video', 'audio', 'image'),
+    )
+
+    # 'mtime' like 1234567890
+    _check_if_exists(doc, ['mtime'],
+        check_time
+    )
+
+    # 'atime' like 1234567890
+    _check_if_exists(doc, ['atime'],
+        check_time
+    )
+
+    # name like 'MVI_5899.MOV'
+    _check_if_exists(doc, ['name'],
+        _check_str,
+    )
+
+    # dir like 'DCIM/100EOS5D2'
+    _check_if_exists(doc, ['dir'],
+        _check_str,
+    )
+
+    # 'meta' like {'iso': 800}
+    _check_if_exists(doc, ['meta'],
+        _check_dict
+    )
+
+    # 'user' like {'title': 'cool sunset'}
+    _check_if_exists(doc, ['user'],
+        _check_dict
+    )
+
+    # 'tags' like {'burp': {'start': 6, 'end': 73}}
+    _check_if_exists(doc, ['tags'],
+        _check_dict
+    )
+
 
 def check_dmedia_store(doc):
     """
@@ -883,6 +1169,7 @@
 
     >>> doc = {
     ...     '_id': 'NZXXMYLDOV2F6ZTUO5PWM5DX',
+    ...     'ver': 0,
     ...     'type': 'dmedia/file',
     ...     'time': 1234567890,
     ...     'plugin': 'filestore',
@@ -896,6 +1183,7 @@
 
     >>> doc = {
     ...     '_id': 'NZXXMYLDOV2F6ZTUO5PWM5DX',
+    ...     'ver': 0,
     ...     'type': 'dmedia/file',
     ...     'time': 1234567890,
     ...     'dispatch': 'filestore',
@@ -925,7 +1213,7 @@
     key = 'copies'
     dc = doc[key]
     _check_int(dc, key)
-    _check_at_least(dc, 1, key)
+    _check_at_least(dc, key, 1)
 
 
 def random_id(random=None):
@@ -950,10 +1238,68 @@
     return b32encode(random)
 
 
-# This should probably be moved
+# FIXME: There is current a recursize import issue with filestore, but FileStore
+# shouldn't deal with the store.json file anyway, should not import
+# `schema.create_store()`
+def tophash_personalization(file_size):
+    return ' '.join(['dmedia/tophash', str(file_size)]).encode('utf-8')
+
+
+def tophash(file_size, leaves):
+    """
+    Initialize hash for a file that is *file_size* bytes.
+    """
+    h = sha1(tophash_personalization(file_size))
+    h.update(leaves)
+    return b32encode(h.digest())
+
+
+def create_file(file_size, leaves, store, copies=0, ext=None, origin='user'):
+    """
+    Create a minimal 'dmedia/file' document.
+
+    :param file_size: an ``int``, the file size in bytes, eg ``20202333``
+    :param leaves: a ``list`` containing the content hash of each leaf
+    :param store: the ID of the store where this file is initially stored, eg
+        ``'Y4J3WQCMKV5GHATOCZZBHF4Y'``
+    :param copies: an ``int`` to represent the durability of the file on this
+        store; default is ``0``
+    :param ext: the file extension, eg ``'mov'``; default is ``None``
+    :param origin: the file's origin (for durability/reclamation purposes);
+        default is ``'user'``
+    """
+    ts = time.time()
+    packed = b''.join(leaves)
+    return {
+        '_id': tophash(file_size, packed),
+        '_attachments': {
+            'leaves': {
+                'data': b64encode(packed),
+                'content_type': 'application/octet-stream',
+            }
+        },
+        'ver': 0,
+        'type': 'dmedia/file',
+        'time': ts,
+        'bytes': file_size,
+        'ext': ext,
+        'origin': origin,
+        'stored': {
+            store: {
+                'copies': copies,
+                'time': ts,
+            }
+        }
+    }
+
+
 def create_store(base, machine_id, copies=1):
+    """
+    Create a 'dmedia/store' document.
+    """
     return {
         '_id': random_id(),
+        'ver': 0,
         'type': 'dmedia/store',
         'time': time.time(),
         'plugin': 'filestore',
@@ -961,3 +1307,52 @@
         'path': base,
         'machine_id': machine_id,
     }
+
+
+def create_batch(machine_id=None):
+    """
+    Create initial 'dmedia/batch' accounting document.
+    """
+    return {
+        '_id': random_id(),
+        'ver': 0,
+        'type': 'dmedia/batch',
+        'time': time.time(),
+        'machine_id': machine_id,
+        'imports': [],
+        'errors': [],
+        'stats': {
+            'considered': {'count': 0, 'bytes': 0},
+            'imported': {'count': 0, 'bytes': 0},
+            'skipped': {'count': 0, 'bytes': 0},
+            'empty': {'count': 0, 'bytes': 0},
+            'error': {'count': 0, 'bytes': 0},
+        }
+    }
+
+
+def create_import(base, batch_id=None, machine_id=None):
+    """
+    Create initial 'dmedia/import' accounting document.
+    """
+    return {
+        '_id': random_id(),
+        'ver': 0,
+        'type': 'dmedia/import',
+        'time': time.time(),
+        'batch_id': batch_id,
+        'machine_id': machine_id,
+        'base': base,
+        'log': {
+            'imported': [],
+            'skipped': [],
+            'empty': [],
+            'error': [],
+        },
+        'stats': {
+            'imported': {'count': 0, 'bytes': 0},
+            'skipped': {'count': 0, 'bytes': 0},
+            'empty': {'count': 0, 'bytes': 0},
+            'error': {'count': 0, 'bytes': 0},
+        }
+    }

=== modified file 'dmedia/tests/helpers.py'
--- dmedia/tests/helpers.py	2011-02-22 14:07:47 +0000
+++ dmedia/tests/helpers.py	2011-04-07 03:13:28 +0000
@@ -34,6 +34,7 @@
 from . import sample_mov, sample_thm
 
 mov_hash = 'TGX33XXWU3EVHEEY5J7NBOJGKBFXLEBK'
+mov_size = 20202333
 mov_leaves = [
     b32decode('IXJTSUCYYFECGSG6JIB2R77CAJVJK4W3'),
     b32decode('MA3IAHUOKXR4TRG7CWAPOO7U4WCV5WJ4'),

=== modified file 'dmedia/tests/test_extractor.py'
--- dmedia/tests/test_extractor.py	2011-01-26 05:14:20 +0000
+++ dmedia/tests/test_extractor.py	2011-04-07 03:13:28 +0000
@@ -411,21 +411,23 @@
             doc,
             dict(
                 ext='mov',
-                width=1920,
-                height=1080,
-                duration=3,
-                codec_video='H.264 / AVC',
-                codec_audio='Raw 16-bit PCM audio',
-                sample_rate=48000,
-                fps=30,
-                channels='Stereo',
-                iso=100,
-                shutter=u'1/100',
-                aperture=11.0,
-                lens=u'Canon EF 70-200mm f/4L IS',
-                camera=u'Canon EOS 5D Mark II',
-                focal_length=u'138.0 mm',
                 mtime=1287520994 + 68 / 100.0,
+                meta=dict(
+                    width=1920,
+                    height=1080,
+                    duration=3,
+                    codec_video='H.264 / AVC',
+                    codec_audio='Raw 16-bit PCM audio',
+                    sample_rate=48000,
+                    fps=30,
+                    channels='Stereo',
+                    iso=100,
+                    shutter=u'1/100',
+                    aperture=11.0,
+                    lens=u'Canon EF 70-200mm f/4L IS',
+                    camera=u'Canon EOS 5D Mark II',
+                    focal_length=u'138.0 mm',
+                ),
             )
         )
 

=== modified file 'dmedia/tests/test_importer.py'
--- dmedia/tests/test_importer.py	2011-03-28 12:38:29 +0000
+++ dmedia/tests/test_importer.py	2011-04-07 03:13:28 +0000
@@ -132,103 +132,6 @@
         )
         os.chmod(subdir, 0o700)
 
-    def test_create_batch(self):
-        f = importer.create_batch
-        machine_id = random_id()
-        doc = f(machine_id)
-
-        self.assertEqual(schema.check_dmedia(doc), None)
-        self.assertTrue(isinstance(doc, dict))
-        self.assertEqual(
-            set(doc),
-            set([
-                '_id',
-                'type',
-                'time',
-                'imports',
-                'errors',
-                'machine_id',
-                'stats',
-            ])
-        )
-        _id = doc['_id']
-        self.assertEqual(b32encode(b32decode(_id)), _id)
-        self.assertEqual(len(_id), 24)
-        self.assertEqual(doc['type'], 'dmedia/batch')
-        self.assertTrue(isinstance(doc['time'], (int, float)))
-        self.assertTrue(doc['time'] <= time.time())
-        self.assertEqual(doc['imports'], [])
-        self.assertEqual(doc['errors'], [])
-        self.assertEqual(doc['machine_id'], machine_id)
-        self.assertEqual(
-            doc['stats'],
-            {
-                'considered': {'count': 0, 'bytes': 0},
-                'imported': {'count': 0, 'bytes': 0},
-                'skipped': {'count': 0, 'bytes': 0},
-                'empty': {'count': 0, 'bytes': 0},
-                'error': {'count': 0, 'bytes': 0},
-            }
-        )
-
-    def test_create_import(self):
-        f = importer.create_import
-
-        base = '/media/EOS_DIGITAL'
-        batch_id = random_id()
-        machine_id = random_id()
-
-        keys = set([
-            '_id',
-            'type',
-            'time',
-            'base',
-            'batch_id',
-            'machine_id',
-            'log',
-            'stats',
-        ])
-
-        doc = f(base, batch_id=batch_id, machine_id=machine_id)
-        self.assertEqual(schema.check_dmedia(doc), None)
-        self.assertTrue(isinstance(doc, dict))
-        self.assertEqual(set(doc), keys)
-
-        _id = doc['_id']
-        self.assertEqual(b32encode(b32decode(_id)), _id)
-        self.assertEqual(len(_id), 24)
-
-        self.assertEqual(doc['type'], 'dmedia/import')
-        self.assertTrue(isinstance(doc['time'], (int, float)))
-        self.assertTrue(doc['time'] <= time.time())
-        self.assertEqual(doc['base'], base)
-        self.assertEqual(doc['batch_id'], batch_id)
-        self.assertEqual(doc['machine_id'], machine_id)
-
-        doc = f(base)
-        self.assertEqual(schema.check_dmedia(doc), None)
-        self.assertEqual(set(doc), keys)
-        self.assertEqual(doc['batch_id'], None)
-        self.assertEqual(doc['machine_id'], None)
-        self.assertEqual(
-            doc['log'],
-            {
-                'imported': [],
-                'skipped': [],
-                'empty': [],
-                'error': [],
-            }
-        )
-        self.assertEqual(
-            doc['stats'],
-            {
-                'imported': {'count': 0, 'bytes': 0},
-                'skipped': {'count': 0, 'bytes': 0},
-                'empty': {'count': 0, 'bytes': 0},
-                'error': {'count': 0, 'bytes': 0},
-            }
-        )
-
     def test_to_dbus_stats(self):
         f = importer.to_dbus_stats
         stats = dict(
@@ -406,6 +309,7 @@
             set([
                 '_id',
                 '_rev',
+                'ver',
                 'type',
                 'time',
                 'base',
@@ -507,6 +411,7 @@
                 '_id',
                 '_rev',
                 '_attachments',
+                'ver',
                 'type',
                 'time',
                 'bytes',
@@ -518,7 +423,8 @@
                 'mtime',
                 'name',
                 'dir',
-                'content_type',
+                'mime',
+                'media',
             ])
         )
         self.assertEqual(schema.check_dmedia_file(doc), None)
@@ -534,7 +440,7 @@
         self.assertEqual(doc['mtime'], path.getmtime(src1))
         self.assertEqual(doc['name'], 'MVI_5751.MOV')
         self.assertEqual(doc['dir'], 'DCIM/100EOS5D2')
-        self.assertEqual(doc['content_type'], 'video/quicktime')
+        self.assertEqual(doc['mime'], 'video/quicktime')
 
         # Test with duplicate
         (action, doc) = inst._import_file(src2)
@@ -834,7 +740,9 @@
         self.assertEqual(
             set(batch),
             set([
-                '_id', '_rev',
+                '_id',
+                '_rev',
+                'ver',
                 'type',
                 'time',
                 'imports',

=== modified file 'dmedia/tests/test_schema.py'
--- dmedia/tests/test_schema.py	2011-03-27 09:05:32 +0000
+++ dmedia/tests/test_schema.py	2011-04-07 03:13:28 +0000
@@ -24,10 +24,10 @@
 """
 
 from unittest import TestCase
-from base64 import b32encode, b32decode
+from base64 import b32encode, b32decode, b64encode
 from copy import deepcopy
 import time
-from .helpers import raises, TempDir
+from .helpers import raises, TempDir, mov_hash, mov_leaves, mov_size
 from dmedia.constants import TYPE_ERROR
 from dmedia.schema import random_id
 from dmedia import schema
@@ -163,13 +163,24 @@
 
         good = {
             '_id': 'MZZG2ZDSOQVSW2TEMVZG643F',
+            'ver': 0,
             'type': 'dmedia/foo',
             'time': 1234567890,
             'foo': 'bar',
         }
         g = deepcopy(good)
         self.assertEqual(f(g), None)
-        for key in ['_id', 'type', 'time']:
+
+        # check with bad ver:
+        bad = deepcopy(good)
+        bad['ver'] = 0.0
+        e = raises(TypeError, f, bad)
+        self.assertEqual(str(e), TYPE_ERROR % ('ver', int, float, 0.0))
+        bad['ver'] = 1
+        e = raises(ValueError, f, bad)
+        self.assertEqual(str(e), "doc['ver'] must be 0; got 1")
+
+        for key in ['_id', 'ver', 'type', 'time']:
             bad = deepcopy(good)
             del bad[key]
             e = raises(ValueError, f, bad)
@@ -190,7 +201,7 @@
         e = raises(ValueError, f, bad)
         self.assertEqual(
             str(e),
-            'doc missing keys: %r' % ['_id', 'time', 'type']
+            'doc missing keys: %r' % ['_id', 'time', 'type', 'ver']
         )
 
     def test_check_stored(self):
@@ -275,11 +286,11 @@
             TYPE_ERROR % (label, int, float, 2.0)
         )
         bad = deepcopy(good)
-        bad['MZZG2ZDSOQVSW2TEMVZG643F']['copies'] = 0
+        bad['MZZG2ZDSOQVSW2TEMVZG643F']['copies'] = -2
         e = raises(ValueError, f, bad)
         self.assertEqual(
             str(e),
-            '%s must be >= 1; got 0' % label
+            '%s must be >= 0; got -2' % label
         )
 
         # Test with bad 'time' type/value:
@@ -420,6 +431,7 @@
         # Test with good doc:
         good = {
             '_id': 'ZR765XWSF6S7JQHLUI4GCG5BHGPE252O',
+            'ver': 0,
             'type': 'dmedia/file',
             'time': 1234567890,
             'bytes': 20202333,
@@ -517,7 +529,103 @@
         e = raises(ValueError, f, bad)
         self.assertEqual(
             str(e),
-            "stored['MZZG2ZDSOQVSW2TEMVZG643F']['copies'] must be >= 1; got -1"
+            "stored['MZZG2ZDSOQVSW2TEMVZG643F']['copies'] must be >= 0; got -1"
+        )
+
+    def test_check_dmedia_file_optional(self):
+        f = schema.check_dmedia_file_optional
+        f({})
+
+        # mime
+        self.assertIsNone(f({'mime': 'video/quicktime'}))
+        e = raises(TypeError, f, {'mime': 42})
+        self.assertEqual(
+            str(e),
+            TYPE_ERROR % ("doc['mime']", basestring, int, 42)
+        )
+
+        # media
+        self.assertIsNone(f({'media': None}))
+        self.assertIsNone(f({'media': 'video'}))
+        self.assertIsNone(f({'media': 'audio'}))
+        self.assertIsNone(f({'media': 'image'}))
+        e = raises(TypeError, f, {'media': 42})
+        self.assertEqual(
+            str(e),
+            TYPE_ERROR % ("doc['media']", basestring, int, 42)
+        )
+        e = raises(ValueError, f, {'media': 'stuff'})
+        self.assertEqual(
+            str(e),
+            "doc['media'] value 'stuff' not in ('video', 'audio', 'image')"
+        )
+
+        # mtime
+        self.assertIsNone(f({'mtime': 1302125982.946627}))
+        self.assertIsNone(f({'mtime': 1234567890}))
+        e = raises(TypeError, f, {'mtime': '1234567890'})
+        self.assertEqual(
+            str(e),
+            TYPE_ERROR % ("doc['mtime']", (int, float), str, '1234567890')
+        )
+        e = raises(ValueError, f, {'mtime': -1})
+        self.assertEqual(
+            str(e),
+            "doc['mtime'] must be >= 0; got -1"
+        )
+
+        # atime
+        self.assertIsNone(f({'atime': 1302125982.946627}))
+        self.assertIsNone(f({'atime': 1234567890}))
+        e = raises(TypeError, f, {'atime': '1234567890'})
+        self.assertEqual(
+            str(e),
+            TYPE_ERROR % ("doc['atime']", (int, float), str, '1234567890')
+        )
+        e = raises(ValueError, f, {'atime': -0.3})
+        self.assertEqual(
+            str(e),
+            "doc['atime'] must be >= 0; got -0.3"
+        )
+
+        # name
+        self.assertIsNone(f({'name': 'MVI_5899.MOV'}))
+        e = raises(TypeError, f, {'name': 42})
+        self.assertEqual(
+            str(e),
+            TYPE_ERROR % ("doc['name']", basestring, int, 42)
+        )
+
+        # dir
+        self.assertIsNone(f({'dir': 'DCIM/100EOS5D2'}))
+        e = raises(TypeError, f, {'dir': 42})
+        self.assertEqual(
+            str(e),
+            TYPE_ERROR % ("doc['dir']", basestring, int, 42)
+        )
+
+        # meta
+        self.assertIsNone(f({'meta': {'iso': 800}}))
+        e = raises(TypeError, f, {'meta': 42})
+        self.assertEqual(
+            str(e),
+            TYPE_ERROR % ("doc['meta']", dict, int, 42)
+        )
+
+        # user
+        self.assertIsNone(f({'user': {'title': 'cool sunset'}}))
+        e = raises(TypeError, f, {'user': 42})
+        self.assertEqual(
+            str(e),
+            TYPE_ERROR % ("doc['user']", dict, int, 42)
+        )
+
+        # tags
+        self.assertIsNone(f({'tags': {'burp': {'start': 6, 'end': 73}}}))
+        e = raises(TypeError, f, {'tags': 42})
+        self.assertEqual(
+            str(e),
+            TYPE_ERROR % ("doc['tags']", dict, int, 42)
         )
 
 
@@ -527,6 +635,7 @@
         # Test with good doc:
         good = {
             '_id': 'ZR765XWSF6S7JQHLUI4GCG5BHGPE252O',
+            'ver': 0,
             'type': 'dmedia/file',
             'time': 1234567890,
             'plugin': 'filestore',
@@ -593,6 +702,64 @@
         self.assertEqual(len(binary), 15)
         self.assertEqual(b32encode(binary), _id)
 
+    def test_create_file(self):
+        f = schema.create_file
+        store = schema.random_id()
+
+        d = f(mov_size, mov_leaves, store)
+        schema.check_dmedia_file(d)
+        self.assertEqual(
+            set(d),
+            set([
+                '_id',
+                '_attachments',
+                'ver',
+                'type',
+                'time',
+                'bytes',
+                'ext',
+                'origin',
+                'stored',
+            ])
+        )
+        self.assertEqual(d['_id'], mov_hash)
+        self.assertEqual(
+            d['_attachments'],
+            {
+                'leaves': {
+                    'data': b64encode(b''.join(mov_leaves)),
+                    'content_type': 'application/octet-stream',
+                }
+            }
+        )
+        self.assertEqual(d['ver'], 0)
+        self.assertEqual(d['type'], 'dmedia/file')
+        self.assertLessEqual(d['time'], time.time())
+        self.assertEqual(d['bytes'], mov_size)
+        self.assertIsNone(d['ext'], None)
+        self.assertEqual(d['origin'], 'user')
+
+        s = d['stored']
+        self.assertIsInstance(s, dict)
+        self.assertEqual(list(s), [store])
+        self.assertEqual(set(s[store]), set(['copies', 'time']))
+        self.assertEqual(s[store]['copies'], 0)
+        self.assertEqual(s[store]['time'], d['time'])
+
+        # Test overriding default kwarg values:
+        d = f(mov_size, mov_leaves, store, copies=2)
+        schema.check_dmedia_file(d)
+        self.assertEqual(d['stored'][store]['copies'], 2)
+
+        d = f(mov_size, mov_leaves, store, ext='mov')
+        schema.check_dmedia_file(d)
+        self.assertEqual(d['ext'], 'mov')
+
+        d = f(mov_size, mov_leaves, store, origin='proxy')
+        schema.check_dmedia_file(d)
+        self.assertEqual(d['origin'], 'proxy')
+
+
     def test_create_store(self):
         f = schema.create_store
         tmp = TempDir()
@@ -605,6 +772,7 @@
             set(doc),
             set([
                 '_id',
+                'ver',
                 'type',
                 'time',
                 'plugin',
@@ -625,6 +793,7 @@
             set(doc),
             set([
                 '_id',
+                'ver',
                 'type',
                 'time',
                 'plugin',
@@ -639,3 +808,102 @@
         self.assertEqual(doc['copies'], 3)
         self.assertEqual(doc['path'], base)
         self.assertEqual(doc['machine_id'], machine_id)
+
+    def test_create_batch(self):
+        f = schema.create_batch
+        machine_id = random_id()
+        doc = f(machine_id)
+
+        self.assertEqual(schema.check_dmedia(doc), None)
+        self.assertTrue(isinstance(doc, dict))
+        self.assertEqual(
+            set(doc),
+            set([
+                '_id',
+                'ver',
+                'type',
+                'time',
+                'imports',
+                'errors',
+                'machine_id',
+                'stats',
+            ])
+        )
+        _id = doc['_id']
+        self.assertEqual(b32encode(b32decode(_id)), _id)
+        self.assertEqual(len(_id), 24)
+        self.assertEqual(doc['type'], 'dmedia/batch')
+        self.assertTrue(isinstance(doc['time'], (int, float)))
+        self.assertTrue(doc['time'] <= time.time())
+        self.assertEqual(doc['imports'], [])
+        self.assertEqual(doc['errors'], [])
+        self.assertEqual(doc['machine_id'], machine_id)
+        self.assertEqual(
+            doc['stats'],
+            {
+                'considered': {'count': 0, 'bytes': 0},
+                'imported': {'count': 0, 'bytes': 0},
+                'skipped': {'count': 0, 'bytes': 0},
+                'empty': {'count': 0, 'bytes': 0},
+                'error': {'count': 0, 'bytes': 0},
+            }
+        )
+
+    def test_create_import(self):
+        f = schema.create_import
+
+        base = '/media/EOS_DIGITAL'
+        batch_id = random_id()
+        machine_id = random_id()
+
+        keys = set([
+            '_id',
+            'ver',
+            'type',
+            'time',
+            'base',
+            'batch_id',
+            'machine_id',
+            'log',
+            'stats',
+        ])
+
+        doc = f(base, batch_id=batch_id, machine_id=machine_id)
+        self.assertEqual(schema.check_dmedia(doc), None)
+        self.assertTrue(isinstance(doc, dict))
+        self.assertEqual(set(doc), keys)
+
+        _id = doc['_id']
+        self.assertEqual(b32encode(b32decode(_id)), _id)
+        self.assertEqual(len(_id), 24)
+
+        self.assertEqual(doc['type'], 'dmedia/import')
+        self.assertTrue(isinstance(doc['time'], (int, float)))
+        self.assertTrue(doc['time'] <= time.time())
+        self.assertEqual(doc['base'], base)
+        self.assertEqual(doc['batch_id'], batch_id)
+        self.assertEqual(doc['machine_id'], machine_id)
+
+        doc = f(base)
+        self.assertEqual(schema.check_dmedia(doc), None)
+        self.assertEqual(set(doc), keys)
+        self.assertEqual(doc['batch_id'], None)
+        self.assertEqual(doc['machine_id'], None)
+        self.assertEqual(
+            doc['log'],
+            {
+                'imported': [],
+                'skipped': [],
+                'empty': [],
+                'error': [],
+            }
+        )
+        self.assertEqual(
+            doc['stats'],
+            {
+                'imported': {'count': 0, 'bytes': 0},
+                'skipped': {'count': 0, 'bytes': 0},
+                'empty': {'count': 0, 'bytes': 0},
+                'error': {'count': 0, 'bytes': 0},
+            }
+        )

=== modified file 'dmedia/webui/data/browser.js'
--- dmedia/webui/data/browser.js	2011-03-31 09:28:58 +0000
+++ dmedia/webui/data/browser.js	2011-04-07 03:13:28 +0000
@@ -53,8 +53,8 @@
 }
 Browser.prototype = {
     run: function() {
-        var r = this.db.view('file', 'ext',
-            {key: 'mov', reduce: false, include_docs: true}
+        var r = this.db.view('user', 'video',
+            {include_docs: true, descending: true}
         );
         this.load(r.rows);
     },
@@ -74,7 +74,7 @@
             };
 
             var time = $el('div', {'class': 'time'});
-            time.textContent = doc.duration + 's';
+            time.textContent = doc.meta.duration + 's';
 
             div.appendChild(img);
             div.appendChild(time);
@@ -90,7 +90,7 @@
         names.forEach(function(n) {
             var el = $('meta.' + n);
             if (el) {
-                el.textContent = doc[n];
+                el.textContent = doc.meta[n];
             }
         });
     },