Merge lp:~jderose/dmedia/project-scan into lp:dmedia

[Jason Gerard DeRose]

* Adds schema.get_project_id() function (and test) to go from dmedia-0-project_id to PROJECT_ID

* Adds core.projects_iter() function (and test) to iterate through all Dmedia project databases

* Changes Core.init_project_views() to use projects_iter() instead of the project/atime view, and to copy over the project doc into dmedia-0 when it doesn't already exist

[lp:~jderose/dmedia/project-scan]

448. By Jason Gerard DeRose on 2012-08-13

Core.init_project_views() now preps project/atime view for performance reasons

[David Jordan (dmj726)]

Approved!

review: Approve

=== modified file 'dmedia/core.py'
--- dmedia/core.py	2012-08-06 11:03:16 +0000
+++ dmedia/core.py	2012-08-13 06:09:20 +0000
@@ -38,7 +38,7 @@
 import logging
 from queue import Queue
 
-from microfiber import Database, NotFound, Conflict, random_id2
+from microfiber import Server, Database, NotFound, Conflict
 from filestore import FileStore, check_root_hash, check_id, _start_thread
 
 import dmedia
@@ -83,6 +83,17 @@
     return (httpd, port)
 
 
+def projects_iter(env):
+    server = Server(env)
+    for name in server.get('_all_dbs'):
+        if name.startswith('_'):
+            continue
+        _id = schema.get_project_id(name)
+        if _id is None:
+            continue
+        yield (name, _id)
+
+
 class Core:
     def __init__(self, env, private=None, shared=None, full_init=True):
         self.env = env
@@ -172,8 +183,20 @@
         self.thread = _start_thread(self._background_worker)
 
     def init_project_views(self):
-        for row in self.db.view('project', 'atime')['rows']:
-            util.get_project_db(row['id'], self.env, True)
+        try:
+            for (name, _id) in projects_iter(self.env):
+                db = util.get_project_db(_id, self.env, True)
+                try:
+                    doc = self.db.get(_id)
+                except NotFound:
+                    log.info('missing project doc for %s', _id)
+                    doc = db.get(_id)
+                    del doc['_rev']
+                    self.db.save(doc)
+        except Exception:
+            log.exception('Error in Core.init_project_views():')
+        log.info('prepping project/atime view...')
+        self.db.view('project', 'atime')
         log.info('Core.init_project_views() complete')
 
     def set_default_store(self, value):

=== modified file 'dmedia/schema.py'
--- dmedia/schema.py	2012-08-06 11:27:03 +0000
+++ dmedia/schema.py	2012-08-13 06:09:20 +0000
@@ -259,6 +259,9 @@
 # Pattern that doc['ext'] must match for dmedia/file
 EXT_PAT = '^[a-z0-9]+(\.[a-z0-9]+)?$'
 
+# Pattern to match a project DB name
+PROJECT_DB_PAT = '^dmedia-0-([234567abcdefghijklmnopqrstuvwxyz]{24})$'
+
 
 # Some private helper functions that don't directly define any schema.
 #
@@ -984,10 +987,33 @@
     >>> project_db_name('HB6YSCKAY27KIWUTWKGKCTNI')
     'dmedia-0-hb6ysckay27kiwutwkgkctni'
 
+    Also see `get_project_id()`.
     """
     return '-'.join(['dmedia', str(VER), _id.lower()])
 
 
+def get_project_id(db_name):
+    """
+    Return project ID from CouchDB database name.
+
+    For example:
+
+    >>> get_project_id('dmedia-0-hb6ysckay27kiwutwkgkctni')
+    'HB6YSCKAY27KIWUTWKGKCTNI'
+
+    If *db_name* doesn't match the expected naming convention, ``None`` is
+    returned:
+
+    >>> get_project_id('dmedia-hb6ysckay27kiwutwkgkctni') is None
+    True
+
+    Also see `project_db_name()`.
+    """
+    match = re.match(PROJECT_DB_PAT, db_name)
+    if match:
+        return match.group(1).upper()
+
+
 def create_project(title=''):
     _id = random_id()
     ts = time.time()

=== modified file 'dmedia/tests/test_core.py'
--- dmedia/tests/test_core.py	2012-07-21 00:23:37 +0000
+++ dmedia/tests/test_core.py	2012-08-13 06:09:20 +0000
@@ -32,13 +32,27 @@
 import filestore
 
 from dmedia.local import LocalStores
-from dmedia.schema import DB_NAME, create_filestore
+from dmedia.schema import DB_NAME, create_filestore, project_db_name
 from dmedia import util, core
 
 from .couch import CouchCase
 from .base import TempDir
 
 
+class TestCouchFunctions(CouchCase):
+    def test_projects_iter(self):
+        self.assertEqual(list(core.projects_iter(self.env)), [])
+        ids = tuple(random_id() for i in range(20))
+        server = microfiber.Server(self.env)
+        for _id in ids:
+            db_name = project_db_name(_id)
+            server.put(None, db_name)
+        self.assertEqual(
+            list(core.projects_iter(self.env)),
+            [(project_db_name(_id), _id) for _id in sorted(ids)]
+        )
+
+
 class TestCore(CouchCase):
     def test_init(self):
         inst = core.Core(self.env)

=== modified file 'dmedia/tests/test_schema.py'
--- dmedia/tests/test_schema.py	2012-08-06 11:27:03 +0000
+++ dmedia/tests/test_schema.py	2012-08-13 06:09:20 +0000
@@ -721,6 +721,22 @@
             'dmedia-0-{}'.format(_id.lower())
         )
 
+    def test_get_project_id(self):
+        self.assertEqual(
+            schema.get_project_id('dmedia-0-aaaaaaaaaaaaaaaaaaaaaaaa'),
+            'AAAAAAAAAAAAAAAAAAAAAAAA'
+        )
+        self.assertIsNone(schema.get_project_id('dmedia-0'))
+        self.assertIsNone(schema.get_project_id('novacut-0'))
+        self.assertIsNone(
+            schema.get_project_id('novacut-0-aaaaaaaaaaaaaaaaaaaaaaaa')
+        )
+        # Make sure we can round-trip with project_db_name():
+        for i in range(1000):
+            _id = random_id()
+            db_name = schema.project_db_name(_id)
+            self.assertEqual(schema.get_project_id(db_name), _id)
+
     def test_create_project(self):
         doc = schema.create_project()
         schema.check_project(doc)