Merge lp:~jderose/dmedia/project-stats into lp:dmedia

[Jason Gerard DeRose]

* Changes dmedia/project schema to include doc.count, doc.bytes

* Changes Importer.on_items() (in dmedia.js) to use count and bytes from project directory docs rather than hitting a view in each project database

* Changes Core.init_project_views() to update the doc.count, doc.type in the project doc in the project database, and also to sync the changes into the project directory doc in dmedia-0 if needed

[James Raymond (jamesmr)]

review: Approve

=== modified file 'dmedia/core.py'
--- dmedia/core.py	2012-10-13 15:50:03 +0000
+++ dmedia/core.py	2012-10-17 21:36:22 +0000
@@ -96,8 +96,8 @@
     log.info('** %.2f to snapshot all database', time.time() - start)
 
 
-def projects_iter(env):
-    server = Server(env)
+def projects_iter(server):
+    assert isinstance(server, Server)
     for name in server.get('_all_dbs'):
         if name.startswith('_'):
             continue
@@ -107,12 +107,21 @@
         yield (name, _id)
 
 
+def update_count_and_bytes(db, doc):
+    stats = db.view('user', 'bytes', reduce=True)['rows'][0]['value']
+    if doc.get('count') != stats['count'] or doc.get('bytes') != stats['sum']:
+        doc['count'] = stats['count']
+        doc['bytes'] = stats['sum']
+        db.save(doc)
+
+
 class Core:
     def __init__(self, env, private=None, shared=None):
         self.env = env
         self._private = (PRIVATE if private is None else private)
         self._shared = (SHARED if shared is None else shared)
         self.db = util.get_db(env, init=True)
+        self.server = self.db.server()
         self.ms = MetaStore(self.db)
         self.stores = LocalStores()
         self.queue = Queue()
@@ -201,21 +210,28 @@
         self.thread = start_thread(self._background_worker)
 
     def init_project_views(self):
+        start = time.time()
         try:
-            for (name, _id) in projects_iter(self.env):
-                db = util.get_project_db(_id, self.env, True)
+            self.db.view('project', 'atime')
+            log.info('%.3f to prep project/atime view', time.time() - start)
+            for (name, _id) in projects_iter(self.server):
+                pdb = util.get_project_db(_id, self.env, True)
+                pdoc = pdb.get(_id, attachments=True)
+                update_count_and_bytes(pdb, pdoc)
+                del pdoc['_rev']
                 try:
-                    doc = self.db.get(_id)
+                    doc = self.db.get(_id, attachments=True)
+                    rev = doc.pop('_rev')
+                    if doc != pdoc:
+                        log.info('syncing project metadata for %s', _id)
+                        pdoc['_rev'] = rev
+                        self.db.save(pdoc)
                 except NotFound:
                     log.info('missing project doc for %s', _id)
-                    doc = db.get(_id)
-                    del doc['_rev']
-                    self.db.save(doc)
+                    self.db.save(pdoc)
+            log.info('%.3f for Core.init_project_views() to complete', time.time() - start)
         except Exception:
             log.exception('Error in Core.init_project_views():')
-        log.info('prepping project/atime view...')
-        self.db.view('project', 'atime')
-        log.info('Core.init_project_views() complete')
 
     def snapshot(self, dumpdir, dbname):
         start_process(snapshot_db, self.env, dumpdir, dbname)
 
=== modified file 'dmedia/schema.py'
--- dmedia/schema.py	2012-08-13 02:04:32 +0000
+++ dmedia/schema.py	2012-10-17 21:36:22 +0000
@@ -960,6 +960,8 @@
     ...     'time': 1234567890,
     ...     'db_name': 'dmedia-0-hb6ysckay27kiwutwkgkctni',
     ...     'title': 'UDS-P',
+    ...     'count': 42,
+    ...     'bytes': 22020096000,
     ... }
     ...
     >>> check_project(doc)
@@ -976,6 +978,12 @@
         (_equals, project_db_name(doc['_id'])),
     )
     _check(doc, ['title'], str),
+    _check(doc, ['count'], int,
+        (_at_least, 0),
+    )
+    _check(doc, ['count'], int,
+        (_at_least, 0),
+    )
 
 
 def project_db_name(_id):
@@ -1024,6 +1032,8 @@
         'atime': ts,
         'db_name': project_db_name(_id),
         'title': title,
+        'count': 0,
+        'bytes': 0,
     }
 
 
 
=== modified file 'dmedia/tests/test_core.py'
--- dmedia/tests/test_core.py	2012-10-10 22:47:53 +0000
+++ dmedia/tests/test_core.py	2012-10-17 21:36:22 +0000
@@ -42,14 +42,14 @@
 
 class TestCouchFunctions(CouchCase):
     def test_projects_iter(self):
-        self.assertEqual(list(core.projects_iter(self.env)), [])
+        server = microfiber.Server(self.env)
+        self.assertEqual(list(core.projects_iter(server)), [])
         ids = tuple(random_id() for i in range(20))
-        server = microfiber.Server(self.env)
         for _id in ids:
             db_name = project_db_name(_id)
             server.put(None, db_name)
         self.assertEqual(
-            list(core.projects_iter(self.env)),
+            list(core.projects_iter(server)),
             [(project_db_name(_id), _id) for _id in sorted(ids)]
         )
 
@@ -59,6 +59,8 @@
         inst = core.Core(self.env)
         self.assertIsInstance(inst.db, microfiber.Database)
         self.assertEqual(inst.db.name, DB_NAME)
+        self.assertIsInstance(inst.server, microfiber.Server)
+        self.assertIs(inst.db.ctx, inst.server.ctx)
         self.assertIsInstance(inst.stores, LocalStores)
         self.assertEqual(inst.local, {'_id': '_local/dmedia', 'stores': {}})
 
 
=== modified file 'ui/dmedia.js'
--- ui/dmedia.js	2012-08-21 18:21:45 +0000
+++ ui/dmedia.js	2012-10-17 21:36:22 +0000
@@ -106,20 +106,16 @@
 Importer.prototype = {
     load_items: function() {
         var callback = $bind(this.on_items, this);
-        db.view(callback, 'project', 'title');
+        db.view(callback, 'project', 'title', {'include_docs': true});
     },
 
     on_items: function(req) {
         this.items.replace(req.read().rows,
             function(row, items) {
-                var pdb = new couch.Database("dmedia-0-" + row.id.toLowerCase());
-                try{
-                    var filecount = pdb.view_sync('doc', 'type', {key: 'dmedia/file'}).rows[0].value;
-                }
-                catch(e){
-                    var filecount = 0;
-                }
-                
+                var doc = row.doc;
+                var count = doc.count ? doc.count : 0;
+                var bytes = doc.bytes ? doc.bytes : 0;
+
                 var li = $el('li', {'class': 'project', 'id': row.id});
 
                 var thumb = $el('div', {'class': 'thumbnail'});
@@ -135,7 +131,7 @@
                 );
 
                 info.appendChild(
-                    $el('p', {'textContent': filecount + ' files'})
+                    $el('p', {'textContent': count_n_size(count, bytes)})
                 );
 
                 li.appendChild(thumb);