Merge lp:~openerp-dev/openobject-server/6.0-dbcreatecache into lp:openobject-server/6.0

[Antony Lesuisse (OpenERP)]

It's freaking fast !!!
Actually, this should be ported into 6.1 and should stay as a patch for the 6.0 serie.

[lp:~openerp-dev/openobject-server/6.0-dbcreatecache]

3442. By Antony Lesuisse (OpenERP) on 2011-06-02

createdbcache DBInitializeCache reload pool for cached db too (thanks dsh)

3443. By Antony Lesuisse (OpenERP) on 2011-06-02

createdbcache DBInitializeCache typo

3444. By Antony Lesuisse (OpenERP) on 2011-06-02

createdbcache DBInitializeCache remove print

[Antony Lesuisse (OpenERP) (al-openerp)]

Bug for password

[lp:~openerp-dev/openobject-server/6.0-dbcreatecache]

3445. By Antony Lesuisse (OpenERP) on 2011-06-06

createdbcache change admin password

3446. By Antony Lesuisse (OpenERP) on 2011-06-09

createdbcache change admin password

3447. By Antony Lesuisse (OpenERP) on 2011-06-10

createdbcache wait pg_restore

Unmerged revisions

3447. By Antony Lesuisse (OpenERP) on 2011-06-10

createdbcache wait pg_restore

3446. By Antony Lesuisse (OpenERP) on 2011-06-09

createdbcache change admin password

3445. By Antony Lesuisse (OpenERP) on 2011-06-06

createdbcache change admin password

3444. By Antony Lesuisse (OpenERP) on 2011-06-02

createdbcache DBInitializeCache remove print

3443. By Antony Lesuisse (OpenERP) on 2011-06-02

createdbcache DBInitializeCache typo

3442. By Antony Lesuisse (OpenERP) on 2011-06-02

createdbcache DBInitializeCache reload pool for cached db too (thanks dsh)

3441. By Antony Lesuisse (OpenERP) on 2011-06-02

createdbcache DBInitializeCache fix databased created using cache

3440. By Antony Lesuisse (OpenERP) on 2011-06-01

createdbcache DBInitializeCache it's freaking fast...

3439. By Antony Lesuisse (OpenERP) on 2011-06-01

createdbcache more cleanup DBInitialize, space and folding cleanups

3438. By Antony Lesuisse (OpenERP) on 2011-06-01

createdbcache cleanup DBInitialize

=== modified file 'bin/service/web_services.py'
--- bin/service/web_services.py	2011-01-15 17:04:10 +0000
+++ bin/service/web_services.py	2011-06-10 15:19:25 +0000
@@ -37,8 +37,104 @@
 import tools
 import locale
 import logging
+import traceback
+import zlib
 from cStringIO import StringIO
 
+DBInitializerCache = {}
+
+class DBInitializer(object):
+    def __init__(self, service, action_id, db_name, db_demo, db_lang, db_user_password='admin'):
+        self.service = service
+        self.action_id = action_id
+        self.action = self.service.actions[action_id]
+        self.db_name = db_name
+        self.db_lang = db_lang
+        self.db_demo = db_demo
+        self.db_user_password = db_user_password
+        self.registry = None
+
+    def __call__(self):
+        self.run()
+
+    def with_cr(self,f):
+        cr = None
+        try:
+            cr = sql_db.db_connect(self.db_name).cursor()
+            f(cr)
+        finally:
+            if cr:
+                cr.close()
+
+    def run(self):
+        try:
+            self.action['progress'] = 0
+            key = (self.db_lang, self.db_demo)
+            if key not in DBInitializerCache:
+                self.run_new(key)
+            else:
+                self.run_cached(key)
+        except Exception, e:
+            e_str = StringIO()
+            traceback.print_exc(file=e_str)
+            traceback_str = e_str.getvalue()
+            e_str.close()
+            netsvc.Logger().notifyChannel('web-services', netsvc.LOG_ERROR, 'CREATE DATABASE\n%s' % (traceback_str))
+            self.action['clean'] = False
+            self.action['exception'] = e
+            self.action['traceback'] = traceback_str
+
+    def run_new(self,key):
+        self.service._create_empty_database(self.db_name)
+        self.with_cr(self.init_base)
+        self.with_cr(self.init_lang)
+        try:
+            s = zlib.compress(self.service.exp_dump(self.db_name), 9)
+            DBInitializerCache[key] = s
+            netsvc.Logger().notifyChannel('web-services', netsvc.LOG_DEBUG, 'CREATE DATABASE cached %r size %d' % (key, len(s)))
+        except Exception,e:
+            pass
+        self.with_cr(self.init_user)
+
+    def run_cached(self,key):
+        s = DBInitializerCache[key]
+        self.service.exp_restore(self.db_name, zlib.decompress(s))
+        self.registry = pooler.restart_pool(self.db_name, self.db_demo, None)[1]
+        self.with_cr(self.init_user)
+        self.with_cr(self.init_cached)
+
+    def init_base(self,cr):
+        tools.init_db(cr)
+        cr.commit()
+        tools.config['lang'] = self.db_lang
+        self.registry = pooler.restart_pool(self.db_name, self.db_demo, self.action, update_module=True)[1]
+
+    def init_lang(self,cr):
+        if self.db_lang:
+            modobj = self.registry.get('ir.module.module')
+            mids = modobj.search(cr, 1, [('state', '=', 'installed')])
+            modobj.update_translations(cr, 1, mids, self.db_lang)
+        cr.commit()
+
+    def init_user(self,cr):
+        cr.execute('UPDATE res_users SET password=%s, context_lang=%s, active=True WHERE login=%s', ( self.db_user_password, self.db_lang, 'admin'))
+        cr.execute('SELECT login, password, name FROM res_users ORDER BY login')
+        self.action['users'] = cr.dictfetchall()
+        self.action['clean'] = True
+        cr.commit()
+
+    def init_cached(self,cr):
+        cr.execute('DELETE from ir_config_parameter')
+        self.registry.get('ir.config_parameter').init(cr)
+        cr.execute('UPDATE res_users SET password=%s WHERE id=1',(self.db_user_password,))
+        for i in self.registry.obj_list():
+            table = self.registry.get(i)._table
+            cr.execute("SELECT c.relname FROM pg_class c, pg_attribute a WHERE c.oid=a.attrelid AND c.relname=%s AND a.attname=%s", (table, 'create_date'))
+            if len(cr.fetchall()) > 0:
+                cr.execute('UPDATE %s SET create_date = NOW() WHERE create_date IS NOT NULL'%table)
+                cr.execute('UPDATE %s SET write_date = NOW() WHERE write_date IS NOT NULL'%table)
+        cr.commit()
+
 class db(netsvc.ExportService):
     def __init__(self, name="db"):
         netsvc.ExportService.__init__(self, name)
@@ -67,6 +163,7 @@
 
     def new_dispatch(self,method,auth,params):
         pass
+
     def _create_empty_database(self, name):
         db = sql_db.db_connect('template1')
         cr = db.cursor()
@@ -84,55 +181,14 @@
 
         self.actions[id] = {'clean': False}
 
-        self._create_empty_database(db_name)
-
-        class DBInitialize(object):
-            def __call__(self, serv, id, db_name, demo, lang, user_password='admin'):
-                cr = None
-                try:
-                    serv.actions[id]['progress'] = 0
-                    cr = sql_db.db_connect(db_name).cursor()
-                    tools.init_db(cr)
-                    tools.config['lang'] = lang
-                    cr.commit()
-                    cr.close()
-                    cr = None
-                    pool = pooler.restart_pool(db_name, demo, serv.actions[id],
-                            update_module=True)[1]
-
-                    cr = sql_db.db_connect(db_name).cursor()
-
-                    if lang:
-                        modobj = pool.get('ir.module.module')
-                        mids = modobj.search(cr, 1, [('state', '=', 'installed')])
-                        modobj.update_translations(cr, 1, mids, lang)
-
-                    cr.execute('UPDATE res_users SET password=%s, context_lang=%s, active=True WHERE login=%s', (
-                        user_password, lang, 'admin'))
-                    cr.execute('SELECT login, password, name ' \
-                               '  FROM res_users ' \
-                               ' ORDER BY login')
-                    serv.actions[id]['users'] = cr.dictfetchall()
-                    serv.actions[id]['clean'] = True
-                    cr.commit()
-                    cr.close()
-                except Exception, e:
-                    serv.actions[id]['clean'] = False
-                    serv.actions[id]['exception'] = e
-                    import traceback
-                    e_str = StringIO()
-                    traceback.print_exc(file=e_str)
-                    traceback_str = e_str.getvalue()
-                    e_str.close()
-                    netsvc.Logger().notifyChannel('web-services', netsvc.LOG_ERROR, 'CREATE DATABASE\n%s' % (traceback_str))
-                    serv.actions[id]['traceback'] = traceback_str
-                    if cr:
-                        cr.close()
+        if self.exp_db_exist(db_name):
+            logger.notifyChannel("web-services", netsvc.LOG_WARNING, 'CREATE DATABASE: %s already exists' % (db_name,))
+            raise Exception, "Database already exists"
+
         logger = netsvc.Logger()
         logger.notifyChannel("web-services", netsvc.LOG_INFO, 'CREATE DATABASE: %s' % (db_name.lower()))
-        dbi = DBInitialize()
-        create_thread = threading.Thread(target=dbi,
-                args=(self, id, db_name, demo, lang, user_password))
+        dbi = DBInitializer(self, id, db_name, demo, lang, user_password)
+        create_thread = threading.Thread(target=dbi)
         create_thread.start()
         self.actions[id]['thread'] = create_thread
         return id
@@ -217,8 +273,7 @@
         self._set_pg_psw_env_var()
 
         if self.exp_db_exist(db_name):
-            logger.notifyChannel("web-services", netsvc.LOG_WARNING,
-                    'RESTORE DB: %s already exists' % (db_name,))
+            logger.notifyChannel("web-services", netsvc.LOG_WARNING, 'RESTORE DB: %s already exists' % (db_name,))
             raise Exception, "Database already exists"
 
         self._create_empty_database(db_name)
@@ -240,11 +295,13 @@
             args2=list(args2)
             args2.append(' ' + tmpfile)
             args2=tuple(args2)
-        stdin, stdout = tools.exec_pg_command_pipe(*args2)
+        p = tools.exec_pg_command_pipe_subprocess(*args2)
         if not os.name == "nt":
-            stdin.write(base64.decodestring(data))
-        stdin.close()
-        res = stdout.close()
+            p.stdin.write(base64.decodestring(data))
+        p.stdin.close()
+        p.stdout.close()
+        res = p.wait()
+        print "WAIT ",res
         if res:
             raise Exception, "Couldn't restore database"
         logger.notifyChannel("web-services", netsvc.LOG_INFO,
@@ -346,6 +403,7 @@
                 l.notifyChannel('web-services', netsvc.LOG_ERROR, tb_s)
                 raise
         return True
+
 db()
 
 class _ObjectService(netsvc.ExportService):
@@ -397,7 +455,6 @@
         fn = getattr(self, 'exp_'+method)
         return fn(*params)
 
-
     def new_dispatch(self,method,auth,params):
         pass
 
@@ -423,16 +480,7 @@
         @param extended: if True then return version info
         @return string if extended is False else tuple
         """
-
-        info = _('''
-
-OpenERP is an ERP+CRM program for small and medium businesses.
-
-The whole source code is distributed under the terms of the
-GNU Public Licence.
-
-(c) 2003-TODAY, Fabien Pinckaers - Tiny sprl''')
-
+        info = 'OpenERP'
         if extended:
             return info, release.version
         return info
@@ -452,7 +500,6 @@
         except tm.RemoteContractException, e:
             self.abortResponse(1, 'Migration Error', 'warning', str(e))
 
-
     def exp_get_migration_scripts(self, contract_id, contract_password):
         l = netsvc.Logger()
         import tools.maintenance as tm
@@ -665,6 +712,7 @@
                 raise Exception, 'AccessDenied'
         else:
             raise Exception, 'WizardNotFound'
+
 wizard()
 
 #
@@ -781,6 +829,4 @@
 
 report_spool()
 
-
 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
-

=== modified file 'bin/tools/misc.py'
--- bin/tools/misc.py	2011-01-20 12:40:20 +0000
+++ bin/tools/misc.py	2011-06-10 15:19:25 +0000
@@ -158,15 +158,19 @@
     
     return subprocess.call(args2)
 
+def exec_pg_command_pipe_subprocess(name, *args):
+    prog = find_pg_tool(name)
+    if not prog:
+        raise Exception('Couldn\'t find %s' % name)
+    # on win32, passing close_fds=True is not compatible
+    # with redirecting std[in/err/out]
+    pop = subprocess.Popen((prog,) + args, bufsize= -1,
+          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+          close_fds=(os.name=="posix"))
+    return pop
+
 def exec_pg_command_pipe(name, *args):
-    prog = find_pg_tool(name)
-    if not prog:
-        raise Exception('Couldn\'t find %s' % name)
-    # on win32, passing close_fds=True is not compatible
-    # with redirecting std[in/err/out]
-    pop = subprocess.Popen((prog,) + args, bufsize= -1,
-          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
-          close_fds=(os.name=="posix"))
+    pop = exec_pg_command_pipe_subprocess(name, *args)
     return (pop.stdin, pop.stdout)
 
 def exec_command_pipe(name, *args):
@@ -799,6 +803,7 @@
             self.fun_default_values = dict(zip(self.fun_arg_names[-len(argspec[3]):], argspec[3]))
 
         def cached_result(self2, cr, *args, **kwargs):
+            
             if time.time()-int(self.timeout) > self.lasttime:
                 self.lasttime = time.time()
                 t = time.time()-int(self.timeout)