Merge lp:~therp-nl/openupgrade-server/6.0-API_and_loading_improvements into lp:openupgrade-server/6.0

[Stefan Rijnhart (Opener)]

This branch ensures that pre and post scripts are only called once. This means that developers do not have to bother checking whether they are being rerun in the scripts themselves. It is now also ensured that new dependencies are installed automatically. Modules that are known to be obsolete will now be removed automatically.

The API is augmented with a decorator that can be used in migration scripts to log any errors that may occur. A new function for renaming XML IDs is provided, and setting defaults provided by a function is faster.

Some bugs in the base module migration scripts are solved.

=== modified file 'bin/addons/__init__.py'
--- bin/addons/__init__.py	2012-05-27 12:34:50 +0000
+++ bin/addons/__init__.py	2012-06-07 14:50:24 +0000
@@ -50,14 +50,7 @@
 
 logger = netsvc.Logger()
 
-### OpenUpgrade
-def table_exists(cr, table):
-    """ Check whether a certain table or view exists """
-    cr.execute(
-        'SELECT count(relname) FROM pg_class WHERE relname = %s',
-        (table,))
-    return cr.fetchone()[0] == 1
-### End of OpenUpgrade
+from openupgrade import openupgrade
 
 _ad = os.path.abspath(opj(tools.ustr(tools.config['root_path']), u'addons'))     # default addons path (base)
 ad_paths= map(lambda m: os.path.abspath(tools.ustr(m.strip())), tools.config['addons_path'].split(','))
@@ -808,7 +801,7 @@
         log any differences and merge the local registry with
         the global one.
         """
-        if not table_exists(cr, 'openupgrade_record'):
+        if not openupgrade.table_exists(cr, 'openupgrade_record'):
             return
         for model, fields in local_registry.items():
             registry.setdefault(model, {})
@@ -848,10 +841,9 @@
     logger.notifyChannel('init', netsvc.LOG_DEBUG, 'loading %d packages..' % len(graph))
 
     for package in graph:
-        if skip_modules and package.name in skip_modules:
-            continue
         logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading objects' % package.name)
-        migrations.migrate_module(package, 'pre')
+        if package.name not in skip_modules['pre']:
+            migrations.migrate_module(package, 'pre')
         register_class(package.name)
         modules = pool.instanciate(package.name, cr)
 
@@ -864,13 +856,14 @@
 
             init_module_objects(cr, package.name, modules)
         cr.commit()
+        skip_modules['pre'].append(package.name)
 
     for package in graph:
         status['progress'] = (float(statusi)+0.1) / len(graph)
         m = package.name
         mid = package.id
 
-        if skip_modules and m in skip_modules:
+        if package.name in skip_modules['post']:
             continue
 
         if modobj is None:
@@ -923,6 +916,7 @@
                 if hasattr(package, kind):
                     delattr(package, kind)
 
+        skip_modules['post'].append(package.name)
         statusi += 1
     cr.commit()
 
@@ -964,6 +958,7 @@
 
     registry = {}
 
+    skip_modules = {'pre': [], 'post': []}
     try:
         processed_modules = []
         report = tools.assertion_report()
@@ -977,7 +972,7 @@
         if not graph:
             logger.notifyChannel('init', netsvc.LOG_CRITICAL, 'module base cannot be loaded! (hint: verify addons-path)')
             raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)'))
-        processed_modules.extend(load_module_graph(cr, graph, status, perform_checks=(not update_module), registry=registry, report=report))
+        processed_modules.extend(load_module_graph(cr, graph, status, perform_checks=(not update_module), registry=registry, report=report, skip_modules=skip_modules))
 
         if tools.config['load_language']:
             for lang in tools.config['load_language'].split(','):
@@ -1021,13 +1016,15 @@
             if not module_list:
                 break
 
+            # OpenUpgrade: forcefeeding module dependencies into the graph
+            module_list = openupgrade.add_module_dependencies(cr, module_list)
             new_modules_in_graph = upgrade_graph(graph, cr, module_list, force)
             if new_modules_in_graph == 0:
                 # nothing to load
                 break
 
             logger.notifyChannel('init', netsvc.LOG_DEBUG, 'Updating graph with %d more modules' % (len(module_list)))
-            processed_modules.extend(load_module_graph(cr, graph, status, registry=registry, report=report, skip_modules=processed_modules))
+            processed_modules.extend(load_module_graph(cr, graph, status, registry=registry, report=report, skip_modules=skip_modules))
 
         # load custom models
         cr.execute('select model from ir_model where state=%s', ('manual',))

=== modified file 'bin/addons/base/migrations/6.0.1.3/post-migration.py'
--- bin/addons/base/migrations/6.0.1.3/post-migration.py	2011-11-30 22:51:16 +0000
+++ bin/addons/base/migrations/6.0.1.3/post-migration.py	2012-06-07 14:50:24 +0000
@@ -1,18 +1,58 @@
 # -*- coding: utf-8 -*-
 
-from osv import osv
 import pooler, logging
 from openupgrade import openupgrade
-log = logging.getLogger('migrate')
+logger = logging.getLogger('migrate')
+MODULE="base"
+
+obsolete_modules = [
+    'account_report',
+    'account_reporting',
+    'account_tax_include',
+    'board_account',
+    'board_sale',
+    'report_account', 
+    'report_analytic',
+    'report_analytic_line',
+    'report_crm',
+    'report_purchase',
+    'report_sale',
+    'pxgo_bank_statement_analytic',
+]
+
+def set_defaults_on_act_window(cr):
+    """ The act window model has a constraint
+    that checks the validity of the model on it.
+    At migration time, this is inconvenient.
+    Therefore, we set the defaults through SQL
+
+    Replaces setting the following defaults
+    #'ir.actions.act_window': [
+    #    ('auto_search', True),
+    #    ('context', '{}'),
+    #    ('multi', False),
+    #    ],    
+
+    """
+    cr.execute("""
+        UPDATE ir_act_window
+        SET auto_search = true,
+            multi = false
+        """)
+    cr.execute("""
+        UPDATE ir_act_window
+        SET context = '{}'
+        WHERE context is NULL
+        """)
+    cr.execute("""
+        UPDATE ir_act_window
+        SET context = '{}'
+        WHERE context is NULL
+        """)
 
 defaults = {
     # False results in column value NULL
     # None value triggers a call to the model's default function 
-    'ir.actions.act_window': [
-        ('auto_search', True),
-        ('context', '{}'),
-        ('multi', False),
-        ],    
     'ir.actions.todo': [
         ('restart', 'onskip'),
         ],
@@ -96,13 +136,14 @@
     # In the pre script, we renamed the old function column, a many2one
     # to res.partner.function
     cr.execute(
-        "UPDATE res_partner_address SET function = res_partner_function.name " +
-        "FROM res_partner_function " +
-        "WHERE res_partner_function.id = res_partner_address.tmp_mgr_function"
-        )
-    cr.execute("ALTER TABLE res_partner_address DROP COLUMN tmp_mgr_function CASCADE")
-    cr.execute("DROP TABLE res_partner_function")
-
+        "UPDATE res_partner_address "
+        "SET function = openupgrade_legacy_res_partner_function.name "
+        "FROM openupgrade_legacy_res_partner_function "
+        "WHERE openupgrade_legacy_res_partner_function.id "
+        "= res_partner_address.tmp_mgr_function")
+    cr.execute(
+        "ALTER TABLE res_partner_address "
+        "DROP COLUMN tmp_mgr_function CASCADE")
     # and the reverse: 'title' is now a many2one on res_partner_title
     cr.execute(
         "SELECT id, tmp_mgr_title FROM res_partner_address WHERE title IS NULL " +
@@ -139,10 +180,9 @@
         if bank_ids:
             bank_id = bank_ids[0]
         else:
-            bank_id = bank_obj.create(cursor, uid, dict(
-                    code = info.code or 'UNKNOW',
-                    name = info.name or _('Unknown Bank'),
-                    ))
+            bank_id = bank_obj.create(cr, 1, dict(
+                    code = 'UNKNOW',
+                    name = 'Unknown Bank')),
         partner_bank_obj.write(cr, 1, partner_bank_ids, {'bank': bank_id})
             
 def mgr_roles_to_groups(cr, pool):
@@ -208,24 +248,35 @@
         "AND ir_actions.usage = 'menu'"
         )
 
+def mark_obsolete_modules(cr):
+    """
+    Remove modules that are known to be obsolete
+    in this version of the OpenERP server.
+    """
+    openupgrade.logged_query(
+        cr, """
+        UPDATE
+            ir_module_module
+        SET 
+            state='to remove'
+        WHERE
+            state='installed'
+            AND name in %s
+        """,
+        (tuple(obsolete_modules),))
+
[e-mail]()
 def migrate(cr, version):
-    try:
-        
-        log.info("post-set-defaults.py now called")
-        # this method called in a try block too
-        pool = pooler.get_pool(cr.dbname)
-
-        openupgrade.set_defaults(cr, pool, defaults)
-
-        mgr_ir_rule(cr, pool)
-        mgr_res_partner_address(cr, pool)
-        mgr_res_partner(cr, pool)
-        mgr_default_bank(cr, pool)
-        mgr_roles_to_groups(cr, pool)
-        mgr_res_currency(cr, pool)
-        mgr_ir_module_module(cr)
-        mgr_res_users(cr)
-    except Exception, e:
-        log.info("Migration: error in post-set-defaults.py: %s" % e)
-        raise
+    pool = pooler.get_pool(cr.dbname)
+    set_defaults_on_act_window(cr)
+    openupgrade.set_defaults(cr, pool, defaults)
+    mgr_ir_rule(cr, pool)
+    mgr_res_partner_address(cr, pool)
+    mgr_res_partner(cr, pool)
+    mgr_default_bank(cr, pool)
+    mgr_roles_to_groups(cr, pool)
+    mgr_res_currency(cr, pool)
+    mgr_ir_module_module(cr)
+    mgr_res_users(cr)
+    mark_obsolete_modules(cr)
 

=== modified file 'bin/addons/base/migrations/6.0.1.3/pre-migration.py'
--- bin/addons/base/migrations/6.0.1.3/pre-migration.py	2012-01-15 11:33:42 +0000
+++ bin/addons/base/migrations/6.0.1.3/pre-migration.py	2012-06-07 14:50:24 +0000
@@ -3,11 +3,18 @@
 # Removal of modules that are deprecated
 # e.g. report_analytic_line (incorporated in hr_timesheet_invoice)
 
-from osv import osv
 import pooler
 import logging
 from openupgrade import openupgrade
+
 log = logging.getLogger('migrate')
+MODULE = 'base'
+
+module_namespec = [
+    # This is a list of tuples (old module name, new module name)
+    ('profile_association', 'association'),
+    ('report_analytic_planning', 'project_planning'),
+]
 
 renames = {
     # this is a mapping per table from old column name
@@ -27,6 +34,11 @@
         ],
     }
 
+renamed_xmlids = [
+    ('sale.group_sale_manager', 'base.group_sale_manager'),
+    ('sale.group_sale_user', 'base.group_sale_salesman'),
+]
+
 def mgr_ir_model_fields(cr):
     cr.execute('ALTER TABLE ir_model_fields ADD COLUMN selectable BOOLEAN')
     cr.execute('UPDATE ir_model_fields SET selectable = FALSE')
@@ -54,20 +66,18 @@
                "AND ir_model_data.model = 'res.currency.rate' " +
                "AND ir_model_data.name = 'rateINR'")
     if not cr.rowcount:
-        import pdb
-        pdb.set_trace()
         raise osv.except_osv("Migration: error setting INR rate in demo data, no row found", "")
 
[e-mail]()
 def migrate(cr, version):
-    try:
-        # this method called in a try block too
-        log.info("base:pre.py now called")
-        pool = pooler.get_pool(cr.dbname)
-        openupgrade.rename_columns(cr, renames)
-        mgr_ir_model_fields(cr)
-        mgr_company_id(cr)
-        mgr_fix_test_results(cr)
-    except Exception, e:
-        log.info("Migration: error in pre-convert-fields.py: %s", e)
-        osv.except_osv("Migration: error in pre-convert-fields.py: %s" % e, "")
-        raise
+    openupgrade.update_module_names(
+        cr, module_namespec
+        )
+    openupgrade.rename_columns(cr, renames)
+    openupgrade.rename_xmlids(cr, renamed_xmlids)
+    mgr_ir_model_fields(cr)
+    mgr_company_id(cr)
+    mgr_fix_test_results(cr)
+    openupgrade.rename_tables(
+        cr, [('res_partner_function',
+              'openupgrade_legacy_res_partner_function')])

=== modified file 'bin/addons/openupgrade_records/lib/apriori.py'
--- bin/addons/openupgrade_records/lib/apriori.py	2012-04-18 20:20:03 +0000
+++ bin/addons/openupgrade_records/lib/apriori.py	2012-06-07 14:50:24 +0000
@@ -3,7 +3,10 @@
 """
 
 renamed_modules = {
+    'association_profile': 'association',
+    'report_analytic_planning': 'project_planning',
     }
 
 renamed_models = {
+    'mrp.procurement': 'procurement.order',
     }

=== modified file 'bin/openupgrade/openupgrade.py'
--- bin/openupgrade/openupgrade.py	2012-05-06 18:56:17 +0000
+++ bin/openupgrade/openupgrade.py	2012-06-07 14:50:24 +0000
@@ -1,5 +1,26 @@
 # -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    This module copyright (C) 2011-2012 Therp BV (<http://therp.nl>)
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+
 import os
+import inspect
 from osv import osv
 import pooler
 import logging
@@ -9,16 +30,20 @@
 logger = logging.getLogger('OpenUpgrade')
 
 __all__ = [
+    'migrate',
     'load_data',
     'rename_columns',
     'rename_tables',
     'drop_columns',
     'table_exists',
     'column_exists',
+    'logged_query',
     'delete_model_workflow',
     'set_defaults',
     'update_module_names',
     'add_ir_model_fields',
+    'rename_models',
+    'rename_xmlids',
 ]    
 
 def load_data(cr, module_name, filename, idref=None, mode='init'):
@@ -94,7 +119,7 @@
 def rename_models(cr, model_spec):
     """
     Rename models. Typically called in the pre script.
-    :param column_spec: a list of tuples (old table name, new table name).
+    :param column_spec: a list of tuples (old model name, new model name).
     
     Use case: if a model changes name, but still implements equivalent
     functionality you will want to update references in for instance
@@ -106,6 +131,24 @@
                     old, new)
         cr.execute('UPDATE ir_model_fields SET relation = %s '
                    'WHERE relation = %s', (new, old,))
+    # TODO: signal where the model occurs in references to ir_model
+
+def rename_xmlids(cr, xmlids_spec):
+    """
+    Rename XML IDs. Typically called in the pre script.
+    One usage example is when an ID changes module. In OpenERP 6 for example,
+    a number of res_groups IDs moved to module base from other modules (
+    although they were still being defined in their respective module).
+    """
+    for (old, new) in xmlids_spec:
+        if not old.split('.') or not new.split('.'):
+            logger.error(
+            'Cannot rename XMLID %s to %s: need the module '
+            'reference to be specified in the IDs' % (old, new))
+        else:
+            query = ("UPDATE ir_model_data SET module = %s, name = %s "
+                     "WHERE module = %s and name = %s")
+            logged_query(cr, query, tuple(new.split('.') + old.split('.')))
 
 def drop_columns(cr, column_spec):
     """
@@ -161,8 +204,8 @@
     """
 
     def write_value(ids, field, value):
-        logger.info("model %s, field %s: setting default value of %d resources to %s",
-                 model, field, len(ids), unicode(value))
+        logger.debug("model %s, field %s: setting default value of resources %s to %s",
+                 model, field, ids, unicode(value))
         obj.write(cr, 1, ids, {field: value})
 
     for model in default_spec.keys():
@@ -170,40 +213,42 @@
         if not obj:
             raise osv.except_osv("Migration: error setting default, no such model: %s" % model, "")
 
-    for field, value in default_spec[model]:
-        domain = not force and [(field, '=', False)] or []
-        ids = obj.search(cr, 1, domain)
-        if not ids:
-            continue
-        if value is None:
-            # Set the value by calling the _defaults of the object.
-            # Typically used for company_id on various models, and in that
-            # case the result depends on the user associated with the object.
-            # We retrieve create_uid for this purpose and need to call the _defaults
-            # function per resource. Otherwise, write all resources at once.
-            if field in obj._defaults:
-                if not callable(obj._defaults[field]):
-                    write_value(ids, field, obj._defaults[field])
+        for field, value in default_spec[model]:
+            domain = not force and [(field, '=', False)] or []
+            ids = obj.search(cr, 1, domain)
+            if not ids:
+                continue
+            if value is None:
+                # Set the value by calling the _defaults of the object.
+                # Typically used for company_id on various models, and in that
+                # case the result depends on the user associated with the object.
+                # We retrieve create_uid for this purpose and need to call the _defaults
+                # function per resource. Otherwise, write all resources at once.
+                if field in obj._defaults:
+                    if not callable(obj._defaults[field]):
+                        write_value(ids, field, obj._defaults[field])
+                    else:
+                        # existence users is covered by foreign keys, so this is not needed
+                        # cr.execute("SELECT %s.id, res_users.id FROM %s LEFT OUTER JOIN res_users ON (%s.create_uid = res_users.id) WHERE %s.id IN %s" %
+                        #                     (obj._table, obj._table, obj._table, obj._table, tuple(ids),))
+                        cr.execute("SELECT id, COALESCE(create_uid, 1) FROM %s " % obj._table + "WHERE id in %s", (tuple(ids),))
+                        # Execute the function once per user_id
+                        user_id_map = {}
+                        for row in cr.fetchall():
+                            user_id_map.setdefault(row[1], []).append(row[0])
+                        for user_id in user_id_map:
+                            write_value(
+                                user_id_map[user_id], field,
+                                obj._defaults[field](obj, cr, user_id, None))
                 else:
-                    # existence users is covered by foreign keys, so this is not needed
-                    # cr.execute("SELECT %s.id, res_users.id FROM %s LEFT OUTER JOIN res_users ON (%s.create_uid = res_users.id) WHERE %s.id IN %s" %
-                    #                     (obj._table, obj._table, obj._table, obj._table, tuple(ids),))
-                    cr.execute("SELECT id, COALESCE(create_uid, 1) FROM %s " % obj._table + "WHERE id in %s", (tuple(ids),))
-                    fetchdict = dict(cr.fetchall())
-                    for id in ids:
-                        write_value([id], field, obj._defaults[field](obj, cr, fetchdict.get(id, 1), None))
-                        if id not in fetchdict:
-                            logger.info("model %s, field %s, id %d: no create_uid defined or user does not exist anymore",
-                                     model, field, id)
+                    error = ("OpenUpgrade: error setting default, field %s with "
+                             "None default value not in %s' _defaults" % (
+                            field, model))
+                    logger.error(error)
+                    # this exeption seems to get lost in a higher up try block
+                    osv.except_osv("OpenUpgrade", error)
             else:
-                error = ("OpenUpgrade: error setting default, field %s with "
-                         "None default value not in %s' _defaults" % (
-                        field, model))
-                logger.error(error)
-                # this exeption seems to get lost in a higher up try block
-                osv.except_osv("OpenUpgrade", error)
-        else:
-            write_value(ids, field, value)
+                write_value(ids, field, value)
     
 def logged_query(cr, query, args=None):
     if args is None:
@@ -257,3 +302,63 @@
         query = 'ALTER TABLE ir_model_fields ADD COLUMN %s %s' % (
             column)
         logged_query(cr, query, [])
+        
+def add_module_dependencies(cr, module_list):
+    """
+    Select (new) dependencies from the modules in the list
+    so that we can inject them into the graph at upgrade
+    time. Used in the modified OpenUpgrade Server,
+    not to be used in migration scripts
+    """
+    if not module_list:
+        return module_list
+    cr.execute("""
+        SELECT ir_module_module_dependency.name
+        FROM
+            ir_module_module,
+            ir_module_module_dependency
+        WHERE
+            module_id = ir_module_module.id
+            AND ir_module_module.name in %s
+        """, (tuple(module_list),))
+    dependencies = [x[0] for x in cr.fetchall()]
+    return list(set(module_list + dependencies))
+
+def migrate():
+    """
+    This is the decorator for the migrate() function
+    in migration scripts.
+    Return when the 'version' argument is not defined,
+    and log execeptions.
+    Retrieve debug context data from the frame above for
+    logging purposes.
+    """
+    def wrap(func):
+        def wrapped_function(cr, version):
+            stage =  'unknown'
+            module = 'unknown'
+            filename = 'unknown'
+            try:
+                frame = inspect.getargvalues(inspect.stack()[1][0])
+                stage = frame.locals['stage']
+                module = frame.locals['pkg'].name
+                filename = frame.locals['fp'].name
+            except Exception, e:
+                logger.error(
+                    "'migrate' decorator: failed to inspect "
+                    "the frame above: %s" % e)
+                pass
+            if not version:
+                return
+            logger.info(
+                "%s: %s-migration script called with version %s" %
+                (module, stage, version))
+            try:
+                # The actual function is called here
+                func(cr, version)
+            except Exception, e:
+                logger.error(
+                    "%s: error in migration script %s: %s" % 
+                    (module, filename, e))
+        return wrapped_function
+    return wrap