Merge lp:~therp-nl/openupgrade-server/6.0-API_and_loading_improvements into lp:openupgrade-server/6.0

Proposed by Stefan Rijnhart (Opener) on 2012-06-07
Status: Merged
Merged at revision: 3489
Proposed branch: lp:~therp-nl/openupgrade-server/6.0-API_and_loading_improvements
Merge into: lp:openupgrade-server/6.0
Diff against target: 580 lines (+268/-102)
5 files modified
bin/addons/__init__.py (+12/-15)
bin/addons/base/migrations/6.0.1.3/post-migration.py (+88/-37)
bin/addons/base/migrations/6.0.1.3/pre-migration.py (+25/-15)
bin/addons/openupgrade_records/lib/apriori.py (+3/-0)
bin/openupgrade/openupgrade.py (+140/-35)
To merge this branch: bzr merge lp:~therp-nl/openupgrade-server/6.0-API_and_loading_improvements
Reviewer Review Type Date Requested Status
OpenUpgrade Committers 2012-06-07 Pending
Review via email: mp+109162@code.launchpad.net

Description of the change

This branch ensures that pre and post scripts are only called once. This means that developers do not have to bother checking whether they are being rerun in the scripts themselves. It is now also ensured that new dependencies are installed automatically. Modules that are known to be obsolete will now be removed automatically.

The API is augmented with a decorator that can be used in migration scripts to log any errors that may occur. A new function for renaming XML IDs is provided, and setting defaults provided by a function is faster.

Some bugs in the base module migration scripts are solved.

To post a comment you must log in.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'bin/addons/__init__.py'
2--- bin/addons/__init__.py 2012-05-27 12:34:50 +0000
3+++ bin/addons/__init__.py 2012-06-07 14:50:24 +0000
4@@ -50,14 +50,7 @@
5
6 logger = netsvc.Logger()
7
8-### OpenUpgrade
9-def table_exists(cr, table):
10- """ Check whether a certain table or view exists """
11- cr.execute(
12- 'SELECT count(relname) FROM pg_class WHERE relname = %s',
13- (table,))
14- return cr.fetchone()[0] == 1
15-### End of OpenUpgrade
16+from openupgrade import openupgrade
17
18 _ad = os.path.abspath(opj(tools.ustr(tools.config['root_path']), u'addons')) # default addons path (base)
19 ad_paths= map(lambda m: os.path.abspath(tools.ustr(m.strip())), tools.config['addons_path'].split(','))
20@@ -808,7 +801,7 @@
21 log any differences and merge the local registry with
22 the global one.
23 """
24- if not table_exists(cr, 'openupgrade_record'):
25+ if not openupgrade.table_exists(cr, 'openupgrade_record'):
26 return
27 for model, fields in local_registry.items():
28 registry.setdefault(model, {})
29@@ -848,10 +841,9 @@
30 logger.notifyChannel('init', netsvc.LOG_DEBUG, 'loading %d packages..' % len(graph))
31
32 for package in graph:
33- if skip_modules and package.name in skip_modules:
34- continue
35 logger.notifyChannel('init', netsvc.LOG_INFO, 'module %s: loading objects' % package.name)
36- migrations.migrate_module(package, 'pre')
37+ if package.name not in skip_modules['pre']:
38+ migrations.migrate_module(package, 'pre')
39 register_class(package.name)
40 modules = pool.instanciate(package.name, cr)
41
42@@ -864,13 +856,14 @@
43
44 init_module_objects(cr, package.name, modules)
45 cr.commit()
46+ skip_modules['pre'].append(package.name)
47
48 for package in graph:
49 status['progress'] = (float(statusi)+0.1) / len(graph)
50 m = package.name
51 mid = package.id
52
53- if skip_modules and m in skip_modules:
54+ if package.name in skip_modules['post']:
55 continue
56
57 if modobj is None:
58@@ -923,6 +916,7 @@
59 if hasattr(package, kind):
60 delattr(package, kind)
61
62+ skip_modules['post'].append(package.name)
63 statusi += 1
64 cr.commit()
65
66@@ -964,6 +958,7 @@
67
68 registry = {}
69
70+ skip_modules = {'pre': [], 'post': []}
71 try:
72 processed_modules = []
73 report = tools.assertion_report()
74@@ -977,7 +972,7 @@
75 if not graph:
76 logger.notifyChannel('init', netsvc.LOG_CRITICAL, 'module base cannot be loaded! (hint: verify addons-path)')
77 raise osv.osv.except_osv(_('Could not load base module'), _('module base cannot be loaded! (hint: verify addons-path)'))
78- processed_modules.extend(load_module_graph(cr, graph, status, perform_checks=(not update_module), registry=registry, report=report))
79+ processed_modules.extend(load_module_graph(cr, graph, status, perform_checks=(not update_module), registry=registry, report=report, skip_modules=skip_modules))
80
81 if tools.config['load_language']:
82 for lang in tools.config['load_language'].split(','):
83@@ -1021,13 +1016,15 @@
84 if not module_list:
85 break
86
87+ # OpenUpgrade: forcefeeding module dependencies into the graph
88+ module_list = openupgrade.add_module_dependencies(cr, module_list)
89 new_modules_in_graph = upgrade_graph(graph, cr, module_list, force)
90 if new_modules_in_graph == 0:
91 # nothing to load
92 break
93
94 logger.notifyChannel('init', netsvc.LOG_DEBUG, 'Updating graph with %d more modules' % (len(module_list)))
95- processed_modules.extend(load_module_graph(cr, graph, status, registry=registry, report=report, skip_modules=processed_modules))
96+ processed_modules.extend(load_module_graph(cr, graph, status, registry=registry, report=report, skip_modules=skip_modules))
97
98 # load custom models
99 cr.execute('select model from ir_model where state=%s', ('manual',))
100
101=== modified file 'bin/addons/base/migrations/6.0.1.3/post-migration.py'
102--- bin/addons/base/migrations/6.0.1.3/post-migration.py 2011-11-30 22:51:16 +0000
103+++ bin/addons/base/migrations/6.0.1.3/post-migration.py 2012-06-07 14:50:24 +0000
104@@ -1,18 +1,58 @@
105 # -*- coding: utf-8 -*-
106
107-from osv import osv
108 import pooler, logging
109 from openupgrade import openupgrade
110-log = logging.getLogger('migrate')
111+logger = logging.getLogger('migrate')
112+MODULE="base"
113+
114+obsolete_modules = [
115+ 'account_report',
116+ 'account_reporting',
117+ 'account_tax_include',
118+ 'board_account',
119+ 'board_sale',
120+ 'report_account',
121+ 'report_analytic',
122+ 'report_analytic_line',
123+ 'report_crm',
124+ 'report_purchase',
125+ 'report_sale',
126+ 'pxgo_bank_statement_analytic',
127+]
128+
129+def set_defaults_on_act_window(cr):
130+ """ The act window model has a constraint
131+ that checks the validity of the model on it.
132+ At migration time, this is inconvenient.
133+ Therefore, we set the defaults through SQL
134+
135+ Replaces setting the following defaults
136+ #'ir.actions.act_window': [
137+ # ('auto_search', True),
138+ # ('context', '{}'),
139+ # ('multi', False),
140+ # ],
141+
142+ """
143+ cr.execute("""
144+ UPDATE ir_act_window
145+ SET auto_search = true,
146+ multi = false
147+ """)
148+ cr.execute("""
149+ UPDATE ir_act_window
150+ SET context = '{}'
151+ WHERE context is NULL
152+ """)
153+ cr.execute("""
154+ UPDATE ir_act_window
155+ SET context = '{}'
156+ WHERE context is NULL
157+ """)
158
159 defaults = {
160 # False results in column value NULL
161 # None value triggers a call to the model's default function
162- 'ir.actions.act_window': [
163- ('auto_search', True),
164- ('context', '{}'),
165- ('multi', False),
166- ],
167 'ir.actions.todo': [
168 ('restart', 'onskip'),
169 ],
170@@ -96,13 +136,14 @@
171 # In the pre script, we renamed the old function column, a many2one
172 # to res.partner.function
173 cr.execute(
174- "UPDATE res_partner_address SET function = res_partner_function.name " +
175- "FROM res_partner_function " +
176- "WHERE res_partner_function.id = res_partner_address.tmp_mgr_function"
177- )
178- cr.execute("ALTER TABLE res_partner_address DROP COLUMN tmp_mgr_function CASCADE")
179- cr.execute("DROP TABLE res_partner_function")
180-
181+ "UPDATE res_partner_address "
182+ "SET function = openupgrade_legacy_res_partner_function.name "
183+ "FROM openupgrade_legacy_res_partner_function "
184+ "WHERE openupgrade_legacy_res_partner_function.id "
185+ "= res_partner_address.tmp_mgr_function")
186+ cr.execute(
187+ "ALTER TABLE res_partner_address "
188+ "DROP COLUMN tmp_mgr_function CASCADE")
189 # and the reverse: 'title' is now a many2one on res_partner_title
190 cr.execute(
191 "SELECT id, tmp_mgr_title FROM res_partner_address WHERE title IS NULL " +
192@@ -139,10 +180,9 @@
193 if bank_ids:
194 bank_id = bank_ids[0]
195 else:
196- bank_id = bank_obj.create(cursor, uid, dict(
197- code = info.code or 'UNKNOW',
198- name = info.name or _('Unknown Bank'),
199- ))
200+ bank_id = bank_obj.create(cr, 1, dict(
201+ code = 'UNKNOW',
202+ name = 'Unknown Bank')),
203 partner_bank_obj.write(cr, 1, partner_bank_ids, {'bank': bank_id})
204
205 def mgr_roles_to_groups(cr, pool):
206@@ -208,24 +248,35 @@
207 "AND ir_actions.usage = 'menu'"
208 )
209
210+def mark_obsolete_modules(cr):
211+ """
212+ Remove modules that are known to be obsolete
213+ in this version of the OpenERP server.
214+ """
215+ openupgrade.logged_query(
216+ cr, """
217+ UPDATE
218+ ir_module_module
219+ SET
220+ state='to remove'
221+ WHERE
222+ state='installed'
223+ AND name in %s
224+ """,
225+ (tuple(obsolete_modules),))
226+
227+@openupgrade.migrate()
228 def migrate(cr, version):
229- try:
230-
231- log.info("post-set-defaults.py now called")
232- # this method called in a try block too
233- pool = pooler.get_pool(cr.dbname)
234-
235- openupgrade.set_defaults(cr, pool, defaults)
236-
237- mgr_ir_rule(cr, pool)
238- mgr_res_partner_address(cr, pool)
239- mgr_res_partner(cr, pool)
240- mgr_default_bank(cr, pool)
241- mgr_roles_to_groups(cr, pool)
242- mgr_res_currency(cr, pool)
243- mgr_ir_module_module(cr)
244- mgr_res_users(cr)
245- except Exception, e:
246- log.info("Migration: error in post-set-defaults.py: %s" % e)
247- raise
248+ pool = pooler.get_pool(cr.dbname)
249+ set_defaults_on_act_window(cr)
250+ openupgrade.set_defaults(cr, pool, defaults)
251+ mgr_ir_rule(cr, pool)
252+ mgr_res_partner_address(cr, pool)
253+ mgr_res_partner(cr, pool)
254+ mgr_default_bank(cr, pool)
255+ mgr_roles_to_groups(cr, pool)
256+ mgr_res_currency(cr, pool)
257+ mgr_ir_module_module(cr)
258+ mgr_res_users(cr)
259+ mark_obsolete_modules(cr)
260
261
262=== modified file 'bin/addons/base/migrations/6.0.1.3/pre-migration.py'
263--- bin/addons/base/migrations/6.0.1.3/pre-migration.py 2012-01-15 11:33:42 +0000
264+++ bin/addons/base/migrations/6.0.1.3/pre-migration.py 2012-06-07 14:50:24 +0000
265@@ -3,11 +3,18 @@
266 # Removal of modules that are deprecated
267 # e.g. report_analytic_line (incorporated in hr_timesheet_invoice)
268
269-from osv import osv
270 import pooler
271 import logging
272 from openupgrade import openupgrade
273+
274 log = logging.getLogger('migrate')
275+MODULE = 'base'
276+
277+module_namespec = [
278+ # This is a list of tuples (old module name, new module name)
279+ ('profile_association', 'association'),
280+ ('report_analytic_planning', 'project_planning'),
281+]
282
283 renames = {
284 # this is a mapping per table from old column name
285@@ -27,6 +34,11 @@
286 ],
287 }
288
289+renamed_xmlids = [
290+ ('sale.group_sale_manager', 'base.group_sale_manager'),
291+ ('sale.group_sale_user', 'base.group_sale_salesman'),
292+]
293+
294 def mgr_ir_model_fields(cr):
295 cr.execute('ALTER TABLE ir_model_fields ADD COLUMN selectable BOOLEAN')
296 cr.execute('UPDATE ir_model_fields SET selectable = FALSE')
297@@ -54,20 +66,18 @@
298 "AND ir_model_data.model = 'res.currency.rate' " +
299 "AND ir_model_data.name = 'rateINR'")
300 if not cr.rowcount:
301- import pdb
302- pdb.set_trace()
303 raise osv.except_osv("Migration: error setting INR rate in demo data, no row found", "")
304
305+@openupgrade.migrate()
306 def migrate(cr, version):
307- try:
308- # this method called in a try block too
309- log.info("base:pre.py now called")
310- pool = pooler.get_pool(cr.dbname)
311- openupgrade.rename_columns(cr, renames)
312- mgr_ir_model_fields(cr)
313- mgr_company_id(cr)
314- mgr_fix_test_results(cr)
315- except Exception, e:
316- log.info("Migration: error in pre-convert-fields.py: %s", e)
317- osv.except_osv("Migration: error in pre-convert-fields.py: %s" % e, "")
318- raise
319+ openupgrade.update_module_names(
320+ cr, module_namespec
321+ )
322+ openupgrade.rename_columns(cr, renames)
323+ openupgrade.rename_xmlids(cr, renamed_xmlids)
324+ mgr_ir_model_fields(cr)
325+ mgr_company_id(cr)
326+ mgr_fix_test_results(cr)
327+ openupgrade.rename_tables(
328+ cr, [('res_partner_function',
329+ 'openupgrade_legacy_res_partner_function')])
330
331=== modified file 'bin/addons/openupgrade_records/lib/apriori.py'
332--- bin/addons/openupgrade_records/lib/apriori.py 2012-04-18 20:20:03 +0000
333+++ bin/addons/openupgrade_records/lib/apriori.py 2012-06-07 14:50:24 +0000
334@@ -3,7 +3,10 @@
335 """
336
337 renamed_modules = {
338+ 'association_profile': 'association',
339+ 'report_analytic_planning': 'project_planning',
340 }
341
342 renamed_models = {
343+ 'mrp.procurement': 'procurement.order',
344 }
345
346=== modified file 'bin/openupgrade/openupgrade.py'
347--- bin/openupgrade/openupgrade.py 2012-05-06 18:56:17 +0000
348+++ bin/openupgrade/openupgrade.py 2012-06-07 14:50:24 +0000
349@@ -1,5 +1,26 @@
350 # -*- coding: utf-8 -*-
351+##############################################################################
352+#
353+# OpenERP, Open Source Management Solution
354+# This module copyright (C) 2011-2012 Therp BV (<http://therp.nl>)
355+#
356+# This program is free software: you can redistribute it and/or modify
357+# it under the terms of the GNU Affero General Public License as
358+# published by the Free Software Foundation, either version 3 of the
359+# License, or (at your option) any later version.
360+#
361+# This program is distributed in the hope that it will be useful,
362+# but WITHOUT ANY WARRANTY; without even the implied warranty of
363+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
364+# GNU Affero General Public License for more details.
365+#
366+# You should have received a copy of the GNU Affero General Public License
367+# along with this program. If not, see <http://www.gnu.org/licenses/>.
368+#
369+##############################################################################
370+
371 import os
372+import inspect
373 from osv import osv
374 import pooler
375 import logging
376@@ -9,16 +30,20 @@
377 logger = logging.getLogger('OpenUpgrade')
378
379 __all__ = [
380+ 'migrate',
381 'load_data',
382 'rename_columns',
383 'rename_tables',
384 'drop_columns',
385 'table_exists',
386 'column_exists',
387+ 'logged_query',
388 'delete_model_workflow',
389 'set_defaults',
390 'update_module_names',
391 'add_ir_model_fields',
392+ 'rename_models',
393+ 'rename_xmlids',
394 ]
395
396 def load_data(cr, module_name, filename, idref=None, mode='init'):
397@@ -94,7 +119,7 @@
398 def rename_models(cr, model_spec):
399 """
400 Rename models. Typically called in the pre script.
401- :param column_spec: a list of tuples (old table name, new table name).
402+ :param column_spec: a list of tuples (old model name, new model name).
403
404 Use case: if a model changes name, but still implements equivalent
405 functionality you will want to update references in for instance
406@@ -106,6 +131,24 @@
407 old, new)
408 cr.execute('UPDATE ir_model_fields SET relation = %s '
409 'WHERE relation = %s', (new, old,))
410+ # TODO: signal where the model occurs in references to ir_model
411+
412+def rename_xmlids(cr, xmlids_spec):
413+ """
414+ Rename XML IDs. Typically called in the pre script.
415+ One usage example is when an ID changes module. In OpenERP 6 for example,
416+ a number of res_groups IDs moved to module base from other modules (
417+ although they were still being defined in their respective module).
418+ """
419+ for (old, new) in xmlids_spec:
420+ if not old.split('.') or not new.split('.'):
421+ logger.error(
422+ 'Cannot rename XMLID %s to %s: need the module '
423+ 'reference to be specified in the IDs' % (old, new))
424+ else:
425+ query = ("UPDATE ir_model_data SET module = %s, name = %s "
426+ "WHERE module = %s and name = %s")
427+ logged_query(cr, query, tuple(new.split('.') + old.split('.')))
428
429 def drop_columns(cr, column_spec):
430 """
431@@ -161,8 +204,8 @@
432 """
433
434 def write_value(ids, field, value):
435- logger.info("model %s, field %s: setting default value of %d resources to %s",
436- model, field, len(ids), unicode(value))
437+ logger.debug("model %s, field %s: setting default value of resources %s to %s",
438+ model, field, ids, unicode(value))
439 obj.write(cr, 1, ids, {field: value})
440
441 for model in default_spec.keys():
442@@ -170,40 +213,42 @@
443 if not obj:
444 raise osv.except_osv("Migration: error setting default, no such model: %s" % model, "")
445
446- for field, value in default_spec[model]:
447- domain = not force and [(field, '=', False)] or []
448- ids = obj.search(cr, 1, domain)
449- if not ids:
450- continue
451- if value is None:
452- # Set the value by calling the _defaults of the object.
453- # Typically used for company_id on various models, and in that
454- # case the result depends on the user associated with the object.
455- # We retrieve create_uid for this purpose and need to call the _defaults
456- # function per resource. Otherwise, write all resources at once.
457- if field in obj._defaults:
458- if not callable(obj._defaults[field]):
459- write_value(ids, field, obj._defaults[field])
460+ for field, value in default_spec[model]:
461+ domain = not force and [(field, '=', False)] or []
462+ ids = obj.search(cr, 1, domain)
463+ if not ids:
464+ continue
465+ if value is None:
466+ # Set the value by calling the _defaults of the object.
467+ # Typically used for company_id on various models, and in that
468+ # case the result depends on the user associated with the object.
469+ # We retrieve create_uid for this purpose and need to call the _defaults
470+ # function per resource. Otherwise, write all resources at once.
471+ if field in obj._defaults:
472+ if not callable(obj._defaults[field]):
473+ write_value(ids, field, obj._defaults[field])
474+ else:
475+ # existence users is covered by foreign keys, so this is not needed
476+ # cr.execute("SELECT %s.id, res_users.id FROM %s LEFT OUTER JOIN res_users ON (%s.create_uid = res_users.id) WHERE %s.id IN %s" %
477+ # (obj._table, obj._table, obj._table, obj._table, tuple(ids),))
478+ cr.execute("SELECT id, COALESCE(create_uid, 1) FROM %s " % obj._table + "WHERE id in %s", (tuple(ids),))
479+ # Execute the function once per user_id
480+ user_id_map = {}
481+ for row in cr.fetchall():
482+ user_id_map.setdefault(row[1], []).append(row[0])
483+ for user_id in user_id_map:
484+ write_value(
485+ user_id_map[user_id], field,
486+ obj._defaults[field](obj, cr, user_id, None))
487 else:
488- # existence users is covered by foreign keys, so this is not needed
489- # cr.execute("SELECT %s.id, res_users.id FROM %s LEFT OUTER JOIN res_users ON (%s.create_uid = res_users.id) WHERE %s.id IN %s" %
490- # (obj._table, obj._table, obj._table, obj._table, tuple(ids),))
491- cr.execute("SELECT id, COALESCE(create_uid, 1) FROM %s " % obj._table + "WHERE id in %s", (tuple(ids),))
492- fetchdict = dict(cr.fetchall())
493- for id in ids:
494- write_value([id], field, obj._defaults[field](obj, cr, fetchdict.get(id, 1), None))
495- if id not in fetchdict:
496- logger.info("model %s, field %s, id %d: no create_uid defined or user does not exist anymore",
497- model, field, id)
498+ error = ("OpenUpgrade: error setting default, field %s with "
499+ "None default value not in %s' _defaults" % (
500+ field, model))
501+ logger.error(error)
502+ # this exeption seems to get lost in a higher up try block
503+ osv.except_osv("OpenUpgrade", error)
504 else:
505- error = ("OpenUpgrade: error setting default, field %s with "
506- "None default value not in %s' _defaults" % (
507- field, model))
508- logger.error(error)
509- # this exeption seems to get lost in a higher up try block
510- osv.except_osv("OpenUpgrade", error)
511- else:
512- write_value(ids, field, value)
513+ write_value(ids, field, value)
514
515 def logged_query(cr, query, args=None):
516 if args is None:
517@@ -257,3 +302,63 @@
518 query = 'ALTER TABLE ir_model_fields ADD COLUMN %s %s' % (
519 column)
520 logged_query(cr, query, [])
521+
522+def add_module_dependencies(cr, module_list):
523+ """
524+ Select (new) dependencies from the modules in the list
525+ so that we can inject them into the graph at upgrade
526+ time. Used in the modified OpenUpgrade Server,
527+ not to be used in migration scripts
528+ """
529+ if not module_list:
530+ return module_list
531+ cr.execute("""
532+ SELECT ir_module_module_dependency.name
533+ FROM
534+ ir_module_module,
535+ ir_module_module_dependency
536+ WHERE
537+ module_id = ir_module_module.id
538+ AND ir_module_module.name in %s
539+ """, (tuple(module_list),))
540+ dependencies = [x[0] for x in cr.fetchall()]
541+ return list(set(module_list + dependencies))
542+
543+def migrate():
544+ """
545+ This is the decorator for the migrate() function
546+ in migration scripts.
547+ Return when the 'version' argument is not defined,
548+ and log execeptions.
549+ Retrieve debug context data from the frame above for
550+ logging purposes.
551+ """
552+ def wrap(func):
553+ def wrapped_function(cr, version):
554+ stage = 'unknown'
555+ module = 'unknown'
556+ filename = 'unknown'
557+ try:
558+ frame = inspect.getargvalues(inspect.stack()[1][0])
559+ stage = frame.locals['stage']
560+ module = frame.locals['pkg'].name
561+ filename = frame.locals['fp'].name
562+ except Exception, e:
563+ logger.error(
564+ "'migrate' decorator: failed to inspect "
565+ "the frame above: %s" % e)
566+ pass
567+ if not version:
568+ return
569+ logger.info(
570+ "%s: %s-migration script called with version %s" %
571+ (module, stage, version))
572+ try:
573+ # The actual function is called here
574+ func(cr, version)
575+ except Exception, e:
576+ logger.error(
577+ "%s: error in migration script %s: %s" %
578+ (module, filename, e))
579+ return wrapped_function
580+ return wrap

Subscribers

People subscribed via source and target branches