Merge lp:~fabien-morin/unifield-server/fm-us-1719 into lp:unifield-server

[jftempo]

=== modified file 'bin/addons/msf_profile/data/patches.xml'
--- bin/addons/msf_profile/data/patches.xml	2017-02-09 17:18:36 +0000
+++ bin/addons/msf_profile/data/patches.xml	2017-02-16 09:58:42 +0000
@@ -1,6 +1,10 @@
 <?xml version="1.0" encoding="utf-8" ?>
 <openerp>
     <data>
+        <record id="remove_not_synchronized_data" model="patch.scripts">
+            <field name="method">remove_not_synchronized_data</field>
+        </record>
+
         <record id="us_2110_patch" model="patch.scripts">
             <field name="method">us_2110_patch</field>
         </record>

=== modified file 'bin/addons/msf_profile/msf_profile.py'
--- bin/addons/msf_profile/msf_profile.py	2017-02-13 11:31:10 +0000
+++ bin/addons/msf_profile/msf_profile.py	2017-02-16 09:58:42 +0000
@@ -46,6 +46,50 @@
         'model': lambda *a: 'patch.scripts',
     }
 
+    def remove_not_synchronized_data(self, cr, uid, *a, **b):
+        '''
+        The list of models to synchronize was wrong. It is now build
+        automatically and is then more exact.
+        This patch will remove all the data from ir_model_data that are not
+        synchronized models.
+        '''
+        from sync_common import WHITE_LIST_MODEL
+        removed_obj = 0
+
+        # if sync_client module is installed, get the list of synchronized models
+        if self.pool.get('sync.client.rule') and\
+                self.pool.get('sync.client.message_rule'):
+            entity_obj = self.pool.get('sync.client.entity')
+            server_model_white_set = entity_obj.get_model_white_list(cr, uid)
+
+            # check that all models from the newly generated list are in the hardcoded white list
+            difference = server_model_white_set.difference(WHITE_LIST_MODEL)
+            if difference:
+                err_msg = 'Warning: Some models used in the synchronization '\
+                'rule are not present in the WHITE_LIST_MODEL: %s'
+                self._logger.error(err_msg)
+                raise osv.except_osv(
+                    'Error',
+                    err_msg,
+                )
+            elif server_model_white_set:
+                # get list of all existing models used in ir_model_data
+                cr.execute('SELECT DISTINCT(model) FROM ir_model_data')
+                model_list = [x and x[0] for x in cr.fetchall()]
+                model_to_remove = (set(model_list).difference(server_model_white_set))
+                import pprint
+                pp = pprint.PrettyPrinter(indent=2)
+                model_to_remove_pp = pp.pformat(model_to_remove)
+                self._logger.warn('%s models should not be part of ir_model_data.' % len(model_to_remove))
+                self._logger.warn('The objects linked to the model(s) %s will be removed from ir_model_data.' % model_to_remove_pp)
+
+                for model in model_to_remove:
+                    cr.execute("DELETE FROM ir_model_data WHERE model='%s' AND module='sd'" % model)
+                    current_count = cr.rowcount
+                    removed_obj += current_count
+                    self._logger.warn('ir.model.data, model=%s, %s objects deleted.' % (model, current_count))
+                self._logger.warn('ir.model.data, total of %s objects deleted.' % removed_obj)
+
     def us_1613_remove_all_track_changes_action(self, cr, uid, *a, **b):
         '''
         each time the msf_audittrail is updated, the subscribe() method is
@@ -126,25 +170,31 @@
     def setup_security_on_sync_server(self, cr, uid, *a, **b):
         update_module = self.pool.get('sync.server.update')
         if not update_module:
-            # this script is exucuted on server side, update the first delete
+            # this script is exucuted on server side only
             return
 
         data_obj = self.pool.get('ir.model.data')
-        group_id = data_obj.get_object_reference(cr, uid, 'base',
-                                                 'group_erp_manager')[1]
-
-        model_obj = self.pool.get('ir.model')
-        model_list_not_to_change = ['res.users', 'res.lang', 'res.widget',
-                                    'res.widget.user', 'res.log', 'publisher_warranty.contract',
-                                    'module.module']
-        model_ids = model_obj.search(cr, uid,
-                                     [('model', 'not like', "ir%"),
-                                      ('model', 'not in', model_list_not_to_change)])
-
-        access_obj = self.pool.get('ir.model.access')
-        no_group_access = access_obj.search(cr, uid, [('group_id', '=', False),
-                                                      ('model_id', 'in', model_ids)])
-        access_obj.write(cr, uid, no_group_access, {'group_id': group_id})
+        group_id = None
+        try:
+            group_id = data_obj.get_object_reference(cr, 1, 'base',
+                    'group_erp_manager')[1]
+        except ValueError:
+            # If these groups does not exists anymore
+            pass
+
+        if group_id:
+            model_obj = self.pool.get('ir.model')
+            model_list_not_to_change = ['res.users', 'res.lang', 'res.widget',
+                                        'res.widget.user', 'res.log', 'publisher_warranty.contract',
+                                        'module.module']
+            model_ids = model_obj.search(cr, uid,
+                                         [('model', 'not like', "ir%"),
+                                          ('model', 'not in', model_list_not_to_change)])
+
+            access_obj = self.pool.get('ir.model.access')
+            no_group_access = access_obj.search(cr, uid, [('group_id', '=', False),
+                                                          ('model_id', 'in', model_ids)])
+            access_obj.write(cr, uid, no_group_access, {'group_id': group_id})
 
     def us_1482_fix_default_code_on_msf_lines(self, cr, uid, *a, **b):
         """
@@ -185,6 +235,8 @@
                                 {'password': encrypted_password})
 
     def us_1610_set_oc_on_all_groups(self, cr, uid, *a, **b):
+        from sync_common import OC_LIST
+        lower_oc_list = [x.lower() for x in OC_LIST]
         logger = logging.getLogger('update')
         update_module = self.pool.get('sync.server.entity_group')
         if update_module:
@@ -196,7 +248,11 @@
                 if 'oc' in group_name:
                     index = group_name.index('oc')
                     oc = group_name[index:index+3]
-                    update_module.write(cr, uid, group['id'], {'oc': oc})
+                    if oc in lower_oc_list:
+                        update_module.write(cr, uid, group['id'], {'oc': oc})
+                    else:
+                        logger.warn("""OC = %s from group '%s' is not in the OC_LIST, please fix
+                                mannualy""" % (oc, group['name']))
                 else:
                     logger.warn('sync.server.entity_group "%s" does not contain '\
                                 '"oc" or "OC" in its name. Please set up the '\
@@ -213,7 +269,11 @@
                 if 'oc' in entity_name:
                     index = entity_name.index('oc')
                     oc = entity_name[index:index+3]
-                    sync_client_module.write(cr, uid, entity['id'], {'oc': oc})
+                    if oc in lower_oc_list:
+                        sync_client_module.write(cr, uid, entity['id'], {'oc': oc})
+                    else:
+                        logger.warn("""OC = %s from group '%s' is not in the OC_LIST, please fix
+                                mannualy""" % (oc, group['name']))
                 else:
                     logger.warn('sync.client.entity "%s" does not contain '\
                                 '"oc" or "OC" in its name. Please set up the '\

=== modified file 'bin/addons/sync_client/ir_model_data.py'
--- bin/addons/sync_client/ir_model_data.py	2016-11-09 09:40:24 +0000
+++ bin/addons/sync_client/ir_model_data.py	2017-02-16 09:58:42 +0000
@@ -24,7 +24,7 @@
 from osv import osv, fields
 import tools
 
-from sync_common import MODELS_TO_IGNORE, MODELS_TO_IGNORE_DOMAIN, normalize_sdref
+from sync_common import WHITE_LIST_MODEL, normalize_sdref
 
 
 class ir_module_module(osv.osv):
@@ -98,16 +98,10 @@
         """
         # loop on objects that don't match the models to ignore domain in sync common
         result = set()
-        ir_model = self.pool.get('ir.model')
-        model_ids = ir_model.search(cr, 1, MODELS_TO_IGNORE_DOMAIN)
-
-        for model in filter(lambda m:m.model not in MODELS_TO_IGNORE,
-                            ir_model.browse(cr, 1, model_ids)):
-
-            obj = self.pool.get(model.model)
-
+        for model in WHITE_LIST_MODEL:
+            obj = self.pool.get(model)
             if obj is None:
-                self._logger.warn('Could not get object %s while creating all missing sdrefs' % model.model)
+                self._logger.warn('Could not get object %s while creating all missing sdrefs' % model)
                 continue
 
             # ignore wizard objects
@@ -251,7 +245,7 @@
 
         # when a module load a specific xmlid, the sdref is updated according
         # that xmlid
-        if values['model'] not in MODELS_TO_IGNORE and \
+        if values['model'] in WHITE_LIST_MODEL and \
            values['module'] not in ('sd', '__export__') and \
            not (values['module'] == 'base' and values['name'].startswith('main_')):
             sdref_name = "%(module)s_%(name)s" % values

=== modified file 'bin/addons/sync_client/orm.py'
--- bin/addons/sync_client/orm.py	2016-08-23 12:54:03 +0000
+++ bin/addons/sync_client/orm.py	2017-02-16 09:58:42 +0000
@@ -8,7 +8,7 @@
 import types
 from datetime import date, datetime
 
-from sync_common import MODELS_TO_IGNORE, xmlid_to_sdref
+from sync_common import WHITE_LIST_MODEL, xmlid_to_sdref
 
 #import cProfile
 ## Helpers ###################################################################
@@ -489,7 +489,7 @@
             funct_field = audit_obj.get_functionnal_fields(cr, 1, self._name, audit_rule_ids)
 
         to_be_synchronized = (
-            self._name not in MODELS_TO_IGNORE and
+            self._name in WHITE_LIST_MODEL and
             (not context.get('sync_update_execution') and
              not context.get('sync_update_creation')))
 
@@ -528,7 +528,7 @@
             funct_field = audit_obj.get_functionnal_fields(cr, 1, self._name, audit_rule_ids)
 
         to_be_synchronized = (
-            self._name not in MODELS_TO_IGNORE and
+            self._name in WHITE_LIST_MODEL and
             (not context.get('sync_update_execution') and
              not context.get('sync_update_creation')))
 
@@ -617,7 +617,7 @@
         # synchronization is made.
         # Otherwise, references are kept and synchronization is triggered
         # ...see?
-        if self._name not in MODELS_TO_IGNORE \
+        if self._name in WHITE_LIST_MODEL \
            and not context.get('sync_update_creation'):
             context = dict(context, avoid_sdref_deletion=True)
             if not context.get('sync_update_execution'):

=== modified file 'bin/addons/sync_client/sync_client.py'
--- bin/addons/sync_client/sync_client.py	2016-11-18 13:31:06 +0000
+++ bin/addons/sync_client/sync_client.py	2017-02-16 09:58:42 +0000
@@ -42,6 +42,7 @@
 
 import functools
 
+from sync_common import WHITE_LIST_MODEL
 from datetime import datetime, timedelta
 
 from sync_common import OC_LIST_TUPLE
@@ -205,6 +206,17 @@
                         self.pool.get('sync.monitor').get_logger(cr, uid, defaults_logger, context=context)
                     context['log_sale_purchase'] = True
 
+                    # generate a white list of models
+                    if self.pool.get('sync.client.rule') and\
+                            self.pool.get('sync.client.message_rule'):
+                        server_model_white_set = self.get_model_white_list(cr, uid)
+                        # check all models are in the hardcoded white list
+                        difference = server_model_white_set.difference(WHITE_LIST_MODEL)
+                        if difference:
+                            msg = 'Warning: Some models used in the synchronization '\
+                            'rule are not present in the WHITE_LIST_MODEL: %s'
+                            logger.append(_(msg) % ' ,'.join(list(difference)))
+
                     # create a specific cursor for the call
                     self.sync_cursor = pooler.get_db(cr.dbname).cursor()
 
@@ -497,6 +509,71 @@
         finally:
             self.renew_lock.release()
 
+    def get_model_white_list(self, cr, uid):
+        '''
+        return a set of all models involved in the synchronization process
+        '''
+        model_field_dict = {}
+
+        # search for model of sync_server.sync_rule
+        if self.pool.get('sync_server.sync_rule'):
+            rule_module = self.pool.get('sync_server.sync_rule')
+            model_field_name = 'model_id'
+        else:
+            rule_module = self.pool.get('sync.client.rule')
+            model_field_name = 'model'
+        obj_ids = rule_module.search(cr, uid, [('active', '=', True)])
+        for obj in rule_module.read(cr, uid, obj_ids, [model_field_name, 'included_fields']):
+            if obj[model_field_name] not in model_field_dict:
+                model_field_dict[obj[model_field_name]] = set()
+            model_field_dict[obj[model_field_name]].update(eval(obj['included_fields']))
+
+        # search for model of sync_server.message_rule
+        if self.pool.get('sync_server.message_rule'):
+            rule_module = self.pool.get('sync_server.message_rule')
+            model_field_name = 'model_id'
+        else:
+            rule_module = self.pool.get('sync.client.message_rule')
+            model_field_name = 'model'
+        obj_ids = rule_module.search(cr, uid, [('active', '=', True)])
+        for obj in rule_module.read(cr, uid, obj_ids, [model_field_name, 'arguments']):
+            if obj[model_field_name] not in model_field_dict:
+                model_field_dict[obj[model_field_name]] = set()
+            model_field_dict[obj[model_field_name]].update(eval(obj['arguments']))
+
+        model_set = set(model_field_dict.keys())
+
+        def get_field_obj(model, field_name):
+            model_obj = self.pool.get(model)
+            field_obj = None
+            if field_name in model_obj._columns:
+                field_obj = model_obj._columns[field_name]
+            elif field_name in model_obj._inherit_fields:
+                field_obj = model_obj._inherit_fields[field_name][2]
+            return field_obj
+
+
+        # for each field corresponding to each model, check if it is a m2m m2o or o2m
+        # if yes, add the model of the relation to the model set
+
+        for model, field_list in model_field_dict.items():
+            field_list_to_parse = [x for x in field_list if '/id' in x]
+            if not field_list_to_parse:
+                continue
+
+            for field in field_list_to_parse:
+                field = field.replace('/id', '')
+                if len(field.split('/')) == 2:
+                    related_field, field = field.split('/')
+                    field_obj = get_field_obj(model, related_field)
+                    related_model = field_obj._obj
+                    field_obj = get_field_obj(related_model, field)
+                else:
+                    field_obj = get_field_obj(model, field)
+                if field_obj._type in ('many2one', 'many2many', 'one2many'):
+                    model_set.add(field_obj._obj)
+        return model_set
+
     @sync_process('data_push')
     def push_update(self, cr, uid, context=None):
         """

=== modified file 'bin/addons/sync_common/common.py'
--- bin/addons/sync_common/common.py	2016-11-18 13:31:06 +0000
+++ bin/addons/sync_common/common.py	2017-02-16 09:58:42 +0000
@@ -8,101 +8,147 @@
 import tools
 from tools.translate import _
 
+# model related to synchronization, this model don't have to be ignored.
+# list build by getting models of update rules and message rules
+WHITE_LIST_MODEL = [
+  'account.account',
+  'account.account.type',
+  'account.analytic.account',
+  'account.analytic.journal',
+  'account.analytic.line',
+  'account.bank.statement',
+  'account.bank.statement.line',
+  'account.bank.statement.line.deleted',
+  'account.cashbox.line',
+  'account.destination.link',
+  'account.fiscal.position',
+  'account.fiscalyear',
+  'account.fiscalyear.state',
+  'account.invoice',
+  'account.journal',
+  'account.move',
+  'account.move.line',
+  'account.move.reconcile',
+  'account.payment.term',
+  'account.period',
+  'account.period.state',
+  'account.target.costcenter',
+  'account.tax',
+  'account.tax.code',
+  'analytic.distribution',
+  'claim.event',
+  'claim.product.line',
+  'composition.item',
+  'composition.kit',
+  'cost.center.distribution.line',
+  'country.export.mapping',
+  'distribution.line',
+  'financing.contract.contract',
+  'financing.contract.donor',
+  'financing.contract.format',
+  'financing.contract.format.line',
+  'financing.contract.funding.pool.line',
+  'free.1.distribution.line',
+  'free.2.distribution.line',
+  'funding.pool.distribution.line',
+  'hq.entries',
+  'hr.employee',
+  'hr.employee.marital.status',
+  'hr.job',
+  'initial.stock.inventory',
+  'initial.stock.inventory.line',
+  'ir.actions.act_window',
+  'ir.filters',
+  'ir.model',
+  'ir.model.access',
+  'ir.model.fields',
+  'ir.rule',
+  'ir.sequence',
+  'ir.translation',
+  'ir.ui.menu',
+  'ir.ui.view',
+  'kit.creation',
+  'kit.creation.to.consume',
+  'monthly.review.consumption',
+  'monthly.review.consumption.line',
+  'msf.budget',
+  'msf.budget.decision.moment',
+  'msf.budget.line',
+  'msf.instance',
+  'msf_button_access_rights.button_access_rule',
+  'msf_field_access_rights.field_access_rule',
+  'msf_field_access_rights.field_access_rule_line',
+  'pack.type',
+  'procurement.order',
+  'product.asset',
+  'product.asset.type',
+  'product.category',
+  'product.cold_chain',
+  'product.heat_sensitive',
+  'product.international.status',
+  'product.justification.code',
+  'product.list',
+  'product.list.line',
+  'product.nomenclature',
+  'product.pricelist',
+  'product.pricelist.type',
+  'product.pricelist.version',
+  'product.product',
+  'product.uom',
+  'product.uom.categ',
+  'purchase.order',
+  'purchase.order.line',
+  'real.average.consumption',
+  'real.average.consumption.line',
+  'res.company',
+  'res.country',
+  'res.country.restriction',
+  'res.country.state',
+  'res.currency',
+  'res.currency.rate',
+  'res.currency.table',
+  'res.groups',
+  'res.partner',
+  'res.partner.address',
+  'res.partner.title',
+  'res.users',
+  'return.claim',
+  'sale.order',
+  'sale.order.line',
+  'sale.order.line.cancel',
+  'shipment',
+  'stock.frequence',
+  'stock.inventory',
+  'stock.inventory.line',
+  'stock.journal',
+  'stock.location',
+  'stock.location.chained.options',
+  'stock.mission.report',
+  'stock.mission.report.line',
+  'stock.move',
+  'stock.picking',
+  'stock.production.lot',
+  'stock.reason.type',
+  'stock.warehouse',
+  'stock.warehouse.automatic.supply',
+  'stock.warehouse.automatic.supply.line',
+  'stock.warehouse.order.cycle',
+  'stock.warehouse.order.cycle.line',
+  'stock.warehouse.orderpoint',
+  'supplier.catalogue',
+  'supplier.catalogue.line',
+  'sync.monitor',
+  'sync.sale.order.line.split',
+  'sync.version.instance.monitor',
+  'tender',
+  'tender.line',
+  'threshold.value',
+  'threshold.value.line',
+]
+
 OC_LIST = ['OCA', 'OCB', 'OCBA', 'OCG', 'OCP']
 OC_LIST_TUPLE = zip([x.lower() for x in OC_LIST], OC_LIST)
 
-MODELS_TO_IGNORE = [
-    'ir.actions.wizard',
-    'ir.actions.act_window.view',
-    'ir.report.custom',
-    'ir.actions.act_window.view',
-    'ir.actions.wizard',
-    'ir.report.custom',
-    'ir.ui.view',
-    'ir.sequence',
-    'ir.actions.url',
-    'ir.values',
-    'ir.report.custom.fields',
-    'ir.cron',
-    'ir.actions.report.xml',
-    'ir.property',
-    'ir.actions.todo',
-    'ir.sequence.type',
-    #'ir.actions.act_window',
-    'ir.module.module',
-    'ir.ui.view',
-    'ir.module.repository',
-    'ir.model.data',
-    'ir.model.fields',
-    'ir.ui.view_sc',
-    'ir.config_parameter',
-
-    #'sync.monitor',
-    'sync.client.rule',
-    'sync.client.push.data.information',
-    'sync.client.update_to_send',
-    'sync.client.update_received',
-    'sync.client.entity',
-    'sync.client.sync_server_connection',
-    'sync.client.message_rule',
-    'sync.client.message_to_send',
-    'sync.client.message_received',
-    'sync.client.message_sync',
-    'sync.client.orm_extended',
-
-    'sync.server.test',
-    'sync_server.version.manager',
-    'sync.server.entity_group',
-    'sync.server.entity',
-    'sync.server.group_type',
-    'sync.server.entity_group',
-    'sync.server.entity',
-    'sync.server.sync_manager',
-    'sync_server.sync_rule',
-    'sync_server.message_rule',
-    'sync_server.sync_rule.forced_values',
-    'sync_server.sync_rule.fallback_values',
-    'sync_server.rule.validation.message',
-    'sync.server.update',
-    'sync.server.message',
-    'sync_server.version',
-    'sync.server.puller_logs',
-    'audittrail.log.sequence',
-    'audittrail.log.line',
-
-    'res.widget',
-    'product.likely.expire.report',
-    'product.likely.expire.report.line',
-    'operations.event',
-]
-
-MODELS_TO_IGNORE_DOMAIN = [
-    'sync_client.%',
-    'sync_server.%',
-    'res.widget%',
-    'base%',
-    'board%',
-    'audittrail%',
-    'workflow%',
-]
-
-def __compile_models_to_ignore():
-    global MODELS_TO_IGNORE_DOMAIN
-    simple_patterns = []
-    exact_models = []
-    for model in MODELS_TO_IGNORE_DOMAIN:
-        if model.find('%') >= 0:
-            simple_patterns.append(model)
-        else:
-            exact_models.append(model)
-    MODELS_TO_IGNORE_DOMAIN[:] = [('model','not in',exact_models)]
-    for pattern in simple_patterns:
-        MODELS_TO_IGNORE_DOMAIN.extend(['!',('model','=like',pattern)])
-
-__compile_models_to_ignore()
-
-
-
 def xmlid_to_sdref(xmlid):
     if not xmlid: return None
     head, sep, tail = xmlid.partition('.')
@@ -112,8 +158,6 @@
     else:
         return head
 
-
-
 # TODO deprecated, should disappear
 def sync_log(obj, message=None, level='debug', ids=None, data=None, tb=False):
     if not hasattr(obj, '_logger'):

=== modified file 'bin/addons/sync_server/rules.py'
--- bin/addons/sync_server/rules.py	2016-11-18 11:26:45 +0000
+++ bin/addons/sync_server/rules.py	2017-02-16 09:58:42 +0000
@@ -27,7 +27,7 @@
 from datetime import datetime
 
 import logging
-from sync_common import MODELS_TO_IGNORE_DOMAIN, sync_log
+from sync_common import sync_log
 
 _field2type = {
     'text'      : 'str',
@@ -829,7 +829,7 @@
 
     def _get_fallback_value(self, cr, uid, context=None):
         model = self.pool.get('ir.model')
-        ids = model.search(cr, uid, MODELS_TO_IGNORE_DOMAIN)
+        ids = model.search(cr, uid, ('model', 'not in', WHITE_LIST_MODEL))
         res = model.read(cr, uid, ids, ['model'], context)
         return [(r['model'], r['model']) for r in res]