Merge lp:~fabien-morin/unifield-server/fm-us-2292 into lp:unifield-server

[Quentin THEURET @Amaris]

Unmerged revisions

4171. By Fabien MORIN on 2017-01-27

US-2292 [IMP] This work in progress. Need review, cleanning and testing
Make the code faster

=== modified file 'bin/addons/consumption_calculation/consumption_calculation.py'
--- bin/addons/consumption_calculation/consumption_calculation.py	2017-01-31 15:03:10 +0000
+++ bin/addons/consumption_calculation/consumption_calculation.py	2017-02-14 10:48:41 +0000
@@ -1642,52 +1642,62 @@
             to_date = context.get('to_date')
             rac_domain.append(('period_to', '<=', to_date))
 
-        # Filter for one or some locations    
+        # Filter for one or some locations
         if context.get('location_id', False):
             if type(context['location_id']) == type(1):
                 location_ids = [context['location_id']]
             elif type(context['location_id']) in (type(''), type(u'')):
-                location_ids = self.pool.get('stock.location').search(cr, uid, [('name','ilike',context['location'])], context=context)
+                location_ids = self.pool.get('stock.location').search(cr, uid,
+                        [('name', 'ilike', context['location'])],
+                        order='NO_ORDER', context=context)
             else:
                 location_ids = context.get('location_id', [])
 
-        for id in ids:
-            res[id] = 0.00
-            if from_date and to_date:
-                rcr_domain = ['&', '&', ('product_id', '=', id), ('rac_id.cons_location_id', 'in', location_ids),
-                              # All lines with a report started out the period and finished in the period 
+        res = dict.fromkeys(ids, 0.00)
+        if from_date and to_date:
+            # We want the average for the entire period
+            if to_date < from_date:
+                raise osv.except_osv(_('Error'), _('You cannot have a \'To Date\' younger than \'From Date\'.'))
+            # Calculate the # of months in the period
+            try:
+                to_date_str = strptime(to_date, '%Y-%m-%d')
+            except ValueError:
+                to_date_str = strptime(to_date, '%Y-%m-%d %H:%M:%S')
+            try:
+                from_date_str = strptime(from_date, '%Y-%m-%d')
+            except ValueError:
+                from_date_str = strptime(from_date, '%Y-%m-%d %H:%M:%S')
+            nb_months = self._get_date_diff(from_date_str, to_date_str)
+            if not nb_months:
+                nb_months = 1
+            uom_id_result = self.read(cr, uid, ids, ['uom_id'], context=context)
+            uom_id_dict = dict((x['id'], x['uom_id'][0]) for x in
+                    uom_id_result)
+            racl_obj = self.pool.get('real.average.consumption.line')
+
+            for prod_id in ids:
+                rcr_domain = ['&', '&', ('product_id', '=', prod_id), ('rac_id.cons_location_id', 'in', location_ids),
+                              # All lines with a report started out the period and finished in the period
                               '|', '&', ('rac_id.period_to', '>=', from_date), ('rac_id.period_to', '<=', to_date),
-                              # All lines with a report started in the period and finished out the period 
+                              # All lines with a report started in the period and finished out the period
                               '|', '&', ('rac_id.period_from', '<=', to_date), ('rac_id.period_from', '>=', from_date),
                               # All lines with a report started before the period  and finished after the period
                               '&', ('rac_id.period_from', '<=', from_date), ('rac_id.period_to', '>=', to_date)]
 
-                rcr_line_ids = self.pool.get('real.average.consumption.line').search(cr, uid, rcr_domain, context=context)
-                for line in self.pool.get('real.average.consumption.line').browse(cr, uid, rcr_line_ids, context=context):
+                rcr_line_ids = racl_obj.search(cr, uid, rcr_domain, order='NO_ORDER', context=context)
+                for line in racl_obj.browse(cr, uid, rcr_line_ids, context=context):
+                    print len(rcr_line_ids)
+                    print 'rcr_line_ids'
+                    #import pdb; pdb.set_trace()
                     cons = self._get_period_consumption(cr, uid, line, from_date, to_date, context=context)
-                    res[id] += uom_obj._compute_qty(cr, uid, line.uom_id.id, cons, line.product_id.uom_id.id)
-
-                # We want the average for the entire period
-                if to_date < from_date:
-                    raise osv.except_osv(_('Error'), _('You cannot have a \'To Date\' younger than \'From Date\'.'))
-                # Calculate the # of months in the period
-                try:
-                    to_date_str = strptime(to_date, '%Y-%m-%d')
-                except ValueError:
-                    to_date_str = strptime(to_date, '%Y-%m-%d %H:%M:%S')
-
-                try:
-                    from_date_str = strptime(from_date, '%Y-%m-%d')
-                except ValueError:
-                    from_date_str = strptime(from_date, '%Y-%m-%d %H:%M:%S')
-
-                nb_months = self._get_date_diff(from_date_str, to_date_str)
-
-                if not nb_months: nb_months = 1
-
-                uom_id = self.browse(cr, uid, ids[0], context=context).uom_id.id
-                res[id] = res[id]/nb_months
-                res[id] = round(self.pool.get('product.uom')._compute_qty(cr, uid, uom_id, res[id], uom_id), 2)
+                    res[prod_id] += uom_obj._compute_qty(cr, uid, line.uom_id.id, cons, line.product_id.uom_id.id)
+
+                if res[prod_id]:
+                    per_month_consumption = res[prod_id]/nb_months
+                    res[prod_id] = round(uom_obj._compute_qty(cr, uid,
+                        uom_id_dict[prod_id],
+                        per_month_consumption,
+                        uom_id_dict[prod_id]), 2)
 
         return res
 

=== modified file 'bin/addons/consumption_calculation/history_consumption.py'
--- bin/addons/consumption_calculation/history_consumption.py	2016-10-12 10:07:50 +0000
+++ bin/addons/consumption_calculation/history_consumption.py	2017-02-14 10:48:41 +0000
@@ -33,6 +33,7 @@
 class product_history_consumption(osv.osv):
     _name = 'product.history.consumption'
     _rec_name = 'location_id'
+    _order = 'requestor_date desc, id desc'
 
     def _get_status(self, cr, uid, ids, field_name, args, context=None):
         '''
@@ -40,9 +41,8 @@
         '''
         res = {}
 
-        for obj in self.browse(cr, uid, ids, context=context):
-            res[obj.id] = obj.status
-
+        read_result = self.read(cr, uid, ids,  ['status'], context=context)
+        res = dict((x['id'], x['status']) for x in read_result)
         return res
 
     _columns = {
@@ -91,7 +91,6 @@
             context = {}
         res = {'value': {}}
         month_obj = self.pool.get('product.history.consumption.month')
-        
         if date_from:
             date_from = (DateFrom(date_from) + RelativeDateTime(day=1)).strftime('%Y-%m-%d')
             res['value'].update({'date_from': date_from})
@@ -127,7 +126,8 @@
 
         # Delete all months out of the period
         del_months = []
-        for month_id in month_obj.search(cr, uid, [('history_id', 'in', ids)], context=context):
+        for month_id in month_obj.search(cr, uid, [('history_id', 'in', ids)],
+                order='NO_ORDER', context=context):
             if month_id not in res['value']['month_ids']:
                 del_months.append(month_id)
         if del_months:
@@ -219,9 +219,10 @@
         import threading
         self.write(cr, uid, ids, {'status': 'in_progress'}, context=context)
         cr.commit()
+        #self._create_lines(cr, uid, ids, product_ids, new_context)
         new_thread = threading.Thread(target=self._create_lines, args=(cr, uid, ids, product_ids, new_context))
         new_thread.start()
-        new_thread.join(10.0)
+        new_thread.join(5.0)
         if new_thread.isAlive():
             view_id = self.pool.get('ir.model.data').get_object_reference(cr, uid, 'consumption_calculation', 'history_consumption_waiting_view')[1]
             return {'type': 'ir.actions.act_window',
@@ -240,31 +241,52 @@
         Create lines in background
         '''
         import pooler
-        new_cr = pooler.get_db(cr.dbname).cursor()
-
-        # split ids into slices to not read a lot record in the same time (memory)
-        ids_len = len(product_ids)
-        slice_len = 500
-        if ids_len > slice_len:
-            slice_count = ids_len / slice_len
-            if ids_len % slice_len:
-                slice_count = slice_count + 1
-            # http://www.garyrobinson.net/2008/04/splitting-a-pyt.html
-            slices = [product_ids[i::slice_count] for i in range(slice_count)]
-        else:
+        try:
+            new_cr = pooler.get_db(cr.dbname).cursor()
+            prod_obj = self.pool.get('product.product')
+            logger = logging.getLogger('history.consumption')
+
+            # split ids into slices to not read a lot record in the same time (memory)
+            #product_ids = product_ids[:435]
+            #if 23009 not in product_ids:
+            #    product_ids.append(23009)
+            #if 9273 not in product_ids:
+            #    product_ids.append(9273)
+            ids_len = len(product_ids)
+            slice_len = 500
+            counter = 0
+            slices = []
+            if ids_len > slice_len:
+                while counter < ids_len:
+                    cur_slice = product_ids[counter:counter+slice_len]
+                    slices.append(cur_slice)
+                    counter += slice_len
+            else:
+                slices = [product_ids]
+
             slices = [product_ids]
 
-        for slice_ids in slices:
-            try:
-                self.pool.get('product.product').read(new_cr, uid, slice_ids, ['average'], context=context)
-            except Exception, e:
-                logging.getLogger('history.consumption').warn('Exception in read average', exc_info=True)
-                new_cr.rollback()
-        self.write(new_cr, uid, ids, {'status': 'ready'}, context=context)
-
-        new_cr.commit()
-        new_cr.close(True)
-
+            slice_count = 0
+            for slice_ids in slices:
+                # chaque slice_ids contient 496 ou 495 ids, il y a 40 slice_ids
+                # dans slices
+                # 19812 product_ids
+                slice_count += 1
+                print 'slice_count: %s' % slice_count
+                start = time.time()
+                try:
+
+                    # XXX pourquoi on fait un read qui semble ne servir à rien ???
+                    prod_obj.read(new_cr, uid, slice_ids, ['average'], context=context)
+                except Exception, e:
+                    logger.warn('Exception in read average', exc_info=True)
+                    new_cr.rollback()
+                print 'slice %s total_time: %s' % (slice_count, str(time.time() - start))
+            self.write(new_cr, uid, ids, {'status': 'ready'}, context=context)
+
+            new_cr.commit()
+        finally:
+            new_cr.close(True)
         return
 
     def open_report(self, cr, uid, ids, context=None):
@@ -495,7 +517,6 @@
             context = {}
 
         res = super(product_product, self).fields_get(cr, uid, fields, context=context)
-        
         if context.get('history_cons', False):
             months = context.get('months', [])
 
@@ -522,8 +543,8 @@
         if context is None:
             context = {}
 
+        res = super(product_product, self).read(cr, uid, ids, vals, context=context, load=load)
         if context.get('history_cons', False):
-            res = super(product_product, self).read(cr, uid, ids, vals, context=context, load=load)
 
             if 'average' not in vals:
                 return res
@@ -538,60 +559,71 @@
                 raise osv.except_osv(_('Error'), _('No months found !'))
 
             obj_id = context.get('obj_id')
-            for r in res:
-                total_consumption = 0.00
-                for month in context.get('months'):
-                    field_name = DateFrom(month.get('date_from')).strftime('%m_%Y')
-                    cons_context = {'from_date': month.get('date_from'), 'to_date': month.get('date_to'), 'location_id': context.get('location_id')}
+            total_consumption_dict = dict.fromkeys(ids, 0.00)
+            for month in context.get('months'):
+                start = time.time()
+                field_name = DateFrom(month.get('date_from')).strftime('%m_%Y')
+                cons_context = {'from_date': month.get('date_from'), 'to_date': month.get('date_to'), 'location_id': context.get('location_id')}
+                if context.get('amc') == 'AMC':
+                    cons_type = 'amc'
+                else:
+                    cons_type = 'fmc'
+                    consumption_result = self.read(cr, uid, ids, ['monthly_consumption'],context=cons_context)
+                    fmc_cons_dict = dict((x['id'], x['monthly_consumption'] or 0.00) for x in consumption_result)
+                for r in res:
+                    #if month == {'date_from': '2016-06-01', 'date_to':'2016-06-30'} and r['id'] == 9273:
+                    #    import pdb; pdb.set_trace()
                     consumption = 0.00
                     cons_prod_domain = [('name', '=', field_name),
                                         ('product_id', '=', r['id']),
                                         ('consumption_id', '=', obj_id)]
-                    if context.get('amc') == 'AMC':
-                        cons_prod_domain.append(('cons_type', '=', 'amc'))
-                        cons_id = cons_prod_obj.search(cr, uid, cons_prod_domain, context=context)
-                        if cons_id:
-                            consumption = cons_prod_obj.browse(cr, uid, cons_id[0], context=context).value
-                        else:
-                            consumption = self.pool.get('product.product').compute_amc(cr, uid, r['id'], context=cons_context) or 0.00
-                            cons_prod_obj.create(cr, uid, {'name': field_name,
-                                                           'product_id': r['id'],
-                                                           'consumption_id': obj_id,
-                                                           'cons_type': 'amc',
-                                                           'value': consumption}, context=context)
+                    cons_prod_domain.append(('cons_type', '=', cons_type))
+                    cons_ids = cons_prod_obj.search(cr, uid, cons_prod_domain, context=context)
+                    if cons_ids:
+                        consumption = cons_prod_obj.read(cr, uid,
+                                cons_ids[0], ['value'],
+                                context=context)['value']
                     else:
-                        cons_prod_domain.append(('cons_type', '=', 'fmc'))
-                        cons_id = cons_prod_obj.search(cr, uid, cons_prod_domain, context=context)
-                        if cons_id:
-                            consumption = cons_prod_obj.browse(cr, uid, cons_id[0], context=context).value
+                        if cons_type == 'amc':
+                            consumption = self.compute_amc(cr, uid, r['id'], context=cons_context) or 0.00
                         else:
-                            consumption = self.pool.get('product.product').browse(cr, uid, r['id'], context=cons_context).monthly_consumption or 0.00
-                            cons_prod_obj.create(cr, uid, {'name': field_name,
-                                                           'product_id': r['id'],
-                                                           'consumption_id': obj_id,
-                                                           'cons_type': 'fmc',
-                                                           'value': consumption}, context=context)
-                    total_consumption += consumption
+                            consumption = fmc_cons_dict[r['id']]
+                        cons_prod_obj.create(cr, uid, {'name': field_name,
+                                                       'product_id': r['id'],
+                                                       'consumption_id': obj_id,
+                                                       'cons_type': cons_type,
+                                                       'value': consumption}, context=context)
+
+                    if consumption:
+                        total_consumption_dict[r['id']] += consumption
                     # Update the value for the month
                     r.update({field_name: consumption})
+                stop = time.time() - start
+                print 'month from %s to %s, %s products : %s' % (month.get('date_from'), month.get('date_to'), len(res), str(stop))
+
+            cons_prod_domain = [('name', '=', 'average'),
+                                ('consumption_id', '=', obj_id),
+                                ('cons_type', '=', context.get('amc') == 'AMC' and 'amc' or 'fmc')]
+            print 'start calculing the averages'
+            start = time.time()
+            for product_id in ids:
+                if not total_consumption_dict[product_id]:
+                    continue
 
                 # Update the average field
-                cons_prod_domain = [('name', '=', 'average'),
-                                    ('product_id', '=', r['id']),
-                                    ('consumption_id', '=', obj_id),
-                                    ('cons_type', '=', context.get('amc') == 'AMC' and 'amc' or 'fmc')]
-                r.update({'average': round(total_consumption/float(len(context.get('months'))),2)})
-                cons_id = cons_prod_obj.search(cr, uid, cons_prod_domain, context=context)
-                if cons_id:
-                    cons_prod_obj.write(cr, uid, cons_id, {'value': r['average']}, context=context)
+                cons_prod_domain.append(('product_id', '=', product_id))
+                r.update({'average': round(total_consumption_dict[product_id]/float(len(context.get('months'))),2)})
+                cons_ids = cons_prod_obj.search(cr, uid, cons_prod_domain, order='NO_ORDER', context=context)
+                if cons_ids:
+                    cons_prod_obj.write(cr, uid, cons_ids, {'value': r['average']}, context=context)
                 else:
                     cons_prod_obj.create(cr, uid, {'name': 'average',
                                                    'product_id': r['id'],
                                                    'consumption_id': obj_id,
                                                    'cons_type': context.get('amc') == 'AMC' and 'amc' or 'fmc',
                                                    'value': r['average']}, context=context)
-        else:
-            res = super(product_product, self).read(cr, uid, ids, vals, context=context, load=load)
+            stop = time.time() - start
+            print 'average calculation finished in %s' % (str(stop))
 
         return res
 
@@ -675,7 +707,8 @@
         Return the result in the same order as given in ids
         '''
         res = super(product_history_consumption_product, self).read(cr, uid, ids, fields, context=context, load=load)
-
+        if not isinstance(res, (list, tuple)):
+            return res
         res_final = [None]*len(ids)
         for r in res:
             r_index = ids.index(r['id'])