Merge lp:~camptocamp/openerp-connector-magento/7.0-next-release-missing-records-1322452 into lp:~openerp-connector-core-editors/openerp-connector-magento/7.0-next-release

[Guewen Baconnier @ Camptocamp]

Introduce a 30 seconds buffer between 2 batches of imports to avoid to miss records.

The 'created_at' date on Magento is set at the beginning of a transaction, and the transaction may be committed seconds after. The buffer will allow to include these records in the imports.

Fixes lp:1322452

[David BEAL (ak) (davidbeal)]

Hi Guewen,

You tell line 44:
          # but this is not a big deal because they will be skipped when
45 + # the last `sync_date` is the same

few lines later the condition is not with sync_date but with :

144 + if self.binder.to_openerp(self.magento_id):
145 + return _('Already imported')

sorry to see this only now

good solution i think.

Thanks

PS : "Magento define the create date at the begining of the transaction" : LOL, Magento is a ... app, no word to qualify it.

[Guewen Baconnier @ Camptocamp (gbaconnier-c2c)]

On Mon, May 26, 2014 at 11:55 AM, David BEAL <email address hidden> wrote:
> Hi Guewen,
>
> You tell line 44:
> # but this is not a big deal because they will be skipped when
> 45 + # the last `sync_date` is the same
>
> few lines later the condition is not with sync_date but with :
>
> 144 + if self.binder.to_openerp(self.magento_id):
> 145 + return _('Already imported')

Yeah that's right, the import of sales orders is a special case since
the subsequent imports are just skipped. For all the other records,
the comment applies. I will change this comment in the part regarding
the sales orders.
>
>
> sorry to see this only now
>
> good solution i think.
>
> Thanks
>
>
> PS : "Magento define the create date at the begining of the transaction" : LOL, Magento is a ... app, no word to qualify it.
>
> --
> https://code.launchpad.net/~camptocamp/openerp-connector-magento/7.0-next-release-missing-records-1322452/+merge/220910
> Your team Camptocamp is subscribed to branch lp:~openerp-connector-core-editors/openerp-connector-magento/7.0-next-release.

=== modified file 'magentoerpconnect/magento_model.py'
--- magentoerpconnect/magento_model.py	2014-05-06 09:33:40 +0000
+++ magentoerpconnect/magento_model.py	2014-05-26 07:37:58 +0000
@@ -21,7 +21,7 @@
 ##############################################################################
 
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta
 from openerp.osv import fields, orm
 from openerp.tools import DEFAULT_SERVER_DATETIME_FORMAT
 import openerp.addons.connector as connector
@@ -44,6 +44,8 @@
 
 _logger = logging.getLogger(__name__)
 
+IMPORT_DELTA_BUFFER = 30  # seconds
+
 
 class magento_backend(orm.Model):
     _name = 'magento.backend'
@@ -230,7 +232,7 @@
             ids = [ids]
         self.check_magento_structure(cr, uid, ids, context=context)
         session = ConnectorSession(cr, uid, context=context)
-        import_start_time = datetime.now().strftime(DEFAULT_SERVER_DATETIME_FORMAT)
+        import_start_time = datetime.now()
         for backend in self.browse(cr, uid, ids, context=context):
             from_date = getattr(backend, from_date_field)
             if from_date:
@@ -240,8 +242,18 @@
                 from_date = None
             import_batch.delay(session, model,
                                backend.id, filters={'from_date': from_date})
-        self.write(cr, uid, ids,
-                   {from_date_field: import_start_time})
+        # Records from Magento are imported based on their `created_at`
+        # date.  This date is set on Magento at the beginning of a
+        # transaction, so if the import is run between the beginning and
+        # the end of a transaction, the import of a record may be
+        # missed.  That's why we add a small buffer back in time where
+        # the eventually missed records will be retrieved.  This also
+        # means that we'll have jobs that import twice the same records,
+        # but this is not a big deal because they will be skipped when
+        # the last `sync_date` is the same.
+        next_time = import_start_time - timedelta(seconds=IMPORT_DELTA_BUFFER)
+        next_time = next_time.strftime(DEFAULT_SERVER_DATETIME_FORMAT)
+        self.write(cr, uid, ids, {from_date_field: next_time}, context=context)
 
     def import_product_categories(self, cr, uid, ids, context=None):
         self._import_from_date(cr, uid, ids, 'magento.product.category',
@@ -344,7 +356,7 @@
         if not hasattr(ids, '__iter__'):
             ids = [ids]
         session = ConnectorSession(cr, uid, context=context)
-        import_start_time = datetime.now().strftime(DEFAULT_SERVER_DATETIME_FORMAT)
+        import_start_time = datetime.now()
         for website in self.browse(cr, uid, ids, context=context):
             backend_id = website.backend_id.id
             if website.import_partners_from_date:
@@ -357,8 +369,19 @@
                 session, 'magento.res.partner', backend_id,
                 {'magento_website_id': website.magento_id,
                     'from_date': from_date})
-        self.write(cr, uid, ids,
-                   {'import_partners_from_date': import_start_time})
+        # Records from Magento are imported based on their `created_at`
+        # date.  This date is set on Magento at the beginning of a
+        # transaction, so if the import is run between the beginning and
+        # the end of a transaction, the import of a record may be
+        # missed.  That's why we add a small buffer back in time where
+        # the eventually missed records will be retrieved.  This also
+        # means that we'll have jobs that import twice the same records,
+        # but this is not a big deal because they will be skipped when
+        # the last `sync_date` is the same.
+        next_time = import_start_time - timedelta(seconds=IMPORT_DELTA_BUFFER)
+        next_time = next_time.strftime(DEFAULT_SERVER_DATETIME_FORMAT)
+        self.write(cr, uid, ids, {'import_partners_from_date': next_time},
+                   context=context)
         return True
 
 
@@ -493,7 +516,7 @@
 
     def import_sale_orders(self, cr, uid, ids, context=None):
         session = ConnectorSession(cr, uid, context=context)
-        import_start_time = datetime.now().strftime(DEFAULT_SERVER_DATETIME_FORMAT)
+        import_start_time = datetime.now()
         for storeview in self.browse(cr, uid, ids, context=context):
             backend_id = storeview.backend_id.id
             if storeview.import_orders_from_date:
@@ -509,7 +532,19 @@
                 {'magento_storeview_id': storeview.magento_id,
                  'from_date': from_date},
                 priority=1)  # executed as soon as possible
-        self.write(cr, uid, ids, {'import_orders_from_date': import_start_time})
+        # Records from Magento are imported based on their `created_at`
+        # date.  This date is set on Magento at the beginning of a
+        # transaction, so if the import is run between the beginning and
+        # the end of a transaction, the import of a record may be
+        # missed.  That's why we add a small buffer back in time where
+        # the eventually missed records will be retrieved.  This also
+        # means that we'll have jobs that import twice the same records,
+        # but this is not a big deal because they will be skipped when
+        # the last `sync_date` is the same.
+        next_time = import_start_time - timedelta(seconds=IMPORT_DELTA_BUFFER)
+        next_time = next_time.strftime(DEFAULT_SERVER_DATETIME_FORMAT)
+        self.write(cr, uid, ids, {'import_orders_from_date': next_time},
+                   context=context)
         return True
 
 

=== modified file 'magentoerpconnect/sale.py'
--- magentoerpconnect/sale.py	2014-05-06 08:36:26 +0000
+++ magentoerpconnect/sale.py	2014-05-26 07:37:58 +0000
@@ -22,8 +22,9 @@
 import logging
 import xmlrpclib
 from datetime import datetime, timedelta
+import openerp.addons.decimal_precision as dp
 from openerp.osv import fields, orm
-import openerp.addons.decimal_precision as dp
+from openerp.tools.translate import _
 from openerp.addons.connector.connector import ConnectorUnit
 from openerp.addons.connector.exception import (NothingToDoJob,
                                                 FailedJobError,
@@ -372,6 +373,21 @@
            self._mapper = self.environment.get_connector_unit(SaleOrderImportMapper)
        return self._mapper
 
+    def _must_skip(self):
+        """ Hook called right after we read the data from the backend.
+
+        If the method returns a message giving a reason for the
+        skipping, the import will be interrupted and the message
+        recorded in the job (if the import is called directly by the
+        job, not by dependencies).
+
+        If it returns None, the import will continue normally.
+
+        :returns: None | str | unicode
+        """
+        if self.binder.to_openerp(self.magento_id):
+            return _('Already imported')
+
     def _clean_magento_items(self, resource):
         """
         Method that clean the sale order line given by magento before importing it