Merge lp:~camptocamp/openerp-connector-magento/7.0-export-dependency+lock into lp:~openerp-connector-core-editors/openerp-connector-magento/7.0

Proposed by Guewen Baconnier @ Camptocamp
Status: Work in progress
Proposed branch: lp:~camptocamp/openerp-connector-magento/7.0-export-dependency+lock
Merge into: lp:~openerp-connector-core-editors/openerp-connector-magento/7.0
Diff against target: 208 lines (+167/-1)
1 file modified
magentoerpconnect/unit/export_synchronizer.py (+167/-1)
To merge this branch: bzr merge lp:~camptocamp/openerp-connector-magento/7.0-export-dependency+lock
Reviewer Review Type Date Requested Status
OpenERP Connector Core Editors Pending
Review via email: mp+223508@code.launchpad.net

Description of the change

Prevent concurrent exports to run on the same binding record by acquiring a lock on the binding record before exporting it
Add a 'export_dependency' helper method in the Exporter to ease the export of dependencies

To post a comment you must log in.

Unmerged revisions

1004. By Guewen Baconnier @ Camptocamp on 2014-06-18

Add a 'export_dependency' helper method in the Exporter to ease the export of dependencies

1003. By Guewen Baconnier @ Camptocamp on 2014-06-18

Prevent concurrent exports to run on the same binding record by acquiring a lock on the binding record before exporting it

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'magentoerpconnect/unit/export_synchronizer.py'
2--- magentoerpconnect/unit/export_synchronizer.py 2014-05-26 09:37:00 +0000
3+++ magentoerpconnect/unit/export_synchronizer.py 2014-06-18 07:34:17 +0000
4@@ -20,12 +20,19 @@
5 ##############################################################################
6
7 import logging
8+
9+from contextlib import contextmanager
10 from datetime import datetime
11+
12+import psycopg2
13+
14+from openerp import SUPERUSER_ID
15 from openerp.tools.translate import _
16 from openerp.tools import DEFAULT_SERVER_DATETIME_FORMAT
17 from openerp.addons.connector.queue.job import job, related_action
18 from openerp.addons.connector.unit.synchronizer import ExportSynchronizer
19-from openerp.addons.connector.exception import IDMissingInBackend
20+from openerp.addons.connector.exception import (IDMissingInBackend,
21+ RetryableJobError)
22 from .import_synchronizer import import_record
23 from ..connector import get_environment
24 from ..related_action import unwrap_binding
25@@ -116,6 +123,11 @@
26 result = self._run(*args, **kwargs)
27
28 self.binder.bind(self.magento_id, self.binding_id)
29+ # Commit so we keep the external ID when there are several
30+ # exports (due to dependencies) and one of them fails.
31+ # The commit will also release the lock acquired on the binding
32+ # record
33+ self.session.commit()
34 return result
35
36 def _run(self):
37@@ -134,10 +146,160 @@
38 super(MagentoExporter, self).__init__(environment)
39 self.binding_record = None
40
41+ def _lock(self):
42+ """ Lock the binding record.
43+
44+ Lock the binding record so we are sure that only one export
45+ job is running for this record if concurrent jobs have to export the
46+ same record.
47+
48+ When concurrent jobs try to export the same record, the first one
49+ will lock and proceed, the others will fail to lock and will be
50+ retried later.
51+
52+ This behavior works also when the export becomes multilevel
53+ with :meth:`_export_dependencies`. Each level will set its own lock
54+ on the binding record it has to export.
55+
56+ """
57+ sql = ("SELECT id FROM %s WHERE ID = %%s FOR UPDATE NOWAIT" %
58+ self.model._table)
59+ try:
60+ self.session.cr.execute(sql, (self.binding_id, ),
61+ log_exceptions=False)
62+ except psycopg2.OperationalError:
63+ _logger.info('A concurrent job is already exporting the same '
64+ 'record (%s with id %s). Job delayed later.',
65+ self.model._name, self.binding_id)
66+ raise RetryableJobError(
67+ 'A concurrent job is already exporting the same record '
68+ '(%s with id %s). The job will be retried later.' %
69+ (self.model._name, self.binding_id))
70+
71 def _has_to_skip(self):
72 """ Return True if the export can be skipped """
73 return False
74
75+ @contextmanager
76+ def _retry_unique_violation(self):
77+ """ Context manager: catch Unique constraint error and retry the
78+ job later.
79+
80+ When we execute several jobs workers concurrently, it happens
81+ that 2 jobs are creating the same record at the same time (binding
82+ record created by :meth:`_export_dependency`), resulting in:
83+
84+ IntegrityError: duplicate key value violates unique
85+ constraint "magento_product_product_openerp_uniq"
86+ DETAIL: Key (backend_id, openerp_id)=(1, 4851) already exists.
87+
88+ In that case, we'll retry the import just later.
89+
90+ .. warning:: The unique constraint must be created on the
91+ binding record to prevent 2 bindings to be created
92+ for the same Magento record.
93+
94+ """
95+ try:
96+ yield
97+ except psycopg2.IntegrityError as err:
98+ if err.pgcode == psycopg2.errorcodes.UNIQUE_VIOLATION:
99+ raise RetryableJobError(
100+ 'A database error caused the failure of the job:\n'
101+ '%s\n\n'
102+ 'Likely due to 2 concurrent jobs wanting to create '
103+ 'the same record. The job will be retried later.' % err)
104+ else:
105+ raise
106+
107+ def _export_dependency(self, relation, binding_model, exporter_class=None,
108+ binding_field='magento_bind_ids'):
109+ """
110+ Export a dependency. The exporter class is a subclass of
111+ ``MagentoExporter``. If a more precise class need to be defined,
112+ it can be passed to the ``exporter_class`` keyword argument.
113+
114+ .. warning:: a commit is done at the end of the export of each
115+ dependency. The reason for that is that we pushed a record
116+ on the backend and we absolutely have to keep its ID.
117+
118+ So you *must* take care not to modify the OpenERP
119+ database during an export, excepted when writing
120+ back the external ID or eventually to store
121+ external data that we have to keep on this side.
122+
123+ You should call this method only at the beginning
124+ of the exporter synchronization,
125+ in :meth:`~._export_dependencies`.
126+
127+ :param relation: record to export if not already exported
128+ :type relation: :py:class:`openerp.osv.orm.browse_record`
129+ :param binding_model: name of the binding model for the relation
130+ :type binding_model: str | unicode
131+ :param exporter_cls: :py:class:`openerp.addons.connector.connector.ConnectorUnit`
132+ class or parent class to use for the export.
133+ By default: MagentoExporter
134+ :type exporter_cls: :py:class:`openerp.addons.connector.connector.MetaConnectorUnit`
135+ :param binding_field: name of the one2many field on a normal
136+ record that points to the binding record
137+ (default: magento_bind_ids).
138+ It is used only when the relation is not
139+ a binding but is a normal record.
140+ :type binding_field: str | unicode
141+ """
142+ if not relation:
143+ return
144+ if exporter_class is None:
145+ exporter_class = MagentoExporter
146+ rel_binder = self.get_binder_for_model(binding_model)
147+ # wrap is typically True if the relation is for instance a
148+ # 'product.product' record but the binding model is
149+ # 'magento.product.product'
150+ wrap = relation._model._name != binding_model
151+
152+ if wrap and hasattr(relation, binding_field):
153+ domain = [('openerp_id', '=', relation.id),
154+ ('backend_id', '=', self.backend_record.id)]
155+ binding_ids = self.session.search(binding_model, domain)
156+ if binding_ids:
157+ assert len(binding_ids) == 1, (
158+ 'only 1 binding for a backend is '
159+ 'supported in _export_dependency')
160+ binding_id = binding_ids[0]
161+ # we are working with a unwrapped record (e.g.
162+ # product.category) and the binding does not exist yet.
163+ # Example: I created a product.product and its binding
164+ # magento.product.product and we are exporting it, but we need to
165+ # create the binding for the product.category on which it
166+ # depends.
167+ else:
168+ with self.session.change_context({'connector_no_export': True}):
169+ with self.session.change_user(SUPERUSER_ID):
170+ bind_values = {'backend_id': self.backend_record.id,
171+ 'openerp_id': relation.id}
172+ # If 2 jobs create it at the same time, retry
173+ # one later. A unique constraint (backend_id,
174+ # openerp_id) should exist on the binding model
175+ with self._retry_unique_violation():
176+ binding_id = self.session.create(binding_model,
177+ bind_values)
178+ # Eager commit to avoid having 2 jobs
179+ # exporting at the same time. The constraint
180+ # will pop if an other job already created
181+ # the same binding. It will be caught and
182+ # raise a RetryableJobError.
183+ self.session.commit()
184+ else:
185+ # If magento_bind_ids does not exist we are typically in a
186+ # "direct" binding (the binding record is the same record).
187+ # If wrap is True, relation is already a binding record.
188+ binding_id = relation.id
189+
190+ if rel_binder.to_backend(binding_id) is None:
191+ exporter = self.get_connector_unit_for_model(exporter_class,
192+ binding_model)
193+ exporter.run(binding_id)
194+
195 def _export_dependencies(self):
196 """ Export the dependencies for the record"""
197 return
198@@ -194,6 +356,10 @@
199 # export the missing linked resources
200 self._export_dependencies()
201
202+ # prevent other jobs to export the same record
203+ # will be released on commit (or rollback)
204+ self._lock()
205+
206 map_record = self._map_data()
207
208 if self.magento_id: