Merge lp:~akretion-team/openerp-connector-magento/openerp-connector-magento-base-catalog-export into lp:~openerp-connector-core-editors/openerp-connector-magento/7.0

[Sébastien BEAU - http://www.akretion.com]

This is change done on the existing stable module for being compatible with the magentoerpconnect_catalog and magentoerpconnect_product_variant modules.

Not ready for being merge yet

[lp:~akretion-team/openerp-connector-magento/openerp-connector-magento-base-catalog-export]

1005. By Sébastien BEAU - http://www.akretion.com on 2014-06-23

[MERGE] merge lock branch from Guewen, this will prevent job to export twice the same record

Unmerged revisions

1005. By Sébastien BEAU - http://www.akretion.com on 2014-06-23

[MERGE] merge lock branch from Guewen, this will prevent job to export twice the same record

1004. By Sébastien BEAU - http://www.akretion.com on 2014-06-18

[REF] remove magentoerpconnect_catalog as the developpement is manage in a separated branch, will be merged soon

1003. By Sébastien BEAU - http://www.akretion.com on 2014-06-18

[IMP] commit work done by Augustin Cisterne-Kaas <email address hidden>, David Beal <email address hidden> Chafique Delli <email address hidden> and myself. This have been extracted from working branch on catalog export. Add the delay_unlink_all_binding, add fields visibility/status at the import. Add backendAdapter method, no refactor have been done yet

=== modified file 'magentoerpconnect/consumer.py'
--- magentoerpconnect/consumer.py	2013-10-09 19:41:58 +0000
+++ magentoerpconnect/consumer.py	2014-06-23 09:58:18 +0000
@@ -66,6 +66,22 @@
                             fields=fields)
 
 
+@on_record_unlink(model_names=_MODEL_NAMES)
+def delay_unlink_all_bindings(session, model_name, record_id):
+    """ Delay jobs which delete all the bindings of a record.
+
+    In this case, it is called on records of normal models and will delay
+    the deletion for all the bindings.
+    """
+    if session.context.get('connector_no_export'):
+        return
+    model = session.pool.get(model_name)
+    record = model.browse(session.cr, session.uid,
+                          record_id, context=session.context)
+    for binding in record.magento_bind_ids:
+        delay_unlink(session, binding._model._name, binding.id)
+
+
 @on_record_unlink(model_names=_BIND_MODEL_NAMES)
 def delay_unlink(session, model_name, record_id):
     """ Delay a job which delete a record on Magento.

=== modified file 'magentoerpconnect/product.py'
--- magentoerpconnect/product.py	2014-06-14 20:30:26 +0000
+++ magentoerpconnect/product.py	2014-06-23 09:58:18 +0000
@@ -88,6 +88,18 @@
         'product_type': fields.selection(_product_type_get,
                                          'Magento Product Type',
                                          required=True),
+        'visibility': fields.selection(
+            [('1', 'Not Visible Individually'),
+             ('2', 'Catalog'),
+             ('3', 'Search'),
+             ('4', 'Catalog, Search')],
+            string='Visibility in Magento',
+            required=True),
+        'status': fields.selection(
+            [('1', 'Enabled'),
+             ('2', 'Disabled')],
+            string='Status in Magento',
+            required=True),
         'manage_stock': fields.selection(
             [('use_default', 'Use Default Config'),
              ('no', 'Do Not Manage Stock'),
@@ -116,6 +128,8 @@
         'manage_stock': 'use_default',
         'backorders': 'use_default',
         'no_stock_sync': False,
+        'visibility': '4',
+        'status': '1',
         }
 
     _sql_constraints = [
@@ -205,6 +219,10 @@
                 in self._call('%s.list' % self._magento_model,
                               [filters] if filters else [{}])]
 
+    def create(self, data):
+        return self._call('%s.create'% self._magento_model,
+            [data.pop('product_type'),data.pop('attrset'),data.pop('sku'),data])
+
     def read(self, id, storeview_id=None, attributes=None):
         """ Returns the information of a record
 
@@ -429,6 +447,8 @@
               ('type_id', 'product_type'),
               (normalize_datetime('created_at'), 'created_at'),
               (normalize_datetime('updated_at'), 'updated_at'),
+              ('visibility', 'visibility'),
+              ('status', 'status')
               ]
 
     @mapping

=== modified file 'magentoerpconnect/product_category.py'
--- magentoerpconnect/product_category.py	2014-05-26 09:37:00 +0000
+++ magentoerpconnect/product_category.py	2014-06-23 09:58:18 +0000
@@ -25,7 +25,8 @@
 from openerp.addons.connector.unit.mapper import (mapping,
                                                   ImportMapper
                                                   )
-from openerp.addons.connector.exception import IDMissingInBackend
+from openerp.addons.connector.exception import (MappingError,
+                                                IDMissingInBackend)
 from .unit.backend_adapter import GenericAdapter
 from .unit.import_synchronizer import (DelayedBatchImport,
                                        MagentoImportSynchronizer,
@@ -49,6 +50,8 @@
                                       required=True,
                                       ondelete='cascade'),
         'description': fields.text('Description', translate=True),
+        'is_active': fields.boolean('Active in magento'),
+        'include_in_menu': fields.boolean('Include in magento menu'),
         'magento_parent_id': fields.many2one(
             'magento.product.category',
              string='Magento Parent Category',
@@ -59,6 +62,11 @@
              string='Magento Child Categories'),
     }
 
+    _defaults = {
+        'is_active': True,
+        'include_in_menu': False,
+    }
+
     _sql_constraints = [
         ('magento_uniq', 'unique(backend_id, magento_id)',
          'A product category with same ID on Magento already exists.'),
@@ -100,6 +108,30 @@
             else:
                 raise
 
+    def update_image(self, data):
+        for name in ['image', 'thumbnail']:
+            if data.get(name + '_binary'):
+                img = self._call('%s.create'% 'ol_catalog_category_media',
+                                 [data[name], data.pop(name + '_binary')])
+                if img == 'Error in file creation':
+                    #TODO FIX error management
+                    raise Exception("Image creation: ",
+                           "Magento tried to insert image (%s) but there is "
+                           "no sufficient grants in the folder "
+                           "'media/catalog/category' if it exists" %data[name])
+        return True
+
+    def create(self, data):
+        self.update_image(data)
+        return self._call('%s.create'% self._magento_model,
+                          [data['parent_id'],data])
+
+    def write(self, id, data, storeview=None):
+        """ Update records on the external system """
+        self.update_image(data)
+        return self._call('%s.update' % self._magento_model,
+                          [int(id), data, storeview])
+
     def search(self, filters=None, from_date=None):
         """ Search records according to some criterias and returns a
         list of ids
@@ -217,6 +249,8 @@
 
     direct = [
             ('description', 'description'),
+            ('is_active','is_active'),
+            ('include_in_menu','include_in_menu')
             ]
 
     @mapping

=== modified file 'magentoerpconnect/product_view.xml'
--- magentoerpconnect/product_view.xml	2014-05-26 09:32:38 +0000
+++ magentoerpconnect/product_view.xml	2014-06-23 09:58:18 +0000
@@ -77,7 +77,10 @@
                         <field name="magento_id"/>
                         <field name="created_at" readonly="1"/>
                         <field name="updated_at" readonly="1"/>
+                        <field name="visibility" />
+                        <field name="status" />
                         <field name="product_type" readonly="1"/>
+                        <field name="website_ids" />
                     </group>
                     <group string="Inventory Options">
                         <field name="no_stock_sync"/>

=== modified file 'magentoerpconnect/unit/export_synchronizer.py'
--- magentoerpconnect/unit/export_synchronizer.py	2014-05-26 09:37:00 +0000
+++ magentoerpconnect/unit/export_synchronizer.py	2014-06-23 09:58:18 +0000
@@ -20,12 +20,19 @@
 ##############################################################################
 
 import logging
+
+from contextlib import contextmanager
 from datetime import datetime
+
+import psycopg2
+
+from openerp import SUPERUSER_ID
 from openerp.tools.translate import _
 from openerp.tools import DEFAULT_SERVER_DATETIME_FORMAT
 from openerp.addons.connector.queue.job import job, related_action
 from openerp.addons.connector.unit.synchronizer import ExportSynchronizer
-from openerp.addons.connector.exception import IDMissingInBackend
+from openerp.addons.connector.exception import (IDMissingInBackend,
+                                                RetryableJobError)
 from .import_synchronizer import import_record
 from ..connector import get_environment
 from ..related_action import unwrap_binding
@@ -96,6 +103,10 @@
         """ Return the raw OpenERP data for ``self.binding_id`` """
         return self.session.browse(self.model._name, self.binding_id)
 
+    def _after_export(self):
+        """ Run the after export"""
+        return
+
     def run(self, binding_id, *args, **kwargs):
         """ Run the synchronization
 
@@ -116,6 +127,12 @@
         result = self._run(*args, **kwargs)
 
         self.binder.bind(self.magento_id, self.binding_id)
+        # Commit so we keep the external ID when there are several
+        # exports (due to dependencies) and one of them fails.
+        # The commit will also release the lock acquired on the binding
+        # record
+        self.session.commit()
+        self._after_export() 
         return result
 
     def _run(self):
@@ -134,10 +151,160 @@
         super(MagentoExporter, self).__init__(environment)
         self.binding_record = None
 
+    def _lock(self):
+        """ Lock the binding record.
+
+        Lock the binding record so we are sure that only one export
+        job is running for this record if concurrent jobs have to export the
+        same record.
+
+        When concurrent jobs try to export the same record, the first one
+        will lock and proceed, the others will fail to lock and will be
+        retried later.
+
+        This behavior works also when the export becomes multilevel
+        with :meth:`_export_dependencies`. Each level will set its own lock
+        on the binding record it has to export.
+
+        """
+        sql = ("SELECT id FROM %s WHERE ID = %%s FOR UPDATE NOWAIT" %
+               self.model._table)
+        try:
+            self.session.cr.execute(sql, (self.binding_id, ),
+                                    log_exceptions=False)
+        except psycopg2.OperationalError:
+            _logger.info('A concurrent job is already exporting the same '
+                         'record (%s with id %s). Job delayed later.',
+                         self.model._name, self.binding_id)
+            raise RetryableJobError(
+                'A concurrent job is already exporting the same record '
+                '(%s with id %s). The job will be retried later.' %
+                (self.model._name, self.binding_id))
+
     def _has_to_skip(self):
         """ Return True if the export can be skipped """
         return False
 
+    @contextmanager
+    def _retry_unique_violation(self):
+        """ Context manager: catch Unique constraint error and retry the
+        job later.
+
+        When we execute several jobs workers concurrently, it happens
+        that 2 jobs are creating the same record at the same time (binding
+        record created by :meth:`_export_dependency`), resulting in:
+
+            IntegrityError: duplicate key value violates unique
+            constraint "magento_product_product_openerp_uniq"
+            DETAIL:  Key (backend_id, openerp_id)=(1, 4851) already exists.
+
+        In that case, we'll retry the import just later.
+
+        .. warning:: The unique constraint must be created on the
+                     binding record to prevent 2 bindings to be created
+                     for the same Magento record.
+
+        """
+        try:
+            yield
+        except psycopg2.IntegrityError as err:
+            if err.pgcode == psycopg2.errorcodes.UNIQUE_VIOLATION:
+                raise RetryableJobError(
+                    'A database error caused the failure of the job:\n'
+                    '%s\n\n'
+                    'Likely due to 2 concurrent jobs wanting to create '
+                    'the same record. The job will be retried later.' % err)
+            else:
+                raise
+
+    def _export_dependency(self, relation, binding_model, exporter_class=None,
+                           binding_field='magento_bind_ids'):
+        """
+        Export a dependency. The exporter class is a subclass of
+        ``MagentoExporter``. If a more precise class need to be defined,
+        it can be passed to the ``exporter_class`` keyword argument.
+
+        .. warning:: a commit is done at the end of the export of each
+                     dependency. The reason for that is that we pushed a record
+                     on the backend and we absolutely have to keep its ID.
+
+                     So you *must* take care not to modify the OpenERP
+                     database during an export, excepted when writing
+                     back the external ID or eventually to store
+                     external data that we have to keep on this side.
+
+                     You should call this method only at the beginning
+                     of the exporter synchronization,
+                     in :meth:`~._export_dependencies`.
+
+        :param relation: record to export if not already exported
+        :type relation: :py:class:`openerp.osv.orm.browse_record`
+        :param binding_model: name of the binding model for the relation
+        :type binding_model: str | unicode
+        :param exporter_cls: :py:class:`openerp.addons.connector.connector.ConnectorUnit`
+                             class or parent class to use for the export.
+                             By default: MagentoExporter
+        :type exporter_cls: :py:class:`openerp.addons.connector.connector.MetaConnectorUnit`
+        :param binding_field: name of the one2many field on a normal
+                              record that points to the binding record
+                              (default: magento_bind_ids).
+                              It is used only when the relation is not
+                              a binding but is a normal record.
+        :type binding_field: str | unicode
+        """
+        if not relation:
+            return
+        if exporter_class is None:
+            exporter_class = MagentoExporter
+        rel_binder = self.get_binder_for_model(binding_model)
+        # wrap is typically True if the relation is for instance a
+        # 'product.product' record but the binding model is
+        # 'magento.product.product'
+        wrap = relation._model._name != binding_model
+
+        if wrap and hasattr(relation, binding_field):
+            domain = [('openerp_id', '=', relation.id),
+                      ('backend_id', '=', self.backend_record.id)]
+            binding_ids = self.session.search(binding_model, domain)
+            if binding_ids:
+                assert len(binding_ids) == 1, (
+                    'only 1 binding for a backend is '
+                    'supported in _export_dependency')
+                binding_id = binding_ids[0]
+            # we are working with a unwrapped record (e.g.
+            # product.category) and the binding does not exist yet.
+            # Example: I created a product.product and its binding
+            # magento.product.product and we are exporting it, but we need to
+            # create the binding for the product.category on which it
+            # depends.
+            else:
+                with self.session.change_context({'connector_no_export': True}):
+                    with self.session.change_user(SUPERUSER_ID):
+                        bind_values = {'backend_id': self.backend_record.id,
+                                       'openerp_id': relation.id}
+                        # If 2 jobs create it at the same time, retry
+                        # one later. A unique constraint (backend_id,
+                        # openerp_id) should exist on the binding model
+                        with self._retry_unique_violation():
+                            binding_id = self.session.create(binding_model,
+                                                             bind_values)
+                            # Eager commit to avoid having 2 jobs
+                            # exporting at the same time. The constraint
+                            # will pop if an other job already created
+                            # the same binding. It will be caught and
+                            # raise a RetryableJobError.
+                            self.session.commit()
+        else:
+            # If magento_bind_ids does not exist we are typically in a
+            # "direct" binding (the binding record is the same record).
+            # If wrap is True, relation is already a binding record.
+            binding_id = relation.id
+
+        if rel_binder.to_backend(binding_id) is None:
+            exporter = self.get_connector_unit_for_model(exporter_class,
+                                                         binding_model)
+            exporter.run(binding_id)
+
     def _export_dependencies(self):
         """ Export the dependencies for the record"""
         return
@@ -194,20 +361,74 @@
         # export the missing linked resources
         self._export_dependencies()
 
+        # prevent other jobs to export the same record
+        # will be released on commit (or rollback)
+        self._lock()
+
         map_record = self._map_data()
 
         if self.magento_id:
             record = self._update_data(map_record, fields=fields)
             if not record:
                 return _('Nothing to export.')
+            # special check on data before export
+            self._validate_data(record)
             self._update(record)
         else:
             record = self._create_data(map_record, fields=fields)
             if not record:
                 return _('Nothing to export.')
             self.magento_id = self._create(record)
+        self.session.cr.commit()
         return _('Record exported with ID %s on Magento.') % self.magento_id
 
+class MagentoTranslationExporter(MagentoExporter):
+    """ A common flow for the exports record with translation to Magento """
+
+    def _get_translatable_field(self, fields):
+        # find the translatable fields of the model
+        # Note we consider that a translatable field in Magento
+        # must be a translatable field in OpenERP and vice-versa
+        # you can change this behaviour in your own module
+        all_fields = self.model.fields_get(self.session.cr, self.session.uid,
+                                   context=self.session.context)
+
+        translatable_fields = [field for field, attrs in all_fields.iteritems()
+                           if attrs.get('translate') and (not fields or field in fields)]
+        return translatable_fields
+
+
+    def _run(self, fields=None):
+        default_lang = self.backend_record.default_lang_id
+        session = self.session
+        if session.context is None:
+            session.context = {}
+        session.context['lang'] = default_lang.code
+        res = super(MagentoTranslationExporter, self)._run(fields)
+        
+        storeview_ids = session.search(
+                'magento.storeview',
+                [('backend_id', '=', self.backend_record.id)])
+        storeviews = session.browse('magento.storeview', storeview_ids)
+        lang_storeviews = [sv for sv in storeviews
+                           if sv.lang_id and sv.lang_id != default_lang]
+        if lang_storeviews:
+            translatable_fields = self._get_translatable_field(fields)   
+            if translatable_fields:
+                for storeview in lang_storeviews:
+                    session.context['lang'] = storeview.lang_id.code
+                    self.binding_record = self._get_openerp_data()
+                    self._map_data(fields=translatable_fields)
+                    record = self.mapper.data
+                    if not record:
+                        return _('nothing to export.')
+                    # special check on data before export
+                    self._validate_data(record)
+                    binder = self.get_binder_for_model('magento.storeview')
+                    magento_storeview_id = binder.to_backend(storeview.id)
+                    self.backend_adapter.write(self.magento_id, record, magento_storeview_id)
+        return res
+
 
 @job
 @related_action(action=unwrap_binding)

=== removed directory 'magentoerpconnect_catalog'
=== removed file 'magentoerpconnect_catalog/__init__.py'
--- magentoerpconnect_catalog/__init__.py	2013-03-19 15:41:34 +0000
+++ magentoerpconnect_catalog/__init__.py	1970-01-01 00:00:00 +0000
@@ -1,1 +0,0 @@
-# -*- coding: utf-8 -*-

=== removed file 'magentoerpconnect_catalog/__openerp__.py'
--- magentoerpconnect_catalog/__openerp__.py	2013-03-19 15:41:34 +0000
+++ magentoerpconnect_catalog/__openerp__.py	1970-01-01 00:00:00 +0000
@@ -1,51 +0,0 @@
-# -*- coding: utf-8 -*-
-##############################################################################
-#
-#    Author: Guewen Baconnier
-#    Copyright 2013 Camptocamp SA
-#    Copyright 2013 Akretion
-#
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU Affero General Public License as
-#    published by the Free Software Foundation, either version 3 of the
-#    License, or (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU Affero General Public License for more details.
-#
-#    You should have received a copy of the GNU Affero General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-##############################################################################
-
-
-{'name': 'Magento Connector - Catalog',
- 'version': '2.0.0',
- 'category': 'Connector',
- 'depends': ['magentoerpconnect',
-             'product_links',
-             'product_images',
-             ],
- 'author': 'MagentoERPconnect Core Editors',
- 'license': 'AGPL-3',
- 'website': 'https://launchpad.net/magentoerpconnect',
- 'description': """
-Magento Connector - Catalog
-===========================
-
-Extension for **Magento Connector**, add management of the product's catalog:
-
-* product links
-* product images
-* export of products, categories, links and images
-
-""",
- 'images': [],
- 'demo': [],
- 'data': [],
- 'installable': True,
- 'application': False,
-}
-