Merge lp:~therp-nl/openupgrade-server/7.0-address_deduplication into lp:openupgrade-server

[Stefan Rijnhart (Opener)]

Don't merge, this is a tentative/optional change.

Due to the way that one of our customers imported their addresses, their database contained a large amount of address duplicates. In the new partner model, duplicate addresses are very annoying. This branch performs a simple deduplication, which is quite transparent within the address-to-partner mechanism.

I don't think this is very useful for general usage though, so setting to work in progress.

Unmerged revisions

4628. By Stefan Rijnhart (Opener) on 2013-08-16

[IMP] Perform address deduplication

=== modified file 'openerp/addons/base/migrations/7.0.1.3/post-migration.py'
--- openerp/addons/base/migrations/7.0.1.3/post-migration.py	2013-07-25 06:42:40 +0000
+++ openerp/addons/base/migrations/7.0.1.3/post-migration.py	2013-08-16 10:40:28 +0000
@@ -19,6 +19,7 @@
 #
 ##############################################################################
 
+import logging
 from openupgrade import openupgrade
 from openerp import pooler, SUPERUSER_ID
 
@@ -76,8 +77,14 @@
         'mobile', 'phone', 'state_id', 'street', 'street2', 'type', 'zip',
         'partner_id', 'name',
         ]
+    dedup_fields = [
+        'birthdate', 'city', 'country_id', 'email', 'fax', 'function',
+        'mobile', 'phone', 'state_id', 'street', 'street2', 'zip',
+        ]
     partner_found = []
     processed_ids = []
+    reprs = {}
+    logger = logging.getLogger('OpenUpgrade.base')
 
     def set_address_partner(address_id, partner_id):
         cr.execute(
@@ -93,17 +100,22 @@
         already in vals. Register the created partner_id
         on the obsolete address table
         """
+
         for key in defaults:
             if key not in vals:
                 vals[key] = defaults[key]
 
         partner_id = partner_obj.create(cr, SUPERUSER_ID, vals)
         set_address_partner(address_id, partner_id)
+        return partner_id
 
     def process_address_type(cr, whereclause, args=None):
         """
         Migrate addresses to partners, based on sql WHERE clause
+        
+        :return: number of squashed duplicates
         """
+        duplicates = 0
         cr.execute(
             "SELECT " + ', '.join(fields) + " FROM res_partner_address "
             "WHERE " + whereclause, args or ())
@@ -126,31 +138,50 @@
                 # not supplier and not customer
                 create_partner(address['id'], partner_vals, partner_defaults)
             else:
+
+                # Create duplication key
+                representation = unicode(
+                    [address[field] for field in dedup_fields])
+                # Squash duplicates
+                if representation in reprs.get(address['partner_id'], {}):
+                    set_address_partner(address['id'], reprs[address['partner_id']][representation])
+                    duplicates += 1
+                    continue
                 if address['partner_id'] not in partner_found:
                     # Main partner address
                     partner_obj.write(
                         cr, SUPERUSER_ID, address['partner_id'], partner_vals)
                     partner_found.append(address['partner_id'])
+                    partner_id = address['partner_id']
                     set_address_partner(address['id'], address['partner_id'])
                 else:
                     # any following address for an existing partner
                     partner_vals.update({
                             'is_company': False,
                             'parent_id': address['partner_id']})
-                    create_partner(
+                    partner_id = create_partner(
                         address['id'], partner_vals, partner_defaults)
+
+                # Fill the deduplication search space with default addresses
+                # or the else the first address found 
+                if (not reprs.get(address['partner_id'])
+                       or not address['type'] or address['type'] == 'default'):
+                    reprs.setdefault(address['partner_id'], {})[representation] = partner_id
+
             processed_ids.append(address['id'])
+        return duplicates
 
     # Process all addresses, default type first 
-    process_address_type(cr, "type = 'default'")
-    process_address_type(cr, "type IS NULL OR type = ''")
-    process_address_type(cr, "id NOT IN %s", (tuple(processed_ids),))
+    duplicates = process_address_type(cr, "type = 'default'")
+    duplicates += process_address_type(cr, "type IS NULL OR type = ''")
+    duplicates += process_address_type(cr, "id NOT IN %s", (tuple(processed_ids),))
 
     # Check that all addresses have been migrated
     cr.execute(
         "SELECT COUNT(*) FROM res_partner_address "
         "WHERE openupgrade_7_migrated_to_partner_id is NULL ")
     assert(not cr.fetchone()[0])
+    logger.debug('Squashed %s address duplicates', duplicates)
 
 def update_users_partner(cr, pool):
     """