Merge lp:~ronnie.vd.c/loco-team-portal/import-live-data-improved into lp:loco-team-portal

[Ronnie]

[Ronnie (ronnie.vd.c)]

Meetings are not imported correct

review: Needs Fixing

[lp:~ronnie.vd.c/loco-team-portal/import-live-data-improved]

530. By Ronnie on 2012-05-28

Fixed importing meetings

[Chris Johnston (cjohnston)]

May also want to look at a couple of the changes that we made to Summit:

http://bazaar.launchpad.net/~summit-hackers/summit/trunk/view/head:/summit/schedule/management/commands/import_live_data.py

If I remember correctly, the users are imported and then you just appends new users to the end of the list.

[Adnane Belmadiaf (daker)]

I have created a new MP with a new code and some improvements https://code.launchpad.net/~daker/loco-team-portal/fix.import-live-data/+merge/138338

[Adnane Belmadiaf (daker)]

Merged in trunk with some modifications

[Adnane Belmadiaf (daker)]

review: Approve

=== modified file 'loco_directory/teams/management/commands/import-live-data.py'
--- loco_directory/teams/management/commands/import-live-data.py	2011-02-18 04:34:02 +0000
+++ loco_directory/teams/management/commands/import-live-data.py	2012-05-28 21:59:17 +0000
@@ -1,9 +1,12 @@
 #!/usr/bin/python
+import os
+import subprocess
 
 from django.core.management.base import NoArgsCommand
 from django.contrib.auth.models import User, Group
-from meetings.models import TeamMeeting, AgendaItem
-from events.models import GlobalEvent, TeamEvent, TeamEventComment, Attendee
+from django.db.models.fields import DateTimeField, DateField
+from meetings.models import BaseMeeting, TeamMeeting, AgendaItem
+from events.models import BaseEvent, GlobalEvent, TeamEvent, TeamEventComment, Attendee
 from teams.models import Language, Continent, Country, Team
 from venues.models import Venue
 from userprofiles.models import UserProfile
@@ -30,6 +33,28 @@
          ('agenda', AgendaItem),
          ]
 
+date_pattern1 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$')
+date_pattern2 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+$')
+
+def get_field_data(model, model_data):
+    fields = {}
+    for field in model._meta.fields:
+        field_value = model_data.get(field.name, None)
+        if field_value:
+            if isinstance(field, DateTimeField):
+                if date_pattern1.match(field_value):
+                    fields[field.name] = field_value
+                elif date_pattern2.match(field_value):
+                    fields[field.name] = field_value.split('.')[0]
+            elif isinstance(field, DateField):
+                if date_pattern1.match(field_value) or date_pattern2.match(field_value):
+                    fields[field.name] = field_value.split(' ')[0]
+            else:
+                fields[field.name] = field_value
+    for field in model._meta.many_to_many:
+        fields[field.name] = model_data.get(field.name, [])
+    return fields
+
 
 class Command(NoArgsCommand):
     help = "Copy the live data to the local instance"
@@ -44,68 +69,47 @@
 
         # Delete all old data, so the id's do not conflict when linking
         print 'Removing local data from database...'
-        for service, Model in ORDER:
-            Model.objects.all().delete()
+        for service, model in ORDER:
+            model.objects.all().delete()
         
+        loaddata_json = []
         services_url = 'http://loco.ubuntu.com/services/'
-        date_pattern1 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$')
-        date_pattern2 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+$')
-
-        for service, Model in ORDER:
+        
+        for service, model in ORDER:
             # Get the json data from the server
             print 'Getting', service, '...'
             json_data = urllib2.urlopen(services_url+service).read()
             
-            # Find out what fields are normal, and which many_to_many
-            local_fields = Model._meta.fields
-            many_to_many = Model._meta.many_to_many
-            
-            for entry in json.loads(json_data):
-                # Create the model without the many_to_many fields
-                params = {}
-                for field in local_fields:
-                    if entry.has_key(field.name):
-                        field_value = entry[field.name]
-                        if field_value == None:
-                            if field.null == True: 
-                                # If field on the server is None, but None is not allowed, get the default value
-                                field_value = field.get_default()
-                        else:
-                            # If the field is a related, grab the object with that id
-                            if field.rel:
-                                rel_model = field.rel.to
-                                try:
-                                    params[field.name] = rel_model.objects.get(pk=field_value)
-                                except rel_model.DoesNotExist:
-                                    print '%(model)s with pk=%(pk)s does not exists' % ({'model':rel_model._meta.object_name, 'pk':field_value})
-                            # Else if it is a date, create a date object
-                            elif isinstance(field_value, unicode) and date_pattern1.match(field_value):
-                                params[field.name] = datetime.datetime.strptime(field_value, '%Y-%m-%d %H:%M:%S')
-                            elif isinstance(field_value, unicode) and date_pattern2.match(field_value):
-                                params[field.name] = datetime.datetime.strptime(field_value, '%Y-%m-%d %H:%M:%S.%f')
-                            # Else: copy the info
-                            else:
-                                params[field.name] = field_value
-                
-                model_instance = Model(**params)
-                model_instance.save()
+            for model_data in json.loads(json_data):
+                loaddata_json.append({'pk': model_data['id'],
+                                      'model': str(model._meta), 
+                                      'fields': get_field_data(model, model_data)})
+                                      
+                # GlobalEvent, TeamEvent are a subclass of BaseEvent, so we need to create the BaseEvent also   
+                if model in [GlobalEvent, TeamEvent]:
+                    loaddata_json.append({'pk': model_data['id'],
+                                          'model': str(BaseEvent._meta), 
+                                          'fields': get_field_data(BaseEvent, model_data)})
+                                          
+                # TeamMeeting is a subclass of BaseMeeting, so we need to create the BaseMeeting also   
+                if model is TeamMeeting:
+                    loaddata_json.append({'pk': model_data['id'],
+                                          'model': str(BaseMeeting._meta), 
+                                          'fields': get_field_data(BaseMeeting, model_data)})
 
-                # Add the related object to the model
-                for field in many_to_many:
-                    if entry.has_key(field.name):
-                        for object_id in entry[field.name]:
-                            many_related_manager = getattr(model_instance, field.name)
-                            rel_model = many_related_manager.model
-                            try:
-                                related_object = rel_model.objects.get(pk=object_id)
-                            except rel_model.DoesNotExist:
-                                print '%(model)s with pk=%(pk)s does not exists' % ({'model':rel_model, 'pk':object_id})
-                            else:
-                                many_related_manager.add(related_object)
-                
-                # Save the model to the database, if changed by many to many fields
-                if many_to_many:
-                    model_instance.save()
+        
+        print 'loading live data into database...'
+        f = open('livedata.json', 'w')
+        f.write(json.dumps(loaddata_json))
+        f.close()
+        
+        p = subprocess.Popen(['python', 'manage.py', 'loaddata', 'livedata.json'], stdout=subprocess.PIPE, 
+                             stderr=subprocess.STDOUT, close_fds=True)
+        log = p.stdout.read()
+        p.communicate()
+        print log
+        
+        os.remove('livedata.json')
         
         # Restore the super user
         if super_user:         
@@ -117,4 +121,3 @@
             new_su.__dict__.update(old_su)
             new_su.save()
         
-