Merge lp:~daker/loco-team-portal/fix.import-live-data into lp:loco-team-portal

[Adnane Belmadiaf]

An improved version of the import-live-data command, props ronnie

[lp:~daker/loco-team-portal/fix.import-live-data]

571. By Adnane Belmadiaf on 2012-12-05

Minor fix

=== added file 'loco_directory/common/management/commands/import-live-data.py'
--- loco_directory/common/management/commands/import-live-data.py	1970-01-01 00:00:00 +0000
+++ loco_directory/common/management/commands/import-live-data.py	2012-12-05 23:30:29 +0000
@@ -0,0 +1,130 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+from django.core.management.base import NoArgsCommand
+from django.contrib.auth.models import User, Group
+from django.db.models.fields import DateTimeField, DateField
+
+from meetings.models import BaseMeeting, TeamMeeting, AgendaItem
+from events.models import BaseEvent, GlobalEvent, TeamEvent, TeamEventComment, Attendee
+from teams.models import Language, Continent, Country, Team
+from venues.models import Venue
+from userprofiles.models import UserProfile
+
+import urllib2
+import json
+import re
+import os
+import sys
+import subprocess
+import time
+
+ORDER = [('languages', Language),
+         ('continents', Continent),
+         ('countries', Country),
+         ('venues', Venue),
+         ('global', GlobalEvent),
+         ('groups', Group),
+         ('users', User),
+         ('profiles', UserProfile),
+         ('teams', Team),
+         ('events', TeamEvent),
+         ('comments', TeamEventComment),
+         ('attendees', Attendee),
+         ('meeting', TeamMeeting),
+         ('agenda', AgendaItem),
+         ]
+
+date_pattern1 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$')
+date_pattern2 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+$')
+
+
+def get_field_data(model, model_data):
+    fields = {}
+    for field in model._meta.fields:
+        field_value = model_data.get(field.name, None)
+        if field_value:
+            if isinstance(field, DateTimeField):
+                if date_pattern1.match(field_value):
+                    fields[field.name] = field_value
+                elif date_pattern2.match(field_value):
+                    fields[field.name] = field_value.split('.')[0]
+            elif isinstance(field, DateField):
+                if date_pattern1.match(field_value) or date_pattern2.match(field_value):
+                    fields[field.name] = field_value.split(' ')[0]
+            else:
+                fields[field.name] = field_value
+    for field in model._meta.many_to_many:
+        fields[field.name] = model_data.get(field.name, [])
+    return fields
+
+
+class Command(NoArgsCommand):
+    help = "Copy the live data to the local instance"
+
+    def handle_noargs(self, **options):
+        t1 = time.time()
+        # Save the super user, so it can be restored later
+        super_user = None
+        try:
+            super_user = User.objects.get(pk=1)
+        except:
+            pass
+
+        # Delete all old data, so the id's do not conflict when linking
+        print "Removing local data from database."
+
+        for service, model in ORDER:
+            model.objects.all().delete()
+
+        loaddata_json = []
+        services_url = 'http://loco.ubuntu.com/services/'
+
+        for service, model in ORDER:
+            # Get the json data from the server
+            print "Getting %s." % service
+            json_data = urllib2.urlopen("%s%s" % (services_url, service)).read()
+
+            for model_data in json.loads(json_data):
+                loaddata_json.append({'pk': model_data['id'],
+                                      'model': str(model._meta),
+                                      'fields': get_field_data(model, model_data)})
+
+                # GlobalEvent, TeamEvent are a subclass of BaseEvent, so we need to create the BaseEvent also
+                if model in [GlobalEvent, TeamEvent]:
+                    loaddata_json.append({'pk': model_data['id'],
+                                          'model': str(BaseEvent._meta),
+                                          'fields': get_field_data(BaseEvent, model_data)})
+
+                # TeamMeeting is a subclass of BaseMeeting, so we need to create the BaseMeeting also
+                if model is TeamMeeting:
+                    loaddata_json.append({'pk': model_data['id'],
+                                          'model': str(BaseMeeting._meta),
+                                          'fields': get_field_data(BaseMeeting, model_data)})
+
+        print "Storing data into a json file."
+        f = open('livedata.json', 'w')
+        f.write(json.dumps(loaddata_json))
+        f.close()
+
+        print "Loading data into database."
+        p = subprocess.Popen([sys.executable, 'manage.py', 'loaddata', 'livedata.json'],
+                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
+        log = p.stdout.read()
+        p.communicate()
+        print "%s" % log
+
+        os.remove('livedata.json')
+
+        # Restore the super user
+        if super_user:
+            print"Restoring the super user."
+            old_su = super_user.__dict__
+            del old_su['id']
+            if old_su.__contains__('_state'):
+                del old_su['_state']
+            new_su, created = User.objects.get_or_create(username=old_su['username'])
+            new_su.__dict__.update(old_su)
+            new_su.save()
+
+        t2 = time.time()
+        print "It took %0fsec to finish" % (t2 - t1)

=== removed file 'loco_directory/teams/management/commands/import-live-data.py'
--- loco_directory/teams/management/commands/import-live-data.py	2011-02-18 04:34:02 +0000
+++ loco_directory/teams/management/commands/import-live-data.py	1970-01-01 00:00:00 +0000
@@ -1,120 +0,0 @@
-#!/usr/bin/python
-
-from django.core.management.base import NoArgsCommand
-from django.contrib.auth.models import User, Group
-from meetings.models import TeamMeeting, AgendaItem
-from events.models import GlobalEvent, TeamEvent, TeamEventComment, Attendee
-from teams.models import Language, Continent, Country, Team
-from venues.models import Venue
-from userprofiles.models import UserProfile
-
-import sys
-import urllib2
-import json
-import datetime
-import re
-
-ORDER = [('languages', Language),
-         ('continents', Continent),
-         ('countries', Country),
-         ('venues', Venue),
-         ('global', GlobalEvent),
-         ('groups', Group),
-         ('users', User),
-         ('profiles', UserProfile),
-         ('teams', Team),
-         ('events', TeamEvent),
-         ('comments', TeamEventComment),
-         ('attendees', Attendee),
-         ('meeting', TeamMeeting),
-         ('agenda', AgendaItem),
-         ]
-
-
-class Command(NoArgsCommand):
-    help = "Copy the live data to the local instance"
-
-    def handle_noargs(self, **options):
-        # Save the super user, so it can be restored later
-        super_user = None
-        try:
-            super_user = User.objects.get(pk=1)
-        except:
-            pass
-
-        # Delete all old data, so the id's do not conflict when linking
-        print 'Removing local data from database...'
-        for service, Model in ORDER:
-            Model.objects.all().delete()
-        
-        services_url = 'http://loco.ubuntu.com/services/'
-        date_pattern1 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$')
-        date_pattern2 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+$')
-
-        for service, Model in ORDER:
-            # Get the json data from the server
-            print 'Getting', service, '...'
-            json_data = urllib2.urlopen(services_url+service).read()
-            
-            # Find out what fields are normal, and which many_to_many
-            local_fields = Model._meta.fields
-            many_to_many = Model._meta.many_to_many
-            
-            for entry in json.loads(json_data):
-                # Create the model without the many_to_many fields
-                params = {}
-                for field in local_fields:
-                    if entry.has_key(field.name):
-                        field_value = entry[field.name]
-                        if field_value == None:
-                            if field.null == True: 
-                                # If field on the server is None, but None is not allowed, get the default value
-                                field_value = field.get_default()
-                        else:
-                            # If the field is a related, grab the object with that id
-                            if field.rel:
-                                rel_model = field.rel.to
-                                try:
-                                    params[field.name] = rel_model.objects.get(pk=field_value)
-                                except rel_model.DoesNotExist:
-                                    print '%(model)s with pk=%(pk)s does not exists' % ({'model':rel_model._meta.object_name, 'pk':field_value})
-                            # Else if it is a date, create a date object
-                            elif isinstance(field_value, unicode) and date_pattern1.match(field_value):
-                                params[field.name] = datetime.datetime.strptime(field_value, '%Y-%m-%d %H:%M:%S')
-                            elif isinstance(field_value, unicode) and date_pattern2.match(field_value):
-                                params[field.name] = datetime.datetime.strptime(field_value, '%Y-%m-%d %H:%M:%S.%f')
-                            # Else: copy the info
-                            else:
-                                params[field.name] = field_value
-                
-                model_instance = Model(**params)
-                model_instance.save()
-
-                # Add the related object to the model
-                for field in many_to_many:
-                    if entry.has_key(field.name):
-                        for object_id in entry[field.name]:
-                            many_related_manager = getattr(model_instance, field.name)
-                            rel_model = many_related_manager.model
-                            try:
-                                related_object = rel_model.objects.get(pk=object_id)
-                            except rel_model.DoesNotExist:
-                                print '%(model)s with pk=%(pk)s does not exists' % ({'model':rel_model, 'pk':object_id})
-                            else:
-                                many_related_manager.add(related_object)
-                
-                # Save the model to the database, if changed by many to many fields
-                if many_to_many:
-                    model_instance.save()
-        
-        # Restore the super user
-        if super_user:         
-            old_su = super_user.__dict__
-            del old_su['id']
-            if old_su.__contains__('_state'):
-                del old_su['_state']
-            new_su, created = User.objects.get_or_create(username=old_su['username'])
-            new_su.__dict__.update(old_su)
-            new_su.save()
-        
-