Merge lp:~ronnie.vd.c/loco-team-portal/611304 into lp:loco-team-portal

[Ronnie]

Importing data from the Live (loco.ubuntu.com) server and put it in the local database

[lp:~ronnie.vd.c/loco-team-portal/611304]

350. By Ronnie on 2010-12-20

Using the same id as the server, and solved Venue.save()

351. By Ronnie on 2010-12-21

Fixed super user collision and missing super user

[Chris Johnston (cjohnston)]

Looks good to me. Would like a second opinion before merging though please.

review: Approve

[Michael Hall (mhall119)]

Getting attendees ...
Traceback (most recent call last):
  File "./manage.py", line 11, in <module>
    execute_manager(settings)
  File "/home/mhall/projects/Ubuntu/locodir/loco-directory/loco_directory/django/core/management/__init__.py", line 362, in execute_manager
    utility.execute()
  File "/home/mhall/projects/Ubuntu/locodir/loco-directory/loco_directory/django/core/management/__init__.py", line 303, in execute
    self.fetch_command(subcommand).run_from_argv(self.argv)
  File "/home/mhall/projects/Ubuntu/locodir/loco-directory/loco_directory/django/core/management/base.py", line 195, in run_from_argv
    self.execute(*args, **options.__dict__)
  File "/home/mhall/projects/Ubuntu/locodir/loco-directory/loco_directory/django/core/management/base.py", line 222, in execute
    output = self.handle(*args, **options)
  File "/home/mhall/projects/Ubuntu/locodir/loco-directory/loco_directory/django/core/management/base.py", line 351, in handle
    return self.handle_noargs(**options)
  File "/home/mhall/projects/Ubuntu/locodir/loco-directory/loco_directory/teams/management/commands/import-live-data.py", line 110, in handle_noargs
    del old_su['_state']
KeyError: '_state'

[Ronnie (ronnie.vd.c)]

it could be that not every user has a '_state' key. I included a check for the key, before deleting it.

It should work now

[lp:~ronnie.vd.c/loco-team-portal/611304]

352. By Ronnie on 2010-12-30

added check for key '_state' in user object

=== added file 'loco_directory/teams/management/commands/import-live-data.py'
--- loco_directory/teams/management/commands/import-live-data.py	1970-01-01 00:00:00 +0000
+++ loco_directory/teams/management/commands/import-live-data.py	2010-12-30 13:28:10 +0000
@@ -0,0 +1,116 @@
+#!/usr/bin/python
+
+from django.core.management.base import NoArgsCommand
+from django.contrib.auth.models import User, Group
+from events.models import GlobalEvent, TeamEvent, TeamEventComment, Attendee
+from teams.models import Language, Continent, Country, Team
+from venues.models import Venue
+from userprofiles.models import UserProfile
+
+import sys
+import urllib2
+import json
+import datetime
+import re
+
+ORDER = [('languages', Language),
+         ('continents', Continent),
+         ('countries', Country),
+         ('venues', Venue),
+         ('global', GlobalEvent),
+         ('groups', Group),
+         ('users', User),
+         ('profiles', UserProfile),
+         ('teams', Team),
+         ('events', TeamEvent),
+         ('comments', TeamEventComment),
+         ('attendees', Attendee),]
+
+
+class Command(NoArgsCommand):
+    help = "Copy the live data to the local instance"
+
+    def handle_noargs(self, **options):
+        # Save the super user, so it can be restored later
+        super_user = None
+        try:
+            super_user = User.objects.get(pk=1)
+        except:
+            pass
+
+        # Delete all old data, so the id's do not conflict when linking
+        print 'Removing local data from database...'
+        for service, Model in ORDER:
+            Model.objects.all().delete()
+        
+        services_url = 'http://loco.ubuntu.com/services/'
+        date_pattern1 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$')
+        date_pattern2 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+$')
+
+        for service, Model in ORDER:
+            # Get the json data from the server
+            print 'Getting', service, '...'
+            json_data = urllib2.urlopen(services_url+service).read()
+            
+            # Find out what fields are normal, and which many_to_many
+            local_fields = Model._meta.fields
+            many_to_many = Model._meta.many_to_many
+            
+            for entry in json.loads(json_data):
+                # Create the model without the many_to_many fields
+                params = {}
+                for field in local_fields:
+                    if entry.has_key(field.name):
+                        field_value = entry[field.name]
+                        if field_value == None:
+                            if field.null == True: 
+                                # If field on the server is None, but None is not allowed, get the default value
+                                field_value = field.get_default()
+                        else:
+                            # If the field is a related, grab the object with that id
+                            if field.rel:
+                                rel_model = field.rel.to
+                                try:
+                                    params[field.name] = rel_model.objects.get(pk=field_value)
+                                except rel_model.DoesNotExist:
+                                    print '%(model)s with pk=%(pk)s does not exists' % ({'model':rel_model._meta.object_name, 'pk':field_value})
+                            # Else if it is a date, create a date object
+                            elif isinstance(field_value, unicode) and date_pattern1.match(field_value):
+                                params[field.name] = datetime.datetime.strptime(field_value, '%Y-%m-%d %H:%M:%S')
+                            elif isinstance(field_value, unicode) and date_pattern2.match(field_value):
+                                params[field.name] = datetime.datetime.strptime(field_value, '%Y-%m-%d %H:%M:%S.%f')
+                            # Else: copy the info
+                            else:
+                                params[field.name] = field_value
+                
+                model_instance = Model(**params)
+                model_instance.save()
+
+                # Add the related object to the model
+                for field in many_to_many:
+                    if entry.has_key(field.name):
+                        for object_id in entry[field.name]:
+                            many_related_manager = getattr(model_instance, field.name)
+                            rel_model = many_related_manager.model
+                            try:
+                                related_object = rel_model.objects.get(pk=object_id)
+                            except rel_model.DoesNotExist:
+                                print '%(model)s with pk=%(pk)s does not exists' % ({'model':rel_model, 'pk':object_id})
+                            else:
+                                many_related_manager.add(related_object)
+                
+                # Save the model to the database, if changed by many to many fields
+                if many_to_many:
+                    model_instance.save()
+        
+        # Restore the super user
+        if super_user:         
+            old_su = super_user.__dict__
+            del old_su['id']
+            if old_su.__contains__('_state'):
+                del old_su['_state']
+            new_su, created = User.objects.get_or_create(username=old_su['username'])
+            new_su.__dict__.update(old_su)
+            new_su.save()
+        
+

=== modified file 'loco_directory/venues/models.py'
--- loco_directory/venues/models.py	2010-11-27 03:36:17 +0000
+++ loco_directory/venues/models.py	2010-12-30 13:28:10 +0000
@@ -64,19 +64,21 @@
     def save(self, *args, **kargs):
         if self.id:
             from events.models import TeamEvent
-            old = Venue.objects.get(id=self.id)
-            retval = super(Venue, self).save(*args, **kargs)
-            if old.tz != self.tz:
-                events = TeamEvent.objects.filter(venue=self)
-                for e in events:
-                    old_offset = old.tolocaltime(e.date_begin).utcoffset()
-                    new_offset = self.tolocaltime(e.date_begin).utcoffset()
-                    e.date_begin = e.date_begin + (old_offset - new_offset)
-                    e.date_end = e.date_end + (old_offset - new_offset)
-                    e.save()
-        else:
-            retval = super(Venue, self).save(*args, **kargs)
-        return retval
+            try:
+                old = Venue.objects.get(id=self.id)
+            except Venue.DoesNotExist:
+                # Object is new, but has a fixed id'
+                pass
+            else:
+                if old.tz != self.tz:
+                    events = TeamEvent.objects.filter(venue=self)
+                    for e in events:
+                        old_offset = old.tolocaltime(e.date_begin).utcoffset()
+                        new_offset = self.tolocaltime(e.date_begin).utcoffset()
+                        e.date_begin = e.date_begin + (old_offset - new_offset)
+                        e.date_end = e.date_end + (old_offset - new_offset)
+                        e.save()
+        return super(Venue, self).save(*args, **kargs)
                 
 def venues_without_country():
     return Venue.objects.filter(country__isnull=True)