LoCo Team Portal

Merge lp:~daker/loco-team-portal/fix.import-live-data into lp:loco-team-portal

  1. fix.import-live-data
  2. Merge into 0.2
Proposed by Adnane Belmadiaf
Status: Merged
Approved by: Adnane Belmadiaf
Approved revision: 571
Merged at revision: 573
Proposed branch: lp:~daker/loco-team-portal/fix.import-live-data
Merge into: lp:loco-team-portal
Diff against target: 259 lines (+130/-120)
2 files modified
loco_directory/common/management/commands/import-live-data.py (+130/-0)
loco_directory/teams/management/commands/import-live-data.py (+0/-120)
To merge this branch: bzr merge lp:~daker/loco-team-portal/fix.import-live-data
Reviewer Review Type Date Requested Status
LoCo Team Portal Developers Pending
Review via email: mp+138338@code.launchpad.net

Commit message

An improved version of the import-live-data command, props ronnie

To post a comment you must log in.
lp:~daker/loco-team-portal/fix.import-live-data updated
571. By Adnane Belmadiaf

Minor fix

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== added file 'loco_directory/common/management/commands/import-live-data.py'
--- loco_directory/common/management/commands/import-live-data.py 1970-01-01 00:00:00 +0000
+++ loco_directory/common/management/commands/import-live-data.py 2012-12-05 23:30:29 +0000
@@ -0,0 +1,130 @@
1#!/usr/bin/python
2# -*- coding: utf-8 -*-
3from django.core.management.base import NoArgsCommand
4from django.contrib.auth.models import User, Group
5from django.db.models.fields import DateTimeField, DateField
6
7from meetings.models import BaseMeeting, TeamMeeting, AgendaItem
8from events.models import BaseEvent, GlobalEvent, TeamEvent, TeamEventComment, Attendee
9from teams.models import Language, Continent, Country, Team
10from venues.models import Venue
11from userprofiles.models import UserProfile
12
13import urllib2
14import json
15import re
16import os
17import sys
18import subprocess
19import time
20
21ORDER = [('languages', Language),
22 ('continents', Continent),
23 ('countries', Country),
24 ('venues', Venue),
25 ('global', GlobalEvent),
26 ('groups', Group),
27 ('users', User),
28 ('profiles', UserProfile),
29 ('teams', Team),
30 ('events', TeamEvent),
31 ('comments', TeamEventComment),
32 ('attendees', Attendee),
33 ('meeting', TeamMeeting),
34 ('agenda', AgendaItem),
35 ]
36
37date_pattern1 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$')
38date_pattern2 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+$')
39
40
41def get_field_data(model, model_data):
42 fields = {}
43 for field in model._meta.fields:
44 field_value = model_data.get(field.name, None)
45 if field_value:
46 if isinstance(field, DateTimeField):
47 if date_pattern1.match(field_value):
48 fields[field.name] = field_value
49 elif date_pattern2.match(field_value):
50 fields[field.name] = field_value.split('.')[0]
51 elif isinstance(field, DateField):
52 if date_pattern1.match(field_value) or date_pattern2.match(field_value):
53 fields[field.name] = field_value.split(' ')[0]
54 else:
55 fields[field.name] = field_value
56 for field in model._meta.many_to_many:
57 fields[field.name] = model_data.get(field.name, [])
58 return fields
59
60
61class Command(NoArgsCommand):
62 help = "Copy the live data to the local instance"
63
64 def handle_noargs(self, **options):
65 t1 = time.time()
66 # Save the super user, so it can be restored later
67 super_user = None
68 try:
69 super_user = User.objects.get(pk=1)
70 except:
71 pass
72
73 # Delete all old data, so the id's do not conflict when linking
74 print "Removing local data from database."
75
76 for service, model in ORDER:
77 model.objects.all().delete()
78
79 loaddata_json = []
80 services_url = 'http://loco.ubuntu.com/services/'
81
82 for service, model in ORDER:
83 # Get the json data from the server
84 print "Getting %s." % service
85 json_data = urllib2.urlopen("%s%s" % (services_url, service)).read()
86
87 for model_data in json.loads(json_data):
88 loaddata_json.append({'pk': model_data['id'],
89 'model': str(model._meta),
90 'fields': get_field_data(model, model_data)})
91
92 # GlobalEvent, TeamEvent are a subclass of BaseEvent, so we need to create the BaseEvent also
93 if model in [GlobalEvent, TeamEvent]:
94 loaddata_json.append({'pk': model_data['id'],
95 'model': str(BaseEvent._meta),
96 'fields': get_field_data(BaseEvent, model_data)})
97
98 # TeamMeeting is a subclass of BaseMeeting, so we need to create the BaseMeeting also
99 if model is TeamMeeting:
100 loaddata_json.append({'pk': model_data['id'],
101 'model': str(BaseMeeting._meta),
102 'fields': get_field_data(BaseMeeting, model_data)})
103
104 print "Storing data into a json file."
105 f = open('livedata.json', 'w')
106 f.write(json.dumps(loaddata_json))
107 f.close()
108
109 print "Loading data into database."
110 p = subprocess.Popen([sys.executable, 'manage.py', 'loaddata', 'livedata.json'],
111 stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
112 log = p.stdout.read()
113 p.communicate()
114 print "%s" % log
115
116 os.remove('livedata.json')
117
118 # Restore the super user
119 if super_user:
120 print"Restoring the super user."
121 old_su = super_user.__dict__
122 del old_su['id']
123 if old_su.__contains__('_state'):
124 del old_su['_state']
125 new_su, created = User.objects.get_or_create(username=old_su['username'])
126 new_su.__dict__.update(old_su)
127 new_su.save()
128
129 t2 = time.time()
130 print "It took %0fsec to finish" % (t2 - t1)
0131
=== removed file 'loco_directory/teams/management/commands/import-live-data.py'
--- loco_directory/teams/management/commands/import-live-data.py 2011-02-18 04:34:02 +0000
+++ loco_directory/teams/management/commands/import-live-data.py 1970-01-01 00:00:00 +0000
@@ -1,120 +0,0 @@
1#!/usr/bin/python
2
3from django.core.management.base import NoArgsCommand
4from django.contrib.auth.models import User, Group
5from meetings.models import TeamMeeting, AgendaItem
6from events.models import GlobalEvent, TeamEvent, TeamEventComment, Attendee
7from teams.models import Language, Continent, Country, Team
8from venues.models import Venue
9from userprofiles.models import UserProfile
10
11import sys
12import urllib2
13import json
14import datetime
15import re
16
17ORDER = [('languages', Language),
18 ('continents', Continent),
19 ('countries', Country),
20 ('venues', Venue),
21 ('global', GlobalEvent),
22 ('groups', Group),
23 ('users', User),
24 ('profiles', UserProfile),
25 ('teams', Team),
26 ('events', TeamEvent),
27 ('comments', TeamEventComment),
28 ('attendees', Attendee),
29 ('meeting', TeamMeeting),
30 ('agenda', AgendaItem),
31 ]
32
33
34class Command(NoArgsCommand):
35 help = "Copy the live data to the local instance"
36
37 def handle_noargs(self, **options):
38 # Save the super user, so it can be restored later
39 super_user = None
40 try:
41 super_user = User.objects.get(pk=1)
42 except:
43 pass
44
45 # Delete all old data, so the id's do not conflict when linking
46 print 'Removing local data from database...'
47 for service, Model in ORDER:
48 Model.objects.all().delete()
49
50 services_url = 'http://loco.ubuntu.com/services/'
51 date_pattern1 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$')
52 date_pattern2 = re.compile('^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+$')
53
54 for service, Model in ORDER:
55 # Get the json data from the server
56 print 'Getting', service, '...'
57 json_data = urllib2.urlopen(services_url+service).read()
58
59 # Find out what fields are normal, and which many_to_many
60 local_fields = Model._meta.fields
61 many_to_many = Model._meta.many_to_many
62
63 for entry in json.loads(json_data):
64 # Create the model without the many_to_many fields
65 params = {}
66 for field in local_fields:
67 if entry.has_key(field.name):
68 field_value = entry[field.name]
69 if field_value == None:
70 if field.null == True:
71 # If field on the server is None, but None is not allowed, get the default value
72 field_value = field.get_default()
73 else:
74 # If the field is a related, grab the object with that id
75 if field.rel:
76 rel_model = field.rel.to
77 try:
78 params[field.name] = rel_model.objects.get(pk=field_value)
79 except rel_model.DoesNotExist:
80 print '%(model)s with pk=%(pk)s does not exists' % ({'model':rel_model._meta.object_name, 'pk':field_value})
81 # Else if it is a date, create a date object
82 elif isinstance(field_value, unicode) and date_pattern1.match(field_value):
83 params[field.name] = datetime.datetime.strptime(field_value, '%Y-%m-%d %H:%M:%S')
84 elif isinstance(field_value, unicode) and date_pattern2.match(field_value):
85 params[field.name] = datetime.datetime.strptime(field_value, '%Y-%m-%d %H:%M:%S.%f')
86 # Else: copy the info
87 else:
88 params[field.name] = field_value
89
90 model_instance = Model(**params)
91 model_instance.save()
92
93 # Add the related object to the model
94 for field in many_to_many:
95 if entry.has_key(field.name):
96 for object_id in entry[field.name]:
97 many_related_manager = getattr(model_instance, field.name)
98 rel_model = many_related_manager.model
99 try:
100 related_object = rel_model.objects.get(pk=object_id)
101 except rel_model.DoesNotExist:
102 print '%(model)s with pk=%(pk)s does not exists' % ({'model':rel_model, 'pk':object_id})
103 else:
104 many_related_manager.add(related_object)
105
106 # Save the model to the database, if changed by many to many fields
107 if many_to_many:
108 model_instance.save()
109
110 # Restore the super user
111 if super_user:
112 old_su = super_user.__dict__
113 del old_su['id']
114 if old_su.__contains__('_state'):
115 del old_su['_state']
116 new_su, created = User.objects.get_or_create(username=old_su['username'])
117 new_su.__dict__.update(old_su)
118 new_su.save()
119
120

Subscribers

People subscribed via source and target branches