Merge lp:~andrewsomething/dat-overview/lp1189808 into lp:dat-overview

[Andrew Starr-Bochicchio]

[Daniel Holbach (dholbach)]

Memory usage still goes up considerably during the process (top said 76% on a machine with 1.5G mem + 2G swap), but at least it makes it through a full pass. :-)

review: Approve

=== modified file 'overview/uploads/management/commands/migrate-upload-data.py'
--- overview/uploads/management/commands/migrate-upload-data.py	2013-06-11 09:08:16 +0000
+++ overview/uploads/management/commands/migrate-upload-data.py	2013-07-10 20:21:25 +0000
@@ -19,7 +19,7 @@
             latest_entry = Uploads.objects.values(t).latest(t)[t]
         except ObjectDoesNotExist:
             latest_entry = None
-        for row in cursor.fetchall():
+        for row in self.row_iter(cursor):
             if latest_entry is None or row[0] > latest_entry:
                 if row[4] == row[6] or row[5] == row[7]:
                     spon_email = ''
@@ -30,10 +30,25 @@
                 uploads = Uploads(timestamp=row[0], release=row[1],
                                   package=row[2], version=row[3],
                                   name_changer=row[4], email_changer=row[5],
-                                  name_sponsor=spon_name, email_sponsor=spon_email,
+                                  name_sponsor=spon_name,
+                                  email_sponsor=spon_email,
                                   lpid_changer='' ,lpid_sponsor='')
-                bulk_insert.append(uploads)
+                if len(bulk_insert) == 1000:
+                    Uploads.objects.bulk_create(bulk_insert)
+                    bulk_insert = []
+                    bulk_insert.append(uploads)
+                else:
+                    bulk_insert.append(uploads)
         Uploads.objects.bulk_create(bulk_insert)
+        cursor.close()
+
+    def row_iter(self, cursor, size=1000):
+        while True:
+            rows = cursor.fetchmany(size)
+            if not rows:
+                break
+            for row in rows:
+                yield row
 
     def add_lpids(self):
         self.launchpad = lp('d-a-t', anonymous=True, lp_service='production')
@@ -42,7 +57,8 @@
                                                      flat=True).distinct()
         for e in changer_emails:
             uploads = Uploads.objects.filter(email_changer=e)
-            if uploads[0].lpid_changer == '' and (uploads[0].email_changer not in ('', 'N/A')):
+            if uploads[0].lpid_changer == '' and (
+                uploads[0].email_changer not in ('', 'N/A')):
                 lpid = self.email_to_lp(e)
                 if lpid != '':
                     for ul in uploads.filter(lpid_changer=''):
@@ -58,7 +74,8 @@
                                                      flat=True).distinct()
         for e in sponsor_emails:
             uploads = Uploads.objects.filter(email_sponsor=e)
-            if uploads[0].lpid_sponsor == '' and (uploads[0].email_sponsor not in ('', 'N/A')):
+            if uploads[0].lpid_sponsor == '' and (
+                uploads[0].email_sponsor not in ('', 'N/A')):
                 lpid = self.email_to_lp(e)
                 if lpid != '':
                     for ul in uploads.filter(lpid_sponsor=''):