Merge lp:~andrewsomething/dat-overview/use_new_dump into lp:dat-overview

[Andrew Starr-Bochicchio]

This uses the new smaller udd dump. This fixes lp: #1188634, as we don't need to read it line by line to "extract_good_parts" I dropped the code that checks whether or not the file needs updating. Running 'manage.py update-all' does its own lock/stamp file dance, so there's no need to bother with it there as well.

I had to start calling "CREATE EXTENSION IF NOT EXISTS debversion" in the udd database after creating it. I'm not sure why that wasn't a problem before...

[Daniel Holbach (dholbach)]

Fantastic work!

review: Approve

=== removed file 'overview/uploads/common/udd.py'
--- overview/uploads/common/udd.py	2013-04-08 17:03:42 +0000
+++ overview/uploads/common/udd.py	1970-01-01 00:00:00 +0000
@@ -1,64 +0,0 @@
-import subprocess
-import urllib
-import codecs
-import time
-import re
-import os
-
-from django.conf import settings
-
-UPDATE_INTERVAL = 60 * 60 * 24
-URL = "http://udd.debian.org/udd.sql.gz"
-LOCAL_FILE = os.path.join(settings.TEMP_PATH, "udd.sql.gz")
-UNZIPPED_FILE = os.path.join(settings.TEMP_PATH, "udd.sql")
-UPDATED_FILE = os.path.join(settings.TEMP_PATH, "new-udd.sql")
-
-def needs_update(data_file):
-    return (not os.path.exists(data_file)) or \
-           (time.time()-os.path.getctime(data_file) >= UPDATE_INTERVAL)
-
-def extract_good_parts():
-    new_script = []
-    stuff = [ "CREATE EXTENSION IF NOT EXISTS debversion WITH SCHEMA public;",
-              "CREATE TABLE ubuntu_upload_history (",
-              "COPY ubuntu_upload_history (",
-              "ALTER TABLE ONLY ubuntu_upload_history",
-              "REVOKE ALL ON TABLE ubuntu_upload_history FROM PUBLIC;",
-              "REVOKE ALL ON TABLE ubuntu_upload_history FROM udd;",
-              "GRANT ALL ON TABLE ubuntu_upload_history TO udd;",
-              "GRANT SELECT ON TABLE ubuntu_upload_history TO PUBLIC;",
-            ]
-    copy = False
-    with codecs.open(UNZIPPED_FILE, mode="r", encoding="utf-8", errors="replace") as f:
-        for line in f:
-            if filter(lambda a: line.startswith(a), stuff):
-                copy = True
-            if not line.strip():
-                copy = False
-            if copy:
-                new_line = re.sub("\\n", "\n", unicode(line.strip()))
-                new_script += [ new_line ]
-    new_script += [""]
-    if os.path.exists(UPDATED_FILE):
-        os.remove(UPDATED_FILE)
-    with codecs.open(UPDATED_FILE, mode="w", encoding="utf-8", errors="replace") as f:
-        f.write(u"\n".join(new_script))
-
-
-def get_new_udd_data():
-    if needs_update(UNZIPPED_FILE):
-        try:
-            sock = urllib.urlopen(URL)
-        except:
-            return None
-        sock.close()
-        if os.path.exists(LOCAL_FILE):
-            os.remove(LOCAL_FILE)
-        urllib.urlretrieve(URL, LOCAL_FILE)
-        if os.path.exists(UNZIPPED_FILE):
-            os.remove(UNZIPPED_FILE)
-        subprocess.call(["gunzip", LOCAL_FILE])
-        extract_good_parts()
-    new_script = UPDATED_FILE
-    return new_script
-

=== modified file 'overview/uploads/management/commands/get-udd-data.py'
--- overview/uploads/management/commands/get-udd-data.py	2013-04-09 02:45:57 +0000
+++ overview/uploads/management/commands/get-udd-data.py	2013-06-11 01:26:32 +0000
@@ -1,18 +1,33 @@
 import subprocess
 import sys, os
+import urllib
 
 from django.core.management.base import NoArgsCommand
 from django.db import connection, transaction
 from django.conf import settings
 import psycopg2 as db
 
-from uploads.common import udd
+URL = "http://alioth.debian.org/~asb/udd/ubuntu_upload_history.sql"
+LOCAL_FILE = os.path.join(settings.TEMP_PATH, "udd.sql")
+UPDATED_FILE = os.path.join(settings.TEMP_PATH, "new-udd.sql")
 
 class Command(NoArgsCommand):
     help = "Update uploads data from UDD."
 
+    def get_new_udd_data(self):
+        try:
+            sock = urllib.urlopen(URL)
+            urllib.urlretrieve(URL, UPDATED_FILE)
+            if os.path.exists(LOCAL_FILE):
+                os.remove(LOCAL_FILE)
+            os.rename(UPDATED_FILE, LOCAL_FILE)
+        except:
+            return None
+        sock.close()
+        return LOCAL_FILE
+
     def handle_noargs(self, **options):
-        new_script = udd.get_new_udd_data()
+        new_script = self.get_new_udd_data()
         if not new_script:
             sys.exit(1)
         dbinfo = settings.DATABASES['udd']
@@ -24,16 +39,24 @@
         cursor.execute("DROP DATABASE IF EXISTS " + dbinfo['NAME'])
         cursor.execute("CREATE DATABASE " + dbinfo['NAME'] + 
                        " WITH TEMPLATE=template0 ENCODING 'SQL_ASCII'")
-#        conn = db.connect(host=dbinfo['HOST'], dbname='udd',
-#                  user=dbinfo['USER'], password=dbinfo['PASSWORD'],
-#                  port=dbinfo['PORT'] or 5432)
-#        with conn.autocommit = True:
-#            cursor.execute(new_script)
-        dump = subprocess.Popen(["cat", new_script], stdout=subprocess.PIPE)
+        conn.commit()
+        cursor.close()
+        conn.close()
+
+        conn = db.connect(host=dbinfo['HOST'], dbname=dbinfo['NAME'],
+                  user=dbinfo['USER'], password=dbinfo['PASSWORD'],
+                  port=dbinfo['PORT'] or 5432)
+        cursor = conn.cursor()
+        cursor.execute("CREATE EXTENSION IF NOT EXISTS debversion")
+        conn.commit()
+        cursor.close()
+        conn.close()
+
         psql_env = os.environ.copy()
         psql_env['PGPASSWORD'] = dbinfo['PASSWORD']
-        psql_cmd = ['psql', "-U", dbinfo['USER'], '-d', dbinfo['NAME']]
-        subprocess.call(psql_cmd, env=psql_env, stdin=dump.stdout,
-                                                stdout=subprocess.PIPE,
-                                                stderr=subprocess.PIPE)
+        psql_cmd = ['psql',
+                    '-U', dbinfo['USER'],
+                    '-d', dbinfo['NAME'],
+                    '-f', new_script]
+        subprocess.call(psql_cmd, env=psql_env)