Merge lp:~jameinel/udd/single-download-524123 into lp:udd

[John A Meinel]

[John A Meinel (jameinel)]

This changes the dget code so that if a file exists locally, it computes the md5sum and only downloads it if the hash misses. (bug #524123.)

I have confirmed that in a single run of 'import_package gnome_panel' I'm getting cache hits (and thus not downloading the same file multiple times.)

=== modified file 'import_package.py'
--- import_package.py	2010-02-19 17:53:11 +0000
+++ import_package.py	2010-02-19 18:03:12 +0000
@@ -2,6 +2,10 @@
 
 import datetime
 import errno
+try:
+    from hashlib import md5
+except ImportError:
+    from md5 import md5
 import operator
 import optparse
 import os
@@ -313,13 +317,11 @@
         return "debian"
 
 
-def grab_file(location, target_dir, possible_transports=None):
-    mutter("fetching %s" % location)
-    location_base = urlutils.dirname(location)
-    location_file = urlutils.basename(location)
-    local_path = os.path.join(target_dir, location_file)
+def grab_file(base_url, name, target_dir, possible_transports=None):
+    mutter("fetching %s/%s" % (base_url, name))
+    local_path = os.path.join(target_dir, name)
     def get_file(transport):
-        return transport.get(location_file)
+        return transport.get(name)
     def redirected(t, e, redirection_notice):
         # _redirected_to has a bug that it doesn't support possible_transports,
         # so we just call get_transport directly, we know we are just getting a
@@ -328,7 +330,9 @@
         t2 = transport.get_transport(base,
                                      possible_transports=possible_transports)
         return t2
-    t = transport.get_transport(location_base,
+    # TODO: We could compute the md5sum while iterating the content, rather
+    #       than re-reading the file.
+    t = transport.get_transport(base_url,
                                 possible_transports=possible_transports)
     location_f = transport.do_catching_redirections(get_file, t, redirected)
     try:
@@ -339,21 +343,53 @@
             local_f.close()
     finally:
         location_f.close()
+    return local_path
+
+
+def _check_md5(target_path, expected_md5sum):
+    """See if the md5sum of the given file matches."""
+    if not os.path.exists(target_path):
+        return False
+    file_md5 = md5()
+    BUFSIZE = 128<<10
+    f = open(target_path, 'rb')
+    try:
+        while True:
+            b = f.read(BUFSIZE)
+            if not b:
+                break
+            file_md5.update(b)
+    finally:
+        f.close()
+    if file_md5.hexdigest() == expected_md5sum:
+        mutter('File at %s matched md5sum, reusing.' % (target_path,))
+        return True
+    mutter('File at %s did not match expected md5sum. Redownloading.'
+           % (target_path,))
+    return False
 
 
 def dget(dsc_location, target_dir, possible_transports=None):
-    grab_file(dsc_location, target_dir, possible_transports=possible_transports)
-    local_dsc_path = os.path.join(target_dir,
-        urlutils.basename(dsc_location))
+    base_url, dsc_name = urlutils.split(dsc_location)
+    local_dsc_path = grab_file(base_url, dsc_name, target_dir,
+                               possible_transports=possible_transports)
     dsc_f = open(local_dsc_path)
     try:
         dsc = deb822.Dsc(dsc_f)
         files = dsc['files']
         for file_info in files:
             name = file_info['name']
-            # TODO: md5 check
-            grab_file(urlutils.join(urlutils.dirname(dsc_location), name),
-                target_dir, possible_transports=possible_transports)
+            target_path = os.path.join(target_dir, name)
+            if _check_md5(target_path, file_info['md5sum']):
+                # The file already exists, and the md5sum matches
+                continue
+            # We need to download the file
+            grab_file(base_url, name, target_dir,
+                      possible_transports=possible_transports)
+            if not _check_md5(target_path, file_info['md5sum']):
+                raise ValueError('The downloaded content for %s did'
+                    ' not match the md5sum in %s'
+                    % (target_path, local_dsc_path))
     finally:
         dsc_f.close()
     return local_dsc_path