Merge lp:~ara/ubuntu/lucid/ubuntu-qa-tools/fix_537941 into lp:~ubuntu-branches/ubuntu/lucid/ubuntu-qa-tools/lucid

[Ara Pulido]

  * dl-ubuntu-test-iso is failing to download lucid images (LP: #537941)
  * Fixes some lintian errors due to script naming
  * debian/control:
    - Removed dependency with python-dev
    - Added two existing scripts to the description of the package

[Daniel Holbach (dholbach)]

Good work!

review: Approve

=== renamed file 'bugs-mailinglist/ml-fixes-report.py' => 'bugs-mailinglist/ml-fixes-report'
=== renamed file 'bugs-mailinglist/ml-team-fixes-report.py' => 'bugs-mailinglist/ml-team-fixes-report'
=== modified file 'debian/changelog'
--- debian/changelog	2009-07-14 08:43:29 +0000
+++ debian/changelog	2010-03-12 09:50:29 +0000
@@ -1,3 +1,13 @@
+ubuntu-qa-tools (0.1.4.1) lucid; urgency=low
+
+  * dl-ubuntu-test-iso is failing to download lucid images (LP: #537941)
+  * Fixes some lintian errors due to script naming 
+  * debian/control:
+    - Removed dependency with python-dev
+    - Added two existing scripts to the description of the package
+
+ -- Ara Pulido <[e-mail]>  Fri, 12 Mar 2010 10:05:58 +0100
+
 ubuntu-qa-tools (0.1.4) karmic; urgency=low
 
   * Added three new scripts:

=== modified file 'debian/control'
--- debian/control	2009-02-11 11:27:52 +0000
+++ debian/control	2010-03-12 09:50:29 +0000
@@ -4,11 +4,11 @@
 Maintainer: Ubuntu MOTU Developers <[e-mail]>
 Vcs-Bzr: http://bazaar.launchpad.net/~ubuntu-bugcontrol/ubuntu-qa-tools/packaging
 Vcs-Browser: http://codebrowse.launchpad.net/~ubuntu-bugcontrol/ubuntu-qa-tools/packaging/changes
-Build-Depends: cdbs (>= 0.4.49), debhelper (>= 6), python-all-dev (>= 2.4)
+Build-Depends: cdbs (>= 0.4.49), debhelper (>= 6), python-all
 Build-Depends-Indep: python-central (>= 0.5)
 XS-Python-Version: >= 2.5
 Homepage: https://launchpad.net/ubuntu-qa-tools/
-Standards-Version: 3.8.0
+Standards-Version: 3.8.4
 
 Package: ubuntu-qa-tools
 Architecture: all
@@ -27,10 +27,14 @@
  . 
   * check-needs-packaging - checks for bugs that require new packaging
   * body-searching - search bugs with a particular text 
-  * count-senders - quantity of messages sent by individuals to the bug mailinglist 
+  * count-senders - quantity of messages sent by individuals
+    to the bug mailinglist 
   * tagged-bugs - search for tagged bugs in Launchpad 
   * triager-query - search for actions of a particular triager 
   * dl-ubuntu-test-iso - downloads ISOs for testing using rsync
   * iso-ripper - rips debs out of downloaded isos
   * debian-bug-search - searches the Debian bug tracking system
   * hugday - simplifies participating on ubuntu hugdays
+  * ml-fixes-report - searches fixes in a bug mailing list
+  * ml-team-fixes-report - searches fixes by team
+

=== modified file 'dl-ubuntu-test-iso/dl-ubuntu-test-iso'
--- dl-ubuntu-test-iso/dl-ubuntu-test-iso	2009-07-14 08:43:29 +0000
+++ dl-ubuntu-test-iso/dl-ubuntu-test-iso	2010-03-12 09:50:29 +0000
@@ -1,384 +1,538 @@
-#!/bin/sh
-#
-# Download, with rsync, Ubuntu ISO images for testing them.
-# This will use a lot of bandwidth and disk space.
-#
-# You can put a configuration file (~/.dl-ubuntu-test-iso)
-# and set some variables to limit what will be downloaded.
-# See sample values below.
-#
-#   * ARCHS: what architectures to download
-#   * FLAVORS: what flavors of Ubuntu? Ubuntu itself, Kubuntu, etc
-#   * VARIANTS: for each flavor, what variants? desktop (= live cd)?
-#     alternate cd?
-#   * OPTS: options for rsync
-#
-# You can get sample configuration files at /usr/share/doc/ubuntu-qa-tools/examples/
-#
-# This is based on a script by davmor2 on #ubuntu-testing. Modified and
-# rewritten to be more easily extensible by Lars Wirzenius. Further
-# abuse was perpetrated by Steve Beattie. This is probably
-# too simple to be covered by copyright, but if not:
-#
-# Copyright 2008, 2009 Canonical Ltd.
-# Licensed under the GNU General Public License, version 3.
-
-set -e
-
-ISOROOT="${HOME}/iso"
-BASEURL="rsync://cdimage.ubuntu.com/cdimage"
-RELEASES="karmic hardy"
-ARCHS="i386 amd64"
-FLAVORS="ubuntu kubuntu edubuntu xubuntu ubuntu-server"
-FLAVORS="$FLAVORS ubuntustudio mythbuntu"
-VARIANTS="desktop alternate server addon dvd"
-EXCLUDE=""
-OPTS="-zthhP"
-QUIET="false"
-NO_ACT=no
-VERIFY="true"
-BUILD="current"
-
-RSYNC=rsync
-ISOINFO=/usr/bin/isoinfo
-WGET=/usr/bin/wget
-CURL=/usr/bin/curl
-CURRENT_RELEASE="karmic"
-VERSIONS="false"
-CONFIGFILE="${HOME}/.dl-ubuntu-test-iso"
-
-if [ -e "${CONFIGFILE}" ]
-then
-    . ${CONFIGFILE}
-elif [ -e "${ISOROOT}/iso.cfg" ]
-then
-    echo "******"
-    echo "****** ${ISOROOT}/iso.cfg is deprecated, please use ${CONFIGFILE} instead"
-    echo "******"
-    . "${ISOROOT}/iso.cfg"
-fi
-
-setup_dir()
-{
-    local dir=$1
-
-    [ -d "$dir" ] || mkdir $dir
-    cd $dir
-}
-
-setup_dir $ISOROOT
-
-die()
-{
-    echo "$@" 1>&2
-    exit 1
-}
-
-no_act()
-{
-    [ "$NO_ACT" = yes ]
-}
-
-debugify_commands()
-{
-    RSYNC="echo $RSYNC"
-    ISOINFO="echo $ISOINFO"
-}
-
-wantflavor()
-{
-    echo " $FLAVORS " | grep " $1 " > /dev/null && 
-    echo " $EXCLUDE " | grep -v " $1 " > /dev/null
-}
-
-wantvariant()
-{
-    echo " $VARIANTS " | grep " $1 " > /dev/null
-}
-
-wantarch()
-{
-    local arch="$1"
-    shift
-    echo " $@ " | grep " $arch " > /dev/null
-}
-
-# Dead code, kept around for historical raisins
-#isold()
-#{
-#    if [ ! -e "$1" ]
-#    then
-#        return 0
-#    else
-#        local mtime=$(stat --printf='%Y\n' "$1")
-#        local old=$(date --date="$MINAGE" +%s)
-#        [ "$mtime" -lt "$old" ]
-#    fi
-#}
-# end dead code
-
-isquiet()
-{
-    [ "$QUIET" = "true" ]
-}
-
-doverify()
-{
-    [ "$VERIFY" = "true" ]
-}
-
-doversions()
-{
-    [ "$VERSIONS" = "true" ]
-}
-
-computepath()
-{
-    local flavor="$1"
-    local release="$2"
-    local dir="$3"
-
-    local releasepath=""
-    local flavorpath=""
-    if [ "$CURRENT_RELEASE" != "$release" ]
-    then
-	releasepath="$release/"
-    fi
-
-    if [ "$flavor" != "ubuntu" ]
-    then
-    	flavorpath="$flavor/"
-    fi
-
-    echo "${flavorpath}${releasepath}${dir}"
-}
-
-has_joliet()
-{
-    local iso="$1"
-    ! isoinfo -d -i "${iso}" | grep -q "^NO Joliet"
-}
-
-report_versions()
-{
-    local dir=$1
-
-    if [ ! -x ${ISOINFO} ]
-    then
-    	echo "Please install the isoinfo package in order to report iso versions"
-	echo "(On ubuntu, 'sudo apt-get install genisoimage' will get it."
-	exit 1
-    fi
-
-    find $dir -name "*.iso" -print | while read isoname
-    do
-	# all ubuntu cds should be joliet, but if non-joliet ones are
-	# stored in the tree as well, it's nice to not report errors due
-	# to that.
-	if has_joliet ${isoname}
-	then
-            DISKINFO=$(${ISOINFO} -J -x "/.disk/info"  -i ${isoname} || true)
-            DISKTYPE=$(${ISOINFO} -J -x "/.disk/cd_type"  -i ${isoname} || true)
-	fi
-	echo "$isoname ${DISKINFO} ${DISKTYPE}"
-    done
-}
-
-# returns boolean if file exists remotely
-remote_file_exists()
-{
-    local url=$(echo "$1" | sed -e 's/rsync/http/')
-    if [ -x "${WGET}" ]
-    then
-	${WGET} -q --spider "${url}"
-    elif [ -x "${CURL}" ]
-    then
-	# meh, curl, why are you a pain?
-	local code=$(${CURL} -s -I --write-out "%{http_code}\n" "${url}" | tail -1)
-	[ "${code}" = "200" ]
-    else
-	return 0
-    fi
-}
-
-get()
-{
-    local flavor="$1"
-    local dir="$2"
-    shift 2
-
-    local output=""
-    if isquiet
-    then
-        output="> /dev/null"
-    fi
-
-    for variant in "$@"
-    do
-        if wantflavor "$flavor" && wantvariant "$variant"
-        then
-            [ -d "$flavor" ] || mkdir "$flavor"
-            for arch in $ARCHS
-            do
-		for release in $RELEASES
-		do
-                    if wantarch "$arch" "$ARCHS"
-                    then
-			local iso="$release-$variant-$arch.iso"
-			local path="$(computepath $flavor $release $dir)"
-			local md5sum="MD5SUMS.$release-$variant"
-			local remote_path="$BASEURL/$path/$iso"
-			if ! remote_file_exists "${remote_path}"
-			then
-                            isquiet || echo "Skipping $release $flavor $variant $arch: not on server"
-			else
-                            isquiet || echo "Synching $release $flavor $variant $arch ($path/$iso)"
-                            if no_act
-                            then
-                            	:
-                            	# echo "(Not really running rsync because of NO_ACT)"
-                            else
-                                $RSYNC $OPTS "${remote_path}" "$flavor/$iso" || true
-                                doverify && $RSYNC $OPTS "$BASEURL/$path/MD5SUMS" "$flavor/$md5sum" || true
-                            fi
-			    if doverify
-			    then
-			    	if [ -f "$flavor/$md5sum" ]
-			    	then
-			    	    local FILEMD5SUM="$(grep "$iso" "$flavor/$md5sum" | sed -e "s/\($iso\)/$flavor\/\\1/")"
-			    	    isquiet || echo -n "Verifying $iso... "
-				    eval "echo \"$FILEMD5SUM\" | md5sum -c - $output" || true
-			    	else
-			    	    isquiet || echo "No checksum file available for $iso, skipping verification"
-			        fi
-			    fi
-			fi
-                    fi
-		done
-            done
-        fi
-    done
-}
-
-print_help()
-{
-    echo "
-    ./dl-ubuntu-test-iso [options]
-
-You may use the following options:
-
-    -n, --no-act
-        Do everything, except don't actually run rsync. You can use this
-        to test configuration settings, etc.
-
-    --debug
-        Similar to --no-act, but echos the actual rsync commands, as you
-        might expect, this is useful for debugging the script or
-        configurations
-
-    -P  Run rsync with the -P option, to get progress reporting.
-
-    --only flavor, --flavor flavor
-        Only download a specific flavor of Ubuntu. Only the last
-        --only option applies. (You can also set FLAVORS in the
-        configuration file.) --only should probably be considered
-        depricated.
-
-    --exclude flavor
-        Do not download a specific flavor of Ubuntu. You can use this
-        option multiple times, and none of the specified flavors will be
-        downloaded. (You can also set EXCLUDE in the configuration
-        file.)
-
-    --release release
-        Only download a specific release target of Ubuntu. Only the last
-        --release option applies. (You can also set RELEASES in the
-        configuration file.) \"hardy\" is the earliest release target
-
-    --bwlimit kbpslimit
-	Rate limit rsync download to a limit in KBytes/second.
-    "
-}
-
-TEMP=`getopt -o nPqh --long no-act,help,force,debug,quiet,no-verify,versions,only:,exclude:,release:,flavor:,bwlimit:,build: -- "$@"`
-eval set -- "$TEMP"
-
-while true
-do
-    case "$1" in
-    -n|--no-act) NO_ACT=yes; shift ;;
-    --debug) debugify_commands ; shift;;
-    -q|--quiet) QUIET=true; shift ;;
-    -P) OPTS="$OPTS -P"; shift ;;
-    --only|--flavor) FLAVORS="$2"; shift 2;;
-    --exclude) EXCLUDE="$EXCLUDE $2"; shift 2;;
-    --release) RELEASES="$2"; shift 2;;
-    --bwlimit) OPTS="$OPTS --bwlimit=$2"; shift 2;;
-    --build) BUILD="$2"; shift 2;;
-    --no-verify) VERIFY="FALSE"; shift ;;
-    --versions) VERSIONS="true"; shift ;;
-    -h|--help) print_help ; die ;; 
-    --) shift; break ;;
-    *) die "Internal error while parsing command line"
-    esac
-done
-
-if isquiet
-then
-    OPTS="$OPTS -q"
-fi
-
-isquiet || cat << eof
-BASEURL=$BASEURL
-RELEASES=$RELEASES
-ARCHS=$ARCHS
-FLAVORS=$FLAVORS
-VARIANTS=$VARIANTS
-EXCLUDE=$EXCLUDE
-OPTS=$OPTS
-NO_ACT=$NO_ACT
-VERIFY=$VERIFY
-BUILD=$BUILD
-ISOROOT=$ISOROOT
-QUIET=$QUIET
-
-eof
-
-if doversions
-then
-    report_versions $ISOROOT
-    exit 0
-fi
-
-get ubuntu daily-live/${BUILD} desktop
-get ubuntu daily/${BUILD} alternate
-get ubuntu dvd/${BUILD} dvd
-
-get ubuntu-server daily/${BUILD} server
-
-get kubuntu daily-live/${BUILD} desktop
-get kubuntu daily/${BUILD} alternate
-get kubuntu dvd/${BUILD} dvd
-
-# deprecated, not enabled by default
-get kubuntu-kde4 daily-live/${BUILD} desktop
-get kubuntu-kde4 daily/${BUILD} alternate
-
-get edubuntu daily/${BUILD} addon
-
-get xubuntu daily-live/${BUILD} desktop
-get xubuntu daily/${BUILD} alternate
-
-get ubuntustudio daily/${BUILD} alternate
-
-__ARCHS=$ARCHS
-# deprecated, not enabled by default
-ARCHS=i386 get jeos daily/${BUILD} jeos
-ARCHS=${__ARCHS}
-
-# deprecated, not enabled by default
-get gobuntu daily/${BUILD} alternate
-
-# alternate deprecated, should probably be removed
-get mythbuntu daily/${BUILD} alternate
-get mythbuntu daily-live/${BUILD} desktop
+#!/usr/bin/python
+from __future__ import with_statement # This isn't required in Python 2.6, but must be in the beginning and thus can't be conditional
+
+import httplib
+import urllib2
+from time import strptime
+from datetime import datetime, timedelta
+from optparse import OptionParser
+from os import environ
+import hashlib
+import os
+import re
+import subprocess
+import sys
+import stat
+
+current_release = 'lucid'
+releasedict = {}
+releasedict['hardy'] = []
+releasedict['intrepid'] = []
+releasedict['jaunty'] = []
+releasedict['karmic'] = []
+releasedict['lucid'] = []
+
+#allreleases = ['hardy', 'intrepid']
+allarchs = ['i386', 'amd64', 'lpia', 'armel']
+
+default = {}
+default['archs'] = ['i386', 'amd64']
+default['variants'] = ['desktop', 'alternate', 'dvd']
+default['build'] = 'current'
+default['host'] = 'cdimage.ubuntu.com'
+default['releases'] = ['hardy', 'lucid']
+default['isoroot'] = environ['HOME'] + '/iso/'
+
+zsync_binary = '/usr/bin/zsync'
+
+class Flavor:
+	def __init__(self, name, use_prefix=True, releases=default['releases'],
+	             variants=default['variants'], archs=default['archs']):
+		self.name = name
+		self.prefix = None
+		if use_prefix:
+			self.prefix = name
+		self.dir = dir
+		self.releases = releases
+		self.variants = variants
+		self.archs = archs
+		for r in releases:
+			releasedict[r].append(self)
+		flavors[name] = self
+
+def my_log(x):
+	print x
+
+## HTML Header, defines CSS and header image ##
+def printHeader():
+        print """
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
+"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+<head>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+  <title>Daily ISO Tracker</title>
+  <style type="text/css">
+    body { background: #FFFFFF; color: black; }
+    a { text-decoration: none; }
+    table.head { border-style: none none; }
+    table.head td { text-align: center; padding-left: 10px; padding-right: 10px}
+    table.body { background: #efe1c3; border-collapse: collapse; border-style: solid solid;
+            border-width: 3px; margin-bottom: 3ex; empty-cells: show; }
+    table.body th { text-align: left; border-style: none none dotted none;
+               border-width: 1px; padding-right: 10px; }
+    table.body td { text-align: left; border-style: none none dotted none;
+               border-width: 1px; padding-right: 10px; }
+    a { color: blue; }
+    a.verified { color: green; font-weight: bold; }
+    a.testing { color: blue; }
+    pre { white-space: pre-wrap; }
+  </style>
+</head>
+<body>
+<table class="head">
+        <tr><td width="30%"><a href="http://www.ubuntu.com/"><img src="http://www.ubuntu.com/themes/ubuntu07/images/ubuntulogo.png" border="0" hspace="0" vspace="0" alt="Ubuntu"></a></td>
+        <td><h1>Daily ISO Tracker</h1></td></tr>
+</table>
+"""
+
+## Print footer ##
+def printFooter():
+        time = datetime.datetime.utcnow()
+        print "<p>Last Updated (UTC): %s by <a href=\"https://code.launchpad.net/~sbeattie/sru-tools/sru-buglist\">sru_buglist</a>" % time.ctime()
+        print "written by <a href=\"mailto:[e-mail]\">Steve Beattie</a>.</p>"
+        print "</body></html>"
+
+def dumpHeaders(headers):
+	for header, value in headers:
+		print header + ': ' + value
+
+unit_names = {"year" : ("year", "years"),
+	      "month" : ("month", "months"),
+	      "week" : ("week", "weeks"),
+	      "day" : ("day", "days"),
+	      "hour" : ("hour", "hours"),
+	      "minute" : ("minute", "minutes"),
+	      "second" : ("second", "seconds")}
+
+def seconds_in_units(seconds):
+	"""
+	Returns a tuple containing the most appropriate unit for the
+	number of seconds supplied and the value in that units form.
+
+		>>> seconds_in_units(7700)
+		(2, 'hour')
+	"""
+
+	unit_limits = [("year", 365 * 24 * 3600),
+		       ("month", 30 * 24 * 3600),
+		       ("week", 7 * 24 * 3600),
+		       ("day", 24 * 3600),
+		       ("hour", 3600),
+		       ("minute", 60)]
+
+	for unit_name, limit in unit_limits:
+		if seconds >= limit:
+			amount = int(round(float(seconds) / limit))
+			return amount, unit_name
+	return seconds, "second"
+
+def stringify_timedelta(td):
+	"""
+	Converts a timedelta into a nicely readable string.
+
+	        >>> td = timedelta(days = 77, seconds = 5)
+		>>> print readable_timedelta(td)
+		two months
+	"""
+	seconds = td.days * 3600 * 24 + td.seconds
+	amount, unit_name = seconds_in_units(seconds)
+
+	str_unit = unit_names[unit_name][1]
+	if amount == 1:
+		str_unit = unit_names[unit_name][0]
+	return "%d %s" % (amount, str_unit)
+
+def parseDate(date):
+	return datetime.strptime(date, "%a, %d %b %Y %H:%M:%S %Z")
+
+def create_dir(path):
+	# XXX: make this more defensive. Sigh.
+	try:
+		stat = os.stat(path)
+	except OSError, e:
+		os.makedirs(path)
+
+def read_config(pathname):
+	'''Read the config file, using shell syntax for backwards
+	   compatability.'''
+	if not os.path.exists(pathname):
+		return None
+
+	shell_vars = ['ISOROOT', 'RELEASES', 'FLAVORS', 'VARIANTS', 'EXCLUDE',
+		      'OPTS', 'QUIET', 'NO_ACT', 'VERIFY', 'BUILD', 'BASEURL', 'ARCHS']
+	command = ". %s; " %(pathname)
+	for var in shell_vars:
+		command += 'if [ ! -z "$%s" ] ; then echo %s=$%s; fi ; ' %(var, var, var)
+	p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
+	out, err = p.communicate()
+
+	config = dict()
+	for line in re.split('\n', out):
+		l = re.split('=', line)
+		if not len(l) == 2:
+			continue
+		if l[0] in ['RELEASES', 'FLAVORS', 'VARIANTS', 'EXCLUDE']:
+			l[1] = l[1].lower()
+		config[str.lower(l[0])] = l[1]
+	return config
+
+def computepath(wanted, flavor, release, variant):
+	path = '/'
+	if flavor.prefix:
+		path += flavor.prefix + '/'
+	if release != current_release:
+		path += release + '/'
+	path += '%s/%s/' %(variantpaths[variant], wanted['build'])
+	return path
+
+def get_status(uri, host, conn=None):
+	close = False
+	result = False, None
+	if not conn:
+		conn = httplib.HTTPConnection(host)
+		conn.set_debuglevel(0)
+		close = True
+	conn.request("HEAD", uri)
+	response = conn.getresponse()
+	if response.status == 200:
+		modified = response.getheader('last-modified')
+		age = datetime.utcnow() - parseDate(modified)
+		result = True, stringify_timedelta(age)
+	else:
+		result = False, response.reason
+	response.read()
+	if close:
+		conn.close()
+	return result
+
+def check_status(wanted, config):
+	conn = httplib.HTTPConnection(config.host)
+	conn.set_debuglevel(0)
+	for count, image in enumerate(image_factory(wanted)):
+		release, flavor, variant, arch = image
+		isoname = release + '-' + variant + '-' + arch + '.iso'
+		path = computepath(wanted, flavor, release, variant)
+		uri = path + isoname
+		#print 'Result for ' + uri + ': ' + str(response.status)
+		rc, msg = get_status(uri + '.zsync', config.host, conn=conn)
+		print 'Result for ' + isoname + '.zsync: ' + msg
+		rc, msg = get_status(uri, config.host, conn=conn)
+		print 'Result for ' + isoname + ': ' + msg
+	conn.close()
+
+def _do_download(config, source, target):
+	result = True
+	conn = None
+	out = None
+
+	try:
+	    conn = urllib2.urlopen('http://' + config.host + source)
+	    out = open(target, 'w')
+	    for data in conn:
+	        out.write(data)
+	except IOError:
+	    result = False
+	finally:
+	    if conn:
+	        conn.close()
+	    if out:
+	        out.close()
+	return result
+
+def _do_zsync(config, meta, src, target):
+	command = [zsync_binary, '-k', meta, '-o', target,
+		   'http://%s/%s' %(config.host, src)]
+	if config.quiet:
+		command.insert(1, '-q')
+	if config.no_act or config.debug:
+		if config.debug:
+			print ' '.join(command)
+		return True
+
+	rc = 0
+	interrupted = False
+	try:
+		rc = subprocess.call(command)
+		# trap external SIGINT death
+		if rc > 128 and rc % 128 == signal.SIGINT:
+			interrupted = True
+	except KeyboardInterrupt:
+		interrupted = True
+	finally:
+		if os.path.exists(target + '.zs-old'):
+			os.remove(target + '.zs-old')
+		if (interrupted or rc != 0) and os.path.exists(target + '.part'):
+			os.rename(target + '.part', target)
+		if os.path.exists(target):
+			os.chmod(target, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
+		if interrupted:
+			raise KeyboardInterrupt
+	return rc == 0
+
+def do_zsync(config, destination, uri, isoname):
+	zsync_uri = uri + '.zsync'
+	zsync_iso = destination + '/' + isoname
+	zsync_local = destination + '/'
+	# hide zsync files when not mirroring tree structure
+	if not config.mirror:
+		zsync_local += '.'
+	zsync_local += isoname + '.zsync'
+	available, msg = get_status(zsync_uri, config.host)
+	if not available:
+		config._log("control file does not exist, skipping zsync: " + msg)
+		return False
+
+	# when config.build is set, we could be grabbing an iso older in
+	# time than what we have now. zsync only updates it's local copy
+	# of the meta data if the remote version is dated newer, so we
+	# manually rsync the version we want.
+	if config.build != default['build']:
+		rc = _do_rsync(config, zsync_uri, zsync_local)
+
+	return _do_zsync(config, zsync_local, zsync_uri, zsync_iso)
+	
+
+def _do_rsync(config, source, target):
+	command = ['/usr/bin/rsync', '-zthP', 'rsync://%s/cdimage%s' %(config.host, source), target]
+	if len(config.bwlimit) > 0:
+		command.insert(2, '--bwlimit=' + config.bwlimit)
+	if config.quiet:
+		command.insert(2, '-q')
+	if config.no_act or config.debug:
+		if config.debug:
+			print ' '.join(command)
+		return True
+	rc = subprocess.call(command)
+	return rc == 0
+	
+def do_rsync(config, destination, uri, isoname):
+	rsync_local = destination + '/' + isoname
+	#rc = cmd(['/usr/bin/zsync', '-o', zsync_local, 'http://cdimages.ubuntu.com/' + zsync_uri])
+	return _do_rsync(config, uri, rsync_local)
+
+def lookup_hash(path, host):
+	hash_table = {
+		"SHA256" : hashlib.sha256,
+		"SHA1"   : hashlib.sha1,
+		"MD5"    : hashlib.md5,
+	}
+	conn = httplib.HTTPConnection(host)
+	conn.set_debuglevel(0)
+	for alg in hash_table.keys():
+		uri = path + alg + 'SUMS'
+		available, msg = get_status(uri, host, conn=conn)
+		if available:
+			conn.close()
+			return alg, hash_table[alg]
+	conn.close()
+	return None, None
+
+def do_verify(config, destination, hashsuffix, path, iso):
+	alg, hashfunc  = lookup_hash(path, config.host)
+	if not alg:
+		return False
+		
+	uri = path + alg + "SUMS"
+	if config.mirror:
+		local = destination + '/' + alg + "SUMS"
+	else:
+		local = destination + '/' + alg + "SUMS" + hashsuffix
+	rc = _do_download(config, uri, local)
+	if not rc:
+		return False
+	
+	digest = None
+	with open(local) as f:
+		for line in f:
+			if iso in line:
+				digest = re.split('\s+', line)[0]
+				break
+	
+	if not digest:
+		return False
+	
+	m = hashfunc()
+	fd = os.open(destination + '/' + iso, os.O_RDONLY)
+	data = os.read(fd, 16 * 256 * 256)
+	while len(data):
+		m.update(data)
+		data = os.read(fd, 16 * 256 * 256)
+	os.close(fd)
+	result = m.hexdigest()
+	#print digest + ' ' + iso
+	#print result + ' ' + iso
+	
+	return digest == result
+
+def iso_download(wanted, config):
+	saved_cwd = os.getcwd()
+	create_dir(wanted['isoroot'])
+	os.chdir(wanted['isoroot'])
+
+	for count, image in enumerate(image_factory(wanted)):
+		release, flavor, variant, arch = image
+		isoname = release + '-' + variant + '-' + arch + '.iso'
+		hashsuffix = "." + release + '-' + variant
+		path = computepath(wanted, flavor, release, variant)
+		if config.mirror:
+			destination = path[1:-1]
+		else:
+			destination = flavor.name
+		create_dir(destination)
+		uri = path + isoname
+		#print 'Result for ' + uri + ': ' + str(response.status)
+
+		config._log("Syncing %s %s %s %s" %(release, flavor.name, variant, arch))
+		available, msg = get_status(uri, config.host)
+		if not available:
+			config._log("iso unavailable from server, skipping: " + msg)
+			continue	
+
+		if not (config.use_zsync and do_zsync(config, destination, uri, isoname)):
+			do_rsync(config, destination, uri, isoname)
+		if config.do_verify:
+			config._log("Verifying " + destination + '/' + isoname)
+			if not (config.debug or config.no_act):
+				if do_verify(config, destination, hashsuffix, path, isoname):
+					config._log("Verification succeeded")
+				else:
+					config._log("!!! Verification failed !!!")
+
+	os.chdir(saved_cwd)
+
+def dump_info(wanted):
+	for count, image in enumerate(image_factory(wanted)):
+		release, flavor, variant, arch = image
+		print "%s %s %s %s %s" %(release, flavor.name, variant, arch, variantpaths[variant])
+
+def image_factory(wanted):
+	for release in releasedict.keys():
+		if release not in wanted['releases']:
+			continue
+		for flavor in releasedict[release]:
+			if flavor.name not in wanted['flavors']:
+				continue
+			for variant in flavor.variants:
+				if variant not in wanted['variants']:
+					continue
+				for arch in flavor.archs:
+					if arch not in wanted['archs']:
+						continue
+					yield (release, flavor, variant, arch)
+					#print "%s %s %s %s %s" %(release, flavor.name, variant, arch, variantpaths[variant])
+
+def merge_config(cval, file, default, value):
+	ret = None
+	if len(cval) > 0:
+		ret = cval
+	elif file and value in file.keys() and len(file[value]) > 0:
+		ret = file[value]
+	elif value in default.keys():
+		ret = default[value]
+	return ret
+
+def main():
+	global releasedict, current_release, variantpaths, flavors
+
+	config = {}
+	wanted = {}
+	flavors = {}
+	variantpaths = {}
+	variantpaths['desktop'] = 'daily-live'
+	variantpaths['alternate'] = 'daily'
+	variantpaths['jeos'] = 'daily'
+	variantpaths['dvd'] = 'dvd'
+	variantpaths['server'] = 'daily'
+	variantpaths['addon'] = 'daily'
+	variantpaths['usb'] = 'daily'
+	variantpaths['netbook-remix'] = 'daily-live'
+	variantpaths['netbook'] = 'daily-live'
+	variantpaths['moblin-remix'] = 'daily-live'
+
+	Flavor('ubuntu', use_prefix=False)
+	Flavor('ubuntu-server', variants=['server'])
+	Flavor('kubuntu')
+	Flavor('kubuntu-kde4', releases=['hardy'], variants=['desktop', 'alternate'])
+	Flavor('edubuntu', variants=['addon', 'dvd'])
+	Flavor('xubuntu', variants=['desktop', 'alternate'])
+	Flavor('ubuntustudio', variants=['alternate'])
+	Flavor('jeos', releases=['hardy'], variants=['jeos'], archs=['i386'])
+	Flavor('gobuntu', releases=['hardy'], variants=['alternate'])
+	Flavor('mythbuntu', variants=['desktop'])
+	Flavor('ubuntu-mid', variants=['usb'], archs=['lpia'])
+	Flavor('ubuntu-netbook', variants=['netbook'], archs=['i386'])
+	Flavor('kubuntu-netbook', variants=['netbook'], archs=['i386'])
+	Flavor('ubuntu-moblin-remix', variants=['moblin-remix'], archs=['i386'])
+
+	default['variants'] = variantpaths.keys()
+	default['flavors'] = flavors.keys()
+
+	parser = OptionParser()
+	parser.add_option('-n', '--no-act', default=False, action='store_true', dest='no_act', help='compute everything but don\'t actually download')
+	parser.add_option('-d', '--debug', default=False, action='store_true')
+	parser.add_option('-q', '--quiet', default=False, action='store_true', help='suppress output except errors')
+	parser.add_option('-P', '--progress', default=False, action='store_true', help='display a progress meter while downloading')
+	parser.add_option('--only', '--flavor', action='append', default=[], dest='flavor', help='select specific flavor to download; multiple --flavor args can be passed')
+	parser.add_option('--exclude', '--exclude-flavor', action='append', default=[], dest='exclude_flavor')
+	parser.add_option('--release', '--only-release', action='append', default=[], dest='release')
+	parser.add_option('--variant', '--only-variant', action='append', default=[], dest='variant')
+	parser.add_option('--exclude-variant', action='append', default=[], dest='exclude_variant')
+	parser.add_option('--arch', '--only-arch', action='append', default=[], dest='arch')
+	parser.add_option('--bwlimit', default='', action="store", type="string", help='rate limit rsync download to BWLIMIT in KBytes/second')
+	parser.add_option('--build', default=default['build'], action="store", type="string")
+	parser.add_option('--no-verify', default=True, action="store_false", dest='do_verify', help='skip comparison against published checksums')
+	parser.add_option('--no-zsync', default=True, action="store_false", dest='use_zsync', help="don't use zsync to download")
+	#parser.add_option('--versions', default=False, action='store_true')
+	parser.add_option('--isoroot', default='', action='store', type='string', help='local directory root to store isos; default is %s' %(default['isoroot']))
+	parser.add_option('--config', default=environ['HOME'] + '/.dl-ubuntu-test-iso', action='store', type='string',
+			  help='choose alternate config file location, default is %default')
+	parser.add_option('--host', default=default['host'], action='store', type='string', help='mirror to pull images from; default is %s' %(default['host']))
+	parser.add_option('--mirror', default=False, action='store_true', dest='mirror', help='mirror tree structure from cdimages') 
+
+	config, args = parser.parse_args()
+	if len(args) != 0:
+		parser.error("incorrect number of arguments")
+	file_cfg =  read_config(config.config)
+
+	wanted['flavors'] = merge_config(config.flavor, file_cfg, default, 'flavors')
+	if len(config.exclude_flavor) > 0:
+		wanted['flavors'] = filter(lambda x: x not in config.exclude_flavor, wanted['flavors'])
+
+	wanted['releases'] = merge_config(config.release, file_cfg, default, 'releases')
+	wanted['variants'] = merge_config(config.variant, file_cfg, default, 'variants')
+	if len(config.exclude_variant) > 0:
+		wanted['variants'] = filter(lambda x: x not in config.exclude_variant, wanted['variants'])
+	wanted['build'] = merge_config(config.build, file_cfg, default, 'build')
+	wanted['isoroot'] = merge_config(config.isoroot, file_cfg, default, 'isoroot')
+
+	wanted['archs'] = merge_config(config.arch, file_cfg, default, 'archs')
+
+	if not config.no_act and file_cfg != None and 'no_act' in file_cfg.keys():
+		config.no_act = ('true' == file_cfg['no_act'].lower())
+
+	if config.do_verify and file_cfg != None and 'verify' in file_cfg.keys():
+		config.no_act = ('true' == file_cfg['verify'].lower())
+
+	if config.quiet:
+		config._log = lambda x: x
+	else:
+		config._log = my_log
+
+	if config.use_zsync and not os.path.exists(zsync_binary):
+		config._log("Warning! zsync is not installed, falling back to rsync")
+		config.use_zsync = False
+
+	if config.use_zsync and len(config.bwlimit) > 0:
+		config._log("Warning! zsync does not support bandwidth limiting")
+
+	#print file_cfg
+	#print config
+	#print wanted
+	#dump_info(wanted, config)
+	#check_status(wanted, config)
+	iso_download(wanted, config)
+
+if __name__ == "__main__":
+	main()
+

=== modified file 'setup.py'
--- setup.py	2009-07-14 08:43:29 +0000
+++ setup.py	2010-03-12 09:50:29 +0000
@@ -20,8 +20,8 @@
                'bugs-mailinglist/count-senders',
                'bugs-mailinglist/tagged-bugs',
                'bugs-mailinglist/triager-query',
-               'bugs-mailinglist/ml-team-fixes-report.py',
-               'bugs-mailinglist/ml-fixes-report.py',
+               'bugs-mailinglist/ml-team-fixes-report',
+               'bugs-mailinglist/ml-fixes-report',
                'bugs-mailinglist/new-bug-description-search',
                'bugtrackers/debian-bug-search',
                'dl-ubuntu-test-iso/dl-ubuntu-test-iso',