Merge lp:~noskcaj/ubuntu/vivid/urlgrabber/3.10.1 into lp:ubuntu/vivid/urlgrabber

[Jackson Doak]

New upstream release, upstreams some patcges

[Daniel Holbach (dholbach)]

daniel@daydream:~/urlgrabber$ bzr merge lp:~noskcaj/ubuntu/vivid/urlgrabber/3.10.1
Unapplying quilt patches to prevent spurious conflicts
+N scripts/urlgrabber-ext-down
 M ChangeLog
 M MANIFEST
 M PKG-INFO
 M README
 M debian/changelog
-D debian/patches/grabber_fix.diff
-D debian/patches/progress_fix.diff
-D debian/patches/progress_object_callback_fix.diff
 M debian/patches/series
 M scripts/urlgrabber
 M setup.py
 M test/base_test_code.py
 M test/munittest.py
 M test/test_byterange.py
 M test/test_grabber.py
 M test/test_mirror.py
 M urlgrabber/__init__.py
 M urlgrabber/byterange.py
 M urlgrabber/grabber.py
 M urlgrabber/mirror.py
 M urlgrabber/progress.py
Text conflict in urlgrabber/grabber.py
1 conflicts encountered.
daniel@daydream:~/urlgrabber$

review: Needs Fixing

Unmerged revisions

12. By Jackson Doak on 2014-12-13

* New upstream release.
* Drop all patches, fixed upstream

=== removed file '.pc/applied-patches'
--- .pc/applied-patches	2011-08-09 17:45:08 +0000
+++ .pc/applied-patches	1970-01-01 00:00:00 +0000
@@ -1,3 +0,0 @@
-grabber_fix.diff
-progress_fix.diff
-progress_object_callback_fix.diff
 
=== removed directory '.pc/grabber_fix.diff'
=== removed directory '.pc/grabber_fix.diff/urlgrabber'
=== removed file '.pc/grabber_fix.diff/urlgrabber/grabber.py'
--- .pc/grabber_fix.diff/urlgrabber/grabber.py	2010-07-08 17:40:08 +0000
+++ .pc/grabber_fix.diff/urlgrabber/grabber.py	1970-01-01 00:00:00 +0000
@@ -1,1730 +0,0 @@
-#   This library is free software; you can redistribute it and/or
-#   modify it under the terms of the GNU Lesser General Public
-#   License as published by the Free Software Foundation; either
-#   version 2.1 of the License, or (at your option) any later version.
-#
-#   This library is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   Lesser General Public License for more details.
-#
-#   You should have received a copy of the GNU Lesser General Public
-#   License along with this library; if not, write to the 
-#      Free Software Foundation, Inc., 
-#      59 Temple Place, Suite 330, 
-#      Boston, MA  02111-1307  USA
-
-# This file is part of urlgrabber, a high-level cross-protocol url-grabber
-# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
-# Copyright 2009 Red Hat inc, pycurl code written by Seth Vidal
-
-"""A high-level cross-protocol url-grabber.
-
-GENERAL ARGUMENTS (kwargs)
-
-  Where possible, the module-level default is indicated, and legal
-  values are provided.
-
-  copy_local = 0   [0|1]
-
-    ignored except for file:// urls, in which case it specifies
-    whether urlgrab should still make a copy of the file, or simply
-    point to the existing copy. The module level default for this
-    option is 0.
-
-  close_connection = 0   [0|1]
-
-    tells URLGrabber to close the connection after a file has been
-    transfered. This is ignored unless the download happens with the
-    http keepalive handler (keepalive=1).  Otherwise, the connection
-    is left open for further use. The module level default for this
-    option is 0 (keepalive connections will not be closed).
-
-  keepalive = 1   [0|1]
-
-    specifies whether keepalive should be used for HTTP/1.1 servers
-    that support it. The module level default for this option is 1
-    (keepalive is enabled).
-
-  progress_obj = None
-
-    a class instance that supports the following methods:
-      po.start(filename, url, basename, length, text)
-      # length will be None if unknown
-      po.update(read) # read == bytes read so far
-      po.end()
-
-  text = None
-  
-    specifies alternative text to be passed to the progress meter
-    object.  If not given, the default progress meter will use the
-    basename of the file.
-
-  throttle = 1.0
-
-    a number - if it's an int, it's the bytes/second throttle limit.
-    If it's a float, it is first multiplied by bandwidth.  If throttle
-    == 0, throttling is disabled.  If None, the module-level default
-    (which can be set on default_grabber.throttle) is used. See
-    BANDWIDTH THROTTLING for more information.
-
-  timeout = None
-
-    a positive float expressing the number of seconds to wait for socket
-    operations. If the value is None or 0.0, socket operations will block
-    forever. Setting this option causes urlgrabber to call the settimeout
-    method on the Socket object used for the request. See the Python
-    documentation on settimeout for more information.
-    http://www.python.org/doc/current/lib/socket-objects.html
-
-  bandwidth = 0
-
-    the nominal max bandwidth in bytes/second.  If throttle is a float
-    and bandwidth == 0, throttling is disabled.  If None, the
-    module-level default (which can be set on
-    default_grabber.bandwidth) is used. See BANDWIDTH THROTTLING for
-    more information.
-
-  range = None
-
-    a tuple of the form (first_byte, last_byte) describing a byte
-    range to retrieve. Either or both of the values may set to
-    None. If first_byte is None, byte offset 0 is assumed. If
-    last_byte is None, the last byte available is assumed. Note that
-    the range specification is python-like in that (0,10) will yeild
-    the first 10 bytes of the file.
-
-    If set to None, no range will be used.
-    
-  reget = None   [None|'simple'|'check_timestamp']
-
-    whether to attempt to reget a partially-downloaded file.  Reget
-    only applies to .urlgrab and (obviously) only if there is a
-    partially downloaded file.  Reget has two modes:
-
-      'simple' -- the local file will always be trusted.  If there
-        are 100 bytes in the local file, then the download will always
-        begin 100 bytes into the requested file.
-
-      'check_timestamp' -- the timestamp of the server file will be
-        compared to the timestamp of the local file.  ONLY if the
-        local file is newer than or the same age as the server file
-        will reget be used.  If the server file is newer, or the
-        timestamp is not returned, the entire file will be fetched.
-
-    NOTE: urlgrabber can do very little to verify that the partial
-    file on disk is identical to the beginning of the remote file.
-    You may want to either employ a custom "checkfunc" or simply avoid
-    using reget in situations where corruption is a concern.
-
-  user_agent = 'urlgrabber/VERSION'
-
-    a string, usually of the form 'AGENT/VERSION' that is provided to
-    HTTP servers in the User-agent header. The module level default
-    for this option is "urlgrabber/VERSION".
-
-  http_headers = None
-
-    a tuple of 2-tuples, each containing a header and value.  These
-    will be used for http and https requests only.  For example, you
-    can do
-      http_headers = (('Pragma', 'no-cache'),)
-
-  ftp_headers = None
-
-    this is just like http_headers, but will be used for ftp requests.
-
-  proxies = None
-
-    a dictionary that maps protocol schemes to proxy hosts. For
-    example, to use a proxy server on host "foo" port 3128 for http
-    and https URLs:
-      proxies={ 'http' : 'http://foo:3128', 'https' : 'http://foo:3128' }
-    note that proxy authentication information may be provided using
-    normal URL constructs:
-      proxies={ 'http' : 'http://user:host@foo:3128' }
-    Lastly, if proxies is None, the default environment settings will
-    be used.
-
-  prefix = None
-
-    a url prefix that will be prepended to all requested urls.  For
-    example:
-      g = URLGrabber(prefix='http://foo.com/mirror/')
-      g.urlgrab('some/file.txt')
-      ## this will fetch 'http://foo.com/mirror/some/file.txt'
-    This option exists primarily to allow identical behavior to
-    MirrorGroup (and derived) instances.  Note: a '/' will be inserted
-    if necessary, so you cannot specify a prefix that ends with a
-    partial file or directory name.
-
-  opener = None
-    No-op when using the curl backend (default)
-
-  cache_openers = True
-    No-op when using the curl backend (default)
-
-  data = None
-
-    Only relevant for the HTTP family (and ignored for other
-    protocols), this allows HTTP POSTs.  When the data kwarg is
-    present (and not None), an HTTP request will automatically become
-    a POST rather than GET.  This is done by direct passthrough to
-    urllib2.  If you use this, you may also want to set the
-    'Content-length' and 'Content-type' headers with the http_headers
-    option.  Note that python 2.2 handles the case of these
-    badly and if you do not use the proper case (shown here), your
-    values will be overridden with the defaults.
-    
-  urlparser = URLParser()
-
-    The URLParser class handles pre-processing of URLs, including
-    auth-handling for user/pass encoded in http urls, file handing
-    (that is, filenames not sent as a URL), and URL quoting.  If you
-    want to override any of this behavior, you can pass in a
-    replacement instance.  See also the 'quote' option.
-
-  quote = None
-
-    Whether or not to quote the path portion of a url.
-      quote = 1    ->  quote the URLs (they're not quoted yet)
-      quote = 0    ->  do not quote them (they're already quoted)
-      quote = None ->  guess what to do
-
-    This option only affects proper urls like 'file:///etc/passwd'; it
-    does not affect 'raw' filenames like '/etc/passwd'.  The latter
-    will always be quoted as they are converted to URLs.  Also, only
-    the path part of a url is quoted.  If you need more fine-grained
-    control, you should probably subclass URLParser and pass it in via
-    the 'urlparser' option.
-
-  ssl_ca_cert = None
-
-    this option can be used if M2Crypto is available and will be
-    ignored otherwise.  If provided, it will be used to create an SSL
-    context.  If both ssl_ca_cert and ssl_context are provided, then
-    ssl_context will be ignored and a new context will be created from
-    ssl_ca_cert.
-
-  ssl_context = None
-
-    No-op when using the curl backend (default)
-   
-
-  self.ssl_verify_peer = True 
-
-    Check the server's certificate to make sure it is valid with what our CA validates
-  
-  self.ssl_verify_host = True
-
-    Check the server's hostname to make sure it matches the certificate DN
-
-  self.ssl_key = None
-
-    Path to the key the client should use to connect/authenticate with
-
-  self.ssl_key_type = 'PEM' 
-
-    PEM or DER - format of key
-     
-  self.ssl_cert = None
-
-    Path to the ssl certificate the client should use to to authenticate with
-
-  self.ssl_cert_type = 'PEM' 
-
-    PEM or DER - format of certificate
-    
-  self.ssl_key_pass = None 
-
-    password to access the ssl_key
-    
-  self.size = None
-
-    size (in bytes) or Maximum size of the thing being downloaded. 
-    This is mostly to keep us from exploding with an endless datastream
-  
-  self.max_header_size = 2097152 
-
-    Maximum size (in bytes) of the headers.
-    
-
-RETRY RELATED ARGUMENTS
-
-  retry = None
-
-    the number of times to retry the grab before bailing.  If this is
-    zero, it will retry forever. This was intentional... really, it
-    was :). If this value is not supplied or is supplied but is None
-    retrying does not occur.
-
-  retrycodes = [-1,2,4,5,6,7]
-
-    a sequence of errorcodes (values of e.errno) for which it should
-    retry. See the doc on URLGrabError for more details on this.  You
-    might consider modifying a copy of the default codes rather than
-    building yours from scratch so that if the list is extended in the
-    future (or one code is split into two) you can still enjoy the
-    benefits of the default list.  You can do that with something like
-    this:
-
-      retrycodes = urlgrabber.grabber.URLGrabberOptions().retrycodes
-      if 12 not in retrycodes:
-          retrycodes.append(12)
-      
-  checkfunc = None
-
-    a function to do additional checks. This defaults to None, which
-    means no additional checking.  The function should simply return
-    on a successful check.  It should raise URLGrabError on an
-    unsuccessful check.  Raising of any other exception will be
-    considered immediate failure and no retries will occur.
-
-    If it raises URLGrabError, the error code will determine the retry
-    behavior.  Negative error numbers are reserved for use by these
-    passed in functions, so you can use many negative numbers for
-    different types of failure.  By default, -1 results in a retry,
-    but this can be customized with retrycodes.
-
-    If you simply pass in a function, it will be given exactly one
-    argument: a CallbackObject instance with the .url attribute
-    defined and either .filename (for urlgrab) or .data (for urlread).
-    For urlgrab, .filename is the name of the local file.  For
-    urlread, .data is the actual string data.  If you need other
-    arguments passed to the callback (program state of some sort), you
-    can do so like this:
-
-      checkfunc=(function, ('arg1', 2), {'kwarg': 3})
-
-    if the downloaded file has filename /tmp/stuff, then this will
-    result in this call (for urlgrab):
-
-      function(obj, 'arg1', 2, kwarg=3)
-      # obj.filename = '/tmp/stuff'
-      # obj.url = 'http://foo.com/stuff'
-      
-    NOTE: both the "args" tuple and "kwargs" dict must be present if
-    you use this syntax, but either (or both) can be empty.
-
-  failure_callback = None
-
-    The callback that gets called during retries when an attempt to
-    fetch a file fails.  The syntax for specifying the callback is
-    identical to checkfunc, except for the attributes defined in the
-    CallbackObject instance.  The attributes for failure_callback are:
-
-      exception = the raised exception
-      url       = the url we're trying to fetch
-      tries     = the number of tries so far (including this one)
-      retry     = the value of the retry option
-
-    The callback is present primarily to inform the calling program of
-    the failure, but if it raises an exception (including the one it's
-    passed) that exception will NOT be caught and will therefore cause
-    future retries to be aborted.
-
-    The callback is called for EVERY failure, including the last one.
-    On the last try, the callback can raise an alternate exception,
-    but it cannot (without severe trickiness) prevent the exception
-    from being raised.
-
-  interrupt_callback = None
-
-    This callback is called if KeyboardInterrupt is received at any
-    point in the transfer.  Basically, this callback can have three
-    impacts on the fetch process based on the way it exits:
-
-      1) raise no exception: the current fetch will be aborted, but
-         any further retries will still take place
-
-      2) raise a URLGrabError: if you're using a MirrorGroup, then
-         this will prompt a failover to the next mirror according to
-         the behavior of the MirrorGroup subclass.  It is recommended
-         that you raise URLGrabError with code 15, 'user abort'.  If
-         you are NOT using a MirrorGroup subclass, then this is the
-         same as (3).
-
-      3) raise some other exception (such as KeyboardInterrupt), which
-         will not be caught at either the grabber or mirror levels.
-         That is, it will be raised up all the way to the caller.
-
-    This callback is very similar to failure_callback.  They are
-    passed the same arguments, so you could use the same function for
-    both.
-      
-BANDWIDTH THROTTLING
-
-  urlgrabber supports throttling via two values: throttle and
-  bandwidth Between the two, you can either specify and absolute
-  throttle threshold or specify a theshold as a fraction of maximum
-  available bandwidth.
-
-  throttle is a number - if it's an int, it's the bytes/second
-  throttle limit.  If it's a float, it is first multiplied by
-  bandwidth.  If throttle == 0, throttling is disabled.  If None, the
-  module-level default (which can be set with set_throttle) is used.
-
-  bandwidth is the nominal max bandwidth in bytes/second.  If throttle
-  is a float and bandwidth == 0, throttling is disabled.  If None, the
-  module-level default (which can be set with set_bandwidth) is used.
-
-  THROTTLING EXAMPLES:
-
-  Lets say you have a 100 Mbps connection.  This is (about) 10^8 bits
-  per second, or 12,500,000 Bytes per second.  You have a number of
-  throttling options:
-
-  *) set_bandwidth(12500000); set_throttle(0.5) # throttle is a float
-
-     This will limit urlgrab to use half of your available bandwidth.
-
-  *) set_throttle(6250000) # throttle is an int
-
-     This will also limit urlgrab to use half of your available
-     bandwidth, regardless of what bandwidth is set to.
-
-  *) set_throttle(6250000); set_throttle(1.0) # float
-
-     Use half your bandwidth
-
-  *) set_throttle(6250000); set_throttle(2.0) # float
-
-    Use up to 12,500,000 Bytes per second (your nominal max bandwidth)
-
-  *) set_throttle(6250000); set_throttle(0) # throttle = 0
-
-     Disable throttling - this is more efficient than a very large
-     throttle setting.
-
-  *) set_throttle(0); set_throttle(1.0) # throttle is float, bandwidth = 0
-
-     Disable throttling - this is the default when the module is loaded.
-
-  SUGGESTED AUTHOR IMPLEMENTATION (THROTTLING)
-
-  While this is flexible, it's not extremely obvious to the user.  I
-  suggest you implement a float throttle as a percent to make the
-  distinction between absolute and relative throttling very explicit.
-
-  Also, you may want to convert the units to something more convenient
-  than bytes/second, such as kbps or kB/s, etc.
-
-"""
-
-
-
-import os
-import sys
-import urlparse
-import time
-import string
-import urllib
-import urllib2
-import mimetools
-import thread
-import types
-import stat
-import pycurl
-from ftplib import parse150
-from StringIO import StringIO
-from httplib import HTTPException
-import socket
-from byterange import range_tuple_normalize, range_tuple_to_header, RangeError
-
-########################################################################
-#                     MODULE INITIALIZATION
-########################################################################
-try:
-    exec('from ' + (__name__.split('.'))[0] + ' import __version__')
-except:
-    __version__ = '???'
-
-########################################################################
-# functions for debugging output.  These functions are here because they
-# are also part of the module initialization.
-DEBUG = None
-def set_logger(DBOBJ):
-    """Set the DEBUG object.  This is called by _init_default_logger when
-    the environment variable URLGRABBER_DEBUG is set, but can also be
-    called by a calling program.  Basically, if the calling program uses
-    the logging module and would like to incorporate urlgrabber logging,
-    then it can do so this way.  It's probably not necessary as most
-    internal logging is only for debugging purposes.
-
-    The passed-in object should be a logging.Logger instance.  It will
-    be pushed into the keepalive and byterange modules if they're
-    being used.  The mirror module pulls this object in on import, so
-    you will need to manually push into it.  In fact, you may find it
-    tidier to simply push your logging object (or objects) into each
-    of these modules independently.
-    """
-
-    global DEBUG
-    DEBUG = DBOBJ
-
-def _init_default_logger(logspec=None):
-    '''Examines the environment variable URLGRABBER_DEBUG and creates
-    a logging object (logging.logger) based on the contents.  It takes
-    the form
-
-      URLGRABBER_DEBUG=level,filename
-      
-    where "level" can be either an integer or a log level from the
-    logging module (DEBUG, INFO, etc).  If the integer is zero or
-    less, logging will be disabled.  Filename is the filename where
-    logs will be sent.  If it is "-", then stdout will be used.  If
-    the filename is empty or missing, stderr will be used.  If the
-    variable cannot be processed or the logging module cannot be
-    imported (python < 2.3) then logging will be disabled.  Here are
-    some examples:
-
-      URLGRABBER_DEBUG=1,debug.txt   # log everything to debug.txt
-      URLGRABBER_DEBUG=WARNING,-     # log warning and higher to stdout
-      URLGRABBER_DEBUG=INFO          # log info and higher to stderr
-      
-    This funtion is called during module initialization.  It is not
-    intended to be called from outside.  The only reason it is a
-    function at all is to keep the module-level namespace tidy and to
-    collect the code into a nice block.'''
-
-    try:
-        if logspec is None:
-            logspec = os.environ['URLGRABBER_DEBUG']
-        dbinfo = logspec.split(',')
-        import logging
-        level = logging._levelNames.get(dbinfo[0], None)
-        if level is None: level = int(dbinfo[0])
-        if level < 1: raise ValueError()
-
-        formatter = logging.Formatter('%(asctime)s %(message)s')
-        if len(dbinfo) > 1: filename = dbinfo[1]
-        else: filename = ''
-        if filename == '': handler = logging.StreamHandler(sys.stderr)
-        elif filename == '-': handler = logging.StreamHandler(sys.stdout)
-        else:  handler = logging.FileHandler(filename)
-        handler.setFormatter(formatter)
-        DBOBJ = logging.getLogger('urlgrabber')
-        DBOBJ.addHandler(handler)
-        DBOBJ.setLevel(level)
-    except (KeyError, ImportError, ValueError):
-        DBOBJ = None
-    set_logger(DBOBJ)
-
-def _log_package_state():
-    if not DEBUG: return
-    DEBUG.info('urlgrabber version  = %s' % __version__)
-    DEBUG.info('trans function "_"  = %s' % _)
-        
-_init_default_logger()
-_log_package_state()
-
-
-# normally this would be from i18n or something like it ...
-def _(st):
-    return st
-
-########################################################################
-#                 END MODULE INITIALIZATION
-########################################################################
-
-
-
-class URLGrabError(IOError):
-    """
-    URLGrabError error codes:
-
-      URLGrabber error codes (0 -- 255)
-        0    - everything looks good (you should never see this)
-        1    - malformed url
-        2    - local file doesn't exist
-        3    - request for non-file local file (dir, etc)
-        4    - IOError on fetch
-        5    - OSError on fetch
-        6    - no content length header when we expected one
-        7    - HTTPException
-        8    - Exceeded read limit (for urlread)
-        9    - Requested byte range not satisfiable.
-        10   - Byte range requested, but range support unavailable
-        11   - Illegal reget mode
-        12   - Socket timeout
-        13   - malformed proxy url
-        14   - HTTPError (includes .code and .exception attributes)
-        15   - user abort
-        16   - error writing to local file
-        
-      MirrorGroup error codes (256 -- 511)
-        256  - No more mirrors left to try
-
-      Custom (non-builtin) classes derived from MirrorGroup (512 -- 767)
-        [ this range reserved for application-specific error codes ]
-
-      Retry codes (< 0)
-        -1   - retry the download, unknown reason
-
-    Note: to test which group a code is in, you can simply do integer
-    division by 256: e.errno / 256
-
-    Negative codes are reserved for use by functions passed in to
-    retrygrab with checkfunc.  The value -1 is built in as a generic
-    retry code and is already included in the retrycodes list.
-    Therefore, you can create a custom check function that simply
-    returns -1 and the fetch will be re-tried.  For more customized
-    retries, you can use other negative number and include them in
-    retry-codes.  This is nice for outputting useful messages about
-    what failed.
-
-    You can use these error codes like so:
-      try: urlgrab(url)
-      except URLGrabError, e:
-         if e.errno == 3: ...
-           # or
-         print e.strerror
-           # or simply
-         print e  #### print '[Errno %i] %s' % (e.errno, e.strerror)
-    """
-    def __init__(self, *args):
-        IOError.__init__(self, *args)
-        self.url = "No url specified"
-
-class CallbackObject:
-    """Container for returned callback data.
-
-    This is currently a dummy class into which urlgrabber can stuff
-    information for passing to callbacks.  This way, the prototype for
-    all callbacks is the same, regardless of the data that will be
-    passed back.  Any function that accepts a callback function as an
-    argument SHOULD document what it will define in this object.
-
-    It is possible that this class will have some greater
-    functionality in the future.
-    """
-    def __init__(self, **kwargs):
-        self.__dict__.update(kwargs)
-
-def urlgrab(url, filename=None, **kwargs):
-    """grab the file at <url> and make a local copy at <filename>
-    If filename is none, the basename of the url is used.
-    urlgrab returns the filename of the local file, which may be different
-    from the passed-in filename if the copy_local kwarg == 0.
-    
-    See module documentation for a description of possible kwargs.
-    """
-    return default_grabber.urlgrab(url, filename, **kwargs)
-
-def urlopen(url, **kwargs):
-    """open the url and return a file object
-    If a progress object or throttle specifications exist, then
-    a special file object will be returned that supports them.
-    The file object can be treated like any other file object.
-    
-    See module documentation for a description of possible kwargs.
-    """
-    return default_grabber.urlopen(url, **kwargs)
-
-def urlread(url, limit=None, **kwargs):
-    """read the url into a string, up to 'limit' bytes
-    If the limit is exceeded, an exception will be thrown.  Note that urlread
-    is NOT intended to be used as a way of saying "I want the first N bytes"
-    but rather 'read the whole file into memory, but don't use too much'
-    
-    See module documentation for a description of possible kwargs.
-    """
-    return default_grabber.urlread(url, limit, **kwargs)
-
-
-class URLParser:
-    """Process the URLs before passing them to urllib2.
-
-    This class does several things:
-
-      * add any prefix
-      * translate a "raw" file to a proper file: url
-      * handle any http or https auth that's encoded within the url
-      * quote the url
-
-    Only the "parse" method is called directly, and it calls sub-methods.
-
-    An instance of this class is held in the options object, which
-    means that it's easy to change the behavior by sub-classing and
-    passing the replacement in.  It need only have a method like:
-
-        url, parts = urlparser.parse(url, opts)
-    """
-
-    def parse(self, url, opts):
-        """parse the url and return the (modified) url and its parts
-
-        Note: a raw file WILL be quoted when it's converted to a URL.
-        However, other urls (ones which come with a proper scheme) may
-        or may not be quoted according to opts.quote
-
-          opts.quote = 1     --> quote it
-          opts.quote = 0     --> do not quote it
-          opts.quote = None  --> guess
-        """
-        quote = opts.quote
-        
-        if opts.prefix:
-            url = self.add_prefix(url, opts.prefix)
-            
-        parts = urlparse.urlparse(url)
-        (scheme, host, path, parm, query, frag) = parts
-
-        if not scheme or (len(scheme) == 1 and scheme in string.letters):
-            # if a scheme isn't specified, we guess that it's "file:"
-            if url[0] not in '/\\': url = os.path.abspath(url)
-            url = 'file:' + urllib.pathname2url(url)
-            parts = urlparse.urlparse(url)
-            quote = 0 # pathname2url quotes, so we won't do it again
-            
-        if scheme in ['http', 'https']:
-            parts = self.process_http(parts, url)
-            
-        if quote is None:
-            quote = self.guess_should_quote(parts)
-        if quote:
-            parts = self.quote(parts)
-        
-        url = urlparse.urlunparse(parts)
-        return url, parts
-
-    def add_prefix(self, url, prefix):
-        if prefix[-1] == '/' or url[0] == '/':
-            url = prefix + url
-        else:
-            url = prefix + '/' + url
-        return url
-
-    def process_http(self, parts, url):
-        (scheme, host, path, parm, query, frag) = parts
-        # TODO: auth-parsing here, maybe? pycurl doesn't really need it
-        return (scheme, host, path, parm, query, frag)
-
-    def quote(self, parts):
-        """quote the URL
-
-        This method quotes ONLY the path part.  If you need to quote
-        other parts, you should override this and pass in your derived
-        class.  The other alternative is to quote other parts before
-        passing into urlgrabber.
-        """
-        (scheme, host, path, parm, query, frag) = parts
-        path = urllib.quote(path)
-        return (scheme, host, path, parm, query, frag)
-
-    hexvals = '0123456789ABCDEF'
-    def guess_should_quote(self, parts):
-        """
-        Guess whether we should quote a path.  This amounts to
-        guessing whether it's already quoted.
-
-        find ' '   ->  1
-        find '%'   ->  1
-        find '%XX' ->  0
-        else       ->  1
-        """
-        (scheme, host, path, parm, query, frag) = parts
-        if ' ' in path:
-            return 1
-        ind = string.find(path, '%')
-        if ind > -1:
-            while ind > -1:
-                if len(path) < ind+3:
-                    return 1
-                code = path[ind+1:ind+3].upper()
-                if     code[0] not in self.hexvals or \
-                       code[1] not in self.hexvals:
-                    return 1
-                ind = string.find(path, '%', ind+1)
-            return 0
-        return 1
-    
-class URLGrabberOptions:
-    """Class to ease kwargs handling."""
-
-    def __init__(self, delegate=None, **kwargs):
-        """Initialize URLGrabberOptions object.
-        Set default values for all options and then update options specified
-        in kwargs.
-        """
-        self.delegate = delegate
-        if delegate is None:
-            self._set_defaults()
-        self._set_attributes(**kwargs)
-    
-    def __getattr__(self, name):
-        if self.delegate and hasattr(self.delegate, name):
-            return getattr(self.delegate, name)
-        raise AttributeError, name
-    
-    def raw_throttle(self):
-        """Calculate raw throttle value from throttle and bandwidth 
-        values.
-        """
-        if self.throttle <= 0:  
-            return 0
-        elif type(self.throttle) == type(0): 
-            return float(self.throttle)
-        else: # throttle is a float
-            return self.bandwidth * self.throttle
-        
-    def derive(self, **kwargs):
-        """Create a derived URLGrabberOptions instance.
-        This method creates a new instance and overrides the
-        options specified in kwargs.
-        """
-        return URLGrabberOptions(delegate=self, **kwargs)
-        
-    def _set_attributes(self, **kwargs):
-        """Update object attributes with those provided in kwargs."""
-        self.__dict__.update(kwargs)
-        if kwargs.has_key('range'):
-            # normalize the supplied range value
-            self.range = range_tuple_normalize(self.range)
-        if not self.reget in [None, 'simple', 'check_timestamp']:
-            raise URLGrabError(11, _('Illegal reget mode: %s') \
-                               % (self.reget, ))
-
-    def _set_defaults(self):
-        """Set all options to their default values. 
-        When adding new options, make sure a default is
-        provided here.
-        """
-        self.progress_obj = None
-        self.throttle = 1.0
-        self.bandwidth = 0
-        self.retry = None
-        self.retrycodes = [-1,2,4,5,6,7]
-        self.checkfunc = None
-        self.copy_local = 0
-        self.close_connection = 0
-        self.range = None
-        self.user_agent = 'urlgrabber/%s' % __version__
-        self.keepalive = 1
-        self.proxies = None
-        self.reget = None
-        self.failure_callback = None
-        self.interrupt_callback = None
-        self.prefix = None
-        self.opener = None
-        self.cache_openers = True
-        self.timeout = None
-        self.text = None
-        self.http_headers = None
-        self.ftp_headers = None
-        self.data = None
-        self.urlparser = URLParser()
-        self.quote = None
-        self.ssl_ca_cert = None # sets SSL_CAINFO - path to certdb
-        self.ssl_context = None # no-op in pycurl
-        self.ssl_verify_peer = True # check peer's cert for authenticityb
-        self.ssl_verify_host = True # make sure who they are and who the cert is for matches
-        self.ssl_key = None # client key
-        self.ssl_key_type = 'PEM' #(or DER)
-        self.ssl_cert = None # client cert
-        self.ssl_cert_type = 'PEM' # (or DER)
-        self.ssl_key_pass = None # password to access the key
-        self.size = None # if we know how big the thing we're getting is going
-                         # to be. this is ultimately a MAXIMUM size for the file
-        self.max_header_size = 2097152 #2mb seems reasonable for maximum header size
-        
-    def __repr__(self):
-        return self.format()
-        
-    def format(self, indent='  '):
-        keys = self.__dict__.keys()
-        if self.delegate is not None:
-            keys.remove('delegate')
-        keys.sort()
-        s = '{\n'
-        for k in keys:
-            s = s + indent + '%-15s: %s,\n' % \
-                (repr(k), repr(self.__dict__[k]))
-        if self.delegate:
-            df = self.delegate.format(indent + '  ')
-            s = s + indent + '%-15s: %s\n' % ("'delegate'", df)
-        s = s + indent + '}'
-        return s
-
-class URLGrabber:
-    """Provides easy opening of URLs with a variety of options.
-    
-    All options are specified as kwargs. Options may be specified when
-    the class is created and may be overridden on a per request basis.
-    
-    New objects inherit default values from default_grabber.
-    """
-    
-    def __init__(self, **kwargs):
-        self.opts = URLGrabberOptions(**kwargs)
-    
-    def _retry(self, opts, func, *args):
-        tries = 0
-        while 1:
-            # there are only two ways out of this loop.  The second has
-            # several "sub-ways"
-            #   1) via the return in the "try" block
-            #   2) by some exception being raised
-            #      a) an excepton is raised that we don't "except"
-            #      b) a callback raises ANY exception
-            #      c) we're not retry-ing or have run out of retries
-            #      d) the URLGrabError code is not in retrycodes
-            # beware of infinite loops :)
-            tries = tries + 1
-            exception = None
-            retrycode = None
-            callback  = None
-            if DEBUG: DEBUG.info('attempt %i/%s: %s',
-                                 tries, opts.retry, args[0])
-            try:
-                r = apply(func, (opts,) + args, {})
-                if DEBUG: DEBUG.info('success')
-                return r
-            except URLGrabError, e:
-                exception = e
-                callback = opts.failure_callback
-                retrycode = e.errno
-            except KeyboardInterrupt, e:
-                exception = e
-                callback = opts.interrupt_callback
-
-            if DEBUG: DEBUG.info('exception: %s', exception)
-            if callback:
-                if DEBUG: DEBUG.info('calling callback: %s', callback)
-                cb_func, cb_args, cb_kwargs = self._make_callback(callback)
-                obj = CallbackObject(exception=exception, url=args[0],
-                                     tries=tries, retry=opts.retry)
-                cb_func(obj, *cb_args, **cb_kwargs)
-
-            if (opts.retry is None) or (tries == opts.retry):
-                if DEBUG: DEBUG.info('retries exceeded, re-raising')
-                raise
-
-            if (retrycode is not None) and (retrycode not in opts.retrycodes):
-                if DEBUG: DEBUG.info('retrycode (%i) not in list %s, re-raising',
-                                     retrycode, opts.retrycodes)
-                raise
-    
-    def urlopen(self, url, **kwargs):
-        """open the url and return a file object
-        If a progress object or throttle value specified when this 
-        object was created, then  a special file object will be 
-        returned that supports them. The file object can be treated 
-        like any other file object.
-        """
-        opts = self.opts.derive(**kwargs)
-        if DEBUG: DEBUG.debug('combined options: %s' % repr(opts))
-        (url,parts) = opts.urlparser.parse(url, opts) 
-        def retryfunc(opts, url):
-            return PyCurlFileObject(url, filename=None, opts=opts)
-        return self._retry(opts, retryfunc, url)
-    
-    def urlgrab(self, url, filename=None, **kwargs):
-        """grab the file at <url> and make a local copy at <filename>
-        If filename is none, the basename of the url is used.
-        urlgrab returns the filename of the local file, which may be 
-        different from the passed-in filename if copy_local == 0.
-        """
-        opts = self.opts.derive(**kwargs)
-        if DEBUG: DEBUG.debug('combined options: %s' % repr(opts))
-        (url,parts) = opts.urlparser.parse(url, opts) 
-        (scheme, host, path, parm, query, frag) = parts
-        if filename is None:
-            filename = os.path.basename( urllib.unquote(path) )
-        if scheme == 'file' and not opts.copy_local:
-            # just return the name of the local file - don't make a 
-            # copy currently
-            path = urllib.url2pathname(path)
-            if host:
-                path = os.path.normpath('//' + host + path)
-            if not os.path.exists(path):
-                err = URLGrabError(2, 
-                      _('Local file does not exist: %s') % (path, ))
-                err.url = url
-                raise err
-            elif not os.path.isfile(path):
-                err = URLGrabError(3, 
-                                 _('Not a normal file: %s') % (path, ))
-                err.url = url
-                raise err
-
-            elif not opts.range:
-                if not opts.checkfunc is None:
-                    cb_func, cb_args, cb_kwargs = \
-                       self._make_callback(opts.checkfunc)
-                    obj = CallbackObject()
-                    obj.filename = path
-                    obj.url = url
-                    apply(cb_func, (obj, )+cb_args, cb_kwargs)        
-                return path
-        
-        def retryfunc(opts, url, filename):
-            fo = PyCurlFileObject(url, filename, opts)
-            try:
-                fo._do_grab()
-                if not opts.checkfunc is None:
-                    cb_func, cb_args, cb_kwargs = \
-                             self._make_callback(opts.checkfunc)
-                    obj = CallbackObject()
-                    obj.filename = filename
-                    obj.url = url
-                    apply(cb_func, (obj, )+cb_args, cb_kwargs)
-            finally:
-                fo.close()
-            return filename
-        
-        return self._retry(opts, retryfunc, url, filename)
-    
-    def urlread(self, url, limit=None, **kwargs):
-        """read the url into a string, up to 'limit' bytes
-        If the limit is exceeded, an exception will be thrown.  Note
-        that urlread is NOT intended to be used as a way of saying 
-        "I want the first N bytes" but rather 'read the whole file 
-        into memory, but don't use too much'
-        """
-        opts = self.opts.derive(**kwargs)
-        if DEBUG: DEBUG.debug('combined options: %s' % repr(opts))
-        (url,parts) = opts.urlparser.parse(url, opts) 
-        if limit is not None:
-            limit = limit + 1
-            
-        def retryfunc(opts, url, limit):
-            fo = PyCurlFileObject(url, filename=None, opts=opts)
-            s = ''
-            try:
-                # this is an unfortunate thing.  Some file-like objects
-                # have a default "limit" of None, while the built-in (real)
-                # file objects have -1.  They each break the other, so for
-                # now, we just force the default if necessary.
-                if limit is None: s = fo.read()
-                else: s = fo.read(limit)
-
-                if not opts.checkfunc is None:
-                    cb_func, cb_args, cb_kwargs = \
-                             self._make_callback(opts.checkfunc)
-                    obj = CallbackObject()
-                    obj.data = s
-                    obj.url = url
-                    apply(cb_func, (obj, )+cb_args, cb_kwargs)
-            finally:
-                fo.close()
-            return s
-            
-        s = self._retry(opts, retryfunc, url, limit)
-        if limit and len(s) > limit:
-            err = URLGrabError(8, 
-                               _('Exceeded limit (%i): %s') % (limit, url))
-            err.url = url
-            raise err
-
-        return s
-        
-    def _make_callback(self, callback_obj):
-        if callable(callback_obj):
-            return callback_obj, (), {}
-        else:
-            return callback_obj
-
-# create the default URLGrabber used by urlXXX functions.
-# NOTE: actual defaults are set in URLGrabberOptions
-default_grabber = URLGrabber()
-
-
-class PyCurlFileObject():
-    def __init__(self, url, filename, opts):
-        self.fo = None
-        self._hdr_dump = ''
-        self._parsed_hdr = None
-        self.url = url
-        self.scheme = urlparse.urlsplit(self.url)[0]
-        self.filename = filename
-        self.append = False
-        self.reget_time = None
-        self.opts = opts
-        if self.opts.reget == 'check_timestamp':
-            raise NotImplementedError, "check_timestamp regets are not implemented in this ver of urlgrabber. Please report this."
-        self._complete = False
-        self._rbuf = ''
-        self._rbufsize = 1024*8
-        self._ttime = time.time()
-        self._tsize = 0
-        self._amount_read = 0
-        self._reget_length = 0
-        self._prog_running = False
-        self._error = (None, None)
-        self.size = None
-        self._do_open()
-        
-        
-    def __getattr__(self, name):
-        """This effectively allows us to wrap at the instance level.
-        Any attribute not found in _this_ object will be searched for
-        in self.fo.  This includes methods."""
-
-        if hasattr(self.fo, name):
-            return getattr(self.fo, name)
-        raise AttributeError, name
-
-    def _retrieve(self, buf):
-        try:
-            if not self._prog_running:
-                if self.opts.progress_obj:
-                    size  = self.size + self._reget_length
-                    self.opts.progress_obj.start(self._prog_reportname, 
-                                                 urllib.unquote(self.url), 
-                                                 self._prog_basename, 
-                                                 size=size,
-                                                 text=self.opts.text)
-                    self._prog_running = True
-                    self.opts.progress_obj.update(self._amount_read)
-
-            self._amount_read += len(buf)
-            self.fo.write(buf)
-            return len(buf)
-        except KeyboardInterrupt:
-            return -1
-            
-    def _hdr_retrieve(self, buf):
-        if self._over_max_size(cur=len(self._hdr_dump), 
-                               max_size=self.opts.max_header_size):
-            return -1            
-        try:
-            self._hdr_dump += buf
-            # we have to get the size before we do the progress obj start
-            # but we can't do that w/o making it do 2 connects, which sucks
-            # so we cheat and stuff it in here in the hdr_retrieve
-            if self.scheme in ['http','https'] and buf.lower().find('content-length') != -1:
-                length = buf.split(':')[1]
-                self.size = int(length)
-            elif self.scheme in ['ftp']:
-                s = None
-                if buf.startswith('213 '):
-                    s = buf[3:].strip()
-                elif buf.startswith('150 '):
-                    s = parse150(buf)
-                if s:
-                    self.size = int(s)
-            
-            return len(buf)
-        except KeyboardInterrupt:
-            return pycurl.READFUNC_ABORT
-
-    def _return_hdr_obj(self):
-        if self._parsed_hdr:
-            return self._parsed_hdr
-        statusend = self._hdr_dump.find('\n')
-        hdrfp = StringIO()
-        hdrfp.write(self._hdr_dump[statusend:])
-        self._parsed_hdr =  mimetools.Message(hdrfp)
-        return self._parsed_hdr
-    
-    hdr = property(_return_hdr_obj)
-    http_code = property(fget=
-                 lambda self: self.curl_obj.getinfo(pycurl.RESPONSE_CODE))
-
-    def _set_opts(self, opts={}):
-        # XXX
-        if not opts:
-            opts = self.opts
-
-
-        # defaults we're always going to set
-        self.curl_obj.setopt(pycurl.NOPROGRESS, False)
-        self.curl_obj.setopt(pycurl.NOSIGNAL, True)
-        self.curl_obj.setopt(pycurl.WRITEFUNCTION, self._retrieve)
-        self.curl_obj.setopt(pycurl.HEADERFUNCTION, self._hdr_retrieve)
-        self.curl_obj.setopt(pycurl.PROGRESSFUNCTION, self._progress_update)
-        self.curl_obj.setopt(pycurl.FAILONERROR, True)
-        self.curl_obj.setopt(pycurl.OPT_FILETIME, True)
-        
-        if DEBUG:
-            self.curl_obj.setopt(pycurl.VERBOSE, True)
-        if opts.user_agent:
-            self.curl_obj.setopt(pycurl.USERAGENT, opts.user_agent)
-        
-        # maybe to be options later
-        self.curl_obj.setopt(pycurl.FOLLOWLOCATION, True)
-        self.curl_obj.setopt(pycurl.MAXREDIRS, 5)
-        
-        # timeouts
-        timeout = 300
-        if opts.timeout:
-            timeout = int(opts.timeout)
-            self.curl_obj.setopt(pycurl.CONNECTTIMEOUT, timeout)
-
-        # ssl options
-        if self.scheme == 'https':
-            if opts.ssl_ca_cert: # this may do ZERO with nss  according to curl docs
-                self.curl_obj.setopt(pycurl.CAPATH, opts.ssl_ca_cert)
-                self.curl_obj.setopt(pycurl.CAINFO, opts.ssl_ca_cert)
-            self.curl_obj.setopt(pycurl.SSL_VERIFYPEER, opts.ssl_verify_peer)
-            self.curl_obj.setopt(pycurl.SSL_VERIFYHOST, opts.ssl_verify_host)
-            if opts.ssl_key:
-                self.curl_obj.setopt(pycurl.SSLKEY, opts.ssl_key)
-            if opts.ssl_key_type:
-                self.curl_obj.setopt(pycurl.SSLKEYTYPE, opts.ssl_key_type)
-            if opts.ssl_cert:
-                self.curl_obj.setopt(pycurl.SSLCERT, opts.ssl_cert)
-            if opts.ssl_cert_type:                
-                self.curl_obj.setopt(pycurl.SSLCERTTYPE, opts.ssl_cert_type)
-            if opts.ssl_key_pass:
-                self.curl_obj.setopt(pycurl.SSLKEYPASSWD, opts.ssl_key_pass)
-
-        #headers:
-        if opts.http_headers and self.scheme in ('http', 'https'):
-            headers = []
-            for (tag, content) in opts.http_headers:
-                headers.append('%s:%s' % (tag, content))
-            self.curl_obj.setopt(pycurl.HTTPHEADER, headers)
-
-        # ranges:
-        if opts.range or opts.reget:
-            range_str = self._build_range()
-            if range_str:
-                self.curl_obj.setopt(pycurl.RANGE, range_str)
-            
-        # throttle/bandwidth
-        if hasattr(opts, 'raw_throttle') and opts.raw_throttle():
-            self.curl_obj.setopt(pycurl.MAX_RECV_SPEED_LARGE, int(opts.raw_throttle()))
-            
-        # proxy settings
-        if opts.proxies:
-            for (scheme, proxy) in opts.proxies.items():
-                if self.scheme in ('ftp'): # only set the ftp proxy for ftp items
-                    if scheme not in ('ftp'):
-                        continue
-                    else:
-                        if proxy == '_none_': proxy = ""
-                        self.curl_obj.setopt(pycurl.PROXY, proxy)
-                elif self.scheme in ('http', 'https'):
-                    if scheme not in ('http', 'https'):
-                        continue
-                    else:
-                        if proxy == '_none_': proxy = ""
-                        self.curl_obj.setopt(pycurl.PROXY, proxy)
-            
-        # FIXME username/password/auth settings
-
-        #posts - simple - expects the fields as they are
-        if opts.data:
-            self.curl_obj.setopt(pycurl.POST, True)
-            self.curl_obj.setopt(pycurl.POSTFIELDS, self._to_utf8(opts.data))
-            
-        # our url
-        self.curl_obj.setopt(pycurl.URL, self.url)
-        
-    
-    def _do_perform(self):
-        if self._complete:
-            return
-        
-        try:
-            self.curl_obj.perform()
-        except pycurl.error, e:
-            # XXX - break some of these out a bit more clearly
-            # to other URLGrabErrors from 
-            # http://curl.haxx.se/libcurl/c/libcurl-errors.html
-            # this covers e.args[0] == 22 pretty well - which will be common
-            
-            code = self.http_code
-            errcode = e.args[0]
-            if self._error[0]:
-                errcode = self._error[0]
-                
-            if errcode == 23 and code >= 200 and code < 299:
-                err = URLGrabError(15, _('User (or something) called abort %s: %s') % (self.url, e))
-                err.url = self.url
-                
-                # this is probably wrong but ultimately this is what happens
-                # we have a legit http code and a pycurl 'writer failed' code
-                # which almost always means something aborted it from outside
-                # since we cannot know what it is -I'm banking on it being
-                # a ctrl-c. XXXX - if there's a way of going back two raises to 
-                # figure out what aborted the pycurl process FIXME
-                raise KeyboardInterrupt
-            
-            elif errcode == 28:
-                err = URLGrabError(12, _('Timeout on %s: %s') % (self.url, e))
-                err.url = self.url
-                raise err
-            elif errcode == 35:
-                msg = _("problem making ssl connection")
-                err = URLGrabError(14, msg)
-                err.url = self.url
-                raise err
-            elif errcode == 37:
-                msg = _("Could not open/read %s") % (self.url)
-                err = URLGrabError(14, msg)
-                err.url = self.url
-                raise err
-                
-            elif errcode == 42:
-                err = URLGrabError(15, _('User (or something) called abort %s: %s') % (self.url, e))
-                err.url = self.url
-                # this is probably wrong but ultimately this is what happens
-                # we have a legit http code and a pycurl 'writer failed' code
-                # which almost always means something aborted it from outside
-                # since we cannot know what it is -I'm banking on it being
-                # a ctrl-c. XXXX - if there's a way of going back two raises to 
-                # figure out what aborted the pycurl process FIXME
-                raise KeyboardInterrupt
-                
-            elif errcode == 58:
-                msg = _("problem with the local client certificate")
-                err = URLGrabError(14, msg)
-                err.url = self.url
-                raise err
-
-            elif errcode == 60:
-                msg = _("client cert cannot be verified or client cert incorrect")
-                err = URLGrabError(14, msg)
-                err.url = self.url
-                raise err
-            
-            elif errcode == 63:
-                if self._error[1]:
-                    msg = self._error[1]
-                else:
-                    msg = _("Max download size exceeded on %s") % (self.url)
-                err = URLGrabError(14, msg)
-                err.url = self.url
-                raise err
-                    
-            elif str(e.args[1]) == '' and self.http_code != 0: # fake it until you make it
-                msg = 'HTTP Error %s : %s ' % (self.http_code, self.url)
-            else:
-                msg = 'PYCURL ERROR %s - "%s"' % (errcode, str(e.args[1]))
-                code = errcode
-            err = URLGrabError(14, msg)
-            err.code = code
-            err.exception = e
-            raise err
-
-    def _do_open(self):
-        self.curl_obj = _curl_cache
-        self.curl_obj.reset() # reset all old settings away, just in case
-        # setup any ranges
-        self._set_opts()
-        self._do_grab()
-        return self.fo
-
-    def _add_headers(self):
-        pass
-        
-    def _build_range(self):
-        reget_length = 0
-        rt = None
-        if self.opts.reget and type(self.filename) in types.StringTypes:
-            # we have reget turned on and we're dumping to a file
-            try:
-                s = os.stat(self.filename)
-            except OSError:
-                pass
-            else:
-                self.reget_time = s[stat.ST_MTIME]
-                reget_length = s[stat.ST_SIZE]
-
-                # Set initial length when regetting
-                self._amount_read = reget_length    
-                self._reget_length = reget_length # set where we started from, too
-
-                rt = reget_length, ''
-                self.append = 1
-                
-        if self.opts.range:
-            rt = self.opts.range
-            if rt[0]: rt = (rt[0] + reget_length, rt[1])
-
-        if rt:
-            header = range_tuple_to_header(rt)
-            if header:
-                return header.split('=')[1]
-
-
-
-    def _make_request(self, req, opener):
-        #XXXX
-        # This doesn't do anything really, but we could use this
-        # instead of do_open() to catch a lot of crap errors as 
-        # mstenner did before here
-        return (self.fo, self.hdr)
-        
-        try:
-            if self.opts.timeout:
-                old_to = socket.getdefaulttimeout()
-                socket.setdefaulttimeout(self.opts.timeout)
-                try:
-                    fo = opener.open(req)
-                finally:
-                    socket.setdefaulttimeout(old_to)
-            else:
-                fo = opener.open(req)
-            hdr = fo.info()
-        except ValueError, e:
-            err = URLGrabError(1, _('Bad URL: %s : %s') % (self.url, e, ))
-            err.url = self.url
-            raise err
-
-        except RangeError, e:
-            err = URLGrabError(9, _('%s on %s') % (e, self.url))
-            err.url = self.url
-            raise err
-        except urllib2.HTTPError, e:
-            new_e = URLGrabError(14, _('%s on %s') % (e, self.url))
-            new_e.code = e.code
-            new_e.exception = e
-            new_e.url = self.url
-            raise new_e
-        except IOError, e:
-            if hasattr(e, 'reason') and isinstance(e.reason, socket.timeout):
-                err = URLGrabError(12, _('Timeout on %s: %s') % (self.url, e))
-                err.url = self.url
-                raise err
-            else:
-                err = URLGrabError(4, _('IOError on %s: %s') % (self.url, e))
-                err.url = self.url
-                raise err
-
-        except OSError, e:
-            err = URLGrabError(5, _('%s on %s') % (e, self.url))
-            err.url = self.url
-            raise err
-
-        except HTTPException, e:
-            err = URLGrabError(7, _('HTTP Exception (%s) on %s: %s') % \
-                            (e.__class__.__name__, self.url, e))
-            err.url = self.url
-            raise err
-
-        else:
-            return (fo, hdr)
-        
-    def _do_grab(self):
-        """dump the file to a filename or StringIO buffer"""
-
-        if self._complete:
-            return
-        _was_filename = False
-        if type(self.filename) in types.StringTypes and self.filename:
-            _was_filename = True
-            self._prog_reportname = str(self.filename)
-            self._prog_basename = os.path.basename(self.filename)
-            
-            if self.append: mode = 'ab'
-            else: mode = 'wb'
-
-            if DEBUG: DEBUG.info('opening local file "%s" with mode %s' % \
-                                 (self.filename, mode))
-            try:
-                self.fo = open(self.filename, mode)
-            except IOError, e:
-                err = URLGrabError(16, _(\
-                  'error opening local file from %s, IOError: %s') % (self.url, e))
-                err.url = self.url
-                raise err
-
-        else:
-            self._prog_reportname = 'MEMORY'
-            self._prog_basename = 'MEMORY'
-
-            
-            self.fo = StringIO()
-            # if this is to be a tempfile instead....
-            # it just makes crap in the tempdir
-            #fh, self._temp_name = mkstemp()
-            #self.fo = open(self._temp_name, 'wb')
-
-            
-        self._do_perform()
-        
-
-
-        if _was_filename:
-            # close it up
-            self.fo.flush()
-            self.fo.close()
-            # set the time
-            mod_time = self.curl_obj.getinfo(pycurl.INFO_FILETIME)
-            if mod_time != -1:
-                os.utime(self.filename, (mod_time, mod_time))
-            # re open it
-            self.fo = open(self.filename, 'r')
-        else:
-            #self.fo = open(self._temp_name, 'r')
-            self.fo.seek(0)
-
-        self._complete = True
-    
-    def _fill_buffer(self, amt=None):
-        """fill the buffer to contain at least 'amt' bytes by reading
-        from the underlying file object.  If amt is None, then it will
-        read until it gets nothing more.  It updates the progress meter
-        and throttles after every self._rbufsize bytes."""
-        # the _rbuf test is only in this first 'if' for speed.  It's not
-        # logically necessary
-        if self._rbuf and not amt is None:
-            L = len(self._rbuf)
-            if amt > L:
-                amt = amt - L
-            else:
-                return
-
-        # if we've made it here, then we don't have enough in the buffer
-        # and we need to read more.
-        
-        if not self._complete: self._do_grab() #XXX cheater - change on ranges
-        
-        buf = [self._rbuf]
-        bufsize = len(self._rbuf)
-        while amt is None or amt:
-            # first, delay if necessary for throttling reasons
-            if self.opts.raw_throttle():
-                diff = self._tsize/self.opts.raw_throttle() - \
-                       (time.time() - self._ttime)
-                if diff > 0: time.sleep(diff)
-                self._ttime = time.time()
-                
-            # now read some data, up to self._rbufsize
-            if amt is None: readamount = self._rbufsize
-            else:           readamount = min(amt, self._rbufsize)
-            try:
-                new = self.fo.read(readamount)
-            except socket.error, e:
-                err = URLGrabError(4, _('Socket Error on %s: %s') % (self.url, e))
-                err.url = self.url
-                raise err
-
-            except socket.timeout, e:
-                raise URLGrabError(12, _('Timeout on %s: %s') % (self.url, e))
-                err.url = self.url
-                raise err
-
-            except IOError, e:
-                raise URLGrabError(4, _('IOError on %s: %s') %(self.url, e))
-                err.url = self.url
-                raise err
-
-            newsize = len(new)
-            if not newsize: break # no more to read
-
-            if amt: amt = amt - newsize
-            buf.append(new)
-            bufsize = bufsize + newsize
-            self._tsize = newsize
-            self._amount_read = self._amount_read + newsize
-            #if self.opts.progress_obj:
-            #    self.opts.progress_obj.update(self._amount_read)
-
-        self._rbuf = string.join(buf, '')
-        return
-
-    def _progress_update(self, download_total, downloaded, upload_total, uploaded):
-        if self._over_max_size(cur=self._amount_read-self._reget_length):
-            return -1
-
-        try:
-            if self._prog_running:
-                downloaded += self._reget_length
-                self.opts.progress_obj.update(downloaded)
-        except KeyboardInterrupt:
-            return -1
-    
-    def _over_max_size(self, cur, max_size=None):
-
-        if not max_size:
-            max_size = self.size
-        if self.opts.size: # if we set an opts size use that, no matter what
-            max_size = self.opts.size
-        if not max_size: return False # if we have None for all of the Max then this is dumb
-        if cur > max_size + max_size*.10:
-
-            msg = _("Downloaded more than max size for %s: %s > %s") \
-                        % (self.url, cur, max_size)
-            self._error = (pycurl.E_FILESIZE_EXCEEDED, msg)
-            return True
-        return False
-        
-    def _to_utf8(self, obj, errors='replace'):
-        '''convert 'unicode' to an encoded utf-8 byte string '''
-        # stolen from yum.i18n
-        if isinstance(obj, unicode):
-            obj = obj.encode('utf-8', errors)
-        return obj
-        
-    def read(self, amt=None):
-        self._fill_buffer(amt)
-        if amt is None:
-            s, self._rbuf = self._rbuf, ''
-        else:
-            s, self._rbuf = self._rbuf[:amt], self._rbuf[amt:]
-        return s
-
-    def readline(self, limit=-1):
-        if not self._complete: self._do_grab()
-        return self.fo.readline()
-        
-        i = string.find(self._rbuf, '\n')
-        while i < 0 and not (0 < limit <= len(self._rbuf)):
-            L = len(self._rbuf)
-            self._fill_buffer(L + self._rbufsize)
-            if not len(self._rbuf) > L: break
-            i = string.find(self._rbuf, '\n', L)
-
-        if i < 0: i = len(self._rbuf)
-        else: i = i+1
-        if 0 <= limit < len(self._rbuf): i = limit
-
-        s, self._rbuf = self._rbuf[:i], self._rbuf[i:]
-        return s
-
-    def close(self):
-        if self._prog_running:
-            self.opts.progress_obj.end(self._amount_read)
-        self.fo.close()
-        
-
-_curl_cache = pycurl.Curl() # make one and reuse it over and over and over
-
-
-#####################################################################
-# DEPRECATED FUNCTIONS
-def set_throttle(new_throttle):
-    """Deprecated. Use: default_grabber.throttle = new_throttle"""
-    default_grabber.throttle = new_throttle
-
-def set_bandwidth(new_bandwidth):
-    """Deprecated. Use: default_grabber.bandwidth = new_bandwidth"""
-    default_grabber.bandwidth = new_bandwidth
-
-def set_progress_obj(new_progress_obj):
-    """Deprecated. Use: default_grabber.progress_obj = new_progress_obj"""
-    default_grabber.progress_obj = new_progress_obj
-
-def set_user_agent(new_user_agent):
-    """Deprecated. Use: default_grabber.user_agent = new_user_agent"""
-    default_grabber.user_agent = new_user_agent
-    
-def retrygrab(url, filename=None, copy_local=0, close_connection=0,
-              progress_obj=None, throttle=None, bandwidth=None,
-              numtries=3, retrycodes=[-1,2,4,5,6,7], checkfunc=None):
-    """Deprecated. Use: urlgrab() with the retry arg instead"""
-    kwargs = {'copy_local' :  copy_local, 
-              'close_connection' : close_connection,
-              'progress_obj' : progress_obj, 
-              'throttle' : throttle, 
-              'bandwidth' : bandwidth,
-              'retry' : numtries,
-              'retrycodes' : retrycodes,
-              'checkfunc' : checkfunc 
-              }
-    return urlgrab(url, filename, **kwargs)
-
-        
-#####################################################################
-#  TESTING
-def _main_test():
-    try: url, filename = sys.argv[1:3]
-    except ValueError:
-        print 'usage:', sys.argv[0], \
-              '<url> <filename> [copy_local=0|1] [close_connection=0|1]'
-        sys.exit()
-
-    kwargs = {}
-    for a in sys.argv[3:]:
-        k, v = string.split(a, '=', 1)
-        kwargs[k] = int(v)
-
-    set_throttle(1.0)
-    set_bandwidth(32 * 1024)
-    print "throttle: %s,  throttle bandwidth: %s B/s" % (default_grabber.throttle, 
-                                                        default_grabber.bandwidth)
-
-    try: from progress import text_progress_meter
-    except ImportError, e: pass
-    else: kwargs['progress_obj'] = text_progress_meter()
-
-    try: name = apply(urlgrab, (url, filename), kwargs)
-    except URLGrabError, e: print e
-    else: print 'LOCAL FILE:', name
-
-
-def _retry_test():
-    try: url, filename = sys.argv[1:3]
-    except ValueError:
-        print 'usage:', sys.argv[0], \
-              '<url> <filename> [copy_local=0|1] [close_connection=0|1]'
-        sys.exit()
-
-    kwargs = {}
-    for a in sys.argv[3:]:
-        k, v = string.split(a, '=', 1)
-        kwargs[k] = int(v)
-
-    try: from progress import text_progress_meter
-    except ImportError, e: pass
-    else: kwargs['progress_obj'] = text_progress_meter()
-
-    def cfunc(filename, hello, there='foo'):
-        print hello, there
-        import random
-        rnum = random.random()
-        if rnum < .5:
-            print 'forcing retry'
-            raise URLGrabError(-1, 'forcing retry')
-        if rnum < .75:
-            print 'forcing failure'
-            raise URLGrabError(-2, 'forcing immediate failure')
-        print 'success'
-        return
-        
-    kwargs['checkfunc'] = (cfunc, ('hello',), {'there':'there'})
-    try: name = apply(retrygrab, (url, filename), kwargs)
-    except URLGrabError, e: print e
-    else: print 'LOCAL FILE:', name
-
-def _file_object_test(filename=None):
-    import cStringIO
-    if filename is None:
-        filename = __file__
-    print 'using file "%s" for comparisons' % filename
-    fo = open(filename)
-    s_input = fo.read()
-    fo.close()
-
-    for testfunc in [_test_file_object_smallread,
-                     _test_file_object_readall,
-                     _test_file_object_readline,
-                     _test_file_object_readlines]:
-        fo_input = cStringIO.StringIO(s_input)
-        fo_output = cStringIO.StringIO()
-        wrapper = PyCurlFileObject(fo_input, None, 0)
-        print 'testing %-30s ' % testfunc.__name__,
-        testfunc(wrapper, fo_output)
-        s_output = fo_output.getvalue()
-        if s_output == s_input: print 'passed'
-        else: print 'FAILED'
-            
-def _test_file_object_smallread(wrapper, fo_output):
-    while 1:
-        s = wrapper.read(23)
-        fo_output.write(s)
-        if not s: return
-
-def _test_file_object_readall(wrapper, fo_output):
-    s = wrapper.read()
-    fo_output.write(s)
-
-def _test_file_object_readline(wrapper, fo_output):
-    while 1:
-        s = wrapper.readline()
-        fo_output.write(s)
-        if not s: return
-
-def _test_file_object_readlines(wrapper, fo_output):
-    li = wrapper.readlines()
-    fo_output.write(string.join(li, ''))
-
-if __name__ == '__main__':
-    _main_test()
-    _retry_test()
-    _file_object_test('test')
 
=== removed directory '.pc/progress_fix.diff'
=== removed directory '.pc/progress_fix.diff/urlgrabber'
=== removed file '.pc/progress_fix.diff/urlgrabber/progress.py'
--- .pc/progress_fix.diff/urlgrabber/progress.py	2010-07-08 17:40:08 +0000
+++ .pc/progress_fix.diff/urlgrabber/progress.py	1970-01-01 00:00:00 +0000
@@ -1,755 +0,0 @@
-#   This library is free software; you can redistribute it and/or
-#   modify it under the terms of the GNU Lesser General Public
-#   License as published by the Free Software Foundation; either
-#   version 2.1 of the License, or (at your option) any later version.
-#
-#   This library is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   Lesser General Public License for more details.
-#
-#   You should have received a copy of the GNU Lesser General Public
-#   License along with this library; if not, write to the 
-#      Free Software Foundation, Inc., 
-#      59 Temple Place, Suite 330, 
-#      Boston, MA  02111-1307  USA
-
-# This file is part of urlgrabber, a high-level cross-protocol url-grabber
-# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
-
-
-import sys
-import time
-import math
-import thread
-import fcntl
-import struct
-import termios
-
-# Code from http://mail.python.org/pipermail/python-list/2000-May/033365.html
-def terminal_width(fd=1):
-    """ Get the real terminal width """
-    try:
-        buf = 'abcdefgh'
-        buf = fcntl.ioctl(fd, termios.TIOCGWINSZ, buf)
-        ret = struct.unpack('hhhh', buf)[1]
-        if ret == 0:
-            return 80
-        # Add minimum too?
-        return ret
-    except: # IOError
-        return 80
-
-_term_width_val  = None
-_term_width_last = None
-def terminal_width_cached(fd=1, cache_timeout=1.000):
-    """ Get the real terminal width, but cache it for a bit. """
-    global _term_width_val
-    global _term_width_last
-
-    now = time.time()
-    if _term_width_val is None or (now - _term_width_last) > cache_timeout:
-        _term_width_val  = terminal_width(fd)
-        _term_width_last = now
-    return _term_width_val
-
-class TerminalLine:
-    """ Help create dynamic progress bars, uses terminal_width_cached(). """
-
-    def __init__(self, min_rest=0, beg_len=None, fd=1, cache_timeout=1.000):
-        if beg_len is None:
-            beg_len = min_rest
-        self._min_len = min_rest
-        self._llen    = terminal_width_cached(fd, cache_timeout)
-        if self._llen < beg_len:
-            self._llen = beg_len
-        self._fin = False
-
-    def __len__(self):
-        """ Usable length for elements. """
-        return self._llen - self._min_len
-
-    def rest_split(self, fixed, elements=2):
-        """ After a fixed length, split the rest of the line length among
-            a number of different elements (default=2). """
-        if self._llen < fixed:
-            return 0
-        return (self._llen - fixed) / elements
-
-    def add(self, element, full_len=None):
-        """ If there is room left in the line, above min_len, add element.
-            Note that as soon as one add fails all the rest will fail too. """
-
-        if full_len is None:
-            full_len = len(element)
-        if len(self) < full_len:
-            self._fin = True
-        if self._fin:
-            return ''
-
-        self._llen -= len(element)
-        return element
-
-    def rest(self):
-        """ Current rest of line, same as .rest_split(fixed=0, elements=1). """
-        return self._llen
-
-class BaseMeter:
-    def __init__(self):
-        self.update_period = 0.3 # seconds
-
-        self.filename   = None
-        self.url        = None
-        self.basename   = None
-        self.text       = None
-        self.size       = None
-        self.start_time = None
-        self.last_amount_read = 0
-        self.last_update_time = None
-        self.re = RateEstimator()
-        
-    def start(self, filename=None, url=None, basename=None,
-              size=None, now=None, text=None):
-        self.filename = filename
-        self.url      = url
-        self.basename = basename
-        self.text     = text
-
-        #size = None #########  TESTING
-        self.size = size
-        if not size is None: self.fsize = format_number(size) + 'B'
-
-        if now is None: now = time.time()
-        self.start_time = now
-        self.re.start(size, now)
-        self.last_amount_read = 0
-        self.last_update_time = now
-        self._do_start(now)
-        
-    def _do_start(self, now=None):
-        pass
-
-    def update(self, amount_read, now=None):
-        # for a real gui, you probably want to override and put a call
-        # to your mainloop iteration function here
-        if now is None: now = time.time()
-        if (now >= self.last_update_time + self.update_period) or \
-               not self.last_update_time:
-            self.re.update(amount_read, now)
-            self.last_amount_read = amount_read
-            self.last_update_time = now
-            self._do_update(amount_read, now)
-
-    def _do_update(self, amount_read, now=None):
-        pass
-
-    def end(self, amount_read, now=None):
-        if now is None: now = time.time()
-        self.re.update(amount_read, now)
-        self.last_amount_read = amount_read
-        self.last_update_time = now
-        self._do_end(amount_read, now)
-
-    def _do_end(self, amount_read, now=None):
-        pass
-        
-#  This is kind of a hack, but progress is gotten from grabber which doesn't
-# know about the total size to download. So we do this so we can get the data
-# out of band here. This will be "fixed" one way or anther soon.
-_text_meter_total_size = 0
-_text_meter_sofar_size = 0
-def text_meter_total_size(size, downloaded=0):
-    global _text_meter_total_size
-    global _text_meter_sofar_size
-    _text_meter_total_size = size
-    _text_meter_sofar_size = downloaded
-
-#
-#       update: No size (minimal: 17 chars)
-#       -----------------------------------
-# <text>                          <rate> | <current size> <elapsed time> 
-#  8-48                          1    8  3             6 1            9 5
-#
-# Order: 1. <text>+<current size> (17)
-#        2. +<elapsed time>       (10, total: 27)
-#        3. +                     ( 5, total: 32)
-#        4. +<rate>               ( 9, total: 41)
-#
-#       update: Size, Single file
-#       -------------------------
-# <text>            <pc>  <bar> <rate> | <current size> <eta time> ETA
-#  8-25            1 3-4 1 6-16 1   8  3             6 1        9 1  3 1
-#
-# Order: 1. <text>+<current size> (17)
-#        2. +<eta time>           (10, total: 27)
-#        3. +ETA                  ( 5, total: 32)
-#        4. +<pc>                 ( 4, total: 36)
-#        5. +<rate>               ( 9, total: 45)
-#        6. +<bar>                ( 7, total: 52)
-#
-#       update: Size, All files
-#       -----------------------
-# <text> <total pc> <pc>  <bar> <rate> | <current size> <eta time> ETA
-#  8-22 1      5-7 1 3-4 1 6-12 1   8  3             6 1        9 1  3 1
-#
-# Order: 1. <text>+<current size> (17)
-#        2. +<eta time>           (10, total: 27)
-#        3. +ETA                  ( 5, total: 32)
-#        4. +<total pc>           ( 5, total: 37)
-#        4. +<pc>                 ( 4, total: 41)
-#        5. +<rate>               ( 9, total: 50)
-#        6. +<bar>                ( 7, total: 57)
-#
-#       end
-#       ---
-# <text>                                 | <current size> <elapsed time> 
-#  8-56                                  3             6 1            9 5
-#
-# Order: 1. <text>                ( 8)
-#        2. +<current size>       ( 9, total: 17)
-#        3. +<elapsed time>       (10, total: 27)
-#        4. +                     ( 5, total: 32)
-#
-
-class TextMeter(BaseMeter):
-    def __init__(self, fo=sys.stderr):
-        BaseMeter.__init__(self)
-        self.fo = fo
-
-    def _do_update(self, amount_read, now=None):
-        etime = self.re.elapsed_time()
-        fetime = format_time(etime)
-        fread = format_number(amount_read)
-        #self.size = None
-        if self.text is not None:
-            text = self.text
-        else:
-            text = self.basename
-
-        ave_dl = format_number(self.re.average_rate())
-        sofar_size = None
-        if _text_meter_total_size:
-            sofar_size = _text_meter_sofar_size + amount_read
-            sofar_pc   = (sofar_size * 100) / _text_meter_total_size
-
-        # Include text + ui_rate in minimal
-        tl = TerminalLine(8, 8+1+8)
-        ui_size = tl.add(' | %5sB' % fread)
-        if self.size is None:
-            ui_time = tl.add(' %9s' % fetime)
-            ui_end  = tl.add(' ' * 5)
-            ui_rate = tl.add(' %5sB/s' % ave_dl)
-            out = '%-*.*s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
-                                        ui_rate, ui_size, ui_time, ui_end)
-        else:
-            rtime = self.re.remaining_time()
-            frtime = format_time(rtime)
-            frac = self.re.fraction_read()
-
-            ui_time = tl.add(' %9s' % frtime)
-            ui_end  = tl.add(' ETA ')
-
-            if sofar_size is None:
-                ui_sofar_pc = ''
-            else:
-                ui_sofar_pc = tl.add(' (%i%%)' % sofar_pc,
-                                     full_len=len(" (100%)"))
-
-            ui_pc   = tl.add(' %2i%%' % (frac*100))
-            ui_rate = tl.add(' %5sB/s' % ave_dl)
-            # Make text grow a bit before we start growing the bar too
-            blen = 4 + tl.rest_split(8 + 8 + 4)
-            bar  = '='*int(blen * frac)
-            if (blen * frac) - int(blen * frac) >= 0.5:
-                bar += '-'
-            ui_bar  = tl.add(' [%-*.*s]' % (blen, blen, bar))
-            out = '%-*.*s%s%s%s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
-                                              ui_sofar_pc, ui_pc, ui_bar,
-                                              ui_rate, ui_size, ui_time, ui_end)
-
-        self.fo.write(out)
-        self.fo.flush()
-
-    def _do_end(self, amount_read, now=None):
-        global _text_meter_total_size
-        global _text_meter_sofar_size
-
-        total_time = format_time(self.re.elapsed_time())
-        total_size = format_number(amount_read)
-        if self.text is not None:
-            text = self.text
-        else:
-            text = self.basename
-
-        tl = TerminalLine(8)
-        ui_size = tl.add(' | %5sB' % total_size)
-        ui_time = tl.add(' %9s' % total_time)
-        not_done = self.size is not None and amount_read != self.size
-        if not_done:
-            ui_end  = tl.add(' ... ')
-        else:
-            ui_end  = tl.add(' ' * 5)
-
-        out = '\r%-*.*s%s%s%s\n' % (tl.rest(), tl.rest(), text,
-                                    ui_size, ui_time, ui_end)
-        self.fo.write(out)
-        self.fo.flush()
-
-        # Don't add size to the sofar size until we have all of it.
-        # If we don't have a size, then just pretend/hope we got all of it.
-        if not_done:
-            return
-
-        if _text_meter_total_size:
-            _text_meter_sofar_size += amount_read
-        if _text_meter_total_size <= _text_meter_sofar_size:
-            _text_meter_total_size = 0
-            _text_meter_sofar_size = 0
-
-text_progress_meter = TextMeter
-
-class MultiFileHelper(BaseMeter):
-    def __init__(self, master):
-        BaseMeter.__init__(self)
-        self.master = master
-
-    def _do_start(self, now):
-        self.master.start_meter(self, now)
-
-    def _do_update(self, amount_read, now):
-        # elapsed time since last update
-        self.master.update_meter(self, now)
-
-    def _do_end(self, amount_read, now):
-        self.ftotal_time = format_time(now - self.start_time)
-        self.ftotal_size = format_number(self.last_amount_read)
-        self.master.end_meter(self, now)
-
-    def failure(self, message, now=None):
-        self.master.failure_meter(self, message, now)
-
-    def message(self, message):
-        self.master.message_meter(self, message)
-
-class MultiFileMeter:
-    helperclass = MultiFileHelper
-    def __init__(self):
-        self.meters = []
-        self.in_progress_meters = []
-        self._lock = thread.allocate_lock()
-        self.update_period = 0.3 # seconds
-        
-        self.numfiles         = None
-        self.finished_files   = 0
-        self.failed_files     = 0
-        self.open_files       = 0
-        self.total_size       = None
-        self.failed_size      = 0
-        self.start_time       = None
-        self.finished_file_size = 0
-        self.last_update_time = None
-        self.re = RateEstimator()
-
-    def start(self, numfiles=None, total_size=None, now=None):
-        if now is None: now = time.time()
-        self.numfiles         = numfiles
-        self.finished_files   = 0
-        self.failed_files     = 0
-        self.open_files       = 0
-        self.total_size       = total_size
-        self.failed_size      = 0
-        self.start_time       = now
-        self.finished_file_size = 0
-        self.last_update_time = now
-        self.re.start(total_size, now)
-        self._do_start(now)
-
-    def _do_start(self, now):
-        pass
-
-    def end(self, now=None):
-        if now is None: now = time.time()
-        self._do_end(now)
-        
-    def _do_end(self, now):
-        pass
-
-    def lock(self): self._lock.acquire()
-    def unlock(self): self._lock.release()
-
-    ###########################################################
-    # child meter creation and destruction
-    def newMeter(self):
-        newmeter = self.helperclass(self)
-        self.meters.append(newmeter)
-        return newmeter
-    
-    def removeMeter(self, meter):
-        self.meters.remove(meter)
-        
-    ###########################################################
-    # child functions - these should only be called by helpers
-    def start_meter(self, meter, now):
-        if not meter in self.meters:
-            raise ValueError('attempt to use orphaned meter')
-        self._lock.acquire()
-        try:
-            if not meter in self.in_progress_meters:
-                self.in_progress_meters.append(meter)
-                self.open_files += 1
-        finally:
-            self._lock.release()
-        self._do_start_meter(meter, now)
-        
-    def _do_start_meter(self, meter, now):
-        pass
-        
-    def update_meter(self, meter, now):
-        if not meter in self.meters:
-            raise ValueError('attempt to use orphaned meter')
-        if (now >= self.last_update_time + self.update_period) or \
-               not self.last_update_time:
-            self.re.update(self._amount_read(), now)
-            self.last_update_time = now
-            self._do_update_meter(meter, now)
-
-    def _do_update_meter(self, meter, now):
-        pass
-
-    def end_meter(self, meter, now):
-        if not meter in self.meters:
-            raise ValueError('attempt to use orphaned meter')
-        self._lock.acquire()
-        try:
-            try: self.in_progress_meters.remove(meter)
-            except ValueError: pass
-            self.open_files     -= 1
-            self.finished_files += 1
-            self.finished_file_size += meter.last_amount_read
-        finally:
-            self._lock.release()
-        self._do_end_meter(meter, now)
-
-    def _do_end_meter(self, meter, now):
-        pass
-
-    def failure_meter(self, meter, message, now):
-        if not meter in self.meters:
-            raise ValueError('attempt to use orphaned meter')
-        self._lock.acquire()
-        try:
-            try: self.in_progress_meters.remove(meter)
-            except ValueError: pass
-            self.open_files     -= 1
-            self.failed_files   += 1
-            if meter.size and self.failed_size is not None:
-                self.failed_size += meter.size
-            else:
-                self.failed_size = None
-        finally:
-            self._lock.release()
-        self._do_failure_meter(meter, message, now)
-
-    def _do_failure_meter(self, meter, message, now):
-        pass
-
-    def message_meter(self, meter, message):
-        pass
-
-    ########################################################
-    # internal functions
-    def _amount_read(self):
-        tot = self.finished_file_size
-        for m in self.in_progress_meters:
-            tot += m.last_amount_read
-        return tot
-
-
-class TextMultiFileMeter(MultiFileMeter):
-    def __init__(self, fo=sys.stderr):
-        self.fo = fo
-        MultiFileMeter.__init__(self)
-
-    # files: ###/### ###%  data: ######/###### ###%  time: ##:##:##/##:##:##
-    def _do_update_meter(self, meter, now):
-        self._lock.acquire()
-        try:
-            format = "files: %3i/%-3i %3i%%   data: %6.6s/%-6.6s %3i%%   " \
-                     "time: %8.8s/%8.8s"
-            df = self.finished_files
-            tf = self.numfiles or 1
-            pf = 100 * float(df)/tf + 0.49
-            dd = self.re.last_amount_read
-            td = self.total_size
-            pd = 100 * (self.re.fraction_read() or 0) + 0.49
-            dt = self.re.elapsed_time()
-            rt = self.re.remaining_time()
-            if rt is None: tt = None
-            else: tt = dt + rt
-
-            fdd = format_number(dd) + 'B'
-            ftd = format_number(td) + 'B'
-            fdt = format_time(dt, 1)
-            ftt = format_time(tt, 1)
-            
-            out = '%-79.79s' % (format % (df, tf, pf, fdd, ftd, pd, fdt, ftt))
-            self.fo.write('\r' + out)
-            self.fo.flush()
-        finally:
-            self._lock.release()
-
-    def _do_end_meter(self, meter, now):
-        self._lock.acquire()
-        try:
-            format = "%-30.30s %6.6s    %8.8s    %9.9s"
-            fn = meter.basename
-            size = meter.last_amount_read
-            fsize = format_number(size) + 'B'
-            et = meter.re.elapsed_time()
-            fet = format_time(et, 1)
-            frate = format_number(size / et) + 'B/s'
-            
-            out = '%-79.79s' % (format % (fn, fsize, fet, frate))
-            self.fo.write('\r' + out + '\n')
-        finally:
-            self._lock.release()
-        self._do_update_meter(meter, now)
-
-    def _do_failure_meter(self, meter, message, now):
-        self._lock.acquire()
-        try:
-            format = "%-30.30s %6.6s %s"
-            fn = meter.basename
-            if type(message) in (type(''), type(u'')):
-                message = message.splitlines()
-            if not message: message = ['']
-            out = '%-79s' % (format % (fn, 'FAILED', message[0] or ''))
-            self.fo.write('\r' + out + '\n')
-            for m in message[1:]: self.fo.write('  ' + m + '\n')
-            self._lock.release()
-        finally:
-            self._do_update_meter(meter, now)
-
-    def message_meter(self, meter, message):
-        self._lock.acquire()
-        try:
-            pass
-        finally:
-            self._lock.release()
-
-    def _do_end(self, now):
-        self._do_update_meter(None, now)
-        self._lock.acquire()
-        try:
-            self.fo.write('\n')
-            self.fo.flush()
-        finally:
-            self._lock.release()
-        
-######################################################################
-# support classes and functions
-
-class RateEstimator:
-    def __init__(self, timescale=5.0):
-        self.timescale = timescale
-
-    def start(self, total=None, now=None):
-        if now is None: now = time.time()
-        self.total = total
-        self.start_time = now
-        self.last_update_time = now
-        self.last_amount_read = 0
-        self.ave_rate = None
-        
-    def update(self, amount_read, now=None):
-        if now is None: now = time.time()
-        if amount_read == 0:
-            # if we just started this file, all bets are off
-            self.last_update_time = now
-            self.last_amount_read = 0
-            self.ave_rate = None
-            return
-
-        #print 'times', now, self.last_update_time
-        time_diff = now         - self.last_update_time
-        read_diff = amount_read - self.last_amount_read
-        # First update, on reget is the file size
-        if self.last_amount_read:
-            self.last_update_time = now
-            self.ave_rate = self._temporal_rolling_ave(\
-                time_diff, read_diff, self.ave_rate, self.timescale)
-        self.last_amount_read = amount_read
-        #print 'results', time_diff, read_diff, self.ave_rate
-        
-    #####################################################################
-    # result methods
-    def average_rate(self):
-        "get the average transfer rate (in bytes/second)"
-        return self.ave_rate
-
-    def elapsed_time(self):
-        "the time between the start of the transfer and the most recent update"
-        return self.last_update_time - self.start_time
-
-    def remaining_time(self):
-        "estimated time remaining"
-        if not self.ave_rate or not self.total: return None
-        return (self.total - self.last_amount_read) / self.ave_rate
-
-    def fraction_read(self):
-        """the fraction of the data that has been read
-        (can be None for unknown transfer size)"""
-        if self.total is None: return None
-        elif self.total == 0: return 1.0
-        else: return float(self.last_amount_read)/self.total
-
-    #########################################################################
-    # support methods
-    def _temporal_rolling_ave(self, time_diff, read_diff, last_ave, timescale):
-        """a temporal rolling average performs smooth averaging even when
-        updates come at irregular intervals.  This is performed by scaling
-        the "epsilon" according to the time since the last update.
-        Specifically, epsilon = time_diff / timescale
-
-        As a general rule, the average will take on a completely new value
-        after 'timescale' seconds."""
-        epsilon = time_diff / timescale
-        if epsilon > 1: epsilon = 1.0
-        return self._rolling_ave(time_diff, read_diff, last_ave, epsilon)
-    
-    def _rolling_ave(self, time_diff, read_diff, last_ave, epsilon):
-        """perform a "rolling average" iteration
-        a rolling average "folds" new data into an existing average with
-        some weight, epsilon.  epsilon must be between 0.0 and 1.0 (inclusive)
-        a value of 0.0 means only the old value (initial value) counts,
-        and a value of 1.0 means only the newest value is considered."""
-        
-        try:
-            recent_rate = read_diff / time_diff
-        except ZeroDivisionError:
-            recent_rate = None
-        if last_ave is None: return recent_rate
-        elif recent_rate is None: return last_ave
-
-        # at this point, both last_ave and recent_rate are numbers
-        return epsilon * recent_rate  +  (1 - epsilon) * last_ave
-
-    def _round_remaining_time(self, rt, start_time=15.0):
-        """round the remaining time, depending on its size
-        If rt is between n*start_time and (n+1)*start_time round downward
-        to the nearest multiple of n (for any counting number n).
-        If rt < start_time, round down to the nearest 1.
-        For example (for start_time = 15.0):
-         2.7  -> 2.0
-         25.2 -> 25.0
-         26.4 -> 26.0
-         35.3 -> 34.0
-         63.6 -> 60.0
-        """
-
-        if rt < 0: return 0.0
-        shift = int(math.log(rt/start_time)/math.log(2))
-        rt = int(rt)
-        if shift <= 0: return rt
-        return float(int(rt) >> shift << shift)
-        
-
-def format_time(seconds, use_hours=0):
-    if seconds is None or seconds < 0:
-        if use_hours: return '--:--:--'
-        else:         return '--:--'
-    else:
-        seconds = int(seconds)
-        minutes = seconds / 60
-        seconds = seconds % 60
-        if use_hours:
-            hours = minutes / 60
-            minutes = minutes % 60
-            return '%02i:%02i:%02i' % (hours, minutes, seconds)
-        else:
-            return '%02i:%02i' % (minutes, seconds)
-            
-def format_number(number, SI=0, space=' '):
-    """Turn numbers into human-readable metric-like numbers"""
-    symbols = ['',  # (none)
-               'k', # kilo
-               'M', # mega
-               'G', # giga
-               'T', # tera
-               'P', # peta
-               'E', # exa
-               'Z', # zetta
-               'Y'] # yotta
-    
-    if SI: step = 1000.0
-    else: step = 1024.0
-
-    thresh = 999
-    depth = 0
-    max_depth = len(symbols) - 1
-    
-    # we want numbers between 0 and thresh, but don't exceed the length
-    # of our list.  In that event, the formatting will be screwed up,
-    # but it'll still show the right number.
-    while number > thresh and depth < max_depth:
-        depth  = depth + 1
-        number = number / step
-
-    if type(number) == type(1) or type(number) == type(1L):
-        # it's an int or a long, which means it didn't get divided,
-        # which means it's already short enough
-        format = '%i%s%s'
-    elif number < 9.95:
-        # must use 9.95 for proper sizing.  For example, 9.99 will be
-        # rounded to 10.0 with the .1f format string (which is too long)
-        format = '%.1f%s%s'
-    else:
-        format = '%.0f%s%s'
-        
-    return(format % (float(number or 0), space, symbols[depth]))
-
-def _tst(fn, cur, tot, beg, size, *args):
-    tm = TextMeter()
-    text = "(%d/%d): %s" % (cur, tot, fn)
-    tm.start(fn, "http://www.example.com/path/to/fn/" + fn, fn, size, text=text)
-    num = beg
-    off = 0
-    for (inc, delay) in args:
-        off += 1
-        while num < ((size * off) / len(args)):
-            num += inc
-            tm.update(num)
-            time.sleep(delay)
-    tm.end(size)
-
-if __name__ == "__main__":
-    # (1/2): subversion-1.4.4-7.x86_64.rpm               2.4 MB /  85 kB/s    00:28     
-    # (2/2): mercurial-0.9.5-6.fc8.x86_64.rpm            924 kB / 106 kB/s    00:08     
-    if len(sys.argv) >= 2 and sys.argv[1] == 'total':
-        text_meter_total_size(1000 + 10000 + 10000 + 1000000 + 1000000 +
-                              1000000 + 10000 + 10000 + 10000 + 1000000)
-    _tst("sm-1.0.0-1.fc8.i386.rpm", 1, 10, 0, 1000,
-         (10, 0.2), (10, 0.1), (100, 0.25))
-    _tst("s-1.0.1-1.fc8.i386.rpm", 2, 10, 0, 10000,
-         (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25))
-    _tst("m-1.0.1-2.fc8.i386.rpm", 3, 10, 5000, 10000,
-         (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25))
-    _tst("large-file-name-Foo-11.8.7-4.5.6.1.fc8.x86_64.rpm", 4, 10, 0, 1000000,
-         (1000, 0.2), (1000, 0.1), (10000, 0.1))
-    _tst("large-file-name-Foo2-11.8.7-4.5.6.2.fc8.x86_64.rpm", 5, 10,
-         500001, 1000000, (1000, 0.2), (1000, 0.1), (10000, 0.1))
-    _tst("large-file-name-Foo3-11.8.7-4.5.6.3.fc8.x86_64.rpm", 6, 10,
-         750002, 1000000, (1000, 0.2), (1000, 0.1), (10000, 0.1))
-    _tst("large-file-name-Foo4-10.8.7-4.5.6.1.fc8.x86_64.rpm", 7, 10, 0, 10000,
-         (100, 0.1))
-    _tst("large-file-name-Foo5-10.8.7-4.5.6.2.fc8.x86_64.rpm", 8, 10,
-         5001, 10000, (100, 0.1))
-    _tst("large-file-name-Foo6-10.8.7-4.5.6.3.fc8.x86_64.rpm", 9, 10,
-         7502, 10000, (1, 0.1))
-    _tst("large-file-name-Foox-9.8.7-4.5.6.1.fc8.x86_64.rpm",  10, 10,
-         0, 1000000, (10, 0.5),
-         (100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1),
-         (100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1),
-         (100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1),
-         (100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1),
-         (100000, 0.1), (1, 0.1))
 
=== removed directory '.pc/progress_object_callback_fix.diff'
=== removed directory '.pc/progress_object_callback_fix.diff/urlgrabber'
=== removed file '.pc/progress_object_callback_fix.diff/urlgrabber/grabber.py'
--- .pc/progress_object_callback_fix.diff/urlgrabber/grabber.py	2011-08-09 17:45:08 +0000
+++ .pc/progress_object_callback_fix.diff/urlgrabber/grabber.py	1970-01-01 00:00:00 +0000
@@ -1,1802 +0,0 @@
-#   This library is free software; you can redistribute it and/or
-#   modify it under the terms of the GNU Lesser General Public
-#   License as published by the Free Software Foundation; either
-#   version 2.1 of the License, or (at your option) any later version.
-#
-#   This library is distributed in the hope that it will be useful,
-#   but WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   Lesser General Public License for more details.
-#
-#   You should have received a copy of the GNU Lesser General Public
-#   License along with this library; if not, write to the 
-#      Free Software Foundation, Inc., 
-#      59 Temple Place, Suite 330, 
-#      Boston, MA  02111-1307  USA
-
-# This file is part of urlgrabber, a high-level cross-protocol url-grabber
-# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
-# Copyright 2009 Red Hat inc, pycurl code written by Seth Vidal
-
-"""A high-level cross-protocol url-grabber.
-
-GENERAL ARGUMENTS (kwargs)
-
-  Where possible, the module-level default is indicated, and legal
-  values are provided.
-
-  copy_local = 0   [0|1]
-
-    ignored except for file:// urls, in which case it specifies
-    whether urlgrab should still make a copy of the file, or simply
-    point to the existing copy. The module level default for this
-    option is 0.
-
-  close_connection = 0   [0|1]
-
-    tells URLGrabber to close the connection after a file has been
-    transfered. This is ignored unless the download happens with the
-    http keepalive handler (keepalive=1).  Otherwise, the connection
-    is left open for further use. The module level default for this
-    option is 0 (keepalive connections will not be closed).
-
-  keepalive = 1   [0|1]
-
-    specifies whether keepalive should be used for HTTP/1.1 servers
-    that support it. The module level default for this option is 1
-    (keepalive is enabled).
-
-  progress_obj = None
-
-    a class instance that supports the following methods:
-      po.start(filename, url, basename, length, text)
-      # length will be None if unknown
-      po.update(read) # read == bytes read so far
-      po.end()
-
-  text = None
-  
-    specifies alternative text to be passed to the progress meter
-    object.  If not given, the default progress meter will use the
-    basename of the file.
-
-  throttle = 1.0
-
-    a number - if it's an int, it's the bytes/second throttle limit.
-    If it's a float, it is first multiplied by bandwidth.  If throttle
-    == 0, throttling is disabled.  If None, the module-level default
-    (which can be set on default_grabber.throttle) is used. See
-    BANDWIDTH THROTTLING for more information.
-
-  timeout = 300
-
-    a positive integer expressing the number of seconds to wait before
-    timing out attempts to connect to a server. If the value is None
-    or 0, connection attempts will not time out. The timeout is passed
-    to the underlying pycurl object as its CONNECTTIMEOUT option, see
-    the curl documentation on CURLOPT_CONNECTTIMEOUT for more information.
-    http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUT
-
-  bandwidth = 0
-
-    the nominal max bandwidth in bytes/second.  If throttle is a float
-    and bandwidth == 0, throttling is disabled.  If None, the
-    module-level default (which can be set on
-    default_grabber.bandwidth) is used. See BANDWIDTH THROTTLING for
-    more information.
-
-  range = None
-
-    a tuple of the form (first_byte, last_byte) describing a byte
-    range to retrieve. Either or both of the values may set to
-    None. If first_byte is None, byte offset 0 is assumed. If
-    last_byte is None, the last byte available is assumed. Note that
-    the range specification is python-like in that (0,10) will yeild
-    the first 10 bytes of the file.
-
-    If set to None, no range will be used.
-    
-  reget = None   [None|'simple'|'check_timestamp']
-
-    whether to attempt to reget a partially-downloaded file.  Reget
-    only applies to .urlgrab and (obviously) only if there is a
-    partially downloaded file.  Reget has two modes:
-
-      'simple' -- the local file will always be trusted.  If there
-        are 100 bytes in the local file, then the download will always
-        begin 100 bytes into the requested file.
-
-      'check_timestamp' -- the timestamp of the server file will be
-        compared to the timestamp of the local file.  ONLY if the
-        local file is newer than or the same age as the server file
-        will reget be used.  If the server file is newer, or the
-        timestamp is not returned, the entire file will be fetched.
-
-    NOTE: urlgrabber can do very little to verify that the partial
-    file on disk is identical to the beginning of the remote file.
-    You may want to either employ a custom "checkfunc" or simply avoid
-    using reget in situations where corruption is a concern.
-
-  user_agent = 'urlgrabber/VERSION'
-
-    a string, usually of the form 'AGENT/VERSION' that is provided to
-    HTTP servers in the User-agent header. The module level default
-    for this option is "urlgrabber/VERSION".
-
-  http_headers = None
-
-    a tuple of 2-tuples, each containing a header and value.  These
-    will be used for http and https requests only.  For example, you
-    can do
-      http_headers = (('Pragma', 'no-cache'),)
-
-  ftp_headers = None
-
-    this is just like http_headers, but will be used for ftp requests.
-
-  proxies = None
-
-    a dictionary that maps protocol schemes to proxy hosts. For
-    example, to use a proxy server on host "foo" port 3128 for http
-    and https URLs:
-      proxies={ 'http' : 'http://foo:3128', 'https' : 'http://foo:3128' }
-    note that proxy authentication information may be provided using
-    normal URL constructs:
-      proxies={ 'http' : 'http://user:host@foo:3128' }
-    Lastly, if proxies is None, the default environment settings will
-    be used.
-
-  prefix = None
-
-    a url prefix that will be prepended to all requested urls.  For
-    example:
-      g = URLGrabber(prefix='http://foo.com/mirror/')
-      g.urlgrab('some/file.txt')
-      ## this will fetch 'http://foo.com/mirror/some/file.txt'
-    This option exists primarily to allow identical behavior to
-    MirrorGroup (and derived) instances.  Note: a '/' will be inserted
-    if necessary, so you cannot specify a prefix that ends with a
-    partial file or directory name.
-
-  opener = None
-    No-op when using the curl backend (default)
-
-  cache_openers = True
-    No-op when using the curl backend (default)
-
-  data = None
-
-    Only relevant for the HTTP family (and ignored for other
-    protocols), this allows HTTP POSTs.  When the data kwarg is
-    present (and not None), an HTTP request will automatically become
-    a POST rather than GET.  This is done by direct passthrough to
-    urllib2.  If you use this, you may also want to set the
-    'Content-length' and 'Content-type' headers with the http_headers
-    option.  Note that python 2.2 handles the case of these
-    badly and if you do not use the proper case (shown here), your
-    values will be overridden with the defaults.
-    
-  urlparser = URLParser()
-
-    The URLParser class handles pre-processing of URLs, including
-    auth-handling for user/pass encoded in http urls, file handing
-    (that is, filenames not sent as a URL), and URL quoting.  If you
-    want to override any of this behavior, you can pass in a
-    replacement instance.  See also the 'quote' option.
-
-  quote = None
-
-    Whether or not to quote the path portion of a url.
-      quote = 1    ->  quote the URLs (they're not quoted yet)
-      quote = 0    ->  do not quote them (they're already quoted)
-      quote = None ->  guess what to do
-
-    This option only affects proper urls like 'file:///etc/passwd'; it
-    does not affect 'raw' filenames like '/etc/passwd'.  The latter
-    will always be quoted as they are converted to URLs.  Also, only
-    the path part of a url is quoted.  If you need more fine-grained
-    control, you should probably subclass URLParser and pass it in via
-    the 'urlparser' option.
-
-  ssl_ca_cert = None
-
-    this option can be used if M2Crypto is available and will be
-    ignored otherwise.  If provided, it will be used to create an SSL
-    context.  If both ssl_ca_cert and ssl_context are provided, then
-    ssl_context will be ignored and a new context will be created from
-    ssl_ca_cert.
-
-  ssl_context = None
-
-    No-op when using the curl backend (default)
-   
-
-  self.ssl_verify_peer = True 
-
-    Check the server's certificate to make sure it is valid with what our CA validates
-  
-  self.ssl_verify_host = True
-
-    Check the server's hostname to make sure it matches the certificate DN
-
-  self.ssl_key = None
-
-    Path to the key the client should use to connect/authenticate with
-
-  self.ssl_key_type = 'PEM' 
-
-    PEM or DER - format of key
-     
-  self.ssl_cert = None
-
-    Path to the ssl certificate the client should use to to authenticate with
-
-  self.ssl_cert_type = 'PEM' 
-
-    PEM or DER - format of certificate
-    
-  self.ssl_key_pass = None 
-
-    password to access the ssl_key
-    
-  self.size = None
-
-    size (in bytes) or Maximum size of the thing being downloaded. 
-    This is mostly to keep us from exploding with an endless datastream
-  
-  self.max_header_size = 2097152 
-
-    Maximum size (in bytes) of the headers.
-    
-
-RETRY RELATED ARGUMENTS
-
-  retry = None
-
-    the number of times to retry the grab before bailing.  If this is
-    zero, it will retry forever. This was intentional... really, it
-    was :). If this value is not supplied or is supplied but is None
-    retrying does not occur.
-
-  retrycodes = [-1,2,4,5,6,7]
-
-    a sequence of errorcodes (values of e.errno) for which it should
-    retry. See the doc on URLGrabError for more details on this.  You
-    might consider modifying a copy of the default codes rather than
-    building yours from scratch so that if the list is extended in the
-    future (or one code is split into two) you can still enjoy the
-    benefits of the default list.  You can do that with something like
-    this:
-
-      retrycodes = urlgrabber.grabber.URLGrabberOptions().retrycodes
-      if 12 not in retrycodes:
-          retrycodes.append(12)
-      
-  checkfunc = None
-
-    a function to do additional checks. This defaults to None, which
-    means no additional checking.  The function should simply return
-    on a successful check.  It should raise URLGrabError on an
-    unsuccessful check.  Raising of any other exception will be
-    considered immediate failure and no retries will occur.
-
-    If it raises URLGrabError, the error code will determine the retry
-    behavior.  Negative error numbers are reserved for use by these
-    passed in functions, so you can use many negative numbers for
-    different types of failure.  By default, -1 results in a retry,
-    but this can be customized with retrycodes.
-
-    If you simply pass in a function, it will be given exactly one
-    argument: a CallbackObject instance with the .url attribute
-    defined and either .filename (for urlgrab) or .data (for urlread).
-    For urlgrab, .filename is the name of the local file.  For
-    urlread, .data is the actual string data.  If you need other
-    arguments passed to the callback (program state of some sort), you
-    can do so like this:
-
-      checkfunc=(function, ('arg1', 2), {'kwarg': 3})
-
-    if the downloaded file has filename /tmp/stuff, then this will
-    result in this call (for urlgrab):
-
-      function(obj, 'arg1', 2, kwarg=3)
-      # obj.filename = '/tmp/stuff'
-      # obj.url = 'http://foo.com/stuff'
-      
-    NOTE: both the "args" tuple and "kwargs" dict must be present if
-    you use this syntax, but either (or both) can be empty.
-
-  failure_callback = None
-
-    The callback that gets called during retries when an attempt to
-    fetch a file fails.  The syntax for specifying the callback is
-    identical to checkfunc, except for the attributes defined in the
-    CallbackObject instance.  The attributes for failure_callback are:
-
-      exception = the raised exception
-      url       = the url we're trying to fetch
-      tries     = the number of tries so far (including this one)
-      retry     = the value of the retry option
-
-    The callback is present primarily to inform the calling program of
-    the failure, but if it raises an exception (including the one it's
-    passed) that exception will NOT be caught and will therefore cause
-    future retries to be aborted.
-
-    The callback is called for EVERY failure, including the last one.
-    On the last try, the callback can raise an alternate exception,
-    but it cannot (without severe trickiness) prevent the exception
-    from being raised.
-
-  interrupt_callback = None
-
-    This callback is called if KeyboardInterrupt is received at any
-    point in the transfer.  Basically, this callback can have three
-    impacts on the fetch process based on the way it exits:
-
-      1) raise no exception: the current fetch will be aborted, but
-         any further retries will still take place
-
-      2) raise a URLGrabError: if you're using a MirrorGroup, then
-         this will prompt a failover to the next mirror according to
-         the behavior of the MirrorGroup subclass.  It is recommended
-         that you raise URLGrabError with code 15, 'user abort'.  If
-         you are NOT using a MirrorGroup subclass, then this is the
-         same as (3).
-
-      3) raise some other exception (such as KeyboardInterrupt), which
-         will not be caught at either the grabber or mirror levels.
-         That is, it will be raised up all the way to the caller.
-
-    This callback is very similar to failure_callback.  They are
-    passed the same arguments, so you could use the same function for
-    both.
-      
-BANDWIDTH THROTTLING
-
-  urlgrabber supports throttling via two values: throttle and
-  bandwidth Between the two, you can either specify and absolute
-  throttle threshold or specify a theshold as a fraction of maximum
-  available bandwidth.
-
-  throttle is a number - if it's an int, it's the bytes/second
-  throttle limit.  If it's a float, it is first multiplied by
-  bandwidth.  If throttle == 0, throttling is disabled.  If None, the
-  module-level default (which can be set with set_throttle) is used.
-
-  bandwidth is the nominal max bandwidth in bytes/second.  If throttle
-  is a float and bandwidth == 0, throttling is disabled.  If None, the
-  module-level default (which can be set with set_bandwidth) is used.
-
-  THROTTLING EXAMPLES:
-
-  Lets say you have a 100 Mbps connection.  This is (about) 10^8 bits
-  per second, or 12,500,000 Bytes per second.  You have a number of
-  throttling options:
-
-  *) set_bandwidth(12500000); set_throttle(0.5) # throttle is a float
-
-     This will limit urlgrab to use half of your available bandwidth.
-
-  *) set_throttle(6250000) # throttle is an int
-
-     This will also limit urlgrab to use half of your available
-     bandwidth, regardless of what bandwidth is set to.
-
-  *) set_throttle(6250000); set_throttle(1.0) # float
-
-     Use half your bandwidth
-
-  *) set_throttle(6250000); set_throttle(2.0) # float
-
-    Use up to 12,500,000 Bytes per second (your nominal max bandwidth)
-
-  *) set_throttle(6250000); set_throttle(0) # throttle = 0
-
-     Disable throttling - this is more efficient than a very large
-     throttle setting.
-
-  *) set_throttle(0); set_throttle(1.0) # throttle is float, bandwidth = 0
-
-     Disable throttling - this is the default when the module is loaded.
-
-  SUGGESTED AUTHOR IMPLEMENTATION (THROTTLING)
-
-  While this is flexible, it's not extremely obvious to the user.  I
-  suggest you implement a float throttle as a percent to make the
-  distinction between absolute and relative throttling very explicit.
-
-  Also, you may want to convert the units to something more convenient
-  than bytes/second, such as kbps or kB/s, etc.
-
-"""
-
-
-
-import os
-import sys
-import urlparse
-import time
-import string
-import urllib
-import urllib2
-import mimetools
-import thread
-import types
-import stat
-import pycurl
-from ftplib import parse150
-from StringIO import StringIO
-from httplib import HTTPException
-import socket
-from byterange import range_tuple_normalize, range_tuple_to_header, RangeError
-
-########################################################################
-#                     MODULE INITIALIZATION
-########################################################################
-try:
-    exec('from ' + (__name__.split('.'))[0] + ' import __version__')
-except:
-    __version__ = '???'
-
-try:
-    # this part isn't going to do much - need to talk to gettext
-    from i18n import _
-except ImportError, msg:
-    def _(st): return st
-    
-########################################################################
-# functions for debugging output.  These functions are here because they
-# are also part of the module initialization.
-DEBUG = None
-def set_logger(DBOBJ):
-    """Set the DEBUG object.  This is called by _init_default_logger when
-    the environment variable URLGRABBER_DEBUG is set, but can also be
-    called by a calling program.  Basically, if the calling program uses
-    the logging module and would like to incorporate urlgrabber logging,
-    then it can do so this way.  It's probably not necessary as most
-    internal logging is only for debugging purposes.
-
-    The passed-in object should be a logging.Logger instance.  It will
-    be pushed into the keepalive and byterange modules if they're
-    being used.  The mirror module pulls this object in on import, so
-    you will need to manually push into it.  In fact, you may find it
-    tidier to simply push your logging object (or objects) into each
-    of these modules independently.
-    """
-
-    global DEBUG
-    DEBUG = DBOBJ
-
-def _init_default_logger(logspec=None):
-    '''Examines the environment variable URLGRABBER_DEBUG and creates
-    a logging object (logging.logger) based on the contents.  It takes
-    the form
-
-      URLGRABBER_DEBUG=level,filename
-      
-    where "level" can be either an integer or a log level from the
-    logging module (DEBUG, INFO, etc).  If the integer is zero or
-    less, logging will be disabled.  Filename is the filename where
-    logs will be sent.  If it is "-", then stdout will be used.  If
-    the filename is empty or missing, stderr will be used.  If the
-    variable cannot be processed or the logging module cannot be
-    imported (python < 2.3) then logging will be disabled.  Here are
-    some examples:
-
-      URLGRABBER_DEBUG=1,debug.txt   # log everything to debug.txt
-      URLGRABBER_DEBUG=WARNING,-     # log warning and higher to stdout
-      URLGRABBER_DEBUG=INFO          # log info and higher to stderr
-      
-    This funtion is called during module initialization.  It is not
-    intended to be called from outside.  The only reason it is a
-    function at all is to keep the module-level namespace tidy and to
-    collect the code into a nice block.'''
-
-    try:
-        if logspec is None:
-            logspec = os.environ['URLGRABBER_DEBUG']
-        dbinfo = logspec.split(',')
-        import logging
-        level = logging._levelNames.get(dbinfo[0], None)
-        if level is None: level = int(dbinfo[0])
-        if level < 1: raise ValueError()
-
-        formatter = logging.Formatter('%(asctime)s %(message)s')
-        if len(dbinfo) > 1: filename = dbinfo[1]
-        else: filename = ''
-        if filename == '': handler = logging.StreamHandler(sys.stderr)
-        elif filename == '-': handler = logging.StreamHandler(sys.stdout)
-        else:  handler = logging.FileHandler(filename)
-        handler.setFormatter(formatter)
-        DBOBJ = logging.getLogger('urlgrabber')
-        DBOBJ.addHandler(handler)
-        DBOBJ.setLevel(level)
-    except (KeyError, ImportError, ValueError):
-        DBOBJ = None
-    set_logger(DBOBJ)
-
-def _log_package_state():
-    if not DEBUG: return
-    DEBUG.info('urlgrabber version  = %s' % __version__)
-    DEBUG.info('trans function "_"  = %s' % _)
-        
-_init_default_logger()
-_log_package_state()
-
-
-# normally this would be from i18n or something like it ...
-def _(st):
-    return st
-
-########################################################################
-#                 END MODULE INITIALIZATION
-########################################################################
-
-
-
-class URLGrabError(IOError):
-    """
-    URLGrabError error codes:
-
-      URLGrabber error codes (0 -- 255)
-        0    - everything looks good (you should never see this)
-        1    - malformed url
-        2    - local file doesn't exist
-        3    - request for non-file local file (dir, etc)
-        4    - IOError on fetch
-        5    - OSError on fetch
-        6    - no content length header when we expected one
-        7    - HTTPException
-        8    - Exceeded read limit (for urlread)
-        9    - Requested byte range not satisfiable.
-        10   - Byte range requested, but range support unavailable
-        11   - Illegal reget mode
-        12   - Socket timeout
-        13   - malformed proxy url
-        14   - HTTPError (includes .code and .exception attributes)
-        15   - user abort
-        16   - error writing to local file
-        
-      MirrorGroup error codes (256 -- 511)
-        256  - No more mirrors left to try
-
-      Custom (non-builtin) classes derived from MirrorGroup (512 -- 767)
-        [ this range reserved for application-specific error codes ]
-
-      Retry codes (< 0)
-        -1   - retry the download, unknown reason
-
-    Note: to test which group a code is in, you can simply do integer
-    division by 256: e.errno / 256
-
-    Negative codes are reserved for use by functions passed in to
-    retrygrab with checkfunc.  The value -1 is built in as a generic
-    retry code and is already included in the retrycodes list.
-    Therefore, you can create a custom check function that simply
-    returns -1 and the fetch will be re-tried.  For more customized
-    retries, you can use other negative number and include them in
-    retry-codes.  This is nice for outputting useful messages about
-    what failed.
-
-    You can use these error codes like so:
-      try: urlgrab(url)
-      except URLGrabError, e:
-         if e.errno == 3: ...
-           # or
-         print e.strerror
-           # or simply
-         print e  #### print '[Errno %i] %s' % (e.errno, e.strerror)
-    """
-    def __init__(self, *args):
-        IOError.__init__(self, *args)
-        self.url = "No url specified"
-
-class CallbackObject:
-    """Container for returned callback data.
-
-    This is currently a dummy class into which urlgrabber can stuff
-    information for passing to callbacks.  This way, the prototype for
-    all callbacks is the same, regardless of the data that will be
-    passed back.  Any function that accepts a callback function as an
-    argument SHOULD document what it will define in this object.
-
-    It is possible that this class will have some greater
-    functionality in the future.
-    """
-    def __init__(self, **kwargs):
-        self.__dict__.update(kwargs)
-
-def urlgrab(url, filename=None, **kwargs):
-    """grab the file at <url> and make a local copy at <filename>
-    If filename is none, the basename of the url is used.
-    urlgrab returns the filename of the local file, which may be different
-    from the passed-in filename if the copy_local kwarg == 0.
-    
-    See module documentation for a description of possible kwargs.
-    """
-    return default_grabber.urlgrab(url, filename, **kwargs)
-
-def urlopen(url, **kwargs):
-    """open the url and return a file object
-    If a progress object or throttle specifications exist, then
-    a special file object will be returned that supports them.
-    The file object can be treated like any other file object.
-    
-    See module documentation for a description of possible kwargs.
-    """
-    return default_grabber.urlopen(url, **kwargs)
-
-def urlread(url, limit=None, **kwargs):
-    """read the url into a string, up to 'limit' bytes
-    If the limit is exceeded, an exception will be thrown.  Note that urlread
-    is NOT intended to be used as a way of saying "I want the first N bytes"
-    but rather 'read the whole file into memory, but don't use too much'
-    
-    See module documentation for a description of possible kwargs.
-    """
-    return default_grabber.urlread(url, limit, **kwargs)
-
-
-class URLParser:
-    """Process the URLs before passing them to urllib2.
-
-    This class does several things:
-
-      * add any prefix
-      * translate a "raw" file to a proper file: url
-      * handle any http or https auth that's encoded within the url
-      * quote the url
-
-    Only the "parse" method is called directly, and it calls sub-methods.
-
-    An instance of this class is held in the options object, which
-    means that it's easy to change the behavior by sub-classing and
-    passing the replacement in.  It need only have a method like:
-
-        url, parts = urlparser.parse(url, opts)
-    """
-
-    def parse(self, url, opts):
-        """parse the url and return the (modified) url and its parts
-
-        Note: a raw file WILL be quoted when it's converted to a URL.
-        However, other urls (ones which come with a proper scheme) may
-        or may not be quoted according to opts.quote
-
-          opts.quote = 1     --> quote it
-          opts.quote = 0     --> do not quote it
-          opts.quote = None  --> guess
-        """
-        quote = opts.quote
-        
-        if opts.prefix:
-            url = self.add_prefix(url, opts.prefix)
-            
-        parts = urlparse.urlparse(url)
-        (scheme, host, path, parm, query, frag) = parts
-
-        if not scheme or (len(scheme) == 1 and scheme in string.letters):
-            # if a scheme isn't specified, we guess that it's "file:"
-            if url[0] not in '/\\': url = os.path.abspath(url)
-            url = 'file:' + urllib.pathname2url(url)
-            parts = urlparse.urlparse(url)
-            quote = 0 # pathname2url quotes, so we won't do it again
-            
-        if scheme in ['http', 'https']:
-            parts = self.process_http(parts, url)
-            
-        if quote is None:
-            quote = self.guess_should_quote(parts)
-        if quote:
-            parts = self.quote(parts)
-        
-        url = urlparse.urlunparse(parts)
-        return url, parts
-
-    def add_prefix(self, url, prefix):
-        if prefix[-1] == '/' or url[0] == '/':
-            url = prefix + url
-        else:
-            url = prefix + '/' + url
-        return url
-
-    def process_http(self, parts, url):
-        (scheme, host, path, parm, query, frag) = parts
-        # TODO: auth-parsing here, maybe? pycurl doesn't really need it
-        return (scheme, host, path, parm, query, frag)
-
-    def quote(self, parts):
-        """quote the URL
-
-        This method quotes ONLY the path part.  If you need to quote
-        other parts, you should override this and pass in your derived
-        class.  The other alternative is to quote other parts before
-        passing into urlgrabber.
-        """
-        (scheme, host, path, parm, query, frag) = parts
-        path = urllib.quote(path)
-        return (scheme, host, path, parm, query, frag)
-
-    hexvals = '0123456789ABCDEF'
-    def guess_should_quote(self, parts):
-        """
-        Guess whether we should quote a path.  This amounts to
-        guessing whether it's already quoted.
-
-        find ' '   ->  1
-        find '%'   ->  1
-        find '%XX' ->  0
-        else       ->  1
-        """
-        (scheme, host, path, parm, query, frag) = parts
-        if ' ' in path:
-            return 1
-        ind = string.find(path, '%')
-        if ind > -1:
-            while ind > -1:
-                if len(path) < ind+3:
-                    return 1
-                code = path[ind+1:ind+3].upper()
-                if     code[0] not in self.hexvals or \
-                       code[1] not in self.hexvals:
-                    return 1
-                ind = string.find(path, '%', ind+1)
-            return 0
-        return 1
-    
-class URLGrabberOptions:
-    """Class to ease kwargs handling."""
-
-    def __init__(self, delegate=None, **kwargs):
-        """Initialize URLGrabberOptions object.
-        Set default values for all options and then update options specified
-        in kwargs.
-        """
-        self.delegate = delegate
-        if delegate is None:
-            self._set_defaults()
-        self._set_attributes(**kwargs)
-    
-    def __getattr__(self, name):
-        if self.delegate and hasattr(self.delegate, name):
-            return getattr(self.delegate, name)
-        raise AttributeError, name
-    
-    def raw_throttle(self):
-        """Calculate raw throttle value from throttle and bandwidth 
-        values.
-        """
-        if self.throttle <= 0:  
-            return 0
-        elif type(self.throttle) == type(0): 
-            return float(self.throttle)
-        else: # throttle is a float
-            return self.bandwidth * self.throttle
-        
-    def derive(self, **kwargs):
-        """Create a derived URLGrabberOptions instance.
-        This method creates a new instance and overrides the
-        options specified in kwargs.
-        """
-        return URLGrabberOptions(delegate=self, **kwargs)
-        
-    def _set_attributes(self, **kwargs):
-        """Update object attributes with those provided in kwargs."""
-        self.__dict__.update(kwargs)
-        if kwargs.has_key('range'):
-            # normalize the supplied range value
-            self.range = range_tuple_normalize(self.range)
-        if not self.reget in [None, 'simple', 'check_timestamp']:
-            raise URLGrabError(11, _('Illegal reget mode: %s') \
-                               % (self.reget, ))
-
-    def _set_defaults(self):
-        """Set all options to their default values. 
-        When adding new options, make sure a default is
-        provided here.
-        """
-        self.progress_obj = None
-        self.throttle = 1.0
-        self.bandwidth = 0
-        self.retry = None
-        self.retrycodes = [-1,2,4,5,6,7]
-        self.checkfunc = None
-        self.copy_local = 0
-        self.close_connection = 0
-        self.range = None
-        self.user_agent = 'urlgrabber/%s' % __version__
-        self.keepalive = 1
-        self.proxies = None
-        self.reget = None
-        self.failure_callback = None
-        self.interrupt_callback = None
-        self.prefix = None
-        self.opener = None
-        self.cache_openers = True
-        self.timeout = 300
-        self.text = None
-        self.http_headers = None
-        self.ftp_headers = None
-        self.data = None
-        self.urlparser = URLParser()
-        self.quote = None
-        self.ssl_ca_cert = None # sets SSL_CAINFO - path to certdb
-        self.ssl_context = None # no-op in pycurl
-        self.ssl_verify_peer = True # check peer's cert for authenticityb
-        self.ssl_verify_host = True # make sure who they are and who the cert is for matches
-        self.ssl_key = None # client key
-        self.ssl_key_type = 'PEM' #(or DER)
-        self.ssl_cert = None # client cert
-        self.ssl_cert_type = 'PEM' # (or DER)
-        self.ssl_key_pass = None # password to access the key
-        self.size = None # if we know how big the thing we're getting is going
-                         # to be. this is ultimately a MAXIMUM size for the file
-        self.max_header_size = 2097152 #2mb seems reasonable for maximum header size
-        
-    def __repr__(self):
-        return self.format()
-        
-    def format(self, indent='  '):
-        keys = self.__dict__.keys()
-        if self.delegate is not None:
-            keys.remove('delegate')
-        keys.sort()
-        s = '{\n'
-        for k in keys:
-            s = s + indent + '%-15s: %s,\n' % \
-                (repr(k), repr(self.__dict__[k]))
-        if self.delegate:
-            df = self.delegate.format(indent + '  ')
-            s = s + indent + '%-15s: %s\n' % ("'delegate'", df)
-        s = s + indent + '}'
-        return s
-
-class URLGrabber:
-    """Provides easy opening of URLs with a variety of options.
-    
-    All options are specified as kwargs. Options may be specified when
-    the class is created and may be overridden on a per request basis.
-    
-    New objects inherit default values from default_grabber.
-    """
-    
-    def __init__(self, **kwargs):
-        self.opts = URLGrabberOptions(**kwargs)
-    
-    def _retry(self, opts, func, *args):
-        tries = 0
-        while 1:
-            # there are only two ways out of this loop.  The second has
-            # several "sub-ways"
-            #   1) via the return in the "try" block
-            #   2) by some exception being raised
-            #      a) an excepton is raised that we don't "except"
-            #      b) a callback raises ANY exception
-            #      c) we're not retry-ing or have run out of retries
-            #      d) the URLGrabError code is not in retrycodes
-            # beware of infinite loops :)
-            tries = tries + 1
-            exception = None
-            retrycode = None
-            callback  = None
-            if DEBUG: DEBUG.info('attempt %i/%s: %s',
-                                 tries, opts.retry, args[0])
-            try:
-                r = apply(func, (opts,) + args, {})
-                if DEBUG: DEBUG.info('success')
-                return r
-            except URLGrabError, e:
-                exception = e
-                callback = opts.failure_callback
-                retrycode = e.errno
-            except KeyboardInterrupt, e:
-                exception = e
-                callback = opts.interrupt_callback
-
-            if DEBUG: DEBUG.info('exception: %s', exception)
-            if callback:
-                if DEBUG: DEBUG.info('calling callback: %s', callback)
-                cb_func, cb_args, cb_kwargs = self._make_callback(callback)
-                obj = CallbackObject(exception=exception, url=args[0],
-                                     tries=tries, retry=opts.retry)
-                cb_func(obj, *cb_args, **cb_kwargs)
-
-            if (opts.retry is None) or (tries == opts.retry):
-                if DEBUG: DEBUG.info('retries exceeded, re-raising')
-                raise
-
-            if (retrycode is not None) and (retrycode not in opts.retrycodes):
-                if DEBUG: DEBUG.info('retrycode (%i) not in list %s, re-raising',
-                                     retrycode, opts.retrycodes)
-                raise
-    
-    def urlopen(self, url, **kwargs):
-        """open the url and return a file object
-        If a progress object or throttle value specified when this 
-        object was created, then  a special file object will be 
-        returned that supports them. The file object can be treated 
-        like any other file object.
-        """
-        opts = self.opts.derive(**kwargs)
-        if DEBUG: DEBUG.debug('combined options: %s' % repr(opts))
-        (url,parts) = opts.urlparser.parse(url, opts) 
-        def retryfunc(opts, url):
-            return PyCurlFileObject(url, filename=None, opts=opts)
-        return self._retry(opts, retryfunc, url)
-    
-    def urlgrab(self, url, filename=None, **kwargs):
-        """grab the file at <url> and make a local copy at <filename>
-        If filename is none, the basename of the url is used.
-        urlgrab returns the filename of the local file, which may be 
-        different from the passed-in filename if copy_local == 0.
-        """
-        opts = self.opts.derive(**kwargs)
-        if DEBUG: DEBUG.debug('combined options: %s' % repr(opts))
-        (url,parts) = opts.urlparser.parse(url, opts) 
-        (scheme, host, path, parm, query, frag) = parts
-        if filename is None:
-            filename = os.path.basename( urllib.unquote(path) )
-        if scheme == 'file' and not opts.copy_local:
-            # just return the name of the local file - don't make a 
-            # copy currently
-            path = urllib.url2pathname(path)
-            if host:
-                path = os.path.normpath('//' + host + path)
-            if not os.path.exists(path):
-                err = URLGrabError(2, 
-                      _('Local file does not exist: %s') % (path, ))
-                err.url = url
-                raise err
-            elif not os.path.isfile(path):
-                err = URLGrabError(3, 
-                                 _('Not a normal file: %s') % (path, ))
-                err.url = url
-                raise err
-
-            elif not opts.range:
-                if not opts.checkfunc is None:
-                    cb_func, cb_args, cb_kwargs = \
-                       self._make_callback(opts.checkfunc)
-                    obj = CallbackObject()
-                    obj.filename = path
-                    obj.url = url
-                    apply(cb_func, (obj, )+cb_args, cb_kwargs)        
-                return path
-        
-        def retryfunc(opts, url, filename):
-            fo = PyCurlFileObject(url, filename, opts)
-            try:
-                fo._do_grab()
-                if not opts.checkfunc is None:
-                    cb_func, cb_args, cb_kwargs = \
-                             self._make_callback(opts.checkfunc)
-                    obj = CallbackObject()
-                    obj.filename = filename
-                    obj.url = url
-                    apply(cb_func, (obj, )+cb_args, cb_kwargs)
-            finally:
-                fo.close()
-            return filename
-        
-        return self._retry(opts, retryfunc, url, filename)
-    
-    def urlread(self, url, limit=None, **kwargs):
-        """read the url into a string, up to 'limit' bytes
-        If the limit is exceeded, an exception will be thrown.  Note
-        that urlread is NOT intended to be used as a way of saying 
-        "I want the first N bytes" but rather 'read the whole file 
-        into memory, but don't use too much'
-        """
-        opts = self.opts.derive(**kwargs)
-        if DEBUG: DEBUG.debug('combined options: %s' % repr(opts))
-        (url,parts) = opts.urlparser.parse(url, opts) 
-        if limit is not None:
-            limit = limit + 1
-            
-        def retryfunc(opts, url, limit):
-            fo = PyCurlFileObject(url, filename=None, opts=opts)
-            s = ''
-            try:
-                # this is an unfortunate thing.  Some file-like objects
-                # have a default "limit" of None, while the built-in (real)
-                # file objects have -1.  They each break the other, so for
-                # now, we just force the default if necessary.
-                if limit is None: s = fo.read()
-                else: s = fo.read(limit)
-
-                if not opts.checkfunc is None:
-                    cb_func, cb_args, cb_kwargs = \
-                             self._make_callback(opts.checkfunc)
-                    obj = CallbackObject()
-                    obj.data = s
-                    obj.url = url
-                    apply(cb_func, (obj, )+cb_args, cb_kwargs)
-            finally:
-                fo.close()
-            return s
-            
-        s = self._retry(opts, retryfunc, url, limit)
-        if limit and len(s) > limit:
-            err = URLGrabError(8, 
-                               _('Exceeded limit (%i): %s') % (limit, url))
-            err.url = url
-            raise err
-
-        return s
-        
-    def _make_callback(self, callback_obj):
-        if callable(callback_obj):
-            return callback_obj, (), {}
-        else:
-            return callback_obj
-
-# create the default URLGrabber used by urlXXX functions.
-# NOTE: actual defaults are set in URLGrabberOptions
-default_grabber = URLGrabber()
-
-
-class PyCurlFileObject():
-    def __init__(self, url, filename, opts):
-        self.fo = None
-        self._hdr_dump = ''
-        self._parsed_hdr = None
-        self.url = url
-        self.scheme = urlparse.urlsplit(self.url)[0]
-        self.filename = filename
-        self.append = False
-        self.reget_time = None
-        self.opts = opts
-        if self.opts.reget == 'check_timestamp':
-            raise NotImplementedError, "check_timestamp regets are not implemented in this ver of urlgrabber. Please report this."
-        self._complete = False
-        self._rbuf = ''
-        self._rbufsize = 1024*8
-        self._ttime = time.time()
-        self._tsize = 0
-        self._amount_read = 0
-        self._reget_length = 0
-        self._prog_running = False
-        self._error = (None, None)
-        self.size = 0
-        self._hdr_ended = False
-        self._do_open()
-        
-
-    def geturl(self):
-        """ Provide the geturl() method, used to be got from
-            urllib.addinfourl, via. urllib.URLopener.* """
-        return self.url
-        
-    def __getattr__(self, name):
-        """This effectively allows us to wrap at the instance level.
-        Any attribute not found in _this_ object will be searched for
-        in self.fo.  This includes methods."""
-
-        if hasattr(self.fo, name):
-            return getattr(self.fo, name)
-        raise AttributeError, name
-
-    def _retrieve(self, buf):
-        try:
-            if not self._prog_running:
-                if self.opts.progress_obj:
-                    size  = self.size + self._reget_length
-                    self.opts.progress_obj.start(self._prog_reportname, 
-                                                 urllib.unquote(self.url), 
-                                                 self._prog_basename, 
-                                                 size=size,
-                                                 text=self.opts.text)
-                    self._prog_running = True
-                    self.opts.progress_obj.update(self._amount_read)
-
-            self._amount_read += len(buf)
-            self.fo.write(buf)
-            return len(buf)
-        except KeyboardInterrupt:
-            return -1
-            
-    def _hdr_retrieve(self, buf):
-        if self._hdr_ended:
-            self._hdr_dump = ''
-            self.size = 0
-            self._hdr_ended = False
-
-        if self._over_max_size(cur=len(self._hdr_dump), 
-                               max_size=self.opts.max_header_size):
-            return -1
-        try:
-            self._hdr_dump += buf
-            # we have to get the size before we do the progress obj start
-            # but we can't do that w/o making it do 2 connects, which sucks
-            # so we cheat and stuff it in here in the hdr_retrieve
-            if self.scheme in ['http','https'] and buf.lower().find('content-length') != -1:
-                length = buf.split(':')[1]
-                self.size = int(length)
-            elif self.scheme in ['ftp']:
-                s = None
-                if buf.startswith('213 '):
-                    s = buf[3:].strip()
-                elif buf.startswith('150 '):
-                    s = parse150(buf)
-                if s:
-                    self.size = int(s)
-                    
-            if buf.lower().find('location') != -1:
-                location = ':'.join(buf.split(':')[1:])
-                location = location.strip()
-                self.scheme = urlparse.urlsplit(location)[0]
-                self.url = location
-                
-            if len(self._hdr_dump) != 0 and buf == '\r\n':
-                self._hdr_ended = True
-                if DEBUG: DEBUG.info('header ended:')
-                
-            return len(buf)
-        except KeyboardInterrupt:
-            return pycurl.READFUNC_ABORT
-
-    def _return_hdr_obj(self):
-        if self._parsed_hdr:
-            return self._parsed_hdr
-        statusend = self._hdr_dump.find('\n')
-        statusend += 1 # ridiculous as it may seem.
-        hdrfp = StringIO()
-        hdrfp.write(self._hdr_dump[statusend:])
-        hdrfp.seek(0)
-        self._parsed_hdr =  mimetools.Message(hdrfp)
-        return self._parsed_hdr
-    
-    hdr = property(_return_hdr_obj)
-    http_code = property(fget=
-                 lambda self: self.curl_obj.getinfo(pycurl.RESPONSE_CODE))
-
-    def _set_opts(self, opts={}):
-        # XXX
-        if not opts:
-            opts = self.opts
-
-
-        # defaults we're always going to set
-        self.curl_obj.setopt(pycurl.NOPROGRESS, False)
-        self.curl_obj.setopt(pycurl.NOSIGNAL, True)
-        self.curl_obj.setopt(pycurl.WRITEFUNCTION, self._retrieve)
-        self.curl_obj.setopt(pycurl.HEADERFUNCTION, self._hdr_retrieve)
-        self.curl_obj.setopt(pycurl.PROGRESSFUNCTION, self._progress_update)
-        self.curl_obj.setopt(pycurl.FAILONERROR, True)
-        self.curl_obj.setopt(pycurl.OPT_FILETIME, True)
-        self.curl_obj.setopt(pycurl.FOLLOWLOCATION, True)
-        
-        if DEBUG:
-            self.curl_obj.setopt(pycurl.VERBOSE, True)
-        if opts.user_agent:
-            self.curl_obj.setopt(pycurl.USERAGENT, opts.user_agent)
-        
-        # maybe to be options later
-        self.curl_obj.setopt(pycurl.FOLLOWLOCATION, True)
-        self.curl_obj.setopt(pycurl.MAXREDIRS, 5)
-        
-        # timeouts
-        timeout = 300
-        if hasattr(opts, 'timeout'):
-            timeout = int(opts.timeout or 0)
-        self.curl_obj.setopt(pycurl.CONNECTTIMEOUT, timeout)
-        self.curl_obj.setopt(pycurl.LOW_SPEED_LIMIT, 1)
-        self.curl_obj.setopt(pycurl.LOW_SPEED_TIME, timeout)
-
-        # ssl options
-        if self.scheme == 'https':
-            if opts.ssl_ca_cert: # this may do ZERO with nss  according to curl docs
-                self.curl_obj.setopt(pycurl.CAPATH, opts.ssl_ca_cert)
-                self.curl_obj.setopt(pycurl.CAINFO, opts.ssl_ca_cert)
-            self.curl_obj.setopt(pycurl.SSL_VERIFYPEER, opts.ssl_verify_peer)
-            self.curl_obj.setopt(pycurl.SSL_VERIFYHOST, opts.ssl_verify_host)
-            if opts.ssl_key:
-                self.curl_obj.setopt(pycurl.SSLKEY, opts.ssl_key)
-            if opts.ssl_key_type:
-                self.curl_obj.setopt(pycurl.SSLKEYTYPE, opts.ssl_key_type)
-            if opts.ssl_cert:
-                self.curl_obj.setopt(pycurl.SSLCERT, opts.ssl_cert)
-            if opts.ssl_cert_type:                
-                self.curl_obj.setopt(pycurl.SSLCERTTYPE, opts.ssl_cert_type)
-            if opts.ssl_key_pass:
-                self.curl_obj.setopt(pycurl.SSLKEYPASSWD, opts.ssl_key_pass)
-
-        #headers:
-        if opts.http_headers and self.scheme in ('http', 'https'):
-            headers = []
-            for (tag, content) in opts.http_headers:
-                headers.append('%s:%s' % (tag, content))
-            self.curl_obj.setopt(pycurl.HTTPHEADER, headers)
-
-        # ranges:
-        if opts.range or opts.reget:
-            range_str = self._build_range()
-            if range_str:
-                self.curl_obj.setopt(pycurl.RANGE, range_str)
-            
-        # throttle/bandwidth
-        if hasattr(opts, 'raw_throttle') and opts.raw_throttle():
-            self.curl_obj.setopt(pycurl.MAX_RECV_SPEED_LARGE, int(opts.raw_throttle()))
-            
-        # proxy settings
-        if opts.proxies:
-            for (scheme, proxy) in opts.proxies.items():
-                if self.scheme in ('ftp'): # only set the ftp proxy for ftp items
-                    if scheme not in ('ftp'):
-                        continue
-                    else:
-                        if proxy == '_none_': proxy = ""
-                        self.curl_obj.setopt(pycurl.PROXY, proxy)
-                elif self.scheme in ('http', 'https'):
-                    if scheme not in ('http', 'https'):
-                        continue
-                    else:
-                        if proxy == '_none_': proxy = ""
-                        self.curl_obj.setopt(pycurl.PROXY, proxy)
-            
-        # FIXME username/password/auth settings
-
-        #posts - simple - expects the fields as they are
-        if opts.data:
-            self.curl_obj.setopt(pycurl.POST, True)
-            self.curl_obj.setopt(pycurl.POSTFIELDS, self._to_utf8(opts.data))
-            
-        # our url
-        self.curl_obj.setopt(pycurl.URL, self.url)
-        
-    
-    def _do_perform(self):
-        if self._complete:
-            return
-        
-        try:
-            self.curl_obj.perform()
-        except pycurl.error, e:
-            # XXX - break some of these out a bit more clearly
-            # to other URLGrabErrors from 
-            # http://curl.haxx.se/libcurl/c/libcurl-errors.html
-            # this covers e.args[0] == 22 pretty well - which will be common
-            
-            code = self.http_code
-            errcode = e.args[0]
-            if self._error[0]:
-                errcode = self._error[0]
-                
-            if errcode == 23 and code >= 200 and code < 299:
-                err = URLGrabError(15, _('User (or something) called abort %s: %s') % (self.url, e))
-                err.url = self.url
-                
-                # this is probably wrong but ultimately this is what happens
-                # we have a legit http code and a pycurl 'writer failed' code
-                # which almost always means something aborted it from outside
-                # since we cannot know what it is -I'm banking on it being
-                # a ctrl-c. XXXX - if there's a way of going back two raises to 
-                # figure out what aborted the pycurl process FIXME
-                raise KeyboardInterrupt
-            
-            elif errcode == 28:
-                err = URLGrabError(12, _('Timeout on %s: %s') % (self.url, e))
-                err.url = self.url
-                raise err
-            elif errcode == 35:
-                msg = _("problem making ssl connection")
-                err = URLGrabError(14, msg)
-                err.url = self.url
-                raise err
-            elif errcode == 37:
-                msg = _("Could not open/read %s") % (self.url)
-                err = URLGrabError(14, msg)
-                err.url = self.url
-                raise err
-                
-            elif errcode == 42:
-                err = URLGrabError(15, _('User (or something) called abort %s: %s') % (self.url, e))
-                err.url = self.url
-                # this is probably wrong but ultimately this is what happens
-                # we have a legit http code and a pycurl 'writer failed' code
-                # which almost always means something aborted it from outside
-                # since we cannot know what it is -I'm banking on it being
-                # a ctrl-c. XXXX - if there's a way of going back two raises to 
-                # figure out what aborted the pycurl process FIXME
-                raise KeyboardInterrupt
-                
-            elif errcode == 58:
-                msg = _("problem with the local client certificate")
-                err = URLGrabError(14, msg)
-                err.url = self.url
-                raise err
-
-            elif errcode == 60:
-                msg = _("Peer cert cannot be verified or peer cert invalid")
-                err = URLGrabError(14, msg)
-                err.url = self.url
-                raise err
-            
-            elif errcode == 63:
-                if self._error[1]:
-                    msg = self._error[1]
-                else:
-                    msg = _("Max download size exceeded on %s") % (self.url)
-                err = URLGrabError(14, msg)
-                err.url = self.url
-                raise err
-                    
-            elif str(e.args[1]) == '' and self.http_code != 0: # fake it until you make it
-                if self.scheme in ['http', 'https']:
-                    msg = 'HTTP Error %s : %s ' % (self.http_code, self.url)
-                elif self.scheme in ['ftp']:
-                    msg = 'FTP Error %s : %s ' % (self.http_code, self.url)
-                else:
-                    msg = "Unknown Error: URL=%s , scheme=%s" % (self.url, self.scheme)
-            else:
-                msg = 'PYCURL ERROR %s - "%s"' % (errcode, str(e.args[1]))
-                code = errcode
-            err = URLGrabError(14, msg)
-            err.code = code
-            err.exception = e
-            raise err
-        else:
-            if self._error[1]:
-                msg = self._error[1]
-                err = URLGRabError(14, msg)
-                err.url = self.url
-                raise err
-
-    def _do_open(self):
-        self.curl_obj = _curl_cache
-        self.curl_obj.reset() # reset all old settings away, just in case
-        # setup any ranges
-        self._set_opts()
-        self._do_grab()
-        return self.fo
-
-    def _add_headers(self):
-        pass
-        
-    def _build_range(self):
-        reget_length = 0
-        rt = None
-        if self.opts.reget and type(self.filename) in types.StringTypes:
-            # we have reget turned on and we're dumping to a file
-            try:
-                s = os.stat(self.filename)
-            except OSError:
-                pass
-            else:
-                self.reget_time = s[stat.ST_MTIME]
-                reget_length = s[stat.ST_SIZE]
-
-                # Set initial length when regetting
-                self._amount_read = reget_length    
-                self._reget_length = reget_length # set where we started from, too
-
-                rt = reget_length, ''
-                self.append = 1
-                
-        if self.opts.range:
-            rt = self.opts.range
-            if rt[0]: rt = (rt[0] + reget_length, rt[1])
-
-        if rt:
-            header = range_tuple_to_header(rt)
-            if header:
-                return header.split('=')[1]
-
-
-
-    def _make_request(self, req, opener):
-        #XXXX
-        # This doesn't do anything really, but we could use this
-        # instead of do_open() to catch a lot of crap errors as 
-        # mstenner did before here
-        return (self.fo, self.hdr)
-        
-        try:
-            if self.opts.timeout:
-                old_to = socket.getdefaulttimeout()
-                socket.setdefaulttimeout(self.opts.timeout)
-                try:
-                    fo = opener.open(req)
-                finally:
-                    socket.setdefaulttimeout(old_to)
-            else:
-                fo = opener.open(req)
-            hdr = fo.info()
-        except ValueError, e:
-            err = URLGrabError(1, _('Bad URL: %s : %s') % (self.url, e, ))
-            err.url = self.url
-            raise err
-
-        except RangeError, e:
-            err = URLGrabError(9, _('%s on %s') % (e, self.url))
-            err.url = self.url
-            raise err
-        except urllib2.HTTPError, e:
-            new_e = URLGrabError(14, _('%s on %s') % (e, self.url))
-            new_e.code = e.code
-            new_e.exception = e
-            new_e.url = self.url
-            raise new_e
-        except IOError, e:
-            if hasattr(e, 'reason') and isinstance(e.reason, socket.timeout):
-                err = URLGrabError(12, _('Timeout on %s: %s') % (self.url, e))
-                err.url = self.url
-                raise err
-            else:
-                err = URLGrabError(4, _('IOError on %s: %s') % (self.url, e))
-                err.url = self.url
-                raise err
-
-        except OSError, e:
-            err = URLGrabError(5, _('%s on %s') % (e, self.url))
-            err.url = self.url
-            raise err
-
-        except HTTPException, e:
-            err = URLGrabError(7, _('HTTP Exception (%s) on %s: %s') % \
-                            (e.__class__.__name__, self.url, e))
-            err.url = self.url
-            raise err
-
-        else:
-            return (fo, hdr)
-        
-    def _do_grab(self):
-        """dump the file to a filename or StringIO buffer"""
-
-        if self._complete:
-            return
-        _was_filename = False
-        if type(self.filename) in types.StringTypes and self.filename:
-            _was_filename = True
-            self._prog_reportname = str(self.filename)
-            self._prog_basename = os.path.basename(self.filename)
-            
-            if self.append: mode = 'ab'
-            else: mode = 'wb'
-
-            if DEBUG: DEBUG.info('opening local file "%s" with mode %s' % \
-                                 (self.filename, mode))
-            try:
-                self.fo = open(self.filename, mode)
-            except IOError, e:
-                err = URLGrabError(16, _(\
-                  'error opening local file from %s, IOError: %s') % (self.url, e))
-                err.url = self.url
-                raise err
-
-        else:
-            self._prog_reportname = 'MEMORY'
-            self._prog_basename = 'MEMORY'
-
-            
-            self.fo = StringIO()
-            # if this is to be a tempfile instead....
-            # it just makes crap in the tempdir
-            #fh, self._temp_name = mkstemp()
-            #self.fo = open(self._temp_name, 'wb')
-
-            
-        self._do_perform()
-        
-
-
-        if _was_filename:
-            # close it up
-            self.fo.flush()
-            self.fo.close()
-            # set the time
-            mod_time = self.curl_obj.getinfo(pycurl.INFO_FILETIME)
-            if mod_time != -1:
-                try:
-                    os.utime(self.filename, (mod_time, mod_time))
-                except OSError, e:
-                    err = URLGrabError(16, _(\
-                      'error setting timestamp on file %s from %s, OSError: %s') 
-                              % (self.filenameself.url, e))
-                    err.url = self.url
-                    raise err
-            # re open it
-            try:
-                self.fo = open(self.filename, 'r')
-            except IOError, e:
-                err = URLGrabError(16, _(\
-                  'error opening file from %s, IOError: %s') % (self.url, e))
-                err.url = self.url
-                raise err
-                
-        else:
-            #self.fo = open(self._temp_name, 'r')
-            self.fo.seek(0)
-
-        self._complete = True
-    
-    def _fill_buffer(self, amt=None):
-        """fill the buffer to contain at least 'amt' bytes by reading
-        from the underlying file object.  If amt is None, then it will
-        read until it gets nothing more.  It updates the progress meter
-        and throttles after every self._rbufsize bytes."""
-        # the _rbuf test is only in this first 'if' for speed.  It's not
-        # logically necessary
-        if self._rbuf and not amt is None:
-            L = len(self._rbuf)
-            if amt > L:
-                amt = amt - L
-            else:
-                return
-
-        # if we've made it here, then we don't have enough in the buffer
-        # and we need to read more.
-        
-        if not self._complete: self._do_grab() #XXX cheater - change on ranges
-        
-        buf = [self._rbuf]
-        bufsize = len(self._rbuf)
-        while amt is None or amt:
-            # first, delay if necessary for throttling reasons
-            if self.opts.raw_throttle():
-                diff = self._tsize/self.opts.raw_throttle() - \
-                       (time.time() - self._ttime)
-                if diff > 0: time.sleep(diff)
-                self._ttime = time.time()
-                
-            # now read some data, up to self._rbufsize
-            if amt is None: readamount = self._rbufsize
-            else:           readamount = min(amt, self._rbufsize)
-            try:
-                new = self.fo.read(readamount)
-            except socket.error, e:
-                err = URLGrabError(4, _('Socket Error on %s: %s') % (self.url, e))
-                err.url = self.url
-                raise err
-
-            except socket.timeout, e:
-                raise URLGrabError(12, _('Timeout on %s: %s') % (self.url, e))
-                err.url = self.url
-                raise err
-
-            except IOError, e:
-                raise URLGrabError(4, _('IOError on %s: %s') %(self.url, e))
-                err.url = self.url
-                raise err
-
-            newsize = len(new)
-            if not newsize: break # no more to read
-
-            if amt: amt = amt - newsize
-            buf.append(new)
-            bufsize = bufsize + newsize
-            self._tsize = newsize
-            self._amount_read = self._amount_read + newsize
-            #if self.opts.progress_obj:
-            #    self.opts.progress_obj.update(self._amount_read)
-
-        self._rbuf = string.join(buf, '')
-        return
-
-    def _progress_update(self, download_total, downloaded, upload_total, uploaded):
-        if self._over_max_size(cur=self._amount_read-self._reget_length):
-            return -1
-
-        try:
-            if self._prog_running:
-                downloaded += self._reget_length
-                self.opts.progress_obj.update(downloaded)
-        except KeyboardInterrupt:
-            return -1
-    
-    def _over_max_size(self, cur, max_size=None):
-
-        if not max_size:
-            if not self.opts.size:
-                max_size = self.size
-            else:
-                max_size = self.opts.size
-
-        if not max_size: return False # if we have None for all of the Max then this is dumb
-
-        if cur > int(float(max_size) * 1.10):
-
-            msg = _("Downloaded more than max size for %s: %s > %s") \
-                        % (self.url, cur, max_size)
-            self._error = (pycurl.E_FILESIZE_EXCEEDED, msg)
-            return True
-        return False
-        
-    def _to_utf8(self, obj, errors='replace'):
-        '''convert 'unicode' to an encoded utf-8 byte string '''
-        # stolen from yum.i18n
-        if isinstance(obj, unicode):
-            obj = obj.encode('utf-8', errors)
-        return obj
-        
-    def read(self, amt=None):
-        self._fill_buffer(amt)
-        if amt is None:
-            s, self._rbuf = self._rbuf, ''
-        else:
-            s, self._rbuf = self._rbuf[:amt], self._rbuf[amt:]
-        return s
-
-    def readline(self, limit=-1):
-        if not self._complete: self._do_grab()
-        return self.fo.readline()
-        
-        i = string.find(self._rbuf, '\n')
-        while i < 0 and not (0 < limit <= len(self._rbuf)):
-            L = len(self._rbuf)
-            self._fill_buffer(L + self._rbufsize)
-            if not len(self._rbuf) > L: break
-            i = string.find(self._rbuf, '\n', L)
-
-        if i < 0: i = len(self._rbuf)
-        else: i = i+1
-        if 0 <= limit < len(self._rbuf): i = limit
-
-        s, self._rbuf = self._rbuf[:i], self._rbuf[i:]
-        return s
-
-    def close(self):
-        if self._prog_running:
-            self.opts.progress_obj.end(self._amount_read)
-        self.fo.close()
-        
-    def geturl(self):
-        """ Provide the geturl() method, used to be got from
-            urllib.addinfourl, via. urllib.URLopener.* """
-        return self.url
-        
-_curl_cache = pycurl.Curl() # make one and reuse it over and over and over
-
-def reset_curl_obj():
-    """To make sure curl has reread the network/dns info we force a reload"""
-    global _curl_cache
-    _curl_cache.close()
-    _curl_cache = pycurl.Curl()
-
-
-    
-
-#####################################################################
-# DEPRECATED FUNCTIONS
-def set_throttle(new_throttle):
-    """Deprecated. Use: default_grabber.throttle = new_throttle"""
-    default_grabber.throttle = new_throttle
-
-def set_bandwidth(new_bandwidth):
-    """Deprecated. Use: default_grabber.bandwidth = new_bandwidth"""
-    default_grabber.bandwidth = new_bandwidth
-
-def set_progress_obj(new_progress_obj):
-    """Deprecated. Use: default_grabber.progress_obj = new_progress_obj"""
-    default_grabber.progress_obj = new_progress_obj
-
-def set_user_agent(new_user_agent):
-    """Deprecated. Use: default_grabber.user_agent = new_user_agent"""
-    default_grabber.user_agent = new_user_agent
-    
-def retrygrab(url, filename=None, copy_local=0, close_connection=0,
-              progress_obj=None, throttle=None, bandwidth=None,
-              numtries=3, retrycodes=[-1,2,4,5,6,7], checkfunc=None):
-    """Deprecated. Use: urlgrab() with the retry arg instead"""
-    kwargs = {'copy_local' :  copy_local, 
-              'close_connection' : close_connection,
-              'progress_obj' : progress_obj, 
-              'throttle' : throttle, 
-              'bandwidth' : bandwidth,
-              'retry' : numtries,
-              'retrycodes' : retrycodes,
-              'checkfunc' : checkfunc 
-              }
-    return urlgrab(url, filename, **kwargs)
-
-        
-#####################################################################
-#  TESTING
-def _main_test():
-    try: url, filename = sys.argv[1:3]
-    except ValueError:
-        print 'usage:', sys.argv[0], \
-              '<url> <filename> [copy_local=0|1] [close_connection=0|1]'
-        sys.exit()
-
-    kwargs = {}
-    for a in sys.argv[3:]:
-        k, v = string.split(a, '=', 1)
-        kwargs[k] = int(v)
-
-    set_throttle(1.0)
-    set_bandwidth(32 * 1024)
-    print "throttle: %s,  throttle bandwidth: %s B/s" % (default_grabber.throttle, 
-                                                        default_grabber.bandwidth)
-
-    try: from progress import text_progress_meter
-    except ImportError, e: pass
-    else: kwargs['progress_obj'] = text_progress_meter()
-
-    try: name = apply(urlgrab, (url, filename), kwargs)
-    except URLGrabError, e: print e
-    else: print 'LOCAL FILE:', name
-
-
-def _retry_test():
-    try: url, filename = sys.argv[1:3]
-    except ValueError:
-        print 'usage:', sys.argv[0], \
-              '<url> <filename> [copy_local=0|1] [close_connection=0|1]'
-        sys.exit()
-
-    kwargs = {}
-    for a in sys.argv[3:]:
-        k, v = string.split(a, '=', 1)
-        kwargs[k] = int(v)
-
-    try: from progress import text_progress_meter
-    except ImportError, e: pass
-    else: kwargs['progress_obj'] = text_progress_meter()
-
-    def cfunc(filename, hello, there='foo'):
-        print hello, there
-        import random
-        rnum = random.random()
-        if rnum < .5:
-            print 'forcing retry'
-            raise URLGrabError(-1, 'forcing retry')
-        if rnum < .75:
-            print 'forcing failure'
-            raise URLGrabError(-2, 'forcing immediate failure')
-        print 'success'
-        return
-        
-    kwargs['checkfunc'] = (cfunc, ('hello',), {'there':'there'})
-    try: name = apply(retrygrab, (url, filename), kwargs)
-    except URLGrabError, e: print e
-    else: print 'LOCAL FILE:', name
-
-def _file_object_test(filename=None):
-    import cStringIO
-    if filename is None:
-        filename = __file__
-    print 'using file "%s" for comparisons' % filename
-    fo = open(filename)
-    s_input = fo.read()
-    fo.close()
-
-    for testfunc in [_test_file_object_smallread,
-                     _test_file_object_readall,
-                     _test_file_object_readline,
-                     _test_file_object_readlines]:
-        fo_input = cStringIO.StringIO(s_input)
-        fo_output = cStringIO.StringIO()
-        wrapper = PyCurlFileObject(fo_input, None, 0)
-        print 'testing %-30s ' % testfunc.__name__,
-        testfunc(wrapper, fo_output)
-        s_output = fo_output.getvalue()
-        if s_output == s_input: print 'passed'
-        else: print 'FAILED'
-            
-def _test_file_object_smallread(wrapper, fo_output):
-    while 1:
-        s = wrapper.read(23)
-        fo_output.write(s)
-        if not s: return
-
-def _test_file_object_readall(wrapper, fo_output):
-    s = wrapper.read()
-    fo_output.write(s)
-
-def _test_file_object_readline(wrapper, fo_output):
-    while 1:
-        s = wrapper.readline()
-        fo_output.write(s)
-        if not s: return
-
-def _test_file_object_readlines(wrapper, fo_output):
-    li = wrapper.readlines()
-    fo_output.write(string.join(li, ''))
-
-if __name__ == '__main__':
-    _main_test()
-    _retry_test()
-    _file_object_test('test')
 
=== modified file 'ChangeLog'
--- ChangeLog	2010-06-21 20:36:19 +0000
+++ ChangeLog	2014-12-13 22:24:13 +0000
@@ -1,3 +1,11 @@
[phone]  Zdenek Pavlas <[e-mail]>
+
+	* lots of enahncements and bugfixes
+	  (parallel downloading, mirror profiling, new options)
+	* updated authors, url
+	* updated unit tests
+	* bump version to 3.10
+
 2009-09-25  Seth Vidal <[e-mail]>
 
 	* urlgrabber/__init__.py: bump version to 3.9.1
 
=== modified file 'MANIFEST'
--- MANIFEST	2010-06-21 20:36:19 +0000
+++ MANIFEST	2014-12-13 22:24:13 +0000
@@ -1,3 +1,4 @@
+# file GENERATED by distutils, do NOT edit
 ChangeLog
 LICENSE
 MANIFEST
@@ -6,6 +7,7 @@
 makefile
 setup.py
 scripts/urlgrabber
+scripts/urlgrabber-ext-down
 test/base_test_code.py
 test/grabberperf.py
 test/munittest.py
 
=== modified file 'PKG-INFO'
--- PKG-INFO	2010-06-21 20:36:19 +0000
+++ PKG-INFO	2014-12-13 22:24:13 +0000
@@ -1,37 +1,37 @@
-Metadata-Version: 1.0
+Metadata-Version: 1.1
 Name: urlgrabber
-Version: 3.9.1
+Version: 3.10.1
 Summary: A high-level cross-protocol url-grabber
-Home-page: http://linux.duke.edu/projects/urlgrabber/
+Home-page: http://urlgrabber.baseurl.org/
 Author: Michael D. Stenner, Ryan Tomayko
-Author-email: [e-mail], [e-mail]
+Author-email: [e-mail], [e-mail]
 License: LGPL
 Description: A high-level cross-protocol url-grabber.
         
         Using urlgrabber, data can be fetched in three basic ways:
         
-        urlgrab(url) copy the file to the local filesystem
-        urlopen(url) open the remote file and return a file object
-        (like urllib2.urlopen)
-        urlread(url) return the contents of the file as a string
+          urlgrab(url) copy the file to the local filesystem
+          urlopen(url) open the remote file and return a file object
+             (like urllib2.urlopen)
+          urlread(url) return the contents of the file as a string
         
         When using these functions (or methods), urlgrabber supports the
         following features:
         
-        * identical behavior for http://, ftp://, and file:// urls
-        * http keepalive - faster downloads of many files by using
-        only a single connection
-        * byte ranges - fetch only a portion of the file
-        * reget - for a urlgrab, resume a partial download
-        * progress meters - the ability to report download progress
-        automatically, even when using urlopen!
-        * throttling - restrict bandwidth usage
-        * retries - automatically retry a download if it fails. The
-        number of retries and failure types are configurable.
-        * authenticated server access for http and ftp
-        * proxy support - support for authenticated http and ftp proxies
-        * mirror groups - treat a list of mirrors as a single source,
-        automatically switching mirrors if there is a failure.
+          * identical behavior for http://, ftp://, and file:// urls
+          * http keepalive - faster downloads of many files by using
+            only a single connection
+          * byte ranges - fetch only a portion of the file
+          * reget - for a urlgrab, resume a partial download
+          * progress meters - the ability to report download progress
+            automatically, even when using urlopen!
+          * throttling - restrict bandwidth usage
+          * retries - automatically retry a download if it fails. The
+            number of retries and failure types are configurable.
+          * authenticated server access for http and ftp
+          * proxy support - support for authenticated http and ftp proxies
+          * mirror groups - treat a list of mirrors as a single source,
+            automatically switching mirrors if there is a failure.
         
 Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 
=== modified file 'README'
--- README	2005-10-23 12:29:28 +0000
+++ README	2014-12-13 22:24:13 +0000
@@ -19,7 +19,7 @@
    python setup.py bdist_rpm 
 
 The rpms (both source and "binary") will be specific to the current
-distrubution/version and may not be portable to others.  This is
+distribution/version and may not be portable to others.  This is
 because they will be built for the currently installed python.
 
 keepalive.py and byterange.py are generic urllib2 extension modules and
 
=== modified file 'debian/changelog'
--- debian/changelog	2014-02-23 13:54:39 +0000
+++ debian/changelog	2014-12-13 22:24:13 +0000
@@ -1,3 +1,10 @@
+urlgrabber (3.10.1-0ubuntu1) vivid; urgency=medium
+
+  * New upstream release.
+  * Drop all patches, fixed upstream
+
+ -- Jackson Doak <[e-mail]>  Sun, 14 Dec 2014 09:12:57 +1100
+
 urlgrabber (3.9.1-4ubuntu3) trusty; urgency=medium
 
   * Rebuild to drop files installed into /usr/share/pyshared.
 
=== removed file 'debian/patches/grabber_fix.diff'
--- debian/patches/grabber_fix.diff	2010-07-08 17:40:08 +0000
+++ debian/patches/grabber_fix.diff	1970-01-01 00:00:00 +0000
@@ -1,236 +0,0 @@
---- urlgrabber-3.9.1/urlgrabber/grabber.py.orig	2010-07-02 21:24:12.000000000 -0400
-+++ urlgrabber-3.9.1/urlgrabber/grabber.py	2010-07-02 20:30:25.000000000 -0400
-@@ -68,14 +68,14 @@
-     (which can be set on default_grabber.throttle) is used. See
-     BANDWIDTH THROTTLING for more information.
- 
--  timeout = None
-+  timeout = 300
- 
--    a positive float expressing the number of seconds to wait for socket
--    operations. If the value is None or 0.0, socket operations will block
--    forever. Setting this option causes urlgrabber to call the settimeout
--    method on the Socket object used for the request. See the Python
--    documentation on settimeout for more information.
--    http://www.python.org/doc/current/lib/socket-objects.html
-+    a positive integer expressing the number of seconds to wait before
-+    timing out attempts to connect to a server. If the value is None
-+    or 0, connection attempts will not time out. The timeout is passed
-+    to the underlying pycurl object as its CONNECTTIMEOUT option, see
-+    the curl documentation on CURLOPT_CONNECTTIMEOUT for more information.
-+    http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTCONNECTTIMEOUT
- 
-   bandwidth = 0
- 
-@@ -439,6 +439,12 @@
- except:
-     __version__ = '???'
- 
-+try:
-+    # this part isn't going to do much - need to talk to gettext
-+    from i18n import _
-+except ImportError, msg:
-+    def _(st): return st
-+    
- ########################################################################
- # functions for debugging output.  These functions are here because they
- # are also part of the module initialization.
-@@ -808,7 +814,7 @@
-         self.prefix = None
-         self.opener = None
-         self.cache_openers = True
--        self.timeout = None
-+        self.timeout = 300
-         self.text = None
-         self.http_headers = None
-         self.ftp_headers = None
-@@ -1052,9 +1058,15 @@
-         self._reget_length = 0
-         self._prog_running = False
-         self._error = (None, None)
--        self.size = None
-+        self.size = 0
-+        self._hdr_ended = False
-         self._do_open()
-         
-+
-+    def geturl(self):
-+        """ Provide the geturl() method, used to be got from
-+            urllib.addinfourl, via. urllib.URLopener.* """
-+        return self.url
-         
-     def __getattr__(self, name):
-         """This effectively allows us to wrap at the instance level.
-@@ -1085,9 +1097,14 @@
-             return -1
-             
-     def _hdr_retrieve(self, buf):
-+        if self._hdr_ended:
-+            self._hdr_dump = ''
-+            self.size = 0
-+            self._hdr_ended = False
-+
-         if self._over_max_size(cur=len(self._hdr_dump), 
-                                max_size=self.opts.max_header_size):
--            return -1            
-+            return -1
-         try:
-             self._hdr_dump += buf
-             # we have to get the size before we do the progress obj start
-@@ -1104,7 +1121,17 @@
-                     s = parse150(buf)
-                 if s:
-                     self.size = int(s)
--            
-+                    
-+            if buf.lower().find('location') != -1:
-+                location = ':'.join(buf.split(':')[1:])
-+                location = location.strip()
-+                self.scheme = urlparse.urlsplit(location)[0]
-+                self.url = location
-+                
-+            if len(self._hdr_dump) != 0 and buf == '\r\n':
-+                self._hdr_ended = True
-+                if DEBUG: DEBUG.info('header ended:')
-+                
-             return len(buf)
-         except KeyboardInterrupt:
-             return pycurl.READFUNC_ABORT
-@@ -1113,8 +1140,10 @@
-         if self._parsed_hdr:
-             return self._parsed_hdr
-         statusend = self._hdr_dump.find('\n')
-+        statusend += 1 # ridiculous as it may seem.
-         hdrfp = StringIO()
-         hdrfp.write(self._hdr_dump[statusend:])
-+        hdrfp.seek(0)
-         self._parsed_hdr =  mimetools.Message(hdrfp)
-         return self._parsed_hdr
-     
-@@ -1136,6 +1165,7 @@
-         self.curl_obj.setopt(pycurl.PROGRESSFUNCTION, self._progress_update)
-         self.curl_obj.setopt(pycurl.FAILONERROR, True)
-         self.curl_obj.setopt(pycurl.OPT_FILETIME, True)
-+        self.curl_obj.setopt(pycurl.FOLLOWLOCATION, True)
-         
-         if DEBUG:
-             self.curl_obj.setopt(pycurl.VERBOSE, True)
-@@ -1148,9 +1178,11 @@
-         
-         # timeouts
-         timeout = 300
--        if opts.timeout:
--            timeout = int(opts.timeout)
--            self.curl_obj.setopt(pycurl.CONNECTTIMEOUT, timeout)
-+        if hasattr(opts, 'timeout'):
-+            timeout = int(opts.timeout or 0)
-+        self.curl_obj.setopt(pycurl.CONNECTTIMEOUT, timeout)
-+        self.curl_obj.setopt(pycurl.LOW_SPEED_LIMIT, 1)
-+        self.curl_obj.setopt(pycurl.LOW_SPEED_TIME, timeout)
- 
-         # ssl options
-         if self.scheme == 'https':
-@@ -1276,7 +1308,7 @@
-                 raise err
- 
-             elif errcode == 60:
--                msg = _("client cert cannot be verified or client cert incorrect")
-+                msg = _("Peer cert cannot be verified or peer cert invalid")
-                 err = URLGrabError(14, msg)
-                 err.url = self.url
-                 raise err
-@@ -1291,7 +1323,12 @@
-                 raise err
-                     
-             elif str(e.args[1]) == '' and self.http_code != 0: # fake it until you make it
--                msg = 'HTTP Error %s : %s ' % (self.http_code, self.url)
-+                if self.scheme in ['http', 'https']:
-+                    msg = 'HTTP Error %s : %s ' % (self.http_code, self.url)
-+                elif self.scheme in ['ftp']:
-+                    msg = 'FTP Error %s : %s ' % (self.http_code, self.url)
-+                else:
-+                    msg = "Unknown Error: URL=%s , scheme=%s" % (self.url, self.scheme)
-             else:
-                 msg = 'PYCURL ERROR %s - "%s"' % (errcode, str(e.args[1]))
-                 code = errcode
-@@ -1299,6 +1336,12 @@
-             err.code = code
-             err.exception = e
-             raise err
-+        else:
-+            if self._error[1]:
-+                msg = self._error[1]
-+                err = URLGRabError(14, msg)
-+                err.url = self.url
-+                raise err
- 
-     def _do_open(self):
-         self.curl_obj = _curl_cache
-@@ -1446,9 +1489,23 @@
-             # set the time
-             mod_time = self.curl_obj.getinfo(pycurl.INFO_FILETIME)
-             if mod_time != -1:
--                os.utime(self.filename, (mod_time, mod_time))
-+                try:
-+                    os.utime(self.filename, (mod_time, mod_time))
-+                except OSError, e:
-+                    err = URLGrabError(16, _(\
-+                      'error setting timestamp on file %s from %s, OSError: %s') 
-+                              % (self.filenameself.url, e))
-+                    err.url = self.url
-+                    raise err
-             # re open it
--            self.fo = open(self.filename, 'r')
-+            try:
-+                self.fo = open(self.filename, 'r')
-+            except IOError, e:
-+                err = URLGrabError(16, _(\
-+                  'error opening file from %s, IOError: %s') % (self.url, e))
-+                err.url = self.url
-+                raise err
-+                
-         else:
-             #self.fo = open(self._temp_name, 'r')
-             self.fo.seek(0)
-@@ -1532,11 +1589,14 @@
-     def _over_max_size(self, cur, max_size=None):
- 
-         if not max_size:
--            max_size = self.size
--        if self.opts.size: # if we set an opts size use that, no matter what
--            max_size = self.opts.size
-+            if not self.opts.size:
-+                max_size = self.size
-+            else:
-+                max_size = self.opts.size
-+
-         if not max_size: return False # if we have None for all of the Max then this is dumb
--        if cur > max_size + max_size*.10:
-+
-+        if cur > int(float(max_size) * 1.10):
- 
-             msg = _("Downloaded more than max size for %s: %s > %s") \
-                         % (self.url, cur, max_size)
-@@ -1582,9 +1642,21 @@
-             self.opts.progress_obj.end(self._amount_read)
-         self.fo.close()
-         
--
-+    def geturl(self):
-+        """ Provide the geturl() method, used to be got from
-+            urllib.addinfourl, via. urllib.URLopener.* """
-+        return self.url
-+        
- _curl_cache = pycurl.Curl() # make one and reuse it over and over and over
- 
-+def reset_curl_obj():
-+    """To make sure curl has reread the network/dns info we force a reload"""
-+    global _curl_cache
-+    _curl_cache.close()
-+    _curl_cache = pycurl.Curl()
-+
-+
-+    
- 
- #####################################################################
- # DEPRECATED FUNCTIONS
 
=== removed file 'debian/patches/progress_fix.diff'
--- debian/patches/progress_fix.diff	2010-07-08 17:40:08 +0000
+++ debian/patches/progress_fix.diff	1970-01-01 00:00:00 +0000
@@ -1,11 +0,0 @@
---- urlgrabber-3.9.1/urlgrabber/progress.py.orig	2010-07-02 21:25:51.000000000 -0400
-+++ urlgrabber-3.9.1/urlgrabber/progress.py	2010-07-02 20:30:25.000000000 -0400
-@@ -658,6 +658,8 @@
-     if seconds is None or seconds < 0:
-         if use_hours: return '--:--:--'
-         else:         return '--:--'
-+    elif seconds == float('inf'):
-+        return 'Infinite'
-     else:
-         seconds = int(seconds)
-         minutes = seconds / 60
 
=== removed file 'debian/patches/progress_object_callback_fix.diff'
--- debian/patches/progress_object_callback_fix.diff	2011-08-09 17:45:08 +0000
+++ debian/patches/progress_object_callback_fix.diff	1970-01-01 00:00:00 +0000
@@ -1,21 +0,0 @@
-From: James Antill <[e-mail]>
-Date: Thu, 19 May 2011 20:17:14 +0000 (-0400)
-Subject: Fix documentation for progress_object callback.
-X-Git-Url: http://yum.baseurl.org/gitweb?p=urlgrabber.git;a=commitdiff_plain;h=674d545ee303aa99701ffb982536851572d8db77
-
-Fix documentation for progress_object callback.
----
-
-diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
-index 36212cf..f6f57bd 100644
---- a/urlgrabber/grabber.py
-+++ b/urlgrabber/grabber.py
-@@ -49,7 +49,7 @@ GENERAL ARGUMENTS (kwargs)
-   progress_obj = None
- 
-     a class instance that supports the following methods:
--      po.start(filename, url, basename, length, text)
-+      po.start(filename, url, basename, size, now, text)
-       # length will be None if unknown
-       po.update(read) # read == bytes read so far
-       po.end()
 
=== modified file 'debian/patches/series'
--- debian/patches/series	2011-08-09 17:45:08 +0000
+++ debian/patches/series	2014-12-13 22:24:13 +0000
@@ -1,3 +0,0 @@
-grabber_fix.diff
-progress_fix.diff
-progress_object_callback_fix.diff
 
=== modified file 'scripts/urlgrabber'
--- scripts/urlgrabber	2010-06-21 20:36:19 +0000
+++ scripts/urlgrabber	2014-12-13 22:24:13 +0000
@@ -115,6 +115,7 @@
                     including quotes in the case of strings.
                     e.g.  --user_agent='"foobar/2.0"'
 
+  --output FILE
   -o FILE           write output to FILE, otherwise the basename of the
                     url will be used
   -O                print the names of saved files to STDOUT
@@ -170,12 +171,17 @@
         return ug_options, ug_defaults
 
     def process_command_line(self):
-        short_options = 'vd:hoOpD'
+        short_options = 'vd:ho:OpD'
         long_options = ['profile', 'repeat=', 'verbose=',
-                        'debug=', 'help', 'progress']
+                        'debug=', 'help', 'progress', 'output=']
         ug_long = [ o + '=' for o in self.ug_options ]
-        optlist, args = getopt.getopt(sys.argv[1:], short_options,
-                                      long_options + ug_long)
+        try:
+            optlist, args = getopt.getopt(sys.argv[1:], short_options,
+                                          long_options + ug_long)
+        except getopt.GetoptError, e:
+            print >>sys.stderr, "Error:", e
+            self.help([], ret=1)
+
         self.verbose = 0
         self.debug = None
         self.outputfile = None
@@ -193,6 +199,7 @@
             if o == '--verbose': self.verbose = v
             if o == '-v':        self.verbose += 1
             if o == '-o':        self.outputfile = v
+            if o == '--output':  self.outputfile = v
             if o == '-p' or o == '--progress': self.progress = 1
             if o == '-d' or o == '--debug': self.debug = v
             if o == '--profile': self.profile = 1
@@ -222,7 +229,7 @@
             print "ERROR: cannot use -o when grabbing multiple files"
             sys.exit(1)
 
-    def help(self, args):
+    def help(self, args, ret=0):
         if not args:
             print MAINHELP
         else:
@@ -234,7 +241,7 @@
                     self.help_ug_option(a)
                 else:
                     print 'ERROR: no help on command "%s"' % a
-        sys.exit(0)
+        sys.exit(ret)
 
     def help_doc(self):
         print __doc__
@@ -294,6 +301,7 @@
                 if self.op.localfile: print f
             except URLGrabError, e:
                 print e
+                sys.exit(1)
         
     def set_debug_logger(self, dbspec):
         try:
 
=== added file 'scripts/urlgrabber-ext-down'
--- scripts/urlgrabber-ext-down	1970-01-01 00:00:00 +0000
+++ scripts/urlgrabber-ext-down	2014-12-13 22:24:13 +0000
@@ -0,0 +1,75 @@
+#! /usr/bin/python
+#  A very simple external downloader
+#  Copyright 2011-2012 Zdenek Pavlas
+
+#   This library is free software; you can redistribute it and/or
+#   modify it under the terms of the GNU Lesser General Public
+#   License as published by the Free Software Foundation; either
+#   version 2.1 of the License, or (at your option) any later version.
+#
+#   This library is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#   Lesser General Public License for more details.
+#
+#   You should have received a copy of the GNU Lesser General Public
+#   License along with this library; if not, write to the
+#      Free Software Foundation, Inc.,
+#      59 Temple Place, Suite 330,
+#      Boston, MA  02111-1307  USA
+
+import time, os, errno, sys
+from urlgrabber.grabber import \
+    _readlines, URLGrabberOptions, _loads, \
+    PyCurlFileObject, URLGrabError
+
+def write(fmt, *arg):
+    try: os.write(1, fmt % arg)
+    except OSError, e:
+        if e.args[0] != errno.EPIPE: raise
+        sys.exit(1)
+
+class ProxyProgress:
+    def start(self, *d1, **d2):
+        self.next_update = 0
+    def update(self, _amount_read):
+        t = time.time()
+        if t < self.next_update: return
+        self.next_update = t + 0.31
+        write('%d %d\n', self._id, _amount_read)
+
+def main():
+    import signal
+    signal.signal(signal.SIGINT, lambda n, f: sys.exit(1))
+    cnt = 0
+    while True:
+        lines = _readlines(0)
+        if not lines: break
+        for line in lines:
+            cnt += 1
+            opts = URLGrabberOptions()
+            opts._id = cnt
+            for k in line.split(' '):
+                k, v = k.split('=', 1)
+                setattr(opts, k, _loads(v))
+            if opts.progress_obj:
+                opts.progress_obj = ProxyProgress()
+                opts.progress_obj._id = cnt
+
+            dlsz = dltm = 0
+            try:
+                fo = PyCurlFileObject(opts.url, opts.filename, opts)
+                fo._do_grab()
+                fo.fo.close()
+                size = fo._amount_read
+                if fo._tm_last:
+                    dlsz = fo._tm_last[0] - fo._tm_first[0]
+                    dltm = fo._tm_last[1] - fo._tm_first[1]
+                ug_err = 'OK'
+            except URLGrabError, e:
+                size = 0
+                ug_err = '%d %d %s' % (e.errno, getattr(e, 'code', 0), e.strerror)
+            write('%d %d %d %.3f %s\n', opts._id, size, dlsz, dltm, ug_err)
+
+if __name__ == '__main__':
+    main()
 
=== modified file 'setup.py'
--- setup.py	2005-10-23 12:29:28 +0000
+++ setup.py	2014-12-13 22:24:13 +0000
@@ -15,8 +15,10 @@
 packages = ['urlgrabber']
 package_dir = {'urlgrabber':'urlgrabber'}
 scripts = ['scripts/urlgrabber']
-data_files = [('share/doc/' + name + '-' + version,
-               ['README','LICENSE', 'TODO', 'ChangeLog'])]
+data_files = [
+    ('share/doc/' + name + '-' + version, ['README','LICENSE', 'TODO', 'ChangeLog']),
+    ('libexec', ['scripts/urlgrabber-ext-down']),
+]
 options = { 'clean' : { 'all' : 1 } }
 classifiers = [
         'Development Status :: 4 - Beta',
 
=== modified file 'test/base_test_code.py'
--- test/base_test_code.py	2005-10-23 12:29:28 +0000
+++ test/base_test_code.py	2014-12-13 22:24:13 +0000
@@ -1,6 +1,6 @@
 from munittest import *
 
-base_http = 'http://www.linux.duke.edu/projects/urlgrabber/test/'
+base_http = 'http://urlgrabber.baseurl.org/test/'
 base_ftp  = 'ftp://localhost/test/'
 
 # set to a proftp server only. we're working around a couple of
 
=== modified file 'test/munittest.py'
--- test/munittest.py	2005-10-23 12:29:28 +0000
+++ test/munittest.py	2014-12-13 22:24:13 +0000
@@ -113,7 +113,7 @@
 __all__ = ['TestResult', 'TestCase', 'TestSuite', 'TextTestRunner',
            'TestLoader', 'FunctionTestCase', 'main', 'defaultTestLoader']
 
-# Expose obsolete functions for backwards compatability
+# Expose obsolete functions for backwards compatibility
 __all__.extend(['getTestCaseNames', 'makeSuite', 'findTestCases'])
 
 
@@ -410,7 +410,7 @@
            (default 7) and comparing to zero.
 
            Note that decimal places (from zero) is usually not the same
-           as significant digits (measured from the most signficant digit).
+           as significant digits (measured from the most significant digit).
         """
         if round(second-first, places) != 0:
             raise self.failureException, \
@@ -422,7 +422,7 @@
            (default 7) and comparing to zero.
 
            Note that decimal places (from zero) is usually not the same
-           as significant digits (measured from the most signficant digit).
+           as significant digits (measured from the most significant digit).
         """
         if round(second-first, places) == 0:
             raise self.failureException, \
 
=== modified file 'test/test_byterange.py'
--- test/test_byterange.py	2005-10-23 12:29:28 +0000
+++ test/test_byterange.py	2014-12-13 22:24:13 +0000
@@ -25,7 +25,7 @@
 
 import sys
 
-from StringIO import StringIO
+from cStringIO import StringIO
 from urlgrabber.byterange import RangeableFileObject
 
 from base_test_code import *
@@ -52,18 +52,6 @@
         self.rfo.seek(1,1)
         self.assertEquals('of', self.rfo.read(2))
     
-    def test_poor_mans_seek(self):
-        """RangeableFileObject.seek() poor mans version..
-        
-        We just delete the seek method from StringIO so we can
-        excercise RangeableFileObject when the file object supplied
-        doesn't support seek.
-        """
-        seek = StringIO.seek
-        del(StringIO.seek)
-        self.test_seek()
-        StringIO.seek = seek
-    
     def test_read(self):
         """RangeableFileObject.read()"""
         self.assertEquals('the', self.rfo.read(3))
 
=== modified file 'test/test_grabber.py'
--- test/test_grabber.py	2010-06-21 20:36:19 +0000
+++ test/test_grabber.py	2014-12-13 22:24:13 +0000
@@ -86,7 +86,7 @@
     
 class HTTPTests(TestCase):
     def test_reference_file(self):
-        "download refernce file via HTTP"
+        "download reference file via HTTP"
         filename = tempfile.mktemp()
         grabber.urlgrab(ref_http, filename)
 
@@ -98,6 +98,7 @@
 
     def test_post(self):
         "do an HTTP post"
+        self.skip() # disabled on server
         headers = (('Content-type', 'text/plain'),)
         ret = grabber.urlread(base_http + 'test_post.php',
                               data=short_reference_data,
 
=== modified file 'test/test_mirror.py'
--- test/test_mirror.py	2005-12-31 15:34:22 +0000
+++ test/test_mirror.py	2014-12-13 22:24:13 +0000
@@ -28,7 +28,7 @@
 import string, tempfile, random, cStringIO, os
 
 import urlgrabber.grabber