Merge lp:~mterry/duplicity/tarfile into lp:duplicity/0.6

[Michael Terry]

[Michael Terry (mterry)]

Due to conversations in the mailing list, I've retooled this branch to, instead of using the system tarfile.py, just updating our internal copy to python2.7's version (with changes for 2.4-compatibility).

Once we drop 2.4 support, it will be a simple change to drop our internal version and use the system version again.

I've run the test suite against this branch (under python 2.4 too), but of course it could use more testing.

[edso (ed.so)]

Does this solve
https://bugs.launchpad.net/duplicity/+bug/690549
?

Or is the patch there invalid, because tarfile.py does not support such big values?

ede

On 23.08.2011 20:36, Michael Terry wrote:
> Due to conversations in the mailing list, I've retooled this branch to, instead of using the system tarfile.py, just updating our internal copy to python2.7's version (with changes for 2.4-compatibility).
>
> Once we drop 2.4 support, it will be a simple change to drop our internal version and use the system version again.
>
> I've run the test suite against this branch (under python 2.4 too), but of course it could use more testing.

[Michael Terry (mterry)]

The patch there looks invalid, because it looks like it just assigns more bits in the tarfile for the gid/uid, which I don't think is part of the standard. Though maybe I'm misinterpreting the patch.

However, I do think this branch would solve that bug, as newer tarfiles seem to have explicit support for negative numbers (see the itn() function in this branch's tarfile.py).

=== modified file 'duplicity/diffdir.py'
--- duplicity/diffdir.py	2011-06-13 15:40:19 +0000
+++ duplicity/diffdir.py	2011-08-23 18:28:26 +0000
@@ -29,6 +29,7 @@
 
 import cStringIO, types
 from duplicity import statistics
+from duplicity import util
 from duplicity.path import * #@UnusedWildImport
 from duplicity.lazy import * #@UnusedWildImport
 
@@ -181,7 +182,7 @@
     """
     collated = collate2iters(new_iter, sig_iter)
     if sig_fileobj:
-        sigTarFile = tarfile.TarFile("arbitrary", "w", sig_fileobj)
+        sigTarFile = util.make_tarfile("w", sig_fileobj)
     else:
         sigTarFile = None
     for new_path, sig_path in collated:
@@ -224,16 +225,17 @@
     """
     Convert signature tar file object open for reading into path iter
     """
-    tf = tarfile.TarFile("Arbitrary Name", "r", sigtarobj)
-    tf.debug = 2
+    tf = util.make_tarfile("r", sigtarobj)
+    tf.debug = 1
     for tarinfo in tf:
+        tiname = util.get_tarinfo_name(tarinfo)
         for prefix in ["signature/", "snapshot/", "deleted/"]:
-            if tarinfo.name.startswith(prefix):
-                # strip prefix and from name and set it to difftype
-                name, difftype = tarinfo.name[len(prefix):], prefix[:-1]
+            if tiname.startswith(prefix):
+                # strip prefix and '/' from name and set it to difftype
+                name, difftype = tiname[len(prefix):], prefix[:-1]
                 break
         else:
-            raise DiffDirException("Bad tarinfo name %s" % (tarinfo.name,))
+            raise DiffDirException("Bad tarinfo name %s" % (tiname,))
 
         index = tuple(name.split("/"))
         if not index[-1]:
@@ -464,16 +466,12 @@
         self.remember_value = None          # holds index of next block
         self.remember_block = None          # holds block of next block
 
-        # We need to instantiate a dummy TarFile just to get access to
-        # some of the functions like _get_full_headers.
-        self.tf = tarfile.TarFromIterator(None)
-
     def tarinfo2tarblock(self, index, tarinfo, file_data = ""):
         """
         Make tarblock out of tarinfo and file data
         """
         tarinfo.size = len(file_data)
-        headers = self.tf._get_full_headers(tarinfo)
+        headers = tarinfo.tobuf()
         blocks, remainder = divmod(tarinfo.size, tarfile.BLOCKSIZE) #@UnusedVariable
         if remainder > 0:
             filler_data = "\0" * (tarfile.BLOCKSIZE - remainder)

=== modified file 'duplicity/dup_temp.py'
--- duplicity/dup_temp.py	2011-08-18 19:17:55 +0000
+++ duplicity/dup_temp.py	2011-08-23 18:28:26 +0000
@@ -207,6 +207,18 @@
         """
         return self.fileobj.read(length)
 
+    def tell(self):
+        """
+        Returns current location of fileobj
+        """
+        return self.fileobj.tell()
+
+    def seek(self, offset):
+        """
+        Seeks to a location of fileobj
+        """
+        return self.fileobj.seek(offset)
+
     def close(self):
         """
         Close fileobj, running hooks right afterwards

=== modified file 'duplicity/gpg.py'
--- duplicity/gpg.py	2011-07-16 18:37:47 +0000
+++ duplicity/gpg.py	2011-08-23 18:28:26 +0000
@@ -96,6 +96,7 @@
         self.logger_fp = tempfile.TemporaryFile()
         self.stderr_fp = tempfile.TemporaryFile()
         self.name = encrypt_path
+        self.byte_count = 0
 
         # Start GPG process - copied from GnuPGInterface docstring.
         gnupg = GnuPGInterface.GnuPG()
@@ -157,6 +158,8 @@
     def read(self, length = -1):
         try:
             res = self.gpg_output.read(length)
+            if res is not None:
+                self.byte_count += len(res)
         except Exception:
             self.gpg_failed()
         return res
@@ -164,10 +167,21 @@
     def write(self, buf):
         try:
             res = self.gpg_input.write(buf)
+            if res is not None:
+                self.byte_count += len(res)
         except Exception:
             self.gpg_failed()
         return res
 
+    def tell(self):
+        return self.byte_count
+
+    def seek(self, offset):
+        assert not self.encrypt
+        assert offset >= self.byte_count, "%d < %d" % (offset, self.byte_count)
+        if offset > self.byte_count:
+            self.read(offset - self.byte_count)
+
     def gpg_failed(self):
         msg = "GPG Failed, see log below:\n"
         msg += "===== Begin GnuPG log =====\n"

=== modified file 'duplicity/patchdir.py'
--- duplicity/patchdir.py	2011-03-06 12:30:14 +0000
+++ duplicity/patchdir.py	2011-08-23 18:28:26 +0000
@@ -141,8 +141,9 @@
     """Return (index, difftype, multivol) pair from tarinfo object"""
     for prefix in ["snapshot/", "diff/", "deleted/",
                    "multivol_diff/", "multivol_snapshot/"]:
-        if tarinfo.name.startswith( prefix ):
-            name = tarinfo.name[len( prefix ):] # strip prefix
+        tiname = util.get_tarinfo_name( tarinfo )
+        if tiname.startswith( prefix ):
+            name = tiname[len( prefix ):] # strip prefix
             if prefix.startswith( "multivol" ):
                 if prefix == "multivol_diff/":
                     difftype = "diff"
@@ -150,28 +151,28 @@
                     difftype = "snapshot"
                 multivol = 1
                 name, num_subs = \
-                      re.subn( "(?s)^multivol_(diff|snapshot)/(.*)/[0-9]+$",
-                              "\\2", tarinfo.name )
+                      re.subn( "(?s)^multivol_(diff|snapshot)/?(.*)/[0-9]+$",
+                              "\\2", tiname )
                 if num_subs != 1:
                     raise PatchDirException( "Unrecognized diff entry %s" %
-                                            ( tarinfo.name, ) )
+                                            ( tiname, ) )
             else:
                 difftype = prefix[:-1] # strip trailing /
-                name = tarinfo.name[len( prefix ):]
+                name = tiname[len( prefix ):]
                 if name.endswith( "/" ):
                     name = name[:-1] # strip trailing /'s
                 multivol = 0
             break
     else:
         raise PatchDirException( "Unrecognized diff entry %s" %
-                                 ( tarinfo.name, ) )
+                                 ( tiname, ) )
     if name == "." or name == "":
         index = ()
     else:
         index = tuple( name.split( "/" ) )
         if '..' in index:
             raise PatchDirException( "Tar entry %s contains '..'.  Security "
-                                    "violation" % ( tarinfo.name, ) )
+                                    "violation" % ( tiname, ) )
     return ( index, difftype, multivol )
 
 
@@ -320,7 +321,7 @@
         if self.current_fp:
             assert not self.current_fp.close()
         self.current_fp = self.fileobj_iter.next()
-        self.tarfile = tarfile.TarFile( "arbitrary", "r", self.current_fp )
+        self.tarfile = util.make_tarfile("r", self.current_fp)
         self.tar_iter = iter( self.tarfile )
 
     def next( self ):

=== modified file 'duplicity/path.py'
--- duplicity/path.py	2011-08-18 20:08:56 +0000
+++ duplicity/path.py	2011-08-23 18:28:26 +0000
@@ -26,12 +26,12 @@
 
 """
 
-import stat, errno, socket, time, re, gzip
+import stat, errno, socket, time, re, gzip, pwd, grp
 
+from duplicity import tarfile
 from duplicity import file_naming
 from duplicity import globals
 from duplicity import gpg
-from duplicity import tarfile
 from duplicity import util
 from duplicity import librsync
 from duplicity import log #@UnusedImport
@@ -206,13 +206,13 @@
         try:
             if globals.numeric_owner:
                 raise KeyError
-            self.stat.st_uid = tarfile.uname2uid(tarinfo.uname)
+            self.stat.st_uid = pwd.getpwnam(tarinfo.uname)[2]
         except KeyError:
             self.stat.st_uid = tarinfo.uid
         try:
             if globals.numeric_owner:
                 raise KeyError
-            self.stat.st_gid = tarfile.gname2gid(tarinfo.gname)
+            self.stat.st_gid = grp.getgrnam(tarinfo.gname)[2]
         except KeyError:
             self.stat.st_gid = tarinfo.gid
 
@@ -284,13 +284,13 @@
                 ti.mtime = int(self.stat.st_mtime)
 
             try:
-                ti.uname = tarfile.uid2uname(ti.uid)
+                ti.uname = pwd.getpwuid(ti.uid)[0]
             except KeyError:
-                pass
+                ti.uname = ''
             try:
-                ti.gname = tarfile.gid2gname(ti.gid)
+                ti.gname = grp.getgrgid(ti.gid)[0]
             except KeyError:
-                pass
+                ti.gname = ''
 
             if ti.type in (tarfile.CHRTYPE, tarfile.BLKTYPE):
                 if hasattr(os, "major") and hasattr(os, "minor"):

=== modified file 'duplicity/tarfile.py'
--- duplicity/tarfile.py	2011-06-23 15:47:44 +0000
+++ duplicity/tarfile.py	2011-08-23 18:28:26 +0000
@@ -1,17 +1,9 @@
-#!/usr/bin/env python
-# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#! /usr/bin/python2.7
+# -*- coding: iso-8859-1 -*-
 #-------------------------------------------------------------------
 # tarfile.py
-#
-# Module for reading and writing .tar and tar.gz files.
-#
-# Needs at least Python version 2.2.
-#
-# Please consult the html documentation in this distribution
-# for further details on how to use tarfile.
-#
 #-------------------------------------------------------------------
-# Copyright (C) 2002 Lars Gustabel <[e-mail]>
+# Copyright (C) 2002 Lars Gustäbel <[e-mail]>
 # All rights reserved.
 #
 # Permission  is  hereby granted,  free  of charge,  to  any person
@@ -38,98 +30,105 @@
 """Read from and write to tar format archives.
 """
 
-__version__ = "$Revision: 1.8 $"
-# $Source: /sources/duplicity/duplicity/duplicity/tarfile.py,v $
+__version__ = "$Revision: 85213 $"
+# $Source$
 
-version     = "0.4.9"
-__author__  = "Lars Gustabel ([e-mail])"
-__date__    = "$Date: 2008/11/16 18:48:15 $"
-__cvsid__   = "$Id: tarfile.py,v 1.8 2008/11/16 18:48:15 loafman Exp $"
-__credits__ = "Gustavo Niemeyer for his support, " \
-              "Detlef Lannert for some early contributions"
+version     = "0.9.0"
+__author__  = "Lars Gustäbel ([e-mail])"
+__date__    = "$Date: 2010-10-04 10:37:53 -0500 (Mon, 04 Oct 2010) $"
+__cvsid__   = "$Id: tarfile.py 85213 2010-10-04 15:37:53Z lars.gustaebel $"
+__credits__ = "Gustavo Niemeyer, Niels Gustäbel, Richard Townsend."
 
 #---------
 # Imports
 #---------
 import sys
 import os
-import __builtin__
 import shutil
 import stat
 import errno
 import time
+import struct
+import copy
+import re
+import operator
 
 try:
     import grp, pwd
 except ImportError:
     grp = pwd = None
-# These are used later to cache user and group names and ids
-gname_dict = uname_dict = uid_dict = gid_dict = None
-
-# We won't need this anymore in Python 2.3
-#
-# We import the _tarfile extension, that contains
-# some useful functions to handle devices and symlinks.
-# We inject them into os module, as if we were under 2.3.
-#
-try:
-    import _tarfile
-    if _tarfile.mknod is None:
-        _tarfile = None
-except ImportError:
-    _tarfile = None
-if _tarfile and not hasattr(os, "mknod"):
-    os.mknod = _tarfile.mknod
-if _tarfile and not hasattr(os, "major"):
-    os.major = _tarfile.major
-if _tarfile and not hasattr(os, "minor"):
-    os.minor = _tarfile.minor
-if _tarfile and not hasattr(os, "makedev"):
-    os.makedev = _tarfile.makedev
-if _tarfile and not hasattr(os, "lchown"):
-    os.lchown = _tarfile.lchown
-
-# XXX remove for release (2.3)
-if sys.version_info[:2] < (2,3):
-    True  = 1
-    False = 0
-
-#---------------------------------------------------------
-# GNUtar constants
-#---------------------------------------------------------
-BLOCKSIZE  = 512                # length of processing blocks
+
+# from tarfile import *
+__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"]
+
+#---------------------------------------------------------
+# tar constants
+#---------------------------------------------------------
+NUL = "\0"                      # the null character
+BLOCKSIZE = 512                 # length of processing blocks
 RECORDSIZE = BLOCKSIZE * 20     # length of records
-MAGIC      = "ustar"            # magic tar string
-VERSION    = "00"               # version number
-
-LENGTH_NAME = 100               # maximal length of a filename
-LENGTH_LINK = 100               # maximal length of a linkname
-
-REGTYPE  = "0"                  # regular file
+GNU_MAGIC = "ustar  \0"         # magic gnu tar string
+POSIX_MAGIC = "ustar\x0000"     # magic posix tar string
+
+LENGTH_NAME = 100               # maximum length of a filename
+LENGTH_LINK = 100               # maximum length of a linkname
+LENGTH_PREFIX = 155             # maximum length of the prefix field
+
+REGTYPE = "0"                   # regular file
 AREGTYPE = "\0"                 # regular file
-LNKTYPE  = "1"                  # link (inside tarfile)
-SYMTYPE  = "2"                  # symbolic link
-CHRTYPE  = "3"                  # character special device
-BLKTYPE  = "4"                  # block special device
-DIRTYPE  = "5"                  # directory
+LNKTYPE = "1"                   # link (inside tarfile)
+SYMTYPE = "2"                   # symbolic link
+CHRTYPE = "3"                   # character special device
+BLKTYPE = "4"                   # block special device
+DIRTYPE = "5"                   # directory
 FIFOTYPE = "6"                  # fifo special device
 CONTTYPE = "7"                  # contiguous file
 
-GNUTYPE_LONGNAME = "L"          # GNU tar extension for longnames
-GNUTYPE_LONGLINK = "K"          # GNU tar extension for longlink
-GNUTYPE_SPARSE   = "S"          # GNU tar extension for sparse file
+GNUTYPE_LONGNAME = "L"          # GNU tar longname
+GNUTYPE_LONGLINK = "K"          # GNU tar longlink
+GNUTYPE_SPARSE = "S"            # GNU tar sparse file
+
+XHDTYPE = "x"                   # POSIX.1-2001 extended header
+XGLTYPE = "g"                   # POSIX.1-2001 global header
+SOLARIS_XHDTYPE = "X"           # Solaris extended header
+
+USTAR_FORMAT = 0                # POSIX.1-1988 (ustar) format
+GNU_FORMAT = 1                  # GNU tar format
+PAX_FORMAT = 2                  # POSIX.1-2001 (pax) format
+DEFAULT_FORMAT = GNU_FORMAT
 
 #---------------------------------------------------------
 # tarfile constants
 #---------------------------------------------------------
-SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE,  # file types that tarfile
-                   SYMTYPE, DIRTYPE, FIFOTYPE,  # can cope with.
-                   CONTTYPE, GNUTYPE_LONGNAME,
-                   GNUTYPE_LONGLINK, GNUTYPE_SPARSE,
-                   CHRTYPE, BLKTYPE)
-
-REGULAR_TYPES = (REGTYPE, AREGTYPE,             # file types that somehow
-                 CONTTYPE, GNUTYPE_SPARSE)      # represent regular files
+# File types that tarfile supports:
+SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE,
+                   SYMTYPE, DIRTYPE, FIFOTYPE,
+                   CONTTYPE, CHRTYPE, BLKTYPE,
+                   GNUTYPE_LONGNAME, GNUTYPE_LONGLINK,
+                   GNUTYPE_SPARSE)
+
+# File types that will be treated as a regular file.
+REGULAR_TYPES = (REGTYPE, AREGTYPE,
+                 CONTTYPE, GNUTYPE_SPARSE)
+
+# File types that are part of the GNU tar format.
+GNU_TYPES = (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK,
+             GNUTYPE_SPARSE)
+
+# Fields from a pax header that override a TarInfo attribute.
+PAX_FIELDS = ("path", "linkpath", "size", "mtime",
+              "uid", "gid", "uname", "gname")
+
+# Fields in a pax header that are numbers, all other fields
+# are treated as strings.
+PAX_NUMBER_FIELDS = {
+    "atime": float,
+    "ctime": float,
+    "mtime": float,
+    "uid": int,
+    "gid": int,
+    "size": int
+}
 
 #---------------------------------------------------------
 # Bits used in the mode field, values in octal.
@@ -145,34 +144,117 @@
 TSGID   = 02000          # set GID on execution
 TSVTX   = 01000          # reserved
 
-TUREAD  = 00400          # read by owner
-TUWRITE = 00200          # write by owner
-TUEXEC  = 00100          # execute/search by owner
-TGREAD  = 00040          # read by group
-TGWRITE = 00020          # write by group
-TGEXEC  = 00010          # execute/search by group
-TOREAD  = 00004          # read by other
-TOWRITE = 00002          # write by other
-TOEXEC  = 00001          # execute/search by other
+TUREAD  = 0400           # read by owner
+TUWRITE = 0200           # write by owner
+TUEXEC  = 0100           # execute/search by owner
+TGREAD  = 0040           # read by group
+TGWRITE = 0020           # write by group
+TGEXEC  = 0010           # execute/search by group
+TOREAD  = 0004           # read by other
+TOWRITE = 0002           # write by other
+TOEXEC  = 0001           # execute/search by other
+
+#---------------------------------------------------------
+# initialization
+#---------------------------------------------------------
+ENCODING = sys.getfilesystemencoding()
+if ENCODING is None:
+    ENCODING = sys.getdefaultencoding()
 
 #---------------------------------------------------------
 # Some useful functions
 #---------------------------------------------------------
+
+def stn(s, length):
+    """Convert a python string to a null-terminated string buffer.
+    """
+    return s[:length] + (length - len(s)) * NUL
+
 def nts(s):
-    """Convert a null-terminated string buffer to a python string.
-    """
-    return s.split("\0", 1)[0]
-
-def calc_chksum(buf):
-    """Calculate the checksum for a member's header. It's a simple addition
-       of all bytes, treating the chksum field as if filled with spaces.
-       buf is a 512 byte long string buffer which holds the header.
-    """
-    chk = 256                           # chksum field is treated as blanks,
-                                        # so the initial value is 8 * ord(" ")
-    for c in buf[:148]: chk += ord(c)   # sum up all bytes before chksum
-    for c in buf[156:]: chk += ord(c)   # sum up all bytes after chksum
-    return chk
+    """Convert a null-terminated string field to a python string.
+    """
+    # Use the string up to the first null char.
+    p = s.find("\0")
+    if p == -1:
+        return s
+    return s[:p]
+
+def nti(s):
+    """Convert a number field to a python number.
+    """
+    # There are two possible encodings for a number field, see
+    # itn() below.
+    if s[0] != chr(0200):
+        try:
+            n = int(nts(s) or "0", 8)
+        except ValueError:
+            raise InvalidHeaderError("invalid header")
+    else:
+        n = 0L
+        for i in xrange(len(s) - 1):
+            n <<= 8
+            n += ord(s[i + 1])
+    return n
+
+def itn(n, digits=8, format=DEFAULT_FORMAT):
+    """Convert a python number to a number field.
+    """
+    # POSIX 1003.1-1988 requires numbers to be encoded as a string of
+    # octal digits followed by a null-byte, this allows values up to
+    # (8**(digits-1))-1. GNU tar allows storing numbers greater than
+    # that if necessary. A leading 0200 byte indicates this particular
+    # encoding, the following digits-1 bytes are a big-endian
+    # representation. This allows values up to (256**(digits-1))-1.
+    if 0 <= n < 8 ** (digits - 1):
+        s = "%0*o" % (digits - 1, n) + NUL
+    else:
+        if format != GNU_FORMAT or n >= 256 ** (digits - 1):
+            raise ValueError("overflow in number field")
+
+        if n < 0:
+            # XXX We mimic GNU tar's behaviour with negative numbers,
+            # this could raise OverflowError.
+            n = struct.unpack("L", struct.pack("l", n))[0]
+
+        s = ""
+        for i in xrange(digits - 1):
+            s = chr(n & 0377) + s
+            n >>= 8
+        s = chr(0200) + s
+    return s
+
+def uts(s, encoding, errors):
+    """Convert a unicode object to a string.
+    """
+    if errors == "utf-8":
+        # An extra error handler similar to the -o invalid=UTF-8 option
+        # in POSIX.1-2001. Replace untranslatable characters with their
+        # UTF-8 representation.
+        try:
+            return s.encode(encoding, "strict")
+        except UnicodeEncodeError:
+            x = []
+            for c in s:
+                try:
+                    x.append(c.encode(encoding, "strict"))
+                except UnicodeEncodeError:
+                    x.append(c.encode("utf8"))
+            return "".join(x)
+    else:
+        return s.encode(encoding, errors)
+
+def calc_chksums(buf):
+    """Calculate the checksum for a member's header by summing up all
+       characters except for the chksum field which is treated as if
+       it was filled with spaces. According to the GNU tar sources,
+       some tars (Sun and NeXT) calculate chksum with signed char,
+       which will be different if there are chars in the buffer with
+       the high bit set. So we calculate two checksums, unsigned and
+       signed.
+    """
+    unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))
+    signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))
+    return unsigned_chksum, signed_chksum
 
 def copyfileobj(src, dst, length=None):
     """Copy length bytes from fileobj src to fileobj dst.
@@ -186,242 +268,1186 @@
 
     BUFSIZE = 16 * 1024
     blocks, remainder = divmod(length, BUFSIZE)
-    for b in range(blocks): #@UnusedVariable
+    for b in xrange(blocks):
         buf = src.read(BUFSIZE)
         if len(buf) < BUFSIZE:
-            raise IOError, "end of file reached"
+            raise IOError("end of file reached")
         dst.write(buf)
 
     if remainder != 0:
         buf = src.read(remainder)
         if len(buf) < remainder:
-            raise IOError, "end of file reached"
+            raise IOError("end of file reached")
         dst.write(buf)
     return
 
 filemode_table = (
-    (S_IFLNK, "l",
-     S_IFREG, "-",
-     S_IFBLK, "b",
-     S_IFDIR, "d",
-     S_IFCHR, "c",
-     S_IFIFO, "p"),
-    (TUREAD,  "r"),
-    (TUWRITE, "w"),
-    (TUEXEC,  "x", TSUID, "S", TUEXEC|TSUID, "s"),
-    (TGREAD,  "r"),
-    (TGWRITE, "w"),
-    (TGEXEC,  "x", TSGID, "S", TGEXEC|TSGID, "s"),
-    (TOREAD,  "r"),
-    (TOWRITE, "w"),
-    (TOEXEC,  "x", TSVTX, "T", TOEXEC|TSVTX, "t"))
+    ((S_IFLNK,      "l"),
+     (S_IFREG,      "-"),
+     (S_IFBLK,      "b"),
+     (S_IFDIR,      "d"),
+     (S_IFCHR,      "c"),
+     (S_IFIFO,      "p")),
+
+    ((TUREAD,       "r"),),
+    ((TUWRITE,      "w"),),
+    ((TUEXEC|TSUID, "s"),
+     (TSUID,        "S"),
+     (TUEXEC,       "x")),
+
+    ((TGREAD,       "r"),),
+    ((TGWRITE,      "w"),),
+    ((TGEXEC|TSGID, "s"),
+     (TSGID,        "S"),
+     (TGEXEC,       "x")),
+
+    ((TOREAD,       "r"),),
+    ((TOWRITE,      "w"),),
+    ((TOEXEC|TSVTX, "t"),
+     (TSVTX,        "T"),
+     (TOEXEC,       "x"))
+)
 
 def filemode(mode):
     """Convert a file's mode to a string of the form
        -rwxrwxrwx.
        Used by TarFile.list()
     """
-    s = ""
-    for t in filemode_table:
-        while 1:
-            if mode & t[0] == t[0]:
-                s += t[1]
-            elif len(t) > 2:
-                t = t[2:]
-                continue
-            else:
-                s += "-"
-            break
-    return s
-
-if os.sep != "/":
-    normpath = lambda path: os.path.normpath(path).replace(os.sep, "/")
-else:
-    normpath = os.path.normpath
+    perm = []
+    for table in filemode_table:
+        for bit, char in table:
+            if mode & bit == bit:
+                perm.append(char)
+                break
+        else:
+            perm.append("-")
+    return "".join(perm)
 
 class TarError(Exception):
-    """Internally used exception"""
-    pass
-
-#--------------------
-# exported functions
-#--------------------
-def open(name, mode="r", fileobj=None):
-    """Open (uncompressed) tar archive name for reading, writing
-       or appending.
-    """
-    return TarFile(name, mode, fileobj)
-
-def gzopen(gzname, gzmode="r", compresslevel=9, fileobj=None):
-    """Open gzip compressed tar archive name for reading or writing.
-       Appending is not allowed.
-    """
-    if gzmode == "a":
-        raise ValueError, "Appending to gzipped archive is not allowed"
-    import gzip
-    pre, ext = os.path.splitext(gzname)
-    pre = os.path.basename(pre)
-    if ext == ".tgz":
-        ext = ".tar"
-    if ext == ".gz":
-        ext = ""
-    tarname = pre + ext
-    mode = gzmode
-    if "b" not in gzmode:
-        gzmode += "b"
-    if mode[0:1] == "w":
-        if not fileobj:
-            fileobj = __builtin__.file(gzname, gzmode)
-        t = TarFile(tarname, mode, gzip.GzipFile(tarname, gzmode,
-                                                 compresslevel, fileobj))
-    else:
-        t = TarFile(tarname, mode, gzip.open(gzname, gzmode, compresslevel))
-    t._extfileobj = 0
-    return t
-
-def is_tarfile(name):
-    """Return True if name points to a tar archive that we
-       are able to handle, else return False.
-    """
-
-    buftoinfo = TarFile.__dict__["_buftoinfo"]
-    try:
-        buf = __builtin__.open(name, "rb").read(BLOCKSIZE)
-        buftoinfo(None, buf)
-        return True
-    except (ValueError, ImportError):
-        pass
-    try:
-        import gzip
-        buf = gzip.open(name, "rb").read(BLOCKSIZE)
-        buftoinfo(None, buf)
-        return True
-    except (IOError, ValueError, ImportError):
-        pass
-    return False
+    """Base exception."""
+    pass
+class ExtractError(TarError):
+    """General exception for extract errors."""
+    pass
+class ReadError(TarError):
+    """Exception for unreadble tar archives."""
+    pass
+class CompressionError(TarError):
+    """Exception for unavailable compression methods."""
+    pass
+class StreamError(TarError):
+    """Exception for unsupported operations on stream-like TarFiles."""
+    pass
+class HeaderError(TarError):
+    """Base exception for header errors."""
+    pass
+class EmptyHeaderError(HeaderError):
+    """Exception for empty headers."""
+    pass
+class TruncatedHeaderError(HeaderError):
+    """Exception for truncated headers."""
+    pass
+class EOFHeaderError(HeaderError):
+    """Exception for end of file headers."""
+    pass
+class InvalidHeaderError(HeaderError):
+    """Exception for invalid headers."""
+    pass
+class SubsequentHeaderError(HeaderError):
+    """Exception for missing and invalid extended headers."""
+    pass
+
+#---------------------------
+# internal stream interface
+#---------------------------
+class _LowLevelFile:
+    """Low-level file object. Supports reading and writing.
+       It is used instead of a regular file object for streaming
+       access.
+    """
+
+    def __init__(self, name, mode):
+        mode = {
+            "r": os.O_RDONLY,
+            "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
+        }[mode]
+        if hasattr(os, "O_BINARY"):
+            mode |= os.O_BINARY
+        self.fd = os.open(name, mode, 0666)
+
+    def close(self):
+        os.close(self.fd)
+
+    def read(self, size):
+        return os.read(self.fd, size)
+
+    def write(self, s):
+        os.write(self.fd, s)
+
+class _Stream:
+    """Class that serves as an adapter between TarFile and
+       a stream-like object.  The stream-like object only
+       needs to have a read() or write() method and is accessed
+       blockwise.  Use of gzip or bzip2 compression is possible.
+       A stream-like object could be for example: sys.stdin,
+       sys.stdout, a socket, a tape device etc.
+
+       _Stream is intended to be used only internally.
+    """
+
+    def __init__(self, name, mode, comptype, fileobj, bufsize):
+        """Construct a _Stream object.
+        """
+        self._extfileobj = True
+        if fileobj is None:
+            fileobj = _LowLevelFile(name, mode)
+            self._extfileobj = False
+
+        if comptype == '*':
+            # Enable transparent compression detection for the
+            # stream interface
+            fileobj = _StreamProxy(fileobj)
+            comptype = fileobj.getcomptype()
+
+        self.name     = name or ""
+        self.mode     = mode
+        self.comptype = comptype
+        self.fileobj  = fileobj
+        self.bufsize  = bufsize
+        self.buf      = ""
+        self.pos      = 0L
+        self.closed   = False
+
+        if comptype == "gz":
+            try:
+                import zlib
+            except ImportError:
+                raise CompressionError("zlib module is not available")
+            self.zlib = zlib
+            self.crc = zlib.crc32("") & 0xffffffffL
+            if mode == "r":
+                self._init_read_gz()
+            else:
+                self._init_write_gz()
+
+        if comptype == "bz2":
+            try:
+                import bz2
+            except ImportError:
+                raise CompressionError("bz2 module is not available")
+            if mode == "r":
+                self.dbuf = ""
+                self.cmp = bz2.BZ2Decompressor()
+            else:
+                self.cmp = bz2.BZ2Compressor()
+
+    def __del__(self):
+        if hasattr(self, "closed") and not self.closed:
+            self.close()
+
+    def _init_write_gz(self):
+        """Initialize for writing with gzip compression.
+        """
+        self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED,
+                                            -self.zlib.MAX_WBITS,
+                                            self.zlib.DEF_MEM_LEVEL,
+                                            0)
+        timestamp = struct.pack("<L", long(time.time()))
+        self.__write("\037\213\010\010%s\002\377" % timestamp)
+        if self.name.endswith(".gz"):
+            self.name = self.name[:-3]
+        self.__write(self.name + NUL)
+
+    def write(self, s):
+        """Write string s to the stream.
+        """
+        if self.comptype == "gz":
+            self.crc = self.zlib.crc32(s, self.crc) & 0xffffffffL
+        self.pos += len(s)
+        if self.comptype != "tar":
+            s = self.cmp.compress(s)
+        self.__write(s)
+
+    def __write(self, s):
+        """Write string s to the stream if a whole new block
+           is ready to be written.
+        """
+        self.buf += s
+        while len(self.buf) > self.bufsize:
+            self.fileobj.write(self.buf[:self.bufsize])
+            self.buf = self.buf[self.bufsize:]
+
+    def close(self):
+        """Close the _Stream object. No operation should be
+           done on it afterwards.
+        """
+        if self.closed:
+            return
+
+        if self.mode == "w" and self.comptype != "tar":
+            self.buf += self.cmp.flush()
+
+        if self.mode == "w" and self.buf:
+            self.fileobj.write(self.buf)
+            self.buf = ""
+            if self.comptype == "gz":
+                # The native zlib crc is an unsigned 32-bit integer, but
+                # the Python wrapper implicitly casts that to a signed C
+                # long.  So, on a 32-bit box self.crc may "look negative",
+                # while the same crc on a 64-bit box may "look positive".
+                # To avoid irksome warnings from the `struct` module, force
+                # it to look positive on all boxes.
+                self.fileobj.write(struct.pack("<L", self.crc & 0xffffffffL))
+                self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFFL))
+
+        if not self._extfileobj:
+            self.fileobj.close()
+
+        self.closed = True
+
+    def _init_read_gz(self):
+        """Initialize for reading a gzip compressed fileobj.
+        """
+        self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS)
+        self.dbuf = ""
+
+        # taken from gzip.GzipFile with some alterations
+        if self.__read(2) != "\037\213":
+            raise ReadError("not a gzip file")
+        if self.__read(1) != "\010":
+            raise CompressionError("unsupported compression method")
+
+        flag = ord(self.__read(1))
+        self.__read(6)
+
+        if flag & 4:
+            xlen = ord(self.__read(1)) + 256 * ord(self.__read(1))
+            self.read(xlen)
+        if flag & 8:
+            while True:
+                s = self.__read(1)
+                if not s or s == NUL:
+                    break
+        if flag & 16:
+            while True:
+                s = self.__read(1)
+                if not s or s == NUL:
+                    break
+        if flag & 2:
+            self.__read(2)
+
+    def tell(self):
+        """Return the stream's file pointer position.
+        """
+        return self.pos
+
+    def seek(self, pos=0):
+        """Set the stream's file pointer to pos. Negative seeking
+           is forbidden.
+        """
+        if pos - self.pos >= 0:
+            blocks, remainder = divmod(pos - self.pos, self.bufsize)
+            for i in xrange(blocks):
+                self.read(self.bufsize)
+            self.read(remainder)
+        else:
+            raise StreamError("seeking backwards is not allowed")
+        return self.pos
+
+    def read(self, size=None):
+        """Return the next size number of bytes from the stream.
+           If size is not defined, return all bytes of the stream
+           up to EOF.
+        """
+        if size is None:
+            t = []
+            while True:
+                buf = self._read(self.bufsize)
+                if not buf:
+                    break
+                t.append(buf)
+            buf = "".join(t)
+        else:
+            buf = self._read(size)
+        self.pos += len(buf)
+        return buf
+
+    def _read(self, size):
+        """Return size bytes from the stream.
+        """
+        if self.comptype == "tar":
+            return self.__read(size)
+
+        c = len(self.dbuf)
+        t = [self.dbuf]
+        while c < size:
+            buf = self.__read(self.bufsize)
+            if not buf:
+                break
+            try:
+                buf = self.cmp.decompress(buf)
+            except IOError:
+                raise ReadError("invalid compressed data")
+            t.append(buf)
+            c += len(buf)
+        t = "".join(t)
+        self.dbuf = t[size:]
+        return t[:size]
+
+    def __read(self, size):
+        """Return size bytes from stream. If internal buffer is empty,
+           read another block from the stream.
+        """
+        c = len(self.buf)
+        t = [self.buf]
+        while c < size:
+            buf = self.fileobj.read(self.bufsize)
+            if not buf:
+                break
+            t.append(buf)
+            c += len(buf)
+        t = "".join(t)
+        self.buf = t[size:]
+        return t[:size]
+# class _Stream
+
+class _StreamProxy(object):
+    """Small proxy class that enables transparent compression
+       detection for the Stream interface (mode 'r|*').
+    """
+
+    def __init__(self, fileobj):
+        self.fileobj = fileobj
+        self.buf = self.fileobj.read(BLOCKSIZE)
+
+    def read(self, size):
+        self.read = self.fileobj.read
+        return self.buf
+
+    def getcomptype(self):
+        if self.buf.startswith("\037\213\010"):
+            return "gz"
+        if self.buf.startswith("BZh91"):
+            return "bz2"
+        return "tar"
+
+    def close(self):
+        self.fileobj.close()
+# class StreamProxy
+
+class _BZ2Proxy(object):
+    """Small proxy class that enables external file object
+       support for "r:bz2" and "w:bz2" modes. This is actually
+       a workaround for a limitation in bz2 module's BZ2File
+       class which (unlike gzip.GzipFile) has no support for
+       a file object argument.
+    """
+
+    blocksize = 16 * 1024
+
+    def __init__(self, fileobj, mode):
+        self.fileobj = fileobj
+        self.mode = mode
+        self.name = getattr(self.fileobj, "name", None)
+        self.init()
+
+    def init(self):
+        import bz2
+        self.pos = 0
+        if self.mode == "r":
+            self.bz2obj = bz2.BZ2Decompressor()
+            self.fileobj.seek(0)
+            self.buf = ""
+        else:
+            self.bz2obj = bz2.BZ2Compressor()
+
+    def read(self, size):
+        b = [self.buf]
+        x = len(self.buf)
+        while x < size:
+            raw = self.fileobj.read(self.blocksize)
+            if not raw:
+                break
+            data = self.bz2obj.decompress(raw)
+            b.append(data)
+            x += len(data)
+        self.buf = "".join(b)
+
+        buf = self.buf[:size]
+        self.buf = self.buf[size:]
+        self.pos += len(buf)
+        return buf
+
+    def seek(self, pos):
+        if pos < self.pos:
+            self.init()
+        self.read(pos - self.pos)
+
+    def tell(self):
+        return self.pos
+
+    def write(self, data):
+        self.pos += len(data)
+        raw = self.bz2obj.compress(data)
+        self.fileobj.write(raw)
+
+    def close(self):
+        if self.mode == "w":
+            raw = self.bz2obj.flush()
+            self.fileobj.write(raw)
+# class _BZ2Proxy
+
+#------------------------
+# Extraction file object
+#------------------------
+class _FileInFile(object):
+    """A thin wrapper around an existing file object that
+       provides a part of its data as an individual file
+       object.
+    """
+
+    def __init__(self, fileobj, offset, size, sparse=None):
+        self.fileobj = fileobj
+        self.offset = offset
+        self.size = size
+        self.sparse = sparse
+        self.position = 0
+
+    def tell(self):
+        """Return the current file position.
+        """
+        return self.position
+
+    def seek(self, position):
+        """Seek to a position in the file.
+        """
+        self.position = position
+
+    def read(self, size=None):
+        """Read data from the file.
+        """
+        if size is None:
+            size = self.size - self.position
+        else:
+            size = min(size, self.size - self.position)
+
+        if self.sparse is None:
+            return self.readnormal(size)
+        else:
+            return self.readsparse(size)
+
+    def readnormal(self, size):
+        """Read operation for regular files.
+        """
+        self.fileobj.seek(self.offset + self.position)
+        self.position += size
+        return self.fileobj.read(size)
+
+    def readsparse(self, size):
+        """Read operation for sparse files.
+        """
+        data = []
+        while size > 0:
+            buf = self.readsparsesection(size)
+            if not buf:
+                break
+            size -= len(buf)
+            data.append(buf)
+        return "".join(data)
+
+    def readsparsesection(self, size):
+        """Read a single section of a sparse file.
+        """
+        section = self.sparse.find(self.position)
+
+        if section is None:
+            return ""
+
+        size = min(size, section.offset + section.size - self.position)
+
+        if isinstance(section, _data):
+            realpos = section.realpos + self.position - section.offset
+            self.fileobj.seek(self.offset + realpos)
+            self.position += size
+            return self.fileobj.read(size)
+        else:
+            self.position += size
+            return NUL * size
+#class _FileInFile
+
+
+class ExFileObject(object):
+    """File-like object for reading an archive member.
+       Is returned by TarFile.extractfile().
+    """
+    blocksize = 1024
+
+    def __init__(self, tarfile, tarinfo):
+        self.fileobj = _FileInFile(tarfile.fileobj,
+                                   tarinfo.offset_data,
+                                   tarinfo.size,
+                                   getattr(tarinfo, "sparse", None))
+        self.name = tarinfo.name
+        self.mode = "r"
+        self.closed = False
+        self.size = tarinfo.size
+
+        self.position = 0
+        self.buffer = ""
+
+    def read(self, size=None):
+        """Read at most size bytes from the file. If size is not
+           present or None, read all data until EOF is reached.
+        """
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+
+        buf = ""
+        if self.buffer:
+            if size is None:
+                buf = self.buffer
+                self.buffer = ""
+            else:
+                buf = self.buffer[:size]
+                self.buffer = self.buffer[size:]
+
+        if size is None:
+            buf += self.fileobj.read()
+        else:
+            buf += self.fileobj.read(size - len(buf))
+
+        self.position += len(buf)
+        return buf
+
+    def readline(self, size=-1):
+        """Read one entire line from the file. If size is present
+           and non-negative, return a string with at most that
+           size, which may be an incomplete line.
+        """
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+
+        if "\n" in self.buffer:
+            pos = self.buffer.find("\n") + 1
+        else:
+            buffers = [self.buffer]
+            while True:
+                buf = self.fileobj.read(self.blocksize)
+                buffers.append(buf)
+                if not buf or "\n" in buf:
+                    self.buffer = "".join(buffers)
+                    pos = self.buffer.find("\n") + 1
+                    if pos == 0:
+                        # no newline found.
+                        pos = len(self.buffer)
+                    break
+
+        if size != -1:
+            pos = min(size, pos)
+
+        buf = self.buffer[:pos]
+        self.buffer = self.buffer[pos:]
+        self.position += len(buf)
+        return buf
+
+    def readlines(self):
+        """Return a list with all remaining lines.
+        """
+        result = []
+        while True:
+            line = self.readline()
+            if not line: break
+            result.append(line)
+        return result
+
+    def tell(self):
+        """Return the current file position.
+        """
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+
+        return self.position
+
+    def seek(self, pos, whence=0):
+        """Seek to a position in the file.
+        """
+        if self.closed:
+            raise ValueError("I/O operation on closed file")
+
+        if whence == 0:
+            self.position = min(max(pos, 0), self.size)
+        elif whence == 1:
+            if pos < 0:
+                self.position = max(self.position + pos, 0)
+            else:
+                self.position = min(self.position + pos, self.size)
+        elif whence == 2:
+            self.position = max(min(self.size + pos, self.size), 0)
+        else:
+            raise ValueError("Invalid argument")
+
+        self.buffer = ""
+        self.fileobj.seek(self.position)
+
+    def close(self):
+        """Close the file object.
+        """
+        self.closed = True
+
+    def __iter__(self):
+        """Get an iterator over the file's lines.
+        """
+        while True:
+            line = self.readline()
+            if not line:
+                break
+            yield line
+#class ExFileObject
 
 #------------------
 # Exported Classes
 #------------------
-class TarInfo:
+class TarInfo(object):
     """Informational class which holds the details about an
        archive member given by a tar header block.
-       TarInfo instances are returned by TarFile.getmember() and
-       TarFile.getmembers() and are usually created internally.
-       If you want to create a TarInfo instance from the outside,
-       you should use TarFile.gettarinfo() if the file already exists,
-       or you can instanciate the class yourself.
+       TarInfo objects are returned by TarFile.getmember(),
+       TarFile.getmembers() and TarFile.gettarinfo() and are
+       usually created internally.
     """
 
     def __init__(self, name=""):
-        """Construct a TarInfo instance. name is the optional name
+        """Construct a TarInfo object. name is the optional name
            of the member.
         """
-
-        self.name     = name       # member name (dirnames must end with '/')
-        self.mode     = 0100666    # file permissions
-        self.uid      = 0          # user id
-        self.gid      = 0          # group id
-        self.size     = 0          # file size
-        self.mtime    = 0          # modification time
-        self.chksum   = 0          # header checksum
-        self.type     = REGTYPE    # member type
-        self.linkname = ""         # link name
-        self.uname    = "user"     # user name
-        self.gname    = "group"    # group name
-        self.devmajor = 0          #-
-        self.devminor = 0          #-for use with CHRTYPE and BLKTYPE
-        self.prefix   = ""         # prefix, holding information
-#                                  # about sparse files
-
-        self.offset   = 0          # the tar header starts here
-        self.offset_data = 0       # the optional file's data starts here
-
-    def init_from_stat(self, statres):
-        """Initialize various attributes from statobj (these are
-        returned by os.stat() and related functions.  Return none on error"""
-        stmd = statres.st_mode
-        if stat.S_ISREG(stmd): type = REGTYPE
-        elif stat.S_ISDIR(stmd):
-            type = DIRTYPE
-            if self.name[-1:] != "/": self.name += "/"
-        elif stat.S_ISFIFO(stmd): type = FIFOTYPE
-        elif stat.S_ISLNK(stmd): type = SYMTYPE
-        elif stat.S_ISCHR(stmd): type = CHRTYPE
-        elif stat.S_ISBLK(stmd): type = BLKTYPE
-        else: return None
-
-        # Fill the TarInfo instance with all
-        # information we can get.
-        self.mode  = stat.S_IMODE(stmd)
-        self.uid   = statres.st_uid
-        self.gid   = statres.st_gid
-        self.size  = statres.st_size
-        self.mtime = statres.st_mtime
-        self.type  = type
-        if pwd:
-            try: self.uname = uid2uname(self.uid)
-            except KeyError: pass
-        if grp:
-            try: self.gname = gid2gname(self.gid)
-            except KeyError: pass
-
-        if type in (CHRTYPE, BLKTYPE):
-            if hasattr(os, "major") and hasattr(os, "minor"):
-                self.devmajor = os.major(statres.st_rdev)
-                self.devminor = os.minor(statres.st_rdev)
-        return 1
-
-    def set_arcname(self, name):
-        """Set the name of the member in the archive.  Backward
-        slashes are converted to forward slashes, Absolute paths are
-        turned to relative paths.
-        """
-        arcname = normpath(name)
-        drv, arcname = os.path.splitdrive(arcname) #@UnusedVariable
-        while arcname[0:1] == "/":
-            arcname = arcname[1:]
-        self.name = arcname
-
-    def getheader(self):
-        """Return a tar header block as a 512 byte string.
-        """
-        if self.uid > 2097151 or self.uid < 0:
-            sys.stderr.write("uid %i of file %s not in range. Setting uid to 60001\n" % (self.uid,self.name))
-            self.uid = 60001
-        if self.gid > 2097151 or self.gid < 0:
-            sys.stderr.write("gid %i of file %s not in range. Setting gid to 60001\n" % (self.gid, self.name))
-            self.gid = 60001
-        # The following code was contributed by Detlef Lannert.
-        parts = []
-        for value, fieldsize in (
-                (self.name, 100),
-                ("%07o" % self.mode, 8),
-                ("%07o" % self.uid, 8),
-                ("%07o" % self.gid, 8),
-                ("%011o" % self.size, 12),
-                ("%011o" % self.mtime, 12),
-                ("        ", 8),
-                (self.type, 1),
-                (self.linkname, 100),
-                (MAGIC, 6),
-                (VERSION, 2),
-                (self.uname, 32),
-                (self.gname, 32),
-                ("%07o" % self.devmajor, 8),
-                ("%07o" % self.devminor, 8),
-                (self.prefix, 155)
-                ):
-            l = len(value)
-            parts.append(value + (fieldsize - l) * "\0")
-
-        buf = "".join(parts)
-        chksum = calc_chksum(buf)
-        buf = buf[:148] + "%06o\0" % chksum + buf[155:]
-        buf += (512 - len(buf)) * "\0"
-        self.buf = buf
+        self.name = name        # member name
+        self.mode = 0644        # file permissions
+        self.uid = 0            # user id
+        self.gid = 0            # group id
+        self.size = 0           # file size
+        self.mtime = 0          # modification time
+        self.chksum = 0         # header checksum
+        self.type = REGTYPE     # member type
+        self.linkname = ""      # link name
+        self.uname = ""         # user name
+        self.gname = ""         # group name
+        self.devmajor = 0       # device major number
+        self.devminor = 0       # device minor number
+
+        self.offset = 0         # the tar header starts here
+        self.offset_data = 0    # the file's data starts here
+
+        self.pax_headers = {}   # pax header information
+
+    # In pax headers the "name" and "linkname" field are called
+    # "path" and "linkpath".
+    def _getpath(self):
+        return self.name
+    def _setpath(self, name):
+        self.name = name
+    path = property(_getpath, _setpath)
+
+    def _getlinkpath(self):
+        return self.linkname
+    def _setlinkpath(self, linkname):
+        self.linkname = linkname
+    linkpath = property(_getlinkpath, _setlinkpath)
+
+    def __repr__(self):
+        return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self))
+
+    def get_info(self, encoding, errors):
+        """Return the TarInfo's attributes as a dictionary.
+        """
+        info = {
+            "name":     self.name,
+            "mode":     self.mode & 07777,
+            "uid":      self.uid,
+            "gid":      self.gid,
+            "size":     self.size,
+            "mtime":    self.mtime,
+            "chksum":   self.chksum,
+            "type":     self.type,
+            "linkname": self.linkname,
+            "uname":    self.uname,
+            "gname":    self.gname,
+            "devmajor": self.devmajor,
+            "devminor": self.devminor
+        }
+
+        if info["type"] == DIRTYPE and not info["name"].endswith("/"):
+            info["name"] += "/"
+
+        for key in ("name", "linkname", "uname", "gname"):
+            if type(info[key]) is unicode:
+                info[key] = info[key].encode(encoding, errors)
+
+        return info
+
+    def tobuf(self, format=DEFAULT_FORMAT, encoding=ENCODING, errors="strict"):
+        """Return a tar header as a string of 512 byte blocks.
+        """
+        info = self.get_info(encoding, errors)
+
+        if format == USTAR_FORMAT:
+            return self.create_ustar_header(info)
+        elif format == GNU_FORMAT:
+            return self.create_gnu_header(info)
+        elif format == PAX_FORMAT:
+            return self.create_pax_header(info, encoding, errors)
+        else:
+            raise ValueError("invalid format")
+
+    def create_ustar_header(self, info):
+        """Return the object as a ustar header block.
+        """
+        info["magic"] = POSIX_MAGIC
+
+        if len(info["linkname"]) > LENGTH_LINK:
+            raise ValueError("linkname is too long")
+
+        if len(info["name"]) > LENGTH_NAME:
+            info["prefix"], info["name"] = self._posix_split_name(info["name"])
+
+        return self._create_header(info, USTAR_FORMAT)
+
+    def create_gnu_header(self, info):
+        """Return the object as a GNU header block sequence.
+        """
+        info["magic"] = GNU_MAGIC
+
+        buf = ""
+        if len(info["linkname"]) > LENGTH_LINK:
+            buf += self._create_gnu_long_header(info["linkname"], GNUTYPE_LONGLINK)
+
+        if len(info["name"]) > LENGTH_NAME:
+            buf += self._create_gnu_long_header(info["name"], GNUTYPE_LONGNAME)
+
+        return buf + self._create_header(info, GNU_FORMAT)
+
+    def create_pax_header(self, info, encoding, errors):
+        """Return the object as a ustar header block. If it cannot be
+           represented this way, prepend a pax extended header sequence
+           with supplement information.
+        """
+        info["magic"] = POSIX_MAGIC
+        pax_headers = self.pax_headers.copy()
+
+        # Test string fields for values that exceed the field length or cannot
+        # be represented in ASCII encoding.
+        for name, hname, length in (
+                ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK),
+                ("uname", "uname", 32), ("gname", "gname", 32)):
+
+            if hname in pax_headers:
+                # The pax header has priority.
+                continue
+
+            val = info[name].decode(encoding, errors)
+
+            # Try to encode the string as ASCII.
+            try:
+                val.encode("ascii")
+            except UnicodeEncodeError:
+                pax_headers[hname] = val
+                continue
+
+            if len(info[name]) > length:
+                pax_headers[hname] = val
+
+        # Test number fields for values that exceed the field limit or values
+        # that like to be stored as float.
+        for name, digits in (("uid", 8), ("gid", 8), ("size", 12), ("mtime", 12)):
+            if name in pax_headers:
+                # The pax header has priority. Avoid overflow.
+                info[name] = 0
+                continue
+
+            val = info[name]
+            if not 0 <= val < 8 ** (digits - 1) or isinstance(val, float):
+                pax_headers[name] = unicode(val)
+                info[name] = 0
+
+        # Create a pax extended header if necessary.
+        if pax_headers:
+            buf = self._create_pax_generic_header(pax_headers)
+        else:
+            buf = ""
+
+        return buf + self._create_header(info, USTAR_FORMAT)
+
+    @classmethod
+    def create_pax_global_header(cls, pax_headers):
+        """Return the object as a pax global header block sequence.
+        """
+        return cls._create_pax_generic_header(pax_headers, type=XGLTYPE)
+
+    def _posix_split_name(self, name):
+        """Split a name longer than 100 chars into a prefix
+           and a name part.
+        """
+        prefix = name[:LENGTH_PREFIX + 1]
+        while prefix and prefix[-1] != "/":
+            prefix = prefix[:-1]
+
+        name = name[len(prefix):]
+        prefix = prefix[:-1]
+
+        if not prefix or len(name) > LENGTH_NAME:
+            raise ValueError("name is too long")
+        return prefix, name
+
+    @staticmethod
+    def _create_header(info, format):
+        """Return a header block. info is a dictionary with file
+           information, format must be one of the *_FORMAT constants.
+        """
+        parts = [
+            stn(info.get("name", ""), 100),
+            itn(info.get("mode", 0) & 07777, 8, format),
+            itn(info.get("uid", 0), 8, format),
+            itn(info.get("gid", 0), 8, format),
+            itn(info.get("size", 0), 12, format),
+            itn(info.get("mtime", 0), 12, format),
+            "        ", # checksum field
+            info.get("type", REGTYPE),
+            stn(info.get("linkname", ""), 100),
+            stn(info.get("magic", POSIX_MAGIC), 8),
+            stn(info.get("uname", ""), 32),
+            stn(info.get("gname", ""), 32),
+            itn(info.get("devmajor", 0), 8, format),
+            itn(info.get("devminor", 0), 8, format),
+            stn(info.get("prefix", ""), 155)
+        ]
+
+        buf = struct.pack("%ds" % BLOCKSIZE, "".join(parts))
+        chksum = calc_chksums(buf[-BLOCKSIZE:])[0]
+        buf = buf[:-364] + "%06o\0" % chksum + buf[-357:]
         return buf
 
+    @staticmethod
+    def _create_payload(payload):
+        """Return the string payload filled with zero bytes
+           up to the next 512 byte border.
+        """
+        blocks, remainder = divmod(len(payload), BLOCKSIZE)
+        if remainder > 0:
+            payload += (BLOCKSIZE - remainder) * NUL
+        return payload
+
+    @classmethod
+    def _create_gnu_long_header(cls, name, type):
+        """Return a GNUTYPE_LONGNAME or GNUTYPE_LONGLINK sequence
+           for name.
+        """
+        name += NUL
+
+        info = {}
+        info["name"] = "././@LongLink"
+        info["type"] = type
+        info["size"] = len(name)
+        info["magic"] = GNU_MAGIC
+
+        # create extended header + name blocks.
+        return cls._create_header(info, USTAR_FORMAT) + \
+                cls._create_payload(name)
+
+    @classmethod
+    def _create_pax_generic_header(cls, pax_headers, type=XHDTYPE):
+        """Return a POSIX.1-2001 extended or global header sequence
+           that contains a list of keyword, value pairs. The values
+           must be unicode objects.
+        """
+        records = []
+        for keyword, value in pax_headers.iteritems():
+            keyword = keyword.encode("utf8")
+            value = value.encode("utf8")
+            l = len(keyword) + len(value) + 3   # ' ' + '=' + '\n'
+            n = p = 0
+            while True:
+                n = l + len(str(p))
+                if n == p:
+                    break
+                p = n
+            records.append("%d %s=%s\n" % (p, keyword, value))
+        records = "".join(records)
+
+        # We use a hardcoded "././@PaxHeader" name like star does
+        # instead of the one that POSIX recommends.
+        info = {}
+        info["name"] = "././@PaxHeader"
+        info["type"] = type
+        info["size"] = len(records)
+        info["magic"] = POSIX_MAGIC
+
+        # Create pax header + record blocks.
+        return cls._create_header(info, USTAR_FORMAT) + \
+                cls._create_payload(records)
+
+    @classmethod
+    def frombuf(cls, buf):
+        """Construct a TarInfo object from a 512 byte string buffer.
+        """
+        if len(buf) == 0:
+            raise EmptyHeaderError("empty header")
+        if len(buf) != BLOCKSIZE:
+            raise TruncatedHeaderError("truncated header")
+        if buf.count(NUL) == BLOCKSIZE:
+            raise EOFHeaderError("end of file header")
+
+        chksum = nti(buf[148:156])
+        if chksum not in calc_chksums(buf):
+            raise InvalidHeaderError("bad checksum")
+
+        obj = cls()
+        obj.buf = buf
+        obj.name = nts(buf[0:100])
+        obj.mode = nti(buf[100:108])
+        obj.uid = nti(buf[108:116])
+        obj.gid = nti(buf[116:124])
+        obj.size = nti(buf[124:136])
+        obj.mtime = nti(buf[136:148])
+        obj.chksum = chksum
+        obj.type = buf[156:157]
+        obj.linkname = nts(buf[157:257])
+        obj.uname = nts(buf[265:297])
+        obj.gname = nts(buf[297:329])
+        obj.devmajor = nti(buf[329:337])
+        obj.devminor = nti(buf[337:345])
+        prefix = nts(buf[345:500])
+
+        # Old V7 tar format represents a directory as a regular
+        # file with a trailing slash.
+        if obj.type == AREGTYPE and obj.name.endswith("/"):
+            obj.type = DIRTYPE
+
+        # Remove redundant slashes from directories.
+        if obj.isdir():
+            obj.name = obj.name.rstrip("/")
+
+        # Reconstruct a ustar longname.
+        if prefix and obj.type not in GNU_TYPES:
+            obj.name = prefix + "/" + obj.name
+        return obj
+
+    @classmethod
+    def fromtarfile(cls, tarfile):
+        """Return the next TarInfo object from TarFile object
+           tarfile.
+        """
+        buf = tarfile.fileobj.read(BLOCKSIZE)
+        obj = cls.frombuf(buf)
+        obj.offset = tarfile.fileobj.tell() - BLOCKSIZE
+        return obj._proc_member(tarfile)
+
+    #--------------------------------------------------------------------------
+    # The following are methods that are called depending on the type of a
+    # member. The entry point is _proc_member() which can be overridden in a
+    # subclass to add custom _proc_*() methods. A _proc_*() method MUST
+    # implement the following
+    # operations:
+    # 1. Set self.offset_data to the position where the data blocks begin,
+    #    if there is data that follows.
+    # 2. Set tarfile.offset to the position where the next member's header will
+    #    begin.
+    # 3. Return self or another valid TarInfo object.
+    def _proc_member(self, tarfile):
+        """Choose the right processing method depending on
+           the type and call it.
+        """
+        if self.type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK):
+            return self._proc_gnulong(tarfile)
+        elif self.type == GNUTYPE_SPARSE:
+            return self._proc_sparse(tarfile)
+        elif self.type in (XHDTYPE, XGLTYPE, SOLARIS_XHDTYPE):
+            return self._proc_pax(tarfile)
+        else:
+            return self._proc_builtin(tarfile)
+
+    def _proc_builtin(self, tarfile):
+        """Process a builtin type or an unknown type which
+           will be treated as a regular file.
+        """
+        self.offset_data = tarfile.fileobj.tell()
+        offset = self.offset_data
+        if self.isreg() or self.type not in SUPPORTED_TYPES:
+            # Skip the following data blocks.
+            offset += self._block(self.size)
+        tarfile.offset = offset
+
+        # Patch the TarInfo object with saved global
+        # header information.
+        self._apply_pax_info(tarfile.pax_headers, tarfile.encoding, tarfile.errors)
+
+        return self
+
+    def _proc_gnulong(self, tarfile):
+        """Process the blocks that hold a GNU longname
+           or longlink member.
+        """
+        buf = tarfile.fileobj.read(self._block(self.size))
+
+        # Fetch the next header and process it.
+        try:
+            next = self.fromtarfile(tarfile)
+        except HeaderError:
+            raise SubsequentHeaderError("missing or bad subsequent header")
+
+        # Patch the TarInfo object from the next header with
+        # the longname information.
+        next.offset = self.offset
+        if self.type == GNUTYPE_LONGNAME:
+            next.name = nts(buf)
+        elif self.type == GNUTYPE_LONGLINK:
+            next.linkname = nts(buf)
+
+        return next
+
+    def _proc_sparse(self, tarfile):
+        """Process a GNU sparse header plus extra headers.
+        """
+        buf = self.buf
+        sp = _ringbuffer()
+        pos = 386
+        lastpos = 0L
+        realpos = 0L
+        # There are 4 possible sparse structs in the
+        # first header.
+        for i in xrange(4):
+            try:
+                offset = nti(buf[pos:pos + 12])
+                numbytes = nti(buf[pos + 12:pos + 24])
+            except ValueError:
+                break
+            if offset > lastpos:
+                sp.append(_hole(lastpos, offset - lastpos))
+            sp.append(_data(offset, numbytes, realpos))
+            realpos += numbytes
+            lastpos = offset + numbytes
+            pos += 24
+
+        isextended = ord(buf[482])
+        origsize = nti(buf[483:495])
+
+        # If the isextended flag is given,
+        # there are extra headers to process.
+        while isextended == 1:
+            buf = tarfile.fileobj.read(BLOCKSIZE)
+            pos = 0
+            for i in xrange(21):
+                try:
+                    offset = nti(buf[pos:pos + 12])
+                    numbytes = nti(buf[pos + 12:pos + 24])
+                except ValueError:
+                    break
+                if offset > lastpos:
+                    sp.append(_hole(lastpos, offset - lastpos))
+                sp.append(_data(offset, numbytes, realpos))
+                realpos += numbytes
+                lastpos = offset + numbytes
+                pos += 24
+            isextended = ord(buf[504])
+
+        if lastpos < origsize:
+            sp.append(_hole(lastpos, origsize - lastpos))
+
+        self.sparse = sp
+
+        self.offset_data = tarfile.fileobj.tell()
+        tarfile.offset = self.offset_data + self._block(self.size)
+        self.size = origsize
+
+        return self
+
+    def _proc_pax(self, tarfile):
+        """Process an extended or global header as described in
+           POSIX.1-2001.
+        """
+        # Read the header information.
+        buf = tarfile.fileobj.read(self._block(self.size))
+
+        # A pax header stores supplemental information for either
+        # the following file (extended) or all following files
+        # (global).
+        if self.type == XGLTYPE:
+            pax_headers = tarfile.pax_headers
+        else:
+            pax_headers = tarfile.pax_headers.copy()
+
+        # Parse pax header information. A record looks like that:
+        # "%d %s=%s\n" % (length, keyword, value). length is the size
+        # of the complete record including the length field itself and
+        # the newline. keyword and value are both UTF-8 encoded strings.
+        regex = re.compile(r"(\d+) ([^=]+)=", re.U)
+        pos = 0
+        while True:
+            match = regex.match(buf, pos)
+            if not match:
+                break
+
+            length, keyword = match.groups()
+            length = int(length)
+            value = buf[match.end(2) + 1:match.start(1) + length - 1]
+
+            keyword = keyword.decode("utf8")
+            value = value.decode("utf8")
+
+            pax_headers[keyword] = value
+            pos += length
+
+        # Fetch the next header.
+        try:
+            next = self.fromtarfile(tarfile)
+        except HeaderError:
+            raise SubsequentHeaderError("missing or bad subsequent header")
+
+        if self.type in (XHDTYPE, SOLARIS_XHDTYPE):
+            # Patch the TarInfo object with the extended header info.
+            next._apply_pax_info(pax_headers, tarfile.encoding, tarfile.errors)
+            next.offset = self.offset
+
+            if "size" in pax_headers:
+                # If the extended header replaces the size field,
+                # we need to recalculate the offset where the next
+                # header starts.
+                offset = next.offset_data
+                if next.isreg() or next.type not in SUPPORTED_TYPES:
+                    offset += next._block(next.size)
+                tarfile.offset = offset
+
+        return next
+
+    def _apply_pax_info(self, pax_headers, encoding, errors):
+        """Replace fields with supplemental information from a previous
+           pax extended or global header.
+        """
+        for keyword, value in pax_headers.iteritems():
+            if keyword not in PAX_FIELDS:
+                continue
+
+            if keyword == "path":
+                value = value.rstrip("/")
+
+            if keyword in PAX_NUMBER_FIELDS:
+                try:
+                    value = PAX_NUMBER_FIELDS[keyword](value)
+                except ValueError:
+                    value = 0
+            else:
+                value = uts(value, encoding, errors)
+
+            setattr(self, keyword, value)
+
+        self.pax_headers = pax_headers.copy()
+
+    def _block(self, count):
+        """Round up a byte count by BLOCKSIZE and return it,
+           e.g. _block(834) => 1024.
+        """
+        blocks, remainder = divmod(count, BLOCKSIZE)
+        if remainder:
+            blocks += 1
+        return blocks * BLOCKSIZE
+
     def isreg(self):
         return self.type in REGULAR_TYPES
     def isfile(self):
@@ -444,10 +1470,10 @@
         return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE)
 # class TarInfo
 
-
-class TarFile:
-    """Class representing a TAR archive file on disk.
+class TarFile(object):
+    """The TarFile Class provides an interface to tar archives.
     """
+
     debug = 0                   # May be set from 0 (no msgs) to 3 (all msgs)
 
     dereference = False         # If true, add content of linked file to the
@@ -456,235 +1482,459 @@
     ignore_zeros = False        # If true, skips empty or invalid blocks and
                                 # continues processing.
 
-    errorlevel = 0              # If 0, fatal errors only appear in debug
+    errorlevel = 1              # If 0, fatal errors only appear in debug
                                 # messages (if debug >= 0). If > 0, errors
                                 # are passed to the caller as exceptions.
 
-    def __init__(self, name=None, mode="r", fileobj=None):
-        self.name = name
-
+    format = DEFAULT_FORMAT     # The format to use when creating an archive.
+
+    encoding = ENCODING         # Encoding for 8-bit character strings.
+
+    errors = None               # Error handler for unicode conversion.
+
+    tarinfo = TarInfo           # The default TarInfo class to use.
+
+    fileobject = ExFileObject   # The default ExFileObject class to use.
+
+    def __init__(self, name=None, mode="r", fileobj=None, format=None,
+            tarinfo=None, dereference=None, ignore_zeros=None, encoding=None,
+            errors=None, pax_headers=None, debug=None, errorlevel=None):
+        """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to
+           read from an existing archive, 'a' to append data to an existing
+           file or 'w' to create a new file overwriting an existing one. `mode'
+           defaults to 'r'.
+           If `fileobj' is given, it is used for reading or writing data. If it
+           can be determined, `mode' is overridden by `fileobj's mode.
+           `fileobj' is not closed, when TarFile is closed.
+        """
         if len(mode) > 1 or mode not in "raw":
-            raise ValueError, "mode must be either 'r', 'a' or 'w', " \
-                                "not '%s'" % mode
-        self._mode = mode
-        self.mode = {"r": "rb", "a": "r+b", "w": "wb"}[mode]
+            raise ValueError("mode must be 'r', 'a' or 'w'")
+        self.mode = mode
+        self._mode = {"r": "rb", "a": "r+b", "w": "wb"}[mode]
 
         if not fileobj:
-            fileobj = __builtin__.file(self.name, self.mode)
-            self._extfileobj = 0
+            if self.mode == "a" and not os.path.exists(name):
+                # Create nonexistent files in append mode.
+                self.mode = "w"
+                self._mode = "wb"
+            fileobj = bltn_open(name, self._mode)
+            self._extfileobj = False
         else:
-            if self.name is None and hasattr(fileobj, "name"):
-                self.name = fileobj.name
+            if name is None and hasattr(fileobj, "name"):
+                name = fileobj.name
             if hasattr(fileobj, "mode"):
-                self.mode = fileobj.mode
-            self._extfileobj = 1
+                self._mode = fileobj.mode
+            self._extfileobj = True
+				if name:
+						self.name = os.path.abspath(name)
+				else:
+						self.name = None
         self.fileobj = fileobj
 
-        self.init_datastructures()
-
-        if self._mode == "a":
-            self.fileobj.seek(0)
-            self._load()
-
-    def init_datastructures(self):
-        # Init datastructures
-        #self.members     = []       # list of members as TarInfo instances
-        #self.membernames = []       # names of members
-        #self.chunks      = [0]      # chunk cache
-        self._loaded     = 0        # flag if all members have been read
-        self.offset      = 0l       # current position in the archive file
-        self.inodes      = {}       # dictionary caching the inodes of
-                                    # archive members already added
-        self.next_chunk = 0 # offset of next tarinfo, used when reading
-
-    def close(self):
-        """Close the TarFile instance and do some cleanup.
-        """
-        if self.fileobj:
-            if self._mode in "aw":
-                # fill up the end with zero-blocks
-                # (like option -b20 for tar does)
-                blocks, remainder = divmod(self.offset, RECORDSIZE) #@UnusedVariable
-                if remainder > 0:
-                    self.fileobj.write("\0" * (RECORDSIZE - remainder))
-
+        # Init attributes.
+        if format is not None:
+            self.format = format
+        if tarinfo is not None:
+            self.tarinfo = tarinfo
+        if dereference is not None:
+            self.dereference = dereference
+        if ignore_zeros is not None:
+            self.ignore_zeros = ignore_zeros
+        if encoding is not None:
+            self.encoding = encoding
+
+        if errors is not None:
+            self.errors = errors
+        elif mode == "r":
+            self.errors = "utf-8"
+        else:
+            self.errors = "strict"
+
+        if pax_headers is not None and self.format == PAX_FORMAT:
+            self.pax_headers = pax_headers
+        else:
+            self.pax_headers = {}
+
+        if debug is not None:
+            self.debug = debug
+        if errorlevel is not None:
+            self.errorlevel = errorlevel
+
+        # Init datastructures.
+        self.closed = False
+        self.members = []       # list of members as TarInfo objects
+        self._loaded = False    # flag if all members have been read
+        self.offset = self.fileobj.tell()
+                                # current position in the archive file
+        self.inodes = {}        # dictionary caching the inodes of
+                                # archive members already added
+
+        try:
+            if self.mode == "r":
+                self.firstmember = None
+                self.firstmember = self.next()
+
+            if self.mode == "a":
+                # Move to the end of the archive,
+                # before the first empty block.
+                while True:
+                    self.fileobj.seek(self.offset)
+                    try:
+                        tarinfo = self.tarinfo.fromtarfile(self)
+                        self.members.append(tarinfo)
+                    except EOFHeaderError:
+                        self.fileobj.seek(self.offset)
+                        break
+                    except HeaderError, e:
+                        raise ReadError(str(e))
+
+            if self.mode in "aw":
+                self._loaded = True
+
+                if self.pax_headers:
+                    buf = self.tarinfo.create_pax_global_header(self.pax_headers.copy())
+                    self.fileobj.write(buf)
+                    self.offset += len(buf)
+        except:
             if not self._extfileobj:
                 self.fileobj.close()
-            self.fileobj = None
-
-    def throwaway_until(self, position):
-        """Read data, throwing it away until we get to position"""
-        bufsize = 16 * 1024
-        bytes_to_read = position - self.offset
-        assert bytes_to_read >= 0
-        while bytes_to_read >= bufsize:
-            self.fileobj.read(bufsize)
-            bytes_to_read -= bufsize
-        self.fileobj.read(bytes_to_read)
-        self.offset = position
-
-    def next(self):
-        """Return the next member from the archive.
-           Return None if the end is reached.
-           Can be used in a while statement, is used
-           for Iteration (see __iter__()) and internally.
+            self.closed = True
+            raise
+
+    def _getposix(self):
+        return self.format == USTAR_FORMAT
+    def _setposix(self, value):
+        import warnings
+        warnings.warn("use the format attribute instead", DeprecationWarning,
+                      2)
+        if value:
+            self.format = USTAR_FORMAT
+        else:
+            self.format = GNU_FORMAT
+    posix = property(_getposix, _setposix)
+
+    #--------------------------------------------------------------------------
+    # Below are the classmethods which act as alternate constructors to the
+    # TarFile class. The open() method is the only one that is needed for
+    # public use; it is the "super"-constructor and is able to select an
+    # adequate "sub"-constructor for a particular compression using the mapping
+    # from OPEN_METH.
+    #
+    # This concept allows one to subclass TarFile without losing the comfort of
+    # the super-constructor. A sub-constructor is registered and made available
+    # by adding it to the mapping in OPEN_METH.
+
+    @classmethod
+    def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs):
+        """Open a tar archive for reading, writing or appending. Return
+           an appropriate TarFile class.
+
+           mode:
+           'r' or 'r:*' open for reading with transparent compression
+           'r:'         open for reading exclusively uncompressed
+           'r:gz'       open for reading with gzip compression
+           'r:bz2'      open for reading with bzip2 compression
+           'a' or 'a:'  open for appending, creating the file if necessary
+           'w' or 'w:'  open for writing without compression
+           'w:gz'       open for writing with gzip compression
+           'w:bz2'      open for writing with bzip2 compression
+
+           'r|*'        open a stream of tar blocks with transparent compression
+           'r|'         open an uncompressed stream of tar blocks for reading
+           'r|gz'       open a gzip compressed stream of tar blocks
+           'r|bz2'      open a bzip2 compressed stream of tar blocks
+           'w|'         open an uncompressed stream for writing
+           'w|gz'       open a gzip compressed stream for writing
+           'w|bz2'      open a bzip2 compressed stream for writing
         """
-        if not self.fileobj:
-            raise ValueError, "I/O operation on closed file"
-        if self._mode not in "ra":
-            raise ValueError, "reading from a write-mode file"
-
-        # Read the next block.
-        # self.fileobj.seek(self.chunks[-1])
-        #self.fileobj.seek(self.next_chunk)
-        #self.offset = self.next_chunk
-        self.throwaway_until(self.next_chunk)
-        while 1:
-            buf = self.fileobj.read(BLOCKSIZE)
-            if not buf:
-                return None
-            try:
-                tarinfo = self._buftoinfo(buf)
-            except ValueError:
-                if self.ignore_zeros:
-                    if buf.count("\0") == BLOCKSIZE:
-                        adj = "empty"
-                    else:
-                        adj = "invalid"
-                    self._dbg(2, "0x%X: %s block\n" % (self.offset, adj))
-                    self.offset += BLOCKSIZE
+
+        if not name and not fileobj:
+            raise ValueError("nothing to open")
+
+        if mode in ("r", "r:*"):
+            # Find out which *open() is appropriate for opening the file.
+            for comptype in cls.OPEN_METH:
+                func = getattr(cls, cls.OPEN_METH[comptype])
+                if fileobj is not None:
+                    saved_pos = fileobj.tell()
+                try:
+                    return func(name, "r", fileobj, **kwargs)
+                except (ReadError, CompressionError), e:
+                    if fileobj is not None:
+                        fileobj.seek(saved_pos)
                     continue
-                else:
-                    return None
-            break
-
-        # If the TarInfo instance contains a GNUTYPE longname or longlink
-        # statement, we must process this first.
-        if tarinfo.type in (GNUTYPE_LONGLINK, GNUTYPE_LONGNAME):
-            tarinfo = self._proc_gnulong(tarinfo, tarinfo.type)
-            if not tarinfo:
-                return None
-
-        if tarinfo.issparse():
-            assert 0, "Sparse file support turned off"
-            # Sparse files need some care,
-            # due to the possible extra headers.
-            tarinfo.offset = self.offset
-            self.offset += BLOCKSIZE
-            origsize = self._proc_sparse(tarinfo)
-            tarinfo.offset_data = self.offset
-            blocks, remainder = divmod(tarinfo.size, BLOCKSIZE)
-            if remainder:
-                blocks += 1
-            self.offset += blocks * BLOCKSIZE
-            tarinfo.size = origsize
+            raise ReadError("file could not be opened successfully")
+
+        elif ":" in mode:
+            filemode, comptype = mode.split(":", 1)
+            filemode = filemode or "r"
+            comptype = comptype or "tar"
+
+            # Select the *open() function according to
+            # given compression.
+            if comptype in cls.OPEN_METH:
+                func = getattr(cls, cls.OPEN_METH[comptype])
+            else:
+                raise CompressionError("unknown compression type %r" % comptype)
+            return func(name, filemode, fileobj, **kwargs)
+
+        elif "|" in mode:
+            filemode, comptype = mode.split("|", 1)
+            filemode = filemode or "r"
+            comptype = comptype or "tar"
+
+            if filemode not in "rw":
+                raise ValueError("mode must be 'r' or 'w'")
+
+            t = cls(name, filemode,
+                    _Stream(name, filemode, comptype, fileobj, bufsize),
+                    **kwargs)
+            t._extfileobj = False
+            return t
+
+        elif mode in "aw":
+            return cls.taropen(name, mode, fileobj, **kwargs)
+
+        raise ValueError("undiscernible mode")
+
+    @classmethod
+    def taropen(cls, name, mode="r", fileobj=None, **kwargs):
+        """Open uncompressed tar archive name for reading or writing.
+        """
+        if len(mode) > 1 or mode not in "raw":
+            raise ValueError("mode must be 'r', 'a' or 'w'")
+        return cls(name, mode, fileobj, **kwargs)
+
+    @classmethod
+    def gzopen(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs):
+        """Open gzip compressed tar archive name for reading or writing.
+           Appending is not allowed.
+        """
+        if len(mode) > 1 or mode not in "rw":
+            raise ValueError("mode must be 'r' or 'w'")
+
+        try:
+            import gzip
+            gzip.GzipFile
+        except (ImportError, AttributeError):
+            raise CompressionError("gzip module is not available")
+
+        if fileobj is None:
+            fileobj = bltn_open(name, mode + "b")
+
+        try:
+            t = cls.taropen(name, mode,
+                gzip.GzipFile(name, mode, compresslevel, fileobj),
+                **kwargs)
+        except IOError:
+            raise ReadError("not a gzip file")
+        t._extfileobj = False
+        return t
+
+    @classmethod
+    def bz2open(cls, name, mode="r", fileobj=None, compresslevel=9, **kwargs):
+        """Open bzip2 compressed tar archive name for reading or writing.
+           Appending is not allowed.
+        """
+        if len(mode) > 1 or mode not in "rw":
+            raise ValueError("mode must be 'r' or 'w'.")
+
+        try:
+            import bz2
+        except ImportError:
+            raise CompressionError("bz2 module is not available")
+
+        if fileobj is not None:
+            fileobj = _BZ2Proxy(fileobj, mode)
         else:
-            tarinfo.offset = self.offset
-            self.offset += BLOCKSIZE
-            tarinfo.offset_data = self.offset
-            if tarinfo.isreg():
-                ## Skip the following data blocks.
-                blocks, remainder = divmod(tarinfo.size, BLOCKSIZE)
-                if remainder:
-                    blocks += 1
-                self.next_chunk = self.offset + (blocks * BLOCKSIZE)
-            else: self.next_chunk = self.offset
-
-        #self.members.append(tarinfo)  These use too much memory
-        #self.membernames.append(tarinfo.name)
-        #self.chunks.append(self.offset)
-        return tarinfo
+            fileobj = bz2.BZ2File(name, mode, compresslevel=compresslevel)
+
+        try:
+            t = cls.taropen(name, mode, fileobj, **kwargs)
+        except (IOError, EOFError):
+            raise ReadError("not a bzip2 file")
+        t._extfileobj = False
+        return t
+
+    # All *open() methods are registered here.
+    OPEN_METH = {
+        "tar": "taropen",   # uncompressed tar
+        "gz":  "gzopen",    # gzip compressed tar
+        "bz2": "bz2open"    # bzip2 compressed tar
+    }
+
+    #--------------------------------------------------------------------------
+    # The public methods which TarFile provides:
+
+    def close(self):
+        """Close the TarFile. In write-mode, two finishing zero blocks are
+           appended to the archive.
+        """
+        if self.closed:
+            return
+
+        if self.mode in "aw":
+            self.fileobj.write(NUL * (BLOCKSIZE * 2))
+            self.offset += (BLOCKSIZE * 2)
+            # fill up the end with zero-blocks
+            # (like option -b20 for tar does)
+            blocks, remainder = divmod(self.offset, RECORDSIZE)
+            if remainder > 0:
+                self.fileobj.write(NUL * (RECORDSIZE - remainder))
+
+        if not self._extfileobj:
+            self.fileobj.close()
+        self.closed = True
 
     def getmember(self, name):
-        """Return a TarInfo instance for member name.
-        """
-        if name not in self.membernames and not self._loaded:
-            self._load()
-        if name not in self.membernames:
-            raise KeyError, "filename `%s' not found in tar archive" % name
-        return self._getmember(name)
-
-    def getinfo(self, name):
-        """Return a TarInfo instance for member name.
-           This method will be deprecated in 0.6,
-           use getmember() instead.
-        """
-        # XXX kick this out in 0.6
-        import warnings
-        warnings.warn("use getmember() instead", DeprecationWarning)
-        return self.getmember(name)
+        """Return a TarInfo object for member `name'. If `name' can not be
+           found in the archive, KeyError is raised. If a member occurs more
+           than once in the archive, its last occurrence is assumed to be the
+           most up-to-date version.
+        """
+        tarinfo = self._getmember(name)
+        if tarinfo is None:
+            raise KeyError("filename %r not found" % name)
+        return tarinfo
 
     def getmembers(self):
-        """Return a list of all members in the archive
-           (as TarInfo instances).
+        """Return the members of the archive as a list of TarInfo objects. The
+           list has the same order as the members in the archive.
         """
+        self._check()
         if not self._loaded:    # if we want to obtain a list of
             self._load()        # all members, we first have to
                                 # scan the whole archive.
         return self.members
 
     def getnames(self):
-        """Return a list of names of all members in the
-           archive.
-        """
-        if not self._loaded:
-            self._load()
-        return self.membernames
-
-    def gettarinfo(self, name, arcname=None):
-        """Create a TarInfo instance from an existing file.
-           Optional arcname defines the name under which the file
-           shall be stored in the archive.
-        """
-        # Now, fill the TarInfo instance with
+        """Return the members of the archive as a list of their names. It has
+           the same order as the list returned by getmembers().
+        """
+        return [tarinfo.name for tarinfo in self.getmembers()]
+
+    def gettarinfo(self, name=None, arcname=None, fileobj=None):
+        """Create a TarInfo object for either the file `name' or the file
+           object `fileobj' (using os.fstat on its file descriptor). You can
+           modify some of the TarInfo's attributes before you add it using
+           addfile(). If given, `arcname' specifies an alternative name for the
+           file in the archive.
+        """
+        self._check("aw")
+
+        # When fileobj is given, replace name by
+        # fileobj's real name.
+        if fileobj is not None:
+            name = fileobj.name
+
+        # Building the name of the member in the archive.
+        # Backward slashes are converted to forward slashes,
+        # Absolute paths are turned to relative paths.
+        if arcname is None:
+            arcname = name
+        drv, arcname = os.path.splitdrive(arcname)
+        arcname = arcname.replace(os.sep, "/")
+        arcname = arcname.lstrip("/")
+
+        # Now, fill the TarInfo object with
         # information specific for the file.
-        tarinfo = TarInfo()
-
-        if arcname is None: tarinfo.set_arcname(name)
-        else: tarinfo.set_arcname(arcname)
+        tarinfo = self.tarinfo()
+        tarinfo.tarfile = self
 
         # Use os.stat or os.lstat, depending on platform
         # and if symlinks shall be resolved.
-        if hasattr(os, "lstat") and not self.dereference:
-            statres = os.lstat(name)
+        if fileobj is None:
+            if hasattr(os, "lstat") and not self.dereference:
+                statres = os.lstat(name)
+            else:
+                statres = os.stat(name)
         else:
-            statres = os.stat(name)
-
-        if not tarinfo.init_from_stat(statres): return None
-
-        if tarinfo.type == REGTYPE:
-            inode = (statres.st_ino, statres.st_dev, statres.st_mtime)
-            if inode in self.inodes.keys() and not self.dereference:
+            statres = os.fstat(fileobj.fileno())
+        linkname = ""
+
+        stmd = statres.st_mode
+        if stat.S_ISREG(stmd):
+            inode = (statres.st_ino, statres.st_dev)
+            if not self.dereference and statres.st_nlink > 1 and \
+                    inode in self.inodes and arcname != self.inodes[inode]:
                 # Is it a hardlink to an already
                 # archived file?
-                tarinfo.type = LNKTYPE
-                tarinfo.linkname = self.inodes[inode]
+                type = LNKTYPE
+                linkname = self.inodes[inode]
             else:
                 # The inode is added only if its valid.
                 # For win32 it is always 0.
-                if inode[0]: self.inodes[inode] = tarinfo.name
-        elif tarinfo.type == SYMTYPE:
-            tarinfo.linkname = os.readlink(name)
-            tarinfo.size = 0
-
+                type = REGTYPE
+                if inode[0]:
+                    self.inodes[inode] = arcname
+        elif stat.S_ISDIR(stmd):
+            type = DIRTYPE
+        elif stat.S_ISFIFO(stmd):
+            type = FIFOTYPE
+        elif stat.S_ISLNK(stmd):
+            type = SYMTYPE
+            linkname = os.readlink(name)
+        elif stat.S_ISCHR(stmd):
+            type = CHRTYPE
+        elif stat.S_ISBLK(stmd):
+            type = BLKTYPE
+        else:
+            return None
+
+        # Fill the TarInfo object with all
+        # information we can get.
+        tarinfo.name = arcname
+        tarinfo.mode = stmd
+        tarinfo.uid = statres.st_uid
+        tarinfo.gid = statres.st_gid
+        if type == REGTYPE:
+            tarinfo.size = statres.st_size
+        else:
+            tarinfo.size = 0L
+        tarinfo.mtime = statres.st_mtime
+        tarinfo.type = type
+        tarinfo.linkname = linkname
+        if pwd:
+            try:
+                tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0]
+            except KeyError:
+                pass
+        if grp:
+            try:
+                tarinfo.gname = grp.getgrgid(tarinfo.gid)[0]
+            except KeyError:
+                pass
+
+        if type in (CHRTYPE, BLKTYPE):
+            if hasattr(os, "major") and hasattr(os, "minor"):
+                tarinfo.devmajor = os.major(statres.st_rdev)
+                tarinfo.devminor = os.minor(statres.st_rdev)
         return tarinfo
 
-    def list(self, verbose=1):
-        """Print a formatted listing of the archive's
-           contents to stdout.
+    def list(self, verbose=True):
+        """Print a table of contents to sys.stdout. If `verbose' is False, only
+           the names of the members are printed. If it is True, an `ls -l'-like
+           output is produced.
         """
+        self._check()
+
         for tarinfo in self:
             if verbose:
                 print filemode(tarinfo.mode),
-                print tarinfo.uname + "/" + tarinfo.gname,
+                print "%s/%s" % (tarinfo.uname or tarinfo.uid,
+                                 tarinfo.gname or tarinfo.gid),
                 if tarinfo.ischr() or tarinfo.isblk():
-                    print "%10s" % (str(tarinfo.devmajor) + "," + str(tarinfo.devminor)),
+                    print "%10s" % ("%d,%d" \
+                                    % (tarinfo.devmajor, tarinfo.devminor)),
                 else:
                     print "%10d" % tarinfo.size,
                 print "%d-%02d-%02d %02d:%02d:%02d" \
-                      % time.gmtime(tarinfo.mtime)[:6],
+                      % time.localtime(tarinfo.mtime)[:6],
 
-            print tarinfo.name,
+						if tarinfo.isdir():
+		            print tarinfo.name + "/",
+						else:
+								print tarinfo.name,
 
             if verbose:
                 if tarinfo.issym():
@@ -693,254 +1943,290 @@
                     print "link to", tarinfo.linkname,
             print
 
-    def add(self, name, arcname=None, recursive=1):
-        """Add a file or a directory to the archive.
-           Directory addition is recursive by default.
+    def add(self, name, arcname=None, recursive=True, exclude=None, filter=None):
+        """Add the file `name' to the archive. `name' may be any type of file
+           (directory, fifo, symbolic link, etc.). If given, `arcname'
+           specifies an alternative name for the file in the archive.
+           Directories are added recursively by default. This can be avoided by
+           setting `recursive' to False. `exclude' is a function that should
+           return True for each filename to be excluded. `filter' is a function
+           that expects a TarInfo object argument and returns the changed
+           TarInfo object, if it returns None the TarInfo object will be
+           excluded from the archive.
         """
-        if not self.fileobj:
-            raise ValueError, "I/O operation on closed file"
-        if self._mode == "r":
-            raise ValueError, "writing to a read-mode file"
+        self._check("aw")
 
         if arcname is None:
             arcname = name
 
+        # Exclude pathnames.
+        if exclude is not None:
+            import warnings
+            warnings.warn("use the filter argument instead",
+                    DeprecationWarning, 2)
+            if exclude(name):
+                self._dbg(2, "tarfile: Excluded %r" % name)
+                return
+
         # Skip if somebody tries to archive the archive...
-        if os.path.abspath(name) == os.path.abspath(self.name):
-            self._dbg(2, "tarfile: Skipped `%s'\n" % name)
-            return
-
-        # Special case: The user wants to add the current
-        # working directory.
-        if name == ".":
-            if recursive:
-                if arcname == ".":
-                    arcname = ""
-                for f in os.listdir("."):
-                    self.add(f, os.path.join(arcname, f))
-            return
-
-        self._dbg(1, "%s\n" % name)
-
-        # Create a TarInfo instance from the file.
+        if self.name is not None and os.path.abspath(name) == self.name:
+            self._dbg(2, "tarfile: Skipped %r" % name)
+            return
+
+        self._dbg(1, name)
+
+        # Create a TarInfo object from the file.
         tarinfo = self.gettarinfo(name, arcname)
 
         if tarinfo is None:
-            self._dbg(1, "tarfile: Unsupported type `%s'\n" % name)
+            self._dbg(1, "tarfile: Unsupported type %r" % name)
+            return
 
+        # Change or exclude the TarInfo object.
+        if filter is not None:
+            tarinfo = filter(tarinfo)
+            if tarinfo is None:
+                self._dbg(2, "tarfile: Excluded %r" % name)
+                return
 
         # Append the tar header and data to the archive.
         if tarinfo.isreg():
-            f = __builtin__.file(name, "rb")
-            self.addfile(tarinfo, fileobj = f)
+            f = bltn_open(name, "rb")
+            self.addfile(tarinfo, f)
             f.close()
 
-        if tarinfo.type in (LNKTYPE, SYMTYPE, FIFOTYPE, CHRTYPE, BLKTYPE):
-            tarinfo.size = 0l
-            self.addfile(tarinfo)
-
-        if tarinfo.isdir():
+        elif tarinfo.isdir():
             self.addfile(tarinfo)
             if recursive:
                 for f in os.listdir(name):
-                    self.add(os.path.join(name, f), os.path.join(arcname, f))
+                    self.add(os.path.join(name, f), os.path.join(arcname, f),
+                            recursive, exclude, filter)
+
+        else:
+            self.addfile(tarinfo)
 
     def addfile(self, tarinfo, fileobj=None):
-        """Add the content of fileobj to the tarfile.
-           The amount of bytes to read is determined by
-           the size attribute in the tarinfo instance.
+        """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is
+           given, tarinfo.size bytes are read from it and added to the archive.
+           You can create TarInfo objects using gettarinfo().
+           On Windows platforms, `fileobj' should always be opened with mode
+           'rb' to avoid irritation about the file size.
         """
-        if not self.fileobj:
-            raise ValueError, "I/O operation on closed file"
-        if self._mode == "r":
-            raise ValueError, "writing to a read-mode file"
-
-        # XXX What was this good for again?
-        #try:
-        #    self.fileobj.seek(self.chunks[-1])
-        #except IOError:
-        #    pass
-
-        full_headers = self._get_full_headers(tarinfo)
-        self.fileobj.write(full_headers)
-        assert len(full_headers) % BLOCKSIZE == 0
-        self.offset += len(full_headers)
+        self._check("aw")
+
+        tarinfo = copy.copy(tarinfo)
+
+        buf = tarinfo.tobuf(self.format, self.encoding, self.errors)
+        self.fileobj.write(buf)
+        self.offset += len(buf)
 
         # If there's data to follow, append it.
         if fileobj is not None:
             copyfileobj(fileobj, self.fileobj, tarinfo.size)
             blocks, remainder = divmod(tarinfo.size, BLOCKSIZE)
             if remainder > 0:
-                self.fileobj.write("\0" * (BLOCKSIZE - remainder))
+                self.fileobj.write(NUL * (BLOCKSIZE - remainder))
                 blocks += 1
             self.offset += blocks * BLOCKSIZE
 
-        #self.members.append(tarinfo)  #These take up too much memory
-        #self.membernames.append(tarinfo.name)
-        #self.chunks.append(self.offset)
-
-    def _get_full_headers(self, tarinfo):
-        """Return string containing headers around tarinfo, including gnulongs
-        """
-        buf = ""
-        # Now we must check if the strings for filename
-        # and linkname fit into the posix header.
-        # (99 chars + "\0" for each)
-        # If not, we must create GNU extension headers.
-        # If both filename and linkname are too long,
-        # the longlink is first to be written out.
-        if len(tarinfo.linkname) >= LENGTH_LINK - 1:
-            buf += self._return_gnulong(tarinfo.linkname, GNUTYPE_LONGLINK)
-            tarinfo.linkname = tarinfo.linkname[:LENGTH_LINK -1]
-        if len(tarinfo.name) >= LENGTH_NAME - 1:
-            buf += self._return_gnulong(tarinfo.name, GNUTYPE_LONGNAME)
-            tarinfo.name = tarinfo.name[:LENGTH_NAME - 1]
-        return buf + tarinfo.getheader()
-
-#    def untar(self, path):
-#        """Untar the whole archive to path.
-#        """
-#        later = []
-#        for tarinfo in self:
-#            if tarinfo.isdir():
-#                later.append(tarinfo)
-#            self.extract(tarinfo, path)
-#        for tarinfo in later:
-#            self._utime(tarinfo, os.path.join(path, tarinfo.name))
-
-    def extractfile(self, member):
-        """Extract member from the archive and return a file-like
-           object. member may be a name or a TarInfo instance.
-        """
-        if not self.fileobj:
-            raise ValueError, "I/O operation on closed file"
-        if self._mode != "r":
-            raise ValueError, "reading from a write-mode file"
-
-        if isinstance(member, TarInfo):
-            tarinfo = member
-        else:
-            tarinfo = self.getmember(member)
-
-        if tarinfo.isreg() or tarinfo.type not in SUPPORTED_TYPES:
-            return _FileObject(self, tarinfo)
-        elif tarinfo.islnk() or tarinfo.issym():
-            return self.extractfile(self._getmember(tarinfo.linkname, tarinfo))
-        else:
-            return None
+        self.members.append(tarinfo)
+
+    def extractall(self, path=".", members=None):
+        """Extract all members from the archive to the current working
+           directory and set owner, modification time and permissions on
+           directories afterwards. `path' specifies a different directory
+           to extract to. `members' is optional and must be a subset of the
+           list returned by getmembers().
+        """
+        directories = []
+
+        if members is None:
+            members = self
+
+        for tarinfo in members:
+            if tarinfo.isdir():
+                # Extract directories with a safe mode.
+                directories.append(tarinfo)
+                tarinfo = copy.copy(tarinfo)
+                tarinfo.mode = 0700
+            self.extract(tarinfo, path)
+
+        # Reverse sort directories.
+        directories.sort(key=operator.attrgetter('name'))
+        directories.reverse()
+
+        # Set correct owner, mtime and filemode on directories.
+        for tarinfo in directories:
+            dirpath = os.path.join(path, tarinfo.name)
+            try:
+                self.chown(tarinfo, dirpath)
+                self.utime(tarinfo, dirpath)
+                self.chmod(tarinfo, dirpath)
+            except ExtractError, e:
+                if self.errorlevel > 1:
+                    raise
+                else:
+                    self._dbg(1, "tarfile: %s" % e)
 
     def extract(self, member, path=""):
-        """Extract member from the archive and write it to
-           current working directory using its full pathname.
-           If optional path is given, it is attached before the
-           pathname.
-           member may be a name or a TarInfo instance.
+        """Extract a member from the archive to the current working directory,
+           using its full name. Its file information is extracted as accurately
+           as possible. `member' may be a filename or a TarInfo object. You can
+           specify a different directory using `path'.
         """
-        if not self.fileobj:
-            raise ValueError, "I/O operation on closed file"
-        if self._mode != "r":
-            raise ValueError, "reading from a write-mode file"
+        self._check("r")
 
-        if isinstance(member, TarInfo):
+        if isinstance(member, basestring):
+            tarinfo = self.getmember(member)
+        else:
             tarinfo = member
-        else:
-            tarinfo = self.getmember(member)
-
-        self._dbg(1, tarinfo.name)
+
+        # Prepare the link target for makelink().
+        if tarinfo.islnk():
+            tarinfo._link_target = os.path.join(path, tarinfo.linkname)
+
         try:
             self._extract_member(tarinfo, os.path.join(path, tarinfo.name))
         except EnvironmentError, e:
             if self.errorlevel > 0:
                 raise
             else:
-                self._dbg(1, "\ntarfile: %s `%s'" % (e.strerror, e.filename))
-        except TarError, e:
+                if e.filename is None:
+                    self._dbg(1, "tarfile: %s" % e.strerror)
+                else:
+                    self._dbg(1, "tarfile: %s %r" % (e.strerror, e.filename))
+        except ExtractError, e:
             if self.errorlevel > 1:
                 raise
             else:
-                self._dbg(1, "\ntarfile: %s" % e)
-        self._dbg(1, "\n")
+                self._dbg(1, "tarfile: %s" % e)
+
+    def extractfile(self, member):
+        """Extract a member from the archive as a file object. `member' may be
+           a filename or a TarInfo object. If `member' is a regular file, a
+           file-like object is returned. If `member' is a link, a file-like
+           object is constructed from the link's target. If `member' is none of
+           the above, None is returned.
+           The file-like object is read-only and provides the following
+           methods: read(), readline(), readlines(), seek() and tell()
+        """
+        self._check("r")
+
+        if isinstance(member, basestring):
+            tarinfo = self.getmember(member)
+        else:
+            tarinfo = member
+
+        if tarinfo.isreg():
+            return self.fileobject(self, tarinfo)
+
+        elif tarinfo.type not in SUPPORTED_TYPES:
+            # If a member's type is unknown, it is treated as a
+            # regular file.
+            return self.fileobject(self, tarinfo)
+
+        elif tarinfo.islnk() or tarinfo.issym():
+            if isinstance(self.fileobj, _Stream):
+                # A small but ugly workaround for the case that someone tries
+                # to extract a (sym)link as a file-object from a non-seekable
+                # stream of tar blocks.
+                raise StreamError("cannot extract (sym)link as file object")
+            else:
+                # A (sym)link's file object is its target's file object.
+                return self.extractfile(self._find_link_target(tarinfo))
+        else:
+            # If there's no data associated with the member (directory, chrdev,
+            # blkdev, etc.), return None instead of a file object.
+            return None
 
     def _extract_member(self, tarinfo, targetpath):
-        """Extract the TarInfo instance tarinfo to a physical
+        """Extract the TarInfo object tarinfo to a physical
            file called targetpath.
         """
-        # Fetch the TarInfo instance for the given name
+        # Fetch the TarInfo object for the given name
         # and build the destination pathname, replacing
         # forward slashes to platform specific separators.
-        if targetpath[-1:] == "/":
-            targetpath = targetpath[:-1]
-        targetpath = os.path.normpath(targetpath)
+        targetpath = targetpath.rstrip("/")
+        targetpath = targetpath.replace("/", os.sep)
 
         # Create all upper directories.
         upperdirs = os.path.dirname(targetpath)
         if upperdirs and not os.path.exists(upperdirs):
-            ti = TarInfo()
-            ti.name  = ""
-            ti.type  = DIRTYPE
-            ti.mode  = 0777
-            ti.mtime = tarinfo.mtime
-            ti.uid   = tarinfo.uid
-            ti.gid   = tarinfo.gid
-            ti.uname = tarinfo.uname
-            ti.gname = tarinfo.gname
-            for d in os.path.split(os.path.splitdrive(upperdirs)[1]):
-                ti.name = os.path.join(ti.name, d)
-                self._extract_member(ti, ti.name)
+            # Create directories that are not part of the archive with
+            # default permissions.
+            os.makedirs(upperdirs)
+
+        if tarinfo.islnk() or tarinfo.issym():
+            self._dbg(1, "%s -> %s" % (tarinfo.name, tarinfo.linkname))
+        else:
+            self._dbg(1, tarinfo.name)
 
         if tarinfo.isreg():
-            self._makefile(tarinfo, targetpath)
+            self.makefile(tarinfo, targetpath)
         elif tarinfo.isdir():
-            self._makedir(tarinfo, targetpath)
+            self.makedir(tarinfo, targetpath)
         elif tarinfo.isfifo():
-            self._makefifo(tarinfo, targetpath)
+            self.makefifo(tarinfo, targetpath)
         elif tarinfo.ischr() or tarinfo.isblk():
-            self._makedev(tarinfo, targetpath)
+            self.makedev(tarinfo, targetpath)
         elif tarinfo.islnk() or tarinfo.issym():
-            self._makelink(tarinfo, targetpath)
+            self.makelink(tarinfo, targetpath)
+        elif tarinfo.type not in SUPPORTED_TYPES:
+            self.makeunknown(tarinfo, targetpath)
         else:
-            self._makefile(tarinfo, targetpath)
-            if tarinfo.type not in SUPPORTED_TYPES:
-                self._dbg(1, "\ntarfile: Unknown file type '%s', " \
-                             "extracted as regular file." % tarinfo.type)
+            self.makefile(tarinfo, targetpath)
 
+        self.chown(tarinfo, targetpath)
         if not tarinfo.issym():
-            self._chown(tarinfo, targetpath)
-            self._chmod(tarinfo, targetpath)
-            if not tarinfo.isdir():
-                self._utime(tarinfo, targetpath)
-
-    def _makedir(self, tarinfo, targetpath):
-        """Make a directory called targetpath out of tarinfo.
+            self.chmod(tarinfo, targetpath)
+            self.utime(tarinfo, targetpath)
+
+    #--------------------------------------------------------------------------
+    # Below are the different file methods. They are called via
+    # _extract_member() when extract() is called. They can be replaced in a
+    # subclass to implement other functionality.
+
+    def makedir(self, tarinfo, targetpath):
+        """Make a directory called targetpath.
         """
         try:
-            os.mkdir(targetpath)
+            # Use a safe mode for the directory, the real mode is set
+            # later in _extract_member().
+            os.mkdir(targetpath, 0700)
         except EnvironmentError, e:
             if e.errno != errno.EEXIST:
                 raise
 
-    def _makefile(self, tarinfo, targetpath):
-        """Make a file called targetpath out of tarinfo.
+    def makefile(self, tarinfo, targetpath):
+        """Make a file called targetpath.
         """
         source = self.extractfile(tarinfo)
-        target = __builtin__.file(targetpath, "wb")
+        target = bltn_open(targetpath, "wb")
         copyfileobj(source, target)
         source.close()
         target.close()
 
-    def _makefifo(self, tarinfo, targetpath):
-        """Make a fifo called targetpath out of tarinfo.
+    def makeunknown(self, tarinfo, targetpath):
+        """Make a file from a TarInfo object with an unknown type
+           at targetpath.
+        """
+        self.makefile(tarinfo, targetpath)
+        self._dbg(1, "tarfile: Unknown file type %r, " \
+                     "extracted as regular file." % tarinfo.type)
+
+    def makefifo(self, tarinfo, targetpath):
+        """Make a fifo called targetpath.
         """
         if hasattr(os, "mkfifo"):
             os.mkfifo(targetpath)
         else:
-            raise TarError, "Fifo not supported by system"
+            raise ExtractError("fifo not supported by system")
 
-    def _makedev(self, tarinfo, targetpath):
-        """Make a character or block device called targetpath out of tarinfo.
+    def makedev(self, tarinfo, targetpath):
+        """Make a character or block device called targetpath.
         """
-        if not hasattr(os, "mknod"):
-            raise TarError, "Special devices not supported by system"
+        if not hasattr(os, "mknod") or not hasattr(os, "makedev"):
+            raise ExtractError("special devices not supported by system")
 
         mode = tarinfo.mode
         if tarinfo.isblk():
@@ -948,109 +2234,187 @@
         else:
             mode |= stat.S_IFCHR
 
-        # This if statement should go away when python-2.3a0-devicemacros
-        # patch succeeds.
-        if hasattr(os, "makedev"):
-            os.mknod(targetpath, mode,
-                     os.makedev(tarinfo.devmajor, tarinfo.devminor))
-        else:
-            os.mknod(targetpath, mode,
-                     tarinfo.devmajor, tarinfo.devminor)
+        os.mknod(targetpath, mode,
+                 os.makedev(tarinfo.devmajor, tarinfo.devminor))
 
-    def _makelink(self, tarinfo, targetpath):
-        """Make a (symbolic) link called targetpath out of tarinfo.
-           If it cannot be made (due to platform or failure), we try
-           to make a copy of the referenced file instead of a link.
+    def makelink(self, tarinfo, targetpath):
+        """Make a (symbolic) link called targetpath. If it cannot be created
+          (platform limitation), we try to make a copy of the referenced file
+          instead of a link.
         """
-        linkpath = tarinfo.linkname
-        self._dbg(1, " -> %s" % linkpath)
-        try:
+        if hasattr(os, "symlink") and hasattr(os, "link"):
+            # For systems that support symbolic and hard links.
             if tarinfo.issym():
-                os.symlink(linkpath, targetpath)
+                os.symlink(tarinfo.linkname, targetpath)
             else:
-                linkpath = os.path.join(os.path.dirname(targetpath),
-                                        linkpath)
-                os.link(linkpath, targetpath)
-        except AttributeError:
-            linkpath = os.path.join(os.path.dirname(tarinfo.name),
-                                    tarinfo.linkname)
-            linkpath = normpath(linkpath)
+                # See extract().
+                if os.path.exists(tarinfo._link_target):
+                    os.link(tarinfo._link_target, targetpath)
+                else:
+                    self._extract_member(self._find_link_target(tarinfo), targetpath)
+        else:
             try:
-                self._extract_member(self.getmember(linkpath), targetpath)
-            except (IOError, OSError, KeyError), e: #@UnusedVariable
-                linkpath = os.path.normpath(linkpath)
-                try:
-                    shutil.copy2(linkpath, targetpath)
-                except EnvironmentError, e: #@UnusedVariable
-                    raise TarError, "Link could not be created"
+                self._extract_member(self._find_link_target(tarinfo), targetpath)
+            except KeyError:
+                raise ExtractError("unable to resolve link inside archive")
 
-    def _chown(self, tarinfo, targetpath):
+    def chown(self, tarinfo, targetpath):
         """Set owner of targetpath according to tarinfo.
         """
-        if pwd and os.geteuid() == 0:
+        if pwd and hasattr(os, "geteuid") and os.geteuid() == 0:
             # We have to be root to do so.
-            try: g = gname2gid(tarinfo.gname)
-            except KeyError:
-                try:
-                    gid2gname(tarinfo.gid) # Make sure gid exists
-                    g = tarinfo.gid
-                except KeyError: g = os.getgid()
-            try: u = uname2uid(tarinfo.uname)
-            except KeyError:
-                try:
-                    uid2uname(tarinfo.uid) # Make sure uid exists
-                    u = tarinfo.uid
-                except KeyError: u = os.getuid()
+            try:
+                g = grp.getgrnam(tarinfo.gname)[2]
+            except KeyError:
+                try:
+                    g = grp.getgrgid(tarinfo.gid)[2]
+                except KeyError:
+                    g = os.getgid()
+            try:
+                u = pwd.getpwnam(tarinfo.uname)[2]
+            except KeyError:
+                try:
+                    u = pwd.getpwuid(tarinfo.uid)[2]
+                except KeyError:
+                    u = os.getuid()
             try:
                 if tarinfo.issym() and hasattr(os, "lchown"):
                     os.lchown(targetpath, u, g)
                 else:
-                    os.chown(targetpath, u, g)
+                    if sys.platform != "os2emx":
+                        os.chown(targetpath, u, g)
             except EnvironmentError, e:
-                self._dbg(2, "\ntarfile: (chown failed), %s `%s'"
-                             % (e.strerror, e.filename))
+                raise ExtractError("could not change owner")
 
-    def _chmod(self, tarinfo, targetpath):
+    def chmod(self, tarinfo, targetpath):
         """Set file permissions of targetpath according to tarinfo.
         """
-        try:
-            os.chmod(targetpath, tarinfo.mode)
-        except EnvironmentError, e:
-            self._dbg(2, "\ntarfile: (chmod failed), %s `%s'"
-                         % (e.strerror, e.filename))
+        if hasattr(os, 'chmod'):
+            try:
+                os.chmod(targetpath, tarinfo.mode)
+            except EnvironmentError, e:
+                raise ExtractError("could not change mode")
 
-    def _utime(self, tarinfo, targetpath):
+    def utime(self, tarinfo, targetpath):
         """Set modification time of targetpath according to tarinfo.
         """
+        if not hasattr(os, 'utime'):
+            return
         try:
             os.utime(targetpath, (tarinfo.mtime, tarinfo.mtime))
         except EnvironmentError, e:
-            self._dbg(2, "\ntarfile: (utime failed), %s `%s'"
-                         % (e.strerror, e.filename))
-
-    def _getmember(self, name, tarinfo=None):
+            raise ExtractError("could not change modification time")
+
+    #--------------------------------------------------------------------------
+    def next(self):
+        """Return the next member of the archive as a TarInfo object, when
+           TarFile is opened for reading. Return None if there is no more
+           available.
+        """
+        self._check("ra")
+        if self.firstmember is not None:
+            m = self.firstmember
+            self.firstmember = None
+            return m
+
+        # Read the next block.
+        self.fileobj.seek(self.offset)
+        tarinfo = None
+        while True:
+            try:
+                tarinfo = self.tarinfo.fromtarfile(self)
+            except EOFHeaderError, e:
+                if self.ignore_zeros:
+                    self._dbg(2, "0x%X: %s" % (self.offset, e))
+                    self.offset += BLOCKSIZE
+                    continue
+            except InvalidHeaderError, e:
+                if self.ignore_zeros:
+                    self._dbg(2, "0x%X: %s" % (self.offset, e))
+                    self.offset += BLOCKSIZE
+                    continue
+                elif self.offset == 0:
+                    raise ReadError(str(e))
+            except EmptyHeaderError:
+                if self.offset == 0:
+                    raise ReadError("empty file")
+            except TruncatedHeaderError, e:
+                if self.offset == 0:
+                    raise ReadError(str(e))
+            except SubsequentHeaderError, e:
+                raise ReadError(str(e))
+            break
+
+        if tarinfo is not None:
+            self.members.append(tarinfo)
+        else:
+            self._loaded = True
+
+        return tarinfo
+
+    #--------------------------------------------------------------------------
+    # Little helper methods:
+
+    def _getmember(self, name, tarinfo=None, normalize=False):
         """Find an archive member by name from bottom to top.
            If tarinfo is given, it is used as the starting point.
         """
-        if tarinfo is None:
-            end = len(self.members)
-        else:
-            end = self.members.index(tarinfo)
-
-        for i in xrange(end - 1, -1, -1):
-            if name == self.membernames[i]:
-                return self.members[i]
+        # Ensure that all members have been loaded.
+        members = self.getmembers()
+
+        # Limit the member search list up to tarinfo.
+        if tarinfo is not None:
+            members = members[:members.index(tarinfo)]
+
+        if normalize:
+            name = os.path.normpath(name)
+
+        for member in reversed(members):
+            if normalize:
+                member_name = os.path.normpath(member.name)
+            else:
+                member_name = member.name
+
+            if name == member_name:
+                return member
 
     def _load(self):
         """Read through the entire archive file and look for readable
            members.
         """
-        while 1:
+        while True:
             tarinfo = self.next()
             if tarinfo is None:
                 break
-        self._loaded = 1
-        return
+        self._loaded = True
+
+    def _check(self, mode=None):
+        """Check if TarFile is still open, and if the operation's mode
+           corresponds to TarFile's mode.
+        """
+        if self.closed:
+            raise IOError("%s is closed" % self.__class__.__name__)
+        if mode is not None and self.mode not in mode:
+            raise IOError("bad operation for mode %r" % self.mode)
+
+    def _find_link_target(self, tarinfo):
+        """Find the target member of a symlink or hardlink member in the
+           archive.
+        """
+        if tarinfo.issym():
+            # Always search the entire archive.
+            linkname = os.path.dirname(tarinfo.name) + "/" + tarinfo.linkname
+            limit = None
+        else:
+            # Search the archive before the link, because a hard link is
+            # just a reference to an already archived file.
+            linkname = tarinfo.linkname
+            limit = tarinfo
+
+        member = self._getmember(linkname, tarinfo=limit, normalize=True)
+        if member is None:
+            raise KeyError("linkname %r not found" % linkname)
+        return member
 
     def __iter__(self):
         """Provide an iterator object.
@@ -1060,136 +2424,25 @@
         else:
             return TarIter(self)
 
-    def _buftoinfo(self, buf):
-        """Transform a 512 byte block to a TarInfo instance.
-        """
-        tarinfo = TarInfo()
-        tarinfo.name = nts(buf[0:100])
-        tarinfo.mode = int(buf[100:107], 8)
-        tarinfo.uid = int(buf[108:115],8)
-        tarinfo.gid = int(buf[116:123],8)
-        tarinfo.size = long(buf[124:135], 8)
-        tarinfo.mtime = long(buf[136:147], 8)
-        # chksum stored as a six digit octal number with
-        # leading zeroes followed by a nul and then a space
-        tarinfo.chksum = int(buf[148:154], 8)
-        tarinfo.type = buf[156:157]
-        tarinfo.linkname = nts(buf[157:257])
-        tarinfo.uname = nts(buf[265:297])
-        tarinfo.gname = nts(buf[297:329])
-        try:
-            tarinfo.devmajor = int(buf[329:336], 8)
-            tarinfo.devminor = int(buf[337:344], 8)
-        except ValueError:
-            tarinfo.devmajor = tarinfo.devmajor = 0
-        tarinfo.prefix = buf[345:500]
-        if tarinfo.chksum != calc_chksum(buf):
-            self._dbg(1, "tarfile: Bad Checksum\n")
-        return tarinfo
-
-    def _proc_gnulong(self, tarinfo, type):
-        """Evaluate the blocks that hold a GNU longname
-           or longlink member.
-        """
-        name = None
-        linkname = None
-        #may be some sanity checking should be done here
-        #assert tarinfo.size < 1000 * BLOCKSIZE, "Filename appears to be too long!"
-        buf = self.fileobj.read(BLOCKSIZE)
-        if not buf: return None
-        namesize = tarinfo.size - BLOCKSIZE
-        self.offset += BLOCKSIZE
-        # may be the whole name should be read with one operation?
-        while namesize > 0:
-            buf += self.fileobj.read(BLOCKSIZE)
-            if not buf: return None
-            self.offset += BLOCKSIZE
-            namesize -= BLOCKSIZE
-        if type == GNUTYPE_LONGNAME: name = nts(buf)
-        if type == GNUTYPE_LONGLINK: linkname = nts(buf)
-
-        buf = self.fileobj.read(BLOCKSIZE)
-        if not buf: return None
-        tarinfo = self._buftoinfo(buf)
-        if tarinfo.type in (GNUTYPE_LONGLINK, GNUTYPE_LONGNAME):
-            tarinfo = self._proc_gnulong(tarinfo, tarinfo.type)
-            if not tarinfo: return None
-        if name is not None:
-            tarinfo.name = name
-        if linkname is not None:
-            tarinfo.linkname = linkname
-        self.offset += BLOCKSIZE
-        return tarinfo
-
-
-
-    def _return_gnulong(self, name, type):
-        """Insert a GNU longname/longlink member into the archive.
-           It consists of a common tar header, with the length
-           of the longname as size, followed by a data block,
-           which contains the longname as a null terminated string.
-        """
-        tarinfo = TarInfo()
-        tarinfo.name = "././@LongLink"
-        tarinfo.type = type
-        tarinfo.mode = 0
-        tarinfo.size = len(name)
-
-        residual = (tarinfo.size % BLOCKSIZE)
-        return "%s%s%s" % (tarinfo.getheader(), name,
-                           "\0" * ((BLOCKSIZE - residual) * (residual > 0)))
-
-    def _proc_sparse(self, tarinfo):
-        """Analyze a GNU sparse header plus extra headers.
-        """
-        buf = tarinfo.getheader()
-        sp = _ringbuffer()
-        pos = 386
-        lastpos = 0l
-        realpos = 0l
-        try:
-            # There are 4 possible sparse structs in the
-            # first header.
-            for i in range(4): #@UnusedVariable
-                offset = int(buf[pos:pos + 12], 8)
-                numbytes = int(buf[pos + 12:pos + 24], 8)
-                if offset > lastpos:
-                    sp.append(_hole(lastpos, offset - lastpos))
-                sp.append(_data(offset, numbytes, realpos))
-                realpos += numbytes
-                lastpos = offset + numbytes
-                pos += 24
-
-            isextended = ord(buf[482])
-            origsize = int(buf[483:495], 8)
-
-            # If the isextended flag is given,
-            # there are extra headers to process.
-            while isextended == 1:
-                buf = self.fileobj.read(BLOCKSIZE)
-                self.offset += BLOCKSIZE
-                pos = 0
-                for i in range(21): #@UnusedVariable
-                    offset = int(buf[pos:pos + 12], 8)
-                    numbytes = int(buf[pos + 12:pos + 24], 8)
-                    if offset > lastpos:
-                        sp.append(_hole(lastpos, offset - lastpos))
-                    sp.append(_data(offset, numbytes, realpos))
-                    realpos += numbytes
-                    lastpos = offset + numbytes
-                    pos += 24
-                isextended = ord(buf[504])
-        except ValueError:
-            pass
-        if lastpos < origsize:
-            sp.append(_hole(lastpos, origsize - lastpos))
-
-        tarinfo.sparse = sp
-        return origsize
-
     def _dbg(self, level, msg):
+        """Write debugging output to sys.stderr.
+        """
         if level <= self.debug:
-            sys.stdout.write(msg)
+            print >> sys.stderr, msg
+
+    def __enter__(self):
+        self._check()
+        return self
+
+    def __exit__(self, type, value, traceback):
+        if type is None:
+            self.close()
+        else:
+            # An exception occurred. We must not call close() because
+            # it would try to write end-of-archive blocks and padding.
+            if not self._extfileobj:
+                self.fileobj.close()
+            self.closed = True
 # class TarFile
 
 class TarIter:
@@ -1200,9 +2453,10 @@
     """
 
     def __init__(self, tarfile):
-        """Construct a TarIter instance.
+        """Construct a TarIter object.
         """
         self.tarfile = tarfile
+        self.index = 0
     def __iter__(self):
         """Return iterator object.
         """
@@ -1211,12 +2465,21 @@
         """Return the next item using TarFile's next() method.
            When all members have been read, set TarFile as _loaded.
         """
-        tarinfo = self.tarfile.next()
-        if not tarinfo:
-            self.tarfile._loaded = 1
-            raise StopIteration
+        # Fix for SF #1100429: Under rare circumstances it can
+        # happen that getmembers() is called during iteration,
+        # which will cause TarIter to stop prematurely.
+        if not self.tarfile._loaded:
+            tarinfo = self.tarfile.next()
+            if not tarinfo:
+                self.tarfile._loaded = True
+                raise StopIteration
+        else:
+            try:
+                tarinfo = self.tarfile.members[self.index]
+            except IndexError:
+                raise StopIteration
+        self.index += 1
         return tarinfo
-# class TarIter
 
 # Helper classes for sparse file support
 class _section:
@@ -1248,7 +2511,7 @@
         self.idx = 0
     def find(self, offset):
         idx = self.idx
-        while 1:
+        while True:
             item = self[idx]
             if offset in item:
                 break
@@ -1261,153 +2524,9 @@
         self.idx = idx
         return item
 
-class _FileObject:
-    """File-like object for reading an archive member,
-       is returned by TarFile.extractfile().
-       Support for sparse files included.
-    """
-
-    def __init__(self, tarfile, tarinfo):
-        self.tarfile = tarfile
-        self.fileobj = tarfile.fileobj
-        self.name    = tarinfo.name
-        self.mode    = "r"
-        self.closed  = 0
-        self.offset  = tarinfo.offset_data
-        self.size    = tarinfo.size
-        self.pos     = 0l
-        self.linebuffer = ""
-        if tarinfo.issparse():
-            self.sparse = tarinfo.sparse
-            self.read = self._readsparse
-        else:
-            self.read = self._readnormal
-
-    def readline(self, size=-1):
-        """Read a line with approx. size.
-           If size is negative, read a whole line.
-           readline() and read() must not be mixed up (!).
-        """
-        if size < 0:
-            size = sys.maxint
-
-        nl = self.linebuffer.find("\n")
-        if nl >= 0:
-            nl = min(nl, size)
-        else:
-            size -= len(self.linebuffer)
-            while nl < 0:
-                buf = self.read(min(size, 100))
-                if not buf:
-                    break
-                self.linebuffer += buf
-                size -= len(buf)
-                if size <= 0:
-                    break
-                nl = self.linebuffer.find("\n")
-            if nl == -1:
-                s = self.linebuffer
-                self.linebuffer = ""
-                return s
-        buf = self.linebuffer[:nl]
-        self.linebuffer = self.linebuffer[nl + 1:]
-        while buf[-1:] == "\r":
-            buf = buf[:-1]
-        return buf + "\n"
-
-    def readlines(self):
-        """Return a list with all (following) lines.
-        """
-        result = []
-        while 1:
-            line = self.readline()
-            if not line: break
-            result.append(line)
-        return result
-
-    def _readnormal(self, size=None):
-        """Read operation for regular files.
-        """
-        if self.closed:
-            raise ValueError, "I/O operation on closed file"
-        #self.fileobj.seek(self.offset + self.pos)
-        bytesleft = self.size - self.pos
-        if size is None:
-            bytestoread = bytesleft
-        else:
-            bytestoread = min(size, bytesleft)
-        self.pos += bytestoread
-        self.tarfile.offset += bytestoread
-        return self.fileobj.read(bytestoread)
-
-    def _readsparse(self, size=None):
-        """Read operation for sparse files.
-        """
-        if self.closed:
-            raise ValueError, "I/O operation on closed file"
-
-        if size is None:
-            size = self.size - self.pos
-
-        data = ""
-        while size > 0:
-            buf = self._readsparsesection(size)
-            if not buf:
-                break
-            size -= len(buf)
-            data += buf
-        return data
-
-    def _readsparsesection(self, size):
-        """Read a single section of a sparse file.
-        """
-        section = self.sparse.find(self.pos)
-
-        if section is None:
-            return ""
-
-        toread = min(size, section.offset + section.size - self.pos)
-        if isinstance(section, _data):
-            realpos = section.realpos + self.pos - section.offset
-            self.pos += toread
-            self.fileobj.seek(self.offset + realpos)
-            return self.fileobj.read(toread)
-        else:
-            self.pos += toread
-            return "\0" * toread
-
-    def tell(self):
-        """Return the current file position.
-        """
-        return self.pos
-
-    def seek(self, pos, whence=0):
-        """Seek to a position in the file.
-        """
-        self.linebuffer = ""
-        if whence == 0:
-            self.pos = min(max(pos, 0), self.size)
-        if whence == 1:
-            if pos < 0:
-                self.pos = max(self.pos + pos, 0)
-            else:
-                self.pos = min(self.pos + pos, self.size)
-        if whence == 2:
-            self.pos = max(min(self.size + pos, self.size), 0)
-
-    def close(self):
-        """Close the file object.
-        """
-        self.closed = 1
-#class _FileObject
-
 #---------------------------------------------
 # zipfile compatible TarFile class
-#
-# for details consult zipfile's documentation
 #---------------------------------------------
-import cStringIO
-
 TAR_PLAIN = 0           # zipfile.ZIP_STORED
 TAR_GZIPPED = 8         # zipfile.ZIP_DEFLATED
 class TarFileCompat:
@@ -1415,16 +2534,18 @@
        ZipFile class.
     """
     def __init__(self, file, mode="r", compression=TAR_PLAIN):
+        from warnings import warnpy3k
+        warnpy3k("the TarFileCompat class has been removed in Python 3.0",
+                stacklevel=2)
         if compression == TAR_PLAIN:
-            self.tarfile = open(file, mode)
+            self.tarfile = TarFile.taropen(file, mode)
         elif compression == TAR_GZIPPED:
-            self.tarfile = gzopen(file, mode)
+            self.tarfile = TarFile.gzopen(file, mode)
         else:
-            raise ValueError, "unknown compression constant"
+            raise ValueError("unknown compression constant")
         if mode[0:1] == "r":
             members = self.tarfile.getmembers()
-            for i in range(len(members)):
-                m = members[i]
+            for m in members:
                 m.filename = m.name
                 m.file_size = m.size
                 m.date_time = time.gmtime(m.mtime)[:6]
@@ -1444,270 +2565,32 @@
     def write(self, filename, arcname=None, compress_type=None):
         self.tarfile.add(filename, arcname)
     def writestr(self, zinfo, bytes):
+        try:
+            from cStringIO import StringIO
+        except ImportError:
+            from StringIO import StringIO
         import calendar
-        zinfo.name = zinfo.filename
-        zinfo.size = zinfo.file_size
-        zinfo.mtime = calendar.timegm(zinfo.date_time)
-        self.tarfile.addfile(zinfo, cStringIO.StringIO(bytes))
+        tinfo = TarInfo(zinfo.filename)
+        tinfo.size = len(bytes)
+        tinfo.mtime = calendar.timegm(zinfo.date_time)
+        self.tarfile.addfile(tinfo, StringIO(bytes))
     def close(self):
         self.tarfile.close()
 #class TarFileCompat
 
-if __name__ == "__main__":
-    # a "light-weight" implementation of GNUtar ;-)
-    usage = """
-Usage: %s [options] [files]
-
--h      display this help message
--c      create a tarfile
--r      append to an existing archive
--x      extract archive
--t      list archive contents
--f FILENAME
-        use archive FILENAME, else STDOUT (-c)
--z      filter archive through gzip
--C DIRNAME
-        with opt -x:     extract to directory DIRNAME
-        with opt -c, -r: put files to archive under DIRNAME
--v      verbose output
--q      quiet
-
-wildcards *, ?, [seq], [!seq] are accepted.
-    """ % sys.argv[0]
-
-    import getopt, glob
+#--------------------
+# exported functions
+#--------------------
+def is_tarfile(name):
+    """Return True if name points to a tar archive that we
+       are able to handle, else return False.
+    """
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "htcrzxf:C:qv")
-    except getopt.GetoptError, e:
-        print
-        print "ERROR:", e
-        print usage
-        sys.exit(0)
-
-    file = None
-    mode = None
-    dir = None
-    comp = 0
-    debug = 0
-    for o, a in opts:
-        if o == "-t": mode = "l"        # list archive
-        if o == "-c": mode = "w"        # write to archive
-        if o == "-r": mode = "a"        # append to archive
-        if o == "-x": mode = "r"        # extract from archive
-        if o == "-f": file = a          # specify filename else use stdout
-        if o == "-C": dir = a           # change to dir
-        if o == "-z": comp = 1          # filter through gzip
-        if o == "-v": debug = 2         # verbose mode
-        if o == "-q": debug = 0         # quiet mode
-        if o == "-h":                   # help message
-            print usage
-            sys.exit(0)
-
-    if not mode:
-        print usage
-        sys.exit(0)
-
-    if comp:
-        func = gzopen
-    else:
-        func = open
-
-    if not file or file == "-":
-        if mode != "w":
-            print usage
-            sys.exit(0)
-        debug = 0
-        # If under Win32, set stdout to binary.
-        try:
-            import msvcrt
-            msvcrt.setmode(1, os.O_BINARY) #@UndefinedVariable
-        except ImportError:
-            pass
-        tarfile = func("sys.stdout.tar", mode, 9, sys.stdout)
-    else:
-        if mode == "l":
-            tarfile = func(file, "r")
-        else:
-            tarfile = func(file, mode)
-
-    tarfile.debug = debug
-
-    if mode == "r":
-        if dir is None:
-            dir = ""
-        for tarinfo in tarfile:
-            tarfile.extract(tarinfo, dir)
-    elif mode == "l":
-        tarfile.list(debug)
-    else:
-        for arg in args:
-            files = glob.glob(arg)
-            for f in files:
-                tarfile.add(f, dir)
-    tarfile.close()
-
-
-class TarFromIterator(TarFile):
-    """Readable tarfile-like object generated from iterator
-    """
-    # These various status numbers indicate what we are in the process
-    # of doing in the tarfile.
-    BEGIN = 0 # next step is to read tarinfo, write new header
-    MIDDLE_OF_FILE = 1 # in process of writing file data
-    END = 2 # end of data
-
-    # Buffer is added to in multiples of following
-    BUFFER_ADDLEN = 64 * 1024
-
-    def __init__(self, pair_iter):
-        """Construct a TarFromIterator instance.  pair_iter is an
-        iterator of (TarInfo, fileobj) objects, which fileobj should
-        be a file-like object opened for reading, or None.  The
-        fileobjs will be closed before the next element in the
-        iterator is read.
-        """
-        self.closed = None
-        self.name = None
-        self.mode = "rb"
-        self.pair_iter = pair_iter
-
-        self.init_datastructures()
-        self.status = self.BEGIN
-        self.cur_tarinfo, self.cur_fileobj = None, None
-        self.cur_pos_in_fileobj = 0
-        self.buffer = ""
-        # holds current position as seen by reading client.  This is
-        # distinct from self.offset.
-        self.tar_iter_offset = 0
-
-    def seek(self, offset):
-        """Seek to current position.  Just read and discard some amount"""
-        if offset < self.tar_iter_offset:
-            raise TarError("Seeks in TarFromIterator must go forwards,\n"
-                           "Instead asking for %s from %s" %
-                           (offset, self.tar_iter_offset))
-        while offset - self.tar_iter_offset >= self.BUFFER_ADDLEN:
-            buf = self.read(self.BUFFER_ADDLEN)
-            if not buf: return # eof
-        self.read(offset - self.tar_iter_offset)
-
-    def read(self, length = -1):
-        """Return next length bytes, or everything if length < 0"""
-        if length < 0:
-            while 1:
-                if not self._addtobuffer(): break
-            result = self.buffer
-            self.buffer = ""
-        else:
-            while len(self.buffer) < length:
-                if not self._addtobuffer(): break
-            # It's possible that length > len(self.buffer)
-            result = self.buffer[:length]
-            self.buffer = self.buffer[length:]
-        self.tar_iter_offset += len(result)
-        return result
-
-    def _addtobuffer(self):
-        """Write more data into the buffer.  Return None if at end"""
-        if self.status == self.BEGIN:
-            # Just write headers into buffer
-            try: self.cur_tarinfo, self.cur_fileobj = self.pair_iter.next()
-            except StopIteration:
-                self._add_final()
-                self.status = self.END
-                return None
-
-            # Zero out tarinfo sizes for various file types
-            if self.cur_tarinfo.type in (LNKTYPE, SYMTYPE,
-                                         FIFOTYPE, CHRTYPE, BLKTYPE):
-                self.cur_tarinfo.size = 0l
-
-            full_headers = self._get_full_headers(self.cur_tarinfo)
-            self.buffer += full_headers
-            self.offset += len(full_headers)
-            assert len(full_headers) % BLOCKSIZE == 0
-
-            if self.cur_fileobj is None: # no data with header
-                self.status = self.BEGIN
-                self._finish_fileobj()
-            else:
-                self.status = self.MIDDLE_OF_FILE
-                self.cur_pos_in_fileobj = 0
-            return 1
-        elif self.status == self.MIDDLE_OF_FILE:
-            # Add next chunk of self.cur_fileobj to self.buffer
-            l = min(self.BUFFER_ADDLEN,
-                    self.cur_tarinfo.size - self.cur_pos_in_fileobj)
-            s = self.cur_fileobj.read(l)
-            self.cur_pos_in_fileobj += len(s)
-            if len(s) == 0:
-                if l != 0: raise IOError, "end of file reached"
-                blocks, remainder = divmod(self.cur_tarinfo.size, BLOCKSIZE)
-                if remainder > 0:
-                    self.buffer += "\0" * (BLOCKSIZE - remainder)
-                    blocks += 1
-                self.cur_fileobj.close()
-                self.offset += blocks * BLOCKSIZE
-                self._finish_fileobj()
-                self.status = self.BEGIN
-            else: self.buffer += s
-            return 1
-        elif self.status == self.END: return None
-        assert 0
-
-    def _finish_fileobj(self):
-        """Update some variables when done writing fileobj"""
-        return # Skip saving tarinfo information to save memory
-        self.members.append(self.cur_tarinfo)
-        self.membernames.append(self.cur_tarinfo.name)
-        self.chunks.append(self.offset)
-
-    def _add_final(self):
-        """Add closing footer to buffer"""
-        blocks, remainder = divmod(self.offset, RECORDSIZE) #@UnusedVariable
-        if remainder > 0: self.buffer += "\0" * (RECORDSIZE - remainder)
-
-    def close(self):
-        """Close file obj"""
-        assert not self.closed
-        self.closed = 1
-
-
-def uid2uname(uid):
-    """Return uname of uid, or raise KeyError if none"""
-    if uid_dict is None: set_pwd_dict()
-    return uid_dict[uid]
-
-def uname2uid(uname):
-    """Return uid of given uname, or raise KeyError if none"""
-    if uname_dict is None: set_pwd_dict()
-    return uname_dict[uname]
-
-def set_pwd_dict():
-    """Set global pwd caching dictionaries uid_dict and uname_dict"""
-    global uid_dict, uname_dict
-    assert uid_dict is None and uname_dict is None and pwd
-    uid_dict = {}; uname_dict = {}
-    for entry in pwd.getpwall():
-        uname = entry[0]; uid = entry[2]
-        uid_dict[uid] = uname
-        uname_dict[uname] = uid
-
-def gid2gname(gid):
-    """Return group name of gid, or raise KeyError if none"""
-    if gid_dict is None: set_grp_dict()
-    return gid_dict[gid]
-
-def gname2gid(gname):
-    """Return gid of given group name, or raise KeyError if none"""
-    if gname_dict is None: set_grp_dict()
-    return gname_dict[gname]
-
-def set_grp_dict():
-    global gid_dict, gname_dict
-    assert gid_dict is None and gname_dict is None and grp
-    gid_dict = {}; gname_dict = {}
-    for entry in grp.getgrall():
-        gname = entry[0]; gid = entry[2]
-        gid_dict[gid] = gname
-        gname_dict[gname] = gid
+        t = open(name)
+        t.close()
+        return True
+    except TarError:
+        return False
+
+bltn_open = open
+open = TarFile.open

=== modified file 'duplicity/util.py'
--- duplicity/util.py	2011-08-18 19:17:55 +0000
+++ duplicity/util.py	2011-08-23 18:28:26 +0000
@@ -28,6 +28,8 @@
 import string
 import traceback
 
+from duplicity import tarfile
+
 import duplicity.globals as globals
 import duplicity.log as log
 
@@ -69,6 +71,31 @@
         else:
             raise
 
+class FakeTarFile:
+    debug = 0
+    def __iter__(self):
+        return iter([])
+    def close(self):
+        pass
+
+def make_tarfile(mode, fp):
+    # We often use 'empty' tarfiles for signatures that haven't been filled out
+    # yet.  So we want to ignore ReadError exceptions, which are used to signal
+    # this.
+    try:
+        return tarfile.TarFile("arbitrary", mode, fp)
+    except tarfile.ReadError:
+        return FakeTarFile()
+
+def get_tarinfo_name(ti):
+    # Python versions before 2.6 ensure that directories end with /, but 2.6
+    # and later ensure they they *don't* have /.  ::shrug::  Internally, we
+    # continue to use pre-2.6 method.
+    if ti.isdir() and not ti.name.endswith("/"):
+        return ti.name + "/"
+    else:
+        return ti.name
+
 def ignore_missing(fn, filename):
     """
     Execute fn on filename.  Ignore ENOENT errors, otherwise raise exception.

=== modified file 'rdiffdir'
--- rdiffdir	2010-11-20 15:39:00 +0000
+++ rdiffdir	2011-08-23 18:28:26 +0000
@@ -33,7 +33,6 @@
 from duplicity import diffdir
 from duplicity import patchdir
 from duplicity import log
-from duplicity import tarfile
 from duplicity import globals
 from duplicity import selection
 from duplicity import path

=== added file 'tarfile-CHANGES'
--- tarfile-CHANGES	1970-01-01 00:00:00 +0000
+++ tarfile-CHANGES	2011-08-23 18:28:26 +0000
@@ -0,0 +1,3 @@
+tarfile.py is a copy of python2.7's tarfile.py.
+
+No changes besides 2.4 compatibility have been made.

=== modified file 'tarfile-LICENSE'
--- tarfile-LICENSE	2002-10-29 01:49:46 +0000
+++ tarfile-LICENSE	2011-08-23 18:28:26 +0000
@@ -89,4 +89,4 @@
 README Version
 --------------
 
-$Id: tarfile-LICENSE,v 1.1 2002/10/29 01:49:46 bescoto Exp $
\ No newline at end of file
+$Id: tarfile-LICENSE,v 1.1 2002/10/29 01:49:46 bescoto Exp $

=== modified file 'testing/diffdirtest.py'
--- testing/diffdirtest.py	2010-11-20 15:32:59 +0000
+++ testing/diffdirtest.py	2011-08-23 18:28:26 +0000
@@ -26,6 +26,7 @@
 from duplicity.path import * #@UnusedWildImport
 from duplicity import diffdir
 from duplicity import selection
+from duplicity import util
 from duplicity import tarfile #@Reimport
 
 config.setup()
@@ -133,8 +134,9 @@
                          "snapshot/file_to_directory/"]
         for tarinfo in tarfile.TarFile("testfiles/output/dir1dir2.difftar",
                                        "r"):
-            if tarinfo.name in changed_files:
-                changed_files.remove(tarinfo.name)
+            tiname = util.get_tarinfo_name(tarinfo)
+            if tiname in changed_files:
+                changed_files.remove(tiname)
         assert not changed_files, ("Following files not found:\n"
                                    + "\n".join(changed_files))
 

=== modified file 'testing/patchdirtest.py'
--- testing/patchdirtest.py	2011-06-17 18:22:28 +0000
+++ testing/patchdirtest.py	2011-08-23 18:28:26 +0000
@@ -145,25 +145,18 @@
 
         def make_bad_tar(filename):
             """Write attack tarfile to filename"""
-            def iterate_one_pair(path):
-                """Iterate one (tarinfo, fp) pair
-
-                file object will be empty, and tarinfo will have path
-                "snapshot/../warning-security-error"
-
-                """
-                path.index = ("diff", "..", "warning-security-error")
-                ti = path.get_tarinfo()
-                fp = cStringIO.StringIO("")
-                yield (ti, fp)
+            tf = tarfile.TarFile(name=filename, mode="w")
+
+            # file object will be empty, and tarinfo will have path
+            # "snapshot/../warning-security-error"
             assert not os.system("cat /dev/null >testfiles/output/file")
-            tf = tarfile.TarFromIterator(iterate_one_pair(
-                Path("testfiles/output/file")))
-            tfbuf = tf.read()
+            path = Path("testfiles/output/file")
+            path.index = ("diff", "..", "warning-security-error")
+            ti = path.get_tarinfo()
+            fp = cStringIO.StringIO("")
+            tf.addfile(ti, fp)
 
-            fout = open(filename, "wb")
-            fout.write(tfbuf)
-            assert not fout.close()
+            tf.close()
 
         self.deltmp()
         make_bad_tar("testfiles/output/bad.tar")

=== modified file 'testing/test_tarfile.py'
--- testing/test_tarfile.py	2010-11-20 15:32:59 +0000
+++ testing/test_tarfile.py	2011-08-23 18:28:26 +0000
@@ -25,7 +25,7 @@
 # $Id: test_tarfile.py,v 1.11 2009/04/02 14:47:12 loafman Exp $
 
 import config
-import sys, os, shutil, StringIO, tempfile, unittest, stat, pwd, grp
+import sys, os, shutil, StringIO, tempfile, unittest, stat
 
 from duplicity import tarfile
 
@@ -187,52 +187,11 @@
             tf.add(filename, filename, 0)
         tf.close()
 
-    def make_temptar_iterator(self):
-        """Tar up tempdir using an iterator"""
-        try:
-            os.lstat("temp2.tar")
-        except OSError:
-            pass
-        else:
-            assert not os.system("rm temp2.tar")
-
-        self.make_tempdir()
-        def generate_pairs(tfi_list):
-            for filename in self.files_in_tempdir:
-                ti = tarfile.TarInfo()
-                ti.set_arcname(filename)
-                ti.init_from_stat(os.lstat(filename))
-                if filename == "tempdir/hardlinked2":
-                    ti.type = tarfile.LNKTYPE
-                    ti.linkname = "tempdir/hardlinked1"
-                    yield (ti, None)
-                elif filename == "tempdir" or filename == "tempdir/fifo":
-                    yield (ti, None)
-                elif filename == "tempdir/symlink":
-                    ti.linkname = os.readlink(filename)
-                    yield (ti, None)
-                else:
-                    yield (ti, open(filename, "rb"))
-        tfi_list = [None]
-        tfi = tarfile.TarFromIterator(generate_pairs(tfi_list))
-        tfi_list[0] = tfi # now generate_pairs can find tfi
-
-        buf = tfi.read()
-        tfi.close()
-        fout = open("temp2.tar", "wb")
-        fout.write(buf)
-        fout.close()
-
     def test_tarfile_creation(self):
         """Create directory, make tarfile, extract using gnutar, compare"""
         self.make_temptar()
         self.extract_and_compare_tarfile()
 
-    def test_tarfile_creation_from_iterator(self):
-        """Same as test_tarfile_creation, but use iterator interface"""
-        self.make_temptar_iterator()
-        self.extract_and_compare_tarfile()
-
     def extract_and_compare_tarfile(self):
         os.system("rm -r tempdir")
         assert not os.system("tar -xf temp2.tar")
@@ -354,51 +313,13 @@
     def seek(self, position):
         #print "Seeking to ", position
         return self.infp.seek(position)
+    def tell(self):
+        #print "Telling"
+        return self.infp.tell()
     def close(self):
         #print "Closing"
         return self.infp.close()
 
 
-class PasswordTest(unittest.TestCase):
-    """Test retrieving, storing password information"""
-    def compare(self, thunk1, thunk2):
-        """Make sure thunk1 and thunk2 return the same"""
-        try: result1 = thunk1()
-        except KeyError, exc1: keyerror = 1 #@UnusedVariable
-        else: keyerror = 0
-
-        try: result2 = thunk2()
-        except KeyError, exc2: #@UnusedVariable
-            assert keyerror, "Got KeyError vs " + str(result2)
-            return
-        else: assert not keyerror, "Got %s vs KeyError" % (str(result1),)
-
-        assert result1 == result2, (result1, result2)
-
-    def test_uid2uname(self):
-        """Test getting unames by uid"""
-        for uid in (0, 500, 789, 0, 0, 500):
-            self.compare(lambda: tarfile.uid2uname(uid),
-                         lambda: pwd.getpwuid(uid)[0])
-
-    def test_gid2gname(self):
-        """Test getting group names by gid"""
-        for gid in (0, 500, 789, 0, 0, 500):
-            self.compare(lambda: tarfile.gid2gname(gid),
-                         lambda: grp.getgrgid(gid)[0])
-
-    def test_gname2gid(self):
-        """Test getting gids from gnames"""
-        for gname in ('root', 'ben', 'bin', 'sanothua', 'root', 'root'):
-            self.compare(lambda: tarfile.gname2gid(gname),
-                         lambda: grp.getgrnam(gname)[2])
-
-    def test_uname2uid(self):
-        """Test getting uids from unames"""
-        for uname in ('root', 'ben', 'bin', 'sanothua', 'root', 'root'):
-            self.compare(lambda: tarfile.uname2uid(uname),
-                         lambda: pwd.getpwnam(uname)[2])
-
-
 if __name__ == "__main__":
     unittest.main()