Merge lp:~jelmer/bzr-builddeb/pristine-tar-refactor into lp:bzr-builddeb

[Jelmer Vernooij]

Move all pristine-tar related code into upstream/pristinetar.py.

This makes import_dsc.py a bit smaller, and makes it a bit easier to understand how pristine tar is involved. I'm looking at this so we can use pristine tar in bzr-builder too.

[James Westby (james-w)]

review: Approve

[lp:~jelmer/bzr-builddeb/pristine-tar-refactor]

563. By Jelmer Vernooij on 2011-06-04

Merge refactoring of pristine tar upstream source.

=== modified file 'cmds.py'
--- cmds.py	2011-05-16 14:21:31 +0000
+++ cmds.py	2011-06-03 13:28:25 +0000
@@ -93,7 +93,6 @@
 from bzrlib.plugins.builddeb.upstream import (
         AptSource,
         GetOrigSourceSource,
-        PristineTarSource,
         SelfSplitSource,
         TarfileSource,
         UScanSource,
@@ -103,6 +102,9 @@
         LazyUpstreamBranchSource,
         UpstreamBranchSource,
         )
+from bzrlib.plugins.builddeb.upstream.pristinetar import (
+        PristineTarSource,
+        )
 from bzrlib.plugins.builddeb.util import (
         FORMAT_3_0_QUILT,
         FORMAT_3_0_NATIVE,
 
=== modified file 'errors.py'
--- errors.py	2011-05-15 17:21:54 +0000
+++ errors.py	2011-06-03 13:28:25 +0000
@@ -163,13 +163,6 @@
         BzrError.__init__(self, location=location)
 
 
-class PristineTarError(BzrError):
-    _fmt = 'There was an error using pristine-tar: %(error)s.'
-
-    def __init__(self, error):
-        BzrError.__init__(self, error=error)
-
-
 class SharedUpstreamConflictsWithTargetPackaging(BzrError):
     _fmt = ('The upstream branches for the merge source and target have '
             'diverged. Unfortunately, the attempt to fix this problem '
 
=== modified file 'import_dsc.py'
--- import_dsc.py	2011-04-29 12:54:58 +0000
+++ import_dsc.py	2011-06-03 13:28:25 +0000
@@ -25,9 +25,6 @@
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #
 
-from base64 import (
-    standard_b64encode,
-    )
 
 import os
 import re
@@ -74,19 +71,18 @@
     FORMAT_3_0_NATIVE,
     export,
     get_commit_info_from_changelog,
-    make_pristine_tar_delta,
     md5sum_filename,
     open_file_via_transport,
     open_transport,
     safe_decode,
     subprocess_setup,
     )
-from bzrlib.plugins.builddeb.upstream import (
-    PristineTarSource,
-    )
 from bzrlib.plugins.builddeb.upstream.branch import (
     UpstreamBranchSource,
     )
+from bzrlib.plugins.builddeb.upstream.pristinetar import (
+    PristineTarSource,
+    )
 
 
 class DscCache(object):
@@ -912,16 +908,12 @@
         self.upstream_tree.set_parent_ids(upstream_parents)
         revprops = {"deb-md5": md5}
         if upstream_tarball is not None:
-            delta = self.make_pristine_tar_delta(
+            delta_revprops = self.pristine_tar_source.create_delta_revprops(
                 self.upstream_tree, upstream_tarball)
-            uuencoded = standard_b64encode(delta)
-            if upstream_tarball.endswith(".tar.bz2"):
-                revprops["deb-pristine-delta-bz2"] = uuencoded
-            else:
-                revprops["deb-pristine-delta"] = uuencoded
+            revprops.update(delta_revprops)
         if author is not None:
             revprops['authors'] = author
-        timezone=None
+        timezone = None
         if timestamp is not None:
             timezone = timestamp[1]
             timestamp = timestamp[0]
@@ -1075,7 +1067,7 @@
         if message is None:
             message = 'Import packaging changes for version %s' % \
                         (str(version),)
-        revprops={"deb-md5":md5}
+        revprops={"deb-md5": md5}
         if native:
             revprops['deb-native'] = "True"
         if authors:
@@ -1470,21 +1462,6 @@
         finally:
             shutil.rmtree(tempdir)
 
-    def make_pristine_tar_delta(self, tree, tarball_path):
-        tmpdir = tempfile.mkdtemp(prefix="builddeb-pristine-")
-        try:
-            dest = os.path.join(tmpdir, "orig")
-            tree.lock_read()
-            try:
-                for (dp, ie) in tree.inventory.iter_entries():
-                    ie._read_tree_state(dp, tree)
-                export(tree, dest, format='dir')
-            finally:
-                tree.unlock()
-            return make_pristine_tar_delta(dest, tarball_path)
-        finally:
-            shutil.rmtree(tmpdir)
-
 
 class SourceExtractor(object):
     """A class to extract a source package to its constituent parts"""
 
=== modified file 'tests/test_upstream.py'
--- tests/test_upstream.py	2011-04-29 12:34:12 +0000
+++ tests/test_upstream.py	2011-06-03 13:28:25 +0000
@@ -48,13 +48,11 @@
     )
 from bzrlib.plugins.builddeb.upstream import (
     AptSource,
-    PristineTarSource,
     StackedUpstreamSource,
     TarfileSource,
     UpstreamProvider,
     UpstreamSource,
     UScanSource,
-    Version,
     extract_tarball_version,
     )
 from bzrlib.plugins.builddeb.upstream.branch import (
@@ -66,6 +64,9 @@
     upstream_tag_to_version,
     upstream_version_add_revision
     )
+from bzrlib.plugins.builddeb.upstream.pristinetar import (
+    PristineTarSource,
+    )
 
 
 # Unless bug #712474 is fixed and available in the minimum bzrlib required, we
 
=== modified file 'upstream/__init__.py'
--- upstream/__init__.py	2011-04-29 12:34:12 +0000
+++ upstream/__init__.py	2011-06-03 13:28:25 +0000
@@ -25,21 +25,12 @@
 import tarfile
 import tempfile
 
-from base64 import (
-    standard_b64decode,
-    )
-
-
 try:
     from debian.changelog import Version
 except ImportError:
     # Prior to 0.1.15 the debian module was called debian_bundle
     from debian_bundle.changelog import Version
 
-from bzrlib.errors import (
-    NoSuchRevision,
-    NoSuchTag,
-    )
 from bzrlib.trace import (
     note,
     warning,
@@ -48,14 +39,11 @@
 from bzrlib.plugins.builddeb.errors import (
     MissingUpstreamTarball,
     PackageVersionNotPresent,
-    PerFileTimestampsNotSupported,
-    PristineTarError,
     WatchFileMissing,
     )
 from bzrlib.plugins.builddeb.repack_tarball import repack_tarball
 from bzrlib.plugins.builddeb.util import (
     export,
-    reconstruct_pristine_tar,
     tarball_name,
     )
 
@@ -106,135 +94,6 @@
                     format=format))
 
 
-class PristineTarSource(UpstreamSource):
-    """Source that uses the pristine-tar revisions in the packaging branch."""
-
-    def __init__(self, tree, branch):
-        self.branch = branch
-        self.tree = tree
-
-    def tag_name(self, version, distro=None):
-        """Gets the tag name for the upstream part of version.
-
-        :param version: the Version object to extract the upstream
-            part of the version number from.
-        :return: a String with the name of the tag.
-        """
-        assert isinstance(version, str)
-        if distro is None:
-            return "upstream-" + version
-        return "upstream-%s-%s" % (distro, version)
-
-    def fetch_tarball(self, package, version, target_dir):
-        revid = self.version_as_revision(package, version)
-        try:
-            rev = self.branch.repository.get_revision(revid)
-        except NoSuchRevision:
-            raise PackageVersionNotPresent(package, version, self)
-        note("Using pristine-tar to reconstruct the needed tarball.")
-        if self.has_pristine_tar_delta(rev):
-            format = self.pristine_tar_format(rev)
-        else:
-            format = 'gz'
-        target_filename = self._tarball_path(package, version,
-                                             target_dir, format=format)
-        try:
-            self.reconstruct_pristine_tar(revid, package, version, target_filename)
-        except PristineTarError:
-            raise PackageVersionNotPresent(package, version, self)
-        except PerFileTimestampsNotSupported:
-            raise PackageVersionNotPresent(package, version, self)
-        return target_filename
-
-    def _has_version(self, tag_name, md5=None):
-        if not self.branch.tags.has_tag(tag_name):
-            return False
-        revid = self.branch.tags.lookup_tag(tag_name)
-        self.branch.lock_read()
-        try:
-            graph = self.branch.repository.get_graph()
-            if not graph.is_ancestor(revid, self.branch.last_revision()):
-                return False
-        finally:
-            self.branch.unlock()
-        if md5 is None:
-            return True
-        rev = self.branch.repository.get_revision(revid)
-        try:
-            return rev.properties['deb-md5'] == md5
-        except KeyError:
-            warning("tag %s present in branch, but there is no "
-                "associated 'deb-md5' property" % tag_name)
-            return True
-
-    def version_as_revision(self, package, version):
-        assert isinstance(version, str)
-        for tag_name in self.possible_tag_names(version):
-            if self._has_version(tag_name):
-                return self.branch.tags.lookup_tag(tag_name)
-        tag_name = self.tag_name(version)
-        try:
-            return self.branch.tags.lookup_tag(tag_name)
-        except NoSuchTag:
-            raise PackageVersionNotPresent(package, version, self)
-
-    def has_version(self, package, version, md5=None):
-        assert isinstance(version, str), str(type(version))
-        for tag_name in self.possible_tag_names(version):
-            if self._has_version(tag_name, md5=md5):
-                return True
-        return False
-
-    def possible_tag_names(self, version):
-        assert isinstance(version, str)
-        tags = [self.tag_name(version),
-                self.tag_name(version, distro="debian"),
-                self.tag_name(version, distro="ubuntu"),
-                "upstream/%s" % version]
-        return tags
-
-    def has_pristine_tar_delta(self, rev):
-        return ('deb-pristine-delta' in rev.properties
-                or 'deb-pristine-delta-bz2' in rev.properties)
-
-    def pristine_tar_format(self, rev):
-        if 'deb-pristine-delta' in rev.properties:
-            return 'gz'
-        elif 'deb-pristine-delta-bz2' in rev.properties:
-            return 'bz2'
-        assert self.has_pristine_tar_delta(rev)
-        raise AssertionError("Not handled new delta type in "
-                "pristine_tar_format")
-
-    def pristine_tar_delta(self, rev):
-        if 'deb-pristine-delta' in rev.properties:
-            uuencoded = rev.properties['deb-pristine-delta']
-        elif 'deb-pristine-delta-bz2' in rev.properties:
-            uuencoded = rev.properties['deb-pristine-delta-bz2']
-        else:
-            assert self.has_pristine_tar_delta(rev)
-            raise AssertionError("Not handled new delta type in "
-                    "pristine_tar_delta")
-        return standard_b64decode(uuencoded)
-
-    def reconstruct_pristine_tar(self, revid, package, version,
-            dest_filename):
-        """Reconstruct a pristine-tar tarball from a bzr revision."""
-        tree = self.branch.repository.revision_tree(revid)
-        tmpdir = tempfile.mkdtemp(prefix="builddeb-pristine-")
-        try:
-            dest = os.path.join(tmpdir, "orig")
-            rev = self.branch.repository.get_revision(revid)
-            if self.has_pristine_tar_delta(rev):
-                export(tree, dest, format='dir')
-                delta = self.pristine_tar_delta(rev)
-                reconstruct_pristine_tar(dest, delta, dest_filename)
-            else:
-                export(tree, dest_filename, require_per_file_timestamps=True)
-        finally:
-            shutil.rmtree(tmpdir)
-
-
 class AptSource(UpstreamSource):
     """Upstream source that uses apt-source."""
 
 
=== added file 'upstream/pristinetar.py'
--- upstream/pristinetar.py	1970-01-01 00:00:00 +0000
+++ upstream/pristinetar.py	2011-06-03 13:28:25 +0000
@@ -0,0 +1,267 @@
+#    pristinetar.py -- Providers of upstream source
+#    Copyright (C) 2009-2011 Canonical Ltd.
+#    Copyright (C) 2009 Jelmer Vernooij <[e-mail]>
+#
+#    This file is part of bzr-builddeb.
+#
+#    bzr-builddeb is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 2 of the License, or
+#    (at your option) any later version.
+#
+#    bzr-builddeb is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with bzr-builddeb; if not, write to the Free Software
+#    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+from base64 import (
+    standard_b64decode,
+    standard_b64encode,
+    )
+import errno
+import os
+import shutil
+import subprocess
+import tempfile
+
+from bzrlib.plugins.builddeb.errors import (
+    PackageVersionNotPresent,
+    PerFileTimestampsNotSupported,
+    )
+from bzrlib.plugins.builddeb.upstream import UpstreamSource
+from bzrlib.plugins.builddeb.util import (
+    export,
+    subprocess_setup,
+    )
+
+from bzrlib import osutils
+from bzrlib.errors import (
+    BzrError,
+    NoSuchRevision,
+    NoSuchTag,
+    )
+from bzrlib.trace import (
+    note,
+    warning,
+    )
+
+
+class PristineTarError(BzrError):
+    _fmt = 'There was an error using pristine-tar: %(error)s.'
+
+    def __init__(self, error):
+        BzrError.__init__(self, error=error)
+
+
+def reconstruct_pristine_tar(dest, delta, dest_filename):
+    """Reconstruct a pristine tarball from a directory and a delta.
+
+    :param dest: Directory to pack
+    :param delta: pristine-tar delta
+    :param dest_filename: Destination filename
+    """
+    command = ["pristine-tar", "gentar", "-",
+               os.path.abspath(dest_filename)]
+    try:
+        proc = subprocess.Popen(command, stdin=subprocess.PIPE,
+                cwd=dest, preexec_fn=subprocess_setup,
+                stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    except OSError, e:
+        if e.errno == errno.ENOENT:
+            raise PristineTarError("pristine-tar is not installed")
+        else:
+            raise
+    (stdout, stderr) = proc.communicate(delta)
+    if proc.returncode != 0:
+        raise PristineTarError("Generating tar from delta failed: %s" % stdout)
+
+
+def make_pristine_tar_delta(dest, tarball_path):
+    """Create a pristine-tar delta for a tarball.
+
+    :param dest: Directory to generate pristine tar delta for
+    :param tarball_path: Path to the tarball
+    :return: pristine-tarball
+    """
+    # If tarball_path is relative, the cwd=dest parameter to Popen will make
+    # pristine-tar faaaail. pristine-tar doesn't use the VFS either, so we
+    # assume local paths.
+    tarball_path = osutils.abspath(tarball_path)
+    command = ["pristine-tar", "gendelta", tarball_path, "-"]
+    try:
+        proc = subprocess.Popen(command, stdout=subprocess.PIPE,
+                cwd=dest, preexec_fn=subprocess_setup,
+                stderr=subprocess.PIPE)
+    except OSError, e:
+        if e.errno == errno.ENOENT:
+            raise PristineTarError("pristine-tar is not installed")
+        else:
+            raise
+    (stdout, stderr) = proc.communicate()
+    if proc.returncode != 0:
+        raise PristineTarError("Generating delta from tar failed: %s" % stderr)
+    return stdout
+
+
+class PristineTarSource(UpstreamSource):
+    """Source that uses the pristine-tar revisions in the packaging branch."""
+
+    def __init__(self, tree, branch):
+        self.branch = branch
+        self.tree = tree
+
+    def tag_name(self, version, distro=None):
+        """Gets the tag name for the upstream part of version.
+
+        :param version: the Version object to extract the upstream
+            part of the version number from.
+        :return: a String with the name of the tag.
+        """
+        assert isinstance(version, str)
+        if distro is None:
+            return "upstream-" + version
+        return "upstream-%s-%s" % (distro, version)
+
+    def fetch_tarball(self, package, version, target_dir):
+        revid = self.version_as_revision(package, version)
+        try:
+            rev = self.branch.repository.get_revision(revid)
+        except NoSuchRevision:
+            raise PackageVersionNotPresent(package, version, self)
+        note("Using pristine-tar to reconstruct the needed tarball.")
+        if self.has_pristine_tar_delta(rev):
+            format = self.pristine_tar_format(rev)
+        else:
+            format = 'gz'
+        target_filename = self._tarball_path(package, version,
+                                             target_dir, format=format)
+        try:
+            self.reconstruct_pristine_tar(revid, package, version, target_filename)
+        except PristineTarError:
+            raise PackageVersionNotPresent(package, version, self)
+        except PerFileTimestampsNotSupported:
+            raise PackageVersionNotPresent(package, version, self)
+        return target_filename
+
+    def _has_version(self, tag_name, md5=None):
+        if not self.branch.tags.has_tag(tag_name):
+            return False
+        revid = self.branch.tags.lookup_tag(tag_name)
+        self.branch.lock_read()
+        try:
+            graph = self.branch.repository.get_graph()
+            if not graph.is_ancestor(revid, self.branch.last_revision()):
+                return False
+        finally:
+            self.branch.unlock()
+        if md5 is None:
+            return True
+        rev = self.branch.repository.get_revision(revid)
+        try:
+            return rev.properties['deb-md5'] == md5
+        except KeyError:
+            warning("tag %s present in branch, but there is no "
+                "associated 'deb-md5' property" % tag_name)
+            return True
+
+    def version_as_revision(self, package, version):
+        assert isinstance(version, str)
+        for tag_name in self.possible_tag_names(version):
+            if self._has_version(tag_name):
+                return self.branch.tags.lookup_tag(tag_name)
+        tag_name = self.tag_name(version)
+        try:
+            return self.branch.tags.lookup_tag(tag_name)
+        except NoSuchTag:
+            raise PackageVersionNotPresent(package, version, self)
+
+    def has_version(self, package, version, md5=None):
+        assert isinstance(version, str), str(type(version))
+        for tag_name in self.possible_tag_names(version):
+            if self._has_version(tag_name, md5=md5):
+                return True
+        return False
+
+    def possible_tag_names(self, version):
+        assert isinstance(version, str)
+        tags = [self.tag_name(version),
+                self.tag_name(version, distro="debian"),
+                self.tag_name(version, distro="ubuntu"),
+                "upstream/%s" % version]
+        return tags
+
+    def has_pristine_tar_delta(self, rev):
+        return ('deb-pristine-delta' in rev.properties
+                or 'deb-pristine-delta-bz2' in rev.properties)
+
+    def pristine_tar_format(self, rev):
+        if 'deb-pristine-delta' in rev.properties:
+            return 'gz'
+        elif 'deb-pristine-delta-bz2' in rev.properties:
+            return 'bz2'
+        assert self.has_pristine_tar_delta(rev)
+        raise AssertionError("Not handled new delta type in "
+                "pristine_tar_format")
+
+    def pristine_tar_delta(self, rev):
+        if 'deb-pristine-delta' in rev.properties:
+            uuencoded = rev.properties['deb-pristine-delta']
+        elif 'deb-pristine-delta-bz2' in rev.properties:
+            uuencoded = rev.properties['deb-pristine-delta-bz2']
+        else:
+            assert self.has_pristine_tar_delta(rev)
+            raise AssertionError("Not handled new delta type in "
+                    "pristine_tar_delta")
+        return standard_b64decode(uuencoded)
+
+    def reconstruct_pristine_tar(self, revid, package, version,
+            dest_filename):
+        """Reconstruct a pristine-tar tarball from a bzr revision."""
+        tree = self.branch.repository.revision_tree(revid)
+        tmpdir = tempfile.mkdtemp(prefix="builddeb-pristine-")
+        try:
+            dest = os.path.join(tmpdir, "orig")
+            rev = self.branch.repository.get_revision(revid)
+            if self.has_pristine_tar_delta(rev):
+                export(tree, dest, format='dir')
+                delta = self.pristine_tar_delta(rev)
+                reconstruct_pristine_tar(dest, delta, dest_filename)
+            else:
+                export(tree, dest_filename, require_per_file_timestamps=True)
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def make_pristine_tar_delta(self, tree, tarball_path):
+        tmpdir = tempfile.mkdtemp(prefix="builddeb-pristine-")
+        try:
+            dest = os.path.join(tmpdir, "orig")
+            tree.lock_read()
+            try:
+                for (dp, ie) in tree.inventory.iter_entries():
+                    ie._read_tree_state(dp, tree)
+                export(tree, dest, format='dir')
+            finally:
+                tree.unlock()
+            return make_pristine_tar_delta(dest, tarball_path)
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def create_delta_revprops(self, tree, tarball):
+        """Create the revision properties with the pristine tar delta.
+
+        :param tree: Bazaar Tree to diff against
+        :param tarball: The pristine tarball
+        :return: Dictionary with extra revision properties
+        """
+        ret = {}
+        delta = self.make_pristine_tar_delta(tree, tarball)
+        uuencoded = standard_b64encode(delta)
+        if tarball.endswith(".tar.bz2"):
+            ret["deb-pristine-delta-bz2"] = uuencoded
+        else:
+            ret["deb-pristine-delta"] = uuencoded
+        return ret
 
=== modified file 'util.py'
--- util.py	2011-04-08 18:47:10 +0000
+++ util.py	2011-06-03 13:28:25 +0000
@@ -22,10 +22,8 @@
     import hashlib as md5
 except ImportError:
     import md5
-import errno
 import signal
 import shutil
-import subprocess
 import tempfile
 import os
 import re
@@ -43,7 +41,6 @@
 from bzrlib import (
     bugtracker,
     errors,
-    osutils,
     urlutils,
     version_info as bzr_version_info,
     )
@@ -69,7 +66,6 @@
     AddChangelogError,
     InconsistentSourceFormatError,
     NoPreviousUpload,
-    PristineTarError,
     UnableToFindPreviousUpload,
     UnparseableChangelog,
     )
@@ -686,51 +682,4 @@
         return BUILD_TYPE_NORMAL
 
 
-def reconstruct_pristine_tar(dest, delta, dest_filename):
-    """Reconstruct a pristine tarball from a directory and a delta.
-
-    :param dest: Directory to pack
-    :param delta: pristine-tar delta
-    :param dest_filename: Destination filename
-    """
-    command = ["pristine-tar", "gentar", "-",
-               os.path.abspath(dest_filename)]
-    try:
-        proc = subprocess.Popen(command, stdin=subprocess.PIPE,
-                cwd=dest, preexec_fn=subprocess_setup,
-                stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    except OSError, e:
-        if e.errno == errno.ENOENT:
-            raise PristineTarError("pristine-tar is not installed")
-        else:
-            raise
-    (stdout, stderr) = proc.communicate(delta)
-    if proc.returncode != 0:
-        raise PristineTarError("Generating tar from delta failed: %s" % stdout)
-
-
-def make_pristine_tar_delta(dest, tarball_path):
-    """Create a pristine-tar delta for a tarball.
-
-    :param dest: Directory to generate pristine tar delta for
-    :param tarball_path: Path to the tarball
-    :return: pristine-tarball
-    """
-    # If tarball_path is relative, the cwd=dest parameter to Popen will make
-    # pristine-tar faaaail. pristine-tar doesn't use the VFS either, so we
-    # assume local paths.
-    tarball_path = osutils.abspath(tarball_path)
-    command = ["pristine-tar", "gendelta", tarball_path, "-"]
-    try:
-        proc = subprocess.Popen(command, stdout=subprocess.PIPE,
-                cwd=dest, preexec_fn=subprocess_setup,
-                stderr=subprocess.PIPE)
-    except OSError, e:
-        if e.errno == errno.ENOENT:
-            raise PristineTarError("pristine-tar is not installed")
-        else:
-            raise
-    (stdout, stderr) = proc.communicate()
-    if proc.returncode != 0:
-        raise PristineTarError("Generating delta from tar failed: %s" % stderr)
-    return stdout
+