Merge lp:~jelmer/bzr-builddeb/multiple-upstream-tarballs-pt2 into lp:bzr-builddeb

[Jelmer Vernooij]

Step 2 of support for multiple upstream tarballs.

Pass lists of orig tarballs along in more places rather than a single upstream tarball and md5 sum.

[lp:~jelmer/bzr-builddeb/multiple-upstream-tarballs-pt2]

577. By Jelmer Vernooij on 2011-06-15

Factor out extracting of orig tarballs.

[James Westby (james-w)]

62 + db.import_upstream_tarball(tarball_filenames, version, parents)

Should this perhaps be import_upstream_tarballs now?

Thanks,

James

=== modified file 'cmds.py'
--- cmds.py	2011-06-14 14:04:53 +0000
+++ cmds.py	2011-06-15 02:38:47 +0000
@@ -69,7 +69,6 @@
     BuildFailedError,
     DchError,
     MissingChangelogError,
-    MultipleUpstreamTarballsNotSupported,
     NoPreviousUpload,
     PackageVersionNotPresent,
     StrictBuildFailed,
@@ -117,6 +116,7 @@
         get_source_format,
         guess_build_type,
         lookup_distribution,
+        md5sum_filename,
         open_file,
         open_file_via_transport,
         tarball_name,
@@ -560,9 +560,7 @@
         db = DistributionBranch(tree.branch, None, tree=tree)
         dbs = DistributionBranchSet()
         dbs.add_branch(db)
-        if len(tarball_filenames) > 1:
-            raise MultipleUpstreamTarballsNotSupported()
-        conflicts = db.merge_upstream(tarball_filenames[0], package, version,
+        conflicts = db.merge_upstream(tarball_filenames, package, version,
                 current_version, upstream_branch=upstream_branch,
                 upstream_revision=upstream_revision,
                 merge_type=merge_type, force=force)
@@ -977,7 +975,8 @@
         else:
             raise BzrCommandError('bzr import-upstream --revision takes exactly'
                                   ' one revision specifier.')
-        tag_name, _ = db.import_upstream_tarball(location, version, parents,
+        tarballs = [(location, md5sum_filename(location))]
+        tag_name, _ = db.import_upstream_tarball(tarballs, version, parents,
             upstream_branch=upstream, upstream_revision=upstream_revid)
         self.outf.write('Imported %s as tag:%s.\n' % (location, tag_name))
 
@@ -1207,7 +1206,7 @@
 
     def run(self, package_name, version, tarball, bzr_only=None, v3=None):
         tree = dh_make.import_upstream(tarball, package_name,
-                version.encode("utf-8"), use_v3=v3)
+            version.encode("utf-8"), use_v3=v3)
         if not bzr_only:
             tree.lock_write()
             try:

=== modified file 'dh_make.py'
--- dh_make.py	2011-06-14 14:04:53 +0000
+++ dh_make.py	2011-06-15 02:38:47 +0000
@@ -87,10 +87,7 @@
             upstream_tree=tree)
     dbs = import_dsc.DistributionBranchSet()
     dbs.add_branch(db)
-    if len(tarball_filenames) > 1:
-        raise errors.MultipleUpstreamTarballsNotSupported()
-    (tarball_filename, md5sum) = tarball_filenames[0]
-    db.import_upstream_tarball(tarball_filename, version, parents, md5sum=md5sum)
+    db.import_upstream_tarball(tarball_filenames, version, parents)
     return tree
 
 

=== modified file 'import_dsc.py'
--- import_dsc.py	2011-06-09 21:43:07 +0000
+++ import_dsc.py	2011-06-15 02:38:47 +0000
@@ -27,7 +27,6 @@
 
 
 import os
-import re
 import shutil
 import stat
 import subprocess
@@ -63,14 +62,14 @@
 from bzrlib.plugins.builddeb.bzrtools_import import import_dir
 from bzrlib.plugins.builddeb.errors import (
     MultipleUpstreamTarballsNotSupported,
-    TarFailed,
     UpstreamAlreadyImported,
     UpstreamBranchAlreadyMerged,
     )
 from bzrlib.plugins.builddeb.util import (
+    FORMAT_1_0,
     FORMAT_3_0_QUILT,
     FORMAT_3_0_NATIVE,
-    export,
+    extract_orig_tarballs,
     get_commit_info_from_changelog,
     md5sum_filename,
     open_file_via_transport,
@@ -301,6 +300,30 @@
                 pass
         return False
 
+    def _has_upstream_version(self, branch, tag_name, tarballs=None):
+        if branch.tags.has_tag(tag_name):
+            revid = branch.tags.lookup_tag(tag_name)
+            branch.lock_read()
+            try:
+                graph = branch.repository.get_graph()
+                if not graph.is_ancestor(revid, branch.last_revision()):
+                    return False
+            finally:
+                branch.unlock()
+            if tarballs is None:
+                return True
+            if len(tarballs) != 1:
+                raise MultipleUpstreamTarballsNotSupported()
+            (filename, md5) = tarballs[0]
+            rev = branch.repository.get_revision(revid)
+            try:
+                return rev.properties['deb-md5'] == md5
+            except KeyError:
+                warning("tag %s present in branch, but there is no "
+                    "associated 'deb-md5' property" % tag_name)
+                pass
+        return False
+
     def has_version(self, version, md5=None):
         """Whether this branch contains the package version specified.
 
@@ -326,7 +349,7 @@
             return True
         return False
 
-    def has_upstream_version(self, version, md5=None):
+    def has_upstream_version(self, version, tarballs=None):
         """Whether this branch contains the upstream version specified.
 
         The version must be judged present by having the appropriate tag
@@ -336,13 +359,13 @@
 
         :param version: a upstream version number to look for in the upstream 
             branch.
-        :param md5: a string with the md5sum that if not None must be
-            associated with the revision.
+        :param tarballs: list of upstream tarballs that should be present,
+            tuples of filename and md5sum
         :return: True if the upstream branch contains the specified upstream
             version of the package. False otherwise.
         """
         for tag_name in self.pristine_tar_source.possible_tag_names(version):
-            if self._has_version(self.upstream_branch, tag_name, md5=md5):
+            if self._has_upstream_version(self.upstream_branch, tag_name, tarballs=tarballs):
                 return True
         return False
 
@@ -590,7 +613,7 @@
         finally:
             self.branch.unlock()
 
-    def branch_to_pull_upstream_from(self, version, md5):
+    def branch_to_pull_upstream_from(self, version, upstream_tarballs):
         """Checks whether this upstream is a pull from a lesser branch.
 
         Looks in all the other upstream branches for the given
@@ -603,20 +626,15 @@
 
         :param version: the upstream version to use when searching in the 
             lesser branches.
-        :param md5: a String containing the md5 associateed with the
-            upstream version.
         :return: a DistributionBranch object to pull the upstream from
             if that is what should be done, otherwise None.
         """
         assert isinstance(version, str)
-        assert md5 is not None, \
-            ("It's not a good idea to use branch_to_pull_upstream_from with "
-             "md5 == None, as you may pull the wrong revision.")
         up_branch = self.upstream_branch
         up_branch.lock_read()
         try:
             for branch in reversed(self.get_lesser_branches()):
-                if branch.has_upstream_version(version, md5=md5):
+                if branch.has_upstream_version(version, tarballs=upstream_tarballs):
                     # Check that they haven't diverged
                     other_up_branch = branch.upstream_branch
                     other_up_branch.lock_read()
@@ -629,7 +647,7 @@
                     finally:
                         other_up_branch.unlock()
             for branch in self.get_greater_branches():
-                if branch.has_upstream_version(version, md5=md5):
+                if branch.has_upstream_version(version, tarballs=upstream_tarballs):
                     # Check that they haven't diverged
                     other_up_branch = branch.upstream_branch
                     other_up_branch.lock_read()
@@ -837,8 +855,8 @@
         self.branch.fetch(self.upstream_branch, last_revision=revid)
         self.upstream_branch.tags.merge_to(self.branch.tags)
 
-    def import_upstream(self, upstream_part, version, md5, upstream_parents,
-            upstream_tarball=None, upstream_branch=None,
+    def import_upstream(self, upstream_part, version, upstream_parents,
+            upstream_tarballs=None, upstream_branch=None,
             upstream_revision=None, timestamp=None, author=None,
             file_ids_from=None):
         """Import an upstream part on to the upstream branch.
@@ -849,7 +867,6 @@
         :param upstream_part: the path of a directory containing the
             unpacked upstream part of the source package.
         :param version: upstream version that is being imported
-        :param md5: the md5 of the upstream part.
         :param upstream_parents: the parents to give the upstream revision
         :param timestamp: a tuple of (timestamp, timezone) to use for
             the commit, or None to use the current time.
@@ -901,8 +918,12 @@
         finally:
             self_tree.unlock()
         self.upstream_tree.set_parent_ids(upstream_parents)
-        revprops = {"deb-md5": md5}
-        if upstream_tarball is not None:
+        revprops = {}
+        if upstream_tarballs is not None:
+            if len(upstream_tarballs) != 1:
+                raise MultipleUpstreamTarballsNotSupported()
+            (upstream_tarball, md5) = upstream_tarballs[0]
+            revprops["deb-md5"] = md5
             delta_revprops = self.pristine_tar_source.create_delta_revprops(
                 self.upstream_tree, upstream_tarball)
             revprops.update(delta_revprops)
@@ -918,8 +939,8 @@
         tag_name, _ = self.tag_upstream_version(version, revid=revid)
         return tag_name, revid
 
-    def import_upstream_tarball(self, tarball_filename, version, parents,
-        md5sum=None, upstream_branch=None, upstream_revision=None):
+    def import_upstream_tarball(self, tarballs, version, parents,
+        upstream_branch=None, upstream_revision=None):
         """Import an upstream part to the upstream branch.
 
         :param tarball_filename: The tarball to import.
@@ -933,12 +954,10 @@
         :param md5sum: hex digest of the md5sum of the tarball, if known.
         :return: (tag_name, revision_id) of the imported tarball.
         """
-        if not md5sum:
-            md5sum = md5sum_filename(tarball_filename)
-        tarball_dir = self._extract_tarball_to_tempdir(tarball_filename)
+        tarball_dir = self._extract_tarballs_to_tempdir(tarballs)
         try:
-            return self.import_upstream(tarball_dir, version, md5sum, parents,
-                upstream_tarball=tarball_filename,
+            return self.import_upstream(tarball_dir, version, parents,
+                tarballs,
                 upstream_branch=upstream_branch,
                 upstream_revision=upstream_revision)
         finally:
@@ -1132,16 +1151,28 @@
         cl.parse_changelog(open(cl_filename).read(), strict=False)
         return cl
 
-    def _do_import_package(self, version, versions, debian_part, md5,
-            upstream_part, upstream_md5, upstream_tarball=None,
-            timestamp=None, author=None, file_ids_from=None,
-            pull_debian=True):
+    def _import_normal_package(self, version, versions, debian_part, md5,
+            upstream_part, upstream_tarballs, timestamp=None, author=None,
+            file_ids_from=None, pull_debian=True):
+        """Import a source package.
+
+        :param version: Full Debian version
+        :param versions: Safe versions from changelog
+        :param debian_part: Path to extracted directory with Debian changes
+        :param unextracted_debian_md5: MD5 sum of unextracted Debian diff/tarball
+        :param upstream_part: Extracted upstream directory
+        :param upstream_tarballs: List of tuples with (upstream tarfile, md5sum)
+        :param timestamp: Version timestamp according to changelog
+        :param author: Author according to changelog
+        :param file_ids_from: Sequence of trees to take file ids from
+        :param pull_debian: Whether to pull from the Debian branch
+        """
         pull_branch = None
         if pull_debian:
             pull_branch = self.branch_to_pull_version_from(version, md5)
         if pull_branch is not None:
             if (self.branch_to_pull_upstream_from(version.upstream_version,
-                        upstream_md5)
+                        upstream_tarballs)
                     is None):
                 pull_branch = None
         if pull_branch is not None:
@@ -1153,7 +1184,7 @@
             if not self.has_upstream_version(version.upstream_version):
                 up_pull_branch = \
                     self.branch_to_pull_upstream_from(version.upstream_version,
-                            upstream_md5)
+                            upstream_tarballs)
                 if up_pull_branch is not None:
                     self.pull_upstream_from_branch(up_pull_branch,
                             version.upstream_version)
@@ -1165,8 +1196,8 @@
                             version.upstream_version)
                     _, new_revid = self.import_upstream(upstream_part,
                             version.upstream_version,
-                            upstream_md5, upstream_parents,
-                            upstream_tarball=upstream_tarball,
+                            upstream_parents,
+                            upstream_tarballs=upstream_tarballs,
                             timestamp=timestamp, author=author,
                             file_ids_from=file_ids_from)
                     self._fetch_upstream_to_branch(new_revid)
@@ -1250,7 +1281,7 @@
         base_path = osutils.dirname(dsc_filename)
         dsc = deb822.Dsc(open(dsc_filename).read())
         version = Version(dsc['Version'])
-        format = dsc.get('Format', '1.0').strip()
+        format = dsc.get('Format', FORMAT_1_0).strip()
         extractor_cls = SOURCE_EXTRACTORS.get(format)
         if extractor_cls is None:
             raise AssertionError("Don't know how to import source format %s yet"
@@ -1274,13 +1305,13 @@
             #TODO: check that the versions list is correctly ordered,
             # as some methods assume that, and it's not clear what
             # should happen if it isn't.
+
             if extractor.extracted_upstream is not None:
-                self._do_import_package(version, versions,
+                self._import_normal_package(version, versions,
                         extractor.extracted_debianised,
                         extractor.unextracted_debian_md5,
                         extractor.extracted_upstream,
-                        extractor.unextracted_upstream_md5,
-                        upstream_tarball=extractor.unextracted_upstream,
+                        extractor.upstream_tarballs,
                         timestamp=timestamp, author=author,
                         file_ids_from=file_ids_from,
                         pull_debian=pull_debian)
@@ -1347,19 +1378,11 @@
         if root_id:
             self.upstream_tree.set_root_id(root_id)
 
-    def _extract_tarball_to_tempdir(self, tarball_filename):
+    def _extract_tarballs_to_tempdir(self, tarballs):
         tempdir = tempfile.mkdtemp()
-        if tarball_filename.endswith(".tar.bz2"):
-            tar_args = 'xjf'
-        else:
-            tar_args = 'xzf'
         try:
-            proc = subprocess.Popen(["tar", tar_args, tarball_filename, "-C",
-                    tempdir, "--strip-components", "1"],
-                    preexec_fn=subprocess_setup)
-            proc.communicate()
-            if proc.returncode != 0:
-                raise TarFailed("extract", tarball_filename)
+            extract_orig_tarballs([fn for (fn, md5) in tarballs], tempdir,
+                strip_components=1)
             return tempdir
         except:
             shutil.rmtree(tempdir)
@@ -1382,7 +1405,7 @@
                 previous_version,
                 self.pristine_tar_source.tag_name(previous_version)))
 
-    def merge_upstream(self, tarball_filename, package, version, previous_version,
+    def merge_upstream(self, tarball_filenames, package, version, previous_version,
             upstream_branch=None, upstream_revision=None, merge_type=None,
             force=False):
         assert self.upstream_branch is None, \
@@ -1412,17 +1435,17 @@
                     if not force and graph.is_ancestor(upstream_revision,
                             self.branch.last_revision()):
                         raise UpstreamBranchAlreadyMerged
-                tarball_filename = os.path.abspath(tarball_filename)
-                md5sum = md5sum_filename(tarball_filename)
-                tarball_dir = self._extract_tarball_to_tempdir(tarball_filename)
+                upstream_tarballs = [
+                    (os.path.abspath(fn), md5sum_filename(fn)) for fn in
+                    tarball_filenames]
+                tarball_dir = self._extract_tarballs_to_tempdir(upstream_tarballs)
                 try:
                     # FIXME: should use upstream_parents()?
                     parents = []
                     if self.upstream_branch.last_revision() != NULL_REVISION:
                         parents = [self.upstream_branch.last_revision()]
                     _, new_revid = self.import_upstream(tarball_dir,
-                            version,
-                            md5sum, parents, upstream_tarball=tarball_filename,
+                            version, parents, upstream_tarballs=upstream_tarballs,
                             upstream_branch=upstream_branch,
                             upstream_revision=upstream_revision)
                     self._fetch_upstream_to_branch(new_revid)
@@ -1466,9 +1489,22 @@
         self.dsc = dsc
         self.extracted_upstream = None
         self.extracted_debianised = None
-        self.unextracted_upstream = None
         self.unextracted_debian_md5 = None
-        self.unextracted_upstream_md5 = None
+        self.upstream_tarballs = []
+        self.tempdir = None
+
+    def extract(self):
+        """Extract the package to a new temporary directory."""
+        raise NotImplementedError(self.extract)
+
+    def cleanup(self):
+        """Cleanup any extracted files."""
+        if self.tempdir is not None and os.path.isdir(self.tempdir):
+            shutil.rmtree(self.tempdir)
+
+
+class OneZeroSourceExtractor(SourceExtractor):
+    """Source extract for the "1.0" source format."""
 
     def extract(self):
         """Extract the package to a new temporary directory."""
@@ -1495,20 +1531,15 @@
                     self.unextracted_debian_md5 = part['md5sum']
             else:
                 if part['name'].endswith(".orig.tar.gz"):
-                    assert self.unextracted_upstream is None, "Two .orig.tar.gz?"
-                    self.unextracted_upstream = os.path.abspath(
+                    self.upstream_tarballs.append((os.path.abspath(
                             os.path.join(osutils.dirname(self.dsc_path),
-                                part['name']))
-                    self.unextracted_upstream_md5 = part['md5sum']
+                                part['name'])), part['md5sum']))
                 elif part['name'].endswith(".diff.gz"):
                     self.unextracted_debian_md5 = part['md5sum']
 
-    def cleanup(self):
-        if os.path.exists(self.tempdir):
-            shutil.rmtree(self.tempdir)
-
 
 class ThreeDotZeroNativeSourceExtractor(SourceExtractor):
+    """Source extractor for the "3.0 (native)" source format."""
 
     def extract(self):
         self.tempdir = tempfile.mkdtemp()
@@ -1531,6 +1562,7 @@
 
 
 class ThreeDotZeroQuiltSourceExtractor(SourceExtractor):
+    """Source extractor for the "3.0 (quilt)" source format."""
 
     def extract(self):
         self.tempdir = tempfile.mkdtemp()
@@ -1560,25 +1592,20 @@
         subprocess.call(["find", self.extracted_debianised, "-perm",
                 "0000", "-exec", "chmod", "644", "{}", ";"])
         for part in self.dsc['files']:
-            if (re.search("\.orig-[^.]+\.tar\.(gz|bz2|lzma)$", part['name'])):
-                raise MultipleUpstreamTarballsNotSupported()
-            if (part['name'].endswith(".orig.tar.gz")
-                    or part['name'].endswith(".orig.tar.bz2")):
-                assert self.unextracted_upstream is None, "Two .orig.tar.(gz|bz2)?"
-                self.unextracted_upstream = os.path.abspath(
+            if part['name'].startswith("%s_%s.orig" % (name, str(version.upstream_version))):
+                self.upstream_tarballs.append((os.path.abspath(
                         os.path.join(osutils.dirname(self.dsc_path),
-                            part['name']))
-                self.unextracted_upstream_md5 = part['md5sum']
+                            part['name'])), part['md5sum']))
             elif (part['name'].endswith(".debian.tar.gz")
                     or part['name'].endswith(".debian.tar.bz2")):
                 self.unextracted_debian_md5 = part['md5sum']
-        assert self.unextracted_upstream is not None, \
+        assert self.upstream_tarballs is not None, \
             "Can't handle non gz|bz2 tarballs yet"
         assert self.unextracted_debian_md5 is not None, \
             "Can't handle non gz|bz2 tarballs yet"
 
 
 SOURCE_EXTRACTORS = {}
-SOURCE_EXTRACTORS["1.0"] = SourceExtractor
+SOURCE_EXTRACTORS[FORMAT_1_0] = OneZeroSourceExtractor
 SOURCE_EXTRACTORS[FORMAT_3_0_NATIVE] = ThreeDotZeroNativeSourceExtractor
 SOURCE_EXTRACTORS[FORMAT_3_0_QUILT] = ThreeDotZeroQuiltSourceExtractor

=== modified file 'source_distiller.py'
--- source_distiller.py	2011-06-14 14:04:53 +0000
+++ source_distiller.py	2011-06-15 02:38:47 +0000
@@ -20,20 +20,15 @@
 import glob
 import os
 import shutil
-import subprocess
 import tempfile
 
 from bzrlib import errors as bzr_errors
 
-from bzrlib.plugins.builddeb.errors import (
-        MultipleUpstreamTarballsNotSupported,
-        TarFailed,
-        )
 from bzrlib.plugins.builddeb.util import (
         export,
+        extract_orig_tarballs,
         get_parent_dir,
         recursive_copy,
-        subprocess_setup,
         )
 
 
@@ -123,16 +118,10 @@
             os.makedirs(parent_dir)
         if not self.use_existing:
             tarballs = self.upstream_provider.provide(parent_dir)
-            if len(tarballs) > 1:
-                raise MultipleUpstreamTarballsNotSupported()
-            tarball = tarballs[0]
             # Extract it to the right place
             tempdir = tempfile.mkdtemp(prefix='builddeb-merge-')
             try:
-                ret = subprocess.call(['tar', '-C', tempdir, '-xf', tarball],
-                        preexec_fn=subprocess_setup)
-                if ret != 0:
-                    raise TarFailed("uncompress", tarball)
+                extract_orig_tarballs(tarballs, tempdir)
                 files = glob.glob(tempdir+'/*')
                 # If everything is in a single dir then move everything up one
                 # level.
@@ -151,7 +140,7 @@
             tempdir = os.path.join(basetempdir,"export")
             if self.larstiq:
                 os.makedirs(tempdir)
-                export_dir = os.path.join(tempdir,'debian')
+                export_dir = os.path.join(tempdir, 'debian')
             else:
                 export_dir = tempdir
             if self.is_working_tree:

=== modified file 'tests/blackbox/test_merge_upstream.py'
--- tests/blackbox/test_merge_upstream.py	2011-06-13 23:01:14 +0000
+++ tests/blackbox/test_merge_upstream.py	2011-06-15 02:38:47 +0000
@@ -30,6 +30,9 @@
     DistributionBranch,
     DistributionBranchSet,
     )
+from bzrlib.plugins.builddeb.util import (
+    md5sum_filename,
+    )
 
 
 class Fixture(object):
@@ -91,10 +94,11 @@
                 upstream_tree=tree)
         dbs = DistributionBranchSet()
         dbs.add_branch(db)
-        db.import_upstream_tarball(self.tar.tarball, str(self.tar.version),
-                [tree.branch.last_revision()])
+        db.import_upstream_tarball(
+            [(self.tar.tarball, md5sum_filename(self.tar.tarball))], str(self.tar.version),
+            [tree.branch.last_revision()])
         package_builder = SourcePackageBuilder("foo",
-                str(self.tar.version)+"-1")
+            str(self.tar.version)+"-1")
         package_builder.add_default_control()
         package_builder.write_debian_files(branchpath)
         tree.smart_add([tree.basedir])

=== modified file 'tests/test_import_dsc.py'
--- tests/test_import_dsc.py	2011-06-10 11:47:25 +0000
+++ tests/test_import_dsc.py	2011-06-15 02:38:47 +0000
@@ -35,13 +35,10 @@
         tests,
         )
 
-from bzrlib.plugins.builddeb.errors import (
-        MultipleUpstreamTarballsNotSupported,
-        )
 from bzrlib.plugins.builddeb.import_dsc import (
         DistributionBranch,
         DistributionBranchSet,
-        SourceExtractor,
+        OneZeroSourceExtractor,
         SOURCE_EXTRACTORS,
         ThreeDotZeroNativeSourceExtractor,
         ThreeDotZeroQuiltSourceExtractor,
@@ -50,6 +47,9 @@
         BuilddebTestCase,
         SourcePackageBuilder,
         )
+from bzrlib.plugins.builddeb.util import (
+        md5sum_filename,
+        )
 
 
 class _PristineTarFeature(tests.Feature):
@@ -175,20 +175,27 @@
         db = self.db1
         version = "0.1"
         self.assertFalse(db.has_upstream_version(version))
-        self.assertFalse(db.has_upstream_version(version, self.fake_md5_1))
+        self.assertFalse(db.has_upstream_version(version,
+            [("foo.tar.gz", self.fake_md5_1)]))
         self.do_commit_with_md5(self.up_tree1, "one", self.fake_md5_1)
         db.tag_upstream_version(version)
         self.assertTrue(db.has_upstream_version(version))
-        self.assertTrue(db.has_upstream_version(version, self.fake_md5_1))
-        self.assertFalse(db.has_upstream_version(version, self.fake_md5_2))
+        self.assertTrue(db.has_upstream_version(
+            version, [("foo.tar.gz", self.fake_md5_1)]))
+        self.assertFalse(db.has_upstream_version(version,
+            [("foo.tar.gz", self.fake_md5_2)]))
         version = "0.1"
         self.assertTrue(db.has_upstream_version(version))
-        self.assertTrue(db.has_upstream_version(version, self.fake_md5_1))
-        self.assertFalse(db.has_upstream_version(version, self.fake_md5_2))
+        self.assertTrue(db.has_upstream_version(version,
+            [("foo.tar.gz", self.fake_md5_1)]))
+        self.assertFalse(db.has_upstream_version(version,
+            [("foo.tar.gz", self.fake_md5_2)]))
         version = "0.2"
         self.assertFalse(db.has_upstream_version(version))
-        self.assertFalse(db.has_upstream_version(version, self.fake_md5_1))
-        self.assertFalse(db.has_upstream_version(version, self.fake_md5_2))
+        self.assertFalse(db.has_upstream_version(version,
+            [("foo.tar.gz", self.fake_md5_1)]))
+        self.assertFalse(db.has_upstream_version(version,
+            [("foo.tar.gz", self.fake_md5_2)]))
 
     def test_revid_of_version(self):
         db = self.db1
@@ -633,46 +640,46 @@
         version2 = Version("0.2-1")
         # With no versions tagged everything is None
         branch = self.db2.branch_to_pull_upstream_from(
-                version1.upstream_version, self.fake_md5_1)
+                version1.upstream_version, [("foo.tar.gz", self.fake_md5_1)])
         self.assertEqual(branch, None)
         branch = self.db2.branch_to_pull_upstream_from(
-                version1.upstream_version, self.fake_md5_2)
+                version1.upstream_version, [("foo.tar.gz", self.fake_md5_2)])
         self.assertEqual(branch, None)
         branch = self.db1.branch_to_pull_upstream_from(
-                version1.upstream_version, self.fake_md5_1)
+                version1.upstream_version, [("foo.tar.gz", self.fake_md5_1)])
         self.assertEqual(branch, None)
         self.do_commit_with_md5(self.up_tree1, "one", self.fake_md5_1)
         self.db1.tag_upstream_version(version1.upstream_version)
         # Version and md5 available, so we get the correct branch.
         branch = self.db2.branch_to_pull_upstream_from(
-                version1.upstream_version, self.fake_md5_1)
+                version1.upstream_version, [("foo.tar.gz", self.fake_md5_1)])
         self.assertEqual(branch, self.db1)
         # Otherwise (different version or md5) then we get None
         branch = self.db2.branch_to_pull_upstream_from(
-                version1.upstream_version, self.fake_md5_2)
-        self.assertEqual(branch, None)
-        branch = self.db2.branch_to_pull_upstream_from(version2.upstream_version,
-                self.fake_md5_1)
-        self.assertEqual(branch, None)
-        branch = self.db2.branch_to_pull_upstream_from(version2.upstream_version,
-                self.fake_md5_2)
+                version1.upstream_version, [("foo.tar.gz", self.fake_md5_2)])
+        self.assertEqual(branch, None)
+        branch = self.db2.branch_to_pull_upstream_from(version2.upstream_version,
+                [("foo.tar.gz", self.fake_md5_1)])
+        self.assertEqual(branch, None)
+        branch = self.db2.branch_to_pull_upstream_from(version2.upstream_version,
+                [("foo.tar.gz", self.fake_md5_2)])
         self.assertEqual(branch, None)
         # And we don't get a branch for the one that already has
         # the version
         branch = self.db1.branch_to_pull_upstream_from(
-                version1.upstream_version, self.fake_md5_1)
+                version1.upstream_version, [("foo.tar.gz", self.fake_md5_1)])
         self.assertEqual(branch, None)
         self.up_tree2.pull(self.up_tree1.branch)
         self.db2.tag_upstream_version(version1.upstream_version)
         # And we get the greatest branch when two lesser branches
         # have what we are looking for.
         branch = self.db3.branch_to_pull_upstream_from(
-                version1.upstream_version, self.fake_md5_1)
+                version1.upstream_version, [("foo.tar.gz", self.fake_md5_1)])
         self.assertEqual(branch, self.db2)
         # If the branches have diverged then we don't get a branch.
         self.up_tree3.commit("three")
         branch = self.db3.branch_to_pull_upstream_from(
-                version1.upstream_version, self.fake_md5_1)
+                version1.upstream_version, [("foo.tar.gz", self.fake_md5_1)])
         self.assertEqual(branch, None)
 
     def test_pull_from_lesser_branch_no_upstream(self):
@@ -764,8 +771,7 @@
             contents = [(basedir + '/' + element[0],) + element[1:] for
                 element in contents]
         self.build_tree_contents(contents)
-        self.db1.import_upstream(basedir, version.upstream_version, 
-            self.fake_md5_1, [])
+        self.db1.import_upstream(basedir, version.upstream_version, [])
         return version
 
     def test_import_upstream(self):
@@ -779,7 +785,7 @@
         rev = branch.repository.get_revision(rh[0])
         self.assertEqual(rev.message,
                 "Import upstream version %s" % str(version.upstream_version))
-        self.assertEqual(rev.properties['deb-md5'], self.fake_md5_1)
+        self.assertEqual(rev.properties.get('deb-md5'), None)
 
     def test_import_upstream_preserves_dot_bzrignore(self):
         self.import_a_tree([('',), ('.bzrignore', '')])
@@ -799,15 +805,13 @@
         write_to_file(os.path.join(basedir, "README"), "Hi\n")
         write_to_file(os.path.join(basedir, "BUGS"), "")
         write_to_file(os.path.join(basedir, "COPYING"), "")
-        self.db1.import_upstream(basedir, version1.upstream_version,
-                self.fake_md5_1, [])
+        self.db1.import_upstream(basedir, version1.upstream_version, [])
         basedir = name + "-" + str(version2.upstream_version)
         os.mkdir(basedir)
         write_to_file(os.path.join(basedir, "README"), "Now even better\n")
         write_to_file(os.path.join(basedir, "BUGS"), "")
         write_to_file(os.path.join(basedir, "NEWS"), "")
-        self.db1.import_upstream(basedir, version2.upstream_version, 
-                self.fake_md5_2,
+        self.db1.import_upstream(basedir, version2.upstream_version,
                 [self.up_tree1.branch.last_revision()])
         tree = self.up_tree1
         branch = tree.branch
@@ -817,7 +821,7 @@
         rev = branch.repository.get_revision(rh[1])
         self.assertEqual(rev.message,
                 "Import upstream version %s" % str(version2.upstream_version))
-        self.assertEqual(rev.properties['deb-md5'], self.fake_md5_2)
+        self.assertIs(rev.properties.get('deb-md5'), None)
         rev_tree1 = branch.repository.revision_tree(rh[0])
         rev_tree2 = branch.repository.revision_tree(rh[1])
         changes = rev_tree2.changes_from(rev_tree1)
@@ -838,8 +842,8 @@
             tf.add(basedir)
         finally:
             tf.close()
-        self.db1.import_upstream(basedir, version.upstream_version, 
-            self.fake_md5_1, [], upstream_tarball=os.path.abspath(tar_path))
+        self.db1.import_upstream(basedir, version.upstream_version, [],
+                upstream_tarballs=[(os.path.abspath(tar_path), self.fake_md5_1)])
         tree = self.up_tree1
         branch = tree.branch
         rh = branch.revision_history()
@@ -866,8 +870,8 @@
             tf.add(basedir)
         finally:
             tf.close()
-        self.db1.import_upstream(basedir, version.upstream_version, 
-            self.fake_md5_1, [], upstream_tarball=os.path.abspath(tar_path))
+        self.db1.import_upstream(basedir, version.upstream_version,
+            [], upstream_tarballs=[(os.path.abspath(tar_path), self.fake_md5_1)])
         tree = self.up_tree1
         branch = tree.branch
         rh = branch.revision_history()
@@ -1496,7 +1500,7 @@
             tf.add("a")
         finally:
             tf.close()
-        conflicts = db.merge_upstream(tarball_filename, "foo", "0.2", "0.1")
+        conflicts = db.merge_upstream([tarball_filename], "foo", "0.2", "0.1")
         self.assertEqual(0,  conflicts)
 
     def test_merge_upstream_initial_with_branch(self):
@@ -1533,7 +1537,7 @@
             tf.add("a")
         finally:
             tf.close()
-        conflicts = db.merge_upstream(tarball_filename, "foo", "0.2", "0.1",
+        conflicts = db.merge_upstream([tarball_filename], "foo", "0.2", "0.1",
             upstream_branch=upstream_tree.branch,
             upstream_revision=upstream_rev)
         self.assertEqual(0,  conflicts)
@@ -1573,7 +1577,7 @@
             tf.add("a")
         finally:
             tf.close()
-        conflicts = db.merge_upstream(tarball_filename, "foo", "0.2", "0.1",
+        conflicts = db.merge_upstream([tarball_filename], "foo", "0.2", "0.1",
             upstream_branch=upstream_tree.branch,
             upstream_revision=upstream_rev)
         # ./debian conflicts.
@@ -1614,7 +1618,7 @@
         dbs.add_branch(db)
         tree.lock_write()
         self.addCleanup(tree.unlock)
-        db.merge_upstream(builder.tar_name(), "package", str(version2),
+        db.merge_upstream([builder.tar_name()], "package", str(version2),
             version1.upstream_version,
             upstream_branch=upstream_tree.branch,
             upstream_revision=upstream_rev)
@@ -1653,7 +1657,7 @@
             tf.add("a")
         finally:
             tf.close()
-        conflicts = db.merge_upstream(tarball_filename, "package", "0.2-1",
+        conflicts = db.merge_upstream([tarball_filename], "package", "0.2-1",
             "0.1")
         # Check that we tagged wiht the dash version
         self.assertTrue(tree.branch.tags.has_tag('upstream-0.2-1'))
@@ -1692,7 +1696,7 @@
         # We don't add the new file upstream, as the new file id would
         # be picked up from there.
         upstream_rev2 = upstream_tree.commit("two")
-        db.merge_upstream(builder.tar_name(), "package",
+        db.merge_upstream([builder.tar_name()], "package",
             version2.upstream_version,
             version1.upstream_version,
             upstream_branch=upstream_tree.branch,
@@ -1729,7 +1733,7 @@
         # We don't add the new file upstream, as the new file id would
         # be picked up from there.
         upstream_rev2 = upstream_tree.commit("two")
-        db.merge_upstream(builder.tar_name(), "package",
+        db.merge_upstream([builder.tar_name()], "package",
             version2.upstream_version,
             version1.upstream_version,
             upstream_branch=upstream_tree.branch,
@@ -1762,7 +1766,7 @@
         builder.add_upstream_file("a", "New a")
         builder.add_upstream_file("b", "Renamed a")
         builder.build()
-        db.merge_upstream(builder.tar_name(), "packaging",
+        db.merge_upstream([builder.tar_name()], "packaging",
             version2.upstream_version, version1.upstream_version)
         self.assertEqual("a-id", packaging_tree.path2id("b"))
         self.assertEqual("other-a-id", packaging_tree.path2id("a"))
@@ -1778,7 +1782,7 @@
         self.db1.import_package(builder.dsc_name())
 
 
-class SourceExtractorTests(tests.TestCaseInTempDir):
+class OneZeroSourceExtractorTests(tests.TestCaseInTempDir):
 
     def test_extract_format1(self):
         version = Version("0.1-1")
@@ -1789,8 +1793,8 @@
         builder.add_default_control()
         builder.build()
         dsc = deb822.Dsc(open(builder.dsc_name()).read())
-        self.assertEqual(SourceExtractor, SOURCE_EXTRACTORS[dsc['Format']])
-        extractor = SourceExtractor(builder.dsc_name(), dsc)
+        self.assertEqual(OneZeroSourceExtractor, SOURCE_EXTRACTORS[dsc['Format']])
+        extractor = OneZeroSourceExtractor(builder.dsc_name(), dsc)
         try:
             extractor.extract()
             unpacked_dir = extractor.extracted_debianised
@@ -1805,7 +1809,7 @@
                             "README")))
             self.assertFalse(os.path.exists(os.path.join(orig_dir,
                             "debian", "control")))
-            self.assertTrue(os.path.exists(extractor.unextracted_upstream))
+            self.assertTrue(os.path.exists(extractor.upstream_tarballs[0][0]))
         finally:
             extractor.cleanup()
 
@@ -1818,8 +1822,8 @@
         builder.add_default_control()
         builder.build()
         dsc = deb822.Dsc(open(builder.dsc_name()).read())
-        self.assertEqual(SourceExtractor, SOURCE_EXTRACTORS[dsc['Format']])
-        extractor = SourceExtractor(builder.dsc_name(), dsc)
+        self.assertEqual(OneZeroSourceExtractor, SOURCE_EXTRACTORS[dsc['Format']])
+        extractor = OneZeroSourceExtractor(builder.dsc_name(), dsc)
         try:
             extractor.extract()
             unpacked_dir = extractor.extracted_debianised
@@ -1886,7 +1890,7 @@
                             "README")))
             self.assertFalse(os.path.exists(os.path.join(orig_dir,
                             "debian", "control")))
-            self.assertTrue(os.path.exists(extractor.unextracted_upstream))
+            self.assertTrue(os.path.exists(extractor.upstream_tarballs[0][0]))
         finally:
             extractor.cleanup()
 
@@ -1916,7 +1920,7 @@
                             "README")))
             self.assertFalse(os.path.exists(os.path.join(orig_dir,
                             "debian", "control")))
-            self.assertTrue(os.path.exists(extractor.unextracted_upstream))
+            self.assertTrue(os.path.exists(extractor.upstream_tarballs[0][0]))
         finally:
             extractor.cleanup()
 
@@ -1936,4 +1940,4 @@
                 SOURCE_EXTRACTORS[dsc['Format']])
         extractor = ThreeDotZeroQuiltSourceExtractor(builder.dsc_name(), dsc)
         self.addCleanup(extractor.cleanup)
-        self.assertRaises(MultipleUpstreamTarballsNotSupported, extractor.extract)
+        self.assertEquals([], extractor.upstream_tarballs)

=== modified file 'util.py'
--- util.py	2011-06-09 12:13:13 +0000
+++ util.py	2011-06-15 02:38:47 +0000
@@ -24,6 +24,7 @@
     import md5
 import signal
 import shutil
+import subprocess
 import tempfile
 import os
 import re
@@ -65,7 +66,9 @@
     MissingChangelogError,
     AddChangelogError,
     InconsistentSourceFormatError,
+    MultipleUpstreamTarballsNotSupported,
     NoPreviousUpload,
+    TarFailed,
     UnableToFindPreviousUpload,
     UnparseableChangelog,
     )
@@ -682,4 +685,24 @@
         return BUILD_TYPE_NORMAL
 
 
+def extract_orig_tarballs(tarballs, target, strip_components=None):
+    """Extract orig tarballs to a directory.
 
+    :param tarballs: List of tarball filenames
+    :param target: Target directory (must already exist)
+    """
+    if len(tarballs) != 1:
+        raise MultipleUpstreamTarballsNotSupported()
+    tarball_filename = tarballs[0]
+    tar_args = ["tar"]
+    if tarball_filename.endswith(".tar.bz2"):
+        tar_args.append('xjf')
+    else:
+        tar_args.append('xzf')
+    tar_args.extend([tarball_filename, "-C", target])
+    if strip_components is not None:
+        tar_args.extend(["--strip-components", "1"])
+    proc = subprocess.Popen(tar_args, preexec_fn=subprocess_setup)
+    proc.communicate()
+    if proc.returncode != 0:
+        raise TarFailed("extract", tarball_filename)