Merge lp:~jelmer/bzr-builddeb/multiple-upstream-tarballs into lp:bzr-builddeb

[Jelmer Vernooij]

Importing and building of packages with multiple upstream tarballs now works.

There are still two caveats:

 * when exporting the base component before generating a pristine tar delta, we don't exclude non-base components. This probably causes the pristine tar delta to be bigger than it needs to be
 * Needs more tests

=== modified file 'cmds.py'
--- cmds.py	2011-10-03 10:49:43 +0000
+++ cmds.py	2011-10-03 10:49:43 +0000
@@ -748,7 +748,7 @@
             if need_upstream_tarball:
                 target_dir = tempfile.mkdtemp() # FIXME: Cleanup?
                 locations = primary_upstream_source.fetch_tarballs(
-                    package, version, target_dir)
+                    package, version, target_dir, components=[None])
                 source_format = get_source_format(tree)
                 v3 = (source_format in [
                     FORMAT_3_0_QUILT, FORMAT_3_0_NATIVE])
 
=== modified file 'debian/changelog'
--- debian/changelog	2011-10-01 20:41:14 +0000
+++ debian/changelog	2011-10-03 10:49:43 +0000
@@ -1,3 +1,10 @@
+bzr-builddeb (2.7.10) UNRELEASED; urgency=low
+
+  * Support importing and building packages with multiple upstream
+    tarballs. LP: #653757, LP: #664834
+
+ -- Jelmer Vernooij <[e-mail]>  Tue, 27 Sep 2011 02:40:03 +0200
+
 bzr-builddeb (2.7.9) unstable; urgency=low
 
   [ Jelmer Vernooij ]
 
=== modified file 'import_dsc.py'
--- import_dsc.py	2011-10-03 10:49:43 +0000
+++ import_dsc.py	2011-10-03 10:49:43 +0000
@@ -878,9 +878,14 @@
                         target_tree=target_tree)
             finally:
                 self_tree.unlock()
+            if component is None:
+                exclude = [tb[1] for tb in upstream_tarballs if tb[1] is not None]
+            else:
+                exclude = []
             (tag, revid) = self.pristine_upstream_source.import_component_tarball(
                 package, version, self.pristine_upstream_tree, parents,
-                component, md5, tarball, author=author, timestamp=timestamp)
+                component, md5, tarball, author=author, timestamp=timestamp,
+                exclude=exclude)
             self.pristine_upstream_branch.generate_revision_history(revid)
             ret.append((component, tag, revid))
             self.branch.fetch(self.pristine_upstream_branch)
 
=== modified file 'upstream/__init__.py'
--- upstream/__init__.py	2011-10-03 10:49:43 +0000
+++ upstream/__init__.py	2011-10-03 10:49:43 +0000
@@ -95,12 +95,14 @@
         """
         raise NotImplementedError(self.has_version)
 
-    def fetch_tarballs(self, package, version, target_dir):
+    def fetch_tarballs(self, package, version, target_dir, components=None):
         """Fetch the source tarball for a particular version.
 
         :param package: Name of the package
         :param version: Version string of the version to fetch
         :param target_dir: Directory in which to store the tarball
+        :param components: List of component names to fetch; may be None,
+            in which case the backend will have to find out.
         :return: Paths of the fetched tarballs
         """
         raise NotImplementedError(self.fetch_tarballs)
@@ -114,7 +116,7 @@
     """Upstream source that uses apt-source."""
 
     def fetch_tarballs(self, package, upstream_version, target_dir,
-            _apt_pkg=None):
+            _apt_pkg=None, components=None):
         if _apt_pkg is None:
             import apt_pkg
         else:
@@ -198,7 +200,7 @@
         note("get-orig-source did not create file with prefix %s", prefix)
         return None
 
-    def fetch_tarballs(self, package, version, target_dir):
+    def fetch_tarballs(self, package, version, target_dir, components=None):
         if self.top_level:
             rules_name = 'rules'
         else:
@@ -280,7 +282,7 @@
             os.unlink(tempfilename)
         return self._xml_report_extract_upstream_version(stdout)
 
-    def fetch_tarballs(self, package, version, target_dir):
+    def fetch_tarballs(self, package, version, target_dir, components=None):
         note("Using uscan to look for the upstream tarball.")
         try:
             tempfilename = self._export_watchfile()
@@ -327,7 +329,7 @@
         finally:
             shutil.rmtree(tmpdir)
 
-    def fetch_tarballs(self, package, version, target_dir):
+    def fetch_tarballs(self, package, version, target_dir, components=None):
         note("Using the current branch without the 'debian' directory "
                 "to create the tarball")
         tarball_path = self._tarball_path(package, version, None, target_dir)
@@ -347,10 +349,10 @@
     def __repr__(self):
         return "%s(%r)" % (self.__class__.__name__, self._sources)
 
-    def fetch_tarballs(self, package, version, target_dir):
+    def fetch_tarballs(self, package, version, target_dir, components=None):
         for source in self._sources:
             try:
-                paths = source.fetch_tarballs(package, version, target_dir)
+                paths = source.fetch_tarballs(package, version, target_dir, components)
             except PackageVersionNotPresent:
                 pass
             else:
@@ -497,7 +499,7 @@
         self.path = path
         self.version = version
 
-    def fetch_tarballs(self, package, version, target_dir):
+    def fetch_tarballs(self, package, version, target_dir, components=None):
         if version != self.version:
             raise PackageVersionNotPresent(package, version, self)
         dest_name = new_tarball_name(package, version, self.path)
@@ -542,7 +544,7 @@
         else:
             self.project = project
 
-    def fetch_tarballs(self, package, version, target_dir):
+    def fetch_tarballs(self, package, version, target_dir, components=None):
         release = self.project.getRelease(version=version)
         if release is None:
             raise PackageVersionNotPresent(package, version, self)
 
=== modified file 'upstream/branch.py'
--- upstream/branch.py	2011-10-03 10:49:43 +0000
+++ upstream/branch.py	2011-10-03 10:49:43 +0000
@@ -264,7 +264,7 @@
         finally:
             self.upstream_branch.unlock()
 
-    def fetch_tarballs(self, package, version, target_dir):
+    def fetch_tarballs(self, package, version, target_dir, components=None):
         self.upstream_branch.lock_read()
         try:
             revid = self.version_as_revision(package, version)
 
=== modified file 'upstream/pristinetar.py'
--- upstream/pristinetar.py	2011-10-03 10:49:43 +0000
+++ upstream/pristinetar.py	2011-10-03 10:49:43 +0000
@@ -171,14 +171,24 @@
         :param component: Component name (None for base)
         :param exclude: Exclude directories
         """
-        if exclude is not None or subdir is not None:
-            raise NotImplementedError
         if exclude is None:
             exclude = []
+        def include_change(c):
+            if not exclude:
+                return True
+            path = c[1][1]
+            if path is None:
+                return True
+            for e in exclude:
+                if path == e or path.startswith(e+"/"):
+                    return False
+            else:
+                return True
         revprops = {}
         if md5 is not None:
             revprops["deb-md5"] = md5
-            delta = self.make_pristine_tar_delta(tree, tarball, subdir=subdir)
+            delta = self.make_pristine_tar_delta(tree, tarball, subdir=subdir,
+                    exclude=exclude)
             uuencoded = standard_b64encode(delta)
             if tarball.endswith(".tar.bz2"):
                 revprops["deb-pristine-delta-bz2"] = uuencoded
@@ -206,8 +216,9 @@
                 revprops=revprops, timestamp=timestamp, timezone=timezone)
             builder.will_record_deletes()
             try:
-                list(builder.record_iter_changes(tree, base_revid,
-                    tree.iter_changes(basis_tree)))
+                changes = [c for c in tree.iter_changes(basis_tree) if
+                           include_change(c)]
+                list(builder.record_iter_changes(tree, base_revid, changes))
                 builder.finish_inventory()
             except:
                 builder.abort()
@@ -228,13 +239,13 @@
             rev = self.branch.repository.get_revision(revid)
         except NoSuchRevision:
             raise PackageVersionNotPresent(package, version, self)
-        note("Using pristine-tar to reconstruct the needed tarball.")
         if self.has_pristine_tar_delta(rev):
             format = self.pristine_tar_format(rev)
         else:
             format = 'gz'
         target_filename = self._tarball_path(package, version, component,
                                              target_dir, format=format)
+        note("Using pristine-tar to reconstruct %s.", os.path.basename(target_filename))
         try:
             self.reconstruct_pristine_tar(revid, package, version, target_filename)
         except PristineTarError:
@@ -243,8 +254,15 @@
             raise PackageVersionNotPresent(package, version, self)
         return target_filename
 
-    def fetch_tarballs(self, package, version, target_dir):
-        return [self.fetch_component_tarball(package, version, None, target_dir)]
+    def fetch_tarballs(self, package, version, target_dir, components=None):
+        if components is None:
+            # Scan tags for components
+            try:
+                components = self._components_by_version()[version].keys()
+            except KeyError:
+                raise PackageVersionNotPresent(package, version, self)
+        return [self.fetch_component_tarball(package, version, component, target_dir)
+                for component in components]
 
     def _has_revision(self, revid, md5=None):
         self.branch.lock_read()
@@ -370,7 +388,7 @@
         finally:
             shutil.rmtree(tmpdir)
 
-    def make_pristine_tar_delta(self, tree, tarball_path, subdir=None):
+    def make_pristine_tar_delta(self, tree, tarball_path, subdir=None, exclude=None):
         tmpdir = tempfile.mkdtemp(prefix="builddeb-pristine-")
         try:
             dest = os.path.join(tmpdir, "orig")
@@ -385,17 +403,21 @@
         finally:
             shutil.rmtree(tmpdir)
 
-    def iter_versions(self):
-        """Iterate over all upstream versions.
-
-        :return: Iterator over (tag_name, version, revid) tuples
-        """
+    def _components_by_version(self):
         ret = {}
         for tag_name, tag_revid in self.branch.tags.get_tag_dict().iteritems():
             if not is_upstream_tag(tag_name):
                 continue
             (component, version) = upstream_tag_version(tag_name)
             ret.setdefault(version, {})[component] = tag_revid
+        return ret
+
+    def iter_versions(self):
+        """Iterate over all upstream versions.
+
+        :return: Iterator over (tag_name, version, revid) tuples
+        """
+        ret = self._components_by_version()
         return ret.iteritems()