Merge lp:~cjwatson/launchpad/tar-xz into lp:launchpad

[Colin Watson]

[r=lifeless,stevenk,wgrant][bug=742408] Add support for xz-compressed tarballs in source packages.

Summary:

dpkg 1.15.5 and newer supports xz compression for source package tarballs. Debian doesn't quite support this yet, so this hasn't been a huge issue for us, but there is pressure for it to do so soon because GNOME is going to switch to shipping tar.xz (http://lists.debian.org/debian-project/2011/03/msg00047.html). Over time, this would be a straightforward way for us to shrink the size of many source packages.

Proposed fix:

Add orig.tar.xz, orig-component.tar.xz, and debian.tar.xz to the locations where corresponding *.bz2 versions are currently mentioned.

Pre-implementation notes:

Discussed with wgrant. *.xz is only available in the 3.0 (quilt) format, which is supported for Ubuntu 9.10 and newer, while *.xz specifically should strictly only be supported for Ubuntu 10.04 LTS and newer. However, such a constraint would be quite a lot of work to add, karmic is going to be EOL soon anyway, and the only downside of uploading an xz-compressed source package to karmic should be that it fails to build; so we agreed that enforcing this constraint in Launchpad is unnecessary.

Tests:

This branch adjusts various tests that already checked bzip2 support to check xz support as well. I ran 'testr run -- -t archiveuploader'.

QA plan:

Convert an Ubuntu-specific package to tar.xz. We could easily do this with some Ubuntu-native package such as ubiquity.

[William Grant (wgrant)]

It's very close to needing generalisation of the compression type checking, but I think duplication is OK until next time.

review: Approve (code*)

[Robert Collins (lifeless)]

review: Approve

[Steve Kowalik (stevenk)]

I agree with William -- it's okay for now.

review: Approve (code)

[Colin Watson (cjwatson)]

I don't have direct landing access, and would appreciate help in getting this landed.

[William Grant (wgrant)]

I'm landing this.

=== modified file 'lib/lp/archiveuploader/dscfile.py'
--- lib/lp/archiveuploader/dscfile.py	2011-03-07 03:08:47 +0000
+++ lib/lp/archiveuploader/dscfile.py	2011-03-25 16:46:32 +0000
@@ -460,6 +460,7 @@
             }
         component_orig_tar_counts = {}
         bzip2_count = 0
+        xz_count = 0
         files_missing = False
 
         for sub_dsc_file in self.files:
@@ -481,6 +482,8 @@
 
             if sub_dsc_file.filename.endswith('.bz2'):
                 bzip2_count += 1
+            elif sub_dsc_file.filename.endswith('.xz'):
+                xz_count += 1
 
             try:
                 library_file, file_archive = self._getFileByName(
@@ -534,7 +537,7 @@
 
         for error in file_checker(
             self.filename, file_type_counts, component_orig_tar_counts,
-            bzip2_count):
+            bzip2_count, xz_count):
             yield error
 
         if files_missing:
@@ -773,16 +776,20 @@
 
 
 def check_format_1_0_files(filename, file_type_counts, component_counts,
-                           bzip2_count):
+                           bzip2_count, xz_count):
     """Check that the given counts of each file type suit format 1.0.
 
     A 1.0 source must be native (with only one tar.gz), or have an orig.tar.gz
-    and a diff.gz. It cannot use bzip2 compression.
+    and a diff.gz. It cannot use bzip2 or xz compression.
     """
     if bzip2_count > 0:
         yield UploadError(
             "%s: is format 1.0 but uses bzip2 compression."
             % filename)
+    if xz_count > 0:
+        yield UploadError(
+            "%s: is format 1.0 but uses xz compression."
+            % filename)
 
     valid_file_type_counts = [
         {
@@ -807,11 +814,11 @@
 
 
 def check_format_3_0_native_files(filename, file_type_counts,
-                                  component_counts, bzip2_count):
+                                  component_counts, bzip2_count, xz_count):
     """Check that the given counts of each file type suit format 3.0 (native).
 
-    A 3.0 (native) source must have only one tar.*. Both gzip and bzip2
-    compression are permissible.
+    A 3.0 (native) source must have only one tar.*. Any of gzip, bzip2, and
+    xz compression are permissible.
     """
 
     valid_file_type_counts = [
@@ -829,12 +836,12 @@
 
 
 def check_format_3_0_quilt_files(filename, file_type_counts,
-                                 component_counts, bzip2_count):
+                                 component_counts, bzip2_count, xz_count):
     """Check that the given counts of each file type suit format 3.0 (native).
 
     A 3.0 (quilt) source must have exactly one orig.tar.*, one debian.tar.*,
-    and at most one orig-COMPONENT.tar.* for each COMPONENT. Both gzip and
-    bzip2 compression are permissible.
+    and at most one orig-COMPONENT.tar.* for each COMPONENT. Any of gzip,
+    bzip2, and xz compression are permissible.
     """
 
     valid_file_type_counts = [

=== modified file 'lib/lp/archiveuploader/tests/data/suite/bar_1.0-1_3.0-quilt/bar_1.0-1.dsc'
--- lib/lp/archiveuploader/tests/data/suite/bar_1.0-1_3.0-quilt/bar_1.0-1.dsc	2009-11-14 02:59:34 +0000
+++ lib/lp/archiveuploader/tests/data/suite/bar_1.0-1_3.0-quilt/bar_1.0-1.dsc	2011-03-25 16:46:32 +0000
@@ -8,15 +8,18 @@
 Checksums-Sha1: 
  e7a106587e0f11220e27ccc5db7527cae05e4054 178 bar_1.0.orig-comp1.tar.gz
  a645cc0856469f549f11bb9bdc4ca6dda4eedd4b 156 bar_1.0.orig-comp2.tar.bz2
+ bda4aae72bc911d25db3837f644c21fa60a30394 192 bar_1.0.orig-comp3.tar.xz
  73a04163fee97fd2257ab266bd48f1d3d528e012 164 bar_1.0.orig.tar.gz
  727de2395d3a3e35c375159db01072935fea02f8 737 bar_1.0-1.debian.tar.bz2
 Checksums-Sha256: 
  aff0bbc01c5883ac76e89cba7a9b6cd79b521183f0de86c93e93ab8804f5a256 178 bar_1.0.orig-comp1.tar.gz
  955131b3a5bd881c008ff822c0f9b7544350647fe1f355f65a138e0e5e5e8d71 156 bar_1.0.orig-comp2.tar.bz2
+ 189ca14ff5de56e9ee1ad119992d8dada416b4d3e527a3c1ea6c068e6c9ab04f 192 bar_1.0.orig-comp3.tar.xz
  f1ecff929899b567f45d6734b69d59a4f3c04dabce3cc8e6ed6d64073eda360e 164 bar_1.0.orig.tar.gz
  3d2137f9f80f4b6c80f73acb8b3ac7986c962c4268d8948f54823d27148a6116 737 bar_1.0-1.debian.tar.bz2
 Files: 
  eed105761436486f7eebaa8c017bc59a 178 bar_1.0.orig-comp1.tar.gz
  0a30b50fa846e75261808ee1f41d1cbe 156 bar_1.0.orig-comp2.tar.bz2
+ 151443692da0d20731ff79df1e22cc37 192 bar_1.0.orig-comp3.tar.xz
  fc1464e5985b962a042d5354452f361d 164 bar_1.0.orig.tar.gz
  e68110184d4d9e7afd81520cc490b396 737 bar_1.0-1.debian.tar.bz2

=== modified file 'lib/lp/archiveuploader/tests/data/suite/bar_1.0-1_3.0-quilt/bar_1.0-1_source.changes'
--- lib/lp/archiveuploader/tests/data/suite/bar_1.0-1_3.0-quilt/bar_1.0-1_source.changes	2009-11-14 02:59:34 +0000
+++ lib/lp/archiveuploader/tests/data/suite/bar_1.0-1_3.0-quilt/bar_1.0-1_source.changes	2011-03-25 16:46:32 +0000
@@ -15,20 +15,23 @@
  .
    * Initial version
 Checksums-Sha1: 
- 762b018e756c1d9be905d40291bc8e975c57a8f0 1115 bar_1.0-1.dsc
+ e7422c6fba68e168e0d19c691a4a125eed1d7e23 1347 bar_1.0-1.dsc
  e7a106587e0f11220e27ccc5db7527cae05e4054 178 bar_1.0.orig-comp1.tar.gz
  a645cc0856469f549f11bb9bdc4ca6dda4eedd4b 156 bar_1.0.orig-comp2.tar.bz2
+ bda4aae72bc911d25db3837f644c21fa60a30394 192 bar_1.0.orig-comp3.tar.xz
  73a04163fee97fd2257ab266bd48f1d3d528e012 164 bar_1.0.orig.tar.gz
  727de2395d3a3e35c375159db01072935fea02f8 737 bar_1.0-1.debian.tar.bz2
 Checksums-Sha256: 
- 49f2656a0e221b0f07dc208f583e2523b09bc30104855372534b5562478efdb5 1115 bar_1.0-1.dsc
+ e400747cc2d24517e9303ddddc008e0be678d622a0d36c929bbb572c11d36bad 1347 bar_1.0-1.dsc
  aff0bbc01c5883ac76e89cba7a9b6cd79b521183f0de86c93e93ab8804f5a256 178 bar_1.0.orig-comp1.tar.gz
  955131b3a5bd881c008ff822c0f9b7544350647fe1f355f65a138e0e5e5e8d71 156 bar_1.0.orig-comp2.tar.bz2
+ 189ca14ff5de56e9ee1ad119992d8dada416b4d3e527a3c1ea6c068e6c9ab04f 192 bar_1.0.orig-comp3.tar.xz
  f1ecff929899b567f45d6734b69d59a4f3c04dabce3cc8e6ed6d64073eda360e 164 bar_1.0.orig.tar.gz
  3d2137f9f80f4b6c80f73acb8b3ac7986c962c4268d8948f54823d27148a6116 737 bar_1.0-1.debian.tar.bz2
 Files: 
- 662977e76501561c902c34bf94a2054d 1115 devel optional bar_1.0-1.dsc
+ 7db2a612a48d87116b7b3651c4a43aa1 1347 devel optional bar_1.0-1.dsc
  eed105761436486f7eebaa8c017bc59a 178 devel optional bar_1.0.orig-comp1.tar.gz
  0a30b50fa846e75261808ee1f41d1cbe 156 devel optional bar_1.0.orig-comp2.tar.bz2
+ 151443692da0d20731ff79df1e22cc37 192 devel optional bar_1.0.orig-comp3.tar.xz
  fc1464e5985b962a042d5354452f361d 164 devel optional bar_1.0.orig.tar.gz
  e68110184d4d9e7afd81520cc490b396 737 devel optional bar_1.0-1.debian.tar.bz2

=== added file 'lib/lp/archiveuploader/tests/data/suite/bar_1.0-1_3.0-quilt/bar_1.0.orig-comp3.tar.xz'
Binary files lib/lp/archiveuploader/tests/data/suite/bar_1.0-1_3.0-quilt/bar_1.0.orig-comp3.tar.xz	1970-01-01 00:00:00 +0000 and lib/lp/archiveuploader/tests/data/suite/bar_1.0-1_3.0-quilt/bar_1.0.orig-comp3.tar.xz	2011-03-25 16:46:32 +0000 differ
=== modified file 'lib/lp/archiveuploader/tests/test_dscfile.py'
--- lib/lp/archiveuploader/tests/test_dscfile.py	2010-12-22 01:08:48 +0000
+++ lib/lp/archiveuploader/tests/test_dscfile.py	2011-03-25 16:46:32 +0000
@@ -143,7 +143,7 @@
 class BaseTestSourceFileVerification(TestCase):
 
     def assertErrorsForFiles(self, expected, files, components={},
-                             bzip2_count=0):
+                             bzip2_count=0, xz_count=0):
         """Check problems with the given set of files for the given format.
 
         :param expected: a list of expected errors, as strings.
@@ -152,6 +152,7 @@
         :param components: a dict mapping orig component tarball components
             to counts.
         :param bzip2_count: number of files using bzip2 compression.
+        :param xz_count: number of files using xz compression.
         """
         full_files = {
             NATIVE_TARBALL: 0,
@@ -163,9 +164,9 @@
         self.assertEquals(
             expected,
             [str(e) for e in format_to_file_checker_map[self.format](
-                'foo_1.dsc', full_files, components, bzip2_count)])
+                'foo_1.dsc', full_files, components, bzip2_count, xz_count)])
 
-    def assertFilesOK(self, files, components={}, bzip2_count=0):
+    def assertFilesOK(self, files, components={}, bzip2_count=0, xz_count=0):
         """Check that the given set of files is OK for the given format.
 
         :param format: the `SourcePackageFormat` to check against.
@@ -173,8 +174,9 @@
         :param components: a dict mapping orig component tarball components
             to counts.
         :param bzip2_count: number of files using bzip2 compression.
+        :param xz_count: number of files using xz compression.
         """
-        self.assertErrorsForFiles([], files, components, bzip2_count)
+        self.assertErrorsForFiles([], files, components, bzip2_count, xz_count)
 
 
 class Test10SourceFormatVerification(BaseTestSourceFileVerification):
@@ -184,6 +186,7 @@
     wrong_files_error = ('foo_1.dsc: must have exactly one tar.gz, or an '
                          'orig.tar.gz and diff.gz')
     bzip2_error = 'foo_1.dsc: is format 1.0 but uses bzip2 compression.'
+    xz_error = 'foo_1.dsc: is format 1.0 but uses xz compression.'
 
     def testFormat10Debian(self):
         # A 1.0 source can contain an original tarball and a Debian diff
@@ -209,7 +212,12 @@
     def testFormat10CannotUseBzip2(self):
         # 1.0 sources cannot use bzip2 compression.
         self.assertErrorsForFiles(
-            [self.bzip2_error], {NATIVE_TARBALL: 1}, {}, 1)
+            [self.bzip2_error], {NATIVE_TARBALL: 1}, {}, 1, 0)
+
+    def testFormat10CannotUseXz(self):
+        # 1.0 sources cannot use xz compression.
+        self.assertErrorsForFiles(
+            [self.xz_error], {NATIVE_TARBALL: 1}, {}, 0, 1)
 
 
 class Test30QuiltSourceFormatVerification(BaseTestSourceFileVerification):
@@ -223,12 +231,13 @@
     def testFormat30Quilt(self):
         # A 3.0 (quilt) source must contain an orig tarball and a debian
         # tarball. It may also contain at most one component tarball for
-        # each component, and can use gzip or bzip2 compression.
+        # each component, and can use gzip, bzip2, or xz compression.
         for components in ({}, {'foo': 1}, {'foo': 1, 'bar': 1}):
             for bzip2_count in (0, 1):
-                self.assertFilesOK(
-                    {ORIG_TARBALL: 1, DEBIAN_TARBALL: 1}, components,
-                    bzip2_count)
+                for xz_count in (0, 1):
+                    self.assertFilesOK(
+                        {ORIG_TARBALL: 1, DEBIAN_TARBALL: 1}, components,
+                        bzip2_count, xz_count)
 
     def testFormat30QuiltCannotHaveConflictingComponentTarballs(self):
         # Multiple conflicting tarballs for a single component are
@@ -253,10 +262,11 @@
 
     def testFormat30Native(self):
         # 3.0 (native) sources must contain just a native tarball. They
-        # may use gzip or bzip2 compression.
+        # may use gzip, bzip2, or xz compression.
         for bzip2_count in (0, 1):
             self.assertFilesOK({NATIVE_TARBALL: 1}, {},
-            bzip2_count)
+            bzip2_count, 0)
+        self.assertFilesOK({NATIVE_TARBALL: 1}, {}, 0, 1)
 
     def testFormat30NativeCannotHaveWrongFiles(self):
         # 3.0 (quilt) sources may not have a diff, Debian tarball, orig

=== modified file 'lib/lp/archiveuploader/tests/test_utils.py'
--- lib/lp/archiveuploader/tests/test_utils.py	2010-12-09 16:20:20 +0000
+++ lib/lp/archiveuploader/tests/test_utils.py	2011-03-25 16:46:32 +0000
@@ -47,6 +47,9 @@
         self.assertEquals(
             determine_source_file_type('foo_1.0.orig.tar.bz2'),
             SourcePackageFileType.ORIG_TARBALL)
+        self.assertEquals(
+            determine_source_file_type('foo_1.0.orig.tar.xz'),
+            SourcePackageFileType.ORIG_TARBALL)
 
         # Component original tarballs too.
         self.assertEquals(
@@ -55,6 +58,9 @@
         self.assertEquals(
             determine_source_file_type('foo_1.0.orig-bar.tar.bz2'),
             SourcePackageFileType.COMPONENT_ORIG_TARBALL)
+        self.assertEquals(
+            determine_source_file_type('foo_1.0.orig-bar.tar.xz'),
+            SourcePackageFileType.COMPONENT_ORIG_TARBALL)
 
         # And Debian tarballs...
         self.assertEquals(
@@ -63,6 +69,9 @@
         self.assertEquals(
             determine_source_file_type('foo_1.0-2.debian.tar.bz2'),
             SourcePackageFileType.DEBIAN_TARBALL)
+        self.assertEquals(
+            determine_source_file_type('foo_1.0-2.debian.tar.xz'),
+            SourcePackageFileType.DEBIAN_TARBALL)
 
         # And even native tarballs!
         self.assertEquals(
@@ -71,6 +80,9 @@
         self.assertEquals(
             determine_source_file_type('foo_1.0.tar.bz2'),
             SourcePackageFileType.NATIVE_TARBALL)
+        self.assertEquals(
+            determine_source_file_type('foo_1.0.tar.xz'),
+            SourcePackageFileType.NATIVE_TARBALL)
 
         self.assertEquals(None, determine_source_file_type('foo_1.0'))
         self.assertEquals(None, determine_source_file_type('foo_1.0.blah.gz'))
@@ -235,9 +247,11 @@
     def test_re_issource(self):
         # Verify that various source extensions match the regexp.
         extensions = (
-            'dsc', 'tar.gz', 'tar.bz2', 'diff.gz', 'orig.tar.gz',
-            'orig.tar.bz2', 'orig-bar.tar.gz', 'orig-bar.tar.bz2',
-            'orig-foo_bar.tar.gz', 'debian.tar.gz', 'debian.tar.bz2')
+            'dsc', 'tar.gz', 'tar.bz2', 'tar.xz', 'diff.gz',
+            'orig.tar.gz', 'orig.tar.bz2', 'orig.tar.xz',
+            'orig-bar.tar.gz', 'orig-bar.tar.bz2', 'orig-bar.tar.xz',
+            'orig-foo_bar.tar.gz',
+            'debian.tar.gz', 'debian.tar.bz2', 'debian.tar.xz')
         for extension in extensions:
             self.assertEquals(
                 ('foo-bar', '1.0', extension),
@@ -255,8 +269,9 @@
         # A badly formatted name also doesn't match.
         self.assertIs(None, re_issource.match('foo-bar.dsc'))
 
-        # bzip2 compression for files which must be gzipped is invalid.
+        # bzip2/xz compression for files which must be gzipped is invalid.
         self.assertIs(None, re_issource.match('foo-bar_1.0.diff.bz2'))
+        self.assertIs(None, re_issource.match('foo-bar_1.0.diff.xz'))
 
 
 class DdpkgExtractSourceTests(TestCase):

=== modified file 'lib/lp/archiveuploader/utils.py'
--- lib/lp/archiveuploader/utils.py	2011-02-17 17:02:54 +0000
+++ lib/lp/archiveuploader/utils.py	2011-03-25 16:46:32 +0000
@@ -57,14 +57,14 @@
 re_isadeb = re.compile(r"(.+?)_(.+?)_(.+)\.(u?d?deb)$")
 
 source_file_exts = [
-    'orig(?:-.+)?\.tar\.(?:gz|bz2)', 'diff.gz',
-    '(?:debian\.)?tar\.(?:gz|bz2)', 'dsc']
+    'orig(?:-.+)?\.tar\.(?:gz|bz2|xz)', 'diff.gz',
+    '(?:debian\.)?tar\.(?:gz|bz2|xz)', 'dsc']
 re_issource = re.compile(
     r"([^_]+)_(.+?)\.(%s)" % "|".join(ext for ext in source_file_exts))
-re_is_component_orig_tar_ext = re.compile(r"^orig-(.+).tar.(?:gz|bz2)$")
-re_is_orig_tar_ext = re.compile(r"^orig.tar.(?:gz|bz2)$")
-re_is_debian_tar_ext = re.compile(r"^debian.tar.(?:gz|bz2)$")
-re_is_native_tar_ext = re.compile(r"^tar.(?:gz|bz2)$")
+re_is_component_orig_tar_ext = re.compile(r"^orig-(.+).tar.(?:gz|bz2|xz)$")
+re_is_orig_tar_ext = re.compile(r"^orig.tar.(?:gz|bz2|xz)$")
+re_is_debian_tar_ext = re.compile(r"^debian.tar.(?:gz|bz2|xz)$")
+re_is_native_tar_ext = re.compile(r"^tar.(?:gz|bz2|xz)$")
 
 re_no_epoch = re.compile(r"^\d+\:")
 re_no_revision = re.compile(r"-[^-]+$")

=== modified file 'lib/lp/soyuz/enums.py'
--- lib/lp/soyuz/enums.py	2010-08-24 15:29:01 +0000
+++ lib/lp/soyuz/enums.py	2011-03-25 16:46:32 +0000
@@ -523,14 +523,14 @@
         3.0 (quilt)
 
         Specifies a non-native package, with an orig.tar.* and a debian.tar.*.
-        Supports gzip and bzip2 compression.
+        Supports gzip, bzip2, and xz compression.
         """)
 
     FORMAT_3_0_NATIVE = DBItem(2, """
         3.0 (native)
 
-        Specifies a native package, with a single tar.*. Supports gzip and
-        bzip2 compression.
+        Specifies a native package, with a single tar.*. Supports gzip,
+        bzip2, and xz compression.
         """)