Merge lp:~cjwatson/launchpad/orig-tarball-signatures into lp:launchpad

[Colin Watson]

Handle original tarball signatures in source packages.

Handle original tarball signatures in source packages.

Some more upload processor tests should exist, but can't quite yet because we first need a dpkg with the necessary cherry-picks to be able to unpack these packages.

Component tarball signature checks could be more accurate (e.g. test that all signatures have a matching tarball), but that was a bit fiddly to wedge into the file checker abstraction and doesn't seem vitally important.

[William Grant (wgrant)]

review: Approve (code)

=== modified file 'lib/lp/archiveuploader/dscfile.py'
--- lib/lp/archiveuploader/dscfile.py	2016-03-18 14:12:07 +0000
+++ lib/lp/archiveuploader/dscfile.py	2016-06-01 02:29:15 +0000
@@ -50,6 +50,7 @@
     parse_maintainer_bytes,
     ParseMaintError,
     re_is_component_orig_tar_ext,
+    re_is_component_orig_tar_ext_sig,
     re_issource,
     re_valid_pkg_name,
     re_valid_version,
@@ -441,7 +442,10 @@
         if (self.policy.archive.purpose == ArchivePurpose.PPA and
             determine_source_file_type(filename) in (
                 SourcePackageFileType.ORIG_TARBALL,
-                SourcePackageFileType.COMPONENT_ORIG_TARBALL)):
+                SourcePackageFileType.COMPONENT_ORIG_TARBALL,
+                SourcePackageFileType.ORIG_TARBALL_SIGNATURE,
+                SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE,
+                )):
             archives = [self.policy.archive, self.policy.distro.main_archive]
         else:
             archives = [self.policy.archive]
@@ -470,10 +474,12 @@
         file_type_counts = {
             SourcePackageFileType.DIFF: 0,
             SourcePackageFileType.ORIG_TARBALL: 0,
+            SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 0,
             SourcePackageFileType.DEBIAN_TARBALL: 0,
             SourcePackageFileType.NATIVE_TARBALL: 0,
             }
         component_orig_tar_counts = {}
+        component_orig_tar_signature_counts = {}
         bzip2_count = 0
         xz_count = 0
         files_missing = False
@@ -492,6 +498,15 @@
                 if component not in component_orig_tar_counts:
                     component_orig_tar_counts[component] = 0
                 component_orig_tar_counts[component] += 1
+            elif (
+                file_type ==
+                    SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE):
+                # Split the count by component name.
+                component = re_is_component_orig_tar_ext_sig.match(
+                    get_source_file_extension(sub_dsc_file.filename)).group(1)
+                if component not in component_orig_tar_signature_counts:
+                    component_orig_tar_signature_counts[component] = 0
+                component_orig_tar_signature_counts[component] += 1
             else:
                 file_type_counts[file_type] += 1
 
@@ -552,7 +567,7 @@
 
         for error in file_checker(
             self.filename, file_type_counts, component_orig_tar_counts,
-            bzip2_count, xz_count):
+            component_orig_tar_signature_counts, bzip2_count, xz_count):
             yield error
 
         if files_missing:
@@ -791,12 +806,14 @@
     return open(changelog_file, 'r').read()
 
 
-def check_format_1_0_files(filename, file_type_counts, component_counts,
+def check_format_1_0_files(filename, file_type_counts,
+                           component_counts, component_signature_counts,
                            bzip2_count, xz_count):
     """Check that the given counts of each file type suit format 1.0.
 
     A 1.0 source must be native (with only one tar.gz), or have an orig.tar.gz
-    and a diff.gz. It cannot use bzip2 or xz compression.
+    and a diff.gz. It cannot use bzip2 or xz compression. If it has an
+    orig.tar.gz, it may have an orig.tar.gz.asc signature.
     """
     if bzip2_count > 0:
         yield UploadError(
@@ -811,11 +828,20 @@
         {
             SourcePackageFileType.NATIVE_TARBALL: 1,
             SourcePackageFileType.ORIG_TARBALL: 0,
+            SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 0,
             SourcePackageFileType.DEBIAN_TARBALL: 0,
             SourcePackageFileType.DIFF: 0,
         },
         {
             SourcePackageFileType.ORIG_TARBALL: 1,
+            SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 0,
+            SourcePackageFileType.DIFF: 1,
+            SourcePackageFileType.NATIVE_TARBALL: 0,
+            SourcePackageFileType.DEBIAN_TARBALL: 0,
+        },
+        {
+            SourcePackageFileType.ORIG_TARBALL: 1,
+            SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 1,
             SourcePackageFileType.DIFF: 1,
             SourcePackageFileType.NATIVE_TARBALL: 0,
             SourcePackageFileType.DEBIAN_TARBALL: 0,
@@ -823,14 +849,15 @@
     ]
 
     if (file_type_counts not in valid_file_type_counts or
-        len(component_counts) > 0):
+        len(component_counts) > 0 or len(component_signature_counts) > 0):
         yield UploadError(
             "%s: must have exactly one tar.gz, or an orig.tar.gz and diff.gz"
             % filename)
 
 
 def check_format_3_0_native_files(filename, file_type_counts,
-                                  component_counts, bzip2_count, xz_count):
+                                  component_counts, component_signature_counts,
+                                  bzip2_count, xz_count):
     """Check that the given counts of each file type suit format 3.0 (native).
 
     A 3.0 (native) source must have only one tar.*. Any of gzip, bzip2, and
@@ -841,28 +868,39 @@
         {
             SourcePackageFileType.NATIVE_TARBALL: 1,
             SourcePackageFileType.ORIG_TARBALL: 0,
+            SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 0,
             SourcePackageFileType.DEBIAN_TARBALL: 0,
             SourcePackageFileType.DIFF: 0,
         },
     ]
 
     if (file_type_counts not in valid_file_type_counts or
-        len(component_counts) > 0):
+        len(component_counts) > 0 or len(component_signature_counts) > 0):
         yield UploadError("%s: must have only a tar.*." % filename)
 
 
 def check_format_3_0_quilt_files(filename, file_type_counts,
-                                 component_counts, bzip2_count, xz_count):
+                                 component_counts, component_signature_counts,
+                                 bzip2_count, xz_count):
     """Check that the given counts of each file type suit format 3.0 (native).
 
     A 3.0 (quilt) source must have exactly one orig.tar.*, one debian.tar.*,
     and at most one orig-COMPONENT.tar.* for each COMPONENT. Any of gzip,
-    bzip2, and xz compression are permissible.
+    bzip2, and xz compression are permissible. It may have orig.tar.*.asc
+    or orig-COMPONENT.tar.* signatures.
     """
 
     valid_file_type_counts = [
         {
             SourcePackageFileType.ORIG_TARBALL: 1,
+            SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 0,
+            SourcePackageFileType.DEBIAN_TARBALL: 1,
+            SourcePackageFileType.NATIVE_TARBALL: 0,
+            SourcePackageFileType.DIFF: 0,
+        },
+        {
+            SourcePackageFileType.ORIG_TARBALL: 1,
+            SourcePackageFileType.ORIG_TARBALL_SIGNATURE: 1,
             SourcePackageFileType.DEBIAN_TARBALL: 1,
             SourcePackageFileType.NATIVE_TARBALL: 0,
             SourcePackageFileType.DIFF: 0,
@@ -879,6 +917,11 @@
             yield UploadError(
                 "%s: has more than one orig-%s.tar.*."
                 % (filename, component))
+    for component in component_signature_counts:
+        if component_signature_counts[component] > 1:
+            yield UploadError(
+                "%s: has more than one orig-%s.tar.*.asc."
+                % (filename, component))
 
 
 format_to_file_checker_map = {

=== modified file 'lib/lp/archiveuploader/tests/test_dscfile.py'
--- lib/lp/archiveuploader/tests/test_dscfile.py	2016-02-05 16:51:12 +0000
+++ lib/lp/archiveuploader/tests/test_dscfile.py	2016-06-01 02:29:15 +0000
@@ -41,6 +41,7 @@
 
 
 ORIG_TARBALL = SourcePackageFileType.ORIG_TARBALL
+ORIG_TARBALL_SIGNATURE = SourcePackageFileType.ORIG_TARBALL_SIGNATURE
 DEBIAN_TARBALL = SourcePackageFileType.DEBIAN_TARBALL
 NATIVE_TARBALL = SourcePackageFileType.NATIVE_TARBALL
 DIFF = SourcePackageFileType.DIFF
@@ -229,21 +230,24 @@
 
 class BaseTestSourceFileVerification(TestCase):
 
-    def assertErrorsForFiles(self, expected, files, components={},
+    def assertErrorsForFiles(self, expected, files,
+                             components={}, component_signatures={},
                              bzip2_count=0, xz_count=0):
         """Check problems with the given set of files for the given format.
 
         :param expected: a list of expected errors, as strings.
         :param format: the `SourcePackageFormat` to check against.
         :param files: a dict mapping `SourcePackageFileType`s to counts.
-        :param components: a dict mapping orig component tarball components
-            to counts.
+        :param components: a dict mapping orig component tarballs to counts.
+        :param component_signatures: a dict mapping orig component tarball
+            signatures to counts.
         :param bzip2_count: number of files using bzip2 compression.
         :param xz_count: number of files using xz compression.
         """
         full_files = {
             NATIVE_TARBALL: 0,
             ORIG_TARBALL: 0,
+            ORIG_TARBALL_SIGNATURE: 0,
             DIFF: 0,
             DEBIAN_TARBALL: 0,
             }
@@ -251,20 +255,23 @@
         self.assertEquals(
             expected,
             [str(e) for e in format_to_file_checker_map[self.format](
-                'foo_1.dsc', full_files, components, bzip2_count, xz_count)])
+                'foo_1.dsc', full_files, components, component_signatures,
+                bzip2_count, xz_count)])
 
-    def assertFilesOK(self, files, components={}, bzip2_count=0, xz_count=0):
+    def assertFilesOK(self, files, components={}, component_signatures={},
+                      bzip2_count=0, xz_count=0):
         """Check that the given set of files is OK for the given format.
 
         :param format: the `SourcePackageFormat` to check against.
         :param files: a dict mapping `SourcePackageFileType`s to counts.
-        :param components: a dict mapping orig component tarball components
-            to counts.
+        :param components: a dict mapping orig component tarball to counts.
+        :param component_signatures: a dict mapping orig component tarball
+            signatures to counts.
         :param bzip2_count: number of files using bzip2 compression.
         :param xz_count: number of files using xz compression.
         """
         self.assertErrorsForFiles(
-            [], files, components, bzip2_count, xz_count)
+            [], files, components, component_signatures, bzip2_count, xz_count)
 
 
 class Test10SourceFormatVerification(BaseTestSourceFileVerification):
@@ -280,6 +287,11 @@
         # A 1.0 source can contain an original tarball and a Debian diff
         self.assertFilesOK({ORIG_TARBALL: 1, DIFF: 1})
 
+    def testFormat10DebianWithOrigSignature(self):
+        # A 1.0 source can contain an original tarball signature.
+        self.assertFilesOK(
+            {ORIG_TARBALL: 1, ORIG_TARBALL_SIGNATURE: 1, DIFF: 1})
+
     def testFormat10Native(self):
         # A 1.0 source can contain a native tarball.
         self.assertFilesOK({NATIVE_TARBALL: 1})
@@ -287,25 +299,33 @@
     def testFormat10CannotHaveWrongFiles(self):
         # A 1.0 source cannot have a combination of native and
         # non-native files, and cannot have just one of the non-native
-        # files.
+        # files or an original tarball signature without an original
+        # tarball.
         for combination in (
-            {DIFF: 1}, {ORIG_TARBALL: 1}, {ORIG_TARBALL: 1, DIFF: 1,
-            NATIVE_TARBALL: 1}):
+            {DIFF: 1}, {ORIG_TARBALL: 1}, {ORIG_TARBALL_SIGNATURE: 1},
+            {ORIG_TARBALL_SIGNATURE: 1, DIFF: 1}):
+            {ORIG_TARBALL: 1, DIFF: 1, NATIVE_TARBALL: 1},
             self.assertErrorsForFiles([self.wrong_files_error], combination)
 
         # A 1.0 source with component tarballs is invalid.
         self.assertErrorsForFiles(
-            [self.wrong_files_error], {ORIG_TARBALL: 1, DIFF: 1}, {'foo': 1})
+            [self.wrong_files_error], {ORIG_TARBALL: 1, DIFF: 1},
+            components={'foo': 1})
+
+        # A 1.0 source with component tarball signatures is invalid.
+        self.assertErrorsForFiles(
+            [self.wrong_files_error], {ORIG_TARBALL: 1, DIFF: 1},
+            component_signatures={'foo': 1})
 
     def testFormat10CannotUseBzip2(self):
         # 1.0 sources cannot use bzip2 compression.
         self.assertErrorsForFiles(
-            [self.bzip2_error], {NATIVE_TARBALL: 1}, {}, 1, 0)
+            [self.bzip2_error], {NATIVE_TARBALL: 1}, {}, bzip2_count=1)
 
     def testFormat10CannotUseXz(self):
         # 1.0 sources cannot use xz compression.
         self.assertErrorsForFiles(
-            [self.xz_error], {NATIVE_TARBALL: 1}, {}, 0, 1)
+            [self.xz_error], {NATIVE_TARBALL: 1}, {}, xz_count=1)
 
 
 class Test30QuiltSourceFormatVerification(BaseTestSourceFileVerification):
@@ -318,14 +338,21 @@
 
     def testFormat30Quilt(self):
         # A 3.0 (quilt) source must contain an orig tarball and a debian
-        # tarball. It may also contain at most one component tarball for
-        # each component, and can use gzip, bzip2, or xz compression.
+        # tarball. It may also contain at most one component tarball (with
+        # optional signature) for each component, and can use gzip, bzip2,
+        # or xz compression.
         for components in ({}, {'foo': 1}, {'foo': 1, 'bar': 1}):
-            for bzip2_count in (0, 1):
-                for xz_count in (0, 1):
-                    self.assertFilesOK(
-                        {ORIG_TARBALL: 1, DEBIAN_TARBALL: 1}, components,
-                        bzip2_count, xz_count)
+            for component_signatures in ({}, components):
+                for orig_signature in (0, 1):
+                    for bzip2_count in (0, 1):
+                        for xz_count in (0, 1):
+                            self.assertFilesOK(
+                                {ORIG_TARBALL: 1,
+                                 ORIG_TARBALL_SIGNATURE: orig_signature,
+                                 DEBIAN_TARBALL: 1},
+                                components=components,
+                                component_signatures=component_signatures,
+                                bzip2_count=bzip2_count, xz_count=xz_count)
 
     def testFormat30QuiltCannotHaveConflictingComponentTarballs(self):
         # Multiple conflicting tarballs for a single component are
@@ -352,19 +379,24 @@
         # 3.0 (native) sources must contain just a native tarball. They
         # may use gzip, bzip2, or xz compression.
         for bzip2_count in (0, 1):
-            self.assertFilesOK({NATIVE_TARBALL: 1}, {},
-            bzip2_count, 0)
-        self.assertFilesOK({NATIVE_TARBALL: 1}, {}, 0, 1)
+            self.assertFilesOK({NATIVE_TARBALL: 1}, bzip2_count=bzip2_count)
+        self.assertFilesOK({NATIVE_TARBALL: 1}, xz_count=1)
 
     def testFormat30NativeCannotHaveWrongFiles(self):
         # 3.0 (quilt) sources may not have a diff, Debian tarball, orig
-        # tarball, or any component tarballs.
-        for filetype in (DIFF, DEBIAN_TARBALL, ORIG_TARBALL):
+        # tarball, orig tarball signature, or any component tarballs.
+        for filetype in (
+                DIFF, DEBIAN_TARBALL, ORIG_TARBALL, ORIG_TARBALL_SIGNATURE):
             self.assertErrorsForFiles(
                 [self.wrong_files_error], {NATIVE_TARBALL: 1, filetype: 1})
         # A 3.0 (native) source with component tarballs is invalid.
         self.assertErrorsForFiles(
-            [self.wrong_files_error], {NATIVE_TARBALL: 1}, {'foo': 1})
+            [self.wrong_files_error], {NATIVE_TARBALL: 1},
+            components={'foo': 1})
+        # A 3.0 (native) source with component tarball signatures is invalid.
+        self.assertErrorsForFiles(
+            [self.wrong_files_error], {NATIVE_TARBALL: 1},
+            component_signatures={'foo': 1})
 
 
 class UnpackedDirTests(TestCase):

=== modified file 'lib/lp/archiveuploader/tests/test_utils.py'
--- lib/lp/archiveuploader/tests/test_utils.py	2015-07-29 07:11:41 +0000
+++ lib/lp/archiveuploader/tests/test_utils.py	2016-06-01 02:29:15 +0000
@@ -1,6 +1,6 @@
 #!/usr/bin/python
 #
-# Copyright 2009-2010 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 # arch-tag: 90e6eb79-83a2-47e8-9f8b-3c687079c923
@@ -87,6 +87,27 @@
             determine_source_file_type('foo_1.0.tar.xz'),
             SourcePackageFileType.NATIVE_TARBALL)
 
+        # (Component) original tarball signatures are detected for any
+        # supported compression method.
+        self.assertEquals(
+            determine_source_file_type('foo_1.0.orig.tar.gz.asc'),
+            SourcePackageFileType.ORIG_TARBALL_SIGNATURE)
+        self.assertEquals(
+            determine_source_file_type('foo_1.0.orig.tar.bz2.asc'),
+            SourcePackageFileType.ORIG_TARBALL_SIGNATURE)
+        self.assertEquals(
+            determine_source_file_type('foo_1.0.orig.tar.xz.asc'),
+            SourcePackageFileType.ORIG_TARBALL_SIGNATURE)
+        self.assertEquals(
+            determine_source_file_type('foo_1.0.orig-foo.tar.gz.asc'),
+            SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE)
+        self.assertEquals(
+            determine_source_file_type('foo_1.0.orig-bar.tar.bz2.asc'),
+            SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE)
+        self.assertEquals(
+            determine_source_file_type('foo_1.0.orig-bar.tar.xz.asc'),
+            SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE)
+
         self.assertEquals(None, determine_source_file_type('foo_1.0'))
         self.assertEquals(None, determine_source_file_type('foo_1.0.blah.gz'))
 

=== modified file 'lib/lp/archiveuploader/utils.py'
--- lib/lp/archiveuploader/utils.py	2015-07-29 07:01:04 +0000
+++ lib/lp/archiveuploader/utils.py	2016-06-01 02:29:15 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2015 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Archive uploader utilities."""
@@ -19,6 +19,7 @@
     're_isadeb',
     're_issource',
     're_is_component_orig_tar_ext',
+    're_is_component_orig_tar_ext_sig',
     're_no_epoch',
     're_no_revision',
     're_valid_version',
@@ -67,12 +68,15 @@
 re_isadeb = re.compile(r"(.+?)_(.+?)_(.+)\.(u?d?deb)$")
 
 source_file_exts = [
-    'orig(?:-.+)?\.tar\.(?:gz|bz2|xz)', 'diff.gz',
+    'orig(?:-.+)?\.tar\.(?:gz|bz2|xz)(?:\.asc)?', 'diff.gz',
     '(?:debian\.)?tar\.(?:gz|bz2|xz)', 'dsc']
 re_issource = re.compile(
     r"([^_]+)_(.+?)\.(%s)" % "|".join(ext for ext in source_file_exts))
 re_is_component_orig_tar_ext = re.compile(r"^orig-(.+).tar.(?:gz|bz2|xz)$")
+re_is_component_orig_tar_ext_sig = re.compile(
+    r"^orig-(.+).tar.(?:gz|bz2|xz)\.asc$")
 re_is_orig_tar_ext = re.compile(r"^orig.tar.(?:gz|bz2|xz)$")
+re_is_orig_tar_ext_sig = re.compile(r"^orig.tar.(?:gz|bz2|xz)\.asc$")
 re_is_debian_tar_ext = re.compile(r"^debian.tar.(?:gz|bz2|xz)$")
 re_is_native_tar_ext = re.compile(r"^tar.(?:gz|bz2|xz)$")
 
@@ -115,6 +119,10 @@
         return SourcePackageFileType.DEBIAN_TARBALL
     elif re_is_native_tar_ext.match(extension):
         return SourcePackageFileType.NATIVE_TARBALL
+    elif re_is_orig_tar_ext_sig.match(extension):
+        return SourcePackageFileType.ORIG_TARBALL_SIGNATURE
+    elif re_is_component_orig_tar_ext_sig.match(extension):
+        return SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE
     else:
         return None
 

=== modified file 'lib/lp/registry/interfaces/sourcepackage.py'
--- lib/lp/registry/interfaces/sourcepackage.py	2015-11-24 01:44:28 +0000
+++ lib/lp/registry/interfaces/sourcepackage.py	2016-06-01 02:29:15 +0000
@@ -411,6 +411,17 @@
 
         This is only part of the 3.0 (quilt) source package format.""")
 
+    ORIG_TARBALL_SIGNATURE = DBItem(9, """
+        Orig Tarball Signature
+
+        This file is a detached signature for an Ubuntu "orig" file.""")
+
+    COMPONENT_ORIG_TARBALL_SIGNATURE = DBItem(10, """
+        Component Orig Tarball Signature
+
+        This file is a detached signature for an Ubuntu component "orig"
+        file.""")
+
 
 class SourcePackageType(DBEnumeratedType):
     """Source Package Format

=== modified file 'lib/lp/soyuz/model/files.py'
--- lib/lp/soyuz/model/files.py	2015-07-08 16:05:11 +0000
+++ lib/lp/soyuz/model/files.py	2016-06-01 02:29:15 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2013 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 __metaclass__ = type
@@ -42,7 +42,9 @@
     def is_orig(self):
         return self.filetype in (
             SourcePackageFileType.ORIG_TARBALL,
-            SourcePackageFileType.COMPONENT_ORIG_TARBALL
+            SourcePackageFileType.COMPONENT_ORIG_TARBALL,
+            SourcePackageFileType.ORIG_TARBALL_SIGNATURE,
+            SourcePackageFileType.COMPONENT_ORIG_TARBALL_SIGNATURE,
             )