Merge lp:~cjwatson/launchpad/xenial-apt-tests into lp:launchpad

[Colin Watson]

Fix various test failures caused by xenial's new apt version.

[William Grant (wgrant)]

review: Approve (code)

=== modified file 'lib/lp/archivepublisher/tests/test_ftparchive.py'
--- lib/lp/archivepublisher/tests/test_ftparchive.py	2016-09-24 04:24:30 +0000
+++ lib/lp/archivepublisher/tests/test_ftparchive.py	2017-01-14 00:21:20 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2017 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Tests for ftparchive.py"""
@@ -13,7 +13,15 @@
 from textwrap import dedent
 import time
 
-from testtools.matchers import LessThan
+from debian.deb822 import (
+    Packages,
+    Sources,
+    )
+from testtools.matchers import (
+    Equals,
+    LessThan,
+    MatchesListwise,
+    )
 from zope.component import getUtility
 
 from lp.archivepublisher.config import getPubConfig
@@ -100,14 +108,13 @@
         super(TestFTPArchive, self).tearDown()
         shutil.rmtree(self._config.distroroot)
 
-    def _verifyFile(self, filename, directory,
-                    result_suffix="", result_open_func=open):
+    def _verifyFile(self, filename, directory):
         """Compare byte-to-byte the given file and the respective sample.
 
         It's a poor way of testing files generated by apt-ftparchive.
         """
-        result_path = os.path.join(directory, filename) + result_suffix
-        with result_open_func(result_path) as result_file:
+        result_path = os.path.join(directory, filename)
+        with open(result_path) as result_file:
             result_text = result_file.read()
         sample_path = os.path.join(self._sampledir, filename)
         with open(sample_path) as sample_file:
@@ -120,6 +127,18 @@
             sample_text.splitlines(), result_text.splitlines())
         self.assertEqual(sample_text, result_text, '\n'.join(diff_lines))
 
+    def _verifyDeb822(self, filename, directory, deb822_factory,
+                      result_suffix="", result_open_func=open):
+        """Compare the given file and the respective sample as deb822 files."""
+        result_path = os.path.join(directory, filename) + result_suffix
+        with result_open_func(result_path) as result_file:
+            result = list(deb822_factory(result_file))
+        sample_path = os.path.join(self._sampledir, filename)
+        with open(sample_path) as sample_file:
+            sample = list(deb822_factory(sample_file))
+        self.assertThat(
+            result, MatchesListwise([Equals(stanza) for stanza in sample]))
+
     def _verifyEmpty(self, path, open_func=open):
         """Assert that the given file is empty."""
         with open_func(path) as result_file:
@@ -441,18 +460,20 @@
         # check'. Although they should remain active in PQM to avoid possible
         # regressions.
         fa.runApt(apt_conf)
-        self._verifyFile(
+        self._verifyDeb822(
             "Packages",
             os.path.join(self._distsdir, "hoary-test", "main", "binary-i386"),
+            Packages.iter_paragraphs,
             result_suffix=".gz", result_open_func=gzip.open)
         self._verifyEmpty(
             os.path.join(
                 self._distsdir, "hoary-test", "main", "debian-installer",
                 "binary-i386", "Packages.gz"),
             open_func=gzip.open)
-        self._verifyFile(
+        self._verifyDeb822(
             "Sources",
             os.path.join(self._distsdir, "hoary-test", "main", "source"),
+            Sources.iter_paragraphs,
             result_suffix=".gz", result_open_func=gzip.open)
 
         # XXX cprov 2007-03-21: see above, byte-to-byte configuration
@@ -666,15 +687,15 @@
         binary_overrides = FakeSelectResult([(
             "bin%d" % i, "main", "misc", "i386",
             PackagePublishingPriority.EXTRA, BinaryPackageFormat.DEB, None)
-            for i in range(10)])
+            for i in range(50)])
         fa.publishOverrides("hoary-test", source_overrides, binary_overrides)
         source_files = FakeSelectResult([("tiny", "tiny_0.1.dsc", "main")])
         binary_files = FakeSelectResult([(
             "bin%d" % i, "bin%d_1_i386.deb" % i, "main", "binary-i386")
-            for i in range(10)])
+            for i in range(50)])
         fa.publishFileLists("hoary-test", source_files, binary_files)
         self._addRepositoryFile("main", "tiny", "tiny_0.1.dsc")
-        for i in range(10):
+        for i in range(50):
             self._addRepositoryFile(
                 "main", "bin%d" % i, "bin%d_1_i386.deb" % i,
                 samplename="tiny_0.1_i386.deb")
@@ -683,7 +704,7 @@
 
         # Remove most of this repository's files so that cleanCaches has
         # something to do.
-        for i in range(9):
+        for i in range(49):
             os.unlink(
                 self._dp.pathFor("main", "bin%d" % i, "bin%d_1_i386.deb" % i))
 

=== modified file 'lib/lp/archiveuploader/tagfiles.py'
--- lib/lp/archiveuploader/tagfiles.py	2011-12-14 11:58:56 +0000
+++ lib/lp/archiveuploader/tagfiles.py	2017-01-14 00:21:20 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2017 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Utility classes for parsing Debian tag files."""
@@ -35,7 +35,10 @@
     with tempfile.TemporaryFile() as f:
         f.write(strip_pgp_signature(content))
         f.seek(0)
-        stanzas = list(apt_pkg.TagFile(f))
+        try:
+            stanzas = list(apt_pkg.TagFile(f))
+        except SystemError as e:
+            raise TagFileParseError("%s: %s" % (filename, e))
     if len(stanzas) != 1:
         raise TagFileParseError(
             "%s: multiple stanzas where only one is expected" % filename)

=== modified file 'lib/lp/soyuz/doc/gina.txt'
--- lib/lp/soyuz/doc/gina.txt	2016-01-26 15:47:37 +0000
+++ lib/lp/soyuz/doc/gina.txt	2017-01-14 00:21:20 +0000
@@ -166,7 +166,6 @@
     MissingRequiredArguments: ['installed_size']
     ERROR   Invalid Sources stanza in /tmp/tmp...
     ...
-    KeyError: 'Bogus, bogus, bogus\n'
     WARNING No changelog file found for mkvmlinuz in mkvmlinuz-14ubuntu1
     WARNING No copyright file found for mkvmlinuz in mkvmlinuz-14ubuntu1
     WARNING Invalid urgency in mkvmlinuz, None, assumed 'low'
@@ -548,7 +547,6 @@
     MissingRequiredArguments: ['installed_size']
     ERROR   Invalid Sources stanza in /tmp/tmp...
     ...
-    KeyError: 'Bogus, bogus, bogus\n'
     ERROR   Error processing package files for python-sqlite
     ...
     PoolFileNotFound: File python-sqlite_1.0.1-2ubuntu1.dsc not in archive

=== modified file 'lib/lp/soyuz/doc/soyuz-upload.txt'
--- lib/lp/soyuz/doc/soyuz-upload.txt	2016-02-08 12:20:20 +0000
+++ lib/lp/soyuz/doc/soyuz-upload.txt	2017-01-14 00:21:20 +0000
@@ -39,7 +39,10 @@
 been uploaded over FTP.
 
     >>> from lp.services.config import config
-    >>> from lp.archiveuploader.tagfiles import parse_tagfile
+    >>> from lp.archiveuploader.tagfiles import (
+    ...     parse_tagfile,
+    ...     TagFileParseError,
+    ...     )
     >>> import glob
     >>> test_files_dir = os.path.join(config.root,
     ...                               "lib/lp/soyuz/scripts/"
@@ -52,7 +55,10 @@
 
     >>> seq = 1
     >>> for changes_filepath in changes:
-    ...     tf = parse_tagfile(changes_filepath)
+    ...     try:
+    ...         tf = parse_tagfile(changes_filepath)
+    ...     except TagFileParseError:
+    ...         tf = {}
     ...
     ...     if "Source" in tf:
     ...         package_names.append(tf["Source"])
@@ -460,30 +466,34 @@
     ...     "/var/tmp/archive/ubuntutest/pool/universe/e/etherwake"))
     3
 
+Define a helper for pretty-printing Deb822 objects, based on Deb822.dump but
+with sorted output.
+
+    >>> def pprint_deb822(deb822):
+    ...     for key in sorted(deb822):
+    ...         value = deb822.get_as_string(key)
+    ...         if not value or value[0] == '\n':
+    ...             print '%s:%s' % (key, value)
+    ...         else:
+    ...             print '%s: %s' % (key, value)
+    ...     print
+
 Check the generation of a correct Sources tag file for the main
 component of ubuntutest/breezy-autotest, containing the only the
 required entry for 'etherwake':
 
     >>> import gzip
+    >>> from debian.deb822 import Sources
 
-    >>> sources = gzip.open(
-    ...    "/var/tmp/archive/ubuntutest/dists/breezy-autotest/universe/source"
-    ...    "/Sources.gz").read()
-    >>> print sources + '\nEND'
-    Package: etherwake
+    >>> with gzip.open(
+    ...        "/var/tmp/archive/ubuntutest/dists/breezy-autotest/universe/"
+    ...        "source/Sources.gz") as sources_file:
+    ...     for source in Sources.iter_paragraphs(sources_file):
+    ...         pprint_deb822(source)
+    ...     print 'END'
+    Architecture: any
     Binary: etherwake
-    Version: 1.08-1
-    Section: universe/net
-    Maintainer: Alain Schroeder <...@...org>
     Build-Depends: debhelper (>> 2.0)
-    Architecture: any
-    Standards-Version: 3.5.10.0
-    Format: 1.0
-    Directory: pool/universe/e/etherwake
-    Files:
-     f13711c5b8261fbb77b43ae0e8ba9360 566 etherwake_1.08-1.dsc
-     c2dc10f98bac012b900fd0b46721fc80 4455 etherwake_1.08.orig.tar.gz
-     95c1e89e3ad7bc8740793bdf7aeb7334 4145 etherwake_1.08-1.diff.gz
     Checksums-Sha1:
      2ddcdc87ab3dc35d5ce8232b0cc76bad8242725f 566 etherwake_1.08-1.dsc
      4d8aa805cf262a613a48597e3638054dae421048 4455 etherwake_1.08.orig.tar.gz
@@ -496,7 +506,17 @@
      51216a36b2ab6fde6ae04d5bcb0b7cefa9a18eb4b2b11552ca8f3abde928159e93729f30c6079e913078e966817368a6095de2cb4239676a3d6ed5d49d9de699 566 etherwake_1.08-1.dsc
      6ab88a579ae3fdbbe0f1904712a3a42fab98fa586c3718243d2380f3cb021158c228312001b0685a77dc7171b0307d591ad971a82cd1ccd3511135b23d95ee21 4455 etherwake_1.08.orig.tar.gz
      814074aa8349936fbec84b3ee703788159a085f0ce4a5e35d2dbef617e1c3c6e60818d155772d47b58e0823ed4bc9af29136f64eac8d643a833660e537145cb1 4145 etherwake_1.08-1.diff.gz
-    <BLANKLINE>
+    Directory: pool/universe/e/etherwake
+    Files:
+     f13711c5b8261fbb77b43ae0e8ba9360 566 etherwake_1.08-1.dsc
+     c2dc10f98bac012b900fd0b46721fc80 4455 etherwake_1.08.orig.tar.gz
+     95c1e89e3ad7bc8740793bdf7aeb7334 4145 etherwake_1.08-1.diff.gz
+    Format: 1.0
+    Maintainer: Alain Schroeder <...@...org>
+    Package: etherwake
+    Section: universe/net
+    Standards-Version: 3.5.10.0
+    Version: 1.08-1
     <BLANKLINE>
     END
 

=== modified file 'lib/lp/soyuz/scripts/gina/archive.py'
--- lib/lp/soyuz/scripts/gina/archive.py	2016-03-20 22:47:48 +0000
+++ lib/lp/soyuz/scripts/gina/archive.py	2017-01-14 00:21:20 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2017 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Archive pool classes.
@@ -202,17 +202,21 @@
             # but we go over it to also cover source packages that only
             # compile for one architecture.
             sources = apt_pkg.TagFile(info_set.srcfile)
-            for section in sources:
-                try:
-                    src_tmp = dict(section)
-                    src_tmp['Component'] = info_set.component
-                    src_name = src_tmp['Package']
-                except KeyError:
-                    log.exception(
-                        "Invalid Sources stanza in %s",
-                        info_set.sources_tagfile)
-                    continue
-                self.src_map[src_name].append(src_tmp)
+            try:
+                for section in sources:
+                    try:
+                        src_tmp = dict(section)
+                        src_tmp['Component'] = info_set.component
+                        src_name = src_tmp['Package']
+                    except KeyError:
+                        log.exception(
+                            "Invalid Sources stanza in %s",
+                            info_set.sources_tagfile)
+                        continue
+                    self.src_map[src_name].append(src_tmp)
+            except SystemError:
+                log.exception(
+                    "Invalid Sources stanza in %s", info_set.sources_tagfile)
 
             # Check if it's in source-only mode.  If so, skip binary index
             # mapping.

=== modified file 'lib/lp/soyuz/tests/test_archivejob.py'
--- lib/lp/soyuz/tests/test_archivejob.py	2015-09-07 15:15:29 +0000
+++ lib/lp/soyuz/tests/test_archivejob.py	2017-01-14 00:21:20 +0000
@@ -1,6 +1,8 @@
-# Copyright 2010-2015 Canonical Ltd.  This software is licensed under the
+# Copyright 2010-2017 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
+from debian.deb822 import Changes
+
 from lp.services.job.runner import JobRunner
 from lp.services.mail.sendmail import format_address_for_person
 from lp.soyuz.enums import (
@@ -97,7 +99,13 @@
         # Running a job produces a notification.  Detailed tests of which
         # notifications go to whom live in the PackageUpload and
         # PackageUploadMailer tests.
-        upload = self.factory.makeSourcePackageUpload()
+        distroseries = self.factory.makeDistroSeries()
+        creator = self.factory.makePerson()
+        changes = Changes({"Changed-By": format_address_for_person(creator)})
+        upload = self.factory.makePackageUpload(
+            distroseries=distroseries, archive=distroseries.main_archive,
+            changes_file_content=changes.dump().encode("UTF-8"))
+        upload.addSource(self.factory.makeSourcePackageRelease())
         self.factory.makeComponentSelection(
             upload.distroseries, upload.sourcepackagerelease.component)
         upload.setAccepted()
@@ -106,8 +114,6 @@
         with dbuser(job.config.dbuser):
             JobRunner([job]).runAll()
         [email] = pop_notifications()
-        self.assertEqual(
-            format_address_for_person(upload.sourcepackagerelease.creator),
-            email['To'])
+        self.assertEqual(format_address_for_person(creator), email['To'])
         self.assertIn('(Accepted)', email['Subject'])
         self.assertIn('Fake summary', email.get_payload()[0].get_payload())

=== modified file 'lib/lp/soyuz/tests/test_packageupload.py'
--- lib/lp/soyuz/tests/test_packageupload.py	2016-05-23 11:23:48 +0000
+++ lib/lp/soyuz/tests/test_packageupload.py	2017-01-14 00:21:20 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2017 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Test Build features."""
@@ -294,13 +294,17 @@
         # Duplicate queue entries are handled sensibly.
         self.test_publisher.prepareBreezyAutotest()
         distroseries = self.test_publisher.distroseries
+        uploader = self.factory.makePerson()
+        changes = Changes({"Changed-By": uploader.preferredemail.email})
         upload_one = self.factory.makePackageUpload(
-            archive=distroseries.main_archive, distroseries=distroseries)
+            archive=distroseries.main_archive, distroseries=distroseries,
+            changes_file_content=changes.dump().encode("UTF-8"))
         upload_one.addSource(self.factory.makeSourcePackageRelease(
             sourcepackagename="cnews", distroseries=distroseries,
             component="main", version="1.0"))
         upload_two = self.factory.makePackageUpload(
-            archive=distroseries.main_archive, distroseries=distroseries)
+            archive=distroseries.main_archive, distroseries=distroseries,
+            changes_file_content=changes.dump().encode("UTF-8"))
         upload_two.addSource(self.factory.makeSourcePackageRelease(
             sourcepackagename="cnews", distroseries=distroseries,
             component="main", version="1.0"))
@@ -947,10 +951,14 @@
 
     def makeSourcePackageUpload(self, person, **kwargs):
         with person_logged_in(person):
-            upload = self.factory.makeSourcePackageUpload(
-                distroseries=self.distroseries, **kwargs)
-            transaction.commit()
-            spr = upload.sourcepackagerelease
+            uploader = self.factory.makePerson()
+            changes = Changes({"Changed-By": uploader.preferredemail.email})
+            upload = self.factory.makePackageUpload(
+                distroseries=self.distroseries,
+                archive=self.distroseries.main_archive,
+                changes_file_content=changes.dump().encode("UTF-8"))
+            spr = self.factory.makeSourcePackageRelease(**kwargs)
+            upload.addSource(spr)
             for extension in ("dsc", "tar.gz"):
                 filename = "%s_%s.%s" % (spr.name, spr.version, extension)
                 lfa = self.factory.makeLibraryFileAlias(filename=filename)