Merge lp:~apw/launchpad/signing-add-sha256-checksums into lp:launchpad

[Andy Whitcroft]

Add Signing custom upload (raw-signing/raw-uefi) result checksumming. This is the first step in providing a trust chain for the signing custom uploads (Bug #1285919).

Once the Signing Custom upload is unpacked and processed we make a pass over the results producing a SHA256 checksum for each file. These are accumulated in a SHA256SUMS file which is added to the custom upload result directory.

Add Signing custom upload (raw-signing/raw-uefi) result checksumming. This is the first step in providing a trust chain for the signing custom uploads (Bug #1285919).

Once the Signing Custom upload is unpacked and processed we make a pass over the results producing a SHA256 checksum for each file. These are accumulated in a SHA256SUMS file which is added to the custom upload result directory.

NOTE: this branch carries a missing options test which we rely on when testing checksumming.

[Andy Whitcroft (apw)]

Updated this branch to take into account the recent changes to restore the raw-uefi custom upload.

[Colin Watson (cjwatson)]

review: Needs Fixing

[Andy Whitcroft (apw)]

Ok created a new DirectoryHash object which checksums files offered to it. Used this to generate the checksum files. The other smaller nits are also applied where they still exist.

[Colin Watson (cjwatson)]

I much prefer the DirectoryHash abstraction, thanks. Here's a boring review full of mostly style nits, after which this should be good to land.

review: Approve

=== modified file 'lib/lp/archivepublisher/publishing.py'
--- lib/lp/archivepublisher/publishing.py	2016-04-29 12:57:58 +0000
+++ lib/lp/archivepublisher/publishing.py	2016-06-06 17:15:44 +0000
@@ -261,8 +261,12 @@
         "subdirectories.")
     lfc_name = Attribute(
         "LibraryFileContent attribute name corresponding to this algorithm.")
+    dh_name = Attribute(
+        "Filename for use when checksumming directories with this algorithm.")
     write_by_hash = Attribute(
         "Whether to write by-hash subdirectories for this algorithm.")
+    write_directory_hash = Attribute(
+        "Whether to write *SUM files for this algorithm for directories.")
 
 
 @implementer(IArchiveHash)
@@ -271,7 +275,9 @@
     deb822_name = "md5sum"
     apt_name = "MD5Sum"
     lfc_name = "md5"
+    dh_name = "MD5SUMS"
     write_by_hash = False
+    write_directory_hash = False
 
 
 @implementer(IArchiveHash)
@@ -280,7 +286,9 @@
     deb822_name = "sha1"
     apt_name = "SHA1"
     lfc_name = "sha1"
+    dh_name = "SHA1SUMS"
     write_by_hash = False
+    write_directory_hash = False
 
 
 @implementer(IArchiveHash)
@@ -289,7 +297,9 @@
     deb822_name = "sha256"
     apt_name = "SHA256"
     lfc_name = "sha256"
+    dh_name = "SHA256SUMS"
     write_by_hash = True
+    write_directory_hash = True
 
 
 archive_hashes = [
@@ -1462,3 +1472,54 @@
             count += 1
         self.archive.name = new_name
         self.log.info("Renamed deleted archive '%s'.", self.archive.reference)
+
+
+class DirectoryHash:
+    """Represents a directory hierarchy for hashing."""
+
+    def __init__(self, root, tmpdir, log):
+        self.root = root
+        self.tmpdir = tmpdir
+        self.log = log
+        self.checksum_hash = []
+
+        for usable in self._usable_archive_hashes:
+            checksum_file = os.path.join(self.root, usable.dh_name)
+            self.checksum_hash.append(
+                (RepositoryIndexFile(checksum_file, self.tmpdir), usable))
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.close()
+
+    @property
+    def _usable_archive_hashes(self):
+        for archive_hash in archive_hashes:
+            if archive_hash.write_directory_hash:
+                yield archive_hash
+
+    def add(self, path):
+        """Add a path to be checksummed."""
+        hashes = [
+            (checksum_file, archive_hash.hash_factory())
+            for (checksum_file, archive_hash) in self.checksum_hash]
+        with open(path, 'rb') as in_file:
+            for chunk in iter(lambda: in_file.read(256 * 1024), ""):
+                for (checksum_file, hashobj) in hashes:
+                    hashobj.update(chunk)
+
+        for (checksum_file, hashobj) in hashes:
+            checksum_file.write("%s *%s\n" %
+                (hashobj.hexdigest(), path[len(self.root) + 1:]))
+
+    def add_dir(self, path):
+        """Recursively add a directory path to be checksummed."""
+        for dirpath, dirnames, filenames in os.walk(path):
+            for filename in filenames:
+                self.add(os.path.join(dirpath, filename))
+
+    def close(self):
+        for (checksum_file, archive_hash) in self.checksum_hash:
+            checksum_file.close()
 
=== modified file 'lib/lp/archivepublisher/signing.py'
--- lib/lp/archivepublisher/signing.py	2016-05-31 12:40:38 +0000
+++ lib/lp/archivepublisher/signing.py	2016-06-06 17:15:44 +0000
@@ -26,6 +26,7 @@
 import textwrap
 
 from lp.archivepublisher.customupload import CustomUpload
+from lp.archivepublisher.utils import RepositoryIndexFile
 from lp.services.osutils import remove_if_exists
 from lp.soyuz.interfaces.queue import CustomUploadError
 
@@ -291,6 +292,9 @@
 
         No actual extraction is required.
         """
+        # Avoid circular import.
+        from lp.archivepublisher.publishing import DirectoryHash
+
         super(SigningUpload, self).extract()
         self.setSigningOptions()
         filehandlers = list(self.findSigningHandlers())
@@ -303,6 +307,10 @@
         if 'tarball' in self.signing_options:
             self.convertToTarball()
 
+        versiondir = os.path.join(self.tmpdir, self.version)
+        with DirectoryHash(versiondir, self.tmpdir, self.logger) as hasher:
+            hasher.add_dir(versiondir)
+
     def shouldInstall(self, filename):
         return filename.startswith("%s/" % self.version)
 
 
=== modified file 'lib/lp/archivepublisher/tests/test_publisher.py'
--- lib/lp/archivepublisher/tests/test_publisher.py	2016-05-14 10:17:36 +0000
+++ lib/lp/archivepublisher/tests/test_publisher.py	2016-06-06 17:15:44 +0000
@@ -37,6 +37,7 @@
     Is,
     LessThan,
     Matcher,
+    MatchesDict,
     MatchesListwise,
     MatchesSetwise,
     MatchesStructure,
@@ -56,6 +57,7 @@
 from lp.archivepublisher.publishing import (
     ByHash,
     ByHashes,
+    DirectoryHash,
     getPublisher,
     I18nIndex,
     Publisher,
@@ -92,7 +94,10 @@
 from lp.soyuz.interfaces.archive import IArchiveSet
 from lp.soyuz.interfaces.archivefile import IArchiveFileSet
 from lp.soyuz.tests.test_publishing import TestNativePublishingBase
-from lp.testing import TestCaseWithFactory
+from lp.testing import (
+    TestCase,
+    TestCaseWithFactory,
+    )
 from lp.testing.fakemethod import FakeMethod
 from lp.testing.gpgkeys import gpgkeysdir
 from lp.testing.keyserver import KeyServerTac
@@ -3160,3 +3165,103 @@
 
         partner = self.factory.makeArchive(purpose=ArchivePurpose.PARTNER)
         self.assertEqual([], self.makePublisher(partner).subcomponents)
+
+
+class TestDirectoryHash(TestCase):
+    """Unit tests for DirectoryHash object."""
+
+    def createTestFile(self, path, content):
+        with open(path, "w") as tfd:
+            tfd.write(content)
+        return hashlib.sha256(content).hexdigest()
+
+    @property
+    def all_hash_files(self):
+        return ['MD5SUMS', 'SHA1SUMS', 'SHA256SUMS']
+
+    @property
+    def expected_hash_files(self):
+        return ['SHA256SUMS']
+
+    def fetchSums(self, rootdir):
+        result = {}
+        for dh_file in self.all_hash_files:
+            checksum_file = os.path.join(rootdir, dh_file)
+            if os.path.exists(checksum_file):
+                with open(checksum_file, "r") as sfd:
+                    for line in sfd:
+                        file_list = result.setdefault(dh_file, [])
+                        file_list.append(line.strip().split(' '))
+        return result
+
+    def test_checksum_files_created(self):
+        tmpdir = unicode(self.makeTemporaryDirectory())
+        rootdir = unicode(self.makeTemporaryDirectory())
+
+        for dh_file in self.all_hash_files:
+            checksum_file = os.path.join(rootdir, dh_file)
+            self.assertFalse(os.path.exists(checksum_file))
+
+        with DirectoryHash(rootdir, tmpdir, None) as dh:
+            pass
+
+        for dh_file in self.all_hash_files:
+            checksum_file = os.path.join(rootdir, dh_file)
+            if dh_file in self.expected_hash_files:
+                self.assertTrue(os.path.exists(checksum_file))
+            else:
+                self.assertFalse(os.path.exists(checksum_file))
+
+    def test_basic_file_add(self):
+        tmpdir = unicode(self.makeTemporaryDirectory())
+        rootdir = unicode(self.makeTemporaryDirectory())
+        test1_file = os.path.join(rootdir, "test1")
+        test1_hash = self.createTestFile(test1_file, "test1")
+
+        test2_file = os.path.join(rootdir, "test2")
+        test2_hash = self.createTestFile(test2_file, "test2")
+
+        os.mkdir(os.path.join(rootdir, "subdir1"))
+
+        test3_file = os.path.join(rootdir, "subdir1", "test3")
+        test3_hash = self.createTestFile(test3_file, "test3")
+
+        with DirectoryHash(rootdir, tmpdir, None) as dh:
+            dh.add(test1_file)
+            dh.add(test2_file)
+            dh.add(test3_file)
+
+        expected = {
+            'SHA256SUMS': MatchesSetwise(
+                Equals([test1_hash, "*test1"]),
+                Equals([test2_hash, "*test2"]),
+                Equals([test3_hash, "*subdir1/test3"]),
+            ),
+        }
+        self.assertThat(self.fetchSums(rootdir), MatchesDict(expected))
+
+    def test_basic_directory_add(self):
+        tmpdir = unicode(self.makeTemporaryDirectory())
+        rootdir = unicode(self.makeTemporaryDirectory())
+        test1_file = os.path.join(rootdir, "test1")
+        test1_hash = self.createTestFile(test1_file, "test1 dir")
+
+        test2_file = os.path.join(rootdir, "test2")
+        test2_hash = self.createTestFile(test2_file, "test2 dir")
+
+        os.mkdir(os.path.join(rootdir, "subdir1"))
+
+        test3_file = os.path.join(rootdir, "subdir1", "test3")
+        test3_hash = self.createTestFile(test3_file, "test3 dir")
+
+        with DirectoryHash(rootdir, tmpdir, None) as dh:
+            dh.add_dir(rootdir)
+
+        expected = {
+            'SHA256SUMS': MatchesSetwise(
+                Equals([test1_hash, "*test1"]),
+                Equals([test2_hash, "*test2"]),
+                Equals([test3_hash, "*subdir1/test3"]),
+            ),
+        }
+        self.assertThat(self.fetchSums(rootdir), MatchesDict(expected))
 
=== modified file 'lib/lp/archivepublisher/tests/test_signing.py'
--- lib/lp/archivepublisher/tests/test_signing.py	2016-05-31 12:23:15 +0000
+++ lib/lp/archivepublisher/tests/test_signing.py	2016-06-06 17:15:44 +0000
@@ -256,6 +256,23 @@
         self.assertContentEqual(['first', 'second'],
             upload.signing_options.keys())
 
+    def test_options_none(self):
+        # Specifying no options should leave us with an open tree.
+        self.setUpUefiKeys()
+        self.setUpKmodKeys()
+        self.openArchive("test", "1.0", "amd64")
+        self.archive.add_file("1.0/empty.efi", "")
+        self.archive.add_file("1.0/empty.ko", "")
+        self.process_emulate()
+        self.assertTrue(os.path.exists(os.path.join(
+            self.getSignedPath("test", "amd64"), "1.0", "empty.efi")))
+        self.assertTrue(os.path.exists(os.path.join(
+            self.getSignedPath("test", "amd64"), "1.0", "empty.efi.signed")))
+        self.assertTrue(os.path.exists(os.path.join(
+            self.getSignedPath("test", "amd64"), "1.0", "empty.ko")))
+        self.assertTrue(os.path.exists(os.path.join(
+            self.getSignedPath("test", "amd64"), "1.0", "empty.ko.sig")))
+
     def test_options_tarball(self):
         # Specifying the "tarball" option should create an tarball in
         # the tmpdir.
@@ -602,6 +619,19 @@
         self.assertEqual(stat.S_IMODE(os.stat(self.kmod_pem).st_mode), 0o600)
         self.assertEqual(stat.S_IMODE(os.stat(self.kmod_x509).st_mode), 0o644)
 
+    def test_checksumming_tree(self):
+        # Specifying no options should leave us with an open tree,
+        # confirm it is checksummed.
+        self.setUpUefiKeys()
+        self.setUpKmodKeys()
+        self.openArchive("test", "1.0", "amd64")
+        self.archive.add_file("1.0/empty.efi", "")
+        self.archive.add_file("1.0/empty.ko", "")
+        self.process_emulate()
+        sha256file = os.path.join(self.getSignedPath("test", "amd64"),
+             "1.0", "SHA256SUMS")
+        self.assertTrue(os.path.exists(sha256file))
+
 
 class TestUefi(TestSigningHelpers):
 
 
=== modified file 'lib/lp/archivepublisher/utils.py'
--- lib/lp/archivepublisher/utils.py	2016-02-05 20:28:29 +0000
+++ lib/lp/archivepublisher/utils.py	2016-06-06 17:15:44 +0000
@@ -113,7 +113,7 @@
     (plain, gzip, bzip2, and xz) transparently and atomically.
     """
 
-    def __init__(self, path, temp_root, compressors):
+    def __init__(self, path, temp_root, compressors=None):
         """Store repositories destinations and filename.
 
         The given 'temp_root' needs to exist; on the other hand, the
@@ -123,6 +123,9 @@
         Additionally creates the needed temporary files in the given
         'temp_root'.
         """
+        if compressors is None:
+            compressors = [IndexCompressionType.UNCOMPRESSED]
+
         self.root, filename = os.path.split(path)
         assert os.path.exists(temp_root), 'Temporary root does not exist.'
 
@@ -135,6 +138,12 @@
                 self.old_index_files.append(
                     cls(temp_root, filename, auto_open=False))
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.close()
+
     def write(self, content):
         """Write contents to all target medias."""
         for index_file in self.index_files: