Merge lp:~apw/launchpad/signing-add-sha256-checksums into lp:launchpad

Proposed by Andy Whitcroft on 2016-05-24
Status: Merged
Merged at revision: 18097
Proposed branch: lp:~apw/launchpad/signing-add-sha256-checksums
Merge into: lp:launchpad
Diff against target: 354 lines (+215/-2)
5 files modified
lib/lp/archivepublisher/publishing.py (+61/-0)
lib/lp/archivepublisher/signing.py (+8/-0)
lib/lp/archivepublisher/tests/test_publisher.py (+106/-1)
lib/lp/archivepublisher/tests/test_signing.py (+30/-0)
lib/lp/archivepublisher/utils.py (+10/-1)
To merge this branch: bzr merge lp:~apw/launchpad/signing-add-sha256-checksums
Reviewer Review Type Date Requested Status
Colin Watson 2016-05-24 Approve on 2016-06-06
Review via email: mp+295615@code.launchpad.net

Commit Message

Add Signing custom upload (raw-signing/raw-uefi) result checksumming. This is the first step in providing a trust chain for the signing custom uploads (Bug #1285919).

Once the Signing Custom upload is unpacked and processed we make a pass over the results producing a SHA256 checksum for each file. These are accumulated in a SHA256SUMS file which is added to the custom upload result directory.

Description of the Change

Add Signing custom upload (raw-signing/raw-uefi) result checksumming. This is the first step in providing a trust chain for the signing custom uploads (Bug #1285919).

Once the Signing Custom upload is unpacked and processed we make a pass over the results producing a SHA256 checksum for each file. These are accumulated in a SHA256SUMS file which is added to the custom upload result directory.

NOTE: this branch carries a missing options test which we rely on when testing checksumming.

To post a comment you must log in.
Andy Whitcroft (apw) wrote :

Updated this branch to take into account the recent changes to restore the raw-uefi custom upload.

Colin Watson (cjwatson) :
review: Needs Fixing
Andy Whitcroft (apw) wrote :

Ok created a new DirectoryHash object which checksums files offered to it. Used this to generate the checksum files. The other smaller nits are also applied where they still exist.

Colin Watson (cjwatson) wrote :

I much prefer the DirectoryHash abstraction, thanks. Here's a boring review full of mostly style nits, after which this should be good to land.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/archivepublisher/publishing.py'
2--- lib/lp/archivepublisher/publishing.py 2016-04-29 12:57:58 +0000
3+++ lib/lp/archivepublisher/publishing.py 2016-06-06 17:15:44 +0000
4@@ -261,8 +261,12 @@
5 "subdirectories.")
6 lfc_name = Attribute(
7 "LibraryFileContent attribute name corresponding to this algorithm.")
8+ dh_name = Attribute(
9+ "Filename for use when checksumming directories with this algorithm.")
10 write_by_hash = Attribute(
11 "Whether to write by-hash subdirectories for this algorithm.")
12+ write_directory_hash = Attribute(
13+ "Whether to write *SUM files for this algorithm for directories.")
14
15
16 @implementer(IArchiveHash)
17@@ -271,7 +275,9 @@
18 deb822_name = "md5sum"
19 apt_name = "MD5Sum"
20 lfc_name = "md5"
21+ dh_name = "MD5SUMS"
22 write_by_hash = False
23+ write_directory_hash = False
24
25
26 @implementer(IArchiveHash)
27@@ -280,7 +286,9 @@
28 deb822_name = "sha1"
29 apt_name = "SHA1"
30 lfc_name = "sha1"
31+ dh_name = "SHA1SUMS"
32 write_by_hash = False
33+ write_directory_hash = False
34
35
36 @implementer(IArchiveHash)
37@@ -289,7 +297,9 @@
38 deb822_name = "sha256"
39 apt_name = "SHA256"
40 lfc_name = "sha256"
41+ dh_name = "SHA256SUMS"
42 write_by_hash = True
43+ write_directory_hash = True
44
45
46 archive_hashes = [
47@@ -1462,3 +1472,54 @@
48 count += 1
49 self.archive.name = new_name
50 self.log.info("Renamed deleted archive '%s'.", self.archive.reference)
51+
52+
53+class DirectoryHash:
54+ """Represents a directory hierarchy for hashing."""
55+
56+ def __init__(self, root, tmpdir, log):
57+ self.root = root
58+ self.tmpdir = tmpdir
59+ self.log = log
60+ self.checksum_hash = []
61+
62+ for usable in self._usable_archive_hashes:
63+ checksum_file = os.path.join(self.root, usable.dh_name)
64+ self.checksum_hash.append(
65+ (RepositoryIndexFile(checksum_file, self.tmpdir), usable))
66+
67+ def __enter__(self):
68+ return self
69+
70+ def __exit__(self, type, value, traceback):
71+ self.close()
72+
73+ @property
74+ def _usable_archive_hashes(self):
75+ for archive_hash in archive_hashes:
76+ if archive_hash.write_directory_hash:
77+ yield archive_hash
78+
79+ def add(self, path):
80+ """Add a path to be checksummed."""
81+ hashes = [
82+ (checksum_file, archive_hash.hash_factory())
83+ for (checksum_file, archive_hash) in self.checksum_hash]
84+ with open(path, 'rb') as in_file:
85+ for chunk in iter(lambda: in_file.read(256 * 1024), ""):
86+ for (checksum_file, hashobj) in hashes:
87+ hashobj.update(chunk)
88+
89+ for (checksum_file, hashobj) in hashes:
90+ checksum_file.write("%s *%s\n" %
91+ (hashobj.hexdigest(), path[len(self.root) + 1:]))
92+
93+ def add_dir(self, path):
94+ """Recursively add a directory path to be checksummed."""
95+ for dirpath, dirnames, filenames in os.walk(path):
96+ for filename in filenames:
97+ self.add(os.path.join(dirpath, filename))
98+
99+ def close(self):
100+ for (checksum_file, archive_hash) in self.checksum_hash:
101+ checksum_file.close()
102
103=== modified file 'lib/lp/archivepublisher/signing.py'
104--- lib/lp/archivepublisher/signing.py 2016-05-31 12:40:38 +0000
105+++ lib/lp/archivepublisher/signing.py 2016-06-06 17:15:44 +0000
106@@ -26,6 +26,7 @@
107 import textwrap
108
109 from lp.archivepublisher.customupload import CustomUpload
110+from lp.archivepublisher.utils import RepositoryIndexFile
111 from lp.services.osutils import remove_if_exists
112 from lp.soyuz.interfaces.queue import CustomUploadError
113
114@@ -291,6 +292,9 @@
115
116 No actual extraction is required.
117 """
118+ # Avoid circular import.
119+ from lp.archivepublisher.publishing import DirectoryHash
120+
121 super(SigningUpload, self).extract()
122 self.setSigningOptions()
123 filehandlers = list(self.findSigningHandlers())
124@@ -303,6 +307,10 @@
125 if 'tarball' in self.signing_options:
126 self.convertToTarball()
127
128+ versiondir = os.path.join(self.tmpdir, self.version)
129+ with DirectoryHash(versiondir, self.tmpdir, self.logger) as hasher:
130+ hasher.add_dir(versiondir)
131+
132 def shouldInstall(self, filename):
133 return filename.startswith("%s/" % self.version)
134
135
136=== modified file 'lib/lp/archivepublisher/tests/test_publisher.py'
137--- lib/lp/archivepublisher/tests/test_publisher.py 2016-05-14 10:17:36 +0000
138+++ lib/lp/archivepublisher/tests/test_publisher.py 2016-06-06 17:15:44 +0000
139@@ -37,6 +37,7 @@
140 Is,
141 LessThan,
142 Matcher,
143+ MatchesDict,
144 MatchesListwise,
145 MatchesSetwise,
146 MatchesStructure,
147@@ -56,6 +57,7 @@
148 from lp.archivepublisher.publishing import (
149 ByHash,
150 ByHashes,
151+ DirectoryHash,
152 getPublisher,
153 I18nIndex,
154 Publisher,
155@@ -92,7 +94,10 @@
156 from lp.soyuz.interfaces.archive import IArchiveSet
157 from lp.soyuz.interfaces.archivefile import IArchiveFileSet
158 from lp.soyuz.tests.test_publishing import TestNativePublishingBase
159-from lp.testing import TestCaseWithFactory
160+from lp.testing import (
161+ TestCase,
162+ TestCaseWithFactory,
163+ )
164 from lp.testing.fakemethod import FakeMethod
165 from lp.testing.gpgkeys import gpgkeysdir
166 from lp.testing.keyserver import KeyServerTac
167@@ -3160,3 +3165,103 @@
168
169 partner = self.factory.makeArchive(purpose=ArchivePurpose.PARTNER)
170 self.assertEqual([], self.makePublisher(partner).subcomponents)
171+
172+
173+class TestDirectoryHash(TestCase):
174+ """Unit tests for DirectoryHash object."""
175+
176+ def createTestFile(self, path, content):
177+ with open(path, "w") as tfd:
178+ tfd.write(content)
179+ return hashlib.sha256(content).hexdigest()
180+
181+ @property
182+ def all_hash_files(self):
183+ return ['MD5SUMS', 'SHA1SUMS', 'SHA256SUMS']
184+
185+ @property
186+ def expected_hash_files(self):
187+ return ['SHA256SUMS']
188+
189+ def fetchSums(self, rootdir):
190+ result = {}
191+ for dh_file in self.all_hash_files:
192+ checksum_file = os.path.join(rootdir, dh_file)
193+ if os.path.exists(checksum_file):
194+ with open(checksum_file, "r") as sfd:
195+ for line in sfd:
196+ file_list = result.setdefault(dh_file, [])
197+ file_list.append(line.strip().split(' '))
198+ return result
199+
200+ def test_checksum_files_created(self):
201+ tmpdir = unicode(self.makeTemporaryDirectory())
202+ rootdir = unicode(self.makeTemporaryDirectory())
203+
204+ for dh_file in self.all_hash_files:
205+ checksum_file = os.path.join(rootdir, dh_file)
206+ self.assertFalse(os.path.exists(checksum_file))
207+
208+ with DirectoryHash(rootdir, tmpdir, None) as dh:
209+ pass
210+
211+ for dh_file in self.all_hash_files:
212+ checksum_file = os.path.join(rootdir, dh_file)
213+ if dh_file in self.expected_hash_files:
214+ self.assertTrue(os.path.exists(checksum_file))
215+ else:
216+ self.assertFalse(os.path.exists(checksum_file))
217+
218+ def test_basic_file_add(self):
219+ tmpdir = unicode(self.makeTemporaryDirectory())
220+ rootdir = unicode(self.makeTemporaryDirectory())
221+ test1_file = os.path.join(rootdir, "test1")
222+ test1_hash = self.createTestFile(test1_file, "test1")
223+
224+ test2_file = os.path.join(rootdir, "test2")
225+ test2_hash = self.createTestFile(test2_file, "test2")
226+
227+ os.mkdir(os.path.join(rootdir, "subdir1"))
228+
229+ test3_file = os.path.join(rootdir, "subdir1", "test3")
230+ test3_hash = self.createTestFile(test3_file, "test3")
231+
232+ with DirectoryHash(rootdir, tmpdir, None) as dh:
233+ dh.add(test1_file)
234+ dh.add(test2_file)
235+ dh.add(test3_file)
236+
237+ expected = {
238+ 'SHA256SUMS': MatchesSetwise(
239+ Equals([test1_hash, "*test1"]),
240+ Equals([test2_hash, "*test2"]),
241+ Equals([test3_hash, "*subdir1/test3"]),
242+ ),
243+ }
244+ self.assertThat(self.fetchSums(rootdir), MatchesDict(expected))
245+
246+ def test_basic_directory_add(self):
247+ tmpdir = unicode(self.makeTemporaryDirectory())
248+ rootdir = unicode(self.makeTemporaryDirectory())
249+ test1_file = os.path.join(rootdir, "test1")
250+ test1_hash = self.createTestFile(test1_file, "test1 dir")
251+
252+ test2_file = os.path.join(rootdir, "test2")
253+ test2_hash = self.createTestFile(test2_file, "test2 dir")
254+
255+ os.mkdir(os.path.join(rootdir, "subdir1"))
256+
257+ test3_file = os.path.join(rootdir, "subdir1", "test3")
258+ test3_hash = self.createTestFile(test3_file, "test3 dir")
259+
260+ with DirectoryHash(rootdir, tmpdir, None) as dh:
261+ dh.add_dir(rootdir)
262+
263+ expected = {
264+ 'SHA256SUMS': MatchesSetwise(
265+ Equals([test1_hash, "*test1"]),
266+ Equals([test2_hash, "*test2"]),
267+ Equals([test3_hash, "*subdir1/test3"]),
268+ ),
269+ }
270+ self.assertThat(self.fetchSums(rootdir), MatchesDict(expected))
271
272=== modified file 'lib/lp/archivepublisher/tests/test_signing.py'
273--- lib/lp/archivepublisher/tests/test_signing.py 2016-05-31 12:23:15 +0000
274+++ lib/lp/archivepublisher/tests/test_signing.py 2016-06-06 17:15:44 +0000
275@@ -256,6 +256,23 @@
276 self.assertContentEqual(['first', 'second'],
277 upload.signing_options.keys())
278
279+ def test_options_none(self):
280+ # Specifying no options should leave us with an open tree.
281+ self.setUpUefiKeys()
282+ self.setUpKmodKeys()
283+ self.openArchive("test", "1.0", "amd64")
284+ self.archive.add_file("1.0/empty.efi", "")
285+ self.archive.add_file("1.0/empty.ko", "")
286+ self.process_emulate()
287+ self.assertTrue(os.path.exists(os.path.join(
288+ self.getSignedPath("test", "amd64"), "1.0", "empty.efi")))
289+ self.assertTrue(os.path.exists(os.path.join(
290+ self.getSignedPath("test", "amd64"), "1.0", "empty.efi.signed")))
291+ self.assertTrue(os.path.exists(os.path.join(
292+ self.getSignedPath("test", "amd64"), "1.0", "empty.ko")))
293+ self.assertTrue(os.path.exists(os.path.join(
294+ self.getSignedPath("test", "amd64"), "1.0", "empty.ko.sig")))
295+
296 def test_options_tarball(self):
297 # Specifying the "tarball" option should create an tarball in
298 # the tmpdir.
299@@ -602,6 +619,19 @@
300 self.assertEqual(stat.S_IMODE(os.stat(self.kmod_pem).st_mode), 0o600)
301 self.assertEqual(stat.S_IMODE(os.stat(self.kmod_x509).st_mode), 0o644)
302
303+ def test_checksumming_tree(self):
304+ # Specifying no options should leave us with an open tree,
305+ # confirm it is checksummed.
306+ self.setUpUefiKeys()
307+ self.setUpKmodKeys()
308+ self.openArchive("test", "1.0", "amd64")
309+ self.archive.add_file("1.0/empty.efi", "")
310+ self.archive.add_file("1.0/empty.ko", "")
311+ self.process_emulate()
312+ sha256file = os.path.join(self.getSignedPath("test", "amd64"),
313+ "1.0", "SHA256SUMS")
314+ self.assertTrue(os.path.exists(sha256file))
315+
316
317 class TestUefi(TestSigningHelpers):
318
319
320=== modified file 'lib/lp/archivepublisher/utils.py'
321--- lib/lp/archivepublisher/utils.py 2016-02-05 20:28:29 +0000
322+++ lib/lp/archivepublisher/utils.py 2016-06-06 17:15:44 +0000
323@@ -113,7 +113,7 @@
324 (plain, gzip, bzip2, and xz) transparently and atomically.
325 """
326
327- def __init__(self, path, temp_root, compressors):
328+ def __init__(self, path, temp_root, compressors=None):
329 """Store repositories destinations and filename.
330
331 The given 'temp_root' needs to exist; on the other hand, the
332@@ -123,6 +123,9 @@
333 Additionally creates the needed temporary files in the given
334 'temp_root'.
335 """
336+ if compressors is None:
337+ compressors = [IndexCompressionType.UNCOMPRESSED]
338+
339 self.root, filename = os.path.split(path)
340 assert os.path.exists(temp_root), 'Temporary root does not exist.'
341
342@@ -135,6 +138,12 @@
343 self.old_index_files.append(
344 cls(temp_root, filename, auto_open=False))
345
346+ def __enter__(self):
347+ return self
348+
349+ def __exit__(self, type, value, traceback):
350+ self.close()
351+
352 def write(self, content):
353 """Write contents to all target medias."""
354 for index_file in self.index_files: