Launchpad itself

Merge lp:~wgrant/launchpad/sha256-indices into lp:launchpad

  1. sha256-indices
  2. Merge into devel
Proposed by William Grant
Status: Merged
Approved by: William Grant
Approved revision: no longer in the source branch.
Merged at revision: 16676
Proposed branch: lp:~wgrant/launchpad/sha256-indices
Merge into: lp:launchpad
Diff against target: 311 lines (+87/-23)
5 files modified
lib/lp/archivepublisher/tests/test_publisher.py (+12/-0)
lib/lp/archiveuploader/dscfile.py (+4/-2)
lib/lp/soyuz/doc/publishing.txt (+6/-0)
lib/lp/soyuz/model/publishing.py (+26/-9)
lib/lp/soyuz/tests/test_publish_archive_indexes.py (+39/-12)
To merge this branch: bzr merge lp:~wgrant/launchpad/sha256-indices
Reviewer Review Type Date Requested Status
Steve Kowalik (community) code Approve
Review via email: mp+169995@code.launchpad.net

Commit message

Add SHA-256 hashes to Sources and Packages generated by NMAF.

Description of the change

Add SHA-256 hashes to Sources and Packages generated by NMAF. This can't be deployed until the columns are fully populated on production.

To post a comment you must log in.
Revision history for this message
Steve Kowalik (stevenk) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/archivepublisher/tests/test_publisher.py'
2--- lib/lp/archivepublisher/tests/test_publisher.py 2013-05-15 10:40:18 +0000
3+++ lib/lp/archivepublisher/tests/test_publisher.py 2013-06-18 07:23:34 +0000
4@@ -706,6 +706,12 @@
5 'Directory: pool/main/f/foo',
6 'Files:',
7 ' 3e25960a79dbc69b674cd4ec67a72c62 11 foo_1.dsc',
8+ 'Checksums-Sha1:',
9+ ' 7b502c3a1f48c8609ae212cdfb639dee39673f5e 11 foo_1.dsc',
10+ 'Checksums-Sha256:',
11+ ' 64ec88ca00b268e5ba1a35678a1b5316d212f4f366b2477232534a8aeca37f'
12+ '3c 11 foo_1.dsc',
13+
14 ''],
15 index_contents)
16
17@@ -732,6 +738,8 @@
18 'Size: 18',
19 'MD5sum: 008409e7feb1c24a6ccab9f6a62d24c5',
20 'SHA1: 30b7b4e583fa380772c5a40e428434628faef8cf',
21+ 'SHA256: 006ca0f356f54b1916c24c282e6fd19961f4356441401f4b0966f2a'
22+ '00bb3e945',
23 'Description: Foo app is great',
24 ' My leading spaces are normalised to a single space but not '
25 'trailing. ',
26@@ -765,6 +773,8 @@
27 'Size: 18',
28 'MD5sum: 008409e7feb1c24a6ccab9f6a62d24c5',
29 'SHA1: 30b7b4e583fa380772c5a40e428434628faef8cf',
30+ 'SHA256: 006ca0f356f54b1916c24c282e6fd19961f4356441401f4b0966f2a'
31+ '00bb3e945',
32 'Description: Foo app is great',
33 ' nice udeb',
34 ''],
35@@ -794,6 +804,8 @@
36 'Size: 18',
37 'MD5sum: 008409e7feb1c24a6ccab9f6a62d24c5',
38 'SHA1: 30b7b4e583fa380772c5a40e428434628faef8cf',
39+ 'SHA256: 006ca0f356f54b1916c24c282e6fd19961f4356441401f4b0966f2a'
40+ '00bb3e945',
41 'Description: Foo app is great',
42 ' My leading spaces are normalised to a single space but not '
43 'trailing. ',
44
45=== modified file 'lib/lp/archiveuploader/dscfile.py'
46--- lib/lp/archiveuploader/dscfile.py 2012-12-26 01:04:05 +0000
47+++ lib/lp/archiveuploader/dscfile.py 2013-06-18 07:23:34 +0000
48@@ -250,9 +250,11 @@
49 "Build-Depends-Indep",
50 "Build-Conflicts",
51 "Build-Conflicts-Indep",
52+ "Checksums-Sha1",
53+ "Checksums-Sha256",
54 "Format",
55+ "Homepage",
56 "Standards-Version",
57- "homepage",
58 ]))
59
60 # Note that files is actually only set inside verify().
61@@ -674,7 +676,7 @@
62 architecturehintlist=encoded.get('Architecture', ''),
63 creator=self.changes.changed_by['person'],
64 urgency=self.changes.converted_urgency,
65- homepage=encoded.get('homepage'),
66+ homepage=encoded.get('Homepage'),
67 dsc=encoded_raw_content,
68 dscsigningkey=self.signingkey,
69 dsc_maintainer_rfc822=encoded['Maintainer'],
70
71=== modified file 'lib/lp/soyuz/doc/publishing.txt'
72--- lib/lp/soyuz/doc/publishing.txt 2013-05-24 01:20:03 +0000
73+++ lib/lp/soyuz/doc/publishing.txt 2013-06-18 07:23:34 +0000
74@@ -349,6 +349,11 @@
75 Directory: pool/main/a/alsa-utils
76 Files:
77 01234567890123456789012345678925 3 alsa-utils_1.0.8-1ubuntu1.dsc
78+ Checksums-Sha1:
79+ a10856bfea3f0bdb09550dd41f3c5bc275da8a33 3 alsa-utils_1.0.8-1ubuntu1.dsc
80+ Checksums-Sha256:
81+ 0123456789012345678901234567890123456789012345678901234567890123 3 alsa-utils_1.0.8-1ubuntu1.dsc
82+
83
84 Empty fields like are suppressed, like 'Build-Depends' or
85 'Build-Depends-Indep' to avoid extra charge on download.
86@@ -940,6 +945,7 @@
87 Size: 3
88 MD5sum: 01234567890123456789012345678926
89 SHA1: 5a04c7b5ea3f0fdbc95d0dd47f3c5bc275da8a33
90+ SHA256: 0123456789012345678901234567890123456789012345678901234567890123
91 Description: Mozilla Firefox Web Browser
92 Mozilla Firefox Web Browser is .....
93
94
95=== modified file 'lib/lp/soyuz/model/publishing.py'
96--- lib/lp/soyuz/model/publishing.py 2013-05-31 04:27:26 +0000
97+++ lib/lp/soyuz/model/publishing.py 2013-06-18 07:23:34 +0000
98@@ -366,6 +366,7 @@
99 """Store and format ordered Index Stanza fields."""
100
101 def __init__(self):
102+ self._names_lower = set()
103 self.fields = []
104
105 def append(self, name, value):
106@@ -373,12 +374,16 @@
107
108 Then we can use the FIFO-like behaviour in makeOutput().
109 """
110+ if name.lower() in self._names_lower:
111+ return
112+ self._names_lower.add(name.lower())
113 self.fields.append((name, value))
114
115 def extend(self, entries):
116 """Extend the internal list with the key-value pairs in entries.
117 """
118- self.fields.extend(entries)
119+ for name, value in entries:
120+ self.append(name, value)
121
122 def makeOutput(self):
123 """Return a line-by-line aggregation of appended fields.
124@@ -391,8 +396,8 @@
125 if not value:
126 continue
127
128- # do not add separation space for the special field 'Files'
129- if name != 'Files':
130+ # do not add separation space for the special file list fields.
131+ if name not in ('Files', 'Checksums-Sha1', 'Checksums-Sha256'):
132 value = ' %s' % value
133
134 # XXX Michael Nelson 20090930 bug=436182. We have an issue
135@@ -721,16 +726,23 @@
136 name = release.sourcepackagename.name
137 return "%s %s in %s" % (name, release.version, self.distroseries.name)
138
139+ def _formatFileList(self, l):
140+ return ''.join('\n %s %s %s' % ((h,) + f) for (h, f) in l)
141+
142 def buildIndexStanzaFields(self):
143 """See `IPublishing`."""
144 # Special fields preparation.
145 spr = self.sourcepackagerelease
146 pool_path = makePoolPath(spr.name, self.component.name)
147- files_subsection = ''.join(
148- ['\n %s %s %s' % (spf.libraryfile.content.md5,
149- spf.libraryfile.content.filesize,
150- spf.libraryfile.filename)
151- for spf in spr.files])
152+ files_list = []
153+ sha1_list = []
154+ sha256_list = []
155+ for spf in spr.files:
156+ common = (
157+ spf.libraryfile.content.filesize, spf.libraryfile.filename)
158+ files_list.append((spf.libraryfile.content.md5, common))
159+ sha1_list.append((spf.libraryfile.content.sha1, common))
160+ sha256_list.append((spf.libraryfile.content.sha256, common))
161 # Filling stanza options.
162 fields = IndexStanzaFields()
163 fields.append('Package', spr.name)
164@@ -746,7 +758,10 @@
165 fields.append('Standards-Version', spr.dsc_standards_version)
166 fields.append('Format', spr.dsc_format)
167 fields.append('Directory', pool_path)
168- fields.append('Files', files_subsection)
169+ fields.append('Files', self._formatFileList(files_list))
170+ fields.append('Checksums-Sha1', self._formatFileList(sha1_list))
171+ fields.append('Checksums-Sha256', self._formatFileList(sha256_list))
172+ fields.append('Homepage', spr.homepage)
173 if spr.user_defined_fields:
174 fields.extend(spr.user_defined_fields)
175
176@@ -1013,6 +1028,7 @@
177 bin_size = bin_file.libraryfile.content.filesize
178 bin_md5 = bin_file.libraryfile.content.md5
179 bin_sha1 = bin_file.libraryfile.content.sha1
180+ bin_sha256 = bin_file.libraryfile.content.sha256
181 bin_filepath = os.path.join(
182 makePoolPath(spr.name, self.component.name), bin_filename)
183 # description field in index is an association of summary and
184@@ -1066,6 +1082,7 @@
185 fields.append('Size', bin_size)
186 fields.append('MD5sum', bin_md5)
187 fields.append('SHA1', bin_sha1)
188+ fields.append('SHA256', bin_sha256)
189 fields.append(
190 'Phased-Update-Percentage', self.phased_update_percentage)
191 fields.append('Description', bin_description)
192
193=== modified file 'lib/lp/soyuz/tests/test_publish_archive_indexes.py'
194--- lib/lp/soyuz/tests/test_publish_archive_indexes.py 2013-01-21 16:28:14 +0000
195+++ lib/lp/soyuz/tests/test_publish_archive_indexes.py 2013-06-18 07:23:34 +0000
196@@ -19,6 +19,15 @@
197
198 class TestNativeArchiveIndexes(TestNativePublishingBase):
199
200+ deb_md5 = '008409e7feb1c24a6ccab9f6a62d24c5'
201+ deb_sha1 = '30b7b4e583fa380772c5a40e428434628faef8cf'
202+ deb_sha256 = (
203+ '006ca0f356f54b1916c24c282e6fd19961f4356441401f4b0966f2a00bb3e945')
204+ dsc_md5 = '5913c3ad52c14a62e6ae7eef51f9ef42'
205+ dsc_sha1 = 'e35e29b2ea94bbaa831882e11d1f456690f04e69'
206+ dsc_sha256 = (
207+ 'ac512102db9724bee18f26945efeeb82fdab89819e64e120fbfda755ca50c2c6')
208+
209 def setUp(self):
210 """Setup global attributes."""
211 TestNativePublishingBase.setUp(self)
212@@ -55,17 +64,28 @@
213 u'Format: 1.0',
214 u'Directory: pool/main/f/foo',
215 u'Files:',
216- u' 5913c3ad52c14a62e6ae7eef51f9ef42 28 foo_666.dsc'],
217+ u' %s 28 foo_666.dsc' % self.dsc_md5,
218+ u'Checksums-Sha1:',
219+ u' %s 28 foo_666.dsc' % self.dsc_sha1,
220+ u'Checksums-Sha256:',
221+ u' %s 28 foo_666.dsc' % self.dsc_sha256,
222+ ],
223 pub_source.getIndexStanza().splitlines())
224
225 def testSourceStanzaCustomFields(self):
226 """Check just-created source publication Index stanza
227 with custom fields (Python-Version).
228+
229+ A field is excluded if its key case-insensitively matches one that's
230+ already there. This mostly affects sources that were uploaded before
231+ Homepage, Checksums-Sha1 or Checksums-Sha256 were excluded.
232 """
233 pub_source = self.getPubSource(
234 builddepends='fooish', builddependsindep='pyfoo',
235 build_conflicts='bar', build_conflicts_indep='pybar',
236- user_defined_fields=[("Python-Version", "< 1.5")])
237+ user_defined_fields=[
238+ ("Python-Version", "< 1.5"),
239+ ("CHECKSUMS-SHA1", "BLAH")])
240
241 self.assertEqual(
242 [u'Package: foo',
243@@ -82,7 +102,11 @@
244 u'Format: 1.0',
245 u'Directory: pool/main/f/foo',
246 u'Files:',
247- u' 5913c3ad52c14a62e6ae7eef51f9ef42 28 foo_666.dsc',
248+ u' %s 28 foo_666.dsc' % self.dsc_md5,
249+ u'Checksums-Sha1:',
250+ u' %s 28 foo_666.dsc' % self.dsc_sha1,
251+ u'Checksums-Sha256:',
252+ u' %s 28 foo_666.dsc' % self.dsc_sha256,
253 u'Python-Version: < 1.5'],
254 pub_source.getIndexStanza().splitlines())
255
256@@ -118,8 +142,9 @@
257 u'Breaks: old-foo',
258 u'Filename: pool/main/f/foo/foo-bin_666_all.deb',
259 u'Size: 18',
260- u'MD5sum: 008409e7feb1c24a6ccab9f6a62d24c5',
261- u'SHA1: 30b7b4e583fa380772c5a40e428434628faef8cf',
262+ u'MD5sum: ' + self.deb_md5,
263+ u'SHA1: ' + self.deb_sha1,
264+ u'SHA256: ' + self.deb_sha256,
265 u'Phased-Update-Percentage: 50',
266 u'Description: Foo app is great',
267 u' Well ...',
268@@ -157,8 +182,9 @@
269 u'Breaks: old-foo',
270 u'Filename: pool/main/f/foo/foo-bin_666_all.deb',
271 u'Size: 18',
272- u'MD5sum: 008409e7feb1c24a6ccab9f6a62d24c5',
273- u'SHA1: 30b7b4e583fa380772c5a40e428434628faef8cf',
274+ u'MD5sum: ' + self.deb_md5,
275+ u'SHA1: ' + self.deb_sha1,
276+ u'SHA256: ' + self.deb_sha256,
277 u'Description: Foo app is great',
278 u' Well ...',
279 u' it does nothing, though',
280@@ -203,8 +229,9 @@
281 u'Version: 666',
282 u'Filename: pool/main/f/foo/foo-bin_666_all.deb',
283 u'Size: 18',
284- u'MD5sum: 008409e7feb1c24a6ccab9f6a62d24c5',
285- u'SHA1: 30b7b4e583fa380772c5a40e428434628faef8cf',
286+ u'MD5sum: ' + self.deb_md5,
287+ u'SHA1: ' + self.deb_sha1,
288+ u'SHA256: ' + self.deb_sha256,
289 u'Description: Foo app is great',
290 u' Normal',
291 u' Normal',
292@@ -239,8 +266,9 @@
293 u'Version: 666',
294 u'Filename: pool/main/f/foo/foo-bin_666_all.deb',
295 u'Size: 18',
296- u'MD5sum: 008409e7feb1c24a6ccab9f6a62d24c5',
297- u'SHA1: 30b7b4e583fa380772c5a40e428434628faef8cf',
298+ u'MD5sum: ' + self.deb_md5,
299+ u'SHA1: ' + self.deb_sha1,
300+ u'SHA256: ' + self.deb_sha256,
301 u'Description: Foo app is great',
302 u' Using non-ascii as: \xe7\xe3\xe9\xf3',
303 ],
304@@ -424,7 +452,6 @@
305 fields.append('one', 'um')
306 fields.append('three', 'tres')
307 fields.append('two', 'dois')
308- fields.append(None, None)
309
310 self.assertEqual(
311 ['one: um', 'three: tres', 'two: dois',