Merge lp:~jderose/filestore/dbase32 into lp:filestore
- dbase32
- Merge into trunk
Status: | Merged | ||||
---|---|---|---|---|---|
Merged at revision: | 304 | ||||
Proposed branch: | lp:~jderose/filestore/dbase32 | ||||
Merge into: | lp:filestore | ||||
Diff against target: |
3659 lines (+1377/-1583) 12 files modified
debian/control (+2/-2) filestore/__init__.py (+41/-58) filestore/data/MD5SUMS.json (+23/-0) filestore/data/V0.json (+24/-0) filestore/data/V1.json (+24/-0) filestore/data/test-vectors.json (+0/-50) filestore/misc.py (+52/-32) filestore/protocols.py (+68/-164) filestore/tests/__init__.py (+67/-180) filestore/tests/test_misc.py (+132/-54) filestore/tests/test_protocols.py (+943/-1042) setup.py (+1/-1) |
||||
To merge this branch: | bzr merge lp:~jderose/filestore/dbase32 | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
David Jordan | Approve | ||
Review via email: mp+149313@code.launchpad.net |
Commit message
Description of the change
For details, see this bug:
https:/
Changes include:
* Use compatibility functions from `dbase32.rfc3548` so that it will be very easy to switch to Dbase32, in particular use `isb32()` instead of set operations with B32ALPHABET; we can just replace with `isdb32()` when needed
* Removed protocol selection-
* `Protocol.
* `Protocol.
* Likewise, `Protocol.
* `Protocol` is now a base class without any hashing implementation itself; real implementations are now in the `SkeinProtocol` and `OldSkeinProtocol` subclasses
* Changed V1 to use a 240-bit digest and DBase32 encoding, updated test vectors accordingly
* Split old test-vectors.json file (which contained both V0 and V1) into separate V0.json and V1.json files
* Added MD5SUMS.json with md5sums of various values used by the test vectors; these are helpful for debugging an implementation, as mention in the specification
* Huge refactor of the unit tests for `filestore.
* misc.build_
Jason Gerard DeRose (jderose) wrote : | # |
Preview Diff
1 | === modified file 'debian/control' |
2 | --- debian/control 2013-02-18 11:42:49 +0000 |
3 | +++ debian/control 2013-02-19 15:39:24 +0000 |
4 | @@ -6,7 +6,7 @@ |
5 | python3-all-dev (>= 3.3), |
6 | python3-sphinx, |
7 | python3-skein (>= 0.7.1), |
8 | - python3-dbase32 (>= 0.3), |
9 | + python3-dbase32 (>= 0.4), |
10 | Standards-Version: 3.9.3 |
11 | X-Python3-Version: >= 3.2 |
12 | Homepage: https://launchpad.net/filestore |
13 | @@ -15,7 +15,7 @@ |
14 | Architecture: any |
15 | Depends: ${python3:Depends}, ${shlibs:Depends}, ${misc:Depends}, |
16 | python3-skein (>= 0.7.1), |
17 | - python3-dbase32 (>= 0.3), |
18 | + python3-dbase32 (>= 0.4), |
19 | Description: Dmedia hashing protocol and file layout |
20 | This is the heart of Dmedia. It has been split out of Dmedia to make it easier |
21 | to review, and in hopes that other apps might use the hashing protocol and |
22 | |
23 | === modified file 'filestore/__init__.py' |
24 | --- filestore/__init__.py 2013-02-18 12:41:32 +0000 |
25 | +++ filestore/__init__.py 2013-02-19 15:39:24 +0000 |
26 | @@ -76,15 +76,17 @@ |
27 | from os import path |
28 | import io |
29 | import stat |
30 | -from base64 import b32encode, b64encode |
31 | +from base64 import b64encode |
32 | from threading import Thread |
33 | from queue import Queue |
34 | from collections import namedtuple |
35 | import logging |
36 | |
37 | from dbase32 import random_id |
38 | +from dbase32.rfc3548 import b32enc, isb32 |
39 | |
40 | -from .protocols import VERSION0, VERSION1 |
41 | +from .protocols import TYPE_ERROR |
42 | +from .protocols import VERSION0 as PROTOCOL |
43 | |
44 | try: |
45 | from _filestore import fallocate, fastread |
46 | @@ -100,13 +102,10 @@ |
47 | |
48 | log = logging.getLogger('filestore') |
49 | |
50 | -LEAF_SIZE = 8 * 1024 * 1024 # 8 MiB leaf size |
51 | +LEAF_SIZE = 8388608 # 8 MiB leaf size |
52 | DIGEST_BITS = 240 |
53 | -DIGEST_BYTES = DIGEST_BITS // 8 |
54 | -DIGEST_B32LEN = DIGEST_BITS // 5 |
55 | - |
56 | -ALLOWED_B32LEN = (48, 56) |
57 | -PROTOCOL = VERSION0 |
58 | +DIGEST_BYTES = 30 |
59 | +DIGEST_B32LEN = 48 |
60 | |
61 | # Handy constants for file layout: |
62 | DOTNAME = '.dmedia' |
63 | @@ -130,9 +129,6 @@ |
64 | |
65 | IO_READABLE = (io.BufferedReader, io.BufferedRandom) |
66 | |
67 | -# Provide very clear TypeError messages: |
68 | -TYPE_ERROR = '{}: need a {!r}; got a {!r}: {!r}' |
69 | - |
70 | # For the README.txt in .dmedia directories: |
71 | README = """This directory is automatically managed by Dmedia. |
72 | |
73 | @@ -155,13 +151,13 @@ |
74 | |
75 | For example: |
76 | |
77 | - >>> b32encode(hash_leaf(2, b'XYZ')) |
78 | - b'D7GIW5I5NB6SLJC5ALAX4WU7S7CNYUB3ULMECPY67FFQG4F7' |
79 | + >>> b32enc(hash_leaf(2, b'XYZ')) |
80 | + 'D7GIW5I5NB6SLJC5ALAX4WU7S7CNYUB3ULMECPY67FFQG4F7' |
81 | |
82 | :param leaf_index: an ``int`` >= 0 |
83 | :param leaf_data: optional ``bytes`` instance with contents of this leaf |
84 | """ |
85 | - return VERSION0.hash_leaf(leaf_index, leaf_data) |
86 | + return PROTOCOL.hash_leaf(leaf_index, leaf_data) |
87 | |
88 | |
89 | def hash_root(file_size, leaf_hashes): |
90 | @@ -177,15 +173,7 @@ |
91 | :param leaf_hashes: a ``bytes`` instance that is the concatenated leaf |
92 | hashes produced by `hash_leaf()` |
93 | """ |
94 | - return VERSION0.hash_root(file_size, leaf_hashes) |
95 | - |
96 | - |
97 | -def get_protocol(_id): |
98 | - check_id(_id) |
99 | - if len(_id) == 48: |
100 | - return VERSION0 |
101 | - assert len(_id) == 56 |
102 | - return VERSION1 |
103 | + return b32enc(PROTOCOL.hash_root(file_size, leaf_hashes)) |
104 | |
105 | |
106 | class Hasher: |
107 | @@ -193,10 +181,9 @@ |
108 | A helper to keep track of state as you hash leaf after leaf. |
109 | """ |
110 | |
111 | - __slots__ = ('protocol', 'file_size', 'leaf_index', 'array', 'closed') |
112 | + __slots__ = ('file_size', 'leaf_index', 'array', 'closed') |
113 | |
114 | - def __init__(self, protocol=PROTOCOL): |
115 | - self.protocol = protocol |
116 | + def __init__(self): |
117 | self.file_size = 0 |
118 | self.leaf_index = 0 |
119 | self.array = bytearray() |
120 | @@ -211,9 +198,9 @@ |
121 | raise Exception('Expected leaf.index {}, got {}'.format( |
122 | self.leaf_index, leaf.index) |
123 | ) |
124 | - if len(leaf.data) < self.protocol.leaf_size: |
125 | + if len(leaf.data) < LEAF_SIZE: |
126 | self.closed = True |
127 | - leaf_hash = self.protocol.hash_leaf(leaf.index, leaf.data) |
128 | + leaf_hash = PROTOCOL.hash_leaf(leaf.index, leaf.data) |
129 | self.array.extend(leaf_hash) |
130 | self.file_size += len(leaf.data) |
131 | self.leaf_index += 1 |
132 | @@ -223,7 +210,7 @@ |
133 | self.closed = True |
134 | leaf_hashes = bytes(self.array) |
135 | return ContentHash( |
136 | - self.protocol.hash_root(self.file_size, leaf_hashes), |
137 | + b32enc(PROTOCOL.hash_root(self.file_size, leaf_hashes)), |
138 | self.file_size, |
139 | leaf_hashes |
140 | ) |
141 | @@ -345,12 +332,12 @@ |
142 | """ |
143 | if not isinstance(_id, str): |
144 | raise TypeError(TYPE_ERROR.format('_id', str, type(_id), _id)) |
145 | - if not (len(_id) in ALLOWED_B32LEN and set(_id).issubset(B32ALPHABET)): |
146 | + if not (len(_id) == DIGEST_B32LEN and isb32(_id)): |
147 | raise IDError(_id) |
148 | return _id |
149 | |
150 | |
151 | -def check_leaf_hashes(leaf_hashes, protocol=PROTOCOL): |
152 | +def check_leaf_hashes(leaf_hashes): |
153 | """ |
154 | Verify that *leaf_hashes* is a ``bytes`` instance of correct length. |
155 | |
156 | @@ -372,7 +359,7 @@ |
157 | 'leaf_hashes', bytes, type(leaf_hashes), leaf_hashes |
158 | ) |
159 | ) |
160 | - if len(leaf_hashes) == 0 or len(leaf_hashes) % protocol.digest_bytes != 0: |
161 | + if len(leaf_hashes) == 0 or len(leaf_hashes) % DIGEST_BYTES != 0: |
162 | raise LeafHashesError(leaf_hashes) |
163 | return leaf_hashes |
164 | |
165 | @@ -400,12 +387,11 @@ |
166 | 22 |
167 | |
168 | """ |
169 | - protocol = get_protocol(_id) |
170 | - computed = protocol.hash_root(file_size, leaf_hashes) |
171 | + computed = hash_root(file_size, leaf_hashes) |
172 | if _id != computed: |
173 | raise RootHashError(_id, file_size, leaf_hashes, computed) |
174 | if unpack: |
175 | - leaf_hashes = tuple(iter_leaf_hashes(leaf_hashes, protocol)) |
176 | + leaf_hashes = tuple(iter_leaf_hashes(leaf_hashes)) |
177 | return ContentHash(_id, file_size, leaf_hashes) |
178 | |
179 | |
180 | @@ -439,7 +425,7 @@ |
181 | ################################################# |
182 | # Utility functions for working with leaf_hashes: |
183 | |
184 | -def enumerate_leaf_hashes(leaf_hashes, protocol=PROTOCOL): |
185 | +def enumerate_leaf_hashes(leaf_hashes): |
186 | """ |
187 | Enumerate *leaf_hashes*. |
188 | |
189 | @@ -451,13 +437,12 @@ |
190 | [(0, b'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'), (1, b'BBBBBBBBBBBBBBBBBBBBBBBBBBBBBB')] |
191 | |
192 | """ |
193 | - check_leaf_hashes(leaf_hashes, protocol) |
194 | - digest_bytes = protocol.digest_bytes |
195 | - for i in range(len(leaf_hashes) // digest_bytes): |
196 | - yield (i, leaf_hashes[i*digest_bytes : (i+1)*digest_bytes]) |
197 | - |
198 | - |
199 | -def iter_leaf_hashes(leaf_hashes, protocol=PROTOCOL): |
200 | + check_leaf_hashes(leaf_hashes) |
201 | + for i in range(len(leaf_hashes) // DIGEST_BYTES): |
202 | + yield (i, leaf_hashes[i*DIGEST_BYTES : (i+1)*DIGEST_BYTES]) |
203 | + |
204 | + |
205 | +def iter_leaf_hashes(leaf_hashes): |
206 | """ |
207 | Iterate through *leaf_hashes*. |
208 | |
209 | @@ -469,13 +454,12 @@ |
210 | [b'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA', b'BBBBBBBBBBBBBBBBBBBBBBBBBBBBBB'] |
211 | |
212 | """ |
213 | - check_leaf_hashes(leaf_hashes, protocol) |
214 | - digest_bytes = protocol.digest_bytes |
215 | - for i in range(len(leaf_hashes) // digest_bytes): |
216 | - yield leaf_hashes[i*digest_bytes : (i+1)*digest_bytes] |
217 | - |
218 | - |
219 | -def get_leaf_hash(leaf_hashes, i, protocol=PROTOCOL): |
220 | + check_leaf_hashes(leaf_hashes) |
221 | + for i in range(len(leaf_hashes) // DIGEST_BYTES): |
222 | + yield leaf_hashes[i*DIGEST_BYTES : (i+1)*DIGEST_BYTES] |
223 | + |
224 | + |
225 | +def get_leaf_hash(leaf_hashes, i): |
226 | """ |
227 | Return a slice containing the i-th leaf_hash from *leaf_hashes*. |
228 | |
229 | @@ -490,8 +474,9 @@ |
230 | b'' |
231 | |
232 | """ |
233 | - digest_bytes = protocol.digest_bytes |
234 | - return leaf_hashes[i*digest_bytes : (i+1)*digest_bytes] |
235 | + start = i * DIGEST_BYTES |
236 | + stop = start + DIGEST_BYTES |
237 | + return leaf_hashes[start:stop] |
238 | |
239 | |
240 | def missing_leaves(fp, leaf_hashes): |
241 | @@ -764,7 +749,7 @@ |
242 | thread.join() # Make sure batch_reader() terminates |
243 | |
244 | |
245 | -def hash_fp(src_fp, dst_fp=None, protocol=PROTOCOL): |
246 | +def hash_fp(src_fp, dst_fp=None): |
247 | """ |
248 | Compute content hash of open file *src_fp*, optionally writing *dst_fp*. |
249 | |
250 | @@ -777,7 +762,7 @@ |
251 | :param src_fp: A file opened in mode "rb" or "r+b" |
252 | :param dst_fp: An optional file opened in mode "wd" |
253 | """ |
254 | - hasher = Hasher(protocol) |
255 | + hasher = Hasher() |
256 | for leaf in reader_iter(src_fp): |
257 | hasher.hash_leaf(leaf) |
258 | if dst_fp: |
259 | @@ -907,9 +892,7 @@ |
260 | subdir = path.join(self.basedir, 'files', prefix) |
261 | for name in sorted(os.listdir(subdir)): |
262 | _id = prefix + name |
263 | - if len(_id) not in ALLOWED_B32LEN: |
264 | - continue |
265 | - if not set(_id).issubset(B32ALPHABET): |
266 | + if len(_id) != DIGEST_B32LEN or not isb32(_id): |
267 | continue |
268 | fullname = path.join(subdir, name) |
269 | st = os.lstat(fullname) |
270 | @@ -1109,7 +1092,7 @@ |
271 | a ContentHash namedtuple |
272 | """ |
273 | src_fp = self.open(_id) |
274 | - c = hash_fp(src_fp, protocol=get_protocol(_id)) |
275 | + c = hash_fp(src_fp) |
276 | if c.id != _id: |
277 | raise self.move_to_corrupt(src_fp, _id, bad_id=c.id) |
278 | if return_fp: |
279 | |
280 | === added file 'filestore/data/MD5SUMS.json' |
281 | --- filestore/data/MD5SUMS.json 1970-01-01 00:00:00 +0000 |
282 | +++ filestore/data/MD5SUMS.json 2013-02-19 15:39:24 +0000 |
283 | @@ -0,0 +1,23 @@ |
284 | +{ |
285 | + "files": { |
286 | + "A": "7fc56270e7a70fa81a5935b72eacbe29", |
287 | + "B": "d2bad3eedb424dd352d65eafbf6c79ba", |
288 | + "C": "5dd3531303dd6764acb93e5f171a4ab8", |
289 | + "CA": "0722f8dc36d75acb602dcee8d0427ce0", |
290 | + "CB": "77264eb6eed7777a1ee03e2601fc9f64", |
291 | + "CC": "1fbfabdaafff31967f9a95f3a3d3c642" |
292 | + }, |
293 | + "integers": { |
294 | + "0": "cfcd208495d565ef66e7dff9f98764da", |
295 | + "1": "c4ca4238a0b923820dcc509a6f75849b", |
296 | + "16777215": "48ac8929ffdc78a66090d179ff1237d5", |
297 | + "16777216": "e3e330499348f791337e9da6b534a386", |
298 | + "8388607": "32433904a755e2b9eb82cf167723b34f", |
299 | + "8388608": "03926fda4e223707d290ac06bb996653", |
300 | + "8388609": "e9b74719ce6b80c5337148d12725db03" |
301 | + }, |
302 | + "personalization": { |
303 | + "leaf": "8aee35e23a5a74147b230f12123ca82e", |
304 | + "root": "0445d91d37383f5384023d49e71cc629" |
305 | + } |
306 | +} |
307 | |
308 | === added file 'filestore/data/V0.json' |
309 | --- filestore/data/V0.json 1970-01-01 00:00:00 +0000 |
310 | +++ filestore/data/V0.json 2013-02-19 15:39:24 +0000 |
311 | @@ -0,0 +1,24 @@ |
312 | +{ |
313 | + "leaf_hashes": { |
314 | + "A": [ |
315 | + "C23GME2TRCBFM2PA3VIZPUOD5DFYAQLQY2VR46ZKBMH4S4WL", |
316 | + "BWRAN4NJSXYJS6V3YQNNV2EKLEEGNAK5VPSIYZKOBXJ3PPJF" |
317 | + ], |
318 | + "B": [ |
319 | + "NLJ2OHB45MJELINPXJCTBCI7VT3424GRTN36BDP5Z3NE2W5Z", |
320 | + "ALAA376NIW54WR5N6JY2EPR6VFJI7AC2V7PJCRETDIGRQJDN" |
321 | + ], |
322 | + "C": [ |
323 | + "ZQ6RUCU75DHFODZEGNL66QJUM34A3UA53TG2YK7LHRU4SUTF", |
324 | + "MCZHG4NBL5VWKCQIARDXR6IHN34ALIDH7OVYAZ5TNDLXIKA4" |
325 | + ] |
326 | + }, |
327 | + "root_hashes": { |
328 | + "A": "O4NQ4LFNI2UCMKOSWWGPRZXSNEPW2M6OOTBFM3AHTBK35D67", |
329 | + "B": "NW6IMGBAU4NTZDQRHGIZA7VYV5CBZ5TUXADHZE43N4GDZGKK", |
330 | + "C": "TRXCE6PVSJTHQX7H4KV7A7F5DKCMSETLMNANNCJIVT624WLU", |
331 | + "CA": "QSUCTH4LJU3B7QG7IJ3FC7HNE4TXDPZVQEJ36B4A45ATTXDM", |
332 | + "CB": "3FWPZ6RBPXGBSTFBNPK5IOCO7DUMSKR5YTWJ3HM2KI4AEF2O", |
333 | + "CC": "CX2K52LIUTT6SSMFHR7NUI6YIJLO3SGKLSRIL6IDY4BP7HUZ" |
334 | + } |
335 | +} |
336 | |
337 | === added file 'filestore/data/V1.json' |
338 | --- filestore/data/V1.json 1970-01-01 00:00:00 +0000 |
339 | +++ filestore/data/V1.json 2013-02-19 15:39:24 +0000 |
340 | @@ -0,0 +1,24 @@ |
341 | +{ |
342 | + "leaf_hashes": { |
343 | + "A": [ |
344 | + "3M3IHYG9TV3UWTIFI37LN5OVYFEB3PDIAUAQ7OKOQF84MIQT", |
345 | + "VACOMMVCMJP9QBIUIEMMYAPXOWRYDFW3JQUJB4NGNMF48IIW" |
346 | + ], |
347 | + "B": [ |
348 | + "YSRG5KWP5IKWCXTHNN58PY7KSEUJ6JYKDA4RT3F7T8UATQC7", |
349 | + "V6QX3QUDUNE9PVJVF8HBUP4RBKY7AXRJ7MBPWASIKTVWYVUD" |
350 | + ], |
351 | + "C": [ |
352 | + "C8BNKNHHPCSONGE785HTVTIXCHFBCT5SY9GYRDELHQ3BVEHV", |
353 | + "EEOJR9LHS46469EYDUWYWOK7BXW5Y486SNEG9KTSHSSPNIMQ" |
354 | + ] |
355 | + }, |
356 | + "root_hashes": { |
357 | + "A": "6VKN463LEKJMD3OP87UYVU5V65XXOSG3HCT3Q6NDJD58TQXK", |
358 | + "B": "GJDVCPR3JPBFABMGP6PDUBOVQIXX5YLF9PG49SSDJVQEKLOR", |
359 | + "C": "NXIA8BHKVQTD9ACIPT3J7QUYM96FYWJBBEL34YKTA6CKE9IC", |
360 | + "CA": "L66Y5M37PX93NBYYWCSSKW6QQBEYH76OOJFUQJ7FQEA7IB9S", |
361 | + "CB": "SFKVIEOWCIKENPMKDEAGATIA9DGH54LF9BW5LCOW7F9YN9BL", |
362 | + "CC": "VCPN5J8EI4G5PH5A4CUWSPCRTEWGD65PB4WMTGQ7CEVJ6XLO" |
363 | + } |
364 | +} |
365 | |
366 | === removed file 'filestore/data/test-vectors.json' |
367 | --- filestore/data/test-vectors.json 2013-01-15 04:18:38 +0000 |
368 | +++ filestore/data/test-vectors.json 1970-01-01 00:00:00 +0000 |
369 | @@ -1,50 +0,0 @@ |
370 | -{ |
371 | - "48": { |
372 | - "leaf_hashes": { |
373 | - "A": [ |
374 | - "C23GME2TRCBFM2PA3VIZPUOD5DFYAQLQY2VR46ZKBMH4S4WL", |
375 | - "BWRAN4NJSXYJS6V3YQNNV2EKLEEGNAK5VPSIYZKOBXJ3PPJF" |
376 | - ], |
377 | - "B": [ |
378 | - "NLJ2OHB45MJELINPXJCTBCI7VT3424GRTN36BDP5Z3NE2W5Z", |
379 | - "ALAA376NIW54WR5N6JY2EPR6VFJI7AC2V7PJCRETDIGRQJDN" |
380 | - ], |
381 | - "C": [ |
382 | - "ZQ6RUCU75DHFODZEGNL66QJUM34A3UA53TG2YK7LHRU4SUTF", |
383 | - "MCZHG4NBL5VWKCQIARDXR6IHN34ALIDH7OVYAZ5TNDLXIKA4" |
384 | - ] |
385 | - }, |
386 | - "root_hashes": { |
387 | - "A": "O4NQ4LFNI2UCMKOSWWGPRZXSNEPW2M6OOTBFM3AHTBK35D67", |
388 | - "B": "NW6IMGBAU4NTZDQRHGIZA7VYV5CBZ5TUXADHZE43N4GDZGKK", |
389 | - "C": "TRXCE6PVSJTHQX7H4KV7A7F5DKCMSETLMNANNCJIVT624WLU", |
390 | - "CA": "QSUCTH4LJU3B7QG7IJ3FC7HNE4TXDPZVQEJ36B4A45ATTXDM", |
391 | - "CB": "3FWPZ6RBPXGBSTFBNPK5IOCO7DUMSKR5YTWJ3HM2KI4AEF2O", |
392 | - "CC": "CX2K52LIUTT6SSMFHR7NUI6YIJLO3SGKLSRIL6IDY4BP7HUZ" |
393 | - } |
394 | - }, |
395 | - "56": { |
396 | - "leaf_hashes": { |
397 | - "A": [ |
398 | - "XZ5I6KJTUSOIWVCEBOKUELTADZUXNHOAYO77NKKHWCIW3HYGYOPMX5JN", |
399 | - "TEC7754ZNM26MTM6YQFI6TMVTTK4RKQEMPAGT2ROQZUBPUIHSJU2DDR3" |
400 | - ], |
401 | - "B": [ |
402 | - "P67PVKU3SCCQHNIRMR2Z5NICEMIP36WCFJG4AW6YBAE6UI4K6BVLY3EI", |
403 | - "ZIFO5S2OYYPZAUN6XQWTWZGCDATXCGR2JYN7UIAX54WMVWETMIUFG7WM" |
404 | - ], |
405 | - "C": [ |
406 | - "RW2GJFIGPQF5WLR53UAK77TPHNRFKMUBYRB23JFS4G2RFRRNHW6OX4CR", |
407 | - "XBVLPYBUX6QD2DKPJTYVUXT23K3AAUAW5J4RMQ543NQNDAHORQJ7GBDE" |
408 | - ] |
409 | - }, |
410 | - "root_hashes": { |
411 | - "A": "FWV6OJYI36C5NN5DC4GS2IGWZXFCZCGJGHK35YV62LKAG7D2Z4LO4Z2S", |
412 | - "B": "OB756PX5V32JMKJAFKIAJ4AFSFPA2WLNIK32ELNO4FJLJPEEEN6DCAAJ", |
413 | - "C": "QSOHXCDH64IQBOG2NM67XEC6MLZKKPGBTISWWRPMCFCJ2EKMA2SMLY46", |
414 | - "CA": "BQ5UTB33ML2VDTCTLVXK6N4VSMGGKKKDYKG24B6DOAFJB6NRSGMB5BNO", |
415 | - "CB": "ER3LDDZ2LHMTDLOPE5XA5GEEZ6OE45VFIFLY42GEMV4TSZ2B7GJJXAIX", |
416 | - "CC": "R6RN5KL7UBNJWR5SK5YPUKIGAOWWFMYYOVESU5DPT34X5MEK75PXXYIX" |
417 | - } |
418 | - } |
419 | -} |
420 | \ No newline at end of file |
421 | |
422 | === modified file 'filestore/misc.py' |
423 | --- filestore/misc.py 2013-02-14 20:08:06 +0000 |
424 | +++ filestore/misc.py 2013-02-19 15:39:24 +0000 |
425 | @@ -28,23 +28,31 @@ |
426 | import tempfile |
427 | import shutil |
428 | from hashlib import md5 |
429 | -from collections import OrderedDict |
430 | - |
431 | + |
432 | +from dbase32 import db32enc |
433 | + |
434 | +from .protocols import PERS_LEAF, PERS_ROOT, VERSION1 |
435 | from . import FileStore |
436 | -from .protocols import b32dumps |
437 | - |
438 | - |
439 | -TEST_VECTORS = path.join( |
440 | - path.dirname(path.abspath(__file__)), 'data', 'test-vectors.json' |
441 | -) |
442 | -assert path.isfile(TEST_VECTORS) |
443 | - |
444 | - |
445 | -def load_test_vectors(): |
446 | - return json.load(open(TEST_VECTORS, 'r')) |
447 | - |
448 | - |
449 | -def build_test_leaves(leaf_size): |
450 | + |
451 | + |
452 | +DATADIR = path.join(path.dirname(path.abspath(__file__)), 'data') |
453 | + |
454 | + |
455 | +def dumps(obj): |
456 | + return json.dumps(obj, |
457 | + ensure_ascii=False, |
458 | + sort_keys=True, |
459 | + separators=(',',': '), |
460 | + indent=4, |
461 | + ) |
462 | + |
463 | + |
464 | +def load_data(name): |
465 | + filename = path.join(DATADIR, name + '.json') |
466 | + return json.load(open(filename, 'r')) |
467 | + |
468 | + |
469 | +def build_leaves(leaf_size): |
470 | return { |
471 | 'A': b'A', |
472 | 'B': b'B' * (leaf_size - 1), |
473 | @@ -52,36 +60,37 @@ |
474 | } |
475 | |
476 | |
477 | -def build_test_vectors(protocol): |
478 | - leaves = build_test_leaves(protocol.leaf_size) |
479 | - |
480 | - def hash_root(letters): |
481 | +def build_vectors(protocol, encoder=db32enc): |
482 | + leaves = build_leaves(protocol.leaf_size) |
483 | + |
484 | + def hash_leaf(i, L): |
485 | + return encoder(protocol.hash_leaf(i, leaves[L])) |
486 | + |
487 | + def hash_root(name): |
488 | leaf_hashes = b'' |
489 | file_size = 0 |
490 | - for (i, L) in enumerate(letters): |
491 | + for (i, L) in enumerate(name): |
492 | data = leaves[L] |
493 | leaf_hashes += protocol.hash_leaf(i, data) |
494 | file_size += len(data) |
495 | - return protocol.hash_root(file_size, leaf_hashes) |
496 | + return encoder(protocol.hash_root(file_size, leaf_hashes)) |
497 | |
498 | vectors = { |
499 | 'leaf_hashes': {}, |
500 | 'root_hashes': {}, |
501 | } |
502 | |
503 | - for (key, data) in leaves.items(): |
504 | - vectors['leaf_hashes'][key] = [ |
505 | - b32dumps(protocol.hash_leaf(i, data)) for i in range(2) |
506 | - ] |
507 | + for L in leaves: |
508 | + vectors['leaf_hashes'][L] = [hash_leaf(i, L) for i in range(2)] |
509 | |
510 | for L in leaves: |
511 | - for key in (L, 'C' + L): |
512 | - vectors['root_hashes'][key] = hash_root(key) |
513 | + for name in (L, 'C' + L): |
514 | + vectors['root_hashes'][name] = hash_root(name) |
515 | |
516 | return vectors |
517 | |
518 | |
519 | -def get_test_integers(leaf_size): |
520 | +def get_integers(leaf_size): |
521 | return [ |
522 | 0, |
523 | 1, |
524 | @@ -93,20 +102,26 @@ |
525 | ] |
526 | |
527 | |
528 | -def build_test_md5(leaf_size): |
529 | +def build_md5sums(leaf_size): |
530 | integers = dict( |
531 | (str(i), md5(str(i).encode('utf-8')).hexdigest()) |
532 | - for i in get_test_integers(leaf_size) |
533 | + for i in get_integers(leaf_size) |
534 | ) |
535 | - leaves = build_test_leaves(leaf_size) |
536 | + |
537 | + leaves = build_leaves(leaf_size) |
538 | C = leaves['C'] |
539 | files = {} |
540 | for (key, data) in leaves.items(): |
541 | files[key] = md5(data).hexdigest() |
542 | files['C' + key] = md5(C + data).hexdigest() |
543 | + |
544 | return { |
545 | 'integers': integers, |
546 | 'files': files, |
547 | + 'personalization': { |
548 | + 'leaf': md5(PERS_LEAF).hexdigest(), |
549 | + 'root': md5(PERS_ROOT).hexdigest(), |
550 | + } |
551 | } |
552 | |
553 | |
554 | @@ -145,3 +160,8 @@ |
555 | if path.isdir(self.parentdir): |
556 | shutil.rmtree(self.parentdir) |
557 | |
558 | + |
559 | +if __name__ == '__main__': |
560 | + vectors = build_vectors(VERSION1) |
561 | + print(dumps(vectors)) |
562 | + |
563 | |
564 | === modified file 'filestore/protocols.py' |
565 | --- filestore/protocols.py 2013-02-16 01:28:23 +0000 |
566 | +++ filestore/protocols.py 2013-02-19 15:39:24 +0000 |
567 | @@ -94,15 +94,16 @@ |
568 | we still feel this is a very import architecture to embrace now. |
569 | """ |
570 | |
571 | -from base64 import b32encode, b32decode |
572 | from _skein import skein512 |
573 | |
574 | |
575 | +__all__ = ('VERSION0', 'VERSION1') |
576 | + |
577 | # Skein personalization strings used in the Dmedia Hashing Protocol: |
578 | PERS_LEAF = b'20110430 jderose@novacut.com dmedia/leaf' |
579 | PERS_ROOT = b'20110430 jderose@novacut.com dmedia/root' |
580 | |
581 | -# Additional Skein personalization strings used only in the old protocol: |
582 | +# Additional Skein personalization strings used only in the old V0 protocol: |
583 | PERS_LEAF_INDEX = b'20110430 jderose@novacut.com dmedia/leaf-index' |
584 | PERS_FILE_SIZE = b'20110430 jderose@novacut.com dmedia/file-size' |
585 | |
586 | @@ -110,48 +111,45 @@ |
587 | TYPE_ERROR = '{}: need a {!r}; got a {!r}: {!r}' |
588 | |
589 | # Handy: |
590 | -MiB = 1024**2 |
591 | +MiB = 1048576 |
592 | |
593 | # For now so we don't have any issues round-tripping values through JavaScript, |
594 | # we're setting a "soft" file-size limit at 2**53 bytes (the max integer that |
595 | # fully works in JavaScript). But this is quite big, approximately 9.01 PB for |
596 | # a single file: |
597 | -MAX_FILE_SIZE = 2**53 |
598 | +MAX_FILE_SIZE = 9007199254740992 |
599 | |
600 | # A real protocol would unlikely use a leaf-size this small, but a 2 MiB |
601 | # minimum means we can use a uint32_t for the leaf_index in C |
602 | # implementations and still reach the above MAX_FILE_SIZE: |
603 | -MIN_LEAF_SIZE = 2 * MiB |
604 | - |
605 | - |
606 | -def b32dumps(data): |
607 | - """ |
608 | - Base32-encode *data*, return as a string. |
609 | - |
610 | - >>> b32dumps(b'bbbbb') |
611 | - 'MJRGEYTC' |
612 | - |
613 | - """ |
614 | - return b32encode(data).decode('utf-8') |
615 | - |
616 | - |
617 | -def b32loads(string): |
618 | - """ |
619 | - Decode base32-encoded data in *string*. |
620 | - |
621 | - >>> b32loads('MJRGEYTC') |
622 | - b'bbbbb' |
623 | - |
624 | - """ |
625 | - return b32decode(string.encode('utf-8')) |
626 | +MIN_LEAF_SIZE = 2097152 |
627 | + |
628 | + |
629 | +def make_key(integer): |
630 | + """ |
631 | + Return *integer* as it's UTF-8 encoded decimal representation. |
632 | + |
633 | + For example: |
634 | + |
635 | + >>> make_key(1776) |
636 | + b'1776' |
637 | + |
638 | + """ |
639 | + if not isinstance(integer, int): |
640 | + raise TypeError( |
641 | + TYPE_ERROR.format('integer', int, type(integer), integer) |
642 | + ) |
643 | + if integer < 0: |
644 | + raise ValueError('integer: must be >= 0; got {}'.format(integer)) |
645 | + return str(integer).encode('utf-8') |
646 | |
647 | |
648 | class Protocol: |
649 | """ |
650 | - Standard API for current and future versions of the Dmedia Hashing Protocol. |
651 | + API for current and future versions of the Dmedia Hashing Protocol. |
652 | |
653 | - This class provides a standard API that abstracts away the details of a |
654 | - particular hashing protocol and allows multiple protocol versions to be |
655 | + This base class provides a standard API that abstracts away the details of |
656 | + a particular hashing protocol and allows multiple protocol versions to be |
657 | supported simultaneously. |
658 | |
659 | The API consists of two methods: |
660 | @@ -174,7 +172,7 @@ |
661 | (while still using the standard hashing functions), you can simply create |
662 | a `Protocol` instance like this: |
663 | |
664 | - >>> experimental = Protocol(16 * MiB, 360) |
665 | + >>> experimental = SkeinProtocol(16 * MiB, 360) |
666 | |
667 | The `digest_bytes` and `digest_b32len` will be derived from the provided |
668 | *digest_bits*: |
669 | @@ -184,23 +182,15 @@ |
670 | >>> experimental.digest_b32len |
671 | 72 |
672 | |
673 | - In order to implement a protocol that uses different underlying hash |
674 | - functions, you'll need to subclass `Protocol` and override the two |
675 | - low-level hashing methods: |
676 | - |
677 | - `Protocol._hash_leaf(leaf_index, leaf_data, challange)` |
678 | - |
679 | - `Protocol._hash_root(file_size, leaf_hashes)` |
680 | + This base class doesn't provide a working protocol, to do that you must |
681 | + create a subclass and implement these two methods: |
682 | + |
683 | + `Protocol._hash_leaf(key_leaf_index, leaf_data, challange)` |
684 | + |
685 | + `Protocol._hash_root(key_file_size, leaf_hashes)` |
686 | """ |
687 | |
688 | def __init__(self, leaf_size, digest_bits): |
689 | - """ |
690 | - Initialize a new `Protocol` instance. |
691 | - |
692 | - :param leaf_size: an ``int`` which is some multiple of MIN_LEAF_SIZE |
693 | - and >= MIN_LEAF_SIZE |
694 | - :param digest_bits: an ``int`` which is some multiple of 40 and >= 200 |
695 | - """ |
696 | if not isinstance(leaf_size, int): |
697 | raise TypeError( |
698 | TYPE_ERROR.format('leaf_size', int, type(leaf_size), leaf_size) |
699 | @@ -238,56 +228,7 @@ |
700 | self.digest_bytes = digest_bits // 8 |
701 | self.digest_b32len = digest_bits // 5 |
702 | |
703 | - def encode(self, digest): |
704 | - if not isinstance(digest, bytes): |
705 | - raise TypeError( |
706 | - TYPE_ERROR.format('digest', bytes, type(digest), digest) |
707 | - ) |
708 | - if len(digest) != self.digest_bytes: |
709 | - raise ValueError('len(digest) must be {}; got {}'.format( |
710 | - self.digest_bytes, len(digest)) |
711 | - ) |
712 | - return self._encode(digest) |
713 | - |
714 | - def _encode(self, digest): |
715 | - return b32dumps(digest) |
716 | - |
717 | - def decode(self, _id): |
718 | - if not isinstance(_id, str): |
719 | - raise TypeError( |
720 | - TYPE_ERROR.format('_id', str, type(_id), _id) |
721 | - ) |
722 | - if len(_id) != self.digest_b32len: |
723 | - raise ValueError('len(_id) must be {}; got {}'.format( |
724 | - self.digest_b32len, len(_id)) |
725 | - ) |
726 | - return self._decode(_id) |
727 | - |
728 | - def _decode(self, _id): |
729 | - return b32loads(_id) |
730 | - |
731 | def hash_leaf(self, leaf_index, leaf_data, challenge=b''): |
732 | - """ |
733 | - Hash the leaf at (zero) index *leaf_index* containing *leaf_data*. |
734 | - |
735 | - For example: |
736 | - |
737 | - >>> p = Protocol(MIN_LEAF_SIZE, 200) |
738 | - >>> b32encode(p.hash_leaf(0, b'The Leaf Data')) |
739 | - b'EFBXKNNDAMRFIIPTIL2DYZ36YYQDAX54AAOYA7WG' |
740 | - |
741 | - The optional *challenge* keyword argument should be a random nonce used |
742 | - to make a remote cloud storage service prove that they actually have |
743 | - the complete, exact file stored. For example: |
744 | - |
745 | - >>> b32encode(p.hash_leaf(0, b'The Leaf Data', b'Random Nonce')) |
746 | - b'LBAECE4GW3UFXREWAFBMBGV2O4GISUTJ3ALZXGJW' |
747 | - |
748 | - :param leaf_index: an ``int`` >= 0 |
749 | - :param leaf_data: a ``bytes`` instance with leaf content |
750 | - :param challenge: a ``bytes`` instance containing a random nonce to be |
751 | - used in a proof-of-storage challenge; default is ``b''`` |
752 | - """ |
753 | if not isinstance(leaf_index, int): |
754 | raise TypeError( |
755 | TYPE_ERROR.format( |
756 | @@ -315,24 +256,15 @@ |
757 | self.leaf_size, len(leaf_data) |
758 | ) |
759 | ) |
760 | - digest = self._hash_leaf(leaf_index, leaf_data, challenge) |
761 | - assert len(digest) == self.digest_bytes |
762 | - return digest |
763 | + return self._hash_leaf(make_key(leaf_index), leaf_data, challenge) |
764 | + |
765 | + def _hash_leaf(self, key_leaf_index, leaf_data, challenge): |
766 | + name = self.__class__.__name__ |
767 | + raise NotImplementedError( |
768 | + '{}._hash_leaf(key_leaf_index, leaf_data, challenge)'.format(name) |
769 | + ) |
770 | |
771 | def hash_root(self, file_size, leaf_hashes): |
772 | - """ |
773 | - Return the root-hash for a file *file_size* bytes with *leaf_hashes*. |
774 | - |
775 | - For example: |
776 | - |
777 | - >>> p = Protocol(MIN_LEAF_SIZE, 200) |
778 | - >>> p.hash_root(1776, b'DDDDDDDDDDDDDDDDDDDDDDDDD') |
779 | - 'SJGXSO43OF5424G6HAEG55X2SFCZRW7L7LG42UKN' |
780 | - |
781 | - :param file_size: an ``int`` >= 1 |
782 | - :param leaf_hashes: a ``bytes`` instance containing the concatenated |
783 | - leaf-hashes produced by `Protocol.hash_leaf()`. |
784 | - """ |
785 | if not isinstance(file_size, int): |
786 | raise TypeError( |
787 | TYPE_ERROR.format('file_size', int, type(file_size), file_size) |
788 | @@ -369,66 +301,41 @@ |
789 | low, high, file_size |
790 | ) |
791 | ) |
792 | - digest = self._hash_root(file_size, leaf_hashes) |
793 | - return self.encode(digest) |
794 | - |
795 | - def _hash_leaf(self, leaf_index, leaf_data, challenge): |
796 | - """ |
797 | - Protocol version 1 leaf-hashing implementation. |
798 | - |
799 | - Subclasses can override this to use different hash functions, |
800 | - configurations, etc. |
801 | - """ |
802 | - assert leaf_index >= 0 |
803 | - assert 1 <= len(leaf_data) <= self.leaf_size |
804 | + return self._hash_root(make_key(file_size), leaf_hashes) |
805 | + |
806 | + def _hash_root(self, key_file_size, leaf_hashes): |
807 | + name = self.__class__.__name__ |
808 | + raise NotImplementedError( |
809 | + '{}._hash_root(key_file_size, leaf_hashes)'.format(name) |
810 | + ) |
811 | + |
812 | + |
813 | +class SkeinProtocol(Protocol): |
814 | + def _hash_leaf(self, key_leaf_index, leaf_data, challenge): |
815 | return skein512(leaf_data, |
816 | digest_bits=self.digest_bits, |
817 | pers=PERS_LEAF, |
818 | - key=str(leaf_index).encode('utf-8'), |
819 | + key=key_leaf_index, |
820 | nonce=challenge, |
821 | ).digest() |
822 | |
823 | - def _hash_root(self, file_size, leaf_hashes): |
824 | - """ |
825 | - Protocol version 1 root-hashing implementation. |
826 | - |
827 | - Subclasses can override this to use different hash functions, |
828 | - configurations, etc. |
829 | - """ |
830 | - assert file_size >= 1 |
831 | - assert len(leaf_hashes) > 0 |
832 | - assert len(leaf_hashes) % self.digest_bytes == 0 |
833 | + def _hash_root(self, key_file_size, leaf_hashes): |
834 | return skein512(leaf_hashes, |
835 | digest_bits=self.digest_bits, |
836 | pers=PERS_ROOT, |
837 | - key=str(file_size).encode('utf-8'), |
838 | + key=key_file_size, |
839 | ).digest() |
840 | |
841 | |
842 | -VERSION1 = Protocol(8 * MiB, 280) |
843 | - |
844 | - |
845 | -class OldProtocol(Protocol): |
846 | - """ |
847 | - The unofficial version zero protocol. |
848 | - |
849 | - Although no formal support commitments were ever made for this version zero |
850 | - protocol, it has been used enough in the wild that we should really |
851 | - continue to support it and provide a nice migration to the version one |
852 | - protocol. |
853 | - """ |
854 | - |
855 | - def _hash_leaf_index(self, leaf_index): |
856 | - assert leaf_index >= 0 |
857 | - return skein512(str(leaf_index).encode('utf-8'), |
858 | +class OldSkeinProtocol(Protocol): |
859 | + def _hash_leaf_index(self, key_leaf_index): |
860 | + return skein512(key_leaf_index, |
861 | digest_bits=self.digest_bits, |
862 | pers=PERS_LEAF_INDEX, |
863 | ).digest() |
864 | |
865 | - def _hash_leaf(self, leaf_index, leaf_data, challenge): |
866 | - assert leaf_index >= 0 |
867 | - assert 1 <= len(leaf_data) <= self.leaf_size |
868 | - skein = skein512(self._hash_leaf_index(leaf_index), |
869 | + def _hash_leaf(self, key_leaf_index, leaf_data, challenge): |
870 | + skein = skein512(self._hash_leaf_index(key_leaf_index), |
871 | digest_bits=self.digest_bits, |
872 | pers=PERS_LEAF, |
873 | nonce=challenge, |
874 | @@ -436,18 +343,14 @@ |
875 | skein.update(leaf_data) |
876 | return skein.digest() |
877 | |
878 | - def _hash_file_size(self, file_size): |
879 | - assert file_size >= 1 |
880 | - return skein512(str(file_size).encode('utf-8'), |
881 | + def _hash_file_size(self, key_file_size): |
882 | + return skein512(key_file_size, |
883 | digest_bits=self.digest_bits, |
884 | pers=PERS_FILE_SIZE, |
885 | ).digest() |
886 | |
887 | - def _hash_root(self, file_size, leaf_hashes): |
888 | - assert file_size >= 1 |
889 | - assert len(leaf_hashes) > 0 |
890 | - assert len(leaf_hashes) % self.digest_bytes == 0 |
891 | - skein = skein512(self._hash_file_size(file_size), |
892 | + def _hash_root(self, key_file_size, leaf_hashes): |
893 | + skein = skein512(self._hash_file_size(key_file_size), |
894 | digest_bits=self.digest_bits, |
895 | pers=PERS_ROOT, |
896 | ) |
897 | @@ -455,6 +358,7 @@ |
898 | return skein.digest() |
899 | |
900 | |
901 | -VERSION0 = OldProtocol(8 * MiB, 240) |
902 | +# Create the Protocol instances: |
903 | +VERSION0 = OldSkeinProtocol(8 * MiB, 240) |
904 | +VERSION1 = SkeinProtocol(8 * MiB, 240) |
905 | |
906 | -PROTOCOLS = (VERSION0, VERSION1) |
907 | |
908 | === modified file 'filestore/tests/__init__.py' |
909 | --- filestore/tests/__init__.py 2013-02-18 12:40:06 +0000 |
910 | +++ filestore/tests/__init__.py 2013-02-19 15:39:24 +0000 |
911 | @@ -35,12 +35,10 @@ |
912 | from random import SystemRandom |
913 | |
914 | from skein import skein512 |
915 | +from dbase32.rfc3548 import b32enc, b32dec |
916 | from dbase32 import isdb32 |
917 | |
918 | -from filestore import protocols |
919 | -from filestore.protocols import PROTOCOLS, VERSION0, VERSION1 |
920 | -from filestore.protocols import b32dumps, b32loads |
921 | -from filestore import misc |
922 | +from filestore import protocols, misc |
923 | import filestore |
924 | |
925 | |
926 | @@ -69,7 +67,7 @@ |
927 | ]) |
928 | ID = filestore.hash_root(FILE_SIZE, LEAF_HASHES) |
929 | CH = filestore.ContentHash(ID, FILE_SIZE, LEAF_HASHES) |
930 | -assert CH.id == 'DMJGE4OTWZVKSX426GHE46GZCGICMQOVWNGRVB7E665Y2RAM', CH.id |
931 | +#assert CH.id == 'DMJGE4OTWZVKSX426GHE46GZCGICMQOVWNGRVB7E665Y2RAM', CH.id |
932 | |
933 | |
934 | def write_sample_file(fp): |
935 | @@ -110,17 +108,13 @@ |
936 | def touch(self, *parts): |
937 | d = self.makedirs(*parts[:-1]) |
938 | f = self.join(*parts) |
939 | - assert not path.exists(f) |
940 | - open(f, 'wb').close() |
941 | - assert path.isfile(f) and not path.islink(f) |
942 | + open(f, 'xb').close() |
943 | return f |
944 | |
945 | def write(self, content, *parts): |
946 | d = self.makedirs(*parts[:-1]) |
947 | f = self.join(*parts) |
948 | - assert not path.exists(f) |
949 | - open(f, 'wb').write(content) |
950 | - assert path.isfile(f) and not path.islink(f) |
951 | + open(f, 'xb').write(content) |
952 | return f |
953 | |
954 | def copy(self, src, *parts): |
955 | @@ -270,6 +264,33 @@ |
956 | self.assertEqual(str(e), "'the id' not in 'the FileStore'") |
957 | |
958 | |
959 | +class TestConstants(TestCase): |
960 | + def test_leaf_size(self): |
961 | + self.assertIsInstance(filestore.LEAF_SIZE, int) |
962 | + self.assertEqual(filestore.LEAF_SIZE, 8 * 1024 * 1024) |
963 | + self.assertEqual(filestore.LEAF_SIZE, 2**23) |
964 | + self.assertEqual(filestore.LEAF_SIZE, filestore.PROTOCOL.leaf_size) |
965 | + self.assertEqual(filestore.LEAF_SIZE, protocols.VERSION0.leaf_size) |
966 | + self.assertEqual(filestore.LEAF_SIZE, protocols.VERSION1.leaf_size) |
967 | + |
968 | + def test_digest_bits(self): |
969 | + self.assertIsInstance(filestore.DIGEST_BITS, int) |
970 | + self.assertEqual(filestore.DIGEST_BITS % 40, 0) |
971 | + self.assertEqual(filestore.DIGEST_BITS, filestore.PROTOCOL.digest_bits) |
972 | + |
973 | + def test_digest_bytes(self): |
974 | + self.assertIsInstance(filestore.DIGEST_BYTES, int) |
975 | + self.assertEqual(filestore.DIGEST_BYTES % 5, 0) |
976 | + self.assertEqual(filestore.DIGEST_BYTES, filestore.DIGEST_BITS // 8) |
977 | + self.assertEqual(filestore.DIGEST_BYTES, filestore.PROTOCOL.digest_bytes) |
978 | + |
979 | + def test_digest_b32len(self): |
980 | + self.assertIsInstance(filestore.DIGEST_B32LEN, int) |
981 | + self.assertEqual(filestore.DIGEST_B32LEN % 8, 0) |
982 | + self.assertEqual(filestore.DIGEST_B32LEN, filestore.DIGEST_BITS // 5) |
983 | + self.assertEqual(filestore.DIGEST_B32LEN, filestore.PROTOCOL.digest_b32len) |
984 | + |
985 | + |
986 | class TestFunctions(TestCase): |
987 | def test_constants(self): |
988 | """ |
989 | @@ -361,7 +382,7 @@ |
990 | digest = f(2, content) |
991 | self.assertEqual( |
992 | digest, |
993 | - skein512(VERSION0._hash_leaf_index(2) + content, |
994 | + skein512(protocols.VERSION0._hash_leaf_index(b'2') + content, |
995 | digest_bits=240, |
996 | pers=protocols.PERS_LEAF, |
997 | ).digest() |
998 | @@ -371,7 +392,7 @@ |
999 | digest = f(2, content) |
1000 | self.assertEqual( |
1001 | digest, |
1002 | - skein512(VERSION0._hash_leaf_index(2) + content, |
1003 | + skein512(protocols.VERSION0._hash_leaf_index(b'2') + content, |
1004 | digest_bits=240, |
1005 | pers=protocols.PERS_LEAF, |
1006 | ).digest() |
1007 | @@ -499,7 +520,7 @@ |
1008 | |
1009 | def test_check_leaf_hashes(self): |
1010 | # Test with wrong type: |
1011 | - leaf_hashes = 'a' * 60 |
1012 | + leaf_hashes = 'a' * 30 |
1013 | with self.assertRaises(TypeError) as cm: |
1014 | filestore.check_leaf_hashes(leaf_hashes) |
1015 | self.assertEqual( |
1016 | @@ -514,49 +535,30 @@ |
1017 | self.assertIs(cm.exception.leaf_hashes, leaf_hashes) |
1018 | |
1019 | # Test with wrong length: |
1020 | - leaf_hashes = b'a' * 61 |
1021 | + leaf_hashes = b'a' * 29 |
1022 | + with self.assertRaises(filestore.LeafHashesError) as cm: |
1023 | + filestore.check_leaf_hashes(leaf_hashes) |
1024 | + self.assertIs(cm.exception.leaf_hashes, leaf_hashes) |
1025 | + |
1026 | + leaf_hashes = b'a' * 31 |
1027 | + with self.assertRaises(filestore.LeafHashesError) as cm: |
1028 | + filestore.check_leaf_hashes(leaf_hashes) |
1029 | + self.assertIs(cm.exception.leaf_hashes, leaf_hashes) |
1030 | + |
1031 | + leaf_hashes = b'a' * (18 * 30 - 1) |
1032 | with self.assertRaises(filestore.LeafHashesError) as cm: |
1033 | filestore.check_leaf_hashes(leaf_hashes) |
1034 | self.assertIs(cm.exception.leaf_hashes, leaf_hashes) |
1035 | |
1036 | # Test with good values: |
1037 | - good30 = os.urandom(30) |
1038 | - self.assertIs(filestore.check_leaf_hashes(good30), good30) |
1039 | - self.assertIs( |
1040 | - filestore.check_leaf_hashes(good30, protocol=VERSION0), |
1041 | - good30 |
1042 | - ) |
1043 | - good35 = os.urandom(35) |
1044 | - self.assertIs( |
1045 | - filestore.check_leaf_hashes(good35, protocol=VERSION1), |
1046 | - good35 |
1047 | - ) |
1048 | - |
1049 | - # Test with wrong protocol version: |
1050 | - with self.assertRaises(filestore.LeafHashesError) as cm: |
1051 | - filestore.check_leaf_hashes(good30, protocol=VERSION1) |
1052 | - self.assertIs(cm.exception.leaf_hashes, good30) |
1053 | - with self.assertRaises(filestore.LeafHashesError) as cm: |
1054 | - filestore.check_leaf_hashes(good35, protocol=VERSION0) |
1055 | - self.assertIs(cm.exception.leaf_hashes, good35) |
1056 | - |
1057 | - # Future proofing: |
1058 | - for protocol in PROTOCOLS: |
1059 | - good = os.urandom(protocol.digest_bytes) |
1060 | - self.assertIs(filestore.check_leaf_hashes(good, protocol), good) |
1061 | - |
1062 | - good = os.urandom(18 * protocol.digest_bytes) |
1063 | - self.assertIs(filestore.check_leaf_hashes(good, protocol), good) |
1064 | - |
1065 | - bad = os.urandom(protocol.digest_bytes + 1) |
1066 | - with self.assertRaises(filestore.LeafHashesError) as cm: |
1067 | - filestore.check_leaf_hashes(bad, protocol) |
1068 | - self.assertIs(cm.exception.leaf_hashes, bad) |
1069 | - |
1070 | - bad = os.urandom(18 * protocol.digest_bytes - 1) |
1071 | - with self.assertRaises(filestore.LeafHashesError) as cm: |
1072 | - filestore.check_leaf_hashes(bad, protocol) |
1073 | - self.assertIs(cm.exception.leaf_hashes, bad) |
1074 | + good = os.urandom(30) |
1075 | + self.assertIs(filestore.check_leaf_hashes(good), good) |
1076 | + |
1077 | + good = os.urandom(60) |
1078 | + self.assertIs(filestore.check_leaf_hashes(good), good) |
1079 | + |
1080 | + good = os.urandom(18 * 30) |
1081 | + self.assertIs(filestore.check_leaf_hashes(good), good) |
1082 | |
1083 | def test_check_root_hash(self): |
1084 | f = filestore.check_root_hash |
1085 | @@ -587,62 +589,6 @@ |
1086 | 'OYESBWEZ4Y2AGSLMNZB4ZF75A2VG7NXVB4R25SSMRGXLN4CR' |
1087 | ) |
1088 | |
1089 | - def test_check_root_hash2(self): |
1090 | - for protocol in PROTOCOLS: |
1091 | - file_size = protocol.leaf_size + 1 |
1092 | - leaf0_hash = os.urandom(protocol.digest_bytes) |
1093 | - leaf1_hash = os.urandom(protocol.digest_bytes) |
1094 | - leaf_hashes = leaf0_hash + leaf1_hash |
1095 | - _id = protocol.hash_root(file_size, leaf_hashes) |
1096 | - |
1097 | - # Everything is correct: |
1098 | - ch = filestore.check_root_hash(_id, file_size, leaf_hashes) |
1099 | - self.assertIsInstance(ch, filestore.ContentHash) |
1100 | - self.assertEqual(ch.id, _id) |
1101 | - self.assertEqual(ch.file_size, file_size) |
1102 | - self.assertEqual(ch.leaf_hashes, leaf_hashes) |
1103 | - self.assertEqual(ch, (_id, file_size, leaf_hashes)) |
1104 | - |
1105 | - # Everything is correct, unpack=True |
1106 | - ch = filestore.check_root_hash(_id, file_size, leaf_hashes, |
1107 | - unpack=True |
1108 | - ) |
1109 | - self.assertIsInstance(ch, filestore.ContentHash) |
1110 | - self.assertEqual(ch.id, _id) |
1111 | - self.assertEqual(ch.file_size, file_size) |
1112 | - self.assertEqual(ch.leaf_hashes, (leaf0_hash, leaf1_hash)) |
1113 | - self.assertEqual(ch, (_id, file_size, (leaf0_hash, leaf1_hash))) |
1114 | - |
1115 | - # Wrong ID: |
1116 | - bad_id = random_id(protocol.digest_bytes) |
1117 | - with self.assertRaises(filestore.RootHashError) as cm: |
1118 | - filestore.check_root_hash(bad_id, file_size, leaf_hashes) |
1119 | - self.assertEqual(cm.exception.id, bad_id) |
1120 | - self.assertEqual(cm.exception.file_size, file_size) |
1121 | - self.assertEqual(cm.exception.leaf_hashes, leaf_hashes) |
1122 | - self.assertEqual(cm.exception.bad_id, _id) |
1123 | - |
1124 | - # Wrong file_size: |
1125 | - with self.assertRaises(filestore.RootHashError) as cm: |
1126 | - filestore.check_root_hash(_id, file_size + 1, leaf_hashes) |
1127 | - self.assertEqual(cm.exception.id, _id) |
1128 | - self.assertEqual(cm.exception.file_size, file_size + 1) |
1129 | - self.assertEqual(cm.exception.leaf_hashes, leaf_hashes) |
1130 | - self.assertEqual(cm.exception.bad_id, |
1131 | - protocol.hash_root(file_size + 1, leaf_hashes) |
1132 | - ) |
1133 | - |
1134 | - # Wrong leaf_hashes |
1135 | - bad_leaf_hashes = leaf0_hash + os.urandom(protocol.digest_bytes) |
1136 | - with self.assertRaises(filestore.RootHashError) as cm: |
1137 | - filestore.check_root_hash(_id, file_size, bad_leaf_hashes) |
1138 | - self.assertEqual(cm.exception.id, _id) |
1139 | - self.assertEqual(cm.exception.file_size, file_size) |
1140 | - self.assertEqual(cm.exception.leaf_hashes, bad_leaf_hashes) |
1141 | - self.assertEqual(cm.exception.bad_id, |
1142 | - protocol.hash_root(file_size, bad_leaf_hashes) |
1143 | - ) |
1144 | - |
1145 | def test_enumerate_leaf_hashes(self): |
1146 | f = filestore.enumerate_leaf_hashes |
1147 | self.assertEqual( |
1148 | @@ -772,7 +718,7 @@ |
1149 | # Test with wrong length: |
1150 | for i in range(321): |
1151 | value = 'N' * i |
1152 | - if len(value) in (48, 56): |
1153 | + if len(value) == 48: |
1154 | self.assertIs(filestore.check_id(value), value) |
1155 | continue |
1156 | with self.assertRaises(filestore.IDError) as cm: |
1157 | @@ -782,18 +728,12 @@ |
1158 | # Test with 48 and 56 character: |
1159 | id48 = random_id(30) |
1160 | self.assertIs(filestore.check_id(id48), id48) |
1161 | - id56 = random_id(35) |
1162 | - self.assertIs(filestore.check_id(id56), id56) |
1163 | |
1164 | # Test case sensitivity: |
1165 | bad48 = id48.lower() |
1166 | with self.assertRaises(filestore.IDError) as cm: |
1167 | filestore.check_id(bad48) |
1168 | self.assertIs(cm.exception.id, bad48) |
1169 | - bad56 = id56.lower() |
1170 | - with self.assertRaises(filestore.IDError) as cm: |
1171 | - filestore.check_id(bad56) |
1172 | - self.assertIs(cm.exception.id, bad56) |
1173 | |
1174 | def test_iter_files(self): |
1175 | """ |
1176 | @@ -1247,30 +1187,18 @@ |
1177 | src_fp = open(dst, 'rb') |
1178 | self.assertEqual(filestore.hash_fp(src_fp), ch) |
1179 | |
1180 | - def test_hash_fp2(self): |
1181 | - obj = misc.load_test_vectors() |
1182 | - for proto in PROTOCOLS: |
1183 | - tmp = TempDir() |
1184 | - leaves = misc.build_test_leaves(proto.leaf_size) |
1185 | - vectors = obj[str(proto.digest_b32len)] |
1186 | - for (key, value) in vectors['root_hashes'].items(): |
1187 | - name = tmp.join(key) |
1188 | - assert not path.exists(name) |
1189 | - fp = open(name, 'wb') |
1190 | - for L in key: |
1191 | - fp.write(leaves[L]) |
1192 | - fp.close() |
1193 | - fp = open(name, 'rb') |
1194 | - ch = filestore.hash_fp(fp, protocol=proto) |
1195 | - self.assertIsInstance(ch, filestore.ContentHash) |
1196 | - self.assertEqual(ch.id, value) |
1197 | - self.assertEqual(ch.file_size, path.getsize(name)) |
1198 | - self.assertEqual(ch.leaf_hashes, |
1199 | - b''.join( |
1200 | - b32loads(vectors['leaf_hashes'][L][i]) |
1201 | - for (i, L) in enumerate(key) |
1202 | - ) |
1203 | - ) |
1204 | + leaves = misc.build_leaves(filestore.LEAF_SIZE) |
1205 | + C = leaves['C'] |
1206 | + for (L, data) in leaves.items(): |
1207 | + tmp.write(data, L) |
1208 | + tmp.write(C + data, 'C' + L) |
1209 | + |
1210 | + vectors = misc.load_data('V0') |
1211 | + for name in ['A', 'B', 'C', 'CA', 'CB', 'CC']: |
1212 | + src_fp = open(tmp.join(name), 'rb') |
1213 | + ch = filestore.hash_fp(src_fp) |
1214 | + self.assertIsInstance(ch, filestore.ContentHash) |
1215 | + self.assertEqual(ch.id, vectors['root_hashes'][name]) |
1216 | |
1217 | def test_ensuredir(self): |
1218 | f = filestore.ensuredir |
1219 | @@ -1594,22 +1522,6 @@ |
1220 | stats.sort(key=lambda s: s.id) |
1221 | self.assertEqual(list(fs), stats) |
1222 | |
1223 | - # Add more valid files in (56 character IDs) |
1224 | - for i in range(1000): |
1225 | - _id = random_id(35) |
1226 | - size = i + 1 |
1227 | - f = fs.path(_id) |
1228 | - assert not path.exists(f) |
1229 | - open(f, 'wb').write(b'N' * size) |
1230 | - os.chmod(f, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) |
1231 | - self.assertEqual(path.getsize(f), size) |
1232 | - st = filestore.Stat(_id, f, size, path.getmtime(f)) |
1233 | - self.assertEqual(fs.stat(_id), st) |
1234 | - stats.append(st) |
1235 | - self.assertNotEqual(list(fs), stats) # Sorting! |
1236 | - stats.sort(key=lambda s: s.id) |
1237 | - self.assertEqual(list(fs), stats) |
1238 | - |
1239 | # Should ignore symlinks, even if to valid files |
1240 | # This makes sure os.lstat() is being used rather than os.stat() |
1241 | for i in range(50): |
1242 | @@ -1989,31 +1901,6 @@ |
1243 | self.assertFalse(path.exists(canonical)) |
1244 | self.assertTrue(path.isfile(corrupt)) |
1245 | |
1246 | - def test_verify2(self): |
1247 | - tmp = TempDir() |
1248 | - fs = filestore.FileStore(tmp.dir) |
1249 | - obj = misc.load_test_vectors() |
1250 | - for proto in PROTOCOLS: |
1251 | - leaves = misc.build_test_leaves(proto.leaf_size) |
1252 | - vectors = obj[str(proto.digest_b32len)] |
1253 | - for (key, value) in vectors['root_hashes'].items(): |
1254 | - name = fs.path(value) |
1255 | - assert not path.exists(name) |
1256 | - fp = open(name, 'wb') |
1257 | - for L in key: |
1258 | - fp.write(leaves[L]) |
1259 | - fp.close() |
1260 | - ch = fs.verify(value) |
1261 | - self.assertIsInstance(ch, filestore.ContentHash) |
1262 | - self.assertEqual(ch.id, value) |
1263 | - self.assertEqual(ch.file_size, path.getsize(name)) |
1264 | - self.assertEqual(ch.leaf_hashes, |
1265 | - b''.join( |
1266 | - b32loads(vectors['leaf_hashes'][L][i]) |
1267 | - for (i, L) in enumerate(key) |
1268 | - ) |
1269 | - ) |
1270 | - |
1271 | def test_verify_iter(self): |
1272 | tmp = TempDir() |
1273 | fs = filestore.FileStore(tmp.dir) |
1274 | |
1275 | === modified file 'filestore/tests/test_misc.py' |
1276 | --- filestore/tests/test_misc.py 2013-02-14 20:08:06 +0000 |
1277 | +++ filestore/tests/test_misc.py 2013-02-19 15:39:24 +0000 |
1278 | @@ -29,28 +29,44 @@ |
1279 | import json |
1280 | from hashlib import md5 |
1281 | |
1282 | +from dbase32 import db32enc, db32dec |
1283 | +from dbase32.rfc3548 import b32enc, b32dec |
1284 | + |
1285 | import filestore |
1286 | -from filestore.protocols import MIN_LEAF_SIZE, Protocol, VERSION0, VERSION1 |
1287 | -from filestore.protocols import b32dumps, b32loads |
1288 | -from filestore import misc |
1289 | +from filestore.protocols import MIN_LEAF_SIZE |
1290 | +from filestore import misc, protocols |
1291 | |
1292 | |
1293 | class TestFunctions(TestCase): |
1294 | - def test_load_test_vectors(self): |
1295 | + def test_load_data(self): |
1296 | pkg = path.dirname(path.abspath(filestore.__file__)) |
1297 | self.assertTrue(path.isdir(pkg)) |
1298 | - self.assertEqual( |
1299 | - misc.TEST_VECTORS, |
1300 | - path.join(pkg, 'data', 'test-vectors.json') |
1301 | - ) |
1302 | - self.assertTrue(path.isfile(misc.TEST_VECTORS)) |
1303 | - self.assertEqual( |
1304 | - misc.load_test_vectors(), |
1305 | - json.loads(open(misc.TEST_VECTORS, 'r').read()) |
1306 | - ) |
1307 | - |
1308 | - def test_build_test_leaves(self): |
1309 | - obj = misc.build_test_leaves(MIN_LEAF_SIZE) |
1310 | + self.assertEqual(misc.DATADIR, path.join(pkg, 'data')) |
1311 | + self.assertTrue(path.isdir(misc.DATADIR)) |
1312 | + |
1313 | + V0 = path.join(misc.DATADIR, 'V0.json') |
1314 | + self.assertTrue(path.isfile(V0)) |
1315 | + self.assertEqual( |
1316 | + misc.load_data('V0'), |
1317 | + json.loads(open(V0, 'r').read()) |
1318 | + ) |
1319 | + |
1320 | + V1 = path.join(misc.DATADIR, 'V1.json') |
1321 | + self.assertTrue(path.isfile(V0)) |
1322 | + self.assertEqual( |
1323 | + misc.load_data('V1'), |
1324 | + json.loads(open(V1, 'r').read()) |
1325 | + ) |
1326 | + |
1327 | + MD5SUMS = path.join(misc.DATADIR, 'MD5SUMS.json') |
1328 | + self.assertTrue(path.isfile(MD5SUMS)) |
1329 | + self.assertEqual( |
1330 | + misc.load_data('MD5SUMS'), |
1331 | + json.loads(open(MD5SUMS, 'r').read()) |
1332 | + ) |
1333 | + |
1334 | + def test_build_leaves(self): |
1335 | + obj = misc.build_leaves(MIN_LEAF_SIZE) |
1336 | self.assertEqual(obj, |
1337 | { |
1338 | 'A': b'A', |
1339 | @@ -62,7 +78,7 @@ |
1340 | self.assertEqual(len(obj['B']), MIN_LEAF_SIZE - 1) |
1341 | self.assertEqual(len(obj['C']), MIN_LEAF_SIZE) |
1342 | |
1343 | - obj = misc.build_test_leaves(2 * MIN_LEAF_SIZE) |
1344 | + obj = misc.build_leaves(2 * MIN_LEAF_SIZE) |
1345 | self.assertEqual(obj, |
1346 | { |
1347 | 'A': b'A', |
1348 | @@ -74,59 +90,90 @@ |
1349 | self.assertEqual(len(obj['B']), 2 * MIN_LEAF_SIZE - 1) |
1350 | self.assertEqual(len(obj['C']), 2 * MIN_LEAF_SIZE) |
1351 | |
1352 | - def test_build_test_vectors(self): |
1353 | - proto = Protocol(MIN_LEAF_SIZE, 200) |
1354 | - leaves = misc.build_test_leaves(proto.leaf_size) |
1355 | + def test_build_vectors(self): |
1356 | + leaf_size = 2 * 1024 * 1024 |
1357 | + proto = protocols.SkeinProtocol(leaf_size, 200) |
1358 | + leaves = misc.build_leaves(leaf_size) |
1359 | + |
1360 | A0 = proto.hash_leaf(0, leaves['A']) |
1361 | A1 = proto.hash_leaf(1, leaves['A']) |
1362 | B0 = proto.hash_leaf(0, leaves['B']) |
1363 | B1 = proto.hash_leaf(1, leaves['B']) |
1364 | C0 = proto.hash_leaf(0, leaves['C']) |
1365 | C1 = proto.hash_leaf(1, leaves['C']) |
1366 | - self.assertEqual( |
1367 | - misc.build_test_vectors(proto), |
1368 | - { |
1369 | - 'leaf_hashes': { |
1370 | - 'A': [b32dumps(A0), b32dumps(A1)], |
1371 | - 'B': [b32dumps(B0), b32dumps(B1)], |
1372 | - 'C': [b32dumps(C0), b32dumps(C1)], |
1373 | - }, |
1374 | - 'root_hashes': { |
1375 | - 'A': proto.hash_root(1, A0), |
1376 | - 'B': proto.hash_root(MIN_LEAF_SIZE - 1, B0), |
1377 | - 'C': proto.hash_root(MIN_LEAF_SIZE, C0), |
1378 | - 'CA': proto.hash_root(MIN_LEAF_SIZE + 1, C0 + A1), |
1379 | - 'CB': proto.hash_root(2 * MIN_LEAF_SIZE - 1, C0 + B1), |
1380 | - 'CC': proto.hash_root(2 * MIN_LEAF_SIZE, C0 + C1), |
1381 | - } |
1382 | - } |
1383 | - ) |
1384 | - |
1385 | - self.assertEqual( |
1386 | - misc.load_test_vectors(), |
1387 | - { |
1388 | - '48': misc.build_test_vectors(VERSION0), |
1389 | - '56': misc.build_test_vectors(VERSION1), |
1390 | - } |
1391 | - ) |
1392 | - |
1393 | - def test_get_test_integers(self): |
1394 | - self.assertEqual( |
1395 | - misc.get_test_integers(100), |
1396 | + |
1397 | + del leaves |
1398 | + |
1399 | + A = proto.hash_root(1, A0) |
1400 | + B = proto.hash_root(leaf_size - 1, B0) |
1401 | + C = proto.hash_root(leaf_size, C0) |
1402 | + CA = proto.hash_root(leaf_size + 1, C0 + A1) |
1403 | + CB = proto.hash_root(2 * leaf_size - 1, C0 + B1) |
1404 | + CC = proto.hash_root(2 * leaf_size, C0 + C1) |
1405 | + |
1406 | + self.assertEqual( |
1407 | + misc.build_vectors(proto), |
1408 | + { |
1409 | + 'leaf_hashes': { |
1410 | + 'A': [db32enc(A0), db32enc(A1)], |
1411 | + 'B': [db32enc(B0), db32enc(B1)], |
1412 | + 'C': [db32enc(C0), db32enc(C1)], |
1413 | + }, |
1414 | + 'root_hashes': { |
1415 | + 'A': db32enc(A), |
1416 | + 'B': db32enc(B), |
1417 | + 'C': db32enc(C), |
1418 | + 'CA': db32enc(CA), |
1419 | + 'CB': db32enc(CB), |
1420 | + 'CC': db32enc(CC), |
1421 | + } |
1422 | + } |
1423 | + ) |
1424 | + |
1425 | + self.assertEqual( |
1426 | + misc.build_vectors(proto, encoder=b32enc), |
1427 | + { |
1428 | + 'leaf_hashes': { |
1429 | + 'A': [b32enc(A0), b32enc(A1)], |
1430 | + 'B': [b32enc(B0), b32enc(B1)], |
1431 | + 'C': [b32enc(C0), b32enc(C1)], |
1432 | + }, |
1433 | + 'root_hashes': { |
1434 | + 'A': b32enc(A), |
1435 | + 'B': b32enc(B), |
1436 | + 'C': b32enc(C), |
1437 | + 'CA': b32enc(CA), |
1438 | + 'CB': b32enc(CB), |
1439 | + 'CC': b32enc(CC), |
1440 | + } |
1441 | + } |
1442 | + ) |
1443 | + |
1444 | + def test_get_integers(self): |
1445 | + self.assertEqual( |
1446 | + misc.get_integers(100), |
1447 | [0, 1, 99, 100, 101, 199, 200] |
1448 | ) |
1449 | self.assertEqual( |
1450 | - misc.get_test_integers(500), |
1451 | + misc.get_integers(500), |
1452 | [0, 1, 499, 500, 501, 999, 1000] |
1453 | ) |
1454 | |
1455 | - def test_build_test_md5(self): |
1456 | + def test_build_md5sums(self): |
1457 | A = b'A' |
1458 | B = b'B' * 99 |
1459 | C = b'C' * 100 |
1460 | self.assertEqual( |
1461 | - misc.build_test_md5(100), |
1462 | + misc.build_md5sums(100), |
1463 | { |
1464 | + 'files': { |
1465 | + 'A': md5(A).hexdigest(), |
1466 | + 'B': md5(B).hexdigest(), |
1467 | + 'C': md5(C).hexdigest(), |
1468 | + 'CA': md5(C + A).hexdigest(), |
1469 | + 'CB': md5(C + B).hexdigest(), |
1470 | + 'CC': md5(C + C).hexdigest(), |
1471 | + }, |
1472 | 'integers': { |
1473 | '0': md5(b'0').hexdigest(), |
1474 | '1': md5(b'1').hexdigest(), |
1475 | @@ -136,6 +183,19 @@ |
1476 | '199': md5(b'199').hexdigest(), |
1477 | '200': md5(b'200').hexdigest(), |
1478 | }, |
1479 | + 'personalization': { |
1480 | + 'leaf': md5(protocols.PERS_LEAF).hexdigest(), |
1481 | + 'root': md5(protocols.PERS_ROOT).hexdigest(), |
1482 | + }, |
1483 | + } |
1484 | + ) |
1485 | + |
1486 | + A = b'A' |
1487 | + B = b'B' * 499 |
1488 | + C = b'C' * 500 |
1489 | + self.assertEqual( |
1490 | + misc.build_md5sums(500), |
1491 | + { |
1492 | 'files': { |
1493 | 'A': md5(A).hexdigest(), |
1494 | 'B': md5(B).hexdigest(), |
1495 | @@ -143,10 +203,28 @@ |
1496 | 'CA': md5(C + A).hexdigest(), |
1497 | 'CB': md5(C + B).hexdigest(), |
1498 | 'CC': md5(C + C).hexdigest(), |
1499 | - } |
1500 | + }, |
1501 | + 'integers': { |
1502 | + '0': md5(b'0').hexdigest(), |
1503 | + '1': md5(b'1').hexdigest(), |
1504 | + '499': md5(b'499').hexdigest(), |
1505 | + '500': md5(b'500').hexdigest(), |
1506 | + '501': md5(b'501').hexdigest(), |
1507 | + '999': md5(b'999').hexdigest(), |
1508 | + '1000': md5(b'1000').hexdigest(), |
1509 | + }, |
1510 | + 'personalization': { |
1511 | + 'leaf': md5(protocols.PERS_LEAF).hexdigest(), |
1512 | + 'root': md5(protocols.PERS_ROOT).hexdigest(), |
1513 | + }, |
1514 | } |
1515 | ) |
1516 | |
1517 | + self.assertEqual( |
1518 | + misc.build_md5sums(8 * 1024 * 1024), |
1519 | + misc.load_data('MD5SUMS') |
1520 | + ) |
1521 | + |
1522 | |
1523 | class TestTempFileStore(TestCase): |
1524 | def test_init(self): |
1525 | |
1526 | === modified file 'filestore/tests/test_protocols.py' |
1527 | --- filestore/tests/test_protocols.py 2013-02-14 20:08:06 +0000 |
1528 | +++ filestore/tests/test_protocols.py 2013-02-19 15:39:24 +0000 |
1529 | @@ -25,73 +25,154 @@ |
1530 | |
1531 | from unittest import TestCase |
1532 | import os |
1533 | -from base64 import b32encode, b32decode |
1534 | -from hashlib import md5 |
1535 | +from random import SystemRandom |
1536 | |
1537 | from skein import skein512 |
1538 | - |
1539 | -from filestore.protocols import b32dumps, b32loads |
1540 | -from filestore import protocols |
1541 | - |
1542 | - |
1543 | +from dbase32 import db32enc, db32dec |
1544 | +from dbase32.rfc3548 import b32enc, b32dec |
1545 | + |
1546 | +from filestore import protocols, misc |
1547 | + |
1548 | + |
1549 | +random = SystemRandom() |
1550 | MiB = 1024 * 1024 |
1551 | TwoMiB = 2 * MiB |
1552 | EightMiB = 8 * MiB |
1553 | +GiB = 1024 * MiB |
1554 | COUNT = 50 * 1000 |
1555 | TYPE_ERROR = '{}: need a {!r}; got a {!r}: {!r}' |
1556 | |
1557 | -A = b'A' |
1558 | -B = b'B' * (EightMiB - 1) |
1559 | -C = b'C' * EightMiB |
1560 | - |
1561 | -A0_B32 = 'XZ5I6KJTUSOIWVCEBOKUELTADZUXNHOAYO77NKKHWCIW3HYGYOPMX5JN' |
1562 | -A1_B32 = 'TEC7754ZNM26MTM6YQFI6TMVTTK4RKQEMPAGT2ROQZUBPUIHSJU2DDR3' |
1563 | -B0_B32 = 'P67PVKU3SCCQHNIRMR2Z5NICEMIP36WCFJG4AW6YBAE6UI4K6BVLY3EI' |
1564 | -B1_B32 = 'ZIFO5S2OYYPZAUN6XQWTWZGCDATXCGR2JYN7UIAX54WMVWETMIUFG7WM' |
1565 | -C0_B32 = 'RW2GJFIGPQF5WLR53UAK77TPHNRFKMUBYRB23JFS4G2RFRRNHW6OX4CR' |
1566 | -C1_B32 = 'XBVLPYBUX6QD2DKPJTYVUXT23K3AAUAW5J4RMQ543NQNDAHORQJ7GBDE' |
1567 | + |
1568 | +class DigestAccum(set): |
1569 | + def add(self, digest): |
1570 | + if not isinstance(digest, bytes): |
1571 | + raise TypeError('digest must be bytes') |
1572 | + if not (25 <= len(digest) <= 35 and len(digest) % 5 == 0): |
1573 | + raise ValueError('digest has bad length {}'.format(len(digest))) |
1574 | + if digest in self: |
1575 | + raise ValueError('digest already in set') |
1576 | + super().add(digest) |
1577 | + return digest |
1578 | + |
1579 | + |
1580 | +class TestAccum(TestCase): |
1581 | + def test_add(self): |
1582 | + accum = DigestAccum() |
1583 | + self.assertEqual(accum, set()) |
1584 | + self.assertEqual(len(accum), 0) |
1585 | + |
1586 | + with self.assertRaises(TypeError) as cm: |
1587 | + accum.add('A' * 30) |
1588 | + self.assertEqual(str(cm.exception), 'digest must be bytes') |
1589 | + |
1590 | + for size in [24, 26, 29, 31, 34, 36]: |
1591 | + bad = os.urandom(size) |
1592 | + with self.assertRaises(ValueError) as cm: |
1593 | + accum.add(bad) |
1594 | + self.assertEqual( |
1595 | + str(cm.exception), |
1596 | + 'digest has bad length {}'.format(size) |
1597 | + ) |
1598 | + |
1599 | + d1 = os.urandom(30) |
1600 | + d2 = os.urandom(30) |
1601 | + |
1602 | + self.assertIs(accum.add(d1), d1) |
1603 | + self.assertEqual(accum, set([d1])) |
1604 | + self.assertEqual(len(accum), 1) |
1605 | + |
1606 | + self.assertIs(accum.add(d2), d2) |
1607 | + self.assertEqual(accum, set([d1, d2])) |
1608 | + self.assertEqual(len(accum), 2) |
1609 | + |
1610 | + with self.assertRaises(ValueError) as cm: |
1611 | + accum.add(d1) |
1612 | + self.assertEqual(str(cm.exception), 'digest already in set') |
1613 | + |
1614 | + self.assertEqual(accum, set([d1, d2])) |
1615 | + self.assertEqual(len(accum), 2) |
1616 | |
1617 | |
1618 | class TestConstants(TestCase): |
1619 | + def test_pers(self): |
1620 | + all_pers = ( |
1621 | + protocols.PERS_LEAF, |
1622 | + protocols.PERS_ROOT, |
1623 | + protocols.PERS_LEAF_INDEX, |
1624 | + protocols.PERS_FILE_SIZE, |
1625 | + ) |
1626 | + for pers in all_pers: |
1627 | + self.assertIsInstance(pers, bytes) |
1628 | + self.assertEqual(pers[:36], |
1629 | + b'20110430 jderose@novacut.com dmedia/' |
1630 | + ) |
1631 | + self.assertEqual(len(set(all_pers)), len(all_pers)) |
1632 | + |
1633 | + # Be extra safe the what's actually used in V1: |
1634 | + self.assertIsInstance(protocols.PERS_LEAF, bytes) |
1635 | + self.assertIsInstance(protocols.PERS_ROOT, bytes) |
1636 | + self.assertNotEqual(protocols.PERS_LEAF, protocols.PERS_ROOT) |
1637 | + |
1638 | + def test_mib(self): |
1639 | + self.assertIsInstance(protocols.MiB, int) |
1640 | + self.assertEqual(protocols.MiB, 1024 * 1024) |
1641 | + self.assertEqual(protocols.MiB, 2**20) |
1642 | + |
1643 | def test_max_file_size(self): |
1644 | # For now, constrain to JavaScript integer limit (2**53): |
1645 | - self.assertEqual(protocols.MAX_FILE_SIZE, 9007199254740992) |
1646 | + self.assertIsInstance(protocols.MAX_FILE_SIZE, int) |
1647 | self.assertEqual(protocols.MAX_FILE_SIZE, 2**53) |
1648 | |
1649 | def test_min_leaf_size(self): |
1650 | - self.assertEqual(protocols.MIN_LEAF_SIZE, 2 * MiB) |
1651 | + self.assertIsInstance(protocols.MIN_LEAF_SIZE, int) |
1652 | + self.assertEqual(protocols.MIN_LEAF_SIZE, 2 * 1024 * 1024) |
1653 | + self.assertEqual(protocols.MIN_LEAF_SIZE, 2**21) |
1654 | self.assertEqual(protocols.MIN_LEAF_SIZE, |
1655 | protocols.MAX_FILE_SIZE // 2**32 |
1656 | ) |
1657 | |
1658 | |
1659 | class TestFunctions(TestCase): |
1660 | - def test_b32dumps(self): |
1661 | - self.assertEqual(b32dumps(b'\x00\x00\x00\x00\x00'), 'AAAAAAAA') |
1662 | - self.assertEqual(b32dumps(b'\xff\xff\xff\xff\xff'), '77777777') |
1663 | - self.assertEqual(b32dumps(b'\x00' * 35), 'A' * 56) |
1664 | - self.assertEqual(b32dumps(b'\xff' * 35), '7' * 56) |
1665 | - for i in range(1000): |
1666 | - data = os.urandom(15) |
1667 | - self.assertEqual( |
1668 | - b32dumps(data), |
1669 | - b32encode(data).decode('utf-8') |
1670 | - ) |
1671 | - |
1672 | - def test_b32loads(self): |
1673 | - self.assertEqual(b32loads('AAAAAAAA'), b'\x00\x00\x00\x00\x00') |
1674 | - self.assertEqual(b32loads('77777777'), b'\xff\xff\xff\xff\xff') |
1675 | - self.assertEqual(b32loads('A' * 56), b'\x00' * 35) |
1676 | - self.assertEqual(b32loads('7' * 56), b'\xff' * 35) |
1677 | - for i in range(1000): |
1678 | - data = os.urandom(15) |
1679 | - b32 = b32encode(data) |
1680 | - self.assertEqual(b32loads(b32.decode('utf-8')), data) |
1681 | - self.assertEqual(b32decode(b32), data) |
1682 | - |
1683 | -class ProtocolTestCase(TestCase): |
1684 | - def check_hash_leaf(self, proto): |
1685 | - |
1686 | + def test_make_key(self): |
1687 | + with self.assertRaises(TypeError) as cm: |
1688 | + protocols.make_key(1.0) |
1689 | + self.assertEqual( |
1690 | + str(cm.exception), |
1691 | + TYPE_ERROR.format('integer', int, float, 1.0) |
1692 | + ) |
1693 | + with self.assertRaises(TypeError) as cm: |
1694 | + protocols.make_key('1') |
1695 | + self.assertEqual( |
1696 | + str(cm.exception), |
1697 | + TYPE_ERROR.format('integer', int, str, '1') |
1698 | + ) |
1699 | + |
1700 | + with self.assertRaises(ValueError) as cm: |
1701 | + protocols.make_key(-1) |
1702 | + self.assertEqual( |
1703 | + str(cm.exception), |
1704 | + 'integer: must be >= 0; got -1' |
1705 | + ) |
1706 | + with self.assertRaises(ValueError) as cm: |
1707 | + protocols.make_key(-17) |
1708 | + self.assertEqual( |
1709 | + str(cm.exception), |
1710 | + 'integer: must be >= 0; got -17' |
1711 | + ) |
1712 | + |
1713 | + self.assertEqual(protocols.make_key(0), b'0') |
1714 | + self.assertEqual(protocols.make_key(1), b'1') |
1715 | + self.assertEqual(protocols.make_key(1776), b'1776') |
1716 | + for i in range(1000): |
1717 | + n = random.randint(0, 16 * GiB) |
1718 | + self.assertEqual(protocols.make_key(n), repr(n).encode('ascii')) |
1719 | + |
1720 | + |
1721 | +class BaseTestCase1(TestCase): |
1722 | + def setUp(self): |
1723 | + self._checks = [] |
1724 | + |
1725 | + def check_hash_leaf_validation(self, proto): |
1726 | # Test with wrong leaf_index type: |
1727 | with self.assertRaises(TypeError) as cm: |
1728 | proto.hash_leaf(0.0, None) |
1729 | @@ -165,63 +246,9 @@ |
1730 | ) |
1731 | ) |
1732 | |
1733 | - # Smallest possible leaf, index=0 |
1734 | - accum = set() |
1735 | - leaf_data = b'D' |
1736 | - digest = proto.hash_leaf(0, leaf_data) |
1737 | - accum.add(digest) |
1738 | - self.assertEqual(proto._hash_leaf(0, leaf_data, b''), digest) |
1739 | - |
1740 | - # Smallest possible leaf, index=1 |
1741 | - digest = proto.hash_leaf(1, leaf_data) |
1742 | - self.assertNotIn(digest, accum) |
1743 | - accum.add(digest) |
1744 | - self.assertEqual(proto._hash_leaf(1, leaf_data, b''), digest) |
1745 | - |
1746 | - # Largest possible leaf, index=0 |
1747 | - leaf_data = b'D' * proto.leaf_size |
1748 | - digest = proto.hash_leaf(0, leaf_data) |
1749 | - self.assertNotIn(digest, accum) |
1750 | - accum.add(digest) |
1751 | - self.assertEqual(proto._hash_leaf(0, leaf_data, b''), digest) |
1752 | - |
1753 | - # Largest possible leaf, index=1 |
1754 | - digest = proto.hash_leaf(1, leaf_data) |
1755 | - self.assertNotIn(digest, accum) |
1756 | - accum.add(digest) |
1757 | - self.assertEqual(proto._hash_leaf(1, leaf_data, b''), digest) |
1758 | - |
1759 | - # Test with challenge, smallest leaf, index=0: |
1760 | - challenge = os.urandom(16) |
1761 | - leaf_data = b'D' |
1762 | - digest = proto.hash_leaf(0, leaf_data, challenge) |
1763 | - self.assertNotIn(digest, accum) |
1764 | - accum.add(digest) |
1765 | - self.assertEqual(proto._hash_leaf(0, leaf_data, challenge), digest) |
1766 | - |
1767 | - # Test with challenge, smallest leaf, index=1: |
1768 | - digest = proto.hash_leaf(1, leaf_data, challenge) |
1769 | - self.assertNotIn(digest, accum) |
1770 | - accum.add(digest) |
1771 | - self.assertEqual(proto._hash_leaf(1, leaf_data, challenge), digest) |
1772 | - |
1773 | - # Test with challenge, largest leaf, index=0: |
1774 | - leaf_data = b'D' * proto.leaf_size |
1775 | - digest = proto.hash_leaf(0, leaf_data, challenge) |
1776 | - self.assertNotIn(digest, accum) |
1777 | - accum.add(digest) |
1778 | - self.assertEqual(proto._hash_leaf(0, leaf_data, challenge), digest) |
1779 | - |
1780 | - # Test with challenge, largest leaf, index=1: |
1781 | - digest = proto.hash_leaf(1, leaf_data, challenge) |
1782 | - self.assertNotIn(digest, accum) |
1783 | - accum.add(digest) |
1784 | - self.assertEqual(proto._hash_leaf(1, leaf_data, challenge), digest) |
1785 | - |
1786 | - # Make sure we didn't goof |
1787 | - self.assertEqual(len(accum), 8) |
1788 | - |
1789 | - def check_hash_root(self, proto): |
1790 | + self._checks.append('hash_leaf_validation') |
1791 | + |
1792 | + def check_hash_root_validation(self, proto): |
1793 | # Test with wrong file_size type: |
1794 | with self.assertRaises(TypeError) as cm: |
1795 | proto.hash_root(1.0, None) |
1796 | @@ -321,99 +348,10 @@ |
1797 | ) |
1798 | ) |
1799 | |
1800 | - # Test with good file_size and leaf_hashes |
1801 | - accum = set() |
1802 | - leaf_hashes = b'D' * proto.digest_bytes |
1803 | - _id = proto.hash_root(1, leaf_hashes) |
1804 | - self.assertNotIn(_id, accum) |
1805 | - accum.add(_id) |
1806 | - self.assertEqual( |
1807 | - proto._hash_root(1, leaf_hashes), |
1808 | - b32decode(_id.encode('utf-8')) |
1809 | - ) |
1810 | - |
1811 | - _id = proto.hash_root(2, leaf_hashes) |
1812 | - self.assertNotIn(_id, accum) |
1813 | - accum.add(_id) |
1814 | - self.assertEqual( |
1815 | - proto._hash_root(2, leaf_hashes), |
1816 | - b32decode(_id.encode('utf-8')) |
1817 | - ) |
1818 | - |
1819 | - leaf_hashes = b'AB' * proto.digest_bytes |
1820 | - _id = proto.hash_root(proto.leaf_size + 1, leaf_hashes) |
1821 | - self.assertNotIn(_id, accum) |
1822 | - accum.add(_id) |
1823 | - self.assertEqual( |
1824 | - proto._hash_root(proto.leaf_size + 1, leaf_hashes), |
1825 | - b32decode(_id.encode('utf-8')) |
1826 | - ) |
1827 | - |
1828 | - _id = proto.hash_root(proto.leaf_size + 2, leaf_hashes) |
1829 | - self.assertNotIn(_id, accum) |
1830 | - accum.add(_id) |
1831 | - self.assertEqual( |
1832 | - proto._hash_root(proto.leaf_size + 2, leaf_hashes), |
1833 | - b32decode(_id.encode('utf-8')) |
1834 | - ) |
1835 | - |
1836 | - # Make sure we didn't goof |
1837 | - self.assertEqual(len(accum), 4) |
1838 | - |
1839 | - def sanity_check_hash_leaf(self, proto): |
1840 | - """ |
1841 | - Random value sanity check on Protocol._hash_leaf(). |
1842 | - """ |
1843 | - |
1844 | - # Sanity check on our crytographic claim that the |
1845 | - # leaf-hash is tied to the leaf-index: |
1846 | - leaf_data = os.urandom(16) |
1847 | - accum = set( |
1848 | - proto._hash_leaf(i, leaf_data, b'') |
1849 | - for i in range(COUNT) |
1850 | - ) |
1851 | - self.assertEqual(len(accum), COUNT) |
1852 | - |
1853 | - # Sanity check on our crytographic claim that the |
1854 | - # leaf-hash is tied to the leaf-data: |
1855 | - accum = set( |
1856 | - proto._hash_leaf(21, os.urandom(16), b'') |
1857 | - for i in range(COUNT) |
1858 | - ) |
1859 | - self.assertEqual(len(accum), COUNT) |
1860 | - |
1861 | - # Sanity check on our crytographic claim that the |
1862 | - # leaf-hash is tied to the challenge: |
1863 | - accum = set( |
1864 | - proto._hash_leaf(21, leaf_data, os.urandom(16)) |
1865 | - for i in range(COUNT) |
1866 | - ) |
1867 | - self.assertEqual(len(accum), COUNT) |
1868 | - |
1869 | - def sanity_check_hash_root(self, proto): |
1870 | - """ |
1871 | - Random value sanity check on Protocol._hash_root(). |
1872 | - """ |
1873 | - |
1874 | - # Sanity check on our crytographic claim that the |
1875 | - # root-hash is tied to the file-size: |
1876 | - leaf_hashes = os.urandom(proto.digest_bytes) |
1877 | - accum = set( |
1878 | - proto._hash_root(size, leaf_hashes) |
1879 | - for size in range(1, COUNT + 1) |
1880 | - ) |
1881 | - self.assertEqual(len(accum), COUNT) |
1882 | - |
1883 | - # Sanity check on our crytographic claim that the |
1884 | - # root-hash is tied to the leaf-hashes: |
1885 | - accum = set( |
1886 | - proto._hash_root(314159, os.urandom(proto.digest_bytes)) |
1887 | - for i in range(COUNT) |
1888 | - ) |
1889 | - self.assertEqual(len(accum), COUNT) |
1890 | - |
1891 | - |
1892 | -class TestProtocol(ProtocolTestCase): |
1893 | + self._checks.append('hash_root_validation') |
1894 | + |
1895 | + |
1896 | +class TestProtocol(BaseTestCase1): |
1897 | def test_init(self): |
1898 | # Wrong `leaf_size` type: |
1899 | with self.assertRaises(TypeError) as cm: |
1900 | @@ -474,7 +412,7 @@ |
1901 | self.assertEqual(proto.digest_bytes, 25) |
1902 | self.assertEqual(proto.digest_b32len, 40) |
1903 | |
1904 | - # Test with the actual v0 protocol values: |
1905 | + # Test with the actual v1 protocol values: |
1906 | proto = protocols.Protocol(8 * MiB, 240) |
1907 | self.assertEqual(proto.leaf_size, 8 * MiB) |
1908 | self.assertEqual(proto.max_leaf_count, 1073741824) |
1909 | @@ -483,928 +421,891 @@ |
1910 | self.assertEqual(proto.digest_bytes, 30) |
1911 | self.assertEqual(proto.digest_b32len, 48) |
1912 | |
1913 | - # Test with the actual v1 protocol values: |
1914 | - proto = protocols.Protocol(8 * MiB, 280) |
1915 | - self.assertEqual(proto.leaf_size, 8 * MiB) |
1916 | - self.assertEqual(proto.max_leaf_count, 1073741824) |
1917 | - self.assertEqual(proto.max_leaf_count, 2**30) |
1918 | + # Test with possible future values: |
1919 | + proto = protocols.Protocol(16 * MiB, 280) |
1920 | + self.assertEqual(proto.leaf_size, 16 * MiB) |
1921 | + self.assertEqual(proto.max_leaf_count, 536870912) |
1922 | + self.assertEqual(proto.max_leaf_count, 2**29) |
1923 | self.assertEqual(proto.digest_bits, 280) |
1924 | self.assertEqual(proto.digest_bytes, 35) |
1925 | self.assertEqual(proto.digest_b32len, 56) |
1926 | |
1927 | - def test_encode(self): |
1928 | - proto = protocols.Protocol(TwoMiB, 200) |
1929 | - |
1930 | - # Wrong type: |
1931 | - digest = 'A' * 25 |
1932 | - with self.assertRaises(TypeError) as cm: |
1933 | - proto.encode(digest) |
1934 | - self.assertEqual( |
1935 | - str(cm.exception), |
1936 | - TYPE_ERROR.format('digest', bytes, str, digest) |
1937 | - ) |
1938 | - |
1939 | - # Wrong length: |
1940 | - digest = b'A' * 30 |
1941 | - with self.assertRaises(ValueError) as cm: |
1942 | - proto.encode(digest) |
1943 | - self.assertEqual(str(cm.exception), 'len(digest) must be 25; got 30') |
1944 | - |
1945 | - digest = os.urandom(25) |
1946 | - self.assertEqual(proto.encode(digest), b32dumps(digest)) |
1947 | - |
1948 | - def test_decode(self): |
1949 | - proto = protocols.Protocol(TwoMiB, 200) |
1950 | - |
1951 | - # Wrong type: |
1952 | - _id = b'A' * 40 |
1953 | - with self.assertRaises(TypeError) as cm: |
1954 | - proto.decode(_id) |
1955 | - self.assertEqual( |
1956 | - str(cm.exception), |
1957 | - TYPE_ERROR.format('_id', str, bytes, _id) |
1958 | - ) |
1959 | - |
1960 | - # Wrong length: |
1961 | - _id = 'A' * 48 |
1962 | - with self.assertRaises(ValueError) as cm: |
1963 | - proto.decode(_id) |
1964 | - self.assertEqual(str(cm.exception), 'len(_id) must be 40; got 48') |
1965 | - |
1966 | - digest = os.urandom(25) |
1967 | - _id = b32dumps(digest) |
1968 | - self.assertEqual(proto.decode(_id), digest) |
1969 | - |
1970 | - # Test round-trip: |
1971 | - for i in range(100): |
1972 | - digest = os.urandom(25) |
1973 | - self.assertEqual(proto.decode(proto.encode(digest)), digest) |
1974 | - |
1975 | - def test_hash_leaf(self): |
1976 | - proto = protocols.Protocol(TwoMiB, 200) |
1977 | + def test_hash_leaf(self): |
1978 | + proto = protocols.Protocol(TwoMiB, 200) |
1979 | + self.check_hash_leaf_validation(proto) |
1980 | + with self.assertRaises(NotImplementedError) as cm: |
1981 | + proto.hash_leaf(0, b'data') |
1982 | + self.assertEqual( |
1983 | + str(cm.exception), |
1984 | + 'Protocol._hash_leaf(key_leaf_index, leaf_data, challenge)' |
1985 | + ) |
1986 | + with self.assertRaises(NotImplementedError) as cm: |
1987 | + proto.hash_leaf(0, b'data', b'secret') |
1988 | + self.assertEqual( |
1989 | + str(cm.exception), |
1990 | + 'Protocol._hash_leaf(key_leaf_index, leaf_data, challenge)' |
1991 | + ) |
1992 | + with self.assertRaises(NotImplementedError) as cm: |
1993 | + proto._hash_leaf(0, b'data', b'secret') |
1994 | + self.assertEqual( |
1995 | + str(cm.exception), |
1996 | + 'Protocol._hash_leaf(key_leaf_index, leaf_data, challenge)' |
1997 | + ) |
1998 | + |
1999 | + class Example(protocols.Protocol): |
2000 | + pass |
2001 | + |
2002 | + proto = Example(TwoMiB, 200) |
2003 | + self.check_hash_leaf_validation(proto) |
2004 | + with self.assertRaises(NotImplementedError) as cm: |
2005 | + proto.hash_leaf(0, b'data') |
2006 | + self.assertEqual( |
2007 | + str(cm.exception), |
2008 | + 'Example._hash_leaf(key_leaf_index, leaf_data, challenge)' |
2009 | + ) |
2010 | + with self.assertRaises(NotImplementedError) as cm: |
2011 | + proto.hash_leaf(0, b'data', b'secret') |
2012 | + self.assertEqual( |
2013 | + str(cm.exception), |
2014 | + 'Example._hash_leaf(key_leaf_index, leaf_data, challenge)' |
2015 | + ) |
2016 | + with self.assertRaises(NotImplementedError) as cm: |
2017 | + proto._hash_leaf(0, b'data', b'secret') |
2018 | + self.assertEqual( |
2019 | + str(cm.exception), |
2020 | + 'Example._hash_leaf(key_leaf_index, leaf_data, challenge)' |
2021 | + ) |
2022 | + |
2023 | + class FooProtocol(protocols.Protocol): |
2024 | + def _hash_leaf(self, *args): |
2025 | + assert not hasattr(self, '_call') |
2026 | + assert not hasattr(self, '_ret') |
2027 | + self._call = args |
2028 | + self._ret = os.urandom(self.digest_bytes) |
2029 | + return self._ret |
2030 | + |
2031 | + foo = FooProtocol(TwoMiB, 200) |
2032 | + self.check_hash_leaf_validation(foo) |
2033 | + digest = foo.hash_leaf(0, b'data') |
2034 | + self.assertEqual(foo._call, (b'0', b'data', b'')) |
2035 | + self.assertIs(digest, foo._ret) |
2036 | + |
2037 | + foo = FooProtocol(TwoMiB, 200) |
2038 | + digest = foo.hash_leaf(17, b'more data', b'secret') |
2039 | + self.assertEqual(foo._call, (b'17', b'more data', b'secret')) |
2040 | + self.assertIs(digest, foo._ret) |
2041 | + |
2042 | + for i in range(100): |
2043 | + index = random.randint(0, MiB) |
2044 | + length = random.randint(1, TwoMiB) |
2045 | + data = os.urandom(1) * length |
2046 | + |
2047 | + foo = FooProtocol(TwoMiB, 200) |
2048 | + digest = foo.hash_leaf(index, data) |
2049 | + self.assertEqual(foo._call, |
2050 | + (str(index).encode('utf-8'), data, b'') |
2051 | + ) |
2052 | + self.assertIs(digest, foo._ret) |
2053 | + |
2054 | + nonce = os.urandom(16) |
2055 | + foo = FooProtocol(TwoMiB, 200) |
2056 | + digest = foo.hash_leaf(index, data, nonce) |
2057 | + self.assertEqual(foo._call, |
2058 | + (str(index).encode('utf-8'), data, nonce) |
2059 | + ) |
2060 | + self.assertIs(digest, foo._ret) |
2061 | + |
2062 | + def test_hash_root(self): |
2063 | + proto = protocols.Protocol(TwoMiB, 200) |
2064 | + self.check_hash_root_validation(proto) |
2065 | + with self.assertRaises(NotImplementedError) as cm: |
2066 | + proto.hash_root(1, b'A' * 25) |
2067 | + self.assertEqual( |
2068 | + str(cm.exception), |
2069 | + 'Protocol._hash_root(key_file_size, leaf_hashes)' |
2070 | + ) |
2071 | + with self.assertRaises(NotImplementedError) as cm: |
2072 | + proto._hash_root(1, b'A' * 25) |
2073 | + self.assertEqual( |
2074 | + str(cm.exception), |
2075 | + 'Protocol._hash_root(key_file_size, leaf_hashes)' |
2076 | + ) |
2077 | + |
2078 | + class Example(protocols.Protocol): |
2079 | + pass |
2080 | + |
2081 | + proto = Example(TwoMiB, 200) |
2082 | + self.check_hash_root_validation(proto) |
2083 | + with self.assertRaises(NotImplementedError) as cm: |
2084 | + proto.hash_root(1, b'A' * 25) |
2085 | + self.assertEqual( |
2086 | + str(cm.exception), |
2087 | + 'Example._hash_root(key_file_size, leaf_hashes)' |
2088 | + ) |
2089 | + with self.assertRaises(NotImplementedError) as cm: |
2090 | + proto._hash_root(1, b'A' * 25) |
2091 | + self.assertEqual( |
2092 | + str(cm.exception), |
2093 | + 'Example._hash_root(key_file_size, leaf_hashes)' |
2094 | + ) |
2095 | + |
2096 | + class FooProtocol(protocols.Protocol): |
2097 | + def _hash_root(self, *args): |
2098 | + assert not hasattr(self, '_call') |
2099 | + assert not hasattr(self, '_ret') |
2100 | + self._call = args |
2101 | + self._ret = os.urandom(self.digest_bytes) |
2102 | + return self._ret |
2103 | + |
2104 | + foo = FooProtocol(TwoMiB, 200) |
2105 | + self.check_hash_root_validation(foo) |
2106 | + digest = foo.hash_root(1, b'A' * 25) |
2107 | + self.assertEqual(foo._call, (b'1', b'A' * 25)) |
2108 | + self.assertIs(digest, foo._ret) |
2109 | + |
2110 | + for i in range(100): |
2111 | + size = random.randint(1, GiB) |
2112 | + count = size // TwoMiB |
2113 | + if size % TwoMiB: |
2114 | + count += 1 |
2115 | + data = os.urandom(25) * count |
2116 | + |
2117 | + foo = FooProtocol(TwoMiB, 200) |
2118 | + digest = foo.hash_root(size, data) |
2119 | + self.assertEqual(foo._call, |
2120 | + (str(size).encode('utf-8'), data) |
2121 | + ) |
2122 | + self.assertIs(digest, foo._ret) |
2123 | + |
2124 | + |
2125 | +class BaseTestCase2(BaseTestCase1): |
2126 | + def check_hash_leaf_simple(self, proto): |
2127 | + accum = DigestAccum() |
2128 | + small = b'D' # Smallest possible leaf |
2129 | + large = b'D' * proto.leaf_size # Largest possible leaf |
2130 | + |
2131 | + # Smallest leaf, index=0 |
2132 | + digest = accum.add(proto.hash_leaf(0, small)) |
2133 | + self.assertEqual(proto._hash_leaf(b'0', small, b''), digest) |
2134 | + |
2135 | + # Smallest leaf, index=1 |
2136 | + digest = accum.add(proto.hash_leaf(1, small)) |
2137 | + self.assertEqual(proto._hash_leaf(b'1', small, b''), digest) |
2138 | + |
2139 | + # Largest leaf, index=0 |
2140 | + digest = accum.add(proto.hash_leaf(0, large)) |
2141 | + self.assertEqual(proto._hash_leaf(b'0', large, b''), digest) |
2142 | + |
2143 | + # Largest leaf, index=1 |
2144 | + digest = accum.add(proto.hash_leaf(1, large)) |
2145 | + self.assertEqual(proto._hash_leaf(b'1', large, b''), digest) |
2146 | + |
2147 | + # Test with challenge, smallest leaf, index=0: |
2148 | + digest = accum.add(proto.hash_leaf(0, small, b'secret')) |
2149 | + self.assertEqual(proto._hash_leaf(b'0', small, b'secret'), digest) |
2150 | + |
2151 | + # Test with challenge, smallest leaf, index=1: |
2152 | + digest = accum.add(proto.hash_leaf(1, small, b'secret')) |
2153 | + self.assertEqual(proto._hash_leaf(b'1', small, b'secret'), digest) |
2154 | + |
2155 | + # Test with challenge, largest leaf, index=0: |
2156 | + digest = accum.add(proto.hash_leaf(0, large, b'secret')) |
2157 | + self.assertEqual(proto._hash_leaf(b'0', large, b'secret'), digest) |
2158 | + |
2159 | + # Test with challenge, largest leaf, index=1: |
2160 | + digest = accum.add(proto.hash_leaf(1, large, b'secret')) |
2161 | + self.assertEqual(proto._hash_leaf(b'1', large, b'secret'), digest) |
2162 | + |
2163 | + # Make sure we didn't goof |
2164 | + self.assertEqual(len(accum), 8) |
2165 | + for digest in accum: |
2166 | + self.assertEqual(len(digest), proto.digest_bytes) |
2167 | + safety = set() |
2168 | + for index in [0, 1]: |
2169 | + for data in [small, large]: |
2170 | + for challenge in [b'', b'secret']: |
2171 | + safety.add(proto.hash_leaf(index, data, challenge)) |
2172 | + self.assertEqual(safety, accum) |
2173 | + |
2174 | + self._checks.append('hash_leaf_simple') |
2175 | + |
2176 | + def check_hash_root_simple(self, proto): |
2177 | + accum = DigestAccum() |
2178 | + one = b'D' * proto.digest_bytes |
2179 | + two = b'AB' * proto.digest_bytes |
2180 | + |
2181 | + digest = accum.add(proto.hash_root(1, one)) |
2182 | + self.assertEqual(proto._hash_root(b'1', one), digest) |
2183 | + |
2184 | + digest = accum.add(proto.hash_root(2, one)) |
2185 | + self.assertEqual(proto._hash_root(b'2', one), digest) |
2186 | + |
2187 | + size = proto.leaf_size + 1 |
2188 | + digest = accum.add(proto.hash_root(size, two)) |
2189 | + self.assertEqual( |
2190 | + proto._hash_root(str(size).encode('utf-8'), two), |
2191 | + digest |
2192 | + ) |
2193 | + |
2194 | + size = proto.leaf_size + 2 |
2195 | + digest = accum.add(proto.hash_root(size, two)) |
2196 | + self.assertEqual( |
2197 | + proto._hash_root(str(size).encode('utf-8'), two), |
2198 | + digest |
2199 | + ) |
2200 | + |
2201 | + size = proto.leaf_size * 2 |
2202 | + digest = accum.add(proto.hash_root(size, two)) |
2203 | + self.assertEqual( |
2204 | + proto._hash_root(str(size).encode('utf-8'), two), |
2205 | + digest |
2206 | + ) |
2207 | + |
2208 | + # Make sure we didn't goof |
2209 | + self.assertEqual(len(accum), 5) |
2210 | + for digest in accum: |
2211 | + self.assertEqual(len(digest), proto.digest_bytes) |
2212 | + safety = set([ |
2213 | + proto.hash_root(1, one), |
2214 | + proto.hash_root(2, one), |
2215 | + proto.hash_root(proto.leaf_size + 1, two), |
2216 | + proto.hash_root(proto.leaf_size + 2, two), |
2217 | + proto.hash_root(proto.leaf_size * 2, two), |
2218 | + ]) |
2219 | + self.assertEqual(safety, accum) |
2220 | + |
2221 | + self._checks.append('hash_root_simple') |
2222 | + |
2223 | + def check_hash_leaf_crypto(self, proto): |
2224 | + """ |
2225 | + Random value sanity check on Protocol.hash_leaf(). |
2226 | + """ |
2227 | + |
2228 | + # Sanity check on our cryptographic claim that the |
2229 | + # leaf-hash is tied to the leaf-index: |
2230 | + leaf_data = os.urandom(16) |
2231 | + accum = set( |
2232 | + proto.hash_leaf(i, leaf_data, b'') |
2233 | + for i in range(COUNT) |
2234 | + ) |
2235 | + self.assertEqual(len(accum), COUNT) |
2236 | + |
2237 | + # Sanity check on our cryptographic claim that the |
2238 | + # leaf-hash is tied to the leaf-data: |
2239 | + accum = set( |
2240 | + proto.hash_leaf(21, os.urandom(16), b'') |
2241 | + for i in range(COUNT) |
2242 | + ) |
2243 | + self.assertEqual(len(accum), COUNT) |
2244 | + |
2245 | + # Sanity check on our cryptographic claim that the |
2246 | + # leaf-hash is tied to the challenge: |
2247 | + accum = set( |
2248 | + proto.hash_leaf(21, leaf_data, os.urandom(16)) |
2249 | + for i in range(COUNT) |
2250 | + ) |
2251 | + self.assertEqual(len(accum), COUNT) |
2252 | + |
2253 | + self._checks.append('hash_leaf_crypto') |
2254 | + |
2255 | + def check_hash_root_crypto(self, proto): |
2256 | + """ |
2257 | + Random value sanity check on Protocol.hash_root(). |
2258 | + """ |
2259 | + |
2260 | + # Sanity check on our cryptographic claim that the |
2261 | + # root-hash is tied to the file-size: |
2262 | + leaf_hashes = os.urandom(proto.digest_bytes) |
2263 | + accum = set( |
2264 | + proto.hash_root(size, leaf_hashes) |
2265 | + for size in range(1, COUNT + 1) |
2266 | + ) |
2267 | + self.assertEqual(len(accum), COUNT) |
2268 | + |
2269 | + # Sanity check on our cryptographic claim that the |
2270 | + # root-hash is tied to the leaf-hashes: |
2271 | + accum = set( |
2272 | + proto.hash_root(314159, os.urandom(proto.digest_bytes)) |
2273 | + for i in range(COUNT) |
2274 | + ) |
2275 | + self.assertEqual(len(accum), COUNT) |
2276 | + |
2277 | + self._checks.append('hash_root_crypto') |
2278 | + |
2279 | + def check_hash_leaf(self, proto): |
2280 | + self.check_hash_leaf_validation(proto) |
2281 | + self.check_hash_leaf_simple(proto) |
2282 | + self.check_hash_leaf_crypto(proto) |
2283 | + |
2284 | + def check_hash_root(self, proto): |
2285 | + self.check_hash_root_validation(proto) |
2286 | + self.check_hash_root_simple(proto) |
2287 | + self.check_hash_root_crypto(proto) |
2288 | + |
2289 | + |
2290 | +class TestSkeinProtocol(BaseTestCase2): |
2291 | + def test_hash_leaf(self): |
2292 | + proto = protocols.SkeinProtocol(TwoMiB, 200) |
2293 | self.check_hash_leaf(proto) |
2294 | - self.sanity_check_hash_leaf(proto) |
2295 | - |
2296 | - # Smallest possible leaf, index=0 |
2297 | - accum = set() |
2298 | - leaf_data = b'D' |
2299 | - digest = proto.hash_leaf(0, leaf_data) |
2300 | - accum.add(digest) |
2301 | - self.assertEqual( |
2302 | - digest, |
2303 | - skein512(leaf_data, |
2304 | - digest_bits=200, |
2305 | - pers=protocols.PERS_LEAF, |
2306 | - key=b'0', |
2307 | - ).digest() |
2308 | - ) |
2309 | - self.assertEqual(proto._hash_leaf(0, leaf_data, b''), digest) |
2310 | - |
2311 | - # Smallest possible leaf, index=1 |
2312 | - digest = proto.hash_leaf(1, leaf_data) |
2313 | - self.assertNotIn(digest, accum) |
2314 | - accum.add(digest) |
2315 | - self.assertEqual( |
2316 | - digest, |
2317 | - skein512(leaf_data, |
2318 | - digest_bits=200, |
2319 | - pers=protocols.PERS_LEAF, |
2320 | - key=b'1', |
2321 | - ).digest() |
2322 | - ) |
2323 | - self.assertEqual(proto._hash_leaf(1, leaf_data, b''), digest) |
2324 | - |
2325 | - # Largest possible leaf, index=0 |
2326 | - leaf_data = b'D' * TwoMiB |
2327 | - digest = proto.hash_leaf(0, leaf_data) |
2328 | - self.assertNotIn(digest, accum) |
2329 | - accum.add(digest) |
2330 | - self.assertEqual( |
2331 | - digest, |
2332 | - skein512(leaf_data, |
2333 | - digest_bits=200, |
2334 | - pers=protocols.PERS_LEAF, |
2335 | - key=b'0', |
2336 | - ).digest() |
2337 | - ) |
2338 | - self.assertEqual(proto._hash_leaf(0, leaf_data, b''), digest) |
2339 | - |
2340 | - # Largest possible leaf, index=1 |
2341 | - digest = proto.hash_leaf(1, leaf_data) |
2342 | - self.assertNotIn(digest, accum) |
2343 | - accum.add(digest) |
2344 | - self.assertEqual( |
2345 | - digest, |
2346 | - skein512(leaf_data, |
2347 | - digest_bits=200, |
2348 | - pers=protocols.PERS_LEAF, |
2349 | - key=b'1', |
2350 | - ).digest() |
2351 | - ) |
2352 | - self.assertEqual(proto._hash_leaf(1, leaf_data, b''), digest) |
2353 | + self.assertEqual(self._checks, [ |
2354 | + 'hash_leaf_validation', |
2355 | + 'hash_leaf_simple', |
2356 | + 'hash_leaf_crypto', |
2357 | + ]) |
2358 | + |
2359 | + accum = DigestAccum() |
2360 | + small = b'A' # Smallest possible leaf |
2361 | + large = b'B' * proto.leaf_size # Largest possible leaf |
2362 | + |
2363 | + # Smallest leaf, index=0 |
2364 | + digest = accum.add(proto.hash_leaf(0, small)) |
2365 | + self.assertEqual(digest, |
2366 | + skein512(small, |
2367 | + digest_bits=200, |
2368 | + pers=protocols.PERS_LEAF, |
2369 | + key=b'0', |
2370 | + ).digest() |
2371 | + ) |
2372 | + self.assertEqual(proto._hash_leaf(b'0', small, b''), digest) |
2373 | + |
2374 | + # Smallest leaf, index=1 |
2375 | + digest = accum.add(proto.hash_leaf(1, small)) |
2376 | + self.assertEqual(digest, |
2377 | + skein512(small, |
2378 | + digest_bits=200, |
2379 | + pers=protocols.PERS_LEAF, |
2380 | + key=b'1', |
2381 | + ).digest() |
2382 | + ) |
2383 | + self.assertEqual(proto._hash_leaf(b'1', small, b''), digest) |
2384 | + |
2385 | + # Largest leaf, index=0 |
2386 | + digest = accum.add(proto.hash_leaf(0, large)) |
2387 | + self.assertEqual(digest, |
2388 | + skein512(large, |
2389 | + digest_bits=200, |
2390 | + pers=protocols.PERS_LEAF, |
2391 | + key=b'0', |
2392 | + ).digest() |
2393 | + ) |
2394 | + self.assertEqual(proto._hash_leaf(b'0', large, b''), digest) |
2395 | + |
2396 | + # Largest leaf, index=1 |
2397 | + digest = accum.add(proto.hash_leaf(1, large)) |
2398 | + self.assertEqual(digest, |
2399 | + skein512(large, |
2400 | + digest_bits=200, |
2401 | + pers=protocols.PERS_LEAF, |
2402 | + key=b'1', |
2403 | + ).digest() |
2404 | + ) |
2405 | + self.assertEqual(proto._hash_leaf(b'1', large, b''), digest) |
2406 | |
2407 | # Test with challenge, smallest leaf, index=0: |
2408 | - challenge = os.urandom(16) |
2409 | - leaf_data = b'D' |
2410 | - digest = proto.hash_leaf(0, leaf_data, challenge) |
2411 | - self.assertNotIn(digest, accum) |
2412 | - accum.add(digest) |
2413 | - self.assertEqual( |
2414 | - digest, |
2415 | - skein512(leaf_data, |
2416 | + digest = accum.add(proto.hash_leaf(0, small, b'secret')) |
2417 | + self.assertEqual(digest, |
2418 | + skein512(small, |
2419 | digest_bits=200, |
2420 | pers=protocols.PERS_LEAF, |
2421 | key=b'0', |
2422 | - nonce=challenge, |
2423 | + nonce=b'secret', |
2424 | ).digest() |
2425 | ) |
2426 | - self.assertEqual(proto._hash_leaf(0, leaf_data, challenge), digest) |
2427 | + self.assertEqual(proto._hash_leaf(b'0', small, b'secret'), digest) |
2428 | |
2429 | # Test with challenge, smallest leaf, index=1: |
2430 | - digest = proto.hash_leaf(1, leaf_data, challenge) |
2431 | - self.assertNotIn(digest, accum) |
2432 | - accum.add(digest) |
2433 | - self.assertEqual( |
2434 | - digest, |
2435 | - skein512(leaf_data, |
2436 | + digest = accum.add(proto.hash_leaf(1, small, b'secret')) |
2437 | + self.assertEqual(digest, |
2438 | + skein512(small, |
2439 | digest_bits=200, |
2440 | pers=protocols.PERS_LEAF, |
2441 | key=b'1', |
2442 | - nonce=challenge, |
2443 | + nonce=b'secret', |
2444 | ).digest() |
2445 | ) |
2446 | - self.assertEqual(proto._hash_leaf(1, leaf_data, challenge), digest) |
2447 | + self.assertEqual(proto._hash_leaf(b'1', small, b'secret'), digest) |
2448 | |
2449 | # Test with challenge, largest leaf, index=0: |
2450 | - leaf_data = b'D' * TwoMiB |
2451 | - digest = proto.hash_leaf(0, leaf_data, challenge) |
2452 | - self.assertNotIn(digest, accum) |
2453 | - accum.add(digest) |
2454 | - self.assertEqual( |
2455 | - digest, |
2456 | - skein512(leaf_data, |
2457 | + digest = accum.add(proto.hash_leaf(0, large, b'secret')) |
2458 | + self.assertEqual(digest, |
2459 | + skein512(large, |
2460 | digest_bits=200, |
2461 | pers=protocols.PERS_LEAF, |
2462 | key=b'0', |
2463 | - nonce=challenge, |
2464 | + nonce=b'secret', |
2465 | ).digest() |
2466 | ) |
2467 | - self.assertEqual(proto._hash_leaf(0, leaf_data, challenge), digest) |
2468 | + self.assertEqual(proto._hash_leaf(b'0', large, b'secret'), digest) |
2469 | |
2470 | # Test with challenge, largest leaf, index=1: |
2471 | - digest = proto.hash_leaf(1, leaf_data, challenge) |
2472 | - self.assertNotIn(digest, accum) |
2473 | - accum.add(digest) |
2474 | - self.assertEqual( |
2475 | - digest, |
2476 | - skein512(leaf_data, |
2477 | + digest = accum.add(proto.hash_leaf(1, large, b'secret')) |
2478 | + self.assertEqual(digest, |
2479 | + skein512(large, |
2480 | digest_bits=200, |
2481 | pers=protocols.PERS_LEAF, |
2482 | key=b'1', |
2483 | - nonce=challenge, |
2484 | + nonce=b'secret', |
2485 | ).digest() |
2486 | ) |
2487 | - self.assertEqual(proto._hash_leaf(1, leaf_data, challenge), digest) |
2488 | + self.assertEqual(proto._hash_leaf(b'1', large, b'secret'), digest) |
2489 | |
2490 | # Make sure we didn't goof |
2491 | + safety = set() |
2492 | + for index in [0, 1]: |
2493 | + for data in [small, large]: |
2494 | + for challenge in [b'', b'secret']: |
2495 | + safety.add(proto.hash_leaf(index, data, challenge)) |
2496 | + self.assertEqual(safety, accum) |
2497 | self.assertEqual(len(accum), 8) |
2498 | + for digest in accum: |
2499 | + self.assertEqual(len(digest), 25) |
2500 | |
2501 | def test_hash_root(self): |
2502 | - proto = protocols.Protocol(TwoMiB, 200) |
2503 | + proto = protocols.SkeinProtocol(TwoMiB, 200) |
2504 | self.check_hash_root(proto) |
2505 | - self.sanity_check_hash_root(proto) |
2506 | + self.assertEqual(self._checks, [ |
2507 | + 'hash_root_validation', |
2508 | + 'hash_root_simple', |
2509 | + 'hash_root_crypto', |
2510 | + ]) |
2511 | |
2512 | - accum = set() |
2513 | + accum = DigestAccum() |
2514 | + one = b'D' * 25 |
2515 | + two = b'D' * 50 |
2516 | |
2517 | # Smallest file-size possible for one leaf: |
2518 | - leaf_hashes = b'D' * 25 |
2519 | - digest = proto._hash_root(1, leaf_hashes) |
2520 | - self.assertNotIn(digest, accum) |
2521 | - accum.add(digest) |
2522 | - self.assertEqual( |
2523 | - digest, |
2524 | - skein512(leaf_hashes, |
2525 | + digest = accum.add(proto.hash_root(1, one)) |
2526 | + self.assertEqual(digest, |
2527 | + skein512(one, |
2528 | digest_bits=200, |
2529 | pers=protocols.PERS_ROOT, |
2530 | key=b'1', |
2531 | ).digest() |
2532 | ) |
2533 | - self.assertEqual( |
2534 | - proto.hash_root(1, leaf_hashes), |
2535 | - b32encode(digest).decode('utf-8') |
2536 | - ) |
2537 | + self.assertEqual(proto._hash_root(b'1', one), digest) |
2538 | |
2539 | # Largest file-size possible for one leaf: |
2540 | - digest = proto._hash_root(TwoMiB, leaf_hashes) |
2541 | - self.assertNotIn(digest, accum) |
2542 | - accum.add(digest) |
2543 | - self.assertEqual( |
2544 | - digest, |
2545 | - skein512(leaf_hashes, |
2546 | + digest = accum.add(proto.hash_root(TwoMiB, one)) |
2547 | + self.assertEqual(digest, |
2548 | + skein512(one, |
2549 | digest_bits=200, |
2550 | pers=protocols.PERS_ROOT, |
2551 | key=b'2097152', |
2552 | ).digest() |
2553 | ) |
2554 | - self.assertEqual( |
2555 | - proto.hash_root(TwoMiB, leaf_hashes), |
2556 | - b32encode(digest).decode('utf-8') |
2557 | - ) |
2558 | + self.assertEqual(proto._hash_root(b'2097152', one), digest) |
2559 | |
2560 | # Smallest file-size possible for two leaves: |
2561 | - leaf_hashes = b'D' * 50 |
2562 | - digest = proto._hash_root(TwoMiB + 1, leaf_hashes) |
2563 | - self.assertNotIn(digest, accum) |
2564 | - accum.add(digest) |
2565 | - self.assertEqual( |
2566 | - digest, |
2567 | - skein512(leaf_hashes, |
2568 | + digest = accum.add(proto.hash_root(TwoMiB + 1, two)) |
2569 | + self.assertEqual(digest, |
2570 | + skein512(two, |
2571 | digest_bits=200, |
2572 | pers=protocols.PERS_ROOT, |
2573 | key=b'2097153', |
2574 | ).digest() |
2575 | ) |
2576 | - self.assertEqual( |
2577 | - proto.hash_root(TwoMiB + 1, leaf_hashes), |
2578 | - b32encode(digest).decode('utf-8') |
2579 | - ) |
2580 | + self.assertEqual(proto._hash_root(b'2097153', two), digest) |
2581 | |
2582 | # Largest file-size possible for two leaves: |
2583 | - digest = proto._hash_root(2 * TwoMiB, leaf_hashes) |
2584 | - self.assertNotIn(digest, accum) |
2585 | - accum.add(digest) |
2586 | - self.assertEqual( |
2587 | - digest, |
2588 | - skein512(leaf_hashes, |
2589 | + digest = accum.add(proto.hash_root(2 * TwoMiB, two)) |
2590 | + self.assertEqual(digest, |
2591 | + skein512(two, |
2592 | digest_bits=200, |
2593 | pers=protocols.PERS_ROOT, |
2594 | key=b'4194304', |
2595 | ).digest() |
2596 | ) |
2597 | - self.assertEqual( |
2598 | - proto.hash_root(2 * TwoMiB, leaf_hashes), |
2599 | - b32encode(digest).decode('utf-8') |
2600 | - ) |
2601 | + self.assertEqual(proto._hash_root(b'4194304', two), digest) |
2602 | |
2603 | # Make sure we didn't goof |
2604 | self.assertEqual(len(accum), 4) |
2605 | - |
2606 | - |
2607 | -class TestVersionOne(ProtocolTestCase): |
2608 | - |
2609 | - def test_md5(self): |
2610 | - # To aid debugging, md5sums of UTF-8 strings needed by test vectors |
2611 | - self.assertEqual( |
2612 | - md5(protocols.PERS_LEAF).hexdigest(), |
2613 | - '8aee35e23a5a74147b230f12123ca82e' |
2614 | - ) |
2615 | - self.assertEqual( |
2616 | - md5(protocols.PERS_ROOT).hexdigest(), |
2617 | - '0445d91d37383f5384023d49e71cc629' |
2618 | - ) |
2619 | - self.assertEqual( |
2620 | - md5(str(0).encode('utf-8')).hexdigest(), |
2621 | - 'cfcd208495d565ef66e7dff9f98764da' |
2622 | - ) |
2623 | - self.assertEqual( |
2624 | - md5(str(1).encode('utf-8')).hexdigest(), |
2625 | - 'c4ca4238a0b923820dcc509a6f75849b' |
2626 | - ) |
2627 | - self.assertEqual( |
2628 | - md5(str(8388607).encode('utf-8')).hexdigest(), |
2629 | - '32433904a755e2b9eb82cf167723b34f' |
2630 | - ) |
2631 | - self.assertEqual( |
2632 | - md5(str(8388608).encode('utf-8')).hexdigest(), |
2633 | - '03926fda4e223707d290ac06bb996653' |
2634 | - ) |
2635 | - self.assertEqual( |
2636 | - md5(str(8388609).encode('utf-8')).hexdigest(), |
2637 | - 'e9b74719ce6b80c5337148d12725db03' |
2638 | - ) |
2639 | - self.assertEqual( |
2640 | - md5(str(16777215).encode('utf-8')).hexdigest(), |
2641 | - '48ac8929ffdc78a66090d179ff1237d5' |
2642 | - ) |
2643 | - self.assertEqual( |
2644 | - md5(str(16777216).encode('utf-8')).hexdigest(), |
2645 | - 'e3e330499348f791337e9da6b534a386' |
2646 | - ) |
2647 | - |
2648 | - # To aid debugging, md5sums of the test vector files: |
2649 | - self.assertEqual( |
2650 | - md5(A).hexdigest(), |
2651 | - '7fc56270e7a70fa81a5935b72eacbe29' |
2652 | - ) |
2653 | - self.assertEqual( |
2654 | - md5(B).hexdigest(), |
2655 | - 'd2bad3eedb424dd352d65eafbf6c79ba' |
2656 | - ) |
2657 | - self.assertEqual( |
2658 | - md5(C).hexdigest(), |
2659 | - '5dd3531303dd6764acb93e5f171a4ab8' |
2660 | - ) |
2661 | - self.assertEqual( |
2662 | - md5(C + A).hexdigest(), |
2663 | - '0722f8dc36d75acb602dcee8d0427ce0' |
2664 | - ) |
2665 | - self.assertEqual( |
2666 | - md5(C + B).hexdigest(), |
2667 | - '77264eb6eed7777a1ee03e2601fc9f64' |
2668 | - ) |
2669 | - self.assertEqual( |
2670 | - md5(C + C).hexdigest(), |
2671 | - '1fbfabdaafff31967f9a95f3a3d3c642' |
2672 | - ) |
2673 | - |
2674 | + safety = set([ |
2675 | + proto.hash_root(1, one), |
2676 | + proto.hash_root(TwoMiB, one), |
2677 | + proto.hash_root(TwoMiB + 1, two), |
2678 | + proto.hash_root(TwoMiB * 2, two), |
2679 | + ]) |
2680 | + self.assertEqual(safety, accum) |
2681 | + for digest in accum: |
2682 | + self.assertEqual(len(digest), 25) |
2683 | + |
2684 | + |
2685 | +class TestOldSkeinProtocol(BaseTestCase2): |
2686 | def test_hash_leaf(self): |
2687 | - proto = protocols.VERSION1 |
2688 | - |
2689 | + proto = protocols.OldSkeinProtocol(TwoMiB, 200) |
2690 | self.check_hash_leaf(proto) |
2691 | - self.sanity_check_hash_leaf(proto) |
2692 | - |
2693 | - # A0 |
2694 | - digest = proto._hash_leaf(0, A, b'') |
2695 | - self.assertEqual(digest, b32loads(A0_B32)) |
2696 | - self.assertEqual( |
2697 | - digest, |
2698 | - skein512(A, |
2699 | - digest_bits=280, |
2700 | - pers=protocols.PERS_LEAF, |
2701 | - key=b'0', |
2702 | - ).digest() |
2703 | - ) |
2704 | - self.assertEqual(proto.hash_leaf(0, A), digest) |
2705 | - |
2706 | - # A1 |
2707 | - digest = proto._hash_leaf(1, A, b'') |
2708 | - self.assertEqual(digest, b32loads(A1_B32)) |
2709 | - self.assertEqual( |
2710 | - digest, |
2711 | - skein512(A, |
2712 | - digest_bits=280, |
2713 | - pers=protocols.PERS_LEAF, |
2714 | - key=b'1', |
2715 | - ).digest() |
2716 | - ) |
2717 | - self.assertEqual(proto.hash_leaf(1, A), digest) |
2718 | - |
2719 | - # B0 |
2720 | - digest = proto._hash_leaf(0, B, b'') |
2721 | - self.assertEqual(digest, b32loads(B0_B32)) |
2722 | - self.assertEqual( |
2723 | - digest, |
2724 | - skein512(B, |
2725 | - digest_bits=280, |
2726 | - pers=protocols.PERS_LEAF, |
2727 | - key=b'0', |
2728 | - ).digest() |
2729 | - ) |
2730 | - self.assertEqual(proto.hash_leaf(0, B), digest) |
2731 | - |
2732 | - # B1 |
2733 | - digest = proto._hash_leaf(1, B, b'') |
2734 | - self.assertEqual(digest, b32loads(B1_B32)) |
2735 | - self.assertEqual( |
2736 | - digest, |
2737 | - skein512(B, |
2738 | - digest_bits=280, |
2739 | - pers=protocols.PERS_LEAF, |
2740 | - key=b'1', |
2741 | - ).digest() |
2742 | - ) |
2743 | - self.assertEqual(proto.hash_leaf(1, B), digest) |
2744 | - |
2745 | - # C0 |
2746 | - digest = proto._hash_leaf(0, C, b'') |
2747 | - self.assertEqual(digest, b32loads(C0_B32)) |
2748 | - self.assertEqual( |
2749 | - digest, |
2750 | - skein512(C, |
2751 | - digest_bits=280, |
2752 | - pers=protocols.PERS_LEAF, |
2753 | - key=b'0', |
2754 | - ).digest() |
2755 | - ) |
2756 | - self.assertEqual(proto.hash_leaf(0, C), digest) |
2757 | - |
2758 | - # C1 |
2759 | - digest = proto._hash_leaf(1, C, b'') |
2760 | - self.assertEqual(digest, b32loads(C1_B32)) |
2761 | - self.assertEqual( |
2762 | - digest, |
2763 | - skein512(C, |
2764 | - digest_bits=280, |
2765 | - pers=protocols.PERS_LEAF, |
2766 | - key=b'1', |
2767 | - ).digest() |
2768 | - ) |
2769 | - self.assertEqual(proto.hash_leaf(1, C), digest) |
2770 | + self.assertEqual(self._checks, [ |
2771 | + 'hash_leaf_validation', |
2772 | + 'hash_leaf_simple', |
2773 | + 'hash_leaf_crypto', |
2774 | + ]) |
2775 | + |
2776 | + accum = DigestAccum() |
2777 | + small = b'A' # Smallest possible leaf |
2778 | + large = b'B' * proto.leaf_size # Largest possible leaf |
2779 | + key0 = proto._hash_leaf_index(b'0') |
2780 | + key1 = proto._hash_leaf_index(b'1') |
2781 | + |
2782 | + # Smallest leaf, index=0 |
2783 | + digest = accum.add(proto.hash_leaf(0, small)) |
2784 | + self.assertEqual(digest, |
2785 | + skein512(key0 + small, |
2786 | + digest_bits=200, |
2787 | + pers=protocols.PERS_LEAF, |
2788 | + ).digest() |
2789 | + ) |
2790 | + self.assertEqual(proto._hash_leaf(b'0', small, b''), digest) |
2791 | + |
2792 | + # Smallest leaf, index=1 |
2793 | + digest = accum.add(proto.hash_leaf(1, small)) |
2794 | + self.assertEqual(digest, |
2795 | + skein512(key1 + small, |
2796 | + digest_bits=200, |
2797 | + pers=protocols.PERS_LEAF, |
2798 | + ).digest() |
2799 | + ) |
2800 | + self.assertEqual(proto._hash_leaf(b'1', small, b''), digest) |
2801 | + |
2802 | + # Largest leaf, index=0 |
2803 | + digest = accum.add(proto.hash_leaf(0, large)) |
2804 | + self.assertEqual(digest, |
2805 | + skein512(key0 + large, |
2806 | + digest_bits=200, |
2807 | + pers=protocols.PERS_LEAF, |
2808 | + ).digest() |
2809 | + ) |
2810 | + self.assertEqual(proto._hash_leaf(b'0', large, b''), digest) |
2811 | + |
2812 | + # Largest leaf, index=1 |
2813 | + digest = accum.add(proto.hash_leaf(1, large)) |
2814 | + self.assertEqual(digest, |
2815 | + skein512(key1 + large, |
2816 | + digest_bits=200, |
2817 | + pers=protocols.PERS_LEAF, |
2818 | + ).digest() |
2819 | + ) |
2820 | + self.assertEqual(proto._hash_leaf(b'1', large, b''), digest) |
2821 | + |
2822 | + # Test with challenge, smallest leaf, index=0: |
2823 | + digest = accum.add(proto.hash_leaf(0, small, b'secret')) |
2824 | + self.assertEqual(digest, |
2825 | + skein512(key0 + small, |
2826 | + digest_bits=200, |
2827 | + pers=protocols.PERS_LEAF, |
2828 | + nonce=b'secret', |
2829 | + ).digest() |
2830 | + ) |
2831 | + self.assertEqual(proto._hash_leaf(b'0', small, b'secret'), digest) |
2832 | + |
2833 | + # Test with challenge, smallest leaf, index=1: |
2834 | + digest = accum.add(proto.hash_leaf(1, small, b'secret')) |
2835 | + self.assertEqual(digest, |
2836 | + skein512(key1 + small, |
2837 | + digest_bits=200, |
2838 | + pers=protocols.PERS_LEAF, |
2839 | + nonce=b'secret', |
2840 | + ).digest() |
2841 | + ) |
2842 | + self.assertEqual(proto._hash_leaf(b'1', small, b'secret'), digest) |
2843 | + |
2844 | + # Test with challenge, largest leaf, index=0: |
2845 | + digest = accum.add(proto.hash_leaf(0, large, b'secret')) |
2846 | + self.assertEqual(digest, |
2847 | + skein512(key0 + large, |
2848 | + digest_bits=200, |
2849 | + pers=protocols.PERS_LEAF, |
2850 | + nonce=b'secret', |
2851 | + ).digest() |
2852 | + ) |
2853 | + self.assertEqual(proto._hash_leaf(b'0', large, b'secret'), digest) |
2854 | + |
2855 | + # Test with challenge, largest leaf, index=1: |
2856 | + digest = accum.add(proto.hash_leaf(1, large, b'secret')) |
2857 | + self.assertEqual(digest, |
2858 | + skein512(key1 + large, |
2859 | + digest_bits=200, |
2860 | + pers=protocols.PERS_LEAF, |
2861 | + nonce=b'secret', |
2862 | + ).digest() |
2863 | + ) |
2864 | + self.assertEqual(proto._hash_leaf(b'1', large, b'secret'), digest) |
2865 | + |
2866 | + # Make sure we didn't goof |
2867 | + safety = set() |
2868 | + for index in [0, 1]: |
2869 | + for data in [small, large]: |
2870 | + for challenge in [b'', b'secret']: |
2871 | + safety.add(proto.hash_leaf(index, data, challenge)) |
2872 | + self.assertEqual(safety, accum) |
2873 | + self.assertEqual(len(accum), 8) |
2874 | + for digest in accum: |
2875 | + self.assertEqual(len(digest), 25) |
2876 | |
2877 | def test_hash_root(self): |
2878 | - proto = protocols.VERSION1 |
2879 | - |
2880 | + proto = protocols.OldSkeinProtocol(TwoMiB, 200) |
2881 | self.check_hash_root(proto) |
2882 | - self.sanity_check_hash_root(proto) |
2883 | - |
2884 | - A0 = b32loads(A0_B32) |
2885 | - A1 = b32loads(A1_B32) |
2886 | - B0 = b32loads(B0_B32) |
2887 | - B1 = b32loads(B1_B32) |
2888 | - C0 = b32loads(C0_B32) |
2889 | - C1 = b32loads(C1_B32) |
2890 | - |
2891 | - # A |
2892 | - digest = proto._hash_root(1, A0) |
2893 | - self.assertEqual( |
2894 | - b32encode(digest).decode('utf-8'), |
2895 | - 'FWV6OJYI36C5NN5DC4GS2IGWZXFCZCGJGHK35YV62LKAG7D2Z4LO4Z2S' |
2896 | - ) |
2897 | - self.assertEqual( |
2898 | - digest, |
2899 | - skein512(A0, |
2900 | - digest_bits=280, |
2901 | - pers=protocols.PERS_ROOT, |
2902 | - key=b'1', |
2903 | - ).digest() |
2904 | - ) |
2905 | - self.assertEqual( |
2906 | - proto.hash_root(1, A0), |
2907 | - b32encode(digest).decode('utf-8') |
2908 | - ) |
2909 | - |
2910 | - # B |
2911 | - digest = proto._hash_root(8388607, B0) |
2912 | - self.assertEqual( |
2913 | - b32encode(digest).decode('utf-8'), |
2914 | - 'OB756PX5V32JMKJAFKIAJ4AFSFPA2WLNIK32ELNO4FJLJPEEEN6DCAAJ' |
2915 | - ) |
2916 | - self.assertEqual( |
2917 | - digest, |
2918 | - skein512(B0, |
2919 | - digest_bits=280, |
2920 | - pers=protocols.PERS_ROOT, |
2921 | - key=b'8388607', |
2922 | - ).digest() |
2923 | - ) |
2924 | - self.assertEqual( |
2925 | - proto.hash_root(8388607, B0), |
2926 | - b32encode(digest).decode('utf-8') |
2927 | - ) |
2928 | - |
2929 | - # C |
2930 | - digest = proto._hash_root(8388608, C0) |
2931 | - self.assertEqual( |
2932 | - b32encode(digest).decode('utf-8'), |
2933 | - 'QSOHXCDH64IQBOG2NM67XEC6MLZKKPGBTISWWRPMCFCJ2EKMA2SMLY46' |
2934 | - ) |
2935 | - self.assertEqual( |
2936 | - digest, |
2937 | - skein512(C0, |
2938 | - digest_bits=280, |
2939 | - pers=protocols.PERS_ROOT, |
2940 | - key=b'8388608', |
2941 | - ).digest() |
2942 | - ) |
2943 | - self.assertEqual( |
2944 | - proto.hash_root(8388608, C0), |
2945 | - b32encode(digest).decode('utf-8') |
2946 | - ) |
2947 | - |
2948 | - # CA |
2949 | - digest = proto._hash_root(8388609, C0 + A1) |
2950 | - self.assertEqual( |
2951 | - b32encode(digest).decode('utf-8'), |
2952 | - 'BQ5UTB33ML2VDTCTLVXK6N4VSMGGKKKDYKG24B6DOAFJB6NRSGMB5BNO' |
2953 | - ) |
2954 | - self.assertEqual( |
2955 | - digest, |
2956 | - skein512(C0 + A1, |
2957 | - digest_bits=280, |
2958 | - pers=protocols.PERS_ROOT, |
2959 | - key=b'8388609', |
2960 | - ).digest() |
2961 | - ) |
2962 | - self.assertEqual( |
2963 | - proto.hash_root(8388609, C0 + A1), |
2964 | - b32encode(digest).decode('utf-8') |
2965 | - ) |
2966 | - |
2967 | - # CB |
2968 | - digest = proto._hash_root(16777215, C0 + B1) |
2969 | - self.assertEqual( |
2970 | - b32encode(digest).decode('utf-8'), |
2971 | - 'ER3LDDZ2LHMTDLOPE5XA5GEEZ6OE45VFIFLY42GEMV4TSZ2B7GJJXAIX' |
2972 | - ) |
2973 | - self.assertEqual( |
2974 | - digest, |
2975 | - skein512(C0 + B1, |
2976 | - digest_bits=280, |
2977 | - pers=protocols.PERS_ROOT, |
2978 | - key=b'16777215', |
2979 | - ).digest() |
2980 | - ) |
2981 | - self.assertEqual( |
2982 | - proto.hash_root(16777215, C0 + B1), |
2983 | - b32encode(digest).decode('utf-8') |
2984 | - ) |
2985 | - |
2986 | - # CC |
2987 | - digest = proto._hash_root(16777216, C0 + C1) |
2988 | - self.assertEqual( |
2989 | - b32encode(digest).decode('utf-8'), |
2990 | - 'R6RN5KL7UBNJWR5SK5YPUKIGAOWWFMYYOVESU5DPT34X5MEK75PXXYIX' |
2991 | - ) |
2992 | - self.assertEqual( |
2993 | - digest, |
2994 | - skein512(C0 + C1, |
2995 | - digest_bits=280, |
2996 | - pers=protocols.PERS_ROOT, |
2997 | - key=b'16777216', |
2998 | - ).digest() |
2999 | - ) |
3000 | - self.assertEqual( |
3001 | - proto.hash_root(16777216, C0 + C1), |
3002 | - b32encode(digest).decode('utf-8') |
3003 | - ) |
3004 | - |
3005 | - |
3006 | -class TestOldProtocol(TestCase): |
3007 | + self.assertEqual(self._checks, [ |
3008 | + 'hash_root_validation', |
3009 | + 'hash_root_simple', |
3010 | + 'hash_root_crypto', |
3011 | + ]) |
3012 | + |
3013 | + accum = DigestAccum() |
3014 | + one = b'D' * 25 |
3015 | + two = b'D' * 50 |
3016 | + key1 = proto._hash_file_size(b'1') |
3017 | + key2 = proto._hash_file_size(b'2097152') |
3018 | + key3 = proto._hash_file_size(b'2097153') |
3019 | + key4 = proto._hash_file_size(b'4194304') |
3020 | + |
3021 | + # Smallest file-size possible for one leaf: |
3022 | + digest = accum.add(proto.hash_root(1, one)) |
3023 | + self.assertEqual(digest, |
3024 | + skein512(key1 + one, |
3025 | + digest_bits=200, |
3026 | + pers=protocols.PERS_ROOT, |
3027 | + ).digest() |
3028 | + ) |
3029 | + self.assertEqual(proto._hash_root(b'1', one), digest) |
3030 | + |
3031 | + # Largest file-size possible for one leaf: |
3032 | + digest = accum.add(proto.hash_root(TwoMiB, one)) |
3033 | + self.assertEqual(digest, |
3034 | + skein512(key2 + one, |
3035 | + digest_bits=200, |
3036 | + pers=protocols.PERS_ROOT, |
3037 | + ).digest() |
3038 | + ) |
3039 | + self.assertEqual(proto._hash_root(b'2097152', one), digest) |
3040 | + |
3041 | + # Smallest file-size possible for two leaves: |
3042 | + digest = accum.add(proto.hash_root(TwoMiB + 1, two)) |
3043 | + self.assertEqual(digest, |
3044 | + skein512(key3 + two, |
3045 | + digest_bits=200, |
3046 | + pers=protocols.PERS_ROOT, |
3047 | + ).digest() |
3048 | + ) |
3049 | + self.assertEqual(proto._hash_root(b'2097153', two), digest) |
3050 | + |
3051 | + # Largest file-size possible for two leaves: |
3052 | + digest = accum.add(proto.hash_root(2 * TwoMiB, two)) |
3053 | + self.assertEqual(digest, |
3054 | + skein512(key4 + two, |
3055 | + digest_bits=200, |
3056 | + pers=protocols.PERS_ROOT, |
3057 | + ).digest() |
3058 | + ) |
3059 | + self.assertEqual(proto._hash_root(b'4194304', two), digest) |
3060 | + |
3061 | + # Make sure we didn't goof |
3062 | + self.assertEqual(len(accum), 4) |
3063 | + safety = set([ |
3064 | + proto.hash_root(1, one), |
3065 | + proto.hash_root(TwoMiB, one), |
3066 | + proto.hash_root(TwoMiB + 1, two), |
3067 | + proto.hash_root(TwoMiB * 2, two), |
3068 | + ]) |
3069 | + self.assertEqual(safety, accum) |
3070 | + for digest in accum: |
3071 | + self.assertEqual(len(digest), 25) |
3072 | + |
3073 | def test_hash_leaf_index(self): |
3074 | - proto = protocols.OldProtocol(TwoMiB, 200) |
3075 | - accum = set() |
3076 | - key = proto._hash_leaf_index(0) |
3077 | - accum.add(key) |
3078 | - self.assertEqual(key, |
3079 | - skein512(b'0', |
3080 | - digest_bits=200, |
3081 | - pers=protocols.PERS_LEAF_INDEX, |
3082 | - ).digest() |
3083 | - ) |
3084 | - |
3085 | - key = proto._hash_leaf_index(17) |
3086 | - self.assertNotIn(key, accum) |
3087 | - accum.add(key) |
3088 | - self.assertEqual(key, |
3089 | - skein512(b'17', |
3090 | - digest_bits=200, |
3091 | - pers=protocols.PERS_LEAF_INDEX, |
3092 | - ).digest() |
3093 | - ) |
3094 | - self.assertEqual(len(accum), 2) |
3095 | - |
3096 | - count = 25 * 1000 |
3097 | - accum = set( |
3098 | - proto._hash_leaf_index(i) for i in range(count) |
3099 | - ) |
3100 | - self.assertEqual(len(accum), count) |
3101 | - |
3102 | - ############################################# |
3103 | - # Again, this time with different digest_bits |
3104 | - proto = protocols.OldProtocol(TwoMiB, 240) |
3105 | - accum = set() |
3106 | - key = proto._hash_leaf_index(0) |
3107 | - accum.add(key) |
3108 | - self.assertEqual(key, |
3109 | - skein512(b'0', |
3110 | - digest_bits=240, |
3111 | - pers=protocols.PERS_LEAF_INDEX, |
3112 | - ).digest() |
3113 | - ) |
3114 | - |
3115 | - key = proto._hash_leaf_index(17) |
3116 | - self.assertNotIn(key, accum) |
3117 | - accum.add(key) |
3118 | - self.assertEqual(key, |
3119 | - skein512(b'17', |
3120 | - digest_bits=240, |
3121 | - pers=protocols.PERS_LEAF_INDEX, |
3122 | - ).digest() |
3123 | - ) |
3124 | - self.assertEqual(len(accum), 2) |
3125 | - |
3126 | - count = 25 * 1000 |
3127 | - accum = set( |
3128 | - proto._hash_leaf_index(i) for i in range(count) |
3129 | - ) |
3130 | - self.assertEqual(len(accum), count) |
3131 | - |
3132 | - def test_hash_leaf(self): |
3133 | - challenge = b'secret foo bar' |
3134 | - proto = protocols.OldProtocol(TwoMiB, 200) |
3135 | - key0 = proto._hash_leaf_index(0) |
3136 | - key17 = proto._hash_leaf_index(17) |
3137 | - |
3138 | - accum = set() |
3139 | - |
3140 | - # Min leaf size, index=0 |
3141 | - leaf_data = b'D' |
3142 | - digest = proto._hash_leaf(0, leaf_data, b'') |
3143 | - self.assertNotIn(digest, accum) |
3144 | - accum.add(digest) |
3145 | - self.assertEqual(digest, |
3146 | - skein512(key0 + leaf_data, |
3147 | - digest_bits=200, |
3148 | - pers=protocols.PERS_LEAF, |
3149 | - ).digest() |
3150 | - ) |
3151 | - |
3152 | - # Min leaf size, index=17 |
3153 | - digest = proto._hash_leaf(17, leaf_data, b'') |
3154 | - self.assertNotIn(digest, accum) |
3155 | - accum.add(digest) |
3156 | - self.assertEqual(digest, |
3157 | - skein512(key17 + leaf_data, |
3158 | - digest_bits=200, |
3159 | - pers=protocols.PERS_LEAF, |
3160 | - ).digest() |
3161 | - ) |
3162 | - |
3163 | - # With challenge, min leaf size, index=0 |
3164 | - digest = proto._hash_leaf(0, leaf_data, challenge) |
3165 | - self.assertNotIn(digest, accum) |
3166 | - accum.add(digest) |
3167 | - self.assertEqual(digest, |
3168 | - skein512(key0 + leaf_data, |
3169 | - digest_bits=200, |
3170 | - pers=protocols.PERS_LEAF, |
3171 | - nonce=challenge, |
3172 | - ).digest() |
3173 | - ) |
3174 | - |
3175 | - # With challenge, min leaf size, index=17 |
3176 | - digest = proto._hash_leaf(17, leaf_data, challenge) |
3177 | - self.assertNotIn(digest, accum) |
3178 | - accum.add(digest) |
3179 | - self.assertEqual(digest, |
3180 | - skein512(key17 + leaf_data, |
3181 | - digest_bits=200, |
3182 | - pers=protocols.PERS_LEAF, |
3183 | - nonce=challenge, |
3184 | - ).digest() |
3185 | - ) |
3186 | - |
3187 | - # Max leaf size, index=0 |
3188 | - leaf_data = b'D' * TwoMiB |
3189 | - digest = proto._hash_leaf(0, leaf_data, b'') |
3190 | - self.assertNotIn(digest, accum) |
3191 | - accum.add(digest) |
3192 | - self.assertEqual(digest, |
3193 | - skein512(key0 + leaf_data, |
3194 | - digest_bits=200, |
3195 | - pers=protocols.PERS_LEAF, |
3196 | - ).digest() |
3197 | - ) |
3198 | - |
3199 | - # Max leaf size, index=17 |
3200 | - digest = proto._hash_leaf(17, leaf_data, b'') |
3201 | - self.assertNotIn(digest, accum) |
3202 | - accum.add(digest) |
3203 | - self.assertEqual(digest, |
3204 | - skein512(key17 + leaf_data, |
3205 | - digest_bits=200, |
3206 | - pers=protocols.PERS_LEAF, |
3207 | - ).digest() |
3208 | - ) |
3209 | - |
3210 | - # With challenge, max leaf size, index=0 |
3211 | - digest = proto._hash_leaf(0, leaf_data, challenge) |
3212 | - self.assertNotIn(digest, accum) |
3213 | - accum.add(digest) |
3214 | - self.assertEqual(digest, |
3215 | - skein512(key0 + leaf_data, |
3216 | - digest_bits=200, |
3217 | - pers=protocols.PERS_LEAF, |
3218 | - nonce=challenge, |
3219 | - ).digest() |
3220 | - ) |
3221 | - |
3222 | - # With challenge, max leaf size, index=17 |
3223 | - digest = proto._hash_leaf(17, leaf_data, challenge) |
3224 | - self.assertNotIn(digest, accum) |
3225 | - accum.add(digest) |
3226 | - self.assertEqual(digest, |
3227 | - skein512(key17 + leaf_data, |
3228 | - digest_bits=200, |
3229 | - pers=protocols.PERS_LEAF, |
3230 | - nonce=challenge, |
3231 | - ).digest() |
3232 | - ) |
3233 | - |
3234 | - # Make sure we didn't goof: |
3235 | - self.assertEqual(len(accum), 8) |
3236 | - |
3237 | - # A 25k value sanity check on our crytographic claim that the |
3238 | - # leaf-hash is tied to the leaf-index: |
3239 | - count = 25 * 1000 |
3240 | - leaf_data = os.urandom(32) |
3241 | - accum = set( |
3242 | - proto._hash_leaf(i, leaf_data, b'') |
3243 | - for i in range(count) |
3244 | - ) |
3245 | - self.assertEqual(len(accum), count) |
3246 | - |
3247 | - # A 25k random value sanity check on our crytographic claim that the |
3248 | - # leaf-hash is tied to the leaf-data: |
3249 | - accum = set( |
3250 | - proto._hash_leaf(21, os.urandom(32), b'') |
3251 | - for i in range(count) |
3252 | - ) |
3253 | - self.assertEqual(len(accum), count) |
3254 | - |
3255 | - # A 25k random value sanity check on our crytographic claim that the |
3256 | - # leaf-hash is tied to the challenge: |
3257 | - accum = set( |
3258 | - proto._hash_leaf(21, leaf_data, os.urandom(16)) |
3259 | - for i in range(count) |
3260 | - ) |
3261 | - self.assertEqual(len(accum), count) |
3262 | + proto = protocols.OldSkeinProtocol(TwoMiB, 200) |
3263 | + accum = DigestAccum() |
3264 | + for key in [b'0', b'1', b'17', b'18', b'21']: |
3265 | + digest = accum.add(proto._hash_leaf_index(key)) |
3266 | + self.assertEqual(digest, |
3267 | + skein512(key, |
3268 | + digest_bits=200, |
3269 | + pers=protocols.PERS_LEAF_INDEX, |
3270 | + ).digest() |
3271 | + ) |
3272 | + self.assertEqual(len(accum), 5) |
3273 | |
3274 | def test_hash_file_size(self): |
3275 | - proto = protocols.OldProtocol(TwoMiB, 200) |
3276 | - accum = set() |
3277 | - |
3278 | - key = proto._hash_file_size(1) |
3279 | - self.assertNotIn(key, accum) |
3280 | - accum.add(key) |
3281 | - self.assertEqual(key, |
3282 | - skein512(b'1', |
3283 | - digest_bits=200, |
3284 | - pers=protocols.PERS_FILE_SIZE, |
3285 | - ).digest() |
3286 | - ) |
3287 | - |
3288 | - key = proto._hash_file_size(18) |
3289 | - self.assertNotIn(key, accum) |
3290 | - accum.add(key) |
3291 | - self.assertEqual(key, |
3292 | - skein512(b'18', |
3293 | - digest_bits=200, |
3294 | - pers=protocols.PERS_FILE_SIZE, |
3295 | - ).digest() |
3296 | - ) |
3297 | - self.assertEqual(len(accum), 2) |
3298 | - |
3299 | - count = 25 * 1000 |
3300 | - accum = set( |
3301 | - proto._hash_file_size(size) for |
3302 | - size in range(1, count + 1) |
3303 | - ) |
3304 | - self.assertEqual(len(accum), count) |
3305 | - |
3306 | - ############################################# |
3307 | - # Again, this time with different digest_bits |
3308 | - proto = protocols.OldProtocol(TwoMiB, 240) |
3309 | - accum = set() |
3310 | - |
3311 | - key = proto._hash_file_size(1) |
3312 | - self.assertNotIn(key, accum) |
3313 | - accum.add(key) |
3314 | - self.assertEqual(key, |
3315 | - skein512(b'1', |
3316 | - digest_bits=240, |
3317 | - pers=protocols.PERS_FILE_SIZE, |
3318 | - ).digest() |
3319 | - ) |
3320 | - |
3321 | - key = proto._hash_file_size(18) |
3322 | - self.assertNotIn(key, accum) |
3323 | - accum.add(key) |
3324 | - self.assertEqual(key, |
3325 | - skein512(b'18', |
3326 | - digest_bits=240, |
3327 | - pers=protocols.PERS_FILE_SIZE, |
3328 | - ).digest() |
3329 | - ) |
3330 | - self.assertEqual(len(accum), 2) |
3331 | - |
3332 | - count = 25 * 1000 |
3333 | - accum = set( |
3334 | - proto._hash_file_size(size) for |
3335 | - size in range(1, count + 1) |
3336 | - ) |
3337 | - self.assertEqual(len(accum), count) |
3338 | - |
3339 | - def test_hash_root(self): |
3340 | - proto = protocols.OldProtocol(TwoMiB, 200) |
3341 | - key1 = proto._hash_file_size(1) |
3342 | - key18 = proto._hash_file_size(18) |
3343 | - keyM = proto._hash_file_size(TwoMiB) |
3344 | - keyM1 = proto._hash_file_size(TwoMiB + 1) |
3345 | - keyM18 = proto._hash_file_size(TwoMiB + 18) |
3346 | - key2M = proto._hash_file_size(2 * TwoMiB) |
3347 | - |
3348 | - accum = set() |
3349 | - |
3350 | - # One leaf, file_size=1 |
3351 | - leaf_hashes = b'D' * 25 |
3352 | - digest = proto._hash_root(1, leaf_hashes) |
3353 | - self.assertNotIn(digest, accum) |
3354 | - accum.add(digest) |
3355 | - self.assertEqual(digest, |
3356 | - skein512(key1 + leaf_hashes, |
3357 | - digest_bits=200, |
3358 | - pers=protocols.PERS_ROOT, |
3359 | - ).digest() |
3360 | - ) |
3361 | - self.assertEqual( |
3362 | - proto.hash_root(1, leaf_hashes), |
3363 | - b32encode(digest).decode('utf-8') |
3364 | - ) |
3365 | - |
3366 | - # One leaf, file_size=18 |
3367 | - digest = proto._hash_root(18, leaf_hashes) |
3368 | - self.assertNotIn(digest, accum) |
3369 | - accum.add(digest) |
3370 | - self.assertEqual(digest, |
3371 | - skein512(key18 + leaf_hashes, |
3372 | - digest_bits=200, |
3373 | - pers=protocols.PERS_ROOT, |
3374 | - ).digest() |
3375 | - ) |
3376 | - self.assertEqual( |
3377 | - proto.hash_root(18, leaf_hashes), |
3378 | - b32encode(digest).decode('utf-8') |
3379 | - ) |
3380 | - |
3381 | - # One leaf, file_size=TwoMiB |
3382 | - digest = proto._hash_root(TwoMiB, leaf_hashes) |
3383 | - self.assertNotIn(digest, accum) |
3384 | - accum.add(digest) |
3385 | - self.assertEqual(digest, |
3386 | - skein512(keyM + leaf_hashes, |
3387 | - digest_bits=200, |
3388 | - pers=protocols.PERS_ROOT, |
3389 | - ).digest() |
3390 | - ) |
3391 | - self.assertEqual( |
3392 | - proto.hash_root(TwoMiB, leaf_hashes), |
3393 | - b32encode(digest).decode('utf-8') |
3394 | - ) |
3395 | - |
3396 | - # Two leaves, file_size=(TwoMiB + 1) |
3397 | - leaf_hashes = b'D' * 50 |
3398 | - digest = proto._hash_root(TwoMiB + 1, leaf_hashes) |
3399 | - self.assertNotIn(digest, accum) |
3400 | - accum.add(digest) |
3401 | - self.assertEqual(digest, |
3402 | - skein512(keyM1 + leaf_hashes, |
3403 | - digest_bits=200, |
3404 | - pers=protocols.PERS_ROOT, |
3405 | - ).digest() |
3406 | - ) |
3407 | - self.assertEqual( |
3408 | - proto.hash_root(TwoMiB + 1, leaf_hashes), |
3409 | - b32encode(digest).decode('utf-8') |
3410 | - ) |
3411 | - |
3412 | - # Two leaves, file_size=(TwoMiB + 18) |
3413 | - digest = proto._hash_root(TwoMiB + 18, leaf_hashes) |
3414 | - self.assertNotIn(digest, accum) |
3415 | - accum.add(digest) |
3416 | - self.assertEqual(digest, |
3417 | - skein512(keyM18 + leaf_hashes, |
3418 | - digest_bits=200, |
3419 | - pers=protocols.PERS_ROOT, |
3420 | - ).digest() |
3421 | - ) |
3422 | - self.assertEqual( |
3423 | - proto.hash_root(TwoMiB + 18, leaf_hashes), |
3424 | - b32encode(digest).decode('utf-8') |
3425 | - ) |
3426 | - |
3427 | - # Two leaves, file_size=(2 * TwoMiB) |
3428 | - digest = proto._hash_root(2 * TwoMiB, leaf_hashes) |
3429 | - self.assertNotIn(digest, accum) |
3430 | - accum.add(digest) |
3431 | - self.assertEqual(digest, |
3432 | - skein512(key2M + leaf_hashes, |
3433 | - digest_bits=200, |
3434 | - pers=protocols.PERS_ROOT, |
3435 | - ).digest() |
3436 | - ) |
3437 | - self.assertEqual( |
3438 | - proto.hash_root(2 * TwoMiB, leaf_hashes), |
3439 | - b32encode(digest).decode('utf-8') |
3440 | - ) |
3441 | - |
3442 | - # Make sure we didn't goof: |
3443 | - self.assertEqual(len(accum), 6) |
3444 | - |
3445 | - # A 25k value sanity check on our crytographic claim that the |
3446 | - # root-hash is tied to the file-size: |
3447 | - count = 25 * 1000 |
3448 | - leaf_hashes = b'D' * 25 |
3449 | - accum = set( |
3450 | - proto._hash_root(size, leaf_hashes) |
3451 | - for size in range(1, count + 1) |
3452 | - ) |
3453 | - self.assertEqual(len(accum), count) |
3454 | - |
3455 | - # A 25k random value sanity check on our crytographic claim that the |
3456 | - # root-hash is tied to the leaf-hashes: |
3457 | - accum = set( |
3458 | - proto._hash_root(21, os.urandom(25)) |
3459 | - for i in range(count) |
3460 | - ) |
3461 | - self.assertEqual(len(accum), count) |
3462 | + proto = protocols.OldSkeinProtocol(TwoMiB, 200) |
3463 | + accum = DigestAccum() |
3464 | + for key in [b'1', b'2', b'2097152', b'2097153', b'4194304']: |
3465 | + digest = accum.add(proto._hash_file_size(key)) |
3466 | + self.assertEqual(digest, |
3467 | + skein512(key, |
3468 | + digest_bits=200, |
3469 | + pers=protocols.PERS_FILE_SIZE, |
3470 | + ).digest() |
3471 | + ) |
3472 | + self.assertEqual(len(accum), 5) |
3473 | + |
3474 | + |
3475 | +class Test_VERSION0(BaseTestCase2): |
3476 | + proto = protocols.VERSION0 |
3477 | + |
3478 | + def test_hash_leaf(self): |
3479 | + self.check_hash_leaf(self.proto) |
3480 | + self.assertEqual(self._checks, [ |
3481 | + 'hash_leaf_validation', |
3482 | + 'hash_leaf_simple', |
3483 | + 'hash_leaf_crypto', |
3484 | + ]) |
3485 | + |
3486 | + leaves = misc.build_leaves(EightMiB) |
3487 | + vectors = misc.load_data('V0') |
3488 | + self.assertEqual( |
3489 | + b32enc(self.proto.hash_leaf(0, leaves['A'])), |
3490 | + vectors['leaf_hashes']['A'][0] |
3491 | + ) |
3492 | + self.assertEqual( |
3493 | + b32enc(self.proto.hash_leaf(1, leaves['A'])), |
3494 | + vectors['leaf_hashes']['A'][1] |
3495 | + ) |
3496 | + self.assertEqual( |
3497 | + b32enc(self.proto.hash_leaf(0, leaves['B'])), |
3498 | + vectors['leaf_hashes']['B'][0] |
3499 | + ) |
3500 | + self.assertEqual( |
3501 | + b32enc(self.proto.hash_leaf(1, leaves['B'])), |
3502 | + vectors['leaf_hashes']['B'][1] |
3503 | + ) |
3504 | + self.assertEqual( |
3505 | + b32enc(self.proto.hash_leaf(0, leaves['C'])), |
3506 | + vectors['leaf_hashes']['C'][0] |
3507 | + ) |
3508 | + self.assertEqual( |
3509 | + b32enc(self.proto.hash_leaf(1, leaves['C'])), |
3510 | + vectors['leaf_hashes']['C'][1] |
3511 | + ) |
3512 | + |
3513 | + def test_hash_root(self): |
3514 | + self.check_hash_root(self.proto) |
3515 | + self.assertEqual(self._checks, [ |
3516 | + 'hash_root_validation', |
3517 | + 'hash_root_simple', |
3518 | + 'hash_root_crypto', |
3519 | + ]) |
3520 | + |
3521 | + vectors = misc.load_data('V0') |
3522 | + A0 = b32dec(vectors['leaf_hashes']['A'][0]) |
3523 | + A1 = b32dec(vectors['leaf_hashes']['A'][1]) |
3524 | + B0 = b32dec(vectors['leaf_hashes']['B'][0]) |
3525 | + B1 = b32dec(vectors['leaf_hashes']['B'][1]) |
3526 | + C0 = b32dec(vectors['leaf_hashes']['C'][0]) |
3527 | + C1 = b32dec(vectors['leaf_hashes']['C'][1]) |
3528 | + self.assertEqual( |
3529 | + b32enc(self.proto.hash_root(1, A0)), |
3530 | + vectors['root_hashes']['A'] |
3531 | + ) |
3532 | + self.assertEqual( |
3533 | + b32enc(self.proto.hash_root(EightMiB - 1, B0)), |
3534 | + vectors['root_hashes']['B'] |
3535 | + ) |
3536 | + self.assertEqual( |
3537 | + b32enc(self.proto.hash_root(EightMiB , C0)), |
3538 | + vectors['root_hashes']['C'] |
3539 | + ) |
3540 | + self.assertEqual( |
3541 | + b32enc(self.proto.hash_root(EightMiB + 1, C0 + A1)), |
3542 | + vectors['root_hashes']['CA'] |
3543 | + ) |
3544 | + self.assertEqual( |
3545 | + b32enc(self.proto.hash_root(2 * EightMiB - 1, C0 + B1)), |
3546 | + vectors['root_hashes']['CB'] |
3547 | + ) |
3548 | + self.assertEqual( |
3549 | + b32enc(self.proto.hash_root(2 * EightMiB , C0 + C1)), |
3550 | + vectors['root_hashes']['CC'] |
3551 | + ) |
3552 | + |
3553 | + def test_vectors(self): |
3554 | + V0 = misc.load_data('V0') |
3555 | + self.assertEqual(misc.build_vectors(self.proto, b32enc), V0) |
3556 | + |
3557 | + |
3558 | +class Test_VERSION1(BaseTestCase2): |
3559 | + proto = protocols.VERSION1 |
3560 | + |
3561 | + def test_hash_leaf(self): |
3562 | + self.check_hash_leaf(self.proto) |
3563 | + self.assertEqual(self._checks, [ |
3564 | + 'hash_leaf_validation', |
3565 | + 'hash_leaf_simple', |
3566 | + 'hash_leaf_crypto', |
3567 | + ]) |
3568 | + |
3569 | + leaves = misc.build_leaves(EightMiB) |
3570 | + vectors = misc.load_data('V1') |
3571 | + self.assertEqual( |
3572 | + db32enc(self.proto.hash_leaf(0, leaves['A'])), |
3573 | + vectors['leaf_hashes']['A'][0] |
3574 | + ) |
3575 | + self.assertEqual( |
3576 | + db32enc(self.proto.hash_leaf(1, leaves['A'])), |
3577 | + vectors['leaf_hashes']['A'][1] |
3578 | + ) |
3579 | + self.assertEqual( |
3580 | + db32enc(self.proto.hash_leaf(0, leaves['B'])), |
3581 | + vectors['leaf_hashes']['B'][0] |
3582 | + ) |
3583 | + self.assertEqual( |
3584 | + db32enc(self.proto.hash_leaf(1, leaves['B'])), |
3585 | + vectors['leaf_hashes']['B'][1] |
3586 | + ) |
3587 | + self.assertEqual( |
3588 | + db32enc(self.proto.hash_leaf(0, leaves['C'])), |
3589 | + vectors['leaf_hashes']['C'][0] |
3590 | + ) |
3591 | + self.assertEqual( |
3592 | + db32enc(self.proto.hash_leaf(1, leaves['C'])), |
3593 | + vectors['leaf_hashes']['C'][1] |
3594 | + ) |
3595 | + |
3596 | + def test_hash_root(self): |
3597 | + self.check_hash_root(self.proto) |
3598 | + self.assertEqual(self._checks, [ |
3599 | + 'hash_root_validation', |
3600 | + 'hash_root_simple', |
3601 | + 'hash_root_crypto', |
3602 | + ]) |
3603 | + |
3604 | + vectors = misc.load_data('V1') |
3605 | + A0 = db32dec(vectors['leaf_hashes']['A'][0]) |
3606 | + A1 = db32dec(vectors['leaf_hashes']['A'][1]) |
3607 | + B0 = db32dec(vectors['leaf_hashes']['B'][0]) |
3608 | + B1 = db32dec(vectors['leaf_hashes']['B'][1]) |
3609 | + C0 = db32dec(vectors['leaf_hashes']['C'][0]) |
3610 | + C1 = db32dec(vectors['leaf_hashes']['C'][1]) |
3611 | + self.assertEqual( |
3612 | + db32enc(self.proto.hash_root(1, A0)), |
3613 | + vectors['root_hashes']['A'] |
3614 | + ) |
3615 | + self.assertEqual( |
3616 | + db32enc(self.proto.hash_root(EightMiB - 1, B0)), |
3617 | + vectors['root_hashes']['B'] |
3618 | + ) |
3619 | + self.assertEqual( |
3620 | + db32enc(self.proto.hash_root(EightMiB , C0)), |
3621 | + vectors['root_hashes']['C'] |
3622 | + ) |
3623 | + self.assertEqual( |
3624 | + db32enc(self.proto.hash_root(EightMiB + 1, C0 + A1)), |
3625 | + vectors['root_hashes']['CA'] |
3626 | + ) |
3627 | + self.assertEqual( |
3628 | + db32enc(self.proto.hash_root(2 * EightMiB - 1, C0 + B1)), |
3629 | + vectors['root_hashes']['CB'] |
3630 | + ) |
3631 | + self.assertEqual( |
3632 | + db32enc(self.proto.hash_root(2 * EightMiB , C0 + C1)), |
3633 | + vectors['root_hashes']['CC'] |
3634 | + ) |
3635 | + |
3636 | + def test_vectors(self): |
3637 | + V1 = misc.load_data('V1') |
3638 | + self.assertEqual(misc.build_vectors(self.proto, db32enc), V1) |
3639 | + self.assertNotEqual(misc.build_vectors(self.proto, b32enc), V1) |
3640 | + self.assertNotEqual(misc.build_vectors(protocols.VERSION0, db32enc), V1) |
3641 | + self.assertNotEqual(misc.build_vectors(protocols.VERSION0, b32enc), V1) |
3642 | + |
3643 | + def test_md5sums(self): |
3644 | + MD5SUMS = misc.load_data('MD5SUMS') |
3645 | + self.assertEqual(misc.build_md5sums(self.proto.leaf_size), MD5SUMS) |
3646 | + |
3647 | |
3648 | === modified file 'setup.py' |
3649 | --- setup.py 2013-02-18 12:58:01 +0000 |
3650 | +++ setup.py 2013-02-19 15:39:24 +0000 |
3651 | @@ -67,7 +67,7 @@ |
3652 | 'filestore', |
3653 | 'filestore.tests' |
3654 | ], |
3655 | - package_data={'filestore': ['data/test-vectors.json']}, |
3656 | + package_data={'filestore': ['data/*.json']}, |
3657 | scripts=['dmediasum'], |
3658 | cmdclass={'test': Test}, |
3659 | ext_modules=[_filestore], |
As this diff is pretty sprawling and noisy, you can probably understand the key changes more easily by just looking at filestore/ protocols. py:
http:// bazaar. launchpad. net/~jderose/ filestore/ dbase32/ view/head: /filestore/ protocols. py