Merge lp:~jderose/dmedia/filestore-tmp-api into lp:dmedia
- filestore-tmp-api
- Merge into trunk
Proposed by
Jason Gerard DeRose
Status: | Merged |
---|---|
Approved by: | Jason Gerard DeRose |
Approved revision: | 170 |
Merged at revision: | 156 |
Proposed branch: | lp:~jderose/dmedia/filestore-tmp-api |
Merge into: | lp:dmedia |
Diff against target: |
582 lines (+250/-66) 4 files modified
dmedia/downloader.py (+2/-2) dmedia/filestore.py (+119/-42) dmedia/tests/test_downloader.py (+1/-1) dmedia/tests/test_filestore.py (+128/-21) |
To merge this branch: | bzr merge lp:~jderose/dmedia/filestore-tmp-api |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
James Raymond | Approve | ||
dmedia Dev | Pending | ||
Review via email: mp+49495@code.launchpad.net |
Commit message
Description of the change
Splits FileStore.
FileStore.
FileStore.
To fit these naming conventions, I also renamed:
FileStore.reltemp() => FileStore.reltmp()
FileStore.temp() => FileStore.tmp()
Meaning to do that for a while, so good excuse.
Probably easier to read the docstrings here - http://
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'dmedia/downloader.py' |
2 | --- dmedia/downloader.py 2011-02-11 16:55:18 +0000 |
3 | +++ dmedia/downloader.py 2011-02-12 11:14:54 +0000 |
4 | @@ -172,12 +172,12 @@ |
5 | self.ext = ext |
6 | |
7 | def get_tmp(self): |
8 | - tmp = self.fs.temp(self.chash, self.ext, create=True) |
9 | + tmp = self.fs.tmp(self.chash, self.ext, create=True) |
10 | log.debug('Writting file to %r', tmp) |
11 | return tmp |
12 | |
13 | def finalize(self): |
14 | - dst = self.fs.finalize_transfer(self.chash, self.ext) |
15 | + dst = self.fs.tmp_verify_move(self.chash, self.ext) |
16 | log.debug('Canonical name is %r', dst) |
17 | return dst |
18 | |
19 | |
20 | === modified file 'dmedia/filestore.py' |
21 | --- dmedia/filestore.py 2011-02-11 17:22:08 +0000 |
22 | +++ dmedia/filestore.py 2011-02-12 11:14:54 +0000 |
23 | @@ -39,6 +39,7 @@ |
24 | |
25 | import os |
26 | from os import path |
27 | +import stat |
28 | import tempfile |
29 | from hashlib import sha1 as HASH |
30 | from base64 import b32encode, b32decode |
31 | @@ -278,8 +279,6 @@ |
32 | if not chunk: |
33 | break |
34 | self.thread.join() |
35 | - if self.dst_fp is not None: |
36 | - os.fchmod(self.dst_fp.fileno(), 0o444) |
37 | return b32encode(self.h.digest()) |
38 | |
39 | |
40 | @@ -450,18 +449,18 @@ |
41 | return (dname, fname) |
42 | |
43 | @staticmethod |
44 | - def reltemp(chash, ext=None): |
45 | + def reltmp(chash, ext=None): |
46 | """ |
47 | Relative path of temporary file with *chash*, ending with *ext*. |
48 | |
49 | For example: |
50 | |
51 | - >>> FileStore.reltemp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW') |
52 | + >>> FileStore.reltmp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW') |
53 | ('transfers', 'NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW') |
54 | |
55 | Or with the file extension *ext*: |
56 | |
57 | - >>> FileStore.reltemp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW', ext='mov') |
58 | + >>> FileStore.reltmp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW', ext='mov') |
59 | ('transfers', 'NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW.mov') |
60 | |
61 | Also see `FileStore.relpath()`. |
62 | @@ -472,6 +471,9 @@ |
63 | return (TRANSFERS_DIR, chash) |
64 | |
65 | def check_path(self, pathname): |
66 | + """ |
67 | + Verify that *pathname* in inside this filestore base directory. |
68 | + """ |
69 | abspath = path.abspath(pathname) |
70 | if abspath.startswith(self.base + os.sep): |
71 | return abspath |
72 | @@ -574,7 +576,7 @@ |
73 | self.create_parent(filename) |
74 | return filename |
75 | |
76 | - def temp(self, chash, ext=None, create=False): |
77 | + def tmp(self, chash, ext=None, create=False): |
78 | """ |
79 | Returns path of temporary file with *chash*, ending with *ext*. |
80 | |
81 | @@ -582,26 +584,26 @@ |
82 | in which case the content-hash is already known. For example: |
83 | |
84 | >>> fs = FileStore() |
85 | - >>> fs.temp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW') #doctest: +ELLIPSIS |
86 | + >>> fs.tmp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW') #doctest: +ELLIPSIS |
87 | '/tmp/store.../transfers/NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW' |
88 | |
89 | |
90 | Or with a file extension: |
91 | |
92 | - >>> fs.temp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW', 'txt') #doctest: +ELLIPSIS |
93 | + >>> fs.tmp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW', 'txt') #doctest: +ELLIPSIS |
94 | '/tmp/store.../transfers/NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW.txt' |
95 | |
96 | |
97 | If called with ``create=True``, the parent directory is created with |
98 | `FileStore.create_parent()`. |
99 | """ |
100 | - filename = self.join(*self.reltemp(chash, ext)) |
101 | + filename = self.join(*self.reltmp(chash, ext)) |
102 | if create: |
103 | self.create_parent(filename) |
104 | return filename |
105 | |
106 | def allocate_for_transfer(self, size, chash, ext=None): |
107 | - filename = self.temp(chash, ext, create=True) |
108 | + filename = self.tmp(chash, ext, create=True) |
109 | fallocate(size, filename) |
110 | try: |
111 | fp = open(filename, 'r+b') |
112 | @@ -620,14 +622,99 @@ |
113 | fallocate(size, filename) |
114 | return open(filename, 'r+b') |
115 | |
116 | - def finalize_transfer(self, chash, ext=None): |
117 | - """ |
118 | - Move canonically named temporary file to its final canonical location. |
119 | + def tmp_move(self, tmp_fp, chash, ext=None): |
120 | + """ |
121 | + Move temporary file into its canonical location. |
122 | + |
123 | + This method will securely and atomically move a temporary file into its |
124 | + canonical location. |
125 | + |
126 | + For example: |
127 | + |
128 | + >>> fs = FileStore() |
129 | + >>> tmp_fp = open(fs.join('foo.mov'), 'wb') |
130 | + >>> chash = 'ZR765XWSF6S7JQHLUI4GCG5BHGPE252O' |
131 | + >>> fs.tmp_move(tmp_fp, chash, 'mov') #doctest: +ELLIPSIS |
132 | + '/tmp/store.../ZR/765XWSF6S7JQHLUI4GCG5BHGPE252O.mov' |
133 | + |
134 | + |
135 | + Note, however, that this method does *not* verify the content hash of |
136 | + the temporary file! This is by design as many operations will compute |
137 | + the content hash as they write to the temporary file. Other operations |
138 | + should use `FileStore.tmp_verify_move()` to verify and move in one step. |
139 | + |
140 | + Regardless, the full content hash should have been verified prior to |
141 | + calling this method. To ensure the content is not modified, operations |
142 | + must take these steps: |
143 | + |
144 | + 1. Open *tmp_fp* and keep it open, thereby retaining a lock on the |
145 | + file |
146 | + |
147 | + 2. Compute the full content hash, which can be done as content is |
148 | + written to *tmp_fp* (open in mode ``'r+b'`` to resume a transfer, |
149 | + but hash of previously transfered leaves must still be verified) |
150 | + |
151 | + 3. With *tmp_fp* still open, move the temporary file into its |
152 | + canonical location using this method. |
153 | + |
154 | + As a simple locking mechanism, this method takes an open ``file`` rather |
155 | + than a filename, thereby preventing the file from being modified during |
156 | + the move. A ``ValueError`` is raised if *tmp_fp* is already closed. |
157 | + |
158 | + For portability reasons, this method requires that *tmp_fp* be opened in |
159 | + a binary mode: ``'rb'``, ``'wb'``, or ``'r+b'``. A ``ValueError`` is |
160 | + raised if opened in any other mode. |
161 | + |
162 | + For security reasons, this method will only move a temporary file |
163 | + located within the ``FileStore.base`` directory or a subdirectory |
164 | + thereof. If an attempt is made to move a file from outside the store, |
165 | + `FileStoreTraversal` is raised. See `FileStore.check_path()`. |
166 | + |
167 | + Just prior to moving the file, a call to ``os.fchmod()`` is made to set |
168 | + read-only permissions (0444). After the move, *tmp_fp* is closed. |
169 | + |
170 | + If the canonical file already exists, `DuplicateFile` is raised. |
171 | + |
172 | + The return value is the absolute path of the canonical file. |
173 | + |
174 | + :param tmp_fp: a ``file`` instance created with ``open()`` |
175 | + :param chash: base32-encoded content-hash |
176 | + :param ext: normalized lowercase file extension, eg ``'mov'`` |
177 | + """ |
178 | + # Validate tmp_fp: |
179 | + if not isinstance(tmp_fp, file): |
180 | + raise TypeError( |
181 | + TYPE_ERROR % ('tmp_fp', file, type(tmp_fp), tmp_fp) |
182 | + ) |
183 | + if tmp_fp.mode not in ('rb', 'wb', 'r+b'): |
184 | + raise ValueError( |
185 | + "tmp_fp: mode must be 'rb', 'wb', or 'r+b'; got %r" % tmp_fp.mode |
186 | + ) |
187 | + if tmp_fp.closed: |
188 | + raise ValueError('tmp_fp is closed, must be open: %r' % tmp_fp.name) |
189 | + self.check_path(tmp_fp.name) |
190 | + |
191 | + # Get canonical name, check for duplicate: |
192 | + dst = self.path(chash, ext, create=True) |
193 | + if path.exists(dst): |
194 | + raise DuplicateFile(chash=chash, src=tmp_fp.name, dst=dst) |
195 | + |
196 | + # Set file to read-only (0444) and move into canonical location |
197 | + os.fchmod(tmp_fp.fileno(), stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) |
198 | + os.rename(tmp_fp.name, dst) |
199 | + tmp_fp.close() |
200 | + |
201 | + # Return canonical filename: |
202 | + return dst |
203 | + |
204 | + def tmp_verify_move(self, chash, ext=None): |
205 | + """ |
206 | + Verify temporary file, then move into its canonical location. |
207 | |
208 | This method will check the content hash of the canonically-named |
209 | temporary file with content hash *chash* and extension *ext*. If the |
210 | - content hash is correct, it will do an ``os.fchmod()`` to set read-only |
211 | - permissions, and then rename the file into its canonical location. |
212 | + content hash is correct, this method will then move the temporary file |
213 | + into its canonical location using `FileStore.tmp_move()`. |
214 | |
215 | If the content hash is incorrect, `IntegrityError` is raised. If the |
216 | canonical file already exists, `DuplicateFile` is raised. Lastly, if |
217 | @@ -639,7 +726,7 @@ |
218 | temporary file name, like this: |
219 | |
220 | >>> fs = FileStore() |
221 | - >>> tmp = fs.temp('ZR765XWSF6S7JQHLUI4GCG5BHGPE252O', 'mov', create=True) |
222 | + >>> tmp = fs.tmp('ZR765XWSF6S7JQHLUI4GCG5BHGPE252O', 'mov', create=True) |
223 | >>> tmp #doctest: +ELLIPSIS |
224 | '/tmp/store.../transfers/ZR765XWSF6S7JQHLUI4GCG5BHGPE252O.mov' |
225 | |
226 | @@ -662,33 +749,23 @@ |
227 | Finally, the downloader will move the temporary file into its canonical |
228 | location: |
229 | |
230 | - >>> dst = fs.finalize_transfer('ZR765XWSF6S7JQHLUI4GCG5BHGPE252O', 'mov') |
231 | + >>> dst = fs.tmp_verify_move('ZR765XWSF6S7JQHLUI4GCG5BHGPE252O', 'mov') |
232 | >>> dst #doctest: +ELLIPSIS |
233 | '/tmp/store.../ZR/765XWSF6S7JQHLUI4GCG5BHGPE252O.mov' |
234 | |
235 | |
236 | - Note above that this method returns the full path of the canonically |
237 | - named file. |
238 | + The return value is the absolute path of the canonical file. |
239 | + |
240 | + :param chash: base32-encoded content-hash |
241 | + :param ext: normalized lowercase file extension, eg ``'mov'`` |
242 | """ |
243 | - # Open temporary file and check content hash: |
244 | - tmp = self.temp(chash, ext) |
245 | + tmp = self.tmp(chash, ext) |
246 | tmp_fp = open(tmp, 'rb') |
247 | h = HashList(tmp_fp) |
248 | got = h.run() |
249 | if got != chash: |
250 | raise IntegrityError(got=got, expected=chash, filename=tmp_fp.name) |
251 | - |
252 | - # Get canonical name, check for duplicate: |
253 | - dst = self.path(chash, ext, create=True) |
254 | - if path.exists(dst): |
255 | - raise DuplicateFile(chash=chash, src=tmp_fp.name, dst=dst) |
256 | - |
257 | - # Set file to read-only and rename into canonical location |
258 | - os.fchmod(tmp_fp.fileno(), 0o444) |
259 | - os.rename(tmp_fp.name, dst) |
260 | - |
261 | - # Return canonical filename: |
262 | - return dst |
263 | + return self.tmp_move(tmp_fp, chash, ext) |
264 | |
265 | def import_file(self, src_fp, ext=None): |
266 | """ |
267 | @@ -696,8 +773,8 @@ |
268 | |
269 | The method will compute the content-hash of *src_fp* as it copies it to |
270 | a temporary file within this store. Once the copying is complete, the |
271 | - file will be renamed to its canonical location in the store, thus |
272 | - ensuring an atomic operation. |
273 | + temporary file will be moved to its canonical location using |
274 | + `FileStore.tmp_move()`. |
275 | |
276 | A `DuplicatedFile` exception will be raised if the file already exists |
277 | in this store. |
278 | @@ -708,15 +785,15 @@ |
279 | |
280 | Note that *src_fp* must have been opened in ``'rb'`` mode. |
281 | |
282 | - :param src_fp: A ``file`` instance created with ``open()`` |
283 | - :param ext: The file's extension, e.g., ``'ogv'`` |
284 | + :param src_fp: a ``file`` instance created with ``open()`` |
285 | + :param ext: normalized lowercase file extension, eg ``'mov'`` |
286 | """ |
287 | size = os.fstat(src_fp.fileno()).st_size |
288 | - tmp_fp = self.allocate_for_import(size, ext=ext) |
289 | + tmp_fp = self.allocate_for_import(size, ext) |
290 | h = HashList(src_fp, tmp_fp) |
291 | chash = h.run() |
292 | - dst = self.path(chash, ext, create=True) |
293 | - if path.exists(dst): |
294 | - raise DuplicateFile(chash=chash, src=src_fp.name, dst=dst) |
295 | - os.rename(tmp_fp.name, dst) |
296 | + try: |
297 | + self.tmp_move(tmp_fp, chash, ext) |
298 | + except DuplicateFile as e: |
299 | + raise DuplicateFile(src=src_fp.name, dst=e.dst, chash=e.chash) |
300 | return (chash, h.leaves) |
301 | |
302 | === modified file 'dmedia/tests/test_downloader.py' |
303 | --- dmedia/tests/test_downloader.py 2011-02-11 15:45:43 +0000 |
304 | +++ dmedia/tests/test_downloader.py 2011-02-12 11:14:54 +0000 |
305 | @@ -212,7 +212,7 @@ |
306 | self.assertFalse(path.exists(dst)) |
307 | |
308 | # Test when transfers/ exists but file does not: |
309 | - self.assertEqual(fs.temp(mov_hash, 'mov', create=True), src) |
310 | + self.assertEqual(fs.tmp(mov_hash, 'mov', create=True), src) |
311 | self.assertTrue(path.isdir(src_d)) |
312 | e = raises(IOError, inst.finalize) |
313 | self.assertFalse(path.exists(src)) |
314 | |
315 | === modified file 'dmedia/tests/test_filestore.py' |
316 | --- dmedia/tests/test_filestore.py 2011-02-11 01:50:20 +0000 |
317 | +++ dmedia/tests/test_filestore.py 2011-02-12 11:14:54 +0000 |
318 | @@ -26,6 +26,7 @@ |
319 | |
320 | import os |
321 | from os import path |
322 | +import stat |
323 | from hashlib import sha1 |
324 | from base64 import b32encode, b32decode |
325 | import shutil |
326 | @@ -530,24 +531,24 @@ |
327 | 'ext %r does not match pattern %r' % (bad, EXT_PAT) |
328 | ) |
329 | |
330 | - def test_reltemp(self): |
331 | + def test_reltmp(self): |
332 | self.assertEqual( |
333 | - self.klass.reltemp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW'), |
334 | + self.klass.reltmp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW'), |
335 | ('transfers', 'NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW') |
336 | ) |
337 | self.assertEqual( |
338 | - self.klass.reltemp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW', ext='ogv'), |
339 | + self.klass.reltmp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW', ext='ogv'), |
340 | ('transfers', 'NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW.ogv') |
341 | ) |
342 | |
343 | # Test to make sure hashes are getting checked with safe_b32(): |
344 | bad = 'NWBNVXVK5..GIOW7MYR4K3KA5K22W7NW' |
345 | - e = raises(ValueError, self.klass.reltemp, bad) |
346 | + e = raises(ValueError, self.klass.reltmp, bad) |
347 | self.assertEqual( |
348 | str(e), |
349 | 'b32: cannot b32decode %r: Non-base32 digit found' % bad |
350 | ) |
351 | - e = raises(ValueError, self.klass.reltemp, bad, ext='ogv') |
352 | + e = raises(ValueError, self.klass.reltmp, bad, ext='ogv') |
353 | self.assertEqual( |
354 | str(e), |
355 | 'b32: cannot b32decode %r: Non-base32 digit found' % bad |
356 | @@ -556,7 +557,7 @@ |
357 | # Test to make sure ext is getting checked with safe_ext(): |
358 | chash = 'NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW' |
359 | bad = '/../../../.ssh/id_pub' |
360 | - e = raises(ValueError, self.klass.reltemp, chash, bad) |
361 | + e = raises(ValueError, self.klass.reltmp, chash, bad) |
362 | self.assertEqual( |
363 | str(e), |
364 | 'ext %r does not match pattern %r' % (bad, EXT_PAT) |
365 | @@ -750,27 +751,27 @@ |
366 | self.assertFalse(path.exists(f)) |
367 | self.assertTrue(path.isdir(d)) |
368 | |
369 | - def test_temp(self): |
370 | + def test_tmp(self): |
371 | tmp = TempDir() |
372 | inst = self.klass(tmp.path) |
373 | |
374 | self.assertEqual( |
375 | - inst.temp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW'), |
376 | + inst.tmp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW'), |
377 | tmp.join('transfers', 'NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW') |
378 | ) |
379 | self.assertEqual( |
380 | - inst.temp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW', ext='ogv'), |
381 | + inst.tmp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW', ext='ogv'), |
382 | tmp.join('transfers', 'NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW.ogv') |
383 | ) |
384 | |
385 | # Test to make sure hashes are getting checked with safe_b32(): |
386 | bad = 'NWBNVXVK5..GIOW7MYR4K3KA5K22W7NW' |
387 | - e = raises(ValueError, inst.temp, bad) |
388 | + e = raises(ValueError, inst.tmp, bad) |
389 | self.assertEqual( |
390 | str(e), |
391 | 'b32: cannot b32decode %r: Non-base32 digit found' % bad |
392 | ) |
393 | - e = raises(ValueError, inst.temp, bad, ext='ogv') |
394 | + e = raises(ValueError, inst.tmp, bad, ext='ogv') |
395 | self.assertEqual( |
396 | str(e), |
397 | 'b32: cannot b32decode %r: Non-base32 digit found' % bad |
398 | @@ -779,7 +780,7 @@ |
399 | # Test to make sure ext is getting checked with safe_ext(): |
400 | chash = 'NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW' |
401 | bad = '/../../../.ssh/id_pub' |
402 | - e = raises(ValueError, inst.temp, chash, bad) |
403 | + e = raises(ValueError, inst.tmp, chash, bad) |
404 | self.assertEqual( |
405 | str(e), |
406 | 'ext %r does not match pattern %r' % (bad, EXT_PAT) |
407 | @@ -794,13 +795,13 @@ |
408 | self.assertFalse(path.exists(f)) |
409 | self.assertFalse(path.exists(d)) |
410 | self.assertEqual( |
411 | - inst.temp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW'), |
412 | + inst.tmp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW'), |
413 | f |
414 | ) |
415 | self.assertFalse(path.exists(f)) |
416 | self.assertFalse(path.exists(d)) |
417 | self.assertEqual( |
418 | - inst.temp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW', create=True), |
419 | + inst.tmp('NWBNVXVK5DQGIOW7MYR4K3KA5K22W7NW', create=True), |
420 | f |
421 | ) |
422 | self.assertFalse(path.exists(f)) |
423 | @@ -894,7 +895,113 @@ |
424 | self.assertEqual(path.dirname(fp.name), imports) |
425 | self.assertTrue(fp.name.endswith('.mov')) |
426 | |
427 | - def test_finalize_transfer(self): |
428 | + def test_tmp_move(self): |
429 | + tmp = TempDir() |
430 | + base = tmp.join('.dmedia') |
431 | + dst_d = tmp.join('.dmedia', mov_hash[:2]) |
432 | + dst = tmp.join('.dmedia', mov_hash[:2], mov_hash[2:] + '.mov') |
433 | + inst = self.klass(base) |
434 | + |
435 | + # Test with wrong tmp_fp type |
436 | + e = raises(TypeError, inst.tmp_move, 17, mov_hash, 'mov') |
437 | + self.assertEqual(str(e), TYPE_ERROR % ('tmp_fp', file, int, 17)) |
438 | + self.assertFalse(path.exists(dst_d)) |
439 | + self.assertFalse(path.exists(dst)) |
440 | + |
441 | + # Test with tmp_fp opened in wrong mode |
442 | + bad = tmp.touch('.dmedia', 'bad1.mov') |
443 | + for mode in ('r', 'w', 'r+'): |
444 | + fp = open(bad, mode) |
445 | + e = raises(ValueError, inst.tmp_move, fp, mov_hash, 'mov') |
446 | + self.assertEqual( |
447 | + str(e), |
448 | + "tmp_fp: mode must be 'rb', 'wb', or 'r+b'; got %r" % mode |
449 | + ) |
450 | + self.assertFalse(path.exists(dst_d)) |
451 | + self.assertFalse(path.exists(dst)) |
452 | + self.assertFalse(fp.closed) |
453 | + |
454 | + # Test when filename isn't contained in base (to ensure that |
455 | + # FileStore.check_path() is used to validate tmp_fp.name) |
456 | + bad = tmp.touch('.dmedia', '..', 'bad2.mov') |
457 | + fp = open(bad, 'rb') |
458 | + e = raises(FileStoreTraversal, inst.tmp_move, fp, mov_hash, 'mov') |
459 | + self.assertEqual(e.pathname, bad) |
460 | + self.assertEqual(e.abspath, tmp.join('bad2.mov')) |
461 | + self.assertEqual(e.base, base) |
462 | + self.assertFalse(path.exists(dst_d)) |
463 | + self.assertFalse(path.exists(dst)) |
464 | + self.assertFalse(fp.closed) |
465 | + |
466 | + # Test that chash is validated with safe_b32() |
467 | + good = tmp.write('yup', '.dmedia', 'imports', 'good.mov') |
468 | + os.chmod(good, 0o660) |
469 | + fp = open(good, 'rb') |
470 | + b32 = 'NWBNVXVK5DQGIOW7MYR4K3KA' |
471 | + e = raises(ValueError, inst.tmp_move, fp, b32, 'mov') |
472 | + self.assertEqual( |
473 | + str(e), |
474 | + "len(b32) must be 32; got 24: 'NWBNVXVK5DQGIOW7MYR4K3KA'" |
475 | + ) |
476 | + self.assertFalse(path.exists(dst_d)) |
477 | + self.assertFalse(path.exists(dst)) |
478 | + self.assertFalse(fp.closed) |
479 | + |
480 | + # Test that ext is validate with safe_ext() |
481 | + bad_ext = '/etc/ssh' |
482 | + e = raises(ValueError, inst.tmp_move, fp, mov_hash, bad_ext) |
483 | + self.assertEqual( |
484 | + str(e), |
485 | + 'ext %r does not match pattern %r' % (bad_ext, EXT_PAT) |
486 | + ) |
487 | + self.assertFalse(path.exists(dst_d)) |
488 | + self.assertFalse(path.exists(dst)) |
489 | + self.assertFalse(fp.closed) |
490 | + |
491 | + # Test when tmp_fp is closed |
492 | + fp.close() |
493 | + e = raises(ValueError, inst.tmp_move, fp, mov_hash, 'mov') |
494 | + self.assertEqual( |
495 | + str(e), |
496 | + 'tmp_fp is closed, must be open: %r' % good |
497 | + ) |
498 | + self.assertFalse(path.exists(dst_d)) |
499 | + self.assertFalse(path.exists(dst)) |
500 | + self.assertTrue(fp.closed) |
501 | + |
502 | + # Test when it's all good |
503 | + fp = open(good, 'rb') |
504 | + self.assertEqual(stat.S_IMODE(os.stat(good).st_mode), 0o660) |
505 | + self.assertEqual(inst.tmp_move(fp, mov_hash, 'mov'), dst) |
506 | + self.assertFalse(path.exists(good)) |
507 | + self.assertTrue(path.isdir(dst_d)) |
508 | + self.assertTrue(path.isfile(dst)) |
509 | + self.assertEqual(stat.S_IMODE(os.stat(dst).st_mode), 0o444) |
510 | + self.assertEqual(open(dst, 'rb').read(), 'yup') |
511 | + self.assertTrue(fp.closed) |
512 | + |
513 | + # Test when it's a duplicate |
514 | + dup = tmp.write('wowza', '.dmedia', 'imports', 'dup') |
515 | + os.chmod(dup, 0o660) |
516 | + fp = open(dup, 'rb') |
517 | + e = raises(DuplicateFile, inst.tmp_move, fp, mov_hash, 'mov') |
518 | + self.assertEqual(e.chash, mov_hash) |
519 | + self.assertEqual(e.src, dup) |
520 | + self.assertEqual(e.dst, dst) |
521 | + self.assertFalse(fp.closed) |
522 | + |
523 | + # Make sure dup wasn't altered |
524 | + self.assertTrue(path.isfile(dup)) |
525 | + self.assertEqual(stat.S_IMODE(os.stat(dup).st_mode), 0o660) |
526 | + self.assertEqual(open(dup, 'rb').read(), 'wowza') |
527 | + |
528 | + # Make sure dst wasn't altered |
529 | + self.assertTrue(path.isdir(dst_d)) |
530 | + self.assertTrue(path.isfile(dst)) |
531 | + self.assertEqual(stat.S_IMODE(os.stat(dst).st_mode), 0o444) |
532 | + self.assertEqual(open(dst, 'rb').read(), 'yup') |
533 | + |
534 | + def test_tmp_verify_move(self): |
535 | tmp = TempDir() |
536 | inst = self.klass(tmp.path) |
537 | |
538 | @@ -904,22 +1011,22 @@ |
539 | dst = tmp.join(mov_hash[:2], mov_hash[2:] + '.mov') |
540 | |
541 | # Test when transfers/ dir doesn't exist: |
542 | - e = raises(IOError, inst.finalize_transfer, mov_hash, 'mov') |
543 | + e = raises(IOError, inst.tmp_verify_move, mov_hash, 'mov') |
544 | self.assertFalse(path.exists(src_d)) |
545 | self.assertFalse(path.exists(dst_d)) |
546 | self.assertFalse(path.exists(dst)) |
547 | |
548 | # Test when transfers/ exists but file does not: |
549 | - self.assertEqual(inst.temp(mov_hash, 'mov', create=True), src) |
550 | + self.assertEqual(inst.tmp(mov_hash, 'mov', create=True), src) |
551 | self.assertTrue(path.isdir(src_d)) |
552 | - e = raises(IOError, inst.finalize_transfer, mov_hash, 'mov') |
553 | + e = raises(IOError, inst.tmp_verify_move, mov_hash, 'mov') |
554 | self.assertFalse(path.exists(src)) |
555 | self.assertFalse(path.exists(dst_d)) |
556 | self.assertFalse(path.exists(dst)) |
557 | |
558 | # Test when file has wrong content hash and wrong size: |
559 | open(src, 'wb').write(open(sample_thm, 'rb').read()) |
560 | - e = raises(IntegrityError, inst.finalize_transfer, mov_hash, 'mov') |
561 | + e = raises(IntegrityError, inst.tmp_verify_move, mov_hash, 'mov') |
562 | self.assertEqual(e.got, thm_hash) |
563 | self.assertEqual(e.expected, mov_hash) |
564 | self.assertEqual(e.filename, src) |
565 | @@ -942,7 +1049,7 @@ |
566 | fp2.close() |
567 | self.assertEqual(path.getsize(sample_mov), path.getsize(src)) |
568 | |
569 | - e = raises(IntegrityError, inst.finalize_transfer, mov_hash, 'mov') |
570 | + e = raises(IntegrityError, inst.tmp_verify_move, mov_hash, 'mov') |
571 | self.assertEqual(e.got, 'UECTT7A7EIHZ2SGGBMMO5WTTSVU4SUWM') |
572 | self.assertEqual(e.expected, mov_hash) |
573 | self.assertEqual(e.filename, src) |
574 | @@ -959,7 +1066,7 @@ |
575 | fp2.write(chunk) |
576 | fp1.close() |
577 | fp2.close() |
578 | - self.assertEqual(inst.finalize_transfer(mov_hash, 'mov'), dst) |
579 | + self.assertEqual(inst.tmp_verify_move(mov_hash, 'mov'), dst) |
580 | self.assertTrue(path.isdir(src_d)) |
581 | self.assertFalse(path.exists(src)) |
582 | self.assertTrue(path.isdir(dst_d)) |
Looks decent :)