Merge lp:~jderose/filestore/purge_tmp into lp:filestore

[Jason Gerard DeRose]

Adds FileStore.purge_tmp() method, its test, and its documentation.

For more info, see the bug:

https://bugs.launchpad.net/filestore/+bug/1014214

[James Raymond (jamesmr)]

review: Approve

=== modified file 'doc/filestore.rst'
--- doc/filestore.rst	2012-03-22 17:07:54 +0000
+++ doc/filestore.rst	2012-06-17 07:47:20 +0000
@@ -539,7 +539,7 @@
     *batch* must be a :class:`Batch` namedtuple as returned by
     :func:`scandir()`.
 
-    You also must supply one of more *filestores*, which must be
+    You also must supply one or more *filestores*, which must be
     :class:`FileStore` instances.
 
     This function is a Python generator.  For each non-empty file in *batch*, it
@@ -761,6 +761,17 @@
     .. method:: remove(_id)
 
         Remove the file with *_id*.
+        
+    .. method:: purge_tmp()
+
+        Remove all files in .dmedia/tmp/
+
+        This method is for periodically purging stale temporary files. The
+        return value is a list with a :class:`File` namedtuple for each file
+        removed.
+
+        Symlinks are ignored and directories are not traversed, neither of which
+        should ever be created by the :class:`FileStore` itself.
 
 
     .. method:: verify(_id, return_fp=False)

=== modified file 'filestore.py'
--- filestore.py	2012-06-16 02:11:24 +0000
+++ filestore.py	2012-06-17 07:47:20 +0000
@@ -1252,6 +1252,28 @@
         log.info('Removing %r from %r', _id, self)
         os.remove(filename)
 
+    def purge_tmp(self):
+        """
+        Remove all files in .dmedia/tmp/
+
+        This method is for periodically purging stale temporary files. The
+        return value is a list with a `File` namedtuple for each file removed.
+
+        Symlinks are ignored and directories are not traversed, neither of which
+        should ever be created by the `FileStore` itself.
+        """
+        purged = []
+        for name in sorted(os.listdir(self.tmp)):
+            fullname = path.join(self.tmp, name)
+            st = os.lstat(fullname)
+            if stat.S_ISREG(st.st_mode):
+                purged.append(File(fullname, st.st_size, st.st_mtime))
+                log.info('Removing stale temporary file %r', fullname)
+                os.remove(fullname)
+            else:
+                log.info('Unexpected non-regular file %r', fullname)
+        return purged
+
     def allocate_tmp(self, size=None):
         """
         Allocate randomly named temporary file for an import or render.

=== modified file 'test_filestore.py'
--- test_filestore.py	2012-06-16 02:13:09 +0000
+++ test_filestore.py	2012-06-17 07:47:20 +0000
@@ -32,12 +32,14 @@
 from subprocess import check_call
 import tempfile
 import shutil
+from random import SystemRandom
 
 from skein import skein512
 
 import filestore
 
 
+random = SystemRandom()
 TYPE_ERROR = '{}: need a {!r}; got a {!r}: {!r}'
 
 B32NAMES = []
@@ -2137,6 +2139,76 @@
         fs.remove(_id)
         self.assertFalse(path.exists(canonical))
 
+    def test_purge_tmp(self):
+        tmp = TempDir()
+        t = tmp.join('.dmedia', 'tmp')
+        fs = filestore.FileStore(tmp.dir)
+        self.assertTrue(path.isdir(t))
+
+        # Test when nothing is in tmp:
+        self.assertEqual(os.listdir(t), [])
+        self.assertEqual(fs.purge_tmp(), [])
+        self.assertEqual(os.listdir(t), [])
+
+        # Test when only regular files are in tmp:
+        purged = []
+        for i in range(20):
+            size = random.randint(1, 1024 * 1024)
+            fp = fs.allocate_tmp(size)
+            purged.append(
+                filestore.File(fp.name, size, path.getmtime(fp.name))
+            )
+        purged.sort(key=lambda f: f.name)
+        self.assertEqual(fs.purge_tmp(), purged)
+        self.assertEqual(os.listdir(t), [])
+
+        # Test that directories aren't traversed:
+        nope = []
+        for d in ['aye', 'bee', 'see']:
+            for n in 'abcd':
+                nope.append(tmp.touch('.dmedia', 'tmp', d, n))
+        self.assertEqual(sorted(os.listdir(t)), ['aye', 'bee', 'see'])
+        self.assertEqual(fs.purge_tmp(), [])
+        self.assertEqual(sorted(os.listdir(t)), ['aye', 'bee', 'see'])
+        for f in nope:
+            self.assertTrue(path.isfile(f))
+
+        # Test that symlinks are ignored:
+        tmp2 = TempDir()
+        fs2 = filestore.FileStore(tmp2.dir)
+        names = ['aye', 'bee', 'see']
+        for i in range(10):
+            size = random.randint(1, 1024 * 1024)
+            fp = fs2.allocate_tmp(size)
+            name = path.basename(fp.name)
+            link = path.join(t, name)
+            os.symlink(fp.name, link)
+            names.append(name)
+            nope.append(link)
+        names.sort()
+        self.assertEqual(sorted(os.listdir(t)), names)
+        for f in nope:
+            self.assertTrue(path.isfile(f))
+
+        # Add some regular files, including zero-sized files:
+        purged = []
+        for i in range(20):
+            size = random.randint(1, 1024 * 1024)
+            fp = fs.allocate_tmp(size)
+            purged.append(
+                filestore.File(fp.name, size, path.getmtime(fp.name))
+            )
+        for i in range(10):
+            fp = fs.allocate_tmp()
+            purged.append(
+                filestore.File(fp.name, 0, path.getmtime(fp.name))
+            )
+        purged.sort(key=lambda f: f.name)
+        self.assertEqual(fs.purge_tmp(), purged)
+        self.assertEqual(sorted(os.listdir(t)), names)
+        for f in nope:
+            self.assertTrue(path.isfile(f))
+
     def test_allocate_tmp(self):
         tmp = TempDir()
         t = tmp.join('.dmedia', 'tmp')