Merge lp:~satwell/duplicity/caching into lp:duplicity/0.6

[Kenneth Loafman]

Add a cache for password and group lookups. This significantly improves runtime with very large password and group configurations.

=== added file 'duplicity/cached_ops.py'
--- duplicity/cached_ops.py	1970-01-01 00:00:00 +0000
+++ duplicity/cached_ops.py	2012-11-11 01:32:21 +0000
@@ -0,0 +1,62 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright 2012 Google Inc.
+#
+# This file is part of duplicity.
+#
+# Duplicity is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# Duplicity is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with duplicity; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+"""Cache-wrapped functions for grp and pwd lookups."""
+
+import grp
+import pwd
+
+
+class CachedCall(object):
+    """Decorator for caching the results of function calls."""
+
+    def __init__(self, f):
+        self.cache = {}
+        self.f = f
+
+    def __call__(self, *args):
+        try:
+            return self.cache[args]
+        except (KeyError, TypeError), e:
+            result = self.f(*args)
+            if not isinstance(e, TypeError):
+                # TypeError most likely means that args is not hashable
+                self.cache[args] = result
+            return result
+
+
+@CachedCall
+def getgrgid(gid):
+    return grp.getgrgid(gid)
+
+
+@CachedCall
+def getgrnam(name):
+    return grp.getgrnam(name)
+
+
+@CachedCall
+def getpwnam(name):
+    return pwd.getpwnam(name)
+
+
+@CachedCall
+def getpwuid(uid):
+    return pwd.getpwuid(uid)

=== modified file 'duplicity/path.py'
--- duplicity/path.py	2011-08-23 18:14:17 +0000
+++ duplicity/path.py	2012-11-11 01:32:21 +0000
@@ -26,7 +26,7 @@
 
 """
 
-import stat, errno, socket, time, re, gzip, pwd, grp
+import stat, errno, socket, time, re, gzip
 
 from duplicity import tarfile
 from duplicity import file_naming
@@ -36,6 +36,7 @@
 from duplicity import librsync
 from duplicity import log #@UnusedImport
 from duplicity import dup_time
+from duplicity import cached_ops
 from duplicity.lazy import * #@UnusedWildImport
 
 _copy_blocksize = 64 * 1024
@@ -206,13 +207,13 @@
         try:
             if globals.numeric_owner:
                 raise KeyError
-            self.stat.st_uid = pwd.getpwnam(tarinfo.uname)[2]
+            self.stat.st_uid = cached_ops.getpwnam(tarinfo.uname)[2]
         except KeyError:
             self.stat.st_uid = tarinfo.uid
         try:
             if globals.numeric_owner:
                 raise KeyError
-            self.stat.st_gid = grp.getgrnam(tarinfo.gname)[2]
+            self.stat.st_gid = cached_ops.getgrnam(tarinfo.gname)[2]
         except KeyError:
             self.stat.st_gid = tarinfo.gid
 
@@ -284,11 +285,11 @@
                 ti.mtime = int(self.stat.st_mtime)
 
             try:
-                ti.uname = pwd.getpwuid(ti.uid)[0]
+                ti.uname = cached_ops.getpwuid(ti.uid)[0]
             except KeyError:
                 ti.uname = ''
             try:
-                ti.gname = grp.getgrgid(ti.gid)[0]
+                ti.gname = cached_ops.getgrgid(ti.gid)[0]
             except KeyError:
                 ti.gname = ''
 

=== modified file 'duplicity/tarfile.py'
--- duplicity/tarfile.py	2011-10-08 17:40:11 +0000
+++ duplicity/tarfile.py	2012-11-11 01:32:21 +0000
@@ -53,10 +53,8 @@
 import re
 import operator
 
-try:
-    import grp, pwd
-except ImportError:
-    grp = pwd = None
+from duplicity import cached_ops
+grp = pwd = cached_ops
 
 # from tarfile import *
 __all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"]