Merge lp:~mvo/software-center/expunge-cache into lp:software-center

[Michael Vogt]

This adds a helper that keeps the cache clean from cached 301/404 etc responses.
It also offers cleanup by mtime but that is currently not used (only tested)

[Gary Lasker (gary-lasker)]

Nice work, thanks mvo!!

review: Approve

=== added symlink 'data/expunge-cache.py'
=== target is u'../utils/expunge-cache.py'
=== modified file 'setup.py'
--- setup.py	2012-01-27 10:47:16 +0000
+++ setup.py	2012-02-29 17:32:19 +0000
@@ -94,6 +94,8 @@
                "utils/update-software-center",
                "utils/update-software-center-channels",
                "utils/update-software-center-agent",
+               # generic helpers
+               "utils/expunge-cache.py",
                ] + glob.glob("utils/piston-helpers/*.py"),
       packages=['softwarecenter',
                 'softwarecenter.backend',

=== modified file 'softwarecenter/ui/gtk3/app.py'
--- softwarecenter/ui/gtk3/app.py	2012-02-29 07:15:46 +0000
+++ softwarecenter/ui/gtk3/app.py	2012-02-29 17:32:19 +0000
@@ -390,6 +390,10 @@
             # running the agent will trigger a db reload so we do it later
             GObject.timeout_add_seconds(30, self._run_software_center_agent)
 
+
+        # keep the cache clean
+        GObject.timeout_add_seconds(15, self._run_expunge_cache_helper)
+
         # TODO: Remove the following two lines once we have remove repository
         #       support in aptdaemon (see LP: #723911)
         file_menu = self.builder.get_object("menu1")
@@ -419,6 +423,16 @@
         GObject.child_watch_add(
             pid, self._on_update_software_center_agent_finished)
 
+    def _run_expunge_cache_helper(self):
+        """ helper that expires the piston-mini-client cache """
+        sc_expunge_cache = os.path.join(
+            self.datadir, "expunge-cache.py")
+        (pid, stdin, stdout, stderr) = GObject.spawn_async(
+            [sc_expunge_cache, 
+             "--by-unsuccessful-http-states",
+             softwarecenter.paths.SOFTWARE_CENTER_CACHE_DIR,
+             ])
+
     def _rebuild_and_reopen_local_db(self, pathname):
         """ helper that rebuilds a db and reopens it """
         from softwarecenter.db.update import rebuild_database

=== modified file 'test/test_utils.py'
--- test/test_utils.py	2012-02-12 20:03:56 +0000
+++ test/test_utils.py	2012-02-29 17:32:19 +0000
@@ -124,6 +124,42 @@
         uuid = get_uuid()
         self.assertTrue(uuid and len(uuid) > 0)
 
+class TestExpungeCache(unittest.TestCase):
+
+    def test_expunge_cache(self):
+        import subprocess
+        import tempfile
+        dirname = tempfile.mkdtemp('s-c-testsuite')
+        for name, content in [ ("foo-301", "status: 301"),
+                               ("foo-200", "status: 200"),
+                               ("foo-random", "random"),
+                             ]:
+            fullpath = os.path.join(dirname, name)
+            open(fullpath, "w").write(content)
+            # set to 1970+1s time to ensure the cleaner finds it
+            os.utime(fullpath, (1,1))
+        res = subprocess.call(["../utils/expunge-cache.py", dirname])
+        # no arguments
+        self.assertEqual(res, 1)
+        # by status
+        res = subprocess.call(["../utils/expunge-cache.py",
+                               "--debug",
+                               "--by-unsuccessful-http-states",
+                               dirname])
+        self.assertFalse(os.path.exists(os.path.join(dirname, "foo-301")))
+        self.assertTrue(os.path.exists(os.path.join(dirname, "foo-200")))
+        self.assertTrue(os.path.exists(os.path.join(dirname, "foo-random")))
+
+        # by time 
+        res = subprocess.call(["../utils/expunge-cache.py",
+                               "--debug",
+                               "--by-days", "1",
+                               dirname])
+        # now we expect the old file to be gone but the unknown one not to
+        # be touched
+        self.assertFalse(os.path.exists(os.path.join(dirname, "foo-200")))
+        self.assertTrue(os.path.exists(os.path.join(dirname, "foo-random")))
+
 
 if __name__ == "__main__":
     import logging

=== added file 'utils/expunge-cache.py'
--- utils/expunge-cache.py	1970-01-01 00:00:00 +0000
+++ utils/expunge-cache.py	2012-02-29 17:32:19 +0000
@@ -0,0 +1,80 @@
+#!/usr/bin/python
+
+"""
+Expunge httplib2 caches
+"""
+
+import argparse
+import logging
+import os
+import time
+import sys
+
+class ExpungeCache(object):
+    def __init__(self, dirs, args):
+        self.dirs = dirs
+        # days to keep data in the cache (0 == disabled)
+        self.keep_time = 60*60*24* args.by_days
+        self.keep_only_http200 = args.by_unsuccessful_http_states
+        self.dry_run = args.dry_run
+
+    def _rm(self, f):
+        if self.dry_run:
+            print "Would delete: %s" % f
+        else:
+            logging.debug("Deleting: %s" % f)
+            os.unlink(f)
+
+    def clean(self):
+        # go over the directories
+        now = time.time()
+        for d in self.dirs:
+            for root, dirs, files in os.walk(d):
+                for f in files:
+                    fullpath = os.path.join(root, f)
+                    header = open(fullpath).readline().strip()
+                    if not header.startswith("status:"):
+                        logging.debug(
+                            "Skipping files with unknown header: '%s'" % f)
+                        continue
+                    if self.keep_only_http200 and header != "status: 200":
+                        self._rm(fullpath)
+                    if self.keep_time:
+                        mtime = os.path.getmtime(fullpath)
+                        logging.debug("mtime of '%s': '%s" % (f, mtime))
+                        if (mtime + self.keep_time) < now:
+                            self._rm(fullpath)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description='clean software-center httplib2 cache')
+    parser.add_argument(
+        '--debug', action="store_true",
+        help='show debug output')
+    parser.add_argument(
+        '--dry-run', action="store_true",
+        help='do not act, just show what would be done')
+    parser.add_argument(
+        'directories', metavar='directory', nargs='+', type=str,
+        help='directories to be checked')
+    parser.add_argument(
+        '--by-days', type=int, default=0,
+        help='expire everything older than N days')
+    parser.add_argument(
+        '--by-unsuccessful-http-states', action="store_true",
+        help='expire any non 200 status responses')
+    args = parser.parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
+    # sanity checking
+    if args.by_days == 0 and not args.by_unsuccessful_http_states:
+        print "Need either --by-days or --by-unsuccessful-http-states argument"
+        sys.exit(1)
+
+    # do it
+    cleaner = ExpungeCache(args.directories, args)
+    cleaner.clean()