Merge lp:~bac/lpbuildbot/cleanup-testr into lp:lpbuildbot

[Brad Crittenden]

Add functionality at startup to clear out old run data from the testrepository directory. The init_testr script is the right place to do it.

I took the opportunity to rewrite the script in python which was faster than trying to do something mildly complicated in bash.

[lp:~bac/lpbuildbot/cleanup-testr]

29. By Brad Crittenden on 2012-05-17

Use os.symlink

=== modified file 'master.cfg'
--- master.cfg	2012-05-08 12:06:48 +0000
+++ master.cfg	2012-05-17 14:21:17 +0000
@@ -92,11 +92,11 @@
         description=['cleanup','previous','lxc','containers'],
         descriptionDone=['cleaned','previous','lxc','containers']))
     fac.addStep(transfer.FileDownload(
-        mastersrc='scripts/init_testr.sh',
+        mastersrc='scripts/init_testr.py',
         slavedest='init_testr.sh',
         mode=0544))
     fac.addStep(bzrbuildbot.shell.ShellCommand(
-        command=['./init_testr.sh'],
+        command=['./init_testr.py'],
         description=['initializing', 'testrepository'],
         descriptionDone=['initialized', 'testrepository']))
     # Make a temp dir, to work around bug 808557.
 
=== added file 'scripts/init_testr.py'
--- scripts/init_testr.py	1970-01-01 00:00:00 +0000
+++ scripts/init_testr.py	2012-05-17 14:21:17 +0000
@@ -0,0 +1,138 @@
+#!/usr/bin/python
+# Copyright 2012 Canonical Ltd. This software is licensed under the GNU
+# Affero General Public License version 3 (see the file LICENSE).
+"""
+Initialize testrepository, re-using the results from a previous run,
+if available.
+
+The results from a run will be stored in the directory above the
+current working directory and symlinked from the cwd.
+"""
+
+import os
+import subprocess
+import sys
+
+
+TESTR_DIR = ".testrepository"
+# Keep ten total runs, current run plus nine old ones.
+NUM_RESULTS_TO_KEEP = 9
+
+
+def run(cmd):
+    args = cmd.split()
+    process = subprocess.Popen(
+        args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = process.communicate()
+    if process.returncode:
+        print "Error running '%s': %d" % (
+            cmd, process.returncode)
+        return process.returncode
+
+
+def is_result_file(fn):
+    """Result files are named as monotonically increasing integers.
+
+    >>> is_result_file('1')
+    True
+    >>> is_result_file('7up')
+    False
+    """
+    try:
+        int(fn)
+    except ValueError:
+        return False
+    return True
+
+
+def find_deletion_candidates(filenames, keepers=NUM_RESULTS_TO_KEEP):
+    """Given a set of files, find those that are result files that should be
+    deleted.
+
+    >>> find_deletion_candidates(['1','2','5','10','7','c','11'], keepers=1)
+    [1, 2, 5, 7, 10]
+    >>> find_deletion_candidates(['1','2','5','10','7','c','11'], keepers=4)
+    [1, 2]
+    >>> find_deletion_candidates(['1','2','5','10','7','c','11'], keepers=40)
+    []
+    """
+    results = sorted([int(fn) for fn in filenames if is_result_file(fn)])
+    return results[:-keepers]
+
+
+def remove_files(names, path):
+    """Given a list of file names and the path to them, remove the files.
+    Return a \n separated list of output messages.
+    >>> import tempfile
+    >>> def makefiles(names, path):
+    ...    for n in names:
+    ...       open(os.path.join(path, n), 'w').close()
+    >>> tempdir = tempfile.mkdtemp()
+    >>> print sorted(os.listdir(tempdir))
+    []
+    >>> filenames = ['1','2','3']
+    >>> makefiles(filenames, tempdir)
+    >>> print sorted(os.listdir(tempdir))
+    ['1', '2', '3']
+    >>> rc, msg = remove_files(filenames, tempdir)
+    >>> print rc
+    0
+    >>> print msg
+    Removing 3 old testrepository runs
+    >>> print sorted(os.listdir(tempdir))
+    []
+    >>> makefiles(filenames, tempdir)
+    >>> rc, msg = remove_files(filenames + ['99'], tempdir)
+    >>> print rc
+    1
+    >>> print msg   # doctest:+ELLIPSIS
+    Removing 4 old testrepository runs
+    [Errno 2] No such file or directory:...
+    >>> os.rmdir(tempdir)
+    """
+    msg = []
+    rc = 0
+    if names:
+        msg.append("Removing %d old testrepository runs" % len(names))
+    for fn in names:
+        try:
+            filepath = os.path.join(path, str(fn))
+            os.unlink(filepath)
+        except OSError as e:
+            msg.append(str(e))
+            rc = 1
+            break
+    return rc, '\n'.join(msg)
+
+
+def main():
+    # If there is no .testrepository directory in the parent directory,
+    # make one.  We keep it there so it will persist the launchpad directory
+    # being removed after a test run.
+    rc = 0
+    target_dir = os.path.join('..', TESTR_DIR)
+    if not os.path.exists(target_dir):
+        cwd = os.getcwd()
+        os.chdir('..')
+        rc = run('testr init')
+        if rc:
+            return rc
+        os.chdir(cwd)
+    # Create a symbolic link to the repository directory in the parent
+    # directory.
+    if not os.path.exists(TESTR_DIR):
+        os.symlink(target_dir, TESTR_DIR)
+
+    # Each run of the slave will create a set of results in the testrepository
+    # directory.  If we never clean up the old ones they will take up too much
+    # space.  We'll keep a limited number and remove the rest.
+    files = os.listdir(target_dir)
+    goners = find_deletion_candidates(files)
+    rc, msg = remove_files(goners, target_dir)
+    if msg:
+        print msg
+    return rc
+
+
+if __name__ == '__main__':
+    sys.exit(main())
 
=== removed file 'scripts/init_testr.sh'
--- scripts/init_testr.sh	2012-04-02 18:52:40 +0000
+++ scripts/init_testr.sh	1970-01-01 00:00:00 +0000
@@ -1,15 +0,0 @@
-#!/bin/sh
-
-# Initialize testrepository, re-using the results from a previous run,
-# if available.
-
-# The results from a run will be stored in the directory above the
-# current working directory and symlinked from the cwd.
-
-# If there is no .testrepository directory in the parent directory,
-# make one.
-if [ ! -d '../.testrepository' ]]; then
-  (cd ..; testr init)
-fi
-
-ln -s ../.testrepository/