Merge lp:~bac/lpbuildbot/cleanup-testr into lp:lpbuildbot

Proposed by Brad Crittenden
Status: Merged
Approved by: Gary Poster
Approved revision: 29
Merged at revision: 29
Proposed branch: lp:~bac/lpbuildbot/cleanup-testr
Merge into: lp:lpbuildbot
Diff against target: 180 lines (+140/-17)
3 files modified
master.cfg (+2/-2)
scripts/init_testr.py (+138/-0)
scripts/init_testr.sh (+0/-15)
To merge this branch: bzr merge lp:~bac/lpbuildbot/cleanup-testr
Reviewer Review Type Date Requested Status
Yellow Squad code Pending
Review via email: mp+106197@code.launchpad.net

Description of the change

Add functionality at startup to clear out old run data from the testrepository directory. The init_testr script is the right place to do it.

I took the opportunity to rewrite the script in python which was faster than trying to do something mildly complicated in bash.

To post a comment you must log in.
lp:~bac/lpbuildbot/cleanup-testr updated
29. By Brad Crittenden on 2012-05-17

Use os.symlink

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'master.cfg'
--- master.cfg 2012-05-08 12:06:48 +0000
+++ master.cfg 2012-05-17 14:21:17 +0000
@@ -92,11 +92,11 @@
92 description=['cleanup','previous','lxc','containers'],92 description=['cleanup','previous','lxc','containers'],
93 descriptionDone=['cleaned','previous','lxc','containers']))93 descriptionDone=['cleaned','previous','lxc','containers']))
94 fac.addStep(transfer.FileDownload(94 fac.addStep(transfer.FileDownload(
95 mastersrc='scripts/init_testr.sh',95 mastersrc='scripts/init_testr.py',
96 slavedest='init_testr.sh',96 slavedest='init_testr.sh',
97 mode=0544))97 mode=0544))
98 fac.addStep(bzrbuildbot.shell.ShellCommand(98 fac.addStep(bzrbuildbot.shell.ShellCommand(
99 command=['./init_testr.sh'],99 command=['./init_testr.py'],
100 description=['initializing', 'testrepository'],100 description=['initializing', 'testrepository'],
101 descriptionDone=['initialized', 'testrepository']))101 descriptionDone=['initialized', 'testrepository']))
102 # Make a temp dir, to work around bug 808557.102 # Make a temp dir, to work around bug 808557.
103103
=== added file 'scripts/init_testr.py'
--- scripts/init_testr.py 1970-01-01 00:00:00 +0000
+++ scripts/init_testr.py 2012-05-17 14:21:17 +0000
@@ -0,0 +1,138 @@
1#!/usr/bin/python
2# Copyright 2012 Canonical Ltd. This software is licensed under the GNU
3# Affero General Public License version 3 (see the file LICENSE).
4"""
5Initialize testrepository, re-using the results from a previous run,
6if available.
7
8The results from a run will be stored in the directory above the
9current working directory and symlinked from the cwd.
10"""
11
12import os
13import subprocess
14import sys
15
16
17TESTR_DIR = ".testrepository"
18# Keep ten total runs, current run plus nine old ones.
19NUM_RESULTS_TO_KEEP = 9
20
21
22def run(cmd):
23 args = cmd.split()
24 process = subprocess.Popen(
25 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
26 stdout, stderr = process.communicate()
27 if process.returncode:
28 print "Error running '%s': %d" % (
29 cmd, process.returncode)
30 return process.returncode
31
32
33def is_result_file(fn):
34 """Result files are named as monotonically increasing integers.
35
36 >>> is_result_file('1')
37 True
38 >>> is_result_file('7up')
39 False
40 """
41 try:
42 int(fn)
43 except ValueError:
44 return False
45 return True
46
47
48def find_deletion_candidates(filenames, keepers=NUM_RESULTS_TO_KEEP):
49 """Given a set of files, find those that are result files that should be
50 deleted.
51
52 >>> find_deletion_candidates(['1','2','5','10','7','c','11'], keepers=1)
53 [1, 2, 5, 7, 10]
54 >>> find_deletion_candidates(['1','2','5','10','7','c','11'], keepers=4)
55 [1, 2]
56 >>> find_deletion_candidates(['1','2','5','10','7','c','11'], keepers=40)
57 []
58 """
59 results = sorted([int(fn) for fn in filenames if is_result_file(fn)])
60 return results[:-keepers]
61
62
63def remove_files(names, path):
64 """Given a list of file names and the path to them, remove the files.
65 Return a \n separated list of output messages.
66 >>> import tempfile
67 >>> def makefiles(names, path):
68 ... for n in names:
69 ... open(os.path.join(path, n), 'w').close()
70 >>> tempdir = tempfile.mkdtemp()
71 >>> print sorted(os.listdir(tempdir))
72 []
73 >>> filenames = ['1','2','3']
74 >>> makefiles(filenames, tempdir)
75 >>> print sorted(os.listdir(tempdir))
76 ['1', '2', '3']
77 >>> rc, msg = remove_files(filenames, tempdir)
78 >>> print rc
79 0
80 >>> print msg
81 Removing 3 old testrepository runs
82 >>> print sorted(os.listdir(tempdir))
83 []
84 >>> makefiles(filenames, tempdir)
85 >>> rc, msg = remove_files(filenames + ['99'], tempdir)
86 >>> print rc
87 1
88 >>> print msg # doctest:+ELLIPSIS
89 Removing 4 old testrepository runs
90 [Errno 2] No such file or directory:...
91 >>> os.rmdir(tempdir)
92 """
93 msg = []
94 rc = 0
95 if names:
96 msg.append("Removing %d old testrepository runs" % len(names))
97 for fn in names:
98 try:
99 filepath = os.path.join(path, str(fn))
100 os.unlink(filepath)
101 except OSError as e:
102 msg.append(str(e))
103 rc = 1
104 break
105 return rc, '\n'.join(msg)
106
107
108def main():
109 # If there is no .testrepository directory in the parent directory,
110 # make one. We keep it there so it will persist the launchpad directory
111 # being removed after a test run.
112 rc = 0
113 target_dir = os.path.join('..', TESTR_DIR)
114 if not os.path.exists(target_dir):
115 cwd = os.getcwd()
116 os.chdir('..')
117 rc = run('testr init')
118 if rc:
119 return rc
120 os.chdir(cwd)
121 # Create a symbolic link to the repository directory in the parent
122 # directory.
123 if not os.path.exists(TESTR_DIR):
124 os.symlink(target_dir, TESTR_DIR)
125
126 # Each run of the slave will create a set of results in the testrepository
127 # directory. If we never clean up the old ones they will take up too much
128 # space. We'll keep a limited number and remove the rest.
129 files = os.listdir(target_dir)
130 goners = find_deletion_candidates(files)
131 rc, msg = remove_files(goners, target_dir)
132 if msg:
133 print msg
134 return rc
135
136
137if __name__ == '__main__':
138 sys.exit(main())
0139
=== removed file 'scripts/init_testr.sh'
--- scripts/init_testr.sh 2012-04-02 18:52:40 +0000
+++ scripts/init_testr.sh 1970-01-01 00:00:00 +0000
@@ -1,15 +0,0 @@
1#!/bin/sh
2
3# Initialize testrepository, re-using the results from a previous run,
4# if available.
5
6# The results from a run will be stored in the directory above the
7# current working directory and symlinked from the cwd.
8
9# If there is no .testrepository directory in the parent directory,
10# make one.
11if [ ! -d '../.testrepository' ]]; then
12 (cd ..; testr init)
13fi
14
15ln -s ../.testrepository/

Subscribers

People subscribed via source and target branches