Merge lp:~bac/lpbuildbot/cleanup-testr into lp:lpbuildbot

Proposed by Brad Crittenden
Status: Merged
Approved by: Gary Poster
Approved revision: 29
Merged at revision: 29
Proposed branch: lp:~bac/lpbuildbot/cleanup-testr
Merge into: lp:lpbuildbot
Diff against target: 180 lines (+140/-17)
3 files modified
master.cfg (+2/-2)
scripts/init_testr.py (+138/-0)
scripts/init_testr.sh (+0/-15)
To merge this branch: bzr merge lp:~bac/lpbuildbot/cleanup-testr
Reviewer Review Type Date Requested Status
Yellow Squad code Pending
Review via email: mp+106197@code.launchpad.net

Description of the change

Add functionality at startup to clear out old run data from the testrepository directory. The init_testr script is the right place to do it.

I took the opportunity to rewrite the script in python which was faster than trying to do something mildly complicated in bash.

To post a comment you must log in.
lp:~bac/lpbuildbot/cleanup-testr updated
29. By Brad Crittenden on 2012-05-17

Use os.symlink

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'master.cfg'
2--- master.cfg 2012-05-08 12:06:48 +0000
3+++ master.cfg 2012-05-17 14:21:17 +0000
4@@ -92,11 +92,11 @@
5 description=['cleanup','previous','lxc','containers'],
6 descriptionDone=['cleaned','previous','lxc','containers']))
7 fac.addStep(transfer.FileDownload(
8- mastersrc='scripts/init_testr.sh',
9+ mastersrc='scripts/init_testr.py',
10 slavedest='init_testr.sh',
11 mode=0544))
12 fac.addStep(bzrbuildbot.shell.ShellCommand(
13- command=['./init_testr.sh'],
14+ command=['./init_testr.py'],
15 description=['initializing', 'testrepository'],
16 descriptionDone=['initialized', 'testrepository']))
17 # Make a temp dir, to work around bug 808557.
18
19=== added file 'scripts/init_testr.py'
20--- scripts/init_testr.py 1970-01-01 00:00:00 +0000
21+++ scripts/init_testr.py 2012-05-17 14:21:17 +0000
22@@ -0,0 +1,138 @@
23+#!/usr/bin/python
24+# Copyright 2012 Canonical Ltd. This software is licensed under the GNU
25+# Affero General Public License version 3 (see the file LICENSE).
26+"""
27+Initialize testrepository, re-using the results from a previous run,
28+if available.
29+
30+The results from a run will be stored in the directory above the
31+current working directory and symlinked from the cwd.
32+"""
33+
34+import os
35+import subprocess
36+import sys
37+
38+
39+TESTR_DIR = ".testrepository"
40+# Keep ten total runs, current run plus nine old ones.
41+NUM_RESULTS_TO_KEEP = 9
42+
43+
44+def run(cmd):
45+ args = cmd.split()
46+ process = subprocess.Popen(
47+ args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
48+ stdout, stderr = process.communicate()
49+ if process.returncode:
50+ print "Error running '%s': %d" % (
51+ cmd, process.returncode)
52+ return process.returncode
53+
54+
55+def is_result_file(fn):
56+ """Result files are named as monotonically increasing integers.
57+
58+ >>> is_result_file('1')
59+ True
60+ >>> is_result_file('7up')
61+ False
62+ """
63+ try:
64+ int(fn)
65+ except ValueError:
66+ return False
67+ return True
68+
69+
70+def find_deletion_candidates(filenames, keepers=NUM_RESULTS_TO_KEEP):
71+ """Given a set of files, find those that are result files that should be
72+ deleted.
73+
74+ >>> find_deletion_candidates(['1','2','5','10','7','c','11'], keepers=1)
75+ [1, 2, 5, 7, 10]
76+ >>> find_deletion_candidates(['1','2','5','10','7','c','11'], keepers=4)
77+ [1, 2]
78+ >>> find_deletion_candidates(['1','2','5','10','7','c','11'], keepers=40)
79+ []
80+ """
81+ results = sorted([int(fn) for fn in filenames if is_result_file(fn)])
82+ return results[:-keepers]
83+
84+
85+def remove_files(names, path):
86+ """Given a list of file names and the path to them, remove the files.
87+ Return a \n separated list of output messages.
88+ >>> import tempfile
89+ >>> def makefiles(names, path):
90+ ... for n in names:
91+ ... open(os.path.join(path, n), 'w').close()
92+ >>> tempdir = tempfile.mkdtemp()
93+ >>> print sorted(os.listdir(tempdir))
94+ []
95+ >>> filenames = ['1','2','3']
96+ >>> makefiles(filenames, tempdir)
97+ >>> print sorted(os.listdir(tempdir))
98+ ['1', '2', '3']
99+ >>> rc, msg = remove_files(filenames, tempdir)
100+ >>> print rc
101+ 0
102+ >>> print msg
103+ Removing 3 old testrepository runs
104+ >>> print sorted(os.listdir(tempdir))
105+ []
106+ >>> makefiles(filenames, tempdir)
107+ >>> rc, msg = remove_files(filenames + ['99'], tempdir)
108+ >>> print rc
109+ 1
110+ >>> print msg # doctest:+ELLIPSIS
111+ Removing 4 old testrepository runs
112+ [Errno 2] No such file or directory:...
113+ >>> os.rmdir(tempdir)
114+ """
115+ msg = []
116+ rc = 0
117+ if names:
118+ msg.append("Removing %d old testrepository runs" % len(names))
119+ for fn in names:
120+ try:
121+ filepath = os.path.join(path, str(fn))
122+ os.unlink(filepath)
123+ except OSError as e:
124+ msg.append(str(e))
125+ rc = 1
126+ break
127+ return rc, '\n'.join(msg)
128+
129+
130+def main():
131+ # If there is no .testrepository directory in the parent directory,
132+ # make one. We keep it there so it will persist the launchpad directory
133+ # being removed after a test run.
134+ rc = 0
135+ target_dir = os.path.join('..', TESTR_DIR)
136+ if not os.path.exists(target_dir):
137+ cwd = os.getcwd()
138+ os.chdir('..')
139+ rc = run('testr init')
140+ if rc:
141+ return rc
142+ os.chdir(cwd)
143+ # Create a symbolic link to the repository directory in the parent
144+ # directory.
145+ if not os.path.exists(TESTR_DIR):
146+ os.symlink(target_dir, TESTR_DIR)
147+
148+ # Each run of the slave will create a set of results in the testrepository
149+ # directory. If we never clean up the old ones they will take up too much
150+ # space. We'll keep a limited number and remove the rest.
151+ files = os.listdir(target_dir)
152+ goners = find_deletion_candidates(files)
153+ rc, msg = remove_files(goners, target_dir)
154+ if msg:
155+ print msg
156+ return rc
157+
158+
159+if __name__ == '__main__':
160+ sys.exit(main())
161
162=== removed file 'scripts/init_testr.sh'
163--- scripts/init_testr.sh 2012-04-02 18:52:40 +0000
164+++ scripts/init_testr.sh 1970-01-01 00:00:00 +0000
165@@ -1,15 +0,0 @@
166-#!/bin/sh
167-
168-# Initialize testrepository, re-using the results from a previous run,
169-# if available.
170-
171-# The results from a run will be stored in the directory above the
172-# current working directory and symlinked from the cwd.
173-
174-# If there is no .testrepository directory in the parent directory,
175-# make one.
176-if [ ! -d '../.testrepository' ]]; then
177- (cd ..; testr init)
178-fi
179-
180-ln -s ../.testrepository/

Subscribers

People subscribed via source and target branches