Merge lp:~wence/fluidity/testharness-tests-from-file into lp:fluidity

[Lawrence Mitchell]

Allow running a set of tests using testharness by specifying a file that lists tests to run. Useful for break/test/debug/maybe-fix cycles.

[Patrick Farrell (pefarrell)]

review: Approve

=== modified file 'tools/testharness.py'
--- tools/testharness.py	2011-11-04 13:20:03 +0000
+++ tools/testharness.py	2012-10-15 10:06:24 +0000
@@ -18,8 +18,10 @@
   import elementtree.ElementTree as etree
 
 class TestHarness:
-    def __init__(self, length="any", parallel=False, exclude_tags=None, tags=None, file="", verbose=True, justtest=False,
-        valgrind=False):
+    def __init__(self, length="any", parallel=False, exclude_tags=None,
+                 tags=None, file="", from_file=None,
+                 verbose=True, justtest=False,
+                 valgrind=False):
         self.tests = []
         self.verbose = verbose
         self.length = length
@@ -71,14 +73,31 @@
         # step 2. if the user has specified a particular file, let's use that.
 
         if file != "":
+          files = [file]
+        elif from_file:
+          try:
+            f = open(from_file, 'r')
+            files = [line[:-1] for line in f.readlines()]
+          except IOError as e:
+            sys.stderr.write("Unable to read tests from file %s: %s" % (from_file, e))
+            sys.exit(1)
+          f.close()
+        else:
+          files = None
+
+        if files:
           for (subdir, xml_file) in [os.path.split(x) for x in xml_files]:
-            if xml_file == file:
+            if xml_file in files:
               testprob = regressiontest.TestProblem(filename=os.path.join(subdir, xml_file),
                            verbose=self.verbose, replace=self.modify_command_line())
-              self.tests = [(subdir, testprob)]
-              return
-          print "Could not find file %s." % file
-          sys.exit(1)
+              self.tests.append((subdir, testprob))
+              files.remove(xml_file)
+          if files != []:
+            print "Could not find the following specified test files:"
+            for f in files:
+              print f
+            sys.exit(1)
+          return
 
         # step 3. form a cut-down list of the xml files matching the correct length and the correct parallelism.
         working_set = []
@@ -328,6 +347,8 @@
     parser.add_option("-e", "--exclude-tags", dest="exclude_tags", help="run only tests that do not have specific tags (takes precidence over -t)", default=[], action="append")
     parser.add_option("-t", "--tags", dest="tags", help="run tests with specific tags", default=[], action="append")
     parser.add_option("-f", "--file", dest="file", help="specific test case to run (by filename)", default="")
+    parser.add_option("--from-file", dest="from_file", default=None,
+                      help="run tests listed in FROM_FILE (one test per line)")
     parser.add_option("-n", "--threads", dest="thread_count", type="int",
                       help="number of tests to run at the same time", default=1)
     parser.add_option("-v", "--valgrind", action="store_true", dest="valgrind")
@@ -367,8 +388,12 @@
     else:
       tags = options.tags
 
-    testharness = TestHarness(length=options.length, parallel=para, exclude_tags=exclude_tags, tags=tags, file=options.file, verbose=True,
-        justtest=options.justtest, valgrind=options.valgrind)
+    testharness = TestHarness(length=options.length, parallel=para,
+                              exclude_tags=exclude_tags, tags=tags,
+                              file=options.file, verbose=True,
+                              justtest=options.justtest,
+                              valgrind=options.valgrind,
+                              from_file=options.from_file)
 
     if options.justlist:
       testharness.list()