Merge lp:~jon-hill/fluidity/longtest_fix into lp:fluidity

[Jon Hill]

Added option --no_pbs to testharness which allows for the testing of longtests on local machines with no hacking of the XML. It will run on the number of processes specified in the XML.

Tim: can you check this doesn't break on CX1, please?

[Tim Greaves (tim-greaves)]

Thanks Jon! I've merged in the two changes to the debug install in the amcgsoft account and am re-running coarse-corner now on those.

[Tim Greaves (tim-greaves)]

That just ran fine on buildbot - so happy to approve!

review: Approve

=== modified file 'python/fluidity/regressiontest.py'
--- python/fluidity/regressiontest.py	2012-08-22 20:02:27 +0000
+++ python/fluidity/regressiontest.py	2013-01-21 14:26:49 +0000
@@ -12,7 +12,7 @@
 
 class TestProblem:
     """A test records input information as well as tests for the output."""
-    def __init__(self, filename, verbose=False, replace=None):
+    def __init__(self, filename, verbose=False, replace=None, no_pbs=False):
         """Read a regression test from filename and record its details."""
         self.name = ""
         self.command = replace
@@ -26,6 +26,7 @@
         self.pass_status = []
         self.warn_status = []
         self.filename = filename.split('/')[-1]
+        self.no_pbs = no_pbs
         # add dir to import path
         sys.path.insert(0, os.path.dirname(filename))
 
@@ -130,7 +131,7 @@
         except OSError:
           self.log("No Makefile, not calling make")
 
-        if self.nprocs > 1 or self.length == "long":
+        if (not self.no_pbs) and (self.nprocs > 1 or self.length == "long"):
             ret = self.call_genpbs(dir)
             self.log("cd "+dir+"; qsub " + self.filename[:-4] + ".pbs: " + self.command_line)
             os.system("cd "+dir+"; qsub " + self.filename[:-4] + ".pbs")

=== modified file 'tools/testharness.py'
--- tools/testharness.py	2012-11-08 17:26:18 +0000
+++ tools/testharness.py	2013-01-21 14:26:49 +0000
@@ -9,6 +9,7 @@
 import traceback
 import threading
 import xml.parsers.expat
+import string
 
 
 sys.path.insert(0, os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), os.pardir, "python"))
@@ -21,7 +22,7 @@
     def __init__(self, length="any", parallel=False, exclude_tags=None,
                  tags=None, file="", from_file=None,
                  verbose=True, justtest=False,
-                 valgrind=False):
+                 valgrind=False, no_pbs=False):
         self.tests = []
         self.verbose = verbose
         self.length = length
@@ -33,6 +34,7 @@
         self.completed_tests = []
         self.justtest = justtest
         self.valgrind = valgrind
+        self.no_pbs = no_pbs
 
         fluidity_command = self.decide_fluidity_command()
 
@@ -87,9 +89,12 @@
 
         if files:
           for (subdir, xml_file) in [os.path.split(x) for x in xml_files]:
-            if xml_file in files:
+            if xml_file == file:
+              p = etree.parse(os.path.join(subdir,xml_file))
+              prob_defn = p.findall("problem_definition")[0]
+              prob_nprocs = int(prob_defn.attrib["nprocs"])                
               testprob = regressiontest.TestProblem(filename=os.path.join(subdir, xml_file),
-                           verbose=self.verbose, replace=self.modify_command_line())
+                           verbose=self.verbose, replace=self.modify_command_line(prob_nprocs), no_pbs=no_pbs)
               self.tests.append((subdir, testprob))
               files.remove(xml_file)
           if files != []:
@@ -157,8 +162,12 @@
           tagged_set = working_set
 
         for (subdir, xml_file) in [os.path.split(x) for x in tagged_set]:
+          # need to grab nprocs here to pass through to modify_command_line
+          p = etree.parse(os.path.join(subdir,xml_file))
+          prob_defn = p.findall("problem_definition")[0]
+          prob_nprocs = int(prob_defn.attrib["nprocs"])
           testprob = regressiontest.TestProblem(filename=os.path.join(subdir, xml_file),
-                       verbose=self.verbose, replace=self.modify_command_line())
+                       verbose=self.verbose, replace=self.modify_command_line(prob_nprocs))
           self.tests.append((subdir, testprob))
 
         if len(self.tests) == 0:
@@ -211,9 +220,9 @@
               
         return None
 
-    def modify_command_line(self):
+    def modify_command_line(self, nprocs):
       flucmd = self.decide_fluidity_command()
-
+      print flucmd
       def f(s):
         if not flucmd in [None, "fluidity"]:
           s = s.replace('fluidity ', flucmd + ' ')
@@ -222,6 +231,18 @@
           s = "valgrind --tool=memcheck --leak-check=full -v" + \
               " --show-reachable=yes --num-callers=8 --error-limit=no " + \
               "--log-file=test.log " + s
+
+        print s
+        if (self.no_pbs):
+            # check for mpiexec and the correct number of cores
+            if (string.find(s, 'mpiexec') == -1):
+                s = s.replace(flucmd+" ", "mpiexec "+flucmd+" ")
+                print s
+
+            if (string.find(s, '-n') == -1): 
+                s = s.replace('mpiexec ', 'mpiexec -n '+str(nprocs)+' ')
+                print s
+
         return s
 
       return f
@@ -355,6 +376,7 @@
     parser.add_option("-c", "--clean", action="store_true", dest="clean", default = False)
     parser.add_option("--just-test", action="store_true", dest="justtest")
     parser.add_option("--just-list", action="store_true", dest="justlist")
+    parser.add_option("--no_pbs", action="store_true", dest="no_pbs")
     (options, args) = parser.parse_args()
 
     if len(args) > 0: parser.error("Too many arguments.")
@@ -393,7 +415,8 @@
                               file=options.file, verbose=True,
                               justtest=options.justtest,
                               valgrind=options.valgrind,
-                              from_file=options.from_file)
+                              from_file=options.from_file,
+                              no_pbs=options.no_pbs)
 
     if options.justlist:
       testharness.list()