Merge lp:~madteam/mg5amcnlo/test-framework-dev into lp:~madteam/mg5amcnlo/framework-dev

[Olivier Mattelaer]

[Olivier Mattelaer (olivier-mattelaer)]

Add the test manager to launch test automaticaly

[Michel Herquet (herquet)]

Perfect! Ok for merging

review: Approve (full)

=== added file 'bin/test_manager.py'
--- bin/test_manager.py	1970-01-01 00:00:00 +0000
+++ bin/test_manager.py	2009-10-05 15:15:19 +0000
@@ -0,0 +1,314 @@
+#!/usr/bin/env python 
+##############################################################################
+#
+# Copyright (c) 2009 The MadGraph Development team and Contributors
+#
+# This file is a part of the MadGraph 5 project, an application which 
+# automatically generates Feynman diagrams and matrix elements for arbitrary
+# high-energy processes in the Standard Model and beyond.
+#
+# It is subject to the MadGraph license which should accompany this 
+# distribution.
+#
+# For more information, please visit: http://madgraph.phys.ucl.ac.be
+#
+##############################################################################
+""" Manager for running the test library 
+
+   This library offer a simple way to launch test.
+   
+   To run a test/class of test/test file/module of test/...
+   you just have to launch 
+   test_manager.run(NAME)
+   or 
+   test_manager.run(LIST_OF_NAME)
+
+   the NAME can contain regular expression (in python re standard format)
+""" 
+
+import unittest
+import re
+import os
+import inspect
+import sys
+sys.path+=['.','..']
+
+##############################################################################
+def run(expression='', re_opt=0, package='./tests/'):
+    """ running the test associated to expression. By default, this launch all 
+    test derivated from TestCase. expression can be the name of directory, 
+    module, class, function or event standard regular expression (in re format)
+    """
+    
+    #init a test suite
+    testsuite = unittest.TestSuite()
+    collect = unittest.TestLoader()
+    for test_fct in Test_in_module(package=package, expression=expression, \
+                                   re_opt=re_opt):
+        data=collect.loadTestsFromName(test_fct)
+        testsuite.addTest(data)
+    unittest.TextTestRunner(verbosity=1).run(testsuite)
+    
+
+##############################################################################
+class Test_in_module(list):
+    """ class introspecting the test module to find the available test.
+
+    The strategy is the following:
+    The routine collect_dir looks in all module/file to find the different 
+    functions in different test class. This produce a list, on wich external 
+    routines can loop on. 
+        
+    In order to authorize definition and loop on this object on the same time,
+    i.e: for test in Test_in_module([opt])-. At each time a loop is started, 
+    we check if a collect_dir runned before, and run it if neccessary. And 
+    that without any problem with future add-on on this class.
+    """
+
+    search_class=unittest.TestCase
+    class ERRORTestManager(Exception): pass
+
+    ##########################################################################
+    def __init__(self, package='./tests/', expression='', re_opt=0):
+        """ initialize global variable for the test """
+
+        list.__init__(self)
+
+        self.package = package
+        if self.package[-1] != '/': self.package+='/'
+        self.restrict_to(expression, re_opt)
+        
+    ##########################################################################
+    def _check_if_obj_build(self):
+        """ Check if a collect is already done 
+            Uses to have smart __iter__ and __contain__ functions
+        """
+        if len(self) == 0:
+            self.collect_dir(self.package, checking=True)
+
+    ##########################################################################
+    def __iter__(self):
+        """ Check that a collect was performed (do it if needed) """
+        self._check_if_obj_build()
+        return list.__iter__(self)
+
+    ##########################################################################
+    def __contains__(self,value):
+        """ Check that a collect was performed (do it if needed) """
+        self._check_if_obj_build()
+        return list.__contains__(self,value)
+
+    ##########################################################################
+    def collect_dir(self, directory, checking=True):
+        """ Find the file and the subpackage in this package """
+
+        for name in os.listdir(directory):
+            local_check = checking
+
+            status = self.status_file(directory + '/' + name)
+            if status is None:
+                continue
+
+            if checking:
+                if self.check_valid(directory + '/' + name):
+                    local_check = False    #since now perform all the test
+
+            if status == 'file':
+                self.collect_file(directory + '/' + name, local_check)
+            elif status == "module":
+                self.collect_dir(directory + '/' + name, local_check)
+        
+    ##########################################################################
+    def collect_file(self, file, checking=True):
+        """ Find the different class instance derivated of TestCase """
+
+        pyname=self.passin_pyformat(file)
+        exec('import '+pyname+' as obj')         
+
+        #look at class
+        for name in dir(obj):
+            exec('class_=obj.'+name)
+            if inspect.isclass(class_) and issubclass(class_,unittest.TestCase):
+                if checking:
+                    if self.check_valid(name):
+                        check_inside=False
+                    else:
+                        check_inside=True
+                else:
+                    check_inside=False
+
+
+                self.collect_function(class_, checking=check_inside, \
+                                          base=pyname)
+
+    ##########################################################################
+    def collect_function(self, class_, checking=True, base=''):
+        """
+        Find the different test function in this class
+        test functions should start with test
+        """
+
+        if not inspect.isclass(class_):
+            raise self.ERRORTestManager, 'wrong input class_'
+        if not issubclass(class_,unittest.TestCase):
+            raise self.ERRORTestManager, 'wrong input class_'
+
+        #devellop the name
+        if base:
+            base+='.'+class_.__name__
+        else:
+            base=class_.__name__
+
+        candidate=[base+'.'+name for name in dir(class_) if \
+                       name.startswith('test')\
+                       and inspect.ismethod(eval('class_.'+name))]
+
+        if not checking:
+            self+=candidate
+        else:
+            self+=[name for name in candidate if self.check_valid(name)]
+
+    ##########################################################################
+    def restrict_to(self, expression, re_opt=0):
+        """ store in global the expression to fill in order to be a valid test """
+        
+        if isinstance(expression,list):
+            pass
+        elif isinstance(expression,basestring):
+            if expression=='':
+                expression=['.*'] #made an re authorizing all regular name
+            else:
+                expression = [expression]
+        else:
+            raise self.ERRORTestManager, 'obj should be list or string'
+        
+        self.rule=[]
+        for expr in expression:
+            if not expr.startswith('^'): expr='^'+expr #fix the begin of the re
+            if not expr.endswith('$'): expr=expr+'$' #fix the end of the re
+            self.rule.append( re.compile(expr, re_opt) )
+
+    ##########################################################################
+    def check_valid(self, name):
+        """ check if the name correspond to the rule """
+        
+        if not isinstance(name,basestring):
+            raise self.ERRORTestManager, 'check valid take a string argument'
+        
+        for specific_format in self.format_possibility(name):
+            for expr in self.rule:
+                if expr.search(specific_format):
+                    return True
+        return False
+
+    ##########################################################################
+    @staticmethod
+    def status_file(name):
+        """ check if a name is a module/a python file and return the status """
+        if os.path.isfile(name):
+            if name.endswith('.py') and '__init__' not in name:
+                return 'file'
+        elif os.path.isdir(name):
+            if os.path.isfile(name+'/__init__.py'):
+                return 'module'
+
+    ##########################################################################
+    @classmethod
+    def passin_pyformat(cls,name):
+        """ transform a relative position in a python import format """
+
+        if not isinstance(name,basestring):
+            raise cls.ERRORTestManager, 'collect_file takes a file position'
+
+        name=name.replace('//','/') #sanity
+        #deal with begin/end
+        if name.startswith('./'):
+            name=name[2:]
+        if not name.endswith('.py'):
+            raise cls.ERRORTestManager,'Python files should have .py extension'
+        else:
+            name=name[:-3]
+
+        if name.startswith('/'):
+            raise cls.ERRORTestManager, 'path must be relative'
+        if '..' in name:
+            raise cls.ERRORTestManager,'relative position with \'..\' is' + \
+                ' not supported for the moment'
+        
+        #replace '/' by points -> Python format
+        name=name.replace('/','.')        
+
+        #position
+        return name
+
+    ##########################################################################
+    def format_possibility(self,name):
+        """ return the different string derivates from name in order to 
+        scan all the different format authorizes for a restrict_to 
+        format authorizes:
+        1) full file position
+        2) name of the file (with extension)
+        3) full file position whithour extension
+        4) name of the file (whithout extension)
+        5) name of the file (but suppose name in python format)
+        6) if name is a python file, try with a './' and with package pos
+        """
+
+        def add_to_possibility(possibility,val):
+            """ add if not exist """
+            if val not in possibility: 
+                possibility.append(val)
+        #end local def
+                
+        #print name
+        #sanity
+        if name.startswith('./'): name = name[2:]
+        name=name.replace('//','/')
+        # init with solution #
+        out=[name]
+
+        # add solution 2
+        new_pos=name.split('/')[-1]
+        add_to_possibility(out,new_pos)
+        
+        #remove extension and add solution3 and 6
+        if name.endswith('.py'):
+            add_to_possibility(out,'./'+name)
+            add_to_possibility(out,self.package+name)
+            name = name[:-3]
+        add_to_possibility(out,name)
+
+        #add solution 4
+        new_pos=name.split('/')[-1]
+        add_to_possibility(out,new_pos)
+
+        #add solution 5
+        new_pos=name.split('.')[-1]
+        add_to_possibility(out,new_pos)
+
+
+
+
+        return out
+
+##############################################################################
+if __name__ == "__main__":
+
+    opt=sys.argv
+    if len(opt)==1:
+        run()
+    elif len(opt)==2:
+        run(opt[1])
+    else:
+        run(opt[1],re_opt=opt[2])
+
+#some example
+#    run('iolibs')
+#    run('test_test_manager.py')
+#    run('./tests/unit_tests/bin/test_test_manager.py')
+#    run('IOLibsMiscTest')
+#    run('Unittest_on_TestinModule')
+#    run('test_check_valid_on_file')
+#    run('test_collect_dir.*') # '.*' stands for all possible char (re format)
+
+

=== added directory 'tests/unit_tests/bin'
=== added file 'tests/unit_tests/bin/__init__.py'
=== added file 'tests/unit_tests/bin/test_test_manager.py'
--- tests/unit_tests/bin/test_test_manager.py	1970-01-01 00:00:00 +0000
+++ tests/unit_tests/bin/test_test_manager.py	2009-10-05 15:15:19 +0000
@@ -0,0 +1,357 @@
+##############################################################################
+#
+# Copyright (c) 2009 The MadGraph Development team and Contributors
+#
+# This file is a part of the MadGraph 5 project, an application which 
+# automatically generates Feynman diagrams and matrix elements for arbitrary
+# high-energy processes in the Standard Model and beyond.
+#
+# It is subject to the MadGraph license which should accompany this 
+# distribution.
+#
+# For more information, please visit: http://madgraph.phys.ucl.ac.be
+#
+##############################################################################
+
+"""Unit test library for the Misc routine library in the I/O package"""
+
+import sys
+sys.path+=['../../../bin','.','./bin']
+
+import unittest
+import test_manager
+import inspect
+
+class Unittest_on_TestinModule(unittest.TestCase):
+    #This class test to find it's own property so it need some shortcut
+    test_path='./tests/unit_tests/bin/' 
+    name='test.manager.Unittest_on_TestinModule'
+    
+    def setUp(self):
+        """ basic building of the class to test """
+        self.testmodule=test_manager.Test_in_module(package=self.test_path)
+
+    def tearDown(self):
+        pass
+
+    def test_buiding_in_iter(self):
+        """ Test_in_module.__iter__ is able to build the list """
+
+        for i in self.testmodule:
+            break
+        self.assert_(len(self.testmodule)>1)
+
+    def test_collect_dir(self):
+        """ Test_in_module.collect_dir should detect subdirectory and file """
+
+        self.testmodule=test_manager.Test_in_module(package= \
+                         './tests/unit_tests')
+        self.assert_('tests.unit_tests.bin.test_test_manager.Unittest_on_TestinModule.test_collect_dir' in self.testmodule)
+
+    def test_collect_dir_with_restriction_file(self):
+        """ Test_in_module.collect_dir pass corectly restriction rule on file"""
+        self.testmodule.restrict_to('test_test_manager.py')
+#        self.testmodule.collect_dir('./tests/unit_tests/bin')
+        self.assert_('tests.unit_tests.bin.test_test_manager.Unittest_on_TestinModule.test_collect_dir'  in self.testmodule)
+
+    def test_collect_dir_with_restriction_file2(self):
+        """ Test_in_module.collect_dir pass corectly restriction rule on file"""
+        self.testmodule.restrict_to('./tests/unit_tests/bin/test_test_manager.py')
+#        self.testmodule.collect_dir('./tests/unit_tests/bin')
+#        print self.testmodule
+        self.assert_('tests.unit_tests.bin.test_test_manager.Unittest_on_TestinModule.test_collect_dir'  in self.testmodule)
+
+    def test_collect_dir_with_restriction_class(self):
+        """ Test_in_module.collect_dir pass corectly restriction rule on class"""
+
+        self.testmodule.restrict_to('Unittest_on_TestinModule')
+#        self.testmodule.collect_dir('./tests/unit_tests/bin')
+        self.assert_('tests.unit_tests.bin.test_test_manager.Unittest_on_TestinModule.test_collect_dir'  in self.testmodule)
+
+    def test_collect_dir_with_restriction_fct(self):
+        """ Test_in_module.collect_dir pass corectly restriction rule on fct"""
+
+        self.testmodule.restrict_to('test_check_valid.*')
+        self.assert_('tests.unit_tests.bin.test_test_manager.Unittest_on_TestinModule.test_collect_dir' not in self.testmodule)
+        self.assert_('tests.unit_tests.bin.test_test_manager.Unittest_on_TestinModule.test_check_valid_on_file' in self.testmodule)
+
+
+    def test_collect_file_wrong_arg(self):
+        """ Test_in_module.collect_file fails on wrong argument"""
+
+        for input in [1,{1:2},[1],str,int,list,'alpha']:
+            self.assertRaises(test_manager.Test_in_module.ERRORTestManager, \
+                              self.testmodule.collect_file, input)
+
+    def test_collect_file(self):
+        """ Test_in_module.collect_file find the different class in a file """
+        
+        self.testmodule.collect_file('./tests/unit_tests/bin/test_test_manager.py')
+        self.assert_('tests.unit_tests.bin.test_test_manager.Unittest_on_TestinModule.test_collect_file' in \
+                         self.testmodule)
+
+    def test_collect_function_wrong_arg(self):
+        """ Test_in_module.collect_function fails on wrong argument """
+
+        for input in [1,{1:2},[1],'alpha',str,int,list]:
+            self.assertRaises(test_manager.Test_in_module.ERRORTestManager, \
+                              self.testmodule.collect_function, input)
+
+
+    def test_collect_function(self):
+        """ Test_in_module.collect_function find the test function """
+        
+        self.testmodule.collect_function(Unittest_on_TestinModule)
+        self.assert_('Unittest_on_TestinModule.test_collect_function' in \
+                         self.testmodule)
+
+        for name in self.testmodule:
+            name=name.split('.')[-1]
+            self.assertTrue(name.startswith('test'))
+
+    def test_output_are_function(self):
+        """ Test_in_module.collect_function returns test funcions only """
+        self.testmodule.collect_function(Unittest_on_TestinModule)
+        mytest = unittest.TestSuite()
+        collect = unittest.TestLoader()
+        for test_fct in self.testmodule:
+            try:
+                collect.loadTestsFromName( \
+                    'tests.unit_tests.bin.test_test_manager.'+test_fct)
+            except:
+                self.fail('non callable object are returned')
+
+            
+    def test_restrict_to_inputcheck(self):
+        """ Test_in_module.restrict_to fail on non string/list input """
+
+        input1={}
+        self.assertRaises(test_manager.Test_in_module.ERRORTestManager, \
+                              self.testmodule.restrict_to, input1)
+        input1=1
+        self.assertRaises(test_manager.Test_in_module.ERRORTestManager, \
+                              self.testmodule.restrict_to, input1)
+        import re
+
+        input1=re.compile('''1''')
+        self.assertRaises(test_manager.Test_in_module.ERRORTestManager, \
+                              self.testmodule.restrict_to, input1)
+
+    def test_check_valid_wrong_arg(self):
+        """ Test_in_module.check_valid raise error if not str in input """
+
+        for input in [1,{1:2},[1]]:
+            self.assertRaises(test_manager.Test_in_module.ERRORTestManager, \
+                              self.testmodule.check_valid, input)
+
+    def test_check_valid_on_module(self):
+        """ Test_in_module.check_valid recognizes module """
+
+        expression = 'bin'
+        self.testmodule.restrict_to(expression)
+        pos = './bin'
+        self.assertTrue(self.testmodule.check_valid(pos))
+        pos = './bin2'
+        self.assertFalse(self.testmodule.check_valid(pos))
+        pos = 'bin'
+        self.assertTrue(self.testmodule.check_valid(pos))
+        pos = '../test/bin'
+        self.assertTrue(self.testmodule.check_valid(pos))
+
+    def test_check_valid_on_file(self):
+        """ Test_in_module.check_valid recognizes file """
+
+        expression = 'test'
+        self.testmodule.restrict_to(expression)
+        pos = 'test.py'
+        self.assertTrue(self.testmodule.check_valid(pos))
+        pos = './test.py'
+        self.assertTrue(self.testmodule.check_valid(pos))
+        pos = '../test/test.py'
+        self.assertTrue(self.testmodule.check_valid(pos))
+        pos = 'test.pyc'
+        self.assertFalse(self.testmodule.check_valid(pos))
+        pos = 'onetest.py'
+        self.assertFalse(self.testmodule.check_valid(pos))
+        pos = 'test/file.py'
+        self.assertFalse(self.testmodule.check_valid(pos))
+
+        expression = 'test.py'
+        self.testmodule.restrict_to(expression)
+        pos = 'test.py'
+        self.assertTrue(self.testmodule.check_valid(pos))
+        pos = './test.py'
+        self.assertTrue(self.testmodule.check_valid(pos))
+        pos = '../test/test.py'
+        self.assertTrue(self.testmodule.check_valid(pos))
+        pos = 'test.pyc'
+        self.assertFalse(self.testmodule.check_valid(pos))
+        pos = 'onetest.py'
+        self.assertFalse(self.testmodule.check_valid(pos))
+        pos = 'test/file.py'
+        self.assertFalse(self.testmodule.check_valid(pos))
+
+        expression = 'test.test.py'
+        self.testmodule.restrict_to(expression)
+        pos = 'test/test.py'
+        self.assertTrue(self.testmodule.check_valid(pos))
+        pos = './test.py'
+        self.assertFalse(self.testmodule.check_valid(pos))
+        pos = 'bin/test/test.py'
+        self.assertFalse(self.testmodule.check_valid(pos))
+        pos = 'test.pyc'
+        self.assertFalse(self.testmodule.check_valid(pos))
+        pos = 'onetest.py'
+        self.assertFalse(self.testmodule.check_valid(pos))
+        pos = 'test/file.py'
+        self.assertFalse(self.testmodule.check_valid(pos))
+
+        expression = './test/test.py'
+        self.testmodule.restrict_to(expression)
+        pos = 'test/test.py'
+        self.assertTrue(self.testmodule.check_valid(pos))
+        pos = './test.py'
+        self.assertFalse(self.testmodule.check_valid(pos))
+        pos = 'bin/test/test.py'
+        self.assertFalse(self.testmodule.check_valid(pos))
+        pos = 'test.pyc'
+        self.assertFalse(self.testmodule.check_valid(pos))
+        pos = 'onetest.py'
+        self.assertFalse(self.testmodule.check_valid(pos))
+        pos = 'test/file.py'
+        self.assertFalse(self.testmodule.check_valid(pos))
+
+
+    def test_check_valid_on_class(self):
+        """ Test_in_module.check_valid recognizes class of test """
+        
+        expression = 'Unittest_on_TestinModule'
+        self.testmodule.restrict_to(expression)
+        name = 'Unittest_on_TestinModule'
+        self.assertTrue(self.testmodule.check_valid(name))
+        name = 'test.Unittest_on_TestinModule'
+        self.assertTrue(self.testmodule.check_valid(name))
+        name = 'I.am.Your.Father'
+        self.assertFalse(self.testmodule.check_valid(name))
+
+    def test_check_valid_on_function(self):
+        """ Test_in_module.check_valid recognizes functions """
+
+        expression='test_search'
+        self.testmodule.restrict_to(expression)
+        name='test_search'
+        self.assertTrue(self.testmodule.check_valid(name))
+        name='test.test_search'
+        self.assertTrue(self.testmodule.check_valid(name))
+        name='It.is.impossible'
+        self.assertFalse(self.testmodule.check_valid(name))
+
+        expression='test_check_valid.*'
+        self.testmodule.restrict_to(expression)
+        name='module.test_check_valid_on_function'
+        self.assertTrue(self.testmodule.check_valid(name))
+
+    def test_check_valid_with_re(self):
+        """ Test_in_module.check_valid should work with re """
+
+        expression = 'test.*'
+        self.testmodule.restrict_to(expression,)
+        name = 'test_search'
+        self.assertTrue(self.testmodule.check_valid(name))
+        name = 'valid.test_search'
+        self.assertTrue(self.testmodule.check_valid(name))
+        name = 'one_test'
+        self.assertFalse(self.testmodule.check_valid(name))
+
+        expression = 'test'
+        import re
+        re_opt = re.I
+        self.testmodule.restrict_to(expression,re_opt)
+        name = 'test'
+        self.assertTrue(self.testmodule.check_valid(name))
+        name = 'TEST'
+        self.assertTrue(self.testmodule.check_valid(name))
+        name = 'one_test'
+        self.assertFalse(self.testmodule.check_valid(name))
+
+    def test_check_valid_with_list_restriction(self):
+        """ Test_in_module.check_valid should work with list in restrict """
+        
+        expression = ['test.*','iolibs']
+        self.testmodule.restrict_to(expression)
+        name = 'test_search'
+        self.assertTrue(self.testmodule.check_valid(name))
+        name = 'iolibs'
+        self.assertTrue(self.testmodule.check_valid(name))
+        name = 'data'
+        self.assertFalse(self.testmodule.check_valid(name))
+
+    def test_status_file_on_file(self):
+        """ Test_in_module.status_file recognizes file """
+
+        status=self.testmodule.status_file(self.test_path+\
+                                               'test_test_manager.py')
+        self.assertEqual(status,'file')
+        status=self.testmodule.status_file(self.test_path+\
+                             '../../../README')
+        self.assertFalse(status)
+        status=self.testmodule.status_file(self.test_path+\
+                             '__init__.py')
+        self.assertFalse(status)
+        status=self.testmodule.status_file(self.test_path+\
+                             '__init__.pyc')
+        self.assertFalse(status)
+
+
+    def test_status_file_on_dir(self):
+        """ Test_in_module.status_file  doesn't detect non module dir """
+
+        status=self.testmodule.status_file(self.test_path+\
+                             '../bin')
+        self.assertEqual(status,'module')
+
+        status=self.testmodule.status_file(self.test_path+\
+                             '../../../apidoc')
+        self.assertFalse(status)
+
+    
+    def test_passin_pyformat(self):
+        """ convert from linux position format to python include """
+
+        input={'test.py':'test', 'Source/test.py':'Source.test', \
+                   'Source//test.py':'Source.test', './test.py':'test'}
+        for key,value in input.items():
+            self.assertEqual( \
+                self.testmodule.passin_pyformat(key),
+                value)
+
+        input=['test','../Source.py','/home/data.py',1,str]
+        for data in input:
+            self.assertRaises(test_manager.Test_in_module.ERRORTestManager
+                              ,self.testmodule.passin_pyformat, data)
+        
+
+    def test_add_to_possibility(self):
+        """ convert name in different matching possibility """
+        # the sanity of the output is checked by check_valid test,
+        # this test the format of the output
+        
+        output=self.testmodule.format_possibility('./bin/test.py')
+        for name in output:
+            self.assertEqual(output.count(name),1)
+        self.assert_(len(output)>3)
+
+        output=self.testmodule.format_possibility('bin.test')
+        for name in output:
+            self.assertEqual(output.count(name),1)
+        self.assert_(len(output)>1)
+
+if __name__ == "__main__":
+    mytest = unittest.TestSuite()
+    suite = unittest.TestLoader()
+    suite=suite.loadTestsFromTestCase(Unittest_on_TestinModule)
+    mytest.addTest(suite)
+    unittest.TextTestRunner(verbosity=2).run(mytest)
+
+
+#    unittest.main()