Merge lp:~patrick-crews/drizzle/dbqp_add_reorder into lp:drizzle/7.0

[Patrick Crews]

This branch adds --reorder and a placeholder --fast option.
We also quell unneeded libtool message when it isn't applicable.

I also merged in my other changes just to test them in conjunction with these changes. (just a warning in case merging presents any messages about this or whatever)

=== modified file 'tests/dbqp.py'
--- tests/dbqp.py	2011-01-25 01:19:49 +0000
+++ tests/dbqp.py	2011-01-27 23:36:53 +0000
@@ -4,6 +4,20 @@
 #
 # Copyright (C) 2010 Patrick Crews
 #
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 
 """ dbqp.py
 

=== modified file 'tests/lib/drizzle_test_run/dtr_test_execution.py'
--- tests/lib/drizzle_test_run/dtr_test_execution.py	2011-01-26 01:13:20 +0000
+++ tests/lib/drizzle_test_run/dtr_test_execution.py	2011-01-27 23:36:53 +0000
@@ -4,6 +4,21 @@
 #
 # Copyright (C) 2010 Patrick Crews
 #
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 """ dtr_test_execution:
     code related to the execution of dtr test cases 
     

=== modified file 'tests/lib/drizzle_test_run/dtr_test_management.py'
--- tests/lib/drizzle_test_run/dtr_test_management.py	2011-01-22 16:51:51 +0000
+++ tests/lib/drizzle_test_run/dtr_test_management.py	2011-01-27 23:36:53 +0000
@@ -4,6 +4,20 @@
 #
 # Copyright (C) 2010 Patrick Crews
 #
+## This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 """ dtr_test_management:
     code related to the gathering / analysis / management of 
     the test cases
@@ -391,3 +405,31 @@
                     self.system_manager.logging.debug("%s says - I'm disabled" %(test_name))
                 return (1, disabled_tests[test_name])
         return (0,None)
+
+    def sort_testcases(self):
+        """ We sort our testcases according to the server_options they have
+            For each testcase, we sort the list of options, so if a test has
+            --plugin-add=csv --abracadabra, we would get 
+            --abracadabra --plugin-add=csv
+            
+            This results in tests that have similar options being run in order
+            this minimizes server restarts which can be costly
+
+        """
+        test_management.testManager.sort_testcases(self)
+        organizer = {}
+        ordered_list = []
+        for testcase in self.test_list:
+            key = " ".join(sorted(testcase.server_options))
+            if key in organizer:
+                organizer[key].append(testcase)
+            else:
+                organizer[key] = [testcase]
+        for key in organizer.keys():
+            print key, len(organizer[key])
+        for value_list in organizer.values():
+            ordered_list = ordered_list + value_list
+        self.test_list = ordered_list
+        
+
+        

=== modified file 'tests/lib/server_mgmt/drizzled.py'
--- tests/lib/server_mgmt/drizzled.py	2011-01-26 01:13:20 +0000
+++ tests/lib/server_mgmt/drizzled.py	2011-01-27 23:36:53 +0000
@@ -4,6 +4,20 @@
 #
 # Copyright (C) 2010 Patrick Crews
 #
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 
 """ drizzled.py:  code to allow a serverManager
     to provision and start up a drizzled server object

=== modified file 'tests/lib/server_mgmt/server_management.py'
--- tests/lib/server_mgmt/server_management.py	2011-01-25 17:47:18 +0000
+++ tests/lib/server_mgmt/server_management.py	2011-01-27 23:36:53 +0000
@@ -4,6 +4,20 @@
 #
 # Copyright (C) 2010 Patrick Crews
 #
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 """server_management.py
    code for dealing with apportioning servers
    to suit the needs of the tests and executors

=== modified file 'tests/lib/sys_mgmt/codeTree.py'
--- tests/lib/sys_mgmt/codeTree.py	2011-01-25 17:47:18 +0000
+++ tests/lib/sys_mgmt/codeTree.py	2011-01-27 23:36:53 +0000
@@ -4,6 +4,20 @@
 #
 # Copyright (C) 2010 Patrick Crews
 #
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 """codeTree
 
    definition of what a code tree should look like

=== modified file 'tests/lib/sys_mgmt/logging_management.py'
--- tests/lib/sys_mgmt/logging_management.py	2011-01-25 17:14:51 +0000
+++ tests/lib/sys_mgmt/logging_management.py	2011-01-27 23:36:53 +0000
@@ -3,12 +3,31 @@
 # vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
 #
 # Copyright (C) 2009 Sun Microsystems
+# Copyright (C) 2011 Patrick Crews
 #
 # Authors:
 #
 #  Jay Pipes <[e-mail]>
 #  Monty Taylor <[e-mail]>
-#
+#  Patrick Crews 
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+#
+# This code is modified from the logging module used in the 
+# drizzle-automation project - https://launchpad.net/drizzle-automation
 
 
 """ Simple replacement for python logging module that doesn't suck """

=== modified file 'tests/lib/sys_mgmt/port_management.py'
--- tests/lib/sys_mgmt/port_management.py	2011-01-21 18:47:56 +0000
+++ tests/lib/sys_mgmt/port_management.py	2011-01-27 23:36:53 +0000
@@ -4,6 +4,20 @@
 #
 # Copyright (C) 2010 Patrick Crews
 #
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 """port_management.py
    code for dealing with the various tasks
    around handing out and managing server ports
@@ -107,7 +121,16 @@
             if entry.startswith('Proto'):
                 good_data = 1
             elif good_data:
-                used_port = int(entry.split()[3].split(':')[-1].strip())
+                # We try to catch additional output
+                # like we see with freebsd
+                if entry.startswith('Active'):
+                    good_data = 0
+                    pass
+                else:
+                    if self.system_manager.cur_os == 'FreeBSD':
+                        used_port = int(entry.split()[3].split('.')[-1].strip())
+                    else:
+                        used_port = int(entry.split()[3].split(':')[-1].strip())
                 if port == used_port:
                     if entry.split()[-1] != "TIME_WAIT":
                         return 1

=== modified file 'tests/lib/sys_mgmt/system_management.py'
--- tests/lib/sys_mgmt/system_management.py	2011-01-26 01:13:20 +0000
+++ tests/lib/sys_mgmt/system_management.py	2011-01-27 23:36:53 +0000
@@ -4,6 +4,20 @@
 #
 # Copyright (C) 2010 Patrick Crews
 #
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 """system_management.py
    code for dealing with system-level 'stuff'.
    This includes setting environment variables, looking for clients,
@@ -49,10 +63,12 @@
         self.env_var_delimiter = ':'
         self.no_shm = variables['noshm']
         self.shm_path = self.find_path(["/dev/shm", "/tmp"], required=0)
+        self.cur_os = os.uname()[0]
         self.symlink_name = 'dtr_work_sym'
         self.workdir = variables['workdir']
         self.start_dirty = variables['startdirty']
         self.valgrind = variables['valgrind']
+        self.gdb = variables['gdb']
 
         # we use this to preface commands in order to run valgrind and such
         self.cmd_prefix = '' 
@@ -385,7 +401,8 @@
         libtool_path = '../libtool'
         if os.path.exists(libtool_path) and os.access( libtool_path
                                                      , os.X_OK):
-            self.logging.info("Using libtool when running valgrind or debugger")
+            if self.valgrind or self.gdb:
+                self.logging.info("Using libtool when running valgrind or debugger")
             return libtool_path
         else:
             return None

=== modified file 'tests/lib/sys_mgmt/time_management.py'
--- tests/lib/sys_mgmt/time_management.py	2011-01-25 17:10:19 +0000
+++ tests/lib/sys_mgmt/time_management.py	2011-01-27 23:36:53 +0000
@@ -4,6 +4,21 @@
 #
 # Copyright (C) 2010 Patrick Crews
 #
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 """time_management.py
    code for dealing with timing, mostly
    we want to keep track of how long certain things

=== modified file 'tests/lib/test_mgmt/execution_management.py'
--- tests/lib/test_mgmt/execution_management.py	2011-01-15 21:27:41 +0000
+++ tests/lib/test_mgmt/execution_management.py	2011-01-27 23:36:53 +0000
@@ -4,6 +4,20 @@
 #
 # Copyright (C) 2010 Patrick Crews
 #
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 """execution_management.py
    code for dealing with test execution.
    The actual nuts and bolts of executing a test lies with the 

=== modified file 'tests/lib/test_mgmt/test_execution.py'
--- tests/lib/test_mgmt/test_execution.py	2011-01-26 01:13:20 +0000
+++ tests/lib/test_mgmt/test_execution.py	2011-01-27 23:36:53 +0000
@@ -4,6 +4,20 @@
 #
 # Copyright (C) 2010 Patrick Crews
 #
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 """ test_execution:
     code related to the execution of est cases 
     

=== modified file 'tests/lib/test_mgmt/test_management.py'
--- tests/lib/test_mgmt/test_management.py	2011-01-25 01:19:49 +0000
+++ tests/lib/test_mgmt/test_management.py	2011-01-27 23:36:53 +0000
@@ -4,6 +4,20 @@
 #
 # Copyright (C) 2010 Patrick Crews
 #
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 """ dtr_test_management:
     code related to the gathering / analysis / management of 
     the test cases
@@ -27,7 +41,8 @@
     """
 
     def __init__( self, verbose, debug, engine, dotest, skiptest
-                , suitelist, suitepaths, system_manager, test_cases):
+                , reorder, suitelist, suitepaths, system_manager
+                , test_cases):
 
         self.system_manager = system_manager
         self.time_manager = system_manager.time_manager
@@ -51,6 +66,7 @@
         self.engine = engine
         self.dotest = dotest
         self.skiptest = skiptest
+        self.reorder = reorder
         self.suitelist = suitelist
         
         self.code_tree = self.system_manager.code_tree
@@ -90,6 +106,10 @@
     
         """
 
+        # See if we need to reorder our test cases
+        if self.reorder:
+            self.sort_testcases()
+
         if self.desired_tests and not self.test_list:
             # We wanted tests, but found none
             # Probably need to make this smarter at some point
@@ -262,3 +282,11 @@
         else:
             return 0
 
+    def sort_testcases(self):
+        """ Sort testcases to optimize test execution.
+            This can be very mode-specific
+
+        """
+  
+        self.logging.verbose("Reordering testcases to optimize test execution...")
+

=== modified file 'tests/lib/test_mode.py'
--- tests/lib/test_mode.py	2011-01-15 21:27:41 +0000
+++ tests/lib/test_mode.py	2011-01-27 23:36:53 +0000
@@ -2,8 +2,20 @@
 # -*- mode: c; c-basic-offset: 2; indent-tabs-mode: nil; -*-
 # vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
 #
-# Copyright (C) 2010 Patrick Crews
-#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 """test_mode.py
    code for dealing with testing modes
    A given mode should have a systemInitializer, testManager, and testExecutor
@@ -31,9 +43,9 @@
         from drizzle_test_run.dtr_test_management import testManager
         test_manager = testManager( variables['verbose'], variables['debug'] 
                                   , variables['engine'], variables['dotest']
-                                  , variables['skiptest'], variables['suitelist']
-                                  , variables['suitepaths'], system_manager
-                                  , variables['test_cases'])
+                                  , variables['skiptest'], variables['reorder']
+                                  , variables['suitelist'], variables['suitepaths']
+                                  , system_manager, variables['test_cases'])
 
         # get our mode-specific testExecutor
         from drizzle_test_run.dtr_test_execution import dtrTestExecutor

=== modified file 'tests/lib/test_run_options.py'
--- tests/lib/test_run_options.py	2011-01-25 21:19:06 +0000
+++ tests/lib/test_run_options.py	2011-01-27 23:36:53 +0000
@@ -4,6 +4,20 @@
 #
 # Copyright (C) 2010 Patrick Crews
 #
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
 
 
 """Processes command line options for Drizzle test-runner"""
@@ -16,7 +30,12 @@
 # functions
 def comma_list_split(option, opt, value, parser):
     """Callback for splitting input expected in list form"""
-    setattr(parser.values, option.dest, value.split(','))
+    cur_list = getattr(parser.values, option.dest,[])
+    if cur_list:
+        value_list = cur_list + value.split(',')
+    else:
+        value_list = value.split(',')
+    setattr(parser.values, option.dest, value_list)
 
 def organize_options(args, test_cases):
     """Put our arguments in a nice dictionary
@@ -91,6 +110,14 @@
   , help="Record a testcase result (if the testing mode supports it) [%default]"
   )
 
+system_control_group.add_option(
+    "--fast"
+  , dest="fast"
+  , action="store_true"
+  , default=False
+  , help="Don't try to cleanup from earlier runs (currently just a placeholder) [%default]"
+  )
+
 parser.add_option_group(system_control_group)
 
 # end system_control_group
@@ -105,7 +132,8 @@
     "--suite"
   , dest="suitelist"
   , type='string'
-  , action="append"
+  , action="callback"
+  , callback=comma_list_split
   , help="The name of the suite containing tests we want. Use one --suite arg for each suite you want to use. [default=autosearch]"
   )
 
@@ -133,6 +161,16 @@
   , help = "input can either be a prefix or a regex.  Will exclude tests that match the provided pattern"
   )
 
+test_control_group.add_option(
+    "--reorder"
+  , dest="reorder"
+  , action="store_true"
+  , default=False
+  , help = "sort the testcases so that they are executed optimally for the given mode [%default]"
+  )
+
+
+
 parser.add_option_group(test_control_group)
 # end test_control_group