Merge lp:~patrick-crews/percona-xtradb-cluster/qp-integrate into lp:~percona-dev/percona-xtradb-cluster/5.5.20
- qp-integrate
- Merge into 5.5.20
Proposed by
Patrick Crews
Status: | Merged |
---|---|
Approved by: | Vadim Tkachenko |
Approved revision: | no longer in the source branch. |
Merged at revision: | 3718 |
Proposed branch: | lp:~patrick-crews/percona-xtradb-cluster/qp-integrate |
Merge into: | lp:~percona-dev/percona-xtradb-cluster/5.5.20 |
Diff against target: | 323164 lines |
To merge this branch: | bzr merge lp:~patrick-crews/percona-xtradb-cluster/qp-integrate |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Vadim Tkachenko | Approve | ||
Review via email: mp+93648@code.launchpad.net |
Commit message
Description of the change
Removed dbqp and integrated kewpie (updated / improved dbqp) into the tree.
Cut down on the size to 25MB from 400+.
To execute tests, simply type ./kewpie.py from the kewpie directory (additional options may apply)
To post a comment you must log in.
Revision history for this message
Vadim Tkachenko (vadim-tk) : | # |
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file '.bzrignore' | |||
2 | --- .bzrignore 2012-02-07 18:26:15 +0000 | |||
3 | +++ .bzrignore 2012-02-17 20:55:33 +0000 | |||
4 | @@ -656,6 +656,8 @@ | |||
5 | 656 | isamchk/*.ds? | 656 | isamchk/*.ds? |
6 | 657 | isamchk/*.vcproj | 657 | isamchk/*.vcproj |
7 | 658 | item_xmlfunc.cc | 658 | item_xmlfunc.cc |
8 | 659 | kewpie/qp_data/uuid | ||
9 | 660 | kewpie/workdir | ||
10 | 659 | lib_debug/* | 661 | lib_debug/* |
11 | 660 | lib_release/* | 662 | lib_release/* |
12 | 661 | libmysql/*.c | 663 | libmysql/*.c |
13 | 662 | 664 | ||
14 | === removed directory 'dbqp' | |||
15 | === removed file 'dbqp/.bzrignore' | |||
16 | --- dbqp/.bzrignore 2012-02-04 01:22:23 +0000 | |||
17 | +++ dbqp/.bzrignore 1970-01-01 00:00:00 +0000 | |||
18 | @@ -1,3 +0,0 @@ | |||
19 | 1 | dbqp | ||
20 | 2 | dbqp_data/uuid | ||
21 | 3 | workdir | ||
22 | 4 | 0 | ||
23 | === removed file 'dbqp/dbqp.py' | |||
24 | --- dbqp/dbqp.py 2012-02-04 01:22:23 +0000 | |||
25 | +++ dbqp/dbqp.py 1970-01-01 00:00:00 +0000 | |||
26 | @@ -1,99 +0,0 @@ | |||
27 | 1 | #! /usr/bin/env python | ||
28 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
29 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
30 | 4 | # | ||
31 | 5 | # Copyright (C) 2010 Patrick Crews | ||
32 | 6 | # | ||
33 | 7 | # This program is free software; you can redistribute it and/or modify | ||
34 | 8 | # it under the terms of the GNU General Public License as published by | ||
35 | 9 | # the Free Software Foundation; either version 2 of the License, or | ||
36 | 10 | # (at your option) any later version. | ||
37 | 11 | # | ||
38 | 12 | # This program is distributed in the hope that it will be useful, | ||
39 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
40 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
41 | 15 | # GNU General Public License for more details. | ||
42 | 16 | # | ||
43 | 17 | # You should have received a copy of the GNU General Public License | ||
44 | 18 | # along with this program; if not, write to the Free Software | ||
45 | 19 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
46 | 20 | |||
47 | 21 | |||
48 | 22 | """ dbqp.py | ||
49 | 23 | |||
50 | 24 | DataBase Quality Platform - system for executing various | ||
51 | 25 | testing systems and the helper code | ||
52 | 26 | |||
53 | 27 | Designed to be a modular test-runner. Different testing tools | ||
54 | 28 | and databases may be plugged into the system via hacking the | ||
55 | 29 | appropriate modules | ||
56 | 30 | |||
57 | 31 | Currently geared towards Drizzle / will expand to MySQL | ||
58 | 32 | """ | ||
59 | 33 | |||
60 | 34 | # imports | ||
61 | 35 | import os | ||
62 | 36 | import sys | ||
63 | 37 | |||
64 | 38 | import lib.opts.test_run_options as test_run_options | ||
65 | 39 | from lib.modes.test_mode import handle_mode | ||
66 | 40 | from lib.server_mgmt.server_management import serverManager | ||
67 | 41 | from lib.sys_mgmt.system_management import systemManager | ||
68 | 42 | from lib.test_mgmt.execution_management import executionManager | ||
69 | 43 | |||
70 | 44 | # functions | ||
71 | 45 | |||
72 | 46 | |||
73 | 47 | # main | ||
74 | 48 | variables = test_run_options.variables | ||
75 | 49 | system_manager = None | ||
76 | 50 | server_manager = None | ||
77 | 51 | test_manager = None | ||
78 | 52 | test_executor = None | ||
79 | 53 | execution_manager = None | ||
80 | 54 | |||
81 | 55 | try: | ||
82 | 56 | # Some system-level work is constant regardless | ||
83 | 57 | # of the test to be run | ||
84 | 58 | system_manager = systemManager(variables) | ||
85 | 59 | |||
86 | 60 | # Create our server_manager | ||
87 | 61 | server_manager = serverManager(system_manager, variables) | ||
88 | 62 | |||
89 | 63 | # Get our mode-specific test_manager and test_executor | ||
90 | 64 | (test_manager,test_executor) = handle_mode(variables, system_manager) | ||
91 | 65 | |||
92 | 66 | # Gather our tests for execution | ||
93 | 67 | test_manager.gather_tests() | ||
94 | 68 | |||
95 | 69 | # Initialize test execution manager | ||
96 | 70 | execution_manager = executionManager(server_manager, system_manager | ||
97 | 71 | , test_manager, test_executor | ||
98 | 72 | , variables) | ||
99 | 73 | |||
100 | 74 | # Execute our tests! | ||
101 | 75 | execution_manager.execute_tests() | ||
102 | 76 | |||
103 | 77 | except Exception, e: | ||
104 | 78 | print Exception, e | ||
105 | 79 | |||
106 | 80 | except KeyboardInterrupt: | ||
107 | 81 | print "\n\nDetected <Ctrl>+c, shutting down and cleaning up..." | ||
108 | 82 | |||
109 | 83 | finally: | ||
110 | 84 | # TODO - make a more robust cleanup | ||
111 | 85 | # At the moment, runaway servers are our biggest concern | ||
112 | 86 | if server_manager and not variables['startandexit']: | ||
113 | 87 | if variables['gdb']: | ||
114 | 88 | server_manager.cleanup_all_servers() | ||
115 | 89 | else: | ||
116 | 90 | server_manager.cleanup() | ||
117 | 91 | if not variables['startandexit']: | ||
118 | 92 | if test_manager: | ||
119 | 93 | fail_count = test_manager.has_failing_tests() | ||
120 | 94 | sys.exit(test_manager.has_failing_tests()) | ||
121 | 95 | else: | ||
122 | 96 | # return 1 as we likely have a problem if we don't have a | ||
123 | 97 | # test_manager | ||
124 | 98 | sys.exit(1) | ||
125 | 99 | |||
126 | 100 | 0 | ||
127 | === removed directory 'dbqp/dbqp_data' | |||
128 | === removed directory 'dbqp/docs' | |||
129 | === removed file 'dbqp/docs/dbqp.rst' | |||
130 | --- dbqp/docs/dbqp.rst 2012-02-04 01:22:23 +0000 | |||
131 | +++ dbqp/docs/dbqp.rst 1970-01-01 00:00:00 +0000 | |||
132 | @@ -1,458 +0,0 @@ | |||
133 | 1 | ********************************** | ||
134 | 2 | dbqp | ||
135 | 3 | ********************************** | ||
136 | 4 | |||
137 | 5 | Synopsis | ||
138 | 6 | ======== | ||
139 | 7 | Drizzle testing tool | ||
140 | 8 | |||
141 | 9 | **./dbqp** [ *OPTIONS* ] [ TESTCASE ] | ||
142 | 10 | |||
143 | 11 | Description | ||
144 | 12 | =========== | ||
145 | 13 | |||
146 | 14 | :program:`dbqp.py` is BETA software. It is intended to provide a standardized | ||
147 | 15 | platform to facilitate Drizzle testing. | ||
148 | 16 | |||
149 | 17 | The default mode is 'dtr' and is used to execute tests from the Drizzle | ||
150 | 18 | test suite. These tests are included with Drizzle distributions and | ||
151 | 19 | provide a way for users to verify that the system will operate according | ||
152 | 20 | to expectations. | ||
153 | 21 | |||
154 | 22 | The dtr tests use a diff-based paradigm, meaning that the test runner executes | ||
155 | 23 | a test and then compares the results received with pre-recorded expected | ||
156 | 24 | results. In the event of a test failure, the program will provide output | ||
157 | 25 | highlighting the differences found between expected and actual results; this | ||
158 | 26 | can be useful for troubleshooting and in bug reports. | ||
159 | 27 | |||
160 | 28 | The program is also integrated with the random query generator testing tool | ||
161 | 29 | and a 'rangden' mode is available - it will execute randgen tests when | ||
162 | 30 | provided a path to a randgen installation. Tests are organized similar to dtr | ||
163 | 31 | tests, but are .cnf file based. | ||
164 | 32 | |||
165 | 33 | A 'cleanup' mode is also available as a convenience - it will simply shutdown | ||
166 | 34 | any servers that may have been started via start-and-exit. | ||
167 | 35 | |||
168 | 36 | While most users are concerned with ensuring general functionality, the | ||
169 | 37 | program also allows a user to quickly spin up a server for ad-hoc testing | ||
170 | 38 | and to run the test-suite against an already running Drizzle server. | ||
171 | 39 | |||
172 | 40 | Running tests | ||
173 | 41 | ========================= | ||
174 | 42 | |||
175 | 43 | There are several different ways to run tests using :program:`dbqp.py`. | ||
176 | 44 | |||
177 | 45 | It should be noted that unless :option:`--force` is used, the program will | ||
178 | 46 | stop execution upon encountering the first failing test. | ||
179 | 47 | :option:`--force` is recommended if you are running several tests - it will | ||
180 | 48 | allow you to view all successes and failures in one run. | ||
181 | 49 | |||
182 | 50 | Running individual tests | ||
183 | 51 | ------------------------ | ||
184 | 52 | If one only wants to run a few, specific tests, they may do so this way:: | ||
185 | 53 | |||
186 | 54 | ./dbqp.py [OPTIONS] test1 [test2 ... testN] | ||
187 | 55 | |||
188 | 56 | Running all tests within a suite | ||
189 | 57 | -------------------------------- | ||
190 | 58 | Many of the tests supplied with Drizzle are organized into suites. | ||
191 | 59 | |||
192 | 60 | The tests within drizzle/tests/t are considered the 'main' suite. | ||
193 | 61 | Other suites are located in either drizzle/tests/suite or within the various | ||
194 | 62 | directories in drizzle/plugin. Tests for a specific plugin should live in | ||
195 | 63 | the plugin's directory - drizzle/plugin/example_plugin/tests | ||
196 | 64 | |||
197 | 65 | To run the tests in a specific suite:: | ||
198 | 66 | |||
199 | 67 | ./dbqp.py [OPTIONS] --suite=SUITENAME | ||
200 | 68 | |||
201 | 69 | Running specific tests within a suite | ||
202 | 70 | -------------------------------------- | ||
203 | 71 | To run a specific set of tests within a suite:: | ||
204 | 72 | |||
205 | 73 | ./dbqp.py [OPTIONS] --suite=SUITENAME TEST1 [TEST2..TESTN] | ||
206 | 74 | |||
207 | 75 | Calling tests using <suitename>.<testname> currently does not work. | ||
208 | 76 | One must specify the test suite via the :option:`--suite` option. | ||
209 | 77 | |||
210 | 78 | |||
211 | 79 | Running all available tests | ||
212 | 80 | --------------------------- | ||
213 | 81 | Currently, the quickest way to execute all tests in all suites is | ||
214 | 82 | to use 'make test-dbqp' from the drizzle root. | ||
215 | 83 | |||
216 | 84 | Otherwise, one should simply name all suites:: | ||
217 | 85 | |||
218 | 86 | ./dbqp.py [OPTIONS] --suite=SUITE1, SUITE2, ...SUITEN | ||
219 | 87 | |||
220 | 88 | Interpreting test results | ||
221 | 89 | ========================= | ||
222 | 90 | The output of the test runner is quite simple. Every test should pass. | ||
223 | 91 | In the event of a test failure, please take the time to file a bug here: | ||
224 | 92 | *https://bugs.launchpad.net/drizzle* | ||
225 | 93 | |||
226 | 94 | During a run, the program will provide the user with: | ||
227 | 95 | * test name (suite + name) | ||
228 | 96 | * test status (pass/fail/skipped) | ||
229 | 97 | * time spent executing each test | ||
230 | 98 | |||
231 | 99 | At the end of a run, the program will provide the user with a listing of: | ||
232 | 100 | * how many tests were run | ||
233 | 101 | * counts and percentages of total exectuted for all test statuses | ||
234 | 102 | * a listing of failing, skipped, or disabled tests | ||
235 | 103 | * total time spent executing the tests | ||
236 | 104 | |||
237 | 105 | Example output:: | ||
238 | 106 | |||
239 | 107 | <snip> | ||
240 | 108 | 30 Jan 2011 16:26:31 : main.small_tmp_table [ pass ] 38 | ||
241 | 109 | 30 Jan 2011 16:26:31 : main.snowman [ pass ] 42 | ||
242 | 110 | 30 Jan 2011 16:26:31 : main.statement_boundaries [ pass ] 47 | ||
243 | 111 | 30 Jan 2011 16:26:31 : main.status [ pass ] 51 | ||
244 | 112 | 30 Jan 2011 16:26:31 : main.strict [ pass ] 138 | ||
245 | 113 | 30 Jan 2011 16:26:43 : main.subselect [ fail ] 12361 | ||
246 | 114 | 30 Jan 2011 16:26:43 : --- drizzle/tests/r/subselect.result 2011-01-30 16:23:54.975776148 -0500 | ||
247 | 115 | 30 Jan 2011 16:26:43 : +++ drizzle/tests/r/subselect.reject 2011-01-30 16:26:43.835519303 -0500 | ||
248 | 116 | 30 Jan 2011 16:26:43 : @@ -5,7 +5,7 @@ | ||
249 | 117 | 30 Jan 2011 16:26:43 : 2 | ||
250 | 118 | 30 Jan 2011 16:26:43 : explain extended select (select 2); | ||
251 | 119 | 30 Jan 2011 16:26:43 : id select_type table type possible_keys key key_len ref rows filtered Extra | ||
252 | 120 | 30 Jan 2011 16:26:43 : -9 PRIMARY NULL NULL NULL NULL NULL NULL NULL NULL No tables used | ||
253 | 121 | 30 Jan 2011 16:26:43 : +1 PRIMARY NULL NULL NULL NULL NULL NULL NULL NULL No tables used | ||
254 | 122 | <snip> | ||
255 | 123 | 30 Jan 2011 16:30:20 : ================================================================================ | ||
256 | 124 | 30 Jan 2011 16:30:20 INFO: Test execution complete in 314 seconds | ||
257 | 125 | 30 Jan 2011 16:30:20 INFO: Summary report: | ||
258 | 126 | 30 Jan 2011 16:30:20 INFO: Executed 552/552 test cases, 100.00 percent | ||
259 | 127 | 30 Jan 2011 16:30:20 INFO: STATUS: FAIL, 1/552 test cases, 0.18 percent executed | ||
260 | 128 | 30 Jan 2011 16:30:20 INFO: STATUS: PASS, 551/552 test cases, 99.82 percent executed | ||
261 | 129 | 30 Jan 2011 16:30:20 INFO: FAIL tests: main.subselect | ||
262 | 130 | 30 Jan 2011 16:30:20 INFO: Spent 308 / 314 seconds on: TEST(s) | ||
263 | 131 | 30 Jan 2011 16:30:20 INFO: Test execution complete | ||
264 | 132 | 30 Jan 2011 16:30:20 INFO: Stopping all running servers... | ||
265 | 133 | |||
266 | 134 | |||
267 | 135 | Additional uses | ||
268 | 136 | =============== | ||
269 | 137 | Starting a server for manual testing | ||
270 | 138 | ------------------------------------ | ||
271 | 139 | |||
272 | 140 | :program:`dbqp.py` allows a user to get a Drizzle server up and running | ||
273 | 141 | quickly. This can be useful for fast ad-hoc testing. | ||
274 | 142 | |||
275 | 143 | To do so call:: | ||
276 | 144 | |||
277 | 145 | ./dbqp.py --start-and-exit [*OPTIONS*] | ||
278 | 146 | |||
279 | 147 | This will start a Drizzle server that you can connect to and query | ||
280 | 148 | |||
281 | 149 | Starting a server against a pre-populated DATADIR | ||
282 | 150 | -------------------------------------------------- | ||
283 | 151 | |||
284 | 152 | Using :option:`--start-dirty` prevents :program:`dbqp.py` from attempting | ||
285 | 153 | to initialize (clean) the datadir. This can be useful if you want to use | ||
286 | 154 | an already-populated datadir for testing. | ||
287 | 155 | |||
288 | 156 | NOTE: This feature is still being tested, use caution with your data!!! | ||
289 | 157 | |||
290 | 158 | Randgen mode / Executing randgen tests | ||
291 | 159 | --------------------------------------- | ||
292 | 160 | |||
293 | 161 | Using :option:`--mode` =randgen and :option:`--randgen-path` =/path/to/randgen | ||
294 | 162 | will cause the randgen tests to execute. This are simple .cnf file-based | ||
295 | 163 | tests that define various randgen command lines that are useful in testing | ||
296 | 164 | the server. Test organization is similar to the dtr tests. Tests live in | ||
297 | 165 | suites, the default suite is 'main' and they all live in | ||
298 | 166 | drizzle/tests/randgen_tests:: | ||
299 | 167 | |||
300 | 168 | ./dbqp.py --mode=randgen --randgen-path=/path/to/randgen | ||
301 | 169 | |||
302 | 170 | A user may specify suites and individual tests to run, just as with dtr-based | ||
303 | 171 | testing. Test output is the same as well:: | ||
304 | 172 | |||
305 | 173 | ./dbqp --mode=randgen --randgen-path=/home/username/repos/randgen | ||
306 | 174 | Setting --no-secure-file-priv=True for randgen mode... | ||
307 | 175 | <snip> | ||
308 | 176 | 23 Feb 2011 11:42:43 INFO: Using testing mode: randgen | ||
309 | 177 | <snip> | ||
310 | 178 | 23 Feb 2011 11:44:58 : ================================================================================ | ||
311 | 179 | 23 Feb 2011 11:44:58 : TEST NAME [ RESULT ] TIME (ms) | ||
312 | 180 | 23 Feb 2011 11:44:58 : ================================================================================ | ||
313 | 181 | 23 Feb 2011 11:44:58 : main.optimizer_subquery [ pass ] 134153 | ||
314 | 182 | 23 Feb 2011 11:45:03 : main.outer_join [ pass ] 5136 | ||
315 | 183 | 23 Feb 2011 11:45:06 : main.simple [ pass ] 2246 | ||
316 | 184 | 23 Feb 2011 11:45:06 : ================================================================================ | ||
317 | 185 | 23 Feb 2011 11:45:06 INFO: Test execution complete in 142 seconds | ||
318 | 186 | 23 Feb 2011 11:45:06 INFO: Summary report: | ||
319 | 187 | 23 Feb 2011 11:45:06 INFO: Executed 3/3 test cases, 100.00 percent | ||
320 | 188 | 23 Feb 2011 11:45:06 INFO: STATUS: PASS, 3/3 test cases, 100.00 percent executed | ||
321 | 189 | 23 Feb 2011 11:45:06 INFO: Spent 141 / 142 seconds on: TEST(s) | ||
322 | 190 | 23 Feb 2011 11:45:06 INFO: Test execution complete | ||
323 | 191 | 23 Feb 2011 11:45:06 INFO: Stopping all running servers... | ||
324 | 192 | |||
325 | 193 | Cleanup mode | ||
326 | 194 | ------------- | ||
327 | 195 | A cleanup mode is provided for user convenience. This simply shuts down | ||
328 | 196 | any servers whose pid files are detected in the dbqp workdir. It is mainly | ||
329 | 197 | intended as a quick cleanup for post-testing with :option:`--start-and-exit`:: | ||
330 | 198 | |||
331 | 199 | ./dbqp.py --mode=cleanup | ||
332 | 200 | |||
333 | 201 | Setting --start-dirty=True for cleanup mode... | ||
334 | 202 | 23 Feb 2011 11:35:59 INFO: Using Drizzle source tree: | ||
335 | 203 | 23 Feb 2011 11:35:59 INFO: basedir: drizzle | ||
336 | 204 | 23 Feb 2011 11:35:59 INFO: clientbindir: drizzle/client | ||
337 | 205 | 23 Feb 2011 11:35:59 INFO: testdir: drizzle/tests | ||
338 | 206 | 23 Feb 2011 11:35:59 INFO: server_version: 2011.02.2188 | ||
339 | 207 | 23 Feb 2011 11:35:59 INFO: server_compile_os: unknown-linux-gnu | ||
340 | 208 | 23 Feb 2011 11:35:59 INFO: server_platform: x86_64 | ||
341 | 209 | 23 Feb 2011 11:35:59 INFO: server_comment: (Source distribution (dbqp_randgen)) | ||
342 | 210 | 23 Feb 2011 11:35:59 INFO: Using --start-dirty, not attempting to touch directories | ||
343 | 211 | 23 Feb 2011 11:35:59 INFO: Using default-storage-engine: innodb | ||
344 | 212 | 23 Feb 2011 11:35:59 INFO: Using testing mode: cleanup | ||
345 | 213 | 23 Feb 2011 11:35:59 INFO: Killing pid 10484 from drizzle/tests/workdir/testbot0/server0/var/run/server0.pid | ||
346 | 214 | 23 Feb 2011 11:35:59 INFO: Stopping all running servers... | ||
347 | 215 | |||
348 | 216 | Program architecture | ||
349 | 217 | ==================== | ||
350 | 218 | |||
351 | 219 | :program:`dbqp.py`'s 'dtr' mode uses a simple diff-based mechanism for testing. | ||
352 | 220 | This is the default mode and where the majority of Drizzle testing occurs. | ||
353 | 221 | It will execute the statements contained in a test and compare the results | ||
354 | 222 | to pre-recorded expected results. In the event of a test failure, you | ||
355 | 223 | will be presented with a diff:: | ||
356 | 224 | |||
357 | 225 | main.exp1 [ fail ] | ||
358 | 226 | --- drizzle/tests/r/exp1.result 2010-11-02 02:10:25.107013998 +0300 | ||
359 | 227 | +++ drizzle/tests/r/exp1.reject 2010-11-02 02:10:32.017013999 +0300 | ||
360 | 228 | @@ -5,4 +5,5 @@ | ||
361 | 229 | a | ||
362 | 230 | 1 | ||
363 | 231 | 2 | ||
364 | 232 | +3 | ||
365 | 233 | DROP TABLE t1; | ||
366 | 234 | |||
367 | 235 | A test case consists of a .test and a .result file. The .test file includes | ||
368 | 236 | the various statements to be executed for a test. The .result file lists | ||
369 | 237 | the expected results for a given test file. These files live in tests/t | ||
370 | 238 | and tests/r, respectively. This structure is the same for all test suites. | ||
371 | 239 | |||
372 | 240 | dbqp.py options | ||
373 | 241 | =================== | ||
374 | 242 | |||
375 | 243 | The :program:`dbqp.py` tool has several available options: | ||
376 | 244 | |||
377 | 245 | ./dbqp.py [ OPTIONS ] [ TESTCASE ] | ||
378 | 246 | |||
379 | 247 | |||
380 | 248 | Options | ||
381 | 249 | ------- | ||
382 | 250 | |||
383 | 251 | .. program:: dbqp.py | ||
384 | 252 | |||
385 | 253 | .. option:: -h, --help | ||
386 | 254 | |||
387 | 255 | show this help message and exit | ||
388 | 256 | |||
389 | 257 | Options for the test-runner itself | ||
390 | 258 | ---------------------------------- | ||
391 | 259 | |||
392 | 260 | .. program:: dbqp.py | ||
393 | 261 | |||
394 | 262 | .. option:: --force | ||
395 | 263 | |||
396 | 264 | Set this to continue test execution beyond the first failed test | ||
397 | 265 | |||
398 | 266 | .. option:: --start-and-exit | ||
399 | 267 | |||
400 | 268 | Spin up the server(s) for the first specified test then exit | ||
401 | 269 | (will leave servers running) | ||
402 | 270 | |||
403 | 271 | .. option:: --verbose | ||
404 | 272 | |||
405 | 273 | Produces extensive output about test-runner state. | ||
406 | 274 | Distinct from --debug | ||
407 | 275 | |||
408 | 276 | .. option:: --debug | ||
409 | 277 | |||
410 | 278 | Provide internal-level debugging output. | ||
411 | 279 | Distinct from --verbose | ||
412 | 280 | |||
413 | 281 | .. option:: --mode=MODE | ||
414 | 282 | |||
415 | 283 | Testing mode. | ||
416 | 284 | We only support dtr...for now >;) | ||
417 | 285 | [dtr] | ||
418 | 286 | |||
419 | 287 | .. option:: --record | ||
420 | 288 | |||
421 | 289 | Record a testcase result | ||
422 | 290 | (if the testing mode supports it) | ||
423 | 291 | [False] | ||
424 | 292 | |||
425 | 293 | .. option:: --fast | ||
426 | 294 | |||
427 | 295 | Don't try to cleanup from earlier runs | ||
428 | 296 | (currently just a placeholder) [False] | ||
429 | 297 | |||
430 | 298 | .. option:: --randgen-path=RANDGENPATH | ||
431 | 299 | |||
432 | 300 | The path to a randgen installation that can be used to | ||
433 | 301 | execute randgen-based tests | ||
434 | 302 | |||
435 | 303 | |||
436 | 304 | Options for controlling which tests are executed | ||
437 | 305 | ------------------------------------------------ | ||
438 | 306 | |||
439 | 307 | .. program:: dbqp.py | ||
440 | 308 | |||
441 | 309 | .. option:: --suite=SUITELIST | ||
442 | 310 | |||
443 | 311 | The name of the suite containing tests we want. | ||
444 | 312 | Can accept comma-separated list (with no spaces). | ||
445 | 313 | Additional --suite args are appended to existing list | ||
446 | 314 | [autosearch] | ||
447 | 315 | |||
448 | 316 | .. option:: --suitepath=SUITEPATHS | ||
449 | 317 | |||
450 | 318 | The path containing the suite(s) you wish to execute. | ||
451 | 319 | Use on --suitepath for each suite you want to use. | ||
452 | 320 | |||
453 | 321 | .. option:: --do-test=DOTEST | ||
454 | 322 | |||
455 | 323 | input can either be a prefix or a regex. | ||
456 | 324 | Will only execute tests that match the provided pattern | ||
457 | 325 | |||
458 | 326 | .. option:: --skip-test=SKIPTEST | ||
459 | 327 | |||
460 | 328 | input can either be a prefix or a regex. | ||
461 | 329 | Will exclude tests that match the provided pattern | ||
462 | 330 | |||
463 | 331 | .. option:: --reorder | ||
464 | 332 | |||
465 | 333 | sort the testcases so that they are executed optimally | ||
466 | 334 | for the given mode [False] | ||
467 | 335 | |||
468 | 336 | .. option:: --repeat=REPEAT | ||
469 | 337 | |||
470 | 338 | Run each test case the specified number of times. For | ||
471 | 339 | a given sequence, the first test will be run n times, | ||
472 | 340 | then the second, etc [1] | ||
473 | 341 | |||
474 | 342 | Options for defining the code that will be under test | ||
475 | 343 | ----------------------------------------------------- | ||
476 | 344 | |||
477 | 345 | .. program:: dbqp.py | ||
478 | 346 | |||
479 | 347 | .. option:: --basedir=BASEDIR | ||
480 | 348 | |||
481 | 349 | Pass this argument to signal to the test-runner | ||
482 | 350 | that this is an in-tree test (not required). | ||
483 | 351 | We automatically set a number of variables | ||
484 | 352 | relative to the argument (client-bindir, | ||
485 | 353 | serverdir, testdir) [../] | ||
486 | 354 | |||
487 | 355 | .. option:: --serverdir=SERVERPATH | ||
488 | 356 | |||
489 | 357 | Path to the server executable. [auto-search] | ||
490 | 358 | |||
491 | 359 | .. option:: --client-bindir=CLIENTBINDIR | ||
492 | 360 | |||
493 | 361 | Path to the directory containing client program | ||
494 | 362 | binaries for use in testing [auto-search] | ||
495 | 363 | |||
496 | 364 | .. option:: --default-storage-engine=DEFAULTENGINE | ||
497 | 365 | |||
498 | 366 | Start drizzled using the specified engine [innodb] | ||
499 | 367 | |||
500 | 368 | Options for defining the testing environment | ||
501 | 369 | -------------------------------------------- | ||
502 | 370 | |||
503 | 371 | .. program:: dbqp.py | ||
504 | 372 | |||
505 | 373 | .. option:: --testdir=TESTDIR | ||
506 | 374 | |||
507 | 375 | Path to the test dir, containing additional files for | ||
508 | 376 | test execution. [pwd] | ||
509 | 377 | |||
510 | 378 | .. option:: --workdir=WORKDIR | ||
511 | 379 | |||
512 | 380 | Path to the directory test-run will use to store | ||
513 | 381 | generated files and directories. | ||
514 | 382 | [basedir/tests/dbqp_work] | ||
515 | 383 | |||
516 | 384 | .. option:: --top-srcdir=TOPSRCDIR | ||
517 | 385 | |||
518 | 386 | build option [basedir_default] | ||
519 | 387 | |||
520 | 388 | .. option:: --top-builddir=TOPBUILDDIR | ||
521 | 389 | |||
522 | 390 | build option [basedir_default] | ||
523 | 391 | |||
524 | 392 | .. option:: --no-shm | ||
525 | 393 | |||
526 | 394 | By default, we symlink workdir to a location in shm. | ||
527 | 395 | Use this flag to not symlink [False] | ||
528 | 396 | |||
529 | 397 | .. option:: --start-dirty | ||
530 | 398 | |||
531 | 399 | Don't try to clean up working directories before test | ||
532 | 400 | execution [False] | ||
533 | 401 | |||
534 | 402 | .. option:: --no-secure-file-priv | ||
535 | 403 | |||
536 | 404 | Turn off the use of --secure-file-priv=vardir for | ||
537 | 405 | started servers | ||
538 | 406 | |||
539 | 407 | Options to pass options on to the server | ||
540 | 408 | ----------------------------------------- | ||
541 | 409 | |||
542 | 410 | .. program:: dbqp.py | ||
543 | 411 | |||
544 | 412 | .. option:: --drizzled=DRIZZLEDOPTIONS | ||
545 | 413 | |||
546 | 414 | Pass additional options to the server. Will be passed | ||
547 | 415 | to all servers for all tests (mostly for --start-and- | ||
548 | 416 | exit) | ||
549 | 417 | |||
550 | 418 | |||
551 | 419 | Options for defining the tools we use for code analysis (valgrind, gprof, gcov, etc) | ||
552 | 420 | ------------------------------------------------------------------------------------ | ||
553 | 421 | |||
554 | 422 | .. program:: dbqp.py | ||
555 | 423 | |||
556 | 424 | .. option:: --valgrind | ||
557 | 425 | |||
558 | 426 | Run drizzletest and drizzled executables using | ||
559 | 427 | valgrind with default options [False] | ||
560 | 428 | |||
561 | 429 | .. option:: --valgrind-option=VALGRINDARGLIST | ||
562 | 430 | |||
563 | 431 | Pass an option to valgrind (overrides/removes default | ||
564 | 432 | valgrind options) | ||
565 | 433 | |||
566 | 434 | Options for controlling the use of debuggers with test execution | ||
567 | 435 | ---------------------------------------------------------------- | ||
568 | 436 | |||
569 | 437 | .. program:: dbqp.py | ||
570 | 438 | |||
571 | 439 | .. option:: --gdb | ||
572 | 440 | |||
573 | 441 | Start the drizzled server(s) in gdb | ||
574 | 442 | |||
575 | 443 | .. option:: --manual-gdb | ||
576 | 444 | |||
577 | 445 | Allows you to start the drizzled server(s) in gdb | ||
578 | 446 | manually (in another window, etc | ||
579 | 447 | |||
580 | 448 | Options to call additional utilities such as datagen | ||
581 | 449 | ------------------------------------------------------ | ||
582 | 450 | |||
583 | 451 | .. program:: dbqp.py | ||
584 | 452 | |||
585 | 453 | .. option:: --gendata=GENDATAFILE | ||
586 | 454 | |||
587 | 455 | Call the randgen's gendata utility to use the | ||
588 | 456 | specified configuration file. This will populate the | ||
589 | 457 | server prior to any test execution | ||
590 | 458 | |||
591 | 459 | 0 | ||
592 | === removed file 'dbqp/docs/index.rst' | |||
593 | --- dbqp/docs/index.rst 2012-02-04 01:22:23 +0000 | |||
594 | +++ dbqp/docs/index.rst 1970-01-01 00:00:00 +0000 | |||
595 | @@ -1,33 +0,0 @@ | |||
596 | 1 | .. dbqp documentation master file, created by | ||
597 | 2 | sphinx-quickstart on Fri Aug 27 08:33:41 2010. | ||
598 | 3 | You can adapt this file completely to your liking, but it should at least | ||
599 | 4 | contain the root `toctree` directive. | ||
600 | 5 | |||
601 | 6 | Welcome to dbqp's documentation! | ||
602 | 7 | =================================== | ||
603 | 8 | |||
604 | 9 | dbqp (DataBase Quality Platform) is designed to facilitate testing of MySQL-based database systems. Its aim is to provide a pluggable system that allows one to run a variety of testing tools and to share standard helper code (server allocation and management / test result reporting / etc) | ||
605 | 10 | |||
606 | 11 | Introduction: | ||
607 | 12 | ------------- | ||
608 | 13 | .. toctree:: | ||
609 | 14 | :maxdepth: 2 | ||
610 | 15 | |||
611 | 16 | Testing: | ||
612 | 17 | -------- | ||
613 | 18 | .. toctree:: | ||
614 | 19 | :maxdepth: 2 | ||
615 | 20 | |||
616 | 21 | dbqp.rst | ||
617 | 22 | test-run.rst | ||
618 | 23 | randgen.rst | ||
619 | 24 | sql-bench.rst | ||
620 | 25 | sysbench.rst | ||
621 | 26 | writing_tests.rst | ||
622 | 27 | |||
623 | 28 | Indices and tables | ||
624 | 29 | ================== | ||
625 | 30 | |||
626 | 31 | * :ref:`genindex` | ||
627 | 32 | * :ref:`search` | ||
628 | 33 | |||
629 | 34 | 0 | ||
630 | === removed file 'dbqp/docs/randgen.rst' | |||
631 | --- dbqp/docs/randgen.rst 2012-02-04 01:22:23 +0000 | |||
632 | +++ dbqp/docs/randgen.rst 1970-01-01 00:00:00 +0000 | |||
633 | @@ -1,197 +0,0 @@ | |||
634 | 1 | ********************************** | ||
635 | 2 | randgen (random query generator) | ||
636 | 3 | ********************************** | ||
637 | 4 | |||
638 | 5 | |||
639 | 6 | |||
640 | 7 | Description | ||
641 | 8 | =========== | ||
642 | 9 | |||
643 | 10 | The randgen aka the random query generator is a database | ||
644 | 11 | testing tool. It uses a grammar-based stochastic model to represent | ||
645 | 12 | some desired set of queries (to exercise the optimizer, for example) | ||
646 | 13 | and generates random queries as allowed by the grammar | ||
647 | 14 | |||
648 | 15 | The primary documentation is here: http://forge.mysql.com/wiki/RandomQueryGenerator | ||
649 | 16 | |||
650 | 17 | This document is intended to help the user set up their environment so that the tool | ||
651 | 18 | may be used in conjunction with the dbqp.py test-runner. The forge documentation | ||
652 | 19 | contains more information on the particulars of the tool itself. | ||
653 | 20 | |||
654 | 21 | Requirements | ||
655 | 22 | ============ | ||
656 | 23 | |||
657 | 24 | DBD::drizzle | ||
658 | 25 | ------------- | ||
659 | 26 | The DBD::drizzle module is required it can be found here http://launchpad.net/dbd-drizzle/ | ||
660 | 27 | |||
661 | 28 | Additional information for installing the module:: | ||
662 | 29 | |||
663 | 30 | Prerequisites | ||
664 | 31 | ---------------- | ||
665 | 32 | * Perl | ||
666 | 33 | * Drizzle (bzr branch lp:drizzle) | ||
667 | 34 | * libdrizzle (bzr branch lp:libdrizzle) | ||
668 | 35 | * C compiler | ||
669 | 36 | |||
670 | 37 | Installation | ||
671 | 38 | ------------- | ||
672 | 39 | You should only have to run the following: | ||
673 | 40 | |||
674 | 41 | perl Makefile.PL --cflags=-I/usr/local/drizzle/include/ --libs=-"L/usr/local/drizzle/lib -ldrizzle" | ||
675 | 42 | |||
676 | 43 | |||
677 | 44 | Depending on where libdrizzle is installed. Also, you'll want to make | ||
678 | 45 | sure that ldconfig has configured libdrizzle to be in your library path | ||
679 | 46 | |||
680 | 47 | Additional information may be found here: http://forge.mysql.com/wiki/RandomQueryGeneratorQuickStart | ||
681 | 48 | |||
682 | 49 | Installing the randgen | ||
683 | 50 | ======================= | ||
684 | 51 | |||
685 | 52 | The code may be branched from launchpad: bzr branch lp:randgen | ||
686 | 53 | |||
687 | 54 | it also may be downloaded from here http://launchpad.net/randgen/+download | ||
688 | 55 | |||
689 | 56 | That is all there is : ) | ||
690 | 57 | |||
691 | 58 | Randgen / dbqp tests | ||
692 | 59 | ==================== | ||
693 | 60 | |||
694 | 61 | These tests are simple .cnf files that can define a few basic variables | ||
695 | 62 | that are needed to execute tests. The most interesting section is test_servers. It is a simple list of python lists | ||
696 | 63 | Each sub-list contains a string of server options that are needed. Each sub-list represents a server that will be started. | ||
697 | 64 | Using an empty sub-list will create a server with the default options:: | ||
698 | 65 | |||
699 | 66 | [test_info] | ||
700 | 67 | comment = does NOT actually test the master-slave replication yet, but it will. | ||
701 | 68 | |||
702 | 69 | [test_command] | ||
703 | 70 | command = ./gentest.pl --gendata=conf/drizzle/drizzle.zz --grammar=conf/drizzle/optimizer_subquery_drizzle.yy --queries=10 --threads=1 | ||
704 | 71 | |||
705 | 72 | [test_servers] | ||
706 | 73 | servers = [[--innodb.replication-log],[--plugin-add=slave --slave.config-file=$MASTER_SERVER_SLAVE_CONFIG]] | ||
707 | 74 | |||
708 | 75 | Running tests | ||
709 | 76 | ========================= | ||
710 | 77 | |||
711 | 78 | There are several different ways to run tests using :doc:`dbqp` 's randgen mode. | ||
712 | 79 | |||
713 | 80 | It should be noted that unless :option:`--force` is used, the program will | ||
714 | 81 | stop execution upon encountering the first failing test. | ||
715 | 82 | :option:`--force` is recommended if you are running several tests - it will | ||
716 | 83 | allow you to view all successes and failures in one run. | ||
717 | 84 | |||
718 | 85 | Running individual tests | ||
719 | 86 | ------------------------ | ||
720 | 87 | If one only wants to run a few, specific tests, they may do so this way:: | ||
721 | 88 | |||
722 | 89 | ./dbqp --mode=randgen --randgen-path=/path/to/randgen [OPTIONS] test1 [test2 ... testN] | ||
723 | 90 | |||
724 | 91 | Running all tests within a suite | ||
725 | 92 | -------------------------------- | ||
726 | 93 | Many of the tests supplied with Drizzle are organized into suites. | ||
727 | 94 | |||
728 | 95 | The tests within drizzle/tests/randgen_tests/main are considered the 'main' suite. | ||
729 | 96 | Other suites are also subdirectories of drizzle/tests/randgen_tests. | ||
730 | 97 | |||
731 | 98 | To run the tests in a specific suite:: | ||
732 | 99 | |||
733 | 100 | ./dbqp --mode=randgen --randgen-path=/path/to/randgen [OPTIONS] --suite=SUITENAME | ||
734 | 101 | |||
735 | 102 | Running specific tests within a suite | ||
736 | 103 | -------------------------------------- | ||
737 | 104 | To run a specific set of tests within a suite:: | ||
738 | 105 | |||
739 | 106 | ./dbqp --mode=randgen --randgen-path=/path/to/randgen [OPTIONS] --suite=SUITENAME TEST1 [TEST2..TESTN] | ||
740 | 107 | |||
741 | 108 | Calling tests using <suitename>.<testname> currently does not work. | ||
742 | 109 | One must specify the test suite via the :option:`--suite` option. | ||
743 | 110 | |||
744 | 111 | |||
745 | 112 | Running all available tests | ||
746 | 113 | --------------------------- | ||
747 | 114 | One would currently have to name all suites, but the majority of the working tests live in the main suite | ||
748 | 115 | Other suites utilize more exotic server combinations and we are currently tweaking them to better integrate with the | ||
749 | 116 | dbqp system. The slave-plugin suite does currently have a good config file for setting up simple replication setups for testing. | ||
750 | 117 | To execute several suites' worth of tests:: | ||
751 | 118 | |||
752 | 119 | ./dbqp --mode=randgen --randgen-path=/path/to/randgen [OPTIONS] --suite=SUITE1, SUITE2, ...SUITEN | ||
753 | 120 | |||
754 | 121 | Interpreting test results | ||
755 | 122 | ========================= | ||
756 | 123 | The output of the test runner is quite simple. Every test should pass. | ||
757 | 124 | In the event of a test failure, please take the time to file a bug here: | ||
758 | 125 | *https://bugs.launchpad.net/drizzle* | ||
759 | 126 | |||
760 | 127 | During a run, the program will provide the user with: | ||
761 | 128 | * test name (suite + name) | ||
762 | 129 | * test status (pass/fail/skipped) | ||
763 | 130 | * time spent executing each test | ||
764 | 131 | |||
765 | 132 | At the end of a run, the program will provide the user with a listing of: | ||
766 | 133 | * how many tests were run | ||
767 | 134 | * how many tests failed | ||
768 | 135 | * percentage of passing tests | ||
769 | 136 | * a listing of failing tests | ||
770 | 137 | * total time spent executing the tests | ||
771 | 138 | |||
772 | 139 | Example output:: | ||
773 | 140 | |||
774 | 141 | 24 Feb 2011 17:27:36 : main.outer_join_portable [ pass ] 7019 | ||
775 | 142 | 24 Feb 2011 17:27:39 : main.repeatable_read [ pass ] 2764 | ||
776 | 143 | 24 Feb 2011 17:28:57 : main.select_stability_validator [ pass ] 77946 | ||
777 | 144 | 24 Feb 2011 17:29:01 : main.subquery [ pass ] 4474 | ||
778 | 145 | 24 Feb 2011 17:30:52 : main.subquery_semijoin [ pass ] 110355 | ||
779 | 146 | 24 Feb 2011 17:31:00 : main.subquery_semijoin_nested [ pass ] 8750 | ||
780 | 147 | 24 Feb 2011 17:31:03 : main.varchar [ pass ] 3048 | ||
781 | 148 | 24 Feb 2011 17:31:03 : ================================================================================ | ||
782 | 149 | 24 Feb 2011 17:31:03 INFO: Test execution complete in 288 seconds | ||
783 | 150 | 24 Feb 2011 17:31:03 INFO: Summary report: | ||
784 | 151 | 24 Feb 2011 17:31:03 INFO: Executed 18/18 test cases, 100.00 percent | ||
785 | 152 | 24 Feb 2011 17:31:03 INFO: STATUS: PASS, 18/18 test cases, 100.00 percent executed | ||
786 | 153 | 24 Feb 2011 17:31:03 INFO: Spent 287 / 288 seconds on: TEST(s) | ||
787 | 154 | 24 Feb 2011 17:31:03 INFO: Test execution complete | ||
788 | 155 | 24 Feb 2011 17:31:03 INFO: Stopping all running servers... | ||
789 | 156 | |||
790 | 157 | |||
791 | 158 | Additional uses | ||
792 | 159 | =============== | ||
793 | 160 | Starting a server for manual testing and (optionally) populating it | ||
794 | 161 | -------------------------------------------------------------------- | ||
795 | 162 | |||
796 | 163 | :doc:`dbqp` 's randgen mode allows a user to get a Drizzle server up and running quickly. This can be useful for fast ad-hoc testing. | ||
797 | 164 | |||
798 | 165 | To do so call:: | ||
799 | 166 | |||
800 | 167 | ./dbqp --mode=randgen --randgen-path=/path/to/randgen --start-and-exit [*OPTIONS*] | ||
801 | 168 | |||
802 | 169 | This will start a Drizzle server that you can connect to and query | ||
803 | 170 | |||
804 | 171 | With the addition of the --gendata option, a user may utilize the randgen's gendata (table creation and population) tool | ||
805 | 172 | to populate a test server. In the following example, the test server is now populated by the 8 tables listed below:: | ||
806 | 173 | |||
807 | 174 | ./dbqp --mode=randgen --randgen-path=/randgen --start-and-exit --gendata=/randgen/conf/drizzle/drizzle.zz | ||
808 | 175 | <snip> | ||
809 | 176 | 24 Feb 2011 17:48:48 INFO: NAME: server0 | ||
810 | 177 | 24 Feb 2011 17:48:48 INFO: MASTER_PORT: 9306 | ||
811 | 178 | 24 Feb 2011 17:48:48 INFO: DRIZZLE_TCP_PORT: 9307 | ||
812 | 179 | 24 Feb 2011 17:48:48 INFO: MC_PORT: 9308 | ||
813 | 180 | 24 Feb 2011 17:48:48 INFO: PBMS_PORT: 9309 | ||
814 | 181 | 24 Feb 2011 17:48:48 INFO: RABBITMQ_NODE_PORT: 9310 | ||
815 | 182 | 24 Feb 2011 17:48:48 INFO: VARDIR: /home/pcrews/bzr/work/dbqp_randgen_updates/tests/workdir/testbot0/server0/var | ||
816 | 183 | 24 Feb 2011 17:48:48 INFO: STATUS: 1 | ||
817 | 184 | # 2011-02-24T17:48:48 Default schema: test | ||
818 | 185 | # 2011-02-24T17:48:48 Executor initialized, id GenTest::Executor::Drizzle 2011.02.2198 () | ||
819 | 186 | # 2011-02-24T17:48:48 # Creating Drizzle table: test.A; engine: ; rows: 0 . | ||
820 | 187 | # 2011-02-24T17:48:48 # Creating Drizzle table: test.B; engine: ; rows: 0 . | ||
821 | 188 | # 2011-02-24T17:48:48 # Creating Drizzle table: test.C; engine: ; rows: 1 . | ||
822 | 189 | # 2011-02-24T17:48:48 # Creating Drizzle table: test.D; engine: ; rows: 1 . | ||
823 | 190 | # 2011-02-24T17:48:48 # Creating Drizzle table: test.AA; engine: ; rows: 10 . | ||
824 | 191 | # 2011-02-24T17:48:48 # Creating Drizzle table: test.BB; engine: ; rows: 10 . | ||
825 | 192 | # 2011-02-24T17:48:48 # Creating Drizzle table: test.CC; engine: ; rows: 100 . | ||
826 | 193 | # 2011-02-24T17:48:49 # Creating Drizzle table: test.DD; engine: ; rows: 100 . | ||
827 | 194 | 24 Feb 2011 17:48:49 INFO: User specified --start-and-exit. dbqp.py exiting and leaving servers running... | ||
828 | 195 | |||
829 | 196 | |||
830 | 197 | |||
831 | 198 | 0 | ||
832 | === removed file 'dbqp/docs/sql-bench.rst' | |||
833 | --- dbqp/docs/sql-bench.rst 2012-02-04 01:22:23 +0000 | |||
834 | +++ dbqp/docs/sql-bench.rst 1970-01-01 00:00:00 +0000 | |||
835 | @@ -1,174 +0,0 @@ | |||
836 | 1 | ********************************** | ||
837 | 2 | sql-bench | ||
838 | 3 | ********************************** | ||
839 | 4 | |||
840 | 5 | |||
841 | 6 | |||
842 | 7 | Description | ||
843 | 8 | =========== | ||
844 | 9 | dbqp can take advantage of two testing modes offered by the sql-bench tool. | ||
845 | 10 | |||
846 | 11 | The Drizzle code has a copy of this tool set in the tree and the test-runner offers two modes:: | ||
847 | 12 | |||
848 | 13 | sql-bench modes | ||
849 | 14 | --------------- | ||
850 | 15 | * sqlbench - runs the entire sql-bench test suite and can take a very long time (~45 minutes) | ||
851 | 16 | * crashme - runs the crash-me tool and reports failure if any tests should not pass | ||
852 | 17 | |||
853 | 18 | |||
854 | 19 | Requirements | ||
855 | 20 | ============ | ||
856 | 21 | DBD::drizzle | ||
857 | 22 | ------------- | ||
858 | 23 | The DBD::drizzle module is required it can be found here http://launchpad.net/dbd-drizzle/ | ||
859 | 24 | |||
860 | 25 | Additional information for installing the module:: | ||
861 | 26 | |||
862 | 27 | Prerequisites | ||
863 | 28 | ---------------- | ||
864 | 29 | * Perl | ||
865 | 30 | * Drizzle (bzr branch lp:drizzle) | ||
866 | 31 | * libdrizzle (bzr branch lp:libdrizzle) | ||
867 | 32 | * C compiler | ||
868 | 33 | |||
869 | 34 | Installation | ||
870 | 35 | ------------- | ||
871 | 36 | You should only have to run the following: | ||
872 | 37 | |||
873 | 38 | perl Makefile.PL --cflags=-I/usr/local/drizzle/include/ --libs=-"L/usr/local/drizzle/lib -ldrizzle" | ||
874 | 39 | |||
875 | 40 | |||
876 | 41 | Depending on where libdrizzle is installed. Also, you'll want to make | ||
877 | 42 | sure that ldconfig has configured libdrizzle to be in your library path | ||
878 | 43 | |||
879 | 44 | |||
880 | 45 | sql-bench / dbqp tests | ||
881 | 46 | ======================= | ||
882 | 47 | |||
883 | 48 | Currently, there are only two sql-bench test cases for dbqp. As one might expect, main.all_sqlbench_tests executes:: | ||
884 | 49 | |||
885 | 50 | run-all-tests --server=drizzle --dir=$DRIZZLE_TEST_WORKDIR --log --connect-options=port=$MASTER_MYPORT --create-options=ENGINE=innodb --user=root | ||
886 | 51 | |||
887 | 52 | against a Drizzle server. The second test case executes the crashme tool against a running server. | ||
888 | 53 | |||
889 | 54 | Test cases are defined in python .cnf files and live in tests/sqlbench_tests. | ||
890 | 55 | |||
891 | 56 | Running tests | ||
892 | 57 | ========================= | ||
893 | 58 | |||
894 | 59 | NOTE: all_sqlbench_tests can take a significant amount of time to execute (45 minutes or so on a decently provisioned laptop) | ||
895 | 60 | |||
896 | 61 | There are several different ways to run tests using :doc:`dbqp` 's sql-bench mode. | ||
897 | 62 | |||
898 | 63 | It should be noted that unless :option:`--force` is used, the program will | ||
899 | 64 | stop execution upon encountering the first failing test. | ||
900 | 65 | :option:`--force` is recommended if you are running several tests - it will | ||
901 | 66 | allow you to view all successes and failures in one run. | ||
902 | 67 | |||
903 | 68 | At present, sql-bench output in a work in progress. It does report a simple pass/fail, but we are working on alternate ways of viewing / storing the results (and for other testing modes as well) | ||
904 | 69 | |||
905 | 70 | |||
906 | 71 | Running all tests within a suite | ||
907 | 72 | -------------------------------- | ||
908 | 73 | At present, there is only one test case per suite for sqlbench and crashme modes - that is all that is needed for these tools. | ||
909 | 74 | To execute the sql-bench test suite:: | ||
910 | 75 | |||
911 | 76 | ./dbqp --mode=sqlbench | ||
912 | 77 | |||
913 | 78 | To execute the crash-me test suite:: | ||
914 | 79 | |||
915 | 80 | ./dbqp --mode=crashme | ||
916 | 81 | |||
917 | 82 | Interpreting test results | ||
918 | 83 | ========================= | ||
919 | 84 | The output of the test runner is quite simple. Every test should pass. | ||
920 | 85 | In the event of a test failure, please take the time to file a bug here: | ||
921 | 86 | *https://bugs.launchpad.net/drizzle* | ||
922 | 87 | |||
923 | 88 | During a run, the program will provide the user with: | ||
924 | 89 | * test name (suite + name) | ||
925 | 90 | * test status (pass/fail/skipped) | ||
926 | 91 | * time spent executing each test | ||
927 | 92 | |||
928 | 93 | Example sqlbench output:: | ||
929 | 94 | |||
930 | 95 | 20110608-135645 =============================================================== | ||
931 | 96 | 20110608-135645 TEST NAME [ RESULT ] TIME (ms) | ||
932 | 97 | 20110608-135645 =============================================================== | ||
933 | 98 | 20110608-135645 main.all_sqlbench_tests [ pass ] 2732007 | ||
934 | 99 | 20110608-135645 Test finished. You can find the result in: | ||
935 | 100 | 20110608-135645 drizzle/tests/workdir/RUN-drizzle-Linux_2.6.38_9_generic_x86_64 | ||
936 | 101 | 20110608-135645 Benchmark DBD suite: 2.15 | ||
937 | 102 | 20110608-135645 Date of test: 2011-06-08 13:11:10 | ||
938 | 103 | 20110608-135645 Running tests on: Linux 2.6.38-9-generic x86_64 | ||
939 | 104 | 20110608-135645 Arguments: --connect-options=port=9306 --create-options=ENGINE=innodb | ||
940 | 105 | 20110608-135645 Comments: | ||
941 | 106 | 20110608-135645 Limits from: | ||
942 | 107 | 20110608-135645 Server version: Drizzle 2011.06.19.2325 | ||
943 | 108 | 20110608-135645 Optimization: None | ||
944 | 109 | 20110608-135645 Hardware: | ||
945 | 110 | 20110608-135645 | ||
946 | 111 | 20110608-135645 alter-table: Total time: 42 wallclock secs ( 0.06 usr 0.04 sys + 0.00 cusr 0.00 csys = 0.10 CPU) | ||
947 | 112 | 20110608-135645 ATIS: Total time: 22 wallclock secs ( 4.01 usr 0.26 sys + 0.00 cusr 0.00 csys = 4.27 CPU) | ||
948 | 113 | 20110608-135645 big-tables: Total time: 24 wallclock secs ( 4.16 usr 0.22 sys + 0.00 cusr 0.00 csys = 4.38 CPU) | ||
949 | 114 | 20110608-135645 connect: Total time: 31 wallclock secs ( 6.81 usr 4.50 sys + 0.00 cusr 0.00 csys = 11.31 CPU) | ||
950 | 115 | 20110608-135645 create: Total time: 59 wallclock secs ( 2.93 usr 1.65 sys + 0.00 cusr 0.00 csys = 4.58 CPU) | ||
951 | 116 | 20110608-135645 insert: Total time: 1962 wallclock secs (270.53 usr 66.35 sys + 0.00 cusr 0.00 csys = 336.88 CPU) | ||
952 | 117 | 20110608-135645 select: Total time: 560 wallclock secs (23.12 usr 4.62 sys + 0.00 cusr 0.00 csys = 27.74 CPU) | ||
953 | 118 | 20110608-135645 transactions: Total time: 21 wallclock secs ( 2.43 usr 1.98 sys + 0.00 cusr 0.00 csys = 4.41 CPU) | ||
954 | 119 | 20110608-135645 wisconsin: Total time: 10 wallclock secs ( 2.11 usr 0.52 sys + 0.00 cusr 0.00 csys = 2.63 CPU) | ||
955 | 120 | 20110608-135645 | ||
956 | 121 | 20110608-135645 All 9 test executed successfully | ||
957 | 122 | 20110608-135645 | ||
958 | 123 | 20110608-135645 Totals per operation: | ||
959 | 124 | 20110608-135645 Operation seconds usr sys cpu tests | ||
960 | 125 | 20110608-135645 alter_table_add 18.00 0.02 0.00 0.02 100 | ||
961 | 126 | 20110608-135645 alter_table_drop 17.00 0.02 0.01 0.03 91 | ||
962 | 127 | 20110608-135645 connect 2.00 1.02 0.51 1.53 2000 | ||
963 | 128 | <snip> | ||
964 | 129 | 20110608-135645 update_rollback 3.00 0.26 0.23 0.49 100 | ||
965 | 130 | 20110608-135645 update_with_key 73.00 6.70 5.23 11.93 300000 | ||
966 | 131 | 20110608-135645 update_with_key_prefix 34.00 4.45 2.30 6.75 100000 | ||
967 | 132 | 20110608-135645 wisc_benchmark 2.00 1.49 0.00 1.49 114 | ||
968 | 133 | 20110608-135645 TOTALS 2865.00 310.26 79.94 390.20 2974250 | ||
969 | 134 | 20110608-135645 | ||
970 | 135 | 20110608-135645 =============================================================== | ||
971 | 136 | 20110608-135645 INFO Test execution complete in 2735 seconds | ||
972 | 137 | 20110608-135645 INFO Summary report: | ||
973 | 138 | 20110608-135645 INFO Executed 1/1 test cases, 100.00 percent | ||
974 | 139 | 20110608-135645 INFO STATUS: PASS, 1/1 test cases, 100.00 percent executed | ||
975 | 140 | 20110608-135645 INFO Spent 2732 / 2735 seconds on: TEST(s) | ||
976 | 141 | 20110608-135645 INFO Test execution complete | ||
977 | 142 | 20110608-135645 INFO Stopping all running servers... | ||
978 | 143 | |||
979 | 144 | Example crashme output:: | ||
980 | 145 | |||
981 | 146 | 20110608-152759 =============================================================== | ||
982 | 147 | 20110608-152759 TEST NAME [ RESULT ] TIME (ms) | ||
983 | 148 | 20110608-152759 =============================================================== | ||
984 | 149 | 20110608-152759 main.crashme [ fail ] 155298 | ||
985 | 150 | 20110608-152759 func_extra_to_days=error # Function TO_DAYS | ||
986 | 151 | 20110608-152759 ### | ||
987 | 152 | 20110608-152759 ###<select to_days('1996-01-01') from crash_me_d | ||
988 | 153 | 20110608-152759 ###>2450084 | ||
989 | 154 | 20110608-152759 ###We expected '729024' but got '2450084' | ||
990 | 155 | 20110608-152759 func_odbc_timestampadd=error # Function TIMESTAMPADD | ||
991 | 156 | 20110608-152759 ### | ||
992 | 157 | 20110608-152759 ###<select timestampadd(SQL_TSI_SECOND,1,'1997-01-01 00:00:00') | ||
993 | 158 | 20110608-152759 ###>1997-01-01 00:00:01.000000 | ||
994 | 159 | 20110608-152759 ###We expected '1997-01-01 00:00:01' but got '1997-01-01 00:00:01.000000' | ||
995 | 160 | 20110608-152759 ### | ||
996 | 161 | 20110608-152759 ###<select {fn timestampadd(SQL_TSI_SECOND,1,{ts '1997-01-01 00:00:00'}) } | ||
997 | 162 | 20110608-152759 ###>1997-01-01 00:00:01.000000 | ||
998 | 163 | 20110608-152759 ###We expected '1997-01-01 00:00:01' but got '1997-01-01 00:00:01.000000' | ||
999 | 164 | 20110608-152759 | ||
1000 | 165 | 20110608-152759 ERROR Failed test. Use --force to execute beyond the first test failure | ||
1001 | 166 | 20110608-152759 =============================================================== | ||
1002 | 167 | 20110608-152759 INFO Test execution complete in 158 seconds | ||
1003 | 168 | 20110608-152759 INFO Summary report: | ||
1004 | 169 | 20110608-152759 INFO Executed 1/1 test cases, 100.00 percent | ||
1005 | 170 | 20110608-152759 INFO STATUS: FAIL, 1/1 test cases, 100.00 percent executed | ||
1006 | 171 | 20110608-152759 INFO FAIL tests: main.crashme | ||
1007 | 172 | 20110608-152759 INFO Spent 155 / 158 seconds on: TEST(s) | ||
1008 | 173 | 20110608-152759 INFO Test execution complete | ||
1009 | 174 | |||
1010 | 175 | 0 | ||
1011 | === removed file 'dbqp/docs/sysbench.rst' | |||
1012 | --- dbqp/docs/sysbench.rst 2012-02-04 01:22:23 +0000 | |||
1013 | +++ dbqp/docs/sysbench.rst 1970-01-01 00:00:00 +0000 | |||
1014 | @@ -1,123 +0,0 @@ | |||
1015 | 1 | ********************************** | ||
1016 | 2 | sysbench | ||
1017 | 3 | ********************************** | ||
1018 | 4 | |||
1019 | 5 | |||
1020 | 6 | |||
1021 | 7 | Description | ||
1022 | 8 | =========== | ||
1023 | 9 | dbqp's sysbench mode allows a user to run a specific iteration of a sysbench test (eg an oltp readonly run at concurrency = 16) | ||
1024 | 10 | |||
1025 | 11 | |||
1026 | 12 | Requirements | ||
1027 | 13 | ============ | ||
1028 | 14 | |||
1029 | 15 | The SYSBENCH command requires that the Drizzle library header be installed. Simply build Drizzle and do:: | ||
1030 | 16 | |||
1031 | 17 | $> sudo make install | ||
1032 | 18 | |||
1033 | 19 | This will install the right headers. | ||
1034 | 20 | |||
1035 | 21 | The SYSBENCH command also requires installation of the drizzle-sysbench program, which can be installed from source like so:: | ||
1036 | 22 | |||
1037 | 23 | $> bzr branch lp:~drizzle-developers/sysbench/trunk drizzle-sysbench | ||
1038 | 24 | $> cd drizzle-sysbench | ||
1039 | 25 | $> ./autogen.sh && ./configure && make && sudo make install | ||
1040 | 26 | |||
1041 | 27 | Make sure sysbench is then in your path | ||
1042 | 28 | |||
1043 | 29 | |||
1044 | 30 | sysbench / dbqp tests | ||
1045 | 31 | ===================== | ||
1046 | 32 | |||
1047 | 33 | A sysbench test defines a run for a particular concurrency. There are suites for readonly and readwrite. | ||
1048 | 34 | They are currently broken down this way as an experiment - we are open to other ways of organizing these tests:: | ||
1049 | 35 | |||
1050 | 36 | [test_info] | ||
1051 | 37 | comment = 16 threads | ||
1052 | 38 | |||
1053 | 39 | [test_command] | ||
1054 | 40 | command = sysbench --max-time=240 --max-requests=0 --test=oltp --db-ps-mode=disable --drizzle-table-engine=innodb --oltp-read-only=on --oltp-table-size=1000000 --drizzle-mysql=on --drizzle-user=root --drizzle-db=test --drizzle-port=$MASTER_MYPORT --drizzle-host=localhost --db-driver=drizzle --num-threads=16 | ||
1055 | 41 | |||
1056 | 42 | [test_servers] | ||
1057 | 43 | servers = [[innodb.buffer-pool-size=256M innodb.log-file-size=64M innodb.log-buffer-size=8M innodb.thread-concurrency=0 innodb.additional-mem-pool-size=16M table-open-cache=4096 table-definition-cache=4096 mysql-protocol.max-connections=2048]] | ||
1058 | 44 | |||
1059 | 45 | Running tests | ||
1060 | 46 | ========================= | ||
1061 | 47 | |||
1062 | 48 | There are several different ways to run tests using :doc:`dbqp` 's sysbench mode. | ||
1063 | 49 | |||
1064 | 50 | It should be noted that unless :option:`--force` is used, the program will | ||
1065 | 51 | stop execution upon encountering the first failing test. | ||
1066 | 52 | :option:`--force` is recommended if you are running several tests - it will | ||
1067 | 53 | allow you to view all successes and failures in one run. | ||
1068 | 54 | |||
1069 | 55 | Running individual tests | ||
1070 | 56 | ------------------------ | ||
1071 | 57 | If one only wants to run a few, specific tests, they may do so this way:: | ||
1072 | 58 | |||
1073 | 59 | ./dbqp --mode=sysbench [OPTIONS] test1 [test2 ... testN] | ||
1074 | 60 | |||
1075 | 61 | Running all tests within a suite | ||
1076 | 62 | -------------------------------- | ||
1077 | 63 | Many of the tests supplied with Drizzle are organized into suites. | ||
1078 | 64 | |||
1079 | 65 | The tests within drizzle/tests/randgen_tests/main are considered the 'main' suite. | ||
1080 | 66 | Other suites are also subdirectories of drizzle/tests/randgen_tests. | ||
1081 | 67 | |||
1082 | 68 | To run the tests in a specific suite:: | ||
1083 | 69 | |||
1084 | 70 | ./dbqp --mode=sysbench [OPTIONS] --suite=SUITENAME | ||
1085 | 71 | |||
1086 | 72 | Running specific tests within a suite | ||
1087 | 73 | -------------------------------------- | ||
1088 | 74 | To run a specific set of tests within a suite:: | ||
1089 | 75 | |||
1090 | 76 | ./dbqp --mode=sysbench [OPTIONS] --suite=SUITENAME TEST1 [TEST2..TESTN] | ||
1091 | 77 | |||
1092 | 78 | Calling tests using <suitename>.<testname> currently does not work. | ||
1093 | 79 | One must specify the test suite via the :option:`--suite` option. | ||
1094 | 80 | |||
1095 | 81 | |||
1096 | 82 | Running all available tests | ||
1097 | 83 | --------------------------- | ||
1098 | 84 | One would currently have to name all suites, but the majority of the working tests live in the main suite | ||
1099 | 85 | Other suites utilize more exotic server combinations and we are currently tweaking them to better integrate with the | ||
1100 | 86 | dbqp system. The slave-plugin suite does currently have a good config file for setting up simple replication setups for testing. | ||
1101 | 87 | To execute several suites' worth of tests:: | ||
1102 | 88 | |||
1103 | 89 | ./dbqp --mode=sysbench [OPTIONS] --suite=SUITE1, SUITE2, ...SUITEN | ||
1104 | 90 | |||
1105 | 91 | Interpreting test results | ||
1106 | 92 | ========================= | ||
1107 | 93 | The output of the test runner is quite simple. Every test should pass. | ||
1108 | 94 | In the event of a test failure, please take the time to file a bug here: | ||
1109 | 95 | *https://bugs.launchpad.net/drizzle* | ||
1110 | 96 | |||
1111 | 97 | During a run, the program will provide the user with: | ||
1112 | 98 | * test name (suite + name) | ||
1113 | 99 | * test status (pass/fail/skipped) | ||
1114 | 100 | * time spent executing each test | ||
1115 | 101 | |||
1116 | 102 | Example output:: | ||
1117 | 103 | |||
1118 | 104 | 20110601-191706 =============================================================== | ||
1119 | 105 | 20110601-191706 TEST NAME [ RESULT ] TIME (ms) | ||
1120 | 106 | 20110601-191706 =============================================================== | ||
1121 | 107 | 20110601-191706 readonly.concurrency_16 [ pass ] 240019 | ||
1122 | 108 | 20110601-191706 max_req_lat_ms: 21.44 | ||
1123 | 109 | 20110601-191706 rwreqps: 4208.2 | ||
1124 | 110 | 20110601-191706 min_req_lat_ms: 6.31 | ||
1125 | 111 | 20110601-191706 deadlocksps: 0.0 | ||
1126 | 112 | 20110601-191706 tps: 150.29 | ||
1127 | 113 | 20110601-191706 avg_req_lat_ms: 6.65 | ||
1128 | 114 | 20110601-191706 95p_req_lat_ms: 7.02 | ||
1129 | 115 | 20110601-191706 =============================================================== | ||
1130 | 116 | 20110601-191706 INFO Test execution complete in 275 seconds | ||
1131 | 117 | 20110601-191706 INFO Summary report: | ||
1132 | 118 | 20110601-191706 INFO Executed 1/1 test cases, 100.00 percent | ||
1133 | 119 | 20110601-191706 INFO STATUS: PASS, 1/1 test cases, 100.00 percent executed | ||
1134 | 120 | 20110601-191706 INFO Spent 240 / 275 seconds on: TEST(s) | ||
1135 | 121 | 20110601-191706 INFO Test execution complete | ||
1136 | 122 | 20110601-191706 INFO Stopping all running servers... | ||
1137 | 123 | |||
1138 | 124 | 0 | ||
1139 | === removed file 'dbqp/docs/test-run.rst' | |||
1140 | --- dbqp/docs/test-run.rst 2012-02-04 01:22:23 +0000 | |||
1141 | +++ dbqp/docs/test-run.rst 1970-01-01 00:00:00 +0000 | |||
1142 | @@ -1,494 +0,0 @@ | |||
1143 | 1 | .. _test-run-label: | ||
1144 | 2 | |||
1145 | 3 | ********************************** | ||
1146 | 4 | test-run.pl - Drizzle testing tool | ||
1147 | 5 | ********************************** | ||
1148 | 6 | |||
1149 | 7 | Synopsis | ||
1150 | 8 | ======== | ||
1151 | 9 | |||
1152 | 10 | **./test-run** [ *OPTIONS* ] [ TESTCASE ] | ||
1153 | 11 | |||
1154 | 12 | Description | ||
1155 | 13 | =========== | ||
1156 | 14 | |||
1157 | 15 | :program:`test-run.pl` (aka test-run, dtr, mtr) is used to execute tests | ||
1158 | 16 | from the Drizzle test suite. These tests are included with Drizzle | ||
1159 | 17 | distributions and provide a way for users to verify that the system will | ||
1160 | 18 | operate according to expectations. | ||
1161 | 19 | |||
1162 | 20 | The tests use a diff-based paradigm, meaning that the test runner executes | ||
1163 | 21 | a test and then compares the results received with pre-recorded expected | ||
1164 | 22 | results. In the event of a test failure, the program will provide output | ||
1165 | 23 | highlighting the differences found between expected and actual results; this | ||
1166 | 24 | can be useful for troubleshooting and in bug reports. | ||
1167 | 25 | |||
1168 | 26 | While most users are concerned with ensuring general functionality, the | ||
1169 | 27 | program also allows a user to quickly spin up a server for ad-hoc testing | ||
1170 | 28 | and to run the test-suite against an already running Drizzle server. | ||
1171 | 29 | |||
1172 | 30 | Running tests | ||
1173 | 31 | ========================= | ||
1174 | 32 | |||
1175 | 33 | There are several different ways to run tests using :program:`test-run.pl`. | ||
1176 | 34 | |||
1177 | 35 | It should be noted that unless :option:`--force` is used, the program will | ||
1178 | 36 | stop execution upon encountering the first failing test. | ||
1179 | 37 | :option:`--force` is recommended if you are running several tests - it will | ||
1180 | 38 | allow you to view all successes and failures in one run. | ||
1181 | 39 | |||
1182 | 40 | Running individual tests | ||
1183 | 41 | ------------------------ | ||
1184 | 42 | If one only wants to run a few, specific tests, they may do so this way:: | ||
1185 | 43 | |||
1186 | 44 | ./test-run [OPTIONS] test1 [test2 ... testN] | ||
1187 | 45 | |||
1188 | 46 | Running all tests within a suite | ||
1189 | 47 | -------------------------------- | ||
1190 | 48 | Many of the tests supplied with Drizzle are organized into suites. | ||
1191 | 49 | |||
1192 | 50 | The tests within drizzle/tests/t are considered the 'main' suite. | ||
1193 | 51 | Other suites are located in either drizzle/tests/suite or within the various | ||
1194 | 52 | directories in drizzle/plugin. Tests for a specific plugin should live in | ||
1195 | 53 | the plugin's directory - drizzle/plugin/example_plugin/tests | ||
1196 | 54 | |||
1197 | 55 | To run the tests in a specific suite:: | ||
1198 | 56 | |||
1199 | 57 | ./test-run [OPTIONS] --suite=SUITENAME | ||
1200 | 58 | |||
1201 | 59 | Running specific tests within a suite | ||
1202 | 60 | -------------------------------------- | ||
1203 | 61 | To run a specific set of tests within a suite:: | ||
1204 | 62 | |||
1205 | 63 | ./test-run [OPTIONS] --suite=SUITENAME TEST1 [TEST2..TESTN] | ||
1206 | 64 | |||
1207 | 65 | Calling tests using <suitename>.<testname> currently does not work. | ||
1208 | 66 | One must specify the test suite via the :option:`--suite` option. | ||
1209 | 67 | |||
1210 | 68 | |||
1211 | 69 | Running all available tests | ||
1212 | 70 | --------------------------- | ||
1213 | 71 | Currently, the quickest way to execute all tests in all suites is | ||
1214 | 72 | to use 'make test' from the drizzle root. | ||
1215 | 73 | |||
1216 | 74 | Otherwise, one should simply name all suites:: | ||
1217 | 75 | |||
1218 | 76 | ./test-run [OPTIONS] --suite=SUITE1, SUITE2, ...SUITEN | ||
1219 | 77 | |||
1220 | 78 | Interpreting test results | ||
1221 | 79 | ========================= | ||
1222 | 80 | The output of the test runner is quite simple. Every test should pass. | ||
1223 | 81 | In the event of a test failure, please take the time to file a bug here: | ||
1224 | 82 | *https://bugs.launchpad.net/drizzle* | ||
1225 | 83 | |||
1226 | 84 | During a run, the program will provide the user with: | ||
1227 | 85 | * test name (suite + name) | ||
1228 | 86 | * test status (pass/fail/skipped) | ||
1229 | 87 | * time spent executing each test | ||
1230 | 88 | |||
1231 | 89 | At the end of a run, the program will provide the user with a listing of: | ||
1232 | 90 | * how many tests were run | ||
1233 | 91 | * how many tests failed | ||
1234 | 92 | * percentage of passing tests | ||
1235 | 93 | * a listing of failing tests | ||
1236 | 94 | * total time spent executing the tests | ||
1237 | 95 | |||
1238 | 96 | Example output:: | ||
1239 | 97 | |||
1240 | 98 | <snip> | ||
1241 | 99 | main.snowman [ pass ] 9 | ||
1242 | 100 | main.statement_boundaries [ pass ] 17 | ||
1243 | 101 | main.status [ pass ] 12 | ||
1244 | 102 | main.strict [ pass ] 50 | ||
1245 | 103 | main.subselect [ pass ] 6778 | ||
1246 | 104 | main.subselect2 [ pass ] 51 | ||
1247 | 105 | main.subselect3 [ fail ] | ||
1248 | 106 | drizzletest: At line 621: query 'select a, (select max(b) from t1) into outfile | ||
1249 | 107 | <snip> | ||
1250 | 108 | -------------------------------------------------------------------------------- | ||
1251 | 109 | Stopping All Servers | ||
1252 | 110 | Failed 10/231 tests, 95.67% were successful. | ||
1253 | 111 | |||
1254 | 112 | The log files in var/log may give you some hint | ||
1255 | 113 | of what went wrong. | ||
1256 | 114 | If you want to report this error, go to: | ||
1257 | 115 | http://bugs.launchpad.net/drizzle | ||
1258 | 116 | The servers were restarted 16 times | ||
1259 | 117 | Spent 64.364 of 115 seconds executing testcases | ||
1260 | 118 | |||
1261 | 119 | drizzle-test-run in default mode: *** Failing the test(s): main.exp1 | ||
1262 | 120 | main.func_str main.loaddata main.null main.outfile main.subselect3 | ||
1263 | 121 | main.warnings jp.like_utf8 jp.select_utf8 jp.where_utf8 | ||
1264 | 122 | |||
1265 | 123 | Additional uses | ||
1266 | 124 | =============== | ||
1267 | 125 | Starting a server for manual testing | ||
1268 | 126 | ------------------------------------ | ||
1269 | 127 | |||
1270 | 128 | :program:`test-run.pl` allows a user to get a Drizzle server up and running | ||
1271 | 129 | quickly. This can be useful for fast ad-hoc testing. | ||
1272 | 130 | |||
1273 | 131 | To do so call:: | ||
1274 | 132 | |||
1275 | 133 | ./test-run --start-and-exit [*OPTIONS*] | ||
1276 | 134 | |||
1277 | 135 | This will start a Drizzle server that you can connect to and query | ||
1278 | 136 | |||
1279 | 137 | Starting a server against a pre-populated DATADIR | ||
1280 | 138 | -------------------------------------------------- | ||
1281 | 139 | |||
1282 | 140 | Using :option:`--start-dirty` prevents :program:`test-run.pl` from attempting | ||
1283 | 141 | to initialize (clean) the datadir. This can be useful if you want to use | ||
1284 | 142 | an already-populated datadir for testing. | ||
1285 | 143 | |||
1286 | 144 | Program architecture | ||
1287 | 145 | ==================== | ||
1288 | 146 | |||
1289 | 147 | :program:`test-run.pl` uses a simple diff-based mechanism for testing. | ||
1290 | 148 | It will execute the statements contained in a test and compare the results | ||
1291 | 149 | to pre-recorded expected results. In the event of a test failure, you | ||
1292 | 150 | will be presented with a diff:: | ||
1293 | 151 | |||
1294 | 152 | main.exp1 [ fail ] | ||
1295 | 153 | --- drizzle/tests/r/exp1.result 2010-11-02 02:10:25.107013998 +0300 | ||
1296 | 154 | +++ drizzle/tests/r/exp1.reject 2010-11-02 02:10:32.017013999 +0300 | ||
1297 | 155 | @@ -5,4 +5,5 @@ | ||
1298 | 156 | a | ||
1299 | 157 | 1 | ||
1300 | 158 | 2 | ||
1301 | 159 | +3 | ||
1302 | 160 | DROP TABLE t1; | ||
1303 | 161 | |||
1304 | 162 | A test case consists of a .test and a .result file. The .test file includes | ||
1305 | 163 | the various statements to be executed for a test. The .result file lists | ||
1306 | 164 | the expected results for a given test file. These files live in tests/t | ||
1307 | 165 | and tests/r, respectively. This structure is the same for all test suites. | ||
1308 | 166 | |||
1309 | 167 | test-run.pl options | ||
1310 | 168 | =================== | ||
1311 | 169 | |||
1312 | 170 | The :program:`test-run.pl` tool has several available options: | ||
1313 | 171 | |||
1314 | 172 | ./test-run [ OPTIONS ] [ TESTCASE ] | ||
1315 | 173 | |||
1316 | 174 | Options to control what engine/variation to run | ||
1317 | 175 | ----------------------------------------------- | ||
1318 | 176 | |||
1319 | 177 | .. program:: test-run | ||
1320 | 178 | |||
1321 | 179 | .. option:: --compress | ||
1322 | 180 | |||
1323 | 181 | Use the compressed protocol between client and server | ||
1324 | 182 | |||
1325 | 183 | .. option:: --bench | ||
1326 | 184 | |||
1327 | 185 | Run the benchmark suite | ||
1328 | 186 | |||
1329 | 187 | .. option:: --small-bench | ||
1330 | 188 | |||
1331 | 189 | Run the benchmarks with --small-tests --small-tables | ||
1332 | 190 | |||
1333 | 191 | Options to control directories to use | ||
1334 | 192 | ------------------------------------- | ||
1335 | 193 | |||
1336 | 194 | .. program:: test-run | ||
1337 | 195 | |||
1338 | 196 | .. option:: --benchdir=DIR | ||
1339 | 197 | |||
1340 | 198 | The directory where the benchmark suite is stored | ||
1341 | 199 | (default: ../../mysql-bench) | ||
1342 | 200 | |||
1343 | 201 | .. option:: --tmpdir=DIR | ||
1344 | 202 | |||
1345 | 203 | The directory where temporary files are stored | ||
1346 | 204 | (default: ./var/tmp). | ||
1347 | 205 | |||
1348 | 206 | .. option:: --vardir=DIR | ||
1349 | 207 | |||
1350 | 208 | The directory where files generated from the test run | ||
1351 | 209 | is stored (default: ./var). Specifying a ramdisk or | ||
1352 | 210 | tmpfs will speed up tests. | ||
1353 | 211 | |||
1354 | 212 | .. option:: --mem | ||
1355 | 213 | |||
1356 | 214 | Run testsuite in "memory" using tmpfs or ramdisk | ||
1357 | 215 | Attempts to find a suitable location | ||
1358 | 216 | using a builtin list of standard locations | ||
1359 | 217 | for tmpfs (/dev/shm) | ||
1360 | 218 | The option can also be set using environment | ||
1361 | 219 | variable :envvar:`DTR_MEM` =[DIR] | ||
1362 | 220 | |||
1363 | 221 | Options to control what test suites or cases to run | ||
1364 | 222 | --------------------------------------------------- | ||
1365 | 223 | |||
1366 | 224 | .. program:: test-run | ||
1367 | 225 | |||
1368 | 226 | .. option:: --force | ||
1369 | 227 | |||
1370 | 228 | Continue to run the suite after failure | ||
1371 | 229 | |||
1372 | 230 | .. option:: --do-test=PREFIX or REGEX | ||
1373 | 231 | |||
1374 | 232 | Run test cases which name are prefixed with PREFIX | ||
1375 | 233 | or fulfills REGEX | ||
1376 | 234 | |||
1377 | 235 | .. option:: --skip-test=PREFIX or REGEX | ||
1378 | 236 | |||
1379 | 237 | Skip test cases which name are prefixed with PREFIX | ||
1380 | 238 | or fulfills REGEX | ||
1381 | 239 | |||
1382 | 240 | .. option:: --start-from=PREFIX | ||
1383 | 241 | |||
1384 | 242 | Run test cases starting from test prefixed with PREFIX | ||
1385 | 243 | suite[s]=NAME1,..,NAMEN Collect tests in suites from the comma separated | ||
1386 | 244 | list of suite names. | ||
1387 | 245 | The default is: "main,jp" | ||
1388 | 246 | |||
1389 | 247 | .. option:: --skip-rpl | ||
1390 | 248 | |||
1391 | 249 | Skip the replication test cases. | ||
1392 | 250 | combination="ARG1 .. ARG2" Specify a set of "drizzled" arguments for one | ||
1393 | 251 | combination. | ||
1394 | 252 | |||
1395 | 253 | .. option:: --skip-combination | ||
1396 | 254 | |||
1397 | 255 | Skip any combination options and combinations files | ||
1398 | 256 | |||
1399 | 257 | .. option:: --repeat-test=n | ||
1400 | 258 | |||
1401 | 259 | How many times to repeat each test (default: 1) | ||
1402 | 260 | |||
1403 | 261 | Options that specify ports | ||
1404 | 262 | -------------------------- | ||
1405 | 263 | |||
1406 | 264 | .. program:: test-run | ||
1407 | 265 | |||
1408 | 266 | .. option:: --master_port=PORT | ||
1409 | 267 | |||
1410 | 268 | Specify the port number used by the first master | ||
1411 | 269 | |||
1412 | 270 | .. option:: --slave_port=PORT | ||
1413 | 271 | |||
1414 | 272 | Specify the port number used by the first slave | ||
1415 | 273 | |||
1416 | 274 | .. option:: --dtr-build-thread=# | ||
1417 | 275 | |||
1418 | 276 | Specify unique collection of ports. Can also be set by | ||
1419 | 277 | setting the environment variable :envvar:`DTR_BUILD_THREAD`. | ||
1420 | 278 | |||
1421 | 279 | Options for test case authoring | ||
1422 | 280 | ------------------------------- | ||
1423 | 281 | |||
1424 | 282 | .. program:: test-run | ||
1425 | 283 | |||
1426 | 284 | .. option:: --record TESTNAME | ||
1427 | 285 | |||
1428 | 286 | (Re)genereate the result file for TESTNAME | ||
1429 | 287 | |||
1430 | 288 | .. option:: --check-testcases | ||
1431 | 289 | |||
1432 | 290 | Check testcases for sideeffects | ||
1433 | 291 | |||
1434 | 292 | .. option:: --mark-progress | ||
1435 | 293 | |||
1436 | 294 | Log line number and elapsed time to <testname>.progress | ||
1437 | 295 | |||
1438 | 296 | Options that pass on options | ||
1439 | 297 | ---------------------------- | ||
1440 | 298 | |||
1441 | 299 | .. program:: test-run | ||
1442 | 300 | |||
1443 | 301 | .. option:: --drizzled=ARGS | ||
1444 | 302 | |||
1445 | 303 | Specify additional arguments to "drizzled" | ||
1446 | 304 | |||
1447 | 305 | Options to run test on running server | ||
1448 | 306 | ------------------------------------- | ||
1449 | 307 | |||
1450 | 308 | .. program:: test-run | ||
1451 | 309 | |||
1452 | 310 | .. option:: --extern | ||
1453 | 311 | |||
1454 | 312 | Use running server for tests | ||
1455 | 313 | |||
1456 | 314 | .. option:: --user=USER | ||
1457 | 315 | |||
1458 | 316 | User for connection to extern server | ||
1459 | 317 | |||
1460 | 318 | Options for debugging the product | ||
1461 | 319 | --------------------------------- | ||
1462 | 320 | |||
1463 | 321 | .. program:: test-run | ||
1464 | 322 | |||
1465 | 323 | .. option:: --client-ddd | ||
1466 | 324 | |||
1467 | 325 | Start drizzletest client in ddd | ||
1468 | 326 | |||
1469 | 327 | .. option:: --client-debugger=NAME | ||
1470 | 328 | |||
1471 | 329 | Start drizzletest in the selected debugger | ||
1472 | 330 | |||
1473 | 331 | .. option:: --client-gdb | ||
1474 | 332 | |||
1475 | 333 | Start drizzletest client in gdb | ||
1476 | 334 | |||
1477 | 335 | .. option:: --ddd | ||
1478 | 336 | |||
1479 | 337 | Start drizzled in ddd | ||
1480 | 338 | |||
1481 | 339 | .. option:: --debug | ||
1482 | 340 | |||
1483 | 341 | Dump trace output for all servers and client programs | ||
1484 | 342 | |||
1485 | 343 | .. option:: --debugger=NAME | ||
1486 | 344 | |||
1487 | 345 | Start drizzled in the selected debugger | ||
1488 | 346 | |||
1489 | 347 | .. option:: --gdb | ||
1490 | 348 | |||
1491 | 349 | Start the drizzled(s) in gdb | ||
1492 | 350 | |||
1493 | 351 | .. option:: --manual-debug | ||
1494 | 352 | |||
1495 | 353 | Let user manually start drizzled in debugger, before running test(s) | ||
1496 | 354 | |||
1497 | 355 | .. option:: --manual-gdb | ||
1498 | 356 | |||
1499 | 357 | Let user manually start drizzled in gdb, before running test(s) | ||
1500 | 358 | |||
1501 | 359 | .. option:: --manual-ddd | ||
1502 | 360 | |||
1503 | 361 | Let user manually start drizzled in ddd, before running test(s) | ||
1504 | 362 | |||
1505 | 363 | .. option:: --master-binary=PATH | ||
1506 | 364 | |||
1507 | 365 | Specify the master "drizzled" to use | ||
1508 | 366 | |||
1509 | 367 | .. option:: --slave-binary=PATH | ||
1510 | 368 | |||
1511 | 369 | Specify the slave "drizzled" to use | ||
1512 | 370 | |||
1513 | 371 | .. option:: --strace-client | ||
1514 | 372 | |||
1515 | 373 | Create strace output for drizzletest client | ||
1516 | 374 | |||
1517 | 375 | .. option:: --max-save-core | ||
1518 | 376 | |||
1519 | 377 | Limit the number of core files saved (to avoid filling up disks for | ||
1520 | 378 | heavily crashing server). Defaults to 5, set to 0 for no limit. | ||
1521 | 379 | |||
1522 | 380 | Options for coverage, profiling etc | ||
1523 | 381 | ----------------------------------- | ||
1524 | 382 | |||
1525 | 383 | .. todo:: | ||
1526 | 384 | |||
1527 | 385 | .. option:: --gcov | ||
1528 | 386 | |||
1529 | 387 | .. program:: test-run | ||
1530 | 388 | |||
1531 | 389 | .. option:: --gprof | ||
1532 | 390 | |||
1533 | 391 | See online documentation on how to use it. | ||
1534 | 392 | |||
1535 | 393 | .. option:: --valgrind | ||
1536 | 394 | |||
1537 | 395 | Run the *drizzletest* and *drizzled* executables using valgrind with | ||
1538 | 396 | default options | ||
1539 | 397 | |||
1540 | 398 | .. option:: --valgrind-all | ||
1541 | 399 | |||
1542 | 400 | Synonym for :option:`--valgrind` | ||
1543 | 401 | |||
1544 | 402 | .. option:: --valgrind-drizzleslap | ||
1545 | 403 | |||
1546 | 404 | Run "drizzleslap" with valgrind. | ||
1547 | 405 | |||
1548 | 406 | .. option:: --valgrind-drizzletest | ||
1549 | 407 | |||
1550 | 408 | Run the *drizzletest* and *drizzle_client_test* executable with valgrind | ||
1551 | 409 | |||
1552 | 410 | .. option:: --valgrind-drizzled | ||
1553 | 411 | |||
1554 | 412 | Run the "drizzled" executable with valgrind | ||
1555 | 413 | |||
1556 | 414 | .. option:: --valgrind-options=ARGS | ||
1557 | 415 | |||
1558 | 416 | Deprecated, use :option:`--valgrind-option` | ||
1559 | 417 | |||
1560 | 418 | .. option:: --valgrind-option=ARGS | ||
1561 | 419 | |||
1562 | 420 | Option to give valgrind, replaces default option(s), | ||
1563 | 421 | can be specified more then once | ||
1564 | 422 | |||
1565 | 423 | .. option:: --valgrind-path=[EXE] | ||
1566 | 424 | |||
1567 | 425 | Path to the valgrind executable | ||
1568 | 426 | |||
1569 | 427 | .. option:: --callgrind | ||
1570 | 428 | |||
1571 | 429 | Instruct valgrind to use callgrind | ||
1572 | 430 | |||
1573 | 431 | .. option:: --massif | ||
1574 | 432 | |||
1575 | 433 | Instruct valgrind to use massif | ||
1576 | 434 | |||
1577 | 435 | Misc options | ||
1578 | 436 | ------------ | ||
1579 | 437 | |||
1580 | 438 | .. program:: test-run | ||
1581 | 439 | |||
1582 | 440 | .. option:: --comment=STR | ||
1583 | 441 | |||
1584 | 442 | Write STR to the output | ||
1585 | 443 | |||
1586 | 444 | .. option:: --notimer | ||
1587 | 445 | |||
1588 | 446 | Don't show test case execution time | ||
1589 | 447 | |||
1590 | 448 | .. option:: --script-debug | ||
1591 | 449 | |||
1592 | 450 | Debug this script itself | ||
1593 | 451 | |||
1594 | 452 | .. option:: --verbose | ||
1595 | 453 | |||
1596 | 454 | More verbose output | ||
1597 | 455 | |||
1598 | 456 | .. option:: --start-and-exit | ||
1599 | 457 | |||
1600 | 458 | Only initialize and start the servers, using the | ||
1601 | 459 | startup settings for the specified test case (if any) | ||
1602 | 460 | |||
1603 | 461 | .. option:: --start-dirty | ||
1604 | 462 | |||
1605 | 463 | Only start the servers (without initialization) for | ||
1606 | 464 | the specified test case (if any) | ||
1607 | 465 | |||
1608 | 466 | .. option:: --fast | ||
1609 | 467 | |||
1610 | 468 | Don't try to clean up from earlier runs | ||
1611 | 469 | |||
1612 | 470 | .. option:: --reorder | ||
1613 | 471 | |||
1614 | 472 | Reorder tests to get fewer server restarts | ||
1615 | 473 | |||
1616 | 474 | .. option:: --help | ||
1617 | 475 | |||
1618 | 476 | Get this help text | ||
1619 | 477 | |||
1620 | 478 | .. option:: --testcase-timeout=MINUTES | ||
1621 | 479 | |||
1622 | 480 | Max test case run time (default 15) | ||
1623 | 481 | |||
1624 | 482 | .. option:: --suite-timeout=MINUTES | ||
1625 | 483 | |||
1626 | 484 | Max test suite run time (default 180) | ||
1627 | 485 | |||
1628 | 486 | .. option:: --warnings | log-warnings | ||
1629 | 487 | |||
1630 | 488 | Pass --log-warnings to drizzled | ||
1631 | 489 | |||
1632 | 490 | .. option:: --sleep=SECONDS | ||
1633 | 491 | |||
1634 | 492 | Passed to drizzletest, will be used as fixed sleep time | ||
1635 | 493 | |||
1636 | 494 | |||
1637 | 495 | 0 | ||
1638 | === removed file 'dbqp/docs/writing_tests.rst' | |||
1639 | --- dbqp/docs/writing_tests.rst 2012-02-04 01:22:23 +0000 | |||
1640 | +++ dbqp/docs/writing_tests.rst 1970-01-01 00:00:00 +0000 | |||
1641 | @@ -1,30 +0,0 @@ | |||
1642 | 1 | ********************************** | ||
1643 | 2 | Writing drizzletest test cases | ||
1644 | 3 | ********************************** | ||
1645 | 4 | |||
1646 | 5 | Synopsis | ||
1647 | 6 | ======== | ||
1648 | 7 | |||
1649 | 8 | Here, we discuss various topics related to writing test cases, with a focus on features | ||
1650 | 9 | that allow for more complex testing scenarios. Additional documentation for other testing | ||
1651 | 10 | tools will come later. | ||
1652 | 11 | |||
1653 | 12 | Using a pre-populated datadir | ||
1654 | 13 | ============================= | ||
1655 | 14 | The experimental test runner, dbqp allows for starting a server with a pre-populated datadir | ||
1656 | 15 | for a test case. This is accomplished via the use of a .cnf file (versus a master.opt file) | ||
1657 | 16 | Over time, this will be the direction for all drizzletest cases. | ||
1658 | 17 | |||
1659 | 18 | The .cnf file shown below tells the test-runner to use the directory drizzle/tests/std_data/backwards_compat_data | ||
1660 | 19 | as the datadir for the first server. If a test uses multiple servers, the .cnf file can have additional sections ([s1]...[sN]):: | ||
1661 | 20 | |||
1662 | 21 | [test_servers] | ||
1663 | 22 | servers = [[]] | ||
1664 | 23 | |||
1665 | 24 | [s0] | ||
1666 | 25 | load-datadir=backwards_compat_data | ||
1667 | 26 | |||
1668 | 27 | |||
1669 | 28 | All datadirs are expected to be in tests/std_data. If there is need for the ability to use datadirs outside of this location, | ||
1670 | 29 | it can be explored. | ||
1671 | 30 | |||
1672 | 31 | 0 | ||
1673 | === removed directory 'dbqp/lib' | |||
1674 | === removed file 'dbqp/lib/__init__.py' | |||
1675 | === removed directory 'dbqp/lib/modes' | |||
1676 | === removed file 'dbqp/lib/modes/__init__.py' | |||
1677 | === removed directory 'dbqp/lib/modes/dtr' | |||
1678 | === removed file 'dbqp/lib/modes/dtr/__init__.py' | |||
1679 | === removed file 'dbqp/lib/modes/dtr/dtr_test_execution.py' | |||
1680 | --- dbqp/lib/modes/dtr/dtr_test_execution.py 2012-02-04 01:22:23 +0000 | |||
1681 | +++ dbqp/lib/modes/dtr/dtr_test_execution.py 1970-01-01 00:00:00 +0000 | |||
1682 | @@ -1,139 +0,0 @@ | |||
1683 | 1 | #! /usr/bin/env python | ||
1684 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
1685 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
1686 | 4 | # | ||
1687 | 5 | # Copyright (C) 2010 Patrick Crews | ||
1688 | 6 | # | ||
1689 | 7 | # | ||
1690 | 8 | # This program is free software; you can redistribute it and/or modify | ||
1691 | 9 | # it under the terms of the GNU General Public License as published by | ||
1692 | 10 | # the Free Software Foundation; either version 2 of the License, or | ||
1693 | 11 | # (at your option) any later version. | ||
1694 | 12 | # | ||
1695 | 13 | # This program is distributed in the hope that it will be useful, | ||
1696 | 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
1697 | 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
1698 | 16 | # GNU General Public License for more details. | ||
1699 | 17 | # | ||
1700 | 18 | # You should have received a copy of the GNU General Public License | ||
1701 | 19 | # along with this program; if not, write to the Free Software | ||
1702 | 20 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
1703 | 21 | |||
1704 | 22 | """ dtr_test_execution: | ||
1705 | 23 | code related to the execution of dtr test cases | ||
1706 | 24 | |||
1707 | 25 | We are provided access to a testManager with | ||
1708 | 26 | dtr-specific testCases. We contact teh executionManager | ||
1709 | 27 | to produce the system and server configurations we need | ||
1710 | 28 | to execute a test. | ||
1711 | 29 | |||
1712 | 30 | """ | ||
1713 | 31 | |||
1714 | 32 | # imports | ||
1715 | 33 | import os | ||
1716 | 34 | import sys | ||
1717 | 35 | import subprocess | ||
1718 | 36 | import commands | ||
1719 | 37 | |||
1720 | 38 | import lib.test_mgmt.test_execution as test_execution | ||
1721 | 39 | |||
1722 | 40 | class testExecutor(test_execution.testExecutor): | ||
1723 | 41 | """ dtr-specific testExecutor | ||
1724 | 42 | We currently execute by sending test-case | ||
1725 | 43 | data to client/drizzletest...for now | ||
1726 | 44 | |||
1727 | 45 | """ | ||
1728 | 46 | |||
1729 | 47 | def execute_testCase (self): | ||
1730 | 48 | """ Execute a dtr testCase via calls to drizzletest (boo) | ||
1731 | 49 | Eventually, we will replace drizzletest with pythonic | ||
1732 | 50 | goodness, but we have these classes stored here for the moment | ||
1733 | 51 | |||
1734 | 52 | """ | ||
1735 | 53 | test_execution.testExecutor.execute_testCase(self) | ||
1736 | 54 | self.status = 0 | ||
1737 | 55 | |||
1738 | 56 | # generate command line | ||
1739 | 57 | drizzletest_cmd = self.generate_drizzletest_call() | ||
1740 | 58 | |||
1741 | 59 | # call drizzletest | ||
1742 | 60 | self.execute_drizzletest(drizzletest_cmd) | ||
1743 | 61 | |||
1744 | 62 | # analyze results | ||
1745 | 63 | self.current_test_status = self.process_drizzletest_output() | ||
1746 | 64 | self.set_server_status(self.current_test_status) | ||
1747 | 65 | |||
1748 | 66 | |||
1749 | 67 | def generate_drizzletest_call(self): | ||
1750 | 68 | """ Produce the command line we use to call drizzletest | ||
1751 | 69 | We have a healthy number of values, so we put this in a | ||
1752 | 70 | nice function | ||
1753 | 71 | |||
1754 | 72 | """ | ||
1755 | 73 | |||
1756 | 74 | drizzletest_arguments = [ '--no-defaults' | ||
1757 | 75 | , '--silent' | ||
1758 | 76 | , '--tmpdir=%s' %(self.master_server.tmpdir) | ||
1759 | 77 | , '--logdir=%s' %(self.master_server.logdir) | ||
1760 | 78 | , '--port=%d' %(self.master_server.master_port) | ||
1761 | 79 | , '--database=test' | ||
1762 | 80 | , '--user=root' | ||
1763 | 81 | , '--password=' | ||
1764 | 82 | #, '--testdir=%s' %(self.test_manager.testdir) | ||
1765 | 83 | , '--test-file=%s' %(self.current_testcase.testpath) | ||
1766 | 84 | , '--tail-lines=20' | ||
1767 | 85 | , '--timer-file=%s' %(self.master_server.timer_file) | ||
1768 | 86 | , '--result-file=%s' %(self.current_testcase.resultpath) | ||
1769 | 87 | ] | ||
1770 | 88 | if self.record_flag: | ||
1771 | 89 | # We want to record a new result | ||
1772 | 90 | drizzletest_arguments.append('--record') | ||
1773 | 91 | drizzletest_cmd = "%s %s %s" %( self.cmd_prefix | ||
1774 | 92 | , self.master_server.code_tree.drizzletest | ||
1775 | 93 | , " ".join(drizzletest_arguments)) | ||
1776 | 94 | return drizzletest_cmd | ||
1777 | 95 | |||
1778 | 96 | def execute_drizzletest(self, drizzletest_cmd): | ||
1779 | 97 | """ Execute the commandline and return the result. | ||
1780 | 98 | We use subprocess as we can pass os.environ dicts and whatnot | ||
1781 | 99 | |||
1782 | 100 | """ | ||
1783 | 101 | testcase_name = self.current_testcase.fullname | ||
1784 | 102 | self.time_manager.start(testcase_name,'test') | ||
1785 | 103 | #retcode, output = self.system_manager.execute_cmd( drizzletest_cmd | ||
1786 | 104 | # , must_pass = 0 ) | ||
1787 | 105 | drizzletest_outfile = os.path.join(self.logdir,'drizzletest.out') | ||
1788 | 106 | drizzletest_output = open(drizzletest_outfile,'w') | ||
1789 | 107 | drizzletest_subproc = subprocess.Popen( drizzletest_cmd | ||
1790 | 108 | , shell=True | ||
1791 | 109 | , cwd=self.system_manager.testdir | ||
1792 | 110 | , env=self.working_environment | ||
1793 | 111 | , stdout = drizzletest_output | ||
1794 | 112 | , stderr = subprocess.STDOUT | ||
1795 | 113 | ) | ||
1796 | 114 | drizzletest_subproc.wait() | ||
1797 | 115 | retcode = drizzletest_subproc.returncode | ||
1798 | 116 | execution_time = int(self.time_manager.stop(testcase_name)*1000) # millisec | ||
1799 | 117 | |||
1800 | 118 | drizzletest_output.close() | ||
1801 | 119 | drizzletest_file = open(drizzletest_outfile,'r') | ||
1802 | 120 | output = ''.join(drizzletest_file.readlines()) | ||
1803 | 121 | drizzletest_file.close() | ||
1804 | 122 | |||
1805 | 123 | self.logging.debug("drizzletest_retcode: %d" %(retcode)) | ||
1806 | 124 | self.current_test_retcode = retcode | ||
1807 | 125 | self.current_test_output = output | ||
1808 | 126 | self.current_test_exec_time = execution_time | ||
1809 | 127 | |||
1810 | 128 | def process_drizzletest_output(self): | ||
1811 | 129 | """ Drizzletest has run, we now check out what we have """ | ||
1812 | 130 | retcode = self.current_test_retcode | ||
1813 | 131 | if retcode == 0: | ||
1814 | 132 | return 'pass' | ||
1815 | 133 | elif retcode == 62 or retcode == 15872: | ||
1816 | 134 | return 'skipped' | ||
1817 | 135 | elif retcode == 63 or retcode == 1: | ||
1818 | 136 | return 'fail' | ||
1819 | 137 | else: | ||
1820 | 138 | return 'fail' | ||
1821 | 139 | |||
1822 | 140 | 0 | ||
1823 | === removed file 'dbqp/lib/modes/dtr/dtr_test_management.py' | |||
1824 | --- dbqp/lib/modes/dtr/dtr_test_management.py 2012-02-04 01:22:23 +0000 | |||
1825 | +++ dbqp/lib/modes/dtr/dtr_test_management.py 1970-01-01 00:00:00 +0000 | |||
1826 | @@ -1,495 +0,0 @@ | |||
1827 | 1 | #! /usr/bin/env python | ||
1828 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
1829 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
1830 | 4 | # | ||
1831 | 5 | # Copyright (C) 2010 Patrick Crews | ||
1832 | 6 | # | ||
1833 | 7 | ## This program is free software; you can redistribute it and/or modify | ||
1834 | 8 | # it under the terms of the GNU General Public License as published by | ||
1835 | 9 | # the Free Software Foundation; either version 2 of the License, or | ||
1836 | 10 | # (at your option) any later version. | ||
1837 | 11 | # | ||
1838 | 12 | # This program is distributed in the hope that it will be useful, | ||
1839 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
1840 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
1841 | 15 | # GNU General Public License for more details. | ||
1842 | 16 | # | ||
1843 | 17 | # You should have received a copy of the GNU General Public License | ||
1844 | 18 | # along with this program; if not, write to the Free Software | ||
1845 | 19 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
1846 | 20 | |||
1847 | 21 | """ dtr_test_management: | ||
1848 | 22 | code related to the gathering / analysis / management of | ||
1849 | 23 | the test cases | ||
1850 | 24 | ie - collecting the list of tests in each suite, then | ||
1851 | 25 | gathering additional, relevant information for the test-runner's dtr | ||
1852 | 26 | mode. (traditional diff-based testing) | ||
1853 | 27 | |||
1854 | 28 | """ | ||
1855 | 29 | |||
1856 | 30 | # imports | ||
1857 | 31 | import os | ||
1858 | 32 | import re | ||
1859 | 33 | import sys | ||
1860 | 34 | from ConfigParser import RawConfigParser | ||
1861 | 35 | |||
1862 | 36 | import lib.test_mgmt.test_management as test_management | ||
1863 | 37 | |||
1864 | 38 | |||
1865 | 39 | |||
1866 | 40 | class testCase: | ||
1867 | 41 | """Holds info on a per .test file basis | ||
1868 | 42 | Attributes contain information necessary to execute / validate | ||
1869 | 43 | the test file when it is executed. | ||
1870 | 44 | |||
1871 | 45 | """ | ||
1872 | 46 | def __init__(self, system_manager, test_case=None, test_name=None, suite_name=None | ||
1873 | 47 | , suite_path=None, test_server_options=[], test_path=None, result_path=None | ||
1874 | 48 | , comment=None, master_sh=None, cnf_path=None | ||
1875 | 49 | , disable=0, innodb_test=1 | ||
1876 | 50 | , need_debug=0, debug=0): | ||
1877 | 51 | self.system_manager = system_manager | ||
1878 | 52 | self.logging = self.system_manager.logging | ||
1879 | 53 | self.skip_keys = ['system_manager'] | ||
1880 | 54 | self.testcase = test_case | ||
1881 | 55 | self.testname = test_name | ||
1882 | 56 | self.suitename = suite_name | ||
1883 | 57 | self.suitepath = suite_path | ||
1884 | 58 | self.fullname = "%s.%s" %(suite_name, test_name) | ||
1885 | 59 | self.testpath = test_path | ||
1886 | 60 | self.resultpath = result_path | ||
1887 | 61 | |||
1888 | 62 | self.skip_flag = 0 | ||
1889 | 63 | self.timezone = "GMT-3" | ||
1890 | 64 | self.component_id = "drizzled" | ||
1891 | 65 | self.slave_count = 0 | ||
1892 | 66 | self.master_count = 1 | ||
1893 | 67 | self.server_options = test_server_options | ||
1894 | 68 | # We will populate this in a better fashion later on | ||
1895 | 69 | # as we allow .cnf files, we need to do a bit of | ||
1896 | 70 | # messing about to make this work right | ||
1897 | 71 | if self.server_options == [] or type(self.server_options[0]) is not list: | ||
1898 | 72 | self.server_requirements=[self.server_options] | ||
1899 | 73 | else: | ||
1900 | 74 | self.server_requirements = self.server_options | ||
1901 | 75 | self.server_options= self.server_options[0][0] | ||
1902 | 76 | self.comment = comment | ||
1903 | 77 | self.master_sh = master_sh | ||
1904 | 78 | self.cnf_path = cnf_path | ||
1905 | 79 | self.disable = disable | ||
1906 | 80 | self.innodb_test = innodb_test | ||
1907 | 81 | self.need_debug = need_debug | ||
1908 | 82 | |||
1909 | 83 | self.system_manager.logging.debug_class(self) | ||
1910 | 84 | |||
1911 | 85 | def should_run(self): | ||
1912 | 86 | if self.skip_flag or self.disable: | ||
1913 | 87 | return 0 | ||
1914 | 88 | else: | ||
1915 | 89 | return 1 | ||
1916 | 90 | |||
1917 | 91 | |||
1918 | 92 | |||
1919 | 93 | |||
1920 | 94 | |||
1921 | 95 | class testManager(test_management.testManager): | ||
1922 | 96 | """Deals with scanning test directories, gathering test cases, and | ||
1923 | 97 | collecting per-test information (opt files, etc) for use by the | ||
1924 | 98 | test-runner | ||
1925 | 99 | |||
1926 | 100 | """ | ||
1927 | 101 | |||
1928 | 102 | def process_suite(self,suite_dir): | ||
1929 | 103 | """Process a test suite. | ||
1930 | 104 | This includes searching for tests in test_list and only | ||
1931 | 105 | working with the named tests (all tests in suite is the default) | ||
1932 | 106 | Further processing includes reading the disabled.def file | ||
1933 | 107 | to know which tests to skip, processing the suite.opt file, | ||
1934 | 108 | and processing the individual test cases for data relevant | ||
1935 | 109 | to the rest of the test-runner | ||
1936 | 110 | |||
1937 | 111 | """ | ||
1938 | 112 | |||
1939 | 113 | self.system_manager.logging.verbose("Processing suite: %s" %(suite_dir)) | ||
1940 | 114 | |||
1941 | 115 | # Generate our test and result files: | ||
1942 | 116 | testdir = os.path.join(suite_dir, 't') | ||
1943 | 117 | resultdir = os.path.join(suite_dir, 'r') | ||
1944 | 118 | |||
1945 | 119 | # Do basic checks to make sure this is worth further work | ||
1946 | 120 | self.check_suite(suite_dir, testdir, resultdir) | ||
1947 | 121 | |||
1948 | 122 | # Get suite-level options | ||
1949 | 123 | suite_options = [] | ||
1950 | 124 | cnf_path, suite_options = self.process_suite_options(suite_dir) | ||
1951 | 125 | |||
1952 | 126 | # Get the 'name' of the suite. This can require some processing | ||
1953 | 127 | # But the name is useful for reporting and whatnot | ||
1954 | 128 | suite_name = self.get_suite_name(suite_dir) | ||
1955 | 129 | |||
1956 | 130 | # Get the contents of the testdir and filter it accordingly | ||
1957 | 131 | # This applies do-test / skip-test filters and any specific | ||
1958 | 132 | # test case names | ||
1959 | 133 | testlist = self.testlist_filter(os.listdir(testdir)) | ||
1960 | 134 | # sort our list if no individual tests were specified | ||
1961 | 135 | if not self.desired_tests: | ||
1962 | 136 | testlist.sort() | ||
1963 | 137 | |||
1964 | 138 | # gather deeper information on tests we are interested in | ||
1965 | 139 | if testlist: | ||
1966 | 140 | # We have tests we want to process, we gather additional information | ||
1967 | 141 | # that is useful at the suite level. This includes disabled tests | ||
1968 | 142 | # Gather disabled test list. | ||
1969 | 143 | # This is used in process_test_file() | ||
1970 | 144 | disabled_tests = {} | ||
1971 | 145 | disabled_tests = self.process_disabled_test_file(testdir) | ||
1972 | 146 | for test_case in testlist: | ||
1973 | 147 | self.add_test(self.process_test_file(suite_dir, | ||
1974 | 148 | suite_name, cnf_path, suite_options | ||
1975 | 149 | , disabled_tests, testdir | ||
1976 | 150 | , resultdir, test_case)) | ||
1977 | 151 | |||
1978 | 152 | def process_test_file(self, suite_dir, suite_name, suite_cnf_path, suite_options | ||
1979 | 153 | , disabled_tests, testdir | ||
1980 | 154 | , resultdir, test_case): | ||
1981 | 155 | """ We generate / find / store all the relevant information per-test. | ||
1982 | 156 | This information will be used when actually executing the test | ||
1983 | 157 | We store the data in a testCase object | ||
1984 | 158 | |||
1985 | 159 | """ | ||
1986 | 160 | |||
1987 | 161 | test_server_options = suite_options | ||
1988 | 162 | test_name = test_case.replace('.test','') | ||
1989 | 163 | self.system_manager.logging.verbose("Processing test: %s.%s" %(suite_name,test_name)) | ||
1990 | 164 | |||
1991 | 165 | |||
1992 | 166 | # Fix this , create a testCase with gather_test_data() passed | ||
1993 | 167 | # as the argument | ||
1994 | 168 | # Ensure we pass everything we need and use it all | ||
1995 | 169 | ( test_path | ||
1996 | 170 | , result_file_name | ||
1997 | 171 | , result_path | ||
1998 | 172 | , comment | ||
1999 | 173 | , master_sh | ||
2000 | 174 | , cnf_path | ||
2001 | 175 | , test_server_options | ||
2002 | 176 | , disable | ||
2003 | 177 | , innodb_test | ||
2004 | 178 | , need_debug) = self.gather_test_data(test_case, test_name, | ||
2005 | 179 | suite_name, test_server_options,testdir, | ||
2006 | 180 | resultdir, disabled_tests) | ||
2007 | 181 | if suite_cnf_path and not cnf_path: | ||
2008 | 182 | cnf_path=suite_cnf_path | ||
2009 | 183 | test_case = testCase(self.system_manager, test_case, test_name, suite_name, | ||
2010 | 184 | suite_dir, test_server_options,test_path, result_path, | ||
2011 | 185 | master_sh=master_sh, cnf_path=cnf_path, debug=self.debug) | ||
2012 | 186 | return test_case | ||
2013 | 187 | |||
2014 | 188 | |||
2015 | 189 | ######################################################################## | ||
2016 | 190 | # utility functions | ||
2017 | 191 | # | ||
2018 | 192 | # Stuff that helps us out and simplifies our main functions | ||
2019 | 193 | # But isn't that important unless you need to dig deep | ||
2020 | 194 | ######################################################################## | ||
2021 | 195 | |||
2022 | 196 | def gather_test_data(self, test_case, test_name, suite_name, | ||
2023 | 197 | test_server_options, testdir, resultdir, disabled_tests): | ||
2024 | 198 | """ We gather all of the data needed to produce a testCase for | ||
2025 | 199 | a given test | ||
2026 | 200 | |||
2027 | 201 | """ | ||
2028 | 202 | |||
2029 | 203 | test_path = os.path.join(testdir,test_case) | ||
2030 | 204 | result_file_name = test_name+'.result' | ||
2031 | 205 | result_path = self.find_result_path(resultdir, result_file_name) | ||
2032 | 206 | comment = None | ||
2033 | 207 | master_sh_path = test_path.replace('.test','-master.sh') | ||
2034 | 208 | if os.path.exists(master_sh_path): | ||
2035 | 209 | master_sh = master_sh_path | ||
2036 | 210 | else: | ||
2037 | 211 | master_sh = None | ||
2038 | 212 | master_opt_path = test_path.replace('.test', '-master.opt') | ||
2039 | 213 | config_file_path = test_path.replace('.test', '.cnf') | ||
2040 | 214 | # NOTE: this currently makes suite level server options additive | ||
2041 | 215 | # to file-level .opt files...not sure if this is the best. | ||
2042 | 216 | test_server_options = test_server_options + self.process_opt_file( | ||
2043 | 217 | master_opt_path) | ||
2044 | 218 | # deal with .cnf files (which supercede master.opt stuff) | ||
2045 | 219 | cnf_options = [] | ||
2046 | 220 | cnf_flag, returned_options = self.process_cnf_file(config_file_path) | ||
2047 | 221 | cnf_options += returned_options | ||
2048 | 222 | if cnf_flag: # we found a proper file and need to override | ||
2049 | 223 | found_options = cnf_options | ||
2050 | 224 | else: | ||
2051 | 225 | config_file_path = None | ||
2052 | 226 | (disable, comment) = self.check_if_disabled(disabled_tests, test_name) | ||
2053 | 227 | innodb_test = 0 | ||
2054 | 228 | need_debug = 0 | ||
2055 | 229 | return (test_path, result_file_name, result_path, comment, master_sh, | ||
2056 | 230 | config_file_path, test_server_options, disable, innodb_test, need_debug) | ||
2057 | 231 | |||
2058 | 232 | def check_suite(self, suite_dir, testdir, resultdir): | ||
2059 | 233 | """Handle basic checks of the suite: | ||
2060 | 234 | does the suite exist? | ||
2061 | 235 | is there a /t dir? | ||
2062 | 236 | is there a /r dir? | ||
2063 | 237 | Error and die if not | ||
2064 | 238 | |||
2065 | 239 | """ | ||
2066 | 240 | # We expect suite to be a path name, no fuzzy searching | ||
2067 | 241 | if not os.path.exists(suite_dir): | ||
2068 | 242 | self.system_manager.logging.error("Suite: %s does not exist" %(suite_dir)) | ||
2069 | 243 | sys.exit(1) | ||
2070 | 244 | |||
2071 | 245 | # Ensure our test and result directories are present | ||
2072 | 246 | if not os.path.exists(testdir): | ||
2073 | 247 | self.system_manager.logging.error("Suite: %s does not have a 't' directory (expected location for test files)" %(suite_dir)) | ||
2074 | 248 | sys.exit(1) | ||
2075 | 249 | if not os.path.exists(resultdir): | ||
2076 | 250 | self.system_manager.logging.error("Suite: %s does not have an 'r' directory (expected location for result files)" %(suite_dir)) | ||
2077 | 251 | sys.exit(1) | ||
2078 | 252 | |||
2079 | 253 | def get_suite_name(self, suite_dir): | ||
2080 | 254 | """ Get the 'name' of the suite | ||
2081 | 255 | This can either be the path basename or one directory up, as | ||
2082 | 256 | in the case of files in the drizzle/plugins directory | ||
2083 | 257 | |||
2084 | 258 | """ | ||
2085 | 259 | |||
2086 | 260 | # We trim any trailing path delimiters as python returns | ||
2087 | 261 | # '' for basedir if the path ends that way | ||
2088 | 262 | # BEGIN horrible hack to accomodate bad location of main suite : / | ||
2089 | 263 | if suite_dir == self.testdir: | ||
2090 | 264 | return 'main' | ||
2091 | 265 | # END horrible hack : / | ||
2092 | 266 | if suite_dir.endswith('/'): | ||
2093 | 267 | suite_dir=suite_dir[:-1] | ||
2094 | 268 | suite_dir_root,suite_dir_basename = os.path.split(suite_dir) | ||
2095 | 269 | if suite_dir_basename == 'tests' or suite_dir_basename == 'drizzle-tests': | ||
2096 | 270 | suite_name = os.path.basename(suite_dir_root) | ||
2097 | 271 | else: | ||
2098 | 272 | suite_name = suite_dir_basename | ||
2099 | 273 | self.system_manager.logging.debug("Suite_name: %s" %(suite_name)) | ||
2100 | 274 | return suite_name | ||
2101 | 275 | |||
2102 | 276 | def process_suite_options(self, suite_dir): | ||
2103 | 277 | """ Process the suite.opt and master.opt files | ||
2104 | 278 | that reside at the suite-level if they exist. | ||
2105 | 279 | Return a list of the options found | ||
2106 | 280 | |||
2107 | 281 | We also process .cnf files - this | ||
2108 | 282 | is currently dbqp-only and is the proper | ||
2109 | 283 | way to do things :P | ||
2110 | 284 | |||
2111 | 285 | """ | ||
2112 | 286 | found_options = [] | ||
2113 | 287 | opt_files = ['t/master.opt','t/suite.opt'] | ||
2114 | 288 | for opt_file in opt_files: | ||
2115 | 289 | found_options = found_options + self.process_opt_file(os.path.join(suite_dir,opt_file)) | ||
2116 | 290 | # We also process the suite-level .cnf file(s). We override | ||
2117 | 291 | # a master.opt file if we have a .cnf file. There is no reason they | ||
2118 | 292 | # should ever be used in conjunction and I am biased towards .cnf ; ) | ||
2119 | 293 | cnf_files = ['t/master.cnf'] | ||
2120 | 294 | cnf_options = [] | ||
2121 | 295 | for cnf_file in cnf_files: | ||
2122 | 296 | config_file_path = os.path.join(suite_dir,cnf_file) | ||
2123 | 297 | cnf_flag, returned_options = self.process_cnf_file(config_file_path) | ||
2124 | 298 | cnf_options += returned_options | ||
2125 | 299 | if cnf_flag: # we found a proper file and need to override | ||
2126 | 300 | found_options = cnf_options | ||
2127 | 301 | else: | ||
2128 | 302 | config_file_path = None | ||
2129 | 303 | return config_file_path, found_options | ||
2130 | 304 | |||
2131 | 305 | def process_disabled_test_file(self, testdir): | ||
2132 | 306 | """ Checks and processes the suite's disabled.def | ||
2133 | 307 | file. This file must reside in the suite/t directory | ||
2134 | 308 | It must be in the format: | ||
2135 | 309 | test-name : comment (eg BugNNNN - bug on hold, test disabled) | ||
2136 | 310 | In reality a test should *never* be disabled. EVER. | ||
2137 | 311 | However, we keep this as a bit of utility | ||
2138 | 312 | |||
2139 | 313 | """ | ||
2140 | 314 | disabled_tests = {} | ||
2141 | 315 | disabled_def_path = os.path.join(testdir,'disabled.def') | ||
2142 | 316 | if not os.path.exists(disabled_def_path): | ||
2143 | 317 | return disabled_tests | ||
2144 | 318 | |||
2145 | 319 | try: | ||
2146 | 320 | disabled_test_file = open(disabled_def_path,'r') | ||
2147 | 321 | except IOError, e: | ||
2148 | 322 | self.system_manager.logging.error("Problem opening disabled.def file: %s" %(disabled_def_path)) | ||
2149 | 323 | sys.exit(1) | ||
2150 | 324 | |||
2151 | 325 | self.system_manager.logging.debug("Processing disabled.def file: %s" %(disabled_def_path)) | ||
2152 | 326 | disabled_bug_pattern = re.compile("[\S]+[\s]+:[\s]+[\S]") | ||
2153 | 327 | |||
2154 | 328 | for line in disabled_test_file: | ||
2155 | 329 | line = line.strip() | ||
2156 | 330 | if not line.startswith('#'): # comment | ||
2157 | 331 | if re.match(disabled_test_pattern,line): | ||
2158 | 332 | self.system_manager.logging.debug("found disabled test - %s" %(line)) | ||
2159 | 333 | test_name, test_comment = line.split(':') | ||
2160 | 334 | disabled_tests[test_name.strip()]=test_comment.strip() | ||
2161 | 335 | |||
2162 | 336 | disabled_test_file.close() | ||
2163 | 337 | return disabled_tests | ||
2164 | 338 | |||
2165 | 339 | |||
2166 | 340 | def process_opt_file(self, opt_file_path): | ||
2167 | 341 | """ Process a test-run '.opt' file. | ||
2168 | 342 | These files contain test and suite-specific server options | ||
2169 | 343 | (ie what options the server needs to use for the test) | ||
2170 | 344 | |||
2171 | 345 | Returns a list of the options (we don't really validate...yet) | ||
2172 | 346 | |||
2173 | 347 | NOTE: test-run.pl allows for server *and* system options | ||
2174 | 348 | (eg timezone, slave_count, etc) in opt files. We don't. | ||
2175 | 349 | None of our tests use this and we should probably avoid it | ||
2176 | 350 | We can introduce server and system .opt files or even better | ||
2177 | 351 | would be to use config files as we do with drizzle-automation | ||
2178 | 352 | This would allow us to specify options for several different | ||
2179 | 353 | things in a single file, but in a clean and standardized manner | ||
2180 | 354 | |||
2181 | 355 | """ | ||
2182 | 356 | found_options = [] | ||
2183 | 357 | if not os.path.exists(opt_file_path): | ||
2184 | 358 | return found_options | ||
2185 | 359 | |||
2186 | 360 | try: | ||
2187 | 361 | opt_file = open(opt_file_path,'r') | ||
2188 | 362 | except IOError, e: | ||
2189 | 363 | self.system_manager.logging.error("Problem opening option file: %s" %(opt_file_path)) | ||
2190 | 364 | sys.exit(1) | ||
2191 | 365 | |||
2192 | 366 | self.system_manager.logging.debug("Processing opt file: %s" %(opt_file_path)) | ||
2193 | 367 | for line in opt_file: | ||
2194 | 368 | options = line.split('--') | ||
2195 | 369 | if options: | ||
2196 | 370 | for option in options: | ||
2197 | 371 | if option: | ||
2198 | 372 | if 'restart' in option or '#' in option: | ||
2199 | 373 | option = 'restart' | ||
2200 | 374 | found_options.append('--%s' %(option.strip())) | ||
2201 | 375 | opt_file.close() | ||
2202 | 376 | return found_options | ||
2203 | 377 | |||
2204 | 378 | def process_cnf_file(self, cnf_file_path): | ||
2205 | 379 | """ We extract meaningful information from a .cnf file | ||
2206 | 380 | if it exists. Currently limited to server allocation | ||
2207 | 381 | needs | ||
2208 | 382 | |||
2209 | 383 | """ | ||
2210 | 384 | |||
2211 | 385 | server_requirements = [] | ||
2212 | 386 | cnf_flag = 0 | ||
2213 | 387 | if os.path.exists(cnf_file_path): | ||
2214 | 388 | cnf_flag = 1 | ||
2215 | 389 | config_reader = RawConfigParser() | ||
2216 | 390 | config_reader.read(cnf_file_path) | ||
2217 | 391 | server_requirements = self.process_server_reqs(config_reader.get('test_servers','servers')) | ||
2218 | 392 | return ( cnf_flag, server_requirements ) | ||
2219 | 393 | |||
2220 | 394 | def process_server_reqs(self,data_string): | ||
2221 | 395 | """ We read in the list of lists as a string, so we need to | ||
2222 | 396 | handle this / break it down into proper chunks | ||
2223 | 397 | |||
2224 | 398 | """ | ||
2225 | 399 | server_reqs = [] | ||
2226 | 400 | # We expect to see a list of lists and throw away the | ||
2227 | 401 | # enclosing brackets | ||
2228 | 402 | option_sets = data_string[1:-1].strip().split(',') | ||
2229 | 403 | for option_set in option_sets: | ||
2230 | 404 | server_reqs.append([option_set[1:-1].strip()]) | ||
2231 | 405 | return server_reqs | ||
2232 | 406 | |||
2233 | 407 | def testlist_filter(self, testlist): | ||
2234 | 408 | """ Filter our list of testdir contents based on several | ||
2235 | 409 | criteria. This looks for user-specified test-cases | ||
2236 | 410 | and applies the do-test and skip-test filters | ||
2237 | 411 | |||
2238 | 412 | Returns the list of tests that we want to execute | ||
2239 | 413 | for further processing | ||
2240 | 414 | |||
2241 | 415 | """ | ||
2242 | 416 | |||
2243 | 417 | # We want only .test files | ||
2244 | 418 | # Possible TODO: allow alternate test extensions | ||
2245 | 419 | testlist = [test_file for test_file in testlist if test_file.endswith('.test')] | ||
2246 | 420 | |||
2247 | 421 | # Search for specific test names | ||
2248 | 422 | if self.desired_tests: # We have specific, named tests we want from the suite(s) | ||
2249 | 423 | tests_to_use = [] | ||
2250 | 424 | for test in self.desired_tests: | ||
2251 | 425 | if test.endswith('.test'): | ||
2252 | 426 | pass | ||
2253 | 427 | else: | ||
2254 | 428 | test = test+'.test' | ||
2255 | 429 | if test in testlist: | ||
2256 | 430 | tests_to_use.append(test) | ||
2257 | 431 | testlist = tests_to_use | ||
2258 | 432 | |||
2259 | 433 | # TODO: Allow for regex? | ||
2260 | 434 | # Apply do-test filter | ||
2261 | 435 | if self.dotest: | ||
2262 | 436 | testlist = [test_file for test_file in testlist if test_file.startswith(self.dotest)] | ||
2263 | 437 | # Apply skip-test filter | ||
2264 | 438 | if self.skiptest: | ||
2265 | 439 | testlist = [test_file for test_file in testlist if not test_file.startswith(self.skiptest)] | ||
2266 | 440 | return testlist | ||
2267 | 441 | |||
2268 | 442 | def find_result_path(self, result_dir, result_file_name): | ||
2269 | 443 | """ This is copied from test-run.pl dtr_cases.pl | ||
2270 | 444 | If we have an engine option passed in and the | ||
2271 | 445 | path resultdir/engine/testname.result exists, that is | ||
2272 | 446 | our .result file | ||
2273 | 447 | |||
2274 | 448 | Need to check if we really need this - maybe PBXT? | ||
2275 | 449 | |||
2276 | 450 | """ | ||
2277 | 451 | result_path = os.path.join(result_dir,result_file_name) | ||
2278 | 452 | if self.default_engine: | ||
2279 | 453 | candidate_path = os.path.join(result_dir, self.default_engine, | ||
2280 | 454 | result_file_name) | ||
2281 | 455 | if os.path.exists(candidate_path): | ||
2282 | 456 | result_path = candidate_path | ||
2283 | 457 | return result_path | ||
2284 | 458 | |||
2285 | 459 | def check_if_disabled(self, disabled_tests, test_name): | ||
2286 | 460 | """ Scan the list of disabled tests if it exists to see | ||
2287 | 461 | if the test is disabled. | ||
2288 | 462 | |||
2289 | 463 | """ | ||
2290 | 464 | |||
2291 | 465 | if disabled_tests: | ||
2292 | 466 | if test_name in disabled_tests: | ||
2293 | 467 | self.system_manager.logging.debug("%s says - I'm disabled" %(test_name)) | ||
2294 | 468 | return (1, disabled_tests[test_name]) | ||
2295 | 469 | return (0,None) | ||
2296 | 470 | |||
2297 | 471 | def sort_testcases(self): | ||
2298 | 472 | """ We sort our testcases according to the server_options they have | ||
2299 | 473 | For each testcase, we sort the list of options, so if a test has | ||
2300 | 474 | --plugin-add=csv --abracadabra, we would get | ||
2301 | 475 | --abracadabra --plugin-add=csv | ||
2302 | 476 | |||
2303 | 477 | This results in tests that have similar options being run in order | ||
2304 | 478 | this minimizes server restarts which can be costly | ||
2305 | 479 | |||
2306 | 480 | """ | ||
2307 | 481 | test_management.testManager.sort_testcases(self) | ||
2308 | 482 | organizer = {} | ||
2309 | 483 | ordered_list = [] | ||
2310 | 484 | for testcase in self.test_list: | ||
2311 | 485 | key = " ".join(sorted(testcase.server_options)) | ||
2312 | 486 | if key in organizer: | ||
2313 | 487 | organizer[key].append(testcase) | ||
2314 | 488 | else: | ||
2315 | 489 | organizer[key] = [testcase] | ||
2316 | 490 | for value_list in organizer.values(): | ||
2317 | 491 | ordered_list = ordered_list + value_list | ||
2318 | 492 | self.test_list = ordered_list | ||
2319 | 493 | |||
2320 | 494 | |||
2321 | 495 | |||
2322 | 496 | 0 | ||
2323 | === removed directory 'dbqp/lib/modes/native' | |||
2324 | === removed file 'dbqp/lib/modes/native/__init__.py' | |||
2325 | === removed file 'dbqp/lib/modes/native/native_test_execution.py' | |||
2326 | --- dbqp/lib/modes/native/native_test_execution.py 2012-02-04 01:22:23 +0000 | |||
2327 | +++ dbqp/lib/modes/native/native_test_execution.py 1970-01-01 00:00:00 +0000 | |||
2328 | @@ -1,100 +0,0 @@ | |||
2329 | 1 | #! /usr/bin/env python | ||
2330 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
2331 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
2332 | 4 | # | ||
2333 | 5 | # Copyright (C) 2011 Patrick Crews | ||
2334 | 6 | # | ||
2335 | 7 | # | ||
2336 | 8 | # This program is free software; you can redistribute it and/or modify | ||
2337 | 9 | # it under the terms of the GNU General Public License as published by | ||
2338 | 10 | # the Free Software Foundation; either version 2 of the License, or | ||
2339 | 11 | # (at your option) any later version. | ||
2340 | 12 | # | ||
2341 | 13 | # This program is distributed in the hope that it will be useful, | ||
2342 | 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
2343 | 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
2344 | 16 | # GNU General Public License for more details. | ||
2345 | 17 | # | ||
2346 | 18 | # You should have received a copy of the GNU General Public License | ||
2347 | 19 | # along with this program; if not, write to the Free Software | ||
2348 | 20 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
2349 | 21 | |||
2350 | 22 | """ native_test_execution: | ||
2351 | 23 | code related to the execution of native test cases | ||
2352 | 24 | |||
2353 | 25 | We are provided access to a testManager with | ||
2354 | 26 | native-specific testCases. | ||
2355 | 27 | |||
2356 | 28 | """ | ||
2357 | 29 | |||
2358 | 30 | # imports | ||
2359 | 31 | import os | ||
2360 | 32 | import re | ||
2361 | 33 | import imp | ||
2362 | 34 | import sys | ||
2363 | 35 | import subprocess | ||
2364 | 36 | import unittest | ||
2365 | 37 | |||
2366 | 38 | import lib.test_mgmt.test_execution as test_execution | ||
2367 | 39 | |||
2368 | 40 | class testExecutor(test_execution.testExecutor): | ||
2369 | 41 | """ native-mode-specific executor """ | ||
2370 | 42 | |||
2371 | 43 | def execute_testCase (self): | ||
2372 | 44 | """ Execute a test module testCase | ||
2373 | 45 | |||
2374 | 46 | """ | ||
2375 | 47 | test_execution.testExecutor.execute_testCase(self) | ||
2376 | 48 | self.status = 0 | ||
2377 | 49 | |||
2378 | 50 | # execute test module | ||
2379 | 51 | self.current_test_status = self.execute_test_module() | ||
2380 | 52 | |||
2381 | 53 | # analyze results | ||
2382 | 54 | self.set_server_status(self.current_test_status) | ||
2383 | 55 | self.server_manager.reset_servers(self.name) | ||
2384 | 56 | |||
2385 | 57 | |||
2386 | 58 | def execute_test_module(self): | ||
2387 | 59 | """ Execute the commandline and return the result. | ||
2388 | 60 | We use subprocess as we can pass os.environ dicts and whatnot | ||
2389 | 61 | |||
2390 | 62 | """ | ||
2391 | 63 | output_file_path = os.path.join(self.logdir,'native.out') | ||
2392 | 64 | output_file = open(output_file_path,'w') | ||
2393 | 65 | testcase_name = self.current_testcase.fullname | ||
2394 | 66 | test_name = self.current_testcase.name | ||
2395 | 67 | |||
2396 | 68 | # import our module and pass it some goodies to play with | ||
2397 | 69 | test_module = imp.load_source(test_name, self.current_testcase.test_path) | ||
2398 | 70 | test_module.servers = self.current_servers | ||
2399 | 71 | test_module.test_executor = self | ||
2400 | 72 | test_module.server_manager = self.server_manager | ||
2401 | 73 | |||
2402 | 74 | # start our test | ||
2403 | 75 | self.time_manager.start(testcase_name,'test') | ||
2404 | 76 | self.logging.subunit_start(testcase_name) | ||
2405 | 77 | suite = unittest.TestLoader().loadTestsFromTestCase(test_module.basicTest) | ||
2406 | 78 | test_result = unittest.TextTestRunner(stream=output_file, verbosity=2).run(suite) | ||
2407 | 79 | execution_time = int(self.time_manager.stop(testcase_name)*1000) # millisec | ||
2408 | 80 | self.current_test_retcode = test_result.wasSuccessful() | ||
2409 | 81 | output_file.close() | ||
2410 | 82 | output_file = open(output_file_path,'r') | ||
2411 | 83 | self.current_test_output = ''.join(output_file.readlines()) | ||
2412 | 84 | output_file.close() | ||
2413 | 85 | |||
2414 | 86 | self.current_test_exec_time = execution_time | ||
2415 | 87 | retval = None | ||
2416 | 88 | if self.current_test_retcode: | ||
2417 | 89 | if not self.verbose: | ||
2418 | 90 | self.current_test_output = None | ||
2419 | 91 | retval = 'pass' | ||
2420 | 92 | else: | ||
2421 | 93 | retval = 'fail' | ||
2422 | 94 | self.logging.subunit_stop( testcase_name | ||
2423 | 95 | , retval | ||
2424 | 96 | , self.current_test_output | ||
2425 | 97 | ) | ||
2426 | 98 | return retval | ||
2427 | 99 | |||
2428 | 100 | |||
2429 | 101 | 0 | ||
2430 | === removed file 'dbqp/lib/modes/native/native_test_management.py' | |||
2431 | --- dbqp/lib/modes/native/native_test_management.py 2012-02-04 01:22:23 +0000 | |||
2432 | +++ dbqp/lib/modes/native/native_test_management.py 1970-01-01 00:00:00 +0000 | |||
2433 | @@ -1,160 +0,0 @@ | |||
2434 | 1 | #! /usr/bin/env python | ||
2435 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
2436 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
2437 | 4 | # | ||
2438 | 5 | # Copyright (C) 2011 Patrick Crews | ||
2439 | 6 | # | ||
2440 | 7 | ## This program is free software; you can redistribute it and/or modify | ||
2441 | 8 | # it under the terms of the GNU General Public License as published by | ||
2442 | 9 | # the Free Software Foundation; either version 2 of the License, or | ||
2443 | 10 | # (at your option) any later version. | ||
2444 | 11 | # | ||
2445 | 12 | # This program is distributed in the hope that it will be useful, | ||
2446 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
2447 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
2448 | 15 | # GNU General Public License for more details. | ||
2449 | 16 | # | ||
2450 | 17 | # You should have received a copy of the GNU General Public License | ||
2451 | 18 | # along with this program; if not, write to the Free Software | ||
2452 | 19 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
2453 | 20 | |||
2454 | 21 | """ crashme_test_management: | ||
2455 | 22 | code related to the gathering / analysis / management of | ||
2456 | 23 | the test cases | ||
2457 | 24 | ie - collecting the list of tests in each suite, then | ||
2458 | 25 | gathering additional, relevant information for crashme mode | ||
2459 | 26 | |||
2460 | 27 | """ | ||
2461 | 28 | |||
2462 | 29 | # imports | ||
2463 | 30 | import os | ||
2464 | 31 | import re | ||
2465 | 32 | import sys | ||
2466 | 33 | import imp | ||
2467 | 34 | |||
2468 | 35 | import lib.test_mgmt.test_management as test_management | ||
2469 | 36 | |||
2470 | 37 | |||
2471 | 38 | |||
2472 | 39 | class testCase: | ||
2473 | 40 | """Holds info on a single crashme test | ||
2474 | 41 | |||
2475 | 42 | """ | ||
2476 | 43 | def __init__( self | ||
2477 | 44 | , system_manager | ||
2478 | 45 | , name=None | ||
2479 | 46 | , fullname = None | ||
2480 | 47 | , server_requirements=[[]] | ||
2481 | 48 | , comment=None | ||
2482 | 49 | , cnf_path=None | ||
2483 | 50 | , request_dict=None | ||
2484 | 51 | , test_path = None | ||
2485 | 52 | , suitename = 'native_tests' | ||
2486 | 53 | , debug=False ): | ||
2487 | 54 | self.system_manager = system_manager | ||
2488 | 55 | self.logging = self.system_manager.logging | ||
2489 | 56 | self.skip_keys = ['system_manager','logging'] | ||
2490 | 57 | self.name = name | ||
2491 | 58 | self.fullname = fullname | ||
2492 | 59 | self.suitename = suitename | ||
2493 | 60 | self.master_sh = None | ||
2494 | 61 | self.comment = comment | ||
2495 | 62 | self.server_requirements = server_requirements | ||
2496 | 63 | self.cnf_path = cnf_path | ||
2497 | 64 | self.server_requests = request_dict | ||
2498 | 65 | self.test_path = test_path | ||
2499 | 66 | if debug: | ||
2500 | 67 | self.system_manager.logging.debug_class(self) | ||
2501 | 68 | |||
2502 | 69 | def should_run(self): | ||
2503 | 70 | if self.skip_flag or self.disable: | ||
2504 | 71 | return 0 | ||
2505 | 72 | else: | ||
2506 | 73 | return 1 | ||
2507 | 74 | |||
2508 | 75 | |||
2509 | 76 | |||
2510 | 77 | |||
2511 | 78 | |||
2512 | 79 | class testManager(test_management.testManager): | ||
2513 | 80 | """Deals with scanning test directories, gathering test cases, and | ||
2514 | 81 | collecting per-test information (opt files, etc) for use by the | ||
2515 | 82 | test-runner | ||
2516 | 83 | |||
2517 | 84 | """ | ||
2518 | 85 | |||
2519 | 86 | def __init__( self, variables, system_manager): | ||
2520 | 87 | super(testManager, self).__init__( variables, system_manager) | ||
2521 | 88 | server_type = variables['defaultservertype'] | ||
2522 | 89 | if server_type == 'mysql' or server_type =='galera': | ||
2523 | 90 | server_type = 'percona' | ||
2524 | 91 | self.suitepaths = [os.path.join(self.testdir,'%s_tests' %(server_type))] | ||
2525 | 92 | if variables['suitelist'] is None: | ||
2526 | 93 | self.suitelist = ['main'] | ||
2527 | 94 | else: | ||
2528 | 95 | self.suitelist = variables['suitelist'] | ||
2529 | 96 | |||
2530 | 97 | def process_suite(self,suite_dir): | ||
2531 | 98 | """Process a test suite. | ||
2532 | 99 | Look for tests, which are nice clean python unittest files | ||
2533 | 100 | |||
2534 | 101 | """ | ||
2535 | 102 | |||
2536 | 103 | # We know this based on how we organize native test conf files | ||
2537 | 104 | suite_name = os.path.basename(suite_dir) | ||
2538 | 105 | self.system_manager.logging.verbose("Processing suite: %s" %(suite_name)) | ||
2539 | 106 | testlist = [os.path.join(suite_dir,test_file) for test_file in sorted(os.listdir(suite_dir)) if test_file.endswith('_test.py')] | ||
2540 | 107 | |||
2541 | 108 | # Search for specific test names | ||
2542 | 109 | if self.desired_tests: # We have specific, named tests we want from the suite(s) | ||
2543 | 110 | tests_to_use = [] | ||
2544 | 111 | for test in self.desired_tests: | ||
2545 | 112 | if test.endswith('.py'): | ||
2546 | 113 | pass | ||
2547 | 114 | else: | ||
2548 | 115 | test = test+'.py' | ||
2549 | 116 | test = os.path.join(suite_dir,test) | ||
2550 | 117 | if test in testlist: | ||
2551 | 118 | tests_to_use.append(test) | ||
2552 | 119 | testlist = tests_to_use | ||
2553 | 120 | for test_case in testlist: | ||
2554 | 121 | self.add_test(self.process_test_file(suite_name, test_case)) | ||
2555 | 122 | |||
2556 | 123 | |||
2557 | 124 | def process_test_file(self, suite_name, testfile): | ||
2558 | 125 | """ We convert the info in a testfile into a testCase object """ | ||
2559 | 126 | |||
2560 | 127 | # test_name = filename - .py...simpler | ||
2561 | 128 | test_name = os.path.basename(testfile).replace('.py','') | ||
2562 | 129 | test_comment = None | ||
2563 | 130 | test_module = imp.load_source(test_name, testfile) | ||
2564 | 131 | server_requirements = test_module.server_requirements | ||
2565 | 132 | try: | ||
2566 | 133 | server_requests = test_module.server_requests | ||
2567 | 134 | except AttributeError, NameError: | ||
2568 | 135 | server_requests = None | ||
2569 | 136 | return testCase( self.system_manager | ||
2570 | 137 | , name = test_name | ||
2571 | 138 | , fullname = "%s.%s" %(suite_name, test_name) | ||
2572 | 139 | , server_requirements = server_requirements | ||
2573 | 140 | , cnf_path = None | ||
2574 | 141 | , request_dict = server_requests | ||
2575 | 142 | , test_path = testfile | ||
2576 | 143 | , debug = self.debug ) | ||
2577 | 144 | |||
2578 | 145 | |||
2579 | 146 | |||
2580 | 147 | def record_test_result(self, test_case, test_status, output, exec_time): | ||
2581 | 148 | """ Accept the results of an executed testCase for further | ||
2582 | 149 | processing. | ||
2583 | 150 | |||
2584 | 151 | """ | ||
2585 | 152 | if test_status not in self.executed_tests: | ||
2586 | 153 | self.executed_tests[test_status] = [test_case] | ||
2587 | 154 | else: | ||
2588 | 155 | self.executed_tests[test_status].append(test_case) | ||
2589 | 156 | # report | ||
2590 | 157 | self.logging.test_report( test_case.fullname, test_status | ||
2591 | 158 | , str(exec_time), output | ||
2592 | 159 | , report_output= True) | ||
2593 | 160 | |||
2594 | 161 | 0 | ||
2595 | === removed directory 'dbqp/lib/modes/sysbench' | |||
2596 | === removed file 'dbqp/lib/modes/sysbench/__init__.py' | |||
2597 | === removed file 'dbqp/lib/modes/sysbench/sysbench_test_execution.py' | |||
2598 | --- dbqp/lib/modes/sysbench/sysbench_test_execution.py 2012-02-04 01:22:23 +0000 | |||
2599 | +++ dbqp/lib/modes/sysbench/sysbench_test_execution.py 1970-01-01 00:00:00 +0000 | |||
2600 | @@ -1,162 +0,0 @@ | |||
2601 | 1 | #! /usr/bin/env python | ||
2602 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
2603 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
2604 | 4 | # | ||
2605 | 5 | # Copyright (C) 2010 Patrick Crews | ||
2606 | 6 | # | ||
2607 | 7 | # | ||
2608 | 8 | # This program is free software; you can redistribute it and/or modify | ||
2609 | 9 | # it under the terms of the GNU General Public License as published by | ||
2610 | 10 | # the Free Software Foundation; either version 2 of the License, or | ||
2611 | 11 | # (at your option) any later version. | ||
2612 | 12 | # | ||
2613 | 13 | # This program is distributed in the hope that it will be useful, | ||
2614 | 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
2615 | 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
2616 | 16 | # GNU General Public License for more details. | ||
2617 | 17 | # | ||
2618 | 18 | # You should have received a copy of the GNU General Public License | ||
2619 | 19 | # along with this program; if not, write to the Free Software | ||
2620 | 20 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
2621 | 21 | |||
2622 | 22 | """ sysbench_test_execution: | ||
2623 | 23 | code related to the execution of sysbench test cases | ||
2624 | 24 | |||
2625 | 25 | We are provided access to a testManager with | ||
2626 | 26 | sysbench-specific testCases. | ||
2627 | 27 | |||
2628 | 28 | """ | ||
2629 | 29 | |||
2630 | 30 | # imports | ||
2631 | 31 | import os | ||
2632 | 32 | import re | ||
2633 | 33 | import sys | ||
2634 | 34 | import subprocess | ||
2635 | 35 | import commands | ||
2636 | 36 | |||
2637 | 37 | import lib.test_mgmt.test_execution as test_execution | ||
2638 | 38 | |||
2639 | 39 | class testExecutor(test_execution.testExecutor): | ||
2640 | 40 | """ sysbench-specific testExecutor | ||
2641 | 41 | |||
2642 | 42 | """ | ||
2643 | 43 | |||
2644 | 44 | def execute_testCase (self): | ||
2645 | 45 | """ Execute a sysbench testCase | ||
2646 | 46 | |||
2647 | 47 | """ | ||
2648 | 48 | test_execution.testExecutor.execute_testCase(self) | ||
2649 | 49 | self.status = 0 | ||
2650 | 50 | |||
2651 | 51 | # prepare the server for sysbench | ||
2652 | 52 | self.prepare_sysbench() | ||
2653 | 53 | |||
2654 | 54 | # execute sysbench | ||
2655 | 55 | self.execute_sysbench() | ||
2656 | 56 | |||
2657 | 57 | # analyze results | ||
2658 | 58 | self.current_test_status = self.process_sysbench_output() | ||
2659 | 59 | self.set_server_status(self.current_test_status) | ||
2660 | 60 | self.server_manager.reset_servers(self.name) | ||
2661 | 61 | |||
2662 | 62 | def prepare_sysbench(self): | ||
2663 | 63 | """ Prepare the server for a sysbench run | ||
2664 | 64 | We use subprocess as we can pass os.environ dicts and whatnot | ||
2665 | 65 | |||
2666 | 66 | """ | ||
2667 | 67 | |||
2668 | 68 | sysbench_outfile = os.path.join(self.logdir,'sysbench.out') | ||
2669 | 69 | sysbench_output = open(sysbench_outfile,'w') | ||
2670 | 70 | sysbench_cmd = ' '.join([self.current_testcase.test_command,'prepare']) | ||
2671 | 71 | self.logging.info("Preparing database for sysbench run...") | ||
2672 | 72 | self.logging.debug(sysbench_cmd) | ||
2673 | 73 | sysbench_subproc = subprocess.Popen( sysbench_cmd | ||
2674 | 74 | , shell=True | ||
2675 | 75 | #, cwd=os.getcwd() | ||
2676 | 76 | , env=self.working_environment | ||
2677 | 77 | , stdout = sysbench_output | ||
2678 | 78 | , stderr = subprocess.STDOUT | ||
2679 | 79 | ) | ||
2680 | 80 | sysbench_subproc.wait() | ||
2681 | 81 | retcode = sysbench_subproc.returncode | ||
2682 | 82 | |||
2683 | 83 | sysbench_output.close() | ||
2684 | 84 | sysbench_file = open(sysbench_outfile,'r') | ||
2685 | 85 | output = ''.join(sysbench_file.readlines()) | ||
2686 | 86 | sysbench_file.close() | ||
2687 | 87 | self.logging.debug("sysbench_retcode: %d" %(retcode)) | ||
2688 | 88 | self.logging.debug(output) | ||
2689 | 89 | if retcode: | ||
2690 | 90 | self.logging.error("sysbench_prepare failed with retcode %d:" %(retcode)) | ||
2691 | 91 | self.logging.error(output) | ||
2692 | 92 | sys.exit(1) | ||
2693 | 93 | |||
2694 | 94 | |||
2695 | 95 | |||
2696 | 96 | |||
2697 | 97 | def execute_sysbench(self): | ||
2698 | 98 | """ Execute the commandline and return the result. | ||
2699 | 99 | We use subprocess as we can pass os.environ dicts and whatnot | ||
2700 | 100 | |||
2701 | 101 | """ | ||
2702 | 102 | |||
2703 | 103 | testcase_name = self.current_testcase.fullname | ||
2704 | 104 | self.time_manager.start(testcase_name,'test') | ||
2705 | 105 | sysbench_outfile = os.path.join(self.logdir,'sysbench.out') | ||
2706 | 106 | sysbench_output = open(sysbench_outfile,'w') | ||
2707 | 107 | sysbench_cmd = ' '.join([self.current_testcase.test_command, 'run']) | ||
2708 | 108 | self.logging.info("Executing sysbench: %s" %(sysbench_cmd)) | ||
2709 | 109 | |||
2710 | 110 | sysbench_subproc = subprocess.Popen( sysbench_cmd | ||
2711 | 111 | , shell=True | ||
2712 | 112 | #, cwd=self.system_manager.sysbench_path | ||
2713 | 113 | , env=self.working_environment | ||
2714 | 114 | , stdout = sysbench_output | ||
2715 | 115 | , stderr = subprocess.STDOUT | ||
2716 | 116 | ) | ||
2717 | 117 | sysbench_subproc.wait() | ||
2718 | 118 | retcode = sysbench_subproc.returncode | ||
2719 | 119 | execution_time = int(self.time_manager.stop(testcase_name)*1000) # millisec | ||
2720 | 120 | |||
2721 | 121 | sysbench_output.close() | ||
2722 | 122 | sysbench_file = open(sysbench_outfile,'r') | ||
2723 | 123 | output = ''.join(sysbench_file.readlines()) | ||
2724 | 124 | self.logging.debug(output) | ||
2725 | 125 | sysbench_file.close() | ||
2726 | 126 | |||
2727 | 127 | self.logging.debug("sysbench_retcode: %d" %(retcode)) | ||
2728 | 128 | self.current_test_retcode = retcode | ||
2729 | 129 | self.current_test_output = output | ||
2730 | 130 | self.current_test_exec_time = execution_time | ||
2731 | 131 | |||
2732 | 132 | def process_sysbench_output(self): | ||
2733 | 133 | """ sysbench has run, we now check out what we have | ||
2734 | 134 | We also output the data from the run | ||
2735 | 135 | |||
2736 | 136 | """ | ||
2737 | 137 | # This slice code taken from drizzle-automation's sysbench handling | ||
2738 | 138 | # Slice up the output report into a matrix and insert into the DB. | ||
2739 | 139 | regexes= { | ||
2740 | 140 | 'tps': re.compile(r".*transactions\:\s+\d+\D*(\d+\.\d+).*") | ||
2741 | 141 | , 'deadlocksps': re.compile(r".*deadlocks\:\s+\d+\D*(\d+\.\d+).*") | ||
2742 | 142 | , 'rwreqps': re.compile(r".*read\/write\s+requests\:\s+\d+\D*(\d+\.\d+).*") | ||
2743 | 143 | , 'min_req_lat_ms': re.compile(r".*min\:\s+(\d*\.\d+)ms.*") | ||
2744 | 144 | , 'max_req_lat_ms': re.compile(r".*max\:\s+(\d*\.\d+)ms.*") | ||
2745 | 145 | , 'avg_req_lat_ms': re.compile(r".*avg\:\s+(\d*\.\d+)ms.*") | ||
2746 | 146 | , '95p_req_lat_ms': re.compile(r".*approx.\s+95\s+percentile\:\s+(\d+\.\d+)ms.*") | ||
2747 | 147 | } | ||
2748 | 148 | run= {} | ||
2749 | 149 | for line in self.current_test_output.split("\n"): | ||
2750 | 150 | for key in regexes.keys(): | ||
2751 | 151 | result= regexes[key].match(line) | ||
2752 | 152 | if result: | ||
2753 | 153 | run[key]= float(result.group(1)) # group(0) is entire match... | ||
2754 | 154 | # we set our test output to the regex'd-up data | ||
2755 | 155 | # we also make it a single string, separated by newlines | ||
2756 | 156 | self.current_test_output = str(run)[1:-1].replace(',','\n').replace("'",'') | ||
2757 | 157 | |||
2758 | 158 | if self.current_test_retcode == 0: | ||
2759 | 159 | return 'pass' | ||
2760 | 160 | else: | ||
2761 | 161 | return 'fail' | ||
2762 | 162 | |||
2763 | 163 | 0 | ||
2764 | === removed file 'dbqp/lib/modes/sysbench/sysbench_test_management.py' | |||
2765 | --- dbqp/lib/modes/sysbench/sysbench_test_management.py 2012-02-04 01:22:23 +0000 | |||
2766 | +++ dbqp/lib/modes/sysbench/sysbench_test_management.py 1970-01-01 00:00:00 +0000 | |||
2767 | @@ -1,159 +0,0 @@ | |||
2768 | 1 | #! /usr/bin/env python | ||
2769 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
2770 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
2771 | 4 | # | ||
2772 | 5 | # Copyright (C) 2010 Patrick Crews | ||
2773 | 6 | # | ||
2774 | 7 | ## This program is free software; you can redistribute it and/or modify | ||
2775 | 8 | # it under the terms of the GNU General Public License as published by | ||
2776 | 9 | # the Free Software Foundation; either version 2 of the License, or | ||
2777 | 10 | # (at your option) any later version. | ||
2778 | 11 | # | ||
2779 | 12 | # This program is distributed in the hope that it will be useful, | ||
2780 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
2781 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
2782 | 15 | # GNU General Public License for more details. | ||
2783 | 16 | # | ||
2784 | 17 | # You should have received a copy of the GNU General Public License | ||
2785 | 18 | # along with this program; if not, write to the Free Software | ||
2786 | 19 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
2787 | 20 | |||
2788 | 21 | """ sysbench_test_management: | ||
2789 | 22 | code related to the gathering / analysis / management of | ||
2790 | 23 | the test cases | ||
2791 | 24 | ie - collecting the list of tests in each suite, then | ||
2792 | 25 | gathering additional, relevant information for sysbench mode | ||
2793 | 26 | |||
2794 | 27 | """ | ||
2795 | 28 | |||
2796 | 29 | # imports | ||
2797 | 30 | import os | ||
2798 | 31 | import re | ||
2799 | 32 | import sys | ||
2800 | 33 | from ConfigParser import RawConfigParser | ||
2801 | 34 | |||
2802 | 35 | import lib.test_mgmt.test_management as test_management | ||
2803 | 36 | |||
2804 | 37 | |||
2805 | 38 | |||
2806 | 39 | class testCase: | ||
2807 | 40 | """Holds info on a single sysbench test | ||
2808 | 41 | |||
2809 | 42 | """ | ||
2810 | 43 | def __init__( self, system_manager, name=None | ||
2811 | 44 | , fullname = None, server_requirements=[[]] | ||
2812 | 45 | , comment=None, test_command=None, cnf_path=None | ||
2813 | 46 | , debug=False ): | ||
2814 | 47 | self.system_manager = system_manager | ||
2815 | 48 | self.logging = self.system_manager.logging | ||
2816 | 49 | self.skip_keys = ['system_manager','logging'] | ||
2817 | 50 | self.name = name | ||
2818 | 51 | self.fullname = fullname | ||
2819 | 52 | self.suitename = 'sysbench_tests' | ||
2820 | 53 | self.master_sh = None | ||
2821 | 54 | self.comment = comment | ||
2822 | 55 | self.server_requirements = server_requirements | ||
2823 | 56 | self.test_command = test_command | ||
2824 | 57 | self.cnf_path = cnf_path | ||
2825 | 58 | |||
2826 | 59 | if debug: | ||
2827 | 60 | self.system_manager.logging.debug_class(self) | ||
2828 | 61 | |||
2829 | 62 | def should_run(self): | ||
2830 | 63 | if self.skip_flag or self.disable: | ||
2831 | 64 | return 0 | ||
2832 | 65 | else: | ||
2833 | 66 | return 1 | ||
2834 | 67 | |||
2835 | 68 | |||
2836 | 69 | |||
2837 | 70 | |||
2838 | 71 | |||
2839 | 72 | class testManager(test_management.testManager): | ||
2840 | 73 | """Deals with scanning test directories, gathering test cases, and | ||
2841 | 74 | collecting per-test information (opt files, etc) for use by the | ||
2842 | 75 | test-runner | ||
2843 | 76 | |||
2844 | 77 | """ | ||
2845 | 78 | |||
2846 | 79 | def __init__( self, variables, system_manager): | ||
2847 | 80 | super(testManager, self).__init__( variables, system_manager) | ||
2848 | 81 | self.suitepaths = [os.path.join(self.testdir,'sysbench_tests')] | ||
2849 | 82 | if variables['suitelist'] is None: | ||
2850 | 83 | self.suitelist = ['readonly'] | ||
2851 | 84 | else: | ||
2852 | 85 | self.suitelist = variables['suitelist'] | ||
2853 | 86 | |||
2854 | 87 | def process_suite(self,suite_dir): | ||
2855 | 88 | """Process a test suite. | ||
2856 | 89 | Look for sysbench tests, which are nice clean conf files | ||
2857 | 90 | |||
2858 | 91 | """ | ||
2859 | 92 | |||
2860 | 93 | # We know this based on how we organize sysbench test conf files | ||
2861 | 94 | suite_name = os.path.basename(suite_dir) | ||
2862 | 95 | self.system_manager.logging.verbose("Processing suite: %s" %(suite_name)) | ||
2863 | 96 | testlist = [os.path.join(suite_dir,test_file) for test_file in sorted(os.listdir(suite_dir)) if test_file.endswith('.cnf')] | ||
2864 | 97 | |||
2865 | 98 | # Search for specific test names | ||
2866 | 99 | if self.desired_tests: # We have specific, named tests we want from the suite(s) | ||
2867 | 100 | tests_to_use = [] | ||
2868 | 101 | for test in self.desired_tests: | ||
2869 | 102 | if test.endswith('.cnf'): | ||
2870 | 103 | pass | ||
2871 | 104 | else: | ||
2872 | 105 | test = test+'.cnf' | ||
2873 | 106 | test = os.path.join(suite_dir,test) | ||
2874 | 107 | if test in testlist: | ||
2875 | 108 | tests_to_use.append(test) | ||
2876 | 109 | testlist = tests_to_use | ||
2877 | 110 | for test_case in testlist: | ||
2878 | 111 | self.add_test(self.process_test_file(suite_name, test_case)) | ||
2879 | 112 | |||
2880 | 113 | |||
2881 | 114 | def process_test_file(self, suite_name, testfile): | ||
2882 | 115 | """ We convert the info in a testfile into a testCase object """ | ||
2883 | 116 | |||
2884 | 117 | config_reader = RawConfigParser() | ||
2885 | 118 | config_reader.read(testfile) | ||
2886 | 119 | # test_name = filename - .cnf...simpler | ||
2887 | 120 | test_name = os.path.basename(testfile).replace('.cnf','') | ||
2888 | 121 | test_comment = config_reader.get('test_info','comment') | ||
2889 | 122 | server_requirements = self.process_server_reqs(config_reader.get('test_servers','servers')) | ||
2890 | 123 | test_command = config_reader.get('test_command','command') | ||
2891 | 124 | return testCase( self.system_manager | ||
2892 | 125 | , name = test_name | ||
2893 | 126 | , fullname = "%s.%s" %(suite_name, test_name) | ||
2894 | 127 | , server_requirements = server_requirements | ||
2895 | 128 | , test_command = test_command | ||
2896 | 129 | , cnf_path = testfile | ||
2897 | 130 | , debug = self.debug ) | ||
2898 | 131 | |||
2899 | 132 | #sys.exit(0) | ||
2900 | 133 | |||
2901 | 134 | def process_server_reqs(self,data_string): | ||
2902 | 135 | """ We read in the list of lists as a string, so we need to | ||
2903 | 136 | handle this / break it down into proper chunks | ||
2904 | 137 | |||
2905 | 138 | """ | ||
2906 | 139 | server_reqs = [] | ||
2907 | 140 | # We expect to see a list of lists and throw away the | ||
2908 | 141 | # enclosing brackets | ||
2909 | 142 | option_sets = data_string[1:-1].strip().split(',') | ||
2910 | 143 | for option_set in option_sets: | ||
2911 | 144 | server_reqs.append([option_set[1:-1].strip()]) | ||
2912 | 145 | return server_reqs | ||
2913 | 146 | |||
2914 | 147 | def record_test_result(self, test_case, test_status, output, exec_time): | ||
2915 | 148 | """ Accept the results of an executed testCase for further | ||
2916 | 149 | processing. | ||
2917 | 150 | |||
2918 | 151 | """ | ||
2919 | 152 | if test_status not in self.executed_tests: | ||
2920 | 153 | self.executed_tests[test_status] = [test_case] | ||
2921 | 154 | else: | ||
2922 | 155 | self.executed_tests[test_status].append(test_case) | ||
2923 | 156 | # report | ||
2924 | 157 | self.logging.test_report( test_case.fullname, test_status | ||
2925 | 158 | , str(exec_time), output | ||
2926 | 159 | , report_output= True) | ||
2927 | 160 | 0 | ||
2928 | === removed file 'dbqp/lib/modes/test_mode.py' | |||
2929 | --- dbqp/lib/modes/test_mode.py 2012-02-04 01:22:23 +0000 | |||
2930 | +++ dbqp/lib/modes/test_mode.py 1970-01-01 00:00:00 +0000 | |||
2931 | @@ -1,68 +0,0 @@ | |||
2932 | 1 | #! /usr/bin/env python | ||
2933 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
2934 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
2935 | 4 | # | ||
2936 | 5 | # Copyright (C) 2010 Patrick Crews | ||
2937 | 6 | # | ||
2938 | 7 | # This program is free software; you can redistribute it and/or modify | ||
2939 | 8 | # it under the terms of the GNU General Public License as published by | ||
2940 | 9 | # the Free Software Foundation; either version 2 of the License, or | ||
2941 | 10 | # (at your option) any later version. | ||
2942 | 11 | # | ||
2943 | 12 | # This program is distributed in the hope that it will be useful, | ||
2944 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
2945 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
2946 | 15 | # GNU General Public License for more details. | ||
2947 | 16 | # | ||
2948 | 17 | # You should have received a copy of the GNU General Public License | ||
2949 | 18 | # along with this program; if not, write to the Free Software | ||
2950 | 19 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
2951 | 20 | |||
2952 | 21 | """test_mode.py | ||
2953 | 22 | code for dealing with testing modes | ||
2954 | 23 | A given mode should have a systemInitializer, testManager, and testExecutor | ||
2955 | 24 | that define how to setup, manage, and execute test cases | ||
2956 | 25 | |||
2957 | 26 | """ | ||
2958 | 27 | |||
2959 | 28 | # imports | ||
2960 | 29 | import sys | ||
2961 | 30 | |||
2962 | 31 | def handle_mode(variables, system_manager): | ||
2963 | 32 | """ Deals with the 'mode' option and returns | ||
2964 | 33 | the appropriate code objects for the test-runner to play with | ||
2965 | 34 | |||
2966 | 35 | """ | ||
2967 | 36 | |||
2968 | 37 | test_mode = variables['mode'].strip() | ||
2969 | 38 | system_manager.logging.info("Using testing mode: %s" %test_mode) | ||
2970 | 39 | |||
2971 | 40 | if test_mode == 'cleanup': | ||
2972 | 41 | # cleanup mode - we try to kill any servers whose pid's we detect | ||
2973 | 42 | # in our workdir. Might extend to other things (file cleanup, etc) | ||
2974 | 43 | # at some later point | ||
2975 | 44 | system_manager.cleanup(exit=True) | ||
2976 | 45 | |||
2977 | 46 | else: # we expect something from dbqp_modes | ||
2978 | 47 | supported_modes = [ 'dtr' | ||
2979 | 48 | , 'randgen' | ||
2980 | 49 | , 'sysbench' | ||
2981 | 50 | , 'sqlbench' | ||
2982 | 51 | , 'crashme' | ||
2983 | 52 | , 'native' | ||
2984 | 53 | ] | ||
2985 | 54 | if test_mode not in supported_modes: | ||
2986 | 55 | system_manager.logging.error("invalid mode argument: %s" %test_mode) | ||
2987 | 56 | sys.exit(1) | ||
2988 | 57 | |||
2989 | 58 | mgmt_module = "lib.modes.%s.%s_test_management" %(test_mode, test_mode) | ||
2990 | 59 | tmp = __import__(mgmt_module, globals(), locals(), ['testManager'], -1) | ||
2991 | 60 | testManager = tmp.testManager | ||
2992 | 61 | |||
2993 | 62 | exec_module = "%s.%s_test_execution" %(test_mode, test_mode) | ||
2994 | 63 | tmp = __import__(exec_module, globals(), locals(), ['testExecutor'], -1) | ||
2995 | 64 | testExecutor = tmp.testExecutor | ||
2996 | 65 | |||
2997 | 66 | test_manager = testManager( variables, system_manager ) | ||
2998 | 67 | return (test_manager, testExecutor) | ||
2999 | 68 | |||
3000 | 69 | 0 | ||
3001 | === removed directory 'dbqp/lib/opts' | |||
3002 | === removed file 'dbqp/lib/opts/__init__.py' | |||
3003 | === removed file 'dbqp/lib/opts/test_run_options.py' | |||
3004 | --- dbqp/lib/opts/test_run_options.py 2012-02-04 01:22:23 +0000 | |||
3005 | +++ dbqp/lib/opts/test_run_options.py 1970-01-01 00:00:00 +0000 | |||
3006 | @@ -1,556 +0,0 @@ | |||
3007 | 1 | #! /usr/bin/env python | ||
3008 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
3009 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
3010 | 4 | # | ||
3011 | 5 | # Copyright (C) 2010, 2011 Patrick Crews | ||
3012 | 6 | # | ||
3013 | 7 | # This program is free software; you can redistribute it and/or modify | ||
3014 | 8 | # it under the terms of the GNU General Public License as published by | ||
3015 | 9 | # the Free Software Foundation; either version 2 of the License, or | ||
3016 | 10 | # (at your option) any later version. | ||
3017 | 11 | # | ||
3018 | 12 | # This program is distributed in the hope that it will be useful, | ||
3019 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
3020 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
3021 | 15 | # GNU General Public License for more details. | ||
3022 | 16 | # | ||
3023 | 17 | # You should have received a copy of the GNU General Public License | ||
3024 | 18 | # along with this program; if not, write to the Free Software | ||
3025 | 19 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
3026 | 20 | |||
3027 | 21 | |||
3028 | 22 | |||
3029 | 23 | """Processes command line options for Drizzle test-runner""" | ||
3030 | 24 | |||
3031 | 25 | import os | ||
3032 | 26 | import sys | ||
3033 | 27 | import copy | ||
3034 | 28 | import exceptions | ||
3035 | 29 | import optparse | ||
3036 | 30 | |||
3037 | 31 | # functions | ||
3038 | 32 | def comma_list_split(option, opt, value, parser): | ||
3039 | 33 | """Callback for splitting input expected in list form""" | ||
3040 | 34 | cur_list = getattr(parser.values, option.dest,[]) | ||
3041 | 35 | input_list = value.split(',') | ||
3042 | 36 | # this is a hack to work with make target - we | ||
3043 | 37 | # don't deal with a dangling ',' in our list | ||
3044 | 38 | if '' in input_list: | ||
3045 | 39 | input_list.remove('') | ||
3046 | 40 | if cur_list: | ||
3047 | 41 | value_list = cur_list + input_list | ||
3048 | 42 | else: | ||
3049 | 43 | value_list = input_list | ||
3050 | 44 | setattr(parser.values, option.dest, value_list) | ||
3051 | 45 | |||
3052 | 46 | def get_abspath(option, opt, value, parser): | ||
3053 | 47 | """ Utility function to make sure we have absolute paths | ||
3054 | 48 | if the user supplies values | ||
3055 | 49 | |||
3056 | 50 | """ | ||
3057 | 51 | the_path = os.path.abspath(value) | ||
3058 | 52 | setattr(parser.values, option.dest, the_path) | ||
3059 | 53 | |||
3060 | 54 | def organize_options(args, test_cases): | ||
3061 | 55 | """Put our arguments in a nice dictionary | ||
3062 | 56 | We use option.dest as dictionary key | ||
3063 | 57 | item = supplied input | ||
3064 | 58 | [' | ||
3065 | 59 | """ | ||
3066 | 60 | variables = {} | ||
3067 | 61 | # we make a copy as the python manual on vars | ||
3068 | 62 | # says we shouldn't alter the dictionary returned | ||
3069 | 63 | # by vars() - could affect symbol table? | ||
3070 | 64 | variables = copy.copy(vars(args)) | ||
3071 | 65 | variables['test_cases']= test_cases | ||
3072 | 66 | # This code should become a function once | ||
3073 | 67 | # enough thought has been given to it | ||
3074 | 68 | if variables['manualgdb']: | ||
3075 | 69 | variables['gdb']=True | ||
3076 | 70 | if variables['repeat'] <= 0: | ||
3077 | 71 | print "Setting --repeat=1. You chose a silly value that I will ignore :P" | ||
3078 | 72 | variables['repeat'] = 1 | ||
3079 | 73 | if variables['mode'] == 'randgen' or variables['gendatafile']: | ||
3080 | 74 | print "Setting --no-secure-file-priv=True for randgen usage..." | ||
3081 | 75 | variables['nosecurefilepriv']=True | ||
3082 | 76 | if variables['mode'] == 'cleanup': | ||
3083 | 77 | print "Setting --start-dirty=True for cleanup mode..." | ||
3084 | 78 | variables['startdirty']=True | ||
3085 | 79 | if variables['libeatmydata'] and os.path.exists(variables['libeatmydatapath']): | ||
3086 | 80 | # We are using libeatmydata vs. shared mem for server speedup | ||
3087 | 81 | print "Using libeatmydata at %s. Setting --no-shm / not using shared memory for testing..." %(variables['libeatmydatapath']) | ||
3088 | 82 | variables['noshm']=True | ||
3089 | 83 | return variables | ||
3090 | 84 | |||
3091 | 85 | def populate_defaults(variables, basedir_default): | ||
3092 | 86 | """ We fill in any default values that need | ||
3093 | 87 | to be put in post-parsing | ||
3094 | 88 | |||
3095 | 89 | """ | ||
3096 | 90 | if not variables['basedir']: | ||
3097 | 91 | # We populate this value with the default now | ||
3098 | 92 | # it allows us to have a default and have user | ||
3099 | 93 | # supplied opts to override them | ||
3100 | 94 | variables['basedir'].append(basedir_default) | ||
3101 | 95 | return variables | ||
3102 | 96 | |||
3103 | 97 | def handle_user_opts(variables, basedir_default, testdir_default, suitepaths_default): | ||
3104 | 98 | """ Some variables are dependent upon default values | ||
3105 | 99 | We do the probably hacky thing of going through | ||
3106 | 100 | and updating them accordingly | ||
3107 | 101 | |||
3108 | 102 | We make the assumption / decision that only | ||
3109 | 103 | the first basedir value supplied should | ||
3110 | 104 | be applicable when searching for tests | ||
3111 | 105 | |||
3112 | 106 | """ | ||
3113 | 107 | master_basedir = os.path.abspath(variables['basedir'][0].split(':type:')[0]) | ||
3114 | 108 | if master_basedir != basedir_default: | ||
3115 | 109 | new_path = os.path.join(master_basedir, 'plugin') | ||
3116 | 110 | search_path = os.path.join(basedir_default,'plugin') | ||
3117 | 111 | tmp = variables['suitepaths'] | ||
3118 | 112 | tmp[tmp.index(search_path)] = new_path | ||
3119 | 113 | variables['suitepaths'] = tmp | ||
3120 | 114 | if variables['testdir'] != testdir_default: | ||
3121 | 115 | new_path = os.path.join(variables['testdir'],'suite') | ||
3122 | 116 | search_path = os.path.join(testdir_default,'suite') | ||
3123 | 117 | tmp = variables['suitepaths'] | ||
3124 | 118 | tmp[tmp.index(search_path)] = new_path | ||
3125 | 119 | variables['suitepaths'] = tmp | ||
3126 | 120 | return variables | ||
3127 | 121 | |||
3128 | 122 | |||
3129 | 123 | # Create the CLI option parser | ||
3130 | 124 | parser= optparse.OptionParser(version='%prog (database quality platform aka project steve austin) version 0.1.1') | ||
3131 | 125 | |||
3132 | 126 | # set some default values | ||
3133 | 127 | testdir_default = os.path.abspath(os.getcwd()) | ||
3134 | 128 | workdir_default = os.path.join(testdir_default,'workdir') | ||
3135 | 129 | clientbindir_default = os.path.abspath(os.path.join(testdir_default,'../client')) | ||
3136 | 130 | basedir_default = os.path.dirname(testdir_default) | ||
3137 | 131 | server_type_default = 'galera' | ||
3138 | 132 | valgrind_suppression_default = os.path.join(testdir_default,'valgrind.supp') | ||
3139 | 133 | suitepaths_default = [ os.path.join(basedir_default,'plugin') | ||
3140 | 134 | , os.path.join(testdir_default,'suite') | ||
3141 | 135 | ] | ||
3142 | 136 | randgen_path_default = os.path.join(testdir_default,'randgen') | ||
3143 | 137 | subunit_file_default = os.path.join(workdir_default,'test_results.subunit') | ||
3144 | 138 | |||
3145 | 139 | |||
3146 | 140 | config_control_group = optparse.OptionGroup(parser, | ||
3147 | 141 | "Configuration controls - allows you to specify a file with a number of options already specified") | ||
3148 | 142 | config_control_group.add_option( | ||
3149 | 143 | "--sys_config_file" | ||
3150 | 144 | , dest="sysconfigfilepath" | ||
3151 | 145 | , action='store' | ||
3152 | 146 | , default=None # We want to have a file that will be our default defaults file... | ||
3153 | 147 | , help="The file that specifies system configuration specs for dbqp to execute tests (not yet implemented)" | ||
3154 | 148 | ) | ||
3155 | 149 | parser.add_option_group(config_control_group) | ||
3156 | 150 | |||
3157 | 151 | |||
3158 | 152 | system_control_group = optparse.OptionGroup(parser, | ||
3159 | 153 | "Options for the test-runner itself - defining the system under test and how to execute tests") | ||
3160 | 154 | |||
3161 | 155 | system_control_group.add_option( | ||
3162 | 156 | "--force" | ||
3163 | 157 | , dest="force" | ||
3164 | 158 | , action="store_true" | ||
3165 | 159 | , default=False | ||
3166 | 160 | , help="Set this to continue test execution beyond the first failed test" | ||
3167 | 161 | ) | ||
3168 | 162 | |||
3169 | 163 | system_control_group.add_option( | ||
3170 | 164 | "--start-and-exit" | ||
3171 | 165 | , dest="startandexit" | ||
3172 | 166 | , action="store_true" | ||
3173 | 167 | , default=False | ||
3174 | 168 | , help="Spin up the server(s) for the first specified test then exit (will leave servers running)" | ||
3175 | 169 | ) | ||
3176 | 170 | |||
3177 | 171 | system_control_group.add_option( | ||
3178 | 172 | "--verbose" | ||
3179 | 173 | , dest="verbose" | ||
3180 | 174 | , action="store_true" | ||
3181 | 175 | , default = False | ||
3182 | 176 | , help="Produces extensive output about test-runner state. Distinct from --debug" | ||
3183 | 177 | ) | ||
3184 | 178 | |||
3185 | 179 | system_control_group.add_option( | ||
3186 | 180 | "--debug" | ||
3187 | 181 | , dest="debug" | ||
3188 | 182 | , action="store_true" | ||
3189 | 183 | , default = False | ||
3190 | 184 | , help="Provide internal-level debugging output. Distinct from --verbose" | ||
3191 | 185 | ) | ||
3192 | 186 | |||
3193 | 187 | system_control_group.add_option( | ||
3194 | 188 | "--mode" | ||
3195 | 189 | , dest="mode" | ||
3196 | 190 | , default="native" | ||
3197 | 191 | , help="Testing mode. We currently support dtr, randgen, sysbench, sqlbench, crashme and cleanup modes. See docs for further details about individual modes [%default]" | ||
3198 | 192 | ) | ||
3199 | 193 | |||
3200 | 194 | system_control_group.add_option( | ||
3201 | 195 | "--record" | ||
3202 | 196 | , dest="record" | ||
3203 | 197 | , action="store_true" | ||
3204 | 198 | , default=False | ||
3205 | 199 | , help="Record a testcase result (if the testing mode supports it) [%default]" | ||
3206 | 200 | ) | ||
3207 | 201 | |||
3208 | 202 | system_control_group.add_option( | ||
3209 | 203 | "--fast" | ||
3210 | 204 | , dest="fast" | ||
3211 | 205 | , action="store_true" | ||
3212 | 206 | , default=False | ||
3213 | 207 | , help="Don't try to cleanup from earlier runs (currently just a placeholder) [%default]" | ||
3214 | 208 | ) | ||
3215 | 209 | |||
3216 | 210 | parser.add_option_group(system_control_group) | ||
3217 | 211 | |||
3218 | 212 | test_control_group = optparse.OptionGroup(parser, | ||
3219 | 213 | "Options for controlling which tests are executed") | ||
3220 | 214 | |||
3221 | 215 | test_control_group.add_option( | ||
3222 | 216 | "--suite" | ||
3223 | 217 | , dest="suitelist" | ||
3224 | 218 | , type='string' | ||
3225 | 219 | , action="callback" | ||
3226 | 220 | , callback=comma_list_split | ||
3227 | 221 | , help="The name of the suite containing tests we want. Can accept comma-separated list (with no spaces). Additional --suite args are appended to existing list [autosearch]" | ||
3228 | 222 | ) | ||
3229 | 223 | |||
3230 | 224 | test_control_group.add_option( | ||
3231 | 225 | "--suitepath" | ||
3232 | 226 | , dest="suitepaths" | ||
3233 | 227 | , type='string' | ||
3234 | 228 | , action="append" | ||
3235 | 229 | , default = suitepaths_default | ||
3236 | 230 | , help="The path containing the suite(s) you wish to execute. Use one --suitepath for each suite you want to use. [%default]" | ||
3237 | 231 | ) | ||
3238 | 232 | |||
3239 | 233 | test_control_group.add_option( | ||
3240 | 234 | "--do-test" | ||
3241 | 235 | , dest="dotest" | ||
3242 | 236 | , type='string' | ||
3243 | 237 | , default = None | ||
3244 | 238 | , help="input can either be a prefix or a regex. Will only execute tests that match the provided pattern" | ||
3245 | 239 | ) | ||
3246 | 240 | |||
3247 | 241 | test_control_group.add_option( | ||
3248 | 242 | "--skip-test" | ||
3249 | 243 | , dest="skiptest" | ||
3250 | 244 | , type='string' | ||
3251 | 245 | , default = None | ||
3252 | 246 | , help = "input can either be a prefix or a regex. Will exclude tests that match the provided pattern" | ||
3253 | 247 | ) | ||
3254 | 248 | |||
3255 | 249 | test_control_group.add_option( | ||
3256 | 250 | "--reorder" | ||
3257 | 251 | , dest="reorder" | ||
3258 | 252 | , action="store_true" | ||
3259 | 253 | , default=False | ||
3260 | 254 | , help = "sort the testcases so that they are executed optimally for the given mode [%default]" | ||
3261 | 255 | ) | ||
3262 | 256 | |||
3263 | 257 | test_control_group.add_option( | ||
3264 | 258 | "--repeat" | ||
3265 | 259 | , dest="repeat" | ||
3266 | 260 | , type='int' | ||
3267 | 261 | , action="store" | ||
3268 | 262 | , default=1 | ||
3269 | 263 | , help = "Run each test case the specified number of times. For a given sequence, the first test will be run n times, then the second, etc [%default]" | ||
3270 | 264 | ) | ||
3271 | 265 | |||
3272 | 266 | parser.add_option_group(test_control_group) | ||
3273 | 267 | |||
3274 | 268 | # test subject control group | ||
3275 | 269 | # terrible name for options tht define the server / code | ||
3276 | 270 | # that is under test | ||
3277 | 271 | |||
3278 | 272 | # find some default values | ||
3279 | 273 | # assume we are in-tree testing in general and operating from root/test(?) | ||
3280 | 274 | testdir_default = os.path.abspath(os.getcwd()) | ||
3281 | 275 | |||
3282 | 276 | basedir_default = os.path.split(testdir_default)[0] | ||
3283 | 277 | |||
3284 | 278 | test_subject_control_group = optparse.OptionGroup(parser, | ||
3285 | 279 | "Options for defining the code that will be under test") | ||
3286 | 280 | |||
3287 | 281 | test_subject_control_group.add_option( | ||
3288 | 282 | "--basedir" | ||
3289 | 283 | , dest="basedir" | ||
3290 | 284 | , type='string' | ||
3291 | 285 | , default = [] | ||
3292 | 286 | , action="append" | ||
3293 | 287 | , help = "Pass this argument to signal to the test-runner that this is an in-tree test. We automatically set a number of variables relative to the argument (client-bindir, serverdir, testdir) [%basedir_default]" | ||
3294 | 288 | ) | ||
3295 | 289 | |||
3296 | 290 | test_subject_control_group.add_option( | ||
3297 | 291 | "--default-server-type" | ||
3298 | 292 | , dest="defaultservertype" | ||
3299 | 293 | , type='string' | ||
3300 | 294 | , default = server_type_default | ||
3301 | 295 | , action='store' | ||
3302 | 296 | , help = "Defines what we consider to be the default server type. We assume a server is default type unless specified otherwise. [%default]" | ||
3303 | 297 | ) | ||
3304 | 298 | |||
3305 | 299 | test_subject_control_group.add_option( | ||
3306 | 300 | "--serverdir" | ||
3307 | 301 | , dest="serverpath" | ||
3308 | 302 | , type='string' | ||
3309 | 303 | , action="callback" | ||
3310 | 304 | , callback=get_abspath | ||
3311 | 305 | , help = "Path to the server executable. [%default]" | ||
3312 | 306 | ) | ||
3313 | 307 | |||
3314 | 308 | test_subject_control_group.add_option( | ||
3315 | 309 | "--client-bindir" | ||
3316 | 310 | , dest="clientbindir" | ||
3317 | 311 | , type = 'string' | ||
3318 | 312 | , action="callback" | ||
3319 | 313 | , callback=get_abspath | ||
3320 | 314 | , help = "Path to the directory containing client program binaries for use in testing [%default]" | ||
3321 | 315 | ) | ||
3322 | 316 | |||
3323 | 317 | |||
3324 | 318 | test_subject_control_group.add_option( | ||
3325 | 319 | "--default-storage-engine" | ||
3326 | 320 | , dest="defaultengine" | ||
3327 | 321 | , default = 'innodb' | ||
3328 | 322 | , help="Start drizzled using the specified engine [%default]" | ||
3329 | 323 | ) | ||
3330 | 324 | |||
3331 | 325 | |||
3332 | 326 | parser.add_option_group(test_subject_control_group) | ||
3333 | 327 | # end test subject control group | ||
3334 | 328 | |||
3335 | 329 | # environment options | ||
3336 | 330 | |||
3337 | 331 | environment_control_group = optparse.OptionGroup(parser, | ||
3338 | 332 | "Options for defining the testing environment") | ||
3339 | 333 | |||
3340 | 334 | environment_control_group.add_option( | ||
3341 | 335 | "--testdir" | ||
3342 | 336 | , dest="testdir" | ||
3343 | 337 | , type = 'string' | ||
3344 | 338 | , default = testdir_default | ||
3345 | 339 | , action="callback" | ||
3346 | 340 | , callback=get_abspath | ||
3347 | 341 | , help = "Path to the test dir, containing additional files for test execution. [%default]" | ||
3348 | 342 | ) | ||
3349 | 343 | |||
3350 | 344 | environment_control_group.add_option( | ||
3351 | 345 | "--workdir" | ||
3352 | 346 | , dest="workdir" | ||
3353 | 347 | , type='string' | ||
3354 | 348 | , default = workdir_default | ||
3355 | 349 | , action="callback" | ||
3356 | 350 | , callback=get_abspath | ||
3357 | 351 | , help = "Path to the directory test-run will use to store generated files and directories. [%default]" | ||
3358 | 352 | ) | ||
3359 | 353 | |||
3360 | 354 | environment_control_group.add_option( | ||
3361 | 355 | "--top-srcdir" | ||
3362 | 356 | , dest="topsrcdir" | ||
3363 | 357 | , type='string' | ||
3364 | 358 | , default = basedir_default | ||
3365 | 359 | , help = "build option [%default]" | ||
3366 | 360 | ) | ||
3367 | 361 | |||
3368 | 362 | environment_control_group.add_option( | ||
3369 | 363 | "--top-builddir" | ||
3370 | 364 | , dest="topbuilddir" | ||
3371 | 365 | , type='string' | ||
3372 | 366 | , default = basedir_default | ||
3373 | 367 | , help = "build option [%default]" | ||
3374 | 368 | ) | ||
3375 | 369 | |||
3376 | 370 | environment_control_group.add_option( | ||
3377 | 371 | "--no-shm" | ||
3378 | 372 | , dest="noshm" | ||
3379 | 373 | , action='store_true' | ||
3380 | 374 | , default=False | ||
3381 | 375 | , help = "By default, we symlink workdir to a location in shm. Use this flag to not symlink [%default]" | ||
3382 | 376 | ) | ||
3383 | 377 | |||
3384 | 378 | environment_control_group.add_option( | ||
3385 | 379 | "--libeatmydata" | ||
3386 | 380 | , dest="libeatmydata" | ||
3387 | 381 | , action='store_true' | ||
3388 | 382 | , default=False | ||
3389 | 383 | , help = "We use libeatmydata (if available) to disable fsyncs and speed up test execution. Implies --no-shm" | ||
3390 | 384 | ) | ||
3391 | 385 | |||
3392 | 386 | environment_control_group.add_option( | ||
3393 | 387 | "--libeatmydata-path" | ||
3394 | 388 | , dest="libeatmydatapath" | ||
3395 | 389 | , action='store' | ||
3396 | 390 | , default='/usr/local/lib/libeatmydata.so' | ||
3397 | 391 | , help = "Path to the libeatmydata install you want to use [%default]" | ||
3398 | 392 | ) | ||
3399 | 393 | |||
3400 | 394 | environment_control_group.add_option( | ||
3401 | 395 | "--start-dirty" | ||
3402 | 396 | , dest="startdirty" | ||
3403 | 397 | , action='store_true' | ||
3404 | 398 | , default=False | ||
3405 | 399 | , help = "Don't try to clean up working directories before test execution [%default]" | ||
3406 | 400 | ) | ||
3407 | 401 | |||
3408 | 402 | environment_control_group.add_option( | ||
3409 | 403 | "--no-secure-file-priv" | ||
3410 | 404 | , dest = "nosecurefilepriv" | ||
3411 | 405 | , action='store_true' | ||
3412 | 406 | , default=False | ||
3413 | 407 | , help = "Turn off the use of --secure-file-priv=vardir for started servers" | ||
3414 | 408 | ) | ||
3415 | 409 | |||
3416 | 410 | environment_control_group.add_option( | ||
3417 | 411 | "--randgen-path" | ||
3418 | 412 | , dest="randgenpath" | ||
3419 | 413 | , action='store' | ||
3420 | 414 | , default=randgen_path_default | ||
3421 | 415 | , help = "The path to a randgen installation that can be used to execute randgen-based tests" | ||
3422 | 416 | ) | ||
3423 | 417 | |||
3424 | 418 | environment_control_group.add_option( | ||
3425 | 419 | "--innobackupex-path" | ||
3426 | 420 | , dest="innobackupexpath" | ||
3427 | 421 | , action='store' | ||
3428 | 422 | , default=None | ||
3429 | 423 | , help = "The path to the innobackupex script that facilitates the use of Xtrabackup" | ||
3430 | 424 | ) | ||
3431 | 425 | |||
3432 | 426 | environment_control_group.add_option( | ||
3433 | 427 | "--xtrabackup-path" | ||
3434 | 428 | , dest="xtrabackuppath" | ||
3435 | 429 | , action='store' | ||
3436 | 430 | , default=None | ||
3437 | 431 | , help = "The path the xtrabackup binary to be tested" | ||
3438 | 432 | ) | ||
3439 | 433 | |||
3440 | 434 | environment_control_group.add_option( | ||
3441 | 435 | "--wsrep-provider-path" | ||
3442 | 436 | , dest="wsrepprovider" | ||
3443 | 437 | , action='store' | ||
3444 | 438 | , default='/usr/lib/galera/libgalera_smm.so' | ||
3445 | 439 | , help = "The path to a wsrep provider library for use with mysql" | ||
3446 | 440 | ) | ||
3447 | 441 | |||
3448 | 442 | environment_control_group.add_option( | ||
3449 | 443 | "--cluster-cnf" | ||
3450 | 444 | , dest="clustercnf" | ||
3451 | 445 | , action='store' | ||
3452 | 446 | , default=None | ||
3453 | 447 | , help = "The path to a config file defining a running cluster (node info)" | ||
3454 | 448 | ) | ||
3455 | 449 | |||
3456 | 450 | environment_control_group.add_option( | ||
3457 | 451 | "--subunit-outfile" | ||
3458 | 452 | , dest="subunitoutfile" | ||
3459 | 453 | , action='store' | ||
3460 | 454 | , default=subunit_file_default | ||
3461 | 455 | , help = "File path where subunit output will be logged [%default]" | ||
3462 | 456 | ) | ||
3463 | 457 | |||
3464 | 458 | parser.add_option_group(environment_control_group) | ||
3465 | 459 | # end environment control group | ||
3466 | 460 | |||
3467 | 461 | option_passing_group = optparse.OptionGroup(parser, | ||
3468 | 462 | "Options to pass options on to the server") | ||
3469 | 463 | |||
3470 | 464 | option_passing_group.add_option( | ||
3471 | 465 | "--drizzled" | ||
3472 | 466 | , dest="drizzledoptions" | ||
3473 | 467 | , type='string' | ||
3474 | 468 | , action='append' | ||
3475 | 469 | , default = [] | ||
3476 | 470 | , help = "Pass additional options to the server. Will be passed to all servers for all tests (mostly for --start-and-exit)" | ||
3477 | 471 | ) | ||
3478 | 472 | |||
3479 | 473 | parser.add_option_group(option_passing_group) | ||
3480 | 474 | # end option passing group | ||
3481 | 475 | |||
3482 | 476 | analysis_control_group = optparse.OptionGroup(parser, | ||
3483 | 477 | "Options for defining the tools we use for code analysis (valgrind, gprof, gcov, etc)") | ||
3484 | 478 | |||
3485 | 479 | analysis_control_group.add_option( | ||
3486 | 480 | "--valgrind" | ||
3487 | 481 | , dest="valgrind" | ||
3488 | 482 | , action='store_true' | ||
3489 | 483 | , default = False | ||
3490 | 484 | , help = "Run drizzletest and drizzled executables using valgrind with default options [%default]" | ||
3491 | 485 | ) | ||
3492 | 486 | |||
3493 | 487 | analysis_control_group.add_option( | ||
3494 | 488 | "--valgrind-option" | ||
3495 | 489 | , dest="valgrindarglist" | ||
3496 | 490 | , type='string' | ||
3497 | 491 | , action="append" | ||
3498 | 492 | , help = "Pass an option to valgrind (overrides/removes default valgrind options)" | ||
3499 | 493 | ) | ||
3500 | 494 | |||
3501 | 495 | analysis_control_group.add_option( | ||
3502 | 496 | "--valgrind-suppressions" | ||
3503 | 497 | , dest="valgrindsuppressions" | ||
3504 | 498 | , type='string' | ||
3505 | 499 | , action='store' | ||
3506 | 500 | , default = valgrind_suppression_default | ||
3507 | 501 | , help = "Point at a valgrind suppression file [%default]" | ||
3508 | 502 | ) | ||
3509 | 503 | |||
3510 | 504 | analysis_control_group.add_option( | ||
3511 | 505 | "--helgrind" | ||
3512 | 506 | , dest="helgrind" | ||
3513 | 507 | , action='store_true' | ||
3514 | 508 | , default=False | ||
3515 | 509 | , help="Use the helgrind tool for valgrind. Implies / will auto-use --valgrind" | ||
3516 | 510 | ) | ||
3517 | 511 | |||
3518 | 512 | parser.add_option_group(analysis_control_group) | ||
3519 | 513 | |||
3520 | 514 | debugger_control_group = optparse.OptionGroup(parser, | ||
3521 | 515 | "Options for controlling the use of debuggers with test execution") | ||
3522 | 516 | |||
3523 | 517 | debugger_control_group.add_option( | ||
3524 | 518 | "--gdb" | ||
3525 | 519 | , dest="gdb" | ||
3526 | 520 | , action='store_true' | ||
3527 | 521 | , default=False | ||
3528 | 522 | , help="Start the drizzled server(s) in gdb" | ||
3529 | 523 | ) | ||
3530 | 524 | |||
3531 | 525 | debugger_control_group.add_option( | ||
3532 | 526 | "--manual-gdb" | ||
3533 | 527 | , dest="manualgdb" | ||
3534 | 528 | , action='store_true' | ||
3535 | 529 | , default=False | ||
3536 | 530 | , help="Allows you to start the drizzled server(s) in gdb manually (in another window, etc)" | ||
3537 | 531 | ) | ||
3538 | 532 | |||
3539 | 533 | parser.add_option_group(debugger_control_group) | ||
3540 | 534 | |||
3541 | 535 | utility_group = optparse.OptionGroup(parser, | ||
3542 | 536 | "Options to call additional utilities such as datagen") | ||
3543 | 537 | |||
3544 | 538 | utility_group.add_option( | ||
3545 | 539 | "--gendata" | ||
3546 | 540 | , dest="gendatafile" | ||
3547 | 541 | , action='store' | ||
3548 | 542 | , type='string' | ||
3549 | 543 | , default=None | ||
3550 | 544 | , help="Call the randgen's gendata utility to use the specified configuration file. This will populate the server prior to any test execution") | ||
3551 | 545 | |||
3552 | 546 | parser.add_option_group(utility_group) | ||
3553 | 547 | |||
3554 | 548 | # supplied will be those arguments matching an option, | ||
3555 | 549 | # and test_cases will be everything else | ||
3556 | 550 | (args, test_cases)= parser.parse_args() | ||
3557 | 551 | |||
3558 | 552 | variables = {} | ||
3559 | 553 | variables = organize_options(args, test_cases) | ||
3560 | 554 | variables = populate_defaults(variables, basedir_default) | ||
3561 | 555 | variables = handle_user_opts(variables, basedir_default, testdir_default, suitepaths_default) | ||
3562 | 556 | |||
3563 | 557 | 0 | ||
3564 | === removed directory 'dbqp/lib/server_mgmt' | |||
3565 | === removed file 'dbqp/lib/server_mgmt/__init__.py' | |||
3566 | === removed file 'dbqp/lib/server_mgmt/drizzled.py' | |||
3567 | --- dbqp/lib/server_mgmt/drizzled.py 2012-02-04 01:22:23 +0000 | |||
3568 | +++ dbqp/lib/server_mgmt/drizzled.py 1970-01-01 00:00:00 +0000 | |||
3569 | @@ -1,227 +0,0 @@ | |||
3570 | 1 | #! /usr/bin/env python | ||
3571 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
3572 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
3573 | 4 | # | ||
3574 | 5 | # Copyright (C) 2010,2011 Patrick Crews | ||
3575 | 6 | # | ||
3576 | 7 | # This program is free software; you can redistribute it and/or modify | ||
3577 | 8 | # it under the terms of the GNU General Public License as published by | ||
3578 | 9 | # the Free Software Foundation; either version 2 of the License, or | ||
3579 | 10 | # (at your option) any later version. | ||
3580 | 11 | # | ||
3581 | 12 | # This program is distributed in the hope that it will be useful, | ||
3582 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
3583 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
3584 | 15 | # GNU General Public License for more details. | ||
3585 | 16 | # | ||
3586 | 17 | # You should have received a copy of the GNU General Public License | ||
3587 | 18 | # along with this program; if not, write to the Free Software | ||
3588 | 19 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
3589 | 20 | |||
3590 | 21 | |||
3591 | 22 | """ drizzled.py: code to allow a serverManager | ||
3592 | 23 | to provision and start up a drizzled server object | ||
3593 | 24 | for test execution | ||
3594 | 25 | |||
3595 | 26 | """ | ||
3596 | 27 | |||
3597 | 28 | # imports | ||
3598 | 29 | import os | ||
3599 | 30 | from lib.server_mgmt.server import Server | ||
3600 | 31 | |||
3601 | 32 | class drizzleServer(Server): | ||
3602 | 33 | """ represents a drizzle server, its possessions | ||
3603 | 34 | (datadir, ports, etc), and methods for controlling | ||
3604 | 35 | and querying it | ||
3605 | 36 | |||
3606 | 37 | TODO: create a base server class that contains | ||
3607 | 38 | standard methods from which we can inherit | ||
3608 | 39 | Currently there are definitely methods / attr | ||
3609 | 40 | which are general | ||
3610 | 41 | |||
3611 | 42 | """ | ||
3612 | 43 | |||
3613 | 44 | def __init__( self, name, server_manager, code_tree, default_storage_engine | ||
3614 | 45 | , server_options, requester, workdir_root): | ||
3615 | 46 | super(drizzleServer, self).__init__( name | ||
3616 | 47 | , server_manager | ||
3617 | 48 | , code_tree | ||
3618 | 49 | , default_storage_engine | ||
3619 | 50 | , server_options | ||
3620 | 51 | , requester | ||
3621 | 52 | , workdir_root) | ||
3622 | 53 | self.preferred_base_port = 9306 | ||
3623 | 54 | |||
3624 | 55 | # client files | ||
3625 | 56 | self.drizzledump = self.code_tree.drizzledump | ||
3626 | 57 | self.drizzle_client = self.code_tree.drizzle_client | ||
3627 | 58 | self.drizzleimport = self.code_tree.drizzleimport | ||
3628 | 59 | self.drizzleslap = self.code_tree.drizzleslap | ||
3629 | 60 | self.server_path = self.code_tree.drizzle_server | ||
3630 | 61 | self.drizzle_client_path = self.code_tree.drizzle_client | ||
3631 | 62 | self.schemawriter = self.code_tree.schemawriter | ||
3632 | 63 | self.trx_reader = self.code_tree.trx_reader | ||
3633 | 64 | |||
3634 | 65 | # Get our ports | ||
3635 | 66 | self.port_block = self.system_manager.port_manager.get_port_block( self.name | ||
3636 | 67 | , self.preferred_base_port | ||
3637 | 68 | , 6 ) | ||
3638 | 69 | self.master_port = self.port_block[0] | ||
3639 | 70 | self.drizzle_tcp_port = self.port_block[1] | ||
3640 | 71 | self.mc_port = self.port_block[2] | ||
3641 | 72 | self.pbms_port = self.port_block[3] | ||
3642 | 73 | self.rabbitmq_node_port = self.port_block[4] | ||
3643 | 74 | self.json_server_port = self.port_block[5] | ||
3644 | 75 | |||
3645 | 76 | # Generate our working directories | ||
3646 | 77 | self.dirset = {'var_%s' %(self.name): {'std_data_ln':( os.path.join(self.code_tree.testdir,'std_data')) | ||
3647 | 78 | ,'log':None | ||
3648 | 79 | ,'run':None | ||
3649 | 80 | ,'tmp':None | ||
3650 | 81 | ,'master-data': {'local': { 'test':None | ||
3651 | 82 | , 'mysql':None | ||
3652 | 83 | } | ||
3653 | 84 | } | ||
3654 | 85 | } | ||
3655 | 86 | } | ||
3656 | 87 | self.workdir = self.system_manager.create_dirset( workdir_root | ||
3657 | 88 | , self.dirset) | ||
3658 | 89 | self.vardir = self.workdir | ||
3659 | 90 | self.tmpdir = os.path.join(self.vardir,'tmp') | ||
3660 | 91 | self.rundir = os.path.join(self.vardir,'run') | ||
3661 | 92 | self.logdir = os.path.join(self.vardir,'log') | ||
3662 | 93 | self.datadir = os.path.join(self.vardir,'master-data') | ||
3663 | 94 | |||
3664 | 95 | self.error_log = os.path.join(self.logdir,('%s.err' %(self.name))) | ||
3665 | 96 | self.pid_file = os.path.join(self.rundir,('%s.pid' %(self.name))) | ||
3666 | 97 | self.socket_file = os.path.join(self.vardir, ('%s.sock' %(self.name))) | ||
3667 | 98 | self.timer_file = os.path.join(self.logdir,('timer')) | ||
3668 | 99 | |||
3669 | 100 | # Do magic to create a config file for use with the slave | ||
3670 | 101 | # plugin | ||
3671 | 102 | self.slave_config_file = os.path.join(self.logdir,'slave.cnf') | ||
3672 | 103 | self.create_slave_config_file() | ||
3673 | 104 | |||
3674 | 105 | self.snapshot_path = os.path.join(self.tmpdir,('snapshot_%s' %(self.master_port))) | ||
3675 | 106 | # We want to use --secure-file-priv = $vardir by default | ||
3676 | 107 | # but there are times / tools when we need to shut this off | ||
3677 | 108 | if self.no_secure_file_priv: | ||
3678 | 109 | self.secure_file_string = '' | ||
3679 | 110 | else: | ||
3680 | 111 | self.secure_file_string = "--secure-file-priv='%s'" %(self.vardir) | ||
3681 | 112 | self.user_string = '--user=root' | ||
3682 | 113 | |||
3683 | 114 | self.initialize_databases() | ||
3684 | 115 | self.take_db_snapshot() | ||
3685 | 116 | |||
3686 | 117 | self.logging.debug_class(self) | ||
3687 | 118 | |||
3688 | 119 | def report(self): | ||
3689 | 120 | """ We print out some general useful info """ | ||
3690 | 121 | report_values = [ 'name' | ||
3691 | 122 | , 'master_port' | ||
3692 | 123 | , 'drizzle_tcp_port' | ||
3693 | 124 | , 'mc_port' | ||
3694 | 125 | , 'pbms_port' | ||
3695 | 126 | , 'rabbitmq_node_port' | ||
3696 | 127 | , 'vardir' | ||
3697 | 128 | , 'status' | ||
3698 | 129 | ] | ||
3699 | 130 | self.logging.info("%s server:" %(self.owner)) | ||
3700 | 131 | for key in report_values: | ||
3701 | 132 | value = vars(self)[key] | ||
3702 | 133 | self.logging.info("%s: %s" %(key.upper(), value)) | ||
3703 | 134 | |||
3704 | 135 | def get_start_cmd(self): | ||
3705 | 136 | """ Return the command string that will start up the server | ||
3706 | 137 | as desired / intended | ||
3707 | 138 | |||
3708 | 139 | """ | ||
3709 | 140 | |||
3710 | 141 | server_args = [ self.process_server_options() | ||
3711 | 142 | , "--mysql-protocol.port=%d" %(self.master_port) | ||
3712 | 143 | , "--mysql-protocol.connect-timeout=60" | ||
3713 | 144 | , "--innodb.data-file-path=ibdata1:20M:autoextend" | ||
3714 | 145 | , "--sort-buffer-size=256K" | ||
3715 | 146 | , "--max-heap-table-size=1M" | ||
3716 | 147 | , "--mysql-unix-socket-protocol.path=%s" %(self.socket_file) | ||
3717 | 148 | , "--pid-file=%s" %(self.pid_file) | ||
3718 | 149 | , "--drizzle-protocol.port=%d" %(self.drizzle_tcp_port) | ||
3719 | 150 | , "--default-storage-engine=%s" %(self.default_storage_engine) | ||
3720 | 151 | , "--datadir=%s" %(self.datadir) | ||
3721 | 152 | , "--tmpdir=%s" %(self.tmpdir) | ||
3722 | 153 | , self.secure_file_string | ||
3723 | 154 | , self.user_string | ||
3724 | 155 | ] | ||
3725 | 156 | |||
3726 | 157 | if self.gdb: | ||
3727 | 158 | server_args.append('--gdb') | ||
3728 | 159 | return self.system_manager.handle_gdb_reqs(self, server_args) | ||
3729 | 160 | else: | ||
3730 | 161 | return "%s %s %s & " % ( self.cmd_prefix | ||
3731 | 162 | , self.server_path | ||
3732 | 163 | , " ".join(server_args) | ||
3733 | 164 | ) | ||
3734 | 165 | |||
3735 | 166 | |||
3736 | 167 | def get_stop_cmd(self): | ||
3737 | 168 | """ Return the command that will shut us down """ | ||
3738 | 169 | |||
3739 | 170 | return "%s --user=root --port=%d --connect-timeout=5 --silent --password= --shutdown " %(self.drizzle_client_path, self.master_port) | ||
3740 | 171 | |||
3741 | 172 | |||
3742 | 173 | def get_ping_cmd(self): | ||
3743 | 174 | """Return the command string that will | ||
3744 | 175 | ping / check if the server is alive | ||
3745 | 176 | |||
3746 | 177 | """ | ||
3747 | 178 | |||
3748 | 179 | return "%s --ping --port=%d --user=root" % (self.drizzle_client_path, self.master_port) | ||
3749 | 180 | |||
3750 | 181 | def is_started(self): | ||
3751 | 182 | """ Determine if the server is up and running - | ||
3752 | 183 | this may vary from server type to server type | ||
3753 | 184 | |||
3754 | 185 | """ | ||
3755 | 186 | |||
3756 | 187 | # We experiment with waiting for a pid file to be created vs. pinging | ||
3757 | 188 | # This is what test-run.pl does and it helps us pass logging_stats tests | ||
3758 | 189 | # while not self.ping_server(server, quiet=True) and timer != timeout: | ||
3759 | 190 | |||
3760 | 191 | return self.system_manager.find_path( [self.pid_file] | ||
3761 | 192 | , required=0) | ||
3762 | 193 | |||
3763 | 194 | def create_slave_config_file(self): | ||
3764 | 195 | """ Create a config file suitable for use | ||
3765 | 196 | with the slave-plugin. This allows | ||
3766 | 197 | us to tie other servers in easily | ||
3767 | 198 | |||
3768 | 199 | """ | ||
3769 | 200 | |||
3770 | 201 | config_data = [ "[master1]" | ||
3771 | 202 | , "master-host=127.0.0.1" | ||
3772 | 203 | , "master-port=%d" %self.master_port | ||
3773 | 204 | , "master-user=root" | ||
3774 | 205 | , "master-pass=''" | ||
3775 | 206 | , "max-reconnects=100" | ||
3776 | 207 | #, "seconds-between-reconnects=20" | ||
3777 | 208 | ] | ||
3778 | 209 | outfile = open(self.slave_config_file,'w') | ||
3779 | 210 | for line in config_data: | ||
3780 | 211 | outfile.write("%s\n" %(line)) | ||
3781 | 212 | outfile.close() | ||
3782 | 213 | |||
3783 | 214 | |||
3784 | 215 | |||
3785 | 216 | |||
3786 | 217 | |||
3787 | 218 | |||
3788 | 219 | |||
3789 | 220 | |||
3790 | 221 | |||
3791 | 222 | |||
3792 | 223 | |||
3793 | 224 | |||
3794 | 225 | |||
3795 | 226 | |||
3796 | 227 | |||
3797 | 228 | 0 | ||
3798 | === removed file 'dbqp/lib/server_mgmt/galera.py' | |||
3799 | --- dbqp/lib/server_mgmt/galera.py 2012-02-04 01:22:23 +0000 | |||
3800 | +++ dbqp/lib/server_mgmt/galera.py 1970-01-01 00:00:00 +0000 | |||
3801 | @@ -1,334 +0,0 @@ | |||
3802 | 1 | #! /usr/bin/env python | ||
3803 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
3804 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
3805 | 4 | # | ||
3806 | 5 | # Copyright (C) 2010,2011 Patrick Crews | ||
3807 | 6 | # | ||
3808 | 7 | # This program is free software; you can redistribute it and/or modify | ||
3809 | 8 | # it under the terms of the GNU General Public License as published by | ||
3810 | 9 | # the Free Software Foundation; either version 2 of the License, or | ||
3811 | 10 | # (at your option) any later version. | ||
3812 | 11 | # | ||
3813 | 12 | # This program is distributed in the hope that it will be useful, | ||
3814 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
3815 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
3816 | 15 | # GNU General Public License for more details. | ||
3817 | 16 | # | ||
3818 | 17 | # You should have received a copy of the GNU General Public License | ||
3819 | 18 | # along with this program; if not, write to the Free Software | ||
3820 | 19 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
3821 | 20 | |||
3822 | 21 | |||
3823 | 22 | """ mysqld.py: code to allow a serverManager | ||
3824 | 23 | to provision and start up a mysqld server object | ||
3825 | 24 | for test execution | ||
3826 | 25 | |||
3827 | 26 | """ | ||
3828 | 27 | |||
3829 | 28 | # imports | ||
3830 | 29 | import os | ||
3831 | 30 | import sys | ||
3832 | 31 | import time | ||
3833 | 32 | import subprocess | ||
3834 | 33 | |||
3835 | 34 | from ConfigParser import RawConfigParser | ||
3836 | 35 | |||
3837 | 36 | from lib.server_mgmt.server import Server | ||
3838 | 37 | |||
3839 | 38 | class mysqlServer(Server): | ||
3840 | 39 | """ represents a mysql server, its possessions | ||
3841 | 40 | (datadir, ports, etc), and methods for controlling | ||
3842 | 41 | and querying it | ||
3843 | 42 | |||
3844 | 43 | TODO: create a base server class that contains | ||
3845 | 44 | standard methods from which we can inherit | ||
3846 | 45 | Currently there are definitely methods / attr | ||
3847 | 46 | which are general | ||
3848 | 47 | |||
3849 | 48 | """ | ||
3850 | 49 | |||
3851 | 50 | def __init__( self, name, server_manager, code_tree, default_storage_engine | ||
3852 | 51 | , server_options, requester, workdir_root): | ||
3853 | 52 | super(mysqlServer, self).__init__( name | ||
3854 | 53 | , server_manager | ||
3855 | 54 | , code_tree | ||
3856 | 55 | , default_storage_engine | ||
3857 | 56 | , server_options | ||
3858 | 57 | , requester | ||
3859 | 58 | , workdir_root) | ||
3860 | 59 | self.preferred_base_port = 9306 | ||
3861 | 60 | |||
3862 | 61 | # client files | ||
3863 | 62 | self.mysqldump = self.code_tree.mysqldump | ||
3864 | 63 | self.mysqladmin = self.code_tree.mysqladmin | ||
3865 | 64 | self.mysql_client = self.code_tree.mysql_client | ||
3866 | 65 | self.mysqlimport = self.code_tree.mysqlimport | ||
3867 | 66 | self.mysqlslap = self.code_tree.mysqlslap | ||
3868 | 67 | self.mysql_upgrade = self.code_tree.mysql_upgrade | ||
3869 | 68 | self.server_path = self.code_tree.mysql_server | ||
3870 | 69 | self.mysql_client_path = self.code_tree.mysql_client | ||
3871 | 70 | |||
3872 | 71 | # galera-specific info | ||
3873 | 72 | self.wsrep_cluster_addr='' | ||
3874 | 73 | |||
3875 | 74 | # important stuff | ||
3876 | 75 | self.langdir = self.code_tree.langdir | ||
3877 | 76 | self.charsetdir = self.code_tree.charsetdir | ||
3878 | 77 | self.bootstrap_file = self.code_tree.bootstrap_path | ||
3879 | 78 | self.bootstrap_cmd = None | ||
3880 | 79 | |||
3881 | 80 | # Get our ports | ||
3882 | 81 | self.port_block = self.system_manager.port_manager.get_port_block( self.name | ||
3883 | 82 | , self.preferred_base_port | ||
3884 | 83 | , 3 ) | ||
3885 | 84 | self.master_port = self.port_block[0] | ||
3886 | 85 | self.galera_listen_port = self.port_block[1] | ||
3887 | 86 | self.galera_recv_port = self.port_block[2] | ||
3888 | 87 | |||
3889 | 88 | # Generate our working directories | ||
3890 | 89 | self.dirset = { 'var_%s' %(self.name): {'std_data_ln':( os.path.join(self.code_tree.testdir,'std_data')) | ||
3891 | 90 | ,'log':None | ||
3892 | 91 | ,'run':None | ||
3893 | 92 | ,'tmp':None | ||
3894 | 93 | ,'master-data': { 'test':None | ||
3895 | 94 | , 'mysql':None | ||
3896 | 95 | } | ||
3897 | 96 | } | ||
3898 | 97 | } | ||
3899 | 98 | self.workdir = self.system_manager.create_dirset( workdir_root | ||
3900 | 99 | , self.dirset) | ||
3901 | 100 | self.vardir = self.workdir | ||
3902 | 101 | self.tmpdir = os.path.join(self.vardir,'tmp') | ||
3903 | 102 | self.rundir = os.path.join(self.vardir,'run') | ||
3904 | 103 | self.logdir = os.path.join(self.vardir,'log') | ||
3905 | 104 | self.std_data = os.path.join(self.vardir,'std_data_ln') | ||
3906 | 105 | self.datadir = os.path.join(self.vardir,'master-data') | ||
3907 | 106 | |||
3908 | 107 | self.error_log = os.path.join(self.logdir,('error.log' )) | ||
3909 | 108 | self.bootstrap_log = os.path.join(self.logdir,('bootstrap.log')) | ||
3910 | 109 | self.pid_file = os.path.join(self.rundir,('my.pid')) | ||
3911 | 110 | self.socket_file = os.path.join(self.vardir, ('my.sock')) | ||
3912 | 111 | self.timer_file = os.path.join(self.logdir,('timer')) | ||
3913 | 112 | self.general_log_file = os.path.join(self.logdir,'mysqld.log') | ||
3914 | 113 | self.slow_query_log_file = os.path.join(self.logdir,'mysqld-slow.log') | ||
3915 | 114 | self.cnf_file = os.path.join(self.vardir,'my.cnf') | ||
3916 | 115 | |||
3917 | 116 | self.snapshot_path = os.path.join(self.tmpdir,('snapshot_%s' %(self.master_port))) | ||
3918 | 117 | # We want to use --secure-file-priv = $vardir by default | ||
3919 | 118 | # but there are times / tools when we need to shut this off | ||
3920 | 119 | if self.no_secure_file_priv: | ||
3921 | 120 | self.secure_file_string = '' | ||
3922 | 121 | else: | ||
3923 | 122 | self.secure_file_string = "--secure-file-priv='%s'" %(self.vardir) | ||
3924 | 123 | self.user_string = '--user=root' | ||
3925 | 124 | |||
3926 | 125 | self.initialize_databases() | ||
3927 | 126 | self.take_db_snapshot() | ||
3928 | 127 | |||
3929 | 128 | self.logging.debug_class(self) | ||
3930 | 129 | |||
3931 | 130 | |||
3932 | 131 | def report(self): | ||
3933 | 132 | """ We print out some general useful info """ | ||
3934 | 133 | report_values = [ 'name' | ||
3935 | 134 | , 'master_port' | ||
3936 | 135 | , 'galera_listen_port' | ||
3937 | 136 | , 'galera_recv_port' | ||
3938 | 137 | , 'socket_file' | ||
3939 | 138 | , 'vardir' | ||
3940 | 139 | , 'status' | ||
3941 | 140 | ] | ||
3942 | 141 | self.logging.info("%s server:" %(self.owner)) | ||
3943 | 142 | for key in report_values: | ||
3944 | 143 | value = vars(self)[key] | ||
3945 | 144 | self.logging.info("%s: %s" %(key.upper(), value)) | ||
3946 | 145 | |||
3947 | 146 | def initialize_databases(self): | ||
3948 | 147 | """ Do the voodoo required to have a working database setup. | ||
3949 | 148 | For MySQL, this is calling the server with the | ||
3950 | 149 | --bootstrap argument. We generate the bootstrap | ||
3951 | 150 | file during codeTree intialization as the file is standard for | ||
3952 | 151 | all MySQL servers that are spawned from a single codeTree | ||
3953 | 152 | |||
3954 | 153 | """ | ||
3955 | 154 | |||
3956 | 155 | # generate the bootstrap startup command | ||
3957 | 156 | if not self.bootstrap_cmd: | ||
3958 | 157 | mysqld_args = [ "--no-defaults" | ||
3959 | 158 | , "--bootstrap" | ||
3960 | 159 | , "--basedir=%s" %(self.code_tree.basedir) | ||
3961 | 160 | , "--datadir=%s" %(self.datadir) | ||
3962 | 161 | , "--loose-skip-falcon" | ||
3963 | 162 | , "--loose-skip-ndbcluster" | ||
3964 | 163 | , "--tmpdir=%s" %(self.tmpdir) | ||
3965 | 164 | , "--core-file" | ||
3966 | 165 | , "--lc-messages-dir=%s" %(self.langdir) | ||
3967 | 166 | , "--character-sets-dir=%s" %(self.charsetdir) | ||
3968 | 167 | ] | ||
3969 | 168 | # We add server_path into the mix this way as we | ||
3970 | 169 | # may alter how we store / handle server args later | ||
3971 | 170 | mysqld_args.insert(0,self.server_path) | ||
3972 | 171 | self.bootstrap_cmd = " ".join(mysqld_args) | ||
3973 | 172 | # execute our command | ||
3974 | 173 | bootstrap_log = open(self.bootstrap_log,'w') | ||
3975 | 174 | # open our bootstrap file | ||
3976 | 175 | bootstrap_in = open(self.bootstrap_file,'r') | ||
3977 | 176 | bootstrap_subproc = subprocess.Popen( self.bootstrap_cmd | ||
3978 | 177 | , shell=True | ||
3979 | 178 | , stdin=bootstrap_in | ||
3980 | 179 | , stdout=bootstrap_log | ||
3981 | 180 | , stderr=bootstrap_log | ||
3982 | 181 | ) | ||
3983 | 182 | bootstrap_subproc.wait() | ||
3984 | 183 | bootstrap_in.close() | ||
3985 | 184 | bootstrap_log.close() | ||
3986 | 185 | bootstrap_retcode = bootstrap_subproc.returncode | ||
3987 | 186 | if bootstrap_retcode: | ||
3988 | 187 | self.logging.error("Received retcode: %s executing command: %s" | ||
3989 | 188 | %(bootstrap_retcode, self.bootstrap_cmd)) | ||
3990 | 189 | self.logging.error("Check the bootstrap log: %s" %(self.bootstrap_log)) | ||
3991 | 190 | sys.exit(1) | ||
3992 | 191 | |||
3993 | 192 | |||
3994 | 193 | def get_start_cmd(self): | ||
3995 | 194 | """ Return the command string that will start up the server | ||
3996 | 195 | as desired / intended | ||
3997 | 196 | |||
3998 | 197 | """ | ||
3999 | 198 | |||
4000 | 199 | # we need to do some wsrep voodoo for galera rpl to work | ||
4001 | 200 | self.ip_address = self.system_manager.get_ip_address() | ||
4002 | 201 | self.listen_addr_string = "?gmcast.listen_addr=tcp://127.0.0.1:%d" %(self.galera_listen_port) | ||
4003 | 202 | # test for wsrep_provider argument | ||
4004 | 203 | wsrep_provider_string ='' | ||
4005 | 204 | if self.system_manager.wsrep_provider_path: | ||
4006 | 205 | wsrep_provider_string = "--wsrep_provider=%s" %(self.system_manager.wsrep_provider_path) | ||
4007 | 206 | # handle wsrep_cluster_address argument | ||
4008 | 207 | wsrep_cluster_string ="--wsrep_cluster_address='gcomm://%s%s'" %( self.wsrep_cluster_addr | ||
4009 | 208 | , self.listen_addr_string) | ||
4010 | 209 | for server_opt in self.server_options: | ||
4011 | 210 | if server_opt.startswith("--wsrep_cluster_address="): | ||
4012 | 211 | # we have a user-specified value, so we don't | ||
4013 | 212 | # try to insert something | ||
4014 | 213 | wsrep_cluster_string = '' | ||
4015 | 214 | # check to see if they have specified a listen port | ||
4016 | 215 | # we want to use our own, so we have a better idea of | ||
4017 | 216 | # what is going on | ||
4018 | 217 | if '?gmcast.listen_addr' in server_opt: | ||
4019 | 218 | pass | ||
4020 | 219 | else: | ||
4021 | 220 | server_opt = server_opt.strip()+ listen_addr | ||
4022 | 221 | server_args = [ self.process_server_options() | ||
4023 | 222 | , "%s" %(wsrep_cluster_string) | ||
4024 | 223 | , "%s" %(wsrep_provider_string) | ||
4025 | 224 | , "--wsrep_debug=ON" | ||
4026 | 225 | , "--wsrep_provider_options='ist.recv_addr=%s:%d'" %(self.ip_address, self.galera_recv_port) | ||
4027 | 226 | , "--wsrep_sst_receive_address='127.0.0.1:%d'" %( self.master_port) | ||
4028 | 227 | , "--wsrep_sst_auth='root:'" | ||
4029 | 228 | , "--wsrep_node_name='node%d'" %(self.galera_listen_port) | ||
4030 | 229 | , "--bind-address=0.0.0.0" | ||
4031 | 230 | , "--innodb_locks_unsafe_for_binlog=1" | ||
4032 | 231 | , "--open-files-limit=1024" | ||
4033 | 232 | , "--query-cache-size=0" | ||
4034 | 233 | , "--local-infile" | ||
4035 | 234 | , "--character-set-server=latin1" | ||
4036 | 235 | , "--connect-timeout=60" | ||
4037 | 236 | , "--log-bin-trust-function-creators=1" | ||
4038 | 237 | , "--key_buffer_size=1M" | ||
4039 | 238 | , "--sort_buffer=256K" | ||
4040 | 239 | , "--max_heap_table_size=1M" | ||
4041 | 240 | , "--loose-innodb_data_file_path=ibdata1:10M:autoextend" | ||
4042 | 241 | , "--loose-innodb_buffer_pool_size=8M" | ||
4043 | 242 | , "--loose-innodb_write_io_threads=2" | ||
4044 | 243 | , "--loose-innodb_read_io_threads=2" | ||
4045 | 244 | , "--loose-innodb_log_buffer_size=1M" | ||
4046 | 245 | , "--loose-innodb_log_file_size=5M" | ||
4047 | 246 | , "--loose-innodb_additional_mem_pool_size=1M" | ||
4048 | 247 | , "--loose-innodb_log_files_in_group=2" | ||
4049 | 248 | , "--loose-innodb_lock_wait_timeout=9999999" | ||
4050 | 249 | , "--slave-net-timeout=120" | ||
4051 | 250 | , "--log-bin=%s" %(os.path.join(self.logdir,"mysqld-bin")) | ||
4052 | 251 | , "--binlog-direct-non-transactional-updates" | ||
4053 | 252 | , "--general_log=1" | ||
4054 | 253 | , "--general_log_file=%s" %(self.general_log_file) | ||
4055 | 254 | , "--slow_query_log=1" | ||
4056 | 255 | , "--slow_query_log_file=%s" %(self.slow_query_log_file) | ||
4057 | 256 | , "--basedir=%s" %(self.code_tree.basedir) | ||
4058 | 257 | , "--datadir=%s" %(self.datadir) | ||
4059 | 258 | , "--tmpdir=%s" %(self.tmpdir) | ||
4060 | 259 | , "--character-sets-dir=%s" %(self.charsetdir) | ||
4061 | 260 | , "--lc-messages-dir=%s" %(self.langdir) | ||
4062 | 261 | , "--port=%d" %(self.master_port) | ||
4063 | 262 | , "--socket=%s" %(self.socket_file) | ||
4064 | 263 | , "--pid-file=%s" %(self.pid_file) | ||
4065 | 264 | , "--default-storage-engine=%s" %(self.default_storage_engine) | ||
4066 | 265 | # server-id maybe needs fixing, but we want to avoid | ||
4067 | 266 | # the server-id=0 and no replication thing... | ||
4068 | 267 | , "--server-id=%d" %(self.get_numeric_server_id()+1) | ||
4069 | 268 | , self.secure_file_string | ||
4070 | 269 | , self.user_string | ||
4071 | 270 | ] | ||
4072 | 271 | self.gen_cnf_file(server_args) | ||
4073 | 272 | |||
4074 | 273 | if self.gdb: | ||
4075 | 274 | server_args.append('--gdb') | ||
4076 | 275 | return self.system_manager.handle_gdb_reqs(self, server_args) | ||
4077 | 276 | else: | ||
4078 | 277 | return "%s %s %s & " % ( self.cmd_prefix | ||
4079 | 278 | , self.server_path | ||
4080 | 279 | , " ".join(server_args) | ||
4081 | 280 | ) | ||
4082 | 281 | |||
4083 | 282 | |||
4084 | 283 | def get_stop_cmd(self): | ||
4085 | 284 | """ Return the command that will shut us down """ | ||
4086 | 285 | |||
4087 | 286 | return "%s --no-defaults --user=root --port=%d --host=127.0.0.1 --protocol=tcp shutdown " %(self.mysqladmin, self.master_port) | ||
4088 | 287 | |||
4089 | 288 | |||
4090 | 289 | def get_ping_cmd(self): | ||
4091 | 290 | """Return the command string that will | ||
4092 | 291 | ping / check if the server is alive | ||
4093 | 292 | |||
4094 | 293 | """ | ||
4095 | 294 | |||
4096 | 295 | return '%s --no-defaults --user=root --port=%d --host=127.0.0.1 --protocol=tcp ping' % (self.mysqladmin, self.master_port) | ||
4097 | 296 | |||
4098 | 297 | def is_started(self): | ||
4099 | 298 | """ Determine if the server is up and running - | ||
4100 | 299 | this may vary from server type to server type | ||
4101 | 300 | |||
4102 | 301 | """ | ||
4103 | 302 | |||
4104 | 303 | # We experiment with waiting for a pid file to be created vs. pinging | ||
4105 | 304 | # This is what test-run.pl does and it helps us pass logging_stats tests | ||
4106 | 305 | # while not self.ping_server(server, quiet=True) and timer != timeout: | ||
4107 | 306 | |||
4108 | 307 | return self.system_manager.find_path( [self.pid_file] | ||
4109 | 308 | , required=0) | ||
4110 | 309 | |||
4111 | 310 | def gen_cnf_file(self, server_args): | ||
4112 | 311 | """ We generate a .cnf file for the server based | ||
4113 | 312 | on the arguments. We currently don't use | ||
4114 | 313 | this for much, but xtrabackup uses it, so | ||
4115 | 314 | we must produce one. This could also be | ||
4116 | 315 | helpful for testing / etc | ||
4117 | 316 | |||
4118 | 317 | """ | ||
4119 | 318 | |||
4120 | 319 | config_file = open(self.cnf_file,'w') | ||
4121 | 320 | config_file.write('[mysqld]') | ||
4122 | 321 | for server_arg in server_args: | ||
4123 | 322 | # We currently have a list of string values | ||
4124 | 323 | # We need to remove any '--' stuff | ||
4125 | 324 | server_arg = server_arg.replace('--','')+'\n' | ||
4126 | 325 | config_file.write(server_arg) | ||
4127 | 326 | config_file.close() | ||
4128 | 327 | |||
4129 | 328 | def set_master(self, master_server): | ||
4130 | 329 | """ We update things so that we use the provided master_server | ||
4131 | 330 | as our reference point for a new cluster / etc | ||
4132 | 331 | |||
4133 | 332 | """ | ||
4134 | 333 | |||
4135 | 334 | self.wsrep_cluster_addr = '127.0.0.1:%d' %(master_server.galera_listen_port) | ||
4136 | 335 | 0 | ||
4137 | === removed file 'dbqp/lib/server_mgmt/mysqld.py' | |||
4138 | --- dbqp/lib/server_mgmt/mysqld.py 2012-02-04 01:22:23 +0000 | |||
4139 | +++ dbqp/lib/server_mgmt/mysqld.py 1970-01-01 00:00:00 +0000 | |||
4140 | @@ -1,295 +0,0 @@ | |||
4141 | 1 | #! /usr/bin/env python | ||
4142 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
4143 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
4144 | 4 | # | ||
4145 | 5 | # Copyright (C) 2010,2011 Patrick Crews | ||
4146 | 6 | # | ||
4147 | 7 | # This program is free software; you can redistribute it and/or modify | ||
4148 | 8 | # it under the terms of the GNU General Public License as published by | ||
4149 | 9 | # the Free Software Foundation; either version 2 of the License, or | ||
4150 | 10 | # (at your option) any later version. | ||
4151 | 11 | # | ||
4152 | 12 | # This program is distributed in the hope that it will be useful, | ||
4153 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
4154 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
4155 | 15 | # GNU General Public License for more details. | ||
4156 | 16 | # | ||
4157 | 17 | # You should have received a copy of the GNU General Public License | ||
4158 | 18 | # along with this program; if not, write to the Free Software | ||
4159 | 19 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
4160 | 20 | |||
4161 | 21 | |||
4162 | 22 | """ mysqld.py: code to allow a serverManager | ||
4163 | 23 | to provision and start up a mysqld server object | ||
4164 | 24 | for test execution | ||
4165 | 25 | |||
4166 | 26 | """ | ||
4167 | 27 | |||
4168 | 28 | # imports | ||
4169 | 29 | import os | ||
4170 | 30 | import sys | ||
4171 | 31 | import time | ||
4172 | 32 | import subprocess | ||
4173 | 33 | |||
4174 | 34 | from ConfigParser import RawConfigParser | ||
4175 | 35 | |||
4176 | 36 | from lib.server_mgmt.server import Server | ||
4177 | 37 | |||
4178 | 38 | class mysqlServer(Server): | ||
4179 | 39 | """ represents a mysql server, its possessions | ||
4180 | 40 | (datadir, ports, etc), and methods for controlling | ||
4181 | 41 | and querying it | ||
4182 | 42 | |||
4183 | 43 | TODO: create a base server class that contains | ||
4184 | 44 | standard methods from which we can inherit | ||
4185 | 45 | Currently there are definitely methods / attr | ||
4186 | 46 | which are general | ||
4187 | 47 | |||
4188 | 48 | """ | ||
4189 | 49 | |||
4190 | 50 | def __init__( self, name, server_manager, code_tree, default_storage_engine | ||
4191 | 51 | , server_options, requester, workdir_root): | ||
4192 | 52 | super(mysqlServer, self).__init__( name | ||
4193 | 53 | , server_manager | ||
4194 | 54 | , code_tree | ||
4195 | 55 | , default_storage_engine | ||
4196 | 56 | , server_options | ||
4197 | 57 | , requester | ||
4198 | 58 | , workdir_root) | ||
4199 | 59 | self.preferred_base_port = 9306 | ||
4200 | 60 | |||
4201 | 61 | # client files | ||
4202 | 62 | self.mysqldump = self.code_tree.mysqldump | ||
4203 | 63 | self.mysqladmin = self.code_tree.mysqladmin | ||
4204 | 64 | self.mysql_client = self.code_tree.mysql_client | ||
4205 | 65 | self.mysqlimport = self.code_tree.mysqlimport | ||
4206 | 66 | self.mysqlslap = self.code_tree.mysqlslap | ||
4207 | 67 | self.mysql_upgrade = self.code_tree.mysql_upgrade | ||
4208 | 68 | self.server_path = self.code_tree.mysql_server | ||
4209 | 69 | self.mysql_client_path = self.code_tree.mysql_client | ||
4210 | 70 | |||
4211 | 71 | # important stuff | ||
4212 | 72 | self.langdir = self.code_tree.langdir | ||
4213 | 73 | self.charsetdir = self.code_tree.charsetdir | ||
4214 | 74 | self.bootstrap_file = self.code_tree.bootstrap_path | ||
4215 | 75 | self.bootstrap_cmd = None | ||
4216 | 76 | |||
4217 | 77 | # Get our ports | ||
4218 | 78 | self.port_block = self.system_manager.port_manager.get_port_block( self.name | ||
4219 | 79 | , self.preferred_base_port | ||
4220 | 80 | , 1 ) | ||
4221 | 81 | self.master_port = self.port_block[0] | ||
4222 | 82 | |||
4223 | 83 | # Generate our working directories | ||
4224 | 84 | self.dirset = { 'var_%s' %(self.name): {'std_data_ln':( os.path.join(self.code_tree.testdir,'std_data')) | ||
4225 | 85 | ,'log':None | ||
4226 | 86 | ,'run':None | ||
4227 | 87 | ,'tmp':None | ||
4228 | 88 | ,'master-data': { 'test':None | ||
4229 | 89 | , 'mysql':None | ||
4230 | 90 | } | ||
4231 | 91 | } | ||
4232 | 92 | } | ||
4233 | 93 | self.workdir = self.system_manager.create_dirset( workdir_root | ||
4234 | 94 | , self.dirset) | ||
4235 | 95 | self.vardir = self.workdir | ||
4236 | 96 | self.tmpdir = os.path.join(self.vardir,'tmp') | ||
4237 | 97 | self.rundir = os.path.join(self.vardir,'run') | ||
4238 | 98 | self.logdir = os.path.join(self.vardir,'log') | ||
4239 | 99 | self.std_data = os.path.join(self.vardir,'std_data_ln') | ||
4240 | 100 | self.datadir = os.path.join(self.vardir,'master-data') | ||
4241 | 101 | |||
4242 | 102 | self.error_log = os.path.join(self.logdir,('%s.err' %(self.name))) | ||
4243 | 103 | self.bootstrap_log = os.path.join(self.logdir,('bootstrap.log')) | ||
4244 | 104 | self.pid_file = os.path.join(self.rundir,('%s.pid' %(self.name))) | ||
4245 | 105 | self.socket_file = os.path.join(self.vardir, ('%s.sock' %(self.name))) | ||
4246 | 106 | self.timer_file = os.path.join(self.logdir,('timer')) | ||
4247 | 107 | self.general_log_file = os.path.join(self.logdir,'mysqld.log') | ||
4248 | 108 | self.slow_query_log_file = os.path.join(self.logdir,'mysqld-slow.log') | ||
4249 | 109 | self.cnf_file = os.path.join(self.vardir,'my.cnf') | ||
4250 | 110 | |||
4251 | 111 | self.snapshot_path = os.path.join(self.tmpdir,('snapshot_%s' %(self.master_port))) | ||
4252 | 112 | # We want to use --secure-file-priv = $vardir by default | ||
4253 | 113 | # but there are times / tools when we need to shut this off | ||
4254 | 114 | if self.no_secure_file_priv: | ||
4255 | 115 | self.secure_file_string = '' | ||
4256 | 116 | else: | ||
4257 | 117 | self.secure_file_string = "--secure-file-priv='%s'" %(self.vardir) | ||
4258 | 118 | self.user_string = '--user=root' | ||
4259 | 119 | |||
4260 | 120 | self.initialize_databases() | ||
4261 | 121 | self.take_db_snapshot() | ||
4262 | 122 | |||
4263 | 123 | self.logging.debug_class(self) | ||
4264 | 124 | |||
4265 | 125 | |||
4266 | 126 | def report(self): | ||
4267 | 127 | """ We print out some general useful info """ | ||
4268 | 128 | report_values = [ 'name' | ||
4269 | 129 | , 'master_port' | ||
4270 | 130 | , 'socket_file' | ||
4271 | 131 | , 'vardir' | ||
4272 | 132 | , 'status' | ||
4273 | 133 | ] | ||
4274 | 134 | self.logging.info("%s server:" %(self.owner)) | ||
4275 | 135 | for key in report_values: | ||
4276 | 136 | value = vars(self)[key] | ||
4277 | 137 | self.logging.info("%s: %s" %(key.upper(), value)) | ||
4278 | 138 | |||
4279 | 139 | def initialize_databases(self): | ||
4280 | 140 | """ Do the voodoo required to have a working database setup. | ||
4281 | 141 | For MySQL, this is calling the server with the | ||
4282 | 142 | --bootstrap argument. We generate the bootstrap | ||
4283 | 143 | file during codeTree intialization as the file is standard for | ||
4284 | 144 | all MySQL servers that are spawned from a single codeTree | ||
4285 | 145 | |||
4286 | 146 | """ | ||
4287 | 147 | |||
4288 | 148 | # generate the bootstrap startup command | ||
4289 | 149 | if not self.bootstrap_cmd: | ||
4290 | 150 | mysqld_args = [ "--no-defaults" | ||
4291 | 151 | , "--bootstrap" | ||
4292 | 152 | , "--basedir=%s" %(self.code_tree.basedir) | ||
4293 | 153 | , "--datadir=%s" %(self.datadir) | ||
4294 | 154 | , "--loose-skip-falcon" | ||
4295 | 155 | , "--loose-skip-ndbcluster" | ||
4296 | 156 | , "--tmpdir=%s" %(self.tmpdir) | ||
4297 | 157 | , "--core-file" | ||
4298 | 158 | , "--lc-messages-dir=%s" %(self.langdir) | ||
4299 | 159 | , "--character-sets-dir=%s" %(self.charsetdir) | ||
4300 | 160 | ] | ||
4301 | 161 | # We add server_path into the mix this way as we | ||
4302 | 162 | # may alter how we store / handle server args later | ||
4303 | 163 | mysqld_args.insert(0,self.server_path) | ||
4304 | 164 | self.bootstrap_cmd = " ".join(mysqld_args) | ||
4305 | 165 | # execute our command | ||
4306 | 166 | bootstrap_log = open(self.bootstrap_log,'w') | ||
4307 | 167 | # open our bootstrap file | ||
4308 | 168 | bootstrap_in = open(self.bootstrap_file,'r') | ||
4309 | 169 | bootstrap_subproc = subprocess.Popen( self.bootstrap_cmd | ||
4310 | 170 | , shell=True | ||
4311 | 171 | , stdin=bootstrap_in | ||
4312 | 172 | , stdout=bootstrap_log | ||
4313 | 173 | , stderr=bootstrap_log | ||
4314 | 174 | ) | ||
4315 | 175 | bootstrap_subproc.wait() | ||
4316 | 176 | bootstrap_in.close() | ||
4317 | 177 | bootstrap_log.close() | ||
4318 | 178 | bootstrap_retcode = bootstrap_subproc.returncode | ||
4319 | 179 | if bootstrap_retcode: | ||
4320 | 180 | self.logging.error("Received retcode: %s executing command: %s" | ||
4321 | 181 | %(bootstrap_retcode, self.bootstrap_cmd)) | ||
4322 | 182 | self.logging.error("Check the bootstrap log: %s" %(self.bootstrap_log)) | ||
4323 | 183 | sys.exit(1) | ||
4324 | 184 | |||
4325 | 185 | |||
4326 | 186 | def get_start_cmd(self): | ||
4327 | 187 | """ Return the command string that will start up the server | ||
4328 | 188 | as desired / intended | ||
4329 | 189 | |||
4330 | 190 | """ | ||
4331 | 191 | |||
4332 | 192 | server_args = [ self.process_server_options() | ||
4333 | 193 | , "--open-files-limit=1024" | ||
4334 | 194 | , "--local-infile" | ||
4335 | 195 | , "--character-set-server=latin1" | ||
4336 | 196 | , "--connect-timeout=60" | ||
4337 | 197 | , "--log-bin-trust-function-creators=1" | ||
4338 | 198 | , "--key_buffer_size=1M" | ||
4339 | 199 | , "--sort_buffer=256K" | ||
4340 | 200 | , "--max_heap_table_size=1M" | ||
4341 | 201 | , "--loose-innodb_data_file_path=ibdata1:10M:autoextend" | ||
4342 | 202 | , "--loose-innodb_buffer_pool_size=8M" | ||
4343 | 203 | , "--loose-innodb_write_io_threads=2" | ||
4344 | 204 | , "--loose-innodb_read_io_threads=2" | ||
4345 | 205 | , "--loose-innodb_log_buffer_size=1M" | ||
4346 | 206 | , "--loose-innodb_log_file_size=5M" | ||
4347 | 207 | , "--loose-innodb_additional_mem_pool_size=1M" | ||
4348 | 208 | , "--loose-innodb_log_files_in_group=2" | ||
4349 | 209 | , "--slave-net-timeout=120" | ||
4350 | 210 | , "--log-bin=%s" %(os.path.join(self.logdir,"mysqld-bin")) | ||
4351 | 211 | , "--loose-enable-performance-schema" | ||
4352 | 212 | , "--loose-performance-schema-max-mutex-instances=10000" | ||
4353 | 213 | , "--loose-performance-schema-max-rwlock-instances=10000" | ||
4354 | 214 | , "--loose-performance-schema-max-table-instances=500" | ||
4355 | 215 | , "--loose-performance-schema-max-table-handles=1000" | ||
4356 | 216 | , "--binlog-direct-non-transactional-updates" | ||
4357 | 217 | , "--loose-enable-performance-schema" | ||
4358 | 218 | , "--general_log=1" | ||
4359 | 219 | , "--general_log_file=%s" %(self.general_log_file) | ||
4360 | 220 | , "--slow_query_log=1" | ||
4361 | 221 | , "--slow_query_log_file=%s" %(self.slow_query_log_file) | ||
4362 | 222 | , "--basedir=%s" %(self.code_tree.basedir) | ||
4363 | 223 | , "--datadir=%s" %(self.datadir) | ||
4364 | 224 | , "--tmpdir=%s" %(self.tmpdir) | ||
4365 | 225 | , "--character-sets-dir=%s" %(self.charsetdir) | ||
4366 | 226 | , "--lc-messages-dir=%s" %(self.langdir) | ||
4367 | 227 | , "--ssl-ca=%s" %(os.path.join(self.std_data,'cacert.pem')) | ||
4368 | 228 | , "--ssl-cert=%s" %(os.path.join(self.std_data,'server-cert.pem')) | ||
4369 | 229 | , "--ssl-key=%s" %(os.path.join(self.std_data,'server-key.pem')) | ||
4370 | 230 | , "--port=%d" %(self.master_port) | ||
4371 | 231 | , "--socket=%s" %(self.socket_file) | ||
4372 | 232 | , "--pid-file=%s" %(self.pid_file) | ||
4373 | 233 | , "--default-storage-engine=%s" %(self.default_storage_engine) | ||
4374 | 234 | # server-id maybe needs fixing, but we want to avoid | ||
4375 | 235 | # the server-id=0 and no replication thing... | ||
4376 | 236 | , "--server-id=%d" %(self.get_numeric_server_id()+1) | ||
4377 | 237 | , self.secure_file_string | ||
4378 | 238 | , self.user_string | ||
4379 | 239 | ] | ||
4380 | 240 | self.gen_cnf_file(server_args) | ||
4381 | 241 | |||
4382 | 242 | if self.gdb: | ||
4383 | 243 | server_args.append('--gdb') | ||
4384 | 244 | return self.system_manager.handle_gdb_reqs(self, server_args) | ||
4385 | 245 | else: | ||
4386 | 246 | return "%s %s %s & " % ( self.cmd_prefix | ||
4387 | 247 | , self.server_path | ||
4388 | 248 | , " ".join(server_args) | ||
4389 | 249 | ) | ||
4390 | 250 | |||
4391 | 251 | |||
4392 | 252 | def get_stop_cmd(self): | ||
4393 | 253 | """ Return the command that will shut us down """ | ||
4394 | 254 | |||
4395 | 255 | return "%s --no-defaults --user=root --port=%d --host=127.0.0.1 --protocol=tcp shutdown " %(self.mysqladmin, self.master_port) | ||
4396 | 256 | |||
4397 | 257 | |||
4398 | 258 | def get_ping_cmd(self): | ||
4399 | 259 | """Return the command string that will | ||
4400 | 260 | ping / check if the server is alive | ||
4401 | 261 | |||
4402 | 262 | """ | ||
4403 | 263 | |||
4404 | 264 | return '%s --no-defaults --user=root --port=%d --host=127.0.0.1 --protocol=tcp ping' % (self.mysqladmin, self.master_port) | ||
4405 | 265 | |||
4406 | 266 | def is_started(self): | ||
4407 | 267 | """ Determine if the server is up and running - | ||
4408 | 268 | this may vary from server type to server type | ||
4409 | 269 | |||
4410 | 270 | """ | ||
4411 | 271 | |||
4412 | 272 | # We experiment with waiting for a pid file to be created vs. pinging | ||
4413 | 273 | # This is what test-run.pl does and it helps us pass logging_stats tests | ||
4414 | 274 | # while not self.ping_server(server, quiet=True) and timer != timeout: | ||
4415 | 275 | |||
4416 | 276 | return self.system_manager.find_path( [self.pid_file] | ||
4417 | 277 | , required=0) | ||
4418 | 278 | |||
4419 | 279 | def gen_cnf_file(self, server_args): | ||
4420 | 280 | """ We generate a .cnf file for the server based | ||
4421 | 281 | on the arguments. We currently don't use | ||
4422 | 282 | this for much, but xtrabackup uses it, so | ||
4423 | 283 | we must produce one. This could also be | ||
4424 | 284 | helpful for testing / etc | ||
4425 | 285 | |||
4426 | 286 | """ | ||
4427 | 287 | |||
4428 | 288 | config_file = open(self.cnf_file,'w') | ||
4429 | 289 | config_file.write('[mysqld]') | ||
4430 | 290 | for server_arg in server_args: | ||
4431 | 291 | # We currently have a list of string values | ||
4432 | 292 | # We need to remove any '--' stuff | ||
4433 | 293 | server_arg = server_arg.replace('--','')+'\n' | ||
4434 | 294 | config_file.write(server_arg) | ||
4435 | 295 | config_file.close() | ||
4436 | 296 | 0 | ||
4437 | === removed file 'dbqp/lib/server_mgmt/percona.py' | |||
4438 | --- dbqp/lib/server_mgmt/percona.py 2012-02-04 01:22:23 +0000 | |||
4439 | +++ dbqp/lib/server_mgmt/percona.py 1970-01-01 00:00:00 +0000 | |||
4440 | @@ -1,273 +0,0 @@ | |||
4441 | 1 | #! /usr/bin/env python | ||
4442 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
4443 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
4444 | 4 | # | ||
4445 | 5 | # Copyright (C) 2010,2011 Patrick Crews | ||
4446 | 6 | # | ||
4447 | 7 | # This program is free software; you can redistribute it and/or modify | ||
4448 | 8 | # it under the terms of the GNU General Public License as published by | ||
4449 | 9 | # the Free Software Foundation; either version 2 of the License, or | ||
4450 | 10 | # (at your option) any later version. | ||
4451 | 11 | # | ||
4452 | 12 | # This program is distributed in the hope that it will be useful, | ||
4453 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
4454 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
4455 | 15 | # GNU General Public License for more details. | ||
4456 | 16 | # | ||
4457 | 17 | # You should have received a copy of the GNU General Public License | ||
4458 | 18 | # along with this program; if not, write to the Free Software | ||
4459 | 19 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
4460 | 20 | |||
4461 | 21 | |||
4462 | 22 | """ mysqld.py: code to allow a serverManager | ||
4463 | 23 | to provision and start up a mysqld server object | ||
4464 | 24 | for test execution | ||
4465 | 25 | |||
4466 | 26 | """ | ||
4467 | 27 | |||
4468 | 28 | # imports | ||
4469 | 29 | import os | ||
4470 | 30 | import sys | ||
4471 | 31 | import subproc | ||
4472 | 32 | |||
4473 | 33 | |||
4474 | 34 | from lib.server_mgmt.server import Server | ||
4475 | 35 | |||
4476 | 36 | class mysqlServer(Server): | ||
4477 | 37 | """ represents a mysql server, its possessions | ||
4478 | 38 | (datadir, ports, etc), and methods for controlling | ||
4479 | 39 | and querying it | ||
4480 | 40 | |||
4481 | 41 | TODO: create a base server class that contains | ||
4482 | 42 | standard methods from which we can inherit | ||
4483 | 43 | Currently there are definitely methods / attr | ||
4484 | 44 | which are general | ||
4485 | 45 | |||
4486 | 46 | """ | ||
4487 | 47 | |||
4488 | 48 | def __init__( self, name, server_manager, code_tree, default_storage_engine | ||
4489 | 49 | , server_options, requester, workdir_root): | ||
4490 | 50 | super(mysqlServer, self).__init__( name | ||
4491 | 51 | , server_manager | ||
4492 | 52 | , code_tree | ||
4493 | 53 | , default_storage_engine | ||
4494 | 54 | , server_options | ||
4495 | 55 | , requester | ||
4496 | 56 | , workdir_root) | ||
4497 | 57 | self.preferred_base_port = 9306 | ||
4498 | 58 | |||
4499 | 59 | # client files | ||
4500 | 60 | self.mysqldump = self.code_tree.mysqldump | ||
4501 | 61 | self.mysqladmin = self.code_tree.mysqladmin | ||
4502 | 62 | self.mysql_client = self.code_tree.mysql_client | ||
4503 | 63 | self.mysqlimport = self.code_tree.mysqlimport | ||
4504 | 64 | self.mysqlslap = self.code_tree.mysqlslap | ||
4505 | 65 | self.server_path = self.code_tree.mysql_server | ||
4506 | 66 | self.mysql_client_path = self.code_tree.mysql_client | ||
4507 | 67 | |||
4508 | 68 | # important stuff | ||
4509 | 69 | self.langdir = self.code_tree.langdir | ||
4510 | 70 | self.charsetdir = self.code_tree.charsetdir | ||
4511 | 71 | self.bootstrap_file = self.code_tree.bootstrap_path | ||
4512 | 72 | self.bootstrap_cmd = None | ||
4513 | 73 | |||
4514 | 74 | # Get our ports | ||
4515 | 75 | self.port_block = self.system_manager.port_manager.get_port_block( self.name | ||
4516 | 76 | , self.preferred_base_port | ||
4517 | 77 | , 1 ) | ||
4518 | 78 | self.master_port = self.port_block[0] | ||
4519 | 79 | |||
4520 | 80 | # Generate our working directories | ||
4521 | 81 | self.dirset = { self.name : { 'var': {'std_data_ln':( os.path.join(self.code_tree.testdir,'std_data')) | ||
4522 | 82 | ,'log':None | ||
4523 | 83 | ,'run':None | ||
4524 | 84 | ,'tmp':None | ||
4525 | 85 | ,'master-data': {'local': { 'test':None | ||
4526 | 86 | , 'mysql':None | ||
4527 | 87 | } | ||
4528 | 88 | } | ||
4529 | 89 | } | ||
4530 | 90 | } | ||
4531 | 91 | } | ||
4532 | 92 | self.workdir = self.system_manager.create_dirset( workdir_root | ||
4533 | 93 | , self.dirset) | ||
4534 | 94 | self.vardir = os.path.join(self.workdir,'var') | ||
4535 | 95 | self.tmpdir = os.path.join(self.vardir,'tmp') | ||
4536 | 96 | self.rundir = os.path.join(self.vardir,'run') | ||
4537 | 97 | self.logdir = os.path.join(self.vardir,'log') | ||
4538 | 98 | self.std_data = os.path.join(self.vardir,'std_data_ln') | ||
4539 | 99 | self.datadir = os.path.join(self.vardir,'master-data') | ||
4540 | 100 | |||
4541 | 101 | self.error_log = os.path.join(self.logdir,('%s.err' %(self.name))) | ||
4542 | 102 | self.bootstrap_log = os.path.join(self.logdir,('bootstrap.log')) | ||
4543 | 103 | self.pid_file = os.path.join(self.rundir,('%s.pid' %(self.name))) | ||
4544 | 104 | self.socket_file = os.path.join(self.vardir, ('%s.sock' %(self.name))) | ||
4545 | 105 | self.timer_file = os.path.join(self.logdir,('timer')) | ||
4546 | 106 | self.general_log_file = os.path.join(self.logdir,'mysqld.log') | ||
4547 | 107 | self.slow_query_log_file = os.path.join(self.logdir,'mysqld-slow.log') | ||
4548 | 108 | |||
4549 | 109 | self.snapshot_path = os.path.join(self.tmpdir,('snapshot_%s' %(self.master_port))) | ||
4550 | 110 | # We want to use --secure-file-priv = $vardir by default | ||
4551 | 111 | # but there are times / tools when we need to shut this off | ||
4552 | 112 | if self.no_secure_file_priv: | ||
4553 | 113 | self.secure_file_string = '' | ||
4554 | 114 | else: | ||
4555 | 115 | self.secure_file_string = "--secure-file-priv='%s'" %(self.vardir) | ||
4556 | 116 | self.user_string = '--user=root' | ||
4557 | 117 | |||
4558 | 118 | self.initialize_databases() | ||
4559 | 119 | self.take_db_snapshot() | ||
4560 | 120 | |||
4561 | 121 | self.logging.debug_class(self) | ||
4562 | 122 | |||
4563 | 123 | def report(self): | ||
4564 | 124 | """ We print out some general useful info """ | ||
4565 | 125 | report_values = [ 'name' | ||
4566 | 126 | , 'master_port' | ||
4567 | 127 | , 'socket_file' | ||
4568 | 128 | , 'vardir' | ||
4569 | 129 | , 'status' | ||
4570 | 130 | ] | ||
4571 | 131 | self.logging.info("%s server:" %(self.owner)) | ||
4572 | 132 | for key in report_values: | ||
4573 | 133 | value = vars(self)[key] | ||
4574 | 134 | self.logging.info("%s: %s" %(key.upper(), value)) | ||
4575 | 135 | |||
4576 | 136 | def initialize_databases(self): | ||
4577 | 137 | """ Do the voodoo required to have a working database setup. | ||
4578 | 138 | For MySQL, this is calling the server with the | ||
4579 | 139 | --bootstrap argument. We generate the bootstrap | ||
4580 | 140 | file during codeTree intialization as the file is standard for | ||
4581 | 141 | all MySQL servers that are spawned from a single codeTree | ||
4582 | 142 | |||
4583 | 143 | """ | ||
4584 | 144 | |||
4585 | 145 | # generate the bootstrap startup command | ||
4586 | 146 | if not self.bootstrap_cmd: | ||
4587 | 147 | mysqld_args = [ "--no-defaults" | ||
4588 | 148 | , "--bootstrap" | ||
4589 | 149 | , "--basedir=%s" %(self.code_tree.basedir) | ||
4590 | 150 | , "--datadir=%s" %(self.datadir) | ||
4591 | 151 | , "--loose-skip-falcon" | ||
4592 | 152 | , "--loose-skip-ndbcluster" | ||
4593 | 153 | , "--tmpdir=%s" %(self.tmpdir) | ||
4594 | 154 | , "--core-file" | ||
4595 | 155 | , "--lc-messages-dir=%s" %(self.langdir) | ||
4596 | 156 | , "--character-sets-dir=%s" %(self.charsetdir) | ||
4597 | 157 | ] | ||
4598 | 158 | # We add server_path into the mix this way as we | ||
4599 | 159 | # may alter how we store / handle server args later | ||
4600 | 160 | mysqld_args = [self.server_path].append(mysqld_args) | ||
4601 | 161 | self.bootstrap_cmd = " ".join(mysqld_args) | ||
4602 | 162 | # execute our command | ||
4603 | 163 | bootstrap_subproc = subprocess.Popen( self.bootstrap_cmd | ||
4604 | 164 | , shell=True | ||
4605 | 165 | , stdout=self.bootstrap_log | ||
4606 | 166 | , stderr=self.bootstrap_log | ||
4607 | 167 | ) | ||
4608 | 168 | bootstrap_subproc.wait() | ||
4609 | 169 | bootstrap_retcode = bootstrap_subproc.returncode | ||
4610 | 170 | if bootstrap_retcode: | ||
4611 | 171 | self.logging.error("Received retcode: %s executing command: %s" | ||
4612 | 172 | %(bootstrap_retcode, self.bootstrap_cmd)) | ||
4613 | 173 | self.logging.error("Check the bootstrap log: %s" %(self.bootstrap_log)) | ||
4614 | 174 | sys.exit(1) | ||
4615 | 175 | |||
4616 | 176 | |||
4617 | 177 | def get_start_cmd(self): | ||
4618 | 178 | """ Return the command string that will start up the server | ||
4619 | 179 | as desired / intended | ||
4620 | 180 | |||
4621 | 181 | """ | ||
4622 | 182 | |||
4623 | 183 | server_args = [ self.process_server_options() | ||
4624 | 184 | , "--open-files-limit=1024" | ||
4625 | 185 | , "--local-infile" | ||
4626 | 186 | , "--character-set-server=latin1" | ||
4627 | 187 | , "--connect-timeout=60" | ||
4628 | 188 | , "--log-bin-trust-function-creators=1" | ||
4629 | 189 | , "--key_buffer_size=1M" | ||
4630 | 190 | , "--sort_buffer=256K" | ||
4631 | 191 | , "--max_heap_table_size=1M" | ||
4632 | 192 | , "--query-cache-size=0" | ||
4633 | 193 | , "--loose-innodb_data_file_path=ibdata1:10M:autoextend" | ||
4634 | 194 | , "--loose-innodb_buffer_pool_size=32M" | ||
4635 | 195 | , "--loose-innodb_write_io_threads=2" | ||
4636 | 196 | , "--loose-innodb_read_io_threads=2" | ||
4637 | 197 | , "--loose-innodb_log_buffer_size=1M" | ||
4638 | 198 | , "--loose-innodb_log_file_size=5M" | ||
4639 | 199 | , "--loose-innodb_additional_mem_pool_size=1M" | ||
4640 | 200 | , "--loose-innodb_log_files_in_group=2" | ||
4641 | 201 | , "--slave-net-timeout=120" | ||
4642 | 202 | , "--log-bin=mysqld-bin" | ||
4643 | 203 | , "--loose-enable-performance-schema" | ||
4644 | 204 | , "--loose-performance-schema-max-mutex-instances=10000" | ||
4645 | 205 | , "--loose-performance-schema-max-rwlock-instances=10000" | ||
4646 | 206 | , "--loose-performance-schema-max-table-instances=500" | ||
4647 | 207 | , "--loose-performance-schema-max-table-handles=1000" | ||
4648 | 208 | , "--binlog-direct-non-transactional-updates" | ||
4649 | 209 | , "--loose-enable-performance-schema" | ||
4650 | 210 | , "--log-bin=master-bin" | ||
4651 | 211 | , "--general_log=1" | ||
4652 | 212 | , "--general_log_file=%s" %(self.general_log_file) | ||
4653 | 213 | , "--slow_query_log=1" | ||
4654 | 214 | , "--slow_query_log_file=%s" %(self.slow_query_log_file) | ||
4655 | 215 | , "--basedir=%s" %(self.codetree.basedir) | ||
4656 | 216 | , "--datadir=%s" %(self.datadir) | ||
4657 | 217 | , "--tmpdir=%s" %(self.tmpdir) | ||
4658 | 218 | , "--character-sets-dir=%s" %(self.charsetdir) | ||
4659 | 219 | , "--lc-messages-dir=%s" %(self.langdir) | ||
4660 | 220 | , "--ssl-ca=%s" %(os.path.join(self.std_data,'cacert.pem')) | ||
4661 | 221 | , "--ssl-cert=%s" %(os.path.join(self.std_data,'server-cert.pem')) | ||
4662 | 222 | , "--ssl-key=%s" %(os.path.join(self.std_data,'server-key.pem')) | ||
4663 | 223 | , "--port=%d" %(self.master_port) | ||
4664 | 224 | , "--mysql-protocol.connect-timeout=60" | ||
4665 | 225 | , "--innodb.data-file-path=ibdata1:20M:autoextend" | ||
4666 | 226 | , "--sort-buffer-size=256K" | ||
4667 | 227 | , "--max-heap-table-size=1M" | ||
4668 | 228 | , "--socket=%s" %(self.socket_file) | ||
4669 | 229 | , "--pid-file=%s" %(self.pid_file) | ||
4670 | 230 | , "--default-storage-engine=%s" %(self.default_storage_engine) | ||
4671 | 231 | , "--server-id=%d" %(self.get_numeric_server_id) | ||
4672 | 232 | , self.secure_file_string | ||
4673 | 233 | , self.user_string | ||
4674 | 234 | ] | ||
4675 | 235 | |||
4676 | 236 | if self.gdb: | ||
4677 | 237 | server_args.append('--gdb') | ||
4678 | 238 | return self.system_manager.handle_gdb_reqs(self, server_args) | ||
4679 | 239 | else: | ||
4680 | 240 | return "%s %s %s & " % ( self.cmd_prefix | ||
4681 | 241 | , self.server_path | ||
4682 | 242 | , " ".join(server_args) | ||
4683 | 243 | ) | ||
4684 | 244 | |||
4685 | 245 | |||
4686 | 246 | def get_stop_cmd(self): | ||
4687 | 247 | """ Return the command that will shut us down """ | ||
4688 | 248 | |||
4689 | 249 | return "%s --user=root --port=%d --connect-timeout=5 --silent --password= --shutdown " %(self.mysql_admin, self.master_port) | ||
4690 | 250 | |||
4691 | 251 | |||
4692 | 252 | def get_ping_cmd(self): | ||
4693 | 253 | """Return the command string that will | ||
4694 | 254 | ping / check if the server is alive | ||
4695 | 255 | |||
4696 | 256 | """ | ||
4697 | 257 | |||
4698 | 258 | return '%s --port=%d --user=root -hlocalhost --protocol=tcp -e ""' % (self.mysql_client_path, self.master_port) | ||
4699 | 259 | |||
4700 | 260 | def is_started(self): | ||
4701 | 261 | """ Determine if the server is up and running - | ||
4702 | 262 | this may vary from server type to server type | ||
4703 | 263 | |||
4704 | 264 | """ | ||
4705 | 265 | |||
4706 | 266 | # We experiment with waiting for a pid file to be created vs. pinging | ||
4707 | 267 | # This is what test-run.pl does and it helps us pass logging_stats tests | ||
4708 | 268 | # while not self.ping_server(server, quiet=True) and timer != timeout: | ||
4709 | 269 | |||
4710 | 270 | return self.system_manager.find_path( [self.pid_file] | ||
4711 | 271 | , required=0) | ||
4712 | 272 | |||
4713 | 273 | |||
4714 | 274 | 0 | ||
4715 | === removed file 'dbqp/lib/server_mgmt/server.py' | |||
4716 | --- dbqp/lib/server_mgmt/server.py 2012-02-04 01:22:23 +0000 | |||
4717 | +++ dbqp/lib/server_mgmt/server.py 1970-01-01 00:00:00 +0000 | |||
4718 | @@ -1,170 +0,0 @@ | |||
4719 | 1 | #! /usr/bin/env python | ||
4720 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
4721 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
4722 | 4 | # | ||
4723 | 5 | # Copyright (C) 2010,2011 Patrick Crews | ||
4724 | 6 | # | ||
4725 | 7 | # This program is free software; you can redistribute it and/or modify | ||
4726 | 8 | # it under the terms of the GNU General Public License as published by | ||
4727 | 9 | # the Free Software Foundation; either version 2 of the License, or | ||
4728 | 10 | # (at your option) any later version. | ||
4729 | 11 | # | ||
4730 | 12 | # This program is distributed in the hope that it will be useful, | ||
4731 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
4732 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
4733 | 15 | # GNU General Public License for more details. | ||
4734 | 16 | # | ||
4735 | 17 | # You should have received a copy of the GNU General Public License | ||
4736 | 18 | # along with this program; if not, write to the Free Software | ||
4737 | 19 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
4738 | 20 | |||
4739 | 21 | |||
4740 | 22 | """ server.py: generic server object used by the server | ||
4741 | 23 | manager. This contains the generic methods for all | ||
4742 | 24 | servers. Specific types (Drizzle, MySQL, etc) should | ||
4743 | 25 | inherit from this guy | ||
4744 | 26 | |||
4745 | 27 | """ | ||
4746 | 28 | |||
4747 | 29 | # imports | ||
4748 | 30 | import os | ||
4749 | 31 | |||
4750 | 32 | class Server(object): | ||
4751 | 33 | """ the server class from which other servers | ||
4752 | 34 | will inherit - contains generic methods | ||
4753 | 35 | certain methods will be overridden by more | ||
4754 | 36 | specific ones | ||
4755 | 37 | |||
4756 | 38 | """ | ||
4757 | 39 | |||
4758 | 40 | def __init__(self | ||
4759 | 41 | , name | ||
4760 | 42 | , server_manager | ||
4761 | 43 | , code_tree | ||
4762 | 44 | , default_storage_engine | ||
4763 | 45 | , server_options | ||
4764 | 46 | , requester | ||
4765 | 47 | , workdir_root): | ||
4766 | 48 | self.skip_keys = [ 'server_manager' | ||
4767 | 49 | , 'system_manager' | ||
4768 | 50 | , 'dirset' | ||
4769 | 51 | , 'preferred_base_port' | ||
4770 | 52 | , 'no_secure_file_priv' | ||
4771 | 53 | , 'secure_file_string' | ||
4772 | 54 | , 'port_block' | ||
4773 | 55 | ] | ||
4774 | 56 | self.debug = server_manager.debug | ||
4775 | 57 | self.verbose = server_manager.verbose | ||
4776 | 58 | self.initial_run = 1 | ||
4777 | 59 | self.owner = requester | ||
4778 | 60 | self.server_options = server_options | ||
4779 | 61 | self.default_storage_engine = default_storage_engine | ||
4780 | 62 | self.server_manager = server_manager | ||
4781 | 63 | # We register with server_manager asap | ||
4782 | 64 | self.server_manager.log_server(self, requester) | ||
4783 | 65 | |||
4784 | 66 | self.system_manager = self.server_manager.system_manager | ||
4785 | 67 | self.code_tree = code_tree | ||
4786 | 68 | self.type = self.code_tree.type | ||
4787 | 69 | self.valgrind = self.system_manager.valgrind | ||
4788 | 70 | self.gdb = self.system_manager.gdb | ||
4789 | 71 | if self.valgrind: | ||
4790 | 72 | self.valgrind_time_buffer = 10 | ||
4791 | 73 | else: | ||
4792 | 74 | self.valgrind_time_buffer = 1 | ||
4793 | 75 | self.cmd_prefix = self.system_manager.cmd_prefix | ||
4794 | 76 | self.logging = self.system_manager.logging | ||
4795 | 77 | self.no_secure_file_priv = self.server_manager.no_secure_file_priv | ||
4796 | 78 | self.name = name | ||
4797 | 79 | self.status = 0 # stopped, 1 = running | ||
4798 | 80 | self.tried_start = 0 | ||
4799 | 81 | self.failed_test = 0 # was the last test a failure? our state is suspect | ||
4800 | 82 | self.server_start_timeout = 60 * self.valgrind_time_buffer | ||
4801 | 83 | self.pid = None | ||
4802 | 84 | self.ip_address = '127.0.0.1' | ||
4803 | 85 | self.need_reset = False | ||
4804 | 86 | |||
4805 | 87 | def initialize_databases(self): | ||
4806 | 88 | """ Call schemawriter to make db.opt files """ | ||
4807 | 89 | databases = [ 'test' | ||
4808 | 90 | , 'mysql' | ||
4809 | 91 | ] | ||
4810 | 92 | for database in databases: | ||
4811 | 93 | db_path = os.path.join(self.datadir,'local',database,'db.opt') | ||
4812 | 94 | cmd = "%s %s %s" %(self.schemawriter, database, db_path) | ||
4813 | 95 | self.system_manager.execute_cmd(cmd) | ||
4814 | 96 | |||
4815 | 97 | def process_server_options(self): | ||
4816 | 98 | """Consume the list of options we have been passed. | ||
4817 | 99 | Return a string with them joined | ||
4818 | 100 | |||
4819 | 101 | """ | ||
4820 | 102 | |||
4821 | 103 | return " ".join(self.server_options) | ||
4822 | 104 | |||
4823 | 105 | def take_db_snapshot(self): | ||
4824 | 106 | """ Take a snapshot of our vardir for quick restores """ | ||
4825 | 107 | |||
4826 | 108 | self.logging.info("Taking clean db snapshot...") | ||
4827 | 109 | if os.path.exists(self.snapshot_path): | ||
4828 | 110 | # We need to remove an existing path as python shutil | ||
4829 | 111 | # doesn't want an existing target | ||
4830 | 112 | self.system_manager.remove_dir(self.snapshot_path) | ||
4831 | 113 | self.system_manager.copy_dir(self.datadir, self.snapshot_path) | ||
4832 | 114 | |||
4833 | 115 | def restore_snapshot(self): | ||
4834 | 116 | """ Restore from a stored snapshot """ | ||
4835 | 117 | |||
4836 | 118 | if not os.path.exists(self.snapshot_path): | ||
4837 | 119 | self.logging.error("Could not find snapshot: %s" %(self.snapshot_path)) | ||
4838 | 120 | self.system_manager.remove_dir(self.datadir) | ||
4839 | 121 | self.system_manager.copy_dir(self.snapshot_path, self.datadir) | ||
4840 | 122 | |||
4841 | 123 | def is_started(self): | ||
4842 | 124 | """ Is the server running? Particulars are server-dependent """ | ||
4843 | 125 | |||
4844 | 126 | return "You need to implement is_started" | ||
4845 | 127 | |||
4846 | 128 | def get_start_cmd(self): | ||
4847 | 129 | """ Return the command the server_manager can use to start me """ | ||
4848 | 130 | |||
4849 | 131 | return "You need to implement get_start_cmd" | ||
4850 | 132 | |||
4851 | 133 | def get_stop_cmd(self): | ||
4852 | 134 | """ Return the command the server_manager can use to stop me """ | ||
4853 | 135 | |||
4854 | 136 | return "You need to implement get_stop_cmd" | ||
4855 | 137 | |||
4856 | 138 | def get_ping_cmd(self): | ||
4857 | 139 | """ Return the command that can be used to 'ping' me | ||
4858 | 140 | Very similar to is_started, but different | ||
4859 | 141 | |||
4860 | 142 | Determining if a server is still running (ping) | ||
4861 | 143 | may differ from the method used to determine | ||
4862 | 144 | server startup | ||
4863 | 145 | |||
4864 | 146 | """ | ||
4865 | 147 | |||
4866 | 148 | return "You need to implement get_ping_cmd" | ||
4867 | 149 | |||
4868 | 150 | def cleanup(self): | ||
4869 | 151 | """ Cleanup - just free ports for now...""" | ||
4870 | 152 | self.system_manager.port_manager.free_ports(self.port_block) | ||
4871 | 153 | |||
4872 | 154 | def set_server_options(self, server_options): | ||
4873 | 155 | """ We update our server_options to the new set """ | ||
4874 | 156 | self.server_options = server_options | ||
4875 | 157 | |||
4876 | 158 | def reset(self): | ||
4877 | 159 | """ Voodoo to reset ourselves """ | ||
4878 | 160 | self.failed_test = 0 | ||
4879 | 161 | self.need_reset = False | ||
4880 | 162 | |||
4881 | 163 | def get_numeric_server_id(self): | ||
4882 | 164 | """ Return the integer value of server-id | ||
4883 | 165 | Mainly for mysql / percona, but may be useful elsewhere | ||
4884 | 166 | |||
4885 | 167 | """ | ||
4886 | 168 | |||
4887 | 169 | return int(self.name.split(self.server_manager.server_base_name)[1]) | ||
4888 | 170 | |||
4889 | 171 | 0 | ||
4890 | === removed file 'dbqp/lib/server_mgmt/server_management.py' | |||
4891 | --- dbqp/lib/server_mgmt/server_management.py 2012-02-04 01:22:23 +0000 | |||
4892 | +++ dbqp/lib/server_mgmt/server_management.py 1970-01-01 00:00:00 +0000 | |||
4893 | @@ -1,611 +0,0 @@ | |||
4894 | 1 | #! /usr/bin/env python | ||
4895 | 2 | # -*- mode: python; indent-tabs-mode: nil; -*- | ||
4896 | 3 | # vim:expandtab:shiftwidth=2:tabstop=2:smarttab: | ||
4897 | 4 | # | ||
4898 | 5 | # Copyright (C) 2010, 2011 Patrick Crews | ||
4899 | 6 | # | ||
4900 | 7 | # This program is free software; you can redistribute it and/or modify | ||
4901 | 8 | # it under the terms of the GNU General Public License as published by | ||
4902 | 9 | # the Free Software Foundation; either version 2 of the License, or | ||
4903 | 10 | # (at your option) any later version. | ||
4904 | 11 | # | ||
4905 | 12 | # This program is distributed in the hope that it will be useful, | ||
4906 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
4907 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
4908 | 15 | # GNU General Public License for more details. | ||
4909 | 16 | # | ||
4910 | 17 | # You should have received a copy of the GNU General Public License | ||
4911 | 18 | # along with this program; if not, write to the Free Software | ||
4912 | 19 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | ||
4913 | 20 | |||
4914 | 21 | """server_management.py | ||
4915 | 22 | code for dealing with apportioning servers | ||
4916 | 23 | to suit the needs of the tests and executors | ||
4917 | 24 | |||
4918 | 25 | """ | ||
4919 | 26 | # imports | ||
4920 | 27 | import thread | ||
4921 | 28 | import time | ||
4922 | 29 | import os | ||
4923 | 30 | import subprocess | ||
4924 | 31 | from ConfigParser import RawConfigParser | ||
4925 | 32 | |||
4926 | 33 | class serverManager: | ||
4927 | 34 | """ code that handles the server objects | ||
4928 | 35 | We use this to track, do mass actions, etc | ||
4929 | 36 | Single point of contact for this business | ||
4930 | 37 | |||
4931 | 38 | """ | ||
4932 | 39 | |||
4933 | 40 | def __init__(self, system_manager, variables): | ||
4934 | 41 | self.skip_keys = [ 'system_manager' | ||
4935 | 42 | , 'env_manager' | ||
4936 | 43 | , 'code_manager' | ||
4937 | 44 | , 'logging' | ||
4938 | 45 | , 'gdb' | ||
4939 | 46 | ] | ||
4940 | 47 | self.debug = variables['debug'] | ||
4941 | 48 | self.verbose = variables['verbose'] | ||
4942 | 49 | self.initial_run = 1 | ||
4943 | 50 | # we try this to shorten things - will see how this works | ||
4944 | 51 | self.server_base_name = 's' | ||
4945 | 52 | self.no_secure_file_priv = variables['nosecurefilepriv'] | ||
4946 | 53 | self.system_manager = system_manager | ||
4947 | 54 | self.code_manager = system_manager.code_manager | ||
4948 | 55 | self.env_manager = system_manager.env_manager | ||
4949 | 56 | self.logging = system_manager.logging | ||
4950 | 57 | self.gdb = self.system_manager.gdb | ||
4951 | 58 | self.default_storage_engine = variables['defaultengine'] | ||
4952 | 59 | self.default_server_type = variables['defaultservertype'] | ||
4953 | 60 | self.user_server_opts = variables['drizzledoptions'] | ||
4954 | 61 | self.servers = {} | ||
4955 | 62 | |||
4956 | 63 | self.mutex = thread.allocate_lock() | ||
4957 | 64 | self.timer_increment = .5 | ||
4958 | 65 | self.libeatmydata = variables['libeatmydata'] | ||
4959 | 66 | self.libeatmydata_path = variables['libeatmydatapath'] | ||
4960 | 67 | |||
4961 | 68 | self.logging.info("Using default-storage-engine: %s" %(self.default_storage_engine)) | ||
4962 | 69 | |||
4963 | 70 | self.logging.debug_class(self) | ||
4964 | 71 | |||
4965 | 72 | def request_servers( self, requester | ||
4966 | 73 | , workdir, cnf_path | ||
4967 | 74 | , server_requests | ||
4968 | 75 | , server_requirements | ||
4969 | 76 | , working_environ, expect_fail = 0): | ||
4970 | 77 | """ We produce the server objects / start the server processes | ||
4971 | 78 | as requested. We report errors and whatnot if we can't | ||
4972 | 79 | That is, unless we expect the server to not start, then | ||
4973 | 80 | we just return a value / message. | ||
4974 | 81 | |||
4975 | 82 | server_requirements is a list of lists. Each list | ||
4976 | 83 | is a set of server options - we create one server | ||
4977 | 84 | for each set of options requested | ||
4978 | 85 | |||
4979 | 86 | """ | ||
4980 | 87 | |||
4981 | 88 | # Make sure our server is in a decent state, if the last test | ||
4982 | 89 | # failed, then we reset the server | ||
4983 | 90 | self.check_server_status(requester) | ||
4984 | 91 | |||
4985 | 92 | # Make sure we have the proper number of servers for this requester | ||
4986 | 93 | self.process_server_count( requester, len(server_requirements) | ||
4987 | 94 | , workdir, server_requirements) | ||
4988 | 95 | |||
4989 | 96 | # Make sure we are running with the correct options | ||
4990 | 97 | self.evaluate_existing_servers( requester | ||
4991 | 98 | , cnf_path | ||
4992 | 99 | , server_requests | ||
4993 | 100 | , server_requirements) | ||
4994 | 101 | |||
4995 | 102 | # Fire our servers up | ||
4996 | 103 | bad_start = self.start_servers( requester, working_environ | ||
4997 | 104 | , expect_fail) | ||
4998 | 105 | |||
4999 | 106 | # Return them to the requester | ||
5000 | 107 | return (self.get_server_list(requester), bad_start) |
The diff has been truncated for viewing.