Merge lp:~smerrill/pressflow/simpletest-xml-junit into lp:pressflow

[Steven Merrill]

[Steven Merrill (smerrill)]

This is an implementation of d.o #602332, which extends the work done on http://www.computerminds.co.uk/hudson-and-simpletest and brings it up to sync with Pressflow r49+ / SimpleTest 6.x-2.9+.

[Steven Merrill (smerrill)]

As an example, this is our after build script in Hudson that leverages this script to output XML test results into the scripts/tests directory:

/usr/bin/php scripts/run-tests.sh --url http://example.com --xml scripts/tests --all

[David Strauss (davidstrauss)]

I'm going to run this past the folks working on Hudson at The Economist (which includes me). We're certainly using jUnit XML results with Pressflow, and I'll see how this proposed merge differs from what we're using.

[David Strauss (davidstrauss)]

One note, though. This needs to stay in the Pressflow version because Pressflow uses the D7 CLI detection method:
simpletest_script_init(NULL);

[Josh Koenig (joshkoenig)]

This looks quite good to me, and is actually somewhat better than the legacy script we are using now.

I like the notion of reading test results from the database after they're complete rather than building in real-time.

I also like using DomDocument rather than SimpleXML as I've seen SimpleXML choke in some cases by being unreasonably strict.

I approve of this merge. I would also like to get Jimmy (boombatower) to think about including something like this in D7, but am happy if Pressflow leads on this. :)

review: Approve

[Josh Koenig (joshkoenig)]

Ah, I see it's on d.o already. Duh. +1'ed it there also.

Nice Work!

> This looks quite good to me, and is actually somewhat better than the legacy
> script we are using now.
>
> I like the notion of reading test results from the database after they're
> complete rather than building in real-time.
>
> I also like using DomDocument rather than SimpleXML as I've seen SimpleXML
> choke in some cases by being unreasonably strict.
>
> I approve of this merge. I would also like to get Jimmy (boombatower) to think
> about including something like this in D7, but am happy if Pressflow leads on
> this. :)

[lp:~smerrill/pressflow/simpletest-xml-junit]

60. By Steven Merrill on 2009-10-15

Code style cleanups.

61. By Steven Merrill on 2009-10-15

A few more small style tweaks.

62. By Steven Merrill on 2009-10-15

Change to use Pressflow/D7-style cli detection.

[Steven Merrill (smerrill)]

I've updated this branch to use the D7-style CLI detection and cleaned up the code style a bit, including removing the commented-out code.

I just double-checked it on a full Hudson-initiated run of all of our SimpleTests and it's working fine.

[David Strauss (davidstrauss)]

Merging now...

review: Approve

=== modified file 'scripts/run-tests.sh'
--- scripts/run-tests.sh	2009-09-20 21:01:42 +0000
+++ scripts/run-tests.sh	2009-10-15 00:29:09 +0000
@@ -98,8 +98,16 @@
 list($last_prefix, $last_test_class) = simpletest_last_test_get($test_id);
 simpletest_log_read($test_id, $last_prefix, $last_test_class);
 
-// Display results before database is cleared.
-simpletest_script_reporter_display_results();
+if ($args['xml']) {
+  echo $args['xml'];
+  // Save results as xml before database is cleared.
+  simpletest_script_reporter_xml_results();
+
+}
+else {
+  // Display results before database is cleared.
+  simpletest_script_reporter_display_results();
+}
 
 // Cleanup our test results.
 simpletest_clean_results_table($test_id);
@@ -149,6 +157,9 @@
 
   --verbose   Output detailed assertion messages in addition to summary.
 
+  --xml       Output verbose test results to a specified directory using the JUnit
+              test reporting format. Useful for integrating with Hudson.
+
   <test1>[,<test2>[,<test3> ...]]
 
               One or more tests to be run. By default, these are interpreted
@@ -194,7 +205,8 @@
     'test_names' => array(),
     // Used internally.
     'test-id' => NULL,
-    'execute-batch' => FALSE
+    'execute-batch' => FALSE,
+    'xml' => '',
   );
 
   // Override with set values.
@@ -274,7 +286,7 @@
   if (!empty($args['url'])) {
     $parsed_url = parse_url($args['url']);
     $host = $parsed_url['host'] . (isset($parsed_url['port']) ? ':' . $parsed_url['port'] : '');
-    $path = $parsed_url['path'];
+    $path = isset($parsed_url['path']) ? $parsed_url['path'] : '';
   }
 
   $_SERVER['HTTP_HOST'] = $host;
@@ -482,6 +494,75 @@
   echo "\n";
 }
 
+/*
+ * Display test results.
+ */
+function simpletest_script_reporter_xml_results() {
+  global $args, $test_id, $results_map;
+
+  echo "\n";
+  $end = timer_stop('run-tests');
+  echo "XML Test run duration: " . format_interval($end['time'] / 1000);
+  echo "\n";
+
+  // Report results.
+  echo "Detailed test results:\n";
+  echo "----------------------\n";
+  echo "\n";
+
+  $results_map = array(
+    'pass' => 'Pass',
+    'fail' => 'Fail',
+    'exception' => 'Exception',
+  );
+
+  $results = db_query("SELECT * FROM {simpletest} WHERE test_id = %d ORDER BY test_class, message_id", $test_id);
+
+  $test_class = '';
+  $xml_files = array();
+
+  while ($result = db_fetch_object($results)) {
+    if (isset($results_map[$result->status])) {
+      if ($result->test_class != $test_class) {
+        // Display test class every time results are for new test class.
+        if (isset($xml_files[$test_class])) {
+          file_put_contents($args['xml'] . '/' . $test_class . '.xml', $xml_files[$test_class]['doc']->saveXML());
+          unset($xml_files[$test_class]);
+        }
+        $test_class = $result->test_class;
+        if (!isset($xml_files[$test_class])) {
+          $doc = new DomDocument('1.0');
+          $root = $doc->createElement('testsuite');
+          $root = $doc->appendChild($root);
+          $xml_files[$test_class] = array('doc' => $doc, 'suite' => $root);
+        }
+      }
+      // Save the result into the XML:
+      $case = $xml_files[$test_class]['doc']->createElement('testcase');
+      $case->setAttribute('classname', $test_class);
+      list($class, $name) = explode('->', $result->function, 2);
+      $case->setAttribute('name', $name);
+
+      if ($result->status == 'fail') {
+        $fail = $xml_files[$test_class]['doc']->createElement('failure');
+        $fail->setAttribute('type', 'failure');
+        $fail->setAttribute('message', $result->message_group);
+        $text = $xml_files[$test_class]['doc']->createTextNode($result->message);
+        $fail->appendChild($text);
+        $case->appendChild($fail);
+      }
+
+      $xml_files[$test_class]['suite']->appendChild($case);
+    }
+  }
+
+  // Save the last one:
+  if (isset($xml_files[$test_class])) {
+    file_put_contents($args['xml'] . '/' . $test_class . '.xml', $xml_files[$test_class]['doc']->saveXML());
+    unset($xml_files[$test_class]);
+  }
+}
+ 
 /**
  * Display test results.
  */