Merge lp:~lifeless/subunit/addSkip into lp:~subunit/subunit/trunk

[Robert Collins]

[Robert Collins (lifeless)]

This includes filters; best to review after that is reviewed. Also when is that getting reviewed ? :)

[lp:~lifeless/subunit/addSkip]

58. By Robert Collins on 2009-03-01

subunit-filter can now filter skips too.

[Jonathan Lange (jml)]

Fine to land, along with the tweaks suggested for lp:~lifeless/subunit/filter.

review: Approve

=== modified file 'README'
--- README	2009-02-15 11:55:00 +0000
+++ README	2009-02-28 09:27:25 +0000
@@ -40,6 +40,8 @@
 stream on-the-fly. Currently subunit provides:
  * tap2subunit - convert perl's TestAnythingProtocol to subunit.
  * subunit2pyunit - convert a subunit stream to pyunit test results.
+ * subunit-filter - filter out tests from a subunit stream.
+ * subunit-ls - list the tests present in a subunit stream.
  * subunit-stats - generate a summary of a subunit stream.
  * subunit-tags - add or remove tags from a stream.
 
@@ -200,8 +202,10 @@
 Currently this is not exposed at the python API layer.
 
 The skip result is used to indicate a test that was found by the runner but not
-fully executed due to some policy or dependency issue. Currently this is
-represented in Python as a successful test.
+fully executed due to some policy or dependency issue. This is represented in
+python using the addSkip interface that testtools
+(https://edge.launchpad.net/testtools) defines. When communicating with a non
+skip aware test result, the test is reported as an error.
 The xfail result is used to indicate a test that was expected to fail failing
 in the expected manner. As this is a normal condition for such tests it is
 represented as a successful test in Python.

=== added file 'filters/subunit-filter'
--- filters/subunit-filter	1970-01-01 00:00:00 +0000
+++ filters/subunit-filter	2009-02-22 07:51:04 +0000
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+#  subunit: extensions to python unittest to get test results from subprocesses.
+#  Copyright (C) 2008  Robert Collins <[e-mail]>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+
+"""Filter a subunit stream to include/exclude tests.
+
+The default is to strip successful tests.
+"""
+
+from optparse import OptionParser
+import sys
+import unittest
+
+from subunit import ProtocolTestCase, TestResultFilter, TestProtocolClient
+
+parser = OptionParser(description=__doc__)
+parser.add_option("--error", action="store_false",
+    help="include errors", default=False, dest="error")
+parser.add_option("-e", "--no-error", action="store_true",
+    help="exclude errors", dest="error")
+parser.add_option("--failure", action="store_false",
+    help="include failures", default=False, dest="failure")
+parser.add_option("-f", "--no-failure", action="store_true",
+    help="include failures", dest="failure")
+parser.add_option("-s", "--success", action="store_false",
+    help="include successes", dest="success")
+parser.add_option("--no-success", action="store_true",
+    help="exclude successes", default=True, dest="success")
+(options, args) = parser.parse_args()
+result = TestProtocolClient(sys.stdout)
+result = TestResultFilter(result, filter_error=options.error, filter_failure=options.failure,
+    filter_success=options.success)
+test = ProtocolTestCase(sys.stdin)
+test.run(result)
+sys.exit(0)

=== added file 'filters/subunit-ls'
--- filters/subunit-ls	1970-01-01 00:00:00 +0000
+++ filters/subunit-ls	2009-02-23 10:54:28 +0000
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+#  subunit: extensions to python unittest to get test results from subprocesses.
+#  Copyright (C) 2008  Robert Collins <[e-mail]>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+
+"""List tests in a subunit stream."""
+
+import sys
+import unittest
+
+from subunit import ProtocolTestCase
+
+class FilterResult(unittest.TestResult):
+    """Filter test objects for display."""
+
+    def __init__(self, stream):
+        """Create a FilterResult object outputting to stream."""
+        unittest.TestResult.__init__(self)
+        self._stream = stream
+        self.failed_tests = 0
+        
+    def addError(self, test, err):
+        self.failed_tests += 1
+        self.reportTest(test)
+
+    def addFailure(self, test, err):
+        self.failed_tests += 1
+        self.reportTest(test)
+
+    def addSuccess(self, test):
+        self.reportTest(test)
+
+    def reportTest(self, test):
+        self._stream.write(test.id() + '\n')
+
+    def wasSuccessful(self):
+        "Tells whether or not this result was a success"
+        return self.failed_tests == 0
+
+
+result = FilterResult(sys.stdout)
+test = ProtocolTestCase(sys.stdin)
+test.run(result)
+if result.wasSuccessful():
+    exit_code = 0
+else:
+    exit_code = 1
+sys.exit(exit_code)

=== modified file 'python/subunit/__init__.py'
--- python/subunit/__init__.py	2009-02-15 11:55:00 +0000
+++ python/subunit/__init__.py	2009-02-28 09:27:25 +0000
@@ -133,7 +133,7 @@
             self.current_test_description == line[offset:-1]):
             self.state = TestProtocolServer.OUTSIDE_TEST
             self.current_test_description = None
-            self.client.addSuccess(self._current_test)
+            self._skip_or_error()
             self.client.stopTest(self._current_test)
         elif (self.state == TestProtocolServer.TEST_STARTED and
             self.current_test_description + " [" == line[offset:-1]):
@@ -142,6 +142,16 @@
         else:
             self.stdOutLineReceived(line)
 
+    def _skip_or_error(self, message=None):
+        """Report the current test as a skip if possible, or else an error."""
+        addSkip = getattr(self.client, 'addSkip', None)
+        if not callable(addSkip):
+            self.client.addError(self._current_test, RemoteError(message))
+        else:
+            if not message:
+                message = "No reason given"
+            addSkip(self._current_test, message)
+
     def _addSuccess(self, offset, line):
         if (self.state == TestProtocolServer.TEST_STARTED and
             self.current_test_description == line[offset:-1]):
@@ -173,8 +183,12 @@
             self.client.addError(self._current_test,
                                  RemoteError(self._message))
             self.client.stopTest(self._current_test)
+        elif self.state == TestProtocolServer.READING_SKIP:
+            self.state = TestProtocolServer.OUTSIDE_TEST
+            self.current_test_description = None
+            self._skip_or_error(self._message)
+            self.client.stopTest(self._current_test)
         elif self.state in (
-            TestProtocolServer.READING_SKIP,
             TestProtocolServer.READING_SUCCESS,
             TestProtocolServer.READING_XFAIL,
             ):
@@ -314,6 +328,12 @@
             self._stream.write("%s\n" % line)
         self._stream.write("]\n")
 
+    def addSkip(self, test, reason):
+        """Report a skipped test."""
+        self._stream.write("skip: %s [\n" % test.id())
+        self._stream.write("%s\n" % reason)
+        self._stream.write("]\n")
+
     def addSuccess(self, test):
         """Report a success in a test."""
         self._stream.write("successful: %s\n" % test.id())
@@ -363,7 +383,7 @@
         return self.__description
 
     def id(self):
-        return "%s.%s" % (self._strclass(), self.__description)
+        return "%s" % (self.__description,)
 
     def __str__(self):
         return "%s (%s)" % (self.__description, self._strclass())
@@ -651,6 +671,7 @@
         unittest.TestResult.__init__(self)
         self._stream = stream
         self.failed_tests = 0
+        self.skipped_tests = 0
         self.tags = set()
 
     @property
@@ -663,16 +684,20 @@
     def addFailure(self, test, err):
         self.failed_tests += 1
 
+    def addSkip(self, test, reason):
+        self.skipped_tests += 1
+
     def formatStats(self):
-        self._stream.write("Total tests:  %5d\n" % self.total_tests)
-        self._stream.write("Passed tests: %5d\n" % self.passed_tests)
-        self._stream.write("Failed tests: %5d\n" % self.failed_tests)
+        self._stream.write("Total tests:   %5d\n" % self.total_tests)
+        self._stream.write("Passed tests:  %5d\n" % self.passed_tests)
+        self._stream.write("Failed tests:  %5d\n" % self.failed_tests)
+        self._stream.write("Skipped tests: %5d\n" % self.skipped_tests)
         tags = sorted(self.tags)
         self._stream.write("Tags: %s\n" % (", ".join(tags)))
 
     @property
     def passed_tests(self):
-        return self.total_tests - self.failed_tests
+        return self.total_tests - self.failed_tests - self.skipped_tests
 
     def stopTest(self, test):
         unittest.TestResult.stopTest(self, test)
@@ -681,3 +706,66 @@
     def wasSuccessful(self):
         """Tells whether or not this result was a success"""
         return self.failed_tests == 0
+
+
+class TestResultFilter(unittest.TestResult):
+    """A pyunit TestResult interface implementation which filters tests.
+
+    Tests that pass the filter are handed on to another TestResult instance
+    for further processing/reporting. To obtain the filtered results, 
+    the other instance must be interrogated.
+
+    :ivar result: The result that tests are passed to after filtering.
+    """
+
+    def __init__(self, result, filter_error=False, filter_failure=False,
+        filter_success=True, filter_skip=False):
+        """Create a FilterResult object filtering to result.
+        
+        :param filter_error: Filter out errors.
+        :param filter_failure: Filter out failures.
+        :param filter_success: Filter out successful tests.
+        :param filter_skip: Filter out skipped tests.
+        """
+        unittest.TestResult.__init__(self)
+        self.result = result
+        self._filter_error = filter_error
+        self._filter_failure = filter_failure
+        self._filter_success = filter_success
+        self._filter_skip = filter_skip
+        
+    def addError(self, test, err):
+        if not self._filter_error:
+            self.result.startTest(test)
+            self.result.addError(test, err)
+            self.result.stopTest(test)
+
+    def addFailure(self, test, err):
+        if not self._filter_failure:
+            self.result.startTest(test)
+            self.result.addFailure(test, err)
+            self.result.stopTest(test)
+
+    def addSkip(self, test, reason):
+        if not self._filter_skip:
+            self.result.startTest(test)
+            # This is duplicated, it would be nice to have on a 'calls
+            # TestResults' mixin perhaps.
+            addSkip = getattr(self.result, 'addSkip', None)
+            if not callable(addSkip):
+                self.result.addError(test, RemoteError(reason))
+            else:
+                self.result.addSkip(test, reason)
+            self.result.stopTest(test)
+
+    def addSuccess(self, test):
+        if not self._filter_success:
+            self.result.startTest(test)
+            self.result.addSuccess(test)
+            self.result.stopTest(test)
+
+    def id_to_orig_id(self, id):
+        if id.startswith("subunit.RemotedTestCase."):
+            return id[len("subunit.RemotedTestCase."):]
+        return id
+

=== modified file 'python/subunit/tests/__init__.py'
--- python/subunit/tests/__init__.py	2008-12-09 01:00:03 +0000
+++ python/subunit/tests/__init__.py	2009-02-22 06:28:08 +0000
@@ -19,6 +19,7 @@
 
 from subunit.tests import (
     TestUtil,
+    test_subunit_filter,
     test_subunit_stats,
     test_subunit_tags,
     test_tap2subunit,
@@ -29,6 +30,7 @@
     result = TestUtil.TestSuite()
     result.addTest(test_test_protocol.test_suite())
     result.addTest(test_tap2subunit.test_suite())
+    result.addTest(test_subunit_filter.test_suite())
     result.addTest(test_subunit_tags.test_suite())
     result.addTest(test_subunit_stats.test_suite())
     return result

=== added file 'python/subunit/tests/test_subunit_filter.py'
--- python/subunit/tests/test_subunit_filter.py	1970-01-01 00:00:00 +0000
+++ python/subunit/tests/test_subunit_filter.py	2009-02-28 09:27:25 +0000
@@ -0,0 +1,125 @@
+#
+#  subunit: extensions to python unittest to get test results from subprocesses.
+#  Copyright (C) 2005  Robert Collins <[e-mail]>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+
+"""Tests for subunit.TestResultFilter."""
+
+import unittest
+from StringIO import StringIO
+
+import subunit
+
+
+class TestTestResultFilter(unittest.TestCase):
+    """Test for TestResultFilter, a TestResult object which filters tests."""
+
+    def _setUp(self):
+        self.output = StringIO()
+
+    def test_default(self):
+        """The default is to exclude success and include everything else."""
+        self.filtered_result = unittest.TestResult()
+        self.filter = subunit.TestResultFilter(self.filtered_result)
+        self.run_tests()
+        # skips are seen as errors by default python TestResult.
+        self.assertEqual(['error', 'skipped'],
+            [error[0].id() for error in self.filtered_result.errors])
+        self.assertEqual(['failed'],
+            [failure[0].id() for failure in
+            self.filtered_result.failures])
+        self.assertEqual(3, self.filtered_result.testsRun)
+
+    def test_exclude_errors(self):
+        self.filtered_result = unittest.TestResult()
+        self.filter = subunit.TestResultFilter(self.filtered_result,
+            filter_error=True)
+        self.run_tests()
+        # skips are seen as errors by default python TestResult.
+        self.assertEqual(['skipped'],
+            [error[0].id() for error in self.filtered_result.errors])
+        self.assertEqual(['failed'],
+            [failure[0].id() for failure in
+            self.filtered_result.failures])
+        self.assertEqual(2, self.filtered_result.testsRun)
+
+    def test_exclude_failure(self):
+        self.filtered_result = unittest.TestResult()
+        self.filter = subunit.TestResultFilter(self.filtered_result,
+            filter_failure=True)
+        self.run_tests()
+        self.assertEqual(['error', 'skipped'],
+            [error[0].id() for error in self.filtered_result.errors])
+        self.assertEqual([],
+            [failure[0].id() for failure in
+            self.filtered_result.failures])
+        self.assertEqual(2, self.filtered_result.testsRun)
+
+    def test_exclude_skips(self):
+        self.filtered_result = subunit.TestResultStats(None)
+        self.filter = subunit.TestResultFilter(self.filtered_result,
+            filter_skip=True)
+        self.run_tests()
+        self.assertEqual(0, self.filtered_result.skipped_tests)
+        self.assertEqual(2, self.filtered_result.failed_tests)
+        self.assertEqual(2, self.filtered_result.testsRun)
+
+    def test_include_success(self):
+        """Success's can be included if requested."""
+        self.filtered_result = unittest.TestResult()
+        self.filter = subunit.TestResultFilter(self.filtered_result,
+            filter_success=False)
+        self.run_tests()
+        self.assertEqual(['error', 'skipped'],
+            [error[0].id() for error in self.filtered_result.errors])
+        self.assertEqual(['failed'],
+            [failure[0].id() for failure in
+            self.filtered_result.failures])
+        self.assertEqual(5, self.filtered_result.testsRun)
+
+    def run_tests(self):
+        self.setUpTestStream()
+        self.test = subunit.ProtocolTestCase(self.input_stream)
+        self.test.run(self.filter)
+
+    def setUpTestStream(self):
+        # While TestResultFilter works on python objects, using a subunit
+        # stream is an easy pithy way of getting a series of test objects to
+        # call into the TestResult, and as TestResultFilter is intended for use
+        # with subunit also has the benefit of detecting any interface skew issues.
+        self.input_stream = StringIO()
+        self.input_stream.write("""tags: global
+test passed
+success passed
+test failed
+tags: local
+failure failed
+test error
+error error
+test skipped
+skip skipped
+test todo
+xfail todo
+""")
+        self.input_stream.seek(0)
+    
+
+
+def test_suite():
+    loader = subunit.tests.TestUtil.TestLoader()
+    result = loader.loadTestsFromName(__name__)
+    return result

=== modified file 'python/subunit/tests/test_subunit_stats.py'
--- python/subunit/tests/test_subunit_stats.py	2008-12-09 01:00:03 +0000
+++ python/subunit/tests/test_subunit_stats.py	2009-02-28 09:27:25 +0000
@@ -62,15 +62,17 @@
         # Statistics are calculated usefully.
         self.setUpUsedStream()
         self.assertEqual(5, self.result.total_tests)
-        self.assertEqual(3, self.result.passed_tests)
+        self.assertEqual(2, self.result.passed_tests)
         self.assertEqual(2, self.result.failed_tests)
+        self.assertEqual(1, self.result.skipped_tests)
         self.assertEqual(set(["global", "local"]), self.result.tags)
 
     def test_stat_formatting(self):
         expected = ("""
-Total tests:      5
-Passed tests:     3
-Failed tests:     2
+Total tests:       5
+Passed tests:      2
+Failed tests:      2
+Skipped tests:     1
 Tags: global, local
 """)[1:]
         self.setUpUsedStream()

=== modified file 'python/subunit/tests/test_test_protocol.py'
--- python/subunit/tests/test_test_protocol.py	2008-12-14 18:37:08 +0000
+++ python/subunit/tests/test_test_protocol.py	2009-02-28 09:27:25 +0000
@@ -32,6 +32,7 @@
             self.end_calls = []
             self.error_calls = []
             self.failure_calls = []
+            self.skip_calls = []
             self.start_calls = []
             self.success_calls = []
             super(MockTestProtocolServerClient, self).__init__()
@@ -42,6 +43,9 @@
         def addFailure(self, test, error):
             self.failure_calls.append((test, error))
 
+        def addSkip(self, test, reason):
+            self.skip_calls.append((test, reason))
+
         def addSuccess(self, test):
             self.success_calls.append(test)
 
@@ -589,8 +593,8 @@
 class TestTestProtocolServerAddSkip(unittest.TestCase):
     """Tests for the skip keyword.
 
-    In Python this thunks through to Success due to stdlib limitations. (See
-    README).
+    In python this meets the testtools extended TestResult contract.
+    (See https://launchpad.net/testtools).
     """
 
     def setUp(self):
@@ -606,7 +610,9 @@
         self.assertEqual(self.client.end_calls, [self.test])
         self.assertEqual(self.client.error_calls, [])
         self.assertEqual(self.client.failure_calls, [])
-        self.assertEqual(self.client.success_calls, [self.test])
+        self.assertEqual(self.client.success_calls, [])
+        self.assertEqual(self.client.skip_calls,
+            [(self.test, 'No reason given')])
 
     def test_simple_skip(self):
         self.simple_skip_keyword("skip")
@@ -621,7 +627,9 @@
         self.assertEqual(self.client.end_calls, [self.test])
         self.assertEqual(self.client.error_calls, [])
         self.assertEqual(self.client.failure_calls, [])
-        self.assertEqual(self.client.success_calls, [self.test])
+        self.assertEqual(self.client.success_calls, [])
+        self.assertEqual(self.client.skip_calls,
+            [(self.test, "No reason given")])
 
     def skip_quoted_bracket(self, keyword):
         # This tests it is accepted, but cannot test it is used today, because
@@ -633,7 +641,9 @@
         self.assertEqual(self.client.end_calls, [self.test])
         self.assertEqual(self.client.error_calls, [])
         self.assertEqual(self.client.failure_calls, [])
-        self.assertEqual(self.client.success_calls, [self.test])
+        self.assertEqual(self.client.success_calls, [])
+        self.assertEqual(self.client.skip_calls,
+            [(self.test, "]\n")])
 
     def test_skip_quoted_bracket(self):
         self.skip_quoted_bracket("skip")
@@ -772,7 +782,7 @@
         self.assertRaises(NotImplementedError, test.tearDown)
         self.assertEqual("A test description",
                          test.shortDescription())
-        self.assertEqual("subunit.RemotedTestCase.A test description",
+        self.assertEqual("A test description",
                          test.id())
         self.assertEqual("A test description (subunit.RemotedTestCase)", "%s" % test)
         self.assertEqual("<subunit.RemotedTestCase description="
@@ -933,7 +943,6 @@
         self.protocol = subunit.TestProtocolClient(self.io)
         self.test = TestTestProtocolClient("test_start_test")
 
-
     def test_start_test(self):
         """Test startTest on a TestProtocolClient."""
         self.protocol.startTest(self.test)
@@ -968,6 +977,14 @@
             "RemoteException: phwoar crikey\n"
             "]\n" % self.test.id())
 
+    def test_add_skip(self):
+        """Test addSkip on a TestProtocolClient."""
+        self.protocol.addSkip(
+            self.test, "Has it really?")
+        self.assertEqual(
+            self.io.getvalue(),
+            'skip: %s [\nHas it really?\n]\n' % self.test.id())
+
 
 def test_suite():
     loader = subunit.tests.TestUtil.TestLoader()