Merge lp:~jml/testtools/test-by-test-result into lp:~testtools-committers/testtools/trunk

[Jonathan Lange]

Add TestByTestResult

Adds TestByTestResult, which is already in subunit, to testtools.

Main problem with this one is that it adds yet another mapping of outcome to string. Starting to think that there should be some properly defined constants, or perhaps even objects.

[lp:~jml/testtools/test-by-test-result]

249. By Jonathan Lange on 2012-04-11

Better tag handling.

=== modified file 'NEWS'
--- NEWS	2012-04-03 03:28:29 +0000
+++ NEWS	2012-04-11 18:32:20 +0000
@@ -16,6 +16,13 @@
 * ``ErrorHolder`` is now just a function - all the logic is in ``PlaceHolder``.
   (Robert Collins)
 
+Improvements
+------------
+
+* ``TestByTestResult``, a ``TestResult`` that calls a method once per test,
+  added.  (Jonathan Lange)
+
+
 0.9.14
 ~~~~~~
 

=== modified file 'testtools/__init__.py'
--- testtools/__init__.py	2012-02-16 10:52:15 +0000
+++ testtools/__init__.py	2012-04-11 18:32:20 +0000
@@ -1,4 +1,4 @@
-# Copyright (c) 2008-2011 testtools developers. See LICENSE for details.
+# Copyright (c) 2008-2012 testtools developers. See LICENSE for details.
 
 """Extensions to the standard Python unittest library."""
 
@@ -16,6 +16,7 @@
     'run_test_with',
     'TestCase',
     'TestCommand',
+    'TestByTestResult',
     'TestResult',
     'TextTestResult',
     'RunTest',
@@ -55,6 +56,7 @@
 from testtools.testresult import (
     ExtendedToOriginalDecorator,
     MultiTestResult,
+    TestByTestResult,
     TestResult,
     TextTestResult,
     ThreadsafeForwardingResult,

=== modified file 'testtools/testresult/__init__.py'
--- testtools/testresult/__init__.py	2011-01-22 17:56:00 +0000
+++ testtools/testresult/__init__.py	2012-04-11 18:32:20 +0000
@@ -1,10 +1,11 @@
-# Copyright (c) 2009 testtools developers. See LICENSE for details.
+# Copyright (c) 2009-2012 testtools developers. See LICENSE for details.
 
 """Test result objects."""
 
 __all__ = [
     'ExtendedToOriginalDecorator',
     'MultiTestResult',
+    'TestByTestResult',
     'TestResult',
     'TextTestResult',
     'ThreadsafeForwardingResult',
@@ -13,6 +14,7 @@
 from testtools.testresult.real import (
     ExtendedToOriginalDecorator,
     MultiTestResult,
+    TestByTestResult,
     TestResult,
     TextTestResult,
     ThreadsafeForwardingResult,

=== modified file 'testtools/testresult/real.py'
--- testtools/testresult/real.py	2012-02-09 17:52:15 +0000
+++ testtools/testresult/real.py	2012-04-11 18:32:20 +0000
@@ -15,7 +15,10 @@
 import unittest
 
 from testtools.compat import all, str_is_unicode, _u
-from testtools.content import TracebackContent
+from testtools.content import (
+    text_content,
+    TracebackContent,
+    )
 
 # From http://docs.python.org/library/datetime.html
 _ZERO = datetime.timedelta(0)
@@ -624,6 +627,83 @@
         return self.decorated.wasSuccessful()
 
 
+class TestByTestResult(TestResult):
+    """Call something every time a test completes."""
+
+    def __init__(self, on_test):
+        """Construct a ``TestByTestResult``.
+
+        :param on_test: A callable that take a test case, a status (one of
+            "success", "failure", "error", "skip", or "xfail"), a start time
+            (a ``datetime`` with timezone), a stop time, an iterable of tags,
+            and a details dict. Is called at the end of each test (i.e. on
+            ``stopTest``) with the accumulated values for that test.
+        """
+        super(TestByTestResult, self).__init__()
+        self._on_test = on_test
+
+    def startTest(self, test):
+        super(TestByTestResult, self).startTest(test)
+        self._start_time = self._now()
+        # There's no supported (i.e. tested) behaviour that relies on these
+        # being set, but it makes me more comfortable all the same. -- jml
+        self._status = None
+        self._details = None
+        self._stop_time = None
+
+    def stopTest(self, test):
+        self._stop_time = self._now()
+        tags = set(self.current_tags)
+        super(TestByTestResult, self).stopTest(test)
+        self._on_test(
+            test=test,
+            status=self._status,
+            start_time=self._start_time,
+            stop_time=self._stop_time,
+            tags=tags,
+            details=self._details)
+
+    def _err_to_details(self, test, err, details):
+        if details:
+            return details
+        return {'traceback': TracebackContent(err, test)}
+
+    def addSuccess(self, test, details=None):
+        super(TestByTestResult, self).addSuccess(test)
+        self._status = 'success'
+        self._details = details
+
+    def addFailure(self, test, err=None, details=None):
+        super(TestByTestResult, self).addFailure(test, err, details)
+        self._status = 'failure'
+        self._details = self._err_to_details(test, err, details)
+
+    def addError(self, test, err=None, details=None):
+        super(TestByTestResult, self).addError(test, err, details)
+        self._status = 'error'
+        self._details = self._err_to_details(test, err, details)
+
+    def addSkip(self, test, reason=None, details=None):
+        super(TestByTestResult, self).addSkip(test, reason, details)
+        self._status = 'skip'
+        if details is None:
+            details = {'reason': text_content(reason)}
+        elif reason:
+            # XXX: What if details already has 'reason' key?
+            details['reason'] = text_content(reason)
+        self._details = details
+
+    def addExpectedFailure(self, test, err=None, details=None):
+        super(TestByTestResult, self).addExpectedFailure(test, err, details)
+        self._status = 'xfail'
+        self._details = self._err_to_details(test, err, details)
+
+    def addUnexpectedSuccess(self, test, details=None):
+        super(TestByTestResult, self).addUnexpectedSuccess(test, details)
+        self._status = 'success'
+        self._details = details
+
+
 class _StringException(Exception):
     """An exception made from an arbitrary string."""
 

=== modified file 'testtools/tests/test_testresult.py'
--- testtools/tests/test_testresult.py	2012-02-09 17:52:15 +0000
+++ testtools/tests/test_testresult.py	2012-04-11 18:32:20 +0000
@@ -1,4 +1,4 @@
-# Copyright (c) 2008-2011 testtools developers. See LICENSE for details.
+# Copyright (c) 2008-2012 testtools developers. See LICENSE for details.
 
 """Test TestResults and related things."""
 
@@ -19,6 +19,7 @@
     MultiTestResult,
     TestCase,
     TestResult,
+    TestByTestResult,
     TextTestResult,
     ThreadsafeForwardingResult,
     testresult,
@@ -1579,6 +1580,164 @@
 """))
 
 
+class TestByTestResultTests(TestCase):
+
+    def setUp(self):
+        super(TestByTestResultTests, self).setUp()
+        self.log = []
+        self.result = TestByTestResult(self.on_test)
+        self.result._now = iter(range(5)).next
+
+    def assertCalled(self, **kwargs):
+        defaults = {
+            'test': self,
+            'tags': set(),
+            'details': None,
+            'start_time': 0,
+            'stop_time': 1,
+            }
+        defaults.update(kwargs)
+        self.assertEqual([defaults], self.log)
+
+    def on_test(self, **kwargs):
+        self.log.append(kwargs)
+
+    def test_no_tests_nothing_reported(self):
+        self.result.startTestRun()
+        self.result.stopTestRun()
+        self.assertEqual([], self.log)
+
+    def test_add_success(self):
+        self.result.startTest(self)
+        self.result.addSuccess(self)
+        self.result.stopTest(self)
+        self.assertCalled(status='success')
+
+    def test_add_success_details(self):
+        self.result.startTest(self)
+        details = {'foo': 'bar'}
+        self.result.addSuccess(self, details=details)
+        self.result.stopTest(self)
+        self.assertCalled(status='success', details=details)
+
+    def test_global_tags(self):
+        self.result.tags(['foo'], [])
+        self.result.startTest(self)
+        self.result.addSuccess(self)
+        self.result.stopTest(self)
+        self.assertCalled(status='success', tags=set(['foo']))
+
+    def test_local_tags(self):
+        self.result.tags(['foo'], [])
+        self.result.startTest(self)
+        self.result.tags(['bar'], [])
+        self.result.addSuccess(self)
+        self.result.stopTest(self)
+        self.assertCalled(status='success', tags=set(['foo', 'bar']))
+
+    def test_add_error(self):
+        self.result.startTest(self)
+        try:
+            1/0
+        except ZeroDivisionError:
+            error = sys.exc_info()
+        self.result.addError(self, error)
+        self.result.stopTest(self)
+        self.assertCalled(
+            status='error',
+            details={'traceback': TracebackContent(error, self)})
+
+    def test_add_error_details(self):
+        self.result.startTest(self)
+        details = {"foo": text_content("bar")}
+        self.result.addError(self, details=details)
+        self.result.stopTest(self)
+        self.assertCalled(status='error', details=details)
+
+    def test_add_failure(self):
+        self.result.startTest(self)
+        try:
+            self.fail("intentional failure")
+        except self.failureException:
+            failure = sys.exc_info()
+        self.result.addFailure(self, failure)
+        self.result.stopTest(self)
+        self.assertCalled(
+            status='failure',
+            details={'traceback': TracebackContent(failure, self)})
+
+    def test_add_failure_details(self):
+        self.result.startTest(self)
+        details = {"foo": text_content("bar")}
+        self.result.addFailure(self, details=details)
+        self.result.stopTest(self)
+        self.assertCalled(status='failure', details=details)
+
+    def test_add_xfail(self):
+        self.result.startTest(self)
+        try:
+            1/0
+        except ZeroDivisionError:
+            error = sys.exc_info()
+        self.result.addExpectedFailure(self, error)
+        self.result.stopTest(self)
+        self.assertCalled(
+            status='xfail',
+            details={'traceback': TracebackContent(error, self)})
+
+    def test_add_xfail_details(self):
+        self.result.startTest(self)
+        details = {"foo": text_content("bar")}
+        self.result.addExpectedFailure(self, details=details)
+        self.result.stopTest(self)
+        self.assertCalled(status='xfail', details=details)
+
+    def test_add_unexpected_success(self):
+        self.result.startTest(self)
+        details = {'foo': 'bar'}
+        self.result.addUnexpectedSuccess(self, details=details)
+        self.result.stopTest(self)
+        self.assertCalled(status='success', details=details)
+
+    def test_add_skip_reason(self):
+        self.result.startTest(self)
+        reason = self.getUniqueString()
+        self.result.addSkip(self, reason)
+        self.result.stopTest(self)
+        self.assertCalled(
+            status='skip', details={'reason': text_content(reason)})
+
+    def test_add_skip_details(self):
+        self.result.startTest(self)
+        details = {'foo': 'bar'}
+        self.result.addSkip(self, details=details)
+        self.result.stopTest(self)
+        self.assertCalled(status='skip', details=details)
+
+    def test_twice(self):
+        self.result.startTest(self)
+        self.result.addSuccess(self, details={'foo': 'bar'})
+        self.result.stopTest(self)
+        self.result.startTest(self)
+        self.result.addSuccess(self)
+        self.result.stopTest(self)
+        self.assertEqual(
+            [{'test': self,
+              'status': 'success',
+              'start_time': 0,
+              'stop_time': 1,
+              'tags': set(),
+              'details': {'foo': 'bar'}},
+             {'test': self,
+              'status': 'success',
+              'start_time': 2,
+              'stop_time': 3,
+              'tags': set(),
+              'details': None},
+             ],
+            self.log)
+
+
 def test_suite():
     from unittest import TestLoader
     return TestLoader().loadTestsFromName(__name__)