Merge lp:~allenap/maas/tests-emit-subunit into lp:~maas-committers/maas/trunk

[Gavin Panella]

Make it possible to emit subunit from test runs.

The nose-subunit plugin at https://github.com/liucougar/nose-subunit seems to have bit-rotted and does not work for me. This is a very simple replacement that seems to work fine, but may need augmenting to cooperate with other plugins, but we can deal with those problems as we encounter them.

[Mike Pontillo (mpontillo)]

I want to approve this but I have no idea what it does and how to use it. ;-)

Can you explain more about what this branch is all about? Maybe add something to HACKING if appropriate?

review: Needs Information

[Gavin Panella (allenap)]

This adds a --with-subunit flag to nose. Subunit is an on-the-wire format for streaming test results. This stream can be used (with the subunit library) to manipulate a TestResult object elsewhere. In other words, it could be used to run a test in another process (perhaps on another machine) but capture results in nose locally without a significant loss of fidelity. This is part of an effort to run the test suite in parallel. This branch adds only the emission of subunit, optionally to another fd using --subunit-fd, so we can be sure of a clean stream.

I've added a paragraph to HACKING.txt.

[Mike Pontillo (mpontillo)]

Very nice! Thanks.

review: Approve

=== modified file 'HACKING.txt'
--- HACKING.txt	2016-12-07 12:46:14 +0000
+++ HACKING.txt	2016-12-20 07:56:25 +0000
@@ -108,6 +108,19 @@
    changing your locales on your workstation to ones present on the
    server will solve the issue.
 
+
+Emitting subunit
+^^^^^^^^^^^^^^^^
+
+Pass the ``--with-subunit`` flag to any of the test runners (e.g.
+``bin/test.rack``) to produce a `subunit`_ stream of test results. This
+may be useful for parallelising test runs, or to allow later analysis of
+a test run. The optional ``--subunit-fd`` flag can be used to direct the
+results to a different file descriptor, to ensure a clean stream.
+
+.. _subunit: https://launchpad.net/subunit/
+
+
 Running JavaScript tests
 ^^^^^^^^^^^^^^^^^^^^^^^^
 

=== modified file 'src/maastesting/noseplug.py'
--- src/maastesting/noseplug.py	2016-07-07 11:59:07 +0000
+++ src/maastesting/noseplug.py	2016-12-20 07:56:25 +0000
@@ -12,6 +12,7 @@
 ]
 
 import inspect
+import io
 import logging
 import unittest
 
@@ -261,6 +262,52 @@
         return inspect.getdoc(self)
 
 
+class Subunit(Plugin):
+    """Emit test results as a subunit stream."""
+
+    name = "subunit"
+    option_fd = "%s_fd" % name
+    log = logging.getLogger('nose.plugins.%s' % name)
+    score = 2000  # Run really early, beating even xunit.
+
+    def options(self, parser, env):
+        """Add options to Nose's parser.
+
+        :attention: This is part of the Nose plugin contract.
+        """
+        super(Subunit, self).options(parser, env)
+        parser.add_option(
+            "--%s-fd" % self.name, type=int,
+            dest=self.option_fd, action="store", default=1, help=(
+                "Emit subunit via a specific numeric file descriptor, "
+                "stdout (1) by default."
+            ),
+            metavar="FD",
+        )
+
+    def configure(self, options, conf):
+        """Configure, based on the parsed options.
+
+        :attention: This is part of the Nose plugin contract.
+        """
+        super(Subunit, self).configure(options, conf)
+        if self.enabled:
+            # Process --${name}-fd.
+            fd = getattr(options, self.option_fd)
+            self.stream = io.open(fd, "wb")
+
+    def prepareTestResult(self, result):
+        from subunit import TestProtocolClient
+        return TestProtocolClient(self.stream)
+
+    def help(self):
+        """Used in the --help text.
+
+        :attention: This is part of the Nose plugin contract.
+        """
+        return inspect.getdoc(self)
+
+
 def main():
     """Invoke Nose's `TestProgram` with extra plugins.
 
@@ -269,5 +316,5 @@
     flags ``--with-crochet``, ``--with-resources``, ``--with-scenarios``,
     and/or ``--with-select``.
     """
-    plugins = Crochet(), Resources(), Scenarios(), Select()
+    plugins = Crochet(), Resources(), Scenarios(), Select(), Subunit()
     return TestProgram(addplugins=plugins)

=== modified file 'src/maastesting/tests/test_noseplug.py'
--- src/maastesting/tests/test_noseplug.py	2016-07-07 11:59:07 +0000
+++ src/maastesting/tests/test_noseplug.py	2016-12-20 07:56:25 +0000
@@ -6,7 +6,10 @@
 __all__ = []
 
 from optparse import OptionParser
-from os import makedirs
+from os import (
+    devnull,
+    makedirs,
+)
 from os.path import (
     dirname,
     join,
@@ -30,14 +33,17 @@
     Resources,
     Scenarios,
     Select,
+    Subunit,
 )
 from maastesting.testcase import MAASTestCase
 import nose.case
+from subunit import TestProtocolClient
 from testresources import OptimisingTestSuite
 from testtools.matchers import (
     AllMatch,
     Equals,
     HasLength,
+    Is,
     IsInstance,
     MatchesListwise,
     MatchesSetwise,
@@ -356,6 +362,48 @@
             join(directory, factory.make_name("other-child"))))
 
 
+class TestSubunit(MAASTestCase):
+
+    def test__options_adds_options(self):
+        select = Subunit()
+        parser = OptionParser()
+        select.options(parser=parser, env={})
+        self.assertThat(
+            parser.option_list[-2:],
+            MatchesListwise([
+                # The --with-subunit option.
+                MatchesStructure.byEquality(
+                    action="store_true", default=None,
+                    dest="enable_plugin_subunit",
+                ),
+                # The --subunit-fd option.
+                MatchesStructure.byEquality(
+                    action="store", default=1, dest="subunit_fd",
+                    metavar="FD", type="int", _short_opts=[],
+                    _long_opts=["--subunit-fd"],
+                )
+            ]))
+
+    def test__configure_opens_stream(self):
+        subunit = Subunit()
+        parser = OptionParser()
+        subunit.add_options(parser=parser, env={})
+        with open(devnull, "wb") as fd:
+            options, rest = parser.parse_args(
+                ["--with-subunit", "--subunit-fd", str(fd.fileno())])
+            subunit.configure(options, sentinel.conf)
+            self.assertThat(subunit.stream.fileno(), Equals(fd.fileno()))
+            self.assertThat(subunit.stream.mode, Equals("wb"))
+
+    def test__prepareTestResult_returns_subunit_client(self):
+        subunit = Subunit()
+        with open(devnull, "wb") as stream:
+            subunit.stream = stream
+            result = subunit.prepareTestResult(sentinel.result)
+            self.assertThat(result, IsInstance(TestProtocolClient))
+            self.assertThat(result, MatchesStructure(_stream=Is(stream)))
+
+
 class TestMain(MAASTestCase):
 
     def test__sets_addplugins(self):
@@ -363,9 +411,10 @@
         noseplug.main()
         self.assertThat(
             noseplug.TestProgram,
-            MockCalledOnceWith(addplugins=(ANY, ANY, ANY, ANY)))
+            MockCalledOnceWith(addplugins=(ANY, ANY, ANY, ANY, ANY)))
         plugins = noseplug.TestProgram.call_args[1]["addplugins"]
         self.assertThat(plugins, MatchesSetwise(
             IsInstance(Crochet), IsInstance(Resources),
             IsInstance(Scenarios), IsInstance(Select),
+            IsInstance(Subunit),
         ))