Merge lp:~sidnei/lazr.testing/yeti into lp:lazr.testing

[Sidnei da Silva]

- Adds a YetiLayer and YetiTestCase, which use a custom branch of yeti
  with subunit output format to better integrate into unittest.

[Paul Hummer (rockstar)]

sidnei, once again, I bow to your prowess.

Land this kthxbai! :)

review: Approve

=== modified file 'src/lazr/testing/tests/test_js.py'
--- src/lazr/testing/tests/test_js.py	2010-09-03 15:03:15 +0000
+++ src/lazr/testing/tests/test_js.py	2010-12-23 16:11:33 +0000
@@ -1,10 +1,12 @@
+import doctest
+import operator
 import os
 import re
-import unittest
-import doctest
 import socket
-import operator
 import sys
+import unittest
+import warnings
+
 from cStringIO import StringIO
 
 from os.path import dirname
@@ -331,6 +333,11 @@
 def test_suite():
     suite = unittest.TestSuite()
 
+    if not "JSTESTDRIVER" in os.environ:
+        warnings.warn("Environment variable 'JSTESTDRIVER' not set. "
+                      "Skipping 'JSTESTDRIVER' tests.")
+        return suite
+
     if os.environ.get("JSTESTDRIVER_SELFTEST"):
         suite.addTests(unittest.makeSuite(JsTestDriverSelfTest))
     else:

=== added file 'src/lazr/testing/tests/test_yeti.py'
--- src/lazr/testing/tests/test_yeti.py	1970-01-01 00:00:00 +0000
+++ src/lazr/testing/tests/test_yeti.py	2010-12-23 16:11:33 +0000
@@ -0,0 +1,272 @@
+import doctest
+import operator
+import os
+import re
+import socket
+import sys
+import unittest
+import warnings
+
+from cStringIO import StringIO
+
+from os.path import dirname
+
+from mocker import ANY, ARGS, KWARGS, MockerTestCase
+
+from zope.testing import testrunner
+
+from lazr.testing.yeti import YetiLayer
+
+
+class YetiLayerErrorTests(MockerTestCase):
+
+    def setUp(self):
+        super(YetiLayerErrorTests, self).setUp()
+        env_keys = [
+            "YETI",
+            "YETI_SERVER",
+            "YETI_PORT",
+            "YETI_CAPTURE_TIMEOUT",
+            "YETI_BROWSER"]
+
+        def cleanup_non_existing_key(some_key):
+            try:
+                del os.environ[some_key]
+            except KeyError:
+                pass
+
+        for key in env_keys:
+            if key in os.environ:
+                self.addCleanup(
+                    operator.setitem, os.environ, key, os.environ[key])
+            else:
+                self.addCleanup(cleanup_non_existing_key, key)
+
+    def test_binding(self):
+        """
+        If a specific port is requested, and a server is already started in the
+        requested port, then the layer setup fails.
+        """
+        s = socket.socket()
+        try:
+            s.bind((socket.gethostbyname(""), 4225))
+        except socket.error:
+            s.close()
+            s = None
+            raise
+
+        if "YETI_SERVER" in os.environ:
+            del os.environ["YETI_SERVER"]
+        os.environ["YETI_PORT"] = "4225"
+        try:
+            try:
+                YetiLayer.setUp()
+            except ValueError, e:
+                msg = str(e)
+                self.assertIn(
+                    "Failed to execute Yeti server on port 4225", msg)
+                self.assertIn(
+                    "Address already in use", msg)
+            else:
+                self.fail("ValueError not raised")
+        finally:
+            YetiLayer.tearDown()
+            s.close()
+
+    def mock_popen(self):
+        """Replace subprocess.Popen and make it return a mock process.
+
+        The mock process is returned.
+        """
+        mock_Popen = self.mocker.replace("subprocess.Popen")
+        self.mock_proc = self.mocker.mock()
+        mock_Popen(ARGS, KWARGS)
+        self.mocker.result(self.mock_proc)
+        return self.mock_proc
+
+    def mock_builtin_open(self):
+        """Replace built-in open and make it return a mock file.
+
+        The mock file is returned.
+        """
+        mock_open = self.mocker.replace("__builtin__.open")
+        mock_open(ANY)
+        self.mock_file = self.mocker.mock()
+        self.mocker.result(self.mock_file)
+        return self.mock_file
+
+    def test_wait_for_server_startup(self):
+        """
+        Even if we don't wait for the browser to be captured, we wait
+        for the server to start up.
+        """
+        mock_proc = self.mock_popen()
+        mock_file = self.mock_builtin_open()
+
+        with self.mocker.order():
+            mock_time = self.mocker.replace("time.time")
+            # The first time is to initialize the start time.
+            mock_time()
+            start_time = 0
+            self.mocker.result(start_time)
+
+            # The second time is to check if the timeout is exceeded in
+            # the while loop.
+            mock_time()
+            self.mocker.result(start_time)
+            # Go one iteration of the while loop, reporting the server
+            # has started.
+            mock_proc.poll()
+            self.mocker.result(None)
+            mock_file.readline()
+            self.mocker.result("to run and report the results")
+
+            # The opened file is closed.
+            mock_file.close()
+            self.mocker.result(None)
+
+            # Last check to make sure the server is running ok.
+            mock_proc.poll()
+            self.mocker.result(None)
+
+        self.mocker.replay()
+
+        os.environ["YETI_BROWSER"] = ""
+        if "YETI_SERVER" in os.environ:
+            del os.environ["YETI_SERVER"]
+        os.environ["YETI_PORT"] = "4225"
+
+        YetiLayer.setUp()
+        self.assertEqual(
+            "http://localhost:4225", os.environ["YETI_SERVER"])
+
+    def test_server_fail(self):
+        """
+        If we a poll of the process returns a non-None value while we
+        are waiting, we report that server couldn't be started.
+        """
+        mock_proc = self.mock_popen()
+        mock_file = self.mock_builtin_open()
+
+        with self.mocker.order():
+            mock_time = self.mocker.replace("time.time")
+            # The first time is to initialize the start time.
+            mock_time()
+            start_time = 0
+            self.mocker.result(start_time)
+
+            # The second time is to check if the timeout is exceeded in
+            # the while loop.
+            mock_time()
+            self.mocker.result(start_time)
+            # Go one iteration of the while loop, reporting the server
+            # is starting up.
+            mock_proc.poll()
+            self.mocker.result(None)
+            mock_file.readline()
+            self.mocker.result("not yeti?")
+
+            # Go another iteration of the while loop, reporting the
+            # server failed to start up.
+            mock_time()
+            self.mocker.result(start_time)
+            mock_proc.poll()
+            self.mocker.result(1)
+
+            # The opened file is closed.
+            mock_file.close()
+            self.mocker.result(None)
+
+        self.mocker.replay()
+
+        if "YETI_SERVER" in os.environ:
+            del os.environ["YETI_SERVER"]
+        os.environ["YETI_PORT"] = "4225"
+
+        try:
+            YetiLayer.setUp()
+        except ValueError, e:
+            msg = str(e)
+            self.assertIn(
+                "Failed to execute Yeti server on port 4225", msg)
+        else:
+            self.fail("ValueError not raised")
+
+    def test_server_timeout(self):
+        """
+        If we don't see that the server is started before the timeout, a
+        ValueError is raised, even if the process is still running.
+        """
+        timeout = 1
+        os.environ["YETI_CAPTURE_TIMEOUT"] = "%s" % timeout
+        os.environ["YETI_BROWSER"] = ""
+        if "YETI_SERVER" in os.environ:
+            del os.environ["YETI_SERVER"]
+        os.environ["YETI_PORT"] = "4225"
+
+        mock_proc = self.mock_popen()
+        mock_file = self.mock_builtin_open()
+
+        with self.mocker.order():
+            mock_time = self.mocker.replace("time.time")
+            # The first time is to initialize the start time.
+            mock_time()
+            start_time = 0
+            self.mocker.result(start_time)
+
+            # The second time is to check if the timeout is exceeded in
+            # the while loop.
+            mock_time()
+            self.mocker.result(start_time)
+            # Go one iteration of the while loop, reporting the server
+            # is starting up.
+            mock_proc.poll()
+            self.mocker.result(None)
+            mock_file.readline()
+            self.mocker.result("not yeti?")
+
+            # Trigger a timeout.
+            mock_time()
+            self.mocker.result(start_time + timeout + 1)
+
+            # The opened file is closed.
+            mock_file.close()
+            self.mocker.result(None)
+
+            # Last check whether the server is still running.
+            mock_proc.poll()
+            self.mocker.result(None)
+
+            # Since the server is running, it gets terminated.
+            mock_proc.terminate()
+            self.mocker.result(None)
+            mock_proc.wait()
+            self.mocker.result(None)
+
+        self.mocker.replay()
+
+        try:
+            YetiLayer.setUp()
+        except ValueError, e:
+            msg = str(e)
+            self.assertIn(
+                "Failed to execute Yeti server in 1 seconds"
+                " on port 4225", msg)
+        else:
+            self.fail("ValueError not raised")
+
+    def tearDown(self):
+        super(YetiLayerErrorTests, self).tearDown()
+        self.mocker.restore()
+
+
+def test_suite():
+    suite = unittest.TestSuite()
+
+    if not "YETI" in os.environ:
+        warnings.warn("Environment variable 'YETI' not set. "
+                      "Skipping 'YETI' tests.")
+        return suite
+
+    suite.addTests(unittest.makeSuite(YetiLayerErrorTests))
+    return suite

=== added file 'src/lazr/testing/yeti.py'
--- src/lazr/testing/yeti.py	1970-01-01 00:00:00 +0000
+++ src/lazr/testing/yeti.py	2010-12-23 16:11:33 +0000
@@ -0,0 +1,149 @@
+import os
+import time
+import fnmatch
+import signal
+import tempfile
+import subprocess
+
+from unittest import TestCase
+from subunit import ProtocolTestCase
+
+
+def startYeti():
+    yeti = os.environ["YETI"]
+    port = os.environ.get("YETI_PORT", "4422")
+
+    capture_timeout = int(os.environ.get(
+        "YETI_CAPTURE_TIMEOUT", "30"))
+
+    cmd = yeti.split() + ["--port", port, "--server"]
+
+    browser = os.environ.get("YETI_BROWSER", "default")
+
+    if browser:
+        cmd.extend(["--browsers", browser])
+
+    # Redirect stderr through a temporary file, so that it doesn't
+    # block and we don't get an IOError on readline(), apparently
+    # caused by an unhandled SIGINT (Google for it. :)
+    fd, name = tempfile.mkstemp()
+    stderr = open(name)
+    server_started = False
+    rc = None
+
+    try:
+        proc = subprocess.Popen(cmd,
+                                shell=False,
+                                stdin=subprocess.PIPE,
+                                stdout=subprocess.PIPE,
+                                stderr=fd,
+                                close_fds=True)
+
+        # Give the server process a few seconds to start, and
+        # capture the browser if needed.
+        output = []
+        start = time.time()
+        while time.time() - start < capture_timeout:
+            rc = proc.poll()
+            if rc is not None:
+                break
+            line = stderr.readline()
+            if not line:
+                continue
+            output.append(line)
+            if line.startswith("Running tests locally with:"):
+                server_started = True
+                break
+            if line.startswith("to run and report the results"):
+                server_started = True
+                break
+    finally:
+        stderr.close()
+
+    if rc is None:
+        rc = proc.poll()
+    if rc is not None:
+        raise ValueError(
+            "Failed to execute Yeti server on port %s:"
+            "\nError: (%s) %s" %
+            (port, rc, "\n".join(output)))
+    if not server_started:
+        terminateProcess(proc)
+        raise ValueError(
+            "Failed to execute Yeti server in %d seconds on port %s:"
+            "\nError: (%s) %s" %
+            (capture_timeout, port, rc, "\n".join(output)))
+    else:
+        os.environ["YETI_SERVER"] = (
+            "http://localhost:%s" % port)
+    return proc
+
+
+def terminateProcess(proc):
+    try:
+        proc.terminate()
+    except AttributeError:
+        os.kill(proc.pid, signal.SIGTERM)
+    proc.wait()
+
+
+class YetiLayer(object):
+    """Manages startup/shutdown of a I{Yeti} server.
+    """
+
+    @classmethod
+    def setUp(cls):
+        cls.proc = None
+        if os.environ.get("YETI_SERVER") is None:
+            cls.proc = startYeti()
+
+    @classmethod
+    def tearDown(cls):
+        if cls.proc is not None:
+            # If the process was created by us, then that means the
+            # environment variable has been set by ourselves too, so
+            # we must unset it.
+            del os.environ["YETI_SERVER"]
+            terminateProcess(cls.proc)
+
+
+class YetiTestCase(TestCase):
+    """Controls a I{Yeti} client for a specific configuration.
+
+    Test output from I{Yeti} is captured and then parsed and
+    reported to unittest through clever subunit usage.
+
+    We require a L{tests_directory} class variable to be set by
+    subclasses, and that's the only configuration needed.
+    """
+    layer = YetiLayer
+
+    def _runTest(self, result):
+        yeti = os.environ["YETI"]
+        port = os.environ.get("YETI_PORT", "4422")
+        cmd = yeti.split() + ["--formatter=subunit",
+                              "--quiet",
+                              "--solo=1",
+                              "--port=%s" % port]
+        for base, dirs, files in os.walk(self.tests_directory):
+            for filename in fnmatch.filter(files, "test_*.html"):
+                cmd.append(os.path.join(base, filename))
+        proc = subprocess.Popen(cmd,
+                                stdin=subprocess.PIPE,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.STDOUT)
+        suite = ProtocolTestCase(proc.stdout)
+        suite.run(result)
+        proc.wait()
+
+    def run(self, result=None):
+        if result is None:
+            result = self.defaultTestResult()
+        self.setUp()
+        try:
+            self._runTest(result)
+        finally:
+            self.tearDown()
+
+    def runTest(self):
+        self.run()