Merge lp:~soren/nova/improve-smoketests into lp:~hudson-openstack/nova/trunk

[Soren Hansen]

Make smoketests' exit code reveal whether they were succesful.

Adjust volume tests to check the exact size of the block device, instead of a rounded-off size of the resulting filesystem.

Make proxy.sh work with both variants of netcat.

With these couple of changes, my automated runs of both sysadmin_smoketests and netadmin_smoketests are now succesful.

[Vish Ishaya (vishvananda)]

lgtm. Nice fixes

review: Approve

[Jesse Andrews (anotherjesse)]

lgtm!

I finally got hardware from RAX to run the smoketests on a real cluster after each deploy

review: Approve

[OpenStack Infra (hudson-openstack)]

Attempt to merge into lp:nova failed due to conflicts:

text conflict in smoketests/test_sysadmin.py

[lp:~soren/nova/improve-smoketests]

824. By Soren Hansen on 2011-03-20

Merge trunk

825. By Soren Hansen on 2011-03-20

pep8

=== modified file 'nova/tests/api/openstack/test_servers.py'
--- nova/tests/api/openstack/test_servers.py	2011-03-17 22:24:08 +0000
+++ nova/tests/api/openstack/test_servers.py	2011-03-20 19:08:31 +0000
@@ -1174,7 +1174,3 @@
         server = dom.childNodes[0]
         self.assertEquals(server.nodeName, 'server')
         self.assertTrue(server.getAttribute('adminPass').startswith('fake'))
-
-
-if __name__ == "__main__":
-    unittest.main()

=== modified file 'run_tests.py'
--- run_tests.py	2011-02-25 01:04:25 +0000
+++ run_tests.py	2011-03-20 19:08:31 +0000
@@ -60,6 +60,8 @@
 import unittest
 import sys
 
+gettext.install('nova', unicode=1)
+
 from nose import config
 from nose import core
 from nose import result

=== modified file 'smoketests/base.py'
--- smoketests/base.py	2011-02-23 02:04:08 +0000
+++ smoketests/base.py	2011-03-20 19:08:31 +0000
@@ -31,17 +31,24 @@
 SUITE_NAMES = '[image, instance, volume]'
 FLAGS = flags.FLAGS
 flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES)
+flags.DEFINE_integer('ssh_tries', 3, 'Numer of times to try ssh')
 boto_v6 = None
 
 
 class SmokeTestCase(unittest.TestCase):
     def connect_ssh(self, ip, key_name):
-        # TODO(devcamcar): set a more reasonable connection timeout time
         key = paramiko.RSAKey.from_private_key_file('/tmp/%s.pem' % key_name)
-        client = paramiko.SSHClient()
-        client.set_missing_host_key_policy(paramiko.WarningPolicy())
-        client.connect(ip, username='root', pkey=key)
-        return client
+        tries = 0
+        while(True):
+            try:
+                client = paramiko.SSHClient()
+                client.set_missing_host_key_policy(paramiko.WarningPolicy())
+                client.connect(ip, username='root', pkey=key, timeout=5)
+                return client
+            except (paramiko.AuthenticationException, paramiko.SSHException):
+                tries += 1
+                if tries == FLAGS.ssh_tries:
+                    raise
 
     def can_ping(self, ip, command="ping"):
         """Attempt to ping the specified IP, and give up after 1 second."""
@@ -147,8 +154,8 @@
         except:
             pass
 
-    def bundle_image(self, image, kernel=False):
-        cmd = 'euca-bundle-image -i %s' % image
+    def bundle_image(self, image, tempdir='/tmp', kernel=False):
+        cmd = 'euca-bundle-image -i %s -d %s' % (image, tempdir)
         if kernel:
             cmd += ' --kernel true'
         status, output = commands.getstatusoutput(cmd)
@@ -157,9 +164,9 @@
             raise Exception(output)
         return True
 
-    def upload_image(self, bucket_name, image):
+    def upload_image(self, bucket_name, image, tempdir='/tmp'):
         cmd = 'euca-upload-bundle -b '
-        cmd += '%s -m /tmp/%s.manifest.xml' % (bucket_name, image)
+        cmd += '%s -m %s/%s.manifest.xml' % (bucket_name, tempdir, image)
         status, output = commands.getstatusoutput(cmd)
         if status != 0:
             print '%s -> \n %s' % (cmd, output)
@@ -183,29 +190,3 @@
         global TEST_DATA
         self.conn = self.connection_for_env()
         self.data = TEST_DATA
-
-
-def run_tests(suites):
-    argv = FLAGS(sys.argv)
-    if FLAGS.use_ipv6:
-        global boto_v6
-        boto_v6 = __import__('boto_v6')
-
-    if not os.getenv('EC2_ACCESS_KEY'):
-        print >> sys.stderr, 'Missing EC2 environment variables. Please ' \
-                             'source the appropriate novarc file before ' \
-                             'running this test.'
-        return 1
-
-    if FLAGS.suite:
-        try:
-            suite = suites[FLAGS.suite]
-        except KeyError:
-            print >> sys.stderr, 'Available test suites:', \
-                                 ', '.join(suites.keys())
-            return 1
-
-        unittest.TextTestRunner(verbosity=2).run(suite)
-    else:
-        for suite in suites.itervalues():
-            unittest.TextTestRunner(verbosity=2).run(suite)

=== modified file 'smoketests/proxy.sh'
--- smoketests/proxy.sh	2011-02-23 02:04:32 +0000
+++ smoketests/proxy.sh	2011-03-20 19:08:31 +0000
@@ -11,12 +11,19 @@
 mkfifo backpipe1
 mkfifo backpipe2
 
+if nc -h 2>&1 | grep -i openbsd
+then
+	NC_LISTEN="nc -l"
+else
+	NC_LISTEN="nc -l -p"
+fi
+
 # NOTE(vish): proxy metadata on port 80
 while true; do
-    nc -l -p 80 0<backpipe1 | nc 169.254.169.254 80 1>backpipe1
+    $NC_LISTEN 80 0<backpipe1 | nc 169.254.169.254 80 1>backpipe1
 done &
 
 # NOTE(vish): proxy google on port 8080
 while true; do
-    nc -l -p 8080 0<backpipe2 | nc 74.125.19.99 80 1>backpipe2
+    $NC_LISTEN 8080 0<backpipe2 | nc 74.125.19.99 80 1>backpipe2
 done &

=== modified file 'smoketests/public_network_smoketests.py'
--- smoketests/public_network_smoketests.py	2011-02-23 02:04:08 +0000
+++ smoketests/public_network_smoketests.py	2011-03-20 19:08:31 +0000
@@ -19,10 +19,8 @@
 import commands
 import os
 import random
-import socket
 import sys
 import time
-import unittest
 
 # If ../nova/__init__.py exists, add ../ to Python search path, so that
 # it will override what happens to be installed in /usr/(local/)lib/python...
@@ -181,7 +179,3 @@
         self.conn.delete_security_group(security_group_name)
         if 'instance_id' in self.data:
             self.conn.terminate_instances([self.data['instance_id']])
-
-if __name__ == "__main__":
-    suites = {'instance': unittest.makeSuite(InstanceTestsFromPublic)}
-    sys.exit(base.run_tests(suites))

=== added file 'smoketests/run_tests.py'
--- smoketests/run_tests.py	1970-01-01 00:00:00 +0000
+++ smoketests/run_tests.py	2011-03-20 19:08:31 +0000
@@ -0,0 +1,310 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+# Colorizer Code is borrowed from Twisted:
+# Copyright (c) 2001-2010 Twisted Matrix Laboratories.
+#
+#    Permission is hereby granted, free of charge, to any person obtaining
+#    a copy of this software and associated documentation files (the
+#    "Software"), to deal in the Software without restriction, including
+#    without limitation the rights to use, copy, modify, merge, publish,
+#    distribute, sublicense, and/or sell copies of the Software, and to
+#    permit persons to whom the Software is furnished to do so, subject to
+#    the following conditions:
+#
+#    The above copyright notice and this permission notice shall be
+#    included in all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+#    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+#    LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+#    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+#    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""Unittest runner for Nova.
+
+To run all tests
+    python run_tests.py
+
+To run a single test:
+    python run_tests.py test_compute:ComputeTestCase.test_run_terminate
+
+To run a single test module:
+    python run_tests.py test_compute
+
+    or
+
+    python run_tests.py api.test_wsgi
+
+"""
+
+import gettext
+import os
+import unittest
+import sys
+
+# If ../nova/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
+    sys.path.insert(0, possible_topdir)
+
+
+gettext.install('nova', unicode=1)
+
+from nose import config
+from nose import core
+from nose import result
+
+from smoketests import flags
+FLAGS = flags.FLAGS
+
+
+class _AnsiColorizer(object):
+    """
+    A colorizer is an object that loosely wraps around a stream, allowing
+    callers to write text to the stream in a particular color.
+
+    Colorizer classes must implement C{supported()} and C{write(text, color)}.
+    """
+    _colors = dict(black=30, red=31, green=32, yellow=33,
+                   blue=34, magenta=35, cyan=36, white=37)
+
+    def __init__(self, stream):
+        self.stream = stream
+
+    def supported(cls, stream=sys.stdout):
+        """
+        A class method that returns True if the current platform supports
+        coloring terminal output using this method. Returns False otherwise.
+        """
+        if not stream.isatty():
+            return False  # auto color only on TTYs
+        try:
+            import curses
+        except ImportError:
+            return False
+        else:
+            try:
+                try:
+                    return curses.tigetnum("colors") > 2
+                except curses.error:
+                    curses.setupterm()
+                    return curses.tigetnum("colors") > 2
+            except:
+                raise
+                # guess false in case of error
+                return False
+    supported = classmethod(supported)
+
+    def write(self, text, color):
+        """
+        Write the given text to the stream in the given color.
+
+        @param text: Text to be written to the stream.
+
+        @param color: A string label for a color. e.g. 'red', 'white'.
+        """
+        color = self._colors[color]
+        self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text))
+
+
+class _Win32Colorizer(object):
+    """
+    See _AnsiColorizer docstring.
+    """
+    def __init__(self, stream):
+        from win32console import GetStdHandle, STD_OUT_HANDLE, \
+             FOREGROUND_RED, FOREGROUND_BLUE, FOREGROUND_GREEN, \
+             FOREGROUND_INTENSITY
+        red, green, blue, bold = (FOREGROUND_RED, FOREGROUND_GREEN,
+                                  FOREGROUND_BLUE, FOREGROUND_INTENSITY)
+        self.stream = stream
+        self.screenBuffer = GetStdHandle(STD_OUT_HANDLE)
+        self._colors = {
+            'normal': red | green | blue,
+            'red': red | bold,
+            'green': green | bold,
+            'blue': blue | bold,
+            'yellow': red | green | bold,
+            'magenta': red | blue | bold,
+            'cyan': green | blue | bold,
+            'white': red | green | blue | bold
+            }
+
+    def supported(cls, stream=sys.stdout):
+        try:
+            import win32console
+            screenBuffer = win32console.GetStdHandle(
+                win32console.STD_OUT_HANDLE)
+        except ImportError:
+            return False
+        import pywintypes
+        try:
+            screenBuffer.SetConsoleTextAttribute(
+                win32console.FOREGROUND_RED |
+                win32console.FOREGROUND_GREEN |
+                win32console.FOREGROUND_BLUE)
+        except pywintypes.error:
+            return False
+        else:
+            return True
+    supported = classmethod(supported)
+
+    def write(self, text, color):
+        color = self._colors[color]
+        self.screenBuffer.SetConsoleTextAttribute(color)
+        self.stream.write(text)
+        self.screenBuffer.SetConsoleTextAttribute(self._colors['normal'])
+
+
+class _NullColorizer(object):
+    """
+    See _AnsiColorizer docstring.
+    """
+    def __init__(self, stream):
+        self.stream = stream
+
+    def supported(cls, stream=sys.stdout):
+        return True
+    supported = classmethod(supported)
+
+    def write(self, text, color):
+        self.stream.write(text)
+
+
+class NovaTestResult(result.TextTestResult):
+    def __init__(self, *args, **kw):
+        result.TextTestResult.__init__(self, *args, **kw)
+        self._last_case = None
+        self.colorizer = None
+        # NOTE(vish): reset stdout for the terminal check
+        stdout = sys.stdout
+        sys.stdout = sys.__stdout__
+        for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]:
+            if colorizer.supported():
+                self.colorizer = colorizer(self.stream)
+                break
+        sys.stdout = stdout
+
+    def getDescription(self, test):
+        return str(test)
+
+    # NOTE(vish): copied from unittest with edit to add color
+    def addSuccess(self, test):
+        unittest.TestResult.addSuccess(self, test)
+        if self.showAll:
+            self.colorizer.write("OK", 'green')
+            self.stream.writeln()
+        elif self.dots:
+            self.stream.write('.')
+            self.stream.flush()
+
+    # NOTE(vish): copied from unittest with edit to add color
+    def addFailure(self, test, err):
+        unittest.TestResult.addFailure(self, test, err)
+        if self.showAll:
+            self.colorizer.write("FAIL", 'red')
+            self.stream.writeln()
+        elif self.dots:
+            self.stream.write('F')
+            self.stream.flush()
+
+    # NOTE(vish): copied from nose with edit to add color
+    def addError(self, test, err):
+        """Overrides normal addError to add support for
+        errorClasses. If the exception is a registered class, the
+        error will be added to the list for that class, not errors.
+        """
+        stream = getattr(self, 'stream', None)
+        ec, ev, tb = err
+        try:
+            exc_info = self._exc_info_to_string(err, test)
+        except TypeError:
+            # 2.3 compat
+            exc_info = self._exc_info_to_string(err)
+        for cls, (storage, label, isfail) in self.errorClasses.items():
+            if result.isclass(ec) and issubclass(ec, cls):
+                if isfail:
+                    test.passed = False
+                storage.append((test, exc_info))
+                # Might get patched into a streamless result
+                if stream is not None:
+                    if self.showAll:
+                        message = [label]
+                        detail = result._exception_detail(err[1])
+                        if detail:
+                            message.append(detail)
+                        stream.writeln(": ".join(message))
+                    elif self.dots:
+                        stream.write(label[:1])
+                return
+        self.errors.append((test, exc_info))
+        test.passed = False
+        if stream is not None:
+            if self.showAll:
+                self.colorizer.write("ERROR", 'red')
+                self.stream.writeln()
+            elif self.dots:
+                stream.write('E')
+
+    def startTest(self, test):
+        unittest.TestResult.startTest(self, test)
+        current_case = test.test.__class__.__name__
+
+        if self.showAll:
+            if current_case != self._last_case:
+                self.stream.writeln(current_case)
+                self._last_case = current_case
+
+            self.stream.write(
+                '    %s' % str(test.test._testMethodName).ljust(60))
+            self.stream.flush()
+
+
+class NovaTestRunner(core.TextTestRunner):
+    def _makeResult(self):
+        return NovaTestResult(self.stream,
+                              self.descriptions,
+                              self.verbosity,
+                              self.config)
+
+
+if __name__ == '__main__':
+    if not os.getenv('EC2_ACCESS_KEY'):
+        print _('Missing EC2 environment variables. Please ' \
+                'source the appropriate novarc file before ' \
+                'running this test.')
+        sys.exit(1)
+
+    argv = FLAGS(sys.argv)
+    testdir = os.path.abspath("./")
+    c = config.Config(stream=sys.stdout,
+                      env=os.environ,
+                      verbosity=3,
+                      workingDir=testdir,
+                      plugins=core.DefaultPluginManager())
+
+    runner = NovaTestRunner(stream=c.stream,
+                            verbosity=c.verbosity,
+                            config=c)
+    sys.exit(not core.run(config=c, testRunner=runner, argv=argv))

=== renamed file 'smoketests/admin_smoketests.py' => 'smoketests/test_admin.py'
--- smoketests/admin_smoketests.py	2011-01-12 08:47:54 +0000
+++ smoketests/test_admin.py	2011-03-20 19:08:31 +0000
@@ -35,10 +35,7 @@
 from smoketests import base
 
 
-SUITE_NAMES = '[user]'
-
 FLAGS = flags.FLAGS
-flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES)
 
 # TODO(devamcar): Use random tempfile
 ZIP_FILENAME = '/tmp/nova-me-x509.zip'
@@ -92,7 +89,3 @@
             os.remove(ZIP_FILENAME)
         except:
             pass
-
-if __name__ == "__main__":
-    suites = {'user': unittest.makeSuite(UserTests)}
-    sys.exit(base.run_tests(suites))

=== renamed file 'smoketests/netadmin_smoketests.py' => 'smoketests/test_netadmin.py'
--- smoketests/netadmin_smoketests.py	2011-02-23 02:04:32 +0000
+++ smoketests/test_netadmin.py	2011-03-20 19:08:31 +0000
@@ -21,7 +21,6 @@
 import random
 import sys
 import time
-import unittest
 
 # If ../nova/__init__.py exists, add ../ to Python search path, so that
 # it will override what happens to be installed in /usr/(local/)lib/python...
@@ -74,8 +73,10 @@
         groups = self.conn.get_all_security_groups(['default'])
         for rule in groups[0].rules:
             if (rule.ip_protocol == 'tcp' and
-                rule.from_port <= 22 and rule.to_port >= 22):
+                int(rule.from_port) <= 22 and
+                int(rule.to_port) >= 22):
                 ssh_authorized = True
+                break
         if not ssh_authorized:
             self.conn.authorize_security_group('default',
                                                ip_protocol='tcp',
@@ -137,11 +138,6 @@
         if not self.wait_for_running(self.data['instance']):
             self.fail('instance failed to start')
         self.data['instance'].update()
-        if not self.wait_for_ping(self.data['instance'].private_dns_name):
-            self.fail('could not ping instance')
-        if not self.wait_for_ssh(self.data['instance'].private_dns_name,
-                                 TEST_KEY):
-            self.fail('could not ssh to instance')
 
     def test_003_can_authorize_security_group_ingress(self):
         self.assertTrue(self.conn.authorize_security_group(TEST_GROUP,
@@ -185,10 +181,3 @@
         self.assertFalse(TEST_GROUP in [group.name for group in groups])
         self.conn.terminate_instances([self.data['instance'].id])
         self.assertTrue(self.conn.release_address(self.data['public_ip']))
-
-
-if __name__ == "__main__":
-    suites = {'address': unittest.makeSuite(AddressTests),
-              'security_group': unittest.makeSuite(SecurityGroupTests)
-              }
-    sys.exit(base.run_tests(suites))

=== renamed file 'smoketests/sysadmin_smoketests.py' => 'smoketests/test_sysadmin.py'
--- smoketests/sysadmin_smoketests.py	2011-03-18 01:54:51 +0000
+++ smoketests/test_sysadmin.py	2011-03-20 19:08:31 +0000
@@ -16,12 +16,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import commands
 import os
 import random
 import sys
 import time
-import unittest
+import tempfile
+import shutil
 
 # If ../nova/__init__.py exists, add ../ to Python search path, so that
 # it will override what happens to be installed in /usr/(local/)lib/python...
@@ -48,10 +48,18 @@
 
 class ImageTests(base.UserSmokeTestCase):
     def test_001_can_bundle_image(self):
-        self.assertTrue(self.bundle_image(FLAGS.bundle_image))
+        self.data['tempdir'] = tempfile.mkdtemp()
+        self.assertTrue(self.bundle_image(FLAGS.bundle_image,
+                                          self.data['tempdir']))
 
     def test_002_can_upload_image(self):
-        self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_image))
+        try:
+            self.assertTrue(self.upload_image(TEST_BUCKET,
+                                              FLAGS.bundle_image,
+                                              self.data['tempdir']))
+        finally:
+            if os.path.exists(self.data['tempdir']):
+                shutil.rmtree(self.data['tempdir'])
 
     def test_003_can_register_image(self):
         image_id = self.conn.register_image('%s/%s.manifest.xml' %
@@ -192,7 +200,7 @@
         self.assertEqual(volume.size, 1)
         self.data['volume'] = volume
         # Give network time to find volume.
-        time.sleep(10)
+        time.sleep(5)
 
     def test_002_can_attach_volume(self):
         volume = self.data['volume']
@@ -205,6 +213,8 @@
         else:
             self.fail('cannot attach volume with state %s' % volume.status)
 
+        # Give volume some time to be ready.
+        time.sleep(5)
         volume.attach(self.data['instance'].id, self.device)
 
         # wait
@@ -219,7 +229,7 @@
         self.assertTrue(volume.status.startswith('in-use'))
 
         # Give instance time to recognize volume.
-        time.sleep(10)
+        time.sleep(5)
 
     def test_003_can_mount_volume(self):
         ip = self.data['instance'].private_dns_name
@@ -256,12 +266,13 @@
         ip = self.data['instance'].private_dns_name
         conn = self.connect_ssh(ip, TEST_KEY)
         stdin, stdout, stderr = conn.exec_command(
-            "df -h | grep %s | awk {'print $2'}" % self.device)
-        out = stdout.read()
+            "blockdev --getsize64 %s" % self.device)
+        out = stdout.read().strip()
         conn.close()
-        if not out.strip() == '1007.9M':
-            self.fail('Volume is not the right size: %s %s' %
-                      (out, stderr.read()))
+        expected_size = 1024 * 1024 * 1024
+        self.assertEquals('%s' % (expected_size,), out,
+                          'Volume is not the right size: %s %s. Expected: %s' %
+                          (out, stderr.read(), expected_size))
 
     def test_006_me_can_umount_volume(self):
         ip = self.data['instance'].private_dns_name
@@ -284,9 +295,3 @@
     def test_999_tearDown(self):
         self.conn.terminate_instances([self.data['instance'].id])
         self.conn.delete_key_pair(TEST_KEY)
-
-if __name__ == "__main__":
-    suites = {'image': unittest.makeSuite(ImageTests),
-              'instance': unittest.makeSuite(InstanceTests),
-              'volume': unittest.makeSuite(VolumeTests)}
-    sys.exit(base.run_tests(suites))