Merge lp:~terceiro/lava-dispatcher/filter-ansi-color-codes into lp:lava-dispatcher

[Antonio Terceiro]

Filter out ANSI color codes when matching for patterns. This shoulf fix #1197582

[Antonio Terceiro (terceiro)]

This does not fix the problem yet - no need for reviewing now. I will resubmit when it's done.

review: Needs Resubmitting

[lp:~terceiro/lava-dispatcher/filter-ansi-color-codes]

646. By Antonio Terceiro on 2013-08-02

Need to filter out non-color codes as well, such as cursor positioning, screen
clearing etc.

647. By Antonio Terceiro on 2013-08-02

Deal with slow/broken serial connection when filtering out ANSI escape sequences

648. By Antonio Terceiro on 2013-08-02

Merge branch for supporting specifying a bootloader on deployment of prebuilt images as well

649. By Antonio Terceiro on 2013-08-07

filter other escape sequences

650. By Antonio Terceiro on 2013-08-07

update from parent branch

651. By Antonio Terceiro on 2013-08-08

update from trunk

Unmerged revisions

651. By Antonio Terceiro on 2013-08-08

update from trunk

650. By Antonio Terceiro on 2013-08-07

update from parent branch

649. By Antonio Terceiro on 2013-08-07

filter other escape sequences

648. By Antonio Terceiro on 2013-08-02

Merge branch for supporting specifying a bootloader on deployment of prebuilt images as well

647. By Antonio Terceiro on 2013-08-02

Deal with slow/broken serial connection when filtering out ANSI escape sequences

646. By Antonio Terceiro on 2013-08-02

Need to filter out non-color codes as well, such as cursor positioning, screen
clearing etc.

645. By Antonio Terceiro on 2013-08-02

Filter out ANSI color escape sequences when reading input from child processes

=== modified file 'lava_dispatcher/client/targetdevice.py'
--- lava_dispatcher/client/targetdevice.py	2013-07-24 16:56:18 +0000
+++ lava_dispatcher/client/targetdevice.py	2013-08-08 15:29:54 +0000
@@ -62,7 +62,7 @@
         if image is None:
             self.target_device.deploy_linaro(hwpack, rootfs, bootloader)
         else:
-            self.target_device.deploy_linaro_prebuilt(image)
+            self.target_device.deploy_linaro_prebuilt(image, bootloader)
 
     def _boot_linaro_image(self):
         if self.proc:
 
=== modified file 'lava_dispatcher/device/fastmodel.py'
--- lava_dispatcher/device/fastmodel.py	2013-07-24 14:04:25 +0000
+++ lava_dispatcher/device/fastmodel.py	2013-08-08 15:29:54 +0000
@@ -171,10 +171,11 @@
         rootfs = download_image(rootfs, self.context, decompress=False)
         odir = os.path.dirname(rootfs)
 
-        self._bootloader = bootloader
 
         generate_fastmodel_image(self.context, hwpack, rootfs, odir, bootloader)
+
         self._sd_image = '%s/sd.img' % odir
+        self._bootloader = bootloader
 
         self._copy_needed_files_from_directory(odir)
         self._copy_needed_files_from_partition(self.config.boot_part, 'rtsm')
@@ -182,8 +183,9 @@
 
         self._customize_linux(self._sd_image)
 
-    def deploy_linaro_prebuilt(self, image):
+    def deploy_linaro_prebuilt(self, image, bootloader='u_boot'):
         self._sd_image = download_image(image, self.context)
+        self._bootloader = bootloader
 
         self._copy_needed_files_from_partition(self.config.boot_part, 'rtsm')
         self._copy_needed_files_from_partition(self.config.root_part, 'boot')
 
=== modified file 'lava_dispatcher/device/ipmi_pxe.py'
--- lava_dispatcher/device/ipmi_pxe.py	2013-07-16 16:06:42 +0000
+++ lava_dispatcher/device/ipmi_pxe.py	2013-08-08 15:29:54 +0000
@@ -79,7 +79,7 @@
         self._customize_linux(image_file)
         self._deploy_image(image_file, '/dev/sda')
 
-    def deploy_linaro_prebuilt(self, image):
+    def deploy_linaro_prebuilt(self, image, bootloader):
         image_file = download_image(image, self.context, self.scratch_dir)
         self._customize_linux(image_file)
         self._deploy_image(image_file, '/dev/sda')
 
=== modified file 'lava_dispatcher/device/master.py'
--- lava_dispatcher/device/master.py	2013-07-17 09:16:46 +0000
+++ lava_dispatcher/device/master.py	2013-08-08 15:29:54 +0000
@@ -149,7 +149,7 @@
         _deploy_linaro_android_system(master, system_url)
         _deploy_linaro_android_data(master, data_url)
 
-    def deploy_linaro_prebuilt(self, image):
+    def deploy_linaro_prebuilt(self, image, bootloader):
         self.boot_master_image()
 
         if self.context.job_data.get('health_check', False):
 
=== modified file 'lava_dispatcher/device/qemu.py'
--- lava_dispatcher/device/qemu.py	2013-07-24 14:04:25 +0000
+++ lava_dispatcher/device/qemu.py	2013-08-08 15:29:54 +0000
@@ -51,7 +51,7 @@
         self._sd_image = generate_image(self, hwpack, rootfs, odir, bootloader)
         self._customize_linux(self._sd_image)
 
-    def deploy_linaro_prebuilt(self, image):
+    def deploy_linaro_prebuilt(self, image, bootloader='u_boot'):
         self._sd_image = download_image(image, self.context)
         self._customize_linux(self._sd_image)
 
 
=== modified file 'lava_dispatcher/device/sdmux.py'
--- lava_dispatcher/device/sdmux.py	2013-07-16 16:04:07 +0000
+++ lava_dispatcher/device/sdmux.py	2013-08-08 15:29:54 +0000
@@ -93,7 +93,7 @@
         self._customize_linux(img)
         self._write_image(img)
 
-    def deploy_linaro_prebuilt(self, image):
+    def deploy_linaro_prebuilt(self, image, bootloader=None):
         img = download_image(image, self.context)
         self._customize_linux(img)
         self._write_image(img)
 
=== modified file 'lava_dispatcher/device/target.py'
--- lava_dispatcher/device/target.py	2013-07-17 09:16:46 +0000
+++ lava_dispatcher/device/target.py	2013-08-08 15:29:54 +0000
@@ -91,7 +91,7 @@
     def deploy_android(self, boot, system, userdata):
         raise NotImplementedError('deploy_android_image')
 
-    def deploy_linaro_prebuilt(self, image):
+    def deploy_linaro_prebuilt(self, image, bootloader):
         raise NotImplementedError('deploy_linaro_prebuilt')
 
     def power_off(self, proc):
 
=== modified file 'lava_dispatcher/tests/__init__.py'
--- lava_dispatcher/tests/__init__.py	2013-07-16 16:09:31 +0000
+++ lava_dispatcher/tests/__init__.py	2013-08-08 15:29:54 +0000
@@ -3,6 +3,7 @@
 
 def test_suite():
     module_names = [
+        'lava_dispatcher.tests.test_utils',
         'lava_dispatcher.tests.test_config',
         'lava_dispatcher.tests.test_device_version',
     ]
 
=== added file 'lava_dispatcher/tests/simulate_uefi_on_fastmodel.py'
--- lava_dispatcher/tests/simulate_uefi_on_fastmodel.py	1970-01-01 00:00:00 +0000
+++ lava_dispatcher/tests/simulate_uefi_on_fastmodel.py	2013-08-08 15:29:54 +0000
@@ -0,0 +1,10 @@
+import sys
+import time
+
+chars = 0
+for char in "\033[23;01H2.0 \033[1m\033[33m\033[40mShell> \033[0m\033[37m\033[40m":
+    sys.stdout.write(char)
+    sys.stdout.flush()
+    chars += 1
+    if chars % 10 == 0:
+        time.sleep(0.1)
 
=== added file 'lava_dispatcher/tests/test_utils.py'
--- lava_dispatcher/tests/test_utils.py	1970-01-01 00:00:00 +0000
+++ lava_dispatcher/tests/test_utils.py	2013-08-08 15:29:54 +0000
@@ -0,0 +1,61 @@
+# Copyright (C) 2013 Linaro Limited
+#
+# Author: Antonio Terceiro <[e-mail]>
+#
+# This file is part of LAVA Dispatcher.
+#
+# LAVA Dispatcher is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# LAVA Dispatcher is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses>.
+
+from unittest import TestCase
+import pexpect
+
+import os
+
+from lava_dispatcher.utils import logging_spawn, remove_ansi_escape_codes
+
+class TestLoggingSpawn(TestCase):
+
+    def test_filter_out_ansi_color_codes(self):
+        proc = logging_spawn('printf "I AM \033[1mOK\033[m\n"')  # OK in bold
+        proc.expect(["I AM OK", pexpect.EOF])
+        self.assertIsNot(proc.match, pexpect.EOF)
+
+    def test_filter_out_multiple_ansi_color_escape_sequences(self):
+        # "I AM" in bold light green, "OK" in bold
+        proc = logging_spawn('printf "\033[1;32;40mI AM\033[m \033[1mOK\033[m\n"')
+        proc.expect(["I AM OK", pexpect.EOF])
+        self.assertIsNot(proc.match, pexpect.EOF)
+
+    def test_filter_out_cursor_positioning_codes(self):
+        proc = logging_spawn('printf "I AM \033[23;01H\033[1mOK\033[m\n"')
+        proc.expect(["I AM OK", pexpect.EOF])
+        self.assertIsNot(proc.match, pexpect.EOF)
+
+    def test_filter_out_uefi_ansi_crap(self):
+        program = os.path.join(os.path.dirname(__file__),
+                               'simulate_uefi_on_fastmodel.py')
+        proc = logging_spawn('python %s' % program)
+        proc.expect(['2.0 Shell>', pexpect.EOF])
+        self.assertIsNot(proc.match, pexpect.EOF)
+
+    def test_filter_out_other_types_of_escape_sequences(self):
+        proc = logging_spawn('printf "\x1b[=3hTHIS \x1b[=3hIS \x1b[=3hOK"')
+        proc.expect(['THIS IS OK'])
+        self.assertIsNot(proc.match, pexpect.EOF)
+
+    def test_filter_out_incomplete_sequences_followed_by_others(self):
+        data = "\x1b[01;01\x1b[=3hHELLO"
+        self.assertEqual(remove_ansi_escape_codes(data), "HELLO")
+
+
 
=== modified file 'lava_dispatcher/utils.py'
--- lava_dispatcher/utils.py	2013-07-24 16:56:18 +0000
+++ lava_dispatcher/utils.py	2013-08-08 15:29:54 +0000
@@ -29,6 +29,7 @@
 import time
 import urlparse
 import subprocess
+import re
 
 from shlex import shlex
 
@@ -213,6 +214,24 @@
                 ['.+', pexpect.EOF, pexpect.TIMEOUT],
                 timeout=1, lava_no_logging=1)
 
+    def read_nonblocking(self, size, timeout):
+        read = super(logging_spawn, self).read_nonblocking
+
+        raw = read(size, timeout)
+        filtered = remove_ansi_escape_codes(raw)
+
+        while '\x1b' in filtered:
+            # This happens when the serial connection is too slow -- or just
+            # broken -- and the escape sequences are split across different
+            # reads. We have to accumulate more reads until we find the entire
+            # sequence to be able to filter it out.
+            raw = raw + read(size, timeout)
+            logging.debug("Input with potentially incomplete ANSI "
+                          "escape sequence: %r" % raw)
+            filtered = remove_ansi_escape_codes(raw)
+
+        return filtered
+
 
 def connect_to_serial(context):
     """
@@ -276,3 +295,6 @@
         logging.debug("Finalizing child proccess with PID %d" % proc.pid)
         proc.kill(9)
         proc.close()
+
+def remove_ansi_escape_codes(string):
+    return re.sub('(\x1b\[[0-9;=]*)+[a-zA-Z]', '', string)