Merge lp:~saviq/unity8/wrap-qmltestrunner into lp:unity8

[Michał Sawicz]

Enable autopkgtest for graphical QML tests.

[lp:~saviq/unity8/wrap-qmltestrunner]

873. By Michał Sawicz on 2014-05-01

Fix timeout handling.

874. By Michał Sawicz on 2014-05-01

Disable lightdm dbus test for now.

Unmerged revisions

874. By Michał Sawicz on 2014-05-01

Disable lightdm dbus test for now.

873. By Michał Sawicz on 2014-05-01

Fix timeout handling.

872. By Michał Sawicz on 2014-05-01

Make wrapper executable.

871. By Michał Sawicz on 2014-05-01

Refactor how qmltestrunner and qmlscene paths are determined to allow overriding.

870. By Michał Sawicz on 2014-05-01

Introduce qmltest_wrapper.py to record a vido of failed qmltest output.

=== modified file 'cmake/modules/QmlTest.cmake'
--- cmake/modules/QmlTest.cmake	2014-04-29 15:17:21 +0000
+++ cmake/modules/QmlTest.cmake	2014-05-01 21:22:23 +0000
@@ -19,13 +19,31 @@
 # qmltest_DEFAULT_IMPORT_PATHS
 # qmltest_DEFAULT_PROPERTIES
 
-find_program(qmltestrunner_exe qmltestrunner)
-
-if(NOT qmltestrunner_exe)
-  msg(FATAL_ERROR "Could not locate qmltestrunner.")
-endif()
-
-set(qmlscene_exe ${CMAKE_BINARY_DIR}/tests/uqmlscene/uqmlscene)
+if(NOT TARGET qmltestrunner)
+    find_program(qmltestrunner_exe qmltestrunner)
+
+    if(NOT qmltestrunner_exe)
+      msg(FATAL_ERROR "Could not locate qmltestrunner.")
+    endif()
+
+    add_executable(qmltestrunner IMPORTED)
+    set_target_properties(qmltestrunner PROPERTIES IMPORTED_LOCATION ${qmltestrunner_exe})
+endif()
+
+if(NOT TARGET qmlscene)
+    find_program(qmlscene_exe qmlscene)
+
+    if(NOT qmlscene_exe)
+      msg(FATAL_ERROR "Could not locate qmlscene.")
+    endif()
+
+    add_executable(qmlscene IMPORTED)
+    set_target_properties(qmlscene PROPERTIES IMPORTED_LOCATION ${qmlscene_exe})
+endif()
+
+if(NOT DEFINED ARTIFACTS_DIR)
+    set(ARTIFACTS_DIR ${CMAKE_BINARY_DIR})
+endif()
 
 macro(add_manual_qml_test SUBPATH COMPONENT_NAME)
     set(options NO_ADD_TEST NO_TARGETS)
@@ -33,6 +51,8 @@
 
     cmake_parse_arguments(qmltest "${options}" "" "${multi_value_keywords}" ${ARGN})
 
+    get_target_property(qmlscene_executable qmlscene LOCATION)
+
     set(qmlscene_TARGET try${COMPONENT_NAME})
     set(qmltest_FILE ${SUBPATH}/tst_${COMPONENT_NAME})
 
@@ -51,7 +71,7 @@
 
     set(qmlscene_command
         env ${qmltest_ENVIRONMENT}
-        ${qmlscene_exe} ${CMAKE_CURRENT_SOURCE_DIR}/${qmltest_FILE}.qml
+        ${qmlscene_executable} ${CMAKE_CURRENT_SOURCE_DIR}/${qmltest_FILE}.qml
             ${qmlscene_imports}
     )
     add_custom_target(${qmlscene_TARGET} ${qmlscene_command})
@@ -76,6 +96,8 @@
     set(qmltest_xvfb_TARGET xvfbtest${COMPONENT_NAME})
     set(qmltest_FILE ${SUBPATH}/tst_${COMPONENT_NAME})
 
+    get_target_property(qmltestrunner_executable qmltestrunner LOCATION)
+
     set(qmltestrunner_imports "")
     if(NOT "${qmltest_IMPORT_PATHS}" STREQUAL "")
         foreach(IMPORT_PATH ${qmltest_IMPORT_PATHS})
@@ -104,10 +126,10 @@
 
     set(qmltest_command
         env ${qmltest_ENVIRONMENT}
-        ${qmltestrunner_exe} -input ${CMAKE_CURRENT_SOURCE_DIR}/${qmltest_FILE}.qml
+        ${qmltestrunner_executable} -input ${CMAKE_CURRENT_SOURCE_DIR}/${qmltest_FILE}.qml
             ${qmltestrunner_imports}
             ${ITERATIONS_STRING}
-            -o ${CMAKE_BINARY_DIR}/${qmltest_TARGET}.xml,xunitxml
+            -o ${ARTIFACTS_DIR}/${qmltest_TARGET}.xml,xunitxml
             -o -,txt
             ${function_ARGS}
     )
@@ -119,9 +141,9 @@
     set(qmltest_xvfb_command
         env ${qmltest_ENVIRONMENT} ${LD_PRELOAD_PATH}
         xvfb-run --server-args "-screen 0 1024x768x24" --auto-servernum
-        ${qmltestrunner_exe} -input ${CMAKE_CURRENT_SOURCE_DIR}/${qmltest_FILE}.qml
+        ${qmltestrunner_executable} -input ${CMAKE_CURRENT_SOURCE_DIR}/${qmltest_FILE}.qml
         ${qmltestrunner_imports}
-            -o ${CMAKE_BINARY_DIR}/${qmltest_TARGET}.xml,xunitxml
+            -o ${ARTIFACTS_DIR}/${qmltest_TARGET}.xml,xunitxml
             -o -,txt
             ${function_ARGS}
     )
@@ -135,7 +157,7 @@
     set(testCommand
           LD_LIBRARY_PATH=${LD_PATH}
           ${CMAKE_CURRENT_BINARY_DIR}/${CLASS_NAME}TestExec
-          -o ${CMAKE_BINARY_DIR}/${CLASSNAME}Test.xml,xunitxml
+          -o ${ARTIFACTS_DIR}/${CLASSNAME}Test.xml,xunitxml
           -o -,txt)
 
     add_qmltest_target(test${CLASS_NAME} "${testCommand}" FALSE TRUE)
@@ -151,7 +173,7 @@
           LD_LIBRARY_PATH=${LD_PATH}
           xvfb-run --server-args "-screen 0 1024x768x24" --auto-servernum
           ${CMAKE_CURRENT_BINARY_DIR}/${CLASS_NAME}TestExec
-          -o ${CMAKE_BINARY_DIR}/${CLASS_NAME}Test.xml,xunitxml
+          -o ${ARTIFACTS_DIR}/${CLASS_NAME}Test.xml,xunitxml
           -o -,txt)
 
     add_qmltest_target(xvfbtest${CLASS_NAME} "${xvfbtestCommand}" FALSE TRUE)

=== modified file 'tests/CMakeLists.txt'
--- tests/CMakeLists.txt	2014-03-18 12:46:07 +0000
+++ tests/CMakeLists.txt	2014-05-01 21:22:23 +0000
@@ -7,9 +7,9 @@
 add_custom_target(qmltests)
 add_dependencies(qmltests qmlunittests qmluitests)
 
+add_subdirectory(uqmlscene)
 add_subdirectory(qmltests)
 add_subdirectory(mocks)
 add_subdirectory(whitespace)
 add_subdirectory(plugins)
 add_subdirectory(utils)
-add_subdirectory(uqmlscene)

=== modified file 'tests/plugins/LightDM/CMakeLists.txt'
--- tests/plugins/LightDM/CMakeLists.txt	2014-01-14 21:01:28 +0000
+++ tests/plugins/LightDM/CMakeLists.txt	2014-05-01 21:22:23 +0000
@@ -21,5 +21,3 @@
 
 add_custom_target(testLightDMDBus dbus-launch env QML2_IMPORT_PATH=${CMAKE_BINARY_DIR}/tests/mocks LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/tests/mocks/LightDM/full ${CMAKE_CURRENT_BINARY_DIR}/test-lightdm-dbus)
 add_dependencies(testLightDMDBus test-lightdm-dbus)
-
-add_dependencies(qmluitests testLightDMDBus)

=== modified file 'tests/qmltests/CMakeLists.txt'
--- tests/qmltests/CMakeLists.txt	2014-04-22 08:51:33 +0000
+++ tests/qmltests/CMakeLists.txt	2014-05-01 21:22:23 +0000
@@ -1,3 +1,10 @@
+add_executable(qmlscene ALIAS uqmlscene)
+
+if(DEFINED QMLTESTRUNNER_EXECUTABLE)
+    add_executable(qmltestrunner IMPORTED)
+    set_target_properties(qmltestrunner PROPERTIES IMPORTED_LOCATION "${QMLTESTRUNNER_EXECUTABLE}")
+endif()
+
 # add_qml_test macro
 include(QmlTest)
 

=== added file 'tools/qmltest_wrapper.py'
--- tools/qmltest_wrapper.py	1970-01-01 00:00:00 +0000
+++ tools/qmltest_wrapper.py	2014-05-01 21:22:23 +0000
@@ -0,0 +1,94 @@
+#!/usr/bin/python3
+
+import os
+import sys
+import subprocess
+import shutil
+import tempfile
+
+ENCODE_LIMIT=120
+GRACE_TIME=5
+
+recordmydesktop_args = [
+    "recordmydesktop",
+    "--quick-subsampling",
+    "--fps=60",
+    "--no-sound",
+    "--overwrite",
+]
+
+if __name__ == "__main__":
+    """
+    This script wraps qmltestrunner, recording the screen as the test runs
+    with recordmydesktop. Maximum encoding time is {0}s, the process is
+    interrupted after that time. If the test succeeds, recording is aborted
+    and no video produced.
+    Output video file name is derived from the first logger option in the
+    "-o /file/path,format" format with *absolute* paths, the file is saved
+    next to the test log file with the extension ".ogv", falling back to
+    "qmltest.ogv" in the current directory.
+    """
+    
+    argv = sys.argv.copy()
+    argv[0] = "qmltestrunner"
+    
+    outfile = os.path.join(os.curdir, "qmltest.ogv")   
+    returncode = 0
+
+    try:
+        for logger in (argv[k+1] for k, v in enumerate(argv) if v == "-o"):
+            log = logger.split(',')[0]
+            if os.path.isabs(log):
+                outfile = "{0}.ogv".format(os.path.splitext(log)[0])
+                break
+    except IndexError as err:
+        raise ValueError("Missing logger definition:\n{0}".format(argv)) from err
+
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        # create a temporary file and close it to let recordmydesktop overwrite it
+        tmpfd, tmpname = tempfile.mkstemp(dir=tmpdir, suffix=".ogv")
+        os.close(tmpfd)
+        recordmydesktop_args.extend(["--workdir={0}".format(tmpdir),
+                                     "--output={0}".format(tmpname)])        
+                
+        with subprocess.Popen(recordmydesktop_args, stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE, universal_newlines=True) as recorder:
+            try:
+                subprocess.check_call(argv)
+            except subprocess.CalledProcessError as err:
+                # store the exit code to return
+                returncode = err.returncode
+                
+                # stop recording, give 60 seconds to encode
+                recorder.terminate()
+                try:
+                    recorder.wait(ENCODE_LIMIT)
+                except subprocess.TimeoutExpired:
+                    print("===== Recorder took too long to encode, recording will be incomplete =====")
+                    recorder.terminate()
+                    try:
+                        recorder.wait(GRACE_TIME)
+                    except subprocess.TimeoutExpired:
+                        pass
+                if recorder.returncode is 0:
+                    # only store the file if recorder exited cleanly
+                    shutil.move(tmpname, outfile)
+            else:
+                # abort the recording, test passed  
+                recorder.send_signal(subprocess.signal.SIGABRT)
+                try:
+                    recorder.wait(GRACE_TIME)
+                except subprocess.TimeoutExpired:
+                    pass
+            finally:
+                recorder.poll()
+                # kill the recording if still running
+                if recorder.returncode is None:
+                    recorder.kill()
+                    print("===== Had to kill recorder =====")
+                elif recorder.returncode is not 0:
+                    # only print recorder output if terminated successfully
+                    print("===== Recorder error =====\nSTDOUT:\n{0}\n\nSTDERR:\n{1}".format(*recorder.communicate()))
+
+    sys.exit(returncode)
\ No newline at end of file