Merge lp:~michihenning/unity/phablet-coverage-tests into lp:unity/phablet

[Michi Henning]

Added coverage-html target for coverage testing.
Added clean-coveraget target. This removes all .gcda files, which is useful if the tests have been run previously, and code changes have come to the point where the new and old .gcda files can't be merged anymore. In this case, run "make clean-coverage" and re-run the tests before generating the coverage report.
Replaced EnableCoverage.cmake with Jussi's coverage.cmake, which is better and simpler.
Fixed overwriting of CMAKE_MODULE_PATH.
Reconciled setting of CXX flags.
Added cppcheck target for static checking of C++ (similar to lint).
Changed build script to build with coverage enabled. (This is an interim measure until we have proper out-of-tree builds with different configuration parameters and can get rid of the build script.)

Added coverage processing for C++ and suppressions to stop unwanted chatter during the build and to ignore things we don't want coverage for (/usr/include and tests).
Added clean-coverage target. This removes all .gcda files, which is useful if the tests have been run previously, and code changes have come to the point where the new and old .gcda files can't be merged anymore. In this case, run "make clean-coverage" and re-run the tests before generating the coverage report.
Fixed overwriting of CMAKE_MODULE_PATH.
Added cppcheck target for static checking of C++ (similar to lint).
Fixed build script to print usage if command-line arguments/options are incorrect and to write to stderr instead of stdout for errors.
Changed build script to allow the build type to be passed in, so we don't have a hard-wired build type.

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:549
http://jenkins.qa.ubuntu.com/job/unity-phablet-ci/338/
Executed test runs:
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-armhf-ci/339/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-i386-ci/338/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-armhf-ci/214/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-i386-ci/214/console

Click here to trigger a rebuild:
http://s-jenkins:8080/job/unity-phablet-ci/338/rebuild

review: Needs Fixing (continuous-integration)

[Michael Zanetti (mzanetti)]

69 -#####################################################################
70 -# Enable code coverage calculation with gcov/gcovr/lcov
71 -# Usage:
72 -# * Switch build type to coverage (use ccmake or cmake-gui)
73 -# * Invoke make, make test, make coverage (or ninja if you use that backend)
74 -# * Find html report in subdir coveragereport
75 -# * Find xml report feasible for jenkins in coverage.xml
76 -#####################################################################
77 -if(cmake_build_type_lower MATCHES coverage)
78 - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage" )
79 - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage" )
80 - set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} --coverage" )
81 - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --coverage" )
82 - ENABLE_COVERAGE_REPORT(TARGETS ${SHELL_APP})
83 -endif()

Removing this will break coverage measurements in jenkins.

review: Needs Fixing

[Michał Sawicz (saviq)]

Please merge trunk, too, there's conflicts.

review: Needs Fixing

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:551
http://jenkins.qa.ubuntu.com/job/unity-phablet-ci/344/
Executed test runs:
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-armhf-ci/345/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-i386-ci/344/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-armhf-ci/220/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-i386-ci/224/console

Click here to trigger a rebuild:
http://s-jenkins:8080/job/unity-phablet-ci/344/rebuild

review: Needs Fixing (continuous-integration)

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:551
http://jenkins.qa.ubuntu.com/job/unity-phablet-ci/346/
Executed test runs:
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-armhf-ci/347/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-i386-ci/346/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-armhf-ci/222/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-i386-ci/226/console

Click here to trigger a rebuild:
http://s-jenkins:8080/job/unity-phablet-ci/346/rebuild

review: Needs Fixing (continuous-integration)

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:552
http://jenkins.qa.ubuntu.com/job/unity-phablet-ci/348/
Executed test runs:
    SUCCESS: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-armhf-ci/349
        deb: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-armhf-ci/349/artifact/work/output/*zip*/output.zip
    SUCCESS: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-i386-ci/348
    SUCCESS: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-armhf-ci/224
        deb: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-armhf-ci/224/artifact/work/output/*zip*/output.zip
    SUCCESS: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-i386-ci/228
    SUCCESS: http://jenkins.qa.ubuntu.com/job/generic-mediumtests-runner/751

Click here to trigger a rebuild:
http://s-jenkins:8080/job/unity-phablet-ci/348/rebuild

review: Needs Fixing (continuous-integration)

[Michi Henning (michihenning)]

>
> Removing this will break coverage measurements in jenkins.

Oh dang, I wasn't even aware of that :-(

OK, maybe we can integrate the C++ code coverage into that?

Basically, what I want is not only coverage for jenkins, but also code coverage that I can run locally. When I write the tests, I can repeatedly build the coverage target to see what I'm missing.

Is that possible with the current jenkins mechanism? What's the magic incantation to get the coverage report and look at it? Once I know that, I can probably figure out how to add coverage for the C++ side of things.

[Jussi Pakkanen (jpakkane)]

The way to do it is to have a separate build directory only for coverage (with -DCMAKE_BUILD_TYPE=coverage).

[Michael Zanetti (mzanetti)]

Yeah. But this already works with the current CMakeLists.txt. No need to change it.

Just run it with -DCMAKE_BUILD_TYPE=coverage as Jussi already said.

The existing CMakeLists.txt (not the modified one in this MP) enables 3 targets:

make coverage-xml
make coverage-html
make coverage

Where the last one just calls the other two to generate xml and html.

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:552
http://jenkins.qa.ubuntu.com/job/unity-phablet-ci/474/
Executed test runs:
    FAILURE: http://s-jenkins:8080/job/unity-phablet-qmluitests/247/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-armhf-ci/475/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-i386-ci/474/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-armhf-ci/350/console
    FAILURE: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-i386-ci/354/console

Click here to trigger a rebuild:
http://s-jenkins:8080/job/unity-phablet-ci/474/rebuild

review: Needs Fixing (continuous-integration)

[Michał Sawicz (saviq)]

Youch, there's conflicts all over here... Did I not mention you should not chain unrelated changes in prerequisites? :P

review: Needs Fixing

[Michi Henning (michihenning)]

So you did :-) Working on it.

[Michi Henning (michihenning)]

I ended up pushing with --overwrite because there were missing revisions from both sides. The diff looks good to me and matches the one I have locally.

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:585
http://jenkins.qa.ubuntu.com/job/unity-phablet-ci/484/
Executed test runs:
    SUCCESS: http://s-jenkins:8080/job/unity-phablet-qmluitests/259
    SUCCESS: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-armhf-ci/485
        deb: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-armhf-ci/485/artifact/work/output/*zip*/output.zip
    SUCCESS: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-i386-ci/484
    SUCCESS: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-armhf-ci/360
        deb: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-armhf-ci/360/artifact/work/output/*zip*/output.zip
    SUCCESS: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-i386-ci/364

Click here to trigger a rebuild:
http://s-jenkins:8080/job/unity-phablet-ci/484/rebuild

review: Needs Fixing (continuous-integration)

[Michał Sawicz (saviq)]

By the looks of it this still does not filter ${CMAKE_BUILD_DIR} from the checks.

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:586
http://jenkins.qa.ubuntu.com/job/unity-phablet-ci/487/
Executed test runs:
    SUCCESS: http://s-jenkins:8080/job/unity-phablet-qmluitests/262
    SUCCESS: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-armhf-ci/488
        deb: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-armhf-ci/488/artifact/work/output/*zip*/output.zip
    SUCCESS: http://jenkins.qa.ubuntu.com/job/unity-phablet-quantal-i386-ci/487
    SUCCESS: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-armhf-ci/363
        deb: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-armhf-ci/363/artifact/work/output/*zip*/output.zip
    SUCCESS: http://jenkins.qa.ubuntu.com/job/unity-phablet-raring-i386-ci/367

Click here to trigger a rebuild:
http://s-jenkins:8080/job/unity-phablet-ci/487/rebuild

review: Needs Fixing (continuous-integration)

[Michi Henning (michihenning)]

Hmmm... How can I look at the console output of the failed ones. Download the zip file, I assume?

[Michi Henning (michihenning)]

Nope, the zip file contains .deb files. I can't make sense of the jenkins output above. It says that it failed and, below, says success for all but two of the entries, which are the ones with .zip links. How can I work out why jenkins thinks it failed?

[Albert Astals Cid (aacid)]

Go http://jenkins.qa.ubuntu.com/job/unity-phablet-ci/487/ click on the console log to end up in https://jenkins.qa.ubuntu.com/job/unity-phablet-ci/487/console read
    generic-mediumtests #1117 completed. Result was FAILURE
then go to https://jenkins.qa.ubuntu.com/job/generic-mediumtests/1117/console and read
    generic-mediumtests-builder #1132 completed. Result was FAILURE
then realize that unfortunately the generic-mediumtests-builder jobs are not published on the public jenkins and end up in http://s-jenkins:8080/job/generic-mediumtests-builder/1132/console where you can read

23/26 Test #23: copyright ...................................***Failed 0.49 sec
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/src/unity-api_automoc.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/HudClient/moc_volumepeakdetector.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/HudClient/moc_hudclient.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/HudClient/moc_plugin.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/HudClient/HudClientQml_automoc.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/Unity/Unity-qml_automoc.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/Unity/moc_categoryfilter.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/Unity/moc_bottombarvisibilitycommunicatorshell.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/Unity/moc_lenses.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/Unity/moc_lens.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/Unity/moc_qsortfilterproxymodelqml.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/Unity/moc_peoplepreviewdata.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/Unity/moc_categories.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/Unity/moc_plugin.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/Utils/moc_ubuntuwindow.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/Utils/moc_qsortfilterproxymodelqml.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/Utils/moc_plugin.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/Utils/tests/qsortfilterproxymodeltest_automoc.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/plugins/Utils/Utils-qml_automoc.cpp: *No copyright* GENERATED FILE
/tmp/buildd/qml...

[Michi Henning (michihenning)]

Thanks for that!

Looking at the output:

/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu/src/unity-api_automoc.cpp: *No copyright* GENERATED FILE

CMAKE_BUILD_DIR in this case is:

/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu

In CMakeLists.txt, the copyright scanner is called like this:

add_test(copyright ${CMAKE_CURRENT_SOURCE_DIR}/check_copyright.sh
    ${CMAKE_SOURCE_DIR} ^${CMAKE_BINARY_DIR})

The ^ in front of ${CMAKE_BINARY_DIR} anchors the string at the start of the line so, if called with a relative short path, we don't match accidentally.

In the shell script, we have:

ignore_pat="\\.sci$"
[ $# -eq 2 ] && {
    ignore_pat="$ignore_pat|$2"
}
licensecheck -i "$ignore_pat" -r "$1" | grep 'No copyright'

$2 is therefore:

^/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu

and the complete pattern passed to licensecheck would be:

\.sci$|^/tmp/buildd/qml-phone-shell-1.71+autopilot0/obj-i686-linux-gnu

Seriously dangerous, because we have meta-characters in the path name ("." and "+"). I suspect something might be going wrong because of that. When I run the test locally in my tree, it ignores CMAKE_BINARY_DIR just fine...

[Michael Zanetti (mzanetti)]

Hey Michi... This is a bit unfortunate... The mediumtests-builder job (which is the one that failed here) is not supposed to fail without the others failing too. It bascially is a copy of the i386-ci build job but just executing less tess tests. Anyways, because of that it was hidden in the test results and so far this has been working great. Only since yesterday I see the mediumtests-builder job alone failing, and somehow only with your branches.

I will check now if there is a difference somewhere. Maybe all this chaining of prerequisites makes it freak out. For now you have to go to the Jenkins sever and find the according mediumtests-builder job yourself to find the complete log. (I understand that jenkins UI is horrible, so if you can't find it immediately, feel free to ping me in IRC and I'm happy to help you)

[Michi Henning (michihenning)]

Thanks for that Michael!

I'm at home, and currently don't have the VPN configured. Will do that.

In the mean time, I followed the link in the jenkins failure email. Realized that I wasn't logged in. Tried to log into the public jenkins (https://jenkins.qa.ubuntu.com/job/unity-phablet-ci/494/), but jenkins won't let me log in. Apparently, I can log into the the jenkins behind the VPN, but not the one that's public :(

This morning, I saw a four jobs being stuck for many hours on the mediumtests-build job. Two or three of those were mine, but there was at least one other one (maybe two), that weren't mine. So, if it gets stuck, it's not just with my branches. There is something else going on, I think.

[Michael Zanetti (mzanetti)]

Ok, I've compared the 2 jobs and I can't spot any difference that could cause this issues. It might be caused by the fact that the mediumtests-builder job starts later than the i386-ci job and something in either trunk or your branch has changed in the meantime. I have increased the number of simultaneous mediumtests-builder jobs to hopefully minimize the time frame where this can happen.

As a quick workaround, I've set it up to send you an email with a link to the logs when it fails. Hope that helps.

[Michi Henning (michihenning)]

Hey, thanks heaps for that! Waiting for the email for the job that should currently be running (which is guaranteed to fail--I just added some trade to figure out what's going on).

[Michael Zanetti (mzanetti)]

21 +# This gets rid of any stale .gcda files. Run this if a running a binary causes lots of messages about
22 +# about a "merge mismatch for summaries".
23 +add_custom_target(clean-coverage COMMAND find ${CMAKE_BINARY_DIR} -name '*.gcda' | xargs rm -f)

Great one! But shouldn't this be rather part of cmake/modules/EnableCoverageReport.cmake? Seems to be generic enough.

Rest looks good to me!

=== modified file 'CMakeLists.txt'
--- CMakeLists.txt	2013-04-12 11:36:33 +0000
+++ CMakeLists.txt	2013-04-15 01:37:27 +0000
@@ -11,7 +11,7 @@
    message(FATAL_ERROR "In-tree build attempt detected, aborting. Set your build dir outside your source dir, delete CMakeCache.txt from source root and try again.")
 endif()
 
-set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules)
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules)
 
 # Instruct CMake to run moc automatically when needed.
 set(CMAKE_AUTOMOC ON)
@@ -36,9 +36,17 @@
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage" )
   set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} --coverage" )
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --coverage" )
-  ENABLE_COVERAGE_REPORT(TARGETS ${SHELL_APP})
+  ENABLE_COVERAGE_REPORT(TARGETS ${SHELL_APP} FILTER /usr/include ${CMAKE_SOURCE_DIR}/tests/* ${CMAKE_BINARY_DIR}/*)
 endif()
 
+# This gets rid of any stale .gcda files. Run this if a running a binary causes lots of messages about
+# about a "merge mismatch for summaries".
+add_custom_target(clean-coverage COMMAND find ${CMAKE_BINARY_DIR} -name '*.gcda' | xargs rm -f)
+
+# Static C++ checks
+add_custom_target(cppcheck COMMAND cppcheck --enable=all -q --error-exitcode=2
+                                       ${CMAKE_SOURCE_DIR}/src ${CMAKE_SOURCE_DIR}/tests)
+
 include(FindPkgConfig)
 find_package(Qt5Qml)
 find_package(Qt5Quick)

=== modified file 'build'
--- build	2013-04-05 12:44:16 +0000
+++ build	2013-04-15 01:37:27 +0000
@@ -7,18 +7,19 @@
 NUM_JOBS=$(( `grep -c ^processor /proc/cpuinfo` + 1 ))
 
 usage() {
-    echo "usage: build [OPTIONS]\n"
-    echo "Script to build the shell.\n"
-    echo "OPTIONS:"
-    echo "  -s, --setup   Setup the build environment and branch Unity"
-    echo "  -c, --clean   Clean the build tree before building"
-    echo ""
+    echo "usage: build [OPTIONS] [BUILD_TYPE]\n" >&2
+    echo "Script to build the shell. If BUILD_TYPE is not specified, it defaults to \"debug\".\n" >&2
+    echo "OPTIONS:" >&2
+    echo " -s, --setup Setup the build environment and branch Unity" >&2
+    echo " -c, --clean Clean the build tree before building" >&2
+    echo >&2
     exit 1
 }
 
-set -- `getopt -n$0 -u -a --longoptions="setup,clean,help" "sch" "$@"`
+ARGS=`getopt -n$0 -u -a --longoptions="setup,clean,help" -o "sch" -- "$@"`
+[ $? -ne 0 ] && usage
+eval set -- "$ARGS"
 
-# FIXME: giving incorrect arguments does not call usage and exit
 while [ $# -gt 0 ]
 do
     case "$1" in
@@ -30,6 +31,11 @@
     shift
 done
 
+[ $# -gt 1 ] && usage
+
+BUILD_TYPE="debug"
+[ $# -eq 1 ] && BUILD_TYPE="$1"
+
 install_dependencies() {
     RARING=false
     grep -q raring /etc/lsb-release
@@ -66,7 +72,7 @@
     if $CLEAN; then rm -rf builddir; fi
     mkdir -p builddir
     cd builddir
-    PKG_CONFIG_PATH=$UNITY_BUILD_DIR/lib/pkgconfig/:$PKG_CONFIG_PATH cmake -DCMAKE_BUILD_TYPE=debug .. ${GENERATOR}
+    PKG_CONFIG_PATH=$UNITY_BUILD_DIR/lib/pkgconfig/:$PKG_CONFIG_PATH cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. ${GENERATOR}
     ${BUILD_COMMAND}
 fi