Merge lp:~canonical-platform-qa/qa-jenkins-jobs/report-to-practitest into lp:qa-jenkins-jobs

[Allan LeSage]

Report test results to PractiTest.

Report test results to PractiTest.

[Max Brustkern (nuclearbob)]

I think starting the job names with "ubuntu-system-tests" is better since that matches up with the launchpad project name. The other job name changes I like.

Some diff comments. I think one or two things could be cleaner, and I'll be suggesting some packaging-related changes later.

[Allan LeSage (allanlesage)]

Max thanks for your review, I made the one small change you suggested.

We do have to use the qakit from the specific revision for the moment, I want to return to the qakit-packaging project but we do need to get this committed so that we can start reporting results first.

The ust_* renaming is I'm afraid necessary for the moment b/c qakit barfs inexplicably on long path names, or this is the only explanation I'm able to supply, happy to demonstrate, intend to get to the bottom of.

In short yes this is a WIP thanks for your merciful review.

[lp:~canonical-platform-qa/qa-jenkins-jobs/report-to-practitest]

48. By Allan LeSage on 2016-03-11

TEST_UPPER_LEVEL only consider leading chars.

[Max Brustkern (nuclearbob)]

Ship it.

review: Approve

=== added file 'scripts/ubuntu-system-tests/report-results-to-practitest.sh'
--- scripts/ubuntu-system-tests/report-results-to-practitest.sh	1970-01-01 00:00:00 +0000
+++ scripts/ubuntu-system-tests/report-results-to-practitest.sh	2016-03-11 20:58:06 +0000
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+set -x
+set -e
+
+export SUBUNIT_RESULTS_PATH=$1
+export TEST_LEVEL=$2
+export LABEL=$3
+
+rm -rf qakit
+bzr branch lp:qakit -r30
+cd qakit
+
+virtualenv --python=python3 ve3
+. ve3/bin/activate
+python3 setup.py develop
+
+bzr cat "$UBUNTU_SYSTEM_TESTS_CONFIG_BRANCH/practitest/config.ini" > config.ini
+
+upload_system_practitest () {
+  adb -s $ANDROID_SERIAL shell system-image-cli -i | tr '\r' ' ' > VERSION_INFO
+  export IMAGE=$(grep "version version" VERSION_INFO | sed 's/^version version: //;s/ $//')
+  export DEVICE=$(grep "device name" VERSION_INFO | sed 's/^device name: \(.\)/\u\1/;s/ $//')
+  export RELEASE_NUMBER=$(adb shell lsb_release -rs | tr -d '\r')
+  export RELEASE_NAME=$(adb shell lsb_release -cs | tr -d '\r')
+  TEST_LEVEL_LOWER=$1
+  TEST_LEVEL_UPPER=$(echo $1 | sed 's/^\(.\)/\u\1/')
+  SUFFIX=$2
+  SUBUNIT_FILE=$3
+
+  if [ $1 = "sanity" ] ; then
+    export MANUAL="--manual=${TEST_LEVEL_UPPER}/${DEVICE}"
+  else
+    export MANUAL=""
+  fi
+
+  subunit-practitest-export $SUBUNIT_FILE create "${TEST_LEVEL_LOWER}-${DEVICE}-${RELEASE_NAME}-${IMAGE}-${SUFFIX}" ${DEVICE} ${TEST_LEVEL_UPPER} ${RELEASE_NUMBER} ${IMAGE} "'$(adb shell system-image-cli -i)'" ${MANUAL}
+
+}
+
+# this enables HTTP to the outside world (e.g. PractiTest) as managed by IS
+export https_proxy="https://squid.internal:3128/"
+
+# looks like:
+# upload_system_practitest sanity uk "${WORKSPACE}/foo/test-results.subunit"
+upload_system_practitest ${TEST_LEVEL} ${LABEL} ${SUBUNIT_RESULTS_PATH}
+
+unset https_proxy
+deactivate

=== modified file 'ubuntu-system-tests/jobs.yaml'
--- ubuntu-system-tests/jobs.yaml	2016-03-07 18:47:31 +0000
+++ ubuntu-system-tests/jobs.yaml	2016-03-11 20:58:06 +0000
@@ -29,14 +29,12 @@
             device_channel: 'bq-aquaris.en'
         - arale:
             device_channel: 'meizu.en'
-    release:
-        - vivid
     jobs:
-        - 'ubuntu-system-tests_{release}_{channel}_{device}_{test_level}_triggered'
-        - 'ubuntu-system-tests_{release}_{channel}_{device}_{test_level}_freestyle'
+        - 'ust_{channel}_{device}_{test_level}_PT'
+        - 'ust_{channel}_{device}_{test_level}'
 
 - job-template:
-    name: 'ubuntu-system-tests_{release}_{channel}_{device}_{test_level}_triggered'
+    name: 'ust_{channel}_{device}_{test_level}_PT'
     description: "Triggered upon image update."
     parameters:
         - ubuntu-system-tests-parameters:
@@ -59,7 +57,7 @@
             results: 'results/artifacts/test-results.xml'
 
 - job-template:
-    name: 'ubuntu-system-tests_{release}_{channel}_{device}_{test_level}_freestyle'
+    name: 'ust_{channel}_{device}_{test_level}'
     description: "Provided for experimentation--e.g. re-running of failed tests--in way that won't contribute to the pass/fail record of the canonical job."
     parameters:
         - ubuntu-system-tests-parameters:
@@ -140,11 +138,10 @@
 
             rm -rf qa-jenkins-jobs
             bzr branch $QA_JENKINS_JOBS_BRANCH qa-jenkins-jobs
-            cd qa-jenkins-jobs
 
             export UBUNTU_SYSTEM_TESTS_CONFIG_BRANCH="lp:ubuntu-system-tests-config"
             export CONFIG_PATH=${WORKSPACE}/ubuntu-system-tests.conf
-            ./scripts/ubuntu-system-tests/write-config.sh
+            ./qa-jenkins-jobs/scripts/ubuntu-system-tests/write-config.sh
             cat >> $CONFIG_PATH <<EOF
             device_serial = $ANDROID_SERIAL
             output_dir = $OUTPUTDIR
@@ -164,12 +161,24 @@
                 ;;
             esac
 
-            ./scripts/get-recovery-image.sh
+            ./qa-jenkins-jobs/scripts/get-recovery-image.sh
 
             recover_device_flash_and_run_tests () {
               # NOTE pass in a list of tests to run
-              ./scripts/recover-device.sh
-              ./scripts/flash-device.sh
-              ./scripts/ubuntu-system-tests/run-ubuntu-system-tests.sh "$1"
+              ./qa-jenkins-jobs/scripts/recover-device.sh
+              ./qa-jenkins-jobs/scripts/flash-device.sh
+              ./qa-jenkins-jobs/scripts/ubuntu-system-tests/run-ubuntu-system-tests.sh "$1"
             }
             recover_device_flash_and_run_tests "$(echo "$TESTS_TO_RUN")"
+
+            # default to sanity for testing, TODO: consider exiting if not sanity or regression
+            case "${TESTS_TO_RUN}" in
+              "regression")
+                export TEST_LEVEL=regression
+                ;;
+              *)
+                export TEST_LEVEL=sanity
+                ;;
+            esac
+            # "1SS" is the name of the lab
+            ./qa-jenkins-jobs/scripts/ubuntu-system-tests/report-results-to-practitest.sh $WORKSPACE/results/artifacts/test-results.subunit $TEST_LEVEL 1SS