Merge lp:~terceiro/lava-dispatcher/modularize-distro-support into lp:lava-dispatcher

[Antonio Terceiro]

This branch makes some changes to the dispatcher in order to ease the addition of new supported distros. I think we can improve the situation even further, but this is already a good start.

This is a pre-requisite for adding Fedora support (bug #1165999) in a more-or-less clean way.

[Fathi Boudra (fboudra)]

Looks good to me. +1

review: Approve

[Fathi Boudra (fboudra)]

+1 still :)

review: Approve

=== modified file 'lava_dispatcher/actions/lava_test_shell.py'
--- lava_dispatcher/actions/lava_test_shell.py	2013-04-07 19:43:05 +0000
+++ lava_dispatcher/actions/lava_test_shell.py	2013-04-15 19:40:37 +0000
@@ -106,7 +106,7 @@
 # to the host and turned into a bundle for submission to the dashboard.
 
 from datetime import datetime
-import glob
+from glob import glob
 import logging
 import os
 import pexpect
@@ -132,38 +132,18 @@
 from lava_dispatcher.downloader import download_image
 
 LAVA_TEST_DIR = '%s/../../lava_test_shell' % os.path.dirname(__file__)
-LAVA_TEST_ANDROID = '%s/lava-test-runner-android' % LAVA_TEST_DIR
-LAVA_TEST_UBUNTU = '%s/lava-test-runner-ubuntu' % LAVA_TEST_DIR
-LAVA_TEST_UPSTART = '%s/lava-test-runner.conf' % LAVA_TEST_DIR
-LAVA_TEST_INITD = '%s/lava-test-runner.init.d' % LAVA_TEST_DIR
-LAVA_TEST_SHELL = '%s/lava-test-shell' % LAVA_TEST_DIR
-LAVA_TEST_CASE = '%s/lava-test-case' % LAVA_TEST_DIR
-LAVA_TEST_CASE_ATTACH = '%s/lava-test-case-attach' % LAVA_TEST_DIR
-LAVA_TEST_RUN_ATTACH = '%s/lava-test-run-attach' % LAVA_TEST_DIR
 
-Target.android_deployment_data['lava_test_runner'] = LAVA_TEST_ANDROID
-Target.android_deployment_data['lava_test_shell'] = LAVA_TEST_SHELL
-Target.android_deployment_data['lava_test_case'] = LAVA_TEST_CASE
-Target.android_deployment_data['lava_test_case_attach'] = LAVA_TEST_CASE_ATTACH
-Target.android_deployment_data['lava_test_run_attach'] = LAVA_TEST_RUN_ATTACH
+Target.android_deployment_data['distro'] = 'android'
 Target.android_deployment_data['lava_test_sh_cmd'] = '/system/bin/mksh'
 Target.android_deployment_data['lava_test_dir'] = '/data/lava'
 Target.android_deployment_data['lava_test_results_part_attr'] = 'data_part_android_org'
 
-Target.ubuntu_deployment_data['lava_test_runner'] = LAVA_TEST_UBUNTU
-Target.ubuntu_deployment_data['lava_test_shell'] = LAVA_TEST_SHELL
-Target.ubuntu_deployment_data['lava_test_case'] = LAVA_TEST_CASE
-Target.ubuntu_deployment_data['lava_test_case_attach'] = LAVA_TEST_CASE_ATTACH
-Target.ubuntu_deployment_data['lava_test_run_attach'] = LAVA_TEST_RUN_ATTACH
+Target.ubuntu_deployment_data['distro'] = 'ubuntu'
 Target.ubuntu_deployment_data['lava_test_sh_cmd'] = '/bin/bash'
 Target.ubuntu_deployment_data['lava_test_dir'] = '/lava'
 Target.ubuntu_deployment_data['lava_test_results_part_attr'] = 'root_part'
 
-Target.oe_deployment_data['lava_test_runner'] = LAVA_TEST_UBUNTU
-Target.oe_deployment_data['lava_test_shell'] = LAVA_TEST_SHELL
-Target.oe_deployment_data['lava_test_case'] = LAVA_TEST_CASE
-Target.oe_deployment_data['lava_test_case_attach'] = LAVA_TEST_CASE_ATTACH
-Target.oe_deployment_data['lava_test_run_attach'] = LAVA_TEST_RUN_ATTACH
+Target.oe_deployment_data['distro'] = 'oe'
 Target.oe_deployment_data['lava_test_sh_cmd'] = '/bin/sh'
 Target.oe_deployment_data['lava_test_dir'] = '/lava'
 Target.oe_deployment_data['lava_test_results_part_attr'] = 'root_part'
@@ -376,11 +356,17 @@
             f.write('set -ex\n')
             f.write('cd %s\n' % targetdir)
 
-            # TODO how should we handle this for Android?
+            distro = self.context.client.target_device.deployment_data['distro']
+
+            # generic dependencies - must be named the same across all distros
+            # supported by the testdef
             deps = self.testdef['install'].get('deps', [])
+
+            # distro-specific dependencies
+            deps = deps + self.testdef['install'].get('deps-' + distro, [])
+
             if deps:
-                f.write('sudo apt-get update\n')
-                f.write('sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -q ')
+                f.write('lava-install-packages ')
                 for dep in deps:
                     f.write('%s ' % dep)
                 f.write('\n')
@@ -484,7 +470,6 @@
 
     def run(self, testdef_urls=None, testdef_repos=None, timeout=-1):
         target = self.client.target_device
-        self._assert_target(target)
 
         testdefs_by_uuid = self._configure_target(target, testdef_urls, testdef_repos)
 
@@ -533,26 +518,25 @@
         return False
 
     def _copy_runner(self, mntdir, target):
-        runner = target.deployment_data['lava_test_runner']
-        shutil.copy(runner, '%s/bin/lava-test-runner' % mntdir)
-        os.chmod('%s/bin/lava-test-runner' % mntdir, XMOD)
-
-        shcmd = target.deployment_data['lava_test_sh_cmd']
-
-        for key in ['lava_test_shell', 'lava_test_case_attach', 'lava_test_run_attach']:
-            fname = target.deployment_data[key]
+        shell = target.deployment_data['lava_test_sh_cmd']
+
+        # Generic scripts
+        scripts_to_copy = glob(os.path.join(LAVA_TEST_DIR, 'lava-*'))
+
+        # Distro-specific scripts override the generic ones
+        distro = target.deployment_data['distro']
+        distro_support_dir = '%s/distro/%s' % (LAVA_TEST_DIR, distro)
+        for script in glob(os.path.join(distro_support_dir, 'lava-*')):
+            scripts_to_copy.append(script)
+
+        for fname in scripts_to_copy:
             with open(fname, 'r') as fin:
-                with open('%s/bin/%s' % (mntdir, os.path.basename(fname)), 'w') as fout:
-                    fout.write("#!%s\n\n" % shcmd)
+                foutname = os.path.basename(fname)
+                with open('%s/bin/%s' % (mntdir, foutname), 'w') as fout:
+                    fout.write("#!%s\n\n" % shell)
                     fout.write(fin.read())
                     os.fchmod(fout.fileno(), XMOD)
 
-        tc = target.deployment_data['lava_test_case']
-        with open(tc, 'r') as fin:
-            with open('%s/bin/lava-test-case' % mntdir, 'w') as fout:
-                fout.write('#!%s\n\n' % shcmd)
-                fout.write(fin.read())
-                os.fchmod(fout.fileno(), XMOD)
 
     def _mk_runner_dirs(self, mntdir):
         utils.ensure_directory('%s/bin' % mntdir)
@@ -618,16 +602,3 @@
         with os.fdopen(fd, 'w') as f:
             DocumentIO.dump(f, bundle)
 
-    def _assert_target(self, target):
-        """ Ensure the target has the proper deployment data required by this
-        action. This allows us to exit the action early rather than going 75%
-        through the steps before discovering something required is missing
-        """
-        if not target.deployment_data:
-            raise RuntimeError('Target includes no deployment_data')
-
-        keys = ['lava_test_runner', 'lava_test_shell', 'lava_test_dir',
-                'lava_test_sh_cmd']
-        for k in keys:
-            if k not in target.deployment_data:
-                raise RuntimeError('Target deployment_data missing %s' % k)

=== added file 'lava_test_shell/README'
--- lava_test_shell/README	1970-01-01 00:00:00 +0000
+++ lava_test_shell/README	2013-04-15 19:40:37 +0000
@@ -0,0 +1,11 @@
+This directory contains support scripts for lava-test-shell.
+
+The scripts in this directory will be copied into the target device and will be
+in $PATH during the lava-test-shell execution.
+
+Distribution-specific scripts can be placed in distro/$distroname, and will
+override the ones in the top level directory. For example,
+distro/android/lava-test-runner will be used on Android instead of the
+lava-test-runner script present at the same directory as this README file.
+
+All scripts have to be named using a "lava-" suffix.

=== added directory 'lava_test_shell/distro'
=== added directory 'lava_test_shell/distro/android'
=== renamed file 'lava_test_shell/lava-test-runner-android' => 'lava_test_shell/distro/android/lava-test-runner'
=== added directory 'lava_test_shell/distro/ubuntu'
=== added file 'lava_test_shell/distro/ubuntu/lava-install-packages'
--- lava_test_shell/distro/ubuntu/lava-install-packages	1970-01-01 00:00:00 +0000
+++ lava_test_shell/distro/ubuntu/lava-install-packages	2013-04-15 19:40:37 +0000
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+sudo DEBIAN_FRONTEND=noninteractive apt-get update
+sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -q "$@"

=== added file 'lava_test_shell/distro/ubuntu/lava-installed-packages'
--- lava_test_shell/distro/ubuntu/lava-installed-packages	1970-01-01 00:00:00 +0000
+++ lava_test_shell/distro/ubuntu/lava-installed-packages	2013-04-15 19:40:37 +0000
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+dpkg-query -W -f '${status} ${package} : ${version}\n' | sed -n 's/^install ok installed/package:/p'

=== added file 'lava_test_shell/distro/ubuntu/lava-os-build'
--- lava_test_shell/distro/ubuntu/lava-os-build	1970-01-01 00:00:00 +0000
+++ lava_test_shell/distro/ubuntu/lava-os-build	2013-04-15 19:40:37 +0000
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cat /etc/lsb-release | grep DESCRIPTION | cut -d\" -f2

=== added file 'lava_test_shell/lava-installed-packages'
--- lava_test_shell/lava-installed-packages	1970-01-01 00:00:00 +0000
+++ lava_test_shell/lava-installed-packages	2013-04-15 19:40:37 +0000
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo 'Unsupported distro: cannot obtain list of installed packages'

=== added file 'lava_test_shell/lava-os-build'
--- lava_test_shell/lava-os-build	1970-01-01 00:00:00 +0000
+++ lava_test_shell/lava-os-build	2013-04-15 19:40:37 +0000
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo 'Unsupported distro: cannot determine build version'

=== renamed file 'lava_test_shell/lava-test-runner-ubuntu' => 'lava_test_shell/lava-test-runner'
--- lava_test_shell/lava-test-runner-ubuntu	2013-03-15 16:50:30 +0000
+++ lava_test_shell/lava-test-runner	2013-04-15 19:40:37 +0000
@@ -5,12 +5,6 @@
 RESULTSDIR="/lava/results"
 BINDIR="/lava/bin"
 
-detect_distro() {
-	test -x /usr/bin/dpkg-query && echo "debian_based" && return
-
-	echo "unknown"
-}
-
 hwcontext()
 {
 	mkdir -p ${RESULTSDIR}/hwcontext
@@ -21,31 +15,17 @@
 	[ -f ${meminfo} ] || cat /proc/meminfo > ${meminfo}
 }
 
-unknown_swcontext() {
-	mkdir -p ${RESULTSDIR}/swcontext
-	build=${RESULTSDIR}/swcontext/build.txt
-	pkgs=${RESULTSDIR}/swcontext/pkgs.txt
-
-	# we don't know about the software context
-	echo 'Unsupported distro: cannot determine build version' > "$build"
-	echo 'Unsupported distro: cannot obtain list of installed packages' > "$pkgs"
-}
-
-debian_based_swcontext()
+swcontext()
 {
 	mkdir -p ${RESULTSDIR}/swcontext
 	build=${RESULTSDIR}/swcontext/build.txt
 	pkgs=${RESULTSDIR}/swcontext/pkgs.txt
 
-	[ -f ${build} ] || cat /etc/lsb-release | grep DESCRIPTION | cut -d\" -f2 > ${build}
-	# this does a query of installed packaged that will look similar to
+	lava-os-build > ${build}
+
+	# this has to print a list of installed packages that will look similar to
 	# what android's package list does
-	[ -f ${pkgs} ]  || dpkg-query -W -f '${status} ${package} : ${version}\n' | sed -n 's/^install ok installed/package:/p'  > ${pkgs}
-}
-
-swcontext() {
-	distro=$(detect_distro)
-	${distro}_swcontext
+	lava-installed-packages  > ${pkgs}
 }
 
 cleanup()