Merge lp:~qzhang/lava-dispatcher/another-inject-kernel into lp:lava-dispatcher

[Spring Zhang]

Another injecting kernel, use l-h-r now.
1. use linaro-hwpack-replace to get a new hwpack with new kernel before deployment.
2. tested once and pass
3. only support new kernel in deb format in normal image and tarball format in Android image.
4. job file like:
  "actions": [
    {
      "command": "deploy_linaro_image",
      "parameters":
        {
          "rootfs":
"http://snapshots.linaro.org/11.05-daily/linaro-nano/20110612/0/images/tar/nano-n-tar-20110612-0.tar.gz",
          "hwpack":
"http://snapshots.linaro.org/11.05-daily/linaro-hwpacks/lt-mx53loco/20110609/0/images/hwpack/hwpack_linaro-lt-mx53loco_20110609-0_armel_supported.tar.gz",
          "kernel_matrix":[
"http://pkgserver/original/linux-image-2.6.38-1000-linaro-lt-mx5_2.6.38-1000.7_armel.deb", "linux-image-2.6.38"]
        }
    },

[Michael Hudson-Doyle (mwhudson)]

This looks basically OK to me, bet it conflicts with trunk now though.

[lp:~qzhang/lava-dispatcher/another-inject-kernel]

101. By Spring Zhang on 2011-09-14

sync with mainline

102. By Spring Zhang on 2011-09-14

sync with mainline

103. By Spring Zhang on 2011-09-14

sync with mainline

104. By Spring Zhang on 2011-09-15

sync with mainline

105. By Spring Zhang on 2011-09-15

fix another OSError 18 link error

106. By Spring Zhang on 2011-09-15

fix minor bugs

=== added file 'doc/android-new-kernel.json'
--- doc/android-new-kernel.json	1970-01-01 00:00:00 +0000
+++ doc/android-new-kernel.json	2011-09-15 07:09:50 +0000
@@ -0,0 +1,37 @@
+{
+  "job_name": "android_new_kernel",
+  "image_type": "android",
+  "target": "panda01",
+  "timeout": 18000,
+  "actions": [
+    {
+      "command": "deploy_linaro_android_image",
+      "parameters":
+        {
+          "boot": "https://android-build.linaro.org/jenkins/job/linaro-android_leb-panda/61/artifact/build/out/target/product/pandaboard/boot.tar.bz2",
+          "system": "https://android-build.linaro.org/jenkins/job/linaro-android_leb-panda/61/artifact/build/out/target/product/pandaboard/system.tar.bz2",
+          "data": "https://android-build.linaro.org/jenkins/job/linaro-android_leb-panda/61/artifact/build/out/target/product/pandaboard/userdata.tar.bz2",
+          "pkg": "https://android-build.linaro.org/jenkins/job/linaro-android_leb-panda/171/artifact/build/out/target/product/pandaboard/boot.tar.bz2"
+        },
+      "metadata":
+        {
+          "rootfs.type": "android",
+          "rootfs.build": "61"
+        }
+    },
+    {
+      "command": "boot_linaro_android_image"
+    },
+    {
+      "command": "test_android_basic"
+    },
+    {
+      "command": "submit_results_on_host",
+      "parameters":
+        {
+          "server": "http://validation.linaro.org/launch-control",
+          "stream": "/anonymous/android-panda01-basic/"
+        }
+    }
+  ]
+}

=== renamed file 'doc/lava-ltp-job' => 'doc/lava-ltp-job.json'
=== added file 'doc/lava-new-kernel.json'
--- doc/lava-new-kernel.json	1970-01-01 00:00:00 +0000
+++ doc/lava-new-kernel.json	2011-09-15 07:09:50 +0000
@@ -0,0 +1,30 @@
+{
+  "job_name": "new-kernel",
+  "target": "mx53loco01",
+  "timeout": 6000,
+  "actions": [
+    {
+      "command": "deploy_linaro_image",
+      "parameters":
+        {
+          "rootfs":
+"http://snapshots.linaro.org/11.05-daily/linaro-nano/20110612/0/images/tar/nano-n-tar-20110612-0.tar.gz",
+          "hwpack":
+"http://snapshots.linaro.org/11.05-daily/linaro-hwpacks/lt-mx53loco/20110609/0/images/hwpack/hwpack_linaro-lt-mx53loco_20110609-0_armel_supported.tar.gz",
+          "kernel_matrix":[
+"http://pkgserver/original/linux-image-2.6.38-1000-linaro-lt-mx5_2.6.38-1000.7_armel.deb", "linux-image-2.6.38"]
+        }
+    },
+    {
+      "command": "boot_linaro_image"
+    },
+    {
+      "command": "submit_results",
+      "parameters":
+        {
+          "server": "http://validation.linaro.org/launch-control",
+          "stream": "/anonymous/testresult/"
+        }
+    }
+  ]
+}

=== modified file 'lava_dispatcher/actions/android_deploy.py'
--- lava_dispatcher/actions/android_deploy.py	2011-09-05 22:39:47 +0000
+++ lava_dispatcher/actions/android_deploy.py	2011-09-15 07:09:50 +0000
@@ -29,7 +29,7 @@
 from lava_dispatcher.client import CriticalError
 
 class cmd_deploy_linaro_android_image(BaseAction):
-    def run(self, boot, system, data, use_cache=True):
+    def run(self, boot, system, data, pkg=None, use_cache=True):
         LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir
         LAVA_IMAGE_URL = self.context.lava_image_url
         client = self.client
@@ -49,8 +49,8 @@
             raise CriticalError("Unable to reach LAVA server, check network")
 
         try:
-            boot_tbz2, system_tbz2, data_tbz2 = self.download_tarballs(boot,
-                system, data, use_cache)
+            boot_tbz2, system_tbz2, data_tbz2, pkg_tbz2 = \
+                self.download_tarballs(boot, system, data, pkg, use_cache)
         except:
             tb = traceback.format_exc()
             client.sio.write(tb)
@@ -66,9 +66,16 @@
             LAVA_IMAGE_URL, system_tarball])
         data_url = '/'.join(u.strip('/') for u in [
             LAVA_IMAGE_URL, data_tarball])
+        if pkg_tbz2:
+            pkg_tarball = pkg_tbz2.replace(LAVA_IMAGE_TMPDIR, '')
+            pkg_url = '/'.join(u.strip('/') for u in [
+                LAVA_IMAGE_URL, pkg_tarball])
 
         try:
-            self.deploy_linaro_android_testboot(boot_url)
+            if pkg_tbz2:
+                self.deploy_linaro_android_testboot(boot_url, pkg_url)
+            else:
+                self.deploy_linaro_android_testboot(boot_url)
             self.deploy_linaro_android_testrootfs(system_url)
             self.purge_linaro_android_sdcard()
         except:
@@ -78,12 +85,14 @@
         finally:
             shutil.rmtree(self.tarball_dir)
 
-    def download_tarballs(self, boot_url, system_url, data_url, use_cache=True):
+    def download_tarballs(self, boot_url, system_url, data_url, pkg_url=None,
+            use_cache=True):
         """Download tarballs from a boot, system and data tarball url
 
         :param boot_url: url of the Linaro Android boot tarball to download
         :param system_url: url of the Linaro Android system tarball to download
         :param data_url: url of the Linaro Android data tarball to download
+        :param pkg_url: url of the custom kernel tarball to download
         :param use_cache: whether or not to use the cached copy (if it exists)
         """
         lava_cachedir = self.context.lava_cachedir
@@ -96,13 +105,21 @@
             boot_path = download_with_cache(boot_url, tarball_dir, lava_cachedir)
             system_path = download_with_cache(system_url, tarball_dir, lava_cachedir)
             data_path = download_with_cache(data_url, tarball_dir, lava_cachedir)
+            if pkg_url:
+                pkg_path = download_with_cache(pkg_url, tarball_dir)
+            else:
+                pkg_path = None
         else:
             boot_path = download(boot_url, tarball_dir)
             system_path = download(system_url, tarball_dir)
             data_path = download(data_url, tarball_dir)
-        return  boot_path, system_path, data_path
+            if pkg_url:
+                pkg_path = download(pkg_url, tarball_dir)
+            else:
+                pkg_path = None
+        return  boot_path, system_path, data_path, pkg_path
 
-    def deploy_linaro_android_testboot(self, boottbz2):
+    def deploy_linaro_android_testboot(self, boottbz2, pkgbz2=None):
         client = self.client
         client.run_cmd_master('mkfs.vfat /dev/disk/by-label/testboot '
                               '-n testboot')
@@ -111,6 +128,10 @@
         client.run_cmd_master('mount /dev/disk/by-label/testboot '
                               '/mnt/lava/boot')
         client.run_cmd_master('wget -qO- %s |tar --numeric-owner -C /mnt/lava -xjf -' % boottbz2)
+        if pkgbz2:
+            client.run_shell_command(
+                'wget -qO- %s |tar --numeric-owner -C /mnt/lava -xjf -' 
+                    % pkgbz2, response = MASTER_STR)
 
         self.recreate_uInitrd()
 

=== modified file 'lava_dispatcher/actions/deploy.py'
--- lava_dispatcher/actions/deploy.py	2011-09-08 23:35:05 +0000
+++ lava_dispatcher/actions/deploy.py	2011-09-15 07:09:50 +0000
@@ -15,11 +15,11 @@
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along
-# with this program; if not, see <http://www.gnu.org/licenses>.
+# along with this program; if not, see <http://www.gnu.org/licenses>.
 
 from commands import getoutput, getstatusoutput
 import os
+import sys
 import re
 import shutil
 import traceback
@@ -31,13 +31,15 @@
 
 
 class cmd_deploy_linaro_image(BaseAction):
-    def run(self, hwpack, rootfs, use_cache=True):
+    def run(self, hwpack, rootfs, kernel_matrix=None, use_cache=True):
         LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir
         LAVA_IMAGE_URL = self.context.lava_image_url
         client = self.client
         print "deploying on %s" % client.hostname
         print "  hwpack: %s" % hwpack
         print "  rootfs: %s" % rootfs
+        if kernel_matrix:
+            print "  package: %s" % kernel_matrix[0]
         print "Booting master image"
         client.boot_master_image()
 
@@ -49,9 +51,17 @@
             client.sio.write(tb)
             raise CriticalError("Unable to reach LAVA server, check network")
 
+        if kernel_matrix:
+            hwpack = self.refresh_hwpack(kernel_matrix, hwpack, use_cache)
+            #make new hwpack downloadable
+            hwpack = hwpack.replace(LAVA_IMAGE_TMPDIR, '')
+            hwpack = '/'.join(u.strip('/') for u in [
+                LAVA_IMAGE_URL, hwpack])
+            print "Hwpack with new kernel: %s" % hwpack
+
         try:
-            boot_tgz, root_tgz = self.generate_tarballs(
-                hwpack, rootfs, use_cache)
+            boot_tgz, root_tgz = self.generate_tarballs(hwpack, rootfs, 
+                use_cache)
         except:
             tb = traceback.format_exc()
             client.sio.write(tb)
@@ -119,6 +129,7 @@
         self.tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
         tarball_dir = self.tarball_dir
         os.chmod(tarball_dir, 0755)
+        #fix me: if url is not http-prefix, copy it to tarball_dir
         if use_cache:
             hwpack_path = download_with_cache(hwpack_url, tarball_dir, lava_cachedir)
             rootfs_path = download_with_cache(rootfs_url, tarball_dir, lava_cachedir)
@@ -184,3 +195,45 @@
             'wget -qO- %s |tar --numeric-owner -C /mnt/boot -xzf -' % bootfs)
         client.run_cmd_master('umount /mnt/boot')
 
+    def refresh_hwpack(self, kernel_matrix, hwpack, use_cache=True):
+        client = self.client
+        LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir
+        print "Deploying new kernel"
+        new_kernel = kernel_matrix[0]
+        deb_prefix = kernel_matrix[1]
+        filesuffix = new_kernel.split(".")[-1]
+
+        if filesuffix != "deb":
+            raise CriticalError("New kernel only support deb kernel package!")
+
+        # download package to local
+        tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR)
+        os.chmod(tarball_dir, 0755)
+        if use_cache:
+            kernel_path = download_with_cache(new_kernel, tarball_dir)
+            hwpack_path = download_with_cache(hwpack, tarball_dir)
+        else:
+            kernel_path = download(new_kernel, tarball_dir)
+            hwpack_path = download(hwpack, tarball_dir)
+
+        cmd = ("sudo linaro-hwpack-replace -t %s -p %s -r %s" 
+                % (hwpack_path, kernel_path, deb_prefix))
+
+        rc, output = getstatusoutput(cmd)
+        if rc:
+            shutil.rmtree(tarball_dir)
+            tb = traceback.format_exc()
+            client.sio.write(tb)
+            raise RuntimeError("linaro-hwpack-replace failed: %s" % output)
+
+        #fix it:l-h-r doesn't make a output option to specify the output hwpack,
+        #so it needs to do manually here
+
+        #remove old hwpack and leave only new hwpack in tarball_dir
+        os.remove(hwpack_path)
+        hwpack_list = os.listdir(tarball_dir)
+        for hp in hwpack_list:
+            if hp.split(".")[-1] == "gz":
+                new_hwpack_path = os.path.join(tarball_dir, hp)
+                return new_hwpack_path
+

=== modified file 'lava_dispatcher/client.py'
--- lava_dispatcher/client.py	2011-09-14 16:05:43 +0000
+++ lava_dispatcher/client.py	2011-09-15 07:09:50 +0000
@@ -130,7 +130,7 @@
         # Details: system PS1 is set in /etc/bash.bashrc and user PS1 is set in
         # /root/.bashrc, it is
         # "${debian_chroot:+($debian_chroot)}\u@\h:\w\$ "
-        self.proc.sendline('export PS1="$PS1 rc=$(echo \$?) "')
+        self.proc.sendline('export PS1="$PS1 [rc=$(echo \$?)]: "')
         self.proc.expect(self.tester_str)
 
     def enter_uboot(self):

=== modified file 'lava_dispatcher/utils.py'
--- lava_dispatcher/utils.py	2011-09-01 03:03:09 +0000
+++ lava_dispatcher/utils.py	2011-09-15 07:09:50 +0000
@@ -45,7 +45,11 @@
     if os.path.exists(cache_loc):
         filename = os.path.basename(cache_loc)
         file_location = os.path.join(path, filename)
-        os.link(cache_loc, file_location)
+        try:
+            os.link(cache_loc, file_location)
+        except OSError, err:
+            if err.errno == 18:
+                shutil.copy(cache_loc, file_location)
     else:
         file_location = download(url, path)
         try: