Merge lp:~fagerstedt-axel/linaro-android-build-tools/build-android-testsuite-restricted into lp:linaro-android-build-tools

[Axel Fagerstedt]

Adds a new build type which builds stand alone testsuites and bundles them with prebuilt android images fetched from snapshots.

The build script assumes that the specified repo manifest will contain a directory "build" with a makefile providing the targets build and install. It further assumes that making the two targets will produce a directory "out" containing the directories "data" and "system" with any build output in a android style directory structure.

The changes have been tested with the job: https://android-build.linaro.org/builds/~linaro-android-restricted/test-private-test-suite

[Georgy Redkozubov (gesha)]

In general looks good.

One thing I would like to be tested is that out/ dir exists and 'thw' I guess is a mistype:

74 + # move thw boot tarball
75 + mv boot.tar.bz2 out/.
76 +
77 + # we are assuming that the makefile has given us a out/ structure
78 + # containing data/ and system/ with the test files

[lp:~fagerstedt-axel/linaro-android-build-tools/build-android-testsuite-restricted]

626. By Axel Fagerstedt on 2013-06-18

build-android-testsuite: Verify that out has been created.

[Axel Fagerstedt (fagerstedt-axel)]

Updated.

[Georgy Redkozubov (gesha)]

Looks good

review: Approve

=== added file 'build-scripts/build-android-testsuite-restricted'
--- build-scripts/build-android-testsuite-restricted	1970-01-01 00:00:00 +0000
+++ build-scripts/build-android-testsuite-restricted	2013-06-18 21:51:24 +0000
@@ -0,0 +1,110 @@
+###############################################################################
+# Copyright (c) 2013 Linaro
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+###############################################################################
+
+export GIT_SSH=/var/run/lava/ssh
+
+source "${BUILD_SCRIPT_ROOT}"/helpers
+
+trap infrastructure_error ERR
+
+# get the source
+if [ -n "$REPO_SEED_URL" ]; then
+    repo-sync-from-seed "${1}"
+else
+    repo-sync-from-mirror "${1}"
+fi
+
+# get toolchain
+if test -n "$TOOLCHAIN_URL"; then
+    wget -nv --no-check-certificate $TOOLCHAIN_URL
+    toolchain_filename=`echo $TOOLCHAIN_URL | sed -e 's/.*\/\([^/]*\)$/\1/'`
+    mkdir toolchain
+    tar -C toolchain --strip-components 1 -xf $toolchain_filename
+    rm -rf $toolchain_filename
+    export PATH=`pwd`/toolchain/bin:$PATH
+    if test -z "$TOOLCHAIN_TRIPLET"; then
+        #assume arm-linux-gnueabihf-
+	echo "WARNING: assuming arm.linux-gnueabihf toolchain"
+        CROSS_COMPILE="CROSS_COMPILE=arm-linux-gnueabihf-"
+    else
+        CROSS_COMPILE="CROSS_COMPILE=$TOOLCHAIN_TRIPLET"
+    fi
+else
+    echo "ERROR: TOOLCHAIN_URL is not optional."
+    exit 1
+fi
+
+# build the tests
+trap - ERR
+if test -e "build/Makefile"; then
+   make $CROSS_COMPILE -C build/ build
+   make $CROSS_COMPILE -C build/ install
+else
+    echo "ERROR: Could not locate build tools, expecting a Makefile in build/ with targets build and install"
+    exit 1
+fi
+
+trap infrastructure_error ERR
+
+# Verify that the build has produced out/
+if [ ! -d out ]; then
+   echo "ERROR: out directory not found."
+   exit 1
+fi
+
+# Combine output with prebuilt android images
+if test -n "$ANDROID_PREBUILT_URL"; then
+    #get prebuilt android
+    ${BUILD_SCRIPT_ROOT}/fetch_prebuilt.py $ANDROID_PREBUILT_URL
+
+    if [ ! -e boot.tar.bz2 ]; then
+       echo "ERROR: boot.tar.bz2 not downloaded"
+       exit 1
+    elif [ ! -e system.tar.bz2 ]; then
+       echo "ERROR: system.tar.bz2 not downloaded"
+       exit 1
+    elif [ ! -e userdata.tar.bz2 ]; then
+       echo "ERROR: userdata.tar.bz2 not downloaded"
+       exit 1
+    fi
+
+    # move the boot tarball
+    mv boot.tar.bz2 out/.
+
+    # we are assuming that the makefile has given us a out/ structure
+    # containing data/ and system/ with the test files
+
+    # uncompress the archives so we can update them
+    bunzip2 --keep userdata.tar.bz2
+    bunzip2 --keep system.tar.bz2
+    mv userdata.tar out/.
+    mv system.tar out/.
+
+    # update the archives with the tests
+    cd out
+    tar -uvpf userdata.tar data
+    tar -uvpf system.tar system
+
+    # compress them again
+    bzip2 userdata.tar
+    bzip2 system.tar
+
+    # create tarballs with just the tests
+    tar -pcjf tests_userdata.tar.bz2 data
+    tar -pcjf tests_system.tar.bz2 system
+
+    #clean up
+    rm -rf data
+    rm -rf system
+    cd ..
+    rm -rf system.tar.bz2
+    rm -rf userdata.tar.bz2
+else
+    echo "ERROR: Could not locate prebuilt android image, please set ANDROID_PREBUILT_URL"
+fi
+

=== added file 'build-scripts/fetch_prebuilt.py'
--- build-scripts/fetch_prebuilt.py	1970-01-01 00:00:00 +0000
+++ build-scripts/fetch_prebuilt.py	2013-06-18 21:51:24 +0000
@@ -0,0 +1,83 @@
+#!/usr/bin/python
+
+import json
+import urlparse
+import shutil
+import urllib2
+import os
+import sys
+
+
+def download(api_urls, files_to_get):
+    """Example of how to use the API to download a/all files in a directory."""
+
+    # Get listing for file(s) pointed to by URL we were given
+    request = urllib2.urlopen(api_urls.ls())
+    listing = json.loads(request.read())["files"]
+
+    for file_info in listing:
+
+        if file_info["type"] == "folder":
+            # Skip folders...
+            continue
+        elif not file_info["name"] in files_to_get:
+            # only grab the specified files
+            continue
+
+        # Get the licenses. They are returned as a JSON document in the form:
+        # {"licenses":
+        #  [{"text": "<license text>", "digest": "<digest of license>"},
+        #   {"text": "<license text>", "digest": "<digest of license>"},
+        #   ...
+        # ]}
+        # Each license has a digest associated with it.
+        request = urllib2.urlopen(api_urls.license(file_info["url"]))
+        licenses = json.loads(request.read())["licenses"]
+
+        if licenses[0] == "Open":
+            headers = {}
+        else:
+            # Dont download any licensed files
+            continue
+
+        # Once the header has been generated, just download the file.
+        req = urllib2.urlopen(urllib2.Request(api_urls.file(file_info["url"]),
+                                              headers=headers))
+        with open(os.path.basename(file_info["url"]), 'wb') as fp:
+            shutil.copyfileobj(req, fp)
+
+
+class ApiUrls():
+    """Since we want to manipulate URLS, but urlsplit returns an immutable
+    object this is a convenience object to perform the manipulations for us"""
+    def __init__(self, input_url):
+        self.parsed_url = [c for c in urlparse.urlsplit(input_url)]
+        self.path = self.parsed_url[2]
+
+    def ls(self, path=None):
+        if not path:
+            path = self.path
+        self.parsed_url[2] = "/api/ls" + path
+        return urlparse.urlunsplit(self.parsed_url)
+
+    def license(self, path):
+        self.parsed_url[2] = "/api/license" + path
+        return urlparse.urlunsplit(self.parsed_url)
+
+    def file(self, path):
+        self.parsed_url[2] = path
+        return urlparse.urlunsplit(self.parsed_url)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+    # Check that a URL has been supplied.
+        print >> sys.stderr, "Usage: fetch_prebuilt.py <URL>"
+        exit(1)
+
+    api_urls = ApiUrls(sys.argv[1])
+
+    # we are only interested in the fs images
+    files_to_get = "boot.tar.bz2", "system.tar.bz2", "userdata.tar.bz2"
+
+    download(api_urls, files_to_get)

=== modified file 'node/prepare_build_config.py'
--- node/prepare_build_config.py	2013-05-10 10:34:40 +0000
+++ node/prepare_build_config.py	2013-06-18 21:51:24 +0000
@@ -60,7 +60,8 @@
 
     if config.get("BUILD_TYPE", "build-android") in ["build-android-private",
                                                      "build-android-restricted",
-                                                     "build-android-toolchain-linaro-restricted"]:
+                                                     "build-android-toolchain-linaro-restricted",
+                                                     "build-android-testsuite-restricted"]:
         build_type_cat = "restricted"
     else:
         build_type_cat = "normal"