Merge lp:~fgimenez/snappy/filter-tests into lp:~snappy-dev/snappy/snappy-moved-to-github

[Federico Gimenez]

Allow filter tests from the command line.

Allow filter tests from the command line.

The filter flag has been added:

$ go run _integration-tests/main-.go -filter installSuite

The filtering itself is done by generating an adt-run control file on the fly, based on a template.

The same approach could be useful for other use cases too. For instance, when we will have different test binaries, with a template we don't need them to be defined both in the wrapper and in the control file.

[Leo Arias (elopio)]

Shouldn't we have a default control file?
I'm thinking about the future when we can specify the requirements as test classes in the control file, so the runner can call adt-run directly without using our main.go.

So I like this a lot, but I was thinking about also parsing the control file to look for test bed requirements. We can talk tomorrow to figure out how to have both, filtering and test bed specs.

Also, remember that we can't call the test binary directly because the reboot will kill it and we'll lose the results. We need to merge this with the approach in https://code.launchpad.net/~elopio/snappy/upgrade-test/+merge/262657 , that uses a bash script that runs the test binary.

[John Lenton (chipaca)]

lgtm :)

review: Approve

[Leo Arias (elopio)]

Awesome! No complaints from my side, but I still can't test it because of the udf bug. I'll hold the top-approve until I can give it a successful run.

review: Approve

[Leo Arias (elopio)]

Successfully ran with udf from the proposed ppa.a

[Snappy Tarmac (snappydevtarmac)]

Attempt to merge into lp:snappy failed due to conflicts:

duplicate in _integration-tests/data.moved

[lp:~fgimenez/snappy/filter-tests]

529. By Federico Gimenez on 2015-07-06

merged trunk

530. By Federico Gimenez on 2015-07-06

merged prerequisite

531. By Federico Gimenez on 2015-07-06

merged prerequisite

532. By Federico Gimenez on 2015-07-06

merged prerequisite

Unmerged revisions

=== added directory '_integration-tests/data/tpl'
=== added file '_integration-tests/data/tpl/control'
--- _integration-tests/data/tpl/control	1970-01-01 00:00:00 +0000
+++ _integration-tests/data/tpl/control	2015-07-06 07:48:48 +0000
@@ -0,0 +1,10 @@
+{{ $filter := .Filter }}
+{{ range $element := .Tests }}
+Test-Command: debian/integration-tests/snappy-test {{ $element }}.test {{ if $filter }}-gocheck.f {{ $filter }}{{ end }}
+Restrictions: allow-stderr
+{{ end }}
+
+{{ if .IncludeShell }}
+Test-Command: ./_integration-tests/snappy-selftest --yes-really
+Depends:
+{{ end }}

=== modified file '_integration-tests/main.go'
--- _integration-tests/main.go	2015-07-01 08:51:59 +0000
+++ _integration-tests/main.go	2015-07-06 07:48:48 +0000
@@ -28,20 +28,19 @@
 	"path/filepath"
 	"strconv"
 	"strings"
+	"text/template"
 )
 
 const (
-	baseDir          = "/tmp/snappy-test"
-	testsBinDir      = "_integration-tests/bin/"
-	defaultRelease   = "rolling"
-	defaultChannel   = "edge"
-	latestRevision   = ""
-	defaultSSHPort   = 22
-	defaultGoArm     = "7"
-	latestTestName   = "command1"
-	failoverTestName = "command2"
-	updateTestName   = "command3"
-	shellTestName    = "command4"
+	baseDir        = "/tmp/snappy-test"
+	testsBinDir    = "_integration-tests/bin/"
+	defaultRelease = "rolling"
+	defaultChannel = "edge"
+	latestRevision = ""
+	defaultSSHPort = 22
+	defaultGoArm   = "7"
+	controlFile    = "debian/integration-tests/control"
+	controlTpl     = "_integration-tests/data/tpl/control"
 )
 
 var (
@@ -53,9 +52,12 @@
 		[]string{
 			"-s", "/usr/share/autopkgtest/ssh-setup/snappy",
 			"--", "-i", imageTarget}...)
+	testPackagesLatest   = []string{"latest", "failover"}
+	testPackagesPrevious = []string{"update"}
+	testPackages         = append(testPackagesLatest, testPackagesPrevious...)
 )
 
-func setupAndRunTests(useSnappyFromBranch bool, arch, testbedIP string, testbedPort int) {
+func setupAndRunTests(useSnappyFromBranch bool, arch, testbedIP, testFilter string, testbedPort int) {
 	prepareTargetDir(testsBinDir)
 
 	if useSnappyFromBranch {
@@ -66,21 +68,21 @@
 	buildTests(arch)
 
 	rootPath := getRootPath()
+
 	if testbedIP == "" {
+		var includeShell bool
+		if testFilter == "" {
+			includeShell = true
+		}
 		createImage(defaultRelease, defaultChannel, latestRevision)
-		latestTests := []string{
-			latestTestName, failoverTestName, shellTestName}
-		for i := range latestTests {
-			adtRun(rootPath, kvmSSHOptions, latestTests[i])
-		}
+		adtRun(rootPath, testFilter, testPackagesLatest, kvmSSHOptions, includeShell)
 
 		createImage(defaultRelease, defaultChannel, "-1")
-		adtRun(rootPath, kvmSSHOptions, updateTestName)
+		adtRun(rootPath, testFilter, testPackagesPrevious, kvmSSHOptions, false)
 	} else {
 		execCommand("ssh-copy-id", "-p", strconv.Itoa(testbedPort),
 			"ubuntu@"+testbedIP)
-		adtRun(rootPath, remoteTestbedSSHOptions(testbedIP, testbedPort),
-			shellTestName)
+		adtRun(rootPath, "", []string{}, remoteTestbedSSHOptions(testbedIP, testbedPort), true)
 	}
 }
 
@@ -103,9 +105,8 @@
 
 func buildTests(arch string) {
 	fmt.Println("Building tests...")
-	tests := []string{"latest", "failover", "update"}
-	for i := range tests {
-		testName := tests[i]
+
+	for _, testName := range testPackages {
 		goCall(arch, "test", "-c",
 			"./_integration-tests/tests/"+testName)
 		// XXX Go test 1.3 does not have the output flag, so we move the
@@ -143,26 +144,56 @@
 	execCommand(append(udfCommand, coreOptions...)...)
 }
 
-func adtRun(rootPath string, testbedOptions []string, testname string) {
+func adtRun(rootPath, testFilter string, testList, testbedOptions []string, includeShell bool) {
+	createControlFile(testFilter, testList, includeShell)
+
 	fmt.Println("Calling adt-run...")
-	outputDir := filepath.Join(baseDir, "output")
+	outputSubdir := getOutputSubdir(testList, includeShell)
+	outputDir := filepath.Join(baseDir, "output", outputSubdir)
 	prepareTargetDir(outputDir)
 
 	cmd := []string{
 		"adt-run", "-B",
-		"--override-control", "debian/integration-tests/control"}
-
-	cmd = append(cmd, "--testname", testname)
-
-	cmd = append(cmd, []string{
 		"--setup-commands", "touch /run/autopkgtest_no_reboot.stamp",
-		"--override-control", "debian/integration-tests/control",
+		"--override-control", controlFile,
 		"--built-tree", rootPath,
-		"--output-dir", outputDir}...)
+		"--output-dir", outputDir}
 
 	execCommand(append(cmd, testbedOptions...)...)
 }
 
+func createControlFile(testFilter string, testList []string, includeShellTest bool) {
+	type controlData struct {
+		Filter       string
+		Tests        []string
+		IncludeShell bool
+	}
+
+	tpl, err := template.ParseFiles(controlTpl)
+	if err != nil {
+		log.Fatalf("Error reading adt-run control template %s", controlTpl)
+	}
+
+	outputFile, err := os.Create(controlFile)
+	if err != nil {
+		log.Fatalf("Error creating control file %s", controlFile)
+	}
+	defer outputFile.Close()
+
+	err = tpl.Execute(outputFile, controlData{Filter: testFilter, Tests: testList, IncludeShell: includeShellTest})
+	if err != nil {
+		log.Fatalf("execution: %s", err)
+	}
+}
+
+func getOutputSubdir(testList []string, includeShell bool) string {
+	output := strings.Join(testList, "-")
+	if includeShell {
+		output = output + "-shell"
+	}
+	return output
+}
+
 func remoteTestbedSSHOptions(testbedIP string, testbedPort int) []string {
 	options := []string{
 		"-H", testbedIP,
@@ -199,9 +230,11 @@
 			"IP of the testbed. If no IP is passed, a virtual machine will be created for the test.")
 		testbedPort = flag.Int("port", defaultSSHPort,
 			"SSH port of the testbed. Defaults to use port "+strconv.Itoa(defaultSSHPort))
+		testFilter = flag.String("filter", "",
+			"Suites or tests to run, for instance MyTestSuite, MyTestSuite.FirstCustomTest or MyTestSuite.*CustomTest")
 	)
 
 	flag.Parse()
 
-	setupAndRunTests(*useSnappyFromBranch, *arch, *testbedIP, *testbedPort)
+	setupAndRunTests(*useSnappyFromBranch, *arch, *testbedIP, *testFilter, *testbedPort)
 }

=== removed file 'debian/integration-tests/control'
--- debian/integration-tests/control	2015-06-30 15:29:28 +0000
+++ debian/integration-tests/control	1970-01-01 00:00:00 +0000
@@ -1,11 +0,0 @@
-Test-Command: debian/integration-tests/snappy-test latest.test
-Restrictions: allow-stderr
-
-Test-Command: debian/integration-tests/snappy-test failover.test
-Restrictions: allow-stderr
-
-Test-Command: debian/integration-tests/snappy-test update.test
-Restrictions: allow-stderr
-
-Test-Command: ./_integration-tests/snappy-selftest --yes-really
-Depends:

=== modified file 'debian/integration-tests/snappy-test'
--- debian/integration-tests/snappy-test	2015-07-03 11:04:00 +0000
+++ debian/integration-tests/snappy-test	2015-07-06 07:48:48 +0000
@@ -23,7 +23,10 @@
 
 export PATH=$(pwd)/_integration-tests/bin:$PATH
 
-${TEST} -check.vv -test.outputdir=$ADT_ARTIFACTS-$ADT_REBOOT_MARK
+# shift to remove the test binary name (first argument) and be able to pass the rest
+# of them to it
+shift
+${TEST} -check.vv -test.outputdir=$ADT_ARTIFACTS-$ADT_REBOOT_MARK "$@"
 
 if [ -e ${NEEDS_REBOOT} ]; then
     mark=`cat ${NEEDS_REBOOT}`