Merge lp:~canonical-platform-qa/ubuntu-system-tests/spike_manage_test_metadata into lp:ubuntu-system-tests

[Sergio Cazzolato]

Experimental approach to manage test metadata

This approach is similar to the one that TestNG uses defining groups of tests/testsuites.

The idea is to manage all the metadata of the test cases in a json file in a similar way that the Richards implementation.

This piece of code is not the whole implementation, it is just what I would change.

---------------
HOW TO TEST IT:
---------------

import suites
man = suites.TestMetadataManager()

print(man.get_tests_and_dependencies(group='Web'))

print(man.get_tests_and_dependencies()

print(man.get_tests_and_dependencies(suite='dialer_app', group='Telephony'))

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:129
No commit message was specified in the merge proposal. Click on the following link and set the commit message (if you want a jenkins rebuild you need to trigger it yourself):
https://code.launchpad.net/~canonical-platform-qa/ubuntu-system-tests/spike_manage_test_metadata/+merge/261791/+edit-commit-message

http://s-jenkins.ubuntu-ci:8080/job/ubuntu-sanity-tests-ci/335/
Executed test runs:
    SUCCESS: http://s-jenkins.ubuntu-ci:8080/job/ubuntu-sanity-tests-wily-amd64-ci/106
    SUCCESS: http://s-jenkins.ubuntu-ci:8080/job/ubuntu-sanity-tests-wily-armhf-ci/107
    SUCCESS: http://s-jenkins.ubuntu-ci:8080/job/ubuntu-sanity-tests-wily-i386-ci/106

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/ubuntu-sanity-tests-ci/335/rebuild

review: Needs Fixing (continuous-integration)

[Brendan Donegan (brendan-donegan)]

Setting back to WIP, please reset to needs review if you still want it to land

Unmerged revisions

129. By Sergio Cazzolato on 2015-06-11

experimental approach to manage test metadata

=== added directory 'ubuntu_system_tests/external_tests/suites'
=== added file 'ubuntu_system_tests/external_tests/suites/__init__.py'
--- ubuntu_system_tests/external_tests/suites/__init__.py	1970-01-01 00:00:00 +0000
+++ ubuntu_system_tests/external_tests/suites/__init__.py	2015-06-11 22:56:31 +0000
@@ -0,0 +1,116 @@
+# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
+
+#
+# Ubuntu System Tests
+# Copyright (C) 2015 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import json
+import os
+
+
+def get_external_suite_dir_path():
+    """Returns path of the directory containing external test suites"""
+    return os.path.dirname(__file__)
+
+
+def test_metadata_set_decoder(dic):
+    return TestMetadataSet(dic['name'],
+                           dic['depends'],
+                           dic['groups'],
+                           dic['tests'])
+
+
+def test_metadata_decoder(dic):
+    return TestMetadata(dic['name'],
+                        dic['depends'],
+                        dic['groups'])
+
+
+class TestMetadataManager():
+
+    def __init__(self):
+        self.test_sets = []
+        self._load_external_tests()
+
+    def _load_set(self, path):
+        with open(path) as f:
+            test_set_dict = json.load(f)
+        self.test_sets.append(test_metadata_set_decoder(test_set_dict))
+
+    def _load_external_tests(self):
+        external_path = get_external_suite_dir_path()
+        json_files = [f for f in os.listdir(external_path) if
+                      os.path.isfile(os.path.join(external_path, f)) and
+                      f.endswith('.json')]
+
+        for json_file in json_files:
+            self._load_set(os.path.join(external_path, json_file))
+
+    def _get_test_set(self, name):
+        for test_set in self.test_sets:
+            if name == test_set.name:
+                return test_set
+
+    def get_tests_and_dependencies(self, group=None, suite=None):
+        if suite:
+            self._get_test_set(suite).get_tests_and_dependencies(group)
+
+        tests = []
+        dependencies = []
+        for test_set in self.test_sets:
+            curr_tests, curr_deps = test_set.get_tests_and_dependencies(group)
+            tests += curr_tests
+            dependencies += curr_deps
+
+        return tests, list(set(dependencies))
+
+
+class TestMetadataSet():
+
+    def __init__(self, name, depends, groups, tests):
+        self.name = name or ""
+        self.depends = depends or []
+        self.groups = groups
+        self.tests = [test_metadata_decoder(test) for test in tests] or []
+
+    def get_tests_and_dependencies(self, group=None):
+        if not group or group in self.groups:
+            return self.get_all_tests_and_dependencies()
+
+        dependencies = []
+        for test in self.tests:
+            if not group or group in test.groups:
+                dependencies += test.depends
+
+        return [test.name for test in self.tests if not group or group in
+                test.groups], list(set(dependencies))
+
+    def get_all_tests_and_dependencies(self):
+        dependencies = []
+        for test in self.tests:
+            dependencies += test.depends
+
+        return [test.name for test in self.tests], list(set(self.depends +
+                                                            dependencies))
+
+
+class TestMetadata():
+
+    def __init__(self, name, depends, groups):
+        self.name = name or ""
+        self.depends = depends or []
+        self.groups = groups or []

=== added file 'ubuntu_system_tests/external_tests/suites/dialer_tests.json'
--- ubuntu_system_tests/external_tests/suites/dialer_tests.json	1970-01-01 00:00:00 +0000
+++ ubuntu_system_tests/external_tests/suites/dialer_tests.json	2015-06-11 22:56:31 +0000
@@ -0,0 +1,26 @@
+{
+  "name": "dialer_app",
+  "depends": ["dialer-app-autopilot", "dialer-app"],
+  "groups": [
+    "Telephony"
+  ],
+  "tests": [
+    {
+      "name": "dialer_app.tests.test_calls",
+      "depends": [],
+      "groups": [
+
+      ]
+    },
+    {
+      "name": "dialer_app.tests.test_dialer",
+      "depends": [],
+      "groups": [
+
+      ]
+    }
+  ]
+}
+
+
+

=== added file 'ubuntu_system_tests/external_tests/suites/webbrowser_app.json'
--- ubuntu_system_tests/external_tests/suites/webbrowser_app.json	1970-01-01 00:00:00 +0000
+++ ubuntu_system_tests/external_tests/suites/webbrowser_app.json	2015-06-11 22:56:31 +0000
@@ -0,0 +1,14 @@
+{
+  "name": "webbrowser_app",
+  "depends": ["webbrowser-app-autopilot"],
+  "groups": [
+    "Web"
+  ],
+  "tests": [
+    {
+      "name": "webbrowser_app.tests.test_backforward.TestBackForward.test_go_forward_after_going_back",
+      "depends": [],
+      "groups": []
+    }
+  ]
+}
\ No newline at end of file