Merge lp:~doanac/ubuntu-test-cases/config-rework into lp:ubuntu-test-cases/touch

[Andy Doan]

dealing with testsuite configuration information in setup_jenkins.py is becoming cumbersome. Additionally, as MEGA jobs currently stand, you need to regenerate jobs to add/remove testsuites. This moves the configuration information around a little bit so that MEGA jobs seldom need reconfiguring.

[lp:~doanac/ubuntu-test-cases/config-rework]

94. By Andy Doan on 2013-10-24

move the mega job logic into a proper script

The jenkins job shell command was growing too complex. This is
being moved to its own shell script ./scripts/run-smoke.

[Paul Larson (pwlars)]

Cool, that's useful

review: Approve

[lp:~doanac/ubuntu-test-cases/config-rework]

95. By Andy Doan on 2013-10-24

break run-smoke into functions

=== removed file 'jenkins/apconfig.py'
--- jenkins/apconfig.py	2013-10-21 18:16:50 +0000
+++ jenkins/apconfig.py	1970-01-01 00:00:00 +0000
@@ -1,104 +0,0 @@
-#!/usr/bin/env python
-
-# Ubuntu Testing Automation Harness
-# Copyright 2013 Canonical Ltd.
-
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 3, as published
-# by the Free Software Foundation.
-
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranties of
-# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
-# PURPOSE.  See the GNU General Public License for more details.
-
-# You should have received a copy of the GNU General Public License along
-# with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-import argparse
-import collections
-
-TestSuite = collections.namedtuple('TestSuite', ['name', 'app', 'pkgs'])
-
-
-def _ap_test(name, app=None, pkgs=None):
-    if not app:
-        # convert share-app-autopilot to share_app
-        app = name.replace('-', '_').replace('_autopilot', '')
-    return TestSuite(name, app, pkgs)
-
-
-TESTSUITES = [
-    _ap_test('friends-app-autopilot', pkgs=['friends-app-autopilot']),
-    _ap_test('mediaplayer-app-autopilot', pkgs=['mediaplayer-app-autopilot']),
-    _ap_test('gallery-app-autopilot', pkgs=['gallery-app-autopilot']),
-    _ap_test('webbrowser-app-autopilot', pkgs=['webbrowser-app-autopilot']),
-    _ap_test('unity8-autopilot', 'unity8', pkgs=['python-gi']),
-    _ap_test('notes-app-autopilot'),
-    _ap_test('camera-app-autopilot', pkgs=['camera-app-autopilot']),
-    _ap_test('dialer-app-autopilot', pkgs=['dialer-app-autopilot']),
-    _ap_test('messaging-app-autopilot', pkgs=['messaging-app-autopilot']),
-    _ap_test('address-book-app-autopilot',
-             pkgs=['address-book-app-autopilot']),
-    _ap_test('calendar-app-autopilot'),
-    _ap_test('music-app-autopilot'),
-    _ap_test('dropping-letters-app-autopilot'),
-    _ap_test('ubuntu-calculator-app-autopilot'),
-    _ap_test('ubuntu-clock-app-autopilot'),
-    _ap_test('ubuntu-filemanager-app-autopilot'),
-    _ap_test('ubuntu-rssreader-app-autopilot'),
-    _ap_test('ubuntu-terminal-app-autopilot'),
-    _ap_test('ubuntu-weather-app-autopilot'),
-    _ap_test('ubuntu-ui-toolkit-autopilot', 'ubuntuuitoolkit',
-             ['ubuntu-ui-toolkit-autopilot']),
-    _ap_test('ubuntu-system-settings-online-accounts-autopilot',
-             'online_accounts_ui',
-             ['ubuntu-system-settings-online-accounts-autopilot']),
-]
-
-
-def _handle_packages(args):
-    pkgs = []
-    for suite in TESTSUITES:
-        if not args.app or suite.app in args.app:
-            if suite.pkgs:
-                pkgs.extend(suite.pkgs)
-    print(' '.join(pkgs))
-    return 0
-
-
-def _handle_apps(args):
-    apps = [t.app for t in TESTSUITES]
-    print(' '.join(apps))
-
-
-def _handle_tests(args):
-    tests = [t.name for t in TESTSUITES]
-    print(' '.join(tests))
-
-
-def _get_parser():
-    parser = argparse.ArgumentParser(
-        description='List information on configured autopilot tests for touch')
-    sub = parser.add_subparsers(title='Commands', metavar='')
-
-    p = sub.add_parser('packages', help='List packages required for apps')
-    p.set_defaults(func=_handle_packages)
-    p.add_argument('-a', '--app', action='append',
-                   help='Autopilot test application. eg share_app')
-
-    p = sub.add_parser('apps', help='List tests by autopilot application name')
-    p.set_defaults(func=_handle_apps)
-
-    p = sub.add_parser('tests', help='List tests by CI test name')
-    p.set_defaults(func=_handle_tests)
-
-    return parser
-
-
-def main():
-    args = _get_parser().parse_args()
-    return args.func(args)
-
-if __name__ == '__main__':
-    exit(main())

=== modified file 'jenkins/production.py'
--- jenkins/production.py	2013-10-22 14:41:50 +0000
+++ jenkins/production.py	2013-10-24 18:45:43 +0000
@@ -3,32 +3,6 @@
 JENKINS = 'http://10.97.0.1:8080'
 
 
-def _custom_test_filter(common_tests, mktest_func):
-    tests = []
-    test_set = [
-        'install-and-boot',
-        'default',
-        'unity8-autopilot',
-        'webbrowser-app-autopilot',
-    ]
-
-    tests = [t for t in common_tests if t.name in test_set]
-    tests.insert(1, mktest_func('customizations'))
-    return tests
-
-
-def _sf4p_test_filter(common_tests, mktest_func):
-    tests = []
-    test_set = [
-        'install-and-boot',
-        'default',
-        'unity8-autopilot',
-    ]
-
-    tests = [t for t in common_tests if t.name in test_set]
-    return tests
-
-
 def _url(channel, device):
     return 'http://system-image.ubuntu.com/%s/%s/index.json' \
            % (channel, device)
@@ -48,7 +22,6 @@
                 'trigger_url': _url('trusty-proposed', 'maguro'),
             },
         ],
-        'filter': _sf4p_test_filter,
     },
     {
         'image-type': 'touch',
@@ -73,7 +46,6 @@
                 'trigger_url': _url('trusty-proposed-customized', 'mako'),
             },
         ],
-        'filter': _custom_test_filter,
         'IMAGE_OPT': 'export IMAGE_OPT="ubuntu-system '
                      '--channel trusty-proposed-customized"'
     },

=== modified file 'jenkins/setup_jenkins.py'
--- jenkins/setup_jenkins.py	2013-10-22 15:59:20 +0000
+++ jenkins/setup_jenkins.py	2013-10-24 18:45:43 +0000
@@ -16,7 +16,6 @@
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import argparse
-import collections
 import imp
 import jenkins
 import jinja2
@@ -26,41 +25,10 @@
 from distro_info import UbuntuDistroInfo
 DEV_SERIES = UbuntuDistroInfo().devel()
 
-import apconfig
+import testconfig
 
 DEFINE_MEGA = os.environ.get('MEGA', False)
 
-DEF_FMT = '{prefix}{series}-{imagetype}-{type}-smoke-{testname}'
-IDLE_FMT = '{prefix}{testname}-{series}-{imagetype}-armhf-install-idle-{type}'
-
-Test = collections.namedtuple('Test', ['name', 'fmt', 'ap'])
-
-
-def _test(name, fmt=DEF_FMT):
-    return Test(name, fmt, False)
-
-
-def _ap_test(name):
-    return Test(name, DEF_FMT, True)
-
-
-TESTS = [
-    _test('install-and-boot'),
-    _test('default'),
-]
-
-TESTS += [_ap_test(t.name) for t in apconfig.TESTSUITES]
-
-TESTS += [
-    _test('click_image_tests'),
-    _test('sdk'),
-    _test('security'),
-    _test('eventstat', IDLE_FMT),
-    _test('smem', IDLE_FMT),
-    _test('memevent',
-          '{prefix}{testname}-{series}-{imagetype}-armhf-default-{type}'),
-]
-
 
 def _get_parser():
     """Create and return command line parser.
@@ -150,13 +118,12 @@
             'serial': device.get('serial', defserial),
             'publish': args.publish,
             'branch': args.branch,
-            'tests': ' '.join([t.name for t in tests if not t.ap]),
             'trigger_url': device['trigger_url'],
             'imagetype': config_item['image-type'],
             'image_opt': config_item.get('IMAGE_OPT', ''),
         }
         # a hack so we can use _get_job_name
-        test = _test('', fmt='{prefix}{series}-{imagetype}-{type}')
+        test = testconfig.Test('', fmt='{prefix}{series}-{imagetype}-{type}')
         job = _get_job_name(args, name, test, config_item['image-type'])
         _publish(instance, env, args, 'touch-smoke.xml.jinja2', job, **params)
 else:
@@ -187,7 +154,8 @@
             'trigger_url': device['trigger_url'],
         }
         image_type = config_item['image-type']
-        job = _get_job_name(args, device['name'], _test('master'), image_type)
+        test = testconfig.Test('master')
+        job = _get_job_name(args, device['name'], test, image_type)
         _publish(instance, env, args, 'touch-master.xml.jinja2', job, **params)
 
         job = 'smoke-master-free'
@@ -227,9 +195,8 @@
 
     for item in config.MATRIX[args.series]:
         for device in item['devices']:
-            tests = TESTS
-            if 'filter' in item:
-                tests = item['filter'](tests, _test)
+            tests = testconfig.TESTSUITES
+            tests = testconfig.filter_tests(tests, item['image-type'])
             _configure_jobs(jenkins_inst, env, args, item, device, tests)
 
 if __name__ == '__main__':

=== modified file 'jenkins/templates/touch-smoke.xml.jinja2'
--- jenkins/templates/touch-smoke.xml.jinja2	2013-10-22 15:59:20 +0000
+++ jenkins/templates/touch-smoke.xml.jinja2	2013-10-24 18:45:43 +0000
@@ -87,7 +87,7 @@
 
 if [ -z $INSTALL_URL ] ; then
   if [ &quot;$PACKAGES&quot; = &quot;ALL&quot; ] ; then
-    PACKAGES=&quot;$(${BZRDIR}/jenkins/apconfig.py packages)&quot;
+    PACKAGES=&quot;$(${BZRDIR}/jenkins/testconfig.py packages -i {{imagetype}})&quot;
   fi
 
   custom_args=&quot;&quot;
@@ -105,12 +105,12 @@
 fi
 
 if [ &quot;$APPS&quot; = &quot;ALL&quot; ] ; then
-  APPS=&quot;$(${BZRDIR}/jenkins/apconfig.py apps)&quot;
+  APPS=&quot;$(${BZRDIR}/jenkins/testconfig.py apps -i {{imagetype}})&quot;
 fi
 [ -z "$APPS" ] || APPS=${APPS} ${BZRDIR}/scripts/run-autopilot-tests.sh
 
 if [ &quot;$TESTS&quot; = &quot;ALL&quot; ] ; then
-  TESTS="{{tests}}"
+  TESTS=&quot;$(${BZRDIR}/jenkins/testconfig.py tests -i {{imagetype}})&quot;
 fi
 
 rc=0

=== added file 'jenkins/testconfig.py'
--- jenkins/testconfig.py	1970-01-01 00:00:00 +0000
+++ jenkins/testconfig.py	2013-10-24 18:45:43 +0000
@@ -0,0 +1,175 @@
+#!/usr/bin/env python
+
+# Ubuntu Testing Automation Harness
+# Copyright 2013 Canonical Ltd.
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+
+DEF_FMT = '{prefix}{series}-{imagetype}-{type}-smoke-{testname}'
+IDLE_FMT = '{prefix}{testname}-{series}-{imagetype}-armhf-install-idle-{type}'
+
+
+class Test(object):
+    def __init__(self, name, fmt=DEF_FMT):
+        self.name = name
+        self.fmt = fmt
+
+
+class APTest(Test):
+    def __init__(self, name, app=None, pkgs=None):
+        Test.__init__(self, name)
+        self.pkgs = pkgs
+        if not app:
+            # convert share-app-autopilot to share_app
+            app = name.replace('-', '_').replace('_autopilot', '')
+        self.app = app
+
+
+TESTSUITES = [
+    Test('install-and-boot'),
+    Test('default'),
+]
+TESTSUITES += [
+    APTest('friends-app-autopilot', pkgs=['friends-app-autopilot']),
+    APTest('mediaplayer-app-autopilot', pkgs=['mediaplayer-app-autopilot']),
+    APTest('gallery-app-autopilot', pkgs=['gallery-app-autopilot']),
+    APTest('webbrowser-app-autopilot', pkgs=['webbrowser-app-autopilot']),
+    APTest('unity8-autopilot', 'unity8', pkgs=['python-gi']),
+    APTest('notes-app-autopilot'),
+    APTest('camera-app-autopilot', pkgs=['camera-app-autopilot']),
+    APTest('dialer-app-autopilot', pkgs=['dialer-app-autopilot']),
+    APTest('messaging-app-autopilot', pkgs=['messaging-app-autopilot']),
+    APTest('address-book-app-autopilot', pkgs=['address-book-app-autopilot']),
+    APTest('calendar-app-autopilot'),
+    APTest('music-app-autopilot'),
+    APTest('dropping-letters-app-autopilot'),
+    APTest('ubuntu-calculator-app-autopilot'),
+    APTest('ubuntu-clock-app-autopilot'),
+    APTest('ubuntu-filemanager-app-autopilot'),
+    APTest('ubuntu-rssreader-app-autopilot'),
+    APTest('ubuntu-terminal-app-autopilot'),
+    APTest('ubuntu-weather-app-autopilot'),
+    APTest('ubuntu-ui-toolkit-autopilot', 'ubuntuuitoolkit',
+           ['ubuntu-ui-toolkit-autopilot']),
+    APTest('ubuntu-system-settings-online-accounts-autopilot',
+           'online_accounts_ui',
+           ['ubuntu-system-settings-online-accounts-autopilot']),
+]
+TESTSUITES += [
+    Test('click_image_tests'),
+    Test('sdk'),
+    Test('security'),
+    Test('eventstat', IDLE_FMT),
+    Test('smem', IDLE_FMT),
+    Test('memevent',
+         '{prefix}{testname}-{series}-{imagetype}-armhf-default-{type}'),
+]
+
+
+def filter_tests(tests, image_type):
+    if image_type:
+        func = globals().get('get_tests_%s' % image_type)
+        if func:
+            tests = func(tests)
+        elif image_type not in ['touch', 'touch_mir']:
+            print('Unsupported image type: %s' % image_type)
+            exit(1)
+    return tests
+
+
+def _get_tests(test_type, image_type):
+    tests = [t for t in TESTSUITES if type(t) == test_type]
+    return filter_tests(tests, image_type)
+
+
+def _handle_utah(args):
+    tests = _get_tests(Test, args.image_type)
+    # NOTE: this is only called by MEGA jobs, so we can skip install-and-boot
+    print(' '.join([t.name for t in tests if t.name != 'install-and-boot']))
+
+
+def _handle_ap_apps(args):
+    tests = _get_tests(APTest, args.image_type)
+    print(' '.join([t.app for t in tests]))
+
+
+def _handle_ap_packages(args):
+    pkgs = []
+    tests = _get_tests(APTest, args.image_type)
+    for test in tests:
+        if not args.app or test.app in args.app:
+            if test.pkgs:
+                pkgs.extend(test.pkgs)
+    print(' '.join(pkgs))
+
+
+def get_tests_touch_sf4p(common_tests):
+    tests = []
+    test_set = [
+        'install-and-boot',
+        'default',
+        'unity8-autopilot',
+    ]
+
+    tests = [t for t in common_tests if t.name in test_set]
+    return tests
+
+
+def get_tests_touch_custom(common_tests):
+    tests = []
+    test_set = [
+        'install-and-boot',
+        'default',
+        'unity8-autopilot',
+        'webbrowser-app-autopilot',
+    ]
+
+    tests = [t for t in common_tests if t.name in test_set]
+    tests.insert(1, Test('customizations'))
+    return tests
+
+
+def _get_parser():
+    parser = argparse.ArgumentParser(
+        description='List information on configured tests for touch')
+    sub = parser.add_subparsers(title='Commands', metavar='')
+
+    p = sub.add_parser('utah', help='List UTAH tests')
+    p.set_defaults(func=_handle_utah)
+    p.add_argument('-i', '--image-type',
+                   help='Return list of test configured for an image type.')
+
+    p = sub.add_parser('apps', help='List autopilot application names')
+    p.set_defaults(func=_handle_ap_apps)
+    p.add_argument('-i', '--image-type',
+                   help='Return list of test configured for an image type.')
+
+    p = sub.add_parser('packages', help='List packages required for autopilot')
+    p.set_defaults(func=_handle_ap_packages)
+    g = p.add_mutually_exclusive_group()
+    g.add_argument('-i', '--image-type',
+                   help='''If no apps are listed, limit to tests for an
+                        image type.''')
+    g.add_argument('-a', '--app', action='append',
+                   help='Autopilot test application. eg share_app')
+    return parser
+
+
+def main():
+    args = _get_parser().parse_args()
+    return args.func(args)
+
+if __name__ == '__main__':
+    exit(main())