Merge lp:~sbaldassin/qakit/ust_worker into lp:qakit

[Santiago Baldassin]

Ubuntu system test worker

Ubuntu system test worker

[Sergio Cazzolato (sergio-j-cazzolato)]

some minor comments inline

review: Needs Fixing

[Allan LeSage (allanlesage)]

Admit that this is all a bit abstract for me :) but can I ask that you move your entry-level ("main") script into your dir? Or if you insist on leaving it in the root dir would you indicate to which sub-project it belongs in its name, e.g.?

review: Needs Fixing

Unmerged revisions

152. By Santiago Baldassin on 2016-08-19

Ubuntu system test worker

=== added directory 'qakit/britney'
=== added file 'qakit/britney/__init__.py'
--- qakit/britney/__init__.py	1970-01-01 00:00:00 +0000
+++ qakit/britney/__init__.py	2016-08-22 14:09:13 +0000
@@ -0,0 +1,19 @@
+# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
+
+#
+# Ubuntu System Tests
+# Copyright (C) 2016 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#

=== added file 'qakit/britney/config.py'
--- qakit/britney/config.py	1970-01-01 00:00:00 +0000
+++ qakit/britney/config.py	2016-08-22 14:09:13 +0000
@@ -0,0 +1,67 @@
+# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
+
+#
+# Ubuntu System Tests
+# Copyright (C) 2016 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from ubuntu_system_tests.common.config import UbuntuSystemTestsConfig
+
+
+class WorkerConfig:
+
+    def __init__(self):
+        self._default = None
+        self._practitest = None
+        self._amqp = None
+        self._britney = None
+        self._swift = None
+
+    @property
+    def default(self):
+        if self._default is None:
+            self._default = self.__get_config('default')
+        return self._default
+
+    @property
+    def practitest(self):
+        if self._practitest is None:
+            self._practitest = self.__get_config('practitest')
+        return self._practitest
+
+    @property
+    def amqp(self):
+        if self._amqp is None:
+            self._amqp = self.__get_config('amqp')
+        return self._amqp
+
+    @property
+    def britney(self):
+        if self._britney is None:
+            self._britney = self.__get_config('britney')
+        return self._britney
+
+    @property
+    def swift(self):
+        if self._swift is None:
+            self._swift = self.__get_config('swift')
+        return self._swift
+
+    @staticmethod
+    def __get_config(section):
+        cfg = UbuntuSystemTestsConfig(section=section)
+        cfg.get_config_from_file()
+        return cfg

=== added file 'qakit/britney/domains.json'
--- qakit/britney/domains.json	1970-01-01 00:00:00 +0000
+++ qakit/britney/domains.json	2016-08-22 14:09:13 +0000
@@ -0,0 +1,8 @@
+{
+    "autopilot": ["63805"],
+    "ubuntu-ui-toolkit": ["63805"],
+    "account-plugins": ["63805"],
+    "signon-plugin-oauth2": ["63805"],
+    "messaging-app": ["63805"],
+    "goget-ubuntu-touch": ["63805"]
+}

=== added file 'qakit/britney/report.py'
--- qakit/britney/report.py	1970-01-01 00:00:00 +0000
+++ qakit/britney/report.py	2016-08-22 14:09:13 +0000
@@ -0,0 +1,120 @@
+# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
+
+#
+# Ubuntu System Tests
+# Copyright (C) 2016 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import gzip
+import os
+from abc import abstractmethod
+
+import subprocess
+import swiftclient
+
+
+class Reporter:
+
+    def __init__(self, output_dir):
+        self.output_dir = output_dir
+
+    @abstractmethod
+    def report(self):
+        pass
+
+
+class SwiftReporter(Reporter):
+
+    def __init__(self, swift_config, britney_config, output_dir):
+        self.container = britney_config.get('container')
+        self.expected_results_files = britney_config.get(
+            'expected_results_files').split(',')
+        self.result_file = britney_config.get('result_file')
+        self.log_file = britney_config.get('log_file')
+        self.artifacts_file = britney_config.get('artifacts_file')
+
+        self.region = swift_config.get('region_name')
+        self.auth_url = swift_config.get('auth_url')
+        self.username = swift_config.get('username')
+        self.tenant = swift_config.get('tenant')
+        self.password = swift_config.get('password')
+
+        super().__init__(output_dir)
+
+    def report(self):
+        '''
+        Reports to swift should include three files:
+        1. artifacts.tar.gz
+        2. log.gz
+        3. result.tar
+
+        This method will connect to the swift backend and will
+        put the files above in the container defined within the
+        configuration file
+
+        '''
+
+        # create it if it does not exist yet
+        swift_con = swiftclient.Connection(authurl=self.auth_url,
+                                           user=self.username,
+                                           key=self.password)
+        # self.process_output_dir()
+
+        try:
+            swift_con.get_container(self.container, limit=1)
+        except swiftclient.exceptions.ClientException:
+            swift_con.put_container(self.container, headers={
+                'X-Container-Read': '.rlistings,.r:*'})
+
+        result_file = os.path.join(self.output_dir, self.result_file)
+        with open(result_file) as contents:
+            swift_con.put_object(self.container,
+                                 self.result_file,
+                                 contents=contents.read())
+
+        log_file = os.path.join(self.output_dir, self.log_file)
+        artifacts_file = os.path.join(self.output_dir, self.artifacts_file)
+        with gzip.open(log_file) as contents:
+            swift_con.put_object(self.container,
+                                 self.log_file,
+                                 contents=contents.read(),
+                                 content_type='text/plain; charset=UTF-8',
+                                 headers={'Content-Encoding': 'utf-8'})
+        with gzip.open(artifacts_file) as contents:
+            swift_con.put_object(self.container,
+                                 self.artifacts_file,
+                                 contents=contents)
+        swift_con.close()
+
+    def process_output_dir(self):
+        '''Post-process output directory'''
+        results_files = list(set(os.listdir(self.output_dir)))
+        for file in self.expected_results_files:
+            if file not in results_files:
+                raise RuntimeError("File {} not found within the "
+                                   "result files".format(file))
+            else:
+                results_files.remove(file)
+
+        subprocess.check_call(['tar', 'cf', 'result.tar'] +
+                              self.expected_results_files,
+                              cwd=self.output_dir)
+
+        subprocess.check_call(['gzip', '-9',
+                               os.path.join(self.output_dir, 'log')])
+        results_files.remove('log')
+
+        subprocess.check_call(['tar', '-czf', 'artifacts.tar.gz'] +
+                              results_files, cwd=self.output_dir)

=== added file 'qakit/britney/utils.py'
--- qakit/britney/utils.py	1970-01-01 00:00:00 +0000
+++ qakit/britney/utils.py	2016-08-22 14:09:13 +0000
@@ -0,0 +1,47 @@
+# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
+
+#
+# Ubuntu System Tests
+# Copyright (C) 2016 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import os
+import json
+
+from launchpadlib.launchpad import Launchpad
+
+
+def get_filters_for_src_pkgs(src_pkgs):
+    britney_dir = os.path.dirname(__file__)
+    domains_file = os.path.join(str(britney_dir), 'domains.json')
+    with open(domains_file) as domains:
+        domains_json = json.loads(domains.read())
+    filters = [domains_json[package] for package in src_pkgs]
+
+    # Flat the lists and remove duplicates
+    filters_list = list(set([item for sublist in filters for item in sublist]))
+    return ','.join(filter for filter in filters_list)
+
+
+def get_source_packages(message_body):
+    msg_json = json.loads(message_body.decode('utf-8'))
+    team_name, ppa_name = msg_json['ppa'].split('/')
+    launchpad = Launchpad.login_anonymously('ci job', 'production')
+    team = launchpad.people.findTeam(text=team_name)[0]
+    archive = team.getPPAByName(name=ppa_name)
+    src_packages = archive.getPublishedSources(status='Published')
+    packages = list(
+        set([package.source_package_name for package in src_packages]))
+    return packages

=== added file 'qakit/britney/worker.py'
--- qakit/britney/worker.py	1970-01-01 00:00:00 +0000
+++ qakit/britney/worker.py	2016-08-22 14:09:13 +0000
@@ -0,0 +1,94 @@
+# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
+
+#
+# Ubuntu System Tests
+# Copyright (C) 2016 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import logging
+
+from pika import BlockingConnection, ConnectionParameters
+from qakit.britney.config import WorkerConfig
+from qakit.britney.report import SwiftReporter
+
+from qakit.britney.utils import (
+    get_filters_for_src_pkgs, get_source_packages)
+from ubuntu_system_tests.host.command_line import (
+    parse_arguments, run_system_tests, setup_system_tests)
+from ubuntu_system_tests.host.practitest import PractitestSession
+
+
+class UstWorker:
+
+    def __init__(self, config):
+        self.config = config
+
+    def start(self):
+        logging.info('Connecting to AMQP server: {}'.format(
+            self.config.amqp.get('host')))
+        connection = BlockingConnection(ConnectionParameters(
+            host=self.config.amqp.get('host')))
+        channel = connection.channel()
+
+        # TODO: Support multiple (queue, request) set in the configuration file
+        queue = self.config.amqp.get('queue')
+        logging.info('Subscribing to queue: {}'.format(
+            self.config.amqp.get('queue')))
+        channel.queue_declare(queue, durable=True, auto_delete=False)
+        channel.basic_consume(
+            self.process_ust_request, queue=queue, no_ack=True)
+        channel.start_consuming()
+
+    def process_ust_request(self, ch, method, properties, body):
+        '''Callback for ubuntu system test request'''
+        # Parse the message body to get the source packages. Get the
+        # practitests filters ids associated to the source packages
+        # and get the tests based on the filters
+        source_packages = get_source_packages(body)
+        filters = get_filters_for_src_pkgs(source_packages)
+
+        # pt = PractitestSession(self.config.practitest.get(
+        #     '_practitest_api_token'),
+        #                        self.config.practitest.get(
+        #                            '_practitest_project_id'))
+        # tests = ','.join(test for test in pt.get_tests(filters))
+
+        tests = 'ubuntu_system_tests.tests.test_calculator'
+        # Setup the device with ubuntu-system-tests and run the tests
+        setup_args = parse_arguments(['setup', tests])
+        setup_system_tests(setup_args)
+
+        run_args = parse_arguments(['run', tests])
+        run_system_tests(run_args)
+
+        # publish results into swift
+        reporter = SwiftReporter(self.config.swift,
+                                 self.config.britney,
+                                 self.config.default.get('output_dir'))
+        reporter.report()
+
+        logging.info('Acknowledging request %s' % body)
+        logging.info(body)
+
+
+def main():
+    worker_config = WorkerConfig()
+    worker = UstWorker(worker_config)
+    worker.start()
+
+
+if __name__ == '__main__':
+    main()

=== added file 'start_worker.py'
--- start_worker.py	1970-01-01 00:00:00 +0000
+++ start_worker.py	2016-08-22 14:09:13 +0000
@@ -0,0 +1,39 @@
+# -*- Mode: Python; coding: utf-8; indent-tabs-mode: nil; tab-width: 4 -*-
+
+#
+# Ubuntu System Tests
+# Copyright (C) 2016 Canonical
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import sys
+import logging
+from qakit.britney import worker
+
+
+root = logging.getLogger()
+root.setLevel(logging.DEBUG)
+
+ch = logging.StreamHandler(sys.stdout)
+ch.setLevel(logging.INFO)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ch.setFormatter(formatter)
+root.addHandler(ch)
+
+def main():
+    return worker.main()
+
+if __name__ == '__main__':
+    sys.exit(main())