Merge lp:~cprov/britney/integration-tests into lp:~canonical-ci-engineering/britney/queued-announce-and-collect

[Celso Providelo]

Adding integration tests for britney/testclient.py

Adding integrations tests for testclient features. The down-side here is to depend on local rabbit installation for running the tests (kombu 'memory://' cannot be shared across processed and we are calling `britney` for tests).

[lp:~cprov/britney/integration-tests]

439. By Celso Providelo on 2015-05-25

Skip tests if no local rabbitmq is available.

[Thomi Richards (thomir-deactivatedaccount)]

A few comments, otherwise looks good.

review: Approve

[Celso Providelo (cprov)]

Thomi,

Thanks for the review, comments addressed.

[lp:~cprov/britney/integration-tests]

440. By Celso Providelo on 2015-05-25

Addressing review comments.

=== modified file 'britney.conf'
--- britney.conf	2015-03-05 14:57:03 +0000
+++ britney.conf	2015-05-25 23:49:18 +0000
@@ -70,3 +70,7 @@
 BOOTTEST_DEBUG    = yes
 BOOTTEST_ARCHES   = armhf amd64
 BOOTTEST_FETCH    = yes
+
+TESTCLIENT_ENABLE = yes
+TESTCLIENT_AMQP_URIS = amqp://guest:guest@162.213.32.181:5672//
+TESTCLIENT_REQUIRED_TESTS = 

=== modified file 'britney.py'
--- britney.py	2015-05-22 02:51:53 +0000
+++ britney.py	2015-05-25 23:49:18 +0000
@@ -2019,22 +2019,23 @@
                     self.hints.search('force-badtest', package=excuse.name))
                 forces = [x for x in hints
                           if same_source(excuse.ver[1], x.version)]
-                for test in testclient.getTests(excuse.name):
+                for test in testclient.getTests(excuse.name, excuse.ver[1]):
                     label = TestClient.EXCUSE_LABELS.get(
-                        test.status, 'UNKNOWN STATUS')
+                        test.get('status'), 'UNKNOWN STATUS')
                     excuse.addhtml(
                         "%s result: %s (<a href=\"%s\">results</a>)" % (
-                            test.name, label, test.result_url))
+                            test.get('name').capitalize(), label,
+                            test.get('url')))
                     if forces:
                         excuse.addhtml(
                             "Should wait for %s %s %s, but forced by "
                             "%s" % (excuse.name, excuse.ver[1],
                                     test.name, forces[0].user))
                         continue
-                    if test.name not in required_tests:
+                    if test.get('name') not in required_tests:
                         continue
-                    if test.status not in TestClient.VALID_STATUSES:
-                        excuse.addreason(test.name)
+                    if test.get('status') not in TestClient.VALID_STATUSES:
+                        excuse.addreason(test.get('name'))
                         if excuse.is_valid:
                             excuse.is_valid = False
                             excuse.addhtml("Not considered")

=== modified file 'testclient.py'
--- testclient.py	2015-05-22 13:55:57 +0000
+++ testclient.py	2015-05-25 23:49:18 +0000
@@ -68,9 +68,9 @@
 
     VALID_STATUSES = ('PASS', 'SKIP')
 
-    LABELS = {
+    EXCUSE_LABELS = {
         "PASS": '<span style="background:#87d96c">Pass</span>',
-        "SKIP": '<span style="background:#ffff00">Skip</span>',
+        "SKIP": '<span style="background:#ffff00">Test skipped</span>',
         "FAIL": '<span style="background:#ff6666">Regression</span>',
         "RUNNING": '<span style="background:#99ddff">Test in progress</span>',
     }
@@ -93,7 +93,7 @@
         """Announce new source candidates.
 
         Post a message to the EXCHANGE_CANDATIDATES for every new given
-        excuses (cache announcementes so excuses do not get re-annouced).
+        excuses (cache announcements so excuses do not get re-annouced).
         """            
         with nested(json_cached_info(self.cache_path),
                     kombu.Connection(self.amqp_uris)) as (cache, connection):

=== modified file 'tests/test_testclient.py'
--- tests/test_testclient.py	2015-05-22 17:39:54 +0000
+++ tests/test_testclient.py	2015-05-25 23:49:18 +0000
@@ -15,6 +15,7 @@
 import kombu
 from kombu.pools import producers
 
+
 PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, PROJECT_DIR)
 
@@ -24,13 +25,14 @@
     make_cache_key,
     TestClient,
 )
+from tests import TestBase
 
 
 class TestJsonCachedInfo(unittest.TestCase):
 
     def setUp(self):
         super(TestJsonCachedInfo, self).setUp()
-        (_dummy, self.test_cache) = tempfile.mkstemp()
+        _, self.test_cache = tempfile.mkstemp()
         self.addCleanup(os.unlink, self.test_cache)
 
     def test_simple(self):
@@ -64,24 +66,27 @@
         self.path = tempfile.mkdtemp(prefix='testclient')
         os.makedirs(os.path.join(self.path, 'testclient/'))
         self.addCleanup(shutil.rmtree, self.path)
+
         os.chdir(self.path)
+        cwd = os.getcwd()
+        self.addCleanup(os.chdir, cwd)
+
+        self.amqp_uris = ['memory://']
 
     def test_announce(self):
         # 'announce' post messages to the EXCHANGE_CANDIDATES exchange and
         # updates its internal cache.
-        amqp_uris = ['memory://']
-        testclient = TestClient('vivid', amqp_uris)
+        testclient = TestClient('vivid', self.amqp_uris)
         test_excuses = [
             make_excuse('foo', '1.0'),
             make_excuse('bar', '2.0'),
         ]
 
-        with kombu.Connection(amqp_uris) as connection:
+        with kombu.Connection(self.amqp_uris) as connection:
             exchange = kombu.Exchange(
                 testclient.EXCHANGE_CANDIDATES, type="fanout")
             queue = kombu.Queue('testing', exchange)
             with connection.SimpleQueue(queue) as q:
-                q.queue.purge()
                 testclient.announce(test_excuses)
                 self.assertEqual(
                     [{'series': 'vivid',
@@ -100,8 +105,7 @@
     def test_collect(self):
         # 'collect' collects test results and aggregates them in its
         # internal cache.
-        amqp_uris = ['memory://']
-        testclient = TestClient('vivid', amqp_uris)
+        testclient = TestClient('vivid', self.amqp_uris)
 
         result_payloads = [
             {'source_name': 'foo',
@@ -131,7 +135,7 @@
              'test_url': 'http://ubuntu.com/foo'},
         ]
 
-        with kombu.Connection(amqp_uris) as connection:
+        with kombu.Connection(self.amqp_uris) as connection:
             with producers[connection].acquire(block=True) as producer:
                 # Just for binding destination queue to the exchange.
                 testclient.collect()
@@ -159,8 +163,7 @@
     def test_cleanup(self):
         # `cleanup` remove cache entries that are not present in the
         # given excuses list (i.e. not relevant for promotion anymore).
-        amqp_uris = ['memory://']
-        testclient = TestClient('vivid', amqp_uris)
+        testclient = TestClient('vivid', self.amqp_uris)
         test_excuses = [
             make_excuse('foo', '1.0'),
             make_excuse('bar', '2.0'),
@@ -181,8 +184,7 @@
     def test_getTests(self):
         # `getTests` yields cached tests results for a given source name
         # and version.
-        amqp_uris = ['memory://']
-        testclient = TestClient('vivid', amqp_uris)
+        testclient = TestClient('vivid', self.amqp_uris)
 
         with json_cached_info(testclient.cache_path) as cache:
             cache[make_cache_key('foo', '1.0')] = [
@@ -207,5 +209,222 @@
             [], list(testclient.getTests('bar', '1.0')))
 
 
+def has_local_rabbitmq():
+    """Whether a local rabbitmq server is available with default creds."""
+    with kombu.Connection('amqp://guest:guest@localhost:5672//',
+                          connect_timeout=.1) as c:
+        try:
+            c.connect()
+        except:
+            return False
+        return True
+
+
[e-mail](has_local_rabbitmq(), 'No local rabbitmq')
+class TestTestClientEnd2End(TestBase):
+    """End2End tests (calling `britney`) for the TestClient usage."""
+
+    def setUp(self):
+        super(TestTestClientEnd2End, self).setUp()
+
+        # XXX cprov 20150525: unfortunately, this test requires a proper
+        # amqp transport/server layer (rabbitmq) because kombu 'memory://'
+        # cannot be shared across processes (britney & tests).
+        self.amqp_uris = ['amqp://guest:guest@localhost:5672//']
+
+        self.path = tempfile.mkdtemp(prefix='testclient')
+        os.makedirs(os.path.join(self.path, 'testclient/'))
+        self.addCleanup(shutil.rmtree, self.path)
+
+        os.chdir(self.path)
+        cwd = os.getcwd()
+        self.addCleanup(os.chdir, cwd)
+
+        # Disable autopkgtests + boottest tests and use local rabbit
+        # for this testing context.
+        self.overrideConfig({
+            'ADT_ENABLE': 'no',
+            'BOOTTEST_ENABLE': 'no',
+            'TESTCLIENT_AMQP_URIS': ' '.join(self.amqp_uris),
+        })
+        
+        # We publish a version of 'foo' source to make it 'known'.
+        self.data.add('foo', False, {'Architecture': 'amd64'})
+
+    def overrideConfig(self, overrides):
+        """Overrides briney configuration based on the given key-value map."""
+        with open(self.britney_conf, 'r') as fp:
+            original_config = fp.read()
+        new_config = []
+        for line in original_config.splitlines():
+            for k, v in overrides.iteritems():
+                if line.startswith(k):
+                    line = '{} = {}'.format(k, v)
+            new_config.append(line)
+        with open(self.britney_conf, 'w') as fp:
+            fp.write('\n'.join(new_config))
+        self.addCleanup(self.restore_config, original_config)
+
+    def publishTestResults(self, results):
+        """Publish the given list of test results."""
+        with kombu.Connection(self.amqp_uris) as connection:
+            results_exchange = kombu.Exchange(
+                TestClient.EXCHANGE_RESULTS, type="fanout")
+            with producers[connection].acquire(block=True) as producer:
+                publisher = connection.ensure(
+                    producer, producer.publish, max_retries=3)
+                for payload in results:
+                    publisher(payload, exchange=results_exchange)
+
+    def getAnnouncements(self):
+        """Yields announcements payloads.""" 
+        with kombu.Connection(self.amqp_uris) as connection:
+            candidates_exchange = kombu.Exchange(
+                TestClient.EXCHANGE_CANDIDATES, type="fanout")
+            queue = kombu.Queue('testing', candidates_exchange)
+            with connection.SimpleQueue(queue) as q:
+                for i in range(len(q)):
+                    msg = q.get()
+                    msg.ack()
+                    yield msg.payload
+
+    def do_test(self, context, expect=None, no_expect=None):
+        """Process the given package context and assert britney results."""
+        for (pkg, fields) in context:
+            self.data.add(pkg, True, fields)
+
+        # Creates a queue for collecting announcements from
+        # 'candidates.exchanges'.
+        with kombu.Connection(self.amqp_uris) as connection:
+            candidates_exchange = kombu.Exchange(
+                TestClient.EXCHANGE_CANDIDATES, type="fanout")
+            queue = kombu.Queue('testing', candidates_exchange)
+            with connection.SimpleQueue(queue) as q:
+                q.queue.purge()
+
+        (excuses, out) = self.run_britney()
+
+        #print('-------\nexcuses: %s\n-----' % excuses)
+        if expect:
+            for re in expect:
+                self.assertRegexpMatches(excuses, re)
+        if no_expect:
+            for re in no_expect:
+                self.assertNotRegexpMatches(excuses, re)
+
+    def test_non_required_test(self):
+        # Non-required test results are collected as part of the excuse
+        # report but do not block source promotion (i.e. the excuse is
+        # a 'Valid candidate' even if the test is 'in progress').
+
+        # Publish 'in-progress' results for 'bazinga for "foo_1.1"'.
+        test_results = [{
+            'source_name': 'foo',
+            'source_version': '1.1',
+            'series': self.data.series,
+            'test_name': 'bazinga',
+            'test_status': 'RUNNING',
+            'test_url': 'http://bazinga.com/foo',
+        }]
+        self.publishTestResults(test_results)
+
+        # Run britney for 'foo_1.1' and valid candidated is recorded.
+        context = [
+            ('foo', {'Source': 'foo', 'Version': '1.1',
+                     'Architecture': 'amd64'}),
+        ]
+        self.do_test(
+            context,
+            [r'\bfoo\b.*>1</a> to .*>1.1<',
+             r'<li>Bazinga result: .*>Test in progress.*'
+             r'href="http://bazinga.com/foo">results',
+             '<li>Valid candidate'])
+
+        # 'foo_1.1' source candidate was announced.
+        self.assertEqual(
+            [{'source_name': 'foo',
+              'source_version': '1.1',
+              'series': self.data.series,
+            }], list(self.getAnnouncements()))
+
+    def test_required_test(self):
+        # A required-test result is collected and blocks source package
+        # promotion while it hasn't passed.
+
+        # Make 'bazinga' a required test.
+        self.overrideConfig({
+            'TESTCLIENT_REQUIRED_TESTS': 'bazinga',
+        })
+
+        # Publish 'in-progress' results for 'bazinga for "foo_1.1"'.
+        test_results = [{
+            'source_name': 'foo',
+            'source_version': '1.1',
+            'series': self.data.series,
+            'test_name': 'bazinga',
+            'test_status': 'RUNNING',
+            'test_url': 'http://bazinga.com/foo',
+        }]
+        self.publishTestResults(test_results)
+
+        # Run britney for 'foo_1.1' and an unconsidered excuse is recorded.
+        context = [
+            ('foo', {'Source': 'foo', 'Version': '1.1',
+                     'Architecture': 'amd64'}),
+        ]
+        self.do_test(
+            context,
+            [r'\bfoo\b.*>1</a> to .*>1.1<',
+             r'<li>Bazinga result: .*>Test in progress.*'
+             r'href="http://bazinga.com/foo">results',
+             '<li>Not considered'])
+
+        # 'foo_1.1' source candidate was announced.
+        self.assertEqual(
+            [{'source_name': 'foo',
+              'source_version': '1.1',
+              'series': self.data.series,
+            }], list(self.getAnnouncements()))
+
+    def test_promoted(self):
+        # When all required tests passed (or were skipped) the source
+        # candidate can be promoted.
+
+        # Make 'bazinga' and 'zoing' required test.
+        self.overrideConfig({
+            'TESTCLIENT_REQUIRED_TESTS': 'bazinga zoing',
+        })
+
+        # Publish 'in-progress' results for 'bazinga for "foo_1.1"'.
+        test_results = [{
+            'source_name': 'foo',
+            'source_version': '1.1',
+            'series': self.data.series,
+            'test_name': 'bazinga',
+            'test_status': 'SKIP',
+            'test_url': 'http://bazinga.com/foo',
+        }, {
+            'source_name': 'foo',
+            'source_version': '1.1',
+            'series': self.data.series,
+            'test_name': 'zoing',
+            'test_status': 'PASS',
+            'test_url': 'http://zoing.com/foo',
+        }]
+        self.publishTestResults(test_results)
+
+        context = [
+            ('foo', {'Source': 'foo', 'Version': '1.1',
+                     'Architecture': 'amd64'}),
+        ]
+        self.do_test(
+            context,
+            [r'\bfoo\b.*>1</a> to .*>1.1<',
+             r'<li>Bazinga result: .*>Test skipped.*'
+             'href="http://bazinga.com/foo">results',
+             r'<li>Zoing result: .*>Pass.*href="http://zoing.com/foo">results',
+             '<li>Valid candidate'])
+
+
 if __name__ == '__main__':
     unittest.main()