Merge ~sylvain-pineau/hwcert-jenkins-tools:tf-checklist-links-to-summary-v2 into hwcert-jenkins-tools:master

[Sylvain Pineau]

Refactor of the two new trello board updater scripts:

trello-board-updater-desktop.py
trello-board-updater-image-testing.py

Both now share common code with the original trello-board-updater.py and finally get the comment links in checklists.

Tested here:

https://trello.com/b/pmU4fQgz/kernel-deb-update-verification-sylvains-sandbox
https://trello.com/b/ZibLQPmC/certification-image-testing-sylvains-sandbox

[Paul Larson (pwlars)]

great!

review: Approve

diff --git a/trello-board-updater-desktop.py b/trello-board-updater-desktop.py
index 48fb316..d04a990 100755
--- a/trello-board-updater-desktop.py
+++ b/trello-board-updater-desktop.py
@@ -6,6 +6,7 @@
 #
 # Written by:
 #        Taihsiang Ho <[e-mail]>
+#        Sylvain Pineau <[e-mail]>
 #
 #
 # The script uses py-trello package 0.9.0. You may want to fetch it from
@@ -15,6 +16,7 @@
 # SUTs
 #
 import argparse
+import importlib
 import os
 import re
 import requests
@@ -34,89 +36,12 @@ from trello.exceptions import ResourceUnavailable
 
 from unittest.mock import MagicMock
 
+tbu = importlib.import_module("trello-board-updater")
+
 format_str = "[ %(funcName)s() ] %(message)s"
 logging.basicConfig(level=logging.INFO, format=format_str)
 
 
-def environ_or_required(key):
-    if os.environ.get(key):
-        return {'default': os.environ.get(key)}
-    else:
-        return {'required': True}
-
-
-def search_card(board, query, card_filter="open"):
-    for card in board.get_cards(card_filter=card_filter):
-        if re.match(query, card.name):
-            return card
-
-
-def find_or_create_checklist(card, checklist_name, items=[]):
-    existing_checklists = card.fetch_checklists()
-    checklist = None
-    for c in existing_checklists:
-        if c.name == checklist_name:
-            checklist = c
-            break
-    if not checklist:
-        checklist = card.add_checklist(checklist_name, [])
-        for item in items:
-            item_msg = item + ' (NO RESULTS)'
-            checklist.add_checklist_item(item_msg)
-    return checklist
-
-
-def change_checklist_item(checklist, name, checked=False):
-    # keep the trailing space so that we don't match the wrong thing later
-    r = re.match('\[*(.*? )\(', name)
-    if r:
-        debname = r.group(1)
-        for item in checklist.items:
-            if debname in item.get('name'):
-                checklist.rename_checklist_item(item.get('name'), name)
-                checklist.set_checklist_item(name, checked)
-                return True
-        else:
-            return False
-    else:
-        print('WARNING: Invalid name specified', name)
-
-
-def no_new_fails_or_skips(summary_data):
-    """Check summary data for new fails or skips
-
-    Return True if there are no new fails or skips detected and if all
-    tests passed
-    """
-    return ("No new failed or skipped tests" in summary_data and
-            "All tests passed" in summary_data)
-
-
-def load_config(configfile):
-    if not configfile:
-        return []
-    try:
-        data = yaml.safe_load(configfile)
-    except (yaml.parser.ParserError, yaml.scanner.ScannerError):
-        print('ERROR: Error parsing', configfile.name)
-        sys.exit(1)
-    return data
-
-
-def attach_labels(board, card, label_list):
-    for labelstr in label_list:
-        for label in board.get_labels():
-            if label.name == labelstr:
-                labels = card.list_labels or []
-                if label not in labels:
-                    # Avoid crash if checking labels fails to find it
-                    try:
-                        card.add_label(label)
-                    except ResourceUnavailable:
-                        pass
-                break
-
-
 def run(args, board, c3_link, jenkins_link):
 
     kernel_stack = args.series
@@ -203,8 +128,8 @@ def run(args, board, c3_link, jenkins_link):
         re.escape(kernel_stack),
         re.escape(deb_kernel_image),
         deb_version)
-    card = search_card(board, pattern)
-    config = load_config(args.config)
+    card = tbu.search_card(board, pattern)
+    config = tbu.load_config(args.config, None)
     expected_tests = config.get(kernel_stack, {}).get('expected_tests', [])
 
     logging.info('SRU type: {}'.format(args.sru_type))
@@ -241,10 +166,11 @@ def run(args, board, c3_link, jenkins_link):
         summary_data = args.summary.read()
         summary += summary_data
         summary += '\n```\n'
-        card.comment(summary)
+        comment = card.comment(summary)
+        comment_link = "{}#comment-{}".format(card.url, comment['id'])
     else:
         summary_data = ""
-    checklist = find_or_create_checklist(card, 'Testflinger', expected_tests)
+    checklist = tbu.find_or_create_checklist(card, 'Testflinger', expected_tests)
     job_name = args.name.split('-')
     cid = job_name[-2] + '-' + job_name[-1]
 
@@ -265,30 +191,32 @@ def run(args, board, c3_link, jenkins_link):
     print('Detected oem xenial run SUT: {}'.format(sut))
 
     if args.cardonly:
-        item_name = "{} ({})".format(sut, 'In progress')
+        item_content = "{} ({})".format(args.name, 'In progress')
     else:
-        item_name = "{} ({})".format(sut, datetime.utcnow().isoformat())
+        item_content = "[{}]({}) ({})".format(
+            args.name, comment_link, datetime.utcnow().isoformat())
     if jenkins_link:
-        item_name += " [[JENKINS]({})]".format(jenkins_link)
+        item_content += " [[JENKINS]({})]".format(jenkins_link)
     if c3_link:
-        item_name += " [[C3]({})]".format(c3_link)
+        item_content += " [[C3]({})]".format(c3_link)
     elif not args.cardonly:
         # If there was no c3_link, it's because the submission failed
-        attach_labels(board, card, ['TESTFLINGER CRASH'])
+        tbu.attach_labels(board, card, ['TESTFLINGER CRASH'])
 
     # debug message
     logging.info('checklist: {}'.format(checklist))
-    logging.info('item_name: {}'.format(item_name))
-    if not change_checklist_item(
-            checklist, item_name, checked=no_new_fails_or_skips(summary_data)):
-        checklist.add_checklist_item(item_name)
+    logging.info('item_content: {}'.format(item_content))
+    if not tbu.change_checklist_item(
+            checklist, args.name, item_content,
+            checked=tbu.no_new_fails_or_skips(summary_data)):
+        checklist.add_checklist_item(item_content)
 
     if not [c for c in card.fetch_checklists() if c.name == 'Sign-Off']:
-        checklist = find_or_create_checklist(card, 'Sign-Off')
+        checklist = tbu.find_or_create_checklist(card, 'Sign-Off')
         checklist.add_checklist_item('Ready for ' + lanes[0], True)
         checklist.add_checklist_item('Ready for ' + lanes[1])
         checklist.add_checklist_item('Can be Archived')
-    checklist = find_or_create_checklist(card, 'Revisions')
+    checklist = tbu.find_or_create_checklist(card, 'Revisions')
     rev = '{} ({})'.format(deb_version, args.arch)
     if rev not in [item['name'] for item in checklist.items]:
         checklist.add_checklist_item(rev)
@@ -313,11 +241,11 @@ def run(args, board, c3_link, jenkins_link):
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('--key', help="Trello API key",
-                        **environ_or_required('TRELLO_API_KEY'))
+                        **tbu.environ_or_required('TRELLO_API_KEY'))
     parser.add_argument('--token', help="Trello OAuth token",
-                        **environ_or_required('TRELLO_TOKEN'))
+                        **tbu.environ_or_required('TRELLO_TOKEN'))
     parser.add_argument('--board', help="Trello board identifier",
-                        **environ_or_required('TRELLO_BOARD'))
+                        **tbu.environ_or_required('TRELLO_BOARD'))
     parser.add_argument('--config', help="Pool configuration",
                         type=argparse.FileType())
     parser.add_argument('-a', '--arch', help="deb architecture",
@@ -363,7 +291,7 @@ class TestTrelloUpdaterKernelDebSRU(unittest.TestCase):
 
     def _mock_factory(self, jenkins_job_template, card_name, packages_info):
         parser = argparse.ArgumentParser()
-        args = parser.parse_args()
+        args = parser.parse_args([])
         args.__dict__.update(jenkins_job_template)
 
         self.args = args
@@ -398,6 +326,7 @@ class TestTrelloUpdaterKernelDebSRU(unittest.TestCase):
 
     def test_stock_xenial_4_4_kernel_stack(self):
         jenkins_job_template = {
+            'cardonly': False,
             'name': 'xenial-desktop-201606-22344',
             'arch': 'amd64',
             'kernel': 'linux-generic',
@@ -418,6 +347,7 @@ class TestTrelloUpdaterKernelDebSRU(unittest.TestCase):
 
     def test_stock_xenial_4_15_kernel_stack(self):
         jenkins_job_template = {
+            'cardonly': False,
             'name': 'xenial-hwe-desktop-201606-22344',
             'arch': 'amd64',
             'kernel': 'linux-generic-hwe-16_04',
@@ -439,6 +369,7 @@ class TestTrelloUpdaterKernelDebSRU(unittest.TestCase):
 
     def test_stock_bionic_4_15_kernel_stack(self):
         jenkins_job_template = {
+            'cardonly': False,
             'name': 'bionic-desktop-201606-22344',
             'arch': 'amd64',
             'kernel': 'linux-generic',
@@ -459,6 +390,7 @@ class TestTrelloUpdaterKernelDebSRU(unittest.TestCase):
 
     def test_oem_xenial_4_4_kernel_stack(self):
         jenkins_job_template = {
+            'cardonly': False,
             'name': 'xenial-desktop-201610-25144',
             'arch': 'amd64',
             'kernel': 'linux-generic',
@@ -479,6 +411,7 @@ class TestTrelloUpdaterKernelDebSRU(unittest.TestCase):
 
     def test_oem_xenial_4_15_kernel_stack(self):
         jenkins_job_template = {
+            'cardonly': False,
             'name': 'xenial-hwe-desktop-201802-26107',
             'arch': 'amd64',
             'kernel': 'linux-generic-hwe-16_04',
@@ -500,6 +433,7 @@ class TestTrelloUpdaterKernelDebSRU(unittest.TestCase):
 
     def test_oem_bionic_4_15_kernel_stack(self):
         jenkins_job_template = {
+            'cardonly': False,
             'name': 'bionic-desktop-201802-26107',
             'arch': 'amd64',
             'kernel': 'linux-oem',
@@ -520,6 +454,7 @@ class TestTrelloUpdaterKernelDebSRU(unittest.TestCase):
 
     def test_oem_osp1_bionic_4_15_kernel_stack(self):
         jenkins_job_template = {
+            'cardonly': False,
             'name': 'bionic-desktop-201906-27089',
             'arch': 'amd64',
             'kernel': 'linux-oem-osp1',
@@ -541,6 +476,7 @@ class TestTrelloUpdaterKernelDebSRU(unittest.TestCase):
 
     def test_argos_dgx_station_xenial_4_4(self):
         jenkins_job_template = {
+            'cardonly': False,
             'name': 'xenial-desktop-201711-25989',
             'arch': 'amd64',
             'kernel': 'linux-generic',
@@ -567,6 +503,7 @@ class TestTrelloUpdaterKernelDebSRU(unittest.TestCase):
 
     def test_argos_dgx_1_xenial_4_4(self):
         jenkins_job_template = {
+            'cardonly': False,
             'name': 'xenial-server-201802-26098',
             'arch': 'amd64',
             'kernel': 'linux-generic',
@@ -593,10 +530,11 @@ class TestTrelloUpdaterKernelDebSRU(unittest.TestCase):
 
     def test_argos_dgx_1_xenial_hwe(self):
         jenkins_job_template = {
+            'cardonly': False,
             'name': 'xenial-hwe-server-201802-26098',
             'arch': 'amd64',
             'kernel': 'linux-generic-hwe-16_04',
-            'series': 'xenial-hwe',
+            'series': 'xenial',
             'sru_type': 'oem',
             'queue': 'argos-201802-26098',
             'config': self.debs_yaml_stream,
diff --git a/trello-board-updater-image-testing.py b/trello-board-updater-image-testing.py
index ed1485b..ca97b92 100755
--- a/trello-board-updater-image-testing.py
+++ b/trello-board-updater-image-testing.py
@@ -1,85 +1,31 @@
 #!/usr/bin/env python3
-# Copyright 2019 Canonical Ltd.
+# Copyright 2019-2020 Canonical Ltd.
 # All rights reserved.
 #
 # Written by:
 #        Paul Larson <[e-mail]>
+#        Sylvain Pineau <[e-mail]>
 
 import argparse
+import importlib
 import os
 import re
 
 from datetime import datetime
 from trello import TrelloClient
 
-
-def environ_or_required(key):
-    if os.environ.get(key):
-        return {'default': os.environ.get(key)}
-    else:
-        return {'required': True}
-
-
-def search_card(board, query, card_filter="open"):
-    for card in board.get_cards(card_filter=card_filter):
-        if re.match(query, card.name):
-            return card
-
-
-def archive_card(card):
-    print('Archiving old card:', card)
-    card.set_closed(True)
-
-
-def find_or_create_checklist(card, checklist_name, items=None):
-    existing_checklists = card.fetch_checklists()
-    checklist = None
-    for c in existing_checklists:
-        if c.name == checklist_name:
-            checklist = c
-            break
-    if not checklist:
-        checklist = card.add_checklist(checklist_name, [])
-        if items:
-            for item in items:
-                checklist.add_checklist_item(item + ' (NO RESULTS)')
-    return checklist
-
-
-def change_checklist_item(checklist, name, checked=False):
-    # keep the trailing space so that we don't match the wrong thing later
-    r = re.match('\[*(.* )\(', name)
-    if r:
-        device_name = r.group(1)
-        for item in checklist.items:
-            if device_name in item.get('name'):
-                checklist.rename_checklist_item(item.get('name'), name)
-                checklist.set_checklist_item(name, checked)
-                return True
-        else:
-            return False
-    else:
-        print('WARNING: Invalid name specified', name)
-
-
-def no_new_fails_or_skips(summary_data):
-    """Check summary data for new fails or skips
-
-    Return True if there are no new fails or skips detected and if all
-    tests passed
-    """
-    return ("No new failed or skipped tests" in summary_data and
-            "All tests passed" in summary_data)
+tbu = importlib.import_module("trello-board-updater")
+tbm = importlib.import_module("trello-board-manager")
 
 
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('--key', help="Trello API key",
-                        **environ_or_required('TRELLO_API_KEY'))
+                        **tbu.environ_or_required('TRELLO_API_KEY'))
     parser.add_argument('--token', help="Trello OAuth token",
-                        **environ_or_required('TRELLO_TOKEN'))
+                        **tbu.environ_or_required('TRELLO_TOKEN'))
     parser.add_argument('--board', help="Trello board identifier",
-                        **environ_or_required('TRELLO_BOARD'))
+                        **tbu.environ_or_required('TRELLO_BOARD'))
     parser.add_argument('-n', '--name', help="SUT device name", required=True)
     parser.add_argument('-i', '--image', help="image name", required=True)
     parser.add_argument('-c', '--channel', help="image name", default="stable")
@@ -96,14 +42,14 @@ def main():
         re.escape(args.channel),
         re.escape(args.version))
     # First, see if this exact card already exists
-    card = search_card(board, pattern)
+    card = tbu.search_card(board, pattern)
 
     # If not, see if there's an older one for this image
     if not card:
         pattern = "{} - {} - .*".format(re.escape(args.image), args.channel)
-        card = search_card(board, pattern)
+        card = tbu.search_card(board, pattern)
         if card:
-            archive_card(card)
+            tbm.archive_card(card)
         # If we get here, then either we just archived the old card, or
         # it didn't exist. We need to create it either way
         lane = None
@@ -122,19 +68,21 @@ def main():
     summary_data = args.summary.read()
     summary += summary_data
     summary += '\n```\n'
-    card.comment(summary)
-    checklist = find_or_create_checklist(card, 'Testflinger')
-    item_name = "{} ({})".format(args.name, datetime.utcnow().isoformat())
+    comment = card.comment(summary)
+    comment_link = "{}#comment-{}".format(card.url, comment['id'])
+    checklist = tbu.find_or_create_checklist(card, 'Testflinger')
+    item_content = "[{}]({}) ({})".format(
+        args.name, comment_link, datetime.utcnow().isoformat())
     if jenkins_link:
-        item_name += " [[JENKINS]({})]".format(jenkins_link)
+        item_content += " [[JENKINS]({})]".format(jenkins_link)
     if c3_link:
-        item_name += " [[C3]({})]".format(c3_link)
+        item_content += " [[C3]({})]".format(c3_link)
 
-    if not change_checklist_item(
-            checklist, item_name,
-            checked=no_new_fails_or_skips(summary_data)):
+    if not tbu.change_checklist_item(
+            checklist, args.name, item_content,
+            checked=tbu.no_new_fails_or_skips(summary_data)):
         checklist.add_checklist_item(
-            item_name, checked=no_new_fails_or_skips(summary_data))
+            item_content, checked=tbu.no_new_fails_or_skips(summary_data))
 
 
 if __name__ == "__main__":