Merge ppa-dev-tools:support-subtest-status-flaky into ppa-dev-tools:main

[Bryce Harrington]

autopkgtest subtests can be given the restriction 'flaky'[1][2], which results in the subtest being given the status 'FLAKY' in the test log[3]. This is not common and fairly unexciting, but it does show up in the wild[4].

1: https://people.debian.org/~eriberto/README.package-tests.html
2: https://git.launchpad.net/ubuntu/+source/rust-clang-sys/tree/debian/tests/control
3: https://autopkgtest.ubuntu.com/results/autopkgtest-oracular/oracular/ppc64el/r/rust-clang-sys/20240506_050646_5005b@/log.gz
4: https://autopkgtest.ubuntu.com/packages/r/rust-clang-sys/oracular/ppc64el

I display the subtests with colored squares from the unicode characters, and unfortunately there are only a handful of colors available. I considered purple, but brown felt like a better conceptual fit, and purple seemed like it could be useful for some other situation.

[Athos Ribeiro (athos-ribeiro)]

LGTM. I am not sure how the contrast between this brown and the yellow/red colors we already use will be seen by users though, but I have no better suggestions here!

review: Approve

Unmerged commits

45c013f... by Bryce Harrington on 2024-05-07

Add support for subtests marked as 'FLAKY'

Subtest icons will be brown, like a flaky chocolate pastry.

Fixes: https://bugs.launchpad.net/ppa-dev-tools/+bug/1989650

diff --git a/ppa/result.py b/ppa/result.py
index 7488f54..5d6f25f 100755
--- a/ppa/result.py
+++ b/ppa/result.py
@@ -205,7 +205,7 @@ class Result:
 
         subtests = []
         result_sum = result_split[1]
-        for line in re.findall("(.*PASS|.*SKIP|.*FAIL|.*BAD)", result_sum):
+        for line in re.findall("(.*PASS|.*SKIP|.*FAIL|.*FLAKY|.*BAD)", result_sum):
             if re.match(r"^[0-9]+s\s", line):
                 # Newer autopkgtest logs are prefixed with the timestamp.  Ignore it.
                 line = line.split(' ', 1)[1]
diff --git a/ppa/subtest.py b/ppa/subtest.py
index 624334f..da3cdb1 100755
--- a/ppa/subtest.py
+++ b/ppa/subtest.py
@@ -26,6 +26,7 @@ class Subtest:
         'PASS': "🟩",
         'SKIP': "🟧",
         'FAIL': "🟥",
+        'FLAKY': "🟫",
         'BAD': "⛔",
         'UNKNOWN': "⚪"
     }
@@ -116,6 +117,7 @@ if __name__ == "__main__":
     print(Subtest('subtest-e   SKIP'))
     print(Subtest('subtest-f   BAD'))
     print(Subtest('librust-clang-sys-dev:clang_10_0 FAIL non-zero exit status 101'))
+    print(Subtest('librust-clang-sys-dev:static FLAKY non-zero exit status 101'))
     print()
 
     print("Invalid cases")
diff --git a/tests/test_result.py b/tests/test_result.py
index 89a6c8f..3c758cc 100644
--- a/tests/test_result.py
+++ b/tests/test_result.py
@@ -85,13 +85,14 @@ def test_str():
     (
         # When at least one Subtest fails, all subtest results should be shown.
         '20030201_040506',
-        [Subtest('x PASS'), Subtest('y FAIL'), Subtest('z SKIP')],
+        [Subtest('x PASS'), Subtest('y FAIL'), Subtest('f FLAKY'), Subtest('z SKIP')],
         True,
         """    + ❌ source on series for arch    @ 01.02.03 04:05:06
       • Log: file://<DATA_DIR>/x
       • Status: FAIL
       • x                         PASS   🟩
       • y                         FAIL   🟥
+      • f                         FLAKY  🟫
       • z                         SKIP   🟧
 """
     ),
@@ -177,10 +178,11 @@ def test_get_triggers(monkeypatch, triggers, name, expected):
             "x: @@@@@@@@@@@@@@@@@@@@ summary\n"
             "test-a          PASS\n"
             "test-b          FAIL ignorable-note\n"
-            "test-c          NoTaVaLiDsTaTuS\n"
+            "test-c          FLAKY some-detail\n"
+            "test-d          NoTaVaLiDsTaTuS\n"
         ),
         None,
-        {'test-a': 'PASS', 'test-b': 'FAIL'}
+        {'test-a': 'PASS', 'test-b': 'FAIL', 'test-c': 'FLAKY'}
     ),
     (
         (
@@ -254,9 +256,13 @@ def test_get_subtests(tmp_path, log_text: str, subtest_name: str, expected: dict
     ([], None, 'PASS'),
     (['PASS'], None, 'PASS'),
     (['FAIL'], None, 'FAIL'),
+    (['SKIP'], None, 'PASS'),
+    (['FLAKY'], None, 'PASS'),
     (['PASS', 'FAIL'], None, 'FAIL'),
     (['FAIL', 'PASS'], None, 'FAIL'),
     (['PASS', 'PASS', 'PASS'], None, 'PASS'),
+    (['PASS', 'FLAKY'], None, 'PASS'),
+    (['PASS', 'SKIP'], None, 'PASS'),
     (['PASS'], 'x', 'BAD'),
     (['FAIL'], 'x', 'BAD'),
 ])
diff --git a/tests/test_subtest.py b/tests/test_subtest.py
index 41ac536..e7b3327 100644
--- a/tests/test_subtest.py
+++ b/tests/test_subtest.py
@@ -87,6 +87,7 @@ def test_desc():
     ('c FAIL', 'FAIL'),
     ('d FAIL non-zero exit status 123', 'FAIL'),
     ('librust-clang-sys-dev:clang_10_0 FAIL non-zero exit status 101', 'FAIL'),
+    ('librust-clang-sys-dev:static FLAKY non-zero exit status 101', 'FLAKY'),
     ('f BAD', 'BAD'),
     ('g UNKNOWN', 'UNKNOWN'),
     ('h invalid', 'UNKNOWN'),
@@ -101,13 +102,14 @@ def test_status(line, status):
 
 
 @pytest.mark.parametrize('line, icon', [
-    ('a PASS', "🟩"),
-    ('b SKIP', "🟧"),
-    ('c FAIL', "🟥"),
-    ('d BAD', "⛔"),
-    ('e UNKNOWN', "⚪"),
-    ('f invalid', "⚪"),
-    ('f bAd', "⚪"),
+    ('x PASS', "🟩"),
+    ('x SKIP', "🟧"),
+    ('x FAIL', "🟥"),
+    ('x FLAKY', "🟫"),
+    ('x BAD', "⛔"),
+    ('x UNKNOWN', "⚪"),
+    ('x invalid', "⚪"),
+    ('x bAd', "⚪"),
 ])
 def test_status_icon(line, icon):
     """Checks Subtest provides correct icon for status.