Merge ppa-dev-tools:support-subtest-status-flaky into ppa-dev-tools:main

Proposed by Bryce Harrington
Status: Merged
Merge reported by: Bryce Harrington
Merged at revision: 45c013f05868d01ce58b2f27f3a1a6656a521e83
Proposed branch: ppa-dev-tools:support-subtest-status-flaky
Merge into: ppa-dev-tools:main
Diff against target: 115 lines (+21/-11)
4 files modified
ppa/result.py (+1/-1)
ppa/subtest.py (+2/-0)
tests/test_result.py (+9/-3)
tests/test_subtest.py (+9/-7)
Reviewer Review Type Date Requested Status
Athos Ribeiro (community) Approve
Canonical Server Reporter Pending
Review via email: mp+465631@code.launchpad.net

Description of the change

autopkgtest subtests can be given the restriction 'flaky'[1][2], which results in the subtest being given the status 'FLAKY' in the test log[3]. This is not common and fairly unexciting, but it does show up in the wild[4].

1: https://people.debian.org/~eriberto/README.package-tests.html
2: https://git.launchpad.net/ubuntu/+source/rust-clang-sys/tree/debian/tests/control
3: https://autopkgtest.ubuntu.com/results/autopkgtest-oracular/oracular/ppc64el/r/rust-clang-sys/20240506_050646_5005b@/log.gz
4: https://autopkgtest.ubuntu.com/packages/r/rust-clang-sys/oracular/ppc64el

I display the subtests with colored squares from the unicode characters, and unfortunately there are only a handful of colors available. I considered purple, but brown felt like a better conceptual fit, and purple seemed like it could be useful for some other situation.

To post a comment you must log in.
Revision history for this message
Athos Ribeiro (athos-ribeiro) wrote :

LGTM. I am not sure how the contrast between this brown and the yellow/red colors we already use will be seen by users though, but I have no better suggestions here!

review: Approve
Revision history for this message
Bryce Harrington (bryce) wrote :

Thanks, landed to main.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
diff --git a/ppa/result.py b/ppa/result.py
index 7488f54..5d6f25f 100755
--- a/ppa/result.py
+++ b/ppa/result.py
@@ -205,7 +205,7 @@ class Result:
205205
206 subtests = []206 subtests = []
207 result_sum = result_split[1]207 result_sum = result_split[1]
208 for line in re.findall("(.*PASS|.*SKIP|.*FAIL|.*BAD)", result_sum):208 for line in re.findall("(.*PASS|.*SKIP|.*FAIL|.*FLAKY|.*BAD)", result_sum):
209 if re.match(r"^[0-9]+s\s", line):209 if re.match(r"^[0-9]+s\s", line):
210 # Newer autopkgtest logs are prefixed with the timestamp. Ignore it.210 # Newer autopkgtest logs are prefixed with the timestamp. Ignore it.
211 line = line.split(' ', 1)[1]211 line = line.split(' ', 1)[1]
diff --git a/ppa/subtest.py b/ppa/subtest.py
index 624334f..da3cdb1 100755
--- a/ppa/subtest.py
+++ b/ppa/subtest.py
@@ -26,6 +26,7 @@ class Subtest:
26 'PASS': "🟩",26 'PASS': "🟩",
27 'SKIP': "🟧",27 'SKIP': "🟧",
28 'FAIL': "🟥",28 'FAIL': "🟥",
29 'FLAKY': "🟫",
29 'BAD': "⛔",30 'BAD': "⛔",
30 'UNKNOWN': "⚪"31 'UNKNOWN': "⚪"
31 }32 }
@@ -116,6 +117,7 @@ if __name__ == "__main__":
116 print(Subtest('subtest-e SKIP'))117 print(Subtest('subtest-e SKIP'))
117 print(Subtest('subtest-f BAD'))118 print(Subtest('subtest-f BAD'))
118 print(Subtest('librust-clang-sys-dev:clang_10_0 FAIL non-zero exit status 101'))119 print(Subtest('librust-clang-sys-dev:clang_10_0 FAIL non-zero exit status 101'))
120 print(Subtest('librust-clang-sys-dev:static FLAKY non-zero exit status 101'))
119 print()121 print()
120122
121 print("Invalid cases")123 print("Invalid cases")
diff --git a/tests/test_result.py b/tests/test_result.py
index 89a6c8f..3c758cc 100644
--- a/tests/test_result.py
+++ b/tests/test_result.py
@@ -85,13 +85,14 @@ def test_str():
85 (85 (
86 # When at least one Subtest fails, all subtest results should be shown.86 # When at least one Subtest fails, all subtest results should be shown.
87 '20030201_040506',87 '20030201_040506',
88 [Subtest('x PASS'), Subtest('y FAIL'), Subtest('z SKIP')],88 [Subtest('x PASS'), Subtest('y FAIL'), Subtest('f FLAKY'), Subtest('z SKIP')],
89 True,89 True,
90 """ + ❌ source on series for arch @ 01.02.03 04:05:0690 """ + ❌ source on series for arch @ 01.02.03 04:05:06
91 • Log: file://<DATA_DIR>/x91 • Log: file://<DATA_DIR>/x
92 • Status: FAIL92 • Status: FAIL
93 • x PASS 🟩93 • x PASS 🟩
94 • y FAIL 🟥94 • y FAIL 🟥
95 • f FLAKY 🟫
95 • z SKIP 🟧96 • z SKIP 🟧
96"""97"""
97 ),98 ),
@@ -177,10 +178,11 @@ def test_get_triggers(monkeypatch, triggers, name, expected):
177 "x: @@@@@@@@@@@@@@@@@@@@ summary\n"178 "x: @@@@@@@@@@@@@@@@@@@@ summary\n"
178 "test-a PASS\n"179 "test-a PASS\n"
179 "test-b FAIL ignorable-note\n"180 "test-b FAIL ignorable-note\n"
180 "test-c NoTaVaLiDsTaTuS\n"181 "test-c FLAKY some-detail\n"
182 "test-d NoTaVaLiDsTaTuS\n"
181 ),183 ),
182 None,184 None,
183 {'test-a': 'PASS', 'test-b': 'FAIL'}185 {'test-a': 'PASS', 'test-b': 'FAIL', 'test-c': 'FLAKY'}
184 ),186 ),
185 (187 (
186 (188 (
@@ -254,9 +256,13 @@ def test_get_subtests(tmp_path, log_text: str, subtest_name: str, expected: dict
254 ([], None, 'PASS'),256 ([], None, 'PASS'),
255 (['PASS'], None, 'PASS'),257 (['PASS'], None, 'PASS'),
256 (['FAIL'], None, 'FAIL'),258 (['FAIL'], None, 'FAIL'),
259 (['SKIP'], None, 'PASS'),
260 (['FLAKY'], None, 'PASS'),
257 (['PASS', 'FAIL'], None, 'FAIL'),261 (['PASS', 'FAIL'], None, 'FAIL'),
258 (['FAIL', 'PASS'], None, 'FAIL'),262 (['FAIL', 'PASS'], None, 'FAIL'),
259 (['PASS', 'PASS', 'PASS'], None, 'PASS'),263 (['PASS', 'PASS', 'PASS'], None, 'PASS'),
264 (['PASS', 'FLAKY'], None, 'PASS'),
265 (['PASS', 'SKIP'], None, 'PASS'),
260 (['PASS'], 'x', 'BAD'),266 (['PASS'], 'x', 'BAD'),
261 (['FAIL'], 'x', 'BAD'),267 (['FAIL'], 'x', 'BAD'),
262])268])
diff --git a/tests/test_subtest.py b/tests/test_subtest.py
index 41ac536..e7b3327 100644
--- a/tests/test_subtest.py
+++ b/tests/test_subtest.py
@@ -87,6 +87,7 @@ def test_desc():
87 ('c FAIL', 'FAIL'),87 ('c FAIL', 'FAIL'),
88 ('d FAIL non-zero exit status 123', 'FAIL'),88 ('d FAIL non-zero exit status 123', 'FAIL'),
89 ('librust-clang-sys-dev:clang_10_0 FAIL non-zero exit status 101', 'FAIL'),89 ('librust-clang-sys-dev:clang_10_0 FAIL non-zero exit status 101', 'FAIL'),
90 ('librust-clang-sys-dev:static FLAKY non-zero exit status 101', 'FLAKY'),
90 ('f BAD', 'BAD'),91 ('f BAD', 'BAD'),
91 ('g UNKNOWN', 'UNKNOWN'),92 ('g UNKNOWN', 'UNKNOWN'),
92 ('h invalid', 'UNKNOWN'),93 ('h invalid', 'UNKNOWN'),
@@ -101,13 +102,14 @@ def test_status(line, status):
101102
102103
103@pytest.mark.parametrize('line, icon', [104@pytest.mark.parametrize('line, icon', [
104 ('a PASS', "🟩"),105 ('x PASS', "🟩"),
105 ('b SKIP', "🟧"),106 ('x SKIP', "🟧"),
106 ('c FAIL', "🟥"),107 ('x FAIL', "🟥"),
107 ('d BAD', "⛔"),108 ('x FLAKY', "🟫"),
108 ('e UNKNOWN', "⚪"),109 ('x BAD', "⛔"),
109 ('f invalid', "⚪"),110 ('x UNKNOWN', "⚪"),
110 ('f bAd', "⚪"),111 ('x invalid', "⚪"),
112 ('x bAd', "⚪"),
111])113])
112def test_status_icon(line, icon):114def test_status_icon(line, icon):
113 """Checks Subtest provides correct icon for status.115 """Checks Subtest provides correct icon for status.

Subscribers

People subscribed via source and target branches

to all changes: