Merge ~hyask/autopkgtest-cloud:skia/improve_browsing into autopkgtest-cloud:master
- Git
- lp:~hyask/autopkgtest-cloud
- skia/improve_browsing
- Merge into master
Proposed by
Skia
Status: | Merged | ||||
---|---|---|---|---|---|
Merged at revision: | df949e5544820f195b5094b402dd6d0f97a2ef8a | ||||
Proposed branch: | ~hyask/autopkgtest-cloud:skia/improve_browsing | ||||
Merge into: | autopkgtest-cloud:master | ||||
Diff against target: |
372 lines (+90/-88) 10 files modified
charms/focal/autopkgtest-web/charmcraft.yaml (+1/-0) charms/focal/autopkgtest-web/reactive/autopkgtest_web.py (+0/-14) charms/focal/autopkgtest-web/webcontrol/browse.cgi (+25/-23) charms/focal/autopkgtest-web/webcontrol/helpers/utils.py (+18/-2) charms/focal/autopkgtest-web/webcontrol/static/bootstrap (+1/-0) charms/focal/autopkgtest-web/webcontrol/templates/browse-admin.html (+4/-19) charms/focal/autopkgtest-web/webcontrol/templates/browse-layout.html (+5/-5) charms/focal/autopkgtest-web/webcontrol/templates/browse-package.html (+6/-0) charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html (+4/-25) charms/focal/autopkgtest-web/webcontrol/templates/macros.html (+26/-0) |
||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Tim Andersson | Approve | ||
Review via email: mp+461022@code.launchpad.net |
Commit message
Description of the change
A bit of refactor to bring the feature of displaying currently running jobs on each package's page.
To post a comment you must log in.
Revision history for this message
Tim Andersson (andersson123) wrote : | # |
Revision history for this message
Tim Andersson (andersson123) wrote : | # |
still going to review though
Revision history for this message
Tim Andersson (andersson123) wrote : | # |
lbjsbootstrap changes to be tested in staging before deployment
Revision history for this message
Tim Andersson (andersson123) wrote : | # |
Skia to implement new Exceptions in helpers/
Also implement all exception handler in browse.cgi for extendability.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/charms/focal/autopkgtest-web/charmcraft.yaml b/charms/focal/autopkgtest-web/charmcraft.yaml |
2 | index 93c0816..546437f 100644 |
3 | --- a/charms/focal/autopkgtest-web/charmcraft.yaml |
4 | +++ b/charms/focal/autopkgtest-web/charmcraft.yaml |
5 | @@ -6,6 +6,7 @@ parts: |
6 | build-snaps: [charm] |
7 | build-packages: |
8 | - libjs-jquery |
9 | + - libjs-bootstrap |
10 | - python3-dev |
11 | bases: |
12 | - build-on: |
13 | diff --git a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py |
14 | index e1be1a2..6920fc1 100644 |
15 | --- a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py |
16 | +++ b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py |
17 | @@ -338,20 +338,6 @@ def clear_github_status_credentials(): |
18 | pass |
19 | |
20 | |
21 | -@when_not("autopkgtest-web.bootstrap-symlinked") |
22 | -def symlink_bootstrap(): |
23 | - try: |
24 | - os.symlink( |
25 | - os.path.join( |
26 | - os.path.sep, "usr", "share", "javascript", "bootstrap" |
27 | - ), |
28 | - os.path.join(charm_dir(), "webcontrol", "static", "bootstrap"), |
29 | - ) |
30 | - set_flag("autopkgtest-web.bootstrap-symlinked") |
31 | - except FileExistsError: |
32 | - pass |
33 | - |
34 | - |
35 | @when_not("autopkgtest-web.runtime-dir-created") |
36 | def make_runtime_tmpfiles(): |
37 | with open("/etc/tmpfiles.d/autopkgtest-web-runtime.conf", "w") as r: |
38 | diff --git a/charms/focal/autopkgtest-web/webcontrol/browse.cgi b/charms/focal/autopkgtest-web/webcontrol/browse.cgi |
39 | index f39b30f..f6be794 100755 |
40 | --- a/charms/focal/autopkgtest-web/webcontrol/browse.cgi |
41 | +++ b/charms/focal/autopkgtest-web/webcontrol/browse.cgi |
42 | @@ -10,9 +10,9 @@ import sqlite3 |
43 | from collections import OrderedDict |
44 | from wsgiref.handlers import CGIHandler |
45 | |
46 | -import distro_info |
47 | import flask |
48 | from helpers.admin import select_abnormally_long_jobs |
49 | +from helpers.utils import get_all_releases, get_supported_releases |
50 | from werkzeug.middleware.proxy_fix import ProxyFix |
51 | |
52 | app = flask.Flask("browse") |
53 | @@ -20,13 +20,11 @@ app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1) |
54 | db_con = None |
55 | swift_container_url = None |
56 | |
57 | -UDI = distro_info.UbuntuDistroInfo() |
58 | -ALL_UBUNTU_RELEASES = UDI.all |
59 | -SUPPORTED_UBUNTU_RELEASES = sorted( |
60 | - set(UDI.supported() + UDI.supported_esm()), key=ALL_UBUNTU_RELEASES.index |
61 | -) |
62 | - |
63 | +ALL_UBUNTU_RELEASES = get_all_releases() |
64 | +SUPPORTED_UBUNTU_RELEASES = get_supported_releases() |
65 | INDEXED_PACKAGES_FP = "" |
66 | +AMQP_QUEUE_CACHE = "/var/lib/cache-amqp/queued.json" |
67 | +RUNNING_CACHE = "/run/amqp-status-collector/running.json" |
68 | |
69 | |
70 | def init_config(): |
71 | @@ -60,6 +58,16 @@ def get_test_id(release, arch, src): |
72 | return None |
73 | |
74 | |
75 | +def get_running_jobs(): |
76 | + try: |
77 | + with open(RUNNING_CACHE) as f: |
78 | + # package -> runhash -> release -> arch -> (params, duration, logtail) |
79 | + running_info = json.load(f) |
80 | + except FileNotFoundError: |
81 | + running_info = {} |
82 | + return running_info |
83 | + |
84 | + |
85 | def render(template, code=200, **kwargs): |
86 | # sort the values passed in, so that releases are in the right order |
87 | try: |
88 | @@ -81,7 +89,7 @@ def render(template, code=200, **kwargs): |
89 | flask.render_template( |
90 | template, |
91 | base_url=flask.url_for("index_root"), |
92 | - static_url=flask.url_for("static", filename="/"), |
93 | + static_url=flask.url_for("static", filename=""), |
94 | **kwargs |
95 | ), |
96 | code, |
97 | @@ -147,7 +155,7 @@ def get_queue_info(): |
98 | Return (releases, arches, context -> release -> arch -> (queue_size, [requests])). |
99 | """ |
100 | |
101 | - with open("/var/lib/cache-amqp/queued.json", "r") as json_file: |
102 | + with open(AMQP_QUEUE_CACHE, "r") as json_file: |
103 | queue_info_j = json.load(json_file) |
104 | |
105 | arches = queue_info_j["arches"] |
106 | @@ -268,6 +276,10 @@ def package_overview(package, _=None): |
107 | arches.add(row[3]) |
108 | results.setdefault(row[2], {})[row[3]] = human_exitcode(row[1]) |
109 | |
110 | + running_info = dict( |
111 | + (k, v) for (k, v) in get_running_jobs().items() if k == package |
112 | + ) |
113 | + |
114 | return render( |
115 | "browse-package.html", |
116 | package=package, |
117 | @@ -280,6 +292,7 @@ def package_overview(package, _=None): |
118 | arches=sorted(arches), |
119 | results=results, |
120 | title_suffix="- %s" % package, |
121 | + running=running_info, |
122 | ) |
123 | |
124 | |
125 | @@ -371,12 +384,7 @@ def running(): |
126 | a |
127 | ] = queue_length |
128 | |
129 | - try: |
130 | - with open("/run/amqp-status-collector/running.json") as f: |
131 | - # package -> runhash -> release -> arch -> (params, duration, logtail) |
132 | - running_info = json.load(f) |
133 | - except FileNotFoundError: |
134 | - running_info = {} |
135 | + running_info = get_running_jobs() |
136 | |
137 | return render( |
138 | "browse-running.html", |
139 | @@ -391,13 +399,7 @@ def running(): |
140 | |
141 | @app.route("/admin") |
142 | def admin(): |
143 | - try: |
144 | - with open("/run/amqp-status-collector/running.json") as f: |
145 | - # package -> runhash -> release -> arch -> (params, duration, logtail) |
146 | - running_info = json.load(f) |
147 | - except FileNotFoundError as exc: |
148 | - running_info = {} |
149 | - raise FileNotFoundError("running.json doesn't exist!") from exc |
150 | + running_info = get_running_jobs() |
151 | pruned_running_info = select_abnormally_long_jobs( |
152 | running_info, get_test_id=get_test_id, db_con=db_con |
153 | ) |
154 | @@ -439,7 +441,7 @@ def queues_json(): |
155 | |
156 | @app.route("/queued.json") |
157 | def return_queued_exactly(): |
158 | - with open("/var/lib/cache-amqp/queued.json") as json_file: |
159 | + with open(AMQP_QUEUE_CACHE) as json_file: |
160 | queue_info = json.load(json_file) |
161 | return queue_info |
162 | |
163 | diff --git a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py |
164 | index 58a9514..12d93b5 100644 |
165 | --- a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py |
166 | +++ b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py |
167 | @@ -8,6 +8,22 @@ import random |
168 | import sqlite3 |
169 | import time |
170 | |
171 | +import distro_info |
172 | + |
173 | + |
174 | +def get_all_releases(): |
175 | + udi = distro_info.UbuntuDistroInfo() |
176 | + return udi.all |
177 | + |
178 | + |
179 | +def get_supported_releases(): |
180 | + udi = distro_info.UbuntuDistroInfo() |
181 | + all_ubuntu_releases = get_all_releases() |
182 | + return sorted( |
183 | + set(udi.supported() + udi.supported_esm()), |
184 | + key=all_ubuntu_releases.index, |
185 | + ) |
186 | + |
187 | |
188 | def setup_key(app, path): |
189 | """Create or load app.secret_key for cookie encryption.""" |
190 | @@ -22,10 +38,10 @@ def setup_key(app, path): |
191 | app.secret_key = key |
192 | |
193 | |
194 | -def init_db(path): |
195 | +def init_db(path, **kwargs): |
196 | """Create DB if it does not exist, and connect to it""" |
197 | |
198 | - db = sqlite3.connect(path) |
199 | + db = sqlite3.connect(path, **kwargs) |
200 | c = db.cursor() |
201 | try: |
202 | c.execute("PRAGMA journal_mode = WAL") |
203 | diff --git a/charms/focal/autopkgtest-web/webcontrol/static/bootstrap b/charms/focal/autopkgtest-web/webcontrol/static/bootstrap |
204 | new file mode 120000 |
205 | index 0000000..fe0f86b |
206 | --- /dev/null |
207 | +++ b/charms/focal/autopkgtest-web/webcontrol/static/bootstrap |
208 | @@ -0,0 +1 @@ |
209 | +/usr/share/javascript/bootstrap |
210 | \ No newline at end of file |
211 | diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-admin.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-admin.html |
212 | index 5bc2459..266f584 100644 |
213 | --- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-admin.html |
214 | +++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-admin.html |
215 | @@ -1,29 +1,14 @@ |
216 | {% extends "browse-layout.html" %} |
217 | +{% import "macros.html" as macros %} |
218 | + |
219 | {% block content %} |
220 | <h1 class="page-header">Admin</h1> |
221 | <p>Click on the package name to jump to the tests of the package for all arches/releases.</p> |
222 | <p>This page is simply a bunch of heuristics filtering all running jobs to try to get the problematic ones. Feel free to come help improve the heuristics <a href="https://code.launchpad.net/~ubuntu-release/autopkgtest-cloud/+git/autopkgtest-cloud/+ref/master">here.</a></p> |
223 | |
224 | <!-- Running tests --> |
225 | - {% for p in running|sort %} |
226 | - <h2 id="pkg-{{p}}"><a href="/packages/{{p}}">{{p}}</a></h2> |
227 | - {% for runhash, relinfo in running[p].items() %} |
228 | - {% for release, archinfo in relinfo.items() %} |
229 | - {% for arch, (params, duration, logtail) in archinfo.items() %} |
230 | - <table class="table-condensed"> |
231 | - <tr><th>Release:</th><td>{{release}}</td></tr> |
232 | - <tr><th>Architecture:</th><td>{{arch}}</td></tr> |
233 | - {% for param, v in params.items() %} |
234 | - <tr><th>{{param|capitalize}}:</th><td>{{v}}</td></tr> |
235 | - {% endfor %} |
236 | - <tr><th>Running for:</th><td>{{duration//3600 }}h {{duration % 3600//60}}m {{duration % 60}}s</td></tr> |
237 | - </table> |
238 | - <pre> |
239 | -{{logtail}} |
240 | - </pre> |
241 | - {% endfor %} |
242 | - {% endfor %} |
243 | - {% endfor %} |
244 | + {% for p, info in running.items()|sort %} |
245 | + {{ macros.display_running_job(p, info) }} |
246 | {% endfor %} |
247 | |
248 | {% endblock %} |
249 | diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-layout.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-layout.html |
250 | index 0f90de3..c91457d 100644 |
251 | --- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-layout.html |
252 | +++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-layout.html |
253 | @@ -6,9 +6,9 @@ |
254 | <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
255 | <title>Ubuntu Autopkgtest Results {{title_suffix}}</title> |
256 | <!-- <link rel="icon" type="image/png" href="/debian.png"/> --> |
257 | - <link rel="stylesheet" type="text/css" href="{{static_url}}/bootstrap/css/bootstrap.css"/> |
258 | - <link rel="stylesheet" type="text/css" href="{{static_url}}/bootstrap/css/bootstrap-theme.css"/> |
259 | - <link rel="stylesheet" type="text/css" href="{{static_url}}/style.css"/> |
260 | + <link rel="stylesheet" type="text/css" href="{{static_url}}bootstrap/css/bootstrap.css"/> |
261 | + <link rel="stylesheet" type="text/css" href="{{static_url}}bootstrap/css/bootstrap-theme.css"/> |
262 | + <link rel="stylesheet" type="text/css" href="{{static_url}}style.css"/> |
263 | </head> |
264 | <body> |
265 | <div id='wrap'> |
266 | @@ -44,7 +44,7 @@ |
267 | {% block content %}{% endblock %} |
268 | </div> |
269 | |
270 | - <script type="text/javascript" src="{{static_url}}/jquery/jquery.min.js"></script> |
271 | - <script type="text/javascript" src="{{static_url}}/bootstrap/js/bootstrap.min.js"></script> |
272 | + <script type="text/javascript" src="{{static_url}}jquery/jquery.min.js"></script> |
273 | + <script type="text/javascript" src="{{static_url}}bootstrap/js/bootstrap.min.js"></script> |
274 | </body> |
275 | </html> |
276 | diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-package.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-package.html |
277 | index 3ac81c3..fb12afa 100644 |
278 | --- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-package.html |
279 | +++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-package.html |
280 | @@ -1,4 +1,6 @@ |
281 | {% extends "browse-layout.html" %} |
282 | +{% import "macros.html" as macros %} |
283 | + |
284 | {% block content %} |
285 | <h2>{{package}}</h2> |
286 | |
287 | @@ -17,4 +19,8 @@ |
288 | </tr> |
289 | {% endfor %} |
290 | </table> |
291 | + |
292 | + {% for p, info in running.items()|sort %} |
293 | + {{ macros.display_running_job(p, info) }} |
294 | + {% endfor %} |
295 | {% endblock %} |
296 | diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html |
297 | index 3711b97..53ae2f9 100644 |
298 | --- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html |
299 | +++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html |
300 | @@ -1,4 +1,6 @@ |
301 | {% extends "browse-layout.html" %} |
302 | +{% import "macros.html" as macros %} |
303 | + |
304 | {% block content %} |
305 | <h1 class="page-header">Currently running tests</h1> |
306 | <p>Click on the package name to jump to the currently running tests of that package.</p> |
307 | @@ -36,31 +38,8 @@ |
308 | {% endfor %} |
309 | |
310 | <!-- Running tests --> |
311 | - {% for p in running|sort %} |
312 | - <h2 id="pkg-{{p}}"><a href="/packages/{{p}}">{{p}}</a></h2> |
313 | - {% for runhash, relinfo in running[p].items() %} |
314 | - {% for release, archinfo in relinfo.items() %} |
315 | - {% for arch, (params, duration, logtail) in archinfo.items() %} |
316 | - <table class="table-condensed"> |
317 | - <tr><th>Release:</th><td>{{release}}</td></tr> |
318 | - <tr><th>Architecture:</th><td>{{arch}}</td></tr> |
319 | - {% for param, v in params.items() %} |
320 | - {% if param == "requester" %} |
321 | - <tr><th>{{param|capitalize}}:</th><td><a href="https://launchpad.net/~{{v}}">{{v}}</a></td></tr> |
322 | - {% elif param == "uuid" %} |
323 | - <tr><th>{{param|capitalize}}:</th><td>{{v}}</td></tr> |
324 | - {% else %} |
325 | - <tr><th>{{param|capitalize}}:</th><td>{{v}}</td></tr> |
326 | - {% endif %} |
327 | - {% endfor %} |
328 | - <tr><th>Running for:</th><td>{{duration//3600 }}h {{duration % 3600//60}}m {{duration % 60}}s</td></tr> |
329 | - </table> |
330 | - <pre> |
331 | -{{logtail}} |
332 | - </pre> |
333 | - {% endfor %} |
334 | - {% endfor %} |
335 | - {% endfor %} |
336 | + {% for p, info in running.items()|sort %} |
337 | + {{ macros.display_running_job(p, info) }} |
338 | {% endfor %} |
339 | |
340 | <!-- queue contents --> |
341 | diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/macros.html b/charms/focal/autopkgtest-web/webcontrol/templates/macros.html |
342 | new file mode 100644 |
343 | index 0000000..7d43d20 |
344 | --- /dev/null |
345 | +++ b/charms/focal/autopkgtest-web/webcontrol/templates/macros.html |
346 | @@ -0,0 +1,26 @@ |
347 | +{% macro display_running_job(package, info) -%} |
348 | +<h2 id="pkg-{{ package }}"><a href="/packages/{{ package }}">{{ package }}</a></h2> |
349 | + {% for runhash, relinfo in info.items() %} |
350 | + {% for release, archinfo in relinfo.items() %} |
351 | + {% for arch, (params, duration, logtail) in archinfo.items() %} |
352 | + <table class="table-condensed"> |
353 | + <tr><th>Release:</th><td>{{ release }}</td></tr> |
354 | + <tr><th>Architecture:</th><td>{{ arch }}</td></tr> |
355 | + {% for param, v in params.items() %} |
356 | + {% if param == "requester" %} |
357 | + <tr><th>{{ param|capitalize }}:</th><td><a href="https://launchpad.net/~{{ v }}">{{ v }}</a></td></tr> |
358 | + {% elif param == "uuid" %} |
359 | + <tr><th>{{ param|upper }}:</th><td>{{ v }}</td></tr> |
360 | + {% else %} |
361 | + <tr><th>{{ param|capitalize }}:</th><td>{{ v }}</td></tr> |
362 | + {% endif %} |
363 | + {% endfor %} |
364 | + <tr><th>Running for:</th><td>{{ duration//3600 }}h {{ duration % 3600//60 }}m {{ duration % 60 }}s ({{ duration }}s)</td></tr> |
365 | + </table> |
366 | + <pre> |
367 | +{{ logtail }} |
368 | + </pre> |
369 | + {% endfor %} |
370 | + {% endfor %} |
371 | + {% endfor %} |
372 | +{%- endmacro %} |
Looks like I accidentally merged this :)