Merge ~hyask/autopkgtest-cloud:skia/improve_browsing into autopkgtest-cloud:master

[Skia]

A bit of refactor to bring the feature of displaying currently running jobs on each package's page.

[Tim Andersson (andersson123)]

Looks like I accidentally merged this :)

[Tim Andersson (andersson123)]

still going to review though

[Tim Andersson (andersson123)]

lbjsbootstrap changes to be tested in staging before deployment

[Tim Andersson (andersson123)]

Skia to implement new Exceptions in helpers/exceptions.py for the new get_running_jobs function.

Also implement all exception handler in browse.cgi for extendability.

[Tim Andersson (andersson123)]

LGTM ig

review: Approve

diff --git a/charms/focal/autopkgtest-web/charmcraft.yaml b/charms/focal/autopkgtest-web/charmcraft.yaml
index 93c0816..546437f 100644
--- a/charms/focal/autopkgtest-web/charmcraft.yaml
+++ b/charms/focal/autopkgtest-web/charmcraft.yaml
@@ -6,6 +6,7 @@ parts:
     build-snaps: [charm]
     build-packages:
       - libjs-jquery
+      - libjs-bootstrap
       - python3-dev
 bases:
   - build-on:
diff --git a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
index e1be1a2..6920fc1 100644
--- a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
+++ b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
@@ -338,20 +338,6 @@ def clear_github_status_credentials():
         pass
 
 
-@when_not("autopkgtest-web.bootstrap-symlinked")
-def symlink_bootstrap():
-    try:
-        os.symlink(
-            os.path.join(
-                os.path.sep, "usr", "share", "javascript", "bootstrap"
-            ),
-            os.path.join(charm_dir(), "webcontrol", "static", "bootstrap"),
-        )
-        set_flag("autopkgtest-web.bootstrap-symlinked")
-    except FileExistsError:
-        pass
-
-
 @when_not("autopkgtest-web.runtime-dir-created")
 def make_runtime_tmpfiles():
     with open("/etc/tmpfiles.d/autopkgtest-web-runtime.conf", "w") as r:
diff --git a/charms/focal/autopkgtest-web/webcontrol/browse.cgi b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
index f39b30f..f6be794 100755
--- a/charms/focal/autopkgtest-web/webcontrol/browse.cgi
+++ b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
@@ -10,9 +10,9 @@ import sqlite3
 from collections import OrderedDict
 from wsgiref.handlers import CGIHandler
 
-import distro_info
 import flask
 from helpers.admin import select_abnormally_long_jobs
+from helpers.utils import get_all_releases, get_supported_releases
 from werkzeug.middleware.proxy_fix import ProxyFix
 
 app = flask.Flask("browse")
@@ -20,13 +20,11 @@ app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)
 db_con = None
 swift_container_url = None
 
-UDI = distro_info.UbuntuDistroInfo()
-ALL_UBUNTU_RELEASES = UDI.all
-SUPPORTED_UBUNTU_RELEASES = sorted(
-    set(UDI.supported() + UDI.supported_esm()), key=ALL_UBUNTU_RELEASES.index
-)
-
+ALL_UBUNTU_RELEASES = get_all_releases()
+SUPPORTED_UBUNTU_RELEASES = get_supported_releases()
 INDEXED_PACKAGES_FP = ""
+AMQP_QUEUE_CACHE = "/var/lib/cache-amqp/queued.json"
+RUNNING_CACHE = "/run/amqp-status-collector/running.json"
 
 
 def init_config():
@@ -60,6 +58,16 @@ def get_test_id(release, arch, src):
         return None
 
 
+def get_running_jobs():
+    try:
+        with open(RUNNING_CACHE) as f:
+            # package -> runhash -> release -> arch -> (params, duration, logtail)
+            running_info = json.load(f)
+    except FileNotFoundError:
+        running_info = {}
+    return running_info
+
+
 def render(template, code=200, **kwargs):
     # sort the values passed in, so that releases are in the right order
     try:
@@ -81,7 +89,7 @@ def render(template, code=200, **kwargs):
         flask.render_template(
             template,
             base_url=flask.url_for("index_root"),
-            static_url=flask.url_for("static", filename="/"),
+            static_url=flask.url_for("static", filename=""),
             **kwargs
         ),
         code,
@@ -147,7 +155,7 @@ def get_queue_info():
     Return (releases, arches, context -> release -> arch -> (queue_size, [requests])).
     """
 
-    with open("/var/lib/cache-amqp/queued.json", "r") as json_file:
+    with open(AMQP_QUEUE_CACHE, "r") as json_file:
         queue_info_j = json.load(json_file)
 
         arches = queue_info_j["arches"]
@@ -268,6 +276,10 @@ def package_overview(package, _=None):
         arches.add(row[3])
         results.setdefault(row[2], {})[row[3]] = human_exitcode(row[1])
 
+    running_info = dict(
+        (k, v) for (k, v) in get_running_jobs().items() if k == package
+    )
+
     return render(
         "browse-package.html",
         package=package,
@@ -280,6 +292,7 @@ def package_overview(package, _=None):
         arches=sorted(arches),
         results=results,
         title_suffix="- %s" % package,
+        running=running_info,
     )
 
 
@@ -371,12 +384,7 @@ def running():
                     a
                 ] = queue_length
 
-    try:
-        with open("/run/amqp-status-collector/running.json") as f:
-            # package -> runhash -> release -> arch -> (params, duration, logtail)
-            running_info = json.load(f)
-    except FileNotFoundError:
-        running_info = {}
+    running_info = get_running_jobs()
 
     return render(
         "browse-running.html",
@@ -391,13 +399,7 @@ def running():
 
 @app.route("/admin")
 def admin():
-    try:
-        with open("/run/amqp-status-collector/running.json") as f:
-            # package -> runhash -> release -> arch -> (params, duration, logtail)
-            running_info = json.load(f)
-    except FileNotFoundError as exc:
-        running_info = {}
-        raise FileNotFoundError("running.json doesn't exist!") from exc
+    running_info = get_running_jobs()
     pruned_running_info = select_abnormally_long_jobs(
         running_info, get_test_id=get_test_id, db_con=db_con
     )
@@ -439,7 +441,7 @@ def queues_json():
 
 @app.route("/queued.json")
 def return_queued_exactly():
-    with open("/var/lib/cache-amqp/queued.json") as json_file:
+    with open(AMQP_QUEUE_CACHE) as json_file:
         queue_info = json.load(json_file)
     return queue_info
 
diff --git a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
index 58a9514..12d93b5 100644
--- a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
+++ b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
@@ -8,6 +8,22 @@ import random
 import sqlite3
 import time
 
+import distro_info
+
+
+def get_all_releases():
+    udi = distro_info.UbuntuDistroInfo()
+    return udi.all
+
+
+def get_supported_releases():
+    udi = distro_info.UbuntuDistroInfo()
+    all_ubuntu_releases = get_all_releases()
+    return sorted(
+        set(udi.supported() + udi.supported_esm()),
+        key=all_ubuntu_releases.index,
+    )
+
 
 def setup_key(app, path):
     """Create or load app.secret_key for cookie encryption."""
@@ -22,10 +38,10 @@ def setup_key(app, path):
         app.secret_key = key
 
 
-def init_db(path):
+def init_db(path, **kwargs):
     """Create DB if it does not exist, and connect to it"""
 
-    db = sqlite3.connect(path)
+    db = sqlite3.connect(path, **kwargs)
     c = db.cursor()
     try:
         c.execute("PRAGMA journal_mode = WAL")
diff --git a/charms/focal/autopkgtest-web/webcontrol/static/bootstrap b/charms/focal/autopkgtest-web/webcontrol/static/bootstrap
 new file mode 120000
index 0000000..fe0f86b
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/static/bootstrap
@@ -0,0 +1 @@
+/usr/share/javascript/bootstrap
 \ No newline at end of file
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-admin.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-admin.html
index 5bc2459..266f584 100644
--- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-admin.html
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-admin.html
@@ -1,29 +1,14 @@
 {% extends "browse-layout.html" %}
+{% import "macros.html" as macros %}
+
 {% block content %}
   <h1 class="page-header">Admin</h1>
   <p>Click on the package name to jump to the tests of the package for all arches/releases.</p>
   <p>This page is simply a bunch of heuristics filtering all running jobs to try to get the problematic ones. Feel free to come help improve the heuristics <a href="https://code.launchpad.net/~ubuntu-release/autopkgtest-cloud/+git/autopkgtest-cloud/+ref/master">here.</a></p>
 
   <!-- Running tests -->
-  {% for p in running|sort %}
-  <h2 id="pkg-{{p}}"><a href="/packages/{{p}}">{{p}}</a></h2>
-    {% for runhash, relinfo in running[p].items() %}
-      {% for release, archinfo in relinfo.items() %}
-        {% for arch, (params, duration, logtail) in archinfo.items() %}
-          <table class="table-condensed">
-            <tr><th>Release:</th><td>{{release}}</td></tr>
-            <tr><th>Architecture:</th><td>{{arch}}</td></tr>
-            {% for param, v in params.items() %}
-            <tr><th>{{param|capitalize}}:</th><td>{{v}}</td></tr>
-            {% endfor %}
-            <tr><th>Running for:</th><td>{{duration//3600 }}h {{duration % 3600//60}}m {{duration % 60}}s</td></tr>
-         </table>
-        <pre>
-{{logtail}}
-        </pre>
-        {% endfor %}
-      {% endfor %}
-    {% endfor %}
+  {% for p, info in running.items()|sort %}
+    {{ macros.display_running_job(p, info) }}
   {% endfor %}
 
 {% endblock %}
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-layout.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-layout.html
index 0f90de3..c91457d 100644
--- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-layout.html
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-layout.html
@@ -6,9 +6,9 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>Ubuntu Autopkgtest Results {{title_suffix}}</title>
     <!-- <link rel="icon" type="image/png" href="/debian.png"/> -->
-    <link rel="stylesheet" type="text/css" href="{{static_url}}/bootstrap/css/bootstrap.css"/>
-    <link rel="stylesheet" type="text/css" href="{{static_url}}/bootstrap/css/bootstrap-theme.css"/>
-    <link rel="stylesheet" type="text/css" href="{{static_url}}/style.css"/>
+    <link rel="stylesheet" type="text/css" href="{{static_url}}bootstrap/css/bootstrap.css"/>
+    <link rel="stylesheet" type="text/css" href="{{static_url}}bootstrap/css/bootstrap-theme.css"/>
+    <link rel="stylesheet" type="text/css" href="{{static_url}}style.css"/>
   </head>
   <body>
     <div id='wrap'>
@@ -44,7 +44,7 @@
 {% block content %}{% endblock %}
 </div>
 
-    <script type="text/javascript" src="{{static_url}}/jquery/jquery.min.js"></script>
-    <script type="text/javascript" src="{{static_url}}/bootstrap/js/bootstrap.min.js"></script>
+    <script type="text/javascript" src="{{static_url}}jquery/jquery.min.js"></script>
+    <script type="text/javascript" src="{{static_url}}bootstrap/js/bootstrap.min.js"></script>
   </body>
 </html>
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-package.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-package.html
index 3ac81c3..fb12afa 100644
--- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-package.html
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-package.html
@@ -1,4 +1,6 @@
 {% extends "browse-layout.html" %}
+{% import "macros.html" as macros %}
+
 {% block content %}
   <h2>{{package}}</h2>
 
@@ -17,4 +19,8 @@
       </tr>
     {% endfor %}
   </table>
+
+  {% for p, info in running.items()|sort %}
+    {{ macros.display_running_job(p, info) }}
+  {% endfor %}
 {% endblock %}
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html b/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
index 3711b97..53ae2f9 100644
--- a/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/browse-running.html
@@ -1,4 +1,6 @@
 {% extends "browse-layout.html" %}
+{% import "macros.html" as macros %}
+
 {% block content %}
   <h1 class="page-header">Currently running tests</h1>
   <p>Click on the package name to jump to the currently running tests of that package.</p>
@@ -36,31 +38,8 @@
   {% endfor %}
 
   <!-- Running tests -->
-  {% for p in running|sort %}
-  <h2 id="pkg-{{p}}"><a href="/packages/{{p}}">{{p}}</a></h2>
-    {% for runhash, relinfo in running[p].items() %}
-      {% for release, archinfo in relinfo.items() %}
-        {% for arch, (params, duration, logtail) in archinfo.items() %}
-          <table class="table-condensed">
-            <tr><th>Release:</th><td>{{release}}</td></tr>
-            <tr><th>Architecture:</th><td>{{arch}}</td></tr>
-            {% for param, v in params.items() %}
-              {% if param == "requester" %}
-            <tr><th>{{param|capitalize}}:</th><td><a href="https://launchpad.net/~{{v}}">{{v}}</a></td></tr>
-              {% elif param == "uuid" %}
-            <tr><th>{{param|capitalize}}:</th><td>{{v}}</td></tr>
-              {% else %}
-            <tr><th>{{param|capitalize}}:</th><td>{{v}}</td></tr>
-              {% endif %}
-            {% endfor %}
-            <tr><th>Running for:</th><td>{{duration//3600 }}h {{duration % 3600//60}}m {{duration % 60}}s</td></tr>
-         </table>
-        <pre>
-{{logtail}}
-        </pre>
-        {% endfor %}
-      {% endfor %}
-    {% endfor %}
+  {% for p, info in running.items()|sort %}
+    {{ macros.display_running_job(p, info) }}
   {% endfor %}
 
   <!-- queue contents -->
diff --git a/charms/focal/autopkgtest-web/webcontrol/templates/macros.html b/charms/focal/autopkgtest-web/webcontrol/templates/macros.html
 new file mode 100644
index 0000000..7d43d20
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/templates/macros.html
@@ -0,0 +1,26 @@
+{% macro display_running_job(package, info) -%}
+<h2 id="pkg-{{ package }}"><a href="/packages/{{ package }}">{{ package }}</a></h2>
+  {% for runhash, relinfo in info.items() %}
+    {% for release, archinfo in relinfo.items() %}
+      {% for arch, (params, duration, logtail) in archinfo.items() %}
+        <table class="table-condensed">
+          <tr><th>Release:</th><td>{{ release }}</td></tr>
+          <tr><th>Architecture:</th><td>{{ arch }}</td></tr>
+          {% for param, v in params.items() %}
+            {% if param == "requester" %}
+            <tr><th>{{ param|capitalize }}:</th><td><a href="https://launchpad.net/~{{ v }}">{{ v }}</a></td></tr>
+              {% elif param == "uuid" %}
+            <tr><th>{{ param|upper }}:</th><td>{{ v }}</td></tr>
+              {% else %}
+            <tr><th>{{ param|capitalize }}:</th><td>{{ v }}</td></tr>
+              {% endif %}
+          {% endfor %}
+          <tr><th>Running for:</th><td>{{ duration//3600 }}h {{ duration % 3600//60 }}m {{ duration % 60 }}s ({{ duration }}s)</td></tr>
+       </table>
+      <pre>
+{{ logtail }}
+      </pre>
+      {% endfor %}
+    {% endfor %}
+  {% endfor %}
+{%- endmacro %}