Merge ~andersson123/qa-jenkins-jobs:daily_image_kpi into qa-jenkins-jobs:master

[Tim Andersson]

80dbcf8... by Tim Andersson on 2023-07-20

idk

Unmerged commits

80dbcf8... by Tim Andersson on 2023-07-20

idk

f6d8ff6... by Tim Andersson on 2023-07-19

feat: Add script and job for gathering metrics on daily image age

diff --git a/jobs/daily_image_age/jobs.yaml b/jobs/daily_image_age/jobs.yaml
new file mode 100644
index 0000000..8983e50
--- /dev/null
+++ b/jobs/daily_image_age/jobs.yaml
@@ -0,0 +1,40 @@
+# vim: sw=4 ts=4 et
+
+# QA Jenkins Jobs
+# Copyright 2016 Canonical Ltd.
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 3, as published
+# by the Free Software Foundation.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranties of
+# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
+# PURPOSE.  See the GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+- job:
+    name: daily-image-kpi-metric-gatherer
+    description: |
+        Runs a script hourly which checks the age of all the daily images
+        at cdimage.ubuntu.com and exposes this data to grafana for a 
+        KPI board.
+    node: 'venonat'
+    triggers:
+        - timed: '@hourly'
+    builders:
+        - clear-artifacts:
+        - shell: |
+            #!/bin/bash
+            set -ex
+            export http_proxy="http://squid.internal:3128"
+            export https_proxy="http://squid.internal:3128"
+            export no_proxy=launchpad.net
+
+            # Perhaps there should be a mechanism which makes sure only venonat can trigger this
+            # I guess it's part of grafana?
+            git clone https://git.launchpad.net/qa-jenkins-jobs
+            cd qa-jenkins-jobs
+            ./scripts/daily_image_age/daily_image_age_metrics
\ No newline at end of file
diff --git a/scripts/daily_image_age/daily_image_age_metrics b/scripts/daily_image_age/daily_image_age_metrics
new file mode 100755
index 0000000..be2defa
--- /dev/null
+++ b/scripts/daily_image_age/daily_image_age_metrics
@@ -0,0 +1,49 @@
+#!/usr/bin/python3
+
+import requests
+import datetime
+import time
+import json
+
+CD_IMAGE_URL = "https://cdimage.ubuntu.com/"
+
+dirs = [ 
+    ["daily-canary"],
+    ["daily-legacy"],
+    ["daily-live"],
+    ["daily-preinstalled"],
+    ["edubuntu", "daily-live"],
+    ["kubuntu", "daily-live"],
+    ["lubuntu", "daily-live"],
+    ["ubuntu-mate", "daily-live"],
+    ["ubuntu-unity", "daily-live"],
+    ["ubuntucinnamon", "daily-live"],
+    ["xubuntu", "daily-live"]
+]
+
+seconds_in_day = 86400
+ages = {}
+
+for d in dirs:
+    url = CD_IMAGE_URL + "/".join(d) + "/" 
+    ages["/".join(d)] = {}
+    index_html = requests.get(url).content.decode("utf-8").splitlines()
+    contents = ""
+    for l in index_html:
+        if '''<img src="/icons/folder.gif" alt="[DIR]"> <a href="current/">current/</a>''' in l:
+            contents = l 
+    epoch_time_of_image = contents[contents.find("</a>")+4:]
+    ep = epoch_time_of_image.split(" ")
+    date_of_iso = [int(i) for i in ep[16].split("-")]
+    time_of_iso = [int(i) for i in ep[17].split(":")]
+    epoch_time_of_image = datetime.datetime(date_of_iso[0],
+                                            date_of_iso[1],
+                                            date_of_iso[2],
+                                            time_of_iso[0],
+                                            time_of_iso[1]).timestamp()
+    age_in_days = (time.time() - epoch_time_of_image) / seconds_in_day
+    ages["/".join(d)]["up-to-date"] = (age_in_days < 1)
+    ages["/".join(d)]["age"] = age_in_days
+
+print(json.dumps(ages, indent=4))
+# need to now expose this data to grafana