Merge lp:~hloeung/ubuntu-repository-cache/juju-run-metrics into lp:ubuntu-repository-cache

[Haw Loeung]

Add metric for juju-run failures

This is in preparation for updating the charm with retry logic.

First, we want metrics to see the frequency of failures and if it's just down to problematic regions.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[James Simpson (jsimpso)]

LGTM

review: Approve (+1)

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 361

=== modified file 'lib/ubuntu_repository_cache/metadata_sync.py'
--- lib/ubuntu_repository_cache/metadata_sync.py	2022-07-22 06:33:43 +0000
+++ lib/ubuntu_repository_cache/metadata_sync.py	2022-07-28 00:39:07 +0000
@@ -8,6 +8,7 @@
 import pwd
 import shutil
 import socket
+import subprocess
 import time
 import traceback
 
@@ -15,8 +16,6 @@
 from datetime import datetime
 from itertools import cycle
 
-from subprocess import CalledProcessError, check_call, check_output, DEVNULL, STDOUT
-
 from .util import (
     FileSemaphore,
     REPO_SYNC_CRON_LOG,
@@ -70,7 +69,7 @@
 # around subprocess - ~tribaal
 
 
-def rsync(from_path, to_path, options, log, flags="-r", check_output=check_output, user=None, timeout=None):
+def rsync(from_path, to_path, options, log, flags="-r", check_output=subprocess.check_output, user=None, timeout=None):
     """
     A wrapper around subprocess to execute an rsync between a source and a
     destination.
@@ -93,10 +92,10 @@
     cmd.append(to_path)
     log(" ".join(cmd))
 
-    return check_output(cmd, stderr=STDOUT).decode('UTF-8').strip()
-
-
-def symlink(source, destination, log=None, check_call=check_call):
+    return check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip()
+
+
+def symlink(source, destination, log=None, check_call=subprocess.check_call):
     """
     A wrapper around subprocess to create a symlink between the source and the
     destination name.
@@ -258,7 +257,7 @@
     return os.path.basename(dest)
 
 
-def main(environment, log, mirror_archive=mirror_archive, check_call=check_call):
+def main(environment, log, mirror_archive=mirror_archive, check_call=subprocess.check_call):
     """The main function for this script.
 
     This is called at the very bottom of this file should the file be executed
@@ -293,7 +292,12 @@
 
     log("Executing hook: {}".format(cmd))
 
-    check_call(cmd, stdout=DEVNULL)
+    try:
+        check_call(cmd, stdout=subprocess.DEVNULL)
+    except subprocess.CalledProcessError:
+        metric_name = 'ubuntu_repository_cache_metadata_sync_failures'
+        send_to_influx(render_influx(metric_name, '', 1))
+        raise
 
     return meta_ver
 
@@ -377,7 +381,9 @@
             log("Keeping {}".format(entry))
 
 
[e-mail]_exception(backoff.expo, CalledProcessError, max_tries=RSYNC_MAX_RETRIES, giveup=rsync_should_giveup)
[e-mail]_exception(
+    backoff.expo, subprocess.CalledProcessError, max_tries=RSYNC_MAX_RETRIES, giveup=rsync_should_giveup
+)
 def _mirror_metadata(archive_ip, dest, link_dest, log, mirror_series, rsync_module='ubuntu'):
     '''Mirror metadata from the archive.'''
 
@@ -415,7 +421,9 @@
     rsync(rsync_source, dest, rsync_options, log, user='www-sync', timeout=4800)
 
 
[e-mail]_exception(backoff.expo, CalledProcessError, max_tries=RSYNC_MAX_RETRIES, giveup=rsync_should_giveup)
[e-mail]_exception(
+    backoff.expo, subprocess.CalledProcessError, max_tries=RSYNC_MAX_RETRIES, giveup=rsync_should_giveup
+)
 @backoff.on_exception(backoff.expo, UpstreamMirrorSyncError, max_tries=UPSTREAM_SYNC_MAX_RETRIES, factor=60)
 def _mirror_base(apache_root, archive_ip, dest, log=None, rsync_module='ubuntu'):
     """Mirror just the root of the archive.
 
=== modified file 'wheelhouse.txt'
--- wheelhouse.txt	2022-07-25 00:15:09 +0000
+++ wheelhouse.txt	2022-07-28 00:39:07 +0000
@@ -1,4 +1,5 @@
 # Include python requirements here
-backoff<=1.7.0
+# Drop to 1.6.0 which dramatically reduces build time (while still supporting Xenial)
+backoff==1.6.0
 # Xenial & py35 support - https://github.com/juju-solutions/layer-basic/pull/197
 MarkupSafe<2.0.0