Merge lp:~tcuthbert/ubuntu-repository-cache/squid-mem-qos into lp:ubuntu-repository-cache

[Thomas Cuthbert]

[Haw Loeung (hloeung)]

Comments inline. I tested and confirmed memory limits are applied - https://pastebin.canonical.com/p/3Jh3mSBJfV/

[Haw Loeung (hloeung)]

FWIW, this supersedes https://code.launchpad.net/~tcuthbert/ubuntu-repository-cache/squid-mem-max/+merge/423754

[Haw Loeung (hloeung)]

... and superseded yet again - https://code.launchpad.net/~tcuthbert/ubuntu-repository-cache/+git/ubuntu-repository-cache/+merge/429926

Unmerged revisions

374. By Thomas Cuthbert on 2022-09-06

WIP: squid systemd memory qos

=== modified file 'config.yaml'
--- config.yaml	2022-08-22 02:45:49 +0000
+++ config.yaml	2022-09-06 23:31:29 +0000
@@ -76,6 +76,24 @@
       effect running units, only newly added units.  An example would be
       '/dev/xvdb,/dev/xvdc' to specify two ephemeral disks for cache
       storage.
+  squid-memory-high:
+    default: 90%
+    type: string
+    description: |
+      Configurable systemd.resource-control for high squid memory. Takes a
+      memory size in either bytes with an optional suffix or a percentage
+      relative to physical system memory. The series must be xenial or
+      higher for this to take effect.
+      See man systemd.resource-control for details.
+  squid-memory-max:
+    default: 95%
+    type: string
+    description: |
+      Configurable systemd.resource-control for maximum squid memory. Takes a
+      memory size in either bytes with an optional suffix or a percentage
+      relative to physical system memory. The series must be xenial or
+      higher for this to take effect.
+      See man systemd.resource-control for details.
   cache-memory-size:
     default: 0
     type: int
 
=== modified file 'lib/ubuntu_repository_cache/service.py'
--- lib/ubuntu_repository_cache/service.py	2022-08-24 04:03:17 +0000
+++ lib/ubuntu_repository_cache/service.py	2022-09-06 23:31:29 +0000
@@ -190,6 +190,7 @@
     """
 
     return {
+        ('/etc/systemd/system/squid-deb-proxy.service.d/', 'squid-deb-proxy.conf'): ['squid-deb-proxy'],
         '/etc/squid-deb-proxy/squid-deb-proxy.conf': ['squid-deb-proxy'],
         '/etc/squid-deb-proxy/allowed-networks-src.acl': ['squid-deb-proxy'],
         ('/etc/squid-deb-proxy/mirror-dstdomain.acl.d/' '99-ubuntu-repository-cache'): ['squid-deb-proxy'],
 
=== modified file 'lib/ubuntu_repository_cache/squid.py'
--- lib/ubuntu_repository_cache/squid.py	2022-08-22 02:49:47 +0000
+++ lib/ubuntu_repository_cache/squid.py	2022-09-06 23:31:29 +0000
@@ -1,8 +1,11 @@
 '''Functions related to squid-deb-proxy use within this charm'''
 
+import filecmp
 import os
+import re
+import shutil
 import subprocess
-import re
+import tempfile
 
 from charmhelpers import fetch
 
@@ -80,6 +83,68 @@
     )
 
 
[e-mail]_as_user('root')
+def configure_systemd_override():
+    """Configure squid-deb-proxy with systemd-resource controls.
+
+    1. shallow[^1] compare config changes from tempdir
+    2. safely[^2] move configs into /etc/systemd/system
+       - cleanup any leftover files from /etc/systemd on error
+    3. call `systemctl daemon-reload`, handling OS related errors where possible
+
+    [^1]: https://docs.python.org/3/library/filecmp.html#the-dircmp-class
+    [^2]: https://docs.python.org/3/library/shutil.html#shutil.move
+    """
+    context = {}
+    config = hookenv.config()
+
+    LOG("Installing squid-deb-proxy systemd service override configuration", hookenv.INFO)
+    context["squid_memory_high"] = config.get("squid-memory-max", "90%")
+    context["squid_memory_max"] = config.get("squid-memory-max", "95%")
+
+    dirname = "/etc/systemd/system/squid-deb-proxy.service.d"
+    configdir_exists = os.path.exists(dirname)
+    cleanup_files = []
+
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        charm_squid_path = os.path.join(os.getcwd(), "templates/squid-deb-proxy.service.d")
+        temp_squid_path = os.path.join(tmpdirname, "templates/squid-deb-proxy.service.d")
+
+        shutil.copytree(charm_squid_path, temp_squid_path)
+        templating.render(
+            "squid-deb-proxy.service.d/" + "squid-deb-proxy.conf",
+            os.path.join(temp_squid_path, "squid-deb-proxy.conf"),
+            context,
+        )
+
+        config_files = [os.path.join(dirname, f) for f in os.listdir(temp_squid_path)]
+        if configdir_exists:
+            config_files = [os.path.join(dirname, f) for f in filecmp.dircmp(temp_squid_path, dirname).diff_files]
+
+        try:
+            shutil.move(temp_squid_path, "/etc/systemd/system/")
+            verified = util.systemd_verify("squid-deb-proxy.service")
+
+        except Exception as e:
+            cleanup_files.extend(config_files)
+            err_msg = "Installing squid-deb-proxy.service.d files failed: {0}, {err}".format(dirname, err=e)
+            LOG(err_msg, hookenv.ERROR)
+            raise e from None
+
+        else:
+            LOG("Reloading systemd daemons", hookenv.INFO)
+            with util.systemctl_daemon_reload() as (reloaded, stderr):
+                if reloaded and verified:
+                    LOG("squid-deb-proxy.service.d overrides reconfigured, restarting squid", hookenv.INFO)
+                    host.service_restart('squid-deb-proxy')
+                else:
+                    LOG("squid-deb-proxy.service.d in an unexpected state: {}".format(stderr), hookenv.ERROR)
+
+        finally:
+            for f in (f for f in cleanup_files):
+                os.remove(f)
+
+
 def install():
     '''Perform the install/config and disable avahi'''
 
@@ -107,6 +172,7 @@
     stop()
 
     render_configs()
+    configure_systemd_override()
     init_caches()
 
 
@@ -246,6 +312,8 @@
         dstdomain_context,
     )
 
+    configure_systemd_override()
+
 
 def update_checks(nrpe_config):
     '''Update Nagios checks for squid serving archive pool content'''
 
=== modified file 'lib/ubuntu_repository_cache/util.py'
--- lib/ubuntu_repository_cache/util.py	2022-08-29 02:29:43 +0000
+++ lib/ubuntu_repository_cache/util.py	2022-09-06 23:31:29 +0000
@@ -7,6 +7,8 @@
 import socket
 import subprocess
 
+from contextlib import contextmanager
+
 from charmhelpers.contrib.hahelpers import cluster
 from charmhelpers.core import (
     hookenv,
@@ -167,6 +169,56 @@
     return int(output[1].split()[1])
 
 
+def has_systemd():
+    release = host.lsb_release()['DISTRIB_RELEASE']
+    rv = True
+    if release < '16.04':  # before xenial
+        rv = False
+    return rv
+
+
+def systemd_verify(unit_file, timeout=5):
+    args = ["systemd-analyze", "verify", "--system", "--generators", unit_file]
+    ok = 0
+    try:
+        return subprocess.check_call(args, timeout=timeout) == ok
+    except subprocess.CalledProcessError as cpe:
+        LOG("failed to verify systemd override configuration: {}".format(cpe), hookenv.ERROR)
+        raise cpe from None
+    except subprocess.TimeoutExpired as te:
+        LOG("timed out verifying systemd override files: {}".format(te), hookenv.ERROR)
+        raise te from None
+
+
+@contextmanager
+def systemctl_daemon_reload(timeout=5):
+    """
+    contextmanager around the systemctl daemon-reload command.
+
+    Defaults with a 5s timeout that will send SIGTERM on expiry.
+    Yields true and stderr if the process returned 0.
+    Attempts to cleanup the process with SIGKILL[^1] if it's wedged.
+
+    [^1]: https://docs.python.org/3/library/subprocess.html#subprocess.Popen.returncode
+    """
+    cmd = ["/bin/systemctl", "daemon-reload"]
+    proc = subprocess.Popen(cmd, stderr=subprocess.PIPE)
+    stderr = None
+    ok = 0
+    try:
+        _, stderr = proc.communicate(timeout=timeout)
+        yield proc.returncode == ok, stderr
+    except subprocess.TimeoutExpired:
+        LOG("Timeout reloading squid-deb-proxy systemd override config", hookenv.ERROR)
+    finally:
+        proc.terminate()
+        try:
+            proc.wait(timeout=2)
+        except subprocess.TimeoutExpired:
+            LOG("systemctl daemon-reload did not terminate, attempting SIGKILL", hookenv.ERROR)
+            proc.kill()
+
+
 def change_aptsources_url(url='http://archive.ubuntu.com/ubuntu'):
     """Change the URL for the current apt sources
 
 
=== added directory 'templates/squid-deb-proxy.service.d'
=== added file 'templates/squid-deb-proxy.service.d/squid-deb-proxy.conf'
--- templates/squid-deb-proxy.service.d/squid-deb-proxy.conf	1970-01-01 00:00:00 +0000
+++ templates/squid-deb-proxy.service.d/squid-deb-proxy.conf	2022-09-06 23:31:29 +0000
@@ -0,0 +1,8 @@
+[Unit]
+StartLimitIntervalSec=3m
+StartLimitBurst=3
+
+[Service]
+MemoryHigh={{ squid_memory_high }}
+MemoryMax={{ squid_memory_max }}
+Restart=on-failure