Merge ~hloeung/content-cache-charm:sysctl into content-cache-charm:master

[Haw Loeung]

Add tuning of net.core.somaxconn

`net.core.somaxconn 4096` is the default as of Linux 5.4.

See also https://code.launchpad.net/~cjwatson/launchpad-mojo-specs/+git/launchpad-mojo-specs/+merge/453051

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Barry Price (barryprice)]

Couple of questions inline about looseversion, LGTM otherwise!

review: Needs Information

[Barry Price (barryprice)]

Answered offline, distutils is included by default on <3.12, looseversion will be included via wheelhouse on 3.12+, so we're all good here. +1 from me.

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision ede1bda2d772b2b0e6dbaab4be3881ff0cae84db

diff --git a/lib/haproxy.py b/lib/haproxy.py
index 8842982..8387382 100644
--- a/lib/haproxy.py
+++ b/lib/haproxy.py
@@ -6,7 +6,11 @@ import subprocess
 import socket
 
 import jinja2
-from distutils.version import LooseVersion
+
+try:
+    from distutils.version import LooseVersion
+except ImportError:
+    from looseversion import LooseVersion
 
 from lib import utils
 
diff --git a/reactive/content_cache.py b/reactive/content_cache.py
index bc51c18..44fe1f3 100644
--- a/reactive/content_cache.py
+++ b/reactive/content_cache.py
@@ -518,6 +518,7 @@ def configure_nagios():
 
 
 _SYSCTL_CORE_DEFAULT_QDISC = '/proc/sys/net/core/default_qdisc'
+_SYSCTL_CORE_SOMAXCONN = '/proc/sys/net/core/somaxconn'
 _SYSCTL_NETFILTER_CONNTRACK_MAX = '/proc/sys/net/nf_conntrack_max'
 
 
@@ -527,6 +528,7 @@ def configure_sysctl():
 
     context = {
         'net_core_default_qdisc': None,
+        'net_core_somax_conn': None,
         'net_ipv4_tcp_congestion_control': None,
     }
 
@@ -544,6 +546,13 @@ def configure_sysctl():
     # https://blog.cloudflare.com/http-2-prioritization-with-nginx/
     context['net_ipv4_tcp_notsent_lowat'] = '16384'
 
+    # Bump, 4096 is the default as of Linux 5.4. Apply same tuning for earlier.
+    somaxconn = 4096
+    if os.path.exists(_SYSCTL_CORE_SOMAXCONN):
+        with open(_SYSCTL_CORE_SOMAXCONN, 'r', encoding='utf-8') as f:
+            if int(f.readline()) < somaxconn:
+                context['net_core_somax_conn'] = somaxconn
+
     base = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
     env = jinja2.Environment(loader=jinja2.FileSystemLoader(base))
     template = env.get_template('templates/sysctl_conf.tmpl')
diff --git a/templates/sysctl_conf.tmpl b/templates/sysctl_conf.tmpl
index 316a34d..f295ad1 100644
--- a/templates/sysctl_conf.tmpl
+++ b/templates/sysctl_conf.tmpl
@@ -8,6 +8,9 @@ net.ipv4.tcp_notsent_lowat = {{ net_ipv4_tcp_notsent_lowat }}
 {% if net_core_default_qdisc -%}
 net.core.default_qdisc = {{ net_core_default_qdisc }}
 {% endif -%}
+{% if net_core_somax_conn -%}
+net.core.somaxconn = {{ net_core_somax_conn }}
+{% endif -%}
 {% if net_ipv4_tcp_mem %}
 # Tune TCP memory
 net.ipv4.tcp_mem = {{ net_ipv4_tcp_mem }}
diff --git a/tests/unit/files/sysctl_core_default_somaxconn_none.conf b/tests/unit/files/sysctl_core_default_somaxconn_none.conf
new file mode 100644
index 0000000..0509bb2
--- /dev/null
+++ b/tests/unit/files/sysctl_core_default_somaxconn_none.conf
@@ -0,0 +1,22 @@
+net.ipv4.tcp_notsent_lowat = 16384
+net.core.default_qdisc = fq
+
+
+# Tune the TCP stack for Fast Failure Detection
+# tcp_syn_retries set to 1, it takes ~9secs to timeout
+# tcp_syn_retries set to 2, it takes ~21secs to timeout
+# tcp_syn_retries set to 3, it takes ~45secs to timeout
+# tcp_syn_retries set to 4, it takes ~93secs to timeout
+# tcp_syn_retries set to 5, it takes ~189secs to timeout (default)
+# Reduce TCP connection timeouts to ~45 secs.
+net.ipv4.tcp_syn_retries = 3
+net.ipv4.tcp_synack_retries = 3
+
+# Reduce TCP KeepAlive timeout from 2 hrs to 30 mins.
+net.ipv4.tcp_keepalive_time = 1800
+
+# Extend the source port range for outgoing TCP connections.
+net.ipv4.ip_local_port_range = 16384 61000
+
+# Allow early reuse of a same source port for outgoing connections.
+net.ipv4.tcp_tw_reuse = 1
diff --git a/tests/unit/files/sysctl_core_somaxconn.conf b/tests/unit/files/sysctl_core_somaxconn.conf
new file mode 100644
index 0000000..0ecfae9
--- /dev/null
+++ b/tests/unit/files/sysctl_core_somaxconn.conf
@@ -0,0 +1,23 @@
+net.ipv4.tcp_notsent_lowat = 16384
+net.core.default_qdisc = fq
+net.core.somaxconn = 4096
+
+
+# Tune the TCP stack for Fast Failure Detection
+# tcp_syn_retries set to 1, it takes ~9secs to timeout
+# tcp_syn_retries set to 2, it takes ~21secs to timeout
+# tcp_syn_retries set to 3, it takes ~45secs to timeout
+# tcp_syn_retries set to 4, it takes ~93secs to timeout
+# tcp_syn_retries set to 5, it takes ~189secs to timeout (default)
+# Reduce TCP connection timeouts to ~45 secs.
+net.ipv4.tcp_syn_retries = 3
+net.ipv4.tcp_synack_retries = 3
+
+# Reduce TCP KeepAlive timeout from 2 hrs to 30 mins.
+net.ipv4.tcp_keepalive_time = 1800
+
+# Extend the source port range for outgoing TCP connections.
+net.ipv4.ip_local_port_range = 16384 61000
+
+# Allow early reuse of a same source port for outgoing connections.
+net.ipv4.tcp_tw_reuse = 1
diff --git a/tests/unit/requirements.txt b/tests/unit/requirements.txt
index 66d1f72..05e9abd 100644
--- a/tests/unit/requirements.txt
+++ b/tests/unit/requirements.txt
@@ -1,6 +1,7 @@
 charmhelpers
 charms.reactive
 freezegun
+looseversion
 mock
 psutil
 pytest
diff --git a/tests/unit/test_content_cache.py b/tests/unit/test_content_cache.py
index f12286e..fe1ae18 100644
--- a/tests/unit/test_content_cache.py
+++ b/tests/unit/test_content_cache.py
@@ -1472,10 +1472,15 @@ site1.local:
         tcp_cc.return_value = None
         process_rlimits.return_value = '1048777'
 
+        somaxconn_path = os.path.join(self.tmpdir, 'somaxconn')
+        with open(somaxconn_path, 'w', encoding='utf-8') as f:
+            f.write('4096\n')
+
         with mock.patch.multiple(
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC='some-file-does-not-exist',
+            _SYSCTL_CORE_SOMAXCONN=somaxconn_path,
             _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
@@ -1513,10 +1518,15 @@ site1.local:
         qdisc_path = '/proc/uptime'
         conntrack_max_path = '/proc/uptime'
 
+        somaxconn_path = os.path.join(self.tmpdir, 'somaxconn')
+        with open(somaxconn_path, 'w', encoding='utf-8') as f:
+            f.write('4096\n')
+
         with mock.patch.multiple(
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_CORE_SOMAXCONN=somaxconn_path,
             _SYSCTL_NETFILTER_CONNTRACK_MAX=conntrack_max_path,
         ):
             content_cache.configure_sysctl()
@@ -1538,12 +1548,17 @@ site1.local:
         tcp_cc.return_value = None
         process_rlimits.return_value = '1048777'
 
+        somaxconn_path = os.path.join(self.tmpdir, 'somaxconn')
+        with open(somaxconn_path, 'w', encoding='utf-8') as f:
+            f.write('4096\n')
+
         # Use '/proc/uptime' for unit test as that will always exist.
         qdisc_path = '/proc/uptime'
         with mock.patch.multiple(
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_CORE_SOMAXCONN=somaxconn_path,
             _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
@@ -1558,6 +1573,7 @@ site1.local:
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_CORE_SOMAXCONN=somaxconn_path,
             _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
@@ -1572,6 +1588,53 @@ site1.local:
     @mock.patch('lib.utils.process_rlimits')
     @mock.patch('lib.utils.select_tcp_congestion_control')
     @mock.patch('lib.utils.tune_tcp_mem')
+    def test_configure_sysctl_somaxconn(self, tune_tcp_mem, tcp_cc, process_rlimits, call, set_flag):
+        sysctl_conf_path = os.path.join(self.tmpdir, '90-content-cache.conf')
+        self.mock_config.return_value = {'tune_tcp_mem_multiplier': 1.5}
+        tune_tcp_mem.return_value = None
+        tcp_cc.return_value = None
+        process_rlimits.return_value = '1048777'
+        # Use '/proc/uptime' for unit test as that will always exist.
+        qdisc_path = '/proc/uptime'
+
+        somaxconn_path = os.path.join(self.tmpdir, 'somaxconn')
+        with open(somaxconn_path, 'w', encoding='utf-8') as f:
+            f.write('1024\n')
+
+        with mock.patch.multiple(
+            'reactive.content_cache',
+            SYSCTL_CONF_PATH=sysctl_conf_path,
+            _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_CORE_SOMAXCONN=somaxconn_path,
+            _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
+        ):
+            content_cache.configure_sysctl()
+            with open('tests/unit/files/sysctl_core_somaxconn.conf', 'r') as f:
+                want = f.read()
+            with open(sysctl_conf_path, 'r') as f:
+                got = f.read()
+            self.assertEqual(got, want)
+
+        somaxconn_path = 'some-file-does-not-exist'
+        with mock.patch.multiple(
+            'reactive.content_cache',
+            SYSCTL_CONF_PATH=sysctl_conf_path,
+            _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_CORE_SOMAXCONN=somaxconn_path,
+            _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
+        ):
+            content_cache.configure_sysctl()
+            with open('tests/unit/files/sysctl_core_default_somaxconn_none.conf', 'r') as f:
+                want = f.read()
+            with open(sysctl_conf_path, 'r') as f:
+                got = f.read()
+            self.assertEqual(got, want)
+
+    @mock.patch('charms.reactive.set_flag')
+    @mock.patch('subprocess.call')
+    @mock.patch('lib.utils.process_rlimits')
+    @mock.patch('lib.utils.select_tcp_congestion_control')
+    @mock.patch('lib.utils.tune_tcp_mem')
     def test_configure_sysctl_nf_conntrack_max(self, tune_tcp_mem, tcp_cc, process_rlimits, call, set_flag):
         sysctl_conf_path = os.path.join(self.tmpdir, '90-content-cache.conf')
         self.mock_config.return_value = {
@@ -1584,10 +1647,15 @@ site1.local:
         # Use '/proc/uptime' for unit test as that will always exist.
         conntrack_max_path = '/proc/uptime'
 
+        somaxconn_path = os.path.join(self.tmpdir, 'somaxconn')
+        with open(somaxconn_path, 'w', encoding='utf-8') as f:
+            f.write('4096\n')
+
         with mock.patch.multiple(
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC='some-file-does-not-exist',
+            _SYSCTL_CORE_SOMAXCONN=somaxconn_path,
             _SYSCTL_NETFILTER_CONNTRACK_MAX=conntrack_max_path,
         ):
             content_cache.configure_sysctl()
@@ -1605,6 +1673,7 @@ site1.local:
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC='some-file-does-not-exist',
+            _SYSCTL_CORE_SOMAXCONN=somaxconn_path,
             _SYSCTL_NETFILTER_CONNTRACK_MAX=conntrack_max_path,
         ):
             content_cache.configure_sysctl()
@@ -1626,11 +1695,16 @@ site1.local:
         process_rlimits.return_value = '1048777'
         qdisc_path = 'some-file-does-not-exist'
 
+        somaxconn_path = os.path.join(self.tmpdir, 'somaxconn')
+        with open(somaxconn_path, 'w', encoding='utf-8') as f:
+            f.write('4096\n')
+
         tcp_cc.return_value = 'bbr'
         with mock.patch.multiple(
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_CORE_SOMAXCONN=somaxconn_path,
             _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
@@ -1645,6 +1719,7 @@ site1.local:
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_CORE_SOMAXCONN=somaxconn_path,
             _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
@@ -1659,6 +1734,7 @@ site1.local:
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_CORE_SOMAXCONN=somaxconn_path,
             _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
@@ -1679,11 +1755,16 @@ site1.local:
         tcp_cc.return_value = None
         process_rlimits.return_value = '1048777'
 
+        somaxconn_path = os.path.join(self.tmpdir, 'somaxconn')
+        with open(somaxconn_path, 'w', encoding='utf-8') as f:
+            f.write('4096\n')
+
         tune_tcp_mem.return_value = '188081 250774 376162'
         with mock.patch.multiple(
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC='some-file-does-not-exist',
+            _SYSCTL_CORE_SOMAXCONN=somaxconn_path,
             _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
@@ -1698,6 +1779,7 @@ site1.local:
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC='some-file-does-not-exist',
+            _SYSCTL_CORE_SOMAXCONN=somaxconn_path,
             _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
diff --git a/wheelhouse.txt b/wheelhouse.txt
index 5b23121..00f7c6d 100644
--- a/wheelhouse.txt
+++ b/wheelhouse.txt
@@ -1,3 +1,5 @@
+looseversion;python_version >= '3.12'
+
 # Include python requirements here
 psutil<5.9.3;python_version < '3.8'
 psutil;python_version >= '3.8'