Merge ~hloeung/content-cache-charm:sysctl into content-cache-charm:master

[Haw Loeung]

Allow overriding Netfilter conntrack_max - LP:1860631

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Paul Collins (pjdc)]

review: Approve (lgtm)

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision f471a5fc678ccc5ca10b71e3465c3785368161a0

diff --git a/config.yaml b/config.yaml
index 0b1f80f..47d9761 100644
--- a/config.yaml
+++ b/config.yaml
@@ -148,9 +148,14 @@ options:
     type: string
     description: >
       Override default ciphers used for TLS/SSL termination (OpenSSL format).
+  tune_nf_conntrack_max:
+    default: 0
+    type: int
+    description: >
+      Tune or set net.nf_conntrack_max. Set to 0 to disable.
   tune_tcp_mem_multiplier:
-    type: float
     default: 1.5
+    type: float
     description: >
       Tune net.ipv4.tcp_mem, apply multiplier.
   worker_connections:
diff --git a/reactive/content_cache.py b/reactive/content_cache.py
index b090311..bc51c18 100644
--- a/reactive/content_cache.py
+++ b/reactive/content_cache.py
@@ -518,6 +518,7 @@ def configure_nagios():
 
 
 _SYSCTL_CORE_DEFAULT_QDISC = '/proc/sys/net/core/default_qdisc'
+_SYSCTL_NETFILTER_CONNTRACK_MAX = '/proc/sys/net/nf_conntrack_max'
 
 
 @reactive.when_not('content_cache.sysctl.configured')
@@ -536,6 +537,9 @@ def configure_sysctl():
     context['net_ipv4_tcp_congestion_control'] = utils.select_tcp_congestion_control(preferred_tcp_cc)
     context['net_ipv4_tcp_mem'] = utils.tune_tcp_mem(config['tune_tcp_mem_multiplier'])
 
+    if os.path.exists(_SYSCTL_NETFILTER_CONNTRACK_MAX):
+        context['net_nf_conntrack_max'] = config['tune_nf_conntrack_max']
+
     # Set or lower tcp_notsent_lowat to optimise HTTP/2 prioritisation.
     # https://blog.cloudflare.com/http-2-prioritization-with-nginx/
     context['net_ipv4_tcp_notsent_lowat'] = '16384'
diff --git a/templates/sysctl_conf.tmpl b/templates/sysctl_conf.tmpl
index 60395f3..316a34d 100644
--- a/templates/sysctl_conf.tmpl
+++ b/templates/sysctl_conf.tmpl
@@ -11,6 +11,9 @@ net.core.default_qdisc = {{ net_core_default_qdisc }}
 {% if net_ipv4_tcp_mem %}
 # Tune TCP memory
 net.ipv4.tcp_mem = {{ net_ipv4_tcp_mem }}
+{% endif -%}
+{% if net_nf_conntrack_max and net_nf_conntrack_max > 0 %}
+net.nf_conntrack_max = {{ net_nf_conntrack_max }}
 {% endif %}
 
 # Tune the TCP stack for Fast Failure Detection
diff --git a/tests/unit/files/sysctl.conf b/tests/unit/files/sysctl.conf
index 399a1bc..f64dbb8 100644
--- a/tests/unit/files/sysctl.conf
+++ b/tests/unit/files/sysctl.conf
@@ -6,6 +6,8 @@ net.core.default_qdisc = fq
 # Tune TCP memory
 net.ipv4.tcp_mem = 92430 123242 184860
 
+net.nf_conntrack_max = 2097152
+
 
 # Tune the TCP stack for Fast Failure Detection
 # tcp_syn_retries set to 1, it takes ~9secs to timeout
diff --git a/tests/unit/files/sysctl_nf_conntrack_max.conf b/tests/unit/files/sysctl_nf_conntrack_max.conf
new file mode 100644
index 0000000..130ce57
--- /dev/null
+++ b/tests/unit/files/sysctl_nf_conntrack_max.conf
@@ -0,0 +1,23 @@
+net.ipv4.tcp_notsent_lowat = 16384
+
+net.nf_conntrack_max = 2097152
+
+
+# Tune the TCP stack for Fast Failure Detection
+# tcp_syn_retries set to 1, it takes ~9secs to timeout
+# tcp_syn_retries set to 2, it takes ~21secs to timeout
+# tcp_syn_retries set to 3, it takes ~45secs to timeout
+# tcp_syn_retries set to 4, it takes ~93secs to timeout
+# tcp_syn_retries set to 5, it takes ~189secs to timeout (default)
+# Reduce TCP connection timeouts to ~45 secs.
+net.ipv4.tcp_syn_retries = 3
+net.ipv4.tcp_synack_retries = 3
+
+# Reduce TCP KeepAlive timeout from 2 hrs to 30 mins.
+net.ipv4.tcp_keepalive_time = 1800
+
+# Extend the source port range for outgoing TCP connections.
+net.ipv4.ip_local_port_range = 16384 61000
+
+# Allow early reuse of a same source port for outgoing connections.
+net.ipv4.tcp_tw_reuse = 1
diff --git a/tests/unit/files/sysctl_nf_conntrack_max_none.conf b/tests/unit/files/sysctl_nf_conntrack_max_none.conf
new file mode 100644
index 0000000..1c7cccc
--- /dev/null
+++ b/tests/unit/files/sysctl_nf_conntrack_max_none.conf
@@ -0,0 +1,21 @@
+net.ipv4.tcp_notsent_lowat = 16384
+
+
+# Tune the TCP stack for Fast Failure Detection
+# tcp_syn_retries set to 1, it takes ~9secs to timeout
+# tcp_syn_retries set to 2, it takes ~21secs to timeout
+# tcp_syn_retries set to 3, it takes ~45secs to timeout
+# tcp_syn_retries set to 4, it takes ~93secs to timeout
+# tcp_syn_retries set to 5, it takes ~189secs to timeout (default)
+# Reduce TCP connection timeouts to ~45 secs.
+net.ipv4.tcp_syn_retries = 3
+net.ipv4.tcp_synack_retries = 3
+
+# Reduce TCP KeepAlive timeout from 2 hrs to 30 mins.
+net.ipv4.tcp_keepalive_time = 1800
+
+# Extend the source port range for outgoing TCP connections.
+net.ipv4.ip_local_port_range = 16384 61000
+
+# Allow early reuse of a same source port for outgoing connections.
+net.ipv4.tcp_tw_reuse = 1
diff --git a/tests/unit/test_content_cache.py b/tests/unit/test_content_cache.py
index 8ea2f21..789b411 100644
--- a/tests/unit/test_content_cache.py
+++ b/tests/unit/test_content_cache.py
@@ -1474,6 +1474,7 @@ site1.local:
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC='some-file-does-not-exist',
+            _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
             # Check contents
@@ -1497,7 +1498,10 @@ site1.local:
     @mock.patch('lib.utils.tune_tcp_mem')
     def test_configure_sysctl_all(self, tune_tcp_mem, tcp_cc, process_rlimits, call, set_flag):
         sysctl_conf_path = os.path.join(self.tmpdir, '90-content-cache.conf')
-        self.mock_config.return_value = {'tune_tcp_mem_multiplier': 1.5}
+        self.mock_config.return_value = {
+            'tune_nf_conntrack_max': 2097152,
+            'tune_tcp_mem_multiplier': 1.5,
+        }
         process_rlimits.return_value = '1048777'
 
         # Test with all 3, qdisc, tcp_congestion_control, and tcp_mem
@@ -1505,11 +1509,13 @@ site1.local:
         tcp_cc.return_value = 'bbr'
         # Use '/proc/uptime' for unit test as that will always exist.
         qdisc_path = '/proc/uptime'
+        conntrack_max_path = '/proc/uptime'
 
         with mock.patch.multiple(
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_NETFILTER_CONNTRACK_MAX=conntrack_max_path,
         ):
             content_cache.configure_sysctl()
             with open('tests/unit/files/sysctl.conf', 'r') as f:
@@ -1536,6 +1542,7 @@ site1.local:
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
             with open('tests/unit/files/sysctl_core_default_qdisc.conf', 'r') as f:
@@ -1549,6 +1556,7 @@ site1.local:
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
             with open('tests/unit/files/sysctl_core_default_qdisc_none.conf', 'r') as f:
@@ -1562,6 +1570,53 @@ site1.local:
     @mock.patch('lib.utils.process_rlimits')
     @mock.patch('lib.utils.select_tcp_congestion_control')
     @mock.patch('lib.utils.tune_tcp_mem')
+    def test_configure_sysctl_nf_conntrack_max(self, tune_tcp_mem, tcp_cc, process_rlimits, call, set_flag):
+        sysctl_conf_path = os.path.join(self.tmpdir, '90-content-cache.conf')
+        self.mock_config.return_value = {
+            'tune_nf_conntrack_max': 2097152,
+            'tune_tcp_mem_multiplier': 1.5,
+        }
+        process_rlimits.return_value = '1048777'
+        tcp_cc.return_value = None
+        tune_tcp_mem.return_value = None
+        # Use '/proc/uptime' for unit test as that will always exist.
+        conntrack_max_path = '/proc/uptime'
+
+        with mock.patch.multiple(
+            'reactive.content_cache',
+            SYSCTL_CONF_PATH=sysctl_conf_path,
+            _SYSCTL_CORE_DEFAULT_QDISC='some-file-does-not-exist',
+            _SYSCTL_NETFILTER_CONNTRACK_MAX=conntrack_max_path,
+        ):
+            content_cache.configure_sysctl()
+            with open('tests/unit/files/sysctl_nf_conntrack_max.conf', 'r') as f:
+                want = f.read()
+            with open(sysctl_conf_path, 'r') as f:
+                got = f.read()
+            self.assertEqual(got, want)
+
+        self.mock_config.return_value = {
+            'tune_nf_conntrack_max': 0,
+            'tune_tcp_mem_multiplier': 1.5,
+        }
+        with mock.patch.multiple(
+            'reactive.content_cache',
+            SYSCTL_CONF_PATH=sysctl_conf_path,
+            _SYSCTL_CORE_DEFAULT_QDISC='some-file-does-not-exist',
+            _SYSCTL_NETFILTER_CONNTRACK_MAX=conntrack_max_path,
+        ):
+            content_cache.configure_sysctl()
+            with open('tests/unit/files/sysctl_nf_conntrack_max_none.conf', 'r') as f:
+                want = f.read()
+            with open(sysctl_conf_path, 'r') as f:
+                got = f.read()
+            self.assertEqual(got, want)
+
+    @mock.patch('charms.reactive.set_flag')
+    @mock.patch('subprocess.call')
+    @mock.patch('lib.utils.process_rlimits')
+    @mock.patch('lib.utils.select_tcp_congestion_control')
+    @mock.patch('lib.utils.tune_tcp_mem')
     def test_configure_sysctl_tcp_congestion_control(self, tune_tcp_mem, tcp_cc, process_rlimits, call, set_flag):
         sysctl_conf_path = os.path.join(self.tmpdir, '90-content-cache.conf')
         self.mock_config.return_value = {'tune_tcp_mem_multiplier': 1.5}
@@ -1574,6 +1629,7 @@ site1.local:
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
             with open('tests/unit/files/sysctl_net_tcp_congestion_control.conf', 'r') as f:
@@ -1587,6 +1643,7 @@ site1.local:
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
             with open('tests/unit/files/sysctl_net_tcp_congestion_control_bbr2.conf', 'r') as f:
@@ -1600,6 +1657,7 @@ site1.local:
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
             _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
             with open('tests/unit/files/sysctl_net_tcp_congestion_control_no_bbr.conf', 'r') as f:
@@ -1618,13 +1676,13 @@ site1.local:
         self.mock_config.return_value = {'tune_tcp_mem_multiplier': 1.5}
         tcp_cc.return_value = None
         process_rlimits.return_value = '1048777'
-        qdisc_path = 'some-file-does-not-exist'
 
         tune_tcp_mem.return_value = '188081 250774 376162'
         with mock.patch.multiple(
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
-            _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_CORE_DEFAULT_QDISC='some-file-does-not-exist',
+            _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
             with open('tests/unit/files/sysctl_net_tcp_mem.conf', 'r') as f:
@@ -1634,11 +1692,11 @@ site1.local:
             self.assertEqual(got, want)
 
         tune_tcp_mem.return_value = None
-        qdisc_path = 'some-file-does-not-exist'
         with mock.patch.multiple(
             'reactive.content_cache',
             SYSCTL_CONF_PATH=sysctl_conf_path,
-            _SYSCTL_CORE_DEFAULT_QDISC=qdisc_path,
+            _SYSCTL_CORE_DEFAULT_QDISC='some-file-does-not-exist',
+            _SYSCTL_NETFILTER_CONNTRACK_MAX='some-file-does-not-exist',
         ):
             content_cache.configure_sysctl()
             with open('tests/unit/files/sysctl_net_tcp_mem_none.conf', 'r') as f: