Merge ~ballot/content-cache-charm/+git/content-cache-charm:worker_connections into content-cache-charm:master

[Benjamin Allot]

Unmerged commits

95f319d... by Benjamin Allot on 2021-03-23

WIP

diff --git a/config.yaml b/config.yaml
index 3ce504a..c843b93 100644
--- a/config.yaml
+++ b/config.yaml
@@ -132,11 +132,12 @@ options:
     description: >
       Override default ciphers used for TLS/SSL termination (OpenSSL format).
   worker_connections:
-    default: 768
+    default: 0
     type: int
     description: >
       Sets the maximum number of simultaneous connections that can be
       opened by an Nginx worker process.
+      Defaults to auto-calculate (0), which is (number of sites * max_connections / worker_processes).
   worker_processes:
     default: 0
     type: int
diff --git a/lib/haproxy.py b/lib/haproxy.py
index 5153d97..814bc89 100644
--- a/lib/haproxy.py
+++ b/lib/haproxy.py
@@ -339,7 +339,8 @@ backend backend-{name}
 
         return rendered_output
 
-    def _calculate_num_procs_threads(self, num_procs, num_threads):
+    @classmethod
+    def calculate_num_procs_threads(self, num_procs, num_threads):
         if num_procs and num_threads:
             ver = utils.package_version('haproxy')
             # With HAProxy 2, nbproc and nbthreads are mutually exclusive.
@@ -359,7 +360,7 @@ backend backend-{name}
         return (num_procs, num_threads)
 
     def render(self, config, num_procs=None, num_threads=None, monitoring_password=None, tls_cipher_suites=None):
-        (num_procs, num_threads) = self._calculate_num_procs_threads(num_procs, num_threads)
+        (num_procs, num_threads) = self.calculate_num_procs_threads(num_procs, num_threads)
 
         listen_stanzas = self.render_stanza_listen(config)
 
diff --git a/reactive/content_cache.py b/reactive/content_cache.py
index 15cf56d..5cc47df 100644
--- a/reactive/content_cache.py
+++ b/reactive/content_cache.py
@@ -7,6 +7,7 @@ import time
 from copy import deepcopy
 
 import jinja2
+import multiprocessing
 import yaml
 
 from charms import reactive
@@ -199,6 +200,17 @@ def configure_nginx(conf_path=None):
 
     connections = config['worker_connections']
     processes = config['worker_processes']
+    # Auto-calculate worker_connections
+    nb_sites = len(sites.keys())
+    (haproxy_proc, haproxy_thread) = HAProxy.calculate_num_procs_threads(None, None)
+    if config['max_connections'] == 0:
+    haproxy_connections
+    if processes == 0:
+        nb_processes = multiprocessing.cpu_count()
+    else:
+        nb_processes = processes
+    if connections == 0:
+        connections = nb_sites * haproxy_max_connections / nb_processes
     if ngx_conf.sync_sites(sites.keys()) or ngx_conf.set_workers(connections, processes):
         hookenv.log('Enabled sites: {}'.format(' '.join(sites.keys())))
         changed = True
diff --git a/tests/unit/test_content_cache.py b/tests/unit/test_content_cache.py
index b8de692..59afe21 100644
--- a/tests/unit/test_content_cache.py
+++ b/tests/unit/test_content_cache.py
@@ -1423,10 +1423,12 @@ site1.local:
             'enable_cache_background_update': False,
             'enable_cache_lock': False,
             'enable_prometheus_metrics': True,
+            'max_connections': 0,
             'metrics_listen_address': '127.0.0.2',
             'reuseport': True,
             'sites': ngx_config,
-            'worker_connections': 768,
+            # 'worker_connections': 768,
+            'worker_connections': 0,
             'worker_processes': 0,
         }
 
@@ -1437,7 +1439,8 @@ site1.local:
             os.mkdir(os.path.join(self.tmpdir, 'conf.d'))
             os.mkdir(os.path.join(self.tmpdir, 'sites-available'))
             os.mkdir(os.path.join(self.tmpdir, 'sites-enabled'))
-            shutil.copyfile('tests/unit/files/nginx.conf', os.path.join(self.tmpdir, 'nginx.conf'))
+            nginx_conf = os.path.join(self.tmpdir, 'nginx.conf')
+            shutil.copyfile('tests/unit/files/nginx.conf', nginx_conf)
 
             opened_ports.return_value = ['80/tcp', '{0}/tcp'.format(nginx.METRICS_PORT)]
             content_cache.configure_nginx(self.tmpdir)
@@ -1451,3 +1454,6 @@ site1.local:
             with open(test_file, 'r', encoding='utf-8') as f:
                 got = f.read()
             self.assertEqual(got, want)
+            with open(nginx_conf, 'r', encoding='utf-8') as f:
+                self.assertRegex(f.read(), r'.*worker_connections = '
+            assert nginx
diff --git a/tests/unit/test_nginx.py b/tests/unit/test_nginx.py
index 2734123..6e6f209 100644
--- a/tests/unit/test_nginx.py
+++ b/tests/unit/test_nginx.py
@@ -4,6 +4,8 @@ import sys
 import tempfile
 import unittest
 
+from unittest import mock
+
 import jinja2
 import yaml
 
@@ -20,6 +22,10 @@ class TestLibNginx(unittest.TestCase):
         self.tmpdir = tempfile.mkdtemp(prefix='charm-unittests-')
         self.addCleanup(shutil.rmtree, self.tmpdir)
         self.charm_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+        patcher = mock.patch('multiprocessing.cpu_count')
+        self.mock_cpu_count = patcher.start()
+        self.addCleanup(patcher.stop)
+        self.mock_cpu_count.return_value = 4
 
     def test_nginx_config_conf_path(self):
         conf_path = '/etc/nginx/conf.d'