Merge lp:~hloeung/ubuntu-repository-cache/auto-apache2-mpm-tuning into lp:ubuntu-repository-cache

[Haw Loeung]

Automatically calculate MPM worker tuning configs

Currently, these are the flavors/instance types generally used:

GCE - https://cloud.google.com/compute/docs/machine-types#n1_machine_types
n1-standard-16 = 16 vCPUs, 60GiB RAM

AWS - https://aws.amazon.com/ec2/instance-types/#Compute_Optimized
c5.xlarge = 4 vCPUs, 8GiB RAM
c3.4xlarge = 16 vCPUs, 30GiB RAM
c4.8xlarge = 36 vCPUs, 60GiB RAM

Azure - https://docs.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series
Standard_D2_v2 = 2 vCPUs, 7GiB RAM
Standard_D3_v2 = 4 vCPUs, 14GiB RAM

[Canonical IS Mergebot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[lp:~hloeung/ubuntu-repository-cache/auto-apache2-mpm-tuning]

296. By Haw Loeung on 2020-11-02

Fixed comments

[Barry Price (barryprice)]

LGTM +1

review: Approve

[Stuart Bishop (stub)]

This all looks great

review: Approve

[Canonical IS Mergebot (canonical-is-mergebot)]

Change successfully merged at revision 289

=== modified file 'config.yaml'
--- config.yaml	2020-09-30 04:32:54 +0000
+++ config.yaml	2020-11-02 03:11:48 +0000
@@ -111,37 +111,37 @@
     description: Select the worker or prefork multi-processing module
   apache2_mpm_startservers:
     type: int
-    default: 2
-    description: Initial number of server processes to start
+    default: 0
+    description: Initial number of server processes to start (0 means auto calculate)
   apache2_mpm_minsparethreads:
     type: int
-    default: 100
-    description: Minimum number of worker threads which are kept spare
+    default: 0
+    description: Minimum number of worker threads which are kept spare (0 means auto calculate)
   apache2_mpm_maxsparethreads:
     type: int
-    default: 200
-    description: Maximum number of worker threads which are kept spare
+    default: 0
+    description: Maximum number of worker threads which are kept spare (0 means auto calculate)
   apache2_mpm_threadlimit:
     type: int
-    default: 64
+    default: 0
     description: |
        Sets the upper limit on the configurable number of threads per
-       child process
+       child process (0 means autocalculate)
   apache2_mpm_threadsperchild:
     type: int
-    default: 64
-    description: Constant number of worker threads in each server process
+    default: 0
+    description: Constant number of worker threads in each server process (0 means autocalculate)
   apache2_mpm_serverlimit:
     type: int
-    default: 256
-    description: Upper limit on configurable number of processes
+    default: 0
+    description: Upper limit on configurable number of processes (0 means autocalculate)
   apache2_mpm_maxrequestworkers:
     type: int
-    default: 16384
-    description: Maximum number of simultaneous client connections
+    default: 0
+    description: Maximum number of simultaneous client connections (0 means autocalculate)
   apache2_mpm_maxconnectionsperchild:
     type: int
-    default: 0
+    default: 10000
     description: Maximum number of requests a server process serves
   logrotate_rotate:
     type: string

=== modified file 'lib/ubuntu_repository_cache/apache.py'
--- lib/ubuntu_repository_cache/apache.py	2020-10-06 10:04:18 +0000
+++ lib/ubuntu_repository_cache/apache.py	2020-11-02 03:11:48 +0000
@@ -1,5 +1,6 @@
 '''Functions related to apache use within this charm'''
 
+import multiprocessing
 import os
 import subprocess
 import json
@@ -71,6 +72,44 @@
     return host.service_running(SERVICE)
 
 
+def tune_mpm_configs(config):
+    cpu_count = multiprocessing.cpu_count()
+
+    if not config['startservers']:
+        config['startservers'] = 1
+
+    if not config['serverlimit']:
+        # Benchmarking shows single process multiple threads performs better
+        # than multiple processes multiple threads.
+        config['serverlimit'] = 1
+
+    if not config['threadsperchild']:
+        # We could take into consideration how much memory the system
+        # has but the ratio of vCPU to RAM between public clouds seems
+        # good enough so this is not needed.
+        config['threadsperchild'] = 256 * cpu_count
+        # Cap at 8192
+        if config['threadsperchild'] > 8192:
+            config['threadsperchild'] = 8192
+
+    if not config['threadlimit']:
+        # Hard limit should match per child limit.
+        config['threadlimit'] = config['threadsperchild']
+
+    if not config['maxrequestworkers']:
+        # Calculate max. workers based on serverlimit and threadsperchild.
+        config['maxrequestworkers'] = config['serverlimit'] * config['threadsperchild']
+
+    if not config['minsparethreads']:
+        # Use 50% of max. workers.
+        config['minsparethreads'] = int(config['maxrequestworkers'] * 0.5)
+
+    if not config['maxsparethreads']:
+        config['maxsparethreads'] = config['maxrequestworkers']
+
+    return config
+
+
 def create_mpm_workerfile():
     '''Create the multi-processing module (MPM) configuration'''
 
@@ -85,6 +124,7 @@
         'maxrequestworkers': config['apache2_mpm_maxrequestworkers'],
         'maxconnectionsperchild': config['apache2_mpm_maxconnectionsperchild'],
     }
+    mpm_context = tune_mpm_configs(mpm_context)
     conf_mpm_worker = '/etc/apache2/conf-available/000mpm-worker.conf'
     templating.render('apache2/mpm_worker.template', conf_mpm_worker, mpm_context)
     subprocess.check_call(['a2enconf', '000mpm-worker'])

=== added file 'lib/ubuntu_repository_cache/tests/test_apache.py'
--- lib/ubuntu_repository_cache/tests/test_apache.py	1970-01-01 00:00:00 +0000
+++ lib/ubuntu_repository_cache/tests/test_apache.py	2020-11-02 03:11:48 +0000
@@ -0,0 +1,141 @@
+import os
+import shutil
+import tempfile
+import unittest
+from unittest import mock
+
+from .. import apache
+
+from charmhelpers.core import unitdata
+
+
+class ApacheTestCase(unittest.TestCase):
+    def setUp(self):
+        self.maxDiff = None
+        patcher = mock.patch('charmhelpers.core.hookenv.config')
+        self.mock_config = patcher.start()
+        self.addCleanup(patcher.stop)
+
+        self.tmpdir = tempfile.mkdtemp(prefix='charm-unittests-')
+        self.addCleanup(shutil.rmtree, self.tmpdir)
+        os.environ['UNIT_STATE_DB'] = os.path.join(self.tmpdir, '.unit-state.db')
+        unitdata.kv().set('squid-cache-disk', 5000)
+        unitdata.kv().set('squid-disk-caches', [('/srv', 2000)])
+
+    @mock.patch('multiprocessing.cpu_count')
+    def test_tune_mpm_configs(self, cpu_count):
+
+        cpu_count.return_value = 4
+        config = {
+            'startservers': 0,
+            'minsparethreads': 0,
+            'maxsparethreads': 0,
+            'threadlimit': 0,
+            'threadsperchild': 0,
+            'serverlimit': 0,
+            'maxrequestworkers': 0,
+            'maxconnectionsperchild': 10000,
+        }
+        expected = {
+            'startservers': 1,
+            'minsparethreads': 512,
+            'maxsparethreads': 1024,
+            'threadlimit': 1024,
+            'threadsperchild': 1024,
+            'serverlimit': 1,
+            'maxrequestworkers': 1024,
+            'maxconnectionsperchild': 10000,
+        }
+        self.assertEqual(apache.tune_mpm_configs(config), expected)
+
+        cpu_count.return_value = 40
+        config = {
+            'startservers': 0,
+            'minsparethreads': 0,
+            'maxsparethreads': 0,
+            'threadlimit': 0,
+            'threadsperchild': 0,
+            'serverlimit': 0,
+            'maxrequestworkers': 0,
+            'maxconnectionsperchild': 10000,
+        }
+        expected = {
+            'startservers': 1,
+            'minsparethreads': 4096,
+            'maxsparethreads': 8192,
+            'threadlimit': 8192,
+            'threadsperchild': 8192,
+            'serverlimit': 1,
+            'maxrequestworkers': 8192,
+            'maxconnectionsperchild': 10000,
+        }
+        self.assertEqual(apache.tune_mpm_configs(config), expected)
+
+        cpu_count.return_value = 1
+        config = {
+            'startservers': 0,
+            'minsparethreads': 0,
+            'maxsparethreads': 0,
+            'threadlimit': 0,
+            'threadsperchild': 0,
+            'serverlimit': 0,
+            'maxrequestworkers': 0,
+            'maxconnectionsperchild': 10000,
+        }
+        expected = {
+            'startservers': 1,
+            'minsparethreads': 128,
+            'maxsparethreads': 256,
+            'threadlimit': 256,
+            'threadsperchild': 256,
+            'serverlimit': 1,
+            'maxrequestworkers': 256,
+            'maxconnectionsperchild': 10000,
+        }
+        self.assertEqual(apache.tune_mpm_configs(config), expected)
+
+        cpu_count.return_value = 2
+        config = {
+            'startservers': 0,
+            'minsparethreads': 0,
+            'maxsparethreads': 0,
+            'threadlimit': 0,
+            'threadsperchild': 0,
+            'serverlimit': 0,
+            'maxrequestworkers': 0,
+            'maxconnectionsperchild': 10000,
+        }
+        expected = {
+            'startservers': 1,
+            'minsparethreads': 256,
+            'maxsparethreads': 512,
+            'threadlimit': 512,
+            'threadsperchild': 512,
+            'serverlimit': 1,
+            'maxrequestworkers': 512,
+            'maxconnectionsperchild': 10000,
+        }
+        self.assertEqual(apache.tune_mpm_configs(config), expected)
+
+        cpu_count.return_value = 4
+        config = {
+            'startservers': 5,
+            'minsparethreads': 10,
+            'maxsparethreads': 100,
+            'threadlimit': 32,
+            'threadsperchild': 32,
+            'serverlimit': 10,
+            'maxrequestworkers': 8192,
+            'maxconnectionsperchild': 5000,
+        }
+        expected = {
+            'startservers': 5,
+            'minsparethreads': 10,
+            'maxsparethreads': 100,
+            'threadlimit': 32,
+            'threadsperchild': 32,
+            'serverlimit': 10,
+            'maxrequestworkers': 8192,
+            'maxconnectionsperchild': 5000,
+        }
+        self.assertEqual(apache.tune_mpm_configs(config), expected)

=== modified file 'templates/apache2/mpm_worker.template'
--- templates/apache2/mpm_worker.template	2015-02-12 00:05:14 +0000
+++ templates/apache2/mpm_worker.template	2020-11-02 03:11:48 +0000
@@ -1,5 +1,5 @@
 #
-# JUJU WARNING: This file is managed by Juju, do NOT edit 
+# JUJU WARNING: This file is managed by Juju, do NOT edit
 #
 
 # worker MPM
@@ -12,10 +12,11 @@
 <IfModule mpm_worker_module>
     StartServers           {{ startservers }}
     MinSpareThreads        {{ minsparethreads }}
-    MaxSpareThreads        {{ maxsparethreads }} 
+    MaxSpareThreads        {{ maxsparethreads }}
     ThreadLimit            {{ threadlimit }}
     ThreadsPerChild        {{ threadsperchild }}
     ServerLimit            {{ serverlimit }}
     MaxRequestWorkers      {{ maxrequestworkers }}
     MaxConnectionsPerChild {{ maxconnectionsperchild }}
 </IfModule>
+

=== modified file 'tox.ini'
--- tox.ini	2020-08-11 04:12:28 +0000
+++ tox.ini	2020-11-02 03:11:48 +0000
@@ -10,7 +10,7 @@
 [testenv:unit]
 commands =
     pytest --ignore {toxinidir}/tests/functional \
-      {posargs:-v  --cov=lib --cov=reactive --cov=actions --cov-report=term-missing --cov-branch}
+      {posargs:-v  --cov=lib --cov=reactive --cov-report=term-missing --cov-branch}
 deps = -r{toxinidir}/tests/unit/requirements.txt
        -r{toxinidir}/requirements.txt
 setenv =