Merge lp:~hloeung/ubuntu-repository-cache/cache-memory-tuning into lp:ubuntu-repository-cache

[Haw Loeung]

Allow runtime tuning of memory used by squid

Before:

$ juju run --application ubuntu-repository-cache 'grep cache_mem /etc/squid-deb-proxy/squid-deb-proxy.conf'
- Stdout: |
    cache_mem 1428 MB
  UnitId: ubuntu-repository-cache/0
- Stdout: |
    cache_mem 256 MB
  UnitId: ubuntu-repository-cache/2
- Stdout: |
    cache_mem 256 MB
  UnitId: ubuntu-repository-cache/1

After:

$ juju run --application ubuntu-repository-cache 'free -g | grep -v 'Swap'; grep -E "^(cache_mem|cache_dir)" /etc/squid-deb-proxy/squid-deb-proxy.conf'
- Stdout: |2
                  total used free shared buff/cache available
    Mem: 13 0 2 0 10 12
    cache_dir aufs /srv/ubuntu-repository-cache/squid 50000 16 256
    cache_mem 3588 MB
  UnitId: ubuntu-repository-cache/0
- Stdout: |2
                  total used free shared buff/cache available
    Mem: 6 0 2 0 3 5
    cache_dir aufs /srv/ubuntu-repository-cache/squid 50000 16 256
    cache_mem 1241 MB
  UnitId: ubuntu-repository-cache/1
- Stdout: |2
                  total used free shared buff/cache available
    Mem: 6 0 2 0 3 5
    cache_dir aufs /srv/ubuntu-repository-cache/squid 50000 16 256
    cache_mem 1241 MB
  UnitId: ubuntu-repository-cache/2

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[lp:~hloeung/ubuntu-repository-cache/cache-memory-tuning]

292. By Haw Loeung on 2020-12-07

Tune squid cache memory usage reducing from 33% to 30%

[Stuart Bishop (stub)]

size_caches and cache_mem could do with some consistency using full stops. Code all looks good. One of your mocks should be adjusted per inline comments, although I doubt it will ever make a real difference to tests.

review: Approve

[lp:~hloeung/ubuntu-repository-cache/cache-memory-tuning]

293. By Haw Loeung on 2020-12-07

Fixed based on reviews

[Haw Loeung (hloeung)]

Fixed description with regards to using full stops.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 291

=== modified file 'config.yaml'
--- config.yaml	2020-10-28 05:21:42 +0000
+++ config.yaml	2020-12-07 03:36:38 +0000
@@ -71,6 +71,12 @@
       effect running units, only newly added units.  An example would be
       '/dev/xvdb,/dev/xvdc' to specify two ephemeral disks for cache
       storage.
+  cache-memory-size:
+    default: 0
+    type: int
+    description: |
+      Configurable option (in MBytes) to tune/override memory used by
+      squid cache. If 0 or unset, then auto-calculate.
   cache-storage-size:
     default: 0
     type: int

=== modified file 'lib/ubuntu_repository_cache/squid.py'
--- lib/ubuntu_repository_cache/squid.py	2020-10-06 10:04:18 +0000
+++ lib/ubuntu_repository_cache/squid.py	2020-12-07 03:36:38 +0000
@@ -35,18 +35,21 @@
     subprocess.check_call(['/usr/sbin/squid3', '-N', '-z', '-f', '/etc/squid-deb-proxy/squid-deb-proxy.conf'])
 
 
-def size_caches():
-    """Determine the sizes for in-memory/on-disk squid caches
-
-    This must be run at installation time.  Runtime tuning is unsupported.
-    """
-
-    avail_disk = 0
-    for cache in unitdata.kv().get('squid-disk-caches'):
-        avail_disk += cache[1]
-    LOG('{}MB of disk cache available for squid.'.format(avail_disk), hookenv.DEBUG)
-
-    memory = util.system_total_mem() / 3
+def size_caches(system_total_memory=0):
+    """Determine the sizes for in-memory/on-disk squid caches."""
+
+    config = hookenv.config()
+    avail_disk = config.get('cache-storage-size', 0)
+    if not avail_disk:
+        for cache in unitdata.kv().get('squid-disk-caches'):
+            avail_disk += cache[1]
+        LOG('{}MB of disk cache available for squid.'.format(avail_disk), hookenv.DEBUG)
+
+    memory = system_total_memory
+    if not system_total_memory:
+        memory = util.system_total_mem()
+    # Set aside 30% of system memory for squid memory cache.
+    memory = memory * 0.30
     # Estimate of 100MB of overhead for squid
     memory = max(memory - 100, 0)
 
@@ -103,7 +106,6 @@
     host.service_pause(squid_service)
     stop()
 
-    size_caches()
     render_configs()
     init_caches()
 
@@ -165,6 +167,22 @@
     return cache_str
 
 
+def cache_mem(system_total_memory=0):
+    """Generate 'cache_mem' lines for the squid configuration file.
+
+    Returns a string containing the cache_mem config
+    """
+    size_caches(system_total_memory)
+
+    config = hookenv.config()
+    cache_mem = config.get('cache-memory-size', 0)
+    if not cache_mem:
+        cache_mem = unitdata.kv().get('squid-cache-mem')
+
+    cache_str = 'cache_mem {} MB'.format(str(cache_mem))
+    return cache_str
+
+
 @util.run_as_user('root')
 def render_configs():
     '''Render the configuration templates for all required services'''
@@ -172,8 +190,8 @@
     LOG('Rendering squid configuration templates')
     config = hookenv.config()
     context = {}
-    context['Cache_Config'] = cache_dirs()
-    context['Cache_Mem'] = unitdata.kv().get('squid-cache-mem')
+    context['Cache_Storage_Config'] = cache_dirs()
+    context['Cache_Memory_Config'] = cache_mem()
     context['Cache_Peers'] = peer_config()
     context['Cache_Hostname'] = hookenv.unit_private_ip()
     if config['squid_snmp']:

=== modified file 'lib/ubuntu_repository_cache/tests/test_squid.py'
--- lib/ubuntu_repository_cache/tests/test_squid.py	2020-08-11 04:12:28 +0000
+++ lib/ubuntu_repository_cache/tests/test_squid.py	2020-12-07 03:36:38 +0000
@@ -12,17 +12,23 @@
 class SquidTestCase(unittest.TestCase):
     def setUp(self):
         self.maxDiff = None
+        self.tmpdir = tempfile.mkdtemp(prefix='charm-unittests-')
+        self.addCleanup(shutil.rmtree, self.tmpdir)
+
         patcher = mock.patch('charmhelpers.core.hookenv.config')
         self.mock_config = patcher.start()
         self.addCleanup(patcher.stop)
 
-        self.tmpdir = tempfile.mkdtemp(prefix='charm-unittests-')
-        self.addCleanup(shutil.rmtree, self.tmpdir)
+        patcher = mock.patch('charmhelpers.core.hookenv.log')
+        self.mock_log = patcher.start()
+        self.addCleanup(patcher.stop)
+
         os.environ['UNIT_STATE_DB'] = os.path.join(self.tmpdir, '.unit-state.db')
+
+    def test_squid_cache_size(self):
         unitdata.kv().set('squid-cache-disk', 5000)
         unitdata.kv().set('squid-disk-caches', [('/srv', 2000)])
 
-    def test_squid_cache_size(self):
         self.mock_config.return_value = {
             'squid-cache-disk': 20 * 1024,
             'squid-disk-caches': [['/srv', 2000]],
@@ -38,3 +44,37 @@
         }
         result = squid.cache_dirs()
         self.assertEqual(result, 'cache_dir aufs /srv 1000 16 256\n')
+
+    def test_squid_cache_mem(self):
+        # Lots of system RAM, 14G per Azure D3_v2. 50GB cache, auto calculate memory usage.
+        self.mock_config.return_value = {
+            'cache-storage-size': 50000,
+            'cache-memory-size': 0,
+        }
+        result = squid.cache_mem(13997)
+        self.assertEqual(result, 'cache_mem 3122 MB')
+
+        # 7GB per Azure D2_v2. 50GB cache, auto calculate memory usage.
+        self.mock_config.return_value = {
+            'cache-storage-size': 50000,
+            'cache-memory-size': 0,
+        }
+        result = squid.cache_mem(6954)
+        self.assertEqual(result, 'cache_mem 1009 MB')
+
+        # 7GB per Azure D2_v2. 50GB cache, fixed 1000 MB.
+        self.mock_config.return_value = {
+            'cache-storage-size': 50000,
+            'cache-memory-size': 1000,
+        }
+        result = squid.cache_mem(6954)
+        self.assertEqual(result, 'cache_mem 1000 MB')
+
+        # 7GB per Azure D2_v2. Disk with 50GB, auto calculate memory usage.
+        unitdata.kv().set('squid-disk-caches', [('/srv', 50000)])
+        self.mock_config.return_value = {
+            'cache-storage-size': 0,
+            'cache-memory-size': 0,
+        }
+        result = squid.cache_mem(6954)
+        self.assertEqual(result, 'cache_mem 1009 MB')

=== modified file 'templates/squid-deb-proxy/squid-deb-proxy.conf'
--- templates/squid-deb-proxy/squid-deb-proxy.conf	2020-04-14 12:55:36 +0000
+++ templates/squid-deb-proxy/squid-deb-proxy.conf	2020-12-07 03:36:38 +0000
@@ -39,7 +39,7 @@
 maximum_object_size 512 MB
 
 # cache directory
-{{ Cache_Config }}
+{{ Cache_Storage_Config }}
 
 # Cache swap more aggressive to avoid having the cache size going above the cache_dir size parameter
 cache_swap_low 80
@@ -51,7 +51,7 @@
 #access_log daemon:/var/log/squid-deb-proxy/access.log
 
 # tweaks to speed things up
-cache_mem {{ Cache_Mem }} MB
+{{ Cache_Memory_Config }}
 maximum_object_size_in_memory 30 MB
 
 collapsed_forwarding on