Merge lp:~gnuoy/charm-helpers/1496746 into lp:charm-helpers

[Liam Young]

[Liam Young (gnuoy)]

Looking at https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt the pagesize option that is used to mount the hugepages should be set in bytes not human readable equivalent and that makes working out shmmax much easier too.

[lp:~gnuoy/charm-helpers/1496746]

451. By Liam Young on 2015-09-17

Add function to convert human readable representation of quantity of bytes (eg 1G) to int

[James Page (james-page)]

review: Approve

=== modified file 'charmhelpers/core/hugepage.py'
--- charmhelpers/core/hugepage.py	2015-08-19 09:40:54 +0000
+++ charmhelpers/core/hugepage.py	2015-09-17 11:04:29 +0000
@@ -25,11 +25,13 @@
     fstab_mount,
     mkdir,
 )
+from charmhelpers.core.strutils import bytes_from_string
+from subprocess import check_output
 
 
 def hugepage_support(user, group='hugetlb', nr_hugepages=256,
                      max_map_count=65536, mnt_point='/run/hugepages/kvm',
-                     pagesize='2MB', mount=True):
+                     pagesize='2MB', mount=True, set_shmmax=False):
     """Enable hugepages on system.
 
     Args:
@@ -49,6 +51,11 @@
         'vm.max_map_count': max_map_count,
         'vm.hugetlb_shm_group': gid,
     }
+    if set_shmmax:
+        shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax']))
+        shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages
+        if shmmax_minsize > shmmax_current:
+            sysctl_settings['kernel.shmmax'] = shmmax_minsize
     sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf')
     mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False)
     lfstab = fstab.Fstab()

=== modified file 'charmhelpers/core/strutils.py'
--- charmhelpers/core/strutils.py	2015-04-13 19:18:40 +0000
+++ charmhelpers/core/strutils.py	2015-09-17 11:04:29 +0000
@@ -18,6 +18,7 @@
 # along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
 
 import six
+import re
 
 
 def bool_from_string(value):
@@ -40,3 +41,32 @@
 
     msg = "Unable to interpret string value '%s' as boolean" % (value)
     raise ValueError(msg)
+
+
+def bytes_from_string(value):
+    """Interpret human readable string value as bytes.
+
+    Returns int
+    """
+    BYTE_POWER = {
+        'K': 1,
+        'KB': 1,
+        'M': 2,
+        'MB': 2,
+        'G': 3,
+        'GB': 3,
+        'T': 4,
+        'TB': 4,
+        'P': 5,
+        'PB': 5,
+    }
+    if isinstance(value, six.string_types):
+        value = six.text_type(value)
+    else:
+        msg = "Unable to interpret non-string value '%s' as boolean" % (value)
+        raise ValueError(msg)
+    matches = re.match("([0-9]+)([a-zA-Z]+)", value)
+    if not matches:
+        msg = "Unable to interpret string value '%s' as bytes" % (value)
+        raise ValueError(msg)
+    return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])

=== modified file 'tests/core/test_hugepage.py'
--- tests/core/test_hugepage.py	2015-08-17 11:37:03 +0000
+++ tests/core/test_hugepage.py	2015-09-17 11:04:29 +0000
@@ -1,6 +1,7 @@
 from testtools import TestCase
 from mock import patch
 from charmhelpers.core import hugepage
+import yaml
 
 TO_PATCH = [
     'fstab',
@@ -9,6 +10,7 @@
     'sysctl',
     'fstab_mount',
     'mkdir',
+    'check_output',
 ]
 
 
@@ -68,15 +70,33 @@
         self.fstab.Fstab().Entry.return_value = 'new fstab entry'
         hugepage.hugepage_support(
             'nova', group='neutron', nr_hugepages=512, max_map_count=70000,
-            mnt_point='/hugepages', pagesize='1GB', mount=False)
-        sysctl_expect = ("{vm.hugetlb_shm_group: '1010', "
-                         "vm.max_map_count: 70000, vm.nr_hugepages: 512}\n")
-        self.sysctl.create.assert_called_with(sysctl_expect,
-                                              '/etc/sysctl.d/10-hugepage.conf')
+            mnt_point='/hugepages', pagesize='1G', mount=False)
+        sysctl_expect = {
+            'vm.hugetlb_shm_group': '1010',
+            'vm.max_map_count': 70000,
+            'vm.nr_hugepages': 512,
+        }
+        sysctl_setting_arg = self.sysctl.create.call_args_list[0][0][0]
+        self.assertEqual(yaml.load(sysctl_setting_arg), sysctl_expect)
         self.mkdir.assert_called_with('/hugepages', owner='root',
                                       group='root', perms=0o755, force=False)
         self.fstab.Fstab().remove_entry.assert_called_with('old fstab entry')
         self.fstab.Fstab().Entry.assert_called_with(
             'nodev', '/hugepages', 'hugetlbfs',
-            'mode=1770,gid=1010,pagesize=1GB', 0, 0)
+            'mode=1770,gid=1010,pagesize=1G', 0, 0)
         self.fstab.Fstab().add_entry.assert_called_with('new fstab entry')
+
+    def test_hugepage_support_set_shmmax(self):
+        self.add_group.return_value = Group()
+        self.fstab.Fstab().get_entry_by_attr.return_value = None
+        self.fstab.Fstab().Entry.return_value = 'new fstab entry'
+        self.check_output.return_value = 2000
+        hugepage.hugepage_support('nova', mount=False, set_shmmax=True)
+        sysctl_expect = {
+            'kernel.shmmax': 536870912,
+            'vm.hugetlb_shm_group': '1010',
+            'vm.max_map_count': 65536,
+            'vm.nr_hugepages': 256
+        }
+        sysctl_setting_arg = self.sysctl.create.call_args_list[0][0][0]
+        self.assertEqual(yaml.load(sysctl_setting_arg), sysctl_expect)

=== modified file 'tests/core/test_strutils.py'
--- tests/core/test_strutils.py	2015-04-13 19:18:40 +0000
+++ tests/core/test_strutils.py	2015-09-17 11:04:29 +0000
@@ -32,3 +32,18 @@
 
         self.assertRaises(ValueError, strutils.bool_from_string, None)
         self.assertRaises(ValueError, strutils.bool_from_string, 'foo')
+
+    def test_bytes_from_string(self):
+        self.assertEqual(strutils.bytes_from_string('3K'), 3072)
+        self.assertEqual(strutils.bytes_from_string('3KB'), 3072)
+        self.assertEqual(strutils.bytes_from_string('3M'), 3145728)
+        self.assertEqual(strutils.bytes_from_string('3MB'), 3145728)
+        self.assertEqual(strutils.bytes_from_string('3G'), 3221225472)
+        self.assertEqual(strutils.bytes_from_string('3GB'), 3221225472)
+        self.assertEqual(strutils.bytes_from_string('3T'), 3298534883328)
+        self.assertEqual(strutils.bytes_from_string('3TB'), 3298534883328)
+        self.assertEqual(strutils.bytes_from_string('3P'), 3377699720527872)
+        self.assertEqual(strutils.bytes_from_string('3PB'), 3377699720527872)
+
+        self.assertRaises(ValueError, strutils.bytes_from_string, None)
+        self.assertRaises(ValueError, strutils.bytes_from_string, 'foo')