Merge lp:~james-page/charm-helpers/misc-percona-changes into lp:charm-helpers

[James Page]

Update default unset memory configuration behaviour and fixup unit tests.

[lp:~james-page/charm-helpers/misc-percona-changes]

698. By James Page on 2017-02-28

Amend NOTE

[James Page (james-page)]

This review also fixes up the mysql tests to work under py2 and py3

[Jorge Niedbalski (niedbalski)]

James,

Thanks for this addition. I agree with your rationale on this change.

Regarding tests, all the unit tests are passing OK.

LGTM

review: Approve

[Ryan Beisner (1chb1n)]

Also good with this, many thanks.

review: Approve

=== modified file 'charmhelpers/contrib/database/mysql.py'
--- charmhelpers/contrib/database/mysql.py	2017-02-16 03:04:17 +0000
+++ charmhelpers/contrib/database/mysql.py	2017-02-28 12:04:55 +0000
@@ -334,6 +334,7 @@
 
     DEFAULT_PAGE_SIZE = 16 * 1024 * 1024
     DEFAULT_INNODB_BUFFER_FACTOR = 0.50
+    DEFAULT_INNODB_BUFFER_SIZE_MAX = 512 * 1024 * 1024
 
     def human_to_bytes(self, human):
         """Convert human readable configuration options to bytes."""
@@ -416,8 +417,14 @@
             innodb_buffer_pool_size = self.human_to_bytes(
                 dataset_bytes)
         else:
-            innodb_buffer_pool_size = int(
-                total_memory * self.DEFAULT_INNODB_BUFFER_FACTOR)
+            # NOTE(jamespage): pick the smallest of 50% of RAM or 512MB
+            #                  to ensure that deployments in containers
+            #                  without constraints don't try to consume
+            #                  silly amounts of memory.
+            innodb_buffer_pool_size = min(
+                int(total_memory * self.DEFAULT_INNODB_BUFFER_FACTOR),
+                self.DEFAULT_INNODB_BUFFER_SIZE_MAX
+            )
 
         if innodb_buffer_pool_size > total_memory:
             log("innodb_buffer_pool_size; {} is greater than system available memory:{}".format(

=== modified file 'test_requirements.txt'
--- test_requirements.txt	2017-01-25 17:11:37 +0000
+++ test_requirements.txt	2017-02-28 12:04:55 +0000
@@ -17,3 +17,4 @@
 netifaces==0.10    # trusty is 0.8, but using py3 compatible version for tests.
 Jinja2==2.6        # precise
 psutil==1.2.1      # trusty
+dnspython==1.11.1  # trusty

=== added file 'tests/contrib/database/__init__.py'
=== modified file 'tests/contrib/database/test_mysql.py'
--- tests/contrib/database/test_mysql.py	2015-06-03 20:20:53 +0000
+++ tests/contrib/database/test_mysql.py	2017-02-28 12:04:55 +0000
@@ -50,23 +50,20 @@
         units = helper.get_allowed_units('dbA', 'userA')
 
         calls = [mock.call('dbA', 'userA', 'hostA'),
-                 mock.call().__nonzero__(),
                  mock.call('dbA', 'userA', '10.0.0.2'),
-                 mock.call().__nonzero__(),
                  mock.call('dbA', 'userA', '2001:db8:1::2'),
-                 mock.call().__nonzero__(),
-                 mock.call('dbA', 'userA', '10.0.0.3'),
-                 mock.call().__nonzero__()]
+                 mock.call('dbA', 'userA', '10.0.0.3')]
 
-        helper.grant_exists.assert_has_calls(calls)
+        helper.grant_exists.assert_has_calls(calls, any_order=True)
         self.assertEqual(units, set(['unit/0', 'unit/1', 'unit/2']))
 
+    @mock.patch('charmhelpers.contrib.network.ip.ns_query')
     @mock.patch('charmhelpers.contrib.network.ip.socket')
     @mock.patch.object(mysql, 'unit_get')
     @mock.patch.object(mysql, 'config_get')
     @mock.patch.object(mysql, 'log')
     def test_normalize_address(self, mock_log, mock_config_get, mock_unit_get,
-                               mock_socket):
+                               mock_socket, mock_ns_query):
         helper = mysql.MySQLHelper('foo', 'bar', host='hostA')
         # prefer-ipv6
         mock_config_get.return_value = False
@@ -88,6 +85,7 @@
         mock_config_get.assert_called_with('prefer-ipv6')
 
         mock_socket.gethostbyname.side_effect = Exception
+        mock_ns_query.return_value = None
         out = helper.normalize_address('unresolvable')
         self.assertEqual('unresolvable', out)
         mock_config_get.assert_called_with('prefer-ipv6')
@@ -187,7 +185,8 @@
                      mock.call('userA.passwd', '2'),
                      mock.call('mysql-userA.passwd', '3')]
 
-            mock_peer_store.assert_has_calls(calls)
+            mock_peer_store.assert_has_calls(calls,
+                                             any_order=True)
         finally:
             os.rmdir(tmpdir)
 
@@ -231,8 +230,7 @@
 
         self.assertEqual(
             mysql_config.get('innodb_buffer_pool_size'),
-            int(helper.human_to_bytes(mem.return_value) *
-                helper.DEFAULT_INNODB_BUFFER_FACTOR))
+            helper.DEFAULT_INNODB_BUFFER_SIZE_MAX)
 
     @mock.patch.object(mysql.PerconaClusterHelper, 'get_mem_total')
     @mock.patch.object(mysql, 'config_get')
@@ -248,7 +246,25 @@
         mysql_config = helper.parse_config()
 
         self.assertEqual(
-            mysql_config.get('innodb-buffer-pool-size'),
+            mysql_config.get('innodb_buffer_pool_size'),
+            helper.DEFAULT_INNODB_BUFFER_SIZE_MAX)
+
+    @mock.patch.object(mysql.PerconaClusterHelper, 'get_mem_total')
+    @mock.patch.object(mysql, 'config_get')
+    @mock.patch.object(mysql, 'log')
+    def test_parse_config_innodb_buffer_unset_small(self, mog,
+                                                    config, mem):
+        mem.return_value = "512M"
+        config.return_value = {
+            'innodb-buffer-pool-size': None,
+            'dataset-size': None,
+        }
+
+        helper = mysql.PerconaClusterHelper()
+        mysql_config = helper.parse_config()
+
+        self.assertEqual(
+            mysql_config.get('innodb_buffer_pool_size'),
             int(helper.human_to_bytes(mem.return_value) *
                 helper.DEFAULT_INNODB_BUFFER_FACTOR))
 
@@ -265,7 +281,7 @@
         mysql_config = helper.parse_config()
 
         self.assertEqual(
-            mysql_config.get('innodb-buffer-pool-size'),
+            mysql_config.get('innodb_buffer_pool_size'),
             int(helper.human_to_bytes("10G")))
 
     @mock.patch.object(mysql.PerconaClusterHelper, 'get_mem_total')