Merge lp:~hopem/charms/trusty/mysql/fix-lp1425999 into lp:charms/trusty/mysql

[Edward Hope-Morley]

[uosci-testing-bot (uosci-testing-bot)]

charm_unit_test #2134 mysql for hopem mp251159
    UNIT FAIL: unit-test missing

UNIT Results (max last 2 lines):
INFO:root:Search string not found in makefile target commands.
ERROR:root:No make target was executed.

Full unit test output: http://paste.ubuntu.com/10435847/
Build: http://10.245.162.77:8080/job/charm_unit_test/2134/

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #2345 mysql for hopem mp251159
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/2345/

[uosci-testing-bot (uosci-testing-bot)]

charm_amulet_test #2265 mysql for hopem mp251159
    AMULET FAIL: amulet-test missing

AMULET Results (max last 2 lines):
INFO:root:Search string not found in makefile target commands.
ERROR:root:No make target was executed.

Full amulet test output: http://paste.ubuntu.com/10435874/
Build: http://10.245.162.77:8080/job/charm_amulet_test/2265/

[lp:~hopem/charms/trusty/mysql/fix-lp1425999]

142. By Edward Hope-Morley on 2015-02-27

synced ch and add unit test

[Jorge Niedbalski (niedbalski)]

Hello Edward,

Thanks for your submission. I fixed the following lint errors:

unit_tests/test_mysql_common.py:6:1: E402 module level import not at top of file
unit_tests/test_mysql_common.py:13:9: F841 local variable 'h' is assigned to but never used
unit_tests/test_mysql_common.py:17:49: W292 no newline at end of file

Merged.

review: Approve

=== modified file 'Makefile'
--- Makefile	2014-09-26 07:17:08 +0000
+++ Makefile	2015-02-27 10:18:28 +0000
@@ -1,15 +1,15 @@
 #!/usr/bin/make
 PYTHON := /usr/bin/env python
+export PYTHONPATH := hooks
 
 lint:
 	@flake8 --exclude hooks/charmhelpers hooks
-#	@flake8 --exclude hooks/charmhelpers unit_tests
+	@flake8 --exclude hooks/charmhelpers unit_tests
 	@charm proof
 
-# TODO: write some unit tests then uncomment this and the flake8 line above.
-#test:
-#	@echo Starting tests...
-#	@$(PYTHON) /usr/bin/nosetests --nologcapture unit_tests
+test:
+	@echo Starting tests...
+	@$(PYTHON) /usr/bin/nosetests --nologcapture unit_tests
 
 bin/charm_helpers_sync.py:
 	@mkdir -p bin

=== modified file 'hooks/charmhelpers/contrib/database/mysql.py'
--- hooks/charmhelpers/contrib/database/mysql.py	2015-02-25 18:43:26 +0000
+++ hooks/charmhelpers/contrib/database/mysql.py	2015-02-27 10:18:28 +0000
@@ -15,14 +15,15 @@
     write_file
 )
 from charmhelpers.core.hookenv import (
+    config as config_get,
     relation_get,
     related_units,
     unit_get,
     log,
     DEBUG,
     INFO,
+    WARNING,
 )
-from charmhelpers.core.hookenv import config as config_get
 from charmhelpers.fetch import (
     apt_install,
     apt_update,
@@ -32,6 +33,7 @@
     peer_store,
     peer_retrieve,
 )
+from charmhelpers.contrib.network.ip import get_host_ip
 
 try:
     import MySQLdb
@@ -220,6 +222,18 @@
         """Retrieve or generate mysql root password for service units."""
         return self.get_mysql_password(username=None, password=password)
 
+    def normalize_address(self, hostname):
+        """Ensure that address returned is an IP address (i.e. not fqdn)"""
+        if config_get('prefer-ipv6'):
+            # TODO: add support for ipv6 dns
+            return hostname
+
+        if hostname != unit_get('private-address'):
+            return get_host_ip(hostname, fallback=hostname)
+
+        # Otherwise assume localhost
+        return '127.0.0.1'
+
     def get_allowed_units(self, database, username, relation_id=None):
         """Get list of units with access grants for database with username.
 
@@ -247,6 +261,7 @@
 
             if hosts:
                 for host in hosts:
+                    host = self.normalize_address(host)
                     if self.grant_exists(database, username, host):
                         log("Grant exists for host '%s' on db '%s'" %
                             (host, database), level=DEBUG)
@@ -262,21 +277,11 @@
 
     def configure_db(self, hostname, database, username, admin=False):
         """Configure access to database for username from hostname."""
-        if config_get('prefer-ipv6'):
-            remote_ip = hostname
-        elif hostname != unit_get('private-address'):
-            try:
-                remote_ip = socket.gethostbyname(hostname)
-            except Exception:
-                # socket.gethostbyname doesn't support ipv6
-                remote_ip = hostname
-        else:
-            remote_ip = '127.0.0.1'
-
         self.connect(password=self.get_mysql_root_password())
         if not self.database_exists(database):
             self.create_database(database)
 
+        remote_ip = self.normalize_address(hostname)
         password = self.get_mysql_password(username)
         if not self.grant_exists(database, username, remote_ip):
             if not admin:
@@ -289,9 +294,11 @@
 
 class PerconaClusterHelper(object):
 
-    # Going for the biggest page size to avoid wasted bytes. InnoDB page size is
-    # 16MB
+    # Going for the biggest page size to avoid wasted bytes.
+    # InnoDB page size is 16MB
+
     DEFAULT_PAGE_SIZE = 16 * 1024 * 1024
+    DEFAULT_INNODB_BUFFER_FACTOR = 0.50
 
     def human_to_bytes(self, human):
         """Convert human readable configuration options to bytes."""
@@ -352,51 +359,27 @@
         if 'max-connections' in config:
             mysql_config['max_connections'] = config['max-connections']
 
-        # Total memory available for dataset
-        dataset_bytes = self.human_to_bytes(config['dataset-size'])
-        mysql_config['dataset_bytes'] = dataset_bytes
-
-        if 'query-cache-type' in config:
-            # Query Cache Configuration
-            mysql_config['query_cache_size'] = config['query-cache-size']
-            if (config['query-cache-size'] == -1 and
-                    config['query-cache-type'] in ['ON', 'DEMAND']):
-                # Calculate the query cache size automatically
-                qcache_bytes = (dataset_bytes * 0.20)
-                qcache_bytes = int(qcache_bytes -
-                                   (qcache_bytes % self.DEFAULT_PAGE_SIZE))
-                mysql_config['query_cache_size'] = qcache_bytes
-                dataset_bytes -= qcache_bytes
-
-            # 5.5 allows the words, but not 5.1
-            if config['query-cache-type'] == 'ON':
-                mysql_config['query_cache_type'] = 1
-            elif config['query-cache-type'] == 'DEMAND':
-                mysql_config['query_cache_type'] = 2
-            else:
-                mysql_config['query_cache_type'] = 0
-
         # Set a sane default key_buffer size
         mysql_config['key_buffer'] = self.human_to_bytes('32M')
-
-        if 'preferred-storage-engine' in config:
-            # Storage engine configuration
-            preferred_engines = config['preferred-storage-engine'].split(',')
-            chunk_size = int(dataset_bytes / len(preferred_engines))
-            mysql_config['innodb_flush_log_at_trx_commit'] = 1
-            mysql_config['sync_binlog'] = 1
-            if 'InnoDB' in preferred_engines:
-                mysql_config['innodb_buffer_pool_size'] = chunk_size
-                if config['tuning-level'] == 'fast':
-                    mysql_config['innodb_flush_log_at_trx_commit'] = 2
-            else:
-                mysql_config['innodb_buffer_pool_size'] = 0
-
-            mysql_config['default_storage_engine'] = preferred_engines[0]
-            if 'MyISAM' in preferred_engines:
-                mysql_config['key_buffer'] = chunk_size
-
-            if config['tuning-level'] == 'fast':
-                mysql_config['sync_binlog'] = 0
-
+        total_memory = self.human_to_bytes(self.get_mem_total())
+
+        log("Option 'dataset-size' has been deprecated, instead by default %d%% of system \
+        available RAM will be used for innodb_buffer_pool_size allocation" %
+            (self.DEFAULT_INNODB_BUFFER_FACTOR * 100), level="WARN")
+
+        innodb_buffer_pool_size = config.get('innodb-buffer-pool-size', None)
+
+        if innodb_buffer_pool_size:
+            innodb_buffer_pool_size = self.human_to_bytes(
+                innodb_buffer_pool_size)
+
+            if innodb_buffer_pool_size > total_memory:
+                log("innodb_buffer_pool_size; {} is greater than system available memory:{}".format(
+                    innodb_buffer_pool_size,
+                    total_memory), level='WARN')
+        else:
+            innodb_buffer_pool_size = int(
+                total_memory * self.DEFAULT_INNODB_BUFFER_FACTOR)
+
+        mysql_config['innodb_buffer_pool_size'] = innodb_buffer_pool_size
         return mysql_config

=== modified file 'hooks/charmhelpers/contrib/network/ip.py'
--- hooks/charmhelpers/contrib/network/ip.py	2015-02-04 19:09:09 +0000
+++ hooks/charmhelpers/contrib/network/ip.py	2015-02-27 10:18:28 +0000
@@ -17,13 +17,16 @@
 import glob
 import re
 import subprocess
+import six
+import socket
 
 from functools import partial
 
 from charmhelpers.core.hookenv import unit_get
 from charmhelpers.fetch import apt_install
 from charmhelpers.core.hookenv import (
-    log
+    log,
+    WARNING,
 )
 
 try:
@@ -365,3 +368,83 @@
             return True
 
     return False
+
+
+def is_ip(address):
+    """
+    Returns True if address is a valid IP address.
+    """
+    try:
+        # Test to see if already an IPv4 address
+        socket.inet_aton(address)
+        return True
+    except socket.error:
+        return False
+
+
+def ns_query(address):
+    try:
+        import dns.resolver
+    except ImportError:
+        apt_install('python-dnspython')
+        import dns.resolver
+
+    if isinstance(address, dns.name.Name):
+        rtype = 'PTR'
+    elif isinstance(address, six.string_types):
+        rtype = 'A'
+    else:
+        return None
+
+    answers = dns.resolver.query(address, rtype)
+    if answers:
+        return str(answers[0])
+    return None
+
+
+def get_host_ip(hostname, fallback=None):
+    """
+    Resolves the IP for a given hostname, or returns
+    the input if it is already an IP.
+    """
+    if is_ip(hostname):
+        return hostname
+
+    ip_addr = ns_query(hostname)
+    if not ip_addr:
+        try:
+            ip_addr = socket.gethostbyname(hostname)
+        except:
+            log("Failed to resolve hostname '%s'" % (hostname),
+                level=WARNING)
+            return fallback
+    return ip_addr
+
+
+def get_hostname(address, fqdn=True):
+    """
+    Resolves hostname for given IP, or returns the input
+    if it is already a hostname.
+    """
+    if is_ip(address):
+        try:
+            import dns.reversename
+        except ImportError:
+            apt_install("python-dnspython")
+            import dns.reversename
+
+        rev = dns.reversename.from_address(address)
+        result = ns_query(rev)
+        if not result:
+            return None
+    else:
+        result = address
+
+    if fqdn:
+        # strip trailing .
+        if result.endswith('.'):
+            return result[:-1]
+        else:
+            return result
+    else:
+        return result.split('.')[0]

=== modified file 'hooks/charmhelpers/core/services/helpers.py'
--- hooks/charmhelpers/core/services/helpers.py	2015-02-04 19:09:09 +0000
+++ hooks/charmhelpers/core/services/helpers.py	2015-02-27 10:18:28 +0000
@@ -45,12 +45,14 @@
     """
     name = None
     interface = None
-    required_keys = []
 
     def __init__(self, name=None, additional_required_keys=None):
+        if not hasattr(self, 'required_keys'):
+            self.required_keys = []
+
         if name is not None:
             self.name = name
-        if additional_required_keys is not None:
+        if additional_required_keys:
             self.required_keys.extend(additional_required_keys)
         self.get_data()
 
@@ -134,7 +136,10 @@
     """
     name = 'db'
     interface = 'mysql'
-    required_keys = ['host', 'user', 'password', 'database']
+
+    def __init__(self, *args, **kwargs):
+        self.required_keys = ['host', 'user', 'password', 'database']
+        super(HttpRelation).__init__(self, *args, **kwargs)
 
 
 class HttpRelation(RelationContext):
@@ -146,7 +151,10 @@
     """
     name = 'website'
     interface = 'http'
-    required_keys = ['host', 'port']
+
+    def __init__(self, *args, **kwargs):
+        self.required_keys = ['host', 'port']
+        super(HttpRelation).__init__(self, *args, **kwargs)
 
     def provide_data(self):
         return {

=== added directory 'unit_tests'
=== added file 'unit_tests/__init__.py'
=== added file 'unit_tests/test_mysql_common.py'
--- unit_tests/test_mysql_common.py	1970-01-01 00:00:00 +0000
+++ unit_tests/test_mysql_common.py	2015-02-27 10:18:28 +0000
@@ -0,0 +1,17 @@
+import mock
+import unittest
+import sys
+
+sys.modules['MySQLdb'] = mock.Mock()
+import common
+
+
+class CommonTests(unittest.TestCase):
+
+    @mock.patch.object(common, 'MySQLHelper', spec=common.MySQLHelper)
+    def test_get_db_helper(self, mock_helper):
+        h = common.get_db_helper()
+        kwargs = {'rpasswdf_template': '/var/lib/mysql/mysql.passwd',
+                  'upasswdf_template': '/var/lib/mysql/mysql-{}.passwd',
+                  'delete_ondisk_passwd_file': False}
+        mock_helper.assert_called_with(**kwargs)
\ No newline at end of file