Merge lp:~hopem/charms/trusty/mysql/fix-lp1425999 into lp:charms/trusty/mysql
- Trusty Tahr (14.04)
- fix-lp1425999
- Merge into trunk
Status: | Merged |
---|---|
Merged at revision: | 141 |
Proposed branch: | lp:~hopem/charms/trusty/mysql/fix-lp1425999 |
Merge into: | lp:charms/trusty/mysql |
Diff against target: |
366 lines (+160/-69) 5 files modified
Makefile (+5/-5) hooks/charmhelpers/contrib/database/mysql.py (+42/-59) hooks/charmhelpers/contrib/network/ip.py (+84/-1) hooks/charmhelpers/core/services/helpers.py (+12/-4) unit_tests/test_mysql_common.py (+17/-0) |
To merge this branch: | bzr merge lp:~hopem/charms/trusty/mysql/fix-lp1425999 |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Jorge Niedbalski (community) | Approve | ||
Review via email:
|
Commit message
Description of the change
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
uosci-testing-bot (uosci-testing-bot) wrote : | # |
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #2345 mysql for hopem mp251159
LINT OK: passed
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #2265 mysql for hopem mp251159
AMULET FAIL: amulet-test missing
AMULET Results (max last 2 lines):
INFO:root:Search string not found in makefile target commands.
ERROR:root:No make target was executed.
Full amulet test output: http://
Build: http://
- 142. By Edward Hope-Morley
-
synced ch and add unit test
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Jorge Niedbalski (niedbalski) wrote : | # |
Hello Edward,
Thanks for your submission. I fixed the following lint errors:
unit_tests/
unit_tests/
unit_tests/
Merged.
Preview Diff
1 | === modified file 'Makefile' |
2 | --- Makefile 2014-09-26 07:17:08 +0000 |
3 | +++ Makefile 2015-02-27 10:18:28 +0000 |
4 | @@ -1,15 +1,15 @@ |
5 | #!/usr/bin/make |
6 | PYTHON := /usr/bin/env python |
7 | +export PYTHONPATH := hooks |
8 | |
9 | lint: |
10 | @flake8 --exclude hooks/charmhelpers hooks |
11 | -# @flake8 --exclude hooks/charmhelpers unit_tests |
12 | + @flake8 --exclude hooks/charmhelpers unit_tests |
13 | @charm proof |
14 | |
15 | -# TODO: write some unit tests then uncomment this and the flake8 line above. |
16 | -#test: |
17 | -# @echo Starting tests... |
18 | -# @$(PYTHON) /usr/bin/nosetests --nologcapture unit_tests |
19 | +test: |
20 | + @echo Starting tests... |
21 | + @$(PYTHON) /usr/bin/nosetests --nologcapture unit_tests |
22 | |
23 | bin/charm_helpers_sync.py: |
24 | @mkdir -p bin |
25 | |
26 | === modified file 'hooks/charmhelpers/contrib/database/mysql.py' |
27 | --- hooks/charmhelpers/contrib/database/mysql.py 2015-02-25 18:43:26 +0000 |
28 | +++ hooks/charmhelpers/contrib/database/mysql.py 2015-02-27 10:18:28 +0000 |
29 | @@ -15,14 +15,15 @@ |
30 | write_file |
31 | ) |
32 | from charmhelpers.core.hookenv import ( |
33 | + config as config_get, |
34 | relation_get, |
35 | related_units, |
36 | unit_get, |
37 | log, |
38 | DEBUG, |
39 | INFO, |
40 | + WARNING, |
41 | ) |
42 | -from charmhelpers.core.hookenv import config as config_get |
43 | from charmhelpers.fetch import ( |
44 | apt_install, |
45 | apt_update, |
46 | @@ -32,6 +33,7 @@ |
47 | peer_store, |
48 | peer_retrieve, |
49 | ) |
50 | +from charmhelpers.contrib.network.ip import get_host_ip |
51 | |
52 | try: |
53 | import MySQLdb |
54 | @@ -220,6 +222,18 @@ |
55 | """Retrieve or generate mysql root password for service units.""" |
56 | return self.get_mysql_password(username=None, password=password) |
57 | |
58 | + def normalize_address(self, hostname): |
59 | + """Ensure that address returned is an IP address (i.e. not fqdn)""" |
60 | + if config_get('prefer-ipv6'): |
61 | + # TODO: add support for ipv6 dns |
62 | + return hostname |
63 | + |
64 | + if hostname != unit_get('private-address'): |
65 | + return get_host_ip(hostname, fallback=hostname) |
66 | + |
67 | + # Otherwise assume localhost |
68 | + return '127.0.0.1' |
69 | + |
70 | def get_allowed_units(self, database, username, relation_id=None): |
71 | """Get list of units with access grants for database with username. |
72 | |
73 | @@ -247,6 +261,7 @@ |
74 | |
75 | if hosts: |
76 | for host in hosts: |
77 | + host = self.normalize_address(host) |
78 | if self.grant_exists(database, username, host): |
79 | log("Grant exists for host '%s' on db '%s'" % |
80 | (host, database), level=DEBUG) |
81 | @@ -262,21 +277,11 @@ |
82 | |
83 | def configure_db(self, hostname, database, username, admin=False): |
84 | """Configure access to database for username from hostname.""" |
85 | - if config_get('prefer-ipv6'): |
86 | - remote_ip = hostname |
87 | - elif hostname != unit_get('private-address'): |
88 | - try: |
89 | - remote_ip = socket.gethostbyname(hostname) |
90 | - except Exception: |
91 | - # socket.gethostbyname doesn't support ipv6 |
92 | - remote_ip = hostname |
93 | - else: |
94 | - remote_ip = '127.0.0.1' |
95 | - |
96 | self.connect(password=self.get_mysql_root_password()) |
97 | if not self.database_exists(database): |
98 | self.create_database(database) |
99 | |
100 | + remote_ip = self.normalize_address(hostname) |
101 | password = self.get_mysql_password(username) |
102 | if not self.grant_exists(database, username, remote_ip): |
103 | if not admin: |
104 | @@ -289,9 +294,11 @@ |
105 | |
106 | class PerconaClusterHelper(object): |
107 | |
108 | - # Going for the biggest page size to avoid wasted bytes. InnoDB page size is |
109 | - # 16MB |
110 | + # Going for the biggest page size to avoid wasted bytes. |
111 | + # InnoDB page size is 16MB |
112 | + |
113 | DEFAULT_PAGE_SIZE = 16 * 1024 * 1024 |
114 | + DEFAULT_INNODB_BUFFER_FACTOR = 0.50 |
115 | |
116 | def human_to_bytes(self, human): |
117 | """Convert human readable configuration options to bytes.""" |
118 | @@ -352,51 +359,27 @@ |
119 | if 'max-connections' in config: |
120 | mysql_config['max_connections'] = config['max-connections'] |
121 | |
122 | - # Total memory available for dataset |
123 | - dataset_bytes = self.human_to_bytes(config['dataset-size']) |
124 | - mysql_config['dataset_bytes'] = dataset_bytes |
125 | - |
126 | - if 'query-cache-type' in config: |
127 | - # Query Cache Configuration |
128 | - mysql_config['query_cache_size'] = config['query-cache-size'] |
129 | - if (config['query-cache-size'] == -1 and |
130 | - config['query-cache-type'] in ['ON', 'DEMAND']): |
131 | - # Calculate the query cache size automatically |
132 | - qcache_bytes = (dataset_bytes * 0.20) |
133 | - qcache_bytes = int(qcache_bytes - |
134 | - (qcache_bytes % self.DEFAULT_PAGE_SIZE)) |
135 | - mysql_config['query_cache_size'] = qcache_bytes |
136 | - dataset_bytes -= qcache_bytes |
137 | - |
138 | - # 5.5 allows the words, but not 5.1 |
139 | - if config['query-cache-type'] == 'ON': |
140 | - mysql_config['query_cache_type'] = 1 |
141 | - elif config['query-cache-type'] == 'DEMAND': |
142 | - mysql_config['query_cache_type'] = 2 |
143 | - else: |
144 | - mysql_config['query_cache_type'] = 0 |
145 | - |
146 | # Set a sane default key_buffer size |
147 | mysql_config['key_buffer'] = self.human_to_bytes('32M') |
148 | - |
149 | - if 'preferred-storage-engine' in config: |
150 | - # Storage engine configuration |
151 | - preferred_engines = config['preferred-storage-engine'].split(',') |
152 | - chunk_size = int(dataset_bytes / len(preferred_engines)) |
153 | - mysql_config['innodb_flush_log_at_trx_commit'] = 1 |
154 | - mysql_config['sync_binlog'] = 1 |
155 | - if 'InnoDB' in preferred_engines: |
156 | - mysql_config['innodb_buffer_pool_size'] = chunk_size |
157 | - if config['tuning-level'] == 'fast': |
158 | - mysql_config['innodb_flush_log_at_trx_commit'] = 2 |
159 | - else: |
160 | - mysql_config['innodb_buffer_pool_size'] = 0 |
161 | - |
162 | - mysql_config['default_storage_engine'] = preferred_engines[0] |
163 | - if 'MyISAM' in preferred_engines: |
164 | - mysql_config['key_buffer'] = chunk_size |
165 | - |
166 | - if config['tuning-level'] == 'fast': |
167 | - mysql_config['sync_binlog'] = 0 |
168 | - |
169 | + total_memory = self.human_to_bytes(self.get_mem_total()) |
170 | + |
171 | + log("Option 'dataset-size' has been deprecated, instead by default %d%% of system \ |
172 | + available RAM will be used for innodb_buffer_pool_size allocation" % |
173 | + (self.DEFAULT_INNODB_BUFFER_FACTOR * 100), level="WARN") |
174 | + |
175 | + innodb_buffer_pool_size = config.get('innodb-buffer-pool-size', None) |
176 | + |
177 | + if innodb_buffer_pool_size: |
178 | + innodb_buffer_pool_size = self.human_to_bytes( |
179 | + innodb_buffer_pool_size) |
180 | + |
181 | + if innodb_buffer_pool_size > total_memory: |
182 | + log("innodb_buffer_pool_size; {} is greater than system available memory:{}".format( |
183 | + innodb_buffer_pool_size, |
184 | + total_memory), level='WARN') |
185 | + else: |
186 | + innodb_buffer_pool_size = int( |
187 | + total_memory * self.DEFAULT_INNODB_BUFFER_FACTOR) |
188 | + |
189 | + mysql_config['innodb_buffer_pool_size'] = innodb_buffer_pool_size |
190 | return mysql_config |
191 | |
192 | === modified file 'hooks/charmhelpers/contrib/network/ip.py' |
193 | --- hooks/charmhelpers/contrib/network/ip.py 2015-02-04 19:09:09 +0000 |
194 | +++ hooks/charmhelpers/contrib/network/ip.py 2015-02-27 10:18:28 +0000 |
195 | @@ -17,13 +17,16 @@ |
196 | import glob |
197 | import re |
198 | import subprocess |
199 | +import six |
200 | +import socket |
201 | |
202 | from functools import partial |
203 | |
204 | from charmhelpers.core.hookenv import unit_get |
205 | from charmhelpers.fetch import apt_install |
206 | from charmhelpers.core.hookenv import ( |
207 | - log |
208 | + log, |
209 | + WARNING, |
210 | ) |
211 | |
212 | try: |
213 | @@ -365,3 +368,83 @@ |
214 | return True |
215 | |
216 | return False |
217 | + |
218 | + |
219 | +def is_ip(address): |
220 | + """ |
221 | + Returns True if address is a valid IP address. |
222 | + """ |
223 | + try: |
224 | + # Test to see if already an IPv4 address |
225 | + socket.inet_aton(address) |
226 | + return True |
227 | + except socket.error: |
228 | + return False |
229 | + |
230 | + |
231 | +def ns_query(address): |
232 | + try: |
233 | + import dns.resolver |
234 | + except ImportError: |
235 | + apt_install('python-dnspython') |
236 | + import dns.resolver |
237 | + |
238 | + if isinstance(address, dns.name.Name): |
239 | + rtype = 'PTR' |
240 | + elif isinstance(address, six.string_types): |
241 | + rtype = 'A' |
242 | + else: |
243 | + return None |
244 | + |
245 | + answers = dns.resolver.query(address, rtype) |
246 | + if answers: |
247 | + return str(answers[0]) |
248 | + return None |
249 | + |
250 | + |
251 | +def get_host_ip(hostname, fallback=None): |
252 | + """ |
253 | + Resolves the IP for a given hostname, or returns |
254 | + the input if it is already an IP. |
255 | + """ |
256 | + if is_ip(hostname): |
257 | + return hostname |
258 | + |
259 | + ip_addr = ns_query(hostname) |
260 | + if not ip_addr: |
261 | + try: |
262 | + ip_addr = socket.gethostbyname(hostname) |
263 | + except: |
264 | + log("Failed to resolve hostname '%s'" % (hostname), |
265 | + level=WARNING) |
266 | + return fallback |
267 | + return ip_addr |
268 | + |
269 | + |
270 | +def get_hostname(address, fqdn=True): |
271 | + """ |
272 | + Resolves hostname for given IP, or returns the input |
273 | + if it is already a hostname. |
274 | + """ |
275 | + if is_ip(address): |
276 | + try: |
277 | + import dns.reversename |
278 | + except ImportError: |
279 | + apt_install("python-dnspython") |
280 | + import dns.reversename |
281 | + |
282 | + rev = dns.reversename.from_address(address) |
283 | + result = ns_query(rev) |
284 | + if not result: |
285 | + return None |
286 | + else: |
287 | + result = address |
288 | + |
289 | + if fqdn: |
290 | + # strip trailing . |
291 | + if result.endswith('.'): |
292 | + return result[:-1] |
293 | + else: |
294 | + return result |
295 | + else: |
296 | + return result.split('.')[0] |
297 | |
298 | === modified file 'hooks/charmhelpers/core/services/helpers.py' |
299 | --- hooks/charmhelpers/core/services/helpers.py 2015-02-04 19:09:09 +0000 |
300 | +++ hooks/charmhelpers/core/services/helpers.py 2015-02-27 10:18:28 +0000 |
301 | @@ -45,12 +45,14 @@ |
302 | """ |
303 | name = None |
304 | interface = None |
305 | - required_keys = [] |
306 | |
307 | def __init__(self, name=None, additional_required_keys=None): |
308 | + if not hasattr(self, 'required_keys'): |
309 | + self.required_keys = [] |
310 | + |
311 | if name is not None: |
312 | self.name = name |
313 | - if additional_required_keys is not None: |
314 | + if additional_required_keys: |
315 | self.required_keys.extend(additional_required_keys) |
316 | self.get_data() |
317 | |
318 | @@ -134,7 +136,10 @@ |
319 | """ |
320 | name = 'db' |
321 | interface = 'mysql' |
322 | - required_keys = ['host', 'user', 'password', 'database'] |
323 | + |
324 | + def __init__(self, *args, **kwargs): |
325 | + self.required_keys = ['host', 'user', 'password', 'database'] |
326 | + super(HttpRelation).__init__(self, *args, **kwargs) |
327 | |
328 | |
329 | class HttpRelation(RelationContext): |
330 | @@ -146,7 +151,10 @@ |
331 | """ |
332 | name = 'website' |
333 | interface = 'http' |
334 | - required_keys = ['host', 'port'] |
335 | + |
336 | + def __init__(self, *args, **kwargs): |
337 | + self.required_keys = ['host', 'port'] |
338 | + super(HttpRelation).__init__(self, *args, **kwargs) |
339 | |
340 | def provide_data(self): |
341 | return { |
342 | |
343 | === added directory 'unit_tests' |
344 | === added file 'unit_tests/__init__.py' |
345 | === added file 'unit_tests/test_mysql_common.py' |
346 | --- unit_tests/test_mysql_common.py 1970-01-01 00:00:00 +0000 |
347 | +++ unit_tests/test_mysql_common.py 2015-02-27 10:18:28 +0000 |
348 | @@ -0,0 +1,17 @@ |
349 | +import mock |
350 | +import unittest |
351 | +import sys |
352 | + |
353 | +sys.modules['MySQLdb'] = mock.Mock() |
354 | +import common |
355 | + |
356 | + |
357 | +class CommonTests(unittest.TestCase): |
358 | + |
359 | + @mock.patch.object(common, 'MySQLHelper', spec=common.MySQLHelper) |
360 | + def test_get_db_helper(self, mock_helper): |
361 | + h = common.get_db_helper() |
362 | + kwargs = {'rpasswdf_template': '/var/lib/mysql/mysql.passwd', |
363 | + 'upasswdf_template': '/var/lib/mysql/mysql-{}.passwd', |
364 | + 'delete_ondisk_passwd_file': False} |
365 | + mock_helper.assert_called_with(**kwargs) |
366 | \ No newline at end of file |
charm_unit_test #2134 mysql for hopem mp251159
UNIT FAIL: unit-test missing
UNIT Results (max last 2 lines):
INFO:root:Search string not found in makefile target commands.
ERROR:root:No make target was executed.
Full unit test output: http:// paste.ubuntu. com/10435847/ 10.245. 162.77: 8080/job/ charm_unit_ test/2134/
Build: http://