Merge lp:~hopem/charms/trusty/mysql/fix-lp1425999 into lp:charms/trusty/mysql
- Trusty Tahr (14.04)
- fix-lp1425999
- Merge into trunk
Status: | Merged |
---|---|
Merged at revision: | 141 |
Proposed branch: | lp:~hopem/charms/trusty/mysql/fix-lp1425999 |
Merge into: | lp:charms/trusty/mysql |
Diff against target: |
366 lines (+160/-69) 5 files modified
Makefile (+5/-5) hooks/charmhelpers/contrib/database/mysql.py (+42/-59) hooks/charmhelpers/contrib/network/ip.py (+84/-1) hooks/charmhelpers/core/services/helpers.py (+12/-4) unit_tests/test_mysql_common.py (+17/-0) |
To merge this branch: | bzr merge lp:~hopem/charms/trusty/mysql/fix-lp1425999 |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Jorge Niedbalski (community) | Approve | ||
Review via email: mp+251159@code.launchpad.net |
Commit message
Description of the change
uosci-testing-bot (uosci-testing-bot) wrote : | # |
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #2345 mysql for hopem mp251159
LINT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #2265 mysql for hopem mp251159
AMULET FAIL: amulet-test missing
AMULET Results (max last 2 lines):
INFO:root:Search string not found in makefile target commands.
ERROR:root:No make target was executed.
Full amulet test output: http://
Build: http://
- 142. By Edward Hope-Morley
-
synced ch and add unit test
Jorge Niedbalski (niedbalski) wrote : | # |
Hello Edward,
Thanks for your submission. I fixed the following lint errors:
unit_tests/
unit_tests/
unit_tests/
Merged.
Preview Diff
1 | === modified file 'Makefile' | |||
2 | --- Makefile 2014-09-26 07:17:08 +0000 | |||
3 | +++ Makefile 2015-02-27 10:18:28 +0000 | |||
4 | @@ -1,15 +1,15 @@ | |||
5 | 1 | #!/usr/bin/make | 1 | #!/usr/bin/make |
6 | 2 | PYTHON := /usr/bin/env python | 2 | PYTHON := /usr/bin/env python |
7 | 3 | export PYTHONPATH := hooks | ||
8 | 3 | 4 | ||
9 | 4 | lint: | 5 | lint: |
10 | 5 | @flake8 --exclude hooks/charmhelpers hooks | 6 | @flake8 --exclude hooks/charmhelpers hooks |
12 | 6 | # @flake8 --exclude hooks/charmhelpers unit_tests | 7 | @flake8 --exclude hooks/charmhelpers unit_tests |
13 | 7 | @charm proof | 8 | @charm proof |
14 | 8 | 9 | ||
19 | 9 | # TODO: write some unit tests then uncomment this and the flake8 line above. | 10 | test: |
20 | 10 | #test: | 11 | @echo Starting tests... |
21 | 11 | # @echo Starting tests... | 12 | @$(PYTHON) /usr/bin/nosetests --nologcapture unit_tests |
18 | 12 | # @$(PYTHON) /usr/bin/nosetests --nologcapture unit_tests | ||
22 | 13 | 13 | ||
23 | 14 | bin/charm_helpers_sync.py: | 14 | bin/charm_helpers_sync.py: |
24 | 15 | @mkdir -p bin | 15 | @mkdir -p bin |
25 | 16 | 16 | ||
26 | === modified file 'hooks/charmhelpers/contrib/database/mysql.py' | |||
27 | --- hooks/charmhelpers/contrib/database/mysql.py 2015-02-25 18:43:26 +0000 | |||
28 | +++ hooks/charmhelpers/contrib/database/mysql.py 2015-02-27 10:18:28 +0000 | |||
29 | @@ -15,14 +15,15 @@ | |||
30 | 15 | write_file | 15 | write_file |
31 | 16 | ) | 16 | ) |
32 | 17 | from charmhelpers.core.hookenv import ( | 17 | from charmhelpers.core.hookenv import ( |
33 | 18 | config as config_get, | ||
34 | 18 | relation_get, | 19 | relation_get, |
35 | 19 | related_units, | 20 | related_units, |
36 | 20 | unit_get, | 21 | unit_get, |
37 | 21 | log, | 22 | log, |
38 | 22 | DEBUG, | 23 | DEBUG, |
39 | 23 | INFO, | 24 | INFO, |
40 | 25 | WARNING, | ||
41 | 24 | ) | 26 | ) |
42 | 25 | from charmhelpers.core.hookenv import config as config_get | ||
43 | 26 | from charmhelpers.fetch import ( | 27 | from charmhelpers.fetch import ( |
44 | 27 | apt_install, | 28 | apt_install, |
45 | 28 | apt_update, | 29 | apt_update, |
46 | @@ -32,6 +33,7 @@ | |||
47 | 32 | peer_store, | 33 | peer_store, |
48 | 33 | peer_retrieve, | 34 | peer_retrieve, |
49 | 34 | ) | 35 | ) |
50 | 36 | from charmhelpers.contrib.network.ip import get_host_ip | ||
51 | 35 | 37 | ||
52 | 36 | try: | 38 | try: |
53 | 37 | import MySQLdb | 39 | import MySQLdb |
54 | @@ -220,6 +222,18 @@ | |||
55 | 220 | """Retrieve or generate mysql root password for service units.""" | 222 | """Retrieve or generate mysql root password for service units.""" |
56 | 221 | return self.get_mysql_password(username=None, password=password) | 223 | return self.get_mysql_password(username=None, password=password) |
57 | 222 | 224 | ||
58 | 225 | def normalize_address(self, hostname): | ||
59 | 226 | """Ensure that address returned is an IP address (i.e. not fqdn)""" | ||
60 | 227 | if config_get('prefer-ipv6'): | ||
61 | 228 | # TODO: add support for ipv6 dns | ||
62 | 229 | return hostname | ||
63 | 230 | |||
64 | 231 | if hostname != unit_get('private-address'): | ||
65 | 232 | return get_host_ip(hostname, fallback=hostname) | ||
66 | 233 | |||
67 | 234 | # Otherwise assume localhost | ||
68 | 235 | return '127.0.0.1' | ||
69 | 236 | |||
70 | 223 | def get_allowed_units(self, database, username, relation_id=None): | 237 | def get_allowed_units(self, database, username, relation_id=None): |
71 | 224 | """Get list of units with access grants for database with username. | 238 | """Get list of units with access grants for database with username. |
72 | 225 | 239 | ||
73 | @@ -247,6 +261,7 @@ | |||
74 | 247 | 261 | ||
75 | 248 | if hosts: | 262 | if hosts: |
76 | 249 | for host in hosts: | 263 | for host in hosts: |
77 | 264 | host = self.normalize_address(host) | ||
78 | 250 | if self.grant_exists(database, username, host): | 265 | if self.grant_exists(database, username, host): |
79 | 251 | log("Grant exists for host '%s' on db '%s'" % | 266 | log("Grant exists for host '%s' on db '%s'" % |
80 | 252 | (host, database), level=DEBUG) | 267 | (host, database), level=DEBUG) |
81 | @@ -262,21 +277,11 @@ | |||
82 | 262 | 277 | ||
83 | 263 | def configure_db(self, hostname, database, username, admin=False): | 278 | def configure_db(self, hostname, database, username, admin=False): |
84 | 264 | """Configure access to database for username from hostname.""" | 279 | """Configure access to database for username from hostname.""" |
85 | 265 | if config_get('prefer-ipv6'): | ||
86 | 266 | remote_ip = hostname | ||
87 | 267 | elif hostname != unit_get('private-address'): | ||
88 | 268 | try: | ||
89 | 269 | remote_ip = socket.gethostbyname(hostname) | ||
90 | 270 | except Exception: | ||
91 | 271 | # socket.gethostbyname doesn't support ipv6 | ||
92 | 272 | remote_ip = hostname | ||
93 | 273 | else: | ||
94 | 274 | remote_ip = '127.0.0.1' | ||
95 | 275 | |||
96 | 276 | self.connect(password=self.get_mysql_root_password()) | 280 | self.connect(password=self.get_mysql_root_password()) |
97 | 277 | if not self.database_exists(database): | 281 | if not self.database_exists(database): |
98 | 278 | self.create_database(database) | 282 | self.create_database(database) |
99 | 279 | 283 | ||
100 | 284 | remote_ip = self.normalize_address(hostname) | ||
101 | 280 | password = self.get_mysql_password(username) | 285 | password = self.get_mysql_password(username) |
102 | 281 | if not self.grant_exists(database, username, remote_ip): | 286 | if not self.grant_exists(database, username, remote_ip): |
103 | 282 | if not admin: | 287 | if not admin: |
104 | @@ -289,9 +294,11 @@ | |||
105 | 289 | 294 | ||
106 | 290 | class PerconaClusterHelper(object): | 295 | class PerconaClusterHelper(object): |
107 | 291 | 296 | ||
110 | 292 | # Going for the biggest page size to avoid wasted bytes. InnoDB page size is | 297 | # Going for the biggest page size to avoid wasted bytes. |
111 | 293 | # 16MB | 298 | # InnoDB page size is 16MB |
112 | 299 | |||
113 | 294 | DEFAULT_PAGE_SIZE = 16 * 1024 * 1024 | 300 | DEFAULT_PAGE_SIZE = 16 * 1024 * 1024 |
114 | 301 | DEFAULT_INNODB_BUFFER_FACTOR = 0.50 | ||
115 | 295 | 302 | ||
116 | 296 | def human_to_bytes(self, human): | 303 | def human_to_bytes(self, human): |
117 | 297 | """Convert human readable configuration options to bytes.""" | 304 | """Convert human readable configuration options to bytes.""" |
118 | @@ -352,51 +359,27 @@ | |||
119 | 352 | if 'max-connections' in config: | 359 | if 'max-connections' in config: |
120 | 353 | mysql_config['max_connections'] = config['max-connections'] | 360 | mysql_config['max_connections'] = config['max-connections'] |
121 | 354 | 361 | ||
122 | 355 | # Total memory available for dataset | ||
123 | 356 | dataset_bytes = self.human_to_bytes(config['dataset-size']) | ||
124 | 357 | mysql_config['dataset_bytes'] = dataset_bytes | ||
125 | 358 | |||
126 | 359 | if 'query-cache-type' in config: | ||
127 | 360 | # Query Cache Configuration | ||
128 | 361 | mysql_config['query_cache_size'] = config['query-cache-size'] | ||
129 | 362 | if (config['query-cache-size'] == -1 and | ||
130 | 363 | config['query-cache-type'] in ['ON', 'DEMAND']): | ||
131 | 364 | # Calculate the query cache size automatically | ||
132 | 365 | qcache_bytes = (dataset_bytes * 0.20) | ||
133 | 366 | qcache_bytes = int(qcache_bytes - | ||
134 | 367 | (qcache_bytes % self.DEFAULT_PAGE_SIZE)) | ||
135 | 368 | mysql_config['query_cache_size'] = qcache_bytes | ||
136 | 369 | dataset_bytes -= qcache_bytes | ||
137 | 370 | |||
138 | 371 | # 5.5 allows the words, but not 5.1 | ||
139 | 372 | if config['query-cache-type'] == 'ON': | ||
140 | 373 | mysql_config['query_cache_type'] = 1 | ||
141 | 374 | elif config['query-cache-type'] == 'DEMAND': | ||
142 | 375 | mysql_config['query_cache_type'] = 2 | ||
143 | 376 | else: | ||
144 | 377 | mysql_config['query_cache_type'] = 0 | ||
145 | 378 | |||
146 | 379 | # Set a sane default key_buffer size | 362 | # Set a sane default key_buffer size |
147 | 380 | mysql_config['key_buffer'] = self.human_to_bytes('32M') | 363 | mysql_config['key_buffer'] = self.human_to_bytes('32M') |
169 | 381 | 364 | total_memory = self.human_to_bytes(self.get_mem_total()) | |
170 | 382 | if 'preferred-storage-engine' in config: | 365 | |
171 | 383 | # Storage engine configuration | 366 | log("Option 'dataset-size' has been deprecated, instead by default %d%% of system \ |
172 | 384 | preferred_engines = config['preferred-storage-engine'].split(',') | 367 | available RAM will be used for innodb_buffer_pool_size allocation" % |
173 | 385 | chunk_size = int(dataset_bytes / len(preferred_engines)) | 368 | (self.DEFAULT_INNODB_BUFFER_FACTOR * 100), level="WARN") |
174 | 386 | mysql_config['innodb_flush_log_at_trx_commit'] = 1 | 369 | |
175 | 387 | mysql_config['sync_binlog'] = 1 | 370 | innodb_buffer_pool_size = config.get('innodb-buffer-pool-size', None) |
176 | 388 | if 'InnoDB' in preferred_engines: | 371 | |
177 | 389 | mysql_config['innodb_buffer_pool_size'] = chunk_size | 372 | if innodb_buffer_pool_size: |
178 | 390 | if config['tuning-level'] == 'fast': | 373 | innodb_buffer_pool_size = self.human_to_bytes( |
179 | 391 | mysql_config['innodb_flush_log_at_trx_commit'] = 2 | 374 | innodb_buffer_pool_size) |
180 | 392 | else: | 375 | |
181 | 393 | mysql_config['innodb_buffer_pool_size'] = 0 | 376 | if innodb_buffer_pool_size > total_memory: |
182 | 394 | 377 | log("innodb_buffer_pool_size; {} is greater than system available memory:{}".format( | |
183 | 395 | mysql_config['default_storage_engine'] = preferred_engines[0] | 378 | innodb_buffer_pool_size, |
184 | 396 | if 'MyISAM' in preferred_engines: | 379 | total_memory), level='WARN') |
185 | 397 | mysql_config['key_buffer'] = chunk_size | 380 | else: |
186 | 398 | 381 | innodb_buffer_pool_size = int( | |
187 | 399 | if config['tuning-level'] == 'fast': | 382 | total_memory * self.DEFAULT_INNODB_BUFFER_FACTOR) |
188 | 400 | mysql_config['sync_binlog'] = 0 | 383 | |
189 | 401 | 384 | mysql_config['innodb_buffer_pool_size'] = innodb_buffer_pool_size | |
190 | 402 | return mysql_config | 385 | return mysql_config |
191 | 403 | 386 | ||
192 | === modified file 'hooks/charmhelpers/contrib/network/ip.py' | |||
193 | --- hooks/charmhelpers/contrib/network/ip.py 2015-02-04 19:09:09 +0000 | |||
194 | +++ hooks/charmhelpers/contrib/network/ip.py 2015-02-27 10:18:28 +0000 | |||
195 | @@ -17,13 +17,16 @@ | |||
196 | 17 | import glob | 17 | import glob |
197 | 18 | import re | 18 | import re |
198 | 19 | import subprocess | 19 | import subprocess |
199 | 20 | import six | ||
200 | 21 | import socket | ||
201 | 20 | 22 | ||
202 | 21 | from functools import partial | 23 | from functools import partial |
203 | 22 | 24 | ||
204 | 23 | from charmhelpers.core.hookenv import unit_get | 25 | from charmhelpers.core.hookenv import unit_get |
205 | 24 | from charmhelpers.fetch import apt_install | 26 | from charmhelpers.fetch import apt_install |
206 | 25 | from charmhelpers.core.hookenv import ( | 27 | from charmhelpers.core.hookenv import ( |
208 | 26 | log | 28 | log, |
209 | 29 | WARNING, | ||
210 | 27 | ) | 30 | ) |
211 | 28 | 31 | ||
212 | 29 | try: | 32 | try: |
213 | @@ -365,3 +368,83 @@ | |||
214 | 365 | return True | 368 | return True |
215 | 366 | 369 | ||
216 | 367 | return False | 370 | return False |
217 | 371 | |||
218 | 372 | |||
219 | 373 | def is_ip(address): | ||
220 | 374 | """ | ||
221 | 375 | Returns True if address is a valid IP address. | ||
222 | 376 | """ | ||
223 | 377 | try: | ||
224 | 378 | # Test to see if already an IPv4 address | ||
225 | 379 | socket.inet_aton(address) | ||
226 | 380 | return True | ||
227 | 381 | except socket.error: | ||
228 | 382 | return False | ||
229 | 383 | |||
230 | 384 | |||
231 | 385 | def ns_query(address): | ||
232 | 386 | try: | ||
233 | 387 | import dns.resolver | ||
234 | 388 | except ImportError: | ||
235 | 389 | apt_install('python-dnspython') | ||
236 | 390 | import dns.resolver | ||
237 | 391 | |||
238 | 392 | if isinstance(address, dns.name.Name): | ||
239 | 393 | rtype = 'PTR' | ||
240 | 394 | elif isinstance(address, six.string_types): | ||
241 | 395 | rtype = 'A' | ||
242 | 396 | else: | ||
243 | 397 | return None | ||
244 | 398 | |||
245 | 399 | answers = dns.resolver.query(address, rtype) | ||
246 | 400 | if answers: | ||
247 | 401 | return str(answers[0]) | ||
248 | 402 | return None | ||
249 | 403 | |||
250 | 404 | |||
251 | 405 | def get_host_ip(hostname, fallback=None): | ||
252 | 406 | """ | ||
253 | 407 | Resolves the IP for a given hostname, or returns | ||
254 | 408 | the input if it is already an IP. | ||
255 | 409 | """ | ||
256 | 410 | if is_ip(hostname): | ||
257 | 411 | return hostname | ||
258 | 412 | |||
259 | 413 | ip_addr = ns_query(hostname) | ||
260 | 414 | if not ip_addr: | ||
261 | 415 | try: | ||
262 | 416 | ip_addr = socket.gethostbyname(hostname) | ||
263 | 417 | except: | ||
264 | 418 | log("Failed to resolve hostname '%s'" % (hostname), | ||
265 | 419 | level=WARNING) | ||
266 | 420 | return fallback | ||
267 | 421 | return ip_addr | ||
268 | 422 | |||
269 | 423 | |||
270 | 424 | def get_hostname(address, fqdn=True): | ||
271 | 425 | """ | ||
272 | 426 | Resolves hostname for given IP, or returns the input | ||
273 | 427 | if it is already a hostname. | ||
274 | 428 | """ | ||
275 | 429 | if is_ip(address): | ||
276 | 430 | try: | ||
277 | 431 | import dns.reversename | ||
278 | 432 | except ImportError: | ||
279 | 433 | apt_install("python-dnspython") | ||
280 | 434 | import dns.reversename | ||
281 | 435 | |||
282 | 436 | rev = dns.reversename.from_address(address) | ||
283 | 437 | result = ns_query(rev) | ||
284 | 438 | if not result: | ||
285 | 439 | return None | ||
286 | 440 | else: | ||
287 | 441 | result = address | ||
288 | 442 | |||
289 | 443 | if fqdn: | ||
290 | 444 | # strip trailing . | ||
291 | 445 | if result.endswith('.'): | ||
292 | 446 | return result[:-1] | ||
293 | 447 | else: | ||
294 | 448 | return result | ||
295 | 449 | else: | ||
296 | 450 | return result.split('.')[0] | ||
297 | 368 | 451 | ||
298 | === modified file 'hooks/charmhelpers/core/services/helpers.py' | |||
299 | --- hooks/charmhelpers/core/services/helpers.py 2015-02-04 19:09:09 +0000 | |||
300 | +++ hooks/charmhelpers/core/services/helpers.py 2015-02-27 10:18:28 +0000 | |||
301 | @@ -45,12 +45,14 @@ | |||
302 | 45 | """ | 45 | """ |
303 | 46 | name = None | 46 | name = None |
304 | 47 | interface = None | 47 | interface = None |
305 | 48 | required_keys = [] | ||
306 | 49 | 48 | ||
307 | 50 | def __init__(self, name=None, additional_required_keys=None): | 49 | def __init__(self, name=None, additional_required_keys=None): |
308 | 50 | if not hasattr(self, 'required_keys'): | ||
309 | 51 | self.required_keys = [] | ||
310 | 52 | |||
311 | 51 | if name is not None: | 53 | if name is not None: |
312 | 52 | self.name = name | 54 | self.name = name |
314 | 53 | if additional_required_keys is not None: | 55 | if additional_required_keys: |
315 | 54 | self.required_keys.extend(additional_required_keys) | 56 | self.required_keys.extend(additional_required_keys) |
316 | 55 | self.get_data() | 57 | self.get_data() |
317 | 56 | 58 | ||
318 | @@ -134,7 +136,10 @@ | |||
319 | 134 | """ | 136 | """ |
320 | 135 | name = 'db' | 137 | name = 'db' |
321 | 136 | interface = 'mysql' | 138 | interface = 'mysql' |
323 | 137 | required_keys = ['host', 'user', 'password', 'database'] | 139 | |
324 | 140 | def __init__(self, *args, **kwargs): | ||
325 | 141 | self.required_keys = ['host', 'user', 'password', 'database'] | ||
326 | 142 | super(HttpRelation).__init__(self, *args, **kwargs) | ||
327 | 138 | 143 | ||
328 | 139 | 144 | ||
329 | 140 | class HttpRelation(RelationContext): | 145 | class HttpRelation(RelationContext): |
330 | @@ -146,7 +151,10 @@ | |||
331 | 146 | """ | 151 | """ |
332 | 147 | name = 'website' | 152 | name = 'website' |
333 | 148 | interface = 'http' | 153 | interface = 'http' |
335 | 149 | required_keys = ['host', 'port'] | 154 | |
336 | 155 | def __init__(self, *args, **kwargs): | ||
337 | 156 | self.required_keys = ['host', 'port'] | ||
338 | 157 | super(HttpRelation).__init__(self, *args, **kwargs) | ||
339 | 150 | 158 | ||
340 | 151 | def provide_data(self): | 159 | def provide_data(self): |
341 | 152 | return { | 160 | return { |
342 | 153 | 161 | ||
343 | === added directory 'unit_tests' | |||
344 | === added file 'unit_tests/__init__.py' | |||
345 | === added file 'unit_tests/test_mysql_common.py' | |||
346 | --- unit_tests/test_mysql_common.py 1970-01-01 00:00:00 +0000 | |||
347 | +++ unit_tests/test_mysql_common.py 2015-02-27 10:18:28 +0000 | |||
348 | @@ -0,0 +1,17 @@ | |||
349 | 1 | import mock | ||
350 | 2 | import unittest | ||
351 | 3 | import sys | ||
352 | 4 | |||
353 | 5 | sys.modules['MySQLdb'] = mock.Mock() | ||
354 | 6 | import common | ||
355 | 7 | |||
356 | 8 | |||
357 | 9 | class CommonTests(unittest.TestCase): | ||
358 | 10 | |||
359 | 11 | @mock.patch.object(common, 'MySQLHelper', spec=common.MySQLHelper) | ||
360 | 12 | def test_get_db_helper(self, mock_helper): | ||
361 | 13 | h = common.get_db_helper() | ||
362 | 14 | kwargs = {'rpasswdf_template': '/var/lib/mysql/mysql.passwd', | ||
363 | 15 | 'upasswdf_template': '/var/lib/mysql/mysql-{}.passwd', | ||
364 | 16 | 'delete_ondisk_passwd_file': False} | ||
365 | 17 | mock_helper.assert_called_with(**kwargs) | ||
366 | 0 | \ No newline at end of file | 18 | \ No newline at end of file |
charm_unit_test #2134 mysql for hopem mp251159
UNIT FAIL: unit-test missing
UNIT Results (max last 2 lines):
INFO:root:Search string not found in makefile target commands.
ERROR:root:No make target was executed.
Full unit test output: http:// paste.ubuntu. com/10435847/ 10.245. 162.77: 8080/job/ charm_unit_ test/2134/
Build: http://