1
=== modified file 'Makefile'
2
--- Makefile	2014-12-11 06:29:02 +0000
3
+++ Makefile	2015-10-08 08:12:47 +0000
4
@@ -14,6 +14,7 @@
5
14
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
14
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
6
15
15
7
16
PYTHON := /usr/bin/env python
16
PYTHON := /usr/bin/env python
8
17
ACTIONS := $(shell grep -slE '\#!(.*)python' actions/*)
9
17
18
10
18
clean:
19
clean:
11
19
	rm -f .coverage
20
	rm -f .coverage
12
@@ -27,11 +28,12 @@
13
27
	.venv/bin/pip install -I -r test_requirements.txt
28
	.venv/bin/pip install -I -r test_requirements.txt
14
28
29
15
29
lint:   .venv
30
lint:   .venv
17
30
	.venv/bin/flake8 --exclude hooks/charmhelpers hooks tests unit_tests
31
	.venv/bin/flake8 --exclude hooks/charmhelpers actions $(ACTIONS) hooks tests unit_tests
18
31
32
19
32
test:  .venv
33
test:  .venv
20
33
	@echo Starting unit tests...
34
	@echo Starting unit tests...
21
34
	.venv/bin/nosetests -s --nologcapture --with-coverage $(EXTRA) unit_tests/
35
	.venv/bin/nosetests -s --nologcapture --with-coverage $(EXTRA) unit_tests/
22
36
	.venv/bin/nosetests -s --nologcapture --with-coverage $(EXTRA) actions/
23
35
37
24
36
functional_test:
38
functional_test:
25
37
	@echo Starting amulet tests...
39
	@echo Starting amulet tests...
26
38
40
27
=== modified file 'actions.yaml'
28
--- actions.yaml	2015-05-07 20:42:32 +0000
29
+++ actions.yaml	2015-10-08 08:12:47 +0000
30
@@ -47,3 +47,25 @@
31
47
        Only use syncDelay in conjunction with mmf set to true.
47
        Only use syncDelay in conjunction with mmf set to true.
32
48
      type: integer
48
      type: integer
33
49
      default: 0
49
      default: 0
34
50
dump:
35
51
  description: Runs the mongodump command
36
52
  params:
37
53
    args:
38
54
      description: The arguments to pass to the mongodump command
39
55
      type: string
40
56
      default: ""
41
57
    workingDir:
42
58
      description: The directory to run the mongodump command from.
43
59
      type: string
44
60
      default: "/tmp/mongodump"
45
61
restore:
46
62
  description: Runs the mongorestore command
47
63
  params:
48
64
    args:
49
65
      description: The arguments to pass to the mongorestore command
50
66
      type: string
51
67
      default: ""
52
68
    workingDir:
53
69
      description: The directory to run the mongorestore command from.
54
70
      type: string
55
71
      default: "/tmp/mongodump"
56
50
72
57
=== added file 'actions/backup.py'
58
--- actions/backup.py	1970-01-01 00:00:00 +0000
59
+++ actions/backup.py	2015-10-08 08:12:47 +0000
60
@@ -0,0 +1,56 @@
61
1
import subprocess
62
2
import os
63
3
try:
64
4
    from charmhelpers.core.hookenv import action_get, action_set, action_fail
65
5
except ImportError:
66
6
    subprocess.check_call(['apt-get', 'install', '-y', 'python-pip'])
67
7
    subprocess.check_call(['pip', 'install', 'charmhelpers'])
68
8
    from charmhelpers.core.hookenv import action_get, action_set, action_fail
69
9
70
10
71
11
def mkdir(dir):
72
12
    return os.makedirs(dir)
73
13
74
14
75
15
def execute(command, current_working_directory):
76
16
    return subprocess.check_output(
77
17
        " ".join(command),
78
18
        cwd=current_working_directory,
79
19
    )
80
20
81
21
82
22
def dump():
83
23
    args = str(action_get("args")).strip()
84
24
    dir = str(action_get("workingDir"))
85
25
    backup_command("mongodump", args, dir)
86
26
87
27
88
28
def restore():
89
29
    args = str(action_get("args")).strip()
90
30
    dir = str(action_get("workingDir"))
91
31
    backup_command("mongorestore", args, dir)
92
32
93
33
94
34
def backup_command(cmd, args, dir):
95
35
    try:
96
36
        mkdir(dir)
97
37
    except OSError, e:
98
38
        pass  # Ignoring, the directory already exists
99
39
    except Exception, e:
100
40
        action_set({"directory creation exception": e})
101
41
        action_fail(e)
102
42
        return
103
43
104
44
    command = cmd + " " + args
105
45
    command = command.strip()
106
46
    action_set({"command": command, "working-dir": dir})
107
47
    try:
108
48
        output = execute(command, dir)
109
49
        action_set({"output": output})
110
50
    except subprocess.CalledProcessError, e:
111
51
        action_set({"error_code": e.returncode,
112
52
                    "exception": e, "output": e.output})
113
53
        action_fail(e)
114
54
    except Exception, e:
115
55
        action_set({"exception": e})
116
56
        action_fail(e)
117
0
57
118
=== added file 'actions/backup_test.py'
119
--- actions/backup_test.py	1970-01-01 00:00:00 +0000
120
+++ actions/backup_test.py	2015-10-08 08:12:47 +0000
121
@@ -0,0 +1,52 @@
122
1
import unittest
123
2
from mock import create_autospec, call
124
3
import backup
125
4
126
5
127
6
def mock_action_get(key):
128
7
    return {"workingDir": "this/dir", "args": "-arg1 -arg2"}[key]
129
8
130
9
131
10
class TestBackups(unittest.TestCase):
132
11
133
12
    def test_dump(self):
134
13
        mkdir = create_autospec(backup.mkdir, return_value=True)
135
14
        execute = create_autospec(backup.execute, return_value="output")
136
15
        action_set = create_autospec(backup.action_set, return_value=None)
137
16
        action_fail = create_autospec(backup.action_fail, return_value=None)
138
17
        backup.mkdir = mkdir
139
18
        backup.execute = execute
140
19
        backup.action_set = action_set
141
20
        backup.action_fail = action_fail
142
21
        backup.action_get = mock_action_get
143
22
144
23
        backup.dump()
145
24
146
25
        mkdir.assert_called_once_with("this/dir")
147
26
        execute.assert_called_once_with("mongodump -arg1 -arg2", "this/dir")
148
27
        action_set.assert_has_calls([
149
28
            call({"command": "mongodump -arg1 -arg2",
150
29
                 "working-dir": "this/dir"}),
151
30
            call({"output": "output"})])
152
31
153
32
    def test_restore(self):
154
33
        mkdir = create_autospec(backup.mkdir, return_value=True)
155
34
        execute = create_autospec(backup.execute, return_value="output")
156
35
        action_set = create_autospec(backup.action_set, return_value=None)
157
36
        action_fail = create_autospec(backup.action_fail, return_value=None)
158
37
        backup.mkdir = mkdir
159
38
        backup.execute = execute
160
39
        backup.action_set = action_set
161
40
        backup.action_fail = action_fail
162
41
163
42
        backup.restore()
164
43
165
44
        mkdir.assert_called_once_with("this/dir")
166
45
        execute.assert_called_once_with("mongorestore -arg1 -arg2", "this/dir")
167
46
        action_set.assert_has_calls([
168
47
            call({"command": "mongodump -arg1 -arg2",
169
48
                 "working-dir": "this/dir"}),
170
49
            call({"output": "output"})])
171
50
172
51
if __name__ == '__main__':
173
52
    unittest.main()
174
0
53
175
=== added symlink 'actions/charmhelpers'
176
=== target is u'../charmhelpers'
177
=== added file 'actions/dump'
178
--- actions/dump	1970-01-01 00:00:00 +0000
179
+++ actions/dump	2015-10-08 08:12:47 +0000
180
@@ -0,0 +1,4 @@
181
1
#!/usr/bin/env python
182
2
import backup
183
3
184
4
backup.dump()
185
0
5
186
=== added file 'actions/restore'
187
--- actions/restore	1970-01-01 00:00:00 +0000
188
+++ actions/restore	2015-10-08 08:12:47 +0000
189
@@ -0,0 +1,4 @@
190
1
#!/usr/bin/env python
191
2
import backup
192
3
193
4
backup.restore()
194
0
5
195
=== modified file 'charm-helpers-sync.yaml'
196
--- charm-helpers-sync.yaml	2015-02-25 00:24:56 +0000
197
+++ charm-helpers-sync.yaml	2015-10-08 08:12:47 +0000
198
@@ -1,5 +1,5 @@
199
1
branch: lp:charm-helpers
1
branch: lp:charm-helpers
201
2
destination: hooks/charmhelpers
2
destination: charmhelpers
202
3
include:
3
include:
203
4
    - core
4
    - core
204
5
    - fetch
5
    - fetch
205
6
6
206
=== renamed directory 'hooks/charmhelpers' => 'charmhelpers'
207
=== modified file 'charmhelpers/core/hookenv.py'
208
--- hooks/charmhelpers/core/hookenv.py	2015-08-19 13:58:52 +0000
209
+++ charmhelpers/core/hookenv.py	2015-10-08 08:12:47 +0000
210
@@ -623,6 +623,38 @@
211
623
    return unit_get('private-address')
623
    return unit_get('private-address')
212
624
624
213
625
625
214
626
@cached
215
627
def storage_get(attribute="", storage_id=""):
216
628
    """Get storage attributes"""
217
629
    _args = ['storage-get', '--format=json']
218
630
    if storage_id:
219
631
        _args.extend(('-s', storage_id))
220
632
    if attribute:
221
633
        _args.append(attribute)
222
634
    try:
223
635
        return json.loads(subprocess.check_output(_args).decode('UTF-8'))
224
636
    except ValueError:
225
637
        return None
226
638
227
639
228
640
@cached
229
641
def storage_list(storage_name=""):
230
642
    """List the storage IDs for the unit"""
231
643
    _args = ['storage-list', '--format=json']
232
644
    if storage_name:
233
645
        _args.append(storage_name)
234
646
    try:
235
647
        return json.loads(subprocess.check_output(_args).decode('UTF-8'))
236
648
    except ValueError:
237
649
        return None
238
650
    except OSError as e:
239
651
        import errno
240
652
        if e.errno == errno.ENOENT:
241
653
            # storage-list does not exist
242
654
            return []
243
655
        raise
244
656
245
657
246
626
class UnregisteredHookError(Exception):
658
class UnregisteredHookError(Exception):
247
627
    """Raised when an undefined hook is called"""
659
    """Raised when an undefined hook is called"""
248
628
    pass
660
    pass
249
@@ -767,21 +799,23 @@
250
767
799
251
768
800
252
769
def status_get():
801
def status_get():
257
770
    """Retrieve the previously set juju workload state
802
    """Retrieve the previously set juju workload state and message
258
771
803
259
772
    If the status-set command is not found then assume this is juju < 1.23 and
804
    If the status-get command is not found then assume this is juju < 1.23 and
260
773
    return 'unknown'
805
    return 'unknown', ""
261
806
262
774
    """
807
    """
264
775
    cmd = ['status-get']
808
    cmd = ['status-get', "--format=json", "--include-data"]
265
776
    try:
809
    try:
269
777
        raw_status = subprocess.check_output(cmd, universal_newlines=True)
810
        raw_status = subprocess.check_output(cmd)
267
778
        status = raw_status.rstrip()
268
779
        return status
270
780
    except OSError as e:
811
    except OSError as e:
271
781
        if e.errno == errno.ENOENT:
812
        if e.errno == errno.ENOENT:
273
782
            return 'unknown'
813
            return ('unknown', "")
274
783
        else:
814
        else:
275
784
            raise
815
            raise
276
816
    else:
277
817
        status = json.loads(raw_status.decode("UTF-8"))
278
818
        return (status["status"], status["message"])
279
785
819
280
786
820
281
787
def translate_exc(from_exc, to_exc):
821
def translate_exc(from_exc, to_exc):
282
788
822
283
=== modified file 'charmhelpers/core/host.py'
284
--- hooks/charmhelpers/core/host.py	2015-08-19 13:58:52 +0000
285
+++ charmhelpers/core/host.py	2015-10-08 08:12:47 +0000
286
@@ -63,32 +63,48 @@
287
63
    return service_result
63
    return service_result
288
64
64
289
65
65
291
66
def service_pause(service_name, init_dir=None):
66
def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"):
292
67
    """Pause a system service.
67
    """Pause a system service.
293
68
68
294
69
    Stop it, and prevent it from starting again at boot."""
69
    Stop it, and prevent it from starting again at boot."""
295
70
    if init_dir is None:
296
71
        init_dir = "/etc/init"
297
72
    stopped = service_stop(service_name)
70
    stopped = service_stop(service_name)
303
73
    # XXX: Support systemd too
71
    upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
304
74
    override_path = os.path.join(
72
    sysv_file = os.path.join(initd_dir, service_name)
305
75
        init_dir, '{}.override'.format(service_name))
73
    if os.path.exists(upstart_file):
306
76
    with open(override_path, 'w') as fh:
74
        override_path = os.path.join(
307
77
        fh.write("manual\n")
75
            init_dir, '{}.override'.format(service_name))
308
76
        with open(override_path, 'w') as fh:
309
77
            fh.write("manual\n")
310
78
    elif os.path.exists(sysv_file):
311
79
        subprocess.check_call(["update-rc.d", service_name, "disable"])
312
80
    else:
313
81
        # XXX: Support SystemD too
314
82
        raise ValueError(
315
83
            "Unable to detect {0} as either Upstart {1} or SysV {2}".format(
316
84
                service_name, upstart_file, sysv_file))
317
78
    return stopped
85
    return stopped
318
79
86
319
80
87
321
81
def service_resume(service_name, init_dir=None):
88
def service_resume(service_name, init_dir="/etc/init",
322
89
                   initd_dir="/etc/init.d"):
323
82
    """Resume a system service.
90
    """Resume a system service.
324
83
91
325
84
    Reenable starting again at boot. Start the service"""
92
    Reenable starting again at boot. Start the service"""
333
85
    # XXX: Support systemd too
93
    upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
334
86
    if init_dir is None:
94
    sysv_file = os.path.join(initd_dir, service_name)
335
87
        init_dir = "/etc/init"
95
    if os.path.exists(upstart_file):
336
88
    override_path = os.path.join(
96
        override_path = os.path.join(
337
89
        init_dir, '{}.override'.format(service_name))
97
            init_dir, '{}.override'.format(service_name))
338
90
    if os.path.exists(override_path):
98
        if os.path.exists(override_path):
339
91
        os.unlink(override_path)
99
            os.unlink(override_path)
340
100
    elif os.path.exists(sysv_file):
341
101
        subprocess.check_call(["update-rc.d", service_name, "enable"])
342
102
    else:
343
103
        # XXX: Support SystemD too
344
104
        raise ValueError(
345
105
            "Unable to detect {0} as either Upstart {1} or SysV {2}".format(
346
106
                service_name, upstart_file, sysv_file))
347
107
348
92
    started = service_start(service_name)
108
    started = service_start(service_name)
349
93
    return started
109
    return started
350
94
110
351
95
111
352
=== modified file 'charmhelpers/core/hugepage.py'
353
--- hooks/charmhelpers/core/hugepage.py	2015-08-19 13:58:52 +0000
354
+++ charmhelpers/core/hugepage.py	2015-10-08 08:12:47 +0000
355
@@ -25,11 +25,13 @@
356
25
    fstab_mount,
25
    fstab_mount,
357
26
    mkdir,
26
    mkdir,
358
27
)
27
)
359
28
from charmhelpers.core.strutils import bytes_from_string
360
29
from subprocess import check_output
361
28
30
362
29
31
363
30
def hugepage_support(user, group='hugetlb', nr_hugepages=256,
32
def hugepage_support(user, group='hugetlb', nr_hugepages=256,
364
31
                     max_map_count=65536, mnt_point='/run/hugepages/kvm',
33
                     max_map_count=65536, mnt_point='/run/hugepages/kvm',
366
32
                     pagesize='2MB', mount=True):
34
                     pagesize='2MB', mount=True, set_shmmax=False):
367
33
    """Enable hugepages on system.
35
    """Enable hugepages on system.
368
34
36
369
35
    Args:
37
    Args:
370
@@ -49,6 +51,11 @@
371
49
        'vm.max_map_count': max_map_count,
51
        'vm.max_map_count': max_map_count,
372
50
        'vm.hugetlb_shm_group': gid,
52
        'vm.hugetlb_shm_group': gid,
373
51
    }
53
    }
374
54
    if set_shmmax:
375
55
        shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax']))
376
56
        shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages
377
57
        if shmmax_minsize > shmmax_current:
378
58
            sysctl_settings['kernel.shmmax'] = shmmax_minsize
379
52
    sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf')
59
    sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf')
380
53
    mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False)
60
    mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False)
381
54
    lfstab = fstab.Fstab()
61
    lfstab = fstab.Fstab()
382
55
62
383
=== added file 'charmhelpers/core/kernel.py'
384
--- charmhelpers/core/kernel.py	1970-01-01 00:00:00 +0000
385
+++ charmhelpers/core/kernel.py	2015-10-08 08:12:47 +0000
386
@@ -0,0 +1,68 @@
387
1
#!/usr/bin/env python
388
2
# -*- coding: utf-8 -*-
389
3
390
4
# Copyright 2014-2015 Canonical Limited.
391
5
#
392
6
# This file is part of charm-helpers.
393
7
#
394
8
# charm-helpers is free software: you can redistribute it and/or modify
395
9
# it under the terms of the GNU Lesser General Public License version 3 as
396
10
# published by the Free Software Foundation.
397
11
#
398
12
# charm-helpers is distributed in the hope that it will be useful,
399
13
# but WITHOUT ANY WARRANTY; without even the implied warranty of
400
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
401
15
# GNU Lesser General Public License for more details.
402
16
#
403
17
# You should have received a copy of the GNU Lesser General Public License
404
18
# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
405
19
406
20
__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
407
21
408
22
from charmhelpers.core.hookenv import (
409
23
    log,
410
24
    INFO
411
25
)
412
26
413
27
from subprocess import check_call, check_output
414
28
import re
415
29
416
30
417
31
def modprobe(module, persist=True):
418
32
    """Load a kernel module and configure for auto-load on reboot."""
419
33
    cmd = ['modprobe', module]
420
34
421
35
    log('Loading kernel module %s' % module, level=INFO)
422
36
423
37
    check_call(cmd)
424
38
    if persist:
425
39
        with open('/etc/modules', 'r+') as modules:
426
40
            if module not in modules.read():
427
41
                modules.write(module)
428
42
429
43
430
44
def rmmod(module, force=False):
431
45
    """Remove a module from the linux kernel"""
432
46
    cmd = ['rmmod']
433
47
    if force:
434
48
        cmd.append('-f')
435
49
    cmd.append(module)
436
50
    log('Removing kernel module %s' % module, level=INFO)
437
51
    return check_call(cmd)
438
52
439
53
440
54
def lsmod():
441
55
    """Shows what kernel modules are currently loaded"""
442
56
    return check_output(['lsmod'],
443
57
                        universal_newlines=True)
444
58
445
59
446
60
def is_module_loaded(module):
447
61
    """Checks if a kernel module is already loaded"""
448
62
    matches = re.findall('^%s[ ]+' % module, lsmod(), re.M)
449
63
    return len(matches) > 0
450
64
451
65
452
66
def update_initramfs(version='all'):
453
67
    """Updates an initramfs image"""
454
68
    return check_call(["update-initramfs", "-k", version, "-u"])
455
0
69
456
=== modified file 'charmhelpers/core/strutils.py'
457
--- hooks/charmhelpers/core/strutils.py	2015-08-19 13:58:52 +0000
458
+++ charmhelpers/core/strutils.py	2015-10-08 08:12:47 +0000
459
@@ -18,6 +18,7 @@
460
18
# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
18
# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
461
19
19
462
20
import six
20
import six
463
21
import re
464
21
22
465
22
23
466
23
def bool_from_string(value):
24
def bool_from_string(value):
467
@@ -40,3 +41,32 @@
468
40
41
469
41
    msg = "Unable to interpret string value '%s' as boolean" % (value)
42
    msg = "Unable to interpret string value '%s' as boolean" % (value)
470
42
    raise ValueError(msg)
43
    raise ValueError(msg)
471
44
472
45
473
46
def bytes_from_string(value):
474
47
    """Interpret human readable string value as bytes.
475
48
476
49
    Returns int
477
50
    """
478
51
    BYTE_POWER = {
479
52
        'K': 1,
480
53
        'KB': 1,
481
54
        'M': 2,
482
55
        'MB': 2,
483
56
        'G': 3,
484
57
        'GB': 3,
485
58
        'T': 4,
486
59
        'TB': 4,
487
60
        'P': 5,
488
61
        'PB': 5,
489
62
    }
490
63
    if isinstance(value, six.string_types):
491
64
        value = six.text_type(value)
492
65
    else:
493
66
        msg = "Unable to interpret non-string value '%s' as boolean" % (value)
494
67
        raise ValueError(msg)
495
68
    matches = re.match("([0-9]+)([a-zA-Z]+)", value)
496
69
    if not matches:
497
70
        msg = "Unable to interpret string value '%s' as bytes" % (value)
498
71
        raise ValueError(msg)
499
72
    return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)])
500
43
73
501
=== added symlink 'hooks/charmhelpers'
502
=== target is u'../charmhelpers'
Status:	Merged
Merged at revision:	78
Proposed branch:	lp:~mattyw/charms/trusty/mongodb/mongodb-backup
Merge into:	lp:charms/trusty/mongodb
Diff against target:	502 lines (+323/-28) 12 files modified Makefile (+3/-1) actions.yaml (+22/-0) actions/backup.py (+56/-0) actions/backup_test.py (+52/-0) actions/dump (+4/-0) actions/restore (+4/-0) charm-helpers-sync.yaml (+1/-1) charmhelpers/core/hookenv.py (+43/-9) charmhelpers/core/host.py (+32/-16) charmhelpers/core/hugepage.py (+8/-1) charmhelpers/core/kernel.py (+68/-0) charmhelpers/core/strutils.py (+30/-0)
To merge this branch:	bzr merge lp:~mattyw/charms/trusty/mongodb/mongodb-backup
Related bugs:	Link a bug report
Reviewer	Date Requested	Status
Charles Butler (community)		Approve on 2015-11-05
Domas Monkus (community)		Approve on 2015-10-08
Marco Ceppi (community)	2015-10-06	Needs Fixing on 2015-10-06
Ryan Beisner (community)		Needs Fixing on 2015-10-06
Review via email: mp+273544@code.launchpad.net