Merge lp:~brad-marshall/charms/trusty/ceph/nagios-fix-servicegroups into lp:~openstack-charmers-archive/charms/trusty/ceph/next
- Trusty Tahr (14.04)
- nagios-fix-servicegroups
- Merge into next
Proposed by
Brad Marshall
Status: | Merged |
---|---|
Merged at revision: | 100 |
Proposed branch: | lp:~brad-marshall/charms/trusty/ceph/nagios-fix-servicegroups |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/ceph/next |
Diff against target: |
1030 lines (+731/-43) 14 files modified
config.yaml (+7/-0) hooks/charmhelpers/contrib/charmsupport/nrpe.py (+41/-7) hooks/charmhelpers/core/fstab.py (+4/-4) hooks/charmhelpers/core/host.py (+5/-5) hooks/charmhelpers/core/strutils.py (+42/-0) hooks/charmhelpers/core/sysctl.py (+13/-7) hooks/charmhelpers/core/templating.py (+3/-3) hooks/charmhelpers/core/unitdata.py (+477/-0) hooks/charmhelpers/fetch/archiveurl.py (+10/-10) hooks/charmhelpers/fetch/giturl.py (+1/-1) metadata.yaml (+0/-1) tests/basic_deployment.py (+1/-1) tests/charmhelpers/contrib/amulet/utils.py (+122/-2) tests/charmhelpers/contrib/openstack/amulet/deployment.py (+5/-2) |
To merge this branch: | bzr merge lp:~brad-marshall/charms/trusty/ceph/nagios-fix-servicegroups |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Liam Young (community) | Approve | ||
Review via email:
|
Commit message
Description of the change
Synced charmhelpers, added nagios_servicegroup config option, and added haproxy nrpe checks.
To post a comment you must log in.
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
uosci-testing-bot (uosci-testing-bot) wrote : | # |
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #2226 ceph-next for brad-marshall mp250711
LINT OK: passed
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #2172 ceph-next for brad-marshall mp250711
AMULET FAIL: amulet-test failed
AMULET Results (max last 2 lines):
ERROR subprocess encountered error code 1
make: *** [test] Error 1
Full amulet test output: http://
Build: http://
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'config.yaml' | |||
2 | --- config.yaml 2015-02-19 10:17:31 +0000 | |||
3 | +++ config.yaml 2015-02-24 06:27:04 +0000 | |||
4 | @@ -168,6 +168,7 @@ | |||
5 | 168 | nagios_context: | 168 | nagios_context: |
6 | 169 | type: string | 169 | type: string |
7 | 170 | default: "juju" | 170 | default: "juju" |
8 | 171 | type: string | ||
9 | 171 | description: | | 172 | description: | |
10 | 172 | Used by the nrpe-external-master subordinate charm. | 173 | Used by the nrpe-external-master subordinate charm. |
11 | 173 | A string that will be prepended to instance name to set the host name | 174 | A string that will be prepended to instance name to set the host name |
12 | @@ -175,3 +176,9 @@ | |||
13 | 175 | juju-myservice-0 | 176 | juju-myservice-0 |
14 | 176 | If you're running multiple environments with the same services in them | 177 | If you're running multiple environments with the same services in them |
15 | 177 | this allows you to differentiate between them. | 178 | this allows you to differentiate between them. |
16 | 179 | nagios_servicegroups: | ||
17 | 180 | default: "" | ||
18 | 181 | type: string | ||
19 | 182 | description: | | ||
20 | 183 | A comma-separated list of nagios servicegroups. | ||
21 | 184 | If left empty, the nagios_context will be used as the servicegroup | ||
22 | 178 | 185 | ||
23 | === modified file 'hooks/charmhelpers/contrib/charmsupport/nrpe.py' | |||
24 | --- hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-01-26 09:46:20 +0000 | |||
25 | +++ hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-02-24 06:27:04 +0000 | |||
26 | @@ -24,6 +24,8 @@ | |||
27 | 24 | import pwd | 24 | import pwd |
28 | 25 | import grp | 25 | import grp |
29 | 26 | import os | 26 | import os |
30 | 27 | import glob | ||
31 | 28 | import shutil | ||
32 | 27 | import re | 29 | import re |
33 | 28 | import shlex | 30 | import shlex |
34 | 29 | import yaml | 31 | import yaml |
35 | @@ -161,7 +163,7 @@ | |||
36 | 161 | log('Check command not found: {}'.format(parts[0])) | 163 | log('Check command not found: {}'.format(parts[0])) |
37 | 162 | return '' | 164 | return '' |
38 | 163 | 165 | ||
40 | 164 | def write(self, nagios_context, hostname, nagios_servicegroups=None): | 166 | def write(self, nagios_context, hostname, nagios_servicegroups): |
41 | 165 | nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( | 167 | nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format( |
42 | 166 | self.command) | 168 | self.command) |
43 | 167 | with open(nrpe_check_file, 'w') as nrpe_check_config: | 169 | with open(nrpe_check_file, 'w') as nrpe_check_config: |
44 | @@ -177,14 +179,11 @@ | |||
45 | 177 | nagios_servicegroups) | 179 | nagios_servicegroups) |
46 | 178 | 180 | ||
47 | 179 | def write_service_config(self, nagios_context, hostname, | 181 | def write_service_config(self, nagios_context, hostname, |
49 | 180 | nagios_servicegroups=None): | 182 | nagios_servicegroups): |
50 | 181 | for f in os.listdir(NRPE.nagios_exportdir): | 183 | for f in os.listdir(NRPE.nagios_exportdir): |
51 | 182 | if re.search('.*{}.cfg'.format(self.command), f): | 184 | if re.search('.*{}.cfg'.format(self.command), f): |
52 | 183 | os.remove(os.path.join(NRPE.nagios_exportdir, f)) | 185 | os.remove(os.path.join(NRPE.nagios_exportdir, f)) |
53 | 184 | 186 | ||
54 | 185 | if not nagios_servicegroups: | ||
55 | 186 | nagios_servicegroups = nagios_context | ||
56 | 187 | |||
57 | 188 | templ_vars = { | 187 | templ_vars = { |
58 | 189 | 'nagios_hostname': hostname, | 188 | 'nagios_hostname': hostname, |
59 | 190 | 'nagios_servicegroup': nagios_servicegroups, | 189 | 'nagios_servicegroup': nagios_servicegroups, |
60 | @@ -211,10 +210,10 @@ | |||
61 | 211 | super(NRPE, self).__init__() | 210 | super(NRPE, self).__init__() |
62 | 212 | self.config = config() | 211 | self.config = config() |
63 | 213 | self.nagios_context = self.config['nagios_context'] | 212 | self.nagios_context = self.config['nagios_context'] |
65 | 214 | if 'nagios_servicegroups' in self.config: | 213 | if 'nagios_servicegroups' in self.config and self.config['nagios_servicegroups']: |
66 | 215 | self.nagios_servicegroups = self.config['nagios_servicegroups'] | 214 | self.nagios_servicegroups = self.config['nagios_servicegroups'] |
67 | 216 | else: | 215 | else: |
69 | 217 | self.nagios_servicegroups = 'juju' | 216 | self.nagios_servicegroups = self.nagios_context |
70 | 218 | self.unit_name = local_unit().replace('/', '-') | 217 | self.unit_name = local_unit().replace('/', '-') |
71 | 219 | if hostname: | 218 | if hostname: |
72 | 220 | self.hostname = hostname | 219 | self.hostname = hostname |
73 | @@ -322,3 +321,38 @@ | |||
74 | 322 | check_cmd='check_status_file.py -f ' | 321 | check_cmd='check_status_file.py -f ' |
75 | 323 | '/var/lib/nagios/service-check-%s.txt' % svc, | 322 | '/var/lib/nagios/service-check-%s.txt' % svc, |
76 | 324 | ) | 323 | ) |
77 | 324 | |||
78 | 325 | |||
79 | 326 | def copy_nrpe_checks(): | ||
80 | 327 | """ | ||
81 | 328 | Copy the nrpe checks into place | ||
82 | 329 | |||
83 | 330 | """ | ||
84 | 331 | NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins' | ||
85 | 332 | nrpe_files_dir = os.path.join(os.getenv('CHARM_DIR'), 'hooks', | ||
86 | 333 | 'charmhelpers', 'contrib', 'openstack', | ||
87 | 334 | 'files') | ||
88 | 335 | |||
89 | 336 | if not os.path.exists(NAGIOS_PLUGINS): | ||
90 | 337 | os.makedirs(NAGIOS_PLUGINS) | ||
91 | 338 | for fname in glob.glob(os.path.join(nrpe_files_dir, "check_*")): | ||
92 | 339 | if os.path.isfile(fname): | ||
93 | 340 | shutil.copy2(fname, | ||
94 | 341 | os.path.join(NAGIOS_PLUGINS, os.path.basename(fname))) | ||
95 | 342 | |||
96 | 343 | |||
97 | 344 | def add_haproxy_checks(nrpe, unit_name): | ||
98 | 345 | """ | ||
99 | 346 | Add checks for each service in list | ||
100 | 347 | |||
101 | 348 | :param NRPE nrpe: NRPE object to add check to | ||
102 | 349 | :param str unit_name: Unit name to use in check description | ||
103 | 350 | """ | ||
104 | 351 | nrpe.add_check( | ||
105 | 352 | shortname='haproxy_servers', | ||
106 | 353 | description='Check HAProxy {%s}' % unit_name, | ||
107 | 354 | check_cmd='check_haproxy.sh') | ||
108 | 355 | nrpe.add_check( | ||
109 | 356 | shortname='haproxy_queue', | ||
110 | 357 | description='Check HAProxy queue depth {%s}' % unit_name, | ||
111 | 358 | check_cmd='check_haproxy_queue_depth.sh') | ||
112 | 325 | 359 | ||
113 | === modified file 'hooks/charmhelpers/core/fstab.py' | |||
114 | --- hooks/charmhelpers/core/fstab.py 2015-01-26 09:46:20 +0000 | |||
115 | +++ hooks/charmhelpers/core/fstab.py 2015-02-24 06:27:04 +0000 | |||
116 | @@ -17,11 +17,11 @@ | |||
117 | 17 | # You should have received a copy of the GNU Lesser General Public License | 17 | # You should have received a copy of the GNU Lesser General Public License |
118 | 18 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. | 18 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
119 | 19 | 19 | ||
120 | 20 | __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' | ||
121 | 21 | |||
122 | 22 | import io | 20 | import io |
123 | 23 | import os | 21 | import os |
124 | 24 | 22 | ||
125 | 23 | __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' | ||
126 | 24 | |||
127 | 25 | 25 | ||
128 | 26 | class Fstab(io.FileIO): | 26 | class Fstab(io.FileIO): |
129 | 27 | """This class extends file in order to implement a file reader/writer | 27 | """This class extends file in order to implement a file reader/writer |
130 | @@ -77,7 +77,7 @@ | |||
131 | 77 | for line in self.readlines(): | 77 | for line in self.readlines(): |
132 | 78 | line = line.decode('us-ascii') | 78 | line = line.decode('us-ascii') |
133 | 79 | try: | 79 | try: |
135 | 80 | if line.strip() and not line.startswith("#"): | 80 | if line.strip() and not line.strip().startswith("#"): |
136 | 81 | yield self._hydrate_entry(line) | 81 | yield self._hydrate_entry(line) |
137 | 82 | except ValueError: | 82 | except ValueError: |
138 | 83 | pass | 83 | pass |
139 | @@ -104,7 +104,7 @@ | |||
140 | 104 | 104 | ||
141 | 105 | found = False | 105 | found = False |
142 | 106 | for index, line in enumerate(lines): | 106 | for index, line in enumerate(lines): |
144 | 107 | if not line.startswith("#"): | 107 | if line.strip() and not line.strip().startswith("#"): |
145 | 108 | if self._hydrate_entry(line) == entry: | 108 | if self._hydrate_entry(line) == entry: |
146 | 109 | found = True | 109 | found = True |
147 | 110 | break | 110 | break |
148 | 111 | 111 | ||
149 | === modified file 'hooks/charmhelpers/core/host.py' | |||
150 | --- hooks/charmhelpers/core/host.py 2015-01-26 09:46:20 +0000 | |||
151 | +++ hooks/charmhelpers/core/host.py 2015-02-24 06:27:04 +0000 | |||
152 | @@ -191,11 +191,11 @@ | |||
153 | 191 | 191 | ||
154 | 192 | 192 | ||
155 | 193 | def write_file(path, content, owner='root', group='root', perms=0o444): | 193 | def write_file(path, content, owner='root', group='root', perms=0o444): |
157 | 194 | """Create or overwrite a file with the contents of a string""" | 194 | """Create or overwrite a file with the contents of a byte string.""" |
158 | 195 | log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) | 195 | log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) |
159 | 196 | uid = pwd.getpwnam(owner).pw_uid | 196 | uid = pwd.getpwnam(owner).pw_uid |
160 | 197 | gid = grp.getgrnam(group).gr_gid | 197 | gid = grp.getgrnam(group).gr_gid |
162 | 198 | with open(path, 'w') as target: | 198 | with open(path, 'wb') as target: |
163 | 199 | os.fchown(target.fileno(), uid, gid) | 199 | os.fchown(target.fileno(), uid, gid) |
164 | 200 | os.fchmod(target.fileno(), perms) | 200 | os.fchmod(target.fileno(), perms) |
165 | 201 | target.write(content) | 201 | target.write(content) |
166 | @@ -305,11 +305,11 @@ | |||
167 | 305 | ceph_client_changed function. | 305 | ceph_client_changed function. |
168 | 306 | """ | 306 | """ |
169 | 307 | def wrap(f): | 307 | def wrap(f): |
171 | 308 | def wrapped_f(*args): | 308 | def wrapped_f(*args, **kwargs): |
172 | 309 | checksums = {} | 309 | checksums = {} |
173 | 310 | for path in restart_map: | 310 | for path in restart_map: |
174 | 311 | checksums[path] = file_hash(path) | 311 | checksums[path] = file_hash(path) |
176 | 312 | f(*args) | 312 | f(*args, **kwargs) |
177 | 313 | restarts = [] | 313 | restarts = [] |
178 | 314 | for path in restart_map: | 314 | for path in restart_map: |
179 | 315 | if checksums[path] != file_hash(path): | 315 | if checksums[path] != file_hash(path): |
180 | @@ -361,7 +361,7 @@ | |||
181 | 361 | ip_output = (line for line in ip_output if line) | 361 | ip_output = (line for line in ip_output if line) |
182 | 362 | for line in ip_output: | 362 | for line in ip_output: |
183 | 363 | if line.split()[1].startswith(int_type): | 363 | if line.split()[1].startswith(int_type): |
185 | 364 | matched = re.search('.*: (bond[0-9]+\.[0-9]+)@.*', line) | 364 | matched = re.search('.*: (' + int_type + r'[0-9]+\.[0-9]+)@.*', line) |
186 | 365 | if matched: | 365 | if matched: |
187 | 366 | interface = matched.groups()[0] | 366 | interface = matched.groups()[0] |
188 | 367 | else: | 367 | else: |
189 | 368 | 368 | ||
190 | === added file 'hooks/charmhelpers/core/strutils.py' | |||
191 | --- hooks/charmhelpers/core/strutils.py 1970-01-01 00:00:00 +0000 | |||
192 | +++ hooks/charmhelpers/core/strutils.py 2015-02-24 06:27:04 +0000 | |||
193 | @@ -0,0 +1,42 @@ | |||
194 | 1 | #!/usr/bin/env python | ||
195 | 2 | # -*- coding: utf-8 -*- | ||
196 | 3 | |||
197 | 4 | # Copyright 2014-2015 Canonical Limited. | ||
198 | 5 | # | ||
199 | 6 | # This file is part of charm-helpers. | ||
200 | 7 | # | ||
201 | 8 | # charm-helpers is free software: you can redistribute it and/or modify | ||
202 | 9 | # it under the terms of the GNU Lesser General Public License version 3 as | ||
203 | 10 | # published by the Free Software Foundation. | ||
204 | 11 | # | ||
205 | 12 | # charm-helpers is distributed in the hope that it will be useful, | ||
206 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
207 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
208 | 15 | # GNU Lesser General Public License for more details. | ||
209 | 16 | # | ||
210 | 17 | # You should have received a copy of the GNU Lesser General Public License | ||
211 | 18 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. | ||
212 | 19 | |||
213 | 20 | import six | ||
214 | 21 | |||
215 | 22 | |||
216 | 23 | def bool_from_string(value): | ||
217 | 24 | """Interpret string value as boolean. | ||
218 | 25 | |||
219 | 26 | Returns True if value translates to True otherwise False. | ||
220 | 27 | """ | ||
221 | 28 | if isinstance(value, six.string_types): | ||
222 | 29 | value = six.text_type(value) | ||
223 | 30 | else: | ||
224 | 31 | msg = "Unable to interpret non-string value '%s' as boolean" % (value) | ||
225 | 32 | raise ValueError(msg) | ||
226 | 33 | |||
227 | 34 | value = value.strip().lower() | ||
228 | 35 | |||
229 | 36 | if value in ['y', 'yes', 'true', 't']: | ||
230 | 37 | return True | ||
231 | 38 | elif value in ['n', 'no', 'false', 'f']: | ||
232 | 39 | return False | ||
233 | 40 | |||
234 | 41 | msg = "Unable to interpret string value '%s' as boolean" % (value) | ||
235 | 42 | raise ValueError(msg) | ||
236 | 0 | 43 | ||
237 | === modified file 'hooks/charmhelpers/core/sysctl.py' | |||
238 | --- hooks/charmhelpers/core/sysctl.py 2015-01-26 09:46:20 +0000 | |||
239 | +++ hooks/charmhelpers/core/sysctl.py 2015-02-24 06:27:04 +0000 | |||
240 | @@ -17,8 +17,6 @@ | |||
241 | 17 | # You should have received a copy of the GNU Lesser General Public License | 17 | # You should have received a copy of the GNU Lesser General Public License |
242 | 18 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. | 18 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
243 | 19 | 19 | ||
244 | 20 | __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' | ||
245 | 21 | |||
246 | 22 | import yaml | 20 | import yaml |
247 | 23 | 21 | ||
248 | 24 | from subprocess import check_call | 22 | from subprocess import check_call |
249 | @@ -26,25 +24,33 @@ | |||
250 | 26 | from charmhelpers.core.hookenv import ( | 24 | from charmhelpers.core.hookenv import ( |
251 | 27 | log, | 25 | log, |
252 | 28 | DEBUG, | 26 | DEBUG, |
253 | 27 | ERROR, | ||
254 | 29 | ) | 28 | ) |
255 | 30 | 29 | ||
256 | 30 | __author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' | ||
257 | 31 | |||
258 | 31 | 32 | ||
259 | 32 | def create(sysctl_dict, sysctl_file): | 33 | def create(sysctl_dict, sysctl_file): |
260 | 33 | """Creates a sysctl.conf file from a YAML associative array | 34 | """Creates a sysctl.conf file from a YAML associative array |
261 | 34 | 35 | ||
264 | 35 | :param sysctl_dict: a dict of sysctl options eg { 'kernel.max_pid': 1337 } | 36 | :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" |
265 | 36 | :type sysctl_dict: dict | 37 | :type sysctl_dict: str |
266 | 37 | :param sysctl_file: path to the sysctl file to be saved | 38 | :param sysctl_file: path to the sysctl file to be saved |
267 | 38 | :type sysctl_file: str or unicode | 39 | :type sysctl_file: str or unicode |
268 | 39 | :returns: None | 40 | :returns: None |
269 | 40 | """ | 41 | """ |
271 | 41 | sysctl_dict = yaml.load(sysctl_dict) | 42 | try: |
272 | 43 | sysctl_dict_parsed = yaml.safe_load(sysctl_dict) | ||
273 | 44 | except yaml.YAMLError: | ||
274 | 45 | log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), | ||
275 | 46 | level=ERROR) | ||
276 | 47 | return | ||
277 | 42 | 48 | ||
278 | 43 | with open(sysctl_file, "w") as fd: | 49 | with open(sysctl_file, "w") as fd: |
280 | 44 | for key, value in sysctl_dict.items(): | 50 | for key, value in sysctl_dict_parsed.items(): |
281 | 45 | fd.write("{}={}\n".format(key, value)) | 51 | fd.write("{}={}\n".format(key, value)) |
282 | 46 | 52 | ||
284 | 47 | log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict), | 53 | log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed), |
285 | 48 | level=DEBUG) | 54 | level=DEBUG) |
286 | 49 | 55 | ||
287 | 50 | check_call(["sysctl", "-p", sysctl_file]) | 56 | check_call(["sysctl", "-p", sysctl_file]) |
288 | 51 | 57 | ||
289 | === modified file 'hooks/charmhelpers/core/templating.py' | |||
290 | --- hooks/charmhelpers/core/templating.py 2015-01-26 09:46:20 +0000 | |||
291 | +++ hooks/charmhelpers/core/templating.py 2015-02-24 06:27:04 +0000 | |||
292 | @@ -21,7 +21,7 @@ | |||
293 | 21 | 21 | ||
294 | 22 | 22 | ||
295 | 23 | def render(source, target, context, owner='root', group='root', | 23 | def render(source, target, context, owner='root', group='root', |
297 | 24 | perms=0o444, templates_dir=None): | 24 | perms=0o444, templates_dir=None, encoding='UTF-8'): |
298 | 25 | """ | 25 | """ |
299 | 26 | Render a template. | 26 | Render a template. |
300 | 27 | 27 | ||
301 | @@ -64,5 +64,5 @@ | |||
302 | 64 | level=hookenv.ERROR) | 64 | level=hookenv.ERROR) |
303 | 65 | raise e | 65 | raise e |
304 | 66 | content = template.render(context) | 66 | content = template.render(context) |
307 | 67 | host.mkdir(os.path.dirname(target), owner, group) | 67 | host.mkdir(os.path.dirname(target), owner, group, perms=0o755) |
308 | 68 | host.write_file(target, content, owner, group, perms) | 68 | host.write_file(target, content.encode(encoding), owner, group, perms) |
309 | 69 | 69 | ||
310 | === added file 'hooks/charmhelpers/core/unitdata.py' | |||
311 | --- hooks/charmhelpers/core/unitdata.py 1970-01-01 00:00:00 +0000 | |||
312 | +++ hooks/charmhelpers/core/unitdata.py 2015-02-24 06:27:04 +0000 | |||
313 | @@ -0,0 +1,477 @@ | |||
314 | 1 | #!/usr/bin/env python | ||
315 | 2 | # -*- coding: utf-8 -*- | ||
316 | 3 | # | ||
317 | 4 | # Copyright 2014-2015 Canonical Limited. | ||
318 | 5 | # | ||
319 | 6 | # This file is part of charm-helpers. | ||
320 | 7 | # | ||
321 | 8 | # charm-helpers is free software: you can redistribute it and/or modify | ||
322 | 9 | # it under the terms of the GNU Lesser General Public License version 3 as | ||
323 | 10 | # published by the Free Software Foundation. | ||
324 | 11 | # | ||
325 | 12 | # charm-helpers is distributed in the hope that it will be useful, | ||
326 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
327 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
328 | 15 | # GNU Lesser General Public License for more details. | ||
329 | 16 | # | ||
330 | 17 | # You should have received a copy of the GNU Lesser General Public License | ||
331 | 18 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. | ||
332 | 19 | # | ||
333 | 20 | # | ||
334 | 21 | # Authors: | ||
335 | 22 | # Kapil Thangavelu <kapil.foss@gmail.com> | ||
336 | 23 | # | ||
337 | 24 | """ | ||
338 | 25 | Intro | ||
339 | 26 | ----- | ||
340 | 27 | |||
341 | 28 | A simple way to store state in units. This provides a key value | ||
342 | 29 | storage with support for versioned, transactional operation, | ||
343 | 30 | and can calculate deltas from previous values to simplify unit logic | ||
344 | 31 | when processing changes. | ||
345 | 32 | |||
346 | 33 | |||
347 | 34 | Hook Integration | ||
348 | 35 | ---------------- | ||
349 | 36 | |||
350 | 37 | There are several extant frameworks for hook execution, including | ||
351 | 38 | |||
352 | 39 | - charmhelpers.core.hookenv.Hooks | ||
353 | 40 | - charmhelpers.core.services.ServiceManager | ||
354 | 41 | |||
355 | 42 | The storage classes are framework agnostic, one simple integration is | ||
356 | 43 | via the HookData contextmanager. It will record the current hook | ||
357 | 44 | execution environment (including relation data, config data, etc.), | ||
358 | 45 | setup a transaction and allow easy access to the changes from | ||
359 | 46 | previously seen values. One consequence of the integration is the | ||
360 | 47 | reservation of particular keys ('rels', 'unit', 'env', 'config', | ||
361 | 48 | 'charm_revisions') for their respective values. | ||
362 | 49 | |||
363 | 50 | Here's a fully worked integration example using hookenv.Hooks:: | ||
364 | 51 | |||
365 | 52 | from charmhelper.core import hookenv, unitdata | ||
366 | 53 | |||
367 | 54 | hook_data = unitdata.HookData() | ||
368 | 55 | db = unitdata.kv() | ||
369 | 56 | hooks = hookenv.Hooks() | ||
370 | 57 | |||
371 | 58 | @hooks.hook | ||
372 | 59 | def config_changed(): | ||
373 | 60 | # Print all changes to configuration from previously seen | ||
374 | 61 | # values. | ||
375 | 62 | for changed, (prev, cur) in hook_data.conf.items(): | ||
376 | 63 | print('config changed', changed, | ||
377 | 64 | 'previous value', prev, | ||
378 | 65 | 'current value', cur) | ||
379 | 66 | |||
380 | 67 | # Get some unit specific bookeeping | ||
381 | 68 | if not db.get('pkg_key'): | ||
382 | 69 | key = urllib.urlopen('https://example.com/pkg_key').read() | ||
383 | 70 | db.set('pkg_key', key) | ||
384 | 71 | |||
385 | 72 | # Directly access all charm config as a mapping. | ||
386 | 73 | conf = db.getrange('config', True) | ||
387 | 74 | |||
388 | 75 | # Directly access all relation data as a mapping | ||
389 | 76 | rels = db.getrange('rels', True) | ||
390 | 77 | |||
391 | 78 | if __name__ == '__main__': | ||
392 | 79 | with hook_data(): | ||
393 | 80 | hook.execute() | ||
394 | 81 | |||
395 | 82 | |||
396 | 83 | A more basic integration is via the hook_scope context manager which simply | ||
397 | 84 | manages transaction scope (and records hook name, and timestamp):: | ||
398 | 85 | |||
399 | 86 | >>> from unitdata import kv | ||
400 | 87 | >>> db = kv() | ||
401 | 88 | >>> with db.hook_scope('install'): | ||
402 | 89 | ... # do work, in transactional scope. | ||
403 | 90 | ... db.set('x', 1) | ||
404 | 91 | >>> db.get('x') | ||
405 | 92 | 1 | ||
406 | 93 | |||
407 | 94 | |||
408 | 95 | Usage | ||
409 | 96 | ----- | ||
410 | 97 | |||
411 | 98 | Values are automatically json de/serialized to preserve basic typing | ||
412 | 99 | and complex data struct capabilities (dicts, lists, ints, booleans, etc). | ||
413 | 100 | |||
414 | 101 | Individual values can be manipulated via get/set:: | ||
415 | 102 | |||
416 | 103 | >>> kv.set('y', True) | ||
417 | 104 | >>> kv.get('y') | ||
418 | 105 | True | ||
419 | 106 | |||
420 | 107 | # We can set complex values (dicts, lists) as a single key. | ||
421 | 108 | >>> kv.set('config', {'a': 1, 'b': True'}) | ||
422 | 109 | |||
423 | 110 | # Also supports returning dictionaries as a record which | ||
424 | 111 | # provides attribute access. | ||
425 | 112 | >>> config = kv.get('config', record=True) | ||
426 | 113 | >>> config.b | ||
427 | 114 | True | ||
428 | 115 | |||
429 | 116 | |||
430 | 117 | Groups of keys can be manipulated with update/getrange:: | ||
431 | 118 | |||
432 | 119 | >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") | ||
433 | 120 | >>> kv.getrange('gui.', strip=True) | ||
434 | 121 | {'z': 1, 'y': 2} | ||
435 | 122 | |||
436 | 123 | When updating values, its very helpful to understand which values | ||
437 | 124 | have actually changed and how have they changed. The storage | ||
438 | 125 | provides a delta method to provide for this:: | ||
439 | 126 | |||
440 | 127 | >>> data = {'debug': True, 'option': 2} | ||
441 | 128 | >>> delta = kv.delta(data, 'config.') | ||
442 | 129 | >>> delta.debug.previous | ||
443 | 130 | None | ||
444 | 131 | >>> delta.debug.current | ||
445 | 132 | True | ||
446 | 133 | >>> delta | ||
447 | 134 | {'debug': (None, True), 'option': (None, 2)} | ||
448 | 135 | |||
449 | 136 | Note the delta method does not persist the actual change, it needs to | ||
450 | 137 | be explicitly saved via 'update' method:: | ||
451 | 138 | |||
452 | 139 | >>> kv.update(data, 'config.') | ||
453 | 140 | |||
454 | 141 | Values modified in the context of a hook scope retain historical values | ||
455 | 142 | associated to the hookname. | ||
456 | 143 | |||
457 | 144 | >>> with db.hook_scope('config-changed'): | ||
458 | 145 | ... db.set('x', 42) | ||
459 | 146 | >>> db.gethistory('x') | ||
460 | 147 | [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), | ||
461 | 148 | (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] | ||
462 | 149 | |||
463 | 150 | """ | ||
464 | 151 | |||
465 | 152 | import collections | ||
466 | 153 | import contextlib | ||
467 | 154 | import datetime | ||
468 | 155 | import json | ||
469 | 156 | import os | ||
470 | 157 | import pprint | ||
471 | 158 | import sqlite3 | ||
472 | 159 | import sys | ||
473 | 160 | |||
474 | 161 | __author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>' | ||
475 | 162 | |||
476 | 163 | |||
477 | 164 | class Storage(object): | ||
478 | 165 | """Simple key value database for local unit state within charms. | ||
479 | 166 | |||
480 | 167 | Modifications are automatically committed at hook exit. That's | ||
481 | 168 | currently regardless of exit code. | ||
482 | 169 | |||
483 | 170 | To support dicts, lists, integer, floats, and booleans values | ||
484 | 171 | are automatically json encoded/decoded. | ||
485 | 172 | """ | ||
486 | 173 | def __init__(self, path=None): | ||
487 | 174 | self.db_path = path | ||
488 | 175 | if path is None: | ||
489 | 176 | self.db_path = os.path.join( | ||
490 | 177 | os.environ.get('CHARM_DIR', ''), '.unit-state.db') | ||
491 | 178 | self.conn = sqlite3.connect('%s' % self.db_path) | ||
492 | 179 | self.cursor = self.conn.cursor() | ||
493 | 180 | self.revision = None | ||
494 | 181 | self._closed = False | ||
495 | 182 | self._init() | ||
496 | 183 | |||
497 | 184 | def close(self): | ||
498 | 185 | if self._closed: | ||
499 | 186 | return | ||
500 | 187 | self.flush(False) | ||
501 | 188 | self.cursor.close() | ||
502 | 189 | self.conn.close() | ||
503 | 190 | self._closed = True | ||
504 | 191 | |||
505 | 192 | def _scoped_query(self, stmt, params=None): | ||
506 | 193 | if params is None: | ||
507 | 194 | params = [] | ||
508 | 195 | return stmt, params | ||
509 | 196 | |||
510 | 197 | def get(self, key, default=None, record=False): | ||
511 | 198 | self.cursor.execute( | ||
512 | 199 | *self._scoped_query( | ||
513 | 200 | 'select data from kv where key=?', [key])) | ||
514 | 201 | result = self.cursor.fetchone() | ||
515 | 202 | if not result: | ||
516 | 203 | return default | ||
517 | 204 | if record: | ||
518 | 205 | return Record(json.loads(result[0])) | ||
519 | 206 | return json.loads(result[0]) | ||
520 | 207 | |||
521 | 208 | def getrange(self, key_prefix, strip=False): | ||
522 | 209 | stmt = "select key, data from kv where key like '%s%%'" % key_prefix | ||
523 | 210 | self.cursor.execute(*self._scoped_query(stmt)) | ||
524 | 211 | result = self.cursor.fetchall() | ||
525 | 212 | |||
526 | 213 | if not result: | ||
527 | 214 | return None | ||
528 | 215 | if not strip: | ||
529 | 216 | key_prefix = '' | ||
530 | 217 | return dict([ | ||
531 | 218 | (k[len(key_prefix):], json.loads(v)) for k, v in result]) | ||
532 | 219 | |||
533 | 220 | def update(self, mapping, prefix=""): | ||
534 | 221 | for k, v in mapping.items(): | ||
535 | 222 | self.set("%s%s" % (prefix, k), v) | ||
536 | 223 | |||
537 | 224 | def unset(self, key): | ||
538 | 225 | self.cursor.execute('delete from kv where key=?', [key]) | ||
539 | 226 | if self.revision and self.cursor.rowcount: | ||
540 | 227 | self.cursor.execute( | ||
541 | 228 | 'insert into kv_revisions values (?, ?, ?)', | ||
542 | 229 | [key, self.revision, json.dumps('DELETED')]) | ||
543 | 230 | |||
544 | 231 | def set(self, key, value): | ||
545 | 232 | serialized = json.dumps(value) | ||
546 | 233 | |||
547 | 234 | self.cursor.execute( | ||
548 | 235 | 'select data from kv where key=?', [key]) | ||
549 | 236 | exists = self.cursor.fetchone() | ||
550 | 237 | |||
551 | 238 | # Skip mutations to the same value | ||
552 | 239 | if exists: | ||
553 | 240 | if exists[0] == serialized: | ||
554 | 241 | return value | ||
555 | 242 | |||
556 | 243 | if not exists: | ||
557 | 244 | self.cursor.execute( | ||
558 | 245 | 'insert into kv (key, data) values (?, ?)', | ||
559 | 246 | (key, serialized)) | ||
560 | 247 | else: | ||
561 | 248 | self.cursor.execute(''' | ||
562 | 249 | update kv | ||
563 | 250 | set data = ? | ||
564 | 251 | where key = ?''', [serialized, key]) | ||
565 | 252 | |||
566 | 253 | # Save | ||
567 | 254 | if not self.revision: | ||
568 | 255 | return value | ||
569 | 256 | |||
570 | 257 | self.cursor.execute( | ||
571 | 258 | 'select 1 from kv_revisions where key=? and revision=?', | ||
572 | 259 | [key, self.revision]) | ||
573 | 260 | exists = self.cursor.fetchone() | ||
574 | 261 | |||
575 | 262 | if not exists: | ||
576 | 263 | self.cursor.execute( | ||
577 | 264 | '''insert into kv_revisions ( | ||
578 | 265 | revision, key, data) values (?, ?, ?)''', | ||
579 | 266 | (self.revision, key, serialized)) | ||
580 | 267 | else: | ||
581 | 268 | self.cursor.execute( | ||
582 | 269 | ''' | ||
583 | 270 | update kv_revisions | ||
584 | 271 | set data = ? | ||
585 | 272 | where key = ? | ||
586 | 273 | and revision = ?''', | ||
587 | 274 | [serialized, key, self.revision]) | ||
588 | 275 | |||
589 | 276 | return value | ||
590 | 277 | |||
591 | 278 | def delta(self, mapping, prefix): | ||
592 | 279 | """ | ||
593 | 280 | return a delta containing values that have changed. | ||
594 | 281 | """ | ||
595 | 282 | previous = self.getrange(prefix, strip=True) | ||
596 | 283 | if not previous: | ||
597 | 284 | pk = set() | ||
598 | 285 | else: | ||
599 | 286 | pk = set(previous.keys()) | ||
600 | 287 | ck = set(mapping.keys()) | ||
601 | 288 | delta = DeltaSet() | ||
602 | 289 | |||
603 | 290 | # added | ||
604 | 291 | for k in ck.difference(pk): | ||
605 | 292 | delta[k] = Delta(None, mapping[k]) | ||
606 | 293 | |||
607 | 294 | # removed | ||
608 | 295 | for k in pk.difference(ck): | ||
609 | 296 | delta[k] = Delta(previous[k], None) | ||
610 | 297 | |||
611 | 298 | # changed | ||
612 | 299 | for k in pk.intersection(ck): | ||
613 | 300 | c = mapping[k] | ||
614 | 301 | p = previous[k] | ||
615 | 302 | if c != p: | ||
616 | 303 | delta[k] = Delta(p, c) | ||
617 | 304 | |||
618 | 305 | return delta | ||
619 | 306 | |||
620 | 307 | @contextlib.contextmanager | ||
621 | 308 | def hook_scope(self, name=""): | ||
622 | 309 | """Scope all future interactions to the current hook execution | ||
623 | 310 | revision.""" | ||
624 | 311 | assert not self.revision | ||
625 | 312 | self.cursor.execute( | ||
626 | 313 | 'insert into hooks (hook, date) values (?, ?)', | ||
627 | 314 | (name or sys.argv[0], | ||
628 | 315 | datetime.datetime.utcnow().isoformat())) | ||
629 | 316 | self.revision = self.cursor.lastrowid | ||
630 | 317 | try: | ||
631 | 318 | yield self.revision | ||
632 | 319 | self.revision = None | ||
633 | 320 | except: | ||
634 | 321 | self.flush(False) | ||
635 | 322 | self.revision = None | ||
636 | 323 | raise | ||
637 | 324 | else: | ||
638 | 325 | self.flush() | ||
639 | 326 | |||
640 | 327 | def flush(self, save=True): | ||
641 | 328 | if save: | ||
642 | 329 | self.conn.commit() | ||
643 | 330 | elif self._closed: | ||
644 | 331 | return | ||
645 | 332 | else: | ||
646 | 333 | self.conn.rollback() | ||
647 | 334 | |||
648 | 335 | def _init(self): | ||
649 | 336 | self.cursor.execute(''' | ||
650 | 337 | create table if not exists kv ( | ||
651 | 338 | key text, | ||
652 | 339 | data text, | ||
653 | 340 | primary key (key) | ||
654 | 341 | )''') | ||
655 | 342 | self.cursor.execute(''' | ||
656 | 343 | create table if not exists kv_revisions ( | ||
657 | 344 | key text, | ||
658 | 345 | revision integer, | ||
659 | 346 | data text, | ||
660 | 347 | primary key (key, revision) | ||
661 | 348 | )''') | ||
662 | 349 | self.cursor.execute(''' | ||
663 | 350 | create table if not exists hooks ( | ||
664 | 351 | version integer primary key autoincrement, | ||
665 | 352 | hook text, | ||
666 | 353 | date text | ||
667 | 354 | )''') | ||
668 | 355 | self.conn.commit() | ||
669 | 356 | |||
670 | 357 | def gethistory(self, key, deserialize=False): | ||
671 | 358 | self.cursor.execute( | ||
672 | 359 | ''' | ||
673 | 360 | select kv.revision, kv.key, kv.data, h.hook, h.date | ||
674 | 361 | from kv_revisions kv, | ||
675 | 362 | hooks h | ||
676 | 363 | where kv.key=? | ||
677 | 364 | and kv.revision = h.version | ||
678 | 365 | ''', [key]) | ||
679 | 366 | if deserialize is False: | ||
680 | 367 | return self.cursor.fetchall() | ||
681 | 368 | return map(_parse_history, self.cursor.fetchall()) | ||
682 | 369 | |||
683 | 370 | def debug(self, fh=sys.stderr): | ||
684 | 371 | self.cursor.execute('select * from kv') | ||
685 | 372 | pprint.pprint(self.cursor.fetchall(), stream=fh) | ||
686 | 373 | self.cursor.execute('select * from kv_revisions') | ||
687 | 374 | pprint.pprint(self.cursor.fetchall(), stream=fh) | ||
688 | 375 | |||
689 | 376 | |||
690 | 377 | def _parse_history(d): | ||
691 | 378 | return (d[0], d[1], json.loads(d[2]), d[3], | ||
692 | 379 | datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) | ||
693 | 380 | |||
694 | 381 | |||
695 | 382 | class HookData(object): | ||
696 | 383 | """Simple integration for existing hook exec frameworks. | ||
697 | 384 | |||
698 | 385 | Records all unit information, and stores deltas for processing | ||
699 | 386 | by the hook. | ||
700 | 387 | |||
701 | 388 | Sample:: | ||
702 | 389 | |||
703 | 390 | from charmhelper.core import hookenv, unitdata | ||
704 | 391 | |||
705 | 392 | changes = unitdata.HookData() | ||
706 | 393 | db = unitdata.kv() | ||
707 | 394 | hooks = hookenv.Hooks() | ||
708 | 395 | |||
709 | 396 | @hooks.hook | ||
710 | 397 | def config_changed(): | ||
711 | 398 | # View all changes to configuration | ||
712 | 399 | for changed, (prev, cur) in changes.conf.items(): | ||
713 | 400 | print('config changed', changed, | ||
714 | 401 | 'previous value', prev, | ||
715 | 402 | 'current value', cur) | ||
716 | 403 | |||
717 | 404 | # Get some unit specific bookeeping | ||
718 | 405 | if not db.get('pkg_key'): | ||
719 | 406 | key = urllib.urlopen('https://example.com/pkg_key').read() | ||
720 | 407 | db.set('pkg_key', key) | ||
721 | 408 | |||
722 | 409 | if __name__ == '__main__': | ||
723 | 410 | with changes(): | ||
724 | 411 | hook.execute() | ||
725 | 412 | |||
726 | 413 | """ | ||
727 | 414 | def __init__(self): | ||
728 | 415 | self.kv = kv() | ||
729 | 416 | self.conf = None | ||
730 | 417 | self.rels = None | ||
731 | 418 | |||
732 | 419 | @contextlib.contextmanager | ||
733 | 420 | def __call__(self): | ||
734 | 421 | from charmhelpers.core import hookenv | ||
735 | 422 | hook_name = hookenv.hook_name() | ||
736 | 423 | |||
737 | 424 | with self.kv.hook_scope(hook_name): | ||
738 | 425 | self._record_charm_version(hookenv.charm_dir()) | ||
739 | 426 | delta_config, delta_relation = self._record_hook(hookenv) | ||
740 | 427 | yield self.kv, delta_config, delta_relation | ||
741 | 428 | |||
742 | 429 | def _record_charm_version(self, charm_dir): | ||
743 | 430 | # Record revisions.. charm revisions are meaningless | ||
744 | 431 | # to charm authors as they don't control the revision. | ||
745 | 432 | # so logic dependnent on revision is not particularly | ||
746 | 433 | # useful, however it is useful for debugging analysis. | ||
747 | 434 | charm_rev = open( | ||
748 | 435 | os.path.join(charm_dir, 'revision')).read().strip() | ||
749 | 436 | charm_rev = charm_rev or '0' | ||
750 | 437 | revs = self.kv.get('charm_revisions', []) | ||
751 | 438 | if charm_rev not in revs: | ||
752 | 439 | revs.append(charm_rev.strip() or '0') | ||
753 | 440 | self.kv.set('charm_revisions', revs) | ||
754 | 441 | |||
755 | 442 | def _record_hook(self, hookenv): | ||
756 | 443 | data = hookenv.execution_environment() | ||
757 | 444 | self.conf = conf_delta = self.kv.delta(data['conf'], 'config') | ||
758 | 445 | self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') | ||
759 | 446 | self.kv.set('env', data['env']) | ||
760 | 447 | self.kv.set('unit', data['unit']) | ||
761 | 448 | self.kv.set('relid', data.get('relid')) | ||
762 | 449 | return conf_delta, rels_delta | ||
763 | 450 | |||
764 | 451 | |||
765 | 452 | class Record(dict): | ||
766 | 453 | |||
767 | 454 | __slots__ = () | ||
768 | 455 | |||
769 | 456 | def __getattr__(self, k): | ||
770 | 457 | if k in self: | ||
771 | 458 | return self[k] | ||
772 | 459 | raise AttributeError(k) | ||
773 | 460 | |||
774 | 461 | |||
775 | 462 | class DeltaSet(Record): | ||
776 | 463 | |||
777 | 464 | __slots__ = () | ||
778 | 465 | |||
779 | 466 | |||
780 | 467 | Delta = collections.namedtuple('Delta', ['previous', 'current']) | ||
781 | 468 | |||
782 | 469 | |||
783 | 470 | _KV = None | ||
784 | 471 | |||
785 | 472 | |||
786 | 473 | def kv(): | ||
787 | 474 | global _KV | ||
788 | 475 | if _KV is None: | ||
789 | 476 | _KV = Storage() | ||
790 | 477 | return _KV | ||
791 | 0 | 478 | ||
792 | === modified file 'hooks/charmhelpers/fetch/archiveurl.py' | |||
793 | --- hooks/charmhelpers/fetch/archiveurl.py 2015-01-26 09:46:20 +0000 | |||
794 | +++ hooks/charmhelpers/fetch/archiveurl.py 2015-02-24 06:27:04 +0000 | |||
795 | @@ -18,6 +18,16 @@ | |||
796 | 18 | import hashlib | 18 | import hashlib |
797 | 19 | import re | 19 | import re |
798 | 20 | 20 | ||
799 | 21 | from charmhelpers.fetch import ( | ||
800 | 22 | BaseFetchHandler, | ||
801 | 23 | UnhandledSource | ||
802 | 24 | ) | ||
803 | 25 | from charmhelpers.payload.archive import ( | ||
804 | 26 | get_archive_handler, | ||
805 | 27 | extract, | ||
806 | 28 | ) | ||
807 | 29 | from charmhelpers.core.host import mkdir, check_hash | ||
808 | 30 | |||
809 | 21 | import six | 31 | import six |
810 | 22 | if six.PY3: | 32 | if six.PY3: |
811 | 23 | from urllib.request import ( | 33 | from urllib.request import ( |
812 | @@ -35,16 +45,6 @@ | |||
813 | 35 | ) | 45 | ) |
814 | 36 | from urlparse import urlparse, urlunparse, parse_qs | 46 | from urlparse import urlparse, urlunparse, parse_qs |
815 | 37 | 47 | ||
816 | 38 | from charmhelpers.fetch import ( | ||
817 | 39 | BaseFetchHandler, | ||
818 | 40 | UnhandledSource | ||
819 | 41 | ) | ||
820 | 42 | from charmhelpers.payload.archive import ( | ||
821 | 43 | get_archive_handler, | ||
822 | 44 | extract, | ||
823 | 45 | ) | ||
824 | 46 | from charmhelpers.core.host import mkdir, check_hash | ||
825 | 47 | |||
826 | 48 | 48 | ||
827 | 49 | def splituser(host): | 49 | def splituser(host): |
828 | 50 | '''urllib.splituser(), but six's support of this seems broken''' | 50 | '''urllib.splituser(), but six's support of this seems broken''' |
829 | 51 | 51 | ||
830 | === modified file 'hooks/charmhelpers/fetch/giturl.py' | |||
831 | --- hooks/charmhelpers/fetch/giturl.py 2015-01-26 09:46:20 +0000 | |||
832 | +++ hooks/charmhelpers/fetch/giturl.py 2015-02-24 06:27:04 +0000 | |||
833 | @@ -32,7 +32,7 @@ | |||
834 | 32 | apt_install("python-git") | 32 | apt_install("python-git") |
835 | 33 | from git import Repo | 33 | from git import Repo |
836 | 34 | 34 | ||
838 | 35 | from git.exc import GitCommandError | 35 | from git.exc import GitCommandError # noqa E402 |
839 | 36 | 36 | ||
840 | 37 | 37 | ||
841 | 38 | class GitUrlFetchHandler(BaseFetchHandler): | 38 | class GitUrlFetchHandler(BaseFetchHandler): |
842 | 39 | 39 | ||
843 | === modified file 'metadata.yaml' | |||
844 | --- metadata.yaml 2014-10-30 06:57:10 +0000 | |||
845 | +++ metadata.yaml 2015-02-24 06:27:04 +0000 | |||
846 | @@ -22,4 +22,3 @@ | |||
847 | 22 | nrpe-external-master: | 22 | nrpe-external-master: |
848 | 23 | interface: nrpe-external-master | 23 | interface: nrpe-external-master |
849 | 24 | scope: container | 24 | scope: container |
850 | 25 | gets: [nagios_hostname, nagios_host_context] | ||
851 | 26 | 25 | ||
852 | === modified file 'tests/basic_deployment.py' | |||
853 | --- tests/basic_deployment.py 2014-09-29 20:41:39 +0000 | |||
854 | +++ tests/basic_deployment.py 2015-02-24 06:27:04 +0000 | |||
855 | @@ -17,7 +17,7 @@ | |||
856 | 17 | class CephBasicDeployment(OpenStackAmuletDeployment): | 17 | class CephBasicDeployment(OpenStackAmuletDeployment): |
857 | 18 | """Amulet tests on a basic ceph deployment.""" | 18 | """Amulet tests on a basic ceph deployment.""" |
858 | 19 | 19 | ||
860 | 20 | def __init__(self, series=None, openstack=None, source=None, stable=False): | 20 | def __init__(self, series=None, openstack=None, source=None, stable=True): |
861 | 21 | """Deploy the entire test environment.""" | 21 | """Deploy the entire test environment.""" |
862 | 22 | super(CephBasicDeployment, self).__init__(series, openstack, source, | 22 | super(CephBasicDeployment, self).__init__(series, openstack, source, |
863 | 23 | stable) | 23 | stable) |
864 | 24 | 24 | ||
865 | === modified file 'tests/charmhelpers/contrib/amulet/utils.py' | |||
866 | --- tests/charmhelpers/contrib/amulet/utils.py 2015-01-26 09:46:20 +0000 | |||
867 | +++ tests/charmhelpers/contrib/amulet/utils.py 2015-02-24 06:27:04 +0000 | |||
868 | @@ -169,8 +169,13 @@ | |||
869 | 169 | cmd = 'pgrep -o -f {}'.format(service) | 169 | cmd = 'pgrep -o -f {}'.format(service) |
870 | 170 | else: | 170 | else: |
871 | 171 | cmd = 'pgrep -o {}'.format(service) | 171 | cmd = 'pgrep -o {}'.format(service) |
874 | 172 | proc_dir = '/proc/{}'.format(sentry_unit.run(cmd)[0].strip()) | 172 | cmd = cmd + ' | grep -v pgrep || exit 0' |
875 | 173 | return self._get_dir_mtime(sentry_unit, proc_dir) | 173 | cmd_out = sentry_unit.run(cmd) |
876 | 174 | self.log.debug('CMDout: ' + str(cmd_out)) | ||
877 | 175 | if cmd_out[0]: | ||
878 | 176 | self.log.debug('Pid for %s %s' % (service, str(cmd_out[0]))) | ||
879 | 177 | proc_dir = '/proc/{}'.format(cmd_out[0].strip()) | ||
880 | 178 | return self._get_dir_mtime(sentry_unit, proc_dir) | ||
881 | 174 | 179 | ||
882 | 175 | def service_restarted(self, sentry_unit, service, filename, | 180 | def service_restarted(self, sentry_unit, service, filename, |
883 | 176 | pgrep_full=False, sleep_time=20): | 181 | pgrep_full=False, sleep_time=20): |
884 | @@ -187,6 +192,121 @@ | |||
885 | 187 | else: | 192 | else: |
886 | 188 | return False | 193 | return False |
887 | 189 | 194 | ||
888 | 195 | def service_restarted_since(self, sentry_unit, mtime, service, | ||
889 | 196 | pgrep_full=False, sleep_time=20, | ||
890 | 197 | retry_count=2): | ||
891 | 198 | """Check if service was been started after a given time. | ||
892 | 199 | |||
893 | 200 | Args: | ||
894 | 201 | sentry_unit (sentry): The sentry unit to check for the service on | ||
895 | 202 | mtime (float): The epoch time to check against | ||
896 | 203 | service (string): service name to look for in process table | ||
897 | 204 | pgrep_full (boolean): Use full command line search mode with pgrep | ||
898 | 205 | sleep_time (int): Seconds to sleep before looking for process | ||
899 | 206 | retry_count (int): If service is not found, how many times to retry | ||
900 | 207 | |||
901 | 208 | Returns: | ||
902 | 209 | bool: True if service found and its start time it newer than mtime, | ||
903 | 210 | False if service is older than mtime or if service was | ||
904 | 211 | not found. | ||
905 | 212 | """ | ||
906 | 213 | self.log.debug('Checking %s restarted since %s' % (service, mtime)) | ||
907 | 214 | time.sleep(sleep_time) | ||
908 | 215 | proc_start_time = self._get_proc_start_time(sentry_unit, service, | ||
909 | 216 | pgrep_full) | ||
910 | 217 | while retry_count > 0 and not proc_start_time: | ||
911 | 218 | self.log.debug('No pid file found for service %s, will retry %i ' | ||
912 | 219 | 'more times' % (service, retry_count)) | ||
913 | 220 | time.sleep(30) | ||
914 | 221 | proc_start_time = self._get_proc_start_time(sentry_unit, service, | ||
915 | 222 | pgrep_full) | ||
916 | 223 | retry_count = retry_count - 1 | ||
917 | 224 | |||
918 | 225 | if not proc_start_time: | ||
919 | 226 | self.log.warn('No proc start time found, assuming service did ' | ||
920 | 227 | 'not start') | ||
921 | 228 | return False | ||
922 | 229 | if proc_start_time >= mtime: | ||
923 | 230 | self.log.debug('proc start time is newer than provided mtime' | ||
924 | 231 | '(%s >= %s)' % (proc_start_time, mtime)) | ||
925 | 232 | return True | ||
926 | 233 | else: | ||
927 | 234 | self.log.warn('proc start time (%s) is older than provided mtime ' | ||
928 | 235 | '(%s), service did not restart' % (proc_start_time, | ||
929 | 236 | mtime)) | ||
930 | 237 | return False | ||
931 | 238 | |||
932 | 239 | def config_updated_since(self, sentry_unit, filename, mtime, | ||
933 | 240 | sleep_time=20): | ||
934 | 241 | """Check if file was modified after a given time. | ||
935 | 242 | |||
936 | 243 | Args: | ||
937 | 244 | sentry_unit (sentry): The sentry unit to check the file mtime on | ||
938 | 245 | filename (string): The file to check mtime of | ||
939 | 246 | mtime (float): The epoch time to check against | ||
940 | 247 | sleep_time (int): Seconds to sleep before looking for process | ||
941 | 248 | |||
942 | 249 | Returns: | ||
943 | 250 | bool: True if file was modified more recently than mtime, False if | ||
944 | 251 | file was modified before mtime, | ||
945 | 252 | """ | ||
946 | 253 | self.log.debug('Checking %s updated since %s' % (filename, mtime)) | ||
947 | 254 | time.sleep(sleep_time) | ||
948 | 255 | file_mtime = self._get_file_mtime(sentry_unit, filename) | ||
949 | 256 | if file_mtime >= mtime: | ||
950 | 257 | self.log.debug('File mtime is newer than provided mtime ' | ||
951 | 258 | '(%s >= %s)' % (file_mtime, mtime)) | ||
952 | 259 | return True | ||
953 | 260 | else: | ||
954 | 261 | self.log.warn('File mtime %s is older than provided mtime %s' | ||
955 | 262 | % (file_mtime, mtime)) | ||
956 | 263 | return False | ||
957 | 264 | |||
958 | 265 | def validate_service_config_changed(self, sentry_unit, mtime, service, | ||
959 | 266 | filename, pgrep_full=False, | ||
960 | 267 | sleep_time=20, retry_count=2): | ||
961 | 268 | """Check service and file were updated after mtime | ||
962 | 269 | |||
963 | 270 | Args: | ||
964 | 271 | sentry_unit (sentry): The sentry unit to check for the service on | ||
965 | 272 | mtime (float): The epoch time to check against | ||
966 | 273 | service (string): service name to look for in process table | ||
967 | 274 | filename (string): The file to check mtime of | ||
968 | 275 | pgrep_full (boolean): Use full command line search mode with pgrep | ||
969 | 276 | sleep_time (int): Seconds to sleep before looking for process | ||
970 | 277 | retry_count (int): If service is not found, how many times to retry | ||
971 | 278 | |||
972 | 279 | Typical Usage: | ||
973 | 280 | u = OpenStackAmuletUtils(ERROR) | ||
974 | 281 | ... | ||
975 | 282 | mtime = u.get_sentry_time(self.cinder_sentry) | ||
976 | 283 | self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'}) | ||
977 | 284 | if not u.validate_service_config_changed(self.cinder_sentry, | ||
978 | 285 | mtime, | ||
979 | 286 | 'cinder-api', | ||
980 | 287 | '/etc/cinder/cinder.conf') | ||
981 | 288 | amulet.raise_status(amulet.FAIL, msg='update failed') | ||
982 | 289 | Returns: | ||
983 | 290 | bool: True if both service and file where updated/restarted after | ||
984 | 291 | mtime, False if service is older than mtime or if service was | ||
985 | 292 | not found or if filename was modified before mtime. | ||
986 | 293 | """ | ||
987 | 294 | self.log.debug('Checking %s restarted since %s' % (service, mtime)) | ||
988 | 295 | time.sleep(sleep_time) | ||
989 | 296 | service_restart = self.service_restarted_since(sentry_unit, mtime, | ||
990 | 297 | service, | ||
991 | 298 | pgrep_full=pgrep_full, | ||
992 | 299 | sleep_time=0, | ||
993 | 300 | retry_count=retry_count) | ||
994 | 301 | config_update = self.config_updated_since(sentry_unit, filename, mtime, | ||
995 | 302 | sleep_time=0) | ||
996 | 303 | return service_restart and config_update | ||
997 | 304 | |||
998 | 305 | def get_sentry_time(self, sentry_unit): | ||
999 | 306 | """Return current epoch time on a sentry""" | ||
1000 | 307 | cmd = "date +'%s'" | ||
1001 | 308 | return float(sentry_unit.run(cmd)[0]) | ||
1002 | 309 | |||
1003 | 190 | def relation_error(self, name, data): | 310 | def relation_error(self, name, data): |
1004 | 191 | return 'unexpected relation data in {} - {}'.format(name, data) | 311 | return 'unexpected relation data in {} - {}'.format(name, data) |
1005 | 192 | 312 | ||
1006 | 193 | 313 | ||
1007 | === modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py' | |||
1008 | --- tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-01-26 09:46:20 +0000 | |||
1009 | +++ tests/charmhelpers/contrib/openstack/amulet/deployment.py 2015-02-24 06:27:04 +0000 | |||
1010 | @@ -71,16 +71,19 @@ | |||
1011 | 71 | services.append(this_service) | 71 | services.append(this_service) |
1012 | 72 | use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', | 72 | use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', |
1013 | 73 | 'ceph-osd', 'ceph-radosgw'] | 73 | 'ceph-osd', 'ceph-radosgw'] |
1014 | 74 | # Openstack subordinate charms do not expose an origin option as that | ||
1015 | 75 | # is controlled by the principle | ||
1016 | 76 | ignore = ['neutron-openvswitch'] | ||
1017 | 74 | 77 | ||
1018 | 75 | if self.openstack: | 78 | if self.openstack: |
1019 | 76 | for svc in services: | 79 | for svc in services: |
1021 | 77 | if svc['name'] not in use_source: | 80 | if svc['name'] not in use_source + ignore: |
1022 | 78 | config = {'openstack-origin': self.openstack} | 81 | config = {'openstack-origin': self.openstack} |
1023 | 79 | self.d.configure(svc['name'], config) | 82 | self.d.configure(svc['name'], config) |
1024 | 80 | 83 | ||
1025 | 81 | if self.source: | 84 | if self.source: |
1026 | 82 | for svc in services: | 85 | for svc in services: |
1028 | 83 | if svc['name'] in use_source: | 86 | if svc['name'] in use_source and svc['name'] not in ignore: |
1029 | 84 | config = {'source': self.source} | 87 | config = {'source': self.source} |
1030 | 85 | self.d.configure(svc['name'], config) | 88 | self.d.configure(svc['name'], config) |
1031 | 86 | 89 |
charm_unit_test #2015 ceph-next for brad-marshall mp250711
UNIT OK: passed
Build: http:// 10.245. 162.77: 8080/job/ charm_unit_ test/2015/