Merge lp:~fginther/landscape-charm/remove-collect-logs into lp:~landscape/landscape-charm/tools
- remove-collect-logs
- Merge into tools
Proposed by
Francis Ginther
Status: | Merged |
---|---|
Approved by: | Francis Ginther |
Approved revision: | 39 |
Merged at revision: | 39 |
Proposed branch: | lp:~fginther/landscape-charm/remove-collect-logs |
Merge into: | lp:~landscape/landscape-charm/tools |
Diff against target: |
1697 lines (+0/-1683) 3 files modified
Makefile (+0/-7) collect-logs (+0/-613) test_collect-logs.py (+0/-1063) |
To merge this branch: | bzr merge lp:~fginther/landscape-charm/remove-collect-logs |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Alberto Donato (community) | Approve | ||
🤖 Landscape Builder | test results | Needs Fixing | |
Review via email: mp+315731@code.launchpad.net |
Commit message
Remove collect-logs, it's now at https:/
Description of the change
Remove collect-logs, it's now at https:/
To post a comment you must log in.
Revision history for this message
🤖 Landscape Builder (landscape-builder) : | # |
review:
Abstain
(executing tests)
Revision history for this message
🤖 Landscape Builder (landscape-builder) wrote : | # |
review:
Needs Fixing
(test results)
Revision history for this message
Alberto Donato (ack) wrote : | # |
+1
you'll need to update the latch config not to vote on this anymore, since there's no makefile anymore
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === removed file 'Makefile' | |||
2 | --- Makefile 2016-09-16 15:31:13 +0000 | |||
3 | +++ Makefile 1970-01-01 00:00:00 +0000 | |||
4 | @@ -1,7 +0,0 @@ | |||
5 | 1 | .PHONY: test | ||
6 | 2 | test: | ||
7 | 3 | python -m unittest test_collect-logs | ||
8 | 4 | |||
9 | 5 | |||
10 | 6 | .PHONY: ci-test | ||
11 | 7 | ci-test: test | ||
12 | 8 | 0 | ||
13 | === removed file 'collect-logs' | |||
14 | --- collect-logs 2017-01-05 17:33:16 +0000 | |||
15 | +++ collect-logs 1970-01-01 00:00:00 +0000 | |||
16 | @@ -1,613 +0,0 @@ | |||
17 | 1 | #!/usr/bin/python | ||
18 | 2 | |||
19 | 3 | from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter | ||
20 | 4 | import errno | ||
21 | 5 | from functools import partial | ||
22 | 6 | import logging | ||
23 | 7 | import multiprocessing | ||
24 | 8 | import os | ||
25 | 9 | import shutil | ||
26 | 10 | from subprocess import ( | ||
27 | 11 | CalledProcessError, check_call, check_output, call, STDOUT) | ||
28 | 12 | import sys | ||
29 | 13 | from tempfile import mkdtemp | ||
30 | 14 | |||
31 | 15 | import yaml | ||
32 | 16 | |||
33 | 17 | |||
34 | 18 | log = logging.getLogger("collect-logs") | ||
35 | 19 | PRG = os.path.abspath(__file__) | ||
36 | 20 | LOGS = [ | ||
37 | 21 | # Basics | ||
38 | 22 | "/var/log", | ||
39 | 23 | "/etc/hosts", | ||
40 | 24 | "/etc/network", | ||
41 | 25 | # for the landscape client | ||
42 | 26 | "/var/lib/landscape/client", | ||
43 | 27 | # for the landscape server charm | ||
44 | 28 | "/etc/apache2", | ||
45 | 29 | "/etc/haproxy", | ||
46 | 30 | # XXX This should be handled by the per-unit logs. | ||
47 | 31 | "/var/lib/lxc/*/rootfs/var/log", | ||
48 | 32 | # logs and configs for juju 1 using LXC (not LXD) | ||
49 | 33 | "/var/lib/juju/containers", | ||
50 | 34 | # for openstack | ||
51 | 35 | "/etc/nova", | ||
52 | 36 | "/etc/swift", | ||
53 | 37 | "/etc/neutron", | ||
54 | 38 | "/etc/ceph", | ||
55 | 39 | "/etc/glance", | ||
56 | 40 | ] | ||
57 | 41 | EXCLUDED = ["/var/lib/landscape/client/package/hash-id", | ||
58 | 42 | "/var/lib/juju/containers/juju-*-lxc-template"] | ||
59 | 43 | LANDSCAPE_JUJU_HOME = "/var/lib/landscape/juju-homes" | ||
60 | 44 | |||
61 | 45 | JUJU1 = "juju" | ||
62 | 46 | # XXX This is going to break once juju-2.1 happens. | ||
63 | 47 | # See https://bugs.launchpad.net/juju-core/+bug/1613864. | ||
64 | 48 | JUJU2 = "juju-2.1" | ||
65 | 49 | JUJU = JUJU1 | ||
66 | 50 | |||
67 | 51 | DEFAULT_MODEL = object() | ||
68 | 52 | |||
69 | 53 | VERBOSE = False | ||
70 | 54 | |||
71 | 55 | |||
72 | 56 | if VERBOSE: | ||
73 | 57 | def call(args, env=None, _call=call): | ||
74 | 58 | print(" running {!r}".format(" ".join(args))) | ||
75 | 59 | return _call(args, env=env) | ||
76 | 60 | |||
77 | 61 | def check_call(args, env=None, _check_call=check_call): | ||
78 | 62 | print(" running {!r}".format(" ".join(args))) | ||
79 | 63 | return _check_call(args, env=env) | ||
80 | 64 | |||
81 | 65 | def check_output(args, stderr=None, env=None, _check_output=check_output): | ||
82 | 66 | print(" running {!r}".format(" ".join(args))) | ||
83 | 67 | return _check_output(args, stderr=stderr, env=env) | ||
84 | 68 | |||
85 | 69 | |||
86 | 70 | class Juju(object): | ||
87 | 71 | """A wrapper around a juju binary.""" | ||
88 | 72 | |||
89 | 73 | def __init__(self, binary_path=None, model=None, cfgdir=None, sudo=None): | ||
90 | 74 | if binary_path is None: | ||
91 | 75 | binary_path = JUJU | ||
92 | 76 | if model is DEFAULT_MODEL: | ||
93 | 77 | model = None | ||
94 | 78 | |||
95 | 79 | self.binary_path = binary_path | ||
96 | 80 | self.model = model | ||
97 | 81 | self.cfgdir = cfgdir | ||
98 | 82 | self.sudo = sudo | ||
99 | 83 | |||
100 | 84 | if binary_path == JUJU1: | ||
101 | 85 | self.envvar = "JUJU_HOME" | ||
102 | 86 | else: | ||
103 | 87 | self.envvar = "JUJU_DATA" | ||
104 | 88 | |||
105 | 89 | self.env = None | ||
106 | 90 | if cfgdir is not None: | ||
107 | 91 | self.env = dict(os.environ, **{self.envvar: cfgdir}) | ||
108 | 92 | |||
109 | 93 | def __repr__(self): | ||
110 | 94 | args = ", ".join("{}={!r}".format(name, getattr(self, name)) | ||
111 | 95 | for name in ("binary_path", "model", "cfgdir")) | ||
112 | 96 | return "{}({})".format(self.__class__.__name__, args) | ||
113 | 97 | |||
114 | 98 | def __eq__(self, other): | ||
115 | 99 | if self.binary_path != other.binary_path: | ||
116 | 100 | return False | ||
117 | 101 | if self.model != other.model: | ||
118 | 102 | return False | ||
119 | 103 | if self.cfgdir != other.cfgdir: | ||
120 | 104 | return False | ||
121 | 105 | if self.sudo != other.sudo: | ||
122 | 106 | return False | ||
123 | 107 | return True | ||
124 | 108 | |||
125 | 109 | def __ne__(self, other): | ||
126 | 110 | return not(self == other) | ||
127 | 111 | |||
128 | 112 | @property | ||
129 | 113 | def envstr(self): | ||
130 | 114 | if not self.cfgdir: | ||
131 | 115 | return "" | ||
132 | 116 | else: | ||
133 | 117 | return "{}={}".format(self.envvar, self.cfgdir) | ||
134 | 118 | |||
135 | 119 | def status_args(self): | ||
136 | 120 | """Return the subprocess.* args for a status command.""" | ||
137 | 121 | args = self._resolve("status", "--format=yaml") | ||
138 | 122 | return args | ||
139 | 123 | |||
140 | 124 | def format_status(self): | ||
141 | 125 | """Return the formatted juju status command.""" | ||
142 | 126 | args = self.status_args() | ||
143 | 127 | return self._format(args) | ||
144 | 128 | |||
145 | 129 | def set_model_config_args(self, key, value): | ||
146 | 130 | item = "{}={}".format(key, value) | ||
147 | 131 | if self.binary_path == JUJU1: | ||
148 | 132 | args = self._resolve("set-env", item) | ||
149 | 133 | else: | ||
150 | 134 | args = self._resolve("model-config", item) | ||
151 | 135 | return args | ||
152 | 136 | |||
153 | 137 | def format_set_model_config(self, key, value): | ||
154 | 138 | """Return the formatted model config command.""" | ||
155 | 139 | args = self.set_model_config_args(key, value) | ||
156 | 140 | return self._format(args) | ||
157 | 141 | |||
158 | 142 | def ssh_args(self, unit, cmd): | ||
159 | 143 | """Return the subprocess.* args for an SSH command.""" | ||
160 | 144 | return self._resolve("ssh", unit, cmd) | ||
161 | 145 | |||
162 | 146 | def pull_args(self, unit, source, target="."): | ||
163 | 147 | """Return the subprocess.* args for an SCP command.""" | ||
164 | 148 | source = "{}:{}".format(unit, source) | ||
165 | 149 | return self._resolve("scp", source, target) | ||
166 | 150 | |||
167 | 151 | def push_args(self, unit, source, target): | ||
168 | 152 | """Return the subprocess.* args for an SCP command.""" | ||
169 | 153 | target = "{}:{}".format(unit, target) | ||
170 | 154 | return self._resolve("scp", source, target) | ||
171 | 155 | |||
172 | 156 | def format(self, cmd, *subargs): | ||
173 | 157 | """Return the formatted command. | ||
174 | 158 | |||
175 | 159 | "sudo" and the juju config dir env var are set if appropriate. | ||
176 | 160 | """ | ||
177 | 161 | args = [cmd] | ||
178 | 162 | args.extend(subargs) | ||
179 | 163 | return self._format(args) | ||
180 | 164 | |||
181 | 165 | def _format(self, args): | ||
182 | 166 | """Return the formatted args. | ||
183 | 167 | |||
184 | 168 | "sudo" and the juju config dir env var are set if appropriate. | ||
185 | 169 | """ | ||
186 | 170 | if self.cfgdir: | ||
187 | 171 | args.insert(0, self.envstr) | ||
188 | 172 | if self.sudo == "" or self.sudo is True: | ||
189 | 173 | args.insert(0, "sudo") | ||
190 | 174 | elif self.sudo: | ||
191 | 175 | args.insert(0, "sudo -u {}".format(self.sudo)) | ||
192 | 176 | return " ".join(args) | ||
193 | 177 | |||
194 | 178 | def _resolve(self, sub, *subargs): | ||
195 | 179 | """Return the subprocess.* args for the juju subcommand.""" | ||
196 | 180 | args = [self.binary_path, sub] | ||
197 | 181 | if self.model: | ||
198 | 182 | if self.binary_path == JUJU1: | ||
199 | 183 | args.append("-e") | ||
200 | 184 | else: | ||
201 | 185 | args.append("-m") | ||
202 | 186 | args.append(self.model) | ||
203 | 187 | args.extend(subargs) | ||
204 | 188 | return args | ||
205 | 189 | |||
206 | 190 | |||
207 | 191 | def format_collect_logs(juju, script, target, inner=True): | ||
208 | 192 | """Return the formatted command for the collect_logs script.""" | ||
209 | 193 | if inner: | ||
210 | 194 | args = [script, "--inner"] | ||
211 | 195 | else: | ||
212 | 196 | args = [script] | ||
213 | 197 | args.extend(["--juju", juju.binary_path]) | ||
214 | 198 | if juju.model and juju.model is not DEFAULT_MODEL: | ||
215 | 199 | args.extend(["--model", juju.model]) | ||
216 | 200 | if juju.cfgdir: | ||
217 | 201 | args.extend(["--cfgdir", juju.cfgdir]) | ||
218 | 202 | args.append(target) | ||
219 | 203 | return juju._format(args) | ||
220 | 204 | |||
221 | 205 | |||
222 | 206 | def juju_status(juju): | ||
223 | 207 | """Return a juju status structure.""" | ||
224 | 208 | output = check_output(juju.status_args(), env=juju.env) | ||
225 | 209 | output = output.decode("utf-8").strip() | ||
226 | 210 | return yaml.load(output) | ||
227 | 211 | |||
228 | 212 | |||
229 | 213 | def get_units(juju, status=None): | ||
230 | 214 | """Return a list with all units.""" | ||
231 | 215 | if status is None: | ||
232 | 216 | status = juju_status(juju) | ||
233 | 217 | units = [] | ||
234 | 218 | if "services" in status: | ||
235 | 219 | applications = status["services"] | ||
236 | 220 | else: | ||
237 | 221 | applications = status["applications"] | ||
238 | 222 | for application in applications: | ||
239 | 223 | # skip subordinate charms | ||
240 | 224 | if "subordinate-to" in applications[application].keys(): | ||
241 | 225 | continue | ||
242 | 226 | if "units" in applications[application]: | ||
243 | 227 | units.extend(applications[application]["units"].keys()) | ||
244 | 228 | if len(units) == 0: | ||
245 | 229 | sys.exit("ERROR, no units found. Make sure the right juju environment" | ||
246 | 230 | "is set.") | ||
247 | 231 | return units | ||
248 | 232 | |||
249 | 233 | |||
250 | 234 | def _create_ps_output_file(juju, unit): | ||
251 | 235 | """List running processes and redirect them to a file.""" | ||
252 | 236 | log.info("Collecting ps output on unit {}".format(unit)) | ||
253 | 237 | ps_cmd = "ps fauxww | sudo tee /var/log/ps-fauxww.txt" | ||
254 | 238 | args = juju.ssh_args(unit, ps_cmd) | ||
255 | 239 | try: | ||
256 | 240 | check_output(args, stderr=STDOUT, env=juju.env) | ||
257 | 241 | except CalledProcessError as e: | ||
258 | 242 | log.warning( | ||
259 | 243 | "Failed to collect running processes on unit {}".format(unit)) | ||
260 | 244 | log.warning(e.output) | ||
261 | 245 | log.warning(e.returncode) | ||
262 | 246 | |||
263 | 247 | |||
264 | 248 | def _create_log_tarball(juju, unit): | ||
265 | 249 | log.info("Creating tarball on unit {}".format(unit)) | ||
266 | 250 | exclude = " ".join(["--exclude=%s" % x for x in EXCLUDED]) | ||
267 | 251 | logs = "$(sudo sh -c \"ls -1d %s 2>/dev/null\")" % " ".join(LOGS) | ||
268 | 252 | # --ignore-failed-read avoids failure for unreadable files (not for files | ||
269 | 253 | # being written) | ||
270 | 254 | tar_cmd = "sudo tar --ignore-failed-read" | ||
271 | 255 | logsuffix = unit.replace("/", "-") | ||
272 | 256 | if unit == "0": | ||
273 | 257 | logsuffix = "bootstrap" | ||
274 | 258 | cmd = "{} {} -cf /tmp/logs_{}.tar {}".format( | ||
275 | 259 | tar_cmd, exclude, logsuffix, logs) | ||
276 | 260 | args = juju.ssh_args(unit, cmd) | ||
277 | 261 | ATTEMPTS = 5 | ||
278 | 262 | for i in range(ATTEMPTS): | ||
279 | 263 | log.info("...attempt {} of {}".format(i+1, ATTEMPTS)) | ||
280 | 264 | try: | ||
281 | 265 | check_output(args, stderr=STDOUT, env=juju.env) | ||
282 | 266 | except CalledProcessError as e: | ||
283 | 267 | # Note: tar command returns 1 for everything it considers a | ||
284 | 268 | # warning, 2 for fatal errors. Since we are backing up | ||
285 | 269 | # log files that are actively being written, or part of a live | ||
286 | 270 | # system, logging and ignoring such warnings (return code 1) is | ||
287 | 271 | # what we can do now. | ||
288 | 272 | # Everything else we might retry as usual. | ||
289 | 273 | if e.returncode == 1: | ||
290 | 274 | log.warning( | ||
291 | 275 | "tar returned 1, proceeding anyway: {}".format(e.output)) | ||
292 | 276 | break | ||
293 | 277 | log.warning( | ||
294 | 278 | "Failed to archive log files on unit {}".format(unit)) | ||
295 | 279 | log.warning(e.output) | ||
296 | 280 | log.warning(e.returncode) | ||
297 | 281 | if i < 4: | ||
298 | 282 | log.warning("...retrying...") | ||
299 | 283 | cmd = "{} {} --update -f /tmp/logs_{}.tar {}".format( | ||
300 | 284 | tar_cmd, exclude, logsuffix, logs) | ||
301 | 285 | args = juju.ssh_args(unit, cmd) | ||
302 | 286 | else: | ||
303 | 287 | # The command succeeded so we stop the retry loop. | ||
304 | 288 | break | ||
305 | 289 | else: | ||
306 | 290 | # Don't bother compressing. | ||
307 | 291 | log.warning("...{} attempts failed; giving up".format(ATTEMPTS)) | ||
308 | 292 | return | ||
309 | 293 | cmd = "sudo gzip -f /tmp/logs_{}.tar".format(logsuffix) | ||
310 | 294 | args = juju.ssh_args(unit, cmd) | ||
311 | 295 | try: | ||
312 | 296 | check_output(args, stderr=STDOUT, env=juju.env) | ||
313 | 297 | except CalledProcessError as e: | ||
314 | 298 | log.warning( | ||
315 | 299 | "Failed to create remote log tarball on unit {}".format(unit)) | ||
316 | 300 | log.warning(e.output) | ||
317 | 301 | log.warning(e.returncode) | ||
318 | 302 | |||
319 | 303 | |||
320 | 304 | def download_log_from_unit(juju, unit): | ||
321 | 305 | log.info("Downloading tarball from unit %s" % unit) | ||
322 | 306 | unit_filename = unit.replace("/", "-") | ||
323 | 307 | if unit == "0": | ||
324 | 308 | unit_filename = "bootstrap" | ||
325 | 309 | remote_filename = "logs_%s.tar.gz" % unit_filename | ||
326 | 310 | try: | ||
327 | 311 | args = juju.pull_args(unit, "/tmp/" + remote_filename) | ||
328 | 312 | call(args, env=juju.env) | ||
329 | 313 | os.mkdir(unit_filename) | ||
330 | 314 | args = ["tar", "-C", unit_filename, "-xzf", remote_filename] | ||
331 | 315 | call(args) | ||
332 | 316 | os.unlink(remote_filename) | ||
333 | 317 | except: | ||
334 | 318 | log.warning("error collecting logs from %s, skipping" % unit) | ||
335 | 319 | finally: | ||
336 | 320 | if os.path.exists(remote_filename): | ||
337 | 321 | os.unlink(remote_filename) | ||
338 | 322 | |||
339 | 323 | |||
340 | 324 | def collect_logs(juju): | ||
341 | 325 | """ | ||
342 | 326 | Remotely, on each unit, create a tarball with the requested log files | ||
343 | 327 | or directories, if they exist. If a requested log does not exist on a | ||
344 | 328 | particular unit, it's ignored. | ||
345 | 329 | After each tarball is created, it's downloaded to the current directory | ||
346 | 330 | and expanded, and the tarball is then deleted. | ||
347 | 331 | """ | ||
348 | 332 | units = get_units(juju) | ||
349 | 333 | # include bootstrap | ||
350 | 334 | units.append("0") | ||
351 | 335 | |||
352 | 336 | log.info("Collecting running processes for all units including bootstrap") | ||
353 | 337 | map(partial(_create_ps_output_file, juju), units) | ||
354 | 338 | |||
355 | 339 | log.info("Creating remote tarball in parallel for units %s" % ( | ||
356 | 340 | ",".join(units))) | ||
357 | 341 | map(partial(_create_log_tarball, juju), units) | ||
358 | 342 | log.info("Downloading logs from units") | ||
359 | 343 | |||
360 | 344 | _mp_map(partial(download_log_from_unit, juju), units) | ||
361 | 345 | |||
362 | 346 | |||
363 | 347 | def _mp_map(func, args): | ||
364 | 348 | pool = multiprocessing.Pool(processes=4) | ||
365 | 349 | pool.map(func, args) | ||
366 | 350 | |||
367 | 351 | |||
368 | 352 | def get_landscape_unit(units): | ||
369 | 353 | """Return the landscape unit among the units list.""" | ||
370 | 354 | units = [ | ||
371 | 355 | unit for unit in units if unit.startswith("landscape-server/") or | ||
372 | 356 | unit.startswith("landscape/")] | ||
373 | 357 | if len(units) == 0: | ||
374 | 358 | return None | ||
375 | 359 | else: | ||
376 | 360 | # XXX we don't yet support multiple landscape units. We would have to | ||
377 | 361 | # find out which one has the juju home | ||
378 | 362 | return units[0] | ||
379 | 363 | |||
380 | 364 | |||
381 | 365 | def get_inner_model(version, inner_model=DEFAULT_MODEL): | ||
382 | 366 | """Return a best-effort guess at the inner model name.""" | ||
383 | 367 | if inner_model is not DEFAULT_MODEL: | ||
384 | 368 | return inner_model | ||
385 | 369 | if version == JUJU1: | ||
386 | 370 | return None | ||
387 | 371 | # We assume that this is a Landscape-bootstrapped controller. | ||
388 | 372 | return "controller" | ||
389 | 373 | |||
390 | 374 | |||
391 | 375 | def disable_inner_ssh_proxy(juju, landscape_unit, inner_model=DEFAULT_MODEL): | ||
392 | 376 | """ | ||
393 | 377 | Workaround for #1607076: disable the proxy-ssh juju environment setting | ||
394 | 378 | for the inner cloud so we can juju ssh into it. | ||
395 | 379 | """ | ||
396 | 380 | log.info("Disabling proxy-ssh in the juju environment on " | ||
397 | 381 | "{}".format(landscape_unit)) | ||
398 | 382 | |||
399 | 383 | cfgdir = "{0}/`sudo ls -rt {0}/ | tail -1`".format(LANDSCAPE_JUJU_HOME) | ||
400 | 384 | key = "proxy-ssh" | ||
401 | 385 | value = "false" | ||
402 | 386 | |||
403 | 387 | # Try Juju 2. | ||
404 | 388 | model2 = get_inner_model(JUJU2, inner_model) | ||
405 | 389 | inner2 = Juju(JUJU2, model=model2, cfgdir=cfgdir, sudo=True) | ||
406 | 390 | cmd2 = inner2.format_set_model_config(key, value) | ||
407 | 391 | args2 = juju.ssh_args(landscape_unit, cmd2) | ||
408 | 392 | try: | ||
409 | 393 | check_output(args2, stderr=STDOUT, env=juju.env) | ||
410 | 394 | except CalledProcessError as e: | ||
411 | 395 | log.warning("Couldn't disable proxy-ssh in the inner environment " | ||
412 | 396 | "using Juju 2, attempting Juju 1.") | ||
413 | 397 | log.warning("Error was:\n{}".format(e.output)) | ||
414 | 398 | else: | ||
415 | 399 | return | ||
416 | 400 | |||
417 | 401 | # Try Juju 1. | ||
418 | 402 | model1 = get_inner_model(JUJU1, inner_model) | ||
419 | 403 | inner1 = Juju(JUJU1, model=model1, cfgdir=cfgdir, sudo=True) | ||
420 | 404 | cmd1 = inner1.format_set_model_config(key, value) | ||
421 | 405 | args1 = juju.ssh_args(landscape_unit, cmd1) | ||
422 | 406 | try: | ||
423 | 407 | check_output(args1, stderr=STDOUT, env=juju.env) | ||
424 | 408 | except CalledProcessError as e: | ||
425 | 409 | log.warning("Couldn't disable proxy-ssh in the inner environment " | ||
426 | 410 | "using Juju 1, collecting inner logs might fail.") | ||
427 | 411 | log.warning("Error was:\n{}".format(e.output)) | ||
428 | 412 | |||
429 | 413 | |||
430 | 414 | |||
431 | 415 | def find_inner_juju(juju, landscape_unit, inner_model=DEFAULT_MODEL): | ||
432 | 416 | """Return the juju dir and binary path, if any.""" | ||
433 | 417 | # Identify the most recent juju "home" that landscape is using. | ||
434 | 418 | cmd = "sudo ls -rt {}/".format(LANDSCAPE_JUJU_HOME) | ||
435 | 419 | args = juju.ssh_args(landscape_unit, cmd) | ||
436 | 420 | try: | ||
437 | 421 | output = check_output(args, env=juju.env).strip() | ||
438 | 422 | except CalledProcessError: | ||
439 | 423 | return None | ||
440 | 424 | if output.startswith("sudo: "): | ||
441 | 425 | _, _, output = output.partition("\r\n") | ||
442 | 426 | indices = [m for m in output.split() if m and m.isdigit()] | ||
443 | 427 | if not indices: | ||
444 | 428 | return None | ||
445 | 429 | index = indices[-1] | ||
446 | 430 | juju_dir = os.path.join(LANDSCAPE_JUJU_HOME, index) | ||
447 | 431 | |||
448 | 432 | # Try Juju 2. | ||
449 | 433 | model2 = get_inner_model(JUJU2, inner_model) | ||
450 | 434 | inner2 = Juju(JUJU2, model=model2, cfgdir=juju_dir, sudo="") | ||
451 | 435 | cmd2 = inner2.format_status() | ||
452 | 436 | args2 = juju.ssh_args(landscape_unit, cmd2) | ||
453 | 437 | if call(args2, env=juju.env) == 0: | ||
454 | 438 | log.info("using Juju 2 for inner model") | ||
455 | 439 | return inner2 | ||
456 | 440 | |||
457 | 441 | # Try Juju 1. | ||
458 | 442 | model1 = get_inner_model(JUJU1, inner_model) | ||
459 | 443 | inner1 = Juju(JUJU1, model=model1, cfgdir=juju_dir, sudo="landscape") | ||
460 | 444 | cmd1 = inner1.format_status() | ||
461 | 445 | args1 = juju.ssh_args(landscape_unit, cmd1) | ||
462 | 446 | if call(args1, env=juju.env) == 0: | ||
463 | 447 | log.info("using Juju 1 for inner model") | ||
464 | 448 | return inner1 | ||
465 | 449 | |||
466 | 450 | # We didn't find an inner model. | ||
467 | 451 | return None | ||
468 | 452 | |||
469 | 453 | |||
470 | 454 | def collect_inner_logs(juju, inner_model=DEFAULT_MODEL): | ||
471 | 455 | """Collect logs from an inner landscape[-server]/0 unit.""" | ||
472 | 456 | log.info("Collecting logs on inner environment") | ||
473 | 457 | units = get_units(juju) | ||
474 | 458 | landscape_unit = get_landscape_unit(units) | ||
475 | 459 | if not landscape_unit: | ||
476 | 460 | log.info("No landscape[-server]/N found, skipping") | ||
477 | 461 | return | ||
478 | 462 | log.info("Found landscape unit {}".format(landscape_unit)) | ||
479 | 463 | |||
480 | 464 | disable_inner_ssh_proxy(juju, landscape_unit, inner_model) | ||
481 | 465 | |||
482 | 466 | # Look up the inner model. | ||
483 | 467 | inner_juju = find_inner_juju(juju, landscape_unit, inner_model) | ||
484 | 468 | if inner_juju is None: | ||
485 | 469 | log.info(("No active inner environment found on {}, skipping" | ||
486 | 470 | ).format(landscape_unit)) | ||
487 | 471 | return | ||
488 | 472 | |||
489 | 473 | # Prepare to get the logs from the inner model. | ||
490 | 474 | collect_logs = "/tmp/collect-logs" | ||
491 | 475 | args = juju.push_args(landscape_unit, PRG, collect_logs) | ||
492 | 476 | call(args, env=juju.env) | ||
493 | 477 | filename = "inner-logs.tar.gz" | ||
494 | 478 | inner_filename = os.path.join("/tmp", filename) | ||
495 | 479 | args = juju.ssh_args(landscape_unit, "sudo rm -rf " + inner_filename) | ||
496 | 480 | call(args, env=juju.env) | ||
497 | 481 | |||
498 | 482 | # Collect the logs for the inner model. | ||
499 | 483 | cmd = format_collect_logs(inner_juju, collect_logs, inner_filename) | ||
500 | 484 | args = juju.ssh_args(landscape_unit, cmd) | ||
501 | 485 | check_call(args, env=juju.env) | ||
502 | 486 | |||
503 | 487 | # Copy the inner logs into a local directory. | ||
504 | 488 | log.info("Copying inner environment back") | ||
505 | 489 | cwd = os.getcwd() | ||
506 | 490 | target = os.path.join(cwd, filename) | ||
507 | 491 | args = juju.pull_args(landscape_unit, inner_filename, target) | ||
508 | 492 | check_call(args, env=juju.env) | ||
509 | 493 | try: | ||
510 | 494 | inner_dir = "landscape-0-inner-logs" | ||
511 | 495 | os.mkdir(inner_dir) | ||
512 | 496 | os.chdir(inner_dir) | ||
513 | 497 | try: | ||
514 | 498 | check_call(["tar", "-zxf", os.path.join(cwd, filename)]) | ||
515 | 499 | finally: | ||
516 | 500 | os.chdir(cwd) | ||
517 | 501 | finally: | ||
518 | 502 | try: | ||
519 | 503 | os.remove(target) | ||
520 | 504 | except OSError as e: | ||
521 | 505 | if e.errno != errno.ENOENT: | ||
522 | 506 | log.warning( | ||
523 | 507 | "failed to remove inner logs tarball: {}".format(e)) | ||
524 | 508 | |||
525 | 509 | |||
526 | 510 | def bundle_logs(tmpdir, tarfile, extrafiles=[]): | ||
527 | 511 | """ | ||
528 | 512 | Create a tarball with the directories under tmpdir and the | ||
529 | 513 | specified extra files. The tar command is executed outside | ||
530 | 514 | of tmpdir, so we need to a) use an absolute path that | ||
531 | 515 | includes tmpdir; and b) strip the tmpdir so it's not part of | ||
532 | 516 | the tarball. | ||
533 | 517 | We don't want absolute paths for extra files because otherwise | ||
534 | 518 | this path would be inside the tarball. | ||
535 | 519 | |||
536 | 520 | This allows you to have a tarball where this: | ||
537 | 521 | /tmp/tmpdir/foo | ||
538 | 522 | /tmp/tmpdir/bar | ||
539 | 523 | /home/ubuntu/data/log | ||
540 | 524 | |||
541 | 525 | Becomes this inside the tarball: | ||
542 | 526 | foo | ||
543 | 527 | bar | ||
544 | 528 | data/log | ||
545 | 529 | |||
546 | 530 | If collect-logs is run with CWD=/home/ubuntu and given data/log as | ||
547 | 531 | the extra file. | ||
548 | 532 | """ | ||
549 | 533 | args = ["tar", "czf", tarfile] | ||
550 | 534 | # get rid of the tmpdir prefix | ||
551 | 535 | args.extend(["--transform", "s,{}/,,".format(tmpdir[1:])]) | ||
552 | 536 | # need absolute paths since tmpdir isn't the cwd | ||
553 | 537 | args.extend(os.path.join(tmpdir, d) for d in sorted(os.listdir(tmpdir))) | ||
554 | 538 | if extrafiles: | ||
555 | 539 | args.extend(extrafiles) | ||
556 | 540 | call(args) | ||
557 | 541 | |||
558 | 542 | |||
559 | 543 | def get_juju(binary_path, model=DEFAULT_MODEL, cfgdir=None, inner=False): | ||
560 | 544 | """Return a Juju for the provided info.""" | ||
561 | 545 | if model is DEFAULT_MODEL and inner and binary_path != JUJU1: | ||
562 | 546 | # We assume that this is a Landscape-bootstrapped controller. | ||
563 | 547 | model = "controller" | ||
564 | 548 | return Juju(binary_path, model=model, cfgdir=cfgdir) | ||
565 | 549 | |||
566 | 550 | |||
567 | 551 | def get_option_parser(): | ||
568 | 552 | description = ("Collect logs from current juju environment and, if an " | ||
569 | 553 | "inner autopilot cloud is detected, include that.") | ||
570 | 554 | parser = ArgumentParser(description=description, | ||
571 | 555 | formatter_class=ArgumentDefaultsHelpFormatter) | ||
572 | 556 | parser.add_argument("--inner", action="store_true", default=False, | ||
573 | 557 | help="Collect logs for an inner model.") | ||
574 | 558 | parser.add_argument("--juju", default=JUJU2, | ||
575 | 559 | help="The Juju binary to use.") | ||
576 | 560 | parser.add_argument("--model", default=DEFAULT_MODEL, | ||
577 | 561 | help="The Juju model to use.") | ||
578 | 562 | parser.add_argument("--inner-model", default=DEFAULT_MODEL, | ||
579 | 563 | help="The Juju model to use for the inner juju.") | ||
580 | 564 | parser.add_argument("--cfgdir", | ||
581 | 565 | help="The Juju config dir to use.") | ||
582 | 566 | parser.add_argument("tarfile", help="Full path to tarfile to create.") | ||
583 | 567 | parser.add_argument("extrafiles", help="Optional full path to extra " | ||
584 | 568 | "logfiles to include, space separated", nargs="*") | ||
585 | 569 | return parser | ||
586 | 570 | |||
587 | 571 | |||
588 | 572 | def main(tarfile, extrafiles, juju=None, inner_model=DEFAULT_MODEL, | ||
589 | 573 | inner=False): | ||
590 | 574 | if juju is None: | ||
591 | 575 | juju = Juju() | ||
592 | 576 | |||
593 | 577 | # we need the absolute path because we will be changing | ||
594 | 578 | # the cwd | ||
595 | 579 | tmpdir = mkdtemp() | ||
596 | 580 | cwd = os.getcwd() | ||
597 | 581 | # logs are collected inside a temporary directory | ||
598 | 582 | os.chdir(tmpdir) | ||
599 | 583 | try: | ||
600 | 584 | collect_logs(juju) | ||
601 | 585 | if not inner: | ||
602 | 586 | try: | ||
603 | 587 | collect_inner_logs(juju, inner_model) | ||
604 | 588 | except: | ||
605 | 589 | log.warning("Collecting inner logs failed, continuing") | ||
606 | 590 | # we create the final tarball outside of tmpdir to we can | ||
607 | 591 | # add the extrafiles to the tarball root | ||
608 | 592 | os.chdir(cwd) | ||
609 | 593 | bundle_logs(tmpdir, tarfile, extrafiles) | ||
610 | 594 | log.info("created: %s" % tarfile) | ||
611 | 595 | finally: | ||
612 | 596 | call(["chmod", "-R", "u+w", tmpdir]) | ||
613 | 597 | shutil.rmtree(tmpdir) | ||
614 | 598 | |||
615 | 599 | |||
616 | 600 | if __name__ == "__main__": | ||
617 | 601 | logging.basicConfig( | ||
618 | 602 | level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s') | ||
619 | 603 | parser = get_option_parser() | ||
620 | 604 | args = parser.parse_args(sys.argv[1:]) | ||
621 | 605 | tarfile = os.path.abspath(args.tarfile) | ||
622 | 606 | juju = get_juju(args.juju, args.model, args.cfgdir, args.inner) | ||
623 | 607 | if args.inner: | ||
624 | 608 | log.info("# start inner ##############################") | ||
625 | 609 | try: | ||
626 | 610 | main(tarfile, args.extrafiles, juju, args.inner_model, args.inner) | ||
627 | 611 | finally: | ||
628 | 612 | if args.inner: | ||
629 | 613 | log.info("# end inner ################################") | ||
630 | 614 | 0 | ||
631 | === removed file 'test_collect-logs.py' | |||
632 | --- test_collect-logs.py 2017-01-05 17:33:16 +0000 | |||
633 | +++ test_collect-logs.py 1970-01-01 00:00:00 +0000 | |||
634 | @@ -1,1063 +0,0 @@ | |||
635 | 1 | # Copyright 2016 Canonical Limited. All rights reserved. | ||
636 | 2 | |||
637 | 3 | # To run: "python -m unittest test_collect-logs" | ||
638 | 4 | |||
639 | 5 | import errno | ||
640 | 6 | import os | ||
641 | 7 | import os.path | ||
642 | 8 | import shutil | ||
643 | 9 | import subprocess | ||
644 | 10 | import sys | ||
645 | 11 | import tempfile | ||
646 | 12 | from unittest import TestCase | ||
647 | 13 | |||
648 | 14 | import mock | ||
649 | 15 | |||
650 | 16 | |||
651 | 17 | __file__ = os.path.abspath(__file__) | ||
652 | 18 | |||
653 | 19 | script = type(sys)("collect-logs") | ||
654 | 20 | script.__file__ = os.path.abspath("collect-logs") | ||
655 | 21 | execfile("collect-logs", script.__dict__) | ||
656 | 22 | |||
657 | 23 | |||
658 | 24 | class FakeError(Exception): | ||
659 | 25 | """A specific error for which to check.""" | ||
660 | 26 | |||
661 | 27 | |||
662 | 28 | def _create_file(filename, data=None): | ||
663 | 29 | """Create (or re-create) the identified file. | ||
664 | 30 | |||
665 | 31 | If data is provided, it is written to the file. Otherwise it | ||
666 | 32 | will be empty. | ||
667 | 33 | |||
668 | 34 | The file's directory is created if necessary. | ||
669 | 35 | """ | ||
670 | 36 | dirname = os.path.dirname(os.path.abspath(filename)) | ||
671 | 37 | try: | ||
672 | 38 | os.makedirs(dirname) | ||
673 | 39 | except OSError as e: | ||
674 | 40 | if e.errno != errno.EEXIST: | ||
675 | 41 | raise | ||
676 | 42 | |||
677 | 43 | with open(filename, "w") as file: | ||
678 | 44 | if data: | ||
679 | 45 | file.write() | ||
680 | 46 | |||
681 | 47 | |||
682 | 48 | class _BaseTestCase(TestCase): | ||
683 | 49 | |||
684 | 50 | MOCKED = None | ||
685 | 51 | |||
686 | 52 | def setUp(self): | ||
687 | 53 | super(_BaseTestCase, self).setUp() | ||
688 | 54 | |||
689 | 55 | self.orig_cwd = os.getcwd() | ||
690 | 56 | self.cwd = tempfile.mkdtemp() | ||
691 | 57 | os.chdir(self.cwd) | ||
692 | 58 | |||
693 | 59 | self.tempdir = os.path.join(self.cwd, "tempdir") | ||
694 | 60 | os.mkdir(self.tempdir) | ||
695 | 61 | |||
696 | 62 | self.orig = {} | ||
697 | 63 | for attr in self.MOCKED or (): | ||
698 | 64 | self.orig[attr] = getattr(script, attr) | ||
699 | 65 | setattr(script, attr, mock.Mock()) | ||
700 | 66 | |||
701 | 67 | self.juju = script.Juju() | ||
702 | 68 | |||
703 | 69 | def tearDown(self): | ||
704 | 70 | for attr in self.MOCKED or (): | ||
705 | 71 | setattr(script, attr, self.orig[attr]) | ||
706 | 72 | |||
707 | 73 | shutil.rmtree(self.cwd) | ||
708 | 74 | os.chdir(self.orig_cwd) | ||
709 | 75 | |||
710 | 76 | super(_BaseTestCase, self).tearDown() | ||
711 | 77 | |||
712 | 78 | def _create_tempfile(self, filename, data=None): | ||
713 | 79 | """Create a file at the identified path, but rooted at the temp dir.""" | ||
714 | 80 | _create_file(os.path.join(self.tempdir, filename), data) | ||
715 | 81 | |||
716 | 82 | def assert_cwd(self, dirname): | ||
717 | 83 | """Ensure that the CWD matches the given directory.""" | ||
718 | 84 | cwd = os.getcwd() | ||
719 | 85 | self.assertEqual(cwd, dirname) | ||
720 | 86 | |||
721 | 87 | |||
722 | 88 | class GetJujuTests(TestCase): | ||
723 | 89 | |||
724 | 90 | def test_juju1_outer(self): | ||
725 | 91 | """ | ||
726 | 92 | get_juju() returns a Juju prepped for a Juju 1 outer model. | ||
727 | 93 | """ | ||
728 | 94 | juju = script.get_juju(script.JUJU1, inner=False) | ||
729 | 95 | |||
730 | 96 | expected = script.Juju("juju", model=None) | ||
731 | 97 | self.assertEqual(juju, expected) | ||
732 | 98 | |||
733 | 99 | def test_juju1_inner(self): | ||
734 | 100 | """ | ||
735 | 101 | get_juju() returns a Juju prepped for a Juju 1 inner model. | ||
736 | 102 | """ | ||
737 | 103 | cfgdir = "/var/lib/landscape/juju-homes/0" | ||
738 | 104 | |||
739 | 105 | juju = script.get_juju(script.JUJU1, model=None, cfgdir=cfgdir, | ||
740 | 106 | inner=True) | ||
741 | 107 | |||
742 | 108 | expected = script.Juju("juju", cfgdir=cfgdir) | ||
743 | 109 | self.assertEqual(juju, expected) | ||
744 | 110 | |||
745 | 111 | def test_juju2_outer(self): | ||
746 | 112 | """ | ||
747 | 113 | get_juju() returns a Juju prepped for a Juju 2 outer model. | ||
748 | 114 | """ | ||
749 | 115 | juju = script.get_juju(script.JUJU2, inner=False) | ||
750 | 116 | |||
751 | 117 | expected = script.Juju("juju-2.1", model=None) | ||
752 | 118 | self.assertEqual(juju, expected) | ||
753 | 119 | |||
754 | 120 | def test_juju2_inner(self): | ||
755 | 121 | """ | ||
756 | 122 | get_juju() returns a Juju prepped for a Juju 2 inner model. | ||
757 | 123 | """ | ||
758 | 124 | cfgdir = "/var/lib/landscape/juju-homes/0" | ||
759 | 125 | |||
760 | 126 | juju = script.get_juju(script.JUJU2, cfgdir=cfgdir, inner=True) | ||
761 | 127 | |||
762 | 128 | expected = script.Juju("juju-2.1", model="controller", cfgdir=cfgdir) | ||
763 | 129 | self.assertEqual(juju, expected) | ||
764 | 130 | |||
765 | 131 | |||
766 | 132 | class MainTestCase(_BaseTestCase): | ||
767 | 133 | |||
768 | 134 | MOCKED = ("collect_logs", "collect_inner_logs", "bundle_logs") | ||
769 | 135 | |||
770 | 136 | def setUp(self): | ||
771 | 137 | super(MainTestCase, self).setUp() | ||
772 | 138 | |||
773 | 139 | self.orig_mkdtemp = script.mkdtemp | ||
774 | 140 | script.mkdtemp = lambda: self.tempdir | ||
775 | 141 | |||
776 | 142 | def tearDown(self): | ||
777 | 143 | script.mkdtemp = self.orig_mkdtemp | ||
778 | 144 | |||
779 | 145 | super(MainTestCase, self).tearDown() | ||
780 | 146 | |||
781 | 147 | def test_success(self): | ||
782 | 148 | """ | ||
783 | 149 | main() calls collect_logs(), collect_inner_logs(), and bundle_logs(). | ||
784 | 150 | """ | ||
785 | 151 | tarfile = "/tmp/logs.tgz" | ||
786 | 152 | extrafiles = ["spam.py"] | ||
787 | 153 | |||
788 | 154 | script.main(tarfile, extrafiles, juju=self.juju) | ||
789 | 155 | |||
790 | 156 | script.collect_logs.assert_called_once_with(self.juju) | ||
791 | 157 | script.collect_inner_logs.assert_called_once_with( | ||
792 | 158 | self.juju, script.DEFAULT_MODEL) | ||
793 | 159 | script.bundle_logs.assert_called_once_with( | ||
794 | 160 | self.tempdir, tarfile, extrafiles) | ||
795 | 161 | self.assertFalse(os.path.exists(self.tempdir)) | ||
796 | 162 | |||
797 | 163 | def test_in_correct_directories(self): | ||
798 | 164 | """ | ||
799 | 165 | main() calls its dependencies while in specific directories. | ||
800 | 166 | """ | ||
801 | 167 | script.collect_logs.side_effect = ( | ||
802 | 168 | lambda _: self.assert_cwd(self.tempdir)) | ||
803 | 169 | script.collect_inner_logs.side_effect = ( | ||
804 | 170 | lambda _: self.assert_cwd(self.tempdir)) | ||
805 | 171 | script.bundle_logs.side_effect = lambda *a: self.assert_cwd(self.cwd) | ||
806 | 172 | tarfile = "/tmp/logs.tgz" | ||
807 | 173 | extrafiles = ["spam.py"] | ||
808 | 174 | |||
809 | 175 | script.main(tarfile, extrafiles, juju=self.juju) | ||
810 | 176 | |||
811 | 177 | def test_no_script_recursion_for_inner_model(self): | ||
812 | 178 | """ | ||
813 | 179 | main() will not call collect_inner_logs() if --inner is True. | ||
814 | 180 | """ | ||
815 | 181 | tarfile = "/tmp/logs.tgz" | ||
816 | 182 | extrafiles = ["spam.py"] | ||
817 | 183 | cfgdir = "/var/lib/landscape/juju-homes/0" | ||
818 | 184 | juju = script.get_juju(script.JUJU2, cfgdir) | ||
819 | 185 | |||
820 | 186 | script.main(tarfile, extrafiles, juju=juju, inner=True) | ||
821 | 187 | |||
822 | 188 | script.collect_logs.assert_called_once_with(juju) | ||
823 | 189 | script.collect_inner_logs.assert_not_called() | ||
824 | 190 | script.bundle_logs.assert_called_once_with( | ||
825 | 191 | self.tempdir, tarfile, extrafiles) | ||
826 | 192 | self.assertFalse(os.path.exists(self.tempdir)) | ||
827 | 193 | |||
828 | 194 | def test_cleanup(self): | ||
829 | 195 | """ | ||
830 | 196 | main() cleans up the temp dir it creates. | ||
831 | 197 | """ | ||
832 | 198 | tarfile = "/tmp/logs.tgz" | ||
833 | 199 | extrafiles = ["spam.py"] | ||
834 | 200 | |||
835 | 201 | script.main(tarfile, extrafiles, juju=self.juju) | ||
836 | 202 | |||
837 | 203 | self.assertFalse(os.path.exists(self.tempdir)) | ||
838 | 204 | |||
839 | 205 | def test_collect_logs_error(self): | ||
840 | 206 | """ | ||
841 | 207 | main() doesn't handle the error when collect_logs() fails. | ||
842 | 208 | |||
843 | 209 | It still cleans up the temp dir. | ||
844 | 210 | """ | ||
845 | 211 | tarfile = "/tmp/logs.tgz" | ||
846 | 212 | extrafiles = ["spam.py"] | ||
847 | 213 | script.collect_logs.side_effect = FakeError() | ||
848 | 214 | |||
849 | 215 | with self.assertRaises(FakeError): | ||
850 | 216 | script.main(tarfile, extrafiles, juju=self.juju) | ||
851 | 217 | |||
852 | 218 | script.collect_logs.assert_called_once_with(self.juju) | ||
853 | 219 | script.collect_inner_logs.assert_not_called() | ||
854 | 220 | script.bundle_logs.assert_not_called() | ||
855 | 221 | self.assertFalse(os.path.exists(self.tempdir)) | ||
856 | 222 | |||
857 | 223 | def test_collect_inner_logs_error(self): | ||
858 | 224 | """ | ||
859 | 225 | main() ignores the error when collect_inner_logs() fails. | ||
860 | 226 | |||
861 | 227 | It still cleans up the temp dir. | ||
862 | 228 | """ | ||
863 | 229 | tarfile = "/tmp/logs.tgz" | ||
864 | 230 | extrafiles = ["spam.py"] | ||
865 | 231 | script.collect_inner_logs.side_effect = FakeError() | ||
866 | 232 | |||
867 | 233 | script.main(tarfile, extrafiles, juju=self.juju) | ||
868 | 234 | |||
869 | 235 | script.collect_logs.assert_called_once_with(self.juju) | ||
870 | 236 | script.collect_inner_logs.assert_called_once_with( | ||
871 | 237 | self.juju, script.DEFAULT_MODEL) | ||
872 | 238 | script.bundle_logs.assert_called_once_with( | ||
873 | 239 | self.tempdir, tarfile, extrafiles) | ||
874 | 240 | self.assertFalse(os.path.exists(self.tempdir)) | ||
875 | 241 | |||
876 | 242 | def test_bundle_logs_error(self): | ||
877 | 243 | """ | ||
878 | 244 | main() doesn't handle the error when bundle_logs() fails. | ||
879 | 245 | |||
880 | 246 | It still cleans up the temp dir. | ||
881 | 247 | """ | ||
882 | 248 | tarfile = "/tmp/logs.tgz" | ||
883 | 249 | extrafiles = ["spam.py"] | ||
884 | 250 | script.bundle_logs.side_effect = FakeError() | ||
885 | 251 | |||
886 | 252 | with self.assertRaises(FakeError): | ||
887 | 253 | script.main(tarfile, extrafiles, juju=self.juju) | ||
888 | 254 | |||
889 | 255 | script.collect_logs.assert_called_once_with(self.juju) | ||
890 | 256 | script.collect_inner_logs.assert_called_once_with( | ||
891 | 257 | self.juju, script.DEFAULT_MODEL) | ||
892 | 258 | script.bundle_logs.assert_called_once_with( | ||
893 | 259 | self.tempdir, tarfile, extrafiles) | ||
894 | 260 | self.assertFalse(os.path.exists(self.tempdir)) | ||
895 | 261 | |||
896 | 262 | |||
897 | 263 | class CollectLogsTestCase(_BaseTestCase): | ||
898 | 264 | |||
899 | 265 | MOCKED = ("get_units", "check_output", "call") | ||
900 | 266 | |||
901 | 267 | def setUp(self): | ||
902 | 268 | super(CollectLogsTestCase, self).setUp() | ||
903 | 269 | |||
904 | 270 | self.units = [ | ||
905 | 271 | "landscape-server/0", | ||
906 | 272 | "postgresql/0", | ||
907 | 273 | "rabbitmq-server/0", | ||
908 | 274 | "haproxy/0", | ||
909 | 275 | ] | ||
910 | 276 | script.get_units.return_value = self.units[:] | ||
911 | 277 | |||
912 | 278 | self.mp_map_orig = script._mp_map | ||
913 | 279 | script._mp_map = lambda f, a: map(f, a) | ||
914 | 280 | |||
915 | 281 | os.chdir(self.tempdir) | ||
916 | 282 | |||
917 | 283 | def tearDown(self): | ||
918 | 284 | script._mp_map = self.mp_map_orig | ||
919 | 285 | |||
920 | 286 | super(CollectLogsTestCase, self).tearDown() | ||
921 | 287 | |||
922 | 288 | def _call_side_effect(self, cmd, env=None): | ||
923 | 289 | """Perform the side effect of calling the mocked-out call().""" | ||
924 | 290 | if cmd[0] == "tar": | ||
925 | 291 | self.assertTrue(os.path.exists(cmd[-1])) | ||
926 | 292 | return | ||
927 | 293 | self.assertEqual(env, self.juju.env) | ||
928 | 294 | self.assertEqual(cmd[0], self.juju.binary_path) | ||
929 | 295 | _create_file(os.path.basename(cmd[2])) | ||
930 | 296 | |||
931 | 297 | def test_success(self): | ||
932 | 298 | """ | ||
933 | 299 | collect_logs() gathers "ps" output and logs from each unit. | ||
934 | 300 | """ | ||
935 | 301 | script.call.side_effect = self._call_side_effect | ||
936 | 302 | |||
937 | 303 | script.collect_logs(self.juju) | ||
938 | 304 | |||
939 | 305 | script.get_units.assert_called_once_with(self.juju) | ||
940 | 306 | expected = [] | ||
941 | 307 | units = self.units + ["0"] | ||
942 | 308 | # for _create_ps_output_file() | ||
943 | 309 | for unit in units: | ||
944 | 310 | cmd = "ps fauxww | sudo tee /var/log/ps-fauxww.txt" | ||
945 | 311 | expected.append(mock.call(["juju", "ssh", unit, cmd], | ||
946 | 312 | stderr=subprocess.STDOUT, | ||
947 | 313 | env=None, | ||
948 | 314 | )) | ||
949 | 315 | # for _create_log_tarball() | ||
950 | 316 | for unit in units: | ||
951 | 317 | tarfile = "/tmp/logs_{}.tar".format(unit.replace("/", "-") | ||
952 | 318 | if unit != "0" | ||
953 | 319 | else "bootstrap") | ||
954 | 320 | cmd = ("sudo tar --ignore-failed-read" | ||
955 | 321 | " --exclude=/var/lib/landscape/client/package/hash-id" | ||
956 | 322 | " --exclude=/var/lib/juju/containers/juju-*-lxc-template" | ||
957 | 323 | " -cf {}" | ||
958 | 324 | " $(sudo sh -c \"ls -1d {} 2>/dev/null\")" | ||
959 | 325 | ).format( | ||
960 | 326 | tarfile, | ||
961 | 327 | " ".join(["/var/log", | ||
962 | 328 | "/etc/hosts", | ||
963 | 329 | "/etc/network", | ||
964 | 330 | "/var/lib/landscape/client", | ||
965 | 331 | "/etc/apache2", | ||
966 | 332 | "/etc/haproxy", | ||
967 | 333 | "/var/lib/lxc/*/rootfs/var/log", | ||
968 | 334 | "/var/lib/juju/containers", | ||
969 | 335 | "/etc/nova", | ||
970 | 336 | "/etc/swift", | ||
971 | 337 | "/etc/neutron", | ||
972 | 338 | "/etc/ceph", | ||
973 | 339 | "/etc/glance", | ||
974 | 340 | ]), | ||
975 | 341 | ) | ||
976 | 342 | expected.append(mock.call(["juju", "ssh", unit, cmd], | ||
977 | 343 | stderr=subprocess.STDOUT, | ||
978 | 344 | env=None, | ||
979 | 345 | )) | ||
980 | 346 | expected.append(mock.call(["juju", "ssh", unit, | ||
981 | 347 | "sudo gzip -f {}".format(tarfile)], | ||
982 | 348 | stderr=subprocess.STDOUT, | ||
983 | 349 | env=None, | ||
984 | 350 | )) | ||
985 | 351 | self.assertEqual(script.check_output.call_count, len(expected)) | ||
986 | 352 | script.check_output.assert_has_calls(expected, any_order=True) | ||
987 | 353 | # for download_log_from_unit() | ||
988 | 354 | expected = [] | ||
989 | 355 | for unit in units: | ||
990 | 356 | name = unit.replace("/", "-") if unit != "0" else "bootstrap" | ||
991 | 357 | filename = "logs_{}.tar.gz".format(name) | ||
992 | 358 | source = "{}:/tmp/{}".format(unit, filename) | ||
993 | 359 | expected.append(mock.call(["juju", "scp", source, "."], env=None)) | ||
994 | 360 | expected.append(mock.call(["tar", "-C", name, "-xzf", filename])) | ||
995 | 361 | self.assertFalse(os.path.exists(filename)) | ||
996 | 362 | self.assertEqual(script.call.call_count, len(expected)) | ||
997 | 363 | script.call.assert_has_calls(expected, any_order=True) | ||
998 | 364 | |||
999 | 365 | def test_inner(self): | ||
1000 | 366 | """ | ||
1001 | 367 | collect_logs() gathers "ps" output and logs from each unit. | ||
1002 | 368 | Running in the inner model produces different commands. | ||
1003 | 369 | """ | ||
1004 | 370 | cfgdir = "/var/lib/landscape/juju-homes/0" | ||
1005 | 371 | juju = script.Juju("juju-2.1", model="controller", cfgdir=cfgdir) | ||
1006 | 372 | self.juju = juju | ||
1007 | 373 | script.call.side_effect = self._call_side_effect | ||
1008 | 374 | |||
1009 | 375 | script.collect_logs(juju) | ||
1010 | 376 | |||
1011 | 377 | script.get_units.assert_called_once_with(juju) | ||
1012 | 378 | expected = [] | ||
1013 | 379 | units = self.units + ["0"] | ||
1014 | 380 | # for _create_ps_output_file() | ||
1015 | 381 | for unit in units: | ||
1016 | 382 | cmd = "ps fauxww | sudo tee /var/log/ps-fauxww.txt" | ||
1017 | 383 | expected.append(mock.call(["juju-2.1", "ssh", | ||
1018 | 384 | "-m", "controller", unit, cmd], | ||
1019 | 385 | stderr=subprocess.STDOUT, | ||
1020 | 386 | env=juju.env, | ||
1021 | 387 | )) | ||
1022 | 388 | # for _create_log_tarball() | ||
1023 | 389 | for unit in units: | ||
1024 | 390 | tarfile = "/tmp/logs_{}.tar".format(unit.replace("/", "-") | ||
1025 | 391 | if unit != "0" | ||
1026 | 392 | else "bootstrap") | ||
1027 | 393 | cmd = ("sudo tar --ignore-failed-read" | ||
1028 | 394 | " --exclude=/var/lib/landscape/client/package/hash-id" | ||
1029 | 395 | " --exclude=/var/lib/juju/containers/juju-*-lxc-template" | ||
1030 | 396 | " -cf {}" | ||
1031 | 397 | " $(sudo sh -c \"ls -1d {} 2>/dev/null\")" | ||
1032 | 398 | ).format( | ||
1033 | 399 | tarfile, | ||
1034 | 400 | " ".join(["/var/log", | ||
1035 | 401 | "/etc/hosts", | ||
1036 | 402 | "/etc/network", | ||
1037 | 403 | "/var/lib/landscape/client", | ||
1038 | 404 | "/etc/apache2", | ||
1039 | 405 | "/etc/haproxy", | ||
1040 | 406 | "/var/lib/lxc/*/rootfs/var/log", | ||
1041 | 407 | "/var/lib/juju/containers", | ||
1042 | 408 | "/etc/nova", | ||
1043 | 409 | "/etc/swift", | ||
1044 | 410 | "/etc/neutron", | ||
1045 | 411 | "/etc/ceph", | ||
1046 | 412 | "/etc/glance", | ||
1047 | 413 | ]), | ||
1048 | 414 | ) | ||
1049 | 415 | expected.append(mock.call( | ||
1050 | 416 | ["juju-2.1", "ssh", "-m", "controller", unit, cmd], | ||
1051 | 417 | stderr=subprocess.STDOUT, | ||
1052 | 418 | env=juju.env, | ||
1053 | 419 | )) | ||
1054 | 420 | expected.append(mock.call( | ||
1055 | 421 | ["juju-2.1", "ssh", "-m", "controller", unit, | ||
1056 | 422 | "sudo gzip -f {}".format(tarfile)], | ||
1057 | 423 | stderr=subprocess.STDOUT, | ||
1058 | 424 | env=juju.env, | ||
1059 | 425 | )) | ||
1060 | 426 | self.assertEqual(script.check_output.call_count, len(expected)) | ||
1061 | 427 | script.check_output.assert_has_calls(expected, any_order=True) | ||
1062 | 428 | # for download_log_from_unit() | ||
1063 | 429 | expected = [] | ||
1064 | 430 | for unit in units: | ||
1065 | 431 | name = unit.replace("/", "-") if unit != "0" else "bootstrap" | ||
1066 | 432 | filename = "logs_{}.tar.gz".format(name) | ||
1067 | 433 | source = "{}:/tmp/{}".format(unit, filename) | ||
1068 | 434 | expected.append(mock.call( | ||
1069 | 435 | ["juju-2.1", "scp", "-m", "controller", source, "."], | ||
1070 | 436 | env=juju.env)) | ||
1071 | 437 | expected.append(mock.call(["tar", "-C", name, "-xzf", filename])) | ||
1072 | 438 | self.assertFalse(os.path.exists(filename)) | ||
1073 | 439 | self.assertEqual(script.call.call_count, len(expected)) | ||
1074 | 440 | script.call.assert_has_calls(expected, any_order=True) | ||
1075 | 441 | |||
1076 | 442 | def test_get_units_failure(self): | ||
1077 | 443 | """ | ||
1078 | 444 | collect_logs() does not handle errors from get_units(). | ||
1079 | 445 | """ | ||
1080 | 446 | script.get_units.side_effect = FakeError() | ||
1081 | 447 | |||
1082 | 448 | with self.assertRaises(FakeError): | ||
1083 | 449 | script.collect_logs(self.juju) | ||
1084 | 450 | |||
1085 | 451 | script.get_units.assert_called_once_with(self.juju) | ||
1086 | 452 | script.check_output.assert_not_called() | ||
1087 | 453 | script.call.assert_not_called() | ||
1088 | 454 | |||
1089 | 455 | def test_check_output_failure(self): | ||
1090 | 456 | """ | ||
1091 | 457 | collect_logs() does not handle errors from check_output(). | ||
1092 | 458 | """ | ||
1093 | 459 | script.check_output.side_effect = [mock.DEFAULT, | ||
1094 | 460 | FakeError(), | ||
1095 | 461 | ] | ||
1096 | 462 | |||
1097 | 463 | with self.assertRaises(FakeError): | ||
1098 | 464 | script.collect_logs(self.juju) | ||
1099 | 465 | |||
1100 | 466 | script.get_units.assert_called_once_with(self.juju) | ||
1101 | 467 | self.assertEqual(script.check_output.call_count, 2) | ||
1102 | 468 | script.call.assert_not_called() | ||
1103 | 469 | |||
1104 | 470 | def test_call_failure(self): | ||
1105 | 471 | """ | ||
1106 | 472 | collect_logs() does not handle errors from call(). | ||
1107 | 473 | """ | ||
1108 | 474 | def call_side_effect(cmd, env=None): | ||
1109 | 475 | # second use of call() for landscape-server/0 | ||
1110 | 476 | if script.call.call_count == 2: | ||
1111 | 477 | raise FakeError() | ||
1112 | 478 | # first use of call() for postgresql/0 | ||
1113 | 479 | if script.call.call_count == 3: | ||
1114 | 480 | raise FakeError() | ||
1115 | 481 | # all other uses of call() default to the normal side effect. | ||
1116 | 482 | return self._call_side_effect(cmd, env=env) | ||
1117 | 483 | script.call.side_effect = call_side_effect | ||
1118 | 484 | |||
1119 | 485 | script.collect_logs(self.juju) | ||
1120 | 486 | |||
1121 | 487 | script.get_units.assert_called_once_with(self.juju) | ||
1122 | 488 | units = self.units + ["0"] | ||
1123 | 489 | self.assertEqual(script.check_output.call_count, len(units) * 3) | ||
1124 | 490 | self.assertEqual(script.call.call_count, len(units) * 2 - 1) | ||
1125 | 491 | for unit in units: | ||
1126 | 492 | name = unit.replace("/", "-") if unit != "0" else "bootstrap" | ||
1127 | 493 | if unit == self.units[1]: | ||
1128 | 494 | self.assertFalse(os.path.exists(name)) | ||
1129 | 495 | else: | ||
1130 | 496 | self.assertTrue(os.path.exists(name)) | ||
1131 | 497 | filename = "logs_{}.tar.gz".format(name) | ||
1132 | 498 | self.assertFalse(os.path.exists(filename)) | ||
1133 | 499 | |||
1134 | 500 | |||
1135 | 501 | class CollectInnerLogsTestCase(_BaseTestCase): | ||
1136 | 502 | |||
1137 | 503 | MOCKED = ("get_units", "check_output", "call", "check_call") | ||
1138 | 504 | |||
1139 | 505 | def setUp(self): | ||
1140 | 506 | super(CollectInnerLogsTestCase, self).setUp() | ||
1141 | 507 | |||
1142 | 508 | self.units = [ | ||
1143 | 509 | "landscape-server/0", | ||
1144 | 510 | "postgresql/0", | ||
1145 | 511 | "rabbitmq-server/0", | ||
1146 | 512 | "haproxy/0", | ||
1147 | 513 | ] | ||
1148 | 514 | script.get_units.return_value = self.units[:] | ||
1149 | 515 | script.check_output.return_value = "0\n" | ||
1150 | 516 | script.call.return_value = 0 | ||
1151 | 517 | |||
1152 | 518 | os.chdir(self.tempdir) | ||
1153 | 519 | |||
1154 | 520 | def assert_clean(self): | ||
1155 | 521 | """Ensure that collect_inner_logs cleaned up after itself.""" | ||
1156 | 522 | self.assert_cwd(self.tempdir) | ||
1157 | 523 | self.assertFalse(os.path.exists("inner-logs.tar.gz")) | ||
1158 | 524 | |||
1159 | 525 | def test_juju_2(self): | ||
1160 | 526 | """ | ||
1161 | 527 | collect_inner_logs() finds the inner model and runs collect-logs | ||
1162 | 528 | inside it. The resulting tarball is downloaded, extracted, and | ||
1163 | 529 | deleted. | ||
1164 | 530 | """ | ||
1165 | 531 | def check_call_side_effect(cmd, env=None): | ||
1166 | 532 | self.assertEqual(env, self.juju.env) | ||
1167 | 533 | if script.check_call.call_count == 4: | ||
1168 | 534 | self.assert_cwd(self.tempdir) | ||
1169 | 535 | self._create_tempfile("inner-logs.tar.gz") | ||
1170 | 536 | elif script.check_call.call_count == 5: | ||
1171 | 537 | cwd = os.path.join(self.tempdir, "landscape-0-inner-logs") | ||
1172 | 538 | self.assert_cwd(cwd) | ||
1173 | 539 | return None | ||
1174 | 540 | script.check_call.side_effect = check_call_side_effect | ||
1175 | 541 | |||
1176 | 542 | script.collect_inner_logs(self.juju) | ||
1177 | 543 | |||
1178 | 544 | # Check get_units() calls. | ||
1179 | 545 | script.get_units.assert_called_once_with(self.juju) | ||
1180 | 546 | # Check check_output() calls. | ||
1181 | 547 | expected = [] | ||
1182 | 548 | cmd = ("sudo JUJU_DATA=/var/lib/landscape/juju-homes/" | ||
1183 | 549 | "`sudo ls -rt /var/lib/landscape/juju-homes/ | tail -1`" | ||
1184 | 550 | " juju-2.1 model-config -m controller proxy-ssh=false") | ||
1185 | 551 | expected.append(mock.call(["juju", "ssh", "landscape-server/0", cmd], | ||
1186 | 552 | stderr=subprocess.STDOUT, | ||
1187 | 553 | env=self.juju.env)) | ||
1188 | 554 | expected.append(mock.call( | ||
1189 | 555 | ["juju", "ssh", "landscape-server/0", | ||
1190 | 556 | "sudo ls -rt /var/lib/landscape/juju-homes/"], | ||
1191 | 557 | env=self.juju.env)) | ||
1192 | 558 | self.assertEqual(script.check_output.call_count, len(expected)) | ||
1193 | 559 | script.check_output.assert_has_calls(expected, any_order=True) | ||
1194 | 560 | # Check call() calls. | ||
1195 | 561 | expected = [ | ||
1196 | 562 | mock.call(["juju", "ssh", "landscape-server/0", | ||
1197 | 563 | ("sudo JUJU_DATA=/var/lib/landscape/juju-homes/0 " | ||
1198 | 564 | "juju-2.1 status -m controller --format=yaml"), | ||
1199 | 565 | ], env=self.juju.env), | ||
1200 | 566 | mock.call(["juju", "scp", | ||
1201 | 567 | os.path.join(os.path.dirname(__file__), "collect-logs"), | ||
1202 | 568 | "landscape-server/0:/tmp/collect-logs", | ||
1203 | 569 | ], env=self.juju.env), | ||
1204 | 570 | mock.call(["juju", "ssh", | ||
1205 | 571 | "landscape-server/0", | ||
1206 | 572 | "sudo rm -rf /tmp/inner-logs.tar.gz", | ||
1207 | 573 | ], env=self.juju.env), | ||
1208 | 574 | ] | ||
1209 | 575 | self.assertEqual(script.call.call_count, len(expected)) | ||
1210 | 576 | script.call.assert_has_calls(expected, any_order=True) | ||
1211 | 577 | # Check check_call() calls. | ||
1212 | 578 | cmd = ("sudo" | ||
1213 | 579 | " JUJU_DATA=/var/lib/landscape/juju-homes/0" | ||
1214 | 580 | " /tmp/collect-logs --inner --juju juju-2.1" | ||
1215 | 581 | " --model controller" | ||
1216 | 582 | " --cfgdir /var/lib/landscape/juju-homes/0" | ||
1217 | 583 | " /tmp/inner-logs.tar.gz") | ||
1218 | 584 | expected = [ | ||
1219 | 585 | mock.call(["juju", "ssh", "landscape-server/0", cmd], | ||
1220 | 586 | env=self.juju.env), | ||
1221 | 587 | mock.call(["juju", "scp", | ||
1222 | 588 | "landscape-server/0:/tmp/inner-logs.tar.gz", | ||
1223 | 589 | os.path.join(self.tempdir, "inner-logs.tar.gz"), | ||
1224 | 590 | ], env=self.juju.env), | ||
1225 | 591 | mock.call(["tar", "-zxf", self.tempdir + "/inner-logs.tar.gz"]), | ||
1226 | 592 | ] | ||
1227 | 593 | self.assertEqual(script.check_call.call_count, len(expected)) | ||
1228 | 594 | script.check_call.assert_has_calls(expected, any_order=True) | ||
1229 | 595 | self.assert_clean() | ||
1230 | 596 | |||
1231 | 597 | def test_juju_1(self): | ||
1232 | 598 | """ | ||
1233 | 599 | collect_inner_logs() finds the inner model and runs collect-logs | ||
1234 | 600 | inside it. The resulting tarball is downloaded, extracted, and | ||
1235 | 601 | deleted. | ||
1236 | 602 | """ | ||
1237 | 603 | def check_call_side_effect(cmd, env=None): | ||
1238 | 604 | self.assertEqual(env, self.juju.env) | ||
1239 | 605 | if script.check_call.call_count == 4: | ||
1240 | 606 | self.assert_cwd(self.tempdir) | ||
1241 | 607 | self._create_tempfile("inner-logs.tar.gz") | ||
1242 | 608 | elif script.check_call.call_count == 5: | ||
1243 | 609 | cwd = os.path.join(self.tempdir, "landscape-0-inner-logs") | ||
1244 | 610 | self.assert_cwd(cwd) | ||
1245 | 611 | return None | ||
1246 | 612 | script.check_call.side_effect = check_call_side_effect | ||
1247 | 613 | script.call.side_effect = [1, 0, 0, 0] | ||
1248 | 614 | err = subprocess.CalledProcessError(1, "...", "<output>") | ||
1249 | 615 | script.check_output.side_effect = [err, | ||
1250 | 616 | mock.DEFAULT, | ||
1251 | 617 | mock.DEFAULT, | ||
1252 | 618 | ] | ||
1253 | 619 | |||
1254 | 620 | script.collect_inner_logs(self.juju) | ||
1255 | 621 | |||
1256 | 622 | # Check get_units() calls. | ||
1257 | 623 | script.get_units.assert_called_once_with(self.juju) | ||
1258 | 624 | # Check check_output() calls. | ||
1259 | 625 | expected = [] | ||
1260 | 626 | cmd = ("sudo JUJU_DATA=/var/lib/landscape/juju-homes/" | ||
1261 | 627 | "`sudo ls -rt /var/lib/landscape/juju-homes/ | tail -1`" | ||
1262 | 628 | " juju-2.1 model-config -m controller proxy-ssh=false") | ||
1263 | 629 | expected.append(mock.call(["juju", "ssh", "landscape-server/0", cmd], | ||
1264 | 630 | stderr=subprocess.STDOUT, | ||
1265 | 631 | env=None)) | ||
1266 | 632 | cmd = ("sudo JUJU_HOME=/var/lib/landscape/juju-homes/" | ||
1267 | 633 | "`sudo ls -rt /var/lib/landscape/juju-homes/ | tail -1`" | ||
1268 | 634 | " juju set-env proxy-ssh=false") | ||
1269 | 635 | expected.append(mock.call(["juju", "ssh", "landscape-server/0", cmd], | ||
1270 | 636 | stderr=subprocess.STDOUT, | ||
1271 | 637 | env=None)) | ||
1272 | 638 | expected.append(mock.call( | ||
1273 | 639 | ["juju", "ssh", "landscape-server/0", | ||
1274 | 640 | "sudo ls -rt /var/lib/landscape/juju-homes/"], | ||
1275 | 641 | env=None)) | ||
1276 | 642 | self.assertEqual(script.check_output.call_count, len(expected)) | ||
1277 | 643 | script.check_output.assert_has_calls(expected, any_order=True) | ||
1278 | 644 | # Check call() calls. | ||
1279 | 645 | expected = [ | ||
1280 | 646 | mock.call(["juju", "ssh", "landscape-server/0", | ||
1281 | 647 | ("sudo JUJU_DATA=/var/lib/landscape/juju-homes/0 " | ||
1282 | 648 | "juju-2.1 status -m controller --format=yaml"), | ||
1283 | 649 | ], env=None), | ||
1284 | 650 | mock.call(["juju", "ssh", "landscape-server/0", | ||
1285 | 651 | ("sudo -u landscape " | ||
1286 | 652 | "JUJU_HOME=/var/lib/landscape/juju-homes/0 " | ||
1287 | 653 | "juju status --format=yaml"), | ||
1288 | 654 | ], env=None), | ||
1289 | 655 | mock.call(["juju", "scp", | ||
1290 | 656 | os.path.join(os.path.dirname(__file__), "collect-logs"), | ||
1291 | 657 | "landscape-server/0:/tmp/collect-logs", | ||
1292 | 658 | ], env=None), | ||
1293 | 659 | mock.call(["juju", "ssh", | ||
1294 | 660 | "landscape-server/0", | ||
1295 | 661 | "sudo rm -rf /tmp/inner-logs.tar.gz", | ||
1296 | 662 | ], env=None), | ||
1297 | 663 | ] | ||
1298 | 664 | self.assertEqual(script.call.call_count, len(expected)) | ||
1299 | 665 | script.call.assert_has_calls(expected, any_order=True) | ||
1300 | 666 | # Check check_call() calls. | ||
1301 | 667 | cmd = ("sudo -u landscape" | ||
1302 | 668 | " JUJU_HOME=/var/lib/landscape/juju-homes/0" | ||
1303 | 669 | " /tmp/collect-logs --inner --juju juju" | ||
1304 | 670 | " --cfgdir /var/lib/landscape/juju-homes/0" | ||
1305 | 671 | " /tmp/inner-logs.tar.gz") | ||
1306 | 672 | expected = [ | ||
1307 | 673 | mock.call(["juju", "ssh", "landscape-server/0", cmd], env=None), | ||
1308 | 674 | mock.call(["juju", "scp", | ||
1309 | 675 | "landscape-server/0:/tmp/inner-logs.tar.gz", | ||
1310 | 676 | os.path.join(self.tempdir, "inner-logs.tar.gz"), | ||
1311 | 677 | ], env=None), | ||
1312 | 678 | mock.call(["tar", "-zxf", self.tempdir + "/inner-logs.tar.gz"]), | ||
1313 | 679 | ] | ||
1314 | 680 | self.assertEqual(script.check_call.call_count, len(expected)) | ||
1315 | 681 | script.check_call.assert_has_calls(expected, any_order=True) | ||
1316 | 682 | self.assert_clean() | ||
1317 | 683 | |||
1318 | 684 | def test_with_legacy_landscape_unit(self): | ||
1319 | 685 | """ | ||
1320 | 686 | collect_inner_logs() correctly supports legacy landscape installations. | ||
1321 | 687 | """ | ||
1322 | 688 | self.units[0] = "landscape/0" | ||
1323 | 689 | script.get_units.return_value = self.units[:] | ||
1324 | 690 | err = subprocess.CalledProcessError(1, "...", "<output>") | ||
1325 | 691 | script.check_output.side_effect = [err, | ||
1326 | 692 | mock.DEFAULT, | ||
1327 | 693 | mock.DEFAULT, | ||
1328 | 694 | ] | ||
1329 | 695 | |||
1330 | 696 | script.collect_inner_logs(self.juju) | ||
1331 | 697 | |||
1332 | 698 | expected = [] | ||
1333 | 699 | cmd = ("sudo JUJU_DATA=/var/lib/landscape/juju-homes/" | ||
1334 | 700 | "`sudo ls -rt /var/lib/landscape/juju-homes/ | tail -1`" | ||
1335 | 701 | " juju-2.1 model-config -m controller proxy-ssh=false") | ||
1336 | 702 | expected.append(mock.call(["juju", "ssh", "landscape/0", cmd], | ||
1337 | 703 | stderr=subprocess.STDOUT, | ||
1338 | 704 | env=None)) | ||
1339 | 705 | cmd = ("sudo JUJU_HOME=/var/lib/landscape/juju-homes/" | ||
1340 | 706 | "`sudo ls -rt /var/lib/landscape/juju-homes/ | tail -1`" | ||
1341 | 707 | " juju set-env proxy-ssh=false") | ||
1342 | 708 | expected.append(mock.call(["juju", "ssh", "landscape/0", cmd], | ||
1343 | 709 | stderr=subprocess.STDOUT, | ||
1344 | 710 | env=None)) | ||
1345 | 711 | expected.append(mock.call( | ||
1346 | 712 | ["juju", "ssh", "landscape/0", | ||
1347 | 713 | "sudo ls -rt /var/lib/landscape/juju-homes/"], | ||
1348 | 714 | env=None)) | ||
1349 | 715 | self.assertEqual(script.check_output.call_count, len(expected)) | ||
1350 | 716 | script.check_output.assert_has_calls(expected, any_order=True) | ||
1351 | 717 | self.assert_clean() | ||
1352 | 718 | |||
1353 | 719 | def test_no_units(self): | ||
1354 | 720 | """ | ||
1355 | 721 | collect_inner_logs() is a noop if no units are found. | ||
1356 | 722 | """ | ||
1357 | 723 | script.get_units.return_value = [] | ||
1358 | 724 | |||
1359 | 725 | script.collect_inner_logs(self.juju) | ||
1360 | 726 | |||
1361 | 727 | script.get_units.assert_called_once_with(self.juju) | ||
1362 | 728 | script.check_output.assert_not_called() | ||
1363 | 729 | script.call.assert_not_called() | ||
1364 | 730 | script.check_call.assert_not_called() | ||
1365 | 731 | self.assert_clean() | ||
1366 | 732 | |||
1367 | 733 | def test_no_landscape_server_unit(self): | ||
1368 | 734 | """ | ||
1369 | 735 | collect_inner_logs() is a noop if the landscape unit isn't found. | ||
1370 | 736 | """ | ||
1371 | 737 | del self.units[0] | ||
1372 | 738 | script.get_units.return_value = self.units[:] | ||
1373 | 739 | |||
1374 | 740 | script.collect_inner_logs(self.juju) | ||
1375 | 741 | |||
1376 | 742 | script.get_units.assert_called_once_with(self.juju) | ||
1377 | 743 | script.check_output.assert_not_called() | ||
1378 | 744 | script.call.assert_not_called() | ||
1379 | 745 | script.check_call.assert_not_called() | ||
1380 | 746 | self.assert_clean() | ||
1381 | 747 | |||
1382 | 748 | def test_no_juju_homes(self): | ||
1383 | 749 | script.get_units.return_value = [] | ||
1384 | 750 | script.check_output.return_value = "" | ||
1385 | 751 | |||
1386 | 752 | script.collect_inner_logs(self.juju) | ||
1387 | 753 | |||
1388 | 754 | script.get_units.assert_called_once_with(self.juju) | ||
1389 | 755 | |||
1390 | 756 | script.get_units.assert_called_once_with(self.juju) | ||
1391 | 757 | script.check_output.assert_not_called() | ||
1392 | 758 | script.call.assert_not_called() | ||
1393 | 759 | script.check_call.assert_not_called() | ||
1394 | 760 | self.assert_clean() | ||
1395 | 761 | |||
1396 | 762 | def test_get_units_failure(self): | ||
1397 | 763 | """ | ||
1398 | 764 | collect_inner_logs() does not handle errors from get_units(). | ||
1399 | 765 | """ | ||
1400 | 766 | script.get_units.side_effect = FakeError() | ||
1401 | 767 | |||
1402 | 768 | with self.assertRaises(FakeError): | ||
1403 | 769 | script.collect_inner_logs(self.juju) | ||
1404 | 770 | |||
1405 | 771 | self.assertEqual(script.get_units.call_count, 1) | ||
1406 | 772 | script.check_output.assert_not_called() | ||
1407 | 773 | script.call.assert_not_called() | ||
1408 | 774 | script.check_call.assert_not_called() | ||
1409 | 775 | self.assert_cwd(self.tempdir) | ||
1410 | 776 | self.assert_clean() | ||
1411 | 777 | |||
1412 | 778 | def test_check_output_failure_1(self): | ||
1413 | 779 | """ | ||
1414 | 780 | collect_inner_logs() does not handle non-CalledProcessError | ||
1415 | 781 | errors when disabling the SSH proxy. | ||
1416 | 782 | """ | ||
1417 | 783 | script.check_output.side_effect = FakeError() | ||
1418 | 784 | |||
1419 | 785 | with self.assertRaises(FakeError): | ||
1420 | 786 | script.collect_inner_logs(self.juju) | ||
1421 | 787 | |||
1422 | 788 | self.assertEqual(script.get_units.call_count, 1) | ||
1423 | 789 | self.assertEqual(script.check_output.call_count, 1) | ||
1424 | 790 | script.call.assert_not_called() | ||
1425 | 791 | script.check_call.assert_not_called() | ||
1426 | 792 | self.assert_cwd(self.tempdir) | ||
1427 | 793 | self.assert_clean() | ||
1428 | 794 | |||
1429 | 795 | def test_check_output_failure_2(self): | ||
1430 | 796 | """ | ||
1431 | 797 | collect_inner_logs() does not handle non-CalledProcessError | ||
1432 | 798 | errors when verifying the inner model is bootstrapped. | ||
1433 | 799 | """ | ||
1434 | 800 | script.check_output.side_effect = [None, | ||
1435 | 801 | FakeError(), | ||
1436 | 802 | ] | ||
1437 | 803 | |||
1438 | 804 | with self.assertRaises(FakeError): | ||
1439 | 805 | script.collect_inner_logs(self.juju) | ||
1440 | 806 | |||
1441 | 807 | self.assertEqual(script.get_units.call_count, 1) | ||
1442 | 808 | self.assertEqual(script.check_output.call_count, 2) | ||
1443 | 809 | script.call.assert_not_called() | ||
1444 | 810 | script.check_call.assert_not_called() | ||
1445 | 811 | self.assert_cwd(self.tempdir) | ||
1446 | 812 | self.assert_clean() | ||
1447 | 813 | |||
1448 | 814 | def test_call_juju2_failure(self): | ||
1449 | 815 | """ | ||
1450 | 816 | collect_inner_logs() does not handle errors from call(). | ||
1451 | 817 | """ | ||
1452 | 818 | script.call.side_effect = FakeError() | ||
1453 | 819 | |||
1454 | 820 | with self.assertRaises(FakeError): | ||
1455 | 821 | script.collect_inner_logs(self.juju) | ||
1456 | 822 | |||
1457 | 823 | self.assertEqual(script.get_units.call_count, 1) | ||
1458 | 824 | self.assertEqual(script.check_output.call_count, 2) | ||
1459 | 825 | self.assertEqual(script.call.call_count, 1) | ||
1460 | 826 | script.check_call.assert_not_called() | ||
1461 | 827 | self.assert_cwd(self.tempdir) | ||
1462 | 828 | self.assert_clean() | ||
1463 | 829 | |||
1464 | 830 | def test_call_juju1_failure(self): | ||
1465 | 831 | """ | ||
1466 | 832 | collect_inner_logs() does not handle errors from call(). | ||
1467 | 833 | """ | ||
1468 | 834 | script.call.side_effect = [1, | ||
1469 | 835 | FakeError(), | ||
1470 | 836 | ] | ||
1471 | 837 | |||
1472 | 838 | with self.assertRaises(FakeError): | ||
1473 | 839 | script.collect_inner_logs(self.juju) | ||
1474 | 840 | |||
1475 | 841 | self.assertEqual(script.get_units.call_count, 1) | ||
1476 | 842 | self.assertEqual(script.check_output.call_count, 2) | ||
1477 | 843 | self.assertEqual(script.call.call_count, 2) | ||
1478 | 844 | script.check_call.assert_not_called() | ||
1479 | 845 | self.assert_cwd(self.tempdir) | ||
1480 | 846 | self.assert_clean() | ||
1481 | 847 | |||
1482 | 848 | def test_call_juju2_nonzero_return(self): | ||
1483 | 849 | """ | ||
1484 | 850 | When no Juju 2 model is detected, Juju 1 is tried. | ||
1485 | 851 | """ | ||
1486 | 852 | script.call.side_effect = [1, | ||
1487 | 853 | mock.DEFAULT, | ||
1488 | 854 | mock.DEFAULT, | ||
1489 | 855 | mock.DEFAULT, | ||
1490 | 856 | ] | ||
1491 | 857 | |||
1492 | 858 | script.collect_inner_logs(self.juju) | ||
1493 | 859 | |||
1494 | 860 | self.assertEqual(script.get_units.call_count, 1) | ||
1495 | 861 | self.assertEqual(script.check_output.call_count, 2) | ||
1496 | 862 | self.assertEqual(script.call.call_count, 4) | ||
1497 | 863 | self.assertEqual(script.check_call.call_count, 3) | ||
1498 | 864 | self.assert_clean() | ||
1499 | 865 | |||
1500 | 866 | def test_call_juju1_nonzero_return(self): | ||
1501 | 867 | """ | ||
1502 | 868 | When no Juju 2 model is detected, Juju 1 is tried. When that | ||
1503 | 869 | is not detected, no inner logs are collected. | ||
1504 | 870 | """ | ||
1505 | 871 | script.call.side_effect = [1, | ||
1506 | 872 | 1, | ||
1507 | 873 | ] | ||
1508 | 874 | |||
1509 | 875 | script.collect_inner_logs(self.juju) | ||
1510 | 876 | |||
1511 | 877 | self.assertEqual(script.get_units.call_count, 1) | ||
1512 | 878 | self.assertEqual(script.check_output.call_count, 2) | ||
1513 | 879 | self.assertEqual(script.call.call_count, 2) | ||
1514 | 880 | script.check_call.assert_not_called() | ||
1515 | 881 | self.assert_clean() | ||
1516 | 882 | |||
1517 | 883 | def test_call_all_nonzero_return(self): | ||
1518 | 884 | """ | ||
1519 | 885 | When no Juju 2 model is detected, Juju 1 is tried. When that | ||
1520 | 886 | is not detected, no inner logs are collected. | ||
1521 | 887 | """ | ||
1522 | 888 | script.call.return_value = 1 | ||
1523 | 889 | |||
1524 | 890 | script.collect_inner_logs(self.juju) | ||
1525 | 891 | |||
1526 | 892 | self.assertEqual(script.get_units.call_count, 1) | ||
1527 | 893 | self.assertEqual(script.check_output.call_count, 2) | ||
1528 | 894 | self.assertEqual(script.call.call_count, 2) | ||
1529 | 895 | script.check_call.assert_not_called() | ||
1530 | 896 | self.assert_clean() | ||
1531 | 897 | |||
1532 | 898 | def test_check_call_failure_1(self): | ||
1533 | 899 | """ | ||
1534 | 900 | collect_inner_logs() does not handle errors when running | ||
1535 | 901 | collect-logs in the inner model. | ||
1536 | 902 | """ | ||
1537 | 903 | script.check_call.side_effect = FakeError() | ||
1538 | 904 | |||
1539 | 905 | with self.assertRaises(FakeError): | ||
1540 | 906 | script.collect_inner_logs(self.juju) | ||
1541 | 907 | |||
1542 | 908 | self.assertEqual(script.get_units.call_count, 1) | ||
1543 | 909 | self.assertEqual(script.check_output.call_count, 2) | ||
1544 | 910 | self.assertEqual(script.call.call_count, 3) | ||
1545 | 911 | self.assertEqual(script.check_call.call_count, 1) | ||
1546 | 912 | self.assert_clean() | ||
1547 | 913 | |||
1548 | 914 | def test_check_call_failure_2(self): | ||
1549 | 915 | """ | ||
1550 | 916 | collect_inner_logs() does not handle errors downloading the | ||
1551 | 917 | collected logs from the inner model. | ||
1552 | 918 | |||
1553 | 919 | It does clean up, however. | ||
1554 | 920 | """ | ||
1555 | 921 | script.check_call.side_effect = [None, | ||
1556 | 922 | FakeError(), | ||
1557 | 923 | ] | ||
1558 | 924 | |||
1559 | 925 | with self.assertRaises(FakeError): | ||
1560 | 926 | script.collect_inner_logs(self.juju) | ||
1561 | 927 | |||
1562 | 928 | self.assertEqual(script.get_units.call_count, 1) | ||
1563 | 929 | self.assertEqual(script.check_output.call_count, 2) | ||
1564 | 930 | self.assertEqual(script.call.call_count, 3) | ||
1565 | 931 | self.assertEqual(script.check_call.call_count, 2) | ||
1566 | 932 | self.assert_clean() | ||
1567 | 933 | |||
1568 | 934 | def test_check_call_failure_3(self): | ||
1569 | 935 | def check_call_side_effect(cmd, env=None): | ||
1570 | 936 | self.assertEqual(env, self.juju.env) | ||
1571 | 937 | if script.check_call.call_count == 3: | ||
1572 | 938 | raise FakeError() | ||
1573 | 939 | if script.check_call.call_count == 2: | ||
1574 | 940 | self._create_tempfile("inner-logs.tar.gz") | ||
1575 | 941 | return None | ||
1576 | 942 | script.check_call.side_effect = check_call_side_effect | ||
1577 | 943 | |||
1578 | 944 | with self.assertRaises(FakeError): | ||
1579 | 945 | script.collect_inner_logs(self.juju) | ||
1580 | 946 | |||
1581 | 947 | self.assertEqual(script.get_units.call_count, 1) | ||
1582 | 948 | self.assertEqual(script.check_output.call_count, 2) | ||
1583 | 949 | self.assertEqual(script.call.call_count, 3) | ||
1584 | 950 | self.assertEqual(script.check_call.call_count, 3) | ||
1585 | 951 | self.assert_clean() | ||
1586 | 952 | |||
1587 | 953 | |||
1588 | 954 | class BundleLogsTestCase(_BaseTestCase): | ||
1589 | 955 | |||
1590 | 956 | MOCKED = ("call",) | ||
1591 | 957 | |||
1592 | 958 | def setUp(self): | ||
1593 | 959 | """ | ||
1594 | 960 | bundle_logs() creates a tarball holding the files in the tempdir. | ||
1595 | 961 | """ | ||
1596 | 962 | super(BundleLogsTestCase, self).setUp() | ||
1597 | 963 | |||
1598 | 964 | os.chdir(self.tempdir) | ||
1599 | 965 | |||
1600 | 966 | self._create_tempfile("bootstrap/var/log/syslog") | ||
1601 | 967 | self._create_tempfile("bootstrap/var/log/juju/all-machines.log") | ||
1602 | 968 | self._create_tempfile( | ||
1603 | 969 | "bootstrap/var/lib/lxc/deadbeef/rootfs/var/log/syslog") | ||
1604 | 970 | self._create_tempfile("bootstrap/var/lib/juju/containers") | ||
1605 | 971 | self._create_tempfile("landscape-server-0/var/log/syslog") | ||
1606 | 972 | self._create_tempfile("postgresql-0/var/log/syslog") | ||
1607 | 973 | self._create_tempfile("rabbitmq-server-0/var/log/syslog") | ||
1608 | 974 | self._create_tempfile("haproxy-0/var/log/syslog") | ||
1609 | 975 | self._create_tempfile( | ||
1610 | 976 | "landscape-0-inner-logs/bootstrap/var/log/syslog") | ||
1611 | 977 | |||
1612 | 978 | self.extrafile = os.path.join(self.cwd, "spam.txt") | ||
1613 | 979 | _create_file(self.extrafile) | ||
1614 | 980 | |||
1615 | 981 | def test_success_with_extra(self): | ||
1616 | 982 | """ | ||
1617 | 983 | bundle_logs() works if extra files are included. | ||
1618 | 984 | """ | ||
1619 | 985 | tarfile = "/tmp/logs.tgz" | ||
1620 | 986 | extrafiles = [self.extrafile] | ||
1621 | 987 | |||
1622 | 988 | script.bundle_logs(self.tempdir, tarfile, extrafiles) | ||
1623 | 989 | |||
1624 | 990 | script.call.assert_called_once_with( | ||
1625 | 991 | ["tar", | ||
1626 | 992 | "czf", tarfile, | ||
1627 | 993 | "--transform", "s,{}/,,".format(self.tempdir[1:]), | ||
1628 | 994 | os.path.join(self.tempdir, "bootstrap"), | ||
1629 | 995 | os.path.join(self.tempdir, "haproxy-0"), | ||
1630 | 996 | os.path.join(self.tempdir, "landscape-0-inner-logs"), | ||
1631 | 997 | os.path.join(self.tempdir, "landscape-server-0"), | ||
1632 | 998 | os.path.join(self.tempdir, "postgresql-0"), | ||
1633 | 999 | os.path.join(self.tempdir, "rabbitmq-server-0"), | ||
1634 | 1000 | self.extrafile, | ||
1635 | 1001 | ], | ||
1636 | 1002 | ) | ||
1637 | 1003 | |||
1638 | 1004 | def test_success_without_extra(self): | ||
1639 | 1005 | """ | ||
1640 | 1006 | bundle_logs() works if there aren't any extra files. | ||
1641 | 1007 | """ | ||
1642 | 1008 | tarfile = "/tmp/logs.tgz" | ||
1643 | 1009 | |||
1644 | 1010 | script.bundle_logs(self.tempdir, tarfile) | ||
1645 | 1011 | |||
1646 | 1012 | script.call.assert_called_once_with( | ||
1647 | 1013 | ["tar", | ||
1648 | 1014 | "czf", tarfile, | ||
1649 | 1015 | "--transform", "s,{}/,,".format(self.tempdir[1:]), | ||
1650 | 1016 | os.path.join(self.tempdir, "bootstrap"), | ||
1651 | 1017 | os.path.join(self.tempdir, "haproxy-0"), | ||
1652 | 1018 | os.path.join(self.tempdir, "landscape-0-inner-logs"), | ||
1653 | 1019 | os.path.join(self.tempdir, "landscape-server-0"), | ||
1654 | 1020 | os.path.join(self.tempdir, "postgresql-0"), | ||
1655 | 1021 | os.path.join(self.tempdir, "rabbitmq-server-0"), | ||
1656 | 1022 | ], | ||
1657 | 1023 | ) | ||
1658 | 1024 | |||
1659 | 1025 | def test_success_no_files(self): | ||
1660 | 1026 | """ | ||
1661 | 1027 | bundle_logs() works even when the temp dir is empty. | ||
1662 | 1028 | """ | ||
1663 | 1029 | for filename in os.listdir(self.tempdir): | ||
1664 | 1030 | shutil.rmtree(os.path.join(self.tempdir, filename)) | ||
1665 | 1031 | tarfile = "/tmp/logs.tgz" | ||
1666 | 1032 | |||
1667 | 1033 | script.bundle_logs(self.tempdir, tarfile) | ||
1668 | 1034 | |||
1669 | 1035 | script.call.assert_called_once_with( | ||
1670 | 1036 | ["tar", | ||
1671 | 1037 | "czf", tarfile, | ||
1672 | 1038 | "--transform", "s,{}/,,".format(self.tempdir[1:]), | ||
1673 | 1039 | ], | ||
1674 | 1040 | ) | ||
1675 | 1041 | |||
1676 | 1042 | def test_call_failure(self): | ||
1677 | 1043 | """ | ||
1678 | 1044 | bundle_logs() does not handle errors when creating the tarball. | ||
1679 | 1045 | """ | ||
1680 | 1046 | script.call.side_effect = FakeError() | ||
1681 | 1047 | tarfile = "/tmp/logs.tgz" | ||
1682 | 1048 | |||
1683 | 1049 | with self.assertRaises(FakeError): | ||
1684 | 1050 | script.bundle_logs(self.tempdir, tarfile) | ||
1685 | 1051 | |||
1686 | 1052 | script.call.assert_called_once_with( | ||
1687 | 1053 | ["tar", | ||
1688 | 1054 | "czf", tarfile, | ||
1689 | 1055 | "--transform", "s,{}/,,".format(self.tempdir[1:]), | ||
1690 | 1056 | os.path.join(self.tempdir, "bootstrap"), | ||
1691 | 1057 | os.path.join(self.tempdir, "haproxy-0"), | ||
1692 | 1058 | os.path.join(self.tempdir, "landscape-0-inner-logs"), | ||
1693 | 1059 | os.path.join(self.tempdir, "landscape-server-0"), | ||
1694 | 1060 | os.path.join(self.tempdir, "postgresql-0"), | ||
1695 | 1061 | os.path.join(self.tempdir, "rabbitmq-server-0"), | ||
1696 | 1062 | ], | ||
1697 | 1063 | ) |
Command: make ci-test /ci.lscape. net/job/ latch-test- xenial/ 1306/
Result: Fail
Revno: 39
Branch: lp:~fginther/landscape-charm/remove-collect-logs
Jenkins: https:/