Merge bootstack-ops:sosreport-python into bootstack-ops:master

[Joe Guo]

convert shell script to python to generate sosreport

ref: https://wiki.canonical.com/CDO/IS/Bootstack/Playbooks/SOS-Report-and-STS-SF-file-attachments

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

Unmerged commits

2d16501... by Joe Guo on 2020-04-21

sosreport.py: generate sosreport with python

convert shell script to python to generate sosreport
ref: https://wiki.canonical.com/CDO/IS/Bootstack/Playbooks/SOS-Report-and-STS-SF-file-attachments

Signed-off-by: Joe Guo <email address hidden>

4ff45ec... by Felipe Reyes on 2020-01-07

Use 'source' in ceph-radosgw when upgrading

Reviewed-on: https://code.launchpad.net/~freyes/bootstack-ops/+git/bootstack-ops/+merge/376537
Reviewed-by: Alvaro Uria <email address hidden>

c19d158... by Felipe Reyes on 2019-12-09

Use 'source' in ceph-radosgw when upgrading

dbec91e... by Peter Sabaini on 2019-10-28

Try harder to bring up ifaces when link checking

Reviewed-on: https://code.launchpad.net/~peter-sabaini/bootstack-ops/+git/bootstack-ops-1/+merge/374784
Reviewed-by: Ryan Farrell <email address hidden>

0699031... by Peter Sabaini on 2019-10-28

Try harder to bring up ifaces when link checking

4727c41... by Peter Sabaini on 2019-10-28

Add Mellanox network link status

Reviewed-on: https://code.launchpad.net/~peter-sabaini/bootstack-ops/+git/bootstack-ops-1/+merge/368552
Reviewed-by: Alvaro Uria <email address hidden>
Reviewed-by: Joel Sing <email address hidden>

bd21389... by Peter Sabaini on 2019-10-28

Address review comments

0b4851a... by Andrea Ieri on 2019-10-16

Changes to Cisco hardware settings:
* clear ipmi encryption key
* disable SD card
* disable strong password policy

4c31caf... by Andrea Ieri on 2019-09-06

Stage pacemaker to allow crm_attribute calls. Bump version.

Reviewed-on: https://code.launchpad.net/~aieri/bootstack-ops/+git/bootstack-ops/+merge/372419
Reviewed-by: David O Neill <email address hidden>

2b6323c... by Andrea Ieri on 2019-09-06

Stage pacemaker to allow crm_attribute calls. Bump version.

diff --git a/bootstack-ops/sosreport.py b/bootstack-ops/sosreport.py
new file mode 100755
index 0000000..9ff82f2
--- /dev/null
+++ b/bootstack-ops/sosreport.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import os
+import shutil
+import subprocess
+
+TIMEOUT = 1800
+
+
+parser = argparse.ArgumentParser(
+    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    description='genererate sosreport')
+
+# make target exclusive, to simplify logic
+group = parser.add_mutually_exclusive_group(required=True)
+group.add_argument('-a', '--app', metavar='APP', dest='apps', help='comma separated application names')
+group.add_argument('-u', '--unit', metavar='UNIT', dest='units', help='comma separated unit names')
+group.add_argument('--machine', metavar='MACHINE', dest='machines', help='comma separated machine ids')
+parser.add_argument('case', metavar='CASE', help='salesforce case number, e.g.: 00272654')
+
+cli = parser.parse_args()
+CASE = cli.case
+
+
+if cli.apps:
+    JUJU_RUN_TARGETS = ['--app', cli.apps]
+    # expand apps to units for juju scp
+    juju_status_json = subprocess.check_output(['juju', 'status', '--format', 'json']).decode('utf8')
+    juju_status_dict = json.loads(juju_status_json)
+    juju_apps = juju_status_dict['applications']
+    JUJU_SCP_TARGETS = set([])
+    for app in cli.apps.split(','):
+        for unit in juju_apps[app]['units']:
+            JUJU_SCP_TARGETS.add(unit)
+elif cli.units:
+    JUJU_RUN_TARGETS = ['--unit', cli.units]
+    JUJU_SCP_TARGETS = set(cli.units.split(','))
+elif cli.machines:
+    JUJU_RUN_TARGETS = ['--machine', cli.machines]
+    JUJU_SCP_TARGETS = set(cli.machines.split(','))
+
+
+def juju_run(cmd):
+    # default itmeout is 5m, which is not enough for some case
+    fullcmd = [
+        'juju', 'run', '--show-log', '--timeout', '{}s'.format(TIMEOUT)
+    ] + JUJU_RUN_TARGETS + ['--', cmd]
+    print(' '.join(fullcmd))
+    subprocess.check_call(fullcmd)
+
+
+def juju_scp(target):
+    fullcmd = [
+        'juju', 'scp', '--no-host-key-checks', '--', '-r',
+        '{target}:{CASE_DIR}/*'.format(target=target, CASE_DIR=CASE_DIR),
+        CASE_DIR,
+    ]
+    print(' '.join(fullcmd))
+    subprocess.check_call(fullcmd)
+
+
+# use same dir path for both infra node and units
+CASE_DIR = '/tmp/sosreport'
+# rm and re-create the dir
+print('deleting local dir {} '.format(CASE_DIR))
+shutil.rmtree(CASE_DIR, ignore_errors=True)
+os.makedirs(CASE_DIR, exist_ok=False)
+
+cmds = [
+    'sudo rm -rf {CASE_DIR}'.format(CASE_DIR=CASE_DIR),
+    'mkdir -p {CASE_DIR}'.format(CASE_DIR=CASE_DIR),
+    'sudo apt-get install --yes sosreport',
+    'sudo ionice -c 3 sosreport --alloptions --all-logs --batch --case-id={CASE} --tmp-dir {CASE_DIR}'.format(CASE=CASE, CASE_DIR=CASE_DIR),
+    'sudo chown -R ubuntu {CASE_DIR}'.format(CASE_DIR=CASE_DIR),
+]
+
+# Do not use f string, which requires python 3.6+, but we still have 3.4 for many clouds
+for cmd in cmds:
+    juju_run(cmd)
+
+
+for target in JUJU_SCP_TARGETS:
+    juju_scp(target)
+
+
+print('removing reports ...')
+juju_run('rm --verbose -r {}'.format(CASE_DIR))
+
+# make sosreports readable to all users
+# in case you need to scp it back to your local machine with your own user
+subprocess.check_call(['chmod', '-R', 'a+r', CASE_DIR])
+print('sosreports are generated successfully and collected into {}:'.format(CASE_DIR))
+subprocess.check_call(['ls', '-alh', CASE_DIR])
+
+
+UPLOAD_TARGET = 'files.support.canonical.com'
+print('checking {UPLOAD_TARGET} ssh'.format(UPLOAD_TARGET=UPLOAD_TARGET))
+returncode = subprocess.call(['nc', '-w3', '-zv', UPLOAD_TARGET, '22'])
+can_upload = (returncode == 0)
+
+UPLOAD_CMD = ('scp '
+              '-o "StrictHostKeyChecking=no" '
+              '-o "PasswordAuthentication=yes" '
+              '-o "ConnectTimeout={TIMEOUT}" '
+              '{CASE_DIR}/* '
+              'ubuntu@{UPLOAD_TARGET}:').format(
+                  UPLOAD_TARGET=UPLOAD_TARGET,
+                  CASE_DIR=CASE_DIR,
+                  TIMEOUT=TIMEOUT)
+
+HELP_YES = """
+Please check files in {CASE_DIR}. If they are ok, upload them with:
+
+    {UPLOAD_CMD}
+
+""".format(CASE_DIR=CASE_DIR, UPLOAD_CMD=UPLOAD_CMD)
+
+HELP_NO = """
+It seems this node has no access to {UPLOAD_TARGET}, run following cmds on your *local machine* to download files and upload from there instead:
+
+    rm -rf {CASE_DIR} && mkdir -p {CASE_DIR}
+    scp -r maas.XXXX.bootstack:{CASE_DIR}/* {CASE_DIR}
+    {UPLOAD_CMD}
+
+""".format(CASE_DIR=CASE_DIR, UPLOAD_TARGET=UPLOAD_TARGET, UPLOAD_CMD=UPLOAD_CMD)
+
+if can_upload:
+    print(HELP_YES)
+else:
+    print(HELP_NO)