Merge ~pieq/bugit/+git/qabro:more-logs-in-reports into bugit:master

[Pierre Equoy]

The major change of this is made by commit e09dc48 and allows sosreport to run commands like lspci, gather the output and store it in the archive sent to Launchpad.

It was a massive headache and it's still far from being the best solution, but it's already a big improvement over current version of qabro snap.

To test:

1. Generate a snap:

    $ snapcraft clean && snapcraft

2. If using core < 2.36, activate snap layouts "experimental" feature:

    $ sudo snap set core experimental.layouts=true

(snap layouts are experimental until snapd 2.36)

3. Install qabro:

    $ sudo snap install qabro_0.7dev_amd64.snap --devmode

4. Launch qabro and use the staging Launchpad instance for testing:

    $ APPORT_LAUNCHPAD_INSTANCE=staging qabro

The generated report should include things like the correct output for pci and usb plugins (i.e. stuff in sos_commands/pci/ and sos_commands/usb/ should not be empty!)

I tested it on:

    - 18.04
    - UC16
    - UC18

[Sylvain Pineau (sylvain-pineau)]

Lots of good stuff, the patch for sosreport is an excellent workaround to get lspci data.*
and 2.36 is in stable now.

+1

review: Approve

diff --git a/config/sos.conf b/config/sos.conf
deleted file mode 100644
index d5b5377..0000000
--- a/config/sos.conf
+++ /dev/null
@@ -1,9 +0,0 @@
-[plugins]
-
-#disable = rpm, selinux, dovecot
-
-[tunables]
-
-#rpm.rpmva = off
-#general.syslogsize = 15
-
diff --git a/patches/sosreport-env-path-snap.patch b/patches/sosreport-env-path-snap.patch
new file mode 100644
index 0000000..b4f95c1
--- /dev/null
+++ b/patches/sosreport-env-path-snap.patch
@@ -0,0 +1,22 @@
+diff -ru /tmp/sos-3.5/sos/policies/ubuntu.py usr/share/sosreport/sos/policies/ubuntu.py
+--- /tmp/sos-3.5/sos/policies/ubuntu.py	        2017-11-02 20:54:47.000000000 +0800
++++ usr/share/sosreport/sos/policies/ubuntu.py	2018-10-24 15:28:46.439206314 +0800
+@@ -3,11 +3,18 @@
+ from sos.plugins import UbuntuPlugin, DebianPlugin
+ from sos.policies.debian import DebianPolicy
+ 
++import os
+ 
+ class UbuntuPolicy(DebianPolicy):
+     distro = "Ubuntu"
+     vendor = "Ubuntu"
+     vendor_url = "http://www.ubuntu.com/"
++    PATH = ("/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
++            ":/usr/local/sbin:/usr/local/bin"
++            ":$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$SNAP/usr/games"
++            ":$SNAP/usr/local/games:$SNAP/usr/local/sbin:$SNAP/usr/local/bin"
++           )
++    PATH = os.path.expandvars(PATH)
+ 
+     def __init__(self, sysroot=None):
+         super(UbuntuPolicy, self).__init__(sysroot=sysroot)
diff --git a/qabro/__init__.py b/qabro/__init__.py
index 358ed45..ace1c53 100644
--- a/qabro/__init__.py
+++ b/qabro/__init__.py
@@ -1,9 +1,12 @@
-import logging, os, subprocess, sys
-from time import sleep
+import logging
+import os
+import subprocess
+import sys
 
 from qabro.bug_assistant import BugAssistant
 from qabro.ui import ReportScreen
 
+
 def main():
     if os.geteuid() == 0:
         sys.exit("This tool cannot be run as super user.")
@@ -21,18 +24,22 @@ def main():
         sys.exit("This tool requires --devmode option to run properly.")
 
     logfile = os.path.join(os.getenv('SNAP_USER_COMMON', '/tmp'), 'qabro.log')
-    logging.basicConfig(filename=logfile,
-                        format='%(asctime)s - %(levelname)s - %(message)s',
-                        level=logging.ERROR
-                       )
+    loglevel = os.getenv('LOGLEVEL', 'ERROR')
+    logformat = ("%(asctime)s - %(levelname)s - "
+                 "%(module)s:%(funcName)s:%(lineno)d - %(message)s")
+    numeric_level = getattr(logging, loglevel.upper(), None)
+    if not isinstance(numeric_level, int):
+        raise ValueError('Invalid log level: %s' % loglevel)
+    logging.basicConfig(filename=logfile, format=logformat, level=numeric_level)
     start_ui()
 
+
 def start_ui():
     ui = ReportScreen()
     ui.run()
     ba = BugAssistant(ui.report())
     while True:
-        try: 
+        try:
             result = ba.file_bug()
         except Exception as exc:
             ui.message.set_text([('error', "Error: "), str(exc)])
diff --git a/qabro/bug_assistant.py b/qabro/bug_assistant.py
index 87cbb62..adce592 100644
--- a/qabro/bug_assistant.py
+++ b/qabro/bug_assistant.py
@@ -1,24 +1,29 @@
-import getpass, os, re, subprocess, sys, tarfile
+import getpass
+import os
+import re
+import subprocess
+import sys
+import tarfile
 from collections import Counter
-from httplib2 import ServerNotFoundError
 
-from launchpadlib.launchpad import (
-    Launchpad, EDGE_SERVICE_ROOT, STAGING_SERVICE_ROOT
-    )
-from launchpadlib.uris import LPNET_WEB_ROOT, STAGING_WEB_ROOT
+from launchpadlib.launchpad import Launchpad
+from launchpadlib.uris import LPNET_WEB_ROOT, STAGING_WEB_ROOT, QASTAGING_WEB_ROOT
+
 
 class BugReport:
     """Bug report.
-    
+
     A bug report contains all the textual information that can be transmitted
     to Launchpad, such as the bug title, description, related project, etc. It
     also contains a dictionary of options.
     """
-    def __init__(self, title, description, project, series, assignee, tags, status, importance, options):
+    def __init__(self, title, description, project, series, assignee, tags,
+                 status, importance, options):
         self.update(title, description, project, series, assignee, tags,
                     status, importance, options)
 
-    def update(self, title, description, project, series, assignee, tags, status, importance, options):
+    def update(self, title, description, project, series, assignee, tags,
+               status, importance, options):
         self.title = title
         self.description = description
         self.project = project
@@ -28,7 +33,7 @@ class BugReport:
         self.status = status
         self.importance = importance
         self.options = options
-    
+
 
 class BugAssistant:
     """Handle a bug report, its attachments and their transmission to Launchpad."""
@@ -51,13 +56,9 @@ class BugAssistant:
 
     def launchpad_login(self):
         """Log in Launchpad and create credentials file if needed."""
-        if ('APPORT_STAGING' in os.environ
-            or os.environ.get('APPORT_LAUNCHPAD_INSTANCE', None) == 'staging'):
-            print('Using staging service root')
-            service_root = STAGING_SERVICE_ROOT
-        else:
-            print('Using edge service root')
-            service_root = EDGE_SERVICE_ROOT
+        # can be 'production', 'staging' or 'qastaging'
+        service_root = os.environ.get('APPORT_LAUNCHPAD_INSTANCE', 'production')
+        print('Using {} service root'.format(service_root))
 
         directory = os.path.expanduser('~/.launchpadlib')
         if not os.path.isdir(directory):
@@ -91,7 +92,6 @@ class BugAssistant:
         try:
             print('Checking project name...')
             project = launchpad.projects[self.lp_project]
-            #logging.debug('Project found: {0}'.format(project))
         except Exception as ex:
             error_message = ('{} launchpad project not found'
                              .format(self.lp_project))
@@ -105,7 +105,6 @@ class BugAssistant:
             if series is None:
                 error_message = ('{} series not found'
                                  .format(series_name))
-                #logging.error(error_message)
                 raise BugAssistantError(error_message)
 
         assignee_name = self.lp_assignee
@@ -119,6 +118,7 @@ class BugAssistant:
                                  .format(assignee_name))
                 raise BugAssistantError(error_message)
 
+        print('Getting standard bug information...')
         std_info = AttachmentAssistant.get_standard_info()
         std_info_str = '\n'.join(['{}: {}'.format(elt, std_info[elt])
                                   for elt in std_info])
@@ -126,9 +126,9 @@ class BugAssistant:
         if not self.bug:
             print('Creating Launchpad bug report...')
             self.bug = launchpad.bugs.createBug(title=self.lp_title,
-                                           description=description,
-                                           tags=self.lp_tags.split(),
-                                           target=project)
+                                                description=description,
+                                                tags=self.lp_tags.split(),
+                                                target=project)
             print('Bug report #{} created.'.format(self.bug.id))
         else:
             print('Updating Launchpad bug report...')
@@ -158,8 +158,10 @@ class BugAssistant:
             nomination = self.bug.addNomination(target=series)
             nomination.approve()
 
-        if ('APPORT_STAGING' in os.environ
-            or os.environ.get('APPORT_LAUNCHPAD_INSTANCE', None) == 'staging'):
+        service_root = os.environ.get('APPORT_LAUNCHPAD_INSTANCE', 'production')
+        if service_root == 'qastaging':
+            bug_url = QASTAGING_WEB_ROOT+'bugs/{}'.format(self.bug.id)
+        elif service_root == 'staging':
             bug_url = STAGING_WEB_ROOT+'bugs/{}'.format(self.bug.id)
         else:
             bug_url = LPNET_WEB_ROOT+'bugs/{}'.format(self.bug.id)
@@ -171,7 +173,8 @@ class BugAssistant:
         aa.run_attachment_methods()
         aa.upload_attachments(launchpad, self.bug.id)
 
-        return 'Bug report #{} and it attachments are available at <{}>'.format(self.bug.id, bug_url)
+        result = "Bug report #{} and its attachments are available at <{}>"
+        return result.format(self.bug.id, bug_url)
 
 
 class BugAssistantError(Exception):
@@ -181,7 +184,7 @@ class BugAssistantError(Exception):
 
 class AttachmentAssistant:
     """Handle attachments and related tasks.
-    
+
     This class provides a dictionary of attachments that is populated by the
     different `attach_*` methods. These attachments can then be uploaded to
     the given launchpad bug using `upload_attachments`.
@@ -215,16 +218,16 @@ class AttachmentAssistant:
         home_dir = os.getenv('HOME')
         user = getpass.getuser()
         p = os.getenv('PATH')
-        conf_file = os.path.expandvars('$SNAP/etc/sos.conf')
+        s = os.getenv('SNAP')
         if p:
             print("Running sosreport...")
             try:
                 # Call sosreport while preserving the user $PATH when using sudo
                 # to have access to the sosreport version from the snap
-                command = ["sudo", "env", "PATH={}".format(p), "sosreport",
-                           "--no-report", "--batch", "--tmp-dir", home_dir,
-                           "--config-file", conf_file]
-                output = subprocess.check_output(command).decode(sys.stdout.encoding)
+                command = ["sudo", "env", "SNAP={}".format(s),
+                           "PATH={}".format(p), "sosreport", "--no-report",
+                           "--batch", "--tmp-dir", home_dir]
+                output = subprocess.check_output(command).decode('utf-8')
                 archive_path = output.splitlines()[-6].strip()
                 print("Report generated: {}".format(archive_path))
                 command = ["sudo", "chown", "{}:{}".format(user, user), archive_path]
@@ -259,11 +262,12 @@ class AttachmentAssistant:
         if os.path.exists(classic_path):
             rootpath = classic_path
         else:
-            out = subprocess.check_output(["snap", "list"]).decode(sys.stdout.encoding)
+            out = subprocess.check_output(["snap", "list"]).decode('utf-8')
             snap_name_list = [e for e in out.split() if 'checkbox' in e]
             if snap_name_list:
                 snap_name = snap_name_list[0]
-                snap_path = os.path.expandvars("/home/$USER/snap/{}/current/plainbox/sessions".format(snap_name))
+                sp = "/home/$USER/snap/{}/current/plainbox/sessions"
+                snap_path = os.path.expandvars(sp.format(snap_name))
                 if os.path.exists(snap_path):
                     rootpath = snap_path
 
@@ -330,7 +334,7 @@ class AttachmentAssistant:
         else:
             # Otherwise, try to use the first version found on the system
             command = ["sudo", "dmidecode", "-s"]
-        
+
         for dmi_value in ['system-manufacturer', 'system-product-name', 'bios-version']:
             standard_info[dmi_value] = subprocess.check_output(
                 command + [dmi_value]).decode(sys.stdout.encoding).strip()
@@ -365,7 +369,7 @@ class AttachmentAssistant:
         cpu_names = Counter()
         for cpu_number, cpu in enumerate(cpus):
             if 'model name' in cpu:
-                cpu_name =  ' '.join(cpu["model name"].split())
+                cpu_name = ' '.join(cpu["model name"].split())
                 cpu_names[cpu_name] += 1
             elif 'Processor' in cpu:
                 cpu_name = ' '.join(cpu["Processor"].split())
@@ -375,4 +379,3 @@ class AttachmentAssistant:
                          for cpu_name, count in cpu_names.items()]
 
         return '\n'.join(cpu_names_str)
-
diff --git a/qabro/ui.py b/qabro/ui.py
index 806f78e..3767d1b 100644
--- a/qabro/ui.py
+++ b/qabro/ui.py
@@ -1,15 +1,12 @@
-from time import sleep
 import sys
 import urwid
 
 from qabro.bug_assistant import BugReport
 from qabro.__version__ import __version__ as qversion, __title__ as qtitle
 
+
 class ReportScreen:
     palette = [
-        #('body', 'light gray', 'black'),
-        #('focus', 'black', 'light gray', 'standout'),
-        #('head', 'black', 'light gray', 'standout'),
         ('foot', 'light gray', 'black'),
         ('title', 'white', 'black', 'bold'),
         ('key', 'yellow,bold', ''),
@@ -29,38 +26,37 @@ class ReportScreen:
                    'Medium',
                    'High',
                    'Critical')
-    bug_types = { "Audio": "hwe-audio",
-                  "Bluetooth": "hwe-bluetooth",
-                  "Camera": "oem-camera",
-                  "External storage": "oem-storage",
-                  "Touchpad": "hwe-touchpad",
-                  "Touchscreen": "oem-touchscreen",
-                  "Install": "hwe-installer",
-                  "Media Card": "hwe-media",
-                  "Networking (ethernet)": "hwe-networking-ethernet",
-                  "Networking (wifi)": "hwe-networking-wifi",
-                  "Networking (modem)": "hwe-networking-modem",
-                  "Power management": "hwe-suspend-resume",
-                  "Checkbox": "checkbox test-case",
-                  "Video": "hwe-graphics",
-                  "Hotkeys": "hwe-hotkeys",
-                  "Firmware": "hwe-firmware",
-                  "Missing driver": "hwe-needs-driver",
-                  "Performance": "oem-performance",
-                  "Other problem": "oem-other"}
+    bug_types = {"Audio": "hwe-audio",
+                 "Bluetooth": "hwe-bluetooth",
+                 "Camera": "oem-camera",
+                 "External storage": "oem-storage",
+                 "Touchpad": "hwe-touchpad",
+                 "Touchscreen": "oem-touchscreen",
+                 "Install": "hwe-installer",
+                 "Media Card": "hwe-media",
+                 "Networking (ethernet)": "hwe-networking-ethernet",
+                 "Networking (wifi)": "hwe-networking-wifi",
+                 "Networking (modem)": "hwe-networking-modem",
+                 "Power management": "hwe-suspend-resume",
+                 "Checkbox": "checkbox test-case",
+                 "Video": "hwe-graphics",
+                 "Hotkeys": "hwe-hotkeys",
+                 "Firmware": "hwe-firmware",
+                 "Missing driver": "hwe-needs-driver",
+                 "Performance": "oem-performance",
+                 "Other problem": "oem-other"}
 
     default_description = ("Summary:\n\n"
                            "Steps to reproduce:\n\n"
                            "Expected result:\n\n"
                            "Actual result:\n\n"
-                           "Failure rate:"
-                          )
+                           "Failure rate:")
 
     def __init__(self):
         self._title = urwid.LineBox(urwid.Edit(), 'Bug Title')
         self._description = urwid.LineBox(urwid.Edit(edit_text=self.default_description,
-                                         multiline=True),
-                                       'Description (including steps to reproduce)')
+                                          multiline=True),
+                                          "Description (including steps to reproduce)")
         self._project = urwid.LineBox(urwid.Edit(), 'Project')
         self._series = urwid.LineBox(urwid.Edit(), 'Series')
         self._status = []
@@ -71,7 +67,8 @@ class ReportScreen:
             rb = urwid.RadioButton(self._importance, i, 'first True', None)
         self._bug_type = []
         for i in sorted(self.bug_types):
-            rb = urwid.RadioButton(self._bug_type, i, state=False, on_state_change=self.add_tag)
+            rb = urwid.RadioButton(self._bug_type, i, state=False,
+                                   on_state_change=self.add_tag)
         self._assignee = urwid.LineBox(urwid.Edit(), 'Assigned To (Launchpad ID)')
         self._tags = urwid.LineBox(urwid.Edit(), 'Tags')
         status_list = urwid.LineBox(urwid.Pile(self._status), 'Status')
@@ -80,13 +77,13 @@ class ReportScreen:
         self.message = urwid.Text('')
         self.footer = urwid.AttrMap(urwid.Text(self.footer_text), 'foot')
         self.pile = urwid.Pile([self.message,
-                                self._title, 
-                                self._description, 
-                                self._project, 
-                                self._series, 
-                                status_list, 
-                                importance_list, 
-                                self._assignee, 
+                                self._title,
+                                self._description,
+                                self._project,
+                                self._series,
+                                status_list,
+                                importance_list,
+                                self._assignee,
                                 self._tags,
                                 bug_types_list])
         self.view = urwid.Frame(urwid.Filler(self.pile, 'top'), footer=self.footer)
@@ -108,8 +105,7 @@ class ReportScreen:
         self.loop = urwid.MainLoop(self.view,
                                    self.palette,
                                    handle_mouse=False,
-                                   unhandled_input=self.unhandled_input
-                                  )
+                                   unhandled_input=self.unhandled_input)
         self.loop.run()
 
     def unhandled_input(self, key):
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
index 94d79e9..415c1e9 100644
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -22,8 +22,14 @@ apps:
     environment:
       LD_LIBRARY_PATH: $LD_LIBRARY_PATH:$SNAP/lib/fwts
 
-  acpidump:
-    command: acpidump-acpica
+  lspci:
+    command: lspci
+
+  lsusb:
+    command: lsusb
+
+  amixer:
+    command: amixer
 
 parts:
   qabro:
@@ -42,30 +48,40 @@ parts:
       - pciutils
 
   sosreport:
+    after: [patches]
     plugin: nil
     stage-packages:
       - sosreport
       - libc6
       - libdb5.3
+    override-build: |
+      snapcraftctl build
+      # Patch to add a bunch of $SNAP-related paths in the $PATH environment
+      # variable used by sosreport to call other commands.
+      patch $SNAPCRAFT_PART_INSTALL/usr/share/sosreport/sos/policies/ubuntu.py $SNAPCRAFT_STAGE/patches/sosreport-env-path-snap.patch
 
   fwts:
-    plugin: autotools
-    source: git://kernel.ubuntu.com/hwe/fwts
     source-tag: V18.06.02
+
+  lspci:
+    plugin: nil
     stage-packages:
-      - libfdt1
-    build-packages:
-      - gcc
-      - make
-      - autoconf
-      - automake
-      - libtool
-      - libjson-c-dev
-      - flex
-      - bison
-      - dh-autoreconf
-      - libglib2.0-dev
-      - libfdt-dev
+      - pciutils
+      - libc6
+      - libkmod2
+      - libpci3
+
+  lsusb:
+    plugin: nil
+    stage-packages:
+      - usbutils
+      - libc6
+
+  alsa:
+    plugin: nil
+    stage-packages:
+      - alsa-utils
+      - libc6
 
   acpidump:
     plugin: nil
@@ -73,8 +89,25 @@ parts:
       - acpica-tools
       - libc6
 
-  config:
+  dmsetup:
+    plugin: nil
+    stage-packages:
+      - dmsetup
+      - libc6
+
+  patches:
+    source: patches
     plugin: dump
-    source: config/
     organize:
-      '*.conf': etc/
+      '*': patches/
+    prime: [-*]
+
+# https://forum.snapcraft.io/t/snap-layouts/7207
+passthrough:
+  layout:
+    /etc/sos.conf:
+      bind-file: $SNAP/etc/sos.conf
+    /var/lib/usbutils/usb.ids:
+      bind-file: $SNAP/var/lib/usbutils/usb.ids
+    /usr/share/alsa:
+      bind: $SNAP/usr/share/alsa