Merge ~fginther/+git/ksc-test-results:fginther/hints into ~canonical-kernel/+git/ksc-test-results:main

[Francis Ginther]

Add support for applying hints and identifying provisioning failures

Add support for applying hints to test failures from a separate hints repo. Includes the ability to refresh the hints and the full reports when the hints have been updated.

Also identifies failures due to known provisioning issues.

Signed-off-by: Francis Ginther <email address hidden>

These will require the hint repo to be in place which will contain the 'compile-hints' script.

diff --git a/check-hint-age b/check-hint-age
new file mode 100755
index 0000000..e1937cf
--- /dev/null
+++ b/check-hint-age
@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+#
+
+import datetime
+import os
+
+from argparse                           import ArgumentParser, RawDescriptionHelpFormatter
+
+
+def main(raw_hints_dir, age):
+    min_age = int(datetime.datetime.now().timestamp()) - (age * 60)
+    print(min_age)
+    for raw_hints in os.listdir(raw_hints_dir):
+        if not os.path.isfile(os.path.join(raw_hints_dir, raw_hints)):
+            continue
+        mtime = int(os.path.getmtime(os.path.join(raw_hints_dir, raw_hints)))
+        if mtime > min_age:
+            return 1
+    return 0
+
+
+if __name__ == '__main__':
+    retval = -1
+
+    # Command line argument setup and initial processing
+    #
+    app_description = '''
+    Returns non-zero if any files in the requested dir are newer than the specified age.
+    '''
+    app_epilog = '''
+    '''
+    parser = ArgumentParser(description=app_description, epilog=app_epilog, formatter_class=RawDescriptionHelpFormatter)
+    parser.add_argument('raw_hints_dir', help='Directory containing the raw hint files')
+    parser.add_argument('age', help='Age to check (in minutes)')
+
+    args = parser.parse_args()
+
+    retval = 0
+    try:
+        retval = main(args.raw_hints_dir, int(args.age))
+
+    except KeyboardInterrupt:
+        pass
+
+    exit(retval)
+
+# VI:SEt ts=4 sw=4 expandtab syntax=python:
diff --git a/combine-results b/combine-results
index 8fafdbc..1852917 100755
--- a/combine-results
+++ b/combine-results
@@ -122,6 +122,12 @@ class Collector():
                 if rsuite['failed'] > 0:
                     status = 'failed'
                     break
+                if rsuite.get('hinted', 0) > 0:
+                    status = 'hinted'
+                    break
+                if rsuite.get('noprov', 0) > 0:
+                    status = 'noprov'
+                    break
             if status == 'passed':
                 o += CS(f'{status:20s}  ', CF('yellow'))
             else:
@@ -130,8 +136,9 @@ class Collector():
             isd = sd['tests']['ops'][op]['flavours'][flavour]['clouds'][cloud]['suites'][suite]['instance-types'][instance_type]
             if 'status' in isd:
                 if isd['status'] != 'failed':
-                    isd['status'] = status
-                    isd['job-name'] = rmeta['job-name']
+                    if isd['status'] != 'hinted':
+                        isd['status'] = status
+                        isd['job-name'] = rmeta['job-name']
             else:
                 isd['status'] = status
                 isd['job-name'] = rmeta['job-name']
diff --git a/compile-individual-job-results b/compile-individual-job-results
index a7a8dee..612bdbe 100755
--- a/compile-individual-job-results
+++ b/compile-individual-job-results
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 #
 
+import re
 import sys
 import os
 import xmltodict
@@ -31,6 +32,26 @@ class NoJunitResultError(Exception):
 class NoConfigXMLError(Exception):
     pass
 
+class HintDB():
+    '''
+    Represents the set of hints to check and apply.
+    '''
+    def __init__(self, codename, kernel_package, hints_root):
+        # Load the nearest hint file which matches this specific series and kernel.
+        # If one doesn't exist, look for a generic fall back.
+        # If none is found, use an empty dictionary so that no hints are applied.
+        self._data = {}
+        for filename in [f'{codename}-{kernel_package}-hints.json', 'hints.json']:
+            hint_path = os.path.join(hints_root, filename)
+            if os.path.exists(hint_path):
+                with open(hint_path, 'r') as hint_file:
+                    self._data = json.loads(hint_file.read())
+                break
+
+    def get_hints(self):
+        return self._data
+
+
 class JobResults():
     '''
     Take the junitResult.xml and job description from a jenkins job and combine them
@@ -38,14 +59,16 @@ class JobResults():
 
     This is a collection which can be iterated over like a dicionary.
     '''
-    def __init__(self, job_root):
+    def __init__(self, job_root, hint_root):
         self._data = {}
         self._description = None
+        self.provisioning_issues = ['Provisioning', 'kernel-results.xml']
 
         self._data = {
-            'results' : self.condensed_results(self.junit_results(job_root)),
-            'meta'    : self.description(job_root),
+            'meta'    : self.description(job_root)
         }
+        self._hints = HintDB(self.series_codename, self.kernel_package, hint_root).get_hints()
+        self._data['results'] = self.condensed_results(self.junit_results(job_root))
 
     def __iter__(self):
         return iter(self._data)
@@ -116,45 +139,136 @@ class JobResults():
 
         return retval
 
+    @property
+    def series_codename(self):
+        return self._data['meta']['series-codename']
+
+    @property
+    def kernel_package(self):
+        return self._data['meta']['kernel-package']
+
+    @property
+    def kernel_version(self):
+        return self._data['meta']['kernel-version']
+
+    @property
+    def kernel_flavour(self):
+        return self._data['meta']['kernel-flavour']
+
+    @property
+    def kernel_arch(self):
+        return self._data['meta']['arch']
+
+    @property
+    def cloud(self):
+        return self._data['meta']['cloud']
+
+    @property
+    def instance_type(self):
+        return self._data['meta']['instance-type']
+
+    def provisioning_failed(self, suite, case):
+        if suite['name'] in self.provisioning_issues:
+            return True
+        return False
+
+    def hinted(self, suite, case):
+        # Select the hints that match this suite's name. This needs to be an exact match.
+        suite_hints = self._hints.get(suite['name'], [])
+        for hint in suite_hints:
+            if not re.search(hint['series'], self.series_codename):
+                continue
+            if not re.search(hint['source'], self.kernel_package):
+                continue
+            if not re.search(hint['version'], self.kernel_version):
+                continue
+            if not re.search(hint['flavour'], self.kernel_flavour):
+                continue
+            if not re.search(hint['arch'], self.kernel_arch):
+                continue
+            if not re.search(hint['cloud'], self.cloud):
+                continue
+            if not re.search(hint['instance-type'], self.instance_type):
+                continue
+            if not re.search(hint['test-case'], case['testName']):
+                continue
+            return True
+        return False
+
     def _suite_results(self, _suite):
         suite = {
             'cases' : []
         }
         suite['name'] = _suite['name'].replace('Autotest - ', '')
+        debug = False
         suite['duration'] = _suite['duration']
         total_failed = 0
         total_skipped = 0
+        total_hinted = 0
+        total_noprov = 0
         cl = _suite['cases']['case']
         if type(cl) is list:
             for case in _suite['cases']['case']:
                 failed  = True if case['failedSince'] != '0'     else False
                 skipped = True if case['skipped']     == 'true'  else False
+                hinted  = False
+                noprov  = False
+                if failed and self.provisioning_failed(suite, case):
+                    # Only check for failed provisioning if the test is failed
+                    # If it did fail to provision, there is no need to check the hints.
+                    noprov = True
+                    failed = False
+                if failed and self.hinted(suite, case):
+                    # Only hint if the test case is otherwise failed
+                    hinted = True
+                    failed = False
                 case = {
                     'test-name' : case['testName'],
                     'failed'    : failed,
+                    'noprov'    : noprov,
                     'skipped'   : skipped,
+                    'hinted'    : hinted,
                     'duration'  : case['duration'],
                 }
                 if failed: total_failed += 1
+                if noprov: total_noprov += 1
                 if skipped: total_skipped += 1
+                if hinted: total_hinted += 1
                 suite['cases'].append(case)
 
         else:
             case    = _suite['cases']['case']
             failed  = True if case['failedSince'] != '0'     else False
             skipped = True if case['skipped']     == 'true'  else False
+            hinted  = False
+            noprov  = False
+            if failed and self.provisioning_failed(suite, case):
+                # Only check for failed provisioning if the test is failed
+                # If it did fail to provision, there is no need to check for hints
+                noprov = True
+                failed = False
+            if failed and self.hinted(suite, case):
+                # Only hint if the test case is otherwise failed
+                hinted = True
+                failed = False
             case = {
                 'test-name' : case['testName'],
                 'failed'    : failed,
+                'noprov'    : noprov,
                 'skipped'   : skipped,
+                'hinted'    : hinted,
                 'duration'  : case['duration'],
             }
             if failed: total_failed += 1
+            if noprov: total_noprov += 1
             if skipped: total_skipped += 1
+            if hinted: total_hinted += 1
             suite['cases'].append(case)
         suite['failed']  = total_failed
+        suite['noprov']  = total_noprov
         suite['skipped'] = total_skipped
-        suite['passed']  = len(suite['cases']) - total_failed
+        suite['hinted']  = total_hinted
+        suite['passed']  = len(suite['cases']) - (total_failed + total_hinted + total_noprov)
         return suite
 
     def condensed_results(self, results):
@@ -201,9 +315,9 @@ class JobResultsSummary():
         # Go through the individual tests results and if any of them have
         # failures then the summary for this kernel is a fail.
         #
-        summary_status = 'failed' if self._failures(results) else 'passed'
+        #summary_status = 'failed' if self._failures(results) else 'passed'
 
-        self._data['summary-status'] = summary_status
+        self._data['summary-status'] = self._status(results)
 
         for key in keepers:
             self._data[key] = results['meta'][key]
@@ -224,6 +338,18 @@ class JobResultsSummary():
                     return True
         return False
 
+    def _status(self, results):
+        status = 'passed'
+        for suite in results['results']['suites']:
+            for case in suite['cases']:
+                if case['failed']:
+                    return 'failed'
+                if case.get('hinted', 0):
+                    status = 'hinted'
+                if case.get('noprov', 0):
+                    status = 'noprov'
+        return status
+
     def get(self, *args, **kwargs):
         return self._data.get(*args, **kwargs)
 
@@ -246,12 +372,13 @@ if __name__ == '__main__':
     '''
     parser = ArgumentParser(description=app_description, epilog=app_epilog, formatter_class=RawDescriptionHelpFormatter)
     parser.add_argument('job', help='')
+    parser.add_argument('hintroot', help='')
     parser.add_argument('destroot', help='')
 
     args = parser.parse_args()
 
     try:
-        job_results = JobResults(args.job)
+        job_results = JobResults(args.job, args.hintroot)
         # pre_json(job_results._data)
 
         dest = dest_path(args.destroot, job_results['meta'])
diff --git a/do-it-all b/do-it-all
index 7374080..c25e9a2 100755
--- a/do-it-all
+++ b/do-it-all
@@ -14,8 +14,18 @@ $KT/import-raw-initiator-job-results initiator-raw imported
 find web -name expectations.json | xargs rm
 $KT/compile-initiator-job-results imported web
 
-for job in `$KT/find-imported-job-results imported`; do
-    $KT/compile-individual-job-results $job web
+if [ -d "hints/raw/.git" ]; then
+    git -C hints/raw/ pull --ff-only
+fi
+ALL=""
+if ! $KT/check-hint-age hints/raw/ 30; then
+    # Hints have updated since last report cycle, refresh all
+    ALL="--all"
+    /testresults/hints/raw/compile-hints hints/raw hints/compiled
+fi
+
+for job in `$KT/find-imported-job-results $ALL imported`; do
+    $KT/compile-individual-job-results $job hints/compiled web
 done
 
 PKGS=`find web -maxdepth 4 -mindepth 4 -type d`
@@ -44,7 +54,7 @@ for pkg in $PKGS; do
     $KSC/rtr-lvl2 $pkg
 done
 
-$KSC/rtr-log .
+$KSC/rtr-log $ALL .
 
 timeout 15m $KSC/adt-cache
 timeout 15m $KSC/refresh-cache
diff --git a/find-imported-job-results b/find-imported-job-results
index 9c4795d..327bb2e 100755
--- a/find-imported-job-results
+++ b/find-imported-job-results
@@ -7,13 +7,13 @@ import sys
 def pre(*args, **kwargs):
     print(*args, file=sys.stderr, **kwargs)
 
-def find_jenkins_job_results(root):
+def find_jenkins_job_results(root, process_all):
     '''
     Recursively descend a directory tree looking for jenkins job results. These are
     marked by having a 'junitResult.xml' file in their root.
     '''
     dlist = os.listdir(root)
-    if 'junitResult.xml' in dlist and '.processed' not in dlist:
+    if 'junitResult.xml' in dlist and ('.processed' not in dlist or process_all):
         # this looks like a winner
         #
         yield root
@@ -21,7 +21,7 @@ def find_jenkins_job_results(root):
         for fid in dlist:
             p = f'{root}/{fid}'
             if os.path.isdir(p):
-                yield from find_jenkins_job_results(p)
+                yield from find_jenkins_job_results(p, process_all)
 
 
 if __name__ == '__main__':
@@ -35,5 +35,5 @@ if __name__ == '__main__':
 
     args = parser.parse_args()
 
-    for jpath in find_jenkins_job_results(args.srcroot):
+    for jpath in find_jenkins_job_results(args.srcroot, args.all):
         print(jpath)
diff --git a/summarize-results b/summarize-results
index 4d6235e..943551f 100755
--- a/summarize-results
+++ b/summarize-results
@@ -37,6 +37,8 @@ class Summ():
         self.__sum = None
 
     def cloud_summary_status(self, cs):
+        hinted = False
+        noprov = False
         incomplete = False
         for suite in cs['suites']:
             for itype in cs['suites'][suite]['instance-types']:
@@ -44,13 +46,22 @@ class Summ():
                 if status == 'failed':
                     return 'failed'
 
+                if status == 'hinted':
+                    hinted = True
+
+                if status == 'noprov':
+                    noprov = True
+
                 if status == 'incomplete':
                     incomplete = True
 
+        if hinted:
+            return 'hinted'
+        if noprov:
+            return 'noprov'
         if incomplete:
             return 'incomplete'
-        else:
-            return 'passed'
+        return 'passed'
 
     def summarize(self):
         pre(CS(f'Summarizing Test Results ({self.root})', CF('green')))