Merge ~mdeslaur/ubuntu-cve-tracker:check-cves-perf1 into ubuntu-cve-tracker:master

[Marc Deslauriers]

commit cb191ce1d077e41c34d4c27766e07bf97a54dc0a
Author: Marc Deslauriers <email address hidden>
Date: Tue Mar 12 15:02:34 2024 -0400

    check-cves: improve performance with sets instead of lists

    CVEIgnoreNotForUsList and CVEIgnoreMistriagedList were gigantic
    lists of CVEs and check-cves keeps searching for cves in them.
    Switching them to sets dramatically improves performance.

commit bcc2a35fad29a1dd31900e70ea4f44f38fd20f31
Author: Marc Deslauriers <email address hidden>
Date: Tue Mar 12 12:55:12 2024 -0400

    check-cves: simplify mistriaged() logic

    We don't need to call source_map.madison here and perform expensive
    searches, we just need to know if we can see this package in the
    source map. Bail out as soon as we do.

This merge proposal dramatically improves check-cves performance:

BEFORE:
$ time ./scripts/check-cves --import-missing-debian

real 3m17.930s
user 3m17.660s
sys 0m0.274s

AFTER:
$ time ./scripts/check-cves --import-missing-debian

real 0m5.845s
user 0m5.571s
sys 0m0.292s

[Steve Beattie (sbeattie)]

Awesome catch, thanks! Merged.

review: Approve

diff --git a/scripts/check-cves b/scripts/check-cves
index 8a9e8fc..2abf4cc 100755
--- a/scripts/check-cves
+++ b/scripts/check-cves
@@ -223,22 +223,24 @@ def import_debian(handler):
     today = datetime_date.today()
     known = set(CVEKnownList + CVEIgnoreList)
 
+    def ever_existed(pkg):
+        for rel in source:
+            if pkg in source[rel]:
+                return True
+        return False
+
     def mistriaged(cve):
-        if cve in CVEIgnoreNotForUsList and \
-           cve not in CVEIgnoreMistriagedList and \
+        if cve in CVEIgnoreNotForUsSet and \
+           cve not in CVEIgnoreMistriagedSet and \
            handler.debian[cve]['state'] == 'FOUND':
             # check that at least one of the assigned packages exist
             # in Ubuntu
-            pkgs = {}
             for pkg in handler.debian[cve]['pkgs'].keys():
                 # ignore package if debian says is not affected
                 if handler.debian[cve]['pkgs'][pkg]['state'] == '<not-affected>':
                     continue
-                answer = source_map.madison(source, pkg)
-                if len(answer) > 0:
-                    pkgs[pkg] = answer
-            if len(pkgs) > 0:
-                return True
+                if ever_existed(pkg):
+                    return True
         return False
 
     # pull in CVEs from data/DSA/list
@@ -1347,7 +1349,7 @@ class CVEHandler(xml.sax.handler.ContentHandler):
     def add_cve(self, cve, packages, priority=None):
         # remove from not-for-us.txt if adding and ensure we remove any
         # mistriaged_hint from the description
-        if cve in CVEIgnoreNotForUsList:
+        if cve in CVEIgnoreNotForUsSet:
             cmd = ['sed', '-i', '/^%s #.*$/d' % cve, './ignored/not-for-us.txt']
             subprocess.call(cmd)
             self.cve_data[cve]['desc'] = self.cve_data[cve]['desc'].replace(mistriaged_hint, '')
@@ -1469,7 +1471,7 @@ class CVEHandler(xml.sax.handler.ContentHandler):
     def ignore_cve(self, cve, reason):
         # Append to ignore list unless is already in CVEIgnoreList and then
         # append to the ignored/ignore-mistriaged.txt
-        txtfile = 'ignore-mistriaged.txt' if cve in CVEIgnoreNotForUsList else 'not-for-us.txt'
+        txtfile = 'ignore-mistriaged.txt' if cve in CVEIgnoreNotForUsSet else 'not-for-us.txt'
         with open('%s/ignored/%s' % (destdir, txtfile), 'a') as f:
             f.write('%s # %s\n' % (cve, reason))
 
@@ -1588,21 +1590,21 @@ class CheckCVETest(unittest.TestCase):
 ignored_notforus_path = 'ignored/not-for-us.txt'
 if destdir != './' and destdir != '.':
     ignored_notforus_path = os.path.join(destdir, ignored_notforus_path)
-# CVEIgnoreNotForUsList is a list of all CVEs that we have previously
+# CVEIgnoreNotForUsSet is a set of all CVEs that we have previously
 # chosen to ignore since they don't apply to software in Ubuntu
-CVEIgnoreNotForUsList = cve_lib.parse_CVEs_from_uri(ignored_notforus_path)
+CVEIgnoreNotForUsSet = set(cve_lib.parse_CVEs_from_uri(ignored_notforus_path))
 
 ignored_mistriaged_path = 'ignored/ignore-mistriaged.txt'
 if destdir != './' and destdir != '.':
     ignored_mistriaged_path = os.path.join(destdir, ignored_mistriaged_path)
-# CVEIgnoreMistriagedList is a list of all CVEs that we want to definitely
+# CVEIgnoreMistriagedSet is a set of all CVEs that we want to definitely
 # ignore when doing mistriaged CVE detection - they should exist in both
 # CVEIgnoreNotForUsList and CVEIgnoreMistriagedList
-CVEIgnoreMistriagedList = cve_lib.parse_CVEs_from_uri(ignored_mistriaged_path)
+CVEIgnoreMistriagedSet = set(cve_lib.parse_CVEs_from_uri(ignored_mistriaged_path))
 
 # CVEIgnoreList is a list of all CVEs we know about already.  These will be
 # ignored when checking MITRE for new CVEs
-CVEIgnoreList = list(CVEIgnoreNotForUsList)
+CVEIgnoreList = list(CVEIgnoreNotForUsSet)
 
 CVEKnownList = []
 CVEKnownList += [cve for cve in os.listdir(destdir + "/ignored/") if cve.startswith('CVE-')]