Merge ubuntu-cve-tracker:check-cves-argparse into ubuntu-cve-tracker:master

[Mark Esler]

Deprecate optparse for argparse.

optparse is dead, long live argparse

> Deprecated since version 3.2: The optparse module is deprecated and will not be developed further; development will continue with the argparse module. [0]

Apologize for not using my fork.

[0] https://docs.python.org/3/library/optparse.html

[Mark Esler (eslerm)]

argparse adds features, such as `choices`, I plan to use later

[Marc Deslauriers (mdeslaur)]

LGTM, ack, thanks!

review: Approve

[Mark Esler (eslerm)]

Thanks Marc :)

[Steve Beattie (sbeattie)]

On Fri, Mar 15, 2024 at 05:42:17AM -0000, Mark Esler wrote:
> argparse adds features, such as `choices`, I plan to use later

Note that this branch made a subtle change to the script's behavior;
when check-cves --import-missing-debian is invoked, it is invoked
without a uri(s) argument; this branch or the followup commit
59653e29fcc (check-cves: allow argparse to use default uri if unset,
2024-03-15) causes the mitre-allitems to be loaded by check-cves.

Now, in an ideal world, this would actually happen and the description
from the mitre index would be used if its available when presenting
CVEs debian's tracker has but UCT does not yet, because debian's
description is significantly truncated. But that's not how check-cves
is currently implemented.

--
Steve Beattie
<email address hidden>

diff --git a/scripts/check-cves b/scripts/check-cves
index a4541ee..2e4ac9b 100755
--- a/scripts/check-cves
+++ b/scripts/check-cves
@@ -18,7 +18,7 @@
 from datetime import datetime, timezone, date as datetime_date
 import json
 import math
-import optparse
+import argparse
 import os
 import os.path
 import random
@@ -44,24 +44,28 @@ from uct.config import read_uct_config
 # load settings, if any
 uct_config = read_uct_config()
 
-parser = optparse.OptionParser()
-parser.add_option("-r", "--report", help="Just report CVEs that need checking", action="store_true")
-parser.add_option("-v", "--verbose", help="Report verbose XML details", action="store_true")
-parser.add_option("-k", "--known", help="Only report CVEs already known", action="store_true")
-parser.add_option("-N", "--skip-nfu", help="Skip any CVEs marked as NFU (used with -k)", action="store_true")
-parser.add_option("-R", "--refresh", help="Refresh CVE descriptions", action="store_true")
-parser.add_option("-S", "--score-refresh", help="Refresh CVSS scores values only", action="store_true")
-parser.add_option("", "--test", help="NO LONGER SUPPORTED, see test_uct_suggestions.py", action="help")
-parser.add_option("--untriaged", help="Process untriaged CVEs from output of locate_cves.py", metavar="FILE")
-parser.add_option("--mbox", help="Process untriaged CVEs from mbox file", metavar="FILE")
-parser.add_option("--rhel8oval", help="Process untriaged RHEL8 CVEs", metavar="FILE")
-parser.add_option("--import-missing-debian", help="Process missing Debian CVEs", action="store_true")
-parser.add_option("--debug", help="Report debugging information", action="store_true")
-parser.add_option("--cve", help="Check only the listed comma-separated CVEs and ignore others", action="store")
-parser.add_option("--mistriaged", help="Process the specified number of possible mistriaged CVEs compared to Debian\n"
-                  "Implies --import-missing-debian",
-                  action="store", type=int, default=0)
-(opt, args) = parser.parse_args()
+# fmt: off
+parser = argparse.ArgumentParser(prog="check_cves", description="check_cves builds UCT with new CVE data")
+parser.add_argument('uris', nargs='+', default=["https://cve.mitre.org/cve/downloads/allitems.xml",])
+parser.add_argument("-r", "--report", help="Just report CVEs that need checking", action="store_true")
+parser.add_argument("-v", "--verbose", help="Report verbose details", action="store_true")
+parser.add_argument("-d", "--debug", help="Report debugging information", action="store_true")
+parser.add_argument("-k", "--known", help="Only report CVEs already known", action="store_true")
+parser.add_argument("-N", "--skip-nfu", help="Skip any CVEs marked as NFU (used with -k)", action="store_true")
+parser.add_argument("-R", "--refresh", help="Refresh CVE descriptions", action="store_true")
+parser.add_argument("-S", "--score-refresh", help="Refresh CVSS scores values only", action="store_true")
+# TODO: implement --refresh choices
+# parser.add_argument("-R", "--refresh", choices=["all", "cvss", "description", "public_date", "urls"], help="Refresh CVE data")
+parser.add_argument("--untriaged", help="Process untriaged CVEs from output of locate_cves.py", metavar="FILE")
+parser.add_argument("--mbox", help="Process untriaged CVEs from mbox file", metavar="FILE")
+parser.add_argument("--import-missing-debian", help="Process missing Debian CVEs", action="store_true")
+# TODO: deprecate --rhel8oval, upstream data stream has ended
+parser.add_argument("--rhel8oval", help="Process untriaged RHEL8 CVEs", metavar="FILE")
+parser.add_argument("--cve", help="Check only the listed comma-separated CVEs and ignore others", action="store")
+parser.add_argument("-e", "--experimental", help="Enable experimental mode", action="store_true")
+parser.add_argument("--mistriaged", help="Process the specified number of possible mistriaged CVEs compared to Debian\nImplies --import-missing-debian", action="store", type=int, default=0)
+args = parser.parse_args()
+# fmt: on
 
 experimental = os.getenv('CHECK_CVES_EXPERIMENTAL', False)
 breakfix = os.getenv('CHECK_CVES_BREAKFIX', False)
@@ -132,7 +136,7 @@ def _spawn_editor(path):
 
 def debug(msg):
     global opt
-    if opt.debug:
+    if args.debug:
         print(msg, file=sys.stderr)
 
 
@@ -245,11 +249,11 @@ def import_debian(handler):
         return False
 
     # pull in CVEs from data/DSA/list
-    dsas = cve_lib.load_debian_dsas(uct_config['secure_testing_path'] + '/data/DSA/list', opt.verbose)
+    dsas = cve_lib.load_debian_dsas(uct_config['secure_testing_path'] + '/data/DSA/list', args.verbose)
     for dsa in dsas:
         for cve in dsas[dsa]['cves']:
             if not cve_lib.CVE_RE.match(cve):
-                if opt.verbose:
+                if args.verbose:
                     print("Skipping %s, not well-formed?" % cve, file=sys.stderr)
                 continue
 
@@ -270,27 +274,27 @@ def import_debian(handler):
             cves[cve]['subject'] = escape(dsas[dsa]['desc'])
             cves[cve]['date'] = dsas[dsa]['date']
 
-            if opt.verbose:
+            if args.verbose:
                 print("Processing %s: %s (%s)" % (dsa, dsas[dsa]['desc'], cves[cve]['date']), file=sys.stderr)
 
     # Now pull in CVEs from the data/CVE/list
     for cve in handler.debian:
-        if opt.verbose:
+        if args.verbose:
             print("[--- Processing %s ---]" % cve, file=sys.stderr)
 
         if cve in cves:
-            if opt.verbose:
+            if args.verbose:
                 print("Skipping %s, already found in DSA" % cve, file=sys.stderr)
             continue
 
         if not cve_lib.CVE_RE.match(cve):
-            if opt.verbose:
+            if args.verbose:
                 print("Skipping %s, not well-formed?" % cve, file=sys.stderr)
             continue
 
         year = int(re.split('-', cve)[1])
         if year < cve_limit:
-            if opt.verbose:
+            if args.verbose:
                 print("Skipping %s, year %d predates %d" % (cve, year, cve_limit), file=sys.stderr)
             continue
 
@@ -300,7 +304,7 @@ def import_debian(handler):
                 # add a note about how this was originally classified
                 handler.debian[cve]['desc'] = mistriaged_hint + handler.debian[cve]['desc']
             else:
-                if opt.verbose:
+                if args.verbose:
                     print("Skipping %s, already known" % cve, file=sys.stderr)
                 continue
 
@@ -316,7 +320,7 @@ def import_debian(handler):
                 date = "%s-%s-%s" % (today.year, today.month, today.day)
             cves[cve]['date'] = datetime.strptime(date, "%Y-%m-%d")
 
-            if opt.verbose:
+            if args.verbose:
                 print("Processing %s: %s (%s)" % (cve, handler.debian[cve]['desc'], cves[cve]['date']), file=sys.stderr)
 
     nvd = convert_to_nvd(cves, lambda cve: cves[cve]['subject'])
@@ -347,7 +351,7 @@ class RHEL8OVALHandler(xml.sax.handler.ContentHandler):
 
     def startElement(self, name, attrs):
         if name == 'oval:timestamp':
-            if opt.verbose:
+            if args.verbose:
                 print("Parsing RHEL8 OVAL schema", file=sys.stderr)
             self._curr_chars_collect = True
             self._curr_chars = ""
@@ -448,11 +452,11 @@ def read_locate_cves_output(f):
                 print("Skipping malformed CVE: '%s' from '%s'" % (cve, f), file=sys.stderr)
                 cve = None
             elif cve in cves:
-                if opt.verbose:
+                if args.verbose:
                     print("Skipping duplicate '%s' from '%s'" % (cve, f), file=sys.stderr)
                 cve = None
             else:
-                if opt.verbose:
+                if args.verbose:
                     print("Adding '%s'" % cve, file=sys.stderr)
                 cves[cve] = dict()
             continue
@@ -600,7 +604,7 @@ class CVEHandler(xml.sax.handler.ContentHandler):
         # Append to timestamp file list
         with open('%s/check-cves.log' % (destdir), 'a') as f:
             f.write('%s UTC - %s added, %s ignored, %s skipped, %s total - files: %s\n' %
-                (timestamp, self.num_added, self.num_ignored, self.num_skipped, self.num_added + self.num_ignored, [os.path.basename(x) for x in args]))
+                (timestamp, self.num_added, self.num_ignored, self.num_skipped, self.num_added + self.num_ignored, [os.path.basename(x) for x in args.uris]))
 
     def printReport(self):
         print('\n============================ Triage summary =============================')
@@ -732,7 +736,7 @@ class CVEHandler(xml.sax.handler.ContentHandler):
 
     def startElement(self, name, attrs):
         if name == "item":
-            if opt.verbose:
+            if args.verbose:
                 print("Parsing Mitre XML schema", file=sys.stderr)
             self.curr_cve = attrs['name']
             self.curr_refs = []
@@ -772,7 +776,7 @@ class CVEHandler(xml.sax.handler.ContentHandler):
             return
 
         limit = cve_limit
-        if not opt.refresh and not opt.score_refresh:
+        if not args.refresh and not args.score_refresh:
             limit = 2005
         if int(self.curr_cve.split("-")[1]) < limit:
             return
@@ -1398,12 +1402,12 @@ CVEKnownList += [cve for cve in os.listdir(destdir + "/retired/") if cve.startsw
 (ActiveList, EmbargoList) = cve_lib.get_cve_list()
 CVEKnownList += [cve for cve in ActiveList if cve not in EmbargoList]
 
-if not opt.refresh and not opt.mistriaged and not opt.score_refresh:
+if not args.refresh and not args.mistriaged and not args.score_refresh:
     CVEIgnoreList += CVEKnownList
 
-if opt.known:
+if args.known:
     cvelist = CVEIgnoreList
-    if opt.skip_nfu:
+    if args.skip_nfu:
         cvelist = CVEKnownList
     for cve in sorted(cvelist):
         print(cve)
@@ -1416,9 +1420,9 @@ parser.setContentHandler(handler)
 # if has specified to triage only specific CVEs, check these are not
 # ignored
 specific_cves = None
-if opt.cve:
+if args.cve:
     specific_cves = set()
-    for cve in opt.cve.split(","):
+    for cve in args.cve.split(","):
         # ignore empty CVE
         if cve.strip() == "":
             continue
@@ -1429,28 +1433,26 @@ if opt.cve:
         specific_cves.add(cve)
 
 untriaged_json = ""
-if opt.untriaged:
-    untriaged_json = read_locate_cves_output(opt.untriaged)
-    args.append(untriaged_json)
+if args.untriaged:
+    untriaged_json = read_locate_cves_output(args.untriaged)
+    args.uris.append(untriaged_json)
 
-if opt.mbox:
-    untriaged_json = read_mbox_file(opt.mbox)
-    args.append(untriaged_json)
+if args.mbox:
+    untriaged_json = read_mbox_file(args.mbox)
+    args.uris.append(untriaged_json)
 
 rhel8oval_import_json = ""
-if opt.rhel8oval:
-    untriaged_json = read_rhel8oval_file(opt.rhel8oval)
-    args.append(untriaged_json)
+if args.rhel8oval:
+    untriaged_json = read_rhel8oval_file(args.rhel8oval)
+    args.uris.append(untriaged_json)
 
 debian_import_json = ""
-if (opt.import_missing_debian or opt.mistriaged) and handler.debian is not None:
+if (args.import_missing_debian or args.mistriaged) and handler.debian is not None:
     debian_import_json = import_debian(handler)
-    args.append(debian_import_json)
+    args.uris.append(debian_import_json)
 
-if len(args) == 0:
-    args.append("https://cve.mitre.org/cve/downloads/allitems.xml")
 
-for uri in args:
+for uri in args.uris:
     print('Loading %s ...' % (uri), file=sys.stderr)
     if '://' in uri:
         readable = urllib.request.urlopen(uri)
@@ -1487,7 +1489,7 @@ def refresh_cves(cve_refresh_list, full_refresh=True):
                 public = handler.cve_data[cve]['public']
             cvsss = handler.cve_data[cve]['cvss']
         except:
-            if opt.verbose:
+            if args.verbose:
                 print('%s not listed in XML' % (cve), file=sys.stderr)
 
         # Find the on-disk CVE file
@@ -1552,17 +1554,17 @@ def refresh_cves(cve_refresh_list, full_refresh=True):
             print("Refreshed %s" % (cvefile), file=sys.stderr)
 
 
-if opt.refresh or opt.score_refresh:
-    if opt.cve and specific_cves is not set():
+if args.refresh or args.score_refresh:
+    if args.cve and specific_cves is not set():
         cve_refresh_list = specific_cves
     else:
         cve_refresh_list = CVEKnownList
 
-    # with OptParse opt.refresh and opt.score_refresh will each
+    # with OptParse args.refresh and args.score_refresh will each
     # either be True or None. We want full_refresh to be False when
-    # opt.score_refresh is True. If both are true, then we'll do a full
+    # args.score_refresh is True. If both are true, then we'll do a full
     # refresh since it's a superset of the score only refresh.
-    full_refresh = opt.refresh or not opt.score_refresh
+    full_refresh = args.refresh or not args.score_refresh
     refresh_cves(cve_refresh_list, full_refresh=full_refresh)
     sys.exit(0)
 
@@ -1575,13 +1577,13 @@ if experimental:
     handler.display_command_file_usage(fout, '# ')
 
 for cve in new_cves:
-    if opt.cve and cve not in specific_cves:
+    if args.cve and cve not in specific_cves:
         # ignore this cve
         continue
     # if this got marked as mistriaged, probablistically choose it for
     # processing
     if mistriaged_hint in handler.cve_data[cve]['desc']:
-        if opt.mistriaged == 0:
+        if args.mistriaged == 0:
             # ignore this one
             continue
         else:
@@ -1596,11 +1598,11 @@ for cve in new_cves:
             if rand > prob:
                 continue
             # selected!
-            opt.mistriaged = opt.mistriaged - 1
+            args.mistriaged = args.mistriaged - 1
 
     count += 1
 
-    if opt.report:
+    if args.report:
         print(cve)
         continue
 
@@ -1653,6 +1655,6 @@ if experimental:
     fout.seek(0)
     handler.process_command_file(fout)
 
-if not opt.report:
+if not args.report:
     handler.updateTimestamp()
 handler.printReport()