Merge ~alexmurray/ubuntu-cve-tracker:python3-12-deprecation-fixups into ubuntu-cve-tracker:master

[Alex Murray]

I suspect we have other scripts that will need similar fixes so will try and capture as many as I can here.

8683bc4... by Alex Murray on 2024-02-26

scripts/oval_lib.py: update for Python 3.12 deprecations

Signed-off-by: Alex Murray <email address hidden>

b50dcd7... by Alex Murray on 2024-02-26

scripts/active_edit: update for Python 3.12 deprecations

Signed-off-by: Alex Murray <email address hidden>

[Steve Beattie (sbeattie)]

For the datetime changes,

- public_date = datetime.datetime.utcnow().strftime("%Y-%m-%d")
+ public_date = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d")

datetime.UTC was introduced in python 3.11 (so ubuntu 23.10, mantic), so jammy and older won't have this. We'll want to use datetime.timezone.utc instead; while most people should be running mantic or newer, we do have team members and systems running older releases than that. Given that datetime.datetime.now(datetime.timezone.utc) is a bit unwieldy, we could do something like:

  from datetime import datetime, timezone
  public_date = datetime.now(timezone.utc).strftime("%Y-%m-%d")

Thanks.

[Steve Beattie (sbeattie)]

A couple more commits to include in https://code.launchpad.net/~sbeattie/ubuntu-cve-tracker/+git/ubuntu-cve-tracker/+ref/collected_fixups/

[Steve Beattie (sbeattie)]

I went ahead and fixed a bunch more and did the conversion to use datetime.timezone.utc instead of datetime.UTC, and then merged everything in https://git.launchpad.net/ubuntu-cve-tracker/commit/?id=92593eaa749bd40f80856e717c2186064f2993f5

Unfortunately, I ended up needing to rebase on top of master due to a merge conflict, so launchpad is not autoclosing this merge request.

review: Approve

[Steve Beattie (sbeattie)]

(I didn't squash the commits, all the individual ones are still there, that's just how cgit shows the merge request.)

diff --git a/scripts/active_edit b/scripts/active_edit
index dadaf35..b02c28e 100755
--- a/scripts/active_edit
+++ b/scripts/active_edit
@@ -99,7 +99,7 @@ def fetch_kernel_fixes(url):
         if re.match("commit [0-9a-f]{40} upstream.", line):
             # This is an LTS backport, skip it
             return []
-        if re.match("\[ Upstream commit [0-9a-f]{40} \]", line):
+        if re.match(r"\[ Upstream commit [0-9a-f]{40} \]", line):
             # This is an LTS backport, skip it
             return []
         if not commit_hash and line.startswith("From "):
@@ -316,7 +316,7 @@ def create_or_update_cve(cve, packages, priority=None, bug_urls=None,
     # Set a default public date only when the CVE isn't being added to the
     # embargoed tree
     if not embargoed and not public_date:
-        public_date = datetime.datetime.utcnow().strftime("%Y-%m-%d")
+        public_date = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d")
 
     if update:
         mode = "a"
diff --git a/scripts/check-cves b/scripts/check-cves
index 322f38b..0865e4a 100755
--- a/scripts/check-cves
+++ b/scripts/check-cves
@@ -183,7 +183,7 @@ def convert_to_nvd(cves=[], desc=""):
     nvd = {"CVE_data_type": "CVE",
            "CVE_data_format": "MITRE",
            "CVE_data_version": "4.0",
-           "CVE_data_timestamp": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%MZ"),
+           "CVE_data_timestamp": datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H:%MZ"),
            "CVE_Items": []}
 
     keys = list(cves.keys())
@@ -474,8 +474,8 @@ def read_locate_cves_output(f):
                 subject += " " + s.strip()
 
         # Try to fake up some urls
-        rhsa_regex = '\[RHSA-\d\d\d\d:\d+-\d+\]'
-        osssec_regex = '\[oss-security\]'
+        rhsa_regex = r'\[RHSA-\d\d\d\d:\d+-\d+\]'
+        osssec_regex = r'\[oss-security\]'
         if re.search(r'' + rhsa_regex, subject):
             rhsa = re.sub(r'.*(%s).*' % rhsa_regex, r'\1', subject).strip('[|]')
             url = "https://rhn.redhat.com/errata/%s-%s.html" % (rhsa.split(':')[0], rhsa.split(':')[1].split('-')[0])
@@ -651,7 +651,7 @@ class CVEHandler(xml.sax.handler.ContentHandler):
                     raise KeyError("NVD JSON in '%s' seems invalid" % fp.name)
 
             metadata = cve["CVE_data_meta"]
-            if not re.match('^CVE-\d{4}-\d{4,}$', metadata["ID"]):
+            if not re.match(r'^CVE-\d{4}-\d{4,}$', metadata["ID"]):
                 print("Ignoring invalid CVE with ID '%s'" % metadata["ID"])
                 return
             self.curr_cve = metadata["ID"]
@@ -850,8 +850,8 @@ class CVEHandler(xml.sax.handler.ContentHandler):
         if suggestion.startswith('a ') and len(phrases) > 1:
             suggestion = phrases[-2]
 
-        version_preps = '(\s+(before|through|prior to|versions?|[<>]=?))+\s*'
-        version_regex = '\s+([a-zA-Z\._\-]*[0-9]+[a-zA-Z\._\-]*)+'
+        version_preps = r'(\s+(before|through|prior to|versions?|[<>]=?))+\s*'
+        version_regex = r'\s+([a-zA-Z\._\-]*[0-9]+[a-zA-Z\._\-]*)+'
         # prefer 'Apple iOS before <version>' or 'Apple Mac OS X through
         # <version' in the last phrase over other suggestions
         if not re.search(r'' + version_preps + version_regex, suggestion):
@@ -1405,7 +1405,7 @@ class CVEHandler(xml.sax.handler.ContentHandler):
         cmd = ['./scripts/active_edit', '-c', cve, '--yes',
                '--public',
                # set date as now
-               datetime.datetime.utcnow().strftime("%Y-%m-%d")]
+               datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%d")]
         if breakfix:
             cmd.extend(['-k'])
         for url in ref_urls:
@@ -1750,7 +1750,7 @@ def refresh_cves(cve_refresh_list, full_refresh=True):
                     debug("updated pubdate for %s" % (cvefile))
 
             # Add CVE Reference, if it's missing
-            if 'References' in data and re.match('^CVE-\d+-\d+$', cve):
+            if 'References' in data and re.match(r'^CVE-\d+-\d+$', cve):
                 mitre_ref = "https://cve.mitre.org/cgi-bin/cvename.cgi?name=" + cve
                 if mitre_ref not in data['References']:
                     cve_lib.add_reference(cvefile, mitre_ref)
@@ -1808,7 +1808,7 @@ for cve in new_cves:
             # pick half as often, if year is 2 years ago we want to pick
             # 1/4 as often etc - so do 1/2^(diff)
             year = int(re.split('-', cve)[1])
-            now = datetime.datetime.utcnow().year
+            now = datetime.datetime.now(datetime.UTC).year
             prob = 1.0 / math.pow(2, now - year)
             rand = random.random()
             if rand > prob:
diff --git a/scripts/oval_lib.py b/scripts/oval_lib.py
old mode 100644
new mode 100755
index d60c6dc..ace0cb3
--- a/scripts/oval_lib.py
+++ b/scripts/oval_lib.py
@@ -29,7 +29,6 @@ import tempfile
 import collections
 import glob
 import xml.etree.cElementTree as etree
-import json
 from xml.dom import minidom
 from typing import Tuple # Needed because of Python < 3.9 and to also support < 3.7
 
@@ -105,15 +104,15 @@ def is_kernel_binaries(binaries):
 """
 def process_kernel_binaries(binaries, oval_format):
     packages = ' '.join(binaries)
-    parts = re.findall('linux-image-[a-z]*-?([\d|\.]+-)\d+(-[\w|-]+)', packages)
+    parts = re.findall(r'linux-image-[a-z]*-?([\d|\.]+-)\d+(-[\w|-]+)', packages)
     if parts:
         values = set(map(lambda x: x[0], parts))
         version = ''.join(values)
         values = sorted(set(map(lambda x: x[1], parts)))
         flavours = '|'.join(values)
-        regex = version + '\d+(' + flavours + ')'
+        regex = version + r'\d+(' + flavours + ')'
         if oval_format == 'oci':
-            regex = 'linux-image-(?:unsigned-)?' + version + '\d+(?:' + flavours + ')'
+            regex = 'linux-image-(?:unsigned-)?' + version + r'\d+(?:' + flavours + ')'
         return regex
 
     return None
@@ -312,7 +311,7 @@ class CVE:
         for url in info['References'].split('\n'):
             if 'https://ubuntu.com/security/notices/USN-' in url:
                 self.usns.append(url[40:])
-            elif re.match("https?:\/\/(bugs\.)?launchpad\.net\/(.*\/\+bug|bugs)\/\d+", url):
+            elif re.match(r"https?:\/\/(bugs\.)?launchpad\.net\/(.*\/\+bug|bugs)\/\d+", url):
                 self.bugs.append(url)
             elif url:
                 self.references.append(url)
@@ -711,7 +710,7 @@ class OvalGenerator:
             instance = etree.SubElement(object, "ind-def:instance",attrib={"datatype": "int"})
 
             filepath.text = "/etc/lsb-release"
-            pattern.text = "^[\s\S]*DISTRIB_CODENAME=([a-z]+)$"
+            pattern.text = r"^[\s\S]*DISTRIB_CODENAME=([a-z]+)$"
             instance.text = "1"
         else:
             family_object = etree.Element("")
@@ -1034,7 +1033,7 @@ class OvalGenerator:
 
             final_binaries = []
             if self.oval_format == 'oci':
-                variable_values = '(?::\w+|)\s+(.*)$'
+                variable_values = r'(?::\w+|)\s+(.*)$'
                 for binary in binaries:
                     final_binaries.append(f'^{binary}{variable_values}')
             else:
@@ -1060,9 +1059,9 @@ class OvalGenerator:
             if self.oval_format == 'oci':
                 if package.is_kernel_pkg:
                     regex = process_kernel_binaries(binaries, 'oci')
-                    final_binaries = [f'^{regex}(?::\w+|)\s+(.*)$']
+                    final_binaries = [rf'^{regex}(?::\w+|)\s+(.*)$']
                 else:
-                    variable_values = '(?::\w+|)\s+(.*)$'
+                    variable_values = r'(?::\w+|)\s+(.*)$'
 
                     final_binaries = []
                     for binary in binaries:
@@ -1098,7 +1097,7 @@ class OvalGenerator:
         concat = etree.SubElement(var, "concat")
         component = etree.SubElement(concat, "literal_component")
         regex = etree.SubElement(concat, "regex_capture", attrib={
-            "pattern": "^([\d|\.]+-\d+)[-|\w]+$"
+            "pattern": r"^([\d|\.]+-\d+)[-|\w]+$"
         })
 
         etree.SubElement(regex, "object_component", attrib={
@@ -1171,7 +1170,7 @@ class OvalGenerator:
         return object
 
     def _generate_state_kernel_element(self, comment, id, version) -> None:
-        patched = re.search('([\d|\.]+-\d+)[\.|\d]+', version)
+        patched = re.search(r'([\d|\.]+-\d+)[\.|\d]+', version)
         if patched:
             patched = patched.group(1)
         else:
@@ -1285,9 +1284,9 @@ class OvalGenerator:
         if self.oval_format == 'oci':
             if package.is_kernel_pkg:
                 regex = process_kernel_binaries(binaries, 'oci')
-                final_binaries = [f'^{regex}(?::\w+|)\s+(.*)$']
+                final_binaries = [rf'^{regex}(?::\w+|)\s+(.*)$']
             else:
-                variable_values = '(?::\w+|)\s+(.*)$'
+                variable_values = r'(?::\w+|)\s+(.*)$'
 
                 final_binaries = []
                 for binary in binaries:
@@ -2065,7 +2064,7 @@ class OvalGeneratorUSN():
     def create_release_object(self):
         if self.oval_format == 'dpkg':
             _object =\
-        f"""
+        rf"""
         <ind:family_object id="{self.ns}:obj:{self.id}" version="1" comment="The singleton family object."/>
         <ind:textfilecontent54_object id="{self.ns}:obj:{self.id+1}" version="1">
             <ind:filepath datatype="string">/etc/lsb-release</ind:filepath>
@@ -2099,7 +2098,7 @@ class OvalGeneratorUSN():
         bugs = ""
 
         for url in urls:
-            is_bug = re.match("https?:\/\/(bugs\.)?launchpad\.net\/(.*\/\+bug|bugs)\/\d+", url)
+            is_bug = re.match(r"https?:\/\/(bugs\.)?launchpad\.net\/(.*\/\+bug|bugs)\/\d+", url)
 
             if is_bug:
                 bug_urls.append(url)
@@ -2373,7 +2372,7 @@ class OvalGeneratorUSN():
 
             elif 'kernelobj' in test_ref:
                 binary_version = test_ref['version']
-                binary_version = re.search('([\d|\.]+-\d+)[\.|\d]+', binary_version)
+                binary_version = re.search(r'([\d|\.]+-\d+)[\.|\d]+', binary_version)
                 mapping['bversion'] = "0:" + binary_version.group(1)
 
                 state = \
@@ -2431,7 +2430,7 @@ class OvalGeneratorUSN():
         if self.oval_format == 'dpkg':
             if 'kernel' in test_ref:
                 variable = \
-            """
+            r"""
         <local_variable id="{ns}:var:{id}" datatype="debian_evr_string" version="1" comment="kernel version in evr format">
             <concat>
                 <literal_component>0:</literal_component>
@@ -2453,7 +2452,7 @@ class OvalGeneratorUSN():
         else:
             for binary in binaries_list:
                 values += \
-            """<value>^{}(?::\w+|)\s+(.*)$</value>
+            r"""<value>^{}(?::\w+|)\s+(.*)$</value>
             """.format(binary)
 
         mapping['values'] = values.strip()
@@ -2504,7 +2503,7 @@ class OvalGeneratorUSN():
         urls = []
         for cve in cves:
             # Takes urls from the list
-            is_url = re.match('(www|http:|https:)+[^\s]+[\w]', cve)
+            is_url = re.match(r'(www|http:|https:)+[^\s]+[\w]', cve)
 
             if is_url:
                 urls.append(cve)