Merge ~ebarretto/ubuntu-cve-tracker:oval-roadmap-3 into ubuntu-cve-tracker:master

[Eduardo Barretto]

The changes here aim to facilitate the oval generation.
For that we add and oval parameter to the subprojects configuration.
Also, rework of product_series function and add needs_oval function.

[Leonidas S. Barbosa (leosilvab)]

lgtm

review: Approve

65fd154... by Eduardo Barretto on 2022-11-29

cve_lib: Fix lunar indentation

diff --git a/scripts/cve_lib.py b/scripts/cve_lib.py
index 09cb6d9..d8d6bf6 100755
--- a/scripts/cve_lib.py
+++ b/scripts/cve_lib.py
@@ -111,6 +111,7 @@ subprojects = {
     },
     "esm/trusty": {
         "eol": False,
+        "oval": True,
         "packages": ["trusty-esm-supported.txt"],
         "name": "Ubuntu 14.04 ESM",
         "codename": "Trusty Tahr",
@@ -122,6 +123,7 @@ subprojects = {
     },
     "esm-infra/xenial": {
         "eol": False,
+        "oval": True,
         "components": ["main", "restricted"],
         "packages": ["esm-infra-xenial-supported.txt"],
         "name": "Ubuntu 16.04 ESM",
@@ -133,6 +135,7 @@ subprojects = {
     },
     "fips/xenial": {
         "eol": False,
+        "oval": True,
         "packages": ["fips-xenial-supported.txt"],
         "name": "Ubuntu 16.04 FIPS Certified",
         "codename": "Xenial Xerus",
@@ -142,6 +145,7 @@ subprojects = {
     },
     "fips/bionic": {
         "eol": False,
+        "oval": True,
         "packages": ["fips-bionic-supported.txt"],
         "name": "Ubuntu 18.04 FIPS Certified",
         "codename": "Bionic Beaver",
@@ -151,6 +155,7 @@ subprojects = {
     },
     "fips/focal": {
         "eol": False,
+        "oval": True,
         "packages": ["fips-focal-supported.txt"],
         "name": "Ubuntu 20.04 FIPS Certified",
         "codename": "Focal Fossa",
@@ -160,6 +165,7 @@ subprojects = {
     },
     "fips-updates/xenial": {
         "eol": False,
+        "oval": True,
         "packages": ["fips-updates-xenial-supported.txt"],
         "name": "Ubuntu 16.04 FIPS Compliant",
         "codename": "Xenial Xerus",
@@ -169,6 +175,7 @@ subprojects = {
     },
     "fips-updates/bionic": {
         "eol": False,
+        "oval": True,
         "packages": ["fips-updates-bionic-supported.txt"],
         "name": "Ubuntu 18.04 FIPS Compliant",
         "codename": "Bionic Beaver",
@@ -178,6 +185,7 @@ subprojects = {
     },
     "fips-updates/focal": {
         "eol": False,
+        "oval": True,
         "packages": ["fips-updates-focal-supported.txt"],
         "name": "Ubuntu 20.04 FIPS Compliant",
         "codename": "Focal Fossa",
@@ -187,6 +195,7 @@ subprojects = {
     },
     "ros-esm/kinetic": {
         "eol": False,
+        "oval": False,
         "packages": ["ros-esm-xenial-kinetic-supported.txt"],
         "name": "Ubuntu 16.04 ROS ESM",
         "codename": "Xenial Xerus",
@@ -198,6 +207,7 @@ subprojects = {
     },
     "ros-esm/melodic": {
         "eol": False,
+        "oval": False,
         "packages": ["ros-esm-bionic-melodic-supported.txt"],
         "name": "Ubuntu 18.04 ROS ESM",
         "codename": "Bionic Beaver",
@@ -452,6 +462,7 @@ subprojects = {
     },
     "ubuntu/bionic": {
         "eol": False,
+        "oval": True,
         "components": ["main", "restricted", "universe", "multiverse", "partner"],
         "name": "Ubuntu 18.04 LTS",
         "codename": "Bionic Beaver",
@@ -488,6 +499,7 @@ subprojects = {
     },
     "ubuntu/focal": {
         "eol": False,
+        "oval": True,
         "components": ["main", "restricted", "universe", "multiverse", "partner"],
         "name": "Ubuntu 20.04 LTS",
         "codename": "Focal Fossa",
@@ -524,6 +536,7 @@ subprojects = {
     },
     "ubuntu/jammy": {
         "eol": False,
+        "oval": True,
         "components": ["main", "restricted", "universe", "multiverse", "partner"],
         "name": "Ubuntu 22.04 LTS",
         "codename": "Jammy Jellyfish",
@@ -533,6 +546,7 @@ subprojects = {
     },
     "ubuntu/kinetic": {
         "eol": False,
+        "oval": True,
         "components": ["main", "restricted", "universe", "multiverse"],
         "name": "Ubuntu 22.10",
         "codename": "Kinetic Kudu",
@@ -542,16 +556,18 @@ subprojects = {
         "stamp": 1666461600,
     },
     "ubuntu/lunar": {
-       "eol": False,
-       "components": ["main", "restricted", "universe", "multiverse"],
-       "name": "Ubuntu 23.04",
-       "codename": "Lunar Lobster",
-       "alias": "lunar",
-       "devel": True,  # there can be only one ⚔
-       "description": "Interim Release",
+        "eol": False,
+        "oval": True,
+        "components": ["main", "restricted", "universe", "multiverse"],
+        "name": "Ubuntu 23.04",
+        "codename": "Lunar Lobster",
+        "alias": "lunar",
+        "devel": True,  # there can be only one ⚔
+        "description": "Interim Release",
     },
     "snap": {
         "eol": False,
+        "oval": False,
         "packages": ["snap-supported.txt"],
     }
 }
@@ -561,9 +577,19 @@ def product_series(rel):
     """Return the product,series tuple for rel."""
     series = ""
     parts = rel.split('/', 1)
-    product = parts[0]
     if len(parts) == 2:
+        product = parts[0]
         series = parts[1]
+        # handle trusty/esm case
+        if product in releases:
+            product, series = series, product
+    elif parts[0] in releases:
+        # by default ubuntu releases have an omitted ubuntu product
+        # this avoids cases like snaps
+        product = PRODUCT_UBUNTU
+        series = parts[0]
+    else:
+        product = parts[0]
     return product, series
 
 # get the subproject details for rel along with it's canonical name, product and series
@@ -673,6 +699,16 @@ def release_ppa(rel):
         pass
     return ppa
 
+def needs_oval(rel):
+    """Return if OVAL should be generated for a given subproject"""
+    oval_type = None
+    _, product, series, details = get_subproject_details(rel)
+    try:
+        oval_type = details["oval"]
+    except (KeyError, TypeError):
+        pass
+    return oval_type
+
 def get_subproject_description(rel):
     """Return the description for a given release."""
     description = "?"
@@ -773,6 +809,7 @@ def load_external_subprojects():
             # subproject settings
             config = read_external_subproject_config(rel)
             subprojects[rel].setdefault("ppa", config["ppa"])
+            subprojects[rel].setdefault("oval", config["oval"])
             subprojects[rel].setdefault("name", config["name"])
             subprojects[rel].setdefault("description", config["description"])
             subprojects[rel].setdefault("parent", config["parent"])
diff --git a/scripts/generate-oval b/scripts/generate-oval
index 58f1dc6..849ef82 100755
--- a/scripts/generate-oval
+++ b/scripts/generate-oval
@@ -36,7 +36,7 @@ import tempfile
 #from launchpadlib.launchpad import Launchpad
 
 import apt_pkg
-from cve_lib import (kernel_srcs, get_orig_rel_name, load_cve, get_subproject_details, PRODUCT_UBUNTU, all_releases, eol_releases, devel_release, release_parent, release_name, release_ppa, release_progenitor)
+from cve_lib import (kernel_srcs, product_series, load_cve, PRODUCT_UBUNTU, all_releases, eol_releases, devel_release, release_parent, release_name, release_ppa, release_progenitor, needs_oval)
 from kernel_lib import (meta_kernels, kernel_package_abi, kernel_package_version)
 import oval_lib
 import lpl_common
@@ -47,16 +47,13 @@ if 'init_system' in dir(apt_pkg):
 else:
     apt_pkg.InitSystem()
 
-supported_products = [PRODUCT_UBUNTU, 'esm', 'esm-infra', 'esm-apps', 'fips', 'fips-updates']
 supported_releases = []
 for r in set(all_releases).difference(set(eol_releases)).difference(set([devel_release])):
-    _, product, _, _ = get_subproject_details(r)
-    if product in supported_products:
-        if r not in supported_releases:
-            supported_releases.append(r)
-        parent = release_parent(r)
-        if parent and parent not in supported_releases:
-            supported_releases.append(parent)
+    if needs_oval(r):
+        supported_releases.append(r)
+    parent = release_parent(r)
+    if parent and parent not in supported_releases:
+        supported_releases.append(parent)
 
 default_cves_to_process = ['active/CVE-*', 'retired/CVE-*']
 
@@ -71,12 +68,12 @@ def main():
     global debug_level
 
     # parse command line options
-    parser = argparse.ArgumentParser(description='Generate CVE OVAL from ' +
+    parser = argparse.ArgumentParser(description='Generate CVE OVAL from '
                                      'CVE metadata files.')
     parser.add_argument('pathname', nargs='*',
-                        help='pathname patterns (globs) specifying CVE ' +
-                             'metadata files to be converted into OVAL ' +
-                             '(default: "./active/CVE-*" "./retired/CVE-*")')
+                        help='pathname patterns (globs) specifying CVE '
+                        'metadata files to be converted into OVAL '
+                        '(default: "./active/CVE-*" "./retired/CVE-*")')
     parser.add_argument('--oci', action='store_true',
                         help='Also generate OVAL files for scanning Official Cloud Image manifests')
     parser.add_argument('--output-dir', nargs='?', default='./',
@@ -86,7 +83,7 @@ def main():
     parser.add_argument('--oci-prefix', nargs='?', default='oci.',
                         help='Prefix to use for OCI manifest OVAL files names (required if oci-output-dir is the same as output-dir)')
     parser.add_argument('--cve-prefix-dir', nargs='?', default='./',
-                        help='location of CVE metadata files to process ' +
+                        help='location of CVE metadata files to process '
                         '(default is ./)')
     parser.add_argument('--no-progress', action='store_true',
                         help='do not show progress meter')
@@ -99,15 +96,15 @@ def main():
     parser.add_argument('--usn-oval', action='store_true',
                         help='generates oval from the USN database')
     parser.add_argument('--usn-db-dir', default='./', type=str,
-                        help='location of USN database.json to process ' +
+                        help='location of USN database.json to process '
                         '(default is ./)')
     parser.add_argument('--usn-number', default=None, type=str,
                         help='if passed specifics a USN for the oval_usn generator')
     parser.add_argument('--usn-oval-release', default=None, type=str,
                         help='specifies a release to generate the oval usn')
     parser.add_argument('--packages', nargs='+', action='store', default=None,
-                        help='generates oval for specific packages. Only for'+
-                        ' CVE OVAL')
+                        help='generates oval for specific packages. Only for '
+                        'CVE OVAL')
 
     args = parser.parse_args()
     pathnames = args.pathname or default_cves_to_process
@@ -356,12 +353,6 @@ def parse_cve_file(filepath, cache, pkg_filter=None):
                 continue
             if rel not in supported_releases:
                 continue
-            try:
-                _, product, _, _ = get_subproject_details(rel)
-                if product not in supported_products:
-                    continue
-            except KeyError:
-                continue
             state, details = data['pkgs'][pkg][rel]
             status_line = state
             if len(details) > 0:
@@ -556,7 +547,7 @@ class PackageCache():
             if parent:
                 self.releases[release] = self.ubuntu.getSeries(name_or_version=parent)
             else:
-                self.releases[release] = self.ubuntu.getSeries(name_or_version=get_orig_rel_name(release))
+                self.releases[release] = self.ubuntu.getSeries(name_or_version=product_series(release)[1])
 
         ppa = release_ppa(release)
         if ppa:
@@ -690,7 +681,7 @@ def generate_oval_usn(outdir, usn, usn_release, cve_dir, usn_db_dir, ociprefix=N
     else:
         for release in supported_releases:
             # for now we don't differentiate products (e.g. esm) in the USN DB
-            _, product, _, _ = get_subproject_details(release)
+            product, series = product_series(release)
             if product != PRODUCT_UBUNTU:
                 continue