Merge ~sahnaseredini/ubuntu-cve-tracker:amir-dev-oval-pkgcve into ubuntu-cve-tracker:master

[Amir Naseredini]

goals:
- to generate one file for all the different product/release
- to keep the previous functionality with an added option

changes:
- added an option to the code expand
  - when’s it’s off all the oval data (from ubuntu, esm-infra, and esm-apps) is
    combined in one output file
  - by default this option is off and can be turned on by adding --expand
    argument when running the code
  - adding --expand would make the code behave like before by generating
    different output files (from ubuntu, esm-infra, and esm-apps) for oval
- a function `_eligible_for_output` is added in order to check what we want to produce as the output file

[Eduardo Barretto (ebarretto)]

Thanks, I'm running some tests and will let you know in case of issues. Nevertheless we should wait to merge this until next week so we can inform users of the upcoming change.

[Eduardo Barretto (ebarretto)]

There's a small fix needed.
In the criterion comment, it is showing up, for example, as 'esm-apps/jammy'. It should show as jammy.

review: Needs Fixing

[Eduardo Barretto (ebarretto)]

lgtm, thanks for addressing my comment.

review: Approve

[Amir Naseredini (sahnaseredini)]

Thanks for the review Eduardo.

diff --git a/scripts/generate-oval b/scripts/generate-oval
index bdb9902..920bba6 100755
--- a/scripts/generate-oval
+++ b/scripts/generate-oval
@@ -103,6 +103,10 @@ def main():
                         'CVE OVAL')
     parser.add_argument('--fixed-only', action='store_true',
                         help='only generate pkg oval for fixed CVEs')
+    parser.add_argument('--expand', action='store_true',
+                        help='avoids combining all the oval data into one '
+                        'file and expands the output oval files into different '
+                        'releases, such as esm-apps, esm-infra, and ubuntu')
 
     args = parser.parse_args()
     pathnames = args.pathname or default_cves_to_process
@@ -150,6 +154,8 @@ def main():
             if release not in supported_releases:
                 error(f"unknown oval release {release}")
 
+    expand = args.expand
+
     cache = {}
     for release in supported_releases:
         cve_cache = {}
@@ -157,15 +163,15 @@ def main():
 
     if args.pkg_oval:
         if args.oci:
-            generate_oval_package(releases, outdir, args.cve_prefix_dir, cache, cve_cache, args.oci, args.no_progress, args.packages, pathnames, args.fixed_only, ocioutdir)
+            generate_oval_package(releases, outdir, args.cve_prefix_dir, cache, cve_cache, args.oci, args.no_progress, args.packages, pathnames, args.fixed_only, ocioutdir, expand)
         else:
-            generate_oval_package(releases, outdir, args.cve_prefix_dir, cache, cve_cache, args.oci, args.no_progress, args.packages, pathnames, args.fixed_only)
+            generate_oval_package(releases, outdir, args.cve_prefix_dir, cache, cve_cache, args.oci, args.no_progress, args.packages, pathnames, args.fixed_only, expand)
         return
 
     if args.oci:
-        generate_oval_cve(releases, outdir, args.cve_prefix_dir, cache, cve_cache, args.oci, args.no_progress, args.packages, pathnames, args.fixed_only, ocioutdir)
+        generate_oval_cve(releases, outdir, args.cve_prefix_dir, cache, cve_cache, args.oci, args.no_progress, args.packages, pathnames, args.fixed_only, ocioutdir, expand)
     else:
-        generate_oval_cve(releases, outdir, args.cve_prefix_dir, cache, cve_cache, args.oci, args.no_progress, args.packages, pathnames, args.fixed_only)
+        generate_oval_cve(releases, outdir, args.cve_prefix_dir, cache, cve_cache, args.oci, args.no_progress, args.packages, pathnames, args.fixed_only, expand)
     return
 
 
@@ -199,7 +205,7 @@ def ignore_source_package(source):
     return False
 
 
-def parse_cve_file(filepath, cache, pkg_filter=None):
+def parse_cve_file(filepath, cache, pkg_filter=None, expand=False):
     """ parse CVE data file into a dictionary using cve_lib """
 
     cve_header_data = {
@@ -220,7 +226,6 @@ def parse_cve_file(filepath, cache, pkg_filter=None):
         'Unknown-Fields': [],
         'Source-note': filepath
     }
-
     data = load_cve(filepath)
     # first try a naive translation of fields
     for f in cve_header_data:
@@ -255,7 +260,7 @@ def parse_cve_file(filepath, cache, pkg_filter=None):
             if rel not in supported_releases:
                 continue
             state, details = data['pkgs'][pkg][rel]
-            packages[pkg]['Releases'][rel] = oval_lib.CVEPkgRelEntry.parse_package_status(rel, pkg, state, details, filepath, cache)
+            packages[pkg]['Releases'][rel] = oval_lib.CVEPkgRelEntry.parse_package_status(rel, pkg, state, details, filepath, cache, oval_lib.find_release_codename(rel), expand)
 
     # add supplemental packages; usually kernels only need this special case.
     for package in [name for name in packages if name in kernel_srcs]:
@@ -435,9 +440,21 @@ def generate_oval_usn(outdir, usn, usn_releases, cve_dir, usn_db_dir, no_progres
 
     return True
 
-def generate_oval_package(releases, outdir, cve_prefix_dir, pkg_cache, cve_cache, oci, no_progress, packages, pathnames, fixed_only, ocioutdir=None):
+def filter_releases_for_output(releases, ov):
+    output_releases = []
+    for release in releases:
+        ov._init_ids(release)
+        if ov._eligible_for_output(release):
+            if ov.expand == False and 'esm' in ov.release:
+                output_releases.append(ov.release_codename)
+            else:
+                output_releases.append(release)
+    return output_releases
+
+
+def generate_oval_package(releases, outdir, cve_prefix_dir, pkg_cache, cve_cache, oci, no_progress, packages, pathnames, fixed_only, ocioutdir=None, expand=False):
     if not no_progress:
-        print(f'[*] Generating OVAL PKG for packages in releases {", ".join(releases)}')
+        print('[*] Initiating OVAL Generation for PKG')
 
     ov = oval_lib.OvalGeneratorPkg(
         releases, 
@@ -449,8 +466,15 @@ def generate_oval_package(releases, outdir, cve_prefix_dir, pkg_cache, cve_cache
         cve_cache=cve_cache, 
         oval_format='dpkg',
         outdir=outdir, 
-        cve_prefix_dir=cve_prefix_dir
+        cve_prefix_dir=cve_prefix_dir,
+        expand=expand
     )
+
+    output_releases = filter_releases_for_output(releases, ov)
+
+    if not no_progress:
+        print(f'[*] Generating OVAL PKG for packages in releases {", ".join(output_releases)}')
+
     ov.generate_oval()
 
     if oci:
@@ -459,11 +483,11 @@ def generate_oval_package(releases, outdir, cve_prefix_dir, pkg_cache, cve_cache
         ov.generate_oval()
 
     if not no_progress:
-        print(f'[X] Done generating OVAL PKG for packages in releases {", ".join(releases)}')
+        print(f'[X] Done generating OVAL PKG for packages in releases {", ".join(output_releases)}')
 
-def generate_oval_cve(releases, outdir, cve_prefix_dir, pkg_cache, cve_cache, oci, no_progress, packages, pathnames, fixed_only, ocioutdir=None):
+def generate_oval_cve(releases, outdir, cve_prefix_dir, pkg_cache, cve_cache, oci, no_progress, packages, pathnames, fixed_only, ocioutdir=None, expand=False):
     if not no_progress:
-        print(f'[*] Generating OVAL CVE for packages in releases {",".join(releases)}')
+        print('[*] Initiating OVAL Generation for CVE')
 
     ov = oval_lib.OvalGeneratorCVE(
         releases, 
@@ -475,8 +499,15 @@ def generate_oval_cve(releases, outdir, cve_prefix_dir, pkg_cache, cve_cache, oc
         cve_cache=cve_cache, 
         oval_format='dpkg',
         outdir=outdir, 
-        cve_prefix_dir=cve_prefix_dir
+        cve_prefix_dir=cve_prefix_dir,
+        expand=expand
     )
+
+    output_releases = filter_releases_for_output(releases, ov)
+
+    if not no_progress:
+        print(f'[*] Generating OVAL CVE for packages in releases {", ".join(output_releases)}')
+
     ov.generate_oval()
 
     if oci:
@@ -485,7 +516,7 @@ def generate_oval_cve(releases, outdir, cve_prefix_dir, pkg_cache, cve_cache, oc
         ov.generate_oval()
 
     if not no_progress:
-        print(f'[X] Done generating OVAL CVE for packages in releases {", ".join(releases)}')
+        print(f'[X] Done generating OVAL CVE for packages in releases {", ".join(output_releases)}')
 
 if __name__ == '__main__':
     main()
diff --git a/scripts/oval_lib.py b/scripts/oval_lib.py
index b30a987..e018d74 100644
--- a/scripts/oval_lib.py
+++ b/scripts/oval_lib.py
@@ -191,20 +191,22 @@ def get_binarypkgs(cache, source_name, release):
     return rel, binaries_map
 
 class CVEPkgRelEntry:
-    def __init__(self, pkg, release, cve, status, note) -> None:
+    def __init__(self, pkg, release, cve, status, note, expand=False) -> None:
         self.pkg = pkg
         self.cve = cve
         self.orig_status = status
         self.orig_note = note
         self.release = release
-        cve_info = CVEPkgRelEntry.parse_package_status(self.release, pkg.name, status, note, cve.number, None)
+        self.expand = expand
+        self.release_codename = find_release_codename(self.release)
+        cve_info = CVEPkgRelEntry.parse_package_status(self.release, pkg.name, status, note, cve.number, None, self.release_codename, self.expand)
 
         self.note = cve_info['note']
         self.status = cve_info['status']
         self.fixed_version = cve_info['fix-version'] if self.status == 'fixed' else None
 
     @staticmethod
-    def parse_package_status(release, package, status_text, note, filepath, cache):
+    def parse_package_status(release, package, status_text, note, filepath, cache, release_codename, expand=False):
         """ parse ubuntu package status string format:
             <status code> (<version/notes>)
             outputs dictionary: {
@@ -216,6 +218,9 @@ class CVEPkgRelEntry:
 
         # TODO fix for CVE Generator
 
+        if expand == False and 'esm' in release:
+            release = release_codename
+
         # break out status code and detail
         code = status_text.lower()
         detail = note.strip('()') if note else None
@@ -355,8 +360,8 @@ class CVE:
         return pkgs
 
 
-    def add_pkg(self, pkg_object, release, state, note):
-        cve_pkg_entry = CVEPkgRelEntry(pkg_object, release, self, state, note)
+    def add_pkg(self, pkg_object, release, state, note, expand=False):
+        cve_pkg_entry = CVEPkgRelEntry(pkg_object, release, self, state, note, expand)
         self.pkg_rel_entries[str(pkg_object)] = cve_pkg_entry
         self.pkgs.append(pkg_object)
         pkg_object.add_cve(self)
@@ -497,7 +502,7 @@ class OvalGenerator:
     supported_oval_elements = ('definition', 'test', 'object', 'state', 'variable')
     generator_version = '2'
     oval_schema_version = '5.11.1'
-    def __init__(self, type, releases, cve_paths, packages, progress, pkg_cache, fixed_only=True, cve_cache=None,  cve_prefix_dir=None, outdir='./', oval_format='dpkg') -> None:
+    def __init__(self, type, releases, cve_paths, packages, progress, pkg_cache, fixed_only=True, cve_cache=None,  cve_prefix_dir=None, outdir='./', oval_format='dpkg', expand=False) -> None:
         self.releases = releases
         self.output_dir = outdir
         self.oval_format = oval_format
@@ -508,6 +513,7 @@ class OvalGenerator:
         self.cve_paths = cve_paths
         self.fixed_only = fixed_only
         self.packages, self.cves = self._load(cve_prefix_dir, packages)
+        self.expand = expand
 
     def _init_ids(self, release):
         # e.g. codename for trusty/esm should be trusty
@@ -538,8 +544,14 @@ class OvalGenerator:
         self.definition_id = self.release_id
         self.definition_step = 1 * 10 ** 5
         self.criterion_step = 10
-        self.output_filepath = \
-            '{0}com.ubuntu.{1}.{2}.oval.xml'.format('oci.' if self.oval_format == 'oci' else '', self.release.replace('/', '_'), self.generator_type)
+        self.output_filepath = ''
+        if self.expand == False and 'esm' in self.release:
+            self.output_filepath = \
+                '{0}com.ubuntu.{1}.{2}.oval.xml'.format('oci.' if self.oval_format == 'oci' else '', self.release_codename, self.generator_type)
+        else:
+            self.output_filepath = \
+                '{0}com.ubuntu.{1}.{2}.oval.xml'.format('oci.' if self.oval_format == 'oci' else '', self.release.replace('/', '_'), self.generator_type)
+
         
     def _add_structure(self, root) -> None:
         structure = {}
@@ -730,7 +742,7 @@ class OvalGenerator:
             packages[package_name] = pkg_obj
 
         pkg_obj = packages[package_name]
-        cve.add_pkg(pkg_obj, release, cve_data['pkgs'][package_name][release][0],cve_data['pkgs'][package_name][release][1])
+        cve.add_pkg(pkg_obj, release, cve_data['pkgs'][package_name][release][0],cve_data['pkgs'][package_name][release][1], self.expand)
 
     def _load(self, cve_prefix_dir, packages_filter=None) -> None:
         cve_lib.load_external_subprojects()
@@ -806,6 +818,14 @@ class OvalGenerator:
             with open(os.path.join(self.output_dir, self.output_filepath), 'w') as file:
                 file.write(xmlstr)
 
+    def _eligible_for_output(self, release):
+        if self.expand == False:
+            if self.release_codename == release and 'LTS' in cve_lib.release_name(release):
+                return False
+            elif 'esm-infra' in release:
+                return False
+        return True
+
     # Object generators
     def _generate_criteria(self) -> etree.Element:
         criteria = etree.Element("criteria")
@@ -1291,8 +1311,9 @@ class OvalGenerator:
 
 
 class OvalGeneratorPkg(OvalGenerator):
-    def __init__(self, releases, cve_paths, packages, progress, pkg_cache, fixed_only=True, cve_cache=None,  cve_prefix_dir=None, outdir='./', oval_format='dpkg') -> None:
-        super().__init__('pkg', releases, cve_paths, packages, progress, pkg_cache, fixed_only, cve_cache,  cve_prefix_dir, outdir, oval_format)
+    def __init__(self, releases, cve_paths, packages, progress, pkg_cache, fixed_only=True, cve_cache=None,  cve_prefix_dir=None, outdir='./', oval_format='dpkg',expand=False) -> None:
+        self.expand = expand
+        super().__init__('pkg', releases, cve_paths, packages, progress, pkg_cache, fixed_only, cve_cache,  cve_prefix_dir, outdir, oval_format, expand)
 
     def _generate_advisory(self, package: Package) -> etree.Element:
         advisory = etree.Element("advisory")
@@ -1488,11 +1509,13 @@ class OvalGeneratorPkg(OvalGenerator):
                 else:
                     self._populate_pkg(all_pkgs[pkg], root_element)
 
-            self._write_oval_xml(xml_tree, root_element)
+            if self._eligible_for_output(release):
+                self._write_oval_xml(xml_tree, root_element)
 
 class OvalGeneratorCVE(OvalGenerator):
-    def __init__(self, releases, cve_paths, packages, progress, pkg_cache, fixed_only=True, cve_cache=None,  cve_prefix_dir=None, outdir='./', oval_format='dpkg') -> None:
-        super().__init__('cve', releases, cve_paths, packages, progress, pkg_cache, fixed_only, cve_cache,  cve_prefix_dir, outdir, oval_format)
+    def __init__(self, releases, cve_paths, packages, progress, pkg_cache, fixed_only=True, cve_cache=None,  cve_prefix_dir=None, outdir='./', oval_format='dpkg', expand=False) -> None:
+        self.expand = expand
+        super().__init__('cve', releases, cve_paths, packages, progress, pkg_cache, fixed_only, cve_cache,  cve_prefix_dir, outdir, oval_format, expand)
 
     # For CVE OVAL, the definition ID is generated
     # from the CVE ID
@@ -1737,7 +1760,8 @@ class OvalGeneratorCVE(OvalGenerator):
                 self._set_definition_id(cve_id=all_cves[cve].number)
                 self._generate_elements_from_cve(all_cves[cve], accepted_releases, root_element, running_kernel_id, pkg_cache, fixed_versions)
 
-            self._write_oval_xml(xml_tree, root_element)
+            if self._eligible_for_output(release):
+                self._write_oval_xml(xml_tree, root_element)
 
 class OvalGeneratorUSNs(OvalGenerator):
     def __init__(self, release, release_name, cve_paths, packages, progress, pkg_cache, usn_db_dir, fixed_only=True, cve_cache=None,  cve_prefix_dir=None, outdir='./', oval_format='dpkg') -> None:
@@ -2646,3 +2670,7 @@ class OvalGeneratorUSN():
         # remove tmp dir if empty
         if not os.listdir(self.tmpdir):
             os.rmdir(self.tmpdir)
+
+def find_release_codename(release):
+    return cve_lib.release_progenitor(release) if cve_lib.release_progenitor(release) else release.replace('/', '_')
+