Merge ~litios/ubuntu-cve-tracker:oval/usns-only-fixed-cves into ubuntu-cve-tracker:master

[David Fernandez Gonzalez]

[OVAL/JSON GEN] Only load CVEs into USN if affecting the release.

USN CVEs are now based on release. This ensures that only the affected
CVEs will be listed when generating for a specific release.

Since we have regressions that do not list a CVE and wrong
USN fixed version entries, there is no way we can retrieve the right
Package from the CVE so we are looking based on release and fixed version.

---------

USN loading is performed in the following way:

1. Get the listed releases in the USN. For each:
    - Identify active ESM releases
    - Get the packages mentioned in the USN released and fixed versions
    - Based on the release, package and fixed version,
      retrieve the appropriate Package and save it with the USN fixed version.

2. Get the CVE/LP bugs entries. For each CVE:
    - Get each PkgRelEntry and check if is marked as fixed (or if USN is a regression)
      and that the package and release are mentioned in the USN, if so, get CVE.

If there are no packages for the requested releases, skip USN.

---------

These changes introduced a diff in JSON generation:

* Only fixed/related CVEs will be listed on the USN entries.
* More USNs listed since now we do better handling of parent cases.

JSON generation code updated and tests updated.

[Eduardo Barretto (ebarretto)]

tested locally and looks good to me!
thanks!

review: Approve

diff --git a/scripts/oval_lib.py b/scripts/oval_lib.py
index 9127dad..a33cca2 100755
--- a/scripts/oval_lib.py
+++ b/scripts/oval_lib.py
@@ -494,23 +494,29 @@ class USN:
 
         self.lp_bugs = lp_bugs
         self.cves = cve_objs
-        self.pkgs = self._generate_pkg_fixed_ver_tuple_dict(pkgs_by_rel)
-
-    def _generate_pkg_fixed_ver_tuple_dict(self, pkgs_by_rel):
-        tup_dict = {}
-        for rel, pkgs in pkgs_by_rel.items():
-            tup_dict[rel] = {}
-            for src_name, pkg_object in pkgs.items():
-                fixed_ver = self.releases[rel]['sources'][src_name]['version']
-                tup_dict[rel][src_name] = (pkg_object, fixed_ver)
-        return tup_dict
+        self.pkgs = pkgs_by_rel
 
     def get_pkg_fixed_version(self, package: Package):
+        """Get the fixed version of a given package in the USN"""
         if not self.is_package_present(package): return None
-        return self.pkgs[package.rel][package.name][1]
+        for rel in self.pkgs:
+            if package.name in self.pkgs[rel]:
+                return self.pkgs[rel][package.name][1]
 
     def is_package_present(self, package: Package):
-        return package.rel in self.pkgs and package.name in self.pkgs[package.rel]
+        """Check if a package is present in the USN, regardless of the release"""
+        for rel in self.pkgs:
+            if package.name in self.pkgs[rel]:
+                return True
+
+        return False
+
+    def get_package_release_in_usn(self, package: Package):
+        """Get the release of a given package in the USN"""
+        if not self.is_package_present(package): return None
+        for rel in self.pkgs:
+            if package.name in self.pkgs[rel]:
+                return self.pkgs[rel][package.name][0].rel
 
     def __str__(self) -> str:
         # return f'description: {self.description}\nid: {self.id}\ncves: {self.cves}\npkgs: {self.pkgs}\n' # TODO: remove this ugly debug print
@@ -593,9 +599,11 @@ class OvalGenerator:
     def _get_objects_including_parents(self, objects) -> dict:
         all_objs = dict()
         for parent_release in self.parent_releases[::-1]:
-            all_objs.update(objects[parent_release])
+            if parent_release in objects:
+                all_objs.update(objects[parent_release])
 
-        all_objs.update(objects[self.release])
+        if self.release in objects:
+            all_objs.update(objects[self.release])
         return all_objs
 
     def _add_structure(self, root) -> None:
@@ -1799,33 +1807,82 @@ class OvalGeneratorUSNs(OvalGenerator):
     def _load_usns(self, usn_database):
         usns = {}
         # go thru every USN in the JSON
-        for usn_id, usn_data in dict(sorted(usn_database.items())).items():
+        for usn_id, usn_data in usn_database.items():
             # take existing CVE and Package objects
-            cve_objs = {}
-            pkg_objs_by_rels = {}
+            cves = {}
+            pkgs = {}
             lp_bugs = []
+            # Package - fixed version loading
             for rel, info in usn_data['releases'].items():
-                # CVE stays the same across releases
-                for cve in usn_data['cves']:
-                    if 'launchpad' in cve:
-                        lp_bugs.append(cve)
-                    elif cve_objs.get(cve) is None:
-                        try:
-                            cve_objs[cve] = self.cves[rel][cve]
-                        except KeyError:
-                            pass
-                pkg_objs = {}
-                for pkg, _ in info['sources'].items():
-                    try:
-                        pkg_objs[pkg] = self.packages[rel][pkg]
-                    except KeyError:
-                        pass
-                if pkg_objs:
-                    pkg_objs_by_rels[rel] = pkg_objs
+                esm_rels = list(filter(lambda esm_rel: rel in esm_rel and rel != esm_rel, cve_lib.get_active_esm_releases()))
+                all_rels = esm_rels.copy()
+                all_rels.append(rel)
+
+                # Sanity check to skip in case none of the releases was requested
+                rel_exists = False
+                for rel in all_rels:
+                    if rel in self.releases: rel_exists = True
+                if not rel_exists: continue
+
+
+                # Get package
+                for pkg, pkg_info in info['sources'].items():
+                    pkg_obj = None
+                    if 'esm' in pkg_info['version']:
+                        for current_rel in esm_rels:
+                            if current_rel not in self.packages: continue
+                            if pkg not in self.packages[current_rel]: continue
+                            pkg_obj = self.packages[current_rel][pkg]
+                            break
+                        if pkg_obj:
+                            pkgs.setdefault(current_rel, {})
+                            pkgs[current_rel][pkg] = (pkg_obj, pkg_info['version'])
+                    else:
+                        if rel not in self.packages: continue
+                        if pkg not in self.packages[rel]: continue
+                        pkg_obj = self.packages[rel][pkg]
+                        pkgs.setdefault(rel, {})
+                        pkgs[rel][pkg] = (pkg_obj, pkg_info['version'])
+
+            # CVE loading
+            for cve_text in usn_data['cves']:
+                if 'launchpad.net' in cve_text:
+                    lp_bugs.append(cve_text)
+                    continue
+                elif not re.search(r'CVE-\d{4}-\d{4,7}', cve_text):
+                    continue
+
+                cve = None
+                for rel in usn_data['releases']:
+                    if rel not in self.cves or \
+                        cve_text not in self.cves[rel]: continue
+                    cve = self.cves[rel][cve_text]
+                    break
+
+                if not cve: continue
+
+                # Was the CVE fixed in this USN for any of the packages of the considered releases?
+                for pkg_rel_entry in cve.pkg_rel_entries.values():
+                    if pkg_rel_entry.orig_status != 'released' and 'regression' not in usn_data['title']: continue
+                    if 'esm' in pkg_rel_entry.release:
+                        rel = cve_lib.get_orig_rel_name(pkg_rel_entry.release)
+                    else:
+                        rel = pkg_rel_entry.release
+                    if rel not in usn_data['releases'] or \
+                        pkg_rel_entry.pkg.name not in usn_data['releases'][rel]['sources']: continue
+                    # We have some cases like -2 USNs or wrong versions
+                    # so we cannot compare against CVE version, TBD once
+                    # we have fixed USN wrong versions.
+                    cves.setdefault(pkg_rel_entry.release, {})
+                    pkg_info = usn_data['releases'][rel]['sources'][pkg_rel_entry.pkg.name]
+                    cves[pkg_rel_entry.release][cve.number] = cve
+
             # create a USN object with fields in the USN and
             # corresponding CVEs and Packages
-            usns[usn_id] = USN(usn_data, cve_objs, pkg_objs_by_rels, lp_bugs)
-        return usns
+            if len(pkgs) == 0: continue
+            usns[usn_id] = USN(usn_data, cves, pkgs, lp_bugs)
+
+        return dict(sorted(usns.items()))
 
     def _generate_advisory(self, usn: USN) -> etree.Element:
         severities = ['low', 'medium', 'high', 'critical']
@@ -1833,7 +1890,7 @@ class OvalGeneratorUSNs(OvalGenerator):
         severity = etree.SubElement(advisory, "severity")
         issued = etree.SubElement(advisory, "issued")
         severity = None
-        for cve in usn.cves:
+        for cve in usn.cves[self.release]:
             cve_obj = self._generate_cve_tag(self.cves[cve])
             advisory.append(cve_obj)
 
@@ -2731,10 +2788,12 @@ class JSONPkgGenerator(OvalGeneratorUSNs):
         self.generator_type = 'json_pkg'
 
     def _generate_usn_info(self, usn: USN):
+        cves = list(self._get_objects_including_parents(usn.cves).keys())
+        if len(cves) == 0 and len(usn.lp_bugs) == 0: return {} # If no CVEs or LP bugs, don't add the USN
         return {
             'description': usn.description,
             'published_at': datetime.fromtimestamp(usn.timestamp).isoformat(timespec="seconds"),
-            'related_cves': list(usn.cves.keys()),
+            'related_cves': cves,
             'related_launchpad_bugs': usn.lp_bugs
         }
 
@@ -2794,15 +2853,21 @@ class JSONPkgGenerator(OvalGeneratorUSNs):
 
     def _generate_cve_pkg_info(self, package: Package) -> dict:
         pkg_cves = {}
+        all_releases = self.parent_releases.copy()
+        all_releases.append(self.release)
+
         for cve in package.cves:
             cve_pkg_rel_entry = None
 
             for pkg_rel_entry_id in cve.pkg_rel_entries:
                 pkg_rel_entry_package, pkg_rel_entry_rel = pkg_rel_entry_id.split('/', maxsplit=1)
                 if pkg_rel_entry_package == package.name and \
-                    pkg_rel_entry_rel == package.rel:
-                    cve_pkg_rel_entry = cve.pkg_rel_entries[pkg_rel_entry_id]
-                    break
+                    pkg_rel_entry_rel in all_releases:
+
+                    if cve_pkg_rel_entry and cve_pkg_rel_entry.status != 'fixed':
+                        cve_pkg_rel_entry = cve.pkg_rel_entries[pkg_rel_entry_id]
+                    elif not cve_pkg_rel_entry:
+                        cve_pkg_rel_entry = cve.pkg_rel_entries[pkg_rel_entry_id]
 
             if not cve_pkg_rel_entry:
                 print(f'CVE entry for {package.name} - {package.rel} in {cve.number} not found')
@@ -2820,11 +2885,12 @@ class JSONPkgGenerator(OvalGeneratorUSNs):
         usns_pkg_info = dict()
         usns_regression_pkg_info = dict()
         for usn_id, usn in self.usns.items():
-            if usn.is_package_present(package):
+            if usn.is_package_present(package) and \
+                (usn.get_package_release_in_usn(package) == self.release or \
+                 usn.get_package_release_in_usn(package) in self.parent_releases):
                 usn_pkg_info = {
                         'source_fixed_version': usn.get_pkg_fixed_version(package)
                     }
-
                 if 'regression' in usn.title.lower():
                     usns_regression_pkg_info[f'USN-{usn_id}'] = usn_pkg_info
                 else:
@@ -2899,9 +2965,9 @@ class JSONPkgGenerator(OvalGeneratorUSNs):
         self.release_codename = self.release_codename.replace('/', '_')
         self.parent_releases = self._get_parent_releases(release)
         if self._expand_release(self.release, self.expand):
-             self.output_file = f'com.ubuntu.{self.release.replace("/", "_")}.pkg.json'
+            self.output_file = f'com.ubuntu.{self.release.replace("/", "_")}.pkg.json'
         else:
-             self.output_file = f'com.ubuntu.{self.release_codename}.pkg.json'
+            self.output_file = f'com.ubuntu.{self.release_codename}.pkg.json'
 
     def generate_json(self) -> None:
         for release in self.releases:
diff --git a/test/test_json_generation.py b/test/test_json_generation.py
index 9f44642..6638b5d 100644
--- a/test/test_json_generation.py
+++ b/test/test_json_generation.py
@@ -129,7 +129,6 @@ class EmptyJSONPkgGenerator(JSONPkgGenerator):
         self.pkg_cache = None
         self.expand = True
         pass
-
 #### TESTS ####
 
 @pytest.mark.parametrize("mock_cve", [
@@ -202,55 +201,6 @@ def test_generate_cves_info():
         assert generated_info['cvss_score'] == (float
         (mock_cve.cvss[0]['baseScore']) if mock_cve.cvss else None)
 
-def test_generate_cves_info_parents():
-    cves_jammy = {
-        'CVE-0001-0001': generate_mock_cve(pkgs=dict(), cve_id='CVE-0001-0001'),
-        'CVE-0001-0002': generate_mock_cve(pkgs=dict(), cve_id='CVE-0001-0002', public_date='2020-08-04 17:00:00 PDT'),
-    }
-
-    cves_jammy_esm_apps = {
-        'CVE-0001-0003': generate_mock_cve(pkgs=dict(), cve_id='CVE-0001-0003', public_date='2020-08-04 17:00:00 PST'),
-    }
-
-    cves_jammy_esm_infra = {
-        'CVE-0001-0004': generate_mock_cve(pkgs=dict(), cve_id='CVE-0001-0004', cvss=None)
-    }
-
-    json_gen = EmptyJSONPkgGenerator()
-    json_gen.cves = {'jammy': {}, 'esm-apps/jammy': {}, 'esm-infra/jammy': {}}
-
-    for cve_id, cve in cves_jammy.items():
-        json_gen.cves['jammy'][cve_id] = cve
-
-    for cve_id, cve in cves_jammy_esm_apps.items():
-        json_gen.cves['esm-apps/jammy'][cve_id] = cve
-
-    for cve_id, cve in cves_jammy_esm_infra.items():
-        json_gen.cves['esm-infra/jammy'][cve_id] = cve
-
-
-    json_gen._init_ids('jammy')
-    generated_info_cves = json_gen._generate_cves_info()
-    assert len(generated_info_cves) == 2
-    for cve_id in cves_jammy.keys():
-        assert cve_id in generated_info_cves.keys()
-
-    json_gen._init_ids('esm-infra/jammy')
-    json_gen.parent_releases = ['jammy']
-    generated_info_cves = json_gen._generate_cves_info()
-    assert len(generated_info_cves) == 3
-    for cve_id in list(cves_jammy.keys()) + list(cves_jammy_esm_infra.keys()):
-        assert cve_id in generated_info_cves.keys()
-
-
-    json_gen._init_ids('esm-apps/jammy')
-    json_gen.parent_releases = ['esm-infra/jammy', 'jammy']
-    generated_info_cves = json_gen._generate_cves_info()
-    assert len(generated_info_cves) == 4
-    for cve_id in list(cves_jammy.keys()) + list(cves_jammy_esm_infra.keys()) + list(cves_jammy_esm_apps.keys()):
-        assert cve_id in generated_info_cves.keys()
-
-
 @pytest.mark.parametrize("status,note,json_status,fixed_version", [
     ("ignored", "not for us", "vulnerable", None),
     ("needed", "", "vulnerable", None),
@@ -268,6 +218,7 @@ def test_generate_cve_pkg_info(status, note, json_status, fixed_version):
     cve = generate_mock_cve(pkgs=dict(), cve_id='CVE-0001-0001', priority='critical')
     cve.add_pkg(package, 'jammy', status, note)
     json_gen = EmptyJSONPkgGenerator()
+    json_gen._init_ids('jammy')
 
     info = json_gen._generate_cve_pkg_info(package)
     info = info['CVE-0001-0001']
@@ -280,29 +231,31 @@ def test_generate_cve_pkg_info(status, note, json_status, fixed_version):
     generate_mock_usn(
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None},
+        cve_objs={'jammy':{'CVE-0001-0001': None, 'CVE-0001-0002': None}},
         lp_bugs=['http://bug1.com', 'http://bug2.com']
     ),
     generate_mock_usn(
         description='this is a test description',
         timestamp=1708590783,
+        cve_objs={'jammy':{}},
         lp_bugs=['http://bug1.com', 'http://bug2.com']
     ),
     generate_mock_usn(
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None}
+        cve_objs={'jammy':{'CVE-0001-0001': None, 'CVE-0001-0002': None}},
     ),
 ])
 def test_generate_usn_info(usn):
     # 1708590783 is ~2024-02-22T09:33:03+00:00
     date = datetime.fromtimestamp(1708590783).isoformat(timespec="seconds")
     json_gen = EmptyJSONPkgGenerator()
+    json_gen._init_ids('jammy')
 
     info = json_gen._generate_usn_info(usn)
     assert info['description'] == usn.description
     assert info['published_at'] == date
-    assert info['related_cves'] == list(usn.cves.keys())
+    assert info['related_cves'] == list(usn.cves['jammy'].keys())
     assert info['related_launchpad_bugs'] == usn.lp_bugs
 
 def test_generate_usns_info():
@@ -312,7 +265,7 @@ def test_generate_usns_info():
         id='1000-1',
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None},
+        cve_objs={'jammy':{'CVE-0001-0001': None, 'CVE-0001-0002': None}},
         lp_bugs=['http://bug1.com', 'http://bug2.com']
     ),
     generate_mock_usn(
@@ -325,7 +278,7 @@ def test_generate_usns_info():
         id='1000-3',
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None}
+        cve_objs={'jammy':{'CVE-0001-0001': None, 'CVE-0001-0002': None}},
     )]
 
     wrong_release_usn = generate_mock_usn(
@@ -333,11 +286,20 @@ def test_generate_usns_info():
         description='this is a test description',
         timestamp=1708590783,
         releases=['bionic'],
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None}
+        cve_objs={'bionic':{'CVE-0001-0001': None, 'CVE-0001-0002': None}},
+    )
+
+    other_release = generate_mock_usn(
+        id='3000-1',
+        description='this is a test description',
+        timestamp=1708590783,
+        releases=['jammy', 'bionic'],
+        cve_objs={'jammy': {'CVE-0001-0001': None, 'CVE-0001-0002': None}, 'bionic': {'CVE-0001-0003'}}
     )
 
     final_usns  = usns.copy()
     final_usns.append(wrong_release_usn)
+    final_usns.append(other_release)
     usns_dict = dict(map(lambda usn: (usn.id, usn), final_usns))
     json_gen = EmptyJSONPkgGenerator()
     json_gen.usns = usns_dict
@@ -345,20 +307,25 @@ def test_generate_usns_info():
     info = json_gen._generate_usns_info()
 
     assert 'USN-2000-1' not in info
+    assert 'USN-3000-1' in info and 'CVE-0001-0003' not in info[f'USN-3000-1']['related_cves']
 
     for usn in usns:
         assert f'USN-{usn.id}' in info
         assert info[f'USN-{usn.id}']['description'] == usn.description
         assert info[f'USN-{usn.id}']['published_at'] == date
-        assert info[f'USN-{usn.id}']['related_cves'] == list(usn.cves.keys())
+        if 'jammy' in usn.cves:
+            assert info[f'USN-{usn.id}']['related_cves'] == list(usn.cves['jammy'].keys())
+        else:
+            assert info[f'USN-{usn.id}']['related_cves'] == []
         assert info[f'USN-{usn.id}']['related_launchpad_bugs'] == usn.lp_bugs
+    
 
 def test_generate_usns_info_parents():
     jammy_usns = [generate_mock_usn(
         id='1000-1',
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None},
+        cve_objs={'jammy': {'CVE-0001-0001': None, 'CVE-0001-0002': None}},
         lp_bugs=['http://bug1.com', 'http://bug2.com']
     ),
     generate_mock_usn(
@@ -371,7 +338,8 @@ def test_generate_usns_info_parents():
         id='1000-3',
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None}
+        releases=['jammy'],
+        cve_objs={'jammy': {'CVE-0001-0001': None, 'CVE-0001-0002': None}, 'bionic': {'CVE-0001-0003'}}
     )]
 
     jammy_infra_usns = [
@@ -379,8 +347,8 @@ def test_generate_usns_info_parents():
             id='2000-1',
             description='this is a test description',
             timestamp=1708590783,
-            releases=['esm-infra/jammy'],
-            cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None}
+            releases=['jammy'],
+            cve_objs={'jammy': {'CVE-0001-0001': None, 'CVE-0001-0002': None}, 'bionic': {'CVE-0001-0003'}}
         )
     ]
 
@@ -389,8 +357,8 @@ def test_generate_usns_info_parents():
             id='3000-1',
             description='this is a test description',
             timestamp=1708590783,
-            releases=['esm-apps/jammy'],
-            cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None}
+            releases=['jammy'],
+            cve_objs={'jammy': {'CVE-0001-0001': None, 'CVE-0001-0002': None}, 'bionic': {'CVE-0001-0003'}}
         )
     ]
 
@@ -400,22 +368,8 @@ def test_generate_usns_info_parents():
     json_gen.usns = usns_dict
     json_gen._init_ids('jammy')
     info = json_gen._generate_usns_info()
-    assert len(info.keys()) == 3
-    for usn in jammy_usns:
-        assert f'USN-{usn.id}' in info
-
-    json_gen._init_ids('esm-infra/jammy')
-    json_gen.parent_releases = ['jammy']
-    info = json_gen._generate_usns_info()
-    assert len(info.keys()) == 4
-    for usn in jammy_usns + jammy_infra_usns:
-        assert f'USN-{usn.id}' in info
-
-    json_gen._init_ids('esm-apps/jammy')
-    json_gen.parent_releases = ['esm-infra/jammy', 'jammy']
-    info = json_gen._generate_usns_info()
     assert len(info.keys()) == 5
-    for usn in jammy_usns + jammy_infra_usns + jammy_apps_usns:
+    for usn in jammy_usns:
         assert f'USN-{usn.id}' in info
 
 def test_generate_usn_pkg_info():
@@ -451,9 +405,9 @@ def test_generate_usn_pkg_info():
         id='1000-1',
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None},
+        cve_objs={'jammy': {'CVE-0001-0001': None, 'CVE-0001-0002': None}},
         lp_bugs=['http://bug1.com', 'http://bug2.com'],
-        pkgs_by_rel={'jammy': {'foo': pkg}},
+        pkgs_by_rel={'jammy': {'foo': (pkg, '1.0.0')}},
         releases=releases_1
     )
 
@@ -462,13 +416,15 @@ def test_generate_usn_pkg_info():
         title='Regression',
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None},
+        cve_objs={'jammy': {'CVE-0001-0001': None, 'CVE-0001-0002': None}},
         lp_bugs=['http://bug1.com', 'http://bug2.com'],
-        pkgs_by_rel={'jammy': {'foo': pkg}},
+        pkgs_by_rel={'jammy': {'foo': (pkg, '2.0.0')}},
         releases=releases_2
     )
 
     json_gen = EmptyJSONPkgGenerator()
+    json_gen._init_ids('jammy')
+
     usns_dict = dict(map(lambda usn: (usn.id, usn), [usn, usn_regression]))
     json_gen.usns = usns_dict
 
@@ -482,6 +438,7 @@ def test_generate_usn_pkg_info():
     assert usn_info['USN-1000-1']['source_fixed_version'] == '1.0.0'
     assert usn_reg_info['USN-1000-2']['source_fixed_version'] == '2.0.0'
 
+    json_gen._init_ids('bionic')
     usn_info, usn_reg_info = json_gen._generate_usn_pkg_info(pkg_not)
 
     assert 'USN-1000-1' not in usn_info
@@ -554,20 +511,20 @@ def test_generate_package_info(mocker):
         id='1000-1',
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None},
+        cve_objs={'jammy': {'CVE-0001-0001': None, 'CVE-0001-0002': None}},
         lp_bugs=['http://bug1.com', 'http://bug2.com'],
-        pkgs_by_rel={'jammy': {'foo': pkg}},
+        pkgs_by_rel={'jammy': {'foo': (pkg, '1.0.0')}},
         releases=releases_1
     )
 
     usn_regression = generate_mock_usn(
         id='1000-2',
-        title='Regression',
+        title='A regression',
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None},
+        cve_objs={'bionic': {'CVE-0001-0001': None, 'CVE-0001-0002': None}},
         lp_bugs=['http://bug1.com', 'http://bug2.com'],
-        pkgs_by_rel={'bionic': {'bar': pkg2}},
+        pkgs_by_rel={'bionic': {'bar': (pkg2, '2.0.0')}},
         releases=releases_2
     )
 
@@ -644,9 +601,9 @@ def test_generate_packages_info(mocker):
         id='1000-1',
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None},
+        cve_objs={'jammy': {'CVE-0001-0001': None, 'CVE-0001-0002': None}},
         lp_bugs=['http://bug1.com', 'http://bug2.com'],
-        pkgs_by_rel={'jammy': {'foo': pkg}},
+        pkgs_by_rel={'jammy': {'foo': (pkg, '1.0.0')}},
         releases=releases_1
     )
 
@@ -655,9 +612,9 @@ def test_generate_packages_info(mocker):
         title='Regression',
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None},
+        cve_objs={'bionic': {'CVE-0001-0001': None, 'CVE-0001-0002': None}},
         lp_bugs=['http://bug1.com', 'http://bug2.com'],
-        pkgs_by_rel={'bionic': {'bar': pkg2}},
+        pkgs_by_rel={'bionic': {'bar': (pkg2, '2.0.0')}},
         releases=releases_2
     )
 
@@ -706,7 +663,7 @@ def test_generate_packages_info_parents(mocker):
     }
 
     releases_infra_jammy = {
-        'esm-infra/jammy': {
+        'jammy': {
             'sources': {
                 'bar': {
                     'version': '2.0.0'
@@ -716,7 +673,7 @@ def test_generate_packages_info_parents(mocker):
     }
 
     releases_apps_jammy = {
-        'esm-apps/jammy': {
+        'jammy': {
             'sources': {
                 'dodo': {
                     'version': '3.0.0'
@@ -747,9 +704,9 @@ def test_generate_packages_info_parents(mocker):
         id='1000-1',
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None},
+        cve_objs={'jammy': {'CVE-0001-0001': None, 'CVE-0001-0002': None}},
         lp_bugs=['http://bug1.com', 'http://bug2.com'],
-        pkgs_by_rel={'jammy': {'foo': pkg}},
+        pkgs_by_rel={'jammy': {'foo': (pkg, '1.0.0')}},
         releases=releases_jammy
     )
 
@@ -758,9 +715,9 @@ def test_generate_packages_info_parents(mocker):
         title='Infra',
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None},
+        cve_objs={'jammy': {'CVE-0001-0001': None, 'CVE-0001-0002': None}},
         lp_bugs=['http://bug1.com', 'http://bug2.com'],
-        pkgs_by_rel={'esm-infra/jammy': {'bar': pkg2}},
+        pkgs_by_rel={'jammy': {'bar': (pkg2, '2.0.0')}},
         releases=releases_infra_jammy
     )
 
@@ -769,9 +726,9 @@ def test_generate_packages_info_parents(mocker):
         title='Apps',
         description='this is a test description',
         timestamp=1708590783,
-        cve_objs={'CVE-0001-0001': None, 'CVE-0001-0002': None},
+        cve_objs={'jammy': {'CVE-0001-0001': None, 'CVE-0001-0002': None}},
         lp_bugs=['http://bug1.com', 'http://bug2.com'],
-        pkgs_by_rel={'esm-apps/jammy': {'dodo': pkg3}},
+        pkgs_by_rel={'jammy': {'dodo': (pkg3, '3.0.0')}},
         releases=releases_apps_jammy
     )
 
@@ -787,10 +744,10 @@ def test_generate_packages_info_parents(mocker):
     json_gen = EmptyJSONPkgGenerator()
     usns_dict = dict(map(lambda usn: (usn.id, usn), [usn, usn_infra, usn_apps]))
     json_gen.usns = usns_dict
-    json_gen.cves = {'jammy': {}, 'esm-apps/jammy': {}, 'esm-infra/jammy': {}}
+    json_gen.cves = {'jammy': {}}
     json_gen.cves['jammy']['CVE-0001-0001'] = cve
-    json_gen.cves['esm-infra/jammy']['CVE-0001-0002'] = cve2
-    json_gen.cves['esm-apps/jammy']['CVE-0001-0003'] = cve3
+    json_gen.cves['jammy']['CVE-0001-0002'] = cve2
+    json_gen.cves['jammy']['CVE-0001-0003'] = cve3
 
     json_gen.packages = {
         'jammy': {
@@ -805,25 +762,10 @@ def test_generate_packages_info_parents(mocker):
         }
     }
 
-    json_gen._init_ids('jammy')
-    info = json_gen._generate_packages_info()
-    assert 'foo' in info
-    assert 'bar' not in info
-    assert 'dodo' not in info
-    assert info['foo']['cves']['CVE-0001-0001']['status'] == 'vulnerable'
-
     json_gen._init_ids('esm-infra/jammy')
-    json_gen.parent_releases = ['jammy']
-    info = json_gen._generate_packages_info()
-    assert 'foo' in info
-    assert 'bar' in info
-    assert 'dodo' not in info
-    assert info['foo']['cves']['CVE-0001-0001']['status'] == 'vulnerable'
-    assert info['bar']['cves']['CVE-0001-0002']['status'] == 'fixed'
-    assert info['bar']['ubuntu_security_notices']['USN-2000-1']['source_fixed_version'] == '2.0.0'
+    json_gen.parent_releases = ['esm-apps/jammy', 'jammy']
+    json_gen.orig_parent = 'jammy'
 
-    json_gen._init_ids('esm-apps/jammy')
-    json_gen.parent_releases = ['esm-infra/jammy', 'jammy']
     info = json_gen._generate_packages_info()
     assert 'foo' in info
     assert 'bar' in info
@@ -832,8 +774,13 @@ def test_generate_packages_info_parents(mocker):
     assert info['bar']['cves']['CVE-0001-0002']['status'] == 'fixed'
     assert info['dodo']['cves']['CVE-0001-0003']['status'] == 'fixed'
     assert info['foo']['cves']['CVE-0001-0001']['source_fixed_version'] == '4'
+    assert info['bar']['cves']['CVE-0001-0002']['source_fixed_version'] == '1'
+    assert info['dodo']['cves']['CVE-0001-0003']['source_fixed_version'] == '2'
+    assert info['foo']['ubuntu_security_notices']['USN-1000-1']['source_fixed_version'] == '1.0.0'
+    assert info['bar']['ubuntu_security_notices']['USN-2000-1']['source_fixed_version'] == '2.0.0'
     assert info['dodo']['ubuntu_security_notices']['USN-3000-1']['source_fixed_version'] == '3.0.0'
 
+
 @pytest.mark.parametrize("expand,release,filename", [
     (False, 'esm-apps/bionic', 'com.ubuntu.bionic.pkg.json'),
     (True, 'esm-apps/bionic', 'com.ubuntu.esm-apps_bionic.pkg.json'),