Merge ~litios/ubuntu-cve-tracker:ppas-tag-migration into ubuntu-cve-tracker:master

[David Fernandez Gonzalez]

Due to https://code.launchpad.net/~litios/ubuntu-security-customer-ppa-tracking/+git/ubuntu-security-customer-ppa-tracking/+merge/453041, we need to update the loading of the external subprojects accordingly to the new format.

[Emilia Torino (emitorino)]

LGTM, thanks!

review: Approve

[Eduardo Barretto (ebarretto)]

As we've spoken, the current changes break generate_pkg_cache.py
It would also be great to handle the pockets correctly in that same script, otherwise everything will show up as "Release" pocket.
Thanks for making these changes

review: Needs Fixing

be2fc23... by David Fernandez Gonzalez on 2023-10-16

Add pockets to subprojects

Signed-off-by: David Fernandez Gonzalez <email address hidden>

8d5aa38... by David Fernandez Gonzalez on 2023-10-26

cve_lib: remove /ubuntu from subproject ppa names

Signed-off-by: David Fernandez Gonzalez <email address hidden>

[Eduardo Barretto (ebarretto)]

lgtm, thanks

CI failures are not related to this PR

review: Approve

diff --git a/scripts/cve_lib.py b/scripts/cve_lib.py
index fb9a744..7299ac0 100755
--- a/scripts/cve_lib.py
+++ b/scripts/cve_lib.py
@@ -106,7 +106,7 @@ subprojects = {
         "name": "Ubuntu 12.04 ESM",
         "codename": "Precise Pangolin",
         "alias": "precise/esm",
-        "ppa": "ubuntu-esm/esm",
+        "ppas": [{ "ppa": "ubuntu-esm/esm", "pocket": "security"}],
         "parent": "ubuntu/precise",
         "description": "Available with UA Infra or UA Desktop: https://ubuntu.com/advantage",
         "stamp": 1493521200,
@@ -118,7 +118,10 @@ subprojects = {
         "name": "Ubuntu 14.04 ESM",
         "codename": "Trusty Tahr",
         "alias": "trusty/esm",
-        "ppa": "ubuntu-esm/esm-infra-security",
+        "ppas": [
+                 {"ppa": "ubuntu-esm/esm-infra-security", "pocket": "security"}, 
+                 {"ppa": "ubuntu-esm/esm-infra-updates",  "pocket": "updates"}
+                ],
         "parent": "ubuntu/trusty",
         "description": "Available with Ubuntu Pro (Infra-only): https://ubuntu.com/pro",
         "stamp": 1556593200,
@@ -130,7 +133,10 @@ subprojects = {
         "packages": ["esm-infra-xenial-supported.txt"],
         "name": "Ubuntu 16.04 ESM",
         "codename": "Xenial Xerus",
-        "ppa": "ubuntu-esm/esm-infra-security",
+        "ppas": [
+                 {"ppa": "ubuntu-esm/esm-infra-security", "pocket": "security"}, 
+                 {"ppa": "ubuntu-esm/esm-infra-updates",  "pocket": "updates"}
+                ],
         "parent": "ubuntu/xenial",
         "description": "Available with Ubuntu Pro (Infra-only): https://ubuntu.com/pro",
         "stamp": 1618963200,
@@ -142,7 +148,10 @@ subprojects = {
         "packages": ["esm-infra-bionic-supported.txt"],
         "name": "Ubuntu 18.04 ESM",
         "codename": "Bionic Beaver",
-        "ppa": "ubuntu-esm/esm-infra-security",
+        "ppas": [
+                 {"ppa": "ubuntu-esm/esm-infra-security", "pocket": "security"}, 
+                 {"ppa": "ubuntu-esm/esm-infra-updates",  "pocket": "updates"}
+                ],
         "parent": "ubuntu/bionic",
         "description": "Available with Ubuntu Pro (Infra-only): https://ubuntu.com/pro",
         "stamp": 1685539024,
@@ -154,7 +163,10 @@ subprojects = {
         "packages": ["esm-apps-xenial-supported.txt"],
         "name": "Ubuntu 16.04 ESM",
         "codename": "Xenial Xerus",
-        "ppa": "ubuntu-esm/esm-apps-security",
+        "ppas": [
+                 {"ppa": "ubuntu-esm/esm-apps-security", "pocket": "security"}, 
+                 {"ppa": "ubuntu-esm/esm-apps-updates",  "pocket": "updates"}
+                ],
         "parent": "esm-infra/xenial",
         "description": "Available with Ubuntu Pro: https://ubuntu.com/pro",
         "stamp": 1618963200,
@@ -166,7 +178,10 @@ subprojects = {
         "packages": ["esm-apps-bionic-supported.txt"],
         "name": "Ubuntu 18.04 ESM",
         "codename": "Bionic Beaver",
-        "ppa": "ubuntu-esm/esm-apps-security",
+        "ppas": [
+                 {"ppa": "ubuntu-esm/esm-apps-security", "pocket": "security"}, 
+                 {"ppa": "ubuntu-esm/esm-apps-updates",  "pocket": "updates"}
+                ],
         "parent": "esm-infra/bionic",
         "description": "Available with Ubuntu Pro: https://ubuntu.com/pro",
         "stamp": 1524870000,
@@ -178,7 +193,10 @@ subprojects = {
         "packages": ["esm-apps-focal-supported.txt"],
         "name": "Ubuntu 20.04 ESM",
         "codename": "Focal Fossa",
-        "ppa": "ubuntu-esm/esm-apps-security",
+        "ppas": [
+                 {"ppa": "ubuntu-esm/esm-apps-security", "pocket": "security"}, 
+                 {"ppa": "ubuntu-esm/esm-apps-updates",  "pocket": "updates"}
+                ],
         "parent": "ubuntu/focal",
         "description": "Available with Ubuntu Pro: https://ubuntu.com/pro",
         "stamp": 1587567600,
@@ -190,7 +208,10 @@ subprojects = {
         "packages": ["esm-apps-jammy-supported.txt"],
         "name": "Ubuntu 22.04 ESM",
         "codename": "Jammy Jellyfish",
-        "ppa": "ubuntu-esm/esm-apps-security",
+        "ppas": [
+                 {"ppa": "ubuntu-esm/esm-apps-security", "pocket": "security"}, 
+                 {"ppa": "ubuntu-esm/esm-apps-updates",  "pocket": "updates"}
+                ],
         "parent": "ubuntu/jammy",
         "description": "Available with Ubuntu Pro: https://ubuntu.com/pro",
         "stamp": 1650693600,
@@ -201,7 +222,7 @@ subprojects = {
         "packages": ["fips-xenial-supported.txt"],
         "name": "Ubuntu 16.04 FIPS Certified",
         "codename": "Xenial Xerus",
-        "ppa": "ubuntu-advantage/fips",
+        "ppas": [{"ppa" : "ubuntu-advantage/fips", "pocket": "security"}],
         "parent": "ubuntu/xenial",
         "description": "Available with Ubuntu Pro: https://ubuntu.com/pro",
     },
@@ -211,7 +232,7 @@ subprojects = {
         "packages": ["fips-bionic-supported.txt"],
         "name": "Ubuntu 18.04 FIPS Certified",
         "codename": "Bionic Beaver",
-        "ppa": "ubuntu-advantage/fips",
+        "ppas": [{"ppa" : "ubuntu-advantage/fips", "pocket": "security"}],
         "parent": "ubuntu/bionic",
         "description": "Available with Ubuntu Pro: https://ubuntu.com/pro",
     },
@@ -221,7 +242,7 @@ subprojects = {
         "packages": ["fips-focal-supported.txt"],
         "name": "Ubuntu 20.04 FIPS Certified",
         "codename": "Focal Fossa",
-        "ppa": "ubuntu-advantage/fips",
+        "ppas": [{"ppa" : "ubuntu-advantage/fips", "pocket": "security"}],
         "parent": "ubuntu/bionic",
         "description": "Available with Ubuntu Pro: https://ubuntu.com/pro",
     },
@@ -231,7 +252,7 @@ subprojects = {
         "packages": ["fips-updates-xenial-supported.txt"],
         "name": "Ubuntu 16.04 FIPS Compliant",
         "codename": "Xenial Xerus",
-        "ppa": "ubuntu-advantage/fips-updates",
+        "ppas": [{"ppa" : "ubuntu-advantage/fips-updates", "pocket": "updates"}],
         "parent": "ubuntu/xenial",
         "description": "Available with Ubuntu Pro: https://ubuntu.com/pro",
     },
@@ -241,7 +262,7 @@ subprojects = {
         "packages": ["fips-updates-bionic-supported.txt"],
         "name": "Ubuntu 18.04 FIPS Compliant",
         "codename": "Bionic Beaver",
-        "ppa": "ubuntu-advantage/fips-updates",
+        "ppas": [{"ppa" : "ubuntu-advantage/fips-updates", "pocket": "updates"}],
         "parent": "ubuntu/bionic",
         "description": "Available with Ubuntu Pro: https://ubuntu.com/pro",
     },
@@ -251,7 +272,7 @@ subprojects = {
         "packages": ["fips-updates-focal-supported.txt"],
         "name": "Ubuntu 20.04 FIPS Compliant",
         "codename": "Focal Fossa",
-        "ppa": "ubuntu-advantage/fips-updates",
+        "ppas": [{"ppa" : "ubuntu-advantage/fips-updates", "pocket": "updates"}],
         "parent": "ubuntu/bionic",
         "description": "Available with Ubuntu Pro: https://ubuntu.com/pro",
     },
@@ -262,7 +283,7 @@ subprojects = {
         "name": "Ubuntu 16.04 ROS ESM",
         "codename": "Xenial Xerus",
         "alias": "ros-esm/xenial",
-        "ppa": "ubuntu-robotics-packagers/ros-security",
+        "ppas": [{"ppa": "ubuntu-robotics-packagers/ros-security", "pocket": "security"}],
         "parent": "ubuntu/xenial",
         "description": "Available with Ubuntu Advantage: https://ubuntu.com/advantage",
     },
@@ -273,7 +294,7 @@ subprojects = {
         "name": "Ubuntu 18.04 ROS ESM",
         "codename": "Bionic Beaver",
         "alias": "ros-esm/bionic",
-        "ppa": "ubuntu-robotics-packagers/ros-security",
+        "ppas": [{"ppa": "ubuntu-robotics-packagers/ros-security", "pocket": "security"}],
         "parent": "ubuntu/bionic",
         "description": "Available with Ubuntu Advantage: https://ubuntu.com/advantage",
     },
@@ -755,17 +776,30 @@ def get_subproject_details_by_ppa_url_and_series(url, series):
         prod, ser = product_series(rel)
         if ser == series:
             try:
-                if subprojects[rel]["ppa"] in url:
-                    product = prod
-                    canon = product + "/" + series
-                    details = subprojects[rel]
-                    break
+                for ppa_pocket in subprojects[rel]["ppas"]: 
+                    if ppa_pocket['ppa'] in url:
+                        product = prod
+                        canon = product + "/" + series
+                        details = subprojects[rel]
+                        break
             except KeyError:
                 pass
             if details is not None:
                 break
     return canon, product, series, details
 
+def release_ppa_pocket(rel, ppa):
+    _, _, _, details = get_subproject_details(rel)
+    pocket = None
+    try:
+        for ppa_pocket in details["ppas"]:
+            if ppa_pocket["ppa"] == ppa:
+                pocket = ppa_pocket["pocket"]
+                break
+    except (KeyError, TypeError):
+        pass
+    return pocket
+
 def release_name(rel):
     name = None
     _, _, _, details = get_subproject_details(rel)
@@ -832,16 +866,16 @@ def release_version(rel):
             if details:
                 version = details["version"]
     return version
-
-def release_ppa(rel):
-    """Return the ppa for a given subproject."""
-    ppa = None
+    
+def release_ppas(rel):
+    """Return the ppas for a given subproject."""
+    ppas = None
     _, _, _, details = get_subproject_details(rel)
     try:
-        ppa = details["ppa"]
+        ppas = details["ppas"]
     except (KeyError, TypeError):
         pass
-    return ppa
+    return ppas
 
 def needs_oval(rel):
     """Return if OVAL should be generated for a given subproject"""
@@ -931,7 +965,7 @@ def find_external_subproject_cves(cve):
 # should follow the same as any other subproject
 # except for the extra 'product' and 'release' keys.
 MANDATORY_EXTERNAL_SUBPROJECT_KEYS = ['cve_triage', 'cve_patching', 'cve_notification', 'security_updates_notification', 'binary_copies_only', 'seg_support', 'owners']
-MANDATORY_EXTERNAL_SUBPROJECT_PPA_KEYS = ['ppa', 'oval', 'product', 'release', 'supported_packages']
+MANDATORY_EXTERNAL_SUBPROJECT_PPA_KEYS = ['ppas', 'oval', 'product', 'release', 'supported_packages']
 OPTIONAL_EXTERNAL_SUBPROJECT_PPA_KEYS =  ['parent', 'name', 'codename', 'description', 'aliases', 'archs']
 
 def load_external_subprojects(strict=False):
@@ -976,8 +1010,8 @@ def load_external_subprojects(strict=False):
             else:
                 support_metadata[key] = main_config[key]
 
-        for ppa in main_config['ppas']:
-            config = main_config['ppas'][ppa]
+        for subproject in main_config['subprojects']:
+            config = main_config['subprojects'][subproject]
             if 'product' not in config or 'release' not in config:
                 if strict:
                     raise ValueError(error_msg)
@@ -985,39 +1019,38 @@ def load_external_subprojects(strict=False):
                     print(error_msg, file=sys.stderr)
                 error_msg = '%s: missing "product" or "release".' % (subproject_path)
 
-            subproject_name = ppa
-            external_releases.append(subproject_name)
-            subprojects.setdefault(subproject_name, {"packages": [],
+            external_releases.append(subproject)
+            subprojects.setdefault(subproject, {"packages": [],
                                         "eol": False})
             # an external subproject can append to an internal one
-            subprojects[subproject_name]["packages"].append(\
+            subprojects[subproject]["packages"].append(\
                 os.path.join(subproject_path, config['supported_packages']))
 
             # check if aliases for packages exist
             if 'aliases' in config:
-                subprojects[subproject_name].setdefault("aliases", \
+                subprojects[subproject].setdefault("aliases", \
                     os.path.join(subproject_path, config['aliases']))
 
             for key in MANDATORY_EXTERNAL_SUBPROJECT_PPA_KEYS + OPTIONAL_EXTERNAL_SUBPROJECT_PPA_KEYS:
                 if key in config:
-                    subprojects[subproject_name].setdefault(key, config[key])
+                    subprojects[subproject].setdefault(key, config[key])
                 elif key in OPTIONAL_EXTERNAL_SUBPROJECT_PPA_KEYS:
-                    _, _, _, original_release_details = get_subproject_details(subprojects[subproject_name]['release'])
+                    _, _, _, original_release_details = get_subproject_details(subprojects[subproject]['release'])
                     if original_release_details and key in original_release_details:
-                        subprojects[subproject_name].setdefault(key, original_release_details[key])
+                        subprojects[subproject].setdefault(key, original_release_details[key])
                 else:
                     error_msg = '%s missing "%s" field.' % (subproject_path, key)
-                    del subprojects[subproject_name]
-                    external_releases.remove(subproject_name)
+                    del subprojects[subproject]
+                    external_releases.remove(subproject)
                     if strict:
                         raise ValueError(error_msg)
                     else:
                         print(error_msg, file=sys.stderr)
 
-            subprojects[subproject_name].setdefault("support_metadata", support_metadata)
-            project = read_external_subproject_details(subproject_name)
+            subprojects[subproject].setdefault("support_metadata", support_metadata)
+            project = read_external_subproject_details(subproject)
             if project:
-                subprojects[subproject_name].setdefault("customer", project)
+                subprojects[subproject].setdefault("customer", project)
 
 load_external_subprojects()
 
@@ -1068,8 +1101,8 @@ def release_is_older_than(release_a, release_b):
 flavor_releases = [
     'lucid', 'precise', 'trusty', 'utopic', 'vivid', 'wily', 'xenial',
     'yakkety', 'zesty', 'artful', 'bionic', 'cosmic', 'disco', 'eoan',
-    'focal', 'groovy', 'hirsute', 'impish', 'jammy', 'kinetic', "lunar",
-    'mantic', 'noble',
+    'focal', 'groovy', 'hirsute', 'impish', 'jammy', 'kinetic', 'lunar',
+    'mantic','noble',
 ]
 
 all_releases = release_sort(all_releases)
diff --git a/scripts/generate_pkg_cache.py b/scripts/generate_pkg_cache.py
index 27404f7..672e614 100644
--- a/scripts/generate_pkg_cache.py
+++ b/scripts/generate_pkg_cache.py
@@ -11,7 +11,7 @@
 # for details.
 #
 
-from cve_lib import (all_releases, devel_release, eol_releases, needs_oval, product_series, release_parent, release_ppa)
+from cve_lib import (all_releases, devel_release, eol_releases, needs_oval, product_series, release_parent, release_ppas, release_ppa_pocket)
 
 import argparse
 import datetime
@@ -55,31 +55,11 @@ def write_to_cache(cache_dir, release, cache):
     except Exception:
         error(f"Could not write to JSON file: {filename}")
 
-
-def update_cache(release, cache, ppa=None, latest_date_created=None):
-    lp = lpl_common.connect(version='devel')
-    ubuntu = lp.distributions['ubuntu']
-    series = ubuntu.getSeries(name_or_version=product_series(release)[1])
-
-    real_threshold = None
-    if latest_date_created:
-        # Allow a grace period to cope with publications arriving out of
-        # order during long transactions.
-        real_threshold = latest_date_created - datetime.timedelta(hours=1)
-
-    if ppa:
-        archive, group, ppa_full_name = lpl_common.get_archive(
-            ppa,
-            lp,
-            False,
-            distribution=ubuntu
-        )
-    else:
-        archive = ubuntu.main_archive
-
+def get_binaries_rel(release, archive, real_threshold, series, cache, latest_date_created, ppa=None):
     debug(f"Retrieving Launchpad publications since {real_threshold}")
     sources = archive.getPublishedSources(order_by_date=True, created_since_date=real_threshold, distro_series=series)
     for s in sources:
+        override_pocket = None
         if s.pocket not in pockets:
             continue
 
@@ -90,15 +70,16 @@ def update_cache(release, cache, ppa=None, latest_date_created=None):
         src = s.source_package_name
         src_ver = s.source_package_version
         src_component = None
-        if not ppa:
+        if ppa:
             src_component = s.component_name
+            override_pocket = release_ppa_pocket(release, ppa)
 
         binaries = s.getPublishedBinaries(active_binaries_only=False)
         for b in binaries:
             bin_name = b.binary_package_name
             bin_version = b.binary_package_version
             bin_component = b.component_name
-            pocket = b.pocket
+            pocket = b.pocket if not ppa else override_pocket
             bin_arch = b.display_name.split(' ')[-1]
 
             if src not in cache:
@@ -120,6 +101,33 @@ def update_cache(release, cache, ppa=None, latest_date_created=None):
 
     return latest_date_created
 
+def update_cache(release, cache, ppas=None, latest_date_created=None):
+    lp = lpl_common.connect(version='devel')
+    ubuntu = lp.distributions['ubuntu']
+    series = ubuntu.getSeries(name_or_version=product_series(release)[1])
+
+    real_threshold = None
+    if latest_date_created:
+        # Allow a grace period to cope with publications arriving out of
+        # order during long transactions.
+        real_threshold = latest_date_created - datetime.timedelta(hours=1)
+
+    if ppas:
+        for ppa_pocket in ppas:
+            ppa = ppa_pocket['ppa']
+            archive, _, _ = lpl_common.get_archive(
+                ppa.split('/ubuntu')[0],
+                lp,
+                False,
+                distribution=ubuntu
+            )
+
+            latest_date_created = get_binaries_rel(release, archive, real_threshold, series, cache, latest_date_created, ppa)
+    else:
+        latest_date_created = get_binaries_rel(release, ubuntu.main_archive, real_threshold, series, cache, latest_date_created)
+
+    return latest_date_created
+
 
 def warn(message):
     """ print a warning message """
@@ -186,8 +194,8 @@ def main():
 
             debug('UPDATING CACHE')
 
-            ppa = release_ppa(release)
-            latest_date_created = update_cache(release, cache, ppa, latest_date_created)
+            ppas = release_ppas(release)
+            latest_date_created = update_cache(release, cache, ppas, latest_date_created)
 
             if latest_date_created is not None:
                 epoch = datetime.datetime.fromtimestamp(0, tz=datetime.timezone.utc)
diff --git a/scripts/test_source_map.py b/scripts/test_source_map.py
index 8e99825..e05117c 100755
--- a/scripts/test_source_map.py
+++ b/scripts/test_source_map.py
@@ -13,7 +13,7 @@ mock_subprojects = {
         "packages": ["test/esm-fake-supported.txt"],
         "name": "Ubuntu 01.01 ESM",
         "codename": "Fabulous Fake",
-        "ppa": "ubuntu-esm/esm-bar-security/ubuntu",
+        "ppas": [{"ppa": "ubuntu-esm/esm-bar-security/ubuntu", "pocket": "main"}],
         "parent": "ubuntu/bar",
         "description": "Available with Ubuntu Pro: https://ubuntu.com/pro",
         "stamp": 1618963200,