Merge ~litios/ubuntu-cve-tracker:unify-subprojects-info into ubuntu-cve-tracker:master

[David Fernandez Gonzalez]

Changes to support the new structure of subprojects. Please check https://code.launchpad.net/~ubuntu-security/ubuntu-security-customer-ppa-tracking/+git/ubuntu-security-customer-ppa-tracking/+merge/449838

[Emilia Torino (emitorino)]

Minor comment, otherwise LGTM!

review: Approve

f0e3987... by David Fernandez Gonzalez on 2023-08-25

cve_lib: updating subproject file names to new ones

Signed-off-by: David Fernandez Gonzalez <email address hidden>

8ccfc81... by David Fernandez Gonzalez on 2023-08-25

cve_lib: update documentation in load_external_subprojects

[David Fernandez Gonzalez (litios)]

A new commit was added to fix the issues described!

Another one was added to also update the documentation inside of the loading function,
as we no longer have the 2 directory structure.

[Emilia Torino (emitorino)]

Thanks!!

diff --git a/scripts/cve_lib.py b/scripts/cve_lib.py
index f37ef73..a8f0516 100755
--- a/scripts/cve_lib.py
+++ b/scripts/cve_lib.py
@@ -876,7 +876,7 @@ def get_external_subproject_dir(subproject):
 
 def read_external_subproject_config(subproject_dir):
     """Read and return the configuration for the given subproject directory."""
-    config_yaml = os.path.join(subproject_dir, "config.yaml")
+    config_yaml = os.path.join(subproject_dir, "config.yml")
     with open(config_yaml) as cfg:
         return yaml.safe_load(cfg)
 
@@ -914,11 +914,12 @@ def find_external_subproject_cves(cve):
                 cves.append(path)
     return cves
 
-# Keys in config.yaml for a external subproject
+# Keys in config.yml for a external subproject
 # should follow the same as any other subproject
 # except for the extra 'product' and 'release' keys.
-MANDATORY_EXTERNAL_SUBPROJECT_KEYS = ['ppa', 'oval', 'product', 'release']
-OPTIONAL_EXTERNAL_SUBPROJECT_KEYS =  ['parent', 'name', 'codename', 'description']
+MANDATORY_EXTERNAL_SUBPROJECT_KEYS = ['cve_triage', 'cve_patching', 'cve_notification', 'security_updates_notification', 'binary_copies_only', 'seg_support', 'owners']
+MANDATORY_EXTERNAL_SUBPROJECT_PPA_KEYS = ['ppa', 'oval', 'product', 'release', 'supported_packages']
+OPTIONAL_EXTERNAL_SUBPROJECT_PPA_KEYS =  ['parent', 'name', 'codename', 'description', 'aliases', 'archs']
 
 def load_external_subprojects():
     """Search for and load subprojects into the global subprojects dict.
@@ -926,29 +927,39 @@ def load_external_subprojects():
     Search for and load subprojects into the global subprojects dict.
 
     A subproject is defined as a directory which resides within
-    subprojects_dir and contains a supported.txt file. It can also contain
-    a project.yml file which specifies configuration directives for the
-    project as well as snippet CVE files. By convention, a subproject is
-    usually defined as the combination of a product and series, ie:
+    subprojects_dir and references a supported.txt file and a PPA.
+    This information is stored in config.yml, which contains all the
+    information in regards the subproject. It can also contain
+    a project.yml file which specifies metadata for the project as well
+    as snippet CVE files. By convention, a subproject is usually defined
+    as the combination of a product and series, ie:
 
     esm-apps/focal
 
     as such in this case there would expect to be within subprojects_dir a
-    directory called esm-apps/ and within that a subdirectory called
-    focal/. Inside this focal/ subdirectory a supported.txt file would list
-    the packages which are supported by the esm-apps/focal subproject. By
-    convention, snippet CVE files should reside within the esm-apps/
-    project directory rather than the esm-apps/focal/ subdirectory to avoid
-    unnecessary fragmentation across different subproject series.
-
+    directory called esm-apps/ and within that, in the config.yml, an entry
+    of type 'esm-apps/focal'. Inside this entry, a reference to the designated
+    supported.txt file, which would list the packages which are supported by
+    the esm-apps/focal subproject. By convention, snippet CVE files should
+    reside within the esm-apps/ project directory.
     """
-    for supported_txt in find_files_recursive(subprojects_dir, "supported.txt"):
-        try:
-            # use config to populate other parts of the
-            # subproject settings
-            subproject_path = supported_txt[:-len("supported.txt")-1]
-            config = read_external_subproject_config(subproject_path)
-
+    for config_yaml in find_files_recursive(subprojects_dir, "config.yml"):
+        subproject_path = config_yaml[:-len("config.yml")-1]
+        # use config to populate other parts of the
+        # subproject settings
+        main_config = read_external_subproject_config(subproject_path)
+        support_metadata = {}
+
+        # Disable this check until we have the information available
+        # for key in MANDATORY_EXTERNAL_SUBPROJECT_KEYS:
+        #     if key not in main_config:
+        #         print('%s missing "%s" field.' % (subproject_path, key))
+        #         raise ValueError
+        #     else:
+        #         support_metadata[key] = main_config[key]
+
+        for ppa in main_config['ppas']:
+            config = main_config['ppas'][ppa]
             if 'product' not in config or 'release' not in config:
                 print('%s: missing "product" or "release".' % (subproject_path))
                 raise ValueError
@@ -956,19 +967,20 @@ def load_external_subprojects():
             subproject_name = '%s/%s' % (config["product"], config["release"])
             external_releases.append(subproject_name)
             subprojects.setdefault(subproject_name, {"packages": [],
-                                         "eol": False})
+                                        "eol": False})
             # an external subproject can append to an internal one
-            subprojects[subproject_name]["packages"].append(supported_txt)
+            subprojects[subproject_name]["packages"].append(\
+                os.path.join(subproject_path, config['supported_packages']))
 
             # check if aliases for packages exist
-            if os.path.isfile(supported_txt[:-len("supported.txt")] + 'aliases.yaml'):
-                subprojects[subproject_name].setdefault("aliases",
-                    supported_txt[:-len("supported.txt")] + 'aliases.yaml')
+            if 'aliases' in config:
+                subprojects[subproject_name].setdefault("aliases", \
+                    os.path.join(subproject_path, config['aliases']))
 
-            for key in MANDATORY_EXTERNAL_SUBPROJECT_KEYS + OPTIONAL_EXTERNAL_SUBPROJECT_KEYS:
+            for key in MANDATORY_EXTERNAL_SUBPROJECT_PPA_KEYS + OPTIONAL_EXTERNAL_SUBPROJECT_PPA_KEYS:
                 if key in config:
                     subprojects[subproject_name].setdefault(key, config[key])
-                elif key in OPTIONAL_EXTERNAL_SUBPROJECT_KEYS:
+                elif key in OPTIONAL_EXTERNAL_SUBPROJECT_PPA_KEYS:
                     _, _, _, original_release_details = get_subproject_details(subprojects[subproject_name]['release'])
                     if original_release_details and key in original_release_details:
                         subprojects[subproject_name].setdefault(key, original_release_details[key])
@@ -978,11 +990,10 @@ def load_external_subprojects():
                     external_releases.remove(subproject_name)
                     raise ValueError
 
+            subprojects[subproject_name].setdefault("support_metadata", support_metadata)
             project = read_external_subproject_details(subproject_name)
-            if project and "customer" in project:
-                subprojects[subproject_name].setdefault("customer", project["customer"])
-        except:
-            pass
+            if project:
+                subprojects[subproject_name].setdefault("customer", project)
 
 load_external_subprojects()