Merge ~emitorino/ubuntu-cve-tracker:add_project_yml_when_loading_subproject into ubuntu-cve-tracker:master

[Emilia Torino]

For the purpose of improving the management of subprojects, adding support to load the project.yml information into subprojects. Unless missed, it seems cve_lib is not yet doing this.

Also, checking configs before setting them into subprojects, otherwise if a given config is not available, the KeyError exception is catch for the first config not found, and the following ones are not loaded at all.

* cve_lib.py:
  - check config before trying to assign it to subprojects[rel]
  - load customer metadata from project.yml into subprojects[rel]
* test_cve_lib:
  - test for changes

[Alex Murray (alexmurray)]

LGTM - thanks Emi (in particular thanks for the unit test - very nice!)

[Alex Murray (alexmurray)]

review: Approve

diff --git a/scripts/cve_lib.py b/scripts/cve_lib.py
index f118f93..79b4225 100755
--- a/scripts/cve_lib.py
+++ b/scripts/cve_lib.py
@@ -856,6 +856,17 @@ def read_external_subproject_config(subproject):
     with open(config_yaml) as cfg:
         return yaml.safe_load(cfg)
 
+
+def read_external_subproject_details(subproject):
+    """Read and return the project details for the given subproject."""
+    sp_dir = get_external_subproject_dir(subproject)
+    # project.yml is located in the top level folder for the subproject
+    project_dir = sp_dir[:sp_dir.rfind("/")]
+    project_yaml = os.path.join(project_dir, "project.yml")
+    if os.path.isfile(project_yaml):
+        with open(project_yaml) as cfg:
+            return yaml.safe_load(cfg)
+
 def find_files_recursive(path, name):
     """Return a list of all files under path with name."""
     matches = []
@@ -920,11 +931,13 @@ def load_external_subprojects():
             # use config to populate other parts of the
             # subproject settings
             config = read_external_subproject_config(rel)
-            subprojects[rel].setdefault("ppa", config["ppa"])
-            subprojects[rel].setdefault("oval", config["oval"])
-            subprojects[rel].setdefault("name", config["name"])
-            subprojects[rel].setdefault("description", config["description"])
-            subprojects[rel].setdefault("parent", config["parent"])
+            config_data = ['ppa', 'oval', 'name', 'description', 'parent']
+            for data in config_data:
+                if data in config:
+                    subprojects[rel].setdefault(data, config[data])
+            project = read_external_subproject_details(rel)
+            if project and "customer" in project:
+                subprojects[rel].setdefault("customer", project["customer"])
         except:
             pass
 
diff --git a/scripts/test_cve_lib.py b/scripts/test_cve_lib.py
index c5130dc..f9bd1d2 100755
--- a/scripts/test_cve_lib.py
+++ b/scripts/test_cve_lib.py
@@ -184,3 +184,24 @@ class TestParseCVEFiles:
                 js = cve_lib.parse_cvss(cvss[0]['vector'])
                 assert cvss[0]['baseScore'] == str(js['baseMetricV3']['cvssV3']['baseScore'])
                 assert cvss[0]['baseSeverity'] == js['baseMetricV3']['cvssV3']['baseSeverity']
+
+
+class TestSubprojects:
+
+    def test_load_subprojects_loads_every_config_available(self):
+        # mimic here how subprojects are loaded to then assert cve_lib.subprojects
+        # contains every config available
+        for supported_txt in cve_lib.find_files_recursive(cve_lib.subprojects_dir, "supported.txt"):
+            # rel name is the path component between subprojects/ and
+            # /supported.txt
+            rel = supported_txt[len(cve_lib.subprojects_dir) + 1:-len("supported.txt") - 1]
+            config = cve_lib.read_external_subproject_config(rel)
+            config_data = ['ppa', 'oval', 'name', 'description', 'parent']
+            for data in config_data:
+                if data in config:
+                    assert cve_lib.subprojects[rel][data] == config[data]
+            project = cve_lib.read_external_subproject_details(rel)
+            if project and "customer" in project:
+                assert cve_lib.subprojects[rel]["customer"] == project["customer"]
+
+