Merge ~alexmurray/ubuntu-cve-tracker:source-map-performance-improvements into ubuntu-cve-tracker:master

[Alex Murray]

This is currently a work-in-progress

e76ede6... by Alex Murray on 2023-07-27

scripts/test_source_map.py: test get_packages_from_generic_name()

Signed-off-by: Alex Murray <email address hidden>

1d08db8... by Alex Murray on 2023-07-27

scripts/test_source_map.py: test get_aliases_of_ubuntu_package()

Signed-off-by: Alex Murray <email address hidden>

[Alex Murray (alexmurray)]

Closing this since the current approach of using a global variable to store the cache directly in source_map doesn't make sense given that source_map in general doesn't store global state like this. Instead will either need to encapsulate the cache in the returned map, or return it in a separate variable.

review: Disapprove

Unmerged commits

1d08db8... by Alex Murray on 2023-07-27

scripts/test_source_map.py: test get_aliases_of_ubuntu_package()

Signed-off-by: Alex Murray <email address hidden>

	unit-tests:0 (build)
	check-cves:0 (build)

1 → 2 of 2 results

First • Previous • Next • Last

e76ede6... by Alex Murray on 2023-07-27

scripts/test_source_map.py: test get_packages_from_generic_name()

Signed-off-by: Alex Murray <email address hidden>

	unit-tests:0 (build)
	check-cves:0 (build)

1 → 2 of 2 results

First • Previous • Next • Last

24c7dec... by Alex Murray on 2023-07-27

scripts/source_map.py: add comments about future proposed caches

Signed-off-by: Alex Murray <email address hidden>

	unit-tests:0 (build)
	check-cves:0 (build)

1 → 2 of 2 results

First • Previous • Next • Last

73c31b7... by Alex Murray on 2023-07-27

scripts/source_map.py: add a cache for generic name lookups

Signed-off-by: Alex Murray <email address hidden>

54f07e4... by Alex Murray on 2023-07-27

scripts: load aliases when loading a subproject

Signed-off-by: Alex Murray <email address hidden>

diff --git a/scripts/cve_lib.py b/scripts/cve_lib.py
index 120dd8f..6439bd5 100755
--- a/scripts/cve_lib.py
+++ b/scripts/cve_lib.py
@@ -950,8 +950,9 @@ def load_external_subprojects():
 
         # check if aliases for packages exist
         if os.path.isfile(supported_txt[:-len("supported.txt")] + 'aliases.yaml'):
-            subprojects[rel].setdefault("aliases",
-                supported_txt[:-len("supported.txt")] + 'aliases.yaml')
+            # load aliases
+            aliases = yaml.safe_load(open(supported_txt[:-len("supported.txt")] + 'aliases.yaml'))
+            subprojects[rel].setdefault("aliases", aliases)
 
         try:
             # use config to populate other parts of the
diff --git a/scripts/source_map.py b/scripts/source_map.py
index 58ef5c3..65f0bdd 100755
--- a/scripts/source_map.py
+++ b/scripts/source_map.py
@@ -17,7 +17,6 @@ import re
 import subprocess
 import sys
 import cve_lib
-import yaml
 
 built_using_tags = ["Built-Using", "Static-Built-Using", "X-Cargo-Built-Using"]
 apt_pkg.init_system()
@@ -388,37 +387,43 @@ def get_built_using_header():
     header += "\n" + "-" * 120
     return header
 
+# TODO - add a cache which we update when we load the sources data so that we
+# don't have to loop through all the pkgs in the source map
 def get_all_aliases(sources, rel):
     aliases = []
 
     if not rel in sources:
         return aliases
 
+    # this is very expensive
     for pkg in sources[rel]:
         if 'aliases' in sources[rel][pkg]:
-            for alias in sources[rel][pkg]['aliases']:
-                aliases.append(alias)
+            aliases += sources[rel][pkg]['aliases']
     return aliases
 
+# TODO - add a cache which we update when we load the sources data so that we
+# don't have to loop through all the pkgs in the source map
 def get_aliases_of_ubuntu_package(sources, pkg_name, rel):
     aliases = []
     if not rel in sources:
         return aliases
 
+    # this is very expensive
     for pkg in sources[rel]:
         if 'aliases' in sources[rel][pkg]:
             if pkg_name in sources[rel][pkg]['aliases']:
                 aliases.append(pkg)
     return aliases
 
+generic_name_cache = {}
 def get_packages_from_generic_name(sources, generic_name, rel):
     pkgs = []
     if not rel in sources:
         return pkgs
 
-    for pkg in sources[rel]:
-        if 'generic_name' in sources[rel][pkg] and sources[rel][pkg]['generic_name'] == generic_name:
-            pkgs.append(pkg)
+    if generic_name in generic_name_cache and rel in generic_name_cache[generic_name]:
+        return generic_name_cache[generic_name][rel]
+
     return pkgs
 
 def load_subprojects_lists(releases=None):
@@ -484,11 +489,13 @@ def load_subprojects_lists(releases=None):
                     if '|' in pkg:
                         main_package_name = pkg.split('|')[0]
                         map[rel][pkg]['generic_name'] = main_package_name
+                        generic_name_cache.setdefault(main_package_name, {})
+                        generic_name_cache[main_package_name].setdefault(rel, [])
+                        generic_name_cache[main_package_name][rel].append(pkg)
 
-            if 'aliases' in details:
-                with open(details['aliases'], 'r') as file:
-                    aliases = yaml.safe_load(file)
 
+            if 'aliases' in details:
+                    aliases = details['aliases']
                     for pkg in aliases:
                         for src_pkg in map[rel]:
                             if pkg == src_pkg or \
diff --git a/scripts/test_source_map.py b/scripts/test_source_map.py
index 8e99825..04316d3 100755
--- a/scripts/test_source_map.py
+++ b/scripts/test_source_map.py
@@ -15,6 +15,9 @@ mock_subprojects = {
         "codename": "Fabulous Fake",
         "ppa": "ubuntu-esm/esm-bar-security/ubuntu",
         "parent": "ubuntu/bar",
+        "aliases": {
+            "fooishbar": ["bar"],
+        },
         "description": "Available with Ubuntu Pro: https://ubuntu.com/pro",
         "stamp": 1618963200,
     }
@@ -26,7 +29,13 @@ class TestSubprojects:
     def test_load_subproject_supported(self, _find_sources_mock):
         _find_sources_mock.return_value = []
         map = source_map.load(releases=['esm-foo/bar'])
-        assert set(map['esm-foo/bar'].keys()) == set(['foo', 'bar'])
+        assert set(map['esm-foo/bar'].keys()) == set(['foo', 'baz|1.0', 'bar', 'fooishbar'])
+
+        pkgs = source_map.get_packages_from_generic_name(map, "baz", "esm-foo/bar")
+        assert pkgs == ['baz|1.0']
+
+        aliases = source_map.get_aliases_of_ubuntu_package(map, "bar", "esm-foo/bar")
+        assert aliases == ['fooishbar']
 
     @pytest.mark.parametrize("release", cve_lib.get_active_esm_releases())
     @mock.patch("source_map._find_sources")
@@ -52,4 +61,4 @@ class TestSubprojects:
                 packages_clean.add(package)
 
         map = source_map.load(releases=[release])
-        assert set(map[release].keys()) == packages_clean
 \ No newline at end of file
+        assert set(map[release].keys()) == packages_clean
diff --git a/test/esm-fake-supported.txt b/test/esm-fake-supported.txt
index a907ec3..9dcc581 100644
--- a/test/esm-fake-supported.txt
+++ b/test/esm-fake-supported.txt
@@ -1,2 +1,4 @@
 foo
-bar
 \ No newline at end of file
+bar
+baz|1.0
+fooishbar