Merge ~mdeslaur/ubuntu-cve-tracker:more-source-map-cache into ubuntu-cve-tracker:master

[Marc Deslauriers]

commit 5661c00df492483a5448327823c31f40c44bf77a
Author: Marc Deslauriers <email address hidden>
Date: Tue Mar 12 12:43:20 2024 -0400

    source_map.py: use detailed cache if available

commit 3335f8ca63d56e8ea81a15f61a217103ec573fb1
Author: Marc Deslauriers <email address hidden>
Date: Tue Mar 12 12:30:11 2024 -0400

    gen-source-map-cache: create cache for detailed source map

This merge request will add a cache for the detailed source map, this should speed up most of the other tools in UCT.

[Alex Murray (alexmurray)]

LGTM!

review: Approve

diff --git a/scripts/gen-source-map-cache b/scripts/gen-source-map-cache
index 1c84cc6..cb86360 100755
--- a/scripts/gen-source-map-cache
+++ b/scripts/gen-source-map-cache
@@ -32,19 +32,29 @@ def main():
         sys.exit(1)
 
     mirror = config['packages_mirror']
+
+    # This is used by check-syntax and active_edit
     sources_pickle = os.path.join(mirror, 'sources_map.pickle')
+
+    # This is used by active_edit
     sources_pickle_dev_proposed = os.path.join(mirror, 'sources_map_dev_proposed.pickle')
 
+    # This is used by most of the other tools
+    sources_pickle_detailed = os.path.join(mirror, 'sources_map_detailed.pickle')
+
     # Remove these first so that source_map.load doesn't load them!
     if os.path.exists(sources_pickle):
         os.unlink(sources_pickle)
     if os.path.exists(sources_pickle_dev_proposed):
         os.unlink(sources_pickle_dev_proposed)
+    if os.path.exists(sources_pickle_detailed):
+        os.unlink(sources_pickle_detailed)
 
     sources = source_map.load(detailed=False)
     sources_dev_proposed = source_map.load(pockets=["-proposed"],
                                            releases=[cve_lib.devel_release],
                                            detailed=False)
+    sources_detailed = source_map.load(detailed=True)
 
     # Use protocol=3 to remain compatible with focal
     with open(sources_pickle, 'wb') as handle:
@@ -53,5 +63,8 @@ def main():
     with open(sources_pickle_dev_proposed, 'wb') as handle:
         pickle.dump(sources_dev_proposed, handle, protocol=3)
 
+    with open(sources_pickle_detailed, 'wb') as handle:
+        pickle.dump(sources_detailed, handle, protocol=3)
+
 if __name__ == '__main__':
     main()
diff --git a/scripts/source_map.py b/scripts/source_map.py
index c5ec9ad..e353fa8 100755
--- a/scripts/source_map.py
+++ b/scripts/source_map.py
@@ -222,14 +222,20 @@ def load(data_type='sources', pockets=None, releases=None, skip_eol_releases=Tru
     if 'packages_mirror' in config:
         mirror = config['packages_mirror']
 
-        if data_type == 'sources' and skip_eol_releases == True and arch == "amd64" and detailed == False:
-            if pockets == None and releases == None:
+        if data_type == 'sources' and skip_eol_releases == True and arch == "amd64":
+            if detailed == False and pockets == None and releases == None:
                 sources_pickle = os.path.join(mirror, 'sources_map.pickle')
                 if os.path.exists(sources_pickle):
                     with open(sources_pickle, 'rb') as handle:
                         map = pickle.load(handle)
                     return map
-            elif pockets == ["-proposed"] and releases == [cve_lib.devel_release]:
+            elif detailed == True and pockets == None and releases == None:
+                sources_pickle = os.path.join(mirror, 'sources_map_detailed.pickle')
+                if os.path.exists(sources_pickle):
+                    with open(sources_pickle, 'rb') as handle:
+                        map = pickle.load(handle)
+                    return map
+            elif detailed == False and pockets == ["-proposed"] and releases == [cve_lib.devel_release]:
                 sources_pickle_dev_proposed = os.path.join(mirror, 'sources_map_dev_proposed.pickle')
                 if os.path.exists(sources_pickle_dev_proposed):
                     with open(sources_pickle_dev_proposed, 'rb') as handle: