Merge ~brian-murray/ubuntu-archive-tools:generate-freeze into ubuntu-archive-tools:main

[Brian Murray]

This modifies generate freeze to use a local apt cache, after adding sources.list entries for every architecture, to find packages which are seeded. Subsequently, packages like linux-firmware-raspi2 (only available on arm) are now added to the freeze-block.

Here's a diff of the packages in the block:
https://paste.ubuntu.com/p/dxJXDMD32f/

f189b63... by Brian Murray on 2021-10-13

use variables in some more spots for the arches

[Steve Langasek (vorlon)]

Great! One thing I think must be fixed before landing (using proper tempdirs), and some suggestions for optimizations; comments inline.

review: Needs Fixing

6b301e1... by Brian Murray on 2021-10-13

only use the release pocket and don't add deb-src lines

8c6e20c... by Brian Murray on 2021-10-13

use ~/.cache/ for storing apt lists

[Brian Murray (brian-murray)]

I think I've addressed all the issues now.

1e8f476... by Brian Murray on 2021-10-13

use the ubuntu-archive-tools cache_dir

[Brian Murray (brian-murray)]

I've gone ahead and put the freeze-block folder under ubuntu-archive-tools so if somebody is browsing their .cache directory it might be clearer what freeze-block is. I've kept it as a separate directory in case some other tool ends up needing an apt cache.

[Steve Langasek (vorlon)]

review: Approve

diff --git a/generate-freeze-block b/generate-freeze-block
index 0bb2862..ba841b4 100755
--- a/generate-freeze-block
+++ b/generate-freeze-block
@@ -3,6 +3,7 @@
 
 # Copyright (C) 2013 Canonical Ltd.
 # Author: Iain Lane <[e-mail]>
+# Author: Brian Murray <[e-mail]>
 
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -20,6 +21,7 @@
 # USA
 
 import argparse
+import apt
 import apt_pkg
 import gzip
 import io
@@ -27,12 +29,16 @@ import json
 import logging
 import os
 import re
-import urllib.request
+import shutil
 import sys
+import urllib.request
+
+from launchpadlib.launchpad import Launchpad
 
 PARSED_SEEDS_URL = \
     'http://qa.ubuntuwire.org/ubuntu-seeded-packages/seeded.json.gz'
 LOGGER = logging.getLogger(os.path.basename(sys.argv[0]))
+CACHE_DIR = os.path.expanduser("~/.cache/ubuntu-archive-tools/freeze-block/")
 
 
 class GetPackage():
@@ -43,10 +49,47 @@ class GetPackage():
         if not GetPackage.apt_cache_initialised:
             apt_pkg.init()
             GetPackage.apt_cache_initialised = True
-
-        self.cache = apt_pkg.Cache(None)
-        self.pkgrecords = apt_pkg.PackageRecords(self.cache)
-        self.depcache = apt_pkg.DepCache(self.cache)
+        try:
+            os.makedirs("%s/apt/lists/partial" % CACHE_DIR, exist_ok=True)
+        except OSError:
+            pass
+        apt_pkg.config.set("Dir::state", "%s/apt/" % CACHE_DIR)
+        apt_pkg.config.set("Dir::Etc", "%s./apt" % CACHE_DIR)
+
+        lp = Launchpad.login_anonymously('generate-freeze-block',
+                                         'production',
+                                         version='devel')
+        ubuntu = lp.distributions['ubuntu']
+        series = ubuntu.current_series.name
+        self.all_arches = [a.architecture_tag
+                          for a in ubuntu.current_series.architectures]
+        main_arches = ("amd64", "i386")
+        port_arches = set(self.all_arches) - set(main_arches)
+
+        comps = ("main", "restricted", "universe", "multiverse")
+        for arch in main_arches:
+            url = "http://archive.ubuntu.com/ubuntu"
+            line = "deb [arch=%s] %s %s %s\n" % \
+                   (arch, url, series, " ".join(comps))
+            with open("%s/apt/sources.list" % CACHE_DIR, "a") as sources_list:
+                sources_list.write(line)
+        for arch in port_arches:
+            url = "http://ports.ubuntu.com/ubuntu-ports"
+            line = "deb [arch=%s] %s %s %s\n" % \
+                   (arch, url, series, " ".join(comps))
+            with open("%s/apt/sources.list" % CACHE_DIR, "a") as sources_list:
+                sources_list.write(line)
+        apt_pkg.config.set("APT::Architecture", "amd64")
+        apt_pkg.config.set("APT::Architectures",
+                           "%s" % " ".join(self.all_arches))
+        self.cache = apt.Cache(None)
+        keyring = "/usr/share/keyrings/ubuntu-archive-keyring.gpg"
+        apt_pkg.config.set("Dir::Etc::Trusted", keyring)
+        self.cache.update(None)
+        self.cache.open()
+        self.ap_cache = apt_pkg.Cache(None)
+        self.pkgrecords = apt_pkg.PackageRecords(self.ap_cache)
+        self.depcache = apt_pkg.DepCache(self.ap_cache)
 
         # Download & parse the seeds
         response = urllib.request.urlopen(PARSED_SEEDS_URL)
@@ -58,9 +101,15 @@ class GetPackage():
 
     def getsourcepackage(self, pkg):
         pkg = re.sub(':.*', '', pkg)
-        try:
-            candidate = self.depcache.get_candidate_ver(self.cache[pkg])
-        except KeyError:  # no package found (arch specific?)
+        candidate = ''
+        for arch in self.all_arches:
+            try:
+                candidate = self.depcache.get_candidate_ver(self.ap_cache[pkg, arch])
+            except KeyError:
+                pass
+            if candidate:
+                break
+        if not candidate:
             return
         try:
             self.pkgrecords.lookup(candidate.file_list[0])
@@ -78,7 +127,7 @@ def main():
                         help='Block only packages unique to FLAVOURS')
     parser.add_argument('--debug', '-d', action='store_true',
                         help='Output some extra debugging')
-    parser.add_argument('--not-unique','-n', action='store_true',
+    parser.add_argument('--not-unique', '-n', action='store_true',
                         help='Block only packages also seeded outside of FLAVOURS')
     args = parser.parse_args()
 
@@ -130,6 +179,9 @@ def main():
             output.add(source_pkg)
 
     print ("block", " ".join(sorted(output)))
+    # only cleanup the sources.list file in case more arches are added
+    if os.path.exists("%s/apt/sources.list" % CACHE_DIR):
+        os.remove("%s/apt/sources.list" % CACHE_DIR)
 
 
 if __name__ == "__main__":