Merge ~cjwatson/launchpad-buildd:apt-lists into launchpad-buildd:master

Proposed by Colin Watson
Status: Merged
Approved by: Colin Watson
Approved revision: b6064e3d6b54ebcfd133887298e1d48842e8aa10
Merge reported by: Otto Co-Pilot
Merged at revision: not available
Proposed branch: ~cjwatson/launchpad-buildd:apt-lists
Merge into: launchpad-buildd:master
Diff against target: 96 lines (+46/-21)
2 files modified
debian/changelog (+3/-0)
lpbuildd/binarypackage.py (+43/-21)
Reviewer Review Type Date Requested Status
Ioana Lasc Approve
Review via email: mp+383060@code.launchpad.net

Commit message

Use apt helpers to read Packages files

Description of the change

lpbuildd/binarypackage.py: Use "apt-get indextargets" and "apt-helper cat-file" where they exist to read Packages files, rather than looking in /var/lib/apt/lists/ directly.

The apt maintainers don't want other code poking around in /var/lib/apt/lists/ directly, not least because they might e.g. want to start lz4-compressing the package lists there in the near future. Use the recommended helpers where available to read Packages files for dep-wait analysis. We still need the old paths because these helpers are only available in rather recent versions of apt.

This is essentially the same as https://code.launchpad.net/~cjwatson/launchpad-buildd/apt-lists/+merge/286751, converted to git and rebased on master.

To post a comment you must log in.
Revision history for this message
Ioana Lasc (ilasc) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/debian/changelog b/debian/changelog
2index 26d4622..9e45fea 100644
3--- a/debian/changelog
4+++ b/debian/changelog
5@@ -27,6 +27,9 @@ launchpad-buildd (190) UNRELEASED; urgency=medium
6 * Fix LXD.run to not default to universal_newlines=True.
7 * Run on Python 3 when built for >= bionic.
8 * Add some notes on the production deployment.
9+ * lpbuildd/binarypackage.py: Use "apt-get indextargets" and "apt-helper
10+ cat-file" where they exist to read Packages files, rather than looking
11+ in /var/lib/apt/lists/ directly.
12
13 [ Dimitri John Ledkov ]
14 * lxd: Add riscv64 to arch table.
15diff --git a/lpbuildd/binarypackage.py b/lpbuildd/binarypackage.py
16index 2111bbe..19daca7 100644
17--- a/lpbuildd/binarypackage.py
18+++ b/lpbuildd/binarypackage.py
19@@ -177,6 +177,34 @@ class BinaryPackageBuildManager(DebianBuildManager):
20 env["DEB_BUILD_OPTIONS"] = "noautodbgsym"
21 self.runSubProcess(self._sbuildpath, args, env=env)
22
23+ def getAptLists(self):
24+ """Yield each of apt's Packages files in turn as a file object."""
25+ apt_helper = "/usr/lib/apt/apt-helper"
26+ if os.path.exists(os.path.join(self.chroot_path, apt_helper[1:])):
27+ paths = subprocess.check_output(
28+ ["sudo", "chroot", self.chroot_path,
29+ "apt-get", "indextargets", "--format", "$(FILENAME)",
30+ "Created-By: Packages"],
31+ universal_newlines=True).splitlines()
32+ for path in paths:
33+ helper = subprocess.Popen(
34+ ["sudo", "chroot", self.chroot_path,
35+ apt_helper, "cat-file", path],
36+ stdout=subprocess.PIPE)
37+ try:
38+ yield helper.stdout
39+ finally:
40+ helper.stdout.read()
41+ helper.wait()
42+ else:
43+ apt_lists = os.path.join(
44+ self.chroot_path, "var", "lib", "apt", "lists")
45+ for name in sorted(os.listdir(apt_lists)):
46+ if name.endswith("_Packages"):
47+ path = os.path.join(apt_lists, name)
48+ with open(path, "rb") as packages_file:
49+ yield packages_file
50+
51 def getAvailablePackages(self):
52 """Return the available binary packages in the chroot.
53
54@@ -184,27 +212,21 @@ class BinaryPackageBuildManager(DebianBuildManager):
55 available versions of each package.
56 """
57 available = defaultdict(set)
58- apt_lists = os.path.join(
59- self.chroot_path, "var", "lib", "apt", "lists")
60- for name in sorted(os.listdir(apt_lists)):
61- if name.endswith("_Packages"):
62- path = os.path.join(apt_lists, name)
63- with open(path, "rb") as packages_file:
64- for section in apt_pkg.TagFile(packages_file):
65- available[section["package"]].add(section["version"])
66- if "provides" in section:
67- provides = apt_pkg.parse_depends(
68- section["provides"])
69- for provide in provides:
70- # Disjunctions are currently undefined here.
71- if len(provide) > 1:
72- continue
73- # Virtual packages may only provide an exact
74- # version or none.
75- if provide[0][1] and provide[0][2] != "=":
76- continue
77- available[provide[0][0]].add(
78- provide[0][1] if provide[0][1] else None)
79+ for packages_file in self.getAptLists():
80+ for section in apt_pkg.TagFile(packages_file):
81+ available[section["package"]].add(section["version"])
82+ if "provides" in section:
83+ provides = apt_pkg.parse_depends(section["provides"])
84+ for provide in provides:
85+ # Disjunctions are currently undefined here.
86+ if len(provide) > 1:
87+ continue
88+ # Virtual packages may only provide an exact version
89+ # or none.
90+ if provide[0][1] and provide[0][2] != "=":
91+ continue
92+ available[provide[0][0]].add(
93+ provide[0][1] if provide[0][1] else None)
94 return available
95
96 def getBuildDepends(self, dscpath, arch_indep):

Subscribers

People subscribed via source and target branches