Merge ~xnox/germinate:foreign-arch-ma-same into germinate:master

Proposed by Dimitri John Ledkov
Status: Needs review
Proposed branch: ~xnox/germinate:foreign-arch-ma-same
Merge into: germinate:master
Diff against target: 358 lines (+171/-31)
4 files modified
germinate/archive.py (+15/-12)
germinate/germinator.py (+87/-15)
germinate/scripts/germinate_main.py (+4/-3)
germinate/tests/test_germinator.py (+65/-1)
Reviewer Review Type Date Requested Status
Colin Watson Needs Fixing
Review via email: mp+374918@code.launchpad.net

Commit message

This branch enables to seed Multi-Arch: same package dependencies from a foreign arch (ie. :i386), making cross arch dependencies, on systems that have executed dpkg --add-architecture.

This does not allow to seed non-Multi-Arch:same packages (ie. application, e.g. wine32). However, this work can be extended to support that (care needs to be taken when expanding non-arch qualified recommends and resolving them from the default architecture).

This branch achieves generating identical list of i386 ship-live packages as hardcoded in the debian-cd branch for eoan release, when executed as

$ ./bin/germinate --arch amd64 --foreign i386 -d eoan -s ubuntu.eoan

To post a comment you must log in.
Revision history for this message
Dimitri John Ledkov (xnox) wrote :

This now passes existing testsuite. It is awkward that testsuite had many packages without an Architecture declared, and then foreign was not set, and like None == None is obviously True!

Revision history for this message
Colin Watson (cjwatson) wrote :

For the record, I'm waiting to review this until you've added the foreign arch tests you mentioned and removed the note that the branch is WIP from the description.

Revision history for this message
Dimitri John Ledkov (xnox) wrote :

Added a test that loads up both amd64&i386 Packages files, and asserts that behavior does not change with single-arch germinator; and does the right thing with foreign arch option set. Fixed a bug that said testcase surfaced.

I think this is now ready for review.

Revision history for this message
Colin Watson (cjwatson) :
review: Needs Fixing
~xnox/germinate:foreign-arch-ma-same updated
3556b42... by Dimitri John Ledkov

Merge remote-tracking branch 'germinate/master' into foreign-arch-ma-same

89bbfe9... by Dimitri John Ledkov

Start addressing review feedback

Unmerged commits

89bbfe9... by Dimitri John Ledkov

Start addressing review feedback

3556b42... by Dimitri John Ledkov

Merge remote-tracking branch 'germinate/master' into foreign-arch-ma-same

b342713... by Dimitri John Ledkov

Add foreign arch parsing, to support seeding M-A:same deps

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/germinate/archive.py b/germinate/archive.py
2index efb5e72..f48bbd2 100644
3--- a/germinate/archive.py
4+++ b/germinate/archive.py
5@@ -85,13 +85,15 @@ class Archive:
6 class TagFile(Archive):
7 """Fetch package lists from a Debian-format archive as apt tag files."""
8
9- def __init__(self, dists, components, arch, mirrors, source_mirrors=None,
10+ def __init__(self, dists, components, arches, mirrors, source_mirrors=None,
11 installer_packages=True, cleanup=False):
12 """Create a representation of a Debian-format apt archive."""
13 if isinstance(dists, six.string_types):
14 dists = [dists]
15 if isinstance(components, six.string_types):
16 components = [components]
17+ if isinstance(arches, six.string_types):
18+ arches = [arches]
19 if isinstance(mirrors, six.string_types):
20 mirrors = [mirrors]
21 if isinstance(source_mirrors, six.string_types):
22@@ -99,7 +101,7 @@ class TagFile(Archive):
23
24 self._dists = dists
25 self._components = components
26- self._arch = arch
27+ self._arches = arches
28 self._mirrors = mirrors
29 self._installer_packages = installer_packages
30 if source_mirrors:
31@@ -221,15 +223,16 @@ class TagFile(Archive):
32 try:
33 for dist in self._dists:
34 for component in self._components:
35- packages = self._open_tag_files(
36- self._mirrors, dirname, "Packages", dist, component,
37- "binary-" + self._arch + "/Packages")
38- for tag_file in packages:
39- try:
40- for section in apt_pkg.TagFile(tag_file):
41- yield (IndexType.PACKAGES, section)
42- finally:
43- tag_file.close()
44+ for arch in self._arches:
45+ packages = self._open_tag_files(
46+ self._mirrors, dirname, arch + "_Packages", dist,
47+ component, "binary-" + arch + "/Packages")
48+ for tag_file in packages:
49+ try:
50+ for section in apt_pkg.TagFile(tag_file):
51+ yield (IndexType.PACKAGES, section)
52+ finally:
53+ tag_file.close()
54
55 sources = self._open_tag_files(
56 self._source_mirrors, dirname, "Sources", dist,
57@@ -247,7 +250,7 @@ class TagFile(Archive):
58 instpackages = self._open_tag_files(
59 self._mirrors, dirname, "InstallerPackages",
60 dist, component,
61- "debian-installer/binary-" + self._arch +
62+ "debian-installer/binary-" + self._arches[0] +
63 "/Packages")
64 except IOError:
65 # can live without these
66diff --git a/germinate/germinator.py b/germinate/germinator.py
67index 946e1a7..5ca31c9 100644
68--- a/germinate/germinator.py
69+++ b/germinate/germinator.py
70@@ -31,6 +31,7 @@ import re
71 import sys
72
73 import apt_pkg
74+import six
75
76 from germinate.archive import IndexType
77 from germinate.seeds import AtomicFile, SeedStructure, _ensure_unicode
78@@ -382,15 +383,26 @@ class Germinator(object):
79 # Initialisation.
80 # ---------------
81
82- def __init__(self, arch):
83+ def __init__(self, arches):
84 """Create a dependency expander.
85
86- Each instance of this class can only process a single architecture,
87- but can process multiple seed structures against it.
88+ Each instance of this class can only process a single main
89+ architecture, and an optional single foreign architecture (for
90+ Multi-Arch:same only packages), but can process multiple seed
91+ structures against it.
92+
93+ arches can be a string, for main arch only. Or a list, in that
94+ case the first element is the main architecture, and the
95+ second element is the foreign arch.
96
97 """
98- self._arch = arch
99- apt_pkg.config.set("APT::Architecture", self._arch)
100+ if isinstance(arches, six.string_types):
101+ arches = [arches]
102+ self._arches = arches
103+
104+ apt_pkg.config.set("APT::Architecture", self._arches[0])
105+ for arch in self._arches:
106+ apt_pkg.config.set("APT::Architectures::", arch)
107
108 # Global hints file.
109 self._hints = {}
110@@ -445,6 +457,17 @@ class Germinator(object):
111 pkg = section["Package"]
112 ver = section["Version"]
113
114+ arch = section.get("Architecture", self._arches[0])
115+ if arch not in self._arches + ["all"]:
116+ # Skip unknown arches
117+ return
118+ if arch != self._arches[0]:
119+ if section.get("Multi-Arch") == "same":
120+ pkg += ":" + arch
121+ else:
122+ # Skip non-M-A-same foreign packages
123+ return
124+
125 # If we have already seen an equal or newer version of this package,
126 # then skip this section.
127 if pkg in self._packages:
128@@ -465,8 +488,7 @@ class Germinator(object):
129
130 self._packages[pkg]["Essential"] = section.get("Essential", "")
131
132- self._packages[pkg]["Architecture"] = \
133- section.get("Architecture", self._arch)
134+ self._packages[pkg]["Architecture"] = arch
135
136 for field in "Pre-Depends", "Depends", "Recommends", "Built-Using":
137 value = section.get(field, "")
138@@ -483,7 +505,7 @@ class Germinator(object):
139 value = section.get(field, "0")
140 self._packages[pkg][field] = int(value)
141
142- src = section.get("Source", pkg)
143+ src = section.get("Source", section["Package"])
144 idx = src.find("(")
145 if idx != -1:
146 src = src[:idx].strip()
147@@ -494,6 +516,38 @@ class Germinator(object):
148
149 self._packages[pkg]["Multi-Arch"] = section.get("Multi-Arch", "none")
150
151+ # if Multi-arch allowed/foreign, and main Architecture add
152+ # :foreign provides
153+ if arch in [self._arches[0], "all"]:
154+ if self._packages[pkg]["Multi-Arch"] in ["allowed", "foreign"]:
155+ for a in self._arches[1:]:
156+ self._packages[pkg]["Provides"].append(
157+ [("%s:%s" % (pkg, a), '', '')])
158+
159+ # if Package is from supplementary-arches qualify dependencies
160+ # as follows:
161+ # no tag, :native are changed to :arch
162+ # :any is dropped, meaning use host arch
163+ if arch in self._arches[1:]:
164+ for field in "Pre-Depends", "Depends", "Recommends", "Provides":
165+ new_deps = []
166+ for dep in self._packages[pkg][field]:
167+ new_dep = []
168+ for alt in dep:
169+ base = alt[0].split(':', 1)
170+ if len(base) > 1:
171+ tag = base[1]
172+ else:
173+ tag = arch
174+ if tag == "native":
175+ tag = arch
176+ if tag == "any":
177+ tag = ''
178+ new_dep.append(
179+ ('%s:%s' % (alt[0], tag), alt[1], alt[2]))
180+ new_deps.append(new_dep)
181+ self._packages[pkg][field] = new_deps
182+
183 self._packages[pkg]["Kernel-Version"] = section.get(
184 "Kernel-Version", "")
185
186@@ -843,11 +897,21 @@ class Germinator(object):
187 if startarchspec != -1:
188 archspec = pkg[startarchspec + 1:-1].split()
189 pkg = pkg[:startarchspec - 1]
190- posarch = [x for x in archspec if not x.startswith('!')]
191- negarch = [x[1:] for x in archspec if x.startswith('!')]
192- if self._arch in negarch:
193- continue
194- if posarch and self._arch not in posarch:
195+ # include only these arches
196+ posarch = set(
197+ [x for x in archspec if not x.startswith('!')])
198+ # exclude these arches
199+ negarch = set(
200+ [x[1:] for x in archspec if x.startswith('!')])
201+ # all available arches for this germinator
202+ archset = set(self._arches)
203+ # remove blacklisted
204+ archset = archset - negarch
205+ # if there is a whitelist, find common only
206+ if posarch:
207+ archset = archset & posarch
208+ # if we left without any usable arches, skip this
209+ if not archset:
210 continue
211
212 pkg = pkg.split()[0]
213@@ -1142,7 +1206,7 @@ class Germinator(object):
214 within any seed.
215
216 """
217- if ":" in depend:
218+ if depend.endswith(":any") or depend.endswith(":native"):
219 depname, depqual = depend.split(":", 1)
220 else:
221 depname = depend
222@@ -1183,7 +1247,11 @@ class Germinator(object):
223 seed, build_depend,
224 always_include_virtual=False):
225 """Get the possible candidates for satisfying a dependency."""
226- plain_depname = depname.split(":", 1)[0]
227+ depname_split = depname.split(":", 1)
228+ plain_depname = depname_split[0]
229+ if depname_split[-1] in self._arches[1:]:
230+ plain_depname = depname
231+
232 candidates = []
233 if plain_depname in self._packages:
234 candidates.append(
235@@ -1192,7 +1260,11 @@ class Germinator(object):
236 candidates.extend(self._provides[plain_depname].items())
237
238 for candpkg, candver in candidates:
239+ candpkg_split = candpkg.split(":", 1)
240 plain_candpkg = candpkg.split(":", 1)[0]
241+ if candpkg_split[-1] in self._arches[1:]:
242+ plain_candpkg = candpkg
243+
244 if plain_candpkg not in self._packages:
245 continue
246 allowed = False
247diff --git a/germinate/scripts/germinate_main.py b/germinate/scripts/germinate_main.py
248index 329c1e6..86c856e 100644
249--- a/germinate/scripts/germinate_main.py
250+++ b/germinate/scripts/germinate_main.py
251@@ -82,7 +82,8 @@ def parse_options(argv):
252 parser.add_option('-d', '--dist', dest='dist',
253 default=germinate.defaults.dist,
254 help='operate on distribution DIST (default: %default)')
255- parser.add_option('-a', '--arch', dest='arch',
256+ parser.add_option('-a', '--arch', dest='arches',
257+ action='append',
258 default=germinate.defaults.arch,
259 help='operate on architecture ARCH (default: %default)')
260 parser.add_option('-c', '--components', dest='components',
261@@ -145,11 +146,11 @@ def main(argv):
262 else:
263 germinate_logging(logging.INFO)
264
265- g = Germinator(options.arch)
266+ g = Germinator(options.arches)
267 g._always_follow_build_depends = options.always_follow_build_depends
268
269 archive = germinate.archive.TagFile(
270- options.dist, options.components, options.arch,
271+ options.dist, options.components, options.arches,
272 options.mirrors, source_mirrors=options.source_mirrors,
273 installer_packages=options.installer, cleanup=options.cleanup)
274 g.parse_archive(archive)
275diff --git a/germinate/tests/test_germinator.py b/germinate/tests/test_germinator.py
276index aaab106..a848cae 100644
277--- a/germinate/tests/test_germinator.py
278+++ b/germinate/tests/test_germinator.py
279@@ -323,6 +323,71 @@ class TestGerminator(TestCase):
280 shutil.rmtree(self.archive_dir)
281 shutil.rmtree(self.seeds_dir)
282
283+ def test_depends_foreign_multiarch(self):
284+ """Test that amd64 packages can recommend multi-arched :i386
285+ packages, although it's unspecified in the MultiarchSpec, it
286+ is possible with apt and is in use for Nvidia drivers
287+
288+ """
289+ for arches in (["amd64"], ["amd64", "i386"]):
290+ self.addSource("focal", "main", "hello", "1.0-1", ["hello"])
291+ self.addSource("focal", "main", "glibc", "2.36", ["libc6"])
292+ self.addSource("focal", "restricted", "nvidia-graphics", "1.0-1",
293+ ["libnvidia-compute"])
294+ self.addPackage("focal", "main", "amd64", "hello", "1.0-1",
295+ fields={
296+ "Architecture": "amd64",
297+ "Depends": "libnvidia-compute",
298+ "Recommends": "libnvidia-compute:i386",
299+ })
300+ self.addPackage("focal", "main", "i386", "hello-32bit", "1.0-1",
301+ fields={
302+ "Architecture": "i386",
303+ "Depends": "libc6",
304+ })
305+ for arch in "amd64", "i386":
306+ self.addPackage("focal", "main", arch, "libc6", "2.36",
307+ fields={
308+ "Architecture": arch,
309+ "Multi-Arch": "same"
310+ })
311+ self.addPackage("focal", "restricted", arch,
312+ "libnvidia-compute",
313+ "1.0-1", fields={
314+ "Architecture": arch,
315+ "Depends": "libc6",
316+ "Multi-Arch": "same",
317+ })
318+
319+ branch = "collection.focal"
320+ self.addStructureLine(branch, "feature follow-recommends")
321+ self.addSeed(branch, "base")
322+ self.addSeedPackage(branch, "base", "hello")
323+ # This package is seeded, but should be excluded
324+ # only M-A:same foreign arch packages are supported atm
325+ self.addSeedPackage(branch, "base", "hello-32bit")
326+ germinator = Germinator(arches)
327+ for arch in "i386", "amd64":
328+ for component in "main", "restricted":
329+ archive = TagFile(
330+ "focal", component, arch,
331+ "file://%s" % self.archive_dir)
332+ germinator.parse_archive(archive)
333+ structure = self.openSeedStructure(branch)
334+ germinator.plant_seeds(structure)
335+ germinator.grow(structure)
336+ expected = set(["hello",
337+ "libc6",
338+ "libnvidia-compute"])
339+ if 'i386' in arches:
340+ expected.add("libc6:i386")
341+ expected.add("libnvidia-compute:i386")
342+ self.assertEqual(
343+ expected, set(germinator.get_all(structure)))
344+
345+ shutil.rmtree(self.archive_dir)
346+ shutil.rmtree(self.seeds_dir)
347+
348 def test_build_depends_all(self):
349 """Confirm build-depends of arch: all packages are recursed when
350 follow-build-depends-all is set, and not when
351@@ -349,7 +414,6 @@ class TestGerminator(TestCase):
352 structure = self.openSeedStructure(branch)
353 germinator.plant_seeds(structure)
354 germinator.grow(structure)
355-
356 expected = set()
357 if sense == "":
358 expected.add("gettext")

Subscribers

People subscribed via source and target branches