Merge ~xnox/germinate:foreign-arch-ma-same into germinate:master

Proposed by Dimitri John Ledkov
Status: Needs review
Proposed branch: ~xnox/germinate:foreign-arch-ma-same
Merge into: germinate:master
Diff against target: 358 lines (+171/-31)
4 files modified
germinate/archive.py (+15/-12)
germinate/germinator.py (+87/-15)
germinate/scripts/germinate_main.py (+4/-3)
germinate/tests/test_germinator.py (+65/-1)
Reviewer Review Type Date Requested Status
Colin Watson Needs Fixing
Review via email: mp+374918@code.launchpad.net

Commit message

This branch enables to seed Multi-Arch: same package dependencies from a foreign arch (ie. :i386), making cross arch dependencies, on systems that have executed dpkg --add-architecture.

This does not allow to seed non-Multi-Arch:same packages (ie. application, e.g. wine32). However, this work can be extended to support that (care needs to be taken when expanding non-arch qualified recommends and resolving them from the default architecture).

This branch achieves generating identical list of i386 ship-live packages as hardcoded in the debian-cd branch for eoan release, when executed as

$ ./bin/germinate --arch amd64 --foreign i386 -d eoan -s ubuntu.eoan

To post a comment you must log in.
Revision history for this message
Dimitri John Ledkov (xnox) wrote :

This now passes existing testsuite. It is awkward that testsuite had many packages without an Architecture declared, and then foreign was not set, and like None == None is obviously True!

Revision history for this message
Colin Watson (cjwatson) wrote :

For the record, I'm waiting to review this until you've added the foreign arch tests you mentioned and removed the note that the branch is WIP from the description.

Revision history for this message
Dimitri John Ledkov (xnox) wrote :

Added a test that loads up both amd64&i386 Packages files, and asserts that behavior does not change with single-arch germinator; and does the right thing with foreign arch option set. Fixed a bug that said testcase surfaced.

I think this is now ready for review.

Revision history for this message
Colin Watson (cjwatson) :
review: Needs Fixing
~xnox/germinate:foreign-arch-ma-same updated
3556b42... by Dimitri John Ledkov

Merge remote-tracking branch 'germinate/master' into foreign-arch-ma-same

89bbfe9... by Dimitri John Ledkov

Start addressing review feedback

Unmerged commits

89bbfe9... by Dimitri John Ledkov

Start addressing review feedback

3556b42... by Dimitri John Ledkov

Merge remote-tracking branch 'germinate/master' into foreign-arch-ma-same

b342713... by Dimitri John Ledkov

Add foreign arch parsing, to support seeding M-A:same deps

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
diff --git a/germinate/archive.py b/germinate/archive.py
index efb5e72..f48bbd2 100644
--- a/germinate/archive.py
+++ b/germinate/archive.py
@@ -85,13 +85,15 @@ class Archive:
85class TagFile(Archive):85class TagFile(Archive):
86 """Fetch package lists from a Debian-format archive as apt tag files."""86 """Fetch package lists from a Debian-format archive as apt tag files."""
8787
88 def __init__(self, dists, components, arch, mirrors, source_mirrors=None,88 def __init__(self, dists, components, arches, mirrors, source_mirrors=None,
89 installer_packages=True, cleanup=False):89 installer_packages=True, cleanup=False):
90 """Create a representation of a Debian-format apt archive."""90 """Create a representation of a Debian-format apt archive."""
91 if isinstance(dists, six.string_types):91 if isinstance(dists, six.string_types):
92 dists = [dists]92 dists = [dists]
93 if isinstance(components, six.string_types):93 if isinstance(components, six.string_types):
94 components = [components]94 components = [components]
95 if isinstance(arches, six.string_types):
96 arches = [arches]
95 if isinstance(mirrors, six.string_types):97 if isinstance(mirrors, six.string_types):
96 mirrors = [mirrors]98 mirrors = [mirrors]
97 if isinstance(source_mirrors, six.string_types):99 if isinstance(source_mirrors, six.string_types):
@@ -99,7 +101,7 @@ class TagFile(Archive):
99101
100 self._dists = dists102 self._dists = dists
101 self._components = components103 self._components = components
102 self._arch = arch104 self._arches = arches
103 self._mirrors = mirrors105 self._mirrors = mirrors
104 self._installer_packages = installer_packages106 self._installer_packages = installer_packages
105 if source_mirrors:107 if source_mirrors:
@@ -221,15 +223,16 @@ class TagFile(Archive):
221 try:223 try:
222 for dist in self._dists:224 for dist in self._dists:
223 for component in self._components:225 for component in self._components:
224 packages = self._open_tag_files(226 for arch in self._arches:
225 self._mirrors, dirname, "Packages", dist, component,227 packages = self._open_tag_files(
226 "binary-" + self._arch + "/Packages")228 self._mirrors, dirname, arch + "_Packages", dist,
227 for tag_file in packages:229 component, "binary-" + arch + "/Packages")
228 try:230 for tag_file in packages:
229 for section in apt_pkg.TagFile(tag_file):231 try:
230 yield (IndexType.PACKAGES, section)232 for section in apt_pkg.TagFile(tag_file):
231 finally:233 yield (IndexType.PACKAGES, section)
232 tag_file.close()234 finally:
235 tag_file.close()
233236
234 sources = self._open_tag_files(237 sources = self._open_tag_files(
235 self._source_mirrors, dirname, "Sources", dist,238 self._source_mirrors, dirname, "Sources", dist,
@@ -247,7 +250,7 @@ class TagFile(Archive):
247 instpackages = self._open_tag_files(250 instpackages = self._open_tag_files(
248 self._mirrors, dirname, "InstallerPackages",251 self._mirrors, dirname, "InstallerPackages",
249 dist, component,252 dist, component,
250 "debian-installer/binary-" + self._arch +253 "debian-installer/binary-" + self._arches[0] +
251 "/Packages")254 "/Packages")
252 except IOError:255 except IOError:
253 # can live without these256 # can live without these
diff --git a/germinate/germinator.py b/germinate/germinator.py
index 946e1a7..5ca31c9 100644
--- a/germinate/germinator.py
+++ b/germinate/germinator.py
@@ -31,6 +31,7 @@ import re
31import sys31import sys
3232
33import apt_pkg33import apt_pkg
34import six
3435
35from germinate.archive import IndexType36from germinate.archive import IndexType
36from germinate.seeds import AtomicFile, SeedStructure, _ensure_unicode37from germinate.seeds import AtomicFile, SeedStructure, _ensure_unicode
@@ -382,15 +383,26 @@ class Germinator(object):
382 # Initialisation.383 # Initialisation.
383 # ---------------384 # ---------------
384385
385 def __init__(self, arch):386 def __init__(self, arches):
386 """Create a dependency expander.387 """Create a dependency expander.
387388
388 Each instance of this class can only process a single architecture,389 Each instance of this class can only process a single main
389 but can process multiple seed structures against it.390 architecture, and an optional single foreign architecture (for
391 Multi-Arch:same only packages), but can process multiple seed
392 structures against it.
393
394 arches can be a string, for main arch only. Or a list, in that
395 case the first element is the main architecture, and the
396 second element is the foreign arch.
390397
391 """398 """
392 self._arch = arch399 if isinstance(arches, six.string_types):
393 apt_pkg.config.set("APT::Architecture", self._arch)400 arches = [arches]
401 self._arches = arches
402
403 apt_pkg.config.set("APT::Architecture", self._arches[0])
404 for arch in self._arches:
405 apt_pkg.config.set("APT::Architectures::", arch)
394406
395 # Global hints file.407 # Global hints file.
396 self._hints = {}408 self._hints = {}
@@ -445,6 +457,17 @@ class Germinator(object):
445 pkg = section["Package"]457 pkg = section["Package"]
446 ver = section["Version"]458 ver = section["Version"]
447459
460 arch = section.get("Architecture", self._arches[0])
461 if arch not in self._arches + ["all"]:
462 # Skip unknown arches
463 return
464 if arch != self._arches[0]:
465 if section.get("Multi-Arch") == "same":
466 pkg += ":" + arch
467 else:
468 # Skip non-M-A-same foreign packages
469 return
470
448 # If we have already seen an equal or newer version of this package,471 # If we have already seen an equal or newer version of this package,
449 # then skip this section.472 # then skip this section.
450 if pkg in self._packages:473 if pkg in self._packages:
@@ -465,8 +488,7 @@ class Germinator(object):
465488
466 self._packages[pkg]["Essential"] = section.get("Essential", "")489 self._packages[pkg]["Essential"] = section.get("Essential", "")
467490
468 self._packages[pkg]["Architecture"] = \491 self._packages[pkg]["Architecture"] = arch
469 section.get("Architecture", self._arch)
470492
471 for field in "Pre-Depends", "Depends", "Recommends", "Built-Using":493 for field in "Pre-Depends", "Depends", "Recommends", "Built-Using":
472 value = section.get(field, "")494 value = section.get(field, "")
@@ -483,7 +505,7 @@ class Germinator(object):
483 value = section.get(field, "0")505 value = section.get(field, "0")
484 self._packages[pkg][field] = int(value)506 self._packages[pkg][field] = int(value)
485507
486 src = section.get("Source", pkg)508 src = section.get("Source", section["Package"])
487 idx = src.find("(")509 idx = src.find("(")
488 if idx != -1:510 if idx != -1:
489 src = src[:idx].strip()511 src = src[:idx].strip()
@@ -494,6 +516,38 @@ class Germinator(object):
494516
495 self._packages[pkg]["Multi-Arch"] = section.get("Multi-Arch", "none")517 self._packages[pkg]["Multi-Arch"] = section.get("Multi-Arch", "none")
496518
519 # if Multi-arch allowed/foreign, and main Architecture add
520 # :foreign provides
521 if arch in [self._arches[0], "all"]:
522 if self._packages[pkg]["Multi-Arch"] in ["allowed", "foreign"]:
523 for a in self._arches[1:]:
524 self._packages[pkg]["Provides"].append(
525 [("%s:%s" % (pkg, a), '', '')])
526
527 # if Package is from supplementary-arches qualify dependencies
528 # as follows:
529 # no tag, :native are changed to :arch
530 # :any is dropped, meaning use host arch
531 if arch in self._arches[1:]:
532 for field in "Pre-Depends", "Depends", "Recommends", "Provides":
533 new_deps = []
534 for dep in self._packages[pkg][field]:
535 new_dep = []
536 for alt in dep:
537 base = alt[0].split(':', 1)
538 if len(base) > 1:
539 tag = base[1]
540 else:
541 tag = arch
542 if tag == "native":
543 tag = arch
544 if tag == "any":
545 tag = ''
546 new_dep.append(
547 ('%s:%s' % (alt[0], tag), alt[1], alt[2]))
548 new_deps.append(new_dep)
549 self._packages[pkg][field] = new_deps
550
497 self._packages[pkg]["Kernel-Version"] = section.get(551 self._packages[pkg]["Kernel-Version"] = section.get(
498 "Kernel-Version", "")552 "Kernel-Version", "")
499553
@@ -843,11 +897,21 @@ class Germinator(object):
843 if startarchspec != -1:897 if startarchspec != -1:
844 archspec = pkg[startarchspec + 1:-1].split()898 archspec = pkg[startarchspec + 1:-1].split()
845 pkg = pkg[:startarchspec - 1]899 pkg = pkg[:startarchspec - 1]
846 posarch = [x for x in archspec if not x.startswith('!')]900 # include only these arches
847 negarch = [x[1:] for x in archspec if x.startswith('!')]901 posarch = set(
848 if self._arch in negarch:902 [x for x in archspec if not x.startswith('!')])
849 continue903 # exclude these arches
850 if posarch and self._arch not in posarch:904 negarch = set(
905 [x[1:] for x in archspec if x.startswith('!')])
906 # all available arches for this germinator
907 archset = set(self._arches)
908 # remove blacklisted
909 archset = archset - negarch
910 # if there is a whitelist, find common only
911 if posarch:
912 archset = archset & posarch
913 # if we left without any usable arches, skip this
914 if not archset:
851 continue915 continue
852916
853 pkg = pkg.split()[0]917 pkg = pkg.split()[0]
@@ -1142,7 +1206,7 @@ class Germinator(object):
1142 within any seed.1206 within any seed.
11431207
1144 """1208 """
1145 if ":" in depend:1209 if depend.endswith(":any") or depend.endswith(":native"):
1146 depname, depqual = depend.split(":", 1)1210 depname, depqual = depend.split(":", 1)
1147 else:1211 else:
1148 depname = depend1212 depname = depend
@@ -1183,7 +1247,11 @@ class Germinator(object):
1183 seed, build_depend,1247 seed, build_depend,
1184 always_include_virtual=False):1248 always_include_virtual=False):
1185 """Get the possible candidates for satisfying a dependency."""1249 """Get the possible candidates for satisfying a dependency."""
1186 plain_depname = depname.split(":", 1)[0]1250 depname_split = depname.split(":", 1)
1251 plain_depname = depname_split[0]
1252 if depname_split[-1] in self._arches[1:]:
1253 plain_depname = depname
1254
1187 candidates = []1255 candidates = []
1188 if plain_depname in self._packages:1256 if plain_depname in self._packages:
1189 candidates.append(1257 candidates.append(
@@ -1192,7 +1260,11 @@ class Germinator(object):
1192 candidates.extend(self._provides[plain_depname].items())1260 candidates.extend(self._provides[plain_depname].items())
11931261
1194 for candpkg, candver in candidates:1262 for candpkg, candver in candidates:
1263 candpkg_split = candpkg.split(":", 1)
1195 plain_candpkg = candpkg.split(":", 1)[0]1264 plain_candpkg = candpkg.split(":", 1)[0]
1265 if candpkg_split[-1] in self._arches[1:]:
1266 plain_candpkg = candpkg
1267
1196 if plain_candpkg not in self._packages:1268 if plain_candpkg not in self._packages:
1197 continue1269 continue
1198 allowed = False1270 allowed = False
diff --git a/germinate/scripts/germinate_main.py b/germinate/scripts/germinate_main.py
index 329c1e6..86c856e 100644
--- a/germinate/scripts/germinate_main.py
+++ b/germinate/scripts/germinate_main.py
@@ -82,7 +82,8 @@ def parse_options(argv):
82 parser.add_option('-d', '--dist', dest='dist',82 parser.add_option('-d', '--dist', dest='dist',
83 default=germinate.defaults.dist,83 default=germinate.defaults.dist,
84 help='operate on distribution DIST (default: %default)')84 help='operate on distribution DIST (default: %default)')
85 parser.add_option('-a', '--arch', dest='arch',85 parser.add_option('-a', '--arch', dest='arches',
86 action='append',
86 default=germinate.defaults.arch,87 default=germinate.defaults.arch,
87 help='operate on architecture ARCH (default: %default)')88 help='operate on architecture ARCH (default: %default)')
88 parser.add_option('-c', '--components', dest='components',89 parser.add_option('-c', '--components', dest='components',
@@ -145,11 +146,11 @@ def main(argv):
145 else:146 else:
146 germinate_logging(logging.INFO)147 germinate_logging(logging.INFO)
147148
148 g = Germinator(options.arch)149 g = Germinator(options.arches)
149 g._always_follow_build_depends = options.always_follow_build_depends150 g._always_follow_build_depends = options.always_follow_build_depends
150151
151 archive = germinate.archive.TagFile(152 archive = germinate.archive.TagFile(
152 options.dist, options.components, options.arch,153 options.dist, options.components, options.arches,
153 options.mirrors, source_mirrors=options.source_mirrors,154 options.mirrors, source_mirrors=options.source_mirrors,
154 installer_packages=options.installer, cleanup=options.cleanup)155 installer_packages=options.installer, cleanup=options.cleanup)
155 g.parse_archive(archive)156 g.parse_archive(archive)
diff --git a/germinate/tests/test_germinator.py b/germinate/tests/test_germinator.py
index aaab106..a848cae 100644
--- a/germinate/tests/test_germinator.py
+++ b/germinate/tests/test_germinator.py
@@ -323,6 +323,71 @@ class TestGerminator(TestCase):
323 shutil.rmtree(self.archive_dir)323 shutil.rmtree(self.archive_dir)
324 shutil.rmtree(self.seeds_dir)324 shutil.rmtree(self.seeds_dir)
325325
326 def test_depends_foreign_multiarch(self):
327 """Test that amd64 packages can recommend multi-arched :i386
328 packages, although it's unspecified in the MultiarchSpec, it
329 is possible with apt and is in use for Nvidia drivers
330
331 """
332 for arches in (["amd64"], ["amd64", "i386"]):
333 self.addSource("focal", "main", "hello", "1.0-1", ["hello"])
334 self.addSource("focal", "main", "glibc", "2.36", ["libc6"])
335 self.addSource("focal", "restricted", "nvidia-graphics", "1.0-1",
336 ["libnvidia-compute"])
337 self.addPackage("focal", "main", "amd64", "hello", "1.0-1",
338 fields={
339 "Architecture": "amd64",
340 "Depends": "libnvidia-compute",
341 "Recommends": "libnvidia-compute:i386",
342 })
343 self.addPackage("focal", "main", "i386", "hello-32bit", "1.0-1",
344 fields={
345 "Architecture": "i386",
346 "Depends": "libc6",
347 })
348 for arch in "amd64", "i386":
349 self.addPackage("focal", "main", arch, "libc6", "2.36",
350 fields={
351 "Architecture": arch,
352 "Multi-Arch": "same"
353 })
354 self.addPackage("focal", "restricted", arch,
355 "libnvidia-compute",
356 "1.0-1", fields={
357 "Architecture": arch,
358 "Depends": "libc6",
359 "Multi-Arch": "same",
360 })
361
362 branch = "collection.focal"
363 self.addStructureLine(branch, "feature follow-recommends")
364 self.addSeed(branch, "base")
365 self.addSeedPackage(branch, "base", "hello")
366 # This package is seeded, but should be excluded
367 # only M-A:same foreign arch packages are supported atm
368 self.addSeedPackage(branch, "base", "hello-32bit")
369 germinator = Germinator(arches)
370 for arch in "i386", "amd64":
371 for component in "main", "restricted":
372 archive = TagFile(
373 "focal", component, arch,
374 "file://%s" % self.archive_dir)
375 germinator.parse_archive(archive)
376 structure = self.openSeedStructure(branch)
377 germinator.plant_seeds(structure)
378 germinator.grow(structure)
379 expected = set(["hello",
380 "libc6",
381 "libnvidia-compute"])
382 if 'i386' in arches:
383 expected.add("libc6:i386")
384 expected.add("libnvidia-compute:i386")
385 self.assertEqual(
386 expected, set(germinator.get_all(structure)))
387
388 shutil.rmtree(self.archive_dir)
389 shutil.rmtree(self.seeds_dir)
390
326 def test_build_depends_all(self):391 def test_build_depends_all(self):
327 """Confirm build-depends of arch: all packages are recursed when392 """Confirm build-depends of arch: all packages are recursed when
328 follow-build-depends-all is set, and not when393 follow-build-depends-all is set, and not when
@@ -349,7 +414,6 @@ class TestGerminator(TestCase):
349 structure = self.openSeedStructure(branch)414 structure = self.openSeedStructure(branch)
350 germinator.plant_seeds(structure)415 germinator.plant_seeds(structure)
351 germinator.grow(structure)416 germinator.grow(structure)
352
353 expected = set()417 expected = set()
354 if sense == "":418 if sense == "":
355 expected.add("gettext")419 expected.add("gettext")

Subscribers

People subscribed via source and target branches