Merge ~mwhudson/ubuntu-cdimage:simplify-germinate into ubuntu-cdimage:main
- Git
- lp:~mwhudson/ubuntu-cdimage
- simplify-germinate
- Merge into main
Proposed by
Michael Hudson-Doyle
Status: | Merged |
---|---|
Merged at revision: | e5056baafa398e289c2774706cfece5be06bd2c0 |
Proposed branch: | ~mwhudson/ubuntu-cdimage:simplify-germinate |
Merge into: | ubuntu-cdimage:main |
Diff against target: |
980 lines (+39/-628) 11 files modified
bin/cron.dvd (+2/-1) dev/null (+0/-46) lib/cdimage/build.py (+1/-12) lib/cdimage/check_installable.py (+0/-17) lib/cdimage/germinate.py (+25/-235) lib/cdimage/livefs.py (+2/-5) lib/cdimage/tests/test_build.py (+0/-10) lib/cdimage/tests/test_check_installable.py (+1/-35) lib/cdimage/tests/test_germinate.py (+6/-249) lib/cdimage/tests/test_tree.py (+0/-4) lib/cdimage/tree.py (+2/-14) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Steve Langasek | Approve | ||
Review via email:
|
Commit message
Description of the change
This strips out I think all of the unnecessary complexity in ubuntu-cdimage's germinate code that can be stripped without making changes to the debian-cd end of things.
It is perhaps too much to land in one big hit but well. Look at that diffstat!!
Probably best to review commit by commit.
To post a comment you must log in.
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Steve Langasek (vorlon) : | # |
review:
Needs Fixing
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Michael Hudson-Doyle (mwhudson) : | # |
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Steve Langasek (vorlon) : | # |
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/bin/cron.daily b/bin/cron.daily | |||
2 | 0 | deleted file mode 100755 | 0 | deleted file mode 100755 |
3 | index d205b69..0000000 | |||
4 | --- a/bin/cron.daily | |||
5 | +++ /dev/null | |||
6 | @@ -1,46 +0,0 @@ | |||
7 | 1 | #! /usr/bin/python3 | ||
8 | 2 | |||
9 | 3 | # Copyright (C) 2013 Canonical Ltd. | ||
10 | 4 | # Author: Colin Watson <cjwatson@ubuntu.com> | ||
11 | 5 | |||
12 | 6 | # This program is free software: you can redistribute it and/or modify | ||
13 | 7 | # it under the terms of the GNU General Public License as published by | ||
14 | 8 | # the Free Software Foundation; version 3 of the License. | ||
15 | 9 | # | ||
16 | 10 | # This program is distributed in the hope that it will be useful, | ||
17 | 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
18 | 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
19 | 13 | # GNU General Public License for more details. | ||
20 | 14 | # | ||
21 | 15 | # You should have received a copy of the GNU General Public License | ||
22 | 16 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
23 | 17 | |||
24 | 18 | """Build a set of alternate/server images.""" | ||
25 | 19 | |||
26 | 20 | from optparse import OptionParser | ||
27 | 21 | import os | ||
28 | 22 | import sys | ||
29 | 23 | |||
30 | 24 | sys.path.insert(0, os.path.join(sys.path[0], os.pardir, "lib")) | ||
31 | 25 | |||
32 | 26 | |||
33 | 27 | def main(): | ||
34 | 28 | from cdimage.build import build_image_set | ||
35 | 29 | from cdimage.config import Config | ||
36 | 30 | |||
37 | 31 | parser = OptionParser("%prog") | ||
38 | 32 | parser.add_option( | ||
39 | 33 | "--live", default=False, action="store_true", | ||
40 | 34 | help="build live filesystems first") | ||
41 | 35 | options, _ = parser.parse_args() | ||
42 | 36 | config = Config(IMAGE_TYPE="daily") | ||
43 | 37 | config["CDIMAGE_INSTALL"] = "1" | ||
44 | 38 | project = config.project | ||
45 | 39 | if project == "ubuntu-server" and not config["CDIMAGE_NO_SQUASHFS_BASE"]: | ||
46 | 40 | config["CDIMAGE_SQUASHFS_BASE"] = "1" | ||
47 | 41 | if not build_image_set(config, options): | ||
48 | 42 | sys.exit(1) | ||
49 | 43 | |||
50 | 44 | |||
51 | 45 | if __name__ == "__main__": | ||
52 | 46 | main() | ||
53 | diff --git a/bin/cron.dvd b/bin/cron.dvd | |||
54 | index 3732b09..5566973 100755 | |||
55 | --- a/bin/cron.dvd | |||
56 | +++ b/bin/cron.dvd | |||
57 | @@ -36,7 +36,8 @@ def main(): | |||
58 | 36 | config = Config(IMAGE_TYPE="dvd") | 36 | config = Config(IMAGE_TYPE="dvd") |
59 | 37 | project = config.project | 37 | project = config.project |
60 | 38 | if project not in ("ubuntu", "ubuntustudio"): | 38 | if project not in ("ubuntu", "ubuntustudio"): |
62 | 39 | config["CDIMAGE_INSTALL"] = "1" | 39 | print("unsupported configuration!") |
63 | 40 | sys.exit(1) | ||
64 | 40 | config["CDIMAGE_LIVE"] = "1" | 41 | config["CDIMAGE_LIVE"] = "1" |
65 | 41 | config["CDIMAGE_DVD"] = "1" | 42 | config["CDIMAGE_DVD"] = "1" |
66 | 42 | if not build_image_set(config, options): | 43 | if not build_image_set(config, options): |
67 | diff --git a/lib/cdimage/build.py b/lib/cdimage/build.py | |||
68 | index cbc07d4..edd53db 100644 | |||
69 | --- a/lib/cdimage/build.py | |||
70 | +++ b/lib/cdimage/build.py | |||
71 | @@ -30,7 +30,6 @@ import traceback | |||
72 | 30 | 30 | ||
73 | 31 | from cdimage import osextras | 31 | from cdimage import osextras |
74 | 32 | from cdimage.build_id import next_build_id | 32 | from cdimage.build_id import next_build_id |
75 | 33 | from cdimage.check_installable import check_installable | ||
76 | 34 | from cdimage.germinate import Germination | 33 | from cdimage.germinate import Germination |
77 | 35 | from cdimage.livefs import ( | 34 | from cdimage.livefs import ( |
78 | 36 | LiveBuildsFailed, | 35 | LiveBuildsFailed, |
79 | @@ -88,9 +87,6 @@ def configure_for_project(config): | |||
80 | 88 | ): | 87 | ): |
81 | 89 | config["CDIMAGE_UNSUPPORTED"] = "1" | 88 | config["CDIMAGE_UNSUPPORTED"] = "1" |
82 | 90 | 89 | ||
83 | 91 | if config["CDIMAGE_INSTALL"]: | ||
84 | 92 | config["CDIMAGE_INSTALL_BASE"] = "1" | ||
85 | 93 | |||
86 | 94 | 90 | ||
87 | 95 | def open_log(config): | 91 | def open_log(config): |
88 | 96 | if config["DEBUG"] or config["CDIMAGE_NOLOG"]: | 92 | if config["DEBUG"] or config["CDIMAGE_NOLOG"]: |
89 | @@ -661,7 +657,7 @@ def build_image_set_locked(config, options): | |||
90 | 661 | log_marker("Checking for other task changes") | 657 | log_marker("Checking for other task changes") |
91 | 662 | germinate_output.update_tasks(date) | 658 | germinate_output.update_tasks(date) |
92 | 663 | 659 | ||
94 | 664 | if (config["CDIMAGE_LIVE"] or config["CDIMAGE_SQUASHFS_BASE"]): | 660 | if config["CDIMAGE_LIVE"]: |
95 | 665 | log_marker("Downloading live filesystem images") | 661 | log_marker("Downloading live filesystem images") |
96 | 666 | download_live_filesystems(config, builds) | 662 | download_live_filesystems(config, builds) |
97 | 667 | 663 | ||
98 | @@ -671,13 +667,6 @@ def build_image_set_locked(config, options): | |||
99 | 671 | copy_netboot_tarballs(config) | 667 | copy_netboot_tarballs(config) |
100 | 672 | fix_permissions(config) | 668 | fix_permissions(config) |
101 | 673 | 669 | ||
102 | 674 | # Temporarily turned off for live builds. | ||
103 | 675 | if (config["CDIMAGE_INSTALL_BASE"] and | ||
104 | 676 | not config["CDIMAGE_ADDON"] and | ||
105 | 677 | not config["CDIMAGE_PREINSTALLED"]): | ||
106 | 678 | log_marker("Producing installability report") | ||
107 | 679 | check_installable(config) | ||
108 | 680 | |||
109 | 681 | if not config["DEBUG"] and not config["CDIMAGE_NOPUBLISH"]: | 670 | if not config["DEBUG"] and not config["CDIMAGE_NOPUBLISH"]: |
110 | 682 | log_marker("Publishing") | 671 | log_marker("Publishing") |
111 | 683 | tree = Tree.get_daily(config) | 672 | tree = Tree.get_daily(config) |
112 | diff --git a/lib/cdimage/check_installable.py b/lib/cdimage/check_installable.py | |||
113 | index 62cbfc6..0aff60e 100644 | |||
114 | --- a/lib/cdimage/check_installable.py | |||
115 | +++ b/lib/cdimage/check_installable.py | |||
116 | @@ -60,23 +60,6 @@ def _prepare_check_installable(config): | |||
117 | 60 | 60 | ||
118 | 61 | packages = os.path.join(data, "Packages_%s" % arch) | 61 | packages = os.path.join(data, "Packages_%s" % arch) |
119 | 62 | with open(packages, "wb") as packages_file: | 62 | with open(packages, "wb") as packages_file: |
120 | 63 | if config["CDIMAGE_SQUASHFS_BASE"]: | ||
121 | 64 | squashfs = os.path.join(live, "%s.squashfs" % fullarch) | ||
122 | 65 | if os.path.exists(squashfs): | ||
123 | 66 | _ensure_tempdir() | ||
124 | 67 | with open("/dev/null", "w") as devnull: | ||
125 | 68 | subprocess.check_call([ | ||
126 | 69 | "unsquashfs", | ||
127 | 70 | "-d", os.path.join(_tempdir, fullarch), | ||
128 | 71 | squashfs, "/var/lib/dpkg/status", | ||
129 | 72 | ], stdout=devnull) | ||
130 | 73 | status_path = os.path.join( | ||
131 | 74 | _tempdir, fullarch, "var", "lib", "dpkg", "status") | ||
132 | 75 | with open(os.path.join(status_path)) as status: | ||
133 | 76 | subprocess.call([ | ||
134 | 77 | "grep-dctrl", "-XFStatus", "install ok installed", | ||
135 | 78 | ], stdin=status, stdout=packages_file) | ||
136 | 79 | |||
137 | 80 | for component in "main", "restricted", "universe", "multiverse": | 63 | for component in "main", "restricted", "universe", "multiverse": |
138 | 81 | packages_gz = os.path.join( | 64 | packages_gz = os.path.join( |
139 | 82 | image_top, "%s-%s" % (config.series, fullarch), "CD1", | 65 | image_top, "%s-%s" % (config.series, fullarch), "CD1", |
140 | diff --git a/lib/cdimage/germinate.py b/lib/cdimage/germinate.py | |||
141 | index dcc5e88..004e2a3 100644 | |||
142 | --- a/lib/cdimage/germinate.py | |||
143 | +++ b/lib/cdimage/germinate.py | |||
144 | @@ -17,10 +17,7 @@ | |||
145 | 17 | 17 | ||
146 | 18 | from __future__ import print_function | 18 | from __future__ import print_function |
147 | 19 | 19 | ||
148 | 20 | from collections import OrderedDict, defaultdict | ||
149 | 21 | import errno | ||
150 | 22 | import os | 20 | import os |
151 | 23 | import re | ||
152 | 24 | import shutil | 21 | import shutil |
153 | 25 | import subprocess | 22 | import subprocess |
154 | 26 | import traceback | 23 | import traceback |
155 | @@ -47,13 +44,9 @@ class Germination: | |||
156 | 47 | 44 | ||
157 | 48 | @property | 45 | @property |
158 | 49 | def germinate_path(self): | 46 | def germinate_path(self): |
166 | 50 | paths = [ | 47 | path = os.path.join(self.config.root, "germinate", "bin", "germinate") |
167 | 51 | os.path.join(self.config.root, "germinate", "bin", "germinate"), | 48 | if os.access(path, os.X_OK): |
168 | 52 | os.path.join(self.config.root, "germinate", "germinate.py"), | 49 | return path |
162 | 53 | ] | ||
163 | 54 | for path in paths: | ||
164 | 55 | if os.access(path, os.X_OK): | ||
165 | 56 | return path | ||
169 | 57 | else: | 50 | else: |
170 | 58 | raise GerminateNotInstalled( | 51 | raise GerminateNotInstalled( |
171 | 59 | "Please check out lp:germinate in %s." % | 52 | "Please check out lp:germinate in %s." % |
172 | @@ -192,12 +185,6 @@ class NoMasterSeeds(Exception): | |||
173 | 192 | pass | 185 | pass |
174 | 193 | 186 | ||
175 | 194 | 187 | ||
176 | 195 | re_not_base = re.compile( | ||
177 | 196 | r"^(linux-(image|restricted|amd64|386|686|k7|power|" | ||
178 | 197 | r"imx51|dove|omap).*|" | ||
179 | 198 | r"nvidia-kernel-common|grub|yaboot|efibootmgr|elilo|silo|palo)$") | ||
180 | 199 | |||
181 | 200 | |||
182 | 201 | class GerminateOutput: | 188 | class GerminateOutput: |
183 | 202 | def __init__(self, config, directory): | 189 | def __init__(self, config, directory): |
184 | 203 | self.config = config | 190 | self.config = config |
185 | @@ -206,102 +193,30 @@ class GerminateOutput: | |||
186 | 206 | self._parse_structure() | 193 | self._parse_structure() |
187 | 207 | 194 | ||
188 | 208 | def _parse_structure(self): | 195 | def _parse_structure(self): |
190 | 209 | self._seeds = OrderedDict() | 196 | self._seeds = [] |
191 | 210 | with open(self.structure) as structure: | 197 | with open(self.structure) as structure: |
192 | 211 | for line in structure: | 198 | for line in structure: |
193 | 212 | line = line.strip() | 199 | line = line.strip() |
194 | 213 | if not line or line.startswith("#") or ":" not in line: | 200 | if not line or line.startswith("#") or ":" not in line: |
195 | 214 | continue | 201 | continue |
216 | 215 | seed, inherit = line.split(":", 1) | 202 | self._seeds.append(line.split(":", 1)[0]) |
217 | 216 | self._seeds[seed] = inherit.split() | 203 | |
218 | 217 | 204 | def pool_seeds(self): | |
219 | 218 | def _expand_inheritance(self, seed, inherit): | 205 | if not (self.config["CDIMAGE_DVD"] or self.config["CDIMAGE_LIVE"]): |
220 | 219 | for s in self._seeds.get(seed, ()): | 206 | raise NoMasterSeeds("No seeds found for master task!") |
201 | 220 | self._expand_inheritance(s, inherit) | ||
202 | 221 | if seed not in inherit: | ||
203 | 222 | inherit.append(seed) | ||
204 | 223 | |||
205 | 224 | def _inheritance(self, seed): | ||
206 | 225 | inherit = [] | ||
207 | 226 | self._expand_inheritance(seed, inherit) | ||
208 | 227 | return inherit | ||
209 | 228 | |||
210 | 229 | def _without_inheritance(self, subtract, seeds): | ||
211 | 230 | subtract_inherit = self._inheritance(subtract) | ||
212 | 231 | remaining = set(seeds) - set(subtract_inherit) | ||
213 | 232 | return [seed for seed in seeds if seed in remaining] | ||
214 | 233 | |||
215 | 234 | def list_seeds(self, mode): | ||
221 | 235 | project = self.config.project | 207 | project = self.config.project |
222 | 236 | series = self.config["DIST"] | ||
223 | 237 | 208 | ||
291 | 238 | if mode == "all": | 209 | if project == "ubuntustudio": |
292 | 239 | for seed in self._seeds: | 210 | yield "dvd" |
293 | 240 | yield seed | 211 | yield "ship-live" |
294 | 241 | elif mode == "tasks": | 212 | elif project == "ubuntu-server": |
295 | 242 | ship = "ship" | 213 | yield "server-ship-live" |
296 | 243 | if "ship-addon" in self._seeds: | 214 | elif project == "ubuntu" and self.config["SUBPROJECT"] == "canary": |
297 | 244 | ship = "ship-addon" | 215 | # ubuntu-desktop-installer |
298 | 245 | in_squashfs = None | 216 | yield "canary-ship-live" |
299 | 246 | if project == "ubuntu-server": | 217 | # TODO: will we need a legacy-ship-live seed? |
300 | 247 | ship = "server-ship" | 218 | else: |
301 | 248 | in_squashfs = ["minimal"] | 219 | yield "ship-live" |
235 | 249 | seeds = self._inheritance(ship) | ||
236 | 250 | if (self.config["CDIMAGE_SQUASHFS_BASE"] and | ||
237 | 251 | in_squashfs is not None): | ||
238 | 252 | for subtract in in_squashfs: | ||
239 | 253 | seeds = self._without_inheritance(subtract, seeds) | ||
240 | 254 | for seed in seeds: | ||
241 | 255 | yield seed | ||
242 | 256 | if self.config["CDIMAGE_DVD"]: | ||
243 | 257 | # TODO cjwatson 2007-04-18: hideous hack to fix DVD tasks | ||
244 | 258 | yield "dns-server" | ||
245 | 259 | yield "lamp-server" | ||
246 | 260 | elif mode == "installer": | ||
247 | 261 | if self.config["CDIMAGE_INSTALL_BASE"]: | ||
248 | 262 | yield "installer" | ||
249 | 263 | elif mode == "debootstrap": | ||
250 | 264 | yield "required" | ||
251 | 265 | yield "minimal" | ||
252 | 266 | elif mode == "base": | ||
253 | 267 | yield "boot" | ||
254 | 268 | yield "required" | ||
255 | 269 | yield "minimal" | ||
256 | 270 | yield "standard" | ||
257 | 271 | elif mode == "ship-live": | ||
258 | 272 | if project == "lubuntu" and series == "bionic": | ||
259 | 273 | yield "ship-live-gtk" | ||
260 | 274 | yield "ship-live-share" | ||
261 | 275 | elif project == "lubuntu-next" and series == "bionic": | ||
262 | 276 | yield "ship-live-qt" | ||
263 | 277 | yield "ship-live-share" | ||
264 | 278 | elif project == "ubuntu-server" and series >= "bionic": | ||
265 | 279 | yield "server-ship-live" | ||
266 | 280 | elif project == "ubuntu" and self.config["SUBPROJECT"] == "canary": | ||
267 | 281 | # ubuntu-desktop-installer | ||
268 | 282 | yield "canary-ship-live" | ||
269 | 283 | # TODO: we will probably need a legacy-ship-live seed | ||
270 | 284 | else: | ||
271 | 285 | yield "ship-live" | ||
272 | 286 | elif mode == "addon": | ||
273 | 287 | ship = self._inheritance("ship") | ||
274 | 288 | ship_addon = self._inheritance("ship-addon") | ||
275 | 289 | for seed in ship_addon: | ||
276 | 290 | if seed not in ship: | ||
277 | 291 | yield seed | ||
278 | 292 | elif mode == "dvd": | ||
279 | 293 | if project == "ubuntu": | ||
280 | 294 | # no inheritance; most of this goes on the live filesystem | ||
281 | 295 | yield "usb-langsupport" | ||
282 | 296 | yield "usb-ship-live" | ||
283 | 297 | elif project == "ubuntustudio": | ||
284 | 298 | # no inheritance; most of this goes on the live filesystem | ||
285 | 299 | yield "dvd" | ||
286 | 300 | if series >= "bionic": | ||
287 | 301 | yield "ship-live" | ||
288 | 302 | else: | ||
289 | 303 | for seed in self._inheritance("dvd"): | ||
290 | 304 | yield seed | ||
302 | 305 | 220 | ||
303 | 306 | def seed_path(self, arch, seed): | 221 | def seed_path(self, arch, seed): |
304 | 307 | return os.path.join(self.directory, arch, seed) | 222 | return os.path.join(self.directory, arch, seed) |
305 | @@ -311,42 +226,12 @@ class GerminateOutput: | |||
306 | 311 | lines = seed_file.read().splitlines()[2:-2] | 226 | lines = seed_file.read().splitlines()[2:-2] |
307 | 312 | return [line.split(None, 1)[0] for line in lines] | 227 | return [line.split(None, 1)[0] for line in lines] |
308 | 313 | 228 | ||
309 | 314 | def master_seeds(self): | ||
310 | 315 | if self.config["CDIMAGE_ADDON"]: | ||
311 | 316 | for seed in self.list_seeds("addon"): | ||
312 | 317 | yield seed | ||
313 | 318 | else: | ||
314 | 319 | for seed in self.list_seeds("installer"): | ||
315 | 320 | yield seed | ||
316 | 321 | if self.config["CDIMAGE_DVD"]: | ||
317 | 322 | for seed in self.list_seeds("dvd"): | ||
318 | 323 | if seed not in ("installer", "casper"): | ||
319 | 324 | yield seed | ||
320 | 325 | elif self.config["CDIMAGE_INSTALL"]: | ||
321 | 326 | for seed in self.list_seeds("tasks"): | ||
322 | 327 | if seed not in ("installer", "casper"): | ||
323 | 328 | yield seed | ||
324 | 329 | else: | ||
325 | 330 | if self.config.get("CDIMAGE_INSTALL_BASE") == "1": | ||
326 | 331 | for seed in self.list_seeds("base"): | ||
327 | 332 | if seed not in ("installer", "casper"): | ||
328 | 333 | yield seed | ||
329 | 334 | if self.config.get("CDIMAGE_LIVE") == "1": | ||
330 | 335 | for seed in self.list_seeds("ship-live"): | ||
331 | 336 | if seed not in ("installer", "casper"): | ||
332 | 337 | yield seed | ||
333 | 338 | |||
334 | 339 | def master_task_entries(self): | 229 | def master_task_entries(self): |
335 | 340 | project = self.config.project | 230 | project = self.config.project |
336 | 341 | series = self.config.series | 231 | series = self.config.series |
337 | 342 | 232 | ||
340 | 343 | found = False | 233 | for seed in self.pool_seeds(): |
339 | 344 | for seed in self.master_seeds(): | ||
341 | 345 | yield "#include <%s/%s/%s>" % (project, series, seed) | 234 | yield "#include <%s/%s/%s>" % (project, series, seed) |
342 | 346 | found = True | ||
343 | 347 | |||
344 | 348 | if not found: | ||
345 | 349 | raise NoMasterSeeds("No seeds found for master task!") | ||
346 | 350 | 235 | ||
347 | 351 | def tasks_output_dir(self): | 236 | def tasks_output_dir(self): |
348 | 352 | return os.path.join( | 237 | return os.path.join( |
349 | @@ -354,116 +239,21 @@ class GerminateOutput: | |||
350 | 354 | self.config.project, self.config.full_series, | 239 | self.config.project, self.config.full_series, |
351 | 355 | self.config.image_type, "tasks") | 240 | self.config.image_type, "tasks") |
352 | 356 | 241 | ||
353 | 357 | def task_packages(self, arch, seed, seedsource): | ||
354 | 358 | """Like seed_packages, but with various special-case hacks.""" | ||
355 | 359 | installer_seeds = set(self.list_seeds("installer")) | ||
356 | 360 | |||
357 | 361 | for package in self.seed_packages(arch, seedsource): | ||
358 | 362 | # Hackily exclude kernel-image-* from the installer and casper | ||
359 | 363 | # tasks. Those udebs only exist to satisfy dependencies when | ||
360 | 364 | # building the debian-installer package. | ||
361 | 365 | if seed in installer_seeds and package.startswith("kernel-image-"): | ||
362 | 366 | continue | ||
363 | 367 | |||
364 | 368 | # Force the use of live-installer rather than bootstrap-base on | ||
365 | 369 | # squashfs-base images. Seed expansion doesn't do the right | ||
366 | 370 | # thing here because the installer seed is expanded before | ||
367 | 371 | # considering server-ship. | ||
368 | 372 | if self.config["CDIMAGE_SQUASHFS_BASE"]: | ||
369 | 373 | if package == "bootstrap-base": | ||
370 | 374 | package = "live-installer" | ||
371 | 375 | |||
372 | 376 | yield package | ||
373 | 377 | |||
374 | 378 | def task_project(self): | ||
375 | 379 | # ubuntu-server really wants ubuntu-* tasks. | ||
376 | 380 | if self.config.project == "ubuntu-server": | ||
377 | 381 | return "ubuntu" | ||
378 | 382 | else: | ||
379 | 383 | return self.config.project | ||
380 | 384 | |||
381 | 385 | def task_headers(self, arch, seed): | ||
382 | 386 | headers = {} | ||
383 | 387 | try: | ||
384 | 388 | with open("%s.seedtext" % self.seed_path(arch, seed)) as seedtext: | ||
385 | 389 | for line in seedtext: | ||
386 | 390 | if not line.lower().startswith("task-"): | ||
387 | 391 | continue | ||
388 | 392 | line = line.rstrip("\n") | ||
389 | 393 | key, value = line.split(":", 1) | ||
390 | 394 | key = key[5:].lower() | ||
391 | 395 | value = value.lstrip() | ||
392 | 396 | headers[key] = value | ||
393 | 397 | except IOError as e: | ||
394 | 398 | if e.errno != errno.ENOENT: | ||
395 | 399 | raise | ||
396 | 400 | return headers | ||
397 | 401 | |||
398 | 402 | def seed_task_mapping(self, arch): | ||
399 | 403 | task_project = self.task_project() | ||
400 | 404 | for seed in self.list_seeds("all"): | ||
401 | 405 | # Tasks implemented via tasksel, with Task-Seeds to indicate | ||
402 | 406 | # task/seed mapping. | ||
403 | 407 | task = seed | ||
404 | 408 | headers = self.task_headers(arch, seed) | ||
405 | 409 | if not headers: | ||
406 | 410 | continue | ||
407 | 411 | input_seeds = [seed] + headers.get("seeds", "").split() | ||
408 | 412 | if "per-derivative" in headers: | ||
409 | 413 | task = "%s-%s" % (task_project, task) | ||
410 | 414 | |||
411 | 415 | yield input_seeds, task | ||
412 | 416 | |||
413 | 417 | def write_tasks(self): | 242 | def write_tasks(self): |
414 | 418 | output_dir = self.tasks_output_dir() | 243 | output_dir = self.tasks_output_dir() |
415 | 419 | osextras.mkemptydir(self.tasks_output_dir()) | 244 | osextras.mkemptydir(self.tasks_output_dir()) |
416 | 420 | 245 | ||
417 | 421 | for arch in self.config.arches: | 246 | for arch in self.config.arches: |
418 | 422 | packages = defaultdict(list) | ||
419 | 423 | cpparch = arch.replace("+", "_").replace("-", "_") | 247 | cpparch = arch.replace("+", "_").replace("-", "_") |
427 | 424 | for seed in self.list_seeds("all"): | 248 | for seed in self._seeds: |
428 | 425 | if seed == "supported": | 249 | if not os.path.exists(self.seed_path(arch, seed)): |
422 | 426 | seedsource = "%s+build-depends" % seed | ||
423 | 427 | else: | ||
424 | 428 | seedsource = seed | ||
425 | 429 | seed_path = self.seed_path(arch, seedsource) | ||
426 | 430 | if not os.path.exists(seed_path): | ||
429 | 431 | continue | 250 | continue |
430 | 432 | with open(os.path.join(output_dir, seed), "a") as task_file: | 251 | with open(os.path.join(output_dir, seed), "a") as task_file: |
431 | 433 | print("#ifdef ARCH_%s" % cpparch, file=task_file) | 252 | print("#ifdef ARCH_%s" % cpparch, file=task_file) |
435 | 434 | for package in sorted( | 253 | for package in sorted(self.seed_packages(arch, seed)): |
433 | 435 | self.task_packages(arch, seed, seedsource)): | ||
434 | 436 | packages[seed].append(package) | ||
436 | 437 | print(package, file=task_file) | 254 | print(package, file=task_file) |
437 | 438 | print("#endif /* ARCH_%s */" % cpparch, file=task_file) | 255 | print("#endif /* ARCH_%s */" % cpparch, file=task_file) |
438 | 439 | 256 | ||
439 | 440 | tasks = defaultdict(list) | ||
440 | 441 | for input_seeds, task in self.seed_task_mapping(arch): | ||
441 | 442 | for input_seed in input_seeds: | ||
442 | 443 | for pkg in packages.get(input_seed, []): | ||
443 | 444 | tasks[pkg].append(task) | ||
444 | 445 | |||
445 | 446 | # Help debian-cd to regenerate Task headers, to make sure that | ||
446 | 447 | # we don't accidentally end up out of sync with the archive and | ||
447 | 448 | # break the package installation step. | ||
448 | 449 | override_path = os.path.join(output_dir, "override.%s" % arch) | ||
449 | 450 | with open(override_path, "w") as override: | ||
450 | 451 | for pkg, tasknames in sorted(tasks.items()): | ||
451 | 452 | print( | ||
452 | 453 | "%s Task %s" % (pkg, ", ".join(tasknames)), | ||
453 | 454 | file=override) | ||
454 | 455 | # Help debian-cd to get priorities in sync with the current base | ||
455 | 456 | # system, so that debootstrap >= 0.3.1 can work out the correct | ||
456 | 457 | # set of packages to install. | ||
457 | 458 | important_path = os.path.join(output_dir, "important.%s" % arch) | ||
458 | 459 | with open(important_path, "w") as important_file: | ||
459 | 460 | important = [] | ||
460 | 461 | for seed in self.list_seeds("debootstrap"): | ||
461 | 462 | important.extend(packages.get(seed, [])) | ||
462 | 463 | for pkg in sorted(important): | ||
463 | 464 | if not re_not_base.match(pkg): | ||
464 | 465 | print(pkg, file=important_file) | ||
465 | 466 | |||
466 | 467 | with open(os.path.join(output_dir, "MASTER"), "w") as master: | 257 | with open(os.path.join(output_dir, "MASTER"), "w") as master: |
467 | 468 | for entry in self.master_task_entries(): | 258 | for entry in self.master_task_entries(): |
468 | 469 | print(entry, file=master) | 259 | print(entry, file=master) |
469 | @@ -471,7 +261,7 @@ class GerminateOutput: | |||
470 | 471 | def diff_tasks(self, output=None): | 261 | def diff_tasks(self, output=None): |
471 | 472 | tasks_dir = self.tasks_output_dir() | 262 | tasks_dir = self.tasks_output_dir() |
472 | 473 | previous_tasks_dir = "%s-previous" % tasks_dir | 263 | previous_tasks_dir = "%s-previous" % tasks_dir |
474 | 474 | for seed in ["MASTER"] + list(self.list_seeds("all")): | 264 | for seed in ["MASTER"] + list(self._seeds): |
475 | 475 | old = os.path.join(previous_tasks_dir, seed) | 265 | old = os.path.join(previous_tasks_dir, seed) |
476 | 476 | new = os.path.join(tasks_dir, seed) | 266 | new = os.path.join(tasks_dir, seed) |
477 | 477 | if os.path.exists(old) and os.path.exists(new): | 267 | if os.path.exists(old) and os.path.exists(new): |
478 | diff --git a/lib/cdimage/livefs.py b/lib/cdimage/livefs.py | |||
479 | index c1b475a..f361745 100644 | |||
480 | --- a/lib/cdimage/livefs.py | |||
481 | +++ b/lib/cdimage/livefs.py | |||
482 | @@ -724,8 +724,7 @@ def download_live_filesystems(config, builds): | |||
483 | 724 | output_dir = live_output_directory(config) | 724 | output_dir = live_output_directory(config) |
484 | 725 | osextras.mkemptydir(output_dir) | 725 | osextras.mkemptydir(output_dir) |
485 | 726 | 726 | ||
488 | 727 | if (config["CDIMAGE_LIVE"] or config["CDIMAGE_SQUASHFS_BASE"] or | 727 | if config["CDIMAGE_LIVE"] or config["CDIMAGE_PREINSTALLED"]: |
487 | 728 | config["CDIMAGE_PREINSTALLED"]): | ||
489 | 729 | got_image = False | 728 | got_image = False |
490 | 730 | for arch in config.arches: | 729 | for arch in config.arches: |
491 | 731 | if config["CDIMAGE_PREINSTALLED"]: | 730 | if config["CDIMAGE_PREINSTALLED"]: |
492 | @@ -780,9 +779,7 @@ def download_live_filesystems(config, builds): | |||
493 | 780 | got_image = True | 779 | got_image = True |
494 | 781 | else: | 780 | else: |
495 | 782 | continue | 781 | continue |
499 | 783 | if (project != "ubuntu-base" and | 782 | if project != "ubuntu-base" and config.subproject != "wubi": |
497 | 784 | not config["CDIMAGE_SQUASHFS_BASE"] and | ||
498 | 785 | config.subproject != "wubi"): | ||
500 | 786 | download_live_items(config, builds, arch, "kernel") | 783 | download_live_items(config, builds, arch, "kernel") |
501 | 787 | download_live_items(config, builds, arch, "initrd") | 784 | download_live_items(config, builds, arch, "initrd") |
502 | 788 | download_live_items(config, builds, arch, "kernel-efi-signed") | 785 | download_live_items(config, builds, arch, "kernel-efi-signed") |
503 | diff --git a/lib/cdimage/tests/test_build.py b/lib/cdimage/tests/test_build.py | |||
504 | index 1769f09..aac35e1 100644 | |||
505 | --- a/lib/cdimage/tests/test_build.py | |||
506 | +++ b/lib/cdimage/tests/test_build.py | |||
507 | @@ -449,16 +449,6 @@ class TestBuildImageSet(TestCase): | |||
508 | 449 | else: | 449 | else: |
509 | 450 | self.assertNotIn("CDIMAGE_UNSUPPORTED", config) | 450 | self.assertNotIn("CDIMAGE_UNSUPPORTED", config) |
510 | 451 | 451 | ||
511 | 452 | def test_configure_install_base(self): | ||
512 | 453 | config = Config(read=False) | ||
513 | 454 | configure_for_project(config) | ||
514 | 455 | self.assertNotIn("CDIMAGE_INSTALL_BASE", config) | ||
515 | 456 | |||
516 | 457 | config = Config(read=False) | ||
517 | 458 | config["CDIMAGE_INSTALL"] = "1" | ||
518 | 459 | configure_for_project(config) | ||
519 | 460 | self.assertEqual("1", config["CDIMAGE_INSTALL_BASE"]) | ||
520 | 461 | |||
521 | 462 | @mock.patch("os.open") | 452 | @mock.patch("os.open") |
522 | 463 | def test_open_log_debug(self, mock_open): | 453 | def test_open_log_debug(self, mock_open): |
523 | 464 | self.config["DEBUG"] = "1" | 454 | self.config["DEBUG"] = "1" |
524 | diff --git a/lib/cdimage/tests/test_check_installable.py b/lib/cdimage/tests/test_check_installable.py | |||
525 | index 902771e..a671e96 100644 | |||
526 | --- a/lib/cdimage/tests/test_check_installable.py | |||
527 | +++ b/lib/cdimage/tests/test_check_installable.py | |||
528 | @@ -21,8 +21,6 @@ from __future__ import print_function | |||
529 | 21 | 21 | ||
530 | 22 | import gzip | 22 | import gzip |
531 | 23 | import os | 23 | import os |
532 | 24 | import subprocess | ||
533 | 25 | from textwrap import dedent | ||
534 | 26 | 24 | ||
535 | 27 | from cdimage.check_installable import ( | 25 | from cdimage.check_installable import ( |
536 | 28 | _check_installable_command, | 26 | _check_installable_command, |
537 | @@ -30,7 +28,7 @@ from cdimage.check_installable import ( | |||
538 | 30 | _prepare_check_installable, | 28 | _prepare_check_installable, |
539 | 31 | ) | 29 | ) |
540 | 32 | from cdimage.config import Config | 30 | from cdimage.config import Config |
542 | 33 | from cdimage.tests.helpers import TestCase, mkfile | 31 | from cdimage.tests.helpers import TestCase |
543 | 34 | 32 | ||
544 | 35 | 33 | ||
545 | 36 | class TestCheckInstallable(TestCase): | 34 | class TestCheckInstallable(TestCase): |
546 | @@ -86,38 +84,6 @@ class TestCheckInstallable(TestCase): | |||
547 | 86 | with open(os.path.join(data, "Packages_i386")) as packages_file: | 84 | with open(os.path.join(data, "Packages_i386")) as packages_file: |
548 | 87 | self.assertEqual("Package: foo\n\n", packages_file.read()) | 85 | self.assertEqual("Package: foo\n\n", packages_file.read()) |
549 | 88 | 86 | ||
550 | 89 | def test_prepare_squashfs_base(self): | ||
551 | 90 | self.config["CDIMAGE_SQUASHFS_BASE"] = "1" | ||
552 | 91 | _, image_top, live, data = _check_installable_dirs(self.config) | ||
553 | 92 | squashfs_dir = os.path.join(self.temp_dir, "squashfs") | ||
554 | 93 | status_path = os.path.join( | ||
555 | 94 | squashfs_dir, "var", "lib", "dpkg", "status") | ||
556 | 95 | fake_packages = dedent("""\ | ||
557 | 96 | Package: base-files | ||
558 | 97 | Status: install ok installed | ||
559 | 98 | Version: 6.5 | ||
560 | 99 | |||
561 | 100 | Package: libc6 | ||
562 | 101 | Status: install ok installed | ||
563 | 102 | Version: 2.15-1 | ||
564 | 103 | Provides: glibc | ||
565 | 104 | |||
566 | 105 | """) | ||
567 | 106 | with mkfile(status_path) as status: | ||
568 | 107 | print(fake_packages, end="", file=status) | ||
569 | 108 | os.makedirs(live) | ||
570 | 109 | with open("/dev/null", "w") as devnull: | ||
571 | 110 | subprocess.check_call( | ||
572 | 111 | ["mksquashfs", squashfs_dir, | ||
573 | 112 | os.path.join(live, "i386.squashfs")], | ||
574 | 113 | stdout=devnull) | ||
575 | 114 | self.capture_logging() | ||
576 | 115 | _prepare_check_installable(self.config) | ||
577 | 116 | self.assertLogEqual([]) | ||
578 | 117 | self.assertCountEqual(["Packages_i386", "Sources"], os.listdir(data)) | ||
579 | 118 | with open(os.path.join(data, "Packages_i386")) as packages_file: | ||
580 | 119 | self.assertEqual(fake_packages, packages_file.read()) | ||
581 | 120 | |||
582 | 121 | def test_command(self): | 87 | def test_command(self): |
583 | 122 | britney, _, _, data = _check_installable_dirs(self.config) | 88 | britney, _, _, data = _check_installable_dirs(self.config) |
584 | 123 | command = _check_installable_command(self.config) | 89 | command = _check_installable_command(self.config) |
585 | diff --git a/lib/cdimage/tests/test_germinate.py b/lib/cdimage/tests/test_germinate.py | |||
586 | index f567eae..7b15027 100644 | |||
587 | --- a/lib/cdimage/tests/test_germinate.py | |||
588 | +++ b/lib/cdimage/tests/test_germinate.py | |||
589 | @@ -29,7 +29,7 @@ try: | |||
590 | 29 | except ImportError: | 29 | except ImportError: |
591 | 30 | import mock | 30 | import mock |
592 | 31 | 31 | ||
594 | 32 | from cdimage.config import Config, all_series | 32 | from cdimage.config import Config |
595 | 33 | from cdimage.germinate import ( | 33 | from cdimage.germinate import ( |
596 | 34 | GerminateNotInstalled, | 34 | GerminateNotInstalled, |
597 | 35 | GerminateOutput, | 35 | GerminateOutput, |
598 | @@ -55,11 +55,6 @@ class TestGermination(TestCase): | |||
599 | 55 | GerminateNotInstalled, getattr, self.germination, "germinate_path") | 55 | GerminateNotInstalled, getattr, self.germination, "germinate_path") |
600 | 56 | 56 | ||
601 | 57 | germinate_dir = os.path.join(self.temp_dir, "germinate") | 57 | germinate_dir = os.path.join(self.temp_dir, "germinate") |
602 | 58 | old_germinate = os.path.join(germinate_dir, "germinate.py") | ||
603 | 59 | touch(old_germinate) | ||
604 | 60 | os.chmod(old_germinate, 0o755) | ||
605 | 61 | self.assertEqual(old_germinate, self.germination.germinate_path) | ||
606 | 62 | |||
607 | 63 | new_germinate = os.path.join(germinate_dir, "bin", "germinate") | 58 | new_germinate = os.path.join(germinate_dir, "bin", "germinate") |
608 | 64 | touch(new_germinate) | 59 | touch(new_germinate) |
609 | 65 | os.chmod(new_germinate, 0o755) | 60 | os.chmod(new_germinate, 0o755) |
610 | @@ -356,106 +351,6 @@ class TestGerminateOutput(TestCase): | |||
611 | 356 | ["active-ship-live", ["ship-live"]], | 351 | ["active-ship-live", ["ship-live"]], |
612 | 357 | ]) | 352 | ]) |
613 | 358 | 353 | ||
614 | 359 | def test_inheritance_recurses(self): | ||
615 | 360 | """_inheritance recurses properly.""" | ||
616 | 361 | self.write_structure([["a", []], ["b", ["a"]], ["c", ["b"]]]) | ||
617 | 362 | output = GerminateOutput(self.config, self.temp_dir) | ||
618 | 363 | self.assertEqual(["a"], output._inheritance("a")) | ||
619 | 364 | self.assertEqual(["a", "b"], output._inheritance("b")) | ||
620 | 365 | self.assertEqual(["a", "b", "c"], output._inheritance("c")) | ||
621 | 366 | |||
622 | 367 | def test_inheritance_avoids_duplicates(self): | ||
623 | 368 | """_inheritance avoids adding a seed more than once.""" | ||
624 | 369 | self.write_structure([["a", []], ["b", ["a"]], ["c", ["a", "b"]]]) | ||
625 | 370 | output = GerminateOutput(self.config, self.temp_dir) | ||
626 | 371 | self.assertEqual(["a", "b", "c"], output._inheritance("c")) | ||
627 | 372 | |||
628 | 373 | def test_without_inheritance(self): | ||
629 | 374 | self.write_structure( | ||
630 | 375 | [["a", []], ["b", ["a"]], ["c", ["b"]], ["d", ["a", "c"]]]) | ||
631 | 376 | output = GerminateOutput(self.config, self.temp_dir) | ||
632 | 377 | inheritance = output._inheritance("d") | ||
633 | 378 | self.assertEqual(["a", "b", "c", "d"], inheritance) | ||
634 | 379 | self.assertEqual( | ||
635 | 380 | ["c", "d"], output._without_inheritance("b", inheritance)) | ||
636 | 381 | |||
637 | 382 | def test_list_seeds_all(self): | ||
638 | 383 | self.write_structure([["a", []], ["b", ["a"]], ["c", []]]) | ||
639 | 384 | output = GerminateOutput(self.config, self.temp_dir) | ||
640 | 385 | self.assertEqual(["a", "b", "c"], list(output.list_seeds("all"))) | ||
641 | 386 | |||
642 | 387 | def test_list_seeds_tasks_ubuntu(self): | ||
643 | 388 | self.write_ubuntu_structure() | ||
644 | 389 | output = GerminateOutput(self.config, self.temp_dir) | ||
645 | 390 | self.config["PROJECT"] = "ubuntu" | ||
646 | 391 | self.config["DIST"] = "bionic" | ||
647 | 392 | expected = [ | ||
648 | 393 | "boot", "installer", "required", "minimal", "standard", | ||
649 | 394 | "desktop-common", "desktop", "d-i-requirements", "ship", | ||
650 | 395 | ] | ||
651 | 396 | self.assertEqual(expected, list(output.list_seeds("tasks"))) | ||
652 | 397 | self.config["CDIMAGE_DVD"] = "1" | ||
653 | 398 | expected.extend(["dns-server", "lamp-server"]) | ||
654 | 399 | self.assertEqual(expected, list(output.list_seeds("tasks"))) | ||
655 | 400 | |||
656 | 401 | def test_list_seeds_tasks_ubuntu_server(self): | ||
657 | 402 | self.write_ubuntu_structure() | ||
658 | 403 | output = GerminateOutput(self.config, self.temp_dir) | ||
659 | 404 | self.config["PROJECT"] = "ubuntu-server" | ||
660 | 405 | expected = [ | ||
661 | 406 | "boot", "installer", "required", "minimal", "standard", | ||
662 | 407 | "dns-server", "lamp-server", "openssh-server", "print-server", | ||
663 | 408 | "samba-server", "postgresql-server", "mail-server", "server", | ||
664 | 409 | "tomcat-server", "virt-host", "d-i-requirements", "server-ship", | ||
665 | 410 | ] | ||
666 | 411 | for series in all_series[4:]: | ||
667 | 412 | self.config["DIST"] = series | ||
668 | 413 | self.assertEqual(expected, list(output.list_seeds("tasks"))) | ||
669 | 414 | |||
670 | 415 | def test_list_seeds_task_ubuntu_server_squashfs(self): | ||
671 | 416 | self.write_ubuntu_structure() | ||
672 | 417 | output = GerminateOutput(self.config, self.temp_dir) | ||
673 | 418 | self.config["PROJECT"] = "ubuntu-server" | ||
674 | 419 | self.config["DIST"] = "bionic" | ||
675 | 420 | self.config["CDIMAGE_SQUASHFS_BASE"] = "1" | ||
676 | 421 | expected = [ | ||
677 | 422 | "boot", "installer", "standard", "dns-server", "lamp-server", | ||
678 | 423 | "openssh-server", "print-server", "samba-server", | ||
679 | 424 | "postgresql-server", "mail-server", "server", "tomcat-server", | ||
680 | 425 | "virt-host", "d-i-requirements", "server-ship", | ||
681 | 426 | ] | ||
682 | 427 | self.assertEqual(expected, list(output.list_seeds("tasks"))) | ||
683 | 428 | |||
684 | 429 | def test_list_seeds_installer(self): | ||
685 | 430 | self.write_structure([["installer", []], ["casper", []]]) | ||
686 | 431 | output = GerminateOutput(self.config, self.temp_dir) | ||
687 | 432 | self.config["CDIMAGE_INSTALL_BASE"] = "1" | ||
688 | 433 | self.assertEqual(["installer"], list(output.list_seeds("installer"))) | ||
689 | 434 | del self.config["CDIMAGE_INSTALL_BASE"] | ||
690 | 435 | self.config["CDIMAGE_LIVE"] = "1" | ||
691 | 436 | self.config["DIST"] = "bionic" | ||
692 | 437 | self.assertEqual([], list(output.list_seeds("installer"))) | ||
693 | 438 | |||
694 | 439 | def test_list_seeds_debootstrap(self): | ||
695 | 440 | self.write_ubuntu_structure() | ||
696 | 441 | output = GerminateOutput(self.config, self.temp_dir) | ||
697 | 442 | for series in all_series[6:]: | ||
698 | 443 | self.config["DIST"] = series | ||
699 | 444 | self.assertEqual( | ||
700 | 445 | ["required", "minimal"], | ||
701 | 446 | list(output.list_seeds("debootstrap"))) | ||
702 | 447 | |||
703 | 448 | def test_list_seeds_base(self): | ||
704 | 449 | self.write_ubuntu_structure() | ||
705 | 450 | output = GerminateOutput(self.config, self.temp_dir) | ||
706 | 451 | for series in all_series[6:]: | ||
707 | 452 | self.config["DIST"] = series | ||
708 | 453 | self.assertEqual( | ||
709 | 454 | ["boot", "required", "minimal", "standard"], | ||
710 | 455 | list(output.list_seeds("base"))) | ||
711 | 456 | |||
712 | 457 | # TODO list_seeds ship-live/addon/dvd untested | ||
713 | 458 | |||
714 | 459 | def test_seed_path(self): | 354 | def test_seed_path(self): |
715 | 460 | self.write_ubuntu_structure() | 355 | self.write_ubuntu_structure() |
716 | 461 | output = GerminateOutput(self.config, self.temp_dir) | 356 | output = GerminateOutput(self.config, self.temp_dir) |
717 | @@ -495,43 +390,8 @@ class TestGerminateOutput(TestCase): | |||
718 | 495 | ["base-files", "base-passwd"], | 390 | ["base-files", "base-passwd"], |
719 | 496 | output.seed_packages("i386", "base")) | 391 | output.seed_packages("i386", "base")) |
720 | 497 | 392 | ||
758 | 498 | # TODO: master_seeds addon untested | 393 | @mock.patch("cdimage.germinate.GerminateOutput.pool_seeds") |
759 | 499 | 394 | def test_master_task_entries(self, mock_pool_seeds): | |
723 | 500 | def test_master_seeds_dvd_ubuntu_bionic(self): | ||
724 | 501 | self.write_ubuntu_structure() | ||
725 | 502 | output = GerminateOutput(self.config, self.temp_dir) | ||
726 | 503 | self.config["PROJECT"] = "ubuntu" | ||
727 | 504 | self.config["DIST"] = "bionic" | ||
728 | 505 | self.config["CDIMAGE_DVD"] = "1" | ||
729 | 506 | self.assertEqual( | ||
730 | 507 | ["usb-langsupport", "usb-ship-live"], list(output.master_seeds())) | ||
731 | 508 | |||
732 | 509 | def test_master_seeds_install_ubuntu_bionic(self): | ||
733 | 510 | self.write_ubuntu_structure() | ||
734 | 511 | output = GerminateOutput(self.config, self.temp_dir) | ||
735 | 512 | self.config["PROJECT"] = "ubuntu" | ||
736 | 513 | self.config["DIST"] = "bionic" | ||
737 | 514 | self.config["CDIMAGE_INSTALL"] = "1" | ||
738 | 515 | self.config["CDIMAGE_INSTALL_BASE"] = "1" | ||
739 | 516 | self.assertEqual([ | ||
740 | 517 | "installer", "boot", "required", "minimal", "standard", | ||
741 | 518 | "desktop-common", "desktop", "d-i-requirements", "ship", | ||
742 | 519 | ], list(output.master_seeds())) | ||
743 | 520 | |||
744 | 521 | def test_master_seeds_live_ubuntu_bionic(self): | ||
745 | 522 | self.write_ubuntu_structure() | ||
746 | 523 | output = GerminateOutput(self.config, self.temp_dir) | ||
747 | 524 | self.config["PROJECT"] = "ubuntu" | ||
748 | 525 | self.config["DIST"] = "bionic" | ||
749 | 526 | self.config["CDIMAGE_INSTALL_BASE"] = "1" | ||
750 | 527 | self.config["CDIMAGE_LIVE"] = "1" | ||
751 | 528 | self.assertEqual([ | ||
752 | 529 | "installer", "boot", "required", "minimal", "standard", | ||
753 | 530 | "ship-live", | ||
754 | 531 | ], list(output.master_seeds())) | ||
755 | 532 | |||
756 | 533 | @mock.patch("cdimage.germinate.GerminateOutput.master_seeds") | ||
757 | 534 | def test_master_task_entries(self, mock_master_seeds): | ||
760 | 535 | def side_effect(): | 395 | def side_effect(): |
761 | 536 | yield "required" | 396 | yield "required" |
762 | 537 | yield "minimal" | 397 | yield "minimal" |
763 | @@ -540,21 +400,19 @@ class TestGerminateOutput(TestCase): | |||
764 | 540 | output = GerminateOutput(self.config, self.temp_dir) | 400 | output = GerminateOutput(self.config, self.temp_dir) |
765 | 541 | self.config["DIST"] = "bionic" | 401 | self.config["DIST"] = "bionic" |
766 | 542 | self.config["PROJECT"] = "ubuntu" | 402 | self.config["PROJECT"] = "ubuntu" |
768 | 543 | mock_master_seeds.side_effect = side_effect | 403 | mock_pool_seeds.side_effect = side_effect |
769 | 544 | self.assertEqual([ | 404 | self.assertEqual([ |
770 | 545 | "#include <ubuntu/bionic/required>", | 405 | "#include <ubuntu/bionic/required>", |
771 | 546 | "#include <ubuntu/bionic/minimal>", | 406 | "#include <ubuntu/bionic/minimal>", |
772 | 547 | ], list(output.master_task_entries())) | 407 | ], list(output.master_task_entries())) |
773 | 548 | 408 | ||
777 | 549 | @mock.patch( | 409 | def test_pool_seeds_invalid_config(self): |
775 | 550 | "cdimage.germinate.GerminateOutput.master_seeds", return_value=[]) | ||
776 | 551 | def test_master_task_entries_no_seeds(self, mock_master_seeds): | ||
778 | 552 | self.write_ubuntu_structure() | 410 | self.write_ubuntu_structure() |
779 | 553 | output = GerminateOutput(self.config, self.temp_dir) | 411 | output = GerminateOutput(self.config, self.temp_dir) |
780 | 554 | self.config["DIST"] = "bionic" | 412 | self.config["DIST"] = "bionic" |
781 | 555 | self.config["PROJECT"] = "ubuntu" | 413 | self.config["PROJECT"] = "ubuntu" |
782 | 556 | self.assertRaises( | 414 | self.assertRaises( |
784 | 557 | NoMasterSeeds, list, output.master_task_entries()) | 415 | NoMasterSeeds, list, output.pool_seeds()) |
785 | 558 | 416 | ||
786 | 559 | def test_tasks_output_dir(self): | 417 | def test_tasks_output_dir(self): |
787 | 560 | self.write_ubuntu_structure() | 418 | self.write_ubuntu_structure() |
788 | @@ -568,83 +426,6 @@ class TestGerminateOutput(TestCase): | |||
789 | 568 | "tasks"), | 426 | "tasks"), |
790 | 569 | output.tasks_output_dir()) | 427 | output.tasks_output_dir()) |
791 | 570 | 428 | ||
792 | 571 | def test_task_packages_plain(self): | ||
793 | 572 | self.write_structure([["base", []]]) | ||
794 | 573 | self.write_seed_output("i386", "base", ["base-files", "base-passwd"]) | ||
795 | 574 | output = GerminateOutput(self.config, self.temp_dir) | ||
796 | 575 | self.assertEqual( | ||
797 | 576 | ["base-files", "base-passwd"], | ||
798 | 577 | list(output.task_packages("i386", "base", "base"))) | ||
799 | 578 | |||
800 | 579 | def test_task_packages_installer(self): | ||
801 | 580 | # kernel-image-* is excluded from the installer seed. | ||
802 | 581 | self.write_structure([["installer", []]]) | ||
803 | 582 | self.write_seed_output( | ||
804 | 583 | "i386", "installer", [ | ||
805 | 584 | "block-modules-3.8.0-6-generic-di", | ||
806 | 585 | "kernel-image-3.8.0-6-generic-di", | ||
807 | 586 | ]) | ||
808 | 587 | self.config["CDIMAGE_INSTALL_BASE"] = "1" | ||
809 | 588 | output = GerminateOutput(self.config, self.temp_dir) | ||
810 | 589 | self.assertEqual( | ||
811 | 590 | ["block-modules-3.8.0-6-generic-di"], | ||
812 | 591 | list(output.task_packages("i386", "installer", "installer"))) | ||
813 | 592 | |||
814 | 593 | def test_task_packages_squashfs(self): | ||
815 | 594 | self.write_ubuntu_structure() | ||
816 | 595 | self.config["PROJECT"] = "ubuntu-server" | ||
817 | 596 | self.config["DIST"] = "bionic" | ||
818 | 597 | self.write_seed_output( | ||
819 | 598 | "i386", "installer", ["base-installer", "bootstrap-base"]) | ||
820 | 599 | output = GerminateOutput(self.config, self.temp_dir) | ||
821 | 600 | self.assertEqual( | ||
822 | 601 | ["base-installer", "bootstrap-base"], | ||
823 | 602 | list(output.task_packages("i386", "installer", "installer"))) | ||
824 | 603 | self.config["CDIMAGE_SQUASHFS_BASE"] = "1" | ||
825 | 604 | self.assertEqual( | ||
826 | 605 | ["base-installer", "live-installer"], | ||
827 | 606 | list(output.task_packages("i386", "installer", "installer"))) | ||
828 | 607 | |||
829 | 608 | # TODO: task_project untested | ||
830 | 609 | |||
831 | 610 | def test_task_headers(self): | ||
832 | 611 | self.write_ubuntu_structure() | ||
833 | 612 | seedtext_path = os.path.join(self.temp_dir, "i386", "desktop.seedtext") | ||
834 | 613 | with mkfile(seedtext_path) as seedtext: | ||
835 | 614 | print(dedent("""\ | ||
836 | 615 | Task-Per-Derivative: 1 | ||
837 | 616 | Task-Key: ubuntu-desktop | ||
838 | 617 | Task-Seeds: desktop-common | ||
839 | 618 | |||
840 | 619 | = Seed text starts here ="""), file=seedtext) | ||
841 | 620 | output = GerminateOutput(self.config, self.temp_dir) | ||
842 | 621 | expected = { | ||
843 | 622 | "per-derivative": "1", | ||
844 | 623 | "key": "ubuntu-desktop", | ||
845 | 624 | "seeds": "desktop-common", | ||
846 | 625 | } | ||
847 | 626 | self.assertEqual(expected, output.task_headers("i386", "desktop")) | ||
848 | 627 | self.assertEqual({}, output.task_headers("i386", "missing")) | ||
849 | 628 | |||
850 | 629 | def test_seed_task_mapping(self): | ||
851 | 630 | self.write_ubuntu_structure() | ||
852 | 631 | seed_dir = os.path.join(self.temp_dir, "i386") | ||
853 | 632 | with mkfile(os.path.join(seed_dir, "standard.seedtext")) as seedtext: | ||
854 | 633 | print("Task-Key: ubuntu-standard", file=seedtext) | ||
855 | 634 | with mkfile(os.path.join(seed_dir, "desktop.seedtext")) as seedtext: | ||
856 | 635 | print(dedent("""\ | ||
857 | 636 | Task-Per-Derivative: 1 | ||
858 | 637 | Task-Seeds: desktop-common"""), file=seedtext) | ||
859 | 638 | self.config["DIST"] = "bionic" | ||
860 | 639 | self.config["PROJECT"] = "ubuntu" | ||
861 | 640 | output = GerminateOutput(self.config, self.temp_dir) | ||
862 | 641 | expected = [ | ||
863 | 642 | (["standard"], "standard"), | ||
864 | 643 | (["desktop", "desktop-common"], "ubuntu-desktop"), | ||
865 | 644 | ] | ||
866 | 645 | self.assertEqual( | ||
867 | 646 | expected, list(output.seed_task_mapping("i386"))) | ||
868 | 647 | |||
869 | 648 | def test_write_tasks(self): | 429 | def test_write_tasks(self): |
870 | 649 | self.write_ubuntu_structure() | 430 | self.write_ubuntu_structure() |
871 | 650 | for arch in "amd64", "i386": | 431 | for arch in "amd64", "i386": |
872 | @@ -673,8 +454,6 @@ class TestGerminateOutput(TestCase): | |||
873 | 673 | "tasks") | 454 | "tasks") |
874 | 674 | self.assertCountEqual([ | 455 | self.assertCountEqual([ |
875 | 675 | "required", "minimal", "desktop", "live", | 456 | "required", "minimal", "desktop", "live", |
876 | 676 | "override.amd64", "override.i386", | ||
877 | 677 | "important.amd64", "important.i386", | ||
878 | 678 | "MASTER", | 457 | "MASTER", |
879 | 679 | ], os.listdir(output_dir)) | 458 | ], os.listdir(output_dir)) |
880 | 680 | with open(os.path.join(output_dir, "required")) as f: | 459 | with open(os.path.join(output_dir, "required")) as f: |
881 | @@ -723,28 +502,6 @@ class TestGerminateOutput(TestCase): | |||
882 | 723 | #endif /* ARCH_i386 */ | 502 | #endif /* ARCH_i386 */ |
883 | 724 | """), | 503 | """), |
884 | 725 | f.read()) | 504 | f.read()) |
885 | 726 | with open(os.path.join(output_dir, "override.amd64")) as f: | ||
886 | 727 | self.assertEqual( | ||
887 | 728 | dedent("""\ | ||
888 | 729 | adduser-amd64 Task minimal | ||
889 | 730 | base-files-amd64 Task minimal | ||
890 | 731 | firefox Task ubuntu-desktop | ||
891 | 732 | xterm Task ubuntu-desktop, ubuntu-live | ||
892 | 733 | """), | ||
893 | 734 | f.read()) | ||
894 | 735 | with open(os.path.join(output_dir, "override.i386")) as f: | ||
895 | 736 | self.assertEqual( | ||
896 | 737 | dedent("""\ | ||
897 | 738 | adduser-i386 Task minimal | ||
898 | 739 | base-files-i386 Task minimal | ||
899 | 740 | firefox Task ubuntu-desktop | ||
900 | 741 | xterm Task ubuntu-desktop, ubuntu-live | ||
901 | 742 | """), | ||
902 | 743 | f.read()) | ||
903 | 744 | with open(os.path.join(output_dir, "important.amd64")) as f: | ||
904 | 745 | self.assertEqual("adduser-amd64\nbase-files-amd64\n", f.read()) | ||
905 | 746 | with open(os.path.join(output_dir, "important.i386")) as f: | ||
906 | 747 | self.assertEqual("adduser-i386\nbase-files-i386\n", f.read()) | ||
907 | 748 | with open(os.path.join(output_dir, "MASTER")) as f: | 505 | with open(os.path.join(output_dir, "MASTER")) as f: |
908 | 749 | self.assertEqual("#include <ubuntu/bionic/ship-live>\n", f.read()) | 506 | self.assertEqual("#include <ubuntu/bionic/ship-live>\n", f.read()) |
909 | 750 | 507 | ||
910 | diff --git a/lib/cdimage/tests/test_tree.py b/lib/cdimage/tests/test_tree.py | |||
911 | index c5e3665..63a5840 100644 | |||
912 | --- a/lib/cdimage/tests/test_tree.py | |||
913 | +++ b/lib/cdimage/tests/test_tree.py | |||
914 | @@ -1866,7 +1866,6 @@ class TestDailyTreePublisher(TestCase): | |||
915 | 1866 | @mock.patch("cdimage.tree.DailyTreePublisher.post_qa") | 1866 | @mock.patch("cdimage.tree.DailyTreePublisher.post_qa") |
916 | 1867 | def test_publish(self, mock_post_qa, *args): | 1867 | def test_publish(self, mock_post_qa, *args): |
917 | 1868 | self.config["ARCHES"] = "i386" | 1868 | self.config["ARCHES"] = "i386" |
918 | 1869 | self.config["CDIMAGE_INSTALL_BASE"] = "1" | ||
919 | 1870 | publisher = self.make_publisher("ubuntu", "daily-live") | 1869 | publisher = self.make_publisher("ubuntu", "daily-live") |
920 | 1871 | source_dir = publisher.image_output("i386") | 1870 | source_dir = publisher.image_output("i386") |
921 | 1872 | touch(os.path.join( | 1871 | touch(os.path.join( |
922 | @@ -1900,7 +1899,6 @@ class TestDailyTreePublisher(TestCase): | |||
923 | 1900 | "%s-desktop-i386.iso" % self.config.series, | 1899 | "%s-desktop-i386.iso" % self.config.series, |
924 | 1901 | "%s-desktop-i386.list" % self.config.series, | 1900 | "%s-desktop-i386.list" % self.config.series, |
925 | 1902 | "%s-desktop-i386.manifest" % self.config.series, | 1901 | "%s-desktop-i386.manifest" % self.config.series, |
926 | 1903 | "report.html", | ||
927 | 1904 | ], os.listdir(target_dir)) | 1902 | ], os.listdir(target_dir)) |
928 | 1905 | self.assertCountEqual( | 1903 | self.assertCountEqual( |
929 | 1906 | [".htaccess", "20120807", "current", "pending"], | 1904 | [".htaccess", "20120807", "current", "pending"], |
930 | @@ -1921,7 +1919,6 @@ class TestDailyTreePublisher(TestCase): | |||
931 | 1921 | def test_publish_subtree(self, mock_post_qa, *args): | 1919 | def test_publish_subtree(self, mock_post_qa, *args): |
932 | 1922 | self.config.subtree = "subtree/test" | 1920 | self.config.subtree = "subtree/test" |
933 | 1923 | self.config["ARCHES"] = "i386" | 1921 | self.config["ARCHES"] = "i386" |
934 | 1924 | self.config["CDIMAGE_INSTALL_BASE"] = "1" | ||
935 | 1925 | publisher = self.make_publisher("ubuntu", "daily-live") | 1922 | publisher = self.make_publisher("ubuntu", "daily-live") |
936 | 1926 | source_dir = publisher.image_output("i386") | 1923 | source_dir = publisher.image_output("i386") |
937 | 1927 | touch(os.path.join( | 1924 | touch(os.path.join( |
938 | @@ -1960,7 +1957,6 @@ class TestDailyTreePublisher(TestCase): | |||
939 | 1960 | "%s-desktop-i386.iso" % self.config.series, | 1957 | "%s-desktop-i386.iso" % self.config.series, |
940 | 1961 | "%s-desktop-i386.list" % self.config.series, | 1958 | "%s-desktop-i386.list" % self.config.series, |
941 | 1962 | "%s-desktop-i386.manifest" % self.config.series, | 1959 | "%s-desktop-i386.manifest" % self.config.series, |
942 | 1963 | "report.html", | ||
943 | 1964 | ], os.listdir(target_dir)) | 1960 | ], os.listdir(target_dir)) |
944 | 1965 | self.assertCountEqual( | 1961 | self.assertCountEqual( |
945 | 1966 | [".htaccess", "20120807", "current", "pending"], | 1962 | [".htaccess", "20120807", "current", "pending"], |
946 | diff --git a/lib/cdimage/tree.py b/lib/cdimage/tree.py | |||
947 | index 0e9ad71..6fac63b 100644 | |||
948 | --- a/lib/cdimage/tree.py | |||
949 | +++ b/lib/cdimage/tree.py | |||
950 | @@ -2084,13 +2084,7 @@ class DailyTreePublisher(Publisher): | |||
951 | 2084 | else: | 2084 | else: |
952 | 2085 | osextras.unlink_force("%s.manifest" % target_prefix) | 2085 | osextras.unlink_force("%s.manifest" % target_prefix) |
953 | 2086 | 2086 | ||
961 | 2087 | if (self.config["CDIMAGE_SQUASHFS_BASE"] and | 2087 | osextras.unlink_force("%s.squashfs" % target_prefix) |
955 | 2088 | os.path.exists("%s.squashfs" % source_prefix)): | ||
956 | 2089 | logger.info("Publishing %s squashfs ..." % arch) | ||
957 | 2090 | shutil.move( | ||
958 | 2091 | "%s.squashfs" % source_prefix, "%s.squashfs" % target_prefix) | ||
959 | 2092 | else: | ||
960 | 2093 | osextras.unlink_force("%s.squashfs" % target_prefix) | ||
962 | 2094 | 2088 | ||
963 | 2095 | if os.path.exists("%s.custom.tar.gz" % source_prefix): | 2089 | if os.path.exists("%s.custom.tar.gz" % source_prefix): |
964 | 2096 | logger.info("Publishing %s custom tarball ..." % arch) | 2090 | logger.info("Publishing %s custom tarball ..." % arch) |
965 | @@ -2616,14 +2610,8 @@ class DailyTreePublisher(Publisher): | |||
966 | 2616 | logger.warning("No images produced!") | 2610 | logger.warning("No images produced!") |
967 | 2617 | return | 2611 | return |
968 | 2618 | 2612 | ||
969 | 2619 | source_report = os.path.join( | ||
970 | 2620 | self.britney_report, "%s_probs.html" % self.config.series) | ||
971 | 2621 | target_report = os.path.join(self.publish_base, date, "report.html") | 2613 | target_report = os.path.join(self.publish_base, date, "report.html") |
977 | 2622 | if (self.config["CDIMAGE_INSTALL_BASE"] and | 2614 | osextras.unlink_force(target_report) |
973 | 2623 | os.path.exists(source_report)): | ||
974 | 2624 | shutil.copy2(source_report, target_report) | ||
975 | 2625 | else: | ||
976 | 2626 | osextras.unlink_force(target_report) | ||
978 | 2627 | 2615 | ||
979 | 2628 | self.polish_directory(date) | 2616 | self.polish_directory(date) |
980 | 2629 | self.link(date, "pending") | 2617 | self.link(date, "pending") |