Merge ~chad.smith/curtin:ubuntu/xenial into curtin:ubuntu/xenial
- Git
- lp:~chad.smith/curtin
- ubuntu/xenial
- Merge into ubuntu/xenial
Proposed by
Chad Smith
Status: | Merged | ||||
---|---|---|---|---|---|
Merged at revision: | 013f9136a90b27ed4e55c9a7ffd0209d340108a0 | ||||
Proposed branch: | ~chad.smith/curtin:ubuntu/xenial | ||||
Merge into: | curtin:ubuntu/xenial | ||||
Diff against target: |
10291 lines (+3953/-1943) 112 files modified
bin/curtin (+1/-1) curtin/__init__.py (+2/-0) curtin/__main__.py (+4/-0) curtin/block/__init__.py (+26/-80) curtin/block/clear_holders.py (+35/-11) curtin/block/deps.py (+103/-0) curtin/block/iscsi.py (+25/-9) curtin/block/lvm.py (+25/-6) curtin/block/mdadm.py (+4/-4) curtin/block/mkfs.py (+5/-4) curtin/block/zfs.py (+20/-8) curtin/commands/__main__.py (+4/-0) curtin/commands/apply_net.py (+4/-3) curtin/commands/apt_config.py (+13/-13) curtin/commands/block_meta.py (+10/-7) curtin/commands/curthooks.py (+396/-210) curtin/commands/extract.py (+1/-1) curtin/commands/features.py (+20/-0) curtin/commands/in_target.py (+2/-2) curtin/commands/install.py (+22/-8) curtin/commands/main.py (+3/-3) curtin/commands/system_install.py (+2/-1) curtin/commands/system_upgrade.py (+3/-2) curtin/deps/__init__.py (+3/-3) curtin/distro.py (+512/-0) curtin/futil.py (+2/-1) curtin/log.py (+43/-0) curtin/net/__init__.py (+0/-59) curtin/net/deps.py (+72/-0) curtin/paths.py (+34/-0) curtin/udev.py (+2/-0) curtin/url_helper.py (+1/-1) curtin/util.py (+31/-299) debian/changelog (+45/-0) dev/null (+0/-96) doc/topics/config.rst (+40/-0) doc/topics/curthooks.rst (+18/-2) doc/topics/integration-testing.rst (+4/-0) doc/topics/storage.rst (+79/-3) examples/tests/dirty_disks_config.yaml (+30/-3) examples/tests/filesystem_battery.yaml (+2/-2) examples/tests/install_disable_unmount.yaml (+2/-2) examples/tests/lvmoverraid.yaml (+98/-0) examples/tests/mirrorboot-msdos-partition.yaml (+2/-2) examples/tests/mirrorboot-uefi.yaml (+4/-4) examples/tests/vmtest_defaults.yaml (+24/-0) helpers/common (+156/-35) tests/unittests/test_apt_custom_sources_list.py (+10/-8) tests/unittests/test_apt_source.py (+8/-7) tests/unittests/test_block.py (+35/-0) tests/unittests/test_block_iscsi.py (+7/-0) tests/unittests/test_block_lvm.py (+16/-15) tests/unittests/test_block_mdadm.py (+22/-16) tests/unittests/test_block_mkfs.py (+3/-2) tests/unittests/test_block_zfs.py (+98/-31) tests/unittests/test_clear_holders.py (+154/-41) tests/unittests/test_commands_apply_net.py (+7/-7) tests/unittests/test_commands_block_meta.py (+4/-3) tests/unittests/test_commands_collect_logs.py (+26/-14) tests/unittests/test_commands_extract.py (+72/-0) tests/unittests/test_commands_install.py (+40/-0) tests/unittests/test_curthooks.py (+103/-78) tests/unittests/test_distro.py (+302/-0) tests/unittests/test_feature.py (+3/-0) tests/unittests/test_pack.py (+2/-0) tests/unittests/test_util.py (+20/-61) tests/vmtests/__init__.py (+304/-88) tests/vmtests/helpers.py (+28/-1) tests/vmtests/image_sync.py (+4/-2) tests/vmtests/releases.py (+21/-22) tests/vmtests/report_webhook_logger.py (+11/-6) tests/vmtests/test_apt_config_cmd.py (+4/-6) tests/vmtests/test_apt_source.py (+2/-4) tests/vmtests/test_basic.py (+143/-159) tests/vmtests/test_bcache_basic.py (+5/-8) tests/vmtests/test_bcache_bug1718699.py (+2/-2) tests/vmtests/test_fs_battery.py (+29/-11) tests/vmtests/test_install_umount.py (+1/-18) tests/vmtests/test_iscsi.py (+12/-8) tests/vmtests/test_journald_reporter.py (+4/-7) tests/vmtests/test_lvm.py (+10/-10) tests/vmtests/test_lvm_iscsi.py (+11/-6) tests/vmtests/test_lvm_raid.py (+51/-0) tests/vmtests/test_lvm_root.py (+33/-32) tests/vmtests/test_mdadm_bcache.py (+58/-39) tests/vmtests/test_mdadm_iscsi.py (+11/-5) tests/vmtests/test_multipath.py (+10/-18) tests/vmtests/test_network.py (+6/-21) tests/vmtests/test_network_alias.py (+5/-5) tests/vmtests/test_network_bonding.py (+18/-29) tests/vmtests/test_network_bridging.py (+22/-30) tests/vmtests/test_network_ipv6.py (+6/-6) tests/vmtests/test_network_ipv6_static.py (+4/-4) tests/vmtests/test_network_ipv6_vlan.py (+4/-4) tests/vmtests/test_network_mtu.py (+9/-16) tests/vmtests/test_network_static.py (+4/-13) tests/vmtests/test_network_static_routes.py (+4/-4) tests/vmtests/test_network_vlan.py (+6/-14) tests/vmtests/test_nvme.py (+34/-60) tests/vmtests/test_old_apt_features.py (+2/-4) tests/vmtests/test_pollinate_useragent.py (+5/-2) tests/vmtests/test_raid5_bcache.py (+8/-13) tests/vmtests/test_simple.py (+7/-20) tests/vmtests/test_ubuntu_core.py (+3/-8) tests/vmtests/test_uefi_basic.py (+31/-32) tests/vmtests/test_zfsroot.py (+11/-23) tools/curtainer (+21/-6) tools/jenkins-runner (+33/-5) tools/vmtest-filter (+57/-0) tools/vmtest-sync-images (+0/-1) tools/xkvm (+5/-1) tox.ini (+28/-2) |
||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Server Team CI bot | continuous-integration | Approve | |
curtin developers | Pending | ||
Review via email: mp+356003@code.launchpad.net |
Commit message
new upstream snapshot for release into xenial
LP: #1795712
Description of the change
To post a comment you must log in.
Revision history for this message
Server Team CI bot (server-team-bot) wrote : | # |
review:
Approve
(continuous-integration)
Revision history for this message
Server Team CI bot (server-team-bot) wrote : | # |
PASSED: Continuous integration, rev:013f9136a90
https:/
Executed test runs:
SUCCESS: https:/
SUCCESS: https:/
SUCCESS: https:/
SUCCESS: https:/
Click here to trigger a rebuild:
https:/
review:
Approve
(continuous-integration)
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/bin/curtin b/bin/curtin |
2 | index 6c4e457..793fbcb 100755 |
3 | --- a/bin/curtin |
4 | +++ b/bin/curtin |
5 | @@ -1,7 +1,7 @@ |
6 | #!/bin/sh |
7 | # This file is part of curtin. See LICENSE file for copyright and license info. |
8 | |
9 | -PY3OR2_MAIN="curtin.commands.main" |
10 | +PY3OR2_MAIN="curtin" |
11 | PY3OR2_MCHECK="curtin.deps.check" |
12 | PY3OR2_PYTHONS=${PY3OR2_PYTHONS:-"python3:python"} |
13 | PYTHON=${PY3OR2_PYTHON} |
14 | diff --git a/curtin/__init__.py b/curtin/__init__.py |
15 | index 002454b..ee35ca3 100644 |
16 | --- a/curtin/__init__.py |
17 | +++ b/curtin/__init__.py |
18 | @@ -10,6 +10,8 @@ KERNEL_CMDLINE_COPY_TO_INSTALL_SEP = "---" |
19 | FEATURES = [ |
20 | # curtin can apply centos networking via centos_apply_network_config |
21 | 'CENTOS_APPLY_NETWORK_CONFIG', |
22 | + # curtin can configure centos storage devices and boot devices |
23 | + 'CENTOS_CURTHOOK_SUPPORT', |
24 | # install supports the 'network' config version 1 |
25 | 'NETWORK_CONFIG_V1', |
26 | # reporter supports 'webhook' type |
27 | diff --git a/curtin/__main__.py b/curtin/__main__.py |
28 | new file mode 100644 |
29 | index 0000000..5b6aeca |
30 | --- /dev/null |
31 | +++ b/curtin/__main__.py |
32 | @@ -0,0 +1,4 @@ |
33 | +if __name__ == '__main__': |
34 | + from .commands.main import main |
35 | + import sys |
36 | + sys.exit(main()) |
37 | diff --git a/curtin/block/__init__.py b/curtin/block/__init__.py |
38 | index a8ee8a6..490c268 100644 |
39 | --- a/curtin/block/__init__.py |
40 | +++ b/curtin/block/__init__.py |
41 | @@ -378,24 +378,28 @@ def stop_all_unused_multipath_devices(): |
42 | LOG.warn("Failed to stop multipath devices: %s", e) |
43 | |
44 | |
45 | -def rescan_block_devices(warn_on_fail=True): |
46 | +def rescan_block_devices(devices=None, warn_on_fail=True): |
47 | """ |
48 | run 'blockdev --rereadpt' for all block devices not currently mounted |
49 | """ |
50 | - unused = get_unused_blockdev_info() |
51 | - devices = [] |
52 | - for devname, data in unused.items(): |
53 | - if data.get('RM') == "1": |
54 | - continue |
55 | - if data.get('RO') != "0" or data.get('TYPE') != "disk": |
56 | - continue |
57 | - devices.append(data['device_path']) |
58 | + if not devices: |
59 | + unused = get_unused_blockdev_info() |
60 | + devices = [] |
61 | + for devname, data in unused.items(): |
62 | + if data.get('RM') == "1": |
63 | + continue |
64 | + if data.get('RO') != "0" or data.get('TYPE') != "disk": |
65 | + continue |
66 | + devices.append(data['device_path']) |
67 | |
68 | if not devices: |
69 | LOG.debug("no devices found to rescan") |
70 | return |
71 | |
72 | - cmd = ['blockdev', '--rereadpt'] + devices |
73 | + # blockdev needs /dev/ parameters, convert if needed |
74 | + cmd = ['blockdev', '--rereadpt'] + [dev if dev.startswith('/dev/') |
75 | + else sysfs_to_devpath(dev) |
76 | + for dev in devices] |
77 | try: |
78 | util.subp(cmd, capture=True) |
79 | except util.ProcessExecutionError as e: |
80 | @@ -999,75 +1003,17 @@ def wipe_volume(path, mode="superblock", exclusive=True): |
81 | raise ValueError("wipe mode %s not supported" % mode) |
82 | |
83 | |
84 | -def storage_config_required_packages(storage_config, mapping): |
85 | - """Read storage configuration dictionary and determine |
86 | - which packages are required for the supplied configuration |
87 | - to function. Return a list of packaged to install. |
88 | - """ |
89 | - |
90 | - if not storage_config or not isinstance(storage_config, dict): |
91 | - raise ValueError('Invalid storage configuration. ' |
92 | - 'Must be a dict:\n %s' % storage_config) |
93 | - |
94 | - if not mapping or not isinstance(mapping, dict): |
95 | - raise ValueError('Invalid storage mapping. Must be a dict') |
96 | - |
97 | - if 'storage' in storage_config: |
98 | - storage_config = storage_config.get('storage') |
99 | - |
100 | - needed_packages = [] |
101 | - |
102 | - # get reqs by device operation type |
103 | - dev_configs = set(operation['type'] |
104 | - for operation in storage_config['config']) |
105 | - |
106 | - for dev_type in dev_configs: |
107 | - if dev_type in mapping: |
108 | - needed_packages.extend(mapping[dev_type]) |
109 | - |
110 | - # for any format operations, check the fstype and |
111 | - # determine if we need any mkfs tools as well. |
112 | - format_configs = set([operation['fstype'] |
113 | - for operation in storage_config['config'] |
114 | - if operation['type'] == 'format']) |
115 | - for format_type in format_configs: |
116 | - if format_type in mapping: |
117 | - needed_packages.extend(mapping[format_type]) |
118 | - |
119 | - return needed_packages |
120 | - |
121 | - |
122 | -def detect_required_packages_mapping(): |
123 | - """Return a dictionary providing a versioned configuration which maps |
124 | - storage configuration elements to the packages which are required |
125 | - for functionality. |
126 | - |
127 | - The mapping key is either a config type value, or an fstype value. |
128 | - |
129 | - """ |
130 | - version = 1 |
131 | - mapping = { |
132 | - version: { |
133 | - 'handler': storage_config_required_packages, |
134 | - 'mapping': { |
135 | - 'bcache': ['bcache-tools'], |
136 | - 'btrfs': ['btrfs-tools'], |
137 | - 'ext2': ['e2fsprogs'], |
138 | - 'ext3': ['e2fsprogs'], |
139 | - 'ext4': ['e2fsprogs'], |
140 | - 'jfs': ['jfsutils'], |
141 | - 'lvm_partition': ['lvm2'], |
142 | - 'lvm_volgroup': ['lvm2'], |
143 | - 'ntfs': ['ntfs-3g'], |
144 | - 'raid': ['mdadm'], |
145 | - 'reiserfs': ['reiserfsprogs'], |
146 | - 'xfs': ['xfsprogs'], |
147 | - 'zfsroot': ['zfsutils-linux', 'zfs-initramfs'], |
148 | - 'zfs': ['zfsutils-linux', 'zfs-initramfs'], |
149 | - 'zpool': ['zfsutils-linux', 'zfs-initramfs'], |
150 | - }, |
151 | - }, |
152 | - } |
153 | - return mapping |
154 | +def get_supported_filesystems(): |
155 | + """ Return a list of filesystems that the kernel currently supports |
156 | + as read from /proc/filesystems. |
157 | + |
158 | + Raises RuntimeError if /proc/filesystems does not exist. |
159 | + """ |
160 | + proc_fs = "/proc/filesystems" |
161 | + if not os.path.exists(proc_fs): |
162 | + raise RuntimeError("Unable to read 'filesystems' from %s" % proc_fs) |
163 | + |
164 | + return [l.split('\t')[1].strip() |
165 | + for l in util.load_file(proc_fs).splitlines()] |
166 | |
167 | # vi: ts=4 expandtab syntax=python |
168 | diff --git a/curtin/block/clear_holders.py b/curtin/block/clear_holders.py |
169 | index 20c572b..a05c9ca 100644 |
170 | --- a/curtin/block/clear_holders.py |
171 | +++ b/curtin/block/clear_holders.py |
172 | @@ -300,12 +300,18 @@ def wipe_superblock(device): |
173 | else: |
174 | raise e |
175 | |
176 | + # gather any partitions |
177 | + partitions = block.get_sysfs_partitions(device) |
178 | + |
179 | # release zfs member by exporting the pool |
180 | - if block.is_zfs_member(blockdev): |
181 | + if zfs.zfs_supported() and block.is_zfs_member(blockdev): |
182 | poolname = zfs.device_to_poolname(blockdev) |
183 | # only export pools that have been imported |
184 | if poolname in zfs.zpool_list(): |
185 | - zfs.zpool_export(poolname) |
186 | + try: |
187 | + zfs.zpool_export(poolname) |
188 | + except util.ProcessExecutionError as e: |
189 | + LOG.warning('Failed to export zpool "%s": %s', poolname, e) |
190 | |
191 | if is_swap_device(blockdev): |
192 | shutdown_swap(blockdev) |
193 | @@ -325,6 +331,27 @@ def wipe_superblock(device): |
194 | |
195 | _wipe_superblock(blockdev) |
196 | |
197 | + # if we had partitions, make sure they've been removed |
198 | + if partitions: |
199 | + LOG.debug('%s had partitions, issuing partition reread', device) |
200 | + retries = [.5, .5, 1, 2, 5, 7] |
201 | + for attempt, wait in enumerate(retries): |
202 | + try: |
203 | + # only rereadpt on wiped device |
204 | + block.rescan_block_devices(devices=[blockdev]) |
205 | + # may raise IOError, OSError due to wiped partition table |
206 | + curparts = block.get_sysfs_partitions(device) |
207 | + if len(curparts) == 0: |
208 | + return |
209 | + except (IOError, OSError): |
210 | + if attempt + 1 >= len(retries): |
211 | + raise |
212 | + |
213 | + LOG.debug("%s partitions still present, rereading pt" |
214 | + " (%s/%s). sleeping %ss before retry", |
215 | + device, attempt + 1, len(retries), wait) |
216 | + time.sleep(wait) |
217 | + |
218 | |
219 | def _wipe_superblock(blockdev, exclusive=True): |
220 | """ No checks, just call wipe_volume """ |
221 | @@ -579,8 +606,6 @@ def clear_holders(base_paths, try_preserve=False): |
222 | dev_info['dev_type']) |
223 | continue |
224 | |
225 | - # scan before we check |
226 | - block.rescan_block_devices(warn_on_fail=False) |
227 | if os.path.exists(dev_info['device']): |
228 | LOG.info("shutdown running on holder type: '%s' syspath: '%s'", |
229 | dev_info['dev_type'], dev_info['device']) |
230 | @@ -602,19 +627,18 @@ def start_clear_holders_deps(): |
231 | # all disks and partitions should be sufficient to remove the mdadm |
232 | # metadata |
233 | mdadm.mdadm_assemble(scan=True, ignore_errors=True) |
234 | + # scan and activate for logical volumes |
235 | + lvm.lvm_scan() |
236 | + lvm.activate_volgroups() |
237 | # the bcache module needs to be present to properly detect bcache devs |
238 | # on some systems (precise without hwe kernel) it may not be possible to |
239 | # lad the bcache module bcause it is not present in the kernel. if this |
240 | # happens then there is no need to halt installation, as the bcache devices |
241 | # will never appear and will never prevent the disk from being reformatted |
242 | util.load_kernel_module('bcache') |
243 | - # the zfs module is needed to find and export devices which may be in-use |
244 | - # and need to be cleared, only on xenial+. |
245 | - try: |
246 | - if zfs.zfs_supported(): |
247 | - util.load_kernel_module('zfs') |
248 | - except RuntimeError as e: |
249 | - LOG.warning('Failed to load zfs kernel module: %s', e) |
250 | + |
251 | + if not zfs.zfs_supported(): |
252 | + LOG.warning('zfs filesystem is not supported in this environment') |
253 | |
254 | |
255 | # anything that is not identified can assumed to be a 'disk' or similar |
256 | diff --git a/curtin/block/deps.py b/curtin/block/deps.py |
257 | new file mode 100644 |
258 | index 0000000..930f764 |
259 | --- /dev/null |
260 | +++ b/curtin/block/deps.py |
261 | @@ -0,0 +1,103 @@ |
262 | +# This file is part of curtin. See LICENSE file for copyright and license info. |
263 | + |
264 | +from curtin.distro import DISTROS |
265 | +from curtin.block import iscsi |
266 | + |
267 | + |
268 | +def storage_config_required_packages(storage_config, mapping): |
269 | + """Read storage configuration dictionary and determine |
270 | + which packages are required for the supplied configuration |
271 | + to function. Return a list of packaged to install. |
272 | + """ |
273 | + |
274 | + if not storage_config or not isinstance(storage_config, dict): |
275 | + raise ValueError('Invalid storage configuration. ' |
276 | + 'Must be a dict:\n %s' % storage_config) |
277 | + |
278 | + if not mapping or not isinstance(mapping, dict): |
279 | + raise ValueError('Invalid storage mapping. Must be a dict') |
280 | + |
281 | + if 'storage' in storage_config: |
282 | + storage_config = storage_config.get('storage') |
283 | + |
284 | + needed_packages = [] |
285 | + |
286 | + # get reqs by device operation type |
287 | + dev_configs = set(operation['type'] |
288 | + for operation in storage_config['config']) |
289 | + |
290 | + for dev_type in dev_configs: |
291 | + if dev_type in mapping: |
292 | + needed_packages.extend(mapping[dev_type]) |
293 | + |
294 | + # for disks with path: iscsi: we need iscsi tools |
295 | + iscsi_vols = iscsi.get_iscsi_volumes_from_config(storage_config) |
296 | + if len(iscsi_vols) > 0: |
297 | + needed_packages.extend(mapping['iscsi']) |
298 | + |
299 | + # for any format operations, check the fstype and |
300 | + # determine if we need any mkfs tools as well. |
301 | + format_configs = set([operation['fstype'] |
302 | + for operation in storage_config['config'] |
303 | + if operation['type'] == 'format']) |
304 | + for format_type in format_configs: |
305 | + if format_type in mapping: |
306 | + needed_packages.extend(mapping[format_type]) |
307 | + |
308 | + return needed_packages |
309 | + |
310 | + |
311 | +def detect_required_packages_mapping(osfamily=DISTROS.debian): |
312 | + """Return a dictionary providing a versioned configuration which maps |
313 | + storage configuration elements to the packages which are required |
314 | + for functionality. |
315 | + |
316 | + The mapping key is either a config type value, or an fstype value. |
317 | + |
318 | + """ |
319 | + distro_mapping = { |
320 | + DISTROS.debian: { |
321 | + 'bcache': ['bcache-tools'], |
322 | + 'btrfs': ['btrfs-tools'], |
323 | + 'ext2': ['e2fsprogs'], |
324 | + 'ext3': ['e2fsprogs'], |
325 | + 'ext4': ['e2fsprogs'], |
326 | + 'jfs': ['jfsutils'], |
327 | + 'iscsi': ['open-iscsi'], |
328 | + 'lvm_partition': ['lvm2'], |
329 | + 'lvm_volgroup': ['lvm2'], |
330 | + 'ntfs': ['ntfs-3g'], |
331 | + 'raid': ['mdadm'], |
332 | + 'reiserfs': ['reiserfsprogs'], |
333 | + 'xfs': ['xfsprogs'], |
334 | + 'zfsroot': ['zfsutils-linux', 'zfs-initramfs'], |
335 | + 'zfs': ['zfsutils-linux', 'zfs-initramfs'], |
336 | + 'zpool': ['zfsutils-linux', 'zfs-initramfs'], |
337 | + }, |
338 | + DISTROS.redhat: { |
339 | + 'bcache': [], |
340 | + 'btrfs': ['btrfs-progs'], |
341 | + 'ext2': ['e2fsprogs'], |
342 | + 'ext3': ['e2fsprogs'], |
343 | + 'ext4': ['e2fsprogs'], |
344 | + 'jfs': [], |
345 | + 'iscsi': ['iscsi-initiator-utils'], |
346 | + 'lvm_partition': ['lvm2'], |
347 | + 'lvm_volgroup': ['lvm2'], |
348 | + 'ntfs': [], |
349 | + 'raid': ['mdadm'], |
350 | + 'reiserfs': [], |
351 | + 'xfs': ['xfsprogs'], |
352 | + 'zfsroot': [], |
353 | + 'zfs': [], |
354 | + 'zpool': [], |
355 | + }, |
356 | + } |
357 | + if osfamily not in distro_mapping: |
358 | + raise ValueError('No block package mapping for distro: %s' % osfamily) |
359 | + |
360 | + return {1: {'handler': storage_config_required_packages, |
361 | + 'mapping': distro_mapping.get(osfamily)}} |
362 | + |
363 | + |
364 | +# vi: ts=4 expandtab syntax=python |
365 | diff --git a/curtin/block/iscsi.py b/curtin/block/iscsi.py |
366 | index 0c666b6..3c46500 100644 |
367 | --- a/curtin/block/iscsi.py |
368 | +++ b/curtin/block/iscsi.py |
369 | @@ -9,7 +9,7 @@ import os |
370 | import re |
371 | import shutil |
372 | |
373 | -from curtin import (util, udev) |
374 | +from curtin import (paths, util, udev) |
375 | from curtin.block import (get_device_slave_knames, |
376 | path_to_kname) |
377 | |
378 | @@ -230,29 +230,45 @@ def connected_disks(): |
379 | return _ISCSI_DISKS |
380 | |
381 | |
382 | -def get_iscsi_disks_from_config(cfg): |
383 | +def get_iscsi_volumes_from_config(cfg): |
384 | """Parse a curtin storage config and return a list |
385 | - of iscsi disk objects for each configuration present |
386 | + of iscsi disk rfc4173 uris for each configuration present. |
387 | """ |
388 | if not cfg: |
389 | cfg = {} |
390 | |
391 | - sconfig = cfg.get('storage', {}).get('config', {}) |
392 | - if not sconfig: |
393 | + if 'storage' in cfg: |
394 | + sconfig = cfg.get('storage', {}).get('config', []) |
395 | + else: |
396 | + sconfig = cfg.get('config', []) |
397 | + if not sconfig or not isinstance(sconfig, list): |
398 | LOG.warning('Configuration dictionary did not contain' |
399 | ' a storage configuration') |
400 | return [] |
401 | |
402 | + return [disk['path'] for disk in sconfig |
403 | + if disk['type'] == 'disk' and |
404 | + disk.get('path', "").startswith('iscsi:')] |
405 | + |
406 | + |
407 | +def get_iscsi_disks_from_config(cfg): |
408 | + """Return a list of IscsiDisk objects for each iscsi volume present.""" |
409 | # Construct IscsiDisk objects for each iscsi volume present |
410 | - iscsi_disks = [IscsiDisk(disk['path']) for disk in sconfig |
411 | - if disk['type'] == 'disk' and |
412 | - disk.get('path', "").startswith('iscsi:')] |
413 | + iscsi_disks = [IscsiDisk(volume) for volume in |
414 | + get_iscsi_volumes_from_config(cfg)] |
415 | LOG.debug('Found %s iscsi disks in storage config', len(iscsi_disks)) |
416 | return iscsi_disks |
417 | |
418 | |
419 | +def get_iscsi_ports_from_config(cfg): |
420 | + """Return a set of ports that may be used when connecting to volumes.""" |
421 | + ports = set([d.port for d in get_iscsi_disks_from_config(cfg)]) |
422 | + LOG.debug('Found iscsi ports in use: %s', ports) |
423 | + return ports |
424 | + |
425 | + |
426 | def disconnect_target_disks(target_root_path=None): |
427 | - target_nodes_path = util.target_path(target_root_path, '/etc/iscsi/nodes') |
428 | + target_nodes_path = paths.target_path(target_root_path, '/etc/iscsi/nodes') |
429 | fails = [] |
430 | if os.path.isdir(target_nodes_path): |
431 | for target in os.listdir(target_nodes_path): |
432 | diff --git a/curtin/block/lvm.py b/curtin/block/lvm.py |
433 | index 8643245..b3f8bcb 100644 |
434 | --- a/curtin/block/lvm.py |
435 | +++ b/curtin/block/lvm.py |
436 | @@ -4,6 +4,7 @@ |
437 | This module provides some helper functions for manipulating lvm devices |
438 | """ |
439 | |
440 | +from curtin import distro |
441 | from curtin import util |
442 | from curtin.log import LOG |
443 | import os |
444 | @@ -57,20 +58,38 @@ def lvmetad_running(): |
445 | '/run/lvmetad.pid')) |
446 | |
447 | |
448 | -def lvm_scan(): |
449 | +def activate_volgroups(): |
450 | + """ |
451 | + Activate available volgroups and logical volumes within. |
452 | + |
453 | + # found |
454 | + % vgchange -ay |
455 | + 1 logical volume(s) in volume group "vg1sdd" now active |
456 | + |
457 | + # none found (no output) |
458 | + % vgchange -ay |
459 | + """ |
460 | + |
461 | + # vgchange handles syncing with udev by default |
462 | + # see man 8 vgchange and flag --noudevsync |
463 | + out, _ = util.subp(['vgchange', '--activate=y'], capture=True) |
464 | + if out: |
465 | + LOG.info(out) |
466 | + |
467 | + |
468 | +def lvm_scan(activate=True): |
469 | """ |
470 | run full scan for volgroups, logical volumes and physical volumes |
471 | """ |
472 | - # the lvm tools lvscan, vgscan and pvscan on ubuntu precise do not |
473 | - # support the flag --cache. the flag is present for the tools in ubuntu |
474 | - # trusty and later. since lvmetad is used in current releases of |
475 | - # ubuntu, the --cache flag is needed to ensure that the data cached by |
476 | + # prior to xenial, lvmetad is not packaged, so even if a tool supports |
477 | + # flag --cache it has no effect. In Xenial and newer the --cache flag is |
478 | + # used (if lvmetad is running) to ensure that the data cached by |
479 | # lvmetad is updated. |
480 | |
481 | # before appending the cache flag though, check if lvmetad is running. this |
482 | # ensures that we do the right thing even if lvmetad is supported but is |
483 | # not running |
484 | - release = util.lsb_release().get('codename') |
485 | + release = distro.lsb_release().get('codename') |
486 | if release in [None, 'UNAVAILABLE']: |
487 | LOG.warning('unable to find release number, assuming xenial or later') |
488 | release = 'xenial' |
489 | diff --git a/curtin/block/mdadm.py b/curtin/block/mdadm.py |
490 | index e0fe0d3..4ad6aa7 100644 |
491 | --- a/curtin/block/mdadm.py |
492 | +++ b/curtin/block/mdadm.py |
493 | @@ -13,6 +13,7 @@ import time |
494 | |
495 | from curtin.block import (dev_short, dev_path, is_valid_device, sys_block_path) |
496 | from curtin.block import get_holders |
497 | +from curtin.distro import lsb_release |
498 | from curtin import (util, udev) |
499 | from curtin.log import LOG |
500 | |
501 | @@ -95,7 +96,7 @@ VALID_RAID_ARRAY_STATES = ( |
502 | checks the mdadm version and will return True if we can use --export |
503 | for key=value list with enough info, false if version is less than |
504 | ''' |
505 | -MDADM_USE_EXPORT = util.lsb_release()['codename'] not in ['precise', 'trusty'] |
506 | +MDADM_USE_EXPORT = lsb_release()['codename'] not in ['precise', 'trusty'] |
507 | |
508 | # |
509 | # mdadm executors |
510 | @@ -184,7 +185,7 @@ def mdadm_create(md_devname, raidlevel, devices, spares=None, md_name=""): |
511 | cmd.append(device) |
512 | |
513 | # Create the raid device |
514 | - util.subp(["udevadm", "settle"]) |
515 | + udev.udevadm_settle() |
516 | util.subp(["udevadm", "control", "--stop-exec-queue"]) |
517 | try: |
518 | util.subp(cmd, capture=True) |
519 | @@ -208,8 +209,7 @@ def mdadm_create(md_devname, raidlevel, devices, spares=None, md_name=""): |
520 | raise |
521 | |
522 | util.subp(["udevadm", "control", "--start-exec-queue"]) |
523 | - util.subp(["udevadm", "settle", |
524 | - "--exit-if-exists=%s" % md_devname]) |
525 | + udev.udevadm_settle(exists=md_devname) |
526 | |
527 | |
528 | def mdadm_examine(devpath, export=MDADM_USE_EXPORT): |
529 | diff --git a/curtin/block/mkfs.py b/curtin/block/mkfs.py |
530 | index a199d05..4a1e1f9 100644 |
531 | --- a/curtin/block/mkfs.py |
532 | +++ b/curtin/block/mkfs.py |
533 | @@ -3,12 +3,13 @@ |
534 | # This module wraps calls to mkfs.<fstype> and determines the appropriate flags |
535 | # for each filesystem type |
536 | |
537 | -from curtin import util |
538 | from curtin import block |
539 | +from curtin import distro |
540 | +from curtin import util |
541 | |
542 | import string |
543 | import os |
544 | -from uuid import uuid1 |
545 | +from uuid import uuid4 |
546 | |
547 | mkfs_commands = { |
548 | "btrfs": "mkfs.btrfs", |
549 | @@ -102,7 +103,7 @@ def valid_fstypes(): |
550 | |
551 | def get_flag_mapping(flag_name, fs_family, param=None, strict=False): |
552 | ret = [] |
553 | - release = util.lsb_release()['codename'] |
554 | + release = distro.lsb_release()['codename'] |
555 | overrides = release_flag_mapping_overrides.get(release, {}) |
556 | if flag_name in overrides and fs_family in overrides[flag_name]: |
557 | flag_sym = overrides[flag_name][fs_family] |
558 | @@ -191,7 +192,7 @@ def mkfs(path, fstype, strict=False, label=None, uuid=None, force=False): |
559 | |
560 | # If uuid is not specified, generate one and try to use it |
561 | if uuid is None: |
562 | - uuid = str(uuid1()) |
563 | + uuid = str(uuid4()) |
564 | cmd.extend(get_flag_mapping("uuid", fs_family, param=uuid, strict=strict)) |
565 | |
566 | if fs_family == "fat": |
567 | diff --git a/curtin/block/zfs.py b/curtin/block/zfs.py |
568 | index cfb07a9..5615144 100644 |
569 | --- a/curtin/block/zfs.py |
570 | +++ b/curtin/block/zfs.py |
571 | @@ -7,8 +7,9 @@ and volumes.""" |
572 | import os |
573 | |
574 | from curtin.config import merge_config |
575 | +from curtin import distro |
576 | from curtin import util |
577 | -from . import blkid |
578 | +from . import blkid, get_supported_filesystems |
579 | |
580 | ZPOOL_DEFAULT_PROPERTIES = { |
581 | 'ashift': 12, |
582 | @@ -73,6 +74,15 @@ def _join_pool_volume(poolname, volume): |
583 | |
584 | |
585 | def zfs_supported(): |
586 | + """Return a boolean indicating if zfs is supported.""" |
587 | + try: |
588 | + zfs_assert_supported() |
589 | + return True |
590 | + except RuntimeError: |
591 | + return False |
592 | + |
593 | + |
594 | +def zfs_assert_supported(): |
595 | """ Determine if the runtime system supports zfs. |
596 | returns: True if system supports zfs |
597 | raises: RuntimeError: if system does not support zfs |
598 | @@ -81,17 +91,19 @@ def zfs_supported(): |
599 | if arch in ZFS_UNSUPPORTED_ARCHES: |
600 | raise RuntimeError("zfs is not supported on architecture: %s" % arch) |
601 | |
602 | - release = util.lsb_release()['codename'] |
603 | + release = distro.lsb_release()['codename'] |
604 | if release in ZFS_UNSUPPORTED_RELEASES: |
605 | raise RuntimeError("zfs is not supported on release: %s" % release) |
606 | |
607 | - try: |
608 | - util.subp(['modinfo', 'zfs'], capture=True) |
609 | - except util.ProcessExecutionError as err: |
610 | - if err.stderr.startswith("modinfo: ERROR: Module zfs not found."): |
611 | - raise RuntimeError("zfs kernel module is not available: %s" % err) |
612 | + if 'zfs' not in get_supported_filesystems(): |
613 | + try: |
614 | + util.load_kernel_module('zfs') |
615 | + except util.ProcessExecutionError as err: |
616 | + raise RuntimeError("Failed to load 'zfs' kernel module: %s" % err) |
617 | |
618 | - return True |
619 | + missing_progs = [p for p in ('zpool', 'zfs') if not util.which(p)] |
620 | + if missing_progs: |
621 | + raise RuntimeError("Missing zfs utils: %s" % ','.join(missing_progs)) |
622 | |
623 | |
624 | def zpool_create(poolname, vdevs, mountpoint=None, altroot=None, |
625 | diff --git a/curtin/commands/__main__.py b/curtin/commands/__main__.py |
626 | new file mode 100644 |
627 | index 0000000..41c6d17 |
628 | --- /dev/null |
629 | +++ b/curtin/commands/__main__.py |
630 | @@ -0,0 +1,4 @@ |
631 | +if __name__ == '__main__': |
632 | + from .main import main |
633 | + import sys |
634 | + sys.exit(main()) |
635 | diff --git a/curtin/commands/apply_net.py b/curtin/commands/apply_net.py |
636 | index ffd474e..ddc5056 100644 |
637 | --- a/curtin/commands/apply_net.py |
638 | +++ b/curtin/commands/apply_net.py |
639 | @@ -7,6 +7,7 @@ from .. import log |
640 | import curtin.net as net |
641 | import curtin.util as util |
642 | from curtin import config |
643 | +from curtin import paths |
644 | from . import populate_one_subcmd |
645 | |
646 | |
647 | @@ -123,7 +124,7 @@ def _patch_ifupdown_ipv6_mtu_hook(target, |
648 | |
649 | for hook in ['prehook', 'posthook']: |
650 | fn = hookfn[hook] |
651 | - cfg = util.target_path(target, path=fn) |
652 | + cfg = paths.target_path(target, path=fn) |
653 | LOG.info('Injecting fix for ipv6 mtu settings: %s', cfg) |
654 | util.write_file(cfg, contents[hook], mode=0o755) |
655 | |
656 | @@ -136,7 +137,7 @@ def _disable_ipv6_privacy_extensions(target, |
657 | Resolve this by allowing the cloud-image setting to win. """ |
658 | |
659 | LOG.debug('Attempting to remove ipv6 privacy extensions') |
660 | - cfg = util.target_path(target, path=path) |
661 | + cfg = paths.target_path(target, path=path) |
662 | if not os.path.exists(cfg): |
663 | LOG.warn('Failed to find ipv6 privacy conf file %s', cfg) |
664 | return |
665 | @@ -182,7 +183,7 @@ def _maybe_remove_legacy_eth0(target, |
666 | - with unknown content, leave it and warn |
667 | """ |
668 | |
669 | - cfg = util.target_path(target, path=path) |
670 | + cfg = paths.target_path(target, path=path) |
671 | if not os.path.exists(cfg): |
672 | LOG.warn('Failed to find legacy network conf file %s', cfg) |
673 | return |
674 | diff --git a/curtin/commands/apt_config.py b/curtin/commands/apt_config.py |
675 | index 41c329e..9ce25b3 100644 |
676 | --- a/curtin/commands/apt_config.py |
677 | +++ b/curtin/commands/apt_config.py |
678 | @@ -13,7 +13,7 @@ import sys |
679 | import yaml |
680 | |
681 | from curtin.log import LOG |
682 | -from curtin import (config, util, gpg) |
683 | +from curtin import (config, distro, gpg, paths, util) |
684 | |
685 | from . import populate_one_subcmd |
686 | |
687 | @@ -61,7 +61,7 @@ def handle_apt(cfg, target=None): |
688 | curthooks if a global apt config was provided or via the "apt" |
689 | standalone command. |
690 | """ |
691 | - release = util.lsb_release(target=target)['codename'] |
692 | + release = distro.lsb_release(target=target)['codename'] |
693 | arch = util.get_architecture(target) |
694 | mirrors = find_apt_mirror_info(cfg, arch) |
695 | LOG.debug("Apt Mirror info: %s", mirrors) |
696 | @@ -148,7 +148,7 @@ def apply_debconf_selections(cfg, target=None): |
697 | pkg = re.sub(r"[:\s].*", "", line) |
698 | pkgs_cfgd.add(pkg) |
699 | |
700 | - pkgs_installed = util.get_installed_packages(target) |
701 | + pkgs_installed = distro.get_installed_packages(target) |
702 | |
703 | LOG.debug("pkgs_cfgd: %s", pkgs_cfgd) |
704 | LOG.debug("pkgs_installed: %s", pkgs_installed) |
705 | @@ -164,7 +164,7 @@ def apply_debconf_selections(cfg, target=None): |
706 | def clean_cloud_init(target): |
707 | """clean out any local cloud-init config""" |
708 | flist = glob.glob( |
709 | - util.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) |
710 | + paths.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) |
711 | |
712 | LOG.debug("cleaning cloud-init config from: %s", flist) |
713 | for dpkg_cfg in flist: |
714 | @@ -194,7 +194,7 @@ def rename_apt_lists(new_mirrors, target=None): |
715 | """rename_apt_lists - rename apt lists to preserve old cache data""" |
716 | default_mirrors = get_default_mirrors(util.get_architecture(target)) |
717 | |
718 | - pre = util.target_path(target, APT_LISTS) |
719 | + pre = paths.target_path(target, APT_LISTS) |
720 | for (name, omirror) in default_mirrors.items(): |
721 | nmirror = new_mirrors.get(name) |
722 | if not nmirror: |
723 | @@ -299,7 +299,7 @@ def generate_sources_list(cfg, release, mirrors, target=None): |
724 | if tmpl is None: |
725 | LOG.info("No custom template provided, fall back to modify" |
726 | "mirrors in %s on the target system", aptsrc) |
727 | - tmpl = util.load_file(util.target_path(target, aptsrc)) |
728 | + tmpl = util.load_file(paths.target_path(target, aptsrc)) |
729 | # Strategy if no custom template was provided: |
730 | # - Only replacing mirrors |
731 | # - no reason to replace "release" as it is from target anyway |
732 | @@ -310,24 +310,24 @@ def generate_sources_list(cfg, release, mirrors, target=None): |
733 | tmpl = mirror_to_placeholder(tmpl, default_mirrors['SECURITY'], |
734 | "$SECURITY") |
735 | |
736 | - orig = util.target_path(target, aptsrc) |
737 | + orig = paths.target_path(target, aptsrc) |
738 | if os.path.exists(orig): |
739 | os.rename(orig, orig + ".curtin.old") |
740 | |
741 | rendered = util.render_string(tmpl, params) |
742 | disabled = disable_suites(cfg.get('disable_suites'), rendered, release) |
743 | - util.write_file(util.target_path(target, aptsrc), disabled, mode=0o644) |
744 | + util.write_file(paths.target_path(target, aptsrc), disabled, mode=0o644) |
745 | |
746 | # protect the just generated sources.list from cloud-init |
747 | cloudfile = "/etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg" |
748 | # this has to work with older cloud-init as well, so use old key |
749 | cloudconf = yaml.dump({'apt_preserve_sources_list': True}, indent=1) |
750 | try: |
751 | - util.write_file(util.target_path(target, cloudfile), |
752 | + util.write_file(paths.target_path(target, cloudfile), |
753 | cloudconf, mode=0o644) |
754 | except IOError: |
755 | LOG.exception("Failed to protect source.list from cloud-init in (%s)", |
756 | - util.target_path(target, cloudfile)) |
757 | + paths.target_path(target, cloudfile)) |
758 | raise |
759 | |
760 | |
761 | @@ -409,7 +409,7 @@ def add_apt_sources(srcdict, target=None, template_params=None, |
762 | raise |
763 | continue |
764 | |
765 | - sourcefn = util.target_path(target, ent['filename']) |
766 | + sourcefn = paths.target_path(target, ent['filename']) |
767 | try: |
768 | contents = "%s\n" % (source) |
769 | util.write_file(sourcefn, contents, omode="a") |
770 | @@ -417,8 +417,8 @@ def add_apt_sources(srcdict, target=None, template_params=None, |
771 | LOG.exception("failed write to file %s: %s", sourcefn, detail) |
772 | raise |
773 | |
774 | - util.apt_update(target=target, force=True, |
775 | - comment="apt-source changed config") |
776 | + distro.apt_update(target=target, force=True, |
777 | + comment="apt-source changed config") |
778 | |
779 | return |
780 | |
781 | diff --git a/curtin/commands/block_meta.py b/curtin/commands/block_meta.py |
782 | index f5b82cf..197c1fd 100644 |
783 | --- a/curtin/commands/block_meta.py |
784 | +++ b/curtin/commands/block_meta.py |
785 | @@ -1,9 +1,10 @@ |
786 | # This file is part of curtin. See LICENSE file for copyright and license info. |
787 | |
788 | from collections import OrderedDict, namedtuple |
789 | -from curtin import (block, config, util) |
790 | +from curtin import (block, config, paths, util) |
791 | from curtin.block import (bcache, mdadm, mkfs, clear_holders, lvm, iscsi, zfs) |
792 | -from curtin.log import LOG |
793 | +from curtin import distro |
794 | +from curtin.log import LOG, logged_time |
795 | from curtin.reporter import events |
796 | |
797 | from . import populate_one_subcmd |
798 | @@ -48,6 +49,7 @@ CMD_ARGUMENTS = ( |
799 | ) |
800 | |
801 | |
802 | +@logged_time("BLOCK_META") |
803 | def block_meta(args): |
804 | # main entry point for the block-meta command. |
805 | state = util.load_command_environment() |
806 | @@ -729,12 +731,12 @@ def mount_fstab_data(fdata, target=None): |
807 | |
808 | :param fdata: a FstabData type |
809 | :return None.""" |
810 | - mp = util.target_path(target, fdata.path) |
811 | + mp = paths.target_path(target, fdata.path) |
812 | if fdata.device: |
813 | device = fdata.device |
814 | else: |
815 | if fdata.spec.startswith("/") and not fdata.spec.startswith("/dev/"): |
816 | - device = util.target_path(target, fdata.spec) |
817 | + device = paths.target_path(target, fdata.spec) |
818 | else: |
819 | device = fdata.spec |
820 | |
821 | @@ -855,7 +857,7 @@ def lvm_partition_handler(info, storage_config): |
822 | # Use 'wipesignatures' (if available) and 'zero' to clear target lv |
823 | # of any fs metadata |
824 | cmd = ["lvcreate", volgroup, "--name", name, "--zero=y"] |
825 | - release = util.lsb_release()['codename'] |
826 | + release = distro.lsb_release()['codename'] |
827 | if release not in ['precise', 'trusty']: |
828 | cmd.extend(["--wipesignatures=y"]) |
829 | |
830 | @@ -1263,7 +1265,7 @@ def zpool_handler(info, storage_config): |
831 | """ |
832 | Create a zpool based in storage_configuration |
833 | """ |
834 | - zfs.zfs_supported() |
835 | + zfs.zfs_assert_supported() |
836 | |
837 | state = util.load_command_environment() |
838 | |
839 | @@ -1298,7 +1300,8 @@ def zfs_handler(info, storage_config): |
840 | """ |
841 | Create a zfs filesystem |
842 | """ |
843 | - zfs.zfs_supported() |
844 | + zfs.zfs_assert_supported() |
845 | + |
846 | state = util.load_command_environment() |
847 | poolname = get_poolname(info, storage_config) |
848 | volume = info.get('volume') |
849 | diff --git a/curtin/commands/curthooks.py b/curtin/commands/curthooks.py |
850 | index d45c3a8..480eca4 100644 |
851 | --- a/curtin/commands/curthooks.py |
852 | +++ b/curtin/commands/curthooks.py |
853 | @@ -11,12 +11,18 @@ import textwrap |
854 | |
855 | from curtin import config |
856 | from curtin import block |
857 | +from curtin import distro |
858 | +from curtin.block import iscsi |
859 | from curtin import net |
860 | from curtin import futil |
861 | from curtin.log import LOG |
862 | +from curtin import paths |
863 | from curtin import swap |
864 | from curtin import util |
865 | from curtin import version as curtin_version |
866 | +from curtin.block import deps as bdeps |
867 | +from curtin.distro import DISTROS |
868 | +from curtin.net import deps as ndeps |
869 | from curtin.reporter import events |
870 | from curtin.commands import apply_net, apt_config |
871 | from curtin.url_helper import get_maas_version |
872 | @@ -173,10 +179,10 @@ def install_kernel(cfg, target): |
873 | # target only has required packages installed. See LP:1640519 |
874 | fk_packages = get_flash_kernel_pkgs() |
875 | if fk_packages: |
876 | - util.install_packages(fk_packages.split(), target=target) |
877 | + distro.install_packages(fk_packages.split(), target=target) |
878 | |
879 | if kernel_package: |
880 | - util.install_packages([kernel_package], target=target) |
881 | + distro.install_packages([kernel_package], target=target) |
882 | return |
883 | |
884 | # uname[2] is kernel name (ie: 3.16.0-7-generic) |
885 | @@ -193,24 +199,24 @@ def install_kernel(cfg, target): |
886 | LOG.warn("Couldn't detect kernel package to install for %s." |
887 | % kernel) |
888 | if kernel_fallback is not None: |
889 | - util.install_packages([kernel_fallback], target=target) |
890 | + distro.install_packages([kernel_fallback], target=target) |
891 | return |
892 | |
893 | package = "linux-{flavor}{map_suffix}".format( |
894 | flavor=flavor, map_suffix=map_suffix) |
895 | |
896 | - if util.has_pkg_available(package, target): |
897 | - if util.has_pkg_installed(package, target): |
898 | + if distro.has_pkg_available(package, target): |
899 | + if distro.has_pkg_installed(package, target): |
900 | LOG.debug("Kernel package '%s' already installed", package) |
901 | else: |
902 | LOG.debug("installing kernel package '%s'", package) |
903 | - util.install_packages([package], target=target) |
904 | + distro.install_packages([package], target=target) |
905 | else: |
906 | if kernel_fallback is not None: |
907 | LOG.info("Kernel package '%s' not available. " |
908 | "Installing fallback package '%s'.", |
909 | package, kernel_fallback) |
910 | - util.install_packages([kernel_fallback], target=target) |
911 | + distro.install_packages([kernel_fallback], target=target) |
912 | else: |
913 | LOG.warn("Kernel package '%s' not available and no fallback." |
914 | " System may not boot.", package) |
915 | @@ -273,7 +279,7 @@ def uefi_reorder_loaders(grubcfg, target): |
916 | LOG.debug("Currently booted UEFI loader might no longer boot.") |
917 | |
918 | |
919 | -def setup_grub(cfg, target): |
920 | +def setup_grub(cfg, target, osfamily=DISTROS.debian): |
921 | # target is the path to the mounted filesystem |
922 | |
923 | # FIXME: these methods need moving to curtin.block |
924 | @@ -292,7 +298,7 @@ def setup_grub(cfg, target): |
925 | storage_cfg_odict = None |
926 | try: |
927 | storage_cfg_odict = extract_storage_ordered_dict(cfg) |
928 | - except ValueError as e: |
929 | + except ValueError: |
930 | pass |
931 | |
932 | if storage_cfg_odict: |
933 | @@ -324,7 +330,7 @@ def setup_grub(cfg, target): |
934 | try: |
935 | (blockdev, part) = block.get_blockdev_for_partition(maybepart) |
936 | blockdevs.add(blockdev) |
937 | - except ValueError as e: |
938 | + except ValueError: |
939 | # if there is no syspath for this device such as a lvm |
940 | # or raid device, then a ValueError is raised here. |
941 | LOG.debug("failed to find block device for %s", maybepart) |
942 | @@ -353,24 +359,6 @@ def setup_grub(cfg, target): |
943 | else: |
944 | instdevs = list(blockdevs) |
945 | |
946 | - # UEFI requires grub-efi-{arch}. If a signed version of that package |
947 | - # exists then it will be installed. |
948 | - if util.is_uefi_bootable(): |
949 | - arch = util.get_architecture() |
950 | - pkgs = ['grub-efi-%s' % arch] |
951 | - |
952 | - # Architecture might support a signed UEFI loader |
953 | - uefi_pkg_signed = 'grub-efi-%s-signed' % arch |
954 | - if util.has_pkg_available(uefi_pkg_signed): |
955 | - pkgs.append(uefi_pkg_signed) |
956 | - |
957 | - # AMD64 has shim-signed for SecureBoot support |
958 | - if arch == "amd64": |
959 | - pkgs.append("shim-signed") |
960 | - |
961 | - # Install the UEFI packages needed for the architecture |
962 | - util.install_packages(pkgs, target=target) |
963 | - |
964 | env = os.environ.copy() |
965 | |
966 | replace_default = grubcfg.get('replace_linux_default', True) |
967 | @@ -399,6 +387,7 @@ def setup_grub(cfg, target): |
968 | else: |
969 | LOG.debug("NOT enabling UEFI nvram updates") |
970 | LOG.debug("Target system may not boot") |
971 | + args.append('--os-family=%s' % osfamily) |
972 | args.append(target) |
973 | |
974 | # capture stdout and stderr joined. |
975 | @@ -435,14 +424,21 @@ def copy_crypttab(crypttab, target): |
976 | shutil.copy(crypttab, os.path.sep.join([target, 'etc/crypttab'])) |
977 | |
978 | |
979 | -def copy_iscsi_conf(nodes_dir, target): |
980 | +def copy_iscsi_conf(nodes_dir, target, target_nodes_dir='etc/iscsi/nodes'): |
981 | if not nodes_dir: |
982 | LOG.warn("nodes directory must be specified, not copying") |
983 | return |
984 | |
985 | LOG.info("copying iscsi nodes database into target") |
986 | - shutil.copytree(nodes_dir, os.path.sep.join([target, |
987 | - 'etc/iscsi/nodes'])) |
988 | + tdir = os.path.sep.join([target, target_nodes_dir]) |
989 | + if not os.path.exists(tdir): |
990 | + shutil.copytree(nodes_dir, tdir) |
991 | + else: |
992 | + # if /etc/iscsi/nodes exists, copy dirs underneath |
993 | + for ndir in os.listdir(nodes_dir): |
994 | + source_dir = os.path.join(nodes_dir, ndir) |
995 | + target_dir = os.path.join(tdir, ndir) |
996 | + shutil.copytree(source_dir, target_dir) |
997 | |
998 | |
999 | def copy_mdadm_conf(mdadm_conf, target): |
1000 | @@ -486,7 +482,7 @@ def copy_dname_rules(rules_d, target): |
1001 | if not rules_d: |
1002 | LOG.warn("no udev rules directory to copy") |
1003 | return |
1004 | - target_rules_dir = util.target_path(target, "etc/udev/rules.d") |
1005 | + target_rules_dir = paths.target_path(target, "etc/udev/rules.d") |
1006 | for rule in os.listdir(rules_d): |
1007 | target_file = os.path.join(target_rules_dir, rule) |
1008 | shutil.copy(os.path.join(rules_d, rule), target_file) |
1009 | @@ -532,11 +528,19 @@ def add_swap(cfg, target, fstab): |
1010 | maxsize=maxsize) |
1011 | |
1012 | |
1013 | -def detect_and_handle_multipath(cfg, target): |
1014 | - DEFAULT_MULTIPATH_PACKAGES = ['multipath-tools-boot'] |
1015 | +def detect_and_handle_multipath(cfg, target, osfamily=DISTROS.debian): |
1016 | + DEFAULT_MULTIPATH_PACKAGES = { |
1017 | + DISTROS.debian: ['multipath-tools-boot'], |
1018 | + DISTROS.redhat: ['device-mapper-multipath'], |
1019 | + } |
1020 | + if osfamily not in DEFAULT_MULTIPATH_PACKAGES: |
1021 | + raise ValueError( |
1022 | + 'No multipath package mapping for distro: %s' % osfamily) |
1023 | + |
1024 | mpcfg = cfg.get('multipath', {}) |
1025 | mpmode = mpcfg.get('mode', 'auto') |
1026 | - mppkgs = mpcfg.get('packages', DEFAULT_MULTIPATH_PACKAGES) |
1027 | + mppkgs = mpcfg.get('packages', |
1028 | + DEFAULT_MULTIPATH_PACKAGES.get(osfamily)) |
1029 | mpbindings = mpcfg.get('overwrite_bindings', True) |
1030 | |
1031 | if isinstance(mppkgs, str): |
1032 | @@ -549,23 +553,28 @@ def detect_and_handle_multipath(cfg, target): |
1033 | return |
1034 | |
1035 | LOG.info("Detected multipath devices. Installing support via %s", mppkgs) |
1036 | + needed = [pkg for pkg in mppkgs if pkg |
1037 | + not in distro.get_installed_packages(target)] |
1038 | + if needed: |
1039 | + distro.install_packages(needed, target=target, osfamily=osfamily) |
1040 | |
1041 | - util.install_packages(mppkgs, target=target) |
1042 | replace_spaces = True |
1043 | - try: |
1044 | - # check in-target version |
1045 | - pkg_ver = util.get_package_version('multipath-tools', target=target) |
1046 | - LOG.debug("get_package_version:\n%s", pkg_ver) |
1047 | - LOG.debug("multipath version is %s (major=%s minor=%s micro=%s)", |
1048 | - pkg_ver['semantic_version'], pkg_ver['major'], |
1049 | - pkg_ver['minor'], pkg_ver['micro']) |
1050 | - # multipath-tools versions < 0.5.0 do _NOT_ want whitespace replaced |
1051 | - # i.e. 0.4.X in Trusty. |
1052 | - if pkg_ver['semantic_version'] < 500: |
1053 | - replace_spaces = False |
1054 | - except Exception as e: |
1055 | - LOG.warn("failed reading multipath-tools version, " |
1056 | - "assuming it wants no spaces in wwids: %s", e) |
1057 | + if osfamily == DISTROS.debian: |
1058 | + try: |
1059 | + # check in-target version |
1060 | + pkg_ver = distro.get_package_version('multipath-tools', |
1061 | + target=target) |
1062 | + LOG.debug("get_package_version:\n%s", pkg_ver) |
1063 | + LOG.debug("multipath version is %s (major=%s minor=%s micro=%s)", |
1064 | + pkg_ver['semantic_version'], pkg_ver['major'], |
1065 | + pkg_ver['minor'], pkg_ver['micro']) |
1066 | + # multipath-tools versions < 0.5.0 do _NOT_ |
1067 | + # want whitespace replaced i.e. 0.4.X in Trusty. |
1068 | + if pkg_ver['semantic_version'] < 500: |
1069 | + replace_spaces = False |
1070 | + except Exception as e: |
1071 | + LOG.warn("failed reading multipath-tools version, " |
1072 | + "assuming it wants no spaces in wwids: %s", e) |
1073 | |
1074 | multipath_cfg_path = os.path.sep.join([target, '/etc/multipath.conf']) |
1075 | multipath_bind_path = os.path.sep.join([target, '/etc/multipath/bindings']) |
1076 | @@ -574,7 +583,7 @@ def detect_and_handle_multipath(cfg, target): |
1077 | if not os.path.isfile(multipath_cfg_path): |
1078 | # Without user_friendly_names option enabled system fails to boot |
1079 | # if any of the disks has spaces in its name. Package multipath-tools |
1080 | - # has bug opened for this issue (LP: 1432062) but it was not fixed yet. |
1081 | + # has bug opened for this issue LP: #1432062 but it was not fixed yet. |
1082 | multipath_cfg_content = '\n'.join( |
1083 | ['# This file was created by curtin while installing the system.', |
1084 | 'defaults {', |
1085 | @@ -593,7 +602,13 @@ def detect_and_handle_multipath(cfg, target): |
1086 | mpname = "mpath0" |
1087 | grub_dev = "/dev/mapper/" + mpname |
1088 | if partno is not None: |
1089 | - grub_dev += "-part%s" % partno |
1090 | + if osfamily == DISTROS.debian: |
1091 | + grub_dev += "-part%s" % partno |
1092 | + elif osfamily == DISTROS.redhat: |
1093 | + grub_dev += "p%s" % partno |
1094 | + else: |
1095 | + raise ValueError( |
1096 | + 'Unknown grub_dev mapping for distro: %s' % osfamily) |
1097 | |
1098 | LOG.debug("configuring multipath install for root=%s wwid=%s", |
1099 | grub_dev, wwid) |
1100 | @@ -606,31 +621,54 @@ def detect_and_handle_multipath(cfg, target): |
1101 | '']) |
1102 | util.write_file(multipath_bind_path, content=multipath_bind_content) |
1103 | |
1104 | - grub_cfg = os.path.sep.join( |
1105 | - [target, '/etc/default/grub.d/50-curtin-multipath.cfg']) |
1106 | + if osfamily == DISTROS.debian: |
1107 | + grub_cfg = os.path.sep.join( |
1108 | + [target, '/etc/default/grub.d/50-curtin-multipath.cfg']) |
1109 | + omode = 'w' |
1110 | + elif osfamily == DISTROS.redhat: |
1111 | + grub_cfg = os.path.sep.join([target, '/etc/default/grub']) |
1112 | + omode = 'a' |
1113 | + else: |
1114 | + raise ValueError( |
1115 | + 'Unknown grub_cfg mapping for distro: %s' % osfamily) |
1116 | + |
1117 | msg = '\n'.join([ |
1118 | - '# Written by curtin for multipath device wwid "%s"' % wwid, |
1119 | + '# Written by curtin for multipath device %s %s' % (mpname, wwid), |
1120 | 'GRUB_DEVICE=%s' % grub_dev, |
1121 | 'GRUB_DISABLE_LINUX_UUID=true', |
1122 | '']) |
1123 | - util.write_file(grub_cfg, content=msg) |
1124 | - |
1125 | + util.write_file(grub_cfg, omode=omode, content=msg) |
1126 | else: |
1127 | LOG.warn("Not sure how this will boot") |
1128 | |
1129 | - # Initrams needs to be updated to include /etc/multipath.cfg |
1130 | - # and /etc/multipath/bindings files. |
1131 | - update_initramfs(target, all_kernels=True) |
1132 | + if osfamily == DISTROS.debian: |
1133 | + # Initrams needs to be updated to include /etc/multipath.cfg |
1134 | + # and /etc/multipath/bindings files. |
1135 | + update_initramfs(target, all_kernels=True) |
1136 | + elif osfamily == DISTROS.redhat: |
1137 | + # Write out initramfs/dracut config for multipath |
1138 | + dracut_conf_multipath = os.path.sep.join( |
1139 | + [target, '/etc/dracut.conf.d/10-curtin-multipath.conf']) |
1140 | + msg = '\n'.join([ |
1141 | + '# Written by curtin for multipath device wwid "%s"' % wwid, |
1142 | + 'force_drivers+=" dm-multipath "', |
1143 | + 'add_dracutmodules+="multipath"', |
1144 | + 'install_items+="/etc/multipath.conf /etc/multipath/bindings"', |
1145 | + '']) |
1146 | + util.write_file(dracut_conf_multipath, content=msg) |
1147 | + else: |
1148 | + raise ValueError( |
1149 | + 'Unknown initramfs mapping for distro: %s' % osfamily) |
1150 | |
1151 | |
1152 | -def detect_required_packages(cfg): |
1153 | +def detect_required_packages(cfg, osfamily=DISTROS.debian): |
1154 | """ |
1155 | detect packages that will be required in-target by custom config items |
1156 | """ |
1157 | |
1158 | mapping = { |
1159 | - 'storage': block.detect_required_packages_mapping(), |
1160 | - 'network': net.detect_required_packages_mapping(), |
1161 | + 'storage': bdeps.detect_required_packages_mapping(osfamily=osfamily), |
1162 | + 'network': ndeps.detect_required_packages_mapping(osfamily=osfamily), |
1163 | } |
1164 | |
1165 | needed_packages = [] |
1166 | @@ -657,16 +695,16 @@ def detect_required_packages(cfg): |
1167 | return needed_packages |
1168 | |
1169 | |
1170 | -def install_missing_packages(cfg, target): |
1171 | +def install_missing_packages(cfg, target, osfamily=DISTROS.debian): |
1172 | ''' describe which operation types will require specific packages |
1173 | |
1174 | 'custom_config_key': { |
1175 | 'pkg1': ['op_name_1', 'op_name_2', ...] |
1176 | } |
1177 | ''' |
1178 | - |
1179 | - installed_packages = util.get_installed_packages(target) |
1180 | - needed_packages = set([pkg for pkg in detect_required_packages(cfg) |
1181 | + installed_packages = distro.get_installed_packages(target) |
1182 | + needed_packages = set([pkg for pkg in |
1183 | + detect_required_packages(cfg, osfamily=osfamily) |
1184 | if pkg not in installed_packages]) |
1185 | |
1186 | arch_packages = { |
1187 | @@ -678,8 +716,35 @@ def install_missing_packages(cfg, target): |
1188 | if pkg not in needed_packages: |
1189 | needed_packages.add(pkg) |
1190 | |
1191 | + # UEFI requires grub-efi-{arch}. If a signed version of that package |
1192 | + # exists then it will be installed. |
1193 | + if util.is_uefi_bootable(): |
1194 | + uefi_pkgs = [] |
1195 | + if osfamily == DISTROS.redhat: |
1196 | + # centos/redhat doesn't support 32-bit? |
1197 | + uefi_pkgs.extend(['grub2-efi-x64-modules']) |
1198 | + elif osfamily == DISTROS.debian: |
1199 | + arch = util.get_architecture() |
1200 | + uefi_pkgs.append('grub-efi-%s' % arch) |
1201 | + |
1202 | + # Architecture might support a signed UEFI loader |
1203 | + uefi_pkg_signed = 'grub-efi-%s-signed' % arch |
1204 | + if distro.has_pkg_available(uefi_pkg_signed): |
1205 | + uefi_pkgs.append(uefi_pkg_signed) |
1206 | + |
1207 | + # AMD64 has shim-signed for SecureBoot support |
1208 | + if arch == "amd64": |
1209 | + uefi_pkgs.append("shim-signed") |
1210 | + else: |
1211 | + raise ValueError('Unknown grub2 package list for distro: %s' % |
1212 | + osfamily) |
1213 | + needed_packages.update([pkg for pkg in uefi_pkgs |
1214 | + if pkg not in installed_packages]) |
1215 | + |
1216 | # Filter out ifupdown network packages on netplan enabled systems. |
1217 | - if 'ifupdown' not in installed_packages and 'nplan' in installed_packages: |
1218 | + has_netplan = ('nplan' in installed_packages or |
1219 | + 'netplan.io' in installed_packages) |
1220 | + if 'ifupdown' not in installed_packages and has_netplan: |
1221 | drops = set(['bridge-utils', 'ifenslave', 'vlan']) |
1222 | if needed_packages.union(drops): |
1223 | LOG.debug("Skipping install of %s. Not needed on netplan system.", |
1224 | @@ -694,10 +759,10 @@ def install_missing_packages(cfg, target): |
1225 | reporting_enabled=True, level="INFO", |
1226 | description="Installing packages on target system: " + |
1227 | str(to_add)): |
1228 | - util.install_packages(to_add, target=target) |
1229 | + distro.install_packages(to_add, target=target, osfamily=osfamily) |
1230 | |
1231 | |
1232 | -def system_upgrade(cfg, target): |
1233 | +def system_upgrade(cfg, target, osfamily=DISTROS.debian): |
1234 | """run system-upgrade (apt-get dist-upgrade) or other in target. |
1235 | |
1236 | config: |
1237 | @@ -716,7 +781,7 @@ def system_upgrade(cfg, target): |
1238 | LOG.debug("system_upgrade disabled by config.") |
1239 | return |
1240 | |
1241 | - util.system_upgrade(target=target) |
1242 | + distro.system_upgrade(target=target, osfamily=osfamily) |
1243 | |
1244 | |
1245 | def inject_pollinate_user_agent_config(ua_cfg, target): |
1246 | @@ -726,7 +791,7 @@ def inject_pollinate_user_agent_config(ua_cfg, target): |
1247 | if not isinstance(ua_cfg, dict): |
1248 | raise ValueError('ua_cfg is not a dictionary: %s', ua_cfg) |
1249 | |
1250 | - pollinate_cfg = util.target_path(target, '/etc/pollinate/add-user-agent') |
1251 | + pollinate_cfg = paths.target_path(target, '/etc/pollinate/add-user-agent') |
1252 | comment = "# written by curtin" |
1253 | content = "\n".join(["%s/%s %s" % (ua_key, ua_val, comment) |
1254 | for ua_key, ua_val in ua_cfg.items()]) + "\n" |
1255 | @@ -749,6 +814,8 @@ def handle_pollinate_user_agent(cfg, target): |
1256 | curtin version |
1257 | maas version (via endpoint URL, if present) |
1258 | """ |
1259 | + if not util.which('pollinate', target=target): |
1260 | + return |
1261 | |
1262 | pcfg = cfg.get('pollinate') |
1263 | if not isinstance(pcfg, dict): |
1264 | @@ -774,6 +841,63 @@ def handle_pollinate_user_agent(cfg, target): |
1265 | inject_pollinate_user_agent_config(uacfg, target) |
1266 | |
1267 | |
1268 | +def configure_iscsi(cfg, state_etcd, target, osfamily=DISTROS.debian): |
1269 | + # If a /etc/iscsi/nodes/... file was created by block_meta then it |
1270 | + # needs to be copied onto the target system |
1271 | + nodes = os.path.join(state_etcd, "nodes") |
1272 | + if not os.path.exists(nodes): |
1273 | + return |
1274 | + |
1275 | + LOG.info('Iscsi configuration found, enabling service') |
1276 | + if osfamily == DISTROS.redhat: |
1277 | + # copy iscsi node config to target image |
1278 | + LOG.debug('Copying iscsi node config to target') |
1279 | + copy_iscsi_conf(nodes, target, target_nodes_dir='var/lib/iscsi/nodes') |
1280 | + |
1281 | + # update in-target config |
1282 | + with util.ChrootableTarget(target) as in_chroot: |
1283 | + # enable iscsid service |
1284 | + LOG.debug('Enabling iscsi daemon') |
1285 | + in_chroot.subp(['chkconfig', 'iscsid', 'on']) |
1286 | + |
1287 | + # update selinux config for iscsi ports required |
1288 | + for port in [str(port) for port in |
1289 | + iscsi.get_iscsi_ports_from_config(cfg)]: |
1290 | + LOG.debug('Adding iscsi port %s to selinux iscsi_port_t list', |
1291 | + port) |
1292 | + in_chroot.subp(['semanage', 'port', '-a', '-t', |
1293 | + 'iscsi_port_t', '-p', 'tcp', port]) |
1294 | + |
1295 | + elif osfamily == DISTROS.debian: |
1296 | + copy_iscsi_conf(nodes, target) |
1297 | + else: |
1298 | + raise ValueError( |
1299 | + 'Unknown iscsi requirements for distro: %s' % osfamily) |
1300 | + |
1301 | + |
1302 | +def configure_mdadm(cfg, state_etcd, target, osfamily=DISTROS.debian): |
1303 | + # If a mdadm.conf file was created by block_meta than it needs |
1304 | + # to be copied onto the target system |
1305 | + mdadm_location = os.path.join(state_etcd, "mdadm.conf") |
1306 | + if not os.path.exists(mdadm_location): |
1307 | + return |
1308 | + |
1309 | + conf_map = { |
1310 | + DISTROS.debian: 'etc/mdadm/mdadm.conf', |
1311 | + DISTROS.redhat: 'etc/mdadm.conf', |
1312 | + } |
1313 | + if osfamily not in conf_map: |
1314 | + raise ValueError( |
1315 | + 'Unknown mdadm conf mapping for distro: %s' % osfamily) |
1316 | + LOG.info('Mdadm configuration found, enabling service') |
1317 | + shutil.copy(mdadm_location, paths.target_path(target, |
1318 | + conf_map[osfamily])) |
1319 | + if osfamily == DISTROS.debian: |
1320 | + # as per LP: #964052 reconfigure mdadm |
1321 | + util.subp(['dpkg-reconfigure', '--frontend=noninteractive', 'mdadm'], |
1322 | + data=None, target=target) |
1323 | + |
1324 | + |
1325 | def handle_cloudconfig(cfg, base_dir=None): |
1326 | """write cloud-init configuration files into base_dir. |
1327 | |
1328 | @@ -843,21 +967,11 @@ def ubuntu_core_curthooks(cfg, target=None): |
1329 | content=config.dump_config({'network': netconfig})) |
1330 | |
1331 | |
1332 | -def rpm_get_dist_id(target): |
1333 | - """Use rpm command to extract the '%rhel' distro macro which returns |
1334 | - the major os version id (6, 7, 8). This works for centos or rhel |
1335 | - """ |
1336 | - with util.ChrootableTarget(target) as in_chroot: |
1337 | - dist, _ = in_chroot.subp(['rpm', '-E', '%rhel'], capture=True) |
1338 | - return dist.rstrip() |
1339 | - |
1340 | - |
1341 | -def centos_apply_network_config(netcfg, target=None): |
1342 | +def redhat_upgrade_cloud_init(netcfg, target=None, osfamily=DISTROS.redhat): |
1343 | """ CentOS images execute built-in curthooks which only supports |
1344 | simple networking configuration. This hook enables advanced |
1345 | network configuration via config passthrough to the target. |
1346 | """ |
1347 | - |
1348 | def cloud_init_repo(version): |
1349 | if not version: |
1350 | raise ValueError('Missing required version parameter') |
1351 | @@ -866,9 +980,9 @@ def centos_apply_network_config(netcfg, target=None): |
1352 | |
1353 | if netcfg: |
1354 | LOG.info('Removing embedded network configuration (if present)') |
1355 | - ifcfgs = glob.glob(util.target_path(target, |
1356 | - 'etc/sysconfig/network-scripts') + |
1357 | - '/ifcfg-*') |
1358 | + ifcfgs = glob.glob( |
1359 | + paths.target_path(target, 'etc/sysconfig/network-scripts') + |
1360 | + '/ifcfg-*') |
1361 | # remove ifcfg-* (except ifcfg-lo) |
1362 | for ifcfg in ifcfgs: |
1363 | if os.path.basename(ifcfg) != "ifcfg-lo": |
1364 | @@ -882,29 +996,27 @@ def centos_apply_network_config(netcfg, target=None): |
1365 | # if in-target cloud-init is not updated, upgrade via cloud-init repo |
1366 | if not passthrough: |
1367 | cloud_init_yum_repo = ( |
1368 | - util.target_path(target, |
1369 | - 'etc/yum.repos.d/curtin-cloud-init.repo')) |
1370 | + paths.target_path(target, |
1371 | + 'etc/yum.repos.d/curtin-cloud-init.repo')) |
1372 | # Inject cloud-init daily yum repo |
1373 | util.write_file(cloud_init_yum_repo, |
1374 | - content=cloud_init_repo(rpm_get_dist_id(target))) |
1375 | + content=cloud_init_repo( |
1376 | + distro.rpm_get_dist_id(target))) |
1377 | |
1378 | # we separate the installation of repository packages (epel, |
1379 | # cloud-init-el-release) as we need a new invocation of yum |
1380 | # to read the newly installed repo files. |
1381 | - YUM_CMD = ['yum', '-y', '--noplugins', 'install'] |
1382 | - retries = [1] * 30 |
1383 | - with util.ChrootableTarget(target) as in_chroot: |
1384 | - # ensure up-to-date ca-certificates to handle https mirror |
1385 | - # connections |
1386 | - in_chroot.subp(YUM_CMD + ['ca-certificates'], capture=True, |
1387 | - log_captured=True, retries=retries) |
1388 | - in_chroot.subp(YUM_CMD + ['epel-release'], capture=True, |
1389 | - log_captured=True, retries=retries) |
1390 | - in_chroot.subp(YUM_CMD + ['cloud-init-el-release'], |
1391 | - log_captured=True, capture=True, |
1392 | - retries=retries) |
1393 | - in_chroot.subp(YUM_CMD + ['cloud-init'], capture=True, |
1394 | - log_captured=True, retries=retries) |
1395 | + |
1396 | + # ensure up-to-date ca-certificates to handle https mirror |
1397 | + # connections |
1398 | + distro.install_packages(['ca-certificates'], target=target, |
1399 | + osfamily=osfamily) |
1400 | + distro.install_packages(['epel-release'], target=target, |
1401 | + osfamily=osfamily) |
1402 | + distro.install_packages(['cloud-init-el-release'], target=target, |
1403 | + osfamily=osfamily) |
1404 | + distro.install_packages(['cloud-init'], target=target, |
1405 | + osfamily=osfamily) |
1406 | |
1407 | # remove cloud-init el-stable bootstrap repo config as the |
1408 | # cloud-init-el-release package points to the correct repo |
1409 | @@ -917,127 +1029,136 @@ def centos_apply_network_config(netcfg, target=None): |
1410 | capture=False, rcs=[0]) |
1411 | except util.ProcessExecutionError: |
1412 | LOG.debug('Image missing bridge-utils package, installing') |
1413 | - in_chroot.subp(YUM_CMD + ['bridge-utils'], capture=True, |
1414 | - log_captured=True, retries=retries) |
1415 | + distro.install_packages(['bridge-utils'], target=target, |
1416 | + osfamily=osfamily) |
1417 | |
1418 | LOG.info('Passing network configuration through to target') |
1419 | net.render_netconfig_passthrough(target, netconfig={'network': netcfg}) |
1420 | |
1421 | |
1422 | -def target_is_ubuntu_core(target): |
1423 | - """Check if Ubuntu-Core specific directory is present at target""" |
1424 | - if target: |
1425 | - return os.path.exists(util.target_path(target, |
1426 | - 'system-data/var/lib/snapd')) |
1427 | - return False |
1428 | - |
1429 | - |
1430 | -def target_is_centos(target): |
1431 | - """Check if CentOS specific file is present at target""" |
1432 | - if target: |
1433 | - return os.path.exists(util.target_path(target, 'etc/centos-release')) |
1434 | +# Public API, maas may call this from internal curthooks |
1435 | +centos_apply_network_config = redhat_upgrade_cloud_init |
1436 | |
1437 | - return False |
1438 | |
1439 | +def redhat_apply_selinux_autorelabel(target): |
1440 | + """Creates file /.autorelabel. |
1441 | |
1442 | -def target_is_rhel(target): |
1443 | - """Check if RHEL specific file is present at target""" |
1444 | - if target: |
1445 | - return os.path.exists(util.target_path(target, 'etc/redhat-release')) |
1446 | + This is used by SELinux to relabel all of the |
1447 | + files on the filesystem to have the correct |
1448 | + security context. Without this SSH login will |
1449 | + fail. |
1450 | + """ |
1451 | + LOG.debug('enabling selinux autorelabel') |
1452 | + open(paths.target_path(target, '.autorelabel'), 'a').close() |
1453 | |
1454 | - return False |
1455 | |
1456 | +def redhat_update_dracut_config(target, cfg): |
1457 | + initramfs_mapping = { |
1458 | + 'lvm': {'conf': 'lvmconf', 'modules': 'lvm'}, |
1459 | + 'raid': {'conf': 'mdadmconf', 'modules': 'mdraid'}, |
1460 | + } |
1461 | |
1462 | -def curthooks(args): |
1463 | - state = util.load_command_environment() |
1464 | + # no need to update initramfs if no custom storage |
1465 | + if 'storage' not in cfg: |
1466 | + return False |
1467 | |
1468 | - if args.target is not None: |
1469 | - target = args.target |
1470 | - else: |
1471 | - target = state['target'] |
1472 | + storage_config = cfg.get('storage', {}).get('config') |
1473 | + if not storage_config: |
1474 | + raise ValueError('Invalid storage config') |
1475 | + |
1476 | + add_conf = set() |
1477 | + add_modules = set() |
1478 | + for scfg in storage_config: |
1479 | + if scfg['type'] == 'raid': |
1480 | + add_conf.add(initramfs_mapping['raid']['conf']) |
1481 | + add_modules.add(initramfs_mapping['raid']['modules']) |
1482 | + elif scfg['type'] in ['lvm_volgroup', 'lvm_partition']: |
1483 | + add_conf.add(initramfs_mapping['lvm']['conf']) |
1484 | + add_modules.add(initramfs_mapping['lvm']['modules']) |
1485 | + |
1486 | + dconfig = ['# Written by curtin for custom storage config'] |
1487 | + dconfig.append('add_dracutmodules+="%s"' % (" ".join(add_modules))) |
1488 | + for conf in add_conf: |
1489 | + dconfig.append('%s="yes"' % conf) |
1490 | + |
1491 | + # Write out initramfs/dracut config for storage config |
1492 | + dracut_conf_storage = os.path.sep.join( |
1493 | + [target, '/etc/dracut.conf.d/50-curtin-storage.conf']) |
1494 | + msg = '\n'.join(dconfig + ['']) |
1495 | + LOG.debug('Updating redhat dracut config') |
1496 | + util.write_file(dracut_conf_storage, content=msg) |
1497 | + return True |
1498 | + |
1499 | + |
1500 | +def redhat_update_initramfs(target, cfg): |
1501 | + if not redhat_update_dracut_config(target, cfg): |
1502 | + LOG.debug('Skipping redhat initramfs update, no custom storage config') |
1503 | + return |
1504 | + kver_cmd = ['rpm', '-q', '--queryformat', |
1505 | + '%{VERSION}-%{RELEASE}.%{ARCH}', 'kernel'] |
1506 | + with util.ChrootableTarget(target) as in_chroot: |
1507 | + LOG.debug('Finding redhat kernel version: %s', kver_cmd) |
1508 | + kver, _err = in_chroot.subp(kver_cmd, capture=True) |
1509 | + LOG.debug('Found kver=%s' % kver) |
1510 | + initramfs = '/boot/initramfs-%s.img' % kver |
1511 | + dracut_cmd = ['dracut', '-f', initramfs, kver] |
1512 | + LOG.debug('Rebuilding initramfs with: %s', dracut_cmd) |
1513 | + in_chroot.subp(dracut_cmd, capture=True) |
1514 | |
1515 | - if target is None: |
1516 | - sys.stderr.write("Unable to find target. " |
1517 | - "Use --target or set TARGET_MOUNT_POINT\n") |
1518 | - sys.exit(2) |
1519 | |
1520 | - cfg = config.load_command_config(args, state) |
1521 | +def builtin_curthooks(cfg, target, state): |
1522 | + LOG.info('Running curtin builtin curthooks') |
1523 | stack_prefix = state.get('report_stack_prefix', '') |
1524 | - |
1525 | - # if curtin-hooks hook exists in target we can defer to the in-target hooks |
1526 | - if util.run_hook_if_exists(target, 'curtin-hooks'): |
1527 | - # For vmtests to force execute centos_apply_network_config, uncomment |
1528 | - # the value in examples/tests/centos_defaults.yaml |
1529 | - if cfg.get('_ammend_centos_curthooks'): |
1530 | - if cfg.get('cloudconfig'): |
1531 | - handle_cloudconfig( |
1532 | - cfg['cloudconfig'], |
1533 | - base_dir=util.target_path(target, 'etc/cloud/cloud.cfg.d')) |
1534 | - |
1535 | - if target_is_centos(target) or target_is_rhel(target): |
1536 | - LOG.info('Detected RHEL/CentOS image, running extra hooks') |
1537 | - with events.ReportEventStack( |
1538 | - name=stack_prefix, reporting_enabled=True, |
1539 | - level="INFO", |
1540 | - description="Configuring CentOS for first boot"): |
1541 | - centos_apply_network_config(cfg.get('network', {}), target) |
1542 | - sys.exit(0) |
1543 | - |
1544 | - if target_is_ubuntu_core(target): |
1545 | - LOG.info('Detected Ubuntu-Core image, running hooks') |
1546 | + state_etcd = os.path.split(state['fstab'])[0] |
1547 | + |
1548 | + distro_info = distro.get_distroinfo(target=target) |
1549 | + if not distro_info: |
1550 | + raise RuntimeError('Failed to determine target distro') |
1551 | + osfamily = distro_info.family |
1552 | + LOG.info('Configuring target system for distro: %s osfamily: %s', |
1553 | + distro_info.variant, osfamily) |
1554 | + if osfamily == DISTROS.debian: |
1555 | with events.ReportEventStack( |
1556 | - name=stack_prefix, reporting_enabled=True, level="INFO", |
1557 | - description="Configuring Ubuntu-Core for first boot"): |
1558 | - ubuntu_core_curthooks(cfg, target) |
1559 | - sys.exit(0) |
1560 | - |
1561 | - with events.ReportEventStack( |
1562 | - name=stack_prefix + '/writing-config', |
1563 | - reporting_enabled=True, level="INFO", |
1564 | - description="configuring apt configuring apt"): |
1565 | - do_apt_config(cfg, target) |
1566 | - disable_overlayroot(cfg, target) |
1567 | + name=stack_prefix + '/writing-apt-config', |
1568 | + reporting_enabled=True, level="INFO", |
1569 | + description="configuring apt configuring apt"): |
1570 | + do_apt_config(cfg, target) |
1571 | + disable_overlayroot(cfg, target) |
1572 | |
1573 | - # LP: #1742560 prevent zfs-dkms from being installed (Xenial) |
1574 | - if util.lsb_release(target=target)['codename'] == 'xenial': |
1575 | - util.apt_update(target=target) |
1576 | - with util.ChrootableTarget(target) as in_chroot: |
1577 | - in_chroot.subp(['apt-mark', 'hold', 'zfs-dkms']) |
1578 | + # LP: #1742560 prevent zfs-dkms from being installed (Xenial) |
1579 | + if distro.lsb_release(target=target)['codename'] == 'xenial': |
1580 | + distro.apt_update(target=target) |
1581 | + with util.ChrootableTarget(target) as in_chroot: |
1582 | + in_chroot.subp(['apt-mark', 'hold', 'zfs-dkms']) |
1583 | |
1584 | # packages may be needed prior to installing kernel |
1585 | with events.ReportEventStack( |
1586 | name=stack_prefix + '/installing-missing-packages', |
1587 | reporting_enabled=True, level="INFO", |
1588 | description="installing missing packages"): |
1589 | - install_missing_packages(cfg, target) |
1590 | + install_missing_packages(cfg, target, osfamily=osfamily) |
1591 | |
1592 | - # If a /etc/iscsi/nodes/... file was created by block_meta then it |
1593 | - # needs to be copied onto the target system |
1594 | - nodes_location = os.path.join(os.path.split(state['fstab'])[0], |
1595 | - "nodes") |
1596 | - if os.path.exists(nodes_location): |
1597 | - copy_iscsi_conf(nodes_location, target) |
1598 | - # do we need to reconfigure open-iscsi? |
1599 | - |
1600 | - # If a mdadm.conf file was created by block_meta than it needs to be copied |
1601 | - # onto the target system |
1602 | - mdadm_location = os.path.join(os.path.split(state['fstab'])[0], |
1603 | - "mdadm.conf") |
1604 | - if os.path.exists(mdadm_location): |
1605 | - copy_mdadm_conf(mdadm_location, target) |
1606 | - # as per https://bugs.launchpad.net/ubuntu/+source/mdadm/+bug/964052 |
1607 | - # reconfigure mdadm |
1608 | - util.subp(['dpkg-reconfigure', '--frontend=noninteractive', 'mdadm'], |
1609 | - data=None, target=target) |
1610 | + with events.ReportEventStack( |
1611 | + name=stack_prefix + '/configuring-iscsi-service', |
1612 | + reporting_enabled=True, level="INFO", |
1613 | + description="configuring iscsi service"): |
1614 | + configure_iscsi(cfg, state_etcd, target, osfamily=osfamily) |
1615 | |
1616 | with events.ReportEventStack( |
1617 | - name=stack_prefix + '/installing-kernel', |
1618 | + name=stack_prefix + '/configuring-mdadm-service', |
1619 | reporting_enabled=True, level="INFO", |
1620 | - description="installing kernel"): |
1621 | - setup_zipl(cfg, target) |
1622 | - install_kernel(cfg, target) |
1623 | - run_zipl(cfg, target) |
1624 | - restore_dist_interfaces(cfg, target) |
1625 | + description="configuring raid (mdadm) service"): |
1626 | + configure_mdadm(cfg, state_etcd, target, osfamily=osfamily) |
1627 | + |
1628 | + if osfamily == DISTROS.debian: |
1629 | + with events.ReportEventStack( |
1630 | + name=stack_prefix + '/installing-kernel', |
1631 | + reporting_enabled=True, level="INFO", |
1632 | + description="installing kernel"): |
1633 | + setup_zipl(cfg, target) |
1634 | + install_kernel(cfg, target) |
1635 | + run_zipl(cfg, target) |
1636 | + restore_dist_interfaces(cfg, target) |
1637 | |
1638 | with events.ReportEventStack( |
1639 | name=stack_prefix + '/setting-up-swap', |
1640 | @@ -1045,6 +1166,23 @@ def curthooks(args): |
1641 | description="setting up swap"): |
1642 | add_swap(cfg, target, state.get('fstab')) |
1643 | |
1644 | + if osfamily == DISTROS.redhat: |
1645 | + # set cloud-init maas datasource for centos images |
1646 | + if cfg.get('cloudconfig'): |
1647 | + handle_cloudconfig( |
1648 | + cfg['cloudconfig'], |
1649 | + base_dir=paths.target_path(target, |
1650 | + 'etc/cloud/cloud.cfg.d')) |
1651 | + |
1652 | + # For vmtests to force execute redhat_upgrade_cloud_init, uncomment |
1653 | + # the value in examples/tests/centos_defaults.yaml |
1654 | + if cfg.get('_ammend_centos_curthooks'): |
1655 | + with events.ReportEventStack( |
1656 | + name=stack_prefix + '/upgrading cloud-init', |
1657 | + reporting_enabled=True, level="INFO", |
1658 | + description="Upgrading cloud-init in target"): |
1659 | + redhat_upgrade_cloud_init(cfg.get('network', {}), target) |
1660 | + |
1661 | with events.ReportEventStack( |
1662 | name=stack_prefix + '/apply-networking-config', |
1663 | reporting_enabled=True, level="INFO", |
1664 | @@ -1061,29 +1199,44 @@ def curthooks(args): |
1665 | name=stack_prefix + '/configuring-multipath', |
1666 | reporting_enabled=True, level="INFO", |
1667 | description="configuring multipath"): |
1668 | - detect_and_handle_multipath(cfg, target) |
1669 | + detect_and_handle_multipath(cfg, target, osfamily=osfamily) |
1670 | |
1671 | with events.ReportEventStack( |
1672 | name=stack_prefix + '/system-upgrade', |
1673 | reporting_enabled=True, level="INFO", |
1674 | description="updating packages on target system"): |
1675 | - system_upgrade(cfg, target) |
1676 | + system_upgrade(cfg, target, osfamily=osfamily) |
1677 | + |
1678 | + if osfamily == DISTROS.redhat: |
1679 | + with events.ReportEventStack( |
1680 | + name=stack_prefix + '/enabling-selinux-autorelabel', |
1681 | + reporting_enabled=True, level="INFO", |
1682 | + description="enabling selinux autorelabel mode"): |
1683 | + redhat_apply_selinux_autorelabel(target) |
1684 | + |
1685 | + with events.ReportEventStack( |
1686 | + name=stack_prefix + '/updating-initramfs-configuration', |
1687 | + reporting_enabled=True, level="INFO", |
1688 | + description="updating initramfs configuration"): |
1689 | + redhat_update_initramfs(target, cfg) |
1690 | |
1691 | with events.ReportEventStack( |
1692 | name=stack_prefix + '/pollinate-user-agent', |
1693 | reporting_enabled=True, level="INFO", |
1694 | - description="configuring pollinate user-agent on target system"): |
1695 | + description="configuring pollinate user-agent on target"): |
1696 | handle_pollinate_user_agent(cfg, target) |
1697 | |
1698 | - # If a crypttab file was created by block_meta than it needs to be copied |
1699 | - # onto the target system, and update_initramfs() needs to be run, so that |
1700 | - # the cryptsetup hooks are properly configured on the installed system and |
1701 | - # it will be able to open encrypted volumes at boot. |
1702 | - crypttab_location = os.path.join(os.path.split(state['fstab'])[0], |
1703 | - "crypttab") |
1704 | - if os.path.exists(crypttab_location): |
1705 | - copy_crypttab(crypttab_location, target) |
1706 | - update_initramfs(target) |
1707 | + if osfamily == DISTROS.debian: |
1708 | + # If a crypttab file was created by block_meta than it needs to be |
1709 | + # copied onto the target system, and update_initramfs() needs to be |
1710 | + # run, so that the cryptsetup hooks are properly configured on the |
1711 | + # installed system and it will be able to open encrypted volumes |
1712 | + # at boot. |
1713 | + crypttab_location = os.path.join(os.path.split(state['fstab'])[0], |
1714 | + "crypttab") |
1715 | + if os.path.exists(crypttab_location): |
1716 | + copy_crypttab(crypttab_location, target) |
1717 | + update_initramfs(target) |
1718 | |
1719 | # If udev dname rules were created, copy them to target |
1720 | udev_rules_d = os.path.join(state['scratch'], "rules.d") |
1721 | @@ -1100,8 +1253,41 @@ def curthooks(args): |
1722 | machine.startswith('aarch64') and not util.is_uefi_bootable()): |
1723 | update_initramfs(target) |
1724 | else: |
1725 | - setup_grub(cfg, target) |
1726 | + setup_grub(cfg, target, osfamily=osfamily) |
1727 | + |
1728 | + |
1729 | +def curthooks(args): |
1730 | + state = util.load_command_environment() |
1731 | + |
1732 | + if args.target is not None: |
1733 | + target = args.target |
1734 | + else: |
1735 | + target = state['target'] |
1736 | + |
1737 | + if target is None: |
1738 | + sys.stderr.write("Unable to find target. " |
1739 | + "Use --target or set TARGET_MOUNT_POINT\n") |
1740 | + sys.exit(2) |
1741 | + |
1742 | + cfg = config.load_command_config(args, state) |
1743 | + stack_prefix = state.get('report_stack_prefix', '') |
1744 | + curthooks_mode = cfg.get('curthooks', {}).get('mode', 'auto') |
1745 | + |
1746 | + # UC is special, handle it first. |
1747 | + if distro.is_ubuntu_core(target): |
1748 | + LOG.info('Detected Ubuntu-Core image, running hooks') |
1749 | + with events.ReportEventStack( |
1750 | + name=stack_prefix, reporting_enabled=True, level="INFO", |
1751 | + description="Configuring Ubuntu-Core for first boot"): |
1752 | + ubuntu_core_curthooks(cfg, target) |
1753 | + sys.exit(0) |
1754 | + |
1755 | + # user asked for target, or auto mode |
1756 | + if curthooks_mode in ['auto', 'target']: |
1757 | + if util.run_hook_if_exists(target, 'curtin-hooks'): |
1758 | + sys.exit(0) |
1759 | |
1760 | + builtin_curthooks(cfg, target, state) |
1761 | sys.exit(0) |
1762 | |
1763 | |
1764 | diff --git a/curtin/commands/extract.py b/curtin/commands/extract.py |
1765 | index 69a9d18..ec7a791 100644 |
1766 | --- a/curtin/commands/extract.py |
1767 | +++ b/curtin/commands/extract.py |
1768 | @@ -59,7 +59,7 @@ def extract_root_tgz_url(url, target): |
1769 | def extract_root_fsimage_url(url, target): |
1770 | path = _path_from_file_url(url) |
1771 | if path != url or os.path.isfile(path): |
1772 | - return _extract_root_fsimage(path(url), target) |
1773 | + return _extract_root_fsimage(path, target) |
1774 | |
1775 | wfp = tempfile.NamedTemporaryFile(suffix=".img", delete=False) |
1776 | wfp.close() |
1777 | diff --git a/curtin/commands/features.py b/curtin/commands/features.py |
1778 | new file mode 100644 |
1779 | index 0000000..0f6085b |
1780 | --- /dev/null |
1781 | +++ b/curtin/commands/features.py |
1782 | @@ -0,0 +1,20 @@ |
1783 | +# This file is part of curtin. See LICENSE file for copyright and license info. |
1784 | +"""List the supported feature names to stdout.""" |
1785 | + |
1786 | +import sys |
1787 | +from .. import FEATURES |
1788 | +from . import populate_one_subcmd |
1789 | + |
1790 | +CMD_ARGUMENTS = ((tuple())) |
1791 | + |
1792 | + |
1793 | +def features_main(args): |
1794 | + sys.stdout.write("\n".join(sorted(FEATURES)) + "\n") |
1795 | + sys.exit(0) |
1796 | + |
1797 | + |
1798 | +def POPULATE_SUBCMD(parser): |
1799 | + populate_one_subcmd(parser, CMD_ARGUMENTS, features_main) |
1800 | + parser.description = __doc__ |
1801 | + |
1802 | +# vi: ts=4 expandtab syntax=python |
1803 | diff --git a/curtin/commands/in_target.py b/curtin/commands/in_target.py |
1804 | index 8e839c0..c6f7abd 100644 |
1805 | --- a/curtin/commands/in_target.py |
1806 | +++ b/curtin/commands/in_target.py |
1807 | @@ -4,7 +4,7 @@ import os |
1808 | import pty |
1809 | import sys |
1810 | |
1811 | -from curtin import util |
1812 | +from curtin import paths, util |
1813 | |
1814 | from . import populate_one_subcmd |
1815 | |
1816 | @@ -41,7 +41,7 @@ def in_target_main(args): |
1817 | sys.exit(2) |
1818 | |
1819 | daemons = args.allow_daemons |
1820 | - if util.target_path(args.target) == "/": |
1821 | + if paths.target_path(args.target) == "/": |
1822 | sys.stderr.write("WARN: Target is /, daemons are allowed.\n") |
1823 | daemons = True |
1824 | cmd = args.command_args |
1825 | diff --git a/curtin/commands/install.py b/curtin/commands/install.py |
1826 | index a8c4cf9..244683c 100644 |
1827 | --- a/curtin/commands/install.py |
1828 | +++ b/curtin/commands/install.py |
1829 | @@ -13,9 +13,11 @@ import tempfile |
1830 | |
1831 | from curtin.block import iscsi |
1832 | from curtin import config |
1833 | +from curtin import distro |
1834 | from curtin import util |
1835 | +from curtin import paths |
1836 | from curtin import version |
1837 | -from curtin.log import LOG |
1838 | +from curtin.log import LOG, logged_time |
1839 | from curtin.reporter.legacy import load_reporter |
1840 | from curtin.reporter import events |
1841 | from . import populate_one_subcmd |
1842 | @@ -80,7 +82,7 @@ def copy_install_log(logfile, target, log_target_path): |
1843 | LOG.debug('Copying curtin install log from %s to target/%s', |
1844 | logfile, log_target_path) |
1845 | util.write_file( |
1846 | - filename=util.target_path(target, log_target_path), |
1847 | + filename=paths.target_path(target, log_target_path), |
1848 | content=util.load_file(logfile, decode=False), |
1849 | mode=0o400, omode="wb") |
1850 | |
1851 | @@ -111,12 +113,22 @@ class WorkingDir(object): |
1852 | def __init__(self, config): |
1853 | top_d = tempfile.mkdtemp() |
1854 | state_d = os.path.join(top_d, 'state') |
1855 | + scratch_d = os.path.join(top_d, 'scratch') |
1856 | + for p in (state_d, scratch_d): |
1857 | + os.mkdir(p) |
1858 | + |
1859 | target_d = config.get('install', {}).get('target') |
1860 | if not target_d: |
1861 | target_d = os.path.join(top_d, 'target') |
1862 | - scratch_d = os.path.join(top_d, 'scratch') |
1863 | - for p in (state_d, target_d, scratch_d): |
1864 | - os.mkdir(p) |
1865 | + try: |
1866 | + util.ensure_dir(target_d) |
1867 | + except OSError as e: |
1868 | + raise ValueError( |
1869 | + "Unable to create target directory '%s': %s" % |
1870 | + (target_d, e)) |
1871 | + if os.listdir(target_d) != []: |
1872 | + raise ValueError( |
1873 | + "Provided target dir '%s' was not empty." % target_d) |
1874 | |
1875 | netconf_f = os.path.join(state_d, 'network_config') |
1876 | netstate_f = os.path.join(state_d, 'network_state') |
1877 | @@ -309,7 +321,7 @@ def apply_kexec(kexec, target): |
1878 | raise TypeError("kexec is not a dict.") |
1879 | |
1880 | if not util.which('kexec'): |
1881 | - util.install_packages('kexec-tools') |
1882 | + distro.install_packages('kexec-tools') |
1883 | |
1884 | if not os.path.isfile(target_grubcfg): |
1885 | raise ValueError("%s does not exist in target" % grubcfg) |
1886 | @@ -380,6 +392,7 @@ def migrate_proxy_settings(cfg): |
1887 | cfg['proxy'] = proxy |
1888 | |
1889 | |
1890 | +@logged_time("INSTALL_COMMAND") |
1891 | def cmd_install(args): |
1892 | from .collect_logs import create_log_tarfile |
1893 | cfg = deepcopy(CONFIG_BUILTIN) |
1894 | @@ -429,6 +442,7 @@ def cmd_install(args): |
1895 | |
1896 | writeline_and_stdout(logfile, INSTALL_START_MSG) |
1897 | args.reportstack.post_files = post_files |
1898 | + workingd = None |
1899 | try: |
1900 | workingd = WorkingDir(cfg) |
1901 | dd_images = util.get_dd_images(cfg.get('sources', {})) |
1902 | @@ -469,12 +483,12 @@ def cmd_install(args): |
1903 | raise e |
1904 | finally: |
1905 | log_target_path = instcfg.get('save_install_log', SAVE_INSTALL_LOG) |
1906 | - if log_target_path: |
1907 | + if log_target_path and workingd: |
1908 | copy_install_log(logfile, workingd.target, log_target_path) |
1909 | |
1910 | if instcfg.get('unmount', "") == "disabled": |
1911 | LOG.info('Skipping unmount: config disabled target unmounting') |
1912 | - else: |
1913 | + elif workingd: |
1914 | # unmount everything (including iscsi disks) |
1915 | util.do_umount(workingd.target, recursive=True) |
1916 | |
1917 | diff --git a/curtin/commands/main.py b/curtin/commands/main.py |
1918 | index 779bb03..bccfc51 100644 |
1919 | --- a/curtin/commands/main.py |
1920 | +++ b/curtin/commands/main.py |
1921 | @@ -16,9 +16,9 @@ VERSIONSTR = version.version_string() |
1922 | SUB_COMMAND_MODULES = [ |
1923 | 'apply_net', 'apt-config', 'block-attach-iscsi', 'block-detach-iscsi', |
1924 | 'block-info', 'block-meta', 'block-wipe', 'clear-holders', 'curthooks', |
1925 | - 'collect-logs', 'extract', 'hook', 'install', 'mkfs', 'in-target', |
1926 | - 'net-meta', 'pack', 'swap', 'system-install', 'system-upgrade', 'unmount', |
1927 | - 'version', |
1928 | + 'collect-logs', 'extract', 'features', |
1929 | + 'hook', 'install', 'mkfs', 'in-target', 'net-meta', 'pack', 'swap', |
1930 | + 'system-install', 'system-upgrade', 'unmount', 'version', |
1931 | ] |
1932 | |
1933 | |
1934 | diff --git a/curtin/commands/system_install.py b/curtin/commands/system_install.py |
1935 | index 05d70af..6d7b736 100644 |
1936 | --- a/curtin/commands/system_install.py |
1937 | +++ b/curtin/commands/system_install.py |
1938 | @@ -7,6 +7,7 @@ import curtin.util as util |
1939 | |
1940 | from . import populate_one_subcmd |
1941 | from curtin.log import LOG |
1942 | +from curtin import distro |
1943 | |
1944 | |
1945 | def system_install_pkgs_main(args): |
1946 | @@ -16,7 +17,7 @@ def system_install_pkgs_main(args): |
1947 | |
1948 | exit_code = 0 |
1949 | try: |
1950 | - util.install_packages( |
1951 | + distro.install_packages( |
1952 | pkglist=args.packages, target=args.target, |
1953 | allow_daemons=args.allow_daemons) |
1954 | except util.ProcessExecutionError as e: |
1955 | diff --git a/curtin/commands/system_upgrade.py b/curtin/commands/system_upgrade.py |
1956 | index fe10fac..d4f6735 100644 |
1957 | --- a/curtin/commands/system_upgrade.py |
1958 | +++ b/curtin/commands/system_upgrade.py |
1959 | @@ -7,6 +7,7 @@ import curtin.util as util |
1960 | |
1961 | from . import populate_one_subcmd |
1962 | from curtin.log import LOG |
1963 | +from curtin import distro |
1964 | |
1965 | |
1966 | def system_upgrade_main(args): |
1967 | @@ -16,8 +17,8 @@ def system_upgrade_main(args): |
1968 | |
1969 | exit_code = 0 |
1970 | try: |
1971 | - util.system_upgrade(target=args.target, |
1972 | - allow_daemons=args.allow_daemons) |
1973 | + distro.system_upgrade(target=args.target, |
1974 | + allow_daemons=args.allow_daemons) |
1975 | except util.ProcessExecutionError as e: |
1976 | LOG.warn("system upgrade failed: %s" % e) |
1977 | exit_code = e.exit_code |
1978 | diff --git a/curtin/deps/__init__.py b/curtin/deps/__init__.py |
1979 | index 7014895..96df4f6 100644 |
1980 | --- a/curtin/deps/__init__.py |
1981 | +++ b/curtin/deps/__init__.py |
1982 | @@ -6,13 +6,13 @@ import sys |
1983 | from curtin.util import ( |
1984 | ProcessExecutionError, |
1985 | get_architecture, |
1986 | - install_packages, |
1987 | is_uefi_bootable, |
1988 | - lsb_release, |
1989 | subp, |
1990 | which, |
1991 | ) |
1992 | |
1993 | +from curtin.distro import install_packages, lsb_release |
1994 | + |
1995 | REQUIRED_IMPORTS = [ |
1996 | # import string to execute, python2 package, python3 package |
1997 | ('import yaml', 'python-yaml', 'python3-yaml'), |
1998 | @@ -177,7 +177,7 @@ def install_deps(verbosity=False, dry_run=False, allow_daemons=True): |
1999 | ret = 0 |
2000 | try: |
2001 | install_packages(missing_pkgs, allow_daemons=allow_daemons, |
2002 | - aptopts=["--no-install-recommends"]) |
2003 | + opts=["--no-install-recommends"]) |
2004 | except ProcessExecutionError as e: |
2005 | sys.stderr.write("%s\n" % e) |
2006 | ret = e.exit_code |
2007 | diff --git a/curtin/distro.py b/curtin/distro.py |
2008 | new file mode 100644 |
2009 | index 0000000..f2a78ed |
2010 | --- /dev/null |
2011 | +++ b/curtin/distro.py |
2012 | @@ -0,0 +1,512 @@ |
2013 | +# This file is part of curtin. See LICENSE file for copyright and license info. |
2014 | +import glob |
2015 | +from collections import namedtuple |
2016 | +import os |
2017 | +import re |
2018 | +import shutil |
2019 | +import tempfile |
2020 | + |
2021 | +from .paths import target_path |
2022 | +from .util import ( |
2023 | + ChrootableTarget, |
2024 | + find_newer, |
2025 | + load_file, |
2026 | + load_shell_content, |
2027 | + ProcessExecutionError, |
2028 | + set_unexecutable, |
2029 | + string_types, |
2030 | + subp, |
2031 | + which |
2032 | +) |
2033 | +from .log import LOG |
2034 | + |
2035 | +DistroInfo = namedtuple('DistroInfo', ('variant', 'family')) |
2036 | +DISTRO_NAMES = ['arch', 'centos', 'debian', 'fedora', 'freebsd', 'gentoo', |
2037 | + 'opensuse', 'redhat', 'rhel', 'sles', 'suse', 'ubuntu'] |
2038 | + |
2039 | + |
2040 | +# python2.7 lacks PEP 435, so we must make use an alternative for py2.7/3.x |
2041 | +# https://stackoverflow.com/questions/36932/how-can-i-represent-an-enum-in-python |
2042 | +def distro_enum(*distros): |
2043 | + return namedtuple('Distros', distros)(*distros) |
2044 | + |
2045 | + |
2046 | +DISTROS = distro_enum(*DISTRO_NAMES) |
2047 | + |
2048 | +OS_FAMILIES = { |
2049 | + DISTROS.debian: [DISTROS.debian, DISTROS.ubuntu], |
2050 | + DISTROS.redhat: [DISTROS.centos, DISTROS.fedora, DISTROS.redhat, |
2051 | + DISTROS.rhel], |
2052 | + DISTROS.gentoo: [DISTROS.gentoo], |
2053 | + DISTROS.freebsd: [DISTROS.freebsd], |
2054 | + DISTROS.suse: [DISTROS.opensuse, DISTROS.sles, DISTROS.suse], |
2055 | + DISTROS.arch: [DISTROS.arch], |
2056 | +} |
2057 | + |
2058 | +# invert the mapping for faster lookup of variants |
2059 | +DISTRO_TO_OSFAMILY = ( |
2060 | + {variant: family for family, variants in OS_FAMILIES.items() |
2061 | + for variant in variants}) |
2062 | + |
2063 | +_LSB_RELEASE = {} |
2064 | + |
2065 | + |
2066 | +def name_to_distro(distname): |
2067 | + try: |
2068 | + return DISTROS[DISTROS.index(distname)] |
2069 | + except (IndexError, AttributeError): |
2070 | + LOG.error('Unknown distro name: %s', distname) |
2071 | + |
2072 | + |
2073 | +def lsb_release(target=None): |
2074 | + if target_path(target) != "/": |
2075 | + # do not use or update cache if target is provided |
2076 | + return _lsb_release(target) |
2077 | + |
2078 | + global _LSB_RELEASE |
2079 | + if not _LSB_RELEASE: |
2080 | + data = _lsb_release() |
2081 | + _LSB_RELEASE.update(data) |
2082 | + return _LSB_RELEASE |
2083 | + |
2084 | + |
2085 | +def os_release(target=None): |
2086 | + data = {} |
2087 | + os_release = target_path(target, 'etc/os-release') |
2088 | + if os.path.exists(os_release): |
2089 | + data = load_shell_content(load_file(os_release), |
2090 | + add_empty=False, empty_val=None) |
2091 | + if not data: |
2092 | + for relfile in [target_path(target, rel) for rel in |
2093 | + ['etc/centos-release', 'etc/redhat-release']]: |
2094 | + data = _parse_redhat_release(release_file=relfile, target=target) |
2095 | + if data: |
2096 | + break |
2097 | + |
2098 | + return data |
2099 | + |
2100 | + |
2101 | +def _parse_redhat_release(release_file=None, target=None): |
2102 | + """Return a dictionary of distro info fields from /etc/redhat-release. |
2103 | + |
2104 | + Dict keys will align with /etc/os-release keys: |
2105 | + ID, VERSION_ID, VERSION_CODENAME |
2106 | + """ |
2107 | + |
2108 | + if not release_file: |
2109 | + release_file = target_path('etc/redhat-release') |
2110 | + if not os.path.exists(release_file): |
2111 | + return {} |
2112 | + redhat_release = load_file(release_file) |
2113 | + redhat_regex = ( |
2114 | + r'(?P<name>.+) release (?P<version>[\d\.]+) ' |
2115 | + r'\((?P<codename>[^)]+)\)') |
2116 | + match = re.match(redhat_regex, redhat_release) |
2117 | + if match: |
2118 | + group = match.groupdict() |
2119 | + group['name'] = group['name'].lower().partition(' linux')[0] |
2120 | + if group['name'] == 'red hat enterprise': |
2121 | + group['name'] = 'redhat' |
2122 | + return {'ID': group['name'], 'VERSION_ID': group['version'], |
2123 | + 'VERSION_CODENAME': group['codename']} |
2124 | + return {} |
2125 | + |
2126 | + |
2127 | +def get_distroinfo(target=None): |
2128 | + variant_name = os_release(target=target)['ID'] |
2129 | + variant = name_to_distro(variant_name) |
2130 | + family = DISTRO_TO_OSFAMILY.get(variant) |
2131 | + return DistroInfo(variant, family) |
2132 | + |
2133 | + |
2134 | +def get_distro(target=None): |
2135 | + distinfo = get_distroinfo(target=target) |
2136 | + return distinfo.variant |
2137 | + |
2138 | + |
2139 | +def get_osfamily(target=None): |
2140 | + distinfo = get_distroinfo(target=target) |
2141 | + return distinfo.family |
2142 | + |
2143 | + |
2144 | +def is_ubuntu_core(target=None): |
2145 | + """Check if Ubuntu-Core specific directory is present at target""" |
2146 | + return os.path.exists(target_path(target, 'system-data/var/lib/snapd')) |
2147 | + |
2148 | + |
2149 | +def is_centos(target=None): |
2150 | + """Check if CentOS specific file is present at target""" |
2151 | + return os.path.exists(target_path(target, 'etc/centos-release')) |
2152 | + |
2153 | + |
2154 | +def is_rhel(target=None): |
2155 | + """Check if RHEL specific file is present at target""" |
2156 | + return os.path.exists(target_path(target, 'etc/redhat-release')) |
2157 | + |
2158 | + |
2159 | +def _lsb_release(target=None): |
2160 | + fmap = {'Codename': 'codename', 'Description': 'description', |
2161 | + 'Distributor ID': 'id', 'Release': 'release'} |
2162 | + |
2163 | + data = {} |
2164 | + try: |
2165 | + out, _ = subp(['lsb_release', '--all'], capture=True, target=target) |
2166 | + for line in out.splitlines(): |
2167 | + fname, _, val = line.partition(":") |
2168 | + if fname in fmap: |
2169 | + data[fmap[fname]] = val.strip() |
2170 | + missing = [k for k in fmap.values() if k not in data] |
2171 | + if len(missing): |
2172 | + LOG.warn("Missing fields in lsb_release --all output: %s", |
2173 | + ','.join(missing)) |
2174 | + |
2175 | + except ProcessExecutionError as err: |
2176 | + LOG.warn("Unable to get lsb_release --all: %s", err) |
2177 | + data = {v: "UNAVAILABLE" for v in fmap.values()} |
2178 | + |
2179 | + return data |
2180 | + |
2181 | + |
2182 | +def apt_update(target=None, env=None, force=False, comment=None, |
2183 | + retries=None): |
2184 | + |
2185 | + marker = "tmp/curtin.aptupdate" |
2186 | + |
2187 | + if env is None: |
2188 | + env = os.environ.copy() |
2189 | + |
2190 | + if retries is None: |
2191 | + # by default run apt-update up to 3 times to allow |
2192 | + # for transient failures |
2193 | + retries = (1, 2, 3) |
2194 | + |
2195 | + if comment is None: |
2196 | + comment = "no comment provided" |
2197 | + |
2198 | + if comment.endswith("\n"): |
2199 | + comment = comment[:-1] |
2200 | + |
2201 | + marker = target_path(target, marker) |
2202 | + # if marker exists, check if there are files that would make it obsolete |
2203 | + listfiles = [target_path(target, "/etc/apt/sources.list")] |
2204 | + listfiles += glob.glob( |
2205 | + target_path(target, "etc/apt/sources.list.d/*.list")) |
2206 | + |
2207 | + if os.path.exists(marker) and not force: |
2208 | + if len(find_newer(marker, listfiles)) == 0: |
2209 | + return |
2210 | + |
2211 | + restore_perms = [] |
2212 | + |
2213 | + abs_tmpdir = tempfile.mkdtemp(dir=target_path(target, "/tmp")) |
2214 | + try: |
2215 | + abs_slist = abs_tmpdir + "/sources.list" |
2216 | + abs_slistd = abs_tmpdir + "/sources.list.d" |
2217 | + ch_tmpdir = "/tmp/" + os.path.basename(abs_tmpdir) |
2218 | + ch_slist = ch_tmpdir + "/sources.list" |
2219 | + ch_slistd = ch_tmpdir + "/sources.list.d" |
2220 | + |
2221 | + # this file gets executed on apt-get update sometimes. (LP: #1527710) |
2222 | + motd_update = target_path( |
2223 | + target, "/usr/lib/update-notifier/update-motd-updates-available") |
2224 | + pmode = set_unexecutable(motd_update) |
2225 | + if pmode is not None: |
2226 | + restore_perms.append((motd_update, pmode),) |
2227 | + |
2228 | + # create tmpdir/sources.list with all lines other than deb-src |
2229 | + # avoid apt complaining by using existing and empty dir for sourceparts |
2230 | + os.mkdir(abs_slistd) |
2231 | + with open(abs_slist, "w") as sfp: |
2232 | + for sfile in listfiles: |
2233 | + with open(sfile, "r") as fp: |
2234 | + contents = fp.read() |
2235 | + for line in contents.splitlines(): |
2236 | + line = line.lstrip() |
2237 | + if not line.startswith("deb-src"): |
2238 | + sfp.write(line + "\n") |
2239 | + |
2240 | + update_cmd = [ |
2241 | + 'apt-get', '--quiet', |
2242 | + '--option=Acquire::Languages=none', |
2243 | + '--option=Dir::Etc::sourcelist=%s' % ch_slist, |
2244 | + '--option=Dir::Etc::sourceparts=%s' % ch_slistd, |
2245 | + 'update'] |
2246 | + |
2247 | + # do not using 'run_apt_command' so we can use 'retries' to subp |
2248 | + with ChrootableTarget(target, allow_daemons=True) as inchroot: |
2249 | + inchroot.subp(update_cmd, env=env, retries=retries) |
2250 | + finally: |
2251 | + for fname, perms in restore_perms: |
2252 | + os.chmod(fname, perms) |
2253 | + if abs_tmpdir: |
2254 | + shutil.rmtree(abs_tmpdir) |
2255 | + |
2256 | + with open(marker, "w") as fp: |
2257 | + fp.write(comment + "\n") |
2258 | + |
2259 | + |
2260 | +def run_apt_command(mode, args=None, opts=None, env=None, target=None, |
2261 | + execute=True, allow_daemons=False): |
2262 | + defopts = ['--quiet', '--assume-yes', |
2263 | + '--option=Dpkg::options::=--force-unsafe-io', |
2264 | + '--option=Dpkg::Options::=--force-confold'] |
2265 | + if args is None: |
2266 | + args = [] |
2267 | + |
2268 | + if opts is None: |
2269 | + opts = [] |
2270 | + |
2271 | + if env is None: |
2272 | + env = os.environ.copy() |
2273 | + env['DEBIAN_FRONTEND'] = 'noninteractive' |
2274 | + |
2275 | + if which('eatmydata', target=target): |
2276 | + emd = ['eatmydata'] |
2277 | + else: |
2278 | + emd = [] |
2279 | + |
2280 | + cmd = emd + ['apt-get'] + defopts + opts + [mode] + args |
2281 | + if not execute: |
2282 | + return env, cmd |
2283 | + |
2284 | + apt_update(target, env=env, comment=' '.join(cmd)) |
2285 | + with ChrootableTarget(target, allow_daemons=allow_daemons) as inchroot: |
2286 | + return inchroot.subp(cmd, env=env) |
2287 | + |
2288 | + |
2289 | +def run_yum_command(mode, args=None, opts=None, env=None, target=None, |
2290 | + execute=True, allow_daemons=False): |
2291 | + defopts = ['--assumeyes', '--quiet'] |
2292 | + |
2293 | + if args is None: |
2294 | + args = [] |
2295 | + |
2296 | + if opts is None: |
2297 | + opts = [] |
2298 | + |
2299 | + cmd = ['yum'] + defopts + opts + [mode] + args |
2300 | + if not execute: |
2301 | + return env, cmd |
2302 | + |
2303 | + if mode in ["install", "update", "upgrade"]: |
2304 | + return yum_install(mode, args, opts=opts, env=env, target=target, |
2305 | + allow_daemons=allow_daemons) |
2306 | + |
2307 | + with ChrootableTarget(target, allow_daemons=allow_daemons) as inchroot: |
2308 | + return inchroot.subp(cmd, env=env) |
2309 | + |
2310 | + |
2311 | +def yum_install(mode, packages=None, opts=None, env=None, target=None, |
2312 | + allow_daemons=False): |
2313 | + |
2314 | + defopts = ['--assumeyes', '--quiet'] |
2315 | + |
2316 | + if packages is None: |
2317 | + packages = [] |
2318 | + |
2319 | + if opts is None: |
2320 | + opts = [] |
2321 | + |
2322 | + if mode not in ['install', 'update', 'upgrade']: |
2323 | + raise ValueError( |
2324 | + 'Unsupported mode "%s" for yum package install/upgrade' % mode) |
2325 | + |
2326 | + # download first, then install/upgrade from cache |
2327 | + cmd = ['yum'] + defopts + opts + [mode] |
2328 | + dl_opts = ['--downloadonly', '--setopt=keepcache=1'] |
2329 | + inst_opts = ['--cacheonly'] |
2330 | + |
2331 | + # rpm requires /dev /sys and /proc be mounted, use ChrootableTarget |
2332 | + with ChrootableTarget(target, allow_daemons=allow_daemons) as inchroot: |
2333 | + inchroot.subp(cmd + dl_opts + packages, |
2334 | + env=env, retries=[1] * 10) |
2335 | + return inchroot.subp(cmd + inst_opts + packages, env=env) |
2336 | + |
2337 | + |
2338 | +def rpm_get_dist_id(target=None): |
2339 | + """Use rpm command to extract the '%rhel' distro macro which returns |
2340 | + the major os version id (6, 7, 8). This works for centos or rhel |
2341 | + """ |
2342 | + with ChrootableTarget(target) as in_chroot: |
2343 | + dist, _ = in_chroot.subp(['rpm', '-E', '%rhel'], capture=True) |
2344 | + return dist.rstrip() |
2345 | + |
2346 | + |
2347 | +def system_upgrade(opts=None, target=None, env=None, allow_daemons=False, |
2348 | + osfamily=None): |
2349 | + LOG.debug("Upgrading system in %s", target) |
2350 | + |
2351 | + distro_cfg = { |
2352 | + DISTROS.debian: {'function': 'run_apt_command', |
2353 | + 'subcommands': ('dist-upgrade', 'autoremove')}, |
2354 | + DISTROS.redhat: {'function': 'run_yum_command', |
2355 | + 'subcommands': ('upgrade')}, |
2356 | + } |
2357 | + if osfamily not in distro_cfg: |
2358 | + raise ValueError('Distro "%s" does not have system_upgrade support', |
2359 | + osfamily) |
2360 | + |
2361 | + for mode in distro_cfg[osfamily]['subcommands']: |
2362 | + ret = distro_cfg[osfamily]['function']( |
2363 | + mode, opts=opts, target=target, |
2364 | + env=env, allow_daemons=allow_daemons) |
2365 | + return ret |
2366 | + |
2367 | + |
2368 | +def install_packages(pkglist, osfamily=None, opts=None, target=None, env=None, |
2369 | + allow_daemons=False): |
2370 | + if isinstance(pkglist, str): |
2371 | + pkglist = [pkglist] |
2372 | + |
2373 | + if not osfamily: |
2374 | + osfamily = get_osfamily(target=target) |
2375 | + |
2376 | + installer_map = { |
2377 | + DISTROS.debian: run_apt_command, |
2378 | + DISTROS.redhat: run_yum_command, |
2379 | + } |
2380 | + |
2381 | + install_cmd = installer_map.get(osfamily) |
2382 | + if not install_cmd: |
2383 | + raise ValueError('No packge install command for distro: %s' % |
2384 | + osfamily) |
2385 | + |
2386 | + return install_cmd('install', args=pkglist, opts=opts, target=target, |
2387 | + env=env, allow_daemons=allow_daemons) |
2388 | + |
2389 | + |
2390 | +def has_pkg_available(pkg, target=None, osfamily=None): |
2391 | + if not osfamily: |
2392 | + osfamily = get_osfamily(target=target) |
2393 | + |
2394 | + if osfamily not in [DISTROS.debian, DISTROS.redhat]: |
2395 | + raise ValueError('has_pkg_available: unsupported distro family: %s', |
2396 | + osfamily) |
2397 | + |
2398 | + if osfamily == DISTROS.debian: |
2399 | + out, _ = subp(['apt-cache', 'pkgnames'], capture=True, target=target) |
2400 | + for item in out.splitlines(): |
2401 | + if pkg == item.strip(): |
2402 | + return True |
2403 | + return False |
2404 | + |
2405 | + if osfamily == DISTROS.redhat: |
2406 | + out, _ = run_yum_command('list', opts=['--cacheonly']) |
2407 | + for item in out.splitlines(): |
2408 | + if item.lower().startswith(pkg.lower()): |
2409 | + return True |
2410 | + return False |
2411 | + |
2412 | + |
2413 | +def get_installed_packages(target=None): |
2414 | + if which('dpkg-query', target=target): |
2415 | + (out, _) = subp(['dpkg-query', '--list'], target=target, capture=True) |
2416 | + elif which('rpm', target=target): |
2417 | + # rpm requires /dev /sys and /proc be mounted, use ChrootableTarget |
2418 | + with ChrootableTarget(target) as in_chroot: |
2419 | + (out, _) = in_chroot.subp(['rpm', '-qa', '--queryformat', |
2420 | + 'ii %{NAME} %{VERSION}-%{RELEASE}\n'], |
2421 | + target=target, capture=True) |
2422 | + if not out: |
2423 | + raise ValueError('No package query tool') |
2424 | + |
2425 | + pkgs_inst = set() |
2426 | + for line in out.splitlines(): |
2427 | + try: |
2428 | + (state, pkg, other) = line.split(None, 2) |
2429 | + except ValueError: |
2430 | + continue |
2431 | + if state.startswith("hi") or state.startswith("ii"): |
2432 | + pkgs_inst.add(re.sub(":.*", "", pkg)) |
2433 | + |
2434 | + return pkgs_inst |
2435 | + |
2436 | + |
2437 | +def has_pkg_installed(pkg, target=None): |
2438 | + try: |
2439 | + out, _ = subp(['dpkg-query', '--show', '--showformat', |
2440 | + '${db:Status-Abbrev}', pkg], |
2441 | + capture=True, target=target) |
2442 | + return out.rstrip() == "ii" |
2443 | + except ProcessExecutionError: |
2444 | + return False |
2445 | + |
2446 | + |
2447 | +def parse_dpkg_version(raw, name=None, semx=None): |
2448 | + """Parse a dpkg version string into various parts and calcualate a |
2449 | + numerical value of the version for use in comparing package versions |
2450 | + |
2451 | + Native packages (without a '-'), will have the package version treated |
2452 | + as the upstream version. |
2453 | + |
2454 | + returns a dictionary with fields: |
2455 | + 'major' (int), 'minor' (int), 'micro' (int), |
2456 | + 'semantic_version' (int), |
2457 | + 'extra' (string), 'raw' (string), 'upstream' (string), |
2458 | + 'name' (present only if name is not None) |
2459 | + """ |
2460 | + if not isinstance(raw, string_types): |
2461 | + raise TypeError( |
2462 | + "Invalid type %s for parse_dpkg_version" % raw.__class__) |
2463 | + |
2464 | + if semx is None: |
2465 | + semx = (10000, 100, 1) |
2466 | + |
2467 | + if "-" in raw: |
2468 | + upstream = raw.rsplit('-', 1)[0] |
2469 | + else: |
2470 | + # this is a native package, package version treated as upstream. |
2471 | + upstream = raw |
2472 | + |
2473 | + match = re.search(r'[^0-9.]', upstream) |
2474 | + if match: |
2475 | + extra = upstream[match.start():] |
2476 | + upstream_base = upstream[:match.start()] |
2477 | + else: |
2478 | + upstream_base = upstream |
2479 | + extra = None |
2480 | + |
2481 | + toks = upstream_base.split(".", 2) |
2482 | + if len(toks) == 3: |
2483 | + major, minor, micro = toks |
2484 | + elif len(toks) == 2: |
2485 | + major, minor, micro = (toks[0], toks[1], 0) |
2486 | + elif len(toks) == 1: |
2487 | + major, minor, micro = (toks[0], 0, 0) |
2488 | + |
2489 | + version = { |
2490 | + 'major': int(major), |
2491 | + 'minor': int(minor), |
2492 | + 'micro': int(micro), |
2493 | + 'extra': extra, |
2494 | + 'raw': raw, |
2495 | + 'upstream': upstream, |
2496 | + } |
2497 | + if name: |
2498 | + version['name'] = name |
2499 | + |
2500 | + if semx: |
2501 | + try: |
2502 | + version['semantic_version'] = int( |
2503 | + int(major) * semx[0] + int(minor) * semx[1] + |
2504 | + int(micro) * semx[2]) |
2505 | + except (ValueError, IndexError): |
2506 | + version['semantic_version'] = None |
2507 | + |
2508 | + return version |
2509 | + |
2510 | + |
2511 | +def get_package_version(pkg, target=None, semx=None): |
2512 | + """Use dpkg-query to extract package pkg's version string |
2513 | + and parse the version string into a dictionary |
2514 | + """ |
2515 | + try: |
2516 | + out, _ = subp(['dpkg-query', '--show', '--showformat', |
2517 | + '${Version}', pkg], capture=True, target=target) |
2518 | + raw = out.rstrip() |
2519 | + return parse_dpkg_version(raw, name=pkg, semx=semx) |
2520 | + except ProcessExecutionError: |
2521 | + return None |
2522 | + |
2523 | + |
2524 | +# vi: ts=4 expandtab syntax=python |
2525 | diff --git a/curtin/futil.py b/curtin/futil.py |
2526 | index 506964e..e603f88 100644 |
2527 | --- a/curtin/futil.py |
2528 | +++ b/curtin/futil.py |
2529 | @@ -5,7 +5,8 @@ import pwd |
2530 | import os |
2531 | import warnings |
2532 | |
2533 | -from .util import write_file, target_path |
2534 | +from .util import write_file |
2535 | +from .paths import target_path |
2536 | from .log import LOG |
2537 | |
2538 | |
2539 | diff --git a/curtin/log.py b/curtin/log.py |
2540 | index 4844460..446ba2c 100644 |
2541 | --- a/curtin/log.py |
2542 | +++ b/curtin/log.py |
2543 | @@ -1,6 +1,9 @@ |
2544 | # This file is part of curtin. See LICENSE file for copyright and license info. |
2545 | |
2546 | import logging |
2547 | +import time |
2548 | + |
2549 | +from functools import wraps |
2550 | |
2551 | # Logging items for easy access |
2552 | getLogger = logging.getLogger |
2553 | @@ -56,6 +59,46 @@ def _getLogger(name='curtin'): |
2554 | if not logging.getLogger().handlers: |
2555 | logging.getLogger().addHandler(NullHandler()) |
2556 | |
2557 | + |
2558 | +def _repr_call(name, *args, **kwargs): |
2559 | + return "%s(%s)" % ( |
2560 | + name, |
2561 | + ', '.join([str(repr(a)) for a in args] + |
2562 | + ["%s=%s" % (k, repr(v)) for k, v in kwargs.items()])) |
2563 | + |
2564 | + |
2565 | +def log_call(func, *args, **kwargs): |
2566 | + return log_time( |
2567 | + "TIMED %s: " % _repr_call(func.__name__, *args, **kwargs), |
2568 | + func, *args, **kwargs) |
2569 | + |
2570 | + |
2571 | +def log_time(msg, func, *args, **kwargs): |
2572 | + start = time.time() |
2573 | + try: |
2574 | + return func(*args, **kwargs) |
2575 | + finally: |
2576 | + LOG.debug(msg + "%.3f", (time.time() - start)) |
2577 | + |
2578 | + |
2579 | +def logged_call(): |
2580 | + def decorator(func): |
2581 | + @wraps(func) |
2582 | + def wrapper(*args, **kwargs): |
2583 | + return log_call(func, *args, **kwargs) |
2584 | + return wrapper |
2585 | + return decorator |
2586 | + |
2587 | + |
2588 | +def logged_time(msg): |
2589 | + def decorator(func): |
2590 | + @wraps(func) |
2591 | + def wrapper(*args, **kwargs): |
2592 | + return log_time("TIMED %s: " % msg, func, *args, **kwargs) |
2593 | + return wrapper |
2594 | + return decorator |
2595 | + |
2596 | + |
2597 | LOG = _getLogger() |
2598 | |
2599 | # vi: ts=4 expandtab syntax=python |
2600 | diff --git a/curtin/net/__init__.py b/curtin/net/__init__.py |
2601 | index b4c9b59..ef2ba26 100644 |
2602 | --- a/curtin/net/__init__.py |
2603 | +++ b/curtin/net/__init__.py |
2604 | @@ -572,63 +572,4 @@ def get_interface_mac(ifname): |
2605 | return read_sys_net(ifname, "address", enoent=False) |
2606 | |
2607 | |
2608 | -def network_config_required_packages(network_config, mapping=None): |
2609 | - |
2610 | - if network_config is None: |
2611 | - network_config = {} |
2612 | - |
2613 | - if not isinstance(network_config, dict): |
2614 | - raise ValueError('Invalid network configuration. Must be a dict') |
2615 | - |
2616 | - if mapping is None: |
2617 | - mapping = {} |
2618 | - |
2619 | - if not isinstance(mapping, dict): |
2620 | - raise ValueError('Invalid network mapping. Must be a dict') |
2621 | - |
2622 | - # allow top-level 'network' key |
2623 | - if 'network' in network_config: |
2624 | - network_config = network_config.get('network') |
2625 | - |
2626 | - # v1 has 'config' key and uses type: devtype elements |
2627 | - if 'config' in network_config: |
2628 | - dev_configs = set(device['type'] |
2629 | - for device in network_config['config']) |
2630 | - else: |
2631 | - # v2 has no config key |
2632 | - dev_configs = set(cfgtype for (cfgtype, cfg) in |
2633 | - network_config.items() if cfgtype not in ['version']) |
2634 | - |
2635 | - needed_packages = [] |
2636 | - for dev_type in dev_configs: |
2637 | - if dev_type in mapping: |
2638 | - needed_packages.extend(mapping[dev_type]) |
2639 | - |
2640 | - return needed_packages |
2641 | - |
2642 | - |
2643 | -def detect_required_packages_mapping(): |
2644 | - """Return a dictionary providing a versioned configuration which maps |
2645 | - network configuration elements to the packages which are required |
2646 | - for functionality. |
2647 | - """ |
2648 | - mapping = { |
2649 | - 1: { |
2650 | - 'handler': network_config_required_packages, |
2651 | - 'mapping': { |
2652 | - 'bond': ['ifenslave'], |
2653 | - 'bridge': ['bridge-utils'], |
2654 | - 'vlan': ['vlan']}, |
2655 | - }, |
2656 | - 2: { |
2657 | - 'handler': network_config_required_packages, |
2658 | - 'mapping': { |
2659 | - 'bonds': ['ifenslave'], |
2660 | - 'bridges': ['bridge-utils'], |
2661 | - 'vlans': ['vlan']} |
2662 | - }, |
2663 | - } |
2664 | - |
2665 | - return mapping |
2666 | - |
2667 | # vi: ts=4 expandtab syntax=python |
2668 | diff --git a/curtin/net/deps.py b/curtin/net/deps.py |
2669 | new file mode 100644 |
2670 | index 0000000..b98961d |
2671 | --- /dev/null |
2672 | +++ b/curtin/net/deps.py |
2673 | @@ -0,0 +1,72 @@ |
2674 | +# This file is part of curtin. See LICENSE file for copyright and license info. |
2675 | + |
2676 | +from curtin.distro import DISTROS |
2677 | + |
2678 | + |
2679 | +def network_config_required_packages(network_config, mapping=None): |
2680 | + |
2681 | + if network_config is None: |
2682 | + network_config = {} |
2683 | + |
2684 | + if not isinstance(network_config, dict): |
2685 | + raise ValueError('Invalid network configuration. Must be a dict') |
2686 | + |
2687 | + if mapping is None: |
2688 | + mapping = {} |
2689 | + |
2690 | + if not isinstance(mapping, dict): |
2691 | + raise ValueError('Invalid network mapping. Must be a dict') |
2692 | + |
2693 | + # allow top-level 'network' key |
2694 | + if 'network' in network_config: |
2695 | + network_config = network_config.get('network') |
2696 | + |
2697 | + # v1 has 'config' key and uses type: devtype elements |
2698 | + if 'config' in network_config: |
2699 | + dev_configs = set(device['type'] |
2700 | + for device in network_config['config']) |
2701 | + else: |
2702 | + # v2 has no config key |
2703 | + dev_configs = set(cfgtype for (cfgtype, cfg) in |
2704 | + network_config.items() if cfgtype not in ['version']) |
2705 | + |
2706 | + needed_packages = [] |
2707 | + for dev_type in dev_configs: |
2708 | + if dev_type in mapping: |
2709 | + needed_packages.extend(mapping[dev_type]) |
2710 | + |
2711 | + return needed_packages |
2712 | + |
2713 | + |
2714 | +def detect_required_packages_mapping(osfamily=DISTROS.debian): |
2715 | + """Return a dictionary providing a versioned configuration which maps |
2716 | + network configuration elements to the packages which are required |
2717 | + for functionality. |
2718 | + """ |
2719 | + # keys ending with 's' are v2 values |
2720 | + distro_mapping = { |
2721 | + DISTROS.debian: { |
2722 | + 'bond': ['ifenslave'], |
2723 | + 'bonds': [], |
2724 | + 'bridge': ['bridge-utils'], |
2725 | + 'bridges': [], |
2726 | + 'vlan': ['vlan'], |
2727 | + 'vlans': []}, |
2728 | + DISTROS.redhat: { |
2729 | + 'bond': [], |
2730 | + 'bonds': [], |
2731 | + 'bridge': [], |
2732 | + 'bridges': [], |
2733 | + 'vlan': [], |
2734 | + 'vlans': []}, |
2735 | + } |
2736 | + if osfamily not in distro_mapping: |
2737 | + raise ValueError('No net package mapping for distro: %s' % osfamily) |
2738 | + |
2739 | + return {1: {'handler': network_config_required_packages, |
2740 | + 'mapping': distro_mapping.get(osfamily)}, |
2741 | + 2: {'handler': network_config_required_packages, |
2742 | + 'mapping': distro_mapping.get(osfamily)}} |
2743 | + |
2744 | + |
2745 | +# vi: ts=4 expandtab syntax=python |
2746 | diff --git a/curtin/paths.py b/curtin/paths.py |
2747 | new file mode 100644 |
2748 | index 0000000..064b060 |
2749 | --- /dev/null |
2750 | +++ b/curtin/paths.py |
2751 | @@ -0,0 +1,34 @@ |
2752 | +# This file is part of curtin. See LICENSE file for copyright and license info. |
2753 | +import os |
2754 | + |
2755 | +try: |
2756 | + string_types = (basestring,) |
2757 | +except NameError: |
2758 | + string_types = (str,) |
2759 | + |
2760 | + |
2761 | +def target_path(target, path=None): |
2762 | + # return 'path' inside target, accepting target as None |
2763 | + if target in (None, ""): |
2764 | + target = "/" |
2765 | + elif not isinstance(target, string_types): |
2766 | + raise ValueError("Unexpected input for target: %s" % target) |
2767 | + else: |
2768 | + target = os.path.abspath(target) |
2769 | + # abspath("//") returns "//" specifically for 2 slashes. |
2770 | + if target.startswith("//"): |
2771 | + target = target[1:] |
2772 | + |
2773 | + if not path: |
2774 | + return target |
2775 | + |
2776 | + if not isinstance(path, string_types): |
2777 | + raise ValueError("Unexpected input for path: %s" % path) |
2778 | + |
2779 | + # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /. |
2780 | + while len(path) and path[0] == "/": |
2781 | + path = path[1:] |
2782 | + |
2783 | + return os.path.join(target, path) |
2784 | + |
2785 | +# vi: ts=4 expandtab syntax=python |
2786 | diff --git a/curtin/udev.py b/curtin/udev.py |
2787 | index 92e38ff..13d9cc5 100644 |
2788 | --- a/curtin/udev.py |
2789 | +++ b/curtin/udev.py |
2790 | @@ -2,6 +2,7 @@ |
2791 | |
2792 | import os |
2793 | from curtin import util |
2794 | +from curtin.log import logged_call |
2795 | |
2796 | |
2797 | def compose_udev_equality(key, value): |
2798 | @@ -40,6 +41,7 @@ def generate_udev_rule(interface, mac): |
2799 | return '%s\n' % rule |
2800 | |
2801 | |
2802 | +@logged_call() |
2803 | def udevadm_settle(exists=None, timeout=None): |
2804 | settle_cmd = ["udevadm", "settle"] |
2805 | if exists: |
2806 | diff --git a/curtin/url_helper.py b/curtin/url_helper.py |
2807 | index d4d43a9..43c5c36 100644 |
2808 | --- a/curtin/url_helper.py |
2809 | +++ b/curtin/url_helper.py |
2810 | @@ -227,7 +227,7 @@ def geturl(url, headers=None, headers_cb=None, exception_cb=None, |
2811 | try: |
2812 | return _geturl(url=url, headers=headers, headers_cb=headers_cb, |
2813 | exception_cb=exception_cb, data=data) |
2814 | - except _ReRaisedException as e: |
2815 | + except _ReRaisedException: |
2816 | raise curexc.exc |
2817 | except Exception as e: |
2818 | curexc = e |
2819 | diff --git a/curtin/util.py b/curtin/util.py |
2820 | index de0eb88..238d7c5 100644 |
2821 | --- a/curtin/util.py |
2822 | +++ b/curtin/util.py |
2823 | @@ -4,7 +4,6 @@ import argparse |
2824 | import collections |
2825 | from contextlib import contextmanager |
2826 | import errno |
2827 | -import glob |
2828 | import json |
2829 | import os |
2830 | import platform |
2831 | @@ -38,15 +37,16 @@ except NameError: |
2832 | # python3 does not have a long type. |
2833 | numeric_types = (int, float) |
2834 | |
2835 | -from .log import LOG |
2836 | +from . import paths |
2837 | +from .log import LOG, log_call |
2838 | |
2839 | _INSTALLED_HELPERS_PATH = 'usr/lib/curtin/helpers' |
2840 | _INSTALLED_MAIN = 'usr/bin/curtin' |
2841 | |
2842 | -_LSB_RELEASE = {} |
2843 | _USES_SYSTEMD = None |
2844 | _HAS_UNSHARE_PID = None |
2845 | |
2846 | + |
2847 | _DNS_REDIRECT_IP = None |
2848 | |
2849 | # matcher used in template rendering functions |
2850 | @@ -61,7 +61,7 @@ def _subp(args, data=None, rcs=None, env=None, capture=False, |
2851 | rcs = [0] |
2852 | devnull_fp = None |
2853 | |
2854 | - tpath = target_path(target) |
2855 | + tpath = paths.target_path(target) |
2856 | chroot_args = [] if tpath == "/" else ['chroot', target] |
2857 | sh_args = ['sh', '-c'] if shell else [] |
2858 | if isinstance(args, string_types): |
2859 | @@ -103,10 +103,11 @@ def _subp(args, data=None, rcs=None, env=None, capture=False, |
2860 | (out, err) = sp.communicate(data) |
2861 | |
2862 | # Just ensure blank instead of none. |
2863 | - if not out and capture: |
2864 | - out = b'' |
2865 | - if not err and capture: |
2866 | - err = b'' |
2867 | + if capture or combine_capture: |
2868 | + if not out: |
2869 | + out = b'' |
2870 | + if not err: |
2871 | + err = b'' |
2872 | if decode: |
2873 | def ldecode(data, m='utf-8'): |
2874 | if not isinstance(data, bytes): |
2875 | @@ -164,7 +165,7 @@ def _get_unshare_pid_args(unshare_pid=None, target=None, euid=None): |
2876 | if euid is None: |
2877 | euid = os.geteuid() |
2878 | |
2879 | - tpath = target_path(target) |
2880 | + tpath = paths.target_path(target) |
2881 | |
2882 | unshare_pid_in = unshare_pid |
2883 | if unshare_pid is None: |
2884 | @@ -206,6 +207,8 @@ def subp(*args, **kwargs): |
2885 | boolean indicating if stderr should be redirected to stdout. When True, |
2886 | interleaved stderr and stdout will be returned as the first element of |
2887 | a tuple. |
2888 | + if combine_capture is True, then output is captured independent of |
2889 | + the value of capture. |
2890 | :param log_captured: |
2891 | boolean indicating if output should be logged on capture. If |
2892 | True, then stderr and stdout will be logged at DEBUG level. If |
2893 | @@ -521,6 +524,8 @@ def do_umount(mountpoint, recursive=False): |
2894 | |
2895 | |
2896 | def ensure_dir(path, mode=None): |
2897 | + if path == "": |
2898 | + path = "." |
2899 | try: |
2900 | os.makedirs(path) |
2901 | except OSError as e: |
2902 | @@ -590,7 +595,7 @@ def disable_daemons_in_root(target): |
2903 | 'done', |
2904 | '']) |
2905 | |
2906 | - fpath = target_path(target, "/usr/sbin/policy-rc.d") |
2907 | + fpath = paths.target_path(target, "/usr/sbin/policy-rc.d") |
2908 | |
2909 | if os.path.isfile(fpath): |
2910 | return False |
2911 | @@ -601,7 +606,7 @@ def disable_daemons_in_root(target): |
2912 | |
2913 | def undisable_daemons_in_root(target): |
2914 | try: |
2915 | - os.unlink(target_path(target, "/usr/sbin/policy-rc.d")) |
2916 | + os.unlink(paths.target_path(target, "/usr/sbin/policy-rc.d")) |
2917 | except OSError as e: |
2918 | if e.errno != errno.ENOENT: |
2919 | raise |
2920 | @@ -613,7 +618,7 @@ class ChrootableTarget(object): |
2921 | def __init__(self, target, allow_daemons=False, sys_resolvconf=True): |
2922 | if target is None: |
2923 | target = "/" |
2924 | - self.target = target_path(target) |
2925 | + self.target = paths.target_path(target) |
2926 | self.mounts = ["/dev", "/proc", "/sys"] |
2927 | self.umounts = [] |
2928 | self.disabled_daemons = False |
2929 | @@ -623,14 +628,14 @@ class ChrootableTarget(object): |
2930 | |
2931 | def __enter__(self): |
2932 | for p in self.mounts: |
2933 | - tpath = target_path(self.target, p) |
2934 | + tpath = paths.target_path(self.target, p) |
2935 | if do_mount(p, tpath, opts='--bind'): |
2936 | self.umounts.append(tpath) |
2937 | |
2938 | if not self.allow_daemons: |
2939 | self.disabled_daemons = disable_daemons_in_root(self.target) |
2940 | |
2941 | - rconf = target_path(self.target, "/etc/resolv.conf") |
2942 | + rconf = paths.target_path(self.target, "/etc/resolv.conf") |
2943 | target_etc = os.path.dirname(rconf) |
2944 | if self.target != "/" and os.path.isdir(target_etc): |
2945 | # never muck with resolv.conf on / |
2946 | @@ -655,13 +660,13 @@ class ChrootableTarget(object): |
2947 | undisable_daemons_in_root(self.target) |
2948 | |
2949 | # if /dev is to be unmounted, udevadm settle (LP: #1462139) |
2950 | - if target_path(self.target, "/dev") in self.umounts: |
2951 | - subp(['udevadm', 'settle']) |
2952 | + if paths.target_path(self.target, "/dev") in self.umounts: |
2953 | + log_call(subp, ['udevadm', 'settle']) |
2954 | |
2955 | for p in reversed(self.umounts): |
2956 | do_umount(p) |
2957 | |
2958 | - rconf = target_path(self.target, "/etc/resolv.conf") |
2959 | + rconf = paths.target_path(self.target, "/etc/resolv.conf") |
2960 | if self.sys_resolvconf and self.rconf_d: |
2961 | os.rename(os.path.join(self.rconf_d, "resolv.conf"), rconf) |
2962 | shutil.rmtree(self.rconf_d) |
2963 | @@ -671,7 +676,7 @@ class ChrootableTarget(object): |
2964 | return subp(*args, **kwargs) |
2965 | |
2966 | def path(self, path): |
2967 | - return target_path(self.target, path) |
2968 | + return paths.target_path(self.target, path) |
2969 | |
2970 | |
2971 | def is_exe(fpath): |
2972 | @@ -680,29 +685,29 @@ def is_exe(fpath): |
2973 | |
2974 | |
2975 | def which(program, search=None, target=None): |
2976 | - target = target_path(target) |
2977 | + target = paths.target_path(target) |
2978 | |
2979 | if os.path.sep in program: |
2980 | # if program had a '/' in it, then do not search PATH |
2981 | # 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls |
2982 | # so effectively we set cwd to / (or target) |
2983 | - if is_exe(target_path(target, program)): |
2984 | + if is_exe(paths.target_path(target, program)): |
2985 | return program |
2986 | |
2987 | if search is None: |
2988 | - paths = [p.strip('"') for p in |
2989 | - os.environ.get("PATH", "").split(os.pathsep)] |
2990 | + candpaths = [p.strip('"') for p in |
2991 | + os.environ.get("PATH", "").split(os.pathsep)] |
2992 | if target == "/": |
2993 | - search = paths |
2994 | + search = candpaths |
2995 | else: |
2996 | - search = [p for p in paths if p.startswith("/")] |
2997 | + search = [p for p in candpaths if p.startswith("/")] |
2998 | |
2999 | # normalize path input |
3000 | search = [os.path.abspath(p) for p in search] |
3001 | |
3002 | for path in search: |
3003 | ppath = os.path.sep.join((path, program)) |
3004 | - if is_exe(target_path(target, ppath)): |
3005 | + if is_exe(paths.target_path(target, ppath)): |
3006 | return ppath |
3007 | |
3008 | return None |
3009 | @@ -768,91 +773,6 @@ def get_architecture(target=None): |
3010 | return out.strip() |
3011 | |
3012 | |
3013 | -def has_pkg_available(pkg, target=None): |
3014 | - out, _ = subp(['apt-cache', 'pkgnames'], capture=True, target=target) |
3015 | - for item in out.splitlines(): |
3016 | - if pkg == item.strip(): |
3017 | - return True |
3018 | - return False |
3019 | - |
3020 | - |
3021 | -def get_installed_packages(target=None): |
3022 | - (out, _) = subp(['dpkg-query', '--list'], target=target, capture=True) |
3023 | - |
3024 | - pkgs_inst = set() |
3025 | - for line in out.splitlines(): |
3026 | - try: |
3027 | - (state, pkg, other) = line.split(None, 2) |
3028 | - except ValueError: |
3029 | - continue |
3030 | - if state.startswith("hi") or state.startswith("ii"): |
3031 | - pkgs_inst.add(re.sub(":.*", "", pkg)) |
3032 | - |
3033 | - return pkgs_inst |
3034 | - |
3035 | - |
3036 | -def has_pkg_installed(pkg, target=None): |
3037 | - try: |
3038 | - out, _ = subp(['dpkg-query', '--show', '--showformat', |
3039 | - '${db:Status-Abbrev}', pkg], |
3040 | - capture=True, target=target) |
3041 | - return out.rstrip() == "ii" |
3042 | - except ProcessExecutionError: |
3043 | - return False |
3044 | - |
3045 | - |
3046 | -def parse_dpkg_version(raw, name=None, semx=None): |
3047 | - """Parse a dpkg version string into various parts and calcualate a |
3048 | - numerical value of the version for use in comparing package versions |
3049 | - |
3050 | - returns a dictionary with the results |
3051 | - """ |
3052 | - if semx is None: |
3053 | - semx = (10000, 100, 1) |
3054 | - |
3055 | - upstream = raw.split('-')[0] |
3056 | - toks = upstream.split(".", 2) |
3057 | - if len(toks) == 3: |
3058 | - major, minor, micro = toks |
3059 | - elif len(toks) == 2: |
3060 | - major, minor, micro = (toks[0], toks[1], 0) |
3061 | - elif len(toks) == 1: |
3062 | - major, minor, micro = (toks[0], 0, 0) |
3063 | - |
3064 | - version = { |
3065 | - 'major': major, |
3066 | - 'minor': minor, |
3067 | - 'micro': micro, |
3068 | - 'raw': raw, |
3069 | - 'upstream': upstream, |
3070 | - } |
3071 | - if name: |
3072 | - version['name'] = name |
3073 | - |
3074 | - if semx: |
3075 | - try: |
3076 | - version['semantic_version'] = int( |
3077 | - int(major) * semx[0] + int(minor) * semx[1] + |
3078 | - int(micro) * semx[2]) |
3079 | - except (ValueError, IndexError): |
3080 | - version['semantic_version'] = None |
3081 | - |
3082 | - return version |
3083 | - |
3084 | - |
3085 | -def get_package_version(pkg, target=None, semx=None): |
3086 | - """Use dpkg-query to extract package pkg's version string |
3087 | - and parse the version string into a dictionary |
3088 | - """ |
3089 | - try: |
3090 | - out, _ = subp(['dpkg-query', '--show', '--showformat', |
3091 | - '${Version}', pkg], capture=True, target=target) |
3092 | - raw = out.rstrip() |
3093 | - return parse_dpkg_version(raw, name=pkg, semx=semx) |
3094 | - except ProcessExecutionError: |
3095 | - return None |
3096 | - |
3097 | - |
3098 | def find_newer(src, files): |
3099 | mtime = os.stat(src).st_mtime |
3100 | return [f for f in files if |
3101 | @@ -877,134 +797,6 @@ def set_unexecutable(fname, strict=False): |
3102 | return cur |
3103 | |
3104 | |
3105 | -def apt_update(target=None, env=None, force=False, comment=None, |
3106 | - retries=None): |
3107 | - |
3108 | - marker = "tmp/curtin.aptupdate" |
3109 | - if target is None: |
3110 | - target = "/" |
3111 | - |
3112 | - if env is None: |
3113 | - env = os.environ.copy() |
3114 | - |
3115 | - if retries is None: |
3116 | - # by default run apt-update up to 3 times to allow |
3117 | - # for transient failures |
3118 | - retries = (1, 2, 3) |
3119 | - |
3120 | - if comment is None: |
3121 | - comment = "no comment provided" |
3122 | - |
3123 | - if comment.endswith("\n"): |
3124 | - comment = comment[:-1] |
3125 | - |
3126 | - marker = target_path(target, marker) |
3127 | - # if marker exists, check if there are files that would make it obsolete |
3128 | - listfiles = [target_path(target, "/etc/apt/sources.list")] |
3129 | - listfiles += glob.glob( |
3130 | - target_path(target, "etc/apt/sources.list.d/*.list")) |
3131 | - |
3132 | - if os.path.exists(marker) and not force: |
3133 | - if len(find_newer(marker, listfiles)) == 0: |
3134 | - return |
3135 | - |
3136 | - restore_perms = [] |
3137 | - |
3138 | - abs_tmpdir = tempfile.mkdtemp(dir=target_path(target, "/tmp")) |
3139 | - try: |
3140 | - abs_slist = abs_tmpdir + "/sources.list" |
3141 | - abs_slistd = abs_tmpdir + "/sources.list.d" |
3142 | - ch_tmpdir = "/tmp/" + os.path.basename(abs_tmpdir) |
3143 | - ch_slist = ch_tmpdir + "/sources.list" |
3144 | - ch_slistd = ch_tmpdir + "/sources.list.d" |
3145 | - |
3146 | - # this file gets executed on apt-get update sometimes. (LP: #1527710) |
3147 | - motd_update = target_path( |
3148 | - target, "/usr/lib/update-notifier/update-motd-updates-available") |
3149 | - pmode = set_unexecutable(motd_update) |
3150 | - if pmode is not None: |
3151 | - restore_perms.append((motd_update, pmode),) |
3152 | - |
3153 | - # create tmpdir/sources.list with all lines other than deb-src |
3154 | - # avoid apt complaining by using existing and empty dir for sourceparts |
3155 | - os.mkdir(abs_slistd) |
3156 | - with open(abs_slist, "w") as sfp: |
3157 | - for sfile in listfiles: |
3158 | - with open(sfile, "r") as fp: |
3159 | - contents = fp.read() |
3160 | - for line in contents.splitlines(): |
3161 | - line = line.lstrip() |
3162 | - if not line.startswith("deb-src"): |
3163 | - sfp.write(line + "\n") |
3164 | - |
3165 | - update_cmd = [ |
3166 | - 'apt-get', '--quiet', |
3167 | - '--option=Acquire::Languages=none', |
3168 | - '--option=Dir::Etc::sourcelist=%s' % ch_slist, |
3169 | - '--option=Dir::Etc::sourceparts=%s' % ch_slistd, |
3170 | - 'update'] |
3171 | - |
3172 | - # do not using 'run_apt_command' so we can use 'retries' to subp |
3173 | - with ChrootableTarget(target, allow_daemons=True) as inchroot: |
3174 | - inchroot.subp(update_cmd, env=env, retries=retries) |
3175 | - finally: |
3176 | - for fname, perms in restore_perms: |
3177 | - os.chmod(fname, perms) |
3178 | - if abs_tmpdir: |
3179 | - shutil.rmtree(abs_tmpdir) |
3180 | - |
3181 | - with open(marker, "w") as fp: |
3182 | - fp.write(comment + "\n") |
3183 | - |
3184 | - |
3185 | -def run_apt_command(mode, args=None, aptopts=None, env=None, target=None, |
3186 | - execute=True, allow_daemons=False): |
3187 | - opts = ['--quiet', '--assume-yes', |
3188 | - '--option=Dpkg::options::=--force-unsafe-io', |
3189 | - '--option=Dpkg::Options::=--force-confold'] |
3190 | - |
3191 | - if args is None: |
3192 | - args = [] |
3193 | - |
3194 | - if aptopts is None: |
3195 | - aptopts = [] |
3196 | - |
3197 | - if env is None: |
3198 | - env = os.environ.copy() |
3199 | - env['DEBIAN_FRONTEND'] = 'noninteractive' |
3200 | - |
3201 | - if which('eatmydata', target=target): |
3202 | - emd = ['eatmydata'] |
3203 | - else: |
3204 | - emd = [] |
3205 | - |
3206 | - cmd = emd + ['apt-get'] + opts + aptopts + [mode] + args |
3207 | - if not execute: |
3208 | - return env, cmd |
3209 | - |
3210 | - apt_update(target, env=env, comment=' '.join(cmd)) |
3211 | - with ChrootableTarget(target, allow_daemons=allow_daemons) as inchroot: |
3212 | - return inchroot.subp(cmd, env=env) |
3213 | - |
3214 | - |
3215 | -def system_upgrade(aptopts=None, target=None, env=None, allow_daemons=False): |
3216 | - LOG.debug("Upgrading system in %s", target) |
3217 | - for mode in ('dist-upgrade', 'autoremove'): |
3218 | - ret = run_apt_command( |
3219 | - mode, aptopts=aptopts, target=target, |
3220 | - env=env, allow_daemons=allow_daemons) |
3221 | - return ret |
3222 | - |
3223 | - |
3224 | -def install_packages(pkglist, aptopts=None, target=None, env=None, |
3225 | - allow_daemons=False): |
3226 | - if isinstance(pkglist, str): |
3227 | - pkglist = [pkglist] |
3228 | - return run_apt_command( |
3229 | - 'install', args=pkglist, |
3230 | - aptopts=aptopts, target=target, env=env, allow_daemons=allow_daemons) |
3231 | - |
3232 | - |
3233 | def is_uefi_bootable(): |
3234 | return os.path.exists('/sys/firmware/efi') is True |
3235 | |
3236 | @@ -1076,7 +868,7 @@ def run_hook_if_exists(target, hook): |
3237 | """ |
3238 | Look for "hook" in "target" and run it |
3239 | """ |
3240 | - target_hook = target_path(target, '/curtin/' + hook) |
3241 | + target_hook = paths.target_path(target, '/curtin/' + hook) |
3242 | if os.path.isfile(target_hook): |
3243 | LOG.debug("running %s" % target_hook) |
3244 | subp([target_hook]) |
3245 | @@ -1231,41 +1023,6 @@ def is_file_not_found_exc(exc): |
3246 | exc.errno in (errno.ENOENT, errno.EIO, errno.ENXIO)) |
3247 | |
3248 | |
3249 | -def _lsb_release(target=None): |
3250 | - fmap = {'Codename': 'codename', 'Description': 'description', |
3251 | - 'Distributor ID': 'id', 'Release': 'release'} |
3252 | - |
3253 | - data = {} |
3254 | - try: |
3255 | - out, _ = subp(['lsb_release', '--all'], capture=True, target=target) |
3256 | - for line in out.splitlines(): |
3257 | - fname, _, val = line.partition(":") |
3258 | - if fname in fmap: |
3259 | - data[fmap[fname]] = val.strip() |
3260 | - missing = [k for k in fmap.values() if k not in data] |
3261 | - if len(missing): |
3262 | - LOG.warn("Missing fields in lsb_release --all output: %s", |
3263 | - ','.join(missing)) |
3264 | - |
3265 | - except ProcessExecutionError as err: |
3266 | - LOG.warn("Unable to get lsb_release --all: %s", err) |
3267 | - data = {v: "UNAVAILABLE" for v in fmap.values()} |
3268 | - |
3269 | - return data |
3270 | - |
3271 | - |
3272 | -def lsb_release(target=None): |
3273 | - if target_path(target) != "/": |
3274 | - # do not use or update cache if target is provided |
3275 | - return _lsb_release(target) |
3276 | - |
3277 | - global _LSB_RELEASE |
3278 | - if not _LSB_RELEASE: |
3279 | - data = _lsb_release() |
3280 | - _LSB_RELEASE.update(data) |
3281 | - return _LSB_RELEASE |
3282 | - |
3283 | - |
3284 | class MergedCmdAppend(argparse.Action): |
3285 | """This appends to a list in order of appearence both the option string |
3286 | and the value""" |
3287 | @@ -1400,31 +1157,6 @@ def is_resolvable_url(url): |
3288 | return is_resolvable(urlparse(url).hostname) |
3289 | |
3290 | |
3291 | -def target_path(target, path=None): |
3292 | - # return 'path' inside target, accepting target as None |
3293 | - if target in (None, ""): |
3294 | - target = "/" |
3295 | - elif not isinstance(target, string_types): |
3296 | - raise ValueError("Unexpected input for target: %s" % target) |
3297 | - else: |
3298 | - target = os.path.abspath(target) |
3299 | - # abspath("//") returns "//" specifically for 2 slashes. |
3300 | - if target.startswith("//"): |
3301 | - target = target[1:] |
3302 | - |
3303 | - if not path: |
3304 | - return target |
3305 | - |
3306 | - if not isinstance(path, string_types): |
3307 | - raise ValueError("Unexpected input for path: %s" % path) |
3308 | - |
3309 | - # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /. |
3310 | - while len(path) and path[0] == "/": |
3311 | - path = path[1:] |
3312 | - |
3313 | - return os.path.join(target, path) |
3314 | - |
3315 | - |
3316 | class RunInChroot(ChrootableTarget): |
3317 | """Backwards compatibility for RunInChroot (LP: #1617375). |
3318 | It needs to work like: |
3319 | diff --git a/debian/changelog b/debian/changelog |
3320 | index eccc322..10e5fbd 100644 |
3321 | --- a/debian/changelog |
3322 | +++ b/debian/changelog |
3323 | @@ -1,3 +1,48 @@ |
3324 | +curtin (18.1-56-g3aafe77d-0ubuntu1~16.04.1) xenial-proposed; urgency=medium |
3325 | + |
3326 | + * New upstream snapshot. (LP: #1795712) |
3327 | + - vmtest: Fix typo in skip-by-date. |
3328 | + - vmtest: kick skip-by-date for 1671951. |
3329 | + - tools/jenkins-runner: Error if both filters and tests are given. |
3330 | + - vmtests: prevent tests from modifying cls.collect_scripts |
3331 | + - Enable custom storage configuration for centos images |
3332 | + - vmtest: ensure we collect /var/log/journal only once |
3333 | + - Don't allow reads of /proc and modprobe zfs through |
3334 | + - clear-holders: handle missing zpool/zfs tools when wiping |
3335 | + - clear-holders: rescan for lvm devices after assembling raid arrays |
3336 | + - vmtest: enable persistent journal and collect at boot time |
3337 | + - Add timing and logging functions. |
3338 | + - parse_dpkg_version: support non-numeric in version string. |
3339 | + - Add main so that 'python3 -m curtin' does the right thing. |
3340 | + - Add subcommand 'features'. |
3341 | + - block: use uuid4 (random) when autogenerating UUIDS for filesystems |
3342 | + - vmtests: Increase size of root filesystems. |
3343 | + - clear-holders: reread ptable after wiping disks with partitions |
3344 | + - vmtest: Skip proposed pocket on dev release when 'proposed' in ADD_REPOS. |
3345 | + - tests: remove Ubuntu Artful [Joshua Powers] |
3346 | + - vmtests: Let a raised SkipTest go through skip_by_date. |
3347 | + - vmtests: Increase root fs to give upgrades to -proposed more space. |
3348 | + - vmtest: Order the vmtest_pollinate late_command earlier. |
3349 | + - vmtest: always add 'curtin/vmtest' to installed pollinate user_agent. |
3350 | + - vmtests: make skip_by_date a decorator that runs and reports. |
3351 | + - vmtests: always declare certain attributes and remove redundant tests. |
3352 | + - vmtests: Add Cosmic release to tests [Joshua Powers] |
3353 | + - vmtests: skip TrustyTestMdadmBcache until 2019-01-22. |
3354 | + - tox: use simplestreams from git repository rather than bzr. |
3355 | + - document that you can set ptable on raids [Michael Hudson-Doyle] |
3356 | + - vmtests: move skip-by date of xfs root and xfs boot out 1 year. |
3357 | + - vmtests: network_mtu move fixby date out 4 months from last value |
3358 | + - Fix WorkingDir class to support already existing target directory. |
3359 | + - Fix extraction of local filesystem image. |
3360 | + - Fix tip-pyflakes imported but unused call to util.get_platform_arch |
3361 | + - subp: update return value of subp with combine_capture=True. |
3362 | + - tox: add a xenial environments, default envlist changes. |
3363 | + - tests: Fix race on utcnow during timestamped curtin-log dir creation |
3364 | + - curtainer: patch source version from --source. |
3365 | + - pyflakes: fix unused variable references identified by pyflakes 2.0.0. |
3366 | + |
3367 | + -- Chad Smith <chad.smith@canonical.com> Tue, 02 Oct 2018 16:47:10 -0600 |
3368 | + |
3369 | curtin (18.1-17-gae48e86f-0ubuntu1~16.04.1) xenial; urgency=medium |
3370 | |
3371 | * New upstream snapshot. (LP: #1772044) |
3372 | diff --git a/doc/topics/config.rst b/doc/topics/config.rst |
3373 | index 76e520d..218bc17 100644 |
3374 | --- a/doc/topics/config.rst |
3375 | +++ b/doc/topics/config.rst |
3376 | @@ -14,6 +14,7 @@ Curtin's top level config keys are as follows: |
3377 | - apt_mirrors (``apt_mirrors``) |
3378 | - apt_proxy (``apt_proxy``) |
3379 | - block-meta (``block``) |
3380 | +- curthooks (``curthooks``) |
3381 | - debconf_selections (``debconf_selections``) |
3382 | - disable_overlayroot (``disable_overlayroot``) |
3383 | - grub (``grub``) |
3384 | @@ -110,6 +111,45 @@ Specify the filesystem label on the boot partition. |
3385 | label: my-boot-partition |
3386 | |
3387 | |
3388 | +curthooks |
3389 | +~~~~~~~~~ |
3390 | +Configure how Curtin determines what :ref:`curthooks` to run during the installation |
3391 | +process. |
3392 | + |
3393 | +**mode**: *<['auto', 'builtin', 'target']>* |
3394 | + |
3395 | +The default mode is ``auto``. |
3396 | + |
3397 | +In ``auto`` mode, curtin will execute curthooks within the image if present. |
3398 | +For images without curthooks inside, curtin will execute its built-in hooks. |
3399 | + |
3400 | +Currently the built-in curthooks support the following OS families: |
3401 | + |
3402 | +- Ubuntu |
3403 | +- Centos |
3404 | + |
3405 | +When specifying ``builtin``, curtin will only run the curthooks present in |
3406 | +Curtin ignoring any curthooks that may be present in the target operating |
3407 | +system. |
3408 | + |
3409 | +When specifying ``target``, curtin will attempt run the curthooks in the target |
3410 | +operating system. If the target does NOT contain any curthooks, then the |
3411 | +built-in curthooks will be run instead. |
3412 | + |
3413 | +Any errors during execution of curthooks (built-in or target) will fail the |
3414 | +installation. |
3415 | + |
3416 | +**Example**:: |
3417 | + |
3418 | + # ignore any target curthooks |
3419 | + curthooks: |
3420 | + mode: builtin |
3421 | + |
3422 | + # Only run target curthooks, fall back to built-in |
3423 | + curthooks: |
3424 | + mode: target |
3425 | + |
3426 | + |
3427 | debconf_selections |
3428 | ~~~~~~~~~~~~~~~~~~ |
3429 | Curtin will update the target with debconf set-selection values. Users will |
3430 | diff --git a/doc/topics/curthooks.rst b/doc/topics/curthooks.rst |
3431 | index e5f341b..c59aeaf 100644 |
3432 | --- a/doc/topics/curthooks.rst |
3433 | +++ b/doc/topics/curthooks.rst |
3434 | @@ -1,7 +1,13 @@ |
3435 | +.. _curthooks: |
3436 | + |
3437 | ======================================== |
3438 | -Curthooks / New OS Support |
3439 | +Curthooks / New OS Support |
3440 | ======================================== |
3441 | -Curtin has built-in support for installation of Ubuntu. |
3442 | +Curtin has built-in support for installation of: |
3443 | + |
3444 | + - Ubuntu |
3445 | + - Centos |
3446 | + |
3447 | Other operating systems are supported through a mechanism called |
3448 | 'curthooks' or 'curtin-hooks'. |
3449 | |
3450 | @@ -47,11 +53,21 @@ details. Specifically interesting to this stage are: |
3451 | - ``CONFIG``: This is a path to the curtin config file. It is provided so |
3452 | that additional configuration could be provided through to the OS |
3453 | customization. |
3454 | + - ``WORKING_DIR``: This is a path to a temporary directory where curtin |
3455 | + stores state and configuration files. |
3456 | |
3457 | .. **TODO**: We should add 'PYTHON' or 'CURTIN_PYTHON' to this environment |
3458 | so that the hook can easily run a python program with the same python |
3459 | that curtin ran with (ie, python2 or python3). |
3460 | |
3461 | +Running built-in hooks |
3462 | +---------------------- |
3463 | + |
3464 | +Curthooks may opt to run the built-in curthooks that are already provided in |
3465 | +curtin itself. To do so, an in-image curthook can import the ``curthooks`` |
3466 | +module and invoke the ``builtin_curthooks`` function passing in the required |
3467 | +parameters: config, target, and state. |
3468 | + |
3469 | |
3470 | Networking configuration |
3471 | ------------------------ |
3472 | diff --git a/doc/topics/integration-testing.rst b/doc/topics/integration-testing.rst |
3473 | index 7753068..6093b55 100644 |
3474 | --- a/doc/topics/integration-testing.rst |
3475 | +++ b/doc/topics/integration-testing.rst |
3476 | @@ -314,6 +314,10 @@ Some environment variables affect the running of vmtest |
3477 | setting (auto), then a upgrade will be done to make sure to include |
3478 | any new packages. |
3479 | |
3480 | + The string 'proposed' is handled specially. It will enable the |
3481 | + Ubuntu -proposed pocket for non-devel releases. If you wish to test |
3482 | + the -proposed pocket for a devel release, use 'PROPOSED'. |
3483 | + |
3484 | - ``CURTIN_VMTEST_SYSTEM_UPGRADE``: default 'auto' |
3485 | The default setting of 'auto' means to do a system upgrade if |
3486 | there are additional repos added. To enable this explicitly, set |
3487 | diff --git a/doc/topics/storage.rst b/doc/topics/storage.rst |
3488 | index ca6253c..b28964b 100644 |
3489 | --- a/doc/topics/storage.rst |
3490 | +++ b/doc/topics/storage.rst |
3491 | @@ -60,9 +60,9 @@ table. A disk command may contain all or some of the following keys: |
3492 | |
3493 | **ptable**: *msdos, gpt* |
3494 | |
3495 | -If the ``ptable`` key is present and a valid type of partition table, curtin |
3496 | -will create an empty partition table of that type on the disk. At the moment, |
3497 | -msdos and gpt partition tables are supported. |
3498 | +If the ``ptable`` key is present and a curtin will create an empty |
3499 | +partition table of that type on the disk. Curtin supports msdos and |
3500 | +gpt partition tables. |
3501 | |
3502 | **serial**: *<serial number>* |
3503 | |
3504 | @@ -613,6 +613,11 @@ The ``spare_devices`` key specifies a list of the devices that will be used for |
3505 | spares in the raid array. Each device must be referenced by ``id`` and the |
3506 | device must be previously defined in the storage configuration. May be empty. |
3507 | |
3508 | +**ptable**: *msdos, gpt* |
3509 | + |
3510 | +To partition the array rather than mounting it directly, the |
3511 | +``ptable`` key must be present and a valid type of partition table, |
3512 | +i.e. msdos or gpt. |
3513 | |
3514 | **Config Example**:: |
3515 | |
3516 | @@ -801,6 +806,7 @@ Learn by examples. |
3517 | - LVM |
3518 | - Bcache |
3519 | - RAID Boot |
3520 | +- Partitioned RAID |
3521 | - RAID5 + Bcache |
3522 | - ZFS Root Simple |
3523 | - ZFS Root |
3524 | @@ -1045,6 +1051,76 @@ RAID Boot |
3525 | path: / |
3526 | device: md_root |
3527 | |
3528 | +Partitioned RAID |
3529 | +~~~~~~~~~~~~~~~~ |
3530 | + |
3531 | +:: |
3532 | + |
3533 | + storage: |
3534 | + config: |
3535 | + - type: disk |
3536 | + id: disk-0 |
3537 | + ptable: gpt |
3538 | + path: /dev/vda |
3539 | + wipe: superblock |
3540 | + grub_device: true |
3541 | + - type: disk |
3542 | + id: disk-1 |
3543 | + path: /dev/vdb |
3544 | + wipe: superblock |
3545 | + - type: disk |
3546 | + id: disk-2 |
3547 | + path: /dev/vdc |
3548 | + wipe: superblock |
3549 | + - type: partition |
3550 | + id: part-0 |
3551 | + device: disk-0 |
3552 | + size: 1048576 |
3553 | + flag: bios_grub |
3554 | + - type: partition |
3555 | + id: part-1 |
3556 | + device: disk-0 |
3557 | + size: 21471690752 |
3558 | + - id: raid-0 |
3559 | + type: raid |
3560 | + name: md0 |
3561 | + raidlevel: 1 |
3562 | + devices: [disk-2, disk-1] |
3563 | + ptable: gpt |
3564 | + - type: partition |
3565 | + id: part-2 |
3566 | + device: raid-0 |
3567 | + size: 10737418240 |
3568 | + - type: partition |
3569 | + id: part-3 |
3570 | + device: raid-0 |
3571 | + size: 10735321088, |
3572 | + - type: format |
3573 | + id: fs-0 |
3574 | + fstype: ext4 |
3575 | + volume: part-1 |
3576 | + - type: format |
3577 | + id: fs-1 |
3578 | + fstype: xfs |
3579 | + volume: part-2 |
3580 | + - type: format |
3581 | + id: fs-2 |
3582 | + fstype: ext4 |
3583 | + volume: part-3 |
3584 | + - type: mount |
3585 | + id: mount-0 |
3586 | + device: fs-0 |
3587 | + path: / |
3588 | + - type: mount |
3589 | + id: mount-1 |
3590 | + device: fs-1 |
3591 | + path: /srv |
3592 | + - type: mount |
3593 | + id: mount-2 |
3594 | + device: fs-2 |
3595 | + path: /home |
3596 | + version: 1 |
3597 | + |
3598 | |
3599 | RAID5 + Bcache |
3600 | ~~~~~~~~~~~~~~ |
3601 | diff --git a/examples/tests/dirty_disks_config.yaml b/examples/tests/dirty_disks_config.yaml |
3602 | index 75d44c3..fb9a0d6 100644 |
3603 | --- a/examples/tests/dirty_disks_config.yaml |
3604 | +++ b/examples/tests/dirty_disks_config.yaml |
3605 | @@ -27,6 +27,31 @@ bucket: |
3606 | # disable any rpools to trigger disks with zfs_member label but inactive |
3607 | # pools |
3608 | zpool export rpool ||: |
3609 | + - &lvm_stop | |
3610 | + #!/bin/sh |
3611 | + # This function disables any existing lvm logical volumes that |
3612 | + # have been created during the early storage config stage |
3613 | + # and simulates the effect of booting into a system with existing |
3614 | + # (but inactive) lvm configuration. |
3615 | + for vg in `pvdisplay -C --separator = -o vg_name --noheadings`; do |
3616 | + vgchange -an $vg ||: |
3617 | + done |
3618 | + # disable the automatic pvscan, we want to test that curtin |
3619 | + # can find/enable logical volumes without this service |
3620 | + command -v systemctl && systemctl mask lvm2-pvscan\@.service |
3621 | + # remove any existing metadata written from early disk config |
3622 | + rm -rf /etc/lvm/archive /etc/lvm/backup |
3623 | + - &mdadm_stop | |
3624 | + #!/bin/sh |
3625 | + # This function disables any existing raid devices which may |
3626 | + # have been created during the early storage config stage |
3627 | + # and simulates the effect of booting into a system with existing |
3628 | + # but inactive mdadm configuration. |
3629 | + for md in /dev/md*; do |
3630 | + mdadm --stop $md ||: |
3631 | + done |
3632 | + # remove any existing metadata written from early disk config |
3633 | + rm -f /etc/mdadm/mdadm.conf |
3634 | |
3635 | early_commands: |
3636 | # running block-meta custom from the install environment |
3637 | @@ -34,9 +59,11 @@ early_commands: |
3638 | # the disks exactly as in this config before the rest of the install |
3639 | # will just blow it all away. We have clean out other environment |
3640 | # that could unintentionally mess things up. |
3641 | - blockmeta: [env, -u, OUTPUT_FSTAB, |
3642 | + 01-blockmeta: [env, -u, OUTPUT_FSTAB, |
3643 | TARGET_MOUNT_POINT=/tmp/my.bdir/target, |
3644 | WORKING_DIR=/tmp/my.bdir/work.d, |
3645 | curtin, --showtrace, -v, block-meta, --umount, custom] |
3646 | - enable_swaps: [sh, -c, *swapon] |
3647 | - disable_rpool: [sh, -c, *zpool_export] |
3648 | + 02-enable_swaps: [sh, -c, *swapon] |
3649 | + 03-disable_rpool: [sh, -c, *zpool_export] |
3650 | + 04-lvm_stop: [sh, -c, *lvm_stop] |
3651 | + 05-mdadm_stop: [sh, -c, *mdadm_stop] |
3652 | diff --git a/examples/tests/filesystem_battery.yaml b/examples/tests/filesystem_battery.yaml |
3653 | index 3b1edbf..4eae5b6 100644 |
3654 | --- a/examples/tests/filesystem_battery.yaml |
3655 | +++ b/examples/tests/filesystem_battery.yaml |
3656 | @@ -113,8 +113,8 @@ storage: |
3657 | - id: bind1 |
3658 | fstype: "none" |
3659 | options: "bind" |
3660 | - path: "/var/lib" |
3661 | - spec: "/my/bind-over-var-lib" |
3662 | + path: "/var/cache" |
3663 | + spec: "/my/bind-over-var-cache" |
3664 | type: mount |
3665 | - id: bind2 |
3666 | fstype: "none" |
3667 | diff --git a/examples/tests/install_disable_unmount.yaml b/examples/tests/install_disable_unmount.yaml |
3668 | index d3e583f..c0cd759 100644 |
3669 | --- a/examples/tests/install_disable_unmount.yaml |
3670 | +++ b/examples/tests/install_disable_unmount.yaml |
3671 | @@ -14,5 +14,5 @@ post_cmds: |
3672 | late_commands: |
3673 | 01_get_proc_mounts: [sh, -c, *cat_proc_mounts] |
3674 | 02_write_out_target: [sh, -c, *echo_target_mp] |
3675 | - 03_unmount_target: [curtin, unmount] |
3676 | - 04_get_proc_mounts: [cat, /proc/mounts] |
3677 | + 99a_unmount_target: [curtin, unmount] |
3678 | + 99b_get_proc_mounts: [cat, /proc/mounts] |
3679 | diff --git a/examples/tests/lvmoverraid.yaml b/examples/tests/lvmoverraid.yaml |
3680 | new file mode 100644 |
3681 | index 0000000..a1d41e9 |
3682 | --- /dev/null |
3683 | +++ b/examples/tests/lvmoverraid.yaml |
3684 | @@ -0,0 +1,98 @@ |
3685 | +storage: |
3686 | + config: |
3687 | + - grub_device: true |
3688 | + id: disk-0 |
3689 | + model: QEMU_HARDDISK |
3690 | + name: 'main_disk' |
3691 | + serial: disk-a |
3692 | + preserve: false |
3693 | + ptable: gpt |
3694 | + type: disk |
3695 | + wipe: superblock |
3696 | + - grub_device: false |
3697 | + id: disk-2 |
3698 | + name: 'disk-2' |
3699 | + serial: disk-b |
3700 | + preserve: false |
3701 | + type: disk |
3702 | + wipe: superblock |
3703 | + - grub_device: false |
3704 | + id: disk-1 |
3705 | + name: 'disk-1' |
3706 | + serial: disk-c |
3707 | + preserve: false |
3708 | + type: disk |
3709 | + wipe: superblock |
3710 | + - grub_device: false |
3711 | + id: disk-3 |
3712 | + name: 'disk-3' |
3713 | + serial: disk-d |
3714 | + preserve: false |
3715 | + type: disk |
3716 | + wipe: superblock |
3717 | + - grub_device: false |
3718 | + id: disk-4 |
3719 | + name: 'disk-4' |
3720 | + serial: disk-e |
3721 | + preserve: false |
3722 | + type: disk |
3723 | + wipe: superblock |
3724 | + - device: disk-0 |
3725 | + flag: bios_grub |
3726 | + id: part-0 |
3727 | + preserve: false |
3728 | + size: 1048576 |
3729 | + type: partition |
3730 | + - device: disk-0 |
3731 | + flag: '' |
3732 | + id: part-1 |
3733 | + preserve: false |
3734 | + size: 4G |
3735 | + type: partition |
3736 | + - devices: |
3737 | + - disk-2 |
3738 | + - disk-1 |
3739 | + id: raid-0 |
3740 | + name: md0 |
3741 | + raidlevel: 1 |
3742 | + spare_devices: [] |
3743 | + type: raid |
3744 | + - devices: |
3745 | + - disk-3 |
3746 | + - disk-4 |
3747 | + id: raid-1 |
3748 | + name: md1 |
3749 | + raidlevel: 1 |
3750 | + spare_devices: [] |
3751 | + type: raid |
3752 | + - devices: |
3753 | + - raid-0 |
3754 | + - raid-1 |
3755 | + id: vg-0 |
3756 | + name: vg0 |
3757 | + type: lvm_volgroup |
3758 | + - id: lv-0 |
3759 | + name: lv-0 |
3760 | + size: 3G |
3761 | + type: lvm_partition |
3762 | + volgroup: vg-0 |
3763 | + - fstype: ext4 |
3764 | + id: fs-0 |
3765 | + preserve: false |
3766 | + type: format |
3767 | + volume: part-1 |
3768 | + - fstype: ext4 |
3769 | + id: fs-1 |
3770 | + preserve: false |
3771 | + type: format |
3772 | + volume: lv-0 |
3773 | + - device: fs-0 |
3774 | + id: mount-0 |
3775 | + path: / |
3776 | + type: mount |
3777 | + - device: fs-1 |
3778 | + id: mount-1 |
3779 | + path: /home |
3780 | + type: mount |
3781 | + version: 1 |
3782 | + |
3783 | diff --git a/examples/tests/mirrorboot-msdos-partition.yaml b/examples/tests/mirrorboot-msdos-partition.yaml |
3784 | index 1a418fa..2b111a7 100644 |
3785 | --- a/examples/tests/mirrorboot-msdos-partition.yaml |
3786 | +++ b/examples/tests/mirrorboot-msdos-partition.yaml |
3787 | @@ -47,7 +47,7 @@ storage: |
3788 | name: md0-part1 |
3789 | number: 1 |
3790 | offset: 4194304B |
3791 | - size: 2GB |
3792 | + size: 3GB |
3793 | type: partition |
3794 | uuid: 4f4fa336-2762-48e4-ae54-9451141665cd |
3795 | wipe: superblock |
3796 | @@ -55,7 +55,7 @@ storage: |
3797 | id: md0-part2 |
3798 | name: md0-part2 |
3799 | number: 2 |
3800 | - size: 2GB |
3801 | + size: 1.5GB |
3802 | type: partition |
3803 | uuid: c2d21fd3-3cde-4432-8eab-f08594bbe76e |
3804 | wipe: superblock |
3805 | diff --git a/examples/tests/mirrorboot-uefi.yaml b/examples/tests/mirrorboot-uefi.yaml |
3806 | index e1f393f..ca55be9 100644 |
3807 | --- a/examples/tests/mirrorboot-uefi.yaml |
3808 | +++ b/examples/tests/mirrorboot-uefi.yaml |
3809 | @@ -30,7 +30,7 @@ storage: |
3810 | id: sda-part2 |
3811 | name: sda-part2 |
3812 | number: 2 |
3813 | - size: 2G |
3814 | + size: 3G |
3815 | type: partition |
3816 | uuid: 47c97eae-f35d-473f-8f3d-d64161d571f1 |
3817 | wipe: superblock |
3818 | @@ -38,7 +38,7 @@ storage: |
3819 | id: sda-part3 |
3820 | name: sda-part3 |
3821 | number: 3 |
3822 | - size: 2G |
3823 | + size: 1G |
3824 | type: partition |
3825 | uuid: e3202633-841c-4936-a520-b18d1f7938ea |
3826 | wipe: superblock |
3827 | @@ -56,7 +56,7 @@ storage: |
3828 | id: sdb-part2 |
3829 | name: sdb-part2 |
3830 | number: 2 |
3831 | - size: 2G |
3832 | + size: 3G |
3833 | type: partition |
3834 | uuid: a33a83dd-d1bf-4940-bf3e-6d931de85dbc |
3835 | wipe: superblock |
3836 | @@ -72,7 +72,7 @@ storage: |
3837 | id: sdb-part3 |
3838 | name: sdb-part3 |
3839 | number: 3 |
3840 | - size: 2G |
3841 | + size: 1G |
3842 | type: partition |
3843 | uuid: 27e29758-fdcf-4c6a-8578-c92f907a8a9d |
3844 | wipe: superblock |
3845 | diff --git a/examples/tests/vmtest_defaults.yaml b/examples/tests/vmtest_defaults.yaml |
3846 | new file mode 100644 |
3847 | index 0000000..b1512a8 |
3848 | --- /dev/null |
3849 | +++ b/examples/tests/vmtest_defaults.yaml |
3850 | @@ -0,0 +1,24 @@ |
3851 | +# this updates pollinate in the installed target to add a vmtest identifier. |
3852 | +# specifically pollinate's user-agent should contain 'curtin/vmtest'. |
3853 | +_vmtest_pollinate: |
3854 | + - &pvmtest | |
3855 | + cfg="/etc/pollinate/add-user-agent" |
3856 | + [ -d "${cfg%/*}" ] || exit 0 |
3857 | + echo curtin/vmtest >> "$cfg" |
3858 | + |
3859 | +# this enables a persitent journald if target system has journald |
3860 | +# and does not have /var/log/journal directory already |
3861 | +_persist_journal: |
3862 | + - &persist_journal | |
3863 | + command -v journalctl && { |
3864 | + jdir=/var/log/journal |
3865 | + [ -e ${jdir} ] || { |
3866 | + mkdir -p ${jdir} |
3867 | + systemd-tmpfiles --create --prefix ${jdir} |
3868 | + } |
3869 | + } |
3870 | + exit 0 |
3871 | + |
3872 | +late_commands: |
3873 | + 01_vmtest_pollinate: ['curtin', 'in-target', '--', 'sh', '-c', *pvmtest] |
3874 | + 02_persist_journal: ['curtin', 'in-target', '--', 'sh', '-c', *persist_journal] |
3875 | diff --git a/helpers/common b/helpers/common |
3876 | index ac2d0f3..f9217b7 100644 |
3877 | --- a/helpers/common |
3878 | +++ b/helpers/common |
3879 | @@ -541,18 +541,18 @@ get_carryover_params() { |
3880 | } |
3881 | |
3882 | install_grub() { |
3883 | - local long_opts="uefi,update-nvram" |
3884 | + local long_opts="uefi,update-nvram,os-family:" |
3885 | local getopt_out="" mp_efi="" |
3886 | getopt_out=$(getopt --name "${0##*/}" \ |
3887 | --options "" --long "${long_opts}" -- "$@") && |
3888 | eval set -- "${getopt_out}" |
3889 | |
3890 | - local uefi=0 |
3891 | - local update_nvram=0 |
3892 | + local uefi=0 update_nvram=0 os_family="" |
3893 | |
3894 | while [ $# -ne 0 ]; do |
3895 | cur="$1"; next="$2"; |
3896 | case "$cur" in |
3897 | + --os-family) os_family=${next};; |
3898 | --uefi) uefi=$((${uefi}+1));; |
3899 | --update-nvram) update_nvram=$((${update_nvram}+1));; |
3900 | --) shift; break;; |
3901 | @@ -595,29 +595,88 @@ install_grub() { |
3902 | error "$mp_dev ($fstype) is not a block device!"; return 1; |
3903 | fi |
3904 | |
3905 | - # get dpkg arch |
3906 | - local dpkg_arch="" |
3907 | - dpkg_arch=$(chroot "$mp" dpkg --print-architecture) |
3908 | - r=$? |
3909 | + local os_variant="" |
3910 | + if [ -e "${mp}/etc/os-release" ]; then |
3911 | + os_variant=$(chroot "$mp" \ |
3912 | + /bin/sh -c 'echo $(. /etc/os-release; echo $ID)') |
3913 | + else |
3914 | + # Centos6 doesn't have os-release, so check for centos/redhat release |
3915 | + # looks like: CentOS release 6.9 (Final) |
3916 | + for rel in $(ls ${mp}/etc/*-release); do |
3917 | + os_variant=$(awk '{print tolower($1)}' $rel) |
3918 | + [ -n "$os_variant" ] && break |
3919 | + done |
3920 | + fi |
3921 | + [ $? != 0 ] && |
3922 | + { error "Failed to read ID from $mp/etc/os-release"; return 1; } |
3923 | + |
3924 | + local rhel_ver="" |
3925 | + case $os_variant in |
3926 | + debian|ubuntu) os_family="debian";; |
3927 | + centos|rhel) |
3928 | + os_family="redhat" |
3929 | + rhel_ver=$(chroot "$mp" rpm -E '%rhel') |
3930 | + ;; |
3931 | + esac |
3932 | + |
3933 | + # ensure we have both settings, family and variant are needed |
3934 | + [ -n "${os_variant}" -a -n "${os_family}" ] || |
3935 | + { error "Failed to determine os variant and family"; return 1; } |
3936 | + |
3937 | + # get target arch |
3938 | + local target_arch="" r="1" |
3939 | + case $os_family in |
3940 | + debian) |
3941 | + target_arch=$(chroot "$mp" dpkg --print-architecture) |
3942 | + r=$? |
3943 | + ;; |
3944 | + redhat) |
3945 | + target_arch=$(chroot "$mp" rpm -E '%_arch') |
3946 | + r=$? |
3947 | + ;; |
3948 | + esac |
3949 | [ $r -eq 0 ] || { |
3950 | - error "failed to get dpkg architecture [$r]" |
3951 | + error "failed to get target architecture [$r]" |
3952 | return 1; |
3953 | } |
3954 | |
3955 | # grub is not the bootloader you are looking for |
3956 | - if [ "${dpkg_arch}" = "s390x" ]; then |
3957 | - return 0; |
3958 | + if [ "${target_arch}" = "s390x" ]; then |
3959 | + return 0; |
3960 | fi |
3961 | |
3962 | # set correct grub package |
3963 | - local grub_name="grub-pc" |
3964 | - local grub_target="i386-pc" |
3965 | - if [ "${dpkg_arch#ppc64}" != "${dpkg_arch}" ]; then |
3966 | + local grub_name="" |
3967 | + local grub_target="" |
3968 | + case "$target_arch" in |
3969 | + i386|amd64) |
3970 | + # debian |
3971 | + grub_name="grub-pc" |
3972 | + grub_target="i386-pc" |
3973 | + ;; |
3974 | + x86_64) |
3975 | + case $rhel_ver in |
3976 | + 6) grub_name="grub";; |
3977 | + 7) grub_name="grub2-pc";; |
3978 | + *) |
3979 | + error "Unknown rhel_ver [$rhel_ver]"; |
3980 | + return 1; |
3981 | + ;; |
3982 | + esac |
3983 | + grub_target="i386-pc" |
3984 | + ;; |
3985 | + esac |
3986 | + if [ "${target_arch#ppc64}" != "${target_arch}" ]; then |
3987 | grub_name="grub-ieee1275" |
3988 | grub_target="powerpc-ieee1275" |
3989 | elif [ "$uefi" -ge 1 ]; then |
3990 | - grub_name="grub-efi-$dpkg_arch" |
3991 | - case "$dpkg_arch" in |
3992 | + grub_name="grub-efi-$target_arch" |
3993 | + case "$target_arch" in |
3994 | + x86_64) |
3995 | + # centos 7+, no centos6 support |
3996 | + grub_name="grub2-efi-x64-modules" |
3997 | + grub_target="x86_64-efi" |
3998 | + ;; |
3999 | amd64) |
4000 | grub_target="x86_64-efi";; |
4001 | arm64) |
4002 | @@ -626,9 +685,19 @@ install_grub() { |
4003 | fi |
4004 | |
4005 | # check that the grub package is installed |
4006 | - tmp=$(chroot "$mp" dpkg-query --show \ |
4007 | - --showformat='${Status}\n' $grub_name) |
4008 | - r=$? |
4009 | + local r=$? |
4010 | + case $os_family in |
4011 | + debian) |
4012 | + tmp=$(chroot "$mp" dpkg-query --show \ |
4013 | + --showformat='${Status}\n' $grub_name) |
4014 | + r=$? |
4015 | + ;; |
4016 | + redhat) |
4017 | + tmp=$(chroot "$mp" rpm -q \ |
4018 | + --queryformat='install ok installed\n' $grub_name) |
4019 | + r=$? |
4020 | + ;; |
4021 | + esac |
4022 | if [ $r -ne 0 -a $r -ne 1 ]; then |
4023 | error "failed to check if $grub_name installed"; |
4024 | return 1; |
4025 | @@ -636,11 +705,16 @@ install_grub() { |
4026 | case "$tmp" in |
4027 | install\ ok\ installed) :;; |
4028 | *) debug 1 "$grub_name not installed, not doing anything"; |
4029 | - return 0;; |
4030 | + return 1;; |
4031 | esac |
4032 | |
4033 | local grub_d="etc/default/grub.d" |
4034 | local mygrub_cfg="$grub_d/50-curtin-settings.cfg" |
4035 | + case $os_family in |
4036 | + redhat) |
4037 | + grub_d="etc/default" |
4038 | + mygrub_cfg="etc/default/grub";; |
4039 | + esac |
4040 | [ -d "$mp/$grub_d" ] || mkdir -p "$mp/$grub_d" || |
4041 | { error "Failed to create $grub_d"; return 1; } |
4042 | |
4043 | @@ -659,14 +733,23 @@ install_grub() { |
4044 | error "Failed to get carryover parrameters from cmdline"; |
4045 | return 1; |
4046 | } |
4047 | + # always append rd.auto=1 for centos |
4048 | + case $os_family in |
4049 | + redhat) |
4050 | + newargs="$newargs rd.auto=1";; |
4051 | + esac |
4052 | debug 1 "carryover command line params: $newargs" |
4053 | |
4054 | - : > "$mp/$mygrub_cfg" || |
4055 | - { error "Failed to write '$mygrub_cfg'"; return 1; } |
4056 | + case $os_family in |
4057 | + debian) |
4058 | + : > "$mp/$mygrub_cfg" || |
4059 | + { error "Failed to write '$mygrub_cfg'"; return 1; } |
4060 | + ;; |
4061 | + esac |
4062 | { |
4063 | [ "${REPLACE_GRUB_LINUX_DEFAULT:-1}" = "0" ] || |
4064 | echo "GRUB_CMDLINE_LINUX_DEFAULT=\"$newargs\"" |
4065 | - echo "# disable grub os prober that might find other OS installs." |
4066 | + echo "# Curtin disable grub os prober that might find other OS installs." |
4067 | echo "GRUB_DISABLE_OS_PROBER=true" |
4068 | echo "GRUB_TERMINAL=console" |
4069 | } >> "$mp/$mygrub_cfg" |
4070 | @@ -692,30 +775,46 @@ install_grub() { |
4071 | nvram="--no-nvram" |
4072 | if [ "$update_nvram" -ge 1 ]; then |
4073 | nvram="" |
4074 | - fi |
4075 | + fi |
4076 | debug 1 "curtin uefi: installing ${grub_name} to: /boot/efi" |
4077 | chroot "$mp" env DEBIAN_FRONTEND=noninteractive sh -exc ' |
4078 | echo "before grub-install efiboot settings" |
4079 | - efibootmgr || echo "WARN: efibootmgr exited $?" |
4080 | - dpkg-reconfigure "$1" |
4081 | - update-grub |
4082 | + efibootmgr -v || echo "WARN: efibootmgr exited $?" |
4083 | + bootid="$4" |
4084 | + grubpost="" |
4085 | + case $bootid in |
4086 | + debian|ubuntu) |
4087 | + grubcmd="grub-install" |
4088 | + dpkg-reconfigure "$1" |
4089 | + update-grub |
4090 | + ;; |
4091 | + centos|redhat|rhel) |
4092 | + grubcmd="grub2-install" |
4093 | + grubpost="grub2-mkconfig -o /boot/grub2/grub.cfg" |
4094 | + ;; |
4095 | + *) |
4096 | + echo "Unsupported OS: $bootid" 1>&2 |
4097 | + exit 1 |
4098 | + ;; |
4099 | + esac |
4100 | # grub-install in 12.04 does not contain --no-nvram, --target, |
4101 | # or --efi-directory |
4102 | target="--target=$2" |
4103 | no_nvram="$3" |
4104 | efi_dir="--efi-directory=/boot/efi" |
4105 | - gi_out=$(grub-install --help 2>&1) |
4106 | + gi_out=$($grubcmd --help 2>&1) |
4107 | echo "$gi_out" | grep -q -- "$no_nvram" || no_nvram="" |
4108 | echo "$gi_out" | grep -q -- "--target" || target="" |
4109 | echo "$gi_out" | grep -q -- "--efi-directory" || efi_dir="" |
4110 | - grub-install $target $efi_dir \ |
4111 | - --bootloader-id=ubuntu --recheck $no_nvram' -- \ |
4112 | - "${grub_name}" "${grub_target}" "$nvram" </dev/null || |
4113 | + $grubcmd $target $efi_dir \ |
4114 | + --bootloader-id=$bootid --recheck $no_nvram |
4115 | + [ -z "$grubpost" ] || $grubpost;' \ |
4116 | + -- "${grub_name}" "${grub_target}" "$nvram" "$os_variant" </dev/null || |
4117 | { error "failed to install grub!"; return 1; } |
4118 | |
4119 | chroot "$mp" sh -exc ' |
4120 | echo "after grub-install efiboot settings" |
4121 | - efibootmgr || echo "WARN: efibootmgr exited $?" |
4122 | + efibootmgr -v || echo "WARN: efibootmgr exited $?" |
4123 | ' -- </dev/null || |
4124 | { error "failed to list efi boot entries!"; return 1; } |
4125 | else |
4126 | @@ -728,10 +827,32 @@ install_grub() { |
4127 | debug 1 "curtin non-uefi: installing ${grub_name} to: ${grubdevs[*]}" |
4128 | chroot "$mp" env DEBIAN_FRONTEND=noninteractive sh -exc ' |
4129 | pkg=$1; shift; |
4130 | - dpkg-reconfigure "$pkg" |
4131 | - update-grub |
4132 | - for d in "$@"; do grub-install "$d" || exit; done' \ |
4133 | - -- "${grub_name}" "${grubdevs[@]}" </dev/null || |
4134 | + bootid=$1; shift; |
4135 | + bootver=$1; shift; |
4136 | + grubpost="" |
4137 | + case $bootid in |
4138 | + debian|ubuntu) |
4139 | + grubcmd="grub-install" |
4140 | + dpkg-reconfigure "$pkg" |
4141 | + update-grub |
4142 | + ;; |
4143 | + centos|redhat|rhel) |
4144 | + case $bootver in |
4145 | + 6) grubcmd="grub-install";; |
4146 | + 7) grubcmd="grub2-install" |
4147 | + grubpost="grub2-mkconfig -o /boot/grub2/grub.cfg";; |
4148 | + esac |
4149 | + ;; |
4150 | + *) |
4151 | + echo "Unsupported OS: $bootid"; 1>&2 |
4152 | + exit 1 |
4153 | + ;; |
4154 | + esac |
4155 | + for d in "$@"; do |
4156 | + echo $grubcmd "$d"; |
4157 | + $grubcmd "$d" || exit; done |
4158 | + [ -z "$grubpost" ] || $grubpost;' \ |
4159 | + -- "${grub_name}" "${os_variant}" "${rhel_ver}" "${grubdevs[@]}" </dev/null || |
4160 | { error "failed to install grub!"; return 1; } |
4161 | fi |
4162 | |
4163 | diff --git a/tests/unittests/test_apt_custom_sources_list.py b/tests/unittests/test_apt_custom_sources_list.py |
4164 | index 5567dd5..a427ae9 100644 |
4165 | --- a/tests/unittests/test_apt_custom_sources_list.py |
4166 | +++ b/tests/unittests/test_apt_custom_sources_list.py |
4167 | @@ -11,6 +11,8 @@ from mock import call |
4168 | import textwrap |
4169 | import yaml |
4170 | |
4171 | +from curtin import distro |
4172 | +from curtin import paths |
4173 | from curtin import util |
4174 | from curtin.commands import apt_config |
4175 | from .helpers import CiTestCase |
4176 | @@ -106,7 +108,7 @@ class TestAptSourceConfigSourceList(CiTestCase): |
4177 | # make test independent to executing system |
4178 | with mock.patch.object(util, 'load_file', |
4179 | return_value=MOCKED_APT_SRC_LIST): |
4180 | - with mock.patch.object(util, 'lsb_release', |
4181 | + with mock.patch.object(distro, 'lsb_release', |
4182 | return_value={'codename': |
4183 | 'fakerel'}): |
4184 | apt_config.handle_apt(cfg, TARGET) |
4185 | @@ -115,10 +117,10 @@ class TestAptSourceConfigSourceList(CiTestCase): |
4186 | |
4187 | cloudfile = '/etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg' |
4188 | cloudconf = yaml.dump({'apt_preserve_sources_list': True}, indent=1) |
4189 | - calls = [call(util.target_path(TARGET, '/etc/apt/sources.list'), |
4190 | + calls = [call(paths.target_path(TARGET, '/etc/apt/sources.list'), |
4191 | expected, |
4192 | mode=0o644), |
4193 | - call(util.target_path(TARGET, cloudfile), |
4194 | + call(paths.target_path(TARGET, cloudfile), |
4195 | cloudconf, |
4196 | mode=0o644)] |
4197 | mockwrite.assert_has_calls(calls) |
4198 | @@ -147,19 +149,19 @@ class TestAptSourceConfigSourceList(CiTestCase): |
4199 | arch = util.get_architecture() |
4200 | # would fail inside the unittest context |
4201 | with mock.patch.object(util, 'get_architecture', return_value=arch): |
4202 | - with mock.patch.object(util, 'lsb_release', |
4203 | + with mock.patch.object(distro, 'lsb_release', |
4204 | return_value={'codename': 'fakerel'}): |
4205 | apt_config.handle_apt(cfg, target) |
4206 | |
4207 | self.assertEqual( |
4208 | EXPECTED_CONVERTED_CONTENT, |
4209 | - util.load_file(util.target_path(target, "/etc/apt/sources.list"))) |
4210 | - cloudfile = util.target_path( |
4211 | + util.load_file(paths.target_path(target, "/etc/apt/sources.list"))) |
4212 | + cloudfile = paths.target_path( |
4213 | target, '/etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg') |
4214 | self.assertEqual({'apt_preserve_sources_list': True}, |
4215 | yaml.load(util.load_file(cloudfile))) |
4216 | |
4217 | - @mock.patch("curtin.util.lsb_release") |
4218 | + @mock.patch("curtin.distro.lsb_release") |
4219 | @mock.patch("curtin.util.get_architecture", return_value="amd64") |
4220 | def test_trusty_source_lists(self, m_get_arch, m_lsb_release): |
4221 | """Support mirror equivalency with and without trailing /. |
4222 | @@ -199,7 +201,7 @@ class TestAptSourceConfigSourceList(CiTestCase): |
4223 | |
4224 | release = 'trusty' |
4225 | comps = 'main universe multiverse restricted' |
4226 | - easl = util.target_path(target, 'etc/apt/sources.list') |
4227 | + easl = paths.target_path(target, 'etc/apt/sources.list') |
4228 | |
4229 | orig_content = tmpl.format( |
4230 | mirror=orig_primary, security=orig_security, |
4231 | diff --git a/tests/unittests/test_apt_source.py b/tests/unittests/test_apt_source.py |
4232 | index 2ede986..353cdf8 100644 |
4233 | --- a/tests/unittests/test_apt_source.py |
4234 | +++ b/tests/unittests/test_apt_source.py |
4235 | @@ -12,8 +12,9 @@ import socket |
4236 | import mock |
4237 | from mock import call |
4238 | |
4239 | -from curtin import util |
4240 | +from curtin import distro |
4241 | from curtin import gpg |
4242 | +from curtin import util |
4243 | from curtin.commands import apt_config |
4244 | from .helpers import CiTestCase |
4245 | |
4246 | @@ -77,7 +78,7 @@ class TestAptSourceConfig(CiTestCase): |
4247 | |
4248 | @staticmethod |
4249 | def _add_apt_sources(*args, **kwargs): |
4250 | - with mock.patch.object(util, 'apt_update'): |
4251 | + with mock.patch.object(distro, 'apt_update'): |
4252 | apt_config.add_apt_sources(*args, **kwargs) |
4253 | |
4254 | @staticmethod |
4255 | @@ -86,7 +87,7 @@ class TestAptSourceConfig(CiTestCase): |
4256 | Get the most basic default mrror and release info to be used in tests |
4257 | """ |
4258 | params = {} |
4259 | - params['RELEASE'] = util.lsb_release()['codename'] |
4260 | + params['RELEASE'] = distro.lsb_release()['codename'] |
4261 | arch = util.get_architecture() |
4262 | params['MIRROR'] = apt_config.get_default_mirrors(arch)["PRIMARY"] |
4263 | return params |
4264 | @@ -472,7 +473,7 @@ class TestAptSourceConfig(CiTestCase): |
4265 | 'uri': |
4266 | 'http://testsec.ubuntu.com/%s/' % component}]} |
4267 | post = ("%s_dists_%s-updates_InRelease" % |
4268 | - (component, util.lsb_release()['codename'])) |
4269 | + (component, distro.lsb_release()['codename'])) |
4270 | fromfn = ("%s/%s_%s" % (pre, archive, post)) |
4271 | tofn = ("%s/test.ubuntu.com_%s" % (pre, post)) |
4272 | |
4273 | @@ -937,7 +938,7 @@ class TestDebconfSelections(CiTestCase): |
4274 | m_set_sel.assert_not_called() |
4275 | |
4276 | @mock.patch("curtin.commands.apt_config.debconf_set_selections") |
4277 | - @mock.patch("curtin.commands.apt_config.util.get_installed_packages") |
4278 | + @mock.patch("curtin.commands.apt_config.distro.get_installed_packages") |
4279 | def test_set_sel_call_has_expected_input(self, m_get_inst, m_set_sel): |
4280 | data = { |
4281 | 'set1': 'pkga pkga/q1 mybool false', |
4282 | @@ -960,7 +961,7 @@ class TestDebconfSelections(CiTestCase): |
4283 | |
4284 | @mock.patch("curtin.commands.apt_config.dpkg_reconfigure") |
4285 | @mock.patch("curtin.commands.apt_config.debconf_set_selections") |
4286 | - @mock.patch("curtin.commands.apt_config.util.get_installed_packages") |
4287 | + @mock.patch("curtin.commands.apt_config.distro.get_installed_packages") |
4288 | def test_reconfigure_if_intersection(self, m_get_inst, m_set_sel, |
4289 | m_dpkg_r): |
4290 | data = { |
4291 | @@ -985,7 +986,7 @@ class TestDebconfSelections(CiTestCase): |
4292 | |
4293 | @mock.patch("curtin.commands.apt_config.dpkg_reconfigure") |
4294 | @mock.patch("curtin.commands.apt_config.debconf_set_selections") |
4295 | - @mock.patch("curtin.commands.apt_config.util.get_installed_packages") |
4296 | + @mock.patch("curtin.commands.apt_config.distro.get_installed_packages") |
4297 | def test_reconfigure_if_no_intersection(self, m_get_inst, m_set_sel, |
4298 | m_dpkg_r): |
4299 | data = {'set1': 'pkga pkga/q1 mybool false'} |
4300 | diff --git a/tests/unittests/test_block.py b/tests/unittests/test_block.py |
4301 | index d9b19a4..9cf8383 100644 |
4302 | --- a/tests/unittests/test_block.py |
4303 | +++ b/tests/unittests/test_block.py |
4304 | @@ -647,4 +647,39 @@ class TestSlaveKnames(CiTestCase): |
4305 | knames = block.get_device_slave_knames(device) |
4306 | self.assertEqual(slaves, knames) |
4307 | |
4308 | + |
4309 | +class TestGetSupportedFilesystems(CiTestCase): |
4310 | + |
4311 | + supported_filesystems = ['sysfs', 'rootfs', 'ramfs', 'ext4'] |
4312 | + |
4313 | + def _proc_filesystems_output(self, supported=None): |
4314 | + if not supported: |
4315 | + supported = self.supported_filesystems |
4316 | + |
4317 | + def devname(fsname): |
4318 | + """ in-use filesystem modules not emit the 'nodev' prefix """ |
4319 | + return '\t' if fsname.startswith('ext') else 'nodev\t' |
4320 | + |
4321 | + return '\n'.join([devname(fs) + fs for fs in supported]) + '\n' |
4322 | + |
4323 | + @mock.patch('curtin.block.util') |
4324 | + @mock.patch('curtin.block.os') |
4325 | + def test_get_supported_filesystems(self, mock_os, mock_util): |
4326 | + """ test parsing /proc/filesystems contents into a filesystem list""" |
4327 | + mock_os.path.exists.return_value = True |
4328 | + mock_util.load_file.return_value = self._proc_filesystems_output() |
4329 | + |
4330 | + result = block.get_supported_filesystems() |
4331 | + self.assertEqual(sorted(self.supported_filesystems), sorted(result)) |
4332 | + |
4333 | + @mock.patch('curtin.block.util') |
4334 | + @mock.patch('curtin.block.os') |
4335 | + def test_get_supported_filesystems_no_proc_path(self, mock_os, mock_util): |
4336 | + """ missing /proc/filesystems raises RuntimeError """ |
4337 | + mock_os.path.exists.return_value = False |
4338 | + with self.assertRaises(RuntimeError): |
4339 | + block.get_supported_filesystems() |
4340 | + self.assertEqual(0, mock_util.load_file.call_count) |
4341 | + |
4342 | + |
4343 | # vi: ts=4 expandtab syntax=python |
4344 | diff --git a/tests/unittests/test_block_iscsi.py b/tests/unittests/test_block_iscsi.py |
4345 | index afaf1f6..f8ef5d8 100644 |
4346 | --- a/tests/unittests/test_block_iscsi.py |
4347 | +++ b/tests/unittests/test_block_iscsi.py |
4348 | @@ -588,6 +588,13 @@ class TestBlockIscsiDiskFromConfig(CiTestCase): |
4349 | # utilize IscsiDisk str method for equality check |
4350 | self.assertEqual(str(expected_iscsi_disk), str(iscsi_disk)) |
4351 | |
4352 | + # test with cfg.get('storage') since caller may already have |
4353 | + # grabbed the 'storage' value from the curtin config |
4354 | + iscsi_disk = iscsi.get_iscsi_disks_from_config( |
4355 | + cfg.get('storage')).pop() |
4356 | + # utilize IscsiDisk str method for equality check |
4357 | + self.assertEqual(str(expected_iscsi_disk), str(iscsi_disk)) |
4358 | + |
4359 | def test_parse_iscsi_disk_from_config_no_iscsi(self): |
4360 | """Test parsing storage config with no iscsi disks included""" |
4361 | cfg = { |
4362 | diff --git a/tests/unittests/test_block_lvm.py b/tests/unittests/test_block_lvm.py |
4363 | index 341f2fa..c92c1ec 100644 |
4364 | --- a/tests/unittests/test_block_lvm.py |
4365 | +++ b/tests/unittests/test_block_lvm.py |
4366 | @@ -73,26 +73,27 @@ class TestBlockLvm(CiTestCase): |
4367 | |
4368 | @mock.patch('curtin.block.lvm.lvmetad_running') |
4369 | @mock.patch('curtin.block.lvm.util') |
4370 | - def test_lvm_scan(self, mock_util, mock_lvmetad): |
4371 | + @mock.patch('curtin.block.lvm.distro') |
4372 | + def test_lvm_scan(self, mock_distro, mock_util, mock_lvmetad): |
4373 | """check that lvm_scan formats commands correctly for each release""" |
4374 | + cmds = [['pvscan'], ['vgscan', '--mknodes']] |
4375 | for (count, (codename, lvmetad_status, use_cache)) in enumerate( |
4376 | - [('precise', False, False), ('precise', True, False), |
4377 | - ('trusty', False, False), ('trusty', True, True), |
4378 | - ('vivid', False, False), ('vivid', True, True), |
4379 | - ('wily', False, False), ('wily', True, True), |
4380 | + [('precise', False, False), |
4381 | + ('trusty', False, False), |
4382 | ('xenial', False, False), ('xenial', True, True), |
4383 | - ('yakkety', True, True), ('UNAVAILABLE', True, True), |
4384 | (None, True, True), (None, False, False)]): |
4385 | - mock_util.lsb_release.return_value = {'codename': codename} |
4386 | + mock_distro.lsb_release.return_value = {'codename': codename} |
4387 | mock_lvmetad.return_value = lvmetad_status |
4388 | lvm.lvm_scan() |
4389 | - self.assertEqual( |
4390 | - len(mock_util.subp.call_args_list), 2 * (count + 1)) |
4391 | - for (expected, actual) in zip( |
4392 | - [['pvscan'], ['vgscan', '--mknodes']], |
4393 | - mock_util.subp.call_args_list[2 * count:2 * count + 2]): |
4394 | - if use_cache: |
4395 | - expected.append('--cache') |
4396 | - self.assertEqual(mock.call(expected, capture=True), actual) |
4397 | + expected = [cmd for cmd in cmds] |
4398 | + for cmd in expected: |
4399 | + if lvmetad_status: |
4400 | + cmd.append('--cache') |
4401 | + |
4402 | + calls = [mock.call(cmd, capture=True) for cmd in expected] |
4403 | + self.assertEqual(len(expected), len(mock_util.subp.call_args_list)) |
4404 | + mock_util.subp.has_calls(calls) |
4405 | + mock_util.subp.reset_mock() |
4406 | + |
4407 | |
4408 | # vi: ts=4 expandtab syntax=python |
4409 | diff --git a/tests/unittests/test_block_mdadm.py b/tests/unittests/test_block_mdadm.py |
4410 | index e2e109c..d017930 100644 |
4411 | --- a/tests/unittests/test_block_mdadm.py |
4412 | +++ b/tests/unittests/test_block_mdadm.py |
4413 | @@ -15,12 +15,13 @@ class TestBlockMdadmAssemble(CiTestCase): |
4414 | def setUp(self): |
4415 | super(TestBlockMdadmAssemble, self).setUp() |
4416 | self.add_patch('curtin.block.mdadm.util', 'mock_util') |
4417 | + self.add_patch('curtin.block.mdadm.lsb_release', 'mock_lsb_release') |
4418 | self.add_patch('curtin.block.mdadm.is_valid_device', 'mock_valid') |
4419 | self.add_patch('curtin.block.mdadm.udev', 'mock_udev') |
4420 | |
4421 | # Common mock settings |
4422 | self.mock_valid.return_value = True |
4423 | - self.mock_util.lsb_release.return_value = {'codename': 'precise'} |
4424 | + self.mock_lsb_release.return_value = {'codename': 'precise'} |
4425 | self.mock_util.subp.return_value = ('', '') |
4426 | |
4427 | def test_mdadm_assemble_scan(self): |
4428 | @@ -88,12 +89,15 @@ class TestBlockMdadmCreate(CiTestCase): |
4429 | def setUp(self): |
4430 | super(TestBlockMdadmCreate, self).setUp() |
4431 | self.add_patch('curtin.block.mdadm.util', 'mock_util') |
4432 | + self.add_patch('curtin.block.mdadm.lsb_release', 'mock_lsb_release') |
4433 | self.add_patch('curtin.block.mdadm.is_valid_device', 'mock_valid') |
4434 | self.add_patch('curtin.block.mdadm.get_holders', 'mock_holders') |
4435 | + self.add_patch('curtin.block.mdadm.udev.udevadm_settle', |
4436 | + 'm_udevadm_settle') |
4437 | |
4438 | # Common mock settings |
4439 | self.mock_valid.return_value = True |
4440 | - self.mock_util.lsb_release.return_value = {'codename': 'precise'} |
4441 | + self.mock_lsb_release.return_value = {'codename': 'precise'} |
4442 | self.mock_holders.return_value = [] |
4443 | |
4444 | def prepare_mock(self, md_devname, raidlevel, devices, spares): |
4445 | @@ -115,8 +119,6 @@ class TestBlockMdadmCreate(CiTestCase): |
4446 | expected_calls.append( |
4447 | call(["mdadm", "--zero-superblock", d], capture=True)) |
4448 | |
4449 | - side_effects.append(("", "")) # udevadm settle |
4450 | - expected_calls.append(call(["udevadm", "settle"])) |
4451 | side_effects.append(("", "")) # udevadm control --stop-exec-queue |
4452 | expected_calls.append(call(["udevadm", "control", |
4453 | "--stop-exec-queue"])) |
4454 | @@ -134,9 +136,6 @@ class TestBlockMdadmCreate(CiTestCase): |
4455 | side_effects.append(("", "")) # udevadm control --start-exec-queue |
4456 | expected_calls.append(call(["udevadm", "control", |
4457 | "--start-exec-queue"])) |
4458 | - side_effects.append(("", "")) # udevadm settle |
4459 | - expected_calls.append(call(["udevadm", "settle", |
4460 | - "--exit-if-exists=%s" % md_devname])) |
4461 | |
4462 | return (side_effects, expected_calls) |
4463 | |
4464 | @@ -154,6 +153,8 @@ class TestBlockMdadmCreate(CiTestCase): |
4465 | mdadm.mdadm_create(md_devname=md_devname, raidlevel=raidlevel, |
4466 | devices=devices, spares=spares) |
4467 | self.mock_util.subp.assert_has_calls(expected_calls) |
4468 | + self.m_udevadm_settle.assert_has_calls( |
4469 | + [call(), call(exists=md_devname)]) |
4470 | |
4471 | def test_mdadm_create_raid0_devshort(self): |
4472 | md_devname = "md0" |
4473 | @@ -237,14 +238,15 @@ class TestBlockMdadmExamine(CiTestCase): |
4474 | def setUp(self): |
4475 | super(TestBlockMdadmExamine, self).setUp() |
4476 | self.add_patch('curtin.block.mdadm.util', 'mock_util') |
4477 | + self.add_patch('curtin.block.mdadm.lsb_release', 'mock_lsb_release') |
4478 | self.add_patch('curtin.block.mdadm.is_valid_device', 'mock_valid') |
4479 | |
4480 | # Common mock settings |
4481 | self.mock_valid.return_value = True |
4482 | - self.mock_util.lsb_release.return_value = {'codename': 'precise'} |
4483 | + self.mock_lsb_release.return_value = {'codename': 'precise'} |
4484 | |
4485 | def test_mdadm_examine_export(self): |
4486 | - self.mock_util.lsb_release.return_value = {'codename': 'xenial'} |
4487 | + self.mock_lsb_release.return_value = {'codename': 'xenial'} |
4488 | self.mock_util.subp.return_value = ( |
4489 | """ |
4490 | MD_LEVEL=raid0 |
4491 | @@ -321,7 +323,7 @@ class TestBlockMdadmExamine(CiTestCase): |
4492 | class TestBlockMdadmStop(CiTestCase): |
4493 | def setUp(self): |
4494 | super(TestBlockMdadmStop, self).setUp() |
4495 | - self.add_patch('curtin.block.mdadm.util.lsb_release', 'mock_util_lsb') |
4496 | + self.add_patch('curtin.block.mdadm.lsb_release', 'mock_lsb_release') |
4497 | self.add_patch('curtin.block.mdadm.util.subp', 'mock_util_subp') |
4498 | self.add_patch('curtin.block.mdadm.util.write_file', |
4499 | 'mock_util_write_file') |
4500 | @@ -334,7 +336,7 @@ class TestBlockMdadmStop(CiTestCase): |
4501 | |
4502 | # Common mock settings |
4503 | self.mock_valid.return_value = True |
4504 | - self.mock_util_lsb.return_value = {'codename': 'xenial'} |
4505 | + self.mock_lsb_release.return_value = {'codename': 'xenial'} |
4506 | self.mock_util_subp.side_effect = iter([ |
4507 | ("", ""), # mdadm stop device |
4508 | ]) |
4509 | @@ -489,11 +491,12 @@ class TestBlockMdadmRemove(CiTestCase): |
4510 | def setUp(self): |
4511 | super(TestBlockMdadmRemove, self).setUp() |
4512 | self.add_patch('curtin.block.mdadm.util', 'mock_util') |
4513 | + self.add_patch('curtin.block.mdadm.lsb_release', 'mock_lsb_release') |
4514 | self.add_patch('curtin.block.mdadm.is_valid_device', 'mock_valid') |
4515 | |
4516 | # Common mock settings |
4517 | self.mock_valid.return_value = True |
4518 | - self.mock_util.lsb_release.return_value = {'codename': 'xenial'} |
4519 | + self.mock_lsb_release.return_value = {'codename': 'xenial'} |
4520 | self.mock_util.subp.side_effect = [ |
4521 | ("", ""), # mdadm remove device |
4522 | ] |
4523 | @@ -515,14 +518,15 @@ class TestBlockMdadmQueryDetail(CiTestCase): |
4524 | def setUp(self): |
4525 | super(TestBlockMdadmQueryDetail, self).setUp() |
4526 | self.add_patch('curtin.block.mdadm.util', 'mock_util') |
4527 | + self.add_patch('curtin.block.mdadm.lsb_release', 'mock_lsb_release') |
4528 | self.add_patch('curtin.block.mdadm.is_valid_device', 'mock_valid') |
4529 | |
4530 | # Common mock settings |
4531 | self.mock_valid.return_value = True |
4532 | - self.mock_util.lsb_release.return_value = {'codename': 'precise'} |
4533 | + self.mock_lsb_release.return_value = {'codename': 'precise'} |
4534 | |
4535 | def test_mdadm_query_detail_export(self): |
4536 | - self.mock_util.lsb_release.return_value = {'codename': 'xenial'} |
4537 | + self.mock_lsb_release.return_value = {'codename': 'xenial'} |
4538 | self.mock_util.subp.return_value = ( |
4539 | """ |
4540 | MD_LEVEL=raid1 |
4541 | @@ -593,13 +597,14 @@ class TestBlockMdadmDetailScan(CiTestCase): |
4542 | def setUp(self): |
4543 | super(TestBlockMdadmDetailScan, self).setUp() |
4544 | self.add_patch('curtin.block.mdadm.util', 'mock_util') |
4545 | + self.add_patch('curtin.block.mdadm.lsb_release', 'mock_lsb_release') |
4546 | self.add_patch('curtin.block.mdadm.is_valid_device', 'mock_valid') |
4547 | |
4548 | # Common mock settings |
4549 | self.scan_output = ("ARRAY /dev/md0 metadata=1.2 spares=2 name=0 " + |
4550 | "UUID=b1eae2ff:69b6b02e:1d63bb53:ddfa6e4a") |
4551 | self.mock_valid.return_value = True |
4552 | - self.mock_util.lsb_release.return_value = {'codename': 'xenial'} |
4553 | + self.mock_lsb_release.return_value = {'codename': 'xenial'} |
4554 | self.mock_util.subp.side_effect = [ |
4555 | (self.scan_output, ""), # mdadm --detail --scan |
4556 | ] |
4557 | @@ -628,10 +633,11 @@ class TestBlockMdadmMdHelpers(CiTestCase): |
4558 | def setUp(self): |
4559 | super(TestBlockMdadmMdHelpers, self).setUp() |
4560 | self.add_patch('curtin.block.mdadm.util', 'mock_util') |
4561 | + self.add_patch('curtin.block.mdadm.lsb_release', 'mock_lsb_release') |
4562 | self.add_patch('curtin.block.mdadm.is_valid_device', 'mock_valid') |
4563 | |
4564 | self.mock_valid.return_value = True |
4565 | - self.mock_util.lsb_release.return_value = {'codename': 'xenial'} |
4566 | + self.mock_lsb_release.return_value = {'codename': 'xenial'} |
4567 | |
4568 | def test_valid_mdname(self): |
4569 | mdname = "/dev/md0" |
4570 | diff --git a/tests/unittests/test_block_mkfs.py b/tests/unittests/test_block_mkfs.py |
4571 | index c756281..679f85b 100644 |
4572 | --- a/tests/unittests/test_block_mkfs.py |
4573 | +++ b/tests/unittests/test_block_mkfs.py |
4574 | @@ -37,11 +37,12 @@ class TestBlockMkfs(CiTestCase): |
4575 | @mock.patch("curtin.block.mkfs.block") |
4576 | @mock.patch("curtin.block.mkfs.os") |
4577 | @mock.patch("curtin.block.mkfs.util") |
4578 | + @mock.patch("curtin.block.mkfs.distro.lsb_release") |
4579 | def _run_mkfs_with_config(self, config, expected_cmd, expected_flags, |
4580 | - mock_util, mock_os, mock_block, |
4581 | + mock_lsb_release, mock_util, mock_os, mock_block, |
4582 | release="wily", strict=False): |
4583 | # Pretend we are on wily as there are no known edge cases for it |
4584 | - mock_util.lsb_release.return_value = {"codename": release} |
4585 | + mock_lsb_release.return_value = {"codename": release} |
4586 | mock_os.path.exists.return_value = True |
4587 | mock_block.get_blockdev_sector_size.return_value = (512, 512) |
4588 | |
4589 | diff --git a/tests/unittests/test_block_zfs.py b/tests/unittests/test_block_zfs.py |
4590 | index c61a6da..9781946 100644 |
4591 | --- a/tests/unittests/test_block_zfs.py |
4592 | +++ b/tests/unittests/test_block_zfs.py |
4593 | @@ -378,15 +378,20 @@ class TestBlockZfsDeviceToPoolname(CiTestCase): |
4594 | self.mock_blkid.assert_called_with(devs=[devname]) |
4595 | |
4596 | |
4597 | -class TestBlockZfsZfsSupported(CiTestCase): |
4598 | +class TestBlockZfsAssertZfsSupported(CiTestCase): |
4599 | |
4600 | def setUp(self): |
4601 | - super(TestBlockZfsZfsSupported, self).setUp() |
4602 | + super(TestBlockZfsAssertZfsSupported, self).setUp() |
4603 | self.add_patch('curtin.block.zfs.util.subp', 'mock_subp') |
4604 | self.add_patch('curtin.block.zfs.util.get_platform_arch', 'mock_arch') |
4605 | - self.add_patch('curtin.block.zfs.util.lsb_release', 'mock_release') |
4606 | - self.mock_release.return_value = {'codename': 'xenial'} |
4607 | + self.add_patch('curtin.block.zfs.distro.lsb_release', 'mock_release') |
4608 | + self.add_patch('curtin.block.zfs.util.which', 'mock_which') |
4609 | + self.add_patch('curtin.block.zfs.get_supported_filesystems', |
4610 | + 'mock_supfs') |
4611 | self.mock_arch.return_value = 'x86_64' |
4612 | + self.mock_release.return_value = {'codename': 'xenial'} |
4613 | + self.mock_supfs.return_value = ['zfs'] |
4614 | + self.mock_which.return_value = True |
4615 | |
4616 | def test_supported_arch(self): |
4617 | self.assertTrue(zfs.zfs_supported()) |
4618 | @@ -394,81 +399,143 @@ class TestBlockZfsZfsSupported(CiTestCase): |
4619 | def test_unsupported_arch(self): |
4620 | self.mock_arch.return_value = 'i386' |
4621 | with self.assertRaises(RuntimeError): |
4622 | - zfs.zfs_supported() |
4623 | + zfs.zfs_assert_supported() |
4624 | |
4625 | def test_unsupported_releases(self): |
4626 | for rel in ['precise', 'trusty']: |
4627 | self.mock_release.return_value = {'codename': rel} |
4628 | with self.assertRaises(RuntimeError): |
4629 | - zfs.zfs_supported() |
4630 | + zfs.zfs_assert_supported() |
4631 | |
4632 | - def test_missing_module(self): |
4633 | - missing = 'modinfo: ERROR: Module zfs not found.\n ' |
4634 | + @mock.patch('curtin.block.zfs.util.is_kmod_loaded') |
4635 | + @mock.patch('curtin.block.zfs.get_supported_filesystems') |
4636 | + def test_missing_module(self, mock_supfs, mock_kmod): |
4637 | + missing = 'modprobe: FATAL: Module zfs not found.\n ' |
4638 | self.mock_subp.side_effect = ProcessExecutionError(stdout='', |
4639 | stderr=missing, |
4640 | exit_code='1') |
4641 | + mock_supfs.return_value = ['ext4'] |
4642 | + mock_kmod.return_value = False |
4643 | with self.assertRaises(RuntimeError): |
4644 | - zfs.zfs_supported() |
4645 | + zfs.zfs_assert_supported() |
4646 | |
4647 | |
4648 | -class TestZfsSupported(CiTestCase): |
4649 | +class TestAssertZfsSupported(CiTestCase): |
4650 | |
4651 | def setUp(self): |
4652 | - super(TestZfsSupported, self).setUp() |
4653 | + super(TestAssertZfsSupported, self).setUp() |
4654 | |
4655 | + @mock.patch('curtin.block.zfs.get_supported_filesystems') |
4656 | + @mock.patch('curtin.block.zfs.distro') |
4657 | @mock.patch('curtin.block.zfs.util') |
4658 | - def test_zfs_supported_returns_true(self, mock_util): |
4659 | - """zfs_supported returns True on supported platforms""" |
4660 | + def test_zfs_assert_supported_returns_true(self, mock_util, mock_distro, |
4661 | + mock_supfs): |
4662 | + """zfs_assert_supported returns True on supported platforms""" |
4663 | mock_util.get_platform_arch.return_value = 'amd64' |
4664 | - mock_util.lsb_release.return_value = {'codename': 'bionic'} |
4665 | + mock_distro.lsb_release.return_value = {'codename': 'bionic'} |
4666 | mock_util.subp.return_value = ("", "") |
4667 | + mock_supfs.return_value = ['zfs'] |
4668 | + mock_util.which.side_effect = iter(['/wark/zpool', '/wark/zfs']) |
4669 | |
4670 | self.assertNotIn(mock_util.get_platform_arch.return_value, |
4671 | zfs.ZFS_UNSUPPORTED_ARCHES) |
4672 | - self.assertNotIn(mock_util.lsb_release.return_value['codename'], |
4673 | + self.assertNotIn(mock_distro.lsb_release.return_value['codename'], |
4674 | zfs.ZFS_UNSUPPORTED_RELEASES) |
4675 | self.assertTrue(zfs.zfs_supported()) |
4676 | |
4677 | + @mock.patch('curtin.block.zfs.distro') |
4678 | @mock.patch('curtin.block.zfs.util') |
4679 | - def test_zfs_supported_raises_exception_on_bad_arch(self, mock_util): |
4680 | - """zfs_supported raises RuntimeError on unspported arches""" |
4681 | - mock_util.lsb_release.return_value = {'codename': 'bionic'} |
4682 | + def test_zfs_assert_supported_raises_exception_on_bad_arch(self, |
4683 | + mock_util, |
4684 | + mock_distro): |
4685 | + """zfs_assert_supported raises RuntimeError on unspported arches""" |
4686 | + mock_distro.lsb_release.return_value = {'codename': 'bionic'} |
4687 | mock_util.subp.return_value = ("", "") |
4688 | for arch in zfs.ZFS_UNSUPPORTED_ARCHES: |
4689 | mock_util.get_platform_arch.return_value = arch |
4690 | with self.assertRaises(RuntimeError): |
4691 | - zfs.zfs_supported() |
4692 | + zfs.zfs_assert_supported() |
4693 | |
4694 | + @mock.patch('curtin.block.zfs.distro') |
4695 | @mock.patch('curtin.block.zfs.util') |
4696 | - def test_zfs_supported_raises_execption_on_bad_releases(self, mock_util): |
4697 | - """zfs_supported raises RuntimeError on unspported releases""" |
4698 | + def test_zfs_assert_supported_raises_exc_on_bad_releases(self, mock_util, |
4699 | + mock_distro): |
4700 | + """zfs_assert_supported raises RuntimeError on unspported releases""" |
4701 | mock_util.get_platform_arch.return_value = 'amd64' |
4702 | mock_util.subp.return_value = ("", "") |
4703 | for release in zfs.ZFS_UNSUPPORTED_RELEASES: |
4704 | - mock_util.lsb_release.return_value = {'codename': release} |
4705 | + mock_distro.lsb_release.return_value = {'codename': release} |
4706 | with self.assertRaises(RuntimeError): |
4707 | - zfs.zfs_supported() |
4708 | + zfs.zfs_assert_supported() |
4709 | |
4710 | @mock.patch('curtin.block.zfs.util.subprocess.Popen') |
4711 | - @mock.patch('curtin.block.zfs.util.lsb_release') |
4712 | + @mock.patch('curtin.block.zfs.util.is_kmod_loaded') |
4713 | + @mock.patch('curtin.block.zfs.get_supported_filesystems') |
4714 | + @mock.patch('curtin.block.zfs.distro.lsb_release') |
4715 | @mock.patch('curtin.block.zfs.util.get_platform_arch') |
4716 | - def test_zfs_supported_raises_exception_on_missing_module(self, |
4717 | - m_arch, |
4718 | - m_release, |
4719 | - m_popen): |
4720 | - """zfs_supported raises RuntimeError on missing zfs module""" |
4721 | + def test_zfs_assert_supported_raises_exc_on_missing_module(self, |
4722 | + m_arch, |
4723 | + m_release, |
4724 | + m_supfs, |
4725 | + m_kmod, |
4726 | + m_popen, |
4727 | + ): |
4728 | + """zfs_assert_supported raises RuntimeError modprobe zfs error""" |
4729 | |
4730 | m_arch.return_value = 'amd64' |
4731 | m_release.return_value = {'codename': 'bionic'} |
4732 | + m_supfs.return_value = ['ext4'] |
4733 | + m_kmod.return_value = False |
4734 | process_mock = mock.Mock() |
4735 | attrs = { |
4736 | 'returncode': 1, |
4737 | 'communicate.return_value': |
4738 | - ('output', "modinfo: ERROR: Module zfs not found."), |
4739 | + ('output', 'modprobe: FATAL: Module zfs not found ...'), |
4740 | } |
4741 | process_mock.configure_mock(**attrs) |
4742 | m_popen.return_value = process_mock |
4743 | with self.assertRaises(RuntimeError): |
4744 | - zfs.zfs_supported() |
4745 | + zfs.zfs_assert_supported() |
4746 | + |
4747 | + @mock.patch('curtin.block.zfs.get_supported_filesystems') |
4748 | + @mock.patch('curtin.block.zfs.util.lsb_release') |
4749 | + @mock.patch('curtin.block.zfs.util.get_platform_arch') |
4750 | + @mock.patch('curtin.block.zfs.util') |
4751 | + def test_zfs_assert_supported_raises_exc_on_missing_binaries(self, |
4752 | + mock_util, |
4753 | + m_arch, |
4754 | + m_release, |
4755 | + m_supfs): |
4756 | + """zfs_assert_supported raises RuntimeError if no zpool or zfs tools""" |
4757 | + mock_util.get_platform_arch.return_value = 'amd64' |
4758 | + mock_util.lsb_release.return_value = {'codename': 'bionic'} |
4759 | + mock_util.subp.return_value = ("", "") |
4760 | + m_supfs.return_value = ['zfs'] |
4761 | + mock_util.which.return_value = None |
4762 | + |
4763 | + with self.assertRaises(RuntimeError): |
4764 | + zfs.zfs_assert_supported() |
4765 | + |
4766 | + |
4767 | +class TestZfsSupported(CiTestCase): |
4768 | + |
4769 | + @mock.patch('curtin.block.zfs.zfs_assert_supported') |
4770 | + def test_zfs_supported(self, m_assert_zfs): |
4771 | + zfs_supported = True |
4772 | + m_assert_zfs.return_value = zfs_supported |
4773 | + |
4774 | + result = zfs.zfs_supported() |
4775 | + self.assertEqual(zfs_supported, result) |
4776 | + self.assertEqual(1, m_assert_zfs.call_count) |
4777 | + |
4778 | + @mock.patch('curtin.block.zfs.zfs_assert_supported') |
4779 | + def test_zfs_supported_returns_false_on_assert_fail(self, m_assert_zfs): |
4780 | + zfs_supported = False |
4781 | + m_assert_zfs.side_effect = RuntimeError('No zfs module') |
4782 | + |
4783 | + result = zfs.zfs_supported() |
4784 | + self.assertEqual(zfs_supported, result) |
4785 | + self.assertEqual(1, m_assert_zfs.call_count) |
4786 | + |
4787 | |
4788 | # vi: ts=4 expandtab syntax=python |
4789 | diff --git a/tests/unittests/test_clear_holders.py b/tests/unittests/test_clear_holders.py |
4790 | index ceb5615..d3f80a0 100644 |
4791 | --- a/tests/unittests/test_clear_holders.py |
4792 | +++ b/tests/unittests/test_clear_holders.py |
4793 | @@ -6,11 +6,12 @@ import os |
4794 | import textwrap |
4795 | |
4796 | from curtin.block import clear_holders |
4797 | +from curtin.util import ProcessExecutionError |
4798 | from .helpers import CiTestCase |
4799 | |
4800 | |
4801 | class TestClearHolders(CiTestCase): |
4802 | - test_blockdev = '/dev/null' |
4803 | + test_blockdev = '/wark/dev/null' |
4804 | test_syspath = '/sys/class/block/null' |
4805 | remove_retries = [0.2] * 150 # clear_holders defaults to 30 seconds |
4806 | example_holders_trees = [ |
4807 | @@ -153,7 +154,7 @@ class TestClearHolders(CiTestCase): |
4808 | # |
4809 | |
4810 | device = self.test_syspath |
4811 | - mock_block.sys_block_path.return_value = '/dev/null' |
4812 | + mock_block.sys_block_path.return_value = self.test_blockdev |
4813 | bcache_cset_uuid = 'c08ae789-a964-46fb-a66e-650f0ae78f94' |
4814 | |
4815 | mock_os.path.exists.return_value = True |
4816 | @@ -189,9 +190,8 @@ class TestClearHolders(CiTestCase): |
4817 | def test_shutdown_bcache_non_sysfs_device(self, mock_get_bcache, mock_log, |
4818 | mock_os, mock_util, |
4819 | mock_get_bcache_block): |
4820 | - device = "/dev/fakenull" |
4821 | with self.assertRaises(ValueError): |
4822 | - clear_holders.shutdown_bcache(device) |
4823 | + clear_holders.shutdown_bcache(self.test_blockdev) |
4824 | |
4825 | self.assertEqual(0, len(mock_get_bcache.call_args_list)) |
4826 | self.assertEqual(0, len(mock_log.call_args_list)) |
4827 | @@ -208,11 +208,10 @@ class TestClearHolders(CiTestCase): |
4828 | def test_shutdown_bcache_no_device(self, mock_get_bcache, mock_log, |
4829 | mock_os, mock_util, |
4830 | mock_get_bcache_block, mock_block): |
4831 | - device = "/sys/class/block/null" |
4832 | - mock_block.sysfs_to_devpath.return_value = '/dev/null' |
4833 | + mock_block.sysfs_to_devpath.return_value = self.test_blockdev |
4834 | mock_os.path.exists.return_value = False |
4835 | |
4836 | - clear_holders.shutdown_bcache(device) |
4837 | + clear_holders.shutdown_bcache(self.test_syspath) |
4838 | |
4839 | self.assertEqual(3, len(mock_log.info.call_args_list)) |
4840 | self.assertEqual(1, len(mock_os.path.exists.call_args_list)) |
4841 | @@ -229,18 +228,17 @@ class TestClearHolders(CiTestCase): |
4842 | def test_shutdown_bcache_no_cset(self, mock_get_bcache, mock_log, |
4843 | mock_os, mock_util, |
4844 | mock_get_bcache_block, mock_block): |
4845 | - device = "/sys/class/block/null" |
4846 | - mock_block.sysfs_to_devpath.return_value = '/dev/null' |
4847 | + mock_block.sysfs_to_devpath.return_value = self.test_blockdev |
4848 | mock_os.path.exists.side_effect = iter([ |
4849 | True, # backing device exists |
4850 | False, # cset device not present (already removed) |
4851 | True, # backing device (still) exists |
4852 | ]) |
4853 | mock_get_bcache.return_value = '/sys/fs/bcache/fake' |
4854 | - mock_get_bcache_block.return_value = device + '/bcache' |
4855 | + mock_get_bcache_block.return_value = self.test_syspath + '/bcache' |
4856 | mock_os.path.join.side_effect = os.path.join |
4857 | |
4858 | - clear_holders.shutdown_bcache(device) |
4859 | + clear_holders.shutdown_bcache(self.test_syspath) |
4860 | |
4861 | self.assertEqual(4, len(mock_log.info.call_args_list)) |
4862 | self.assertEqual(3, len(mock_os.path.exists.call_args_list)) |
4863 | @@ -249,14 +247,15 @@ class TestClearHolders(CiTestCase): |
4864 | self.assertEqual(1, len(mock_util.write_file.call_args_list)) |
4865 | self.assertEqual(2, len(mock_util.wait_for_removal.call_args_list)) |
4866 | |
4867 | - mock_get_bcache.assert_called_with(device, strict=False) |
4868 | - mock_get_bcache_block.assert_called_with(device, strict=False) |
4869 | - mock_util.write_file.assert_called_with(device + '/bcache/stop', |
4870 | - '1', mode=None) |
4871 | + mock_get_bcache.assert_called_with(self.test_syspath, strict=False) |
4872 | + mock_get_bcache_block.assert_called_with(self.test_syspath, |
4873 | + strict=False) |
4874 | + mock_util.write_file.assert_called_with( |
4875 | + self.test_syspath + '/bcache/stop', '1', mode=None) |
4876 | retries = self.remove_retries |
4877 | mock_util.wait_for_removal.assert_has_calls([ |
4878 | - mock.call(device, retries=retries), |
4879 | - mock.call(device + '/bcache', retries=retries)]) |
4880 | + mock.call(self.test_syspath, retries=retries), |
4881 | + mock.call(self.test_syspath + '/bcache', retries=retries)]) |
4882 | |
4883 | @mock.patch('curtin.block.clear_holders.block') |
4884 | @mock.patch('curtin.block.clear_holders.udev.udevadm_settle') |
4885 | @@ -271,8 +270,7 @@ class TestClearHolders(CiTestCase): |
4886 | mock_get_bcache_block, |
4887 | mock_udevadm_settle, |
4888 | mock_block): |
4889 | - device = "/sys/class/block/null" |
4890 | - mock_block.sysfs_to_devpath.return_value = '/dev/null' |
4891 | + mock_block.sysfs_to_devpath.return_value = self.test_blockdev |
4892 | mock_os.path.exists.side_effect = iter([ |
4893 | True, # backing device exists |
4894 | True, # cset device not present (already removed) |
4895 | @@ -280,10 +278,10 @@ class TestClearHolders(CiTestCase): |
4896 | ]) |
4897 | cset = '/sys/fs/bcache/fake' |
4898 | mock_get_bcache.return_value = cset |
4899 | - mock_get_bcache_block.return_value = device + '/bcache' |
4900 | + mock_get_bcache_block.return_value = self.test_syspath + '/bcache' |
4901 | mock_os.path.join.side_effect = os.path.join |
4902 | |
4903 | - clear_holders.shutdown_bcache(device) |
4904 | + clear_holders.shutdown_bcache(self.test_syspath) |
4905 | |
4906 | self.assertEqual(4, len(mock_log.info.call_args_list)) |
4907 | self.assertEqual(3, len(mock_os.path.exists.call_args_list)) |
4908 | @@ -292,14 +290,15 @@ class TestClearHolders(CiTestCase): |
4909 | self.assertEqual(2, len(mock_util.write_file.call_args_list)) |
4910 | self.assertEqual(3, len(mock_util.wait_for_removal.call_args_list)) |
4911 | |
4912 | - mock_get_bcache.assert_called_with(device, strict=False) |
4913 | - mock_get_bcache_block.assert_called_with(device, strict=False) |
4914 | + mock_get_bcache.assert_called_with(self.test_syspath, strict=False) |
4915 | + mock_get_bcache_block.assert_called_with(self.test_syspath, |
4916 | + strict=False) |
4917 | mock_util.write_file.assert_has_calls([ |
4918 | mock.call(cset + '/stop', '1', mode=None), |
4919 | - mock.call(device + '/bcache/stop', '1', mode=None)]) |
4920 | + mock.call(self.test_syspath + '/bcache/stop', '1', mode=None)]) |
4921 | mock_util.wait_for_removal.assert_has_calls([ |
4922 | mock.call(cset, retries=self.remove_retries), |
4923 | - mock.call(device, retries=self.remove_retries) |
4924 | + mock.call(self.test_syspath, retries=self.remove_retries) |
4925 | ]) |
4926 | |
4927 | @mock.patch('curtin.block.clear_holders.block') |
4928 | @@ -315,8 +314,7 @@ class TestClearHolders(CiTestCase): |
4929 | mock_get_bcache_block, |
4930 | mock_udevadm_settle, |
4931 | mock_block): |
4932 | - device = "/sys/class/block/null" |
4933 | - mock_block.sysfs_to_devpath.return_value = '/dev/null' |
4934 | + mock_block.sysfs_to_devpath.return_value = self.test_blockdev |
4935 | mock_os.path.exists.side_effect = iter([ |
4936 | True, # backing device exists |
4937 | True, # cset device not present (already removed) |
4938 | @@ -324,10 +322,10 @@ class TestClearHolders(CiTestCase): |
4939 | ]) |
4940 | cset = '/sys/fs/bcache/fake' |
4941 | mock_get_bcache.return_value = cset |
4942 | - mock_get_bcache_block.return_value = device + '/bcache' |
4943 | + mock_get_bcache_block.return_value = self.test_syspath + '/bcache' |
4944 | mock_os.path.join.side_effect = os.path.join |
4945 | |
4946 | - clear_holders.shutdown_bcache(device) |
4947 | + clear_holders.shutdown_bcache(self.test_syspath) |
4948 | |
4949 | self.assertEqual(4, len(mock_log.info.call_args_list)) |
4950 | self.assertEqual(3, len(mock_os.path.exists.call_args_list)) |
4951 | @@ -336,7 +334,7 @@ class TestClearHolders(CiTestCase): |
4952 | self.assertEqual(1, len(mock_util.write_file.call_args_list)) |
4953 | self.assertEqual(1, len(mock_util.wait_for_removal.call_args_list)) |
4954 | |
4955 | - mock_get_bcache.assert_called_with(device, strict=False) |
4956 | + mock_get_bcache.assert_called_with(self.test_syspath, strict=False) |
4957 | mock_util.write_file.assert_has_calls([ |
4958 | mock.call(cset + '/stop', '1', mode=None), |
4959 | ]) |
4960 | @@ -361,8 +359,7 @@ class TestClearHolders(CiTestCase): |
4961 | mock_wipe, |
4962 | mock_block): |
4963 | """Test writes sysfs write failures pass if file not present""" |
4964 | - device = "/sys/class/block/null" |
4965 | - mock_block.sysfs_to_devpath.return_value = '/dev/null' |
4966 | + mock_block.sysfs_to_devpath.return_value = self.test_blockdev |
4967 | mock_os.path.exists.side_effect = iter([ |
4968 | True, # backing device exists |
4969 | True, # cset device not present (already removed) |
4970 | @@ -371,14 +368,14 @@ class TestClearHolders(CiTestCase): |
4971 | ]) |
4972 | cset = '/sys/fs/bcache/fake' |
4973 | mock_get_bcache.return_value = cset |
4974 | - mock_get_bcache_block.return_value = device + '/bcache' |
4975 | + mock_get_bcache_block.return_value = self.test_syspath + '/bcache' |
4976 | mock_os.path.join.side_effect = os.path.join |
4977 | |
4978 | # make writes to sysfs fail |
4979 | mock_util.write_file.side_effect = IOError(errno.ENOENT, |
4980 | "File not found") |
4981 | |
4982 | - clear_holders.shutdown_bcache(device) |
4983 | + clear_holders.shutdown_bcache(self.test_syspath) |
4984 | |
4985 | self.assertEqual(4, len(mock_log.info.call_args_list)) |
4986 | self.assertEqual(3, len(mock_os.path.exists.call_args_list)) |
4987 | @@ -387,7 +384,7 @@ class TestClearHolders(CiTestCase): |
4988 | self.assertEqual(1, len(mock_util.write_file.call_args_list)) |
4989 | self.assertEqual(1, len(mock_util.wait_for_removal.call_args_list)) |
4990 | |
4991 | - mock_get_bcache.assert_called_with(device, strict=False) |
4992 | + mock_get_bcache.assert_called_with(self.test_syspath, strict=False) |
4993 | mock_util.write_file.assert_has_calls([ |
4994 | mock.call(cset + '/stop', '1', mode=None), |
4995 | ]) |
4996 | @@ -528,10 +525,15 @@ class TestClearHolders(CiTestCase): |
4997 | self.assertTrue(mock_log.debug.called) |
4998 | self.assertTrue(mock_log.critical.called) |
4999 | |
5000 | + @mock.patch('curtin.block.clear_holders.is_swap_device') |
The diff has been truncated for viewing.
PASSED: Continuous integration, rev:be0f90a7fa3 477255b1994dd46 731481f5580072 /jenkins. ubuntu. com/server/ job/curtin- ci/1069/ /jenkins. ubuntu. com/server/ job/curtin- ci/nodes= metal-arm64/ 1069 /jenkins. ubuntu. com/server/ job/curtin- ci/nodes= metal-ppc64el/ 1069 /jenkins. ubuntu. com/server/ job/curtin- ci/nodes= metal-s390x/ 1069 /jenkins. ubuntu. com/server/ job/curtin- ci/nodes= torkoal/ 1069
https:/
Executed test runs:
SUCCESS: https:/
SUCCESS: https:/
SUCCESS: https:/
SUCCESS: https:/
Click here to trigger a rebuild: /jenkins. ubuntu. com/server/ job/curtin- ci/1069/ rebuild
https:/