Merge ~dbungert/curtin:ubuntu/devel into curtin:ubuntu/devel

Proposed by Dan Bungert
Status: Merged
Merged at revision: 90b8f87ec035980c51af88de84b36e1275532fe7
Proposed branch: ~dbungert/curtin:ubuntu/devel
Merge into: curtin:ubuntu/devel
Diff against target: 4095 lines (+2041/-504)
64 files modified
curtin/__init__.py (+1/-1)
curtin/block/__init__.py (+11/-9)
curtin/block/deps.py (+6/-2)
curtin/block/schemas.py (+10/-0)
curtin/commands/apt_config.py (+50/-4)
curtin/commands/block_meta.py (+44/-8)
curtin/commands/block_meta_v2.py (+422/-0)
curtin/commands/install_grub.py (+3/-0)
curtin/distro.py (+6/-2)
curtin/storage_config.py (+15/-3)
curtin/util.py (+7/-0)
debian/changelog (+69/-0)
debian/control (+4/-0)
dev/null (+0/-101)
doc/topics/apt_source.rst (+2/-0)
doc/topics/storage.rst (+81/-12)
examples/apt-source.yaml (+14/-2)
examples/tests/basic.yaml (+1/-1)
examples/tests/basic_iscsi.yaml (+1/-1)
examples/tests/bcache-partitions.yaml (+0/-1)
examples/tests/lvm.yaml (+1/-1)
examples/tests/lvm_iscsi.yaml (+3/-3)
examples/tests/mirrorboot.yaml (+2/-2)
examples/tests/multipath-lvm-part-wipe.yaml (+1/-1)
examples/tests/multipath-reuse.yaml (+5/-3)
examples/tests/multipath.yaml (+1/-1)
examples/tests/partition-existing-raid.yaml (+4/-2)
examples/tests/preserve-bcache.yaml (+3/-0)
examples/tests/preserve-lvm.yaml (+2/-0)
examples/tests/preserve-partition-wipe-vg-simple.yaml (+2/-0)
examples/tests/preserve-partition-wipe-vg.yaml (+4/-0)
examples/tests/preserve-raid.yaml (+3/-1)
examples/tests/preserve.yaml (+6/-3)
examples/tests/reuse-lvm-member-partition.yaml (+2/-0)
examples/tests/reuse-msdos-partitions.yaml (+4/-0)
examples/tests/reuse-raid-member-wipe-partition.yaml (+2/-0)
examples/tests/uefi_basic.yaml (+1/-1)
examples/tests/uefi_reuse_esp.yaml (+3/-2)
pylintrc (+1/-1)
test-requirements.txt (+1/-0)
tests/integration/test_block_meta.py (+840/-58)
tests/unittests/test_apt_source.py (+49/-0)
tests/unittests/test_block.py (+8/-0)
tests/unittests/test_commands_block_meta.py (+264/-2)
tests/unittests/test_storage_config.py (+60/-15)
tests/vmtests/__init__.py (+0/-35)
tests/vmtests/releases.py (+1/-26)
tests/vmtests/test_basic.py (+1/-26)
tests/vmtests/test_network.py (+0/-5)
tests/vmtests/test_network_alias.py (+0/-5)
tests/vmtests/test_network_bonding.py (+0/-5)
tests/vmtests/test_network_bridging.py (+0/-7)
tests/vmtests/test_network_ipv6.py (+0/-5)
tests/vmtests/test_network_ipv6_static.py (+0/-5)
tests/vmtests/test_network_ipv6_vlan.py (+0/-5)
tests/vmtests/test_network_mtu.py (+0/-5)
tests/vmtests/test_network_static.py (+0/-5)
tests/vmtests/test_network_static_routes.py (+0/-5)
tests/vmtests/test_network_vlan.py (+0/-5)
tests/vmtests/test_preserve_raid.py (+3/-0)
tests/vmtests/test_simple.py (+0/-9)
tests/vmtests/test_uefi_basic.py (+5/-19)
tools/build-deb (+10/-41)
tox.ini (+2/-48)
Reviewer Review Type Date Requested Status
Server Team CI bot continuous-integration Approve
curtin developers Pending
Review via email: mp+425672@code.launchpad.net

Commit message

Version 22.1

To post a comment you must log in.
Revision history for this message
Server Team CI bot (server-team-bot) wrote :
review: Approve (continuous-integration)
Revision history for this message
Dan Bungert (dbungert) wrote :

Merging manually so we don't get the autolander squash.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/curtin/__init__.py b/curtin/__init__.py
2index 8a3e850..036cd5d 100644
3--- a/curtin/__init__.py
4+++ b/curtin/__init__.py
5@@ -40,6 +40,6 @@ FEATURES = [
6 'FSTAB_DEFAULT_FSCK_ON_BLK'
7 ]
8
9-__version__ = "21.3"
10+__version__ = "22.1"
11
12 # vi: ts=4 expandtab syntax=python
13diff --git a/curtin/block/__init__.py b/curtin/block/__init__.py
14index ca0bc10..49b062f 100644
15--- a/curtin/block/__init__.py
16+++ b/curtin/block/__init__.py
17@@ -993,19 +993,12 @@ def sysfs_partition_data(blockdev=None, sysfs_path=None):
18 else:
19 raise ValueError("Blockdev and sysfs_path cannot both be None")
20
21- # queue property is only on parent devices, ie, we can't read
22- # /sys/class/block/vda/vda1/queue/* as queue is only on the
23- # parent device
24 sysfs_prefix = sysfs_path
25 (parent, partnum) = get_blockdev_for_partition(blockdev)
26 if partnum:
27 sysfs_prefix = sys_block_path(parent)
28 partnum = int(partnum)
29
30- block_size = int(util.load_file(os.path.join(
31- sysfs_prefix, 'queue/logical_block_size')))
32- unit = block_size
33-
34 ptdata = []
35 for part_sysfs in get_sysfs_partitions(sysfs_prefix):
36 data = {}
37@@ -1015,8 +1008,12 @@ def sysfs_partition_data(blockdev=None, sysfs_path=None):
38 continue
39 data[sfile] = int(util.load_file(dfile))
40 if partnum is None or data['partition'] == partnum:
41- ptdata.append((path_to_kname(part_sysfs), data['partition'],
42- data['start'] * unit, data['size'] * unit,))
43+ ptdata.append((
44+ path_to_kname(part_sysfs),
45+ data['partition'],
46+ data['start'] * SECTOR_SIZE_BYTES,
47+ data['size'] * SECTOR_SIZE_BYTES,
48+ ))
49
50 return ptdata
51
52@@ -1371,4 +1368,9 @@ def discover():
53 return {}
54
55
56+def get_resize_fstypes():
57+ from curtin.commands.block_meta_v2 import resizers
58+ return {fstype for fstype in resizers.keys()}
59+
60+
61 # vi: ts=4 expandtab syntax=python
62diff --git a/curtin/block/deps.py b/curtin/block/deps.py
63index 38581a8..db449d8 100644
64--- a/curtin/block/deps.py
65+++ b/curtin/block/deps.py
66@@ -96,8 +96,12 @@ def detect_required_packages_mapping(osfamily=DISTROS.debian):
67 if osfamily not in distro_mapping:
68 raise ValueError('No block package mapping for distro: %s' % osfamily)
69
70- return {1: {'handler': storage_config_required_packages,
71- 'mapping': distro_mapping.get(osfamily)}}
72+ cfg_map = {
73+ 'handler': storage_config_required_packages,
74+ 'mapping': distro_mapping.get(osfamily),
75+ }
76+
77+ return {1: cfg_map, 2: cfg_map}
78
79
80 # vi: ts=4 expandtab syntax=python
81diff --git a/curtin/block/schemas.py b/curtin/block/schemas.py
82index 84a5279..92f88d0 100644
83--- a/curtin/block/schemas.py
84+++ b/curtin/block/schemas.py
85@@ -284,8 +284,13 @@ PARTITION = {
86 'properties': {
87 'id': {'$ref': '#/definitions/id'},
88 'multipath': {'type': 'string'},
89+ # Permit path to device as output.
90+ # This value is ignored for input.
91+ 'path': {'type': 'string',
92+ 'pattern': _path_dev},
93 'name': {'$ref': '#/definitions/name'},
94 'offset': {'$ref': '#/definitions/size'}, # XXX: This is not used
95+ 'resize': {'type': 'boolean'},
96 'preserve': {'$ref': '#/definitions/preserve'},
97 'size': {'$ref': '#/definitions/size'},
98 'uuid': {'$ref': '#/definitions/uuid'}, # XXX: This is not used
99@@ -299,6 +304,11 @@ PARTITION = {
100 'enum': ['bios_grub', 'boot', 'extended', 'home', 'linux',
101 'logical', 'lvm', 'mbr', 'prep', 'raid', 'swap',
102 '']},
103+ 'partition_type': {'type': 'string',
104+ 'oneOf': [
105+ {'pattern': r'^0x[0-9a-fA-F]{1,2}$'},
106+ {'$ref': '#/definitions/uuid'},
107+ ]},
108 'grub_device': {
109 'type': ['boolean', 'integer'],
110 'minimum': 0,
111diff --git a/curtin/commands/apt_config.py b/curtin/commands/apt_config.py
112index 9ea2d30..4f62a86 100644
113--- a/curtin/commands/apt_config.py
114+++ b/curtin/commands/apt_config.py
115@@ -28,6 +28,9 @@ APT_LISTS = "/var/lib/apt/lists"
116 APT_CONFIG_FN = "/etc/apt/apt.conf.d/94curtin-config"
117 APT_PROXY_FN = "/etc/apt/apt.conf.d/90curtin-aptproxy"
118
119+# Files to store pinning information
120+APT_PREFERENCES_FN = "/etc/apt/preferences.d/90curtin.pref"
121+
122 # Default keyserver to use
123 DEFAULT_KEYSERVER = "keyserver.ubuntu.com"
124
125@@ -37,7 +40,7 @@ PRIMARY_ARCH_MIRRORS = {"PRIMARY": "http://archive.ubuntu.com/ubuntu/",
126 PORTS_MIRRORS = {"PRIMARY": "http://ports.ubuntu.com/ubuntu-ports",
127 "SECURITY": "http://ports.ubuntu.com/ubuntu-ports"}
128 PRIMARY_ARCHES = ['amd64', 'i386']
129-PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el']
130+PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el', 'riscv64']
131
132 APT_SOURCES_PROPOSED = (
133 "deb $MIRROR $RELEASE-proposed main restricted universe multiverse")
134@@ -81,6 +84,11 @@ def handle_apt(cfg, target=None):
135 except (IOError, OSError):
136 LOG.exception("Failed to apply proxy or apt config info:")
137
138+ try:
139+ apply_apt_preferences(cfg, target + APT_PREFERENCES_FN)
140+ except (IOError, OSError):
141+ LOG.exception("Failed to apply apt preferences.")
142+
143 # Process 'apt_source -> sources {dict}'
144 if 'sources' in cfg:
145 params = mirrors
146@@ -571,7 +579,7 @@ def find_apt_mirror_info(cfg, arch=None):
147
148 def apply_apt_proxy_config(cfg, proxy_fname, config_fname):
149 """apply_apt_proxy_config
150- Applies any apt*proxy config from if specified
151+ Applies any apt*proxy from config if specified
152 """
153 # Set up any apt proxy
154 cfgs = (('proxy', 'Acquire::http::Proxy "%s";'),
155@@ -584,8 +592,14 @@ def apply_apt_proxy_config(cfg, proxy_fname, config_fname):
156 LOG.debug("write apt proxy info to %s", proxy_fname)
157 util.write_file(proxy_fname, '\n'.join(proxies) + '\n')
158 elif os.path.isfile(proxy_fname):
159- util.del_file(proxy_fname)
160- LOG.debug("no apt proxy configured, removed %s", proxy_fname)
161+ # When $ curtin apt-config is called with no proxy set, it makes
162+ # sense to remove the proxy file (if present). Having said that,
163+ # this code is also called automatically at the curthooks stage with an
164+ # empty configuration. Since the installation of external packages and
165+ # execution of unattended-upgrades (which happen after executing the
166+ # curthooks) need to use the proxy if specified, we must not let the
167+ # curthooks remove the proxy file.
168+ pass
169
170 if cfg.get('conf', None):
171 LOG.debug("write apt config info to %s", config_fname)
172@@ -595,6 +609,38 @@ def apply_apt_proxy_config(cfg, proxy_fname, config_fname):
173 LOG.debug("no apt config configured, removed %s", config_fname)
174
175
176+def preference_to_str(preference):
177+ """ Return a textual representation of a given preference as specified in
178+ apt_preferences(5).
179+ """
180+
181+ return """\
182+Package: {package}
183+Pin: {pin}
184+Pin-Priority: {pin_priority}
185+""".format(package=preference["package"],
186+ pin=preference["pin"],
187+ pin_priority=preference["pin-priority"])
188+
189+
190+def apply_apt_preferences(cfg, pref_fname):
191+ """ Apply apt preferences if any is provided.
192+ """
193+
194+ prefs = cfg.get("preferences")
195+ if not prefs:
196+ # When $ curtin apt-config is called with no preferences set, it makes
197+ # sense to remove the preferences file (if present). Having said that,
198+ # this code is also called automatically at the curthooks stage with an
199+ # empty configuration. Since the installation of packages (which
200+ # happens after executing the curthooks) needs to honor the preferences
201+ # set, we must not let the curthooks remove the preferences file.
202+ return
203+ prefs_as_strings = [preference_to_str(pref) for pref in prefs]
204+ LOG.debug("write apt preferences info to %s.", pref_fname)
205+ util.write_file(pref_fname, "\n".join(prefs_as_strings))
206+
207+
208 def apt_command(args):
209 """ Main entry point for curtin apt-config standalone command
210 This does not read the global config as handled by curthooks, but
211diff --git a/curtin/commands/block_meta.py b/curtin/commands/block_meta.py
212index 1913cb4..5614883 100644
213--- a/curtin/commands/block_meta.py
214+++ b/curtin/commands/block_meta.py
215@@ -552,16 +552,30 @@ DEVS = set()
216
217 def image_handler(info, storage_config, handlers):
218 path = info['path']
219- if os.path.exists(path):
220- os.unlink(path)
221+ size = int(util.human2bytes(info['size']))
222+ sector_size = str(int(util.human2bytes(info.get('sector_size', 512))))
223+ if info.get('preserve', False):
224+ actual_size = os.stat(path).st_size
225+ if size != actual_size:
226+ raise RuntimeError(
227+ 'image at {} was size {} not {} as expected.'.format(
228+ path, actual_size, size))
229+ else:
230+ if os.path.exists(path):
231+ os.unlink(path)
232+ try:
233+ with open(path, 'wb') as fp:
234+ fp.truncate(size)
235+ except BaseException:
236+ if os.path.exists(path):
237+ os.unlink(path)
238+ raise
239 try:
240- with open(path, 'wb') as fp:
241- fp.truncate(int(util.human2bytes(info['size'])))
242 dev = util.subp([
243- 'losetup', '--show', '--find', path],
244+ 'losetup', '--show', '--sector-size', sector_size, '--find', path],
245 capture=True)[0].strip()
246 except BaseException:
247- if os.path.exists(path):
248+ if os.path.exists(path) and not info.get('preserve'):
249 os.unlink(path)
250 raise
251 info['dev'] = dev
252@@ -765,12 +779,17 @@ def verify_exists(devpath):
253 raise RuntimeError("Device %s does not exist" % devpath)
254
255
256-def verify_size(devpath, expected_size_bytes, part_info):
257+def get_part_size_bytes(devpath, part_info):
258 (found_type, _code) = ptable_uuid_to_flag_entry(part_info.get('type'))
259 if found_type == 'extended':
260 found_size_bytes = int(part_info['size']) * 512
261 else:
262 found_size_bytes = block.read_sys_block_size_bytes(devpath)
263+ return found_size_bytes
264+
265+
266+def verify_size(devpath, expected_size_bytes, part_info):
267+ found_size_bytes = get_part_size_bytes(devpath, part_info)
268 msg = (
269 'Verifying %s size, expecting %s bytes, found %s bytes' % (
270 devpath, expected_size_bytes, found_size_bytes))
271@@ -807,7 +826,7 @@ def verify_ptable_flag(devpath, expected_flag, label, part_info):
272
273
274 def partition_verify_sfdisk(part_action, label, sfdisk_part_info):
275- devpath = sfdisk_part_info['node']
276+ devpath = os.path.realpath(sfdisk_part_info['node'])
277 verify_size(
278 devpath, int(util.human2bytes(part_action['size'])), sfdisk_part_info)
279 expected_flag = part_action.get('flag')
280@@ -1110,6 +1129,12 @@ def _get_volume_type(device_path):
281 return lsblock[kname]['TYPE']
282
283
284+def _get_volume_fstype(device_path):
285+ lsblock = block._lsblock([device_path])
286+ kname = block.path_to_kname(device_path)
287+ return lsblock[kname]['FSTYPE']
288+
289+
290 def get_volume_spec(device_path):
291 """
292 Return the most reliable spec for a device per Ubuntu FSTAB wiki
293@@ -2004,6 +2029,17 @@ def meta_custom(args):
294
295 storage_config_dict = extract_storage_ordered_dict(cfg)
296
297+ version = cfg['storage']['version']
298+ if version > 1:
299+ from curtin.commands.block_meta_v2 import (
300+ disk_handler_v2,
301+ partition_handler_v2,
302+ )
303+ command_handlers.update({
304+ 'disk': disk_handler_v2,
305+ 'partition': partition_handler_v2,
306+ })
307+
308 storage_config_dict = zfsroot_update_storage_config(storage_config_dict)
309
310 # set up reportstack
311diff --git a/curtin/commands/block_meta_v2.py b/curtin/commands/block_meta_v2.py
312new file mode 100644
313index 0000000..b4838f9
314--- /dev/null
315+++ b/curtin/commands/block_meta_v2.py
316@@ -0,0 +1,422 @@
317+# This file is part of curtin. See LICENSE file for copyright and license info.
318+
319+import os
320+from typing import Optional
321+
322+import attr
323+
324+from curtin import (block, util)
325+from curtin.commands.block_meta import (
326+ _get_volume_fstype,
327+ disk_handler as disk_handler_v1,
328+ get_path_to_storage_volume,
329+ make_dname,
330+ partition_handler as partition_handler_v1,
331+ verify_ptable_flag,
332+ verify_size,
333+ )
334+from curtin.log import LOG
335+from curtin.storage_config import (
336+ GPT_GUID_TO_CURTIN_MAP,
337+ select_configs,
338+ )
339+from curtin.udev import udevadm_settle
340+
341+
342+@attr.s(auto_attribs=True)
343+class PartTableEntry:
344+ number: int
345+ start: int
346+ size: int
347+ type: str
348+ uuid: Optional[str]
349+ bootable: bool = False
350+
351+ def render(self):
352+ r = f'{self.number}: '
353+ for a in 'start', 'size', 'type', 'uuid':
354+ v = getattr(self, a)
355+ if v is not None:
356+ r += f' {a}={v}'
357+ if self.bootable:
358+ r += ' bootable'
359+ return r
360+
361+
362+ONE_MIB_BYTES = 1 << 20
363+
364+
365+def align_up(size, block_size):
366+ return (size + block_size - 1) & ~(block_size - 1)
367+
368+
369+def align_down(size, block_size):
370+ return size & ~(block_size - 1)
371+
372+
373+def resize_ext(path, size):
374+ util.subp(['e2fsck', '-p', '-f', path])
375+ size_k = size // 1024
376+ util.subp(['resize2fs', path, f'{size_k}k'])
377+
378+
379+def resize_ntfs(path, size):
380+ util.subp(['ntfsresize', '-s', str(size), path])
381+
382+
383+def perform_resize(kname, resize):
384+ path = block.kname_to_path(kname)
385+ fstype = resize['fstype']
386+ size = resize['size']
387+ direction = resize['direction']
388+ LOG.debug('Resizing %s of type %s %s to %s',
389+ path, fstype, direction, size)
390+ resizers[fstype](path, size)
391+
392+
393+resizers = {
394+ 'ext2': resize_ext,
395+ 'ext3': resize_ext,
396+ 'ext4': resize_ext,
397+ 'ntfs': resize_ntfs,
398+}
399+
400+
401+FLAG_TO_GUID = {
402+ flag: guid for (guid, (flag, typecode)) in GPT_GUID_TO_CURTIN_MAP.items()
403+ }
404+FLAG_TO_MBR_TYPE = {
405+ flag: typecode[:2].upper() for (guid, (flag, typecode))
406+ in GPT_GUID_TO_CURTIN_MAP.items()
407+ }
408+FLAG_TO_MBR_TYPE['extended'] = '05'
409+
410+
411+class SFDiskPartTable:
412+
413+ label = None
414+
415+ def __init__(self, sector_bytes):
416+ self.entries = []
417+ self.label_id = None
418+ self._sector_bytes = sector_bytes
419+ if ONE_MIB_BYTES % sector_bytes != 0:
420+ raise Exception(
421+ f"sector_bytes {sector_bytes} does not divide 1MiB, cannot "
422+ "continue!")
423+ self.one_mib_sectors = ONE_MIB_BYTES // sector_bytes
424+
425+ def bytes2sectors(self, amount):
426+ return int(util.human2bytes(amount)) // self._sector_bytes
427+
428+ def sectors2bytes(self, amount):
429+ return amount * self._sector_bytes
430+
431+ def render(self):
432+ r = ['label: ' + self.label]
433+ if self.label_id:
434+ r.extend(['label-id: ' + self.label_id])
435+ r.extend([''])
436+ r.extend([e.render() for e in self.entries])
437+ return '\n'.join(r)
438+
439+ def apply(self, device):
440+ sfdisk_script = self.render()
441+ LOG.debug("sfdisk input:\n---\n%s\n---\n", sfdisk_script)
442+ util.subp(
443+ ['sfdisk', '--no-tell-kernel', '--no-reread', device],
444+ data=sfdisk_script.encode('ascii'))
445+ util.subp(['partprobe', device])
446+ # sfdisk and partprobe (as invoked here) use ioctls to inform the
447+ # kernel that the partition table has changed so it can add and remove
448+ # device nodes for the partitions as needed. Unfortunately this is
449+ # asynchronous: we can return before the nodes are present in /dev (or
450+ # /sys for that matter). Calling "udevadm settle" is slightly
451+ # incoherent as udev has nothing to do with creating these nodes, but
452+ # at the same time, udev won't finish processing the events triggered
453+ # by the sfdisk until after the nodes for the partitions have been
454+ # updated by the kernel.
455+ udevadm_settle()
456+
457+
458+class GPTPartTable(SFDiskPartTable):
459+
460+ label = 'gpt'
461+
462+ def add(self, action):
463+ number = action.get('number', len(self.entries) + 1)
464+ if 'offset' in action:
465+ start = self.bytes2sectors(action['offset'])
466+ else:
467+ if self.entries:
468+ prev = self.entries[-1]
469+ start = align_up(prev.start + prev.size, self.one_mib_sectors)
470+ else:
471+ start = self.one_mib_sectors
472+ size = self.bytes2sectors(action['size'])
473+ uuid = action.get('uuid')
474+ type = action.get('partition_type',
475+ FLAG_TO_GUID.get(action.get('flag')))
476+ entry = PartTableEntry(number, start, size, type, uuid)
477+ self.entries.append(entry)
478+ return entry
479+
480+
481+class DOSPartTable(SFDiskPartTable):
482+
483+ label = 'dos'
484+ _extended = None
485+
486+ def add(self, action):
487+ flag = action.get('flag', None)
488+ start = action.get('offset', None)
489+ if start is not None:
490+ start = self.bytes2sectors(start)
491+ if flag == 'logical':
492+ if self._extended is None:
493+ raise Exception("logical partition without extended partition")
494+ prev = None
495+ for entry in reversed(self.entries):
496+ if entry.number > 4:
497+ prev = entry
498+ break
499+ # The number of an logical partition cannot be specified (so the
500+ # 'number' from the action is completely ignored here) as the
501+ # partitions are numbered by the order they are found in the linked
502+ # list of logical partitions. sfdisk just cares that we put a
503+ # number > 4 here, in fact we could "number" every logical
504+ # partition as "5" but it's not hard to put the number that the
505+ # partition will end up getting into the sfdisk input.
506+ if prev is None:
507+ number = 5
508+ if start is None:
509+ start = align_up(
510+ self._extended.start + self.one_mib_sectors,
511+ self.one_mib_sectors)
512+ else:
513+ number = prev.number + 1
514+ if start is None:
515+ start = align_up(
516+ prev.start + prev.size + self.one_mib_sectors,
517+ self.one_mib_sectors)
518+ else:
519+ number = action.get('number', len(self.entries) + 1)
520+ if number > 4:
521+ raise Exception(
522+ "primary partition cannot have number %s" % (number,))
523+ if start is None:
524+ prev = None
525+ for entry in self.entries:
526+ if entry.number <= 4:
527+ prev = entry
528+ if prev is None:
529+ start = self.one_mib_sectors
530+ else:
531+ start = align_up(
532+ prev.start + prev.size,
533+ self.one_mib_sectors)
534+ size = self.bytes2sectors(action['size'])
535+ type = action.get('partition_type', FLAG_TO_MBR_TYPE.get(flag))
536+ if flag == 'boot':
537+ bootable = True
538+ else:
539+ bootable = None
540+ entry = PartTableEntry(
541+ number, start, size, type, uuid=None, bootable=bootable)
542+ if flag == 'extended':
543+ self._extended = entry
544+ self.entries.append(entry)
545+ return entry
546+
547+
548+def _find_part_info(sfdisk_info, offset):
549+ for part in sfdisk_info['partitions']:
550+ if part['start'] == offset:
551+ return part
552+ else:
553+ raise Exception(
554+ "could not find existing partition by offset")
555+
556+
557+def _wipe_for_action(action):
558+ # If a wipe action is specified, do that.
559+ if 'wipe' in action:
560+ return action['wipe']
561+ # Existing partitions are left alone by default.
562+ if action.get('preserve', False):
563+ return None
564+ # New partitions are wiped by default apart from extended partitions, where
565+ # it would destroy the EBR.
566+ if action.get('flag') == 'extended':
567+ return None
568+ return 'superblock'
569+
570+
571+def _prepare_resize(storage_config, part_action, table, part_info):
572+ if not part_action.get('preserve') or not part_action.get('resize'):
573+ return None
574+
575+ devpath = os.path.realpath(part_info['node'])
576+ fstype = _get_volume_fstype(devpath)
577+ if fstype == '':
578+ return None
579+
580+ volume = part_action['id']
581+ format_actions = select_configs(storage_config, type='format',
582+ volume=volume)
583+ if len(format_actions) > 1:
584+ raise Exception(f'too many format actions for volume {volume}')
585+
586+ if len(format_actions) == 1:
587+ if not format_actions[0].get('preserve'):
588+ return None
589+
590+ target_fstype = format_actions[0]['fstype']
591+ msg = (
592+ 'Verifying %s format, expecting %s, found %s' % (
593+ devpath, fstype, target_fstype))
594+ LOG.debug(msg)
595+ if fstype != target_fstype:
596+ raise RuntimeError(msg)
597+
598+ msg = 'Resize requested for format %s' % (fstype, )
599+ LOG.debug(msg)
600+ if fstype not in resizers:
601+ raise RuntimeError(msg + ' is unsupported')
602+
603+ start = table.sectors2bytes(part_info['size'])
604+ end = int(util.human2bytes(part_action['size']))
605+ if start > end:
606+ direction = 'down'
607+ elif start < end:
608+ direction = 'up'
609+ else:
610+ return None
611+
612+ return {
613+ 'fstype': fstype,
614+ 'size': end,
615+ 'direction': direction,
616+ }
617+
618+
619+def verify_offset(devpath, part_action, current_info, table):
620+ if 'offset' not in part_action:
621+ return
622+ current_offset = table.sectors2bytes(current_info['start'])
623+ action_offset = int(util.human2bytes(part_action['offset']))
624+ msg = (
625+ 'Verifying %s offset, expecting %s, found %s' % (
626+ devpath, current_offset, action_offset))
627+ LOG.debug(msg)
628+ if current_offset != action_offset:
629+ raise RuntimeError(msg)
630+
631+
632+def partition_verify_sfdisk_v2(part_action, label, sfdisk_part_info,
633+ storage_config, table):
634+ devpath = os.path.realpath(sfdisk_part_info['node'])
635+ if not part_action.get('resize'):
636+ verify_size(devpath, int(util.human2bytes(part_action['size'])),
637+ sfdisk_part_info)
638+ verify_offset(devpath, part_action, sfdisk_part_info, table)
639+ expected_flag = part_action.get('flag')
640+ if expected_flag:
641+ verify_ptable_flag(devpath, expected_flag, label, sfdisk_part_info)
642+
643+
644+def disk_handler_v2(info, storage_config, handlers):
645+ disk_handler_v1(info, storage_config, handlers)
646+
647+ part_actions = []
648+
649+ for action in storage_config.values():
650+ if action['type'] == 'partition' and action['device'] == info['id']:
651+ part_actions.append(action)
652+
653+ table_cls = {
654+ 'msdos': DOSPartTable,
655+ 'gpt': GPTPartTable,
656+ }.get(info.get('ptable'))
657+
658+ if table_cls is None:
659+ for action in part_actions:
660+ partition_handler_v1(action, storage_config, handlers)
661+ return
662+
663+ disk = get_path_to_storage_volume(info.get('id'), storage_config)
664+ (sector_size, _) = block.get_blockdev_sector_size(disk)
665+
666+ table = table_cls(sector_size)
667+ preserved_offsets = set()
668+ wipes = {}
669+ resizes = {}
670+
671+ sfdisk_info = None
672+ for action in part_actions:
673+ entry = table.add(action)
674+ if action.get('preserve', False):
675+ if sfdisk_info is None:
676+ # Lazily computing sfdisk_info is slightly more efficient but
677+ # the real reason for doing this is that calling sfdisk_info on
678+ # a disk with no partition table logs messages that makes the
679+ # vmtest infrastructure unhappy.
680+ sfdisk_info = block.sfdisk_info(disk)
681+ part_info = _find_part_info(sfdisk_info, entry.start)
682+ partition_verify_sfdisk_v2(action, sfdisk_info['label'], part_info,
683+ storage_config, table)
684+ resizes[entry.start] = _prepare_resize(storage_config, action,
685+ table, part_info)
686+ preserved_offsets.add(entry.start)
687+ wipes[entry.start] = _wipe_for_action(action)
688+
689+ # preserve disk label ids
690+ if info.get('preserve') and sfdisk_info is not None:
691+ table.label_id = sfdisk_info['id']
692+
693+ for kname, nr, offset, size in block.sysfs_partition_data(disk):
694+ offset_sectors = table.bytes2sectors(offset)
695+ resize = resizes.get(offset_sectors)
696+ if resize and resize['direction'] == 'down':
697+ perform_resize(kname, resize)
698+
699+ for kname, nr, offset, size in block.sysfs_partition_data(disk):
700+ offset_sectors = table.bytes2sectors(offset)
701+ if offset_sectors not in preserved_offsets:
702+ # Do a superblock wipe of any partitions that are being deleted.
703+ block.wipe_volume(block.kname_to_path(kname), 'superblock')
704+ elif wipes.get(offset_sectors) is not None:
705+ # We do a quick wipe of where any new partitions will be,
706+ # because if there is bcache or other metadata there, this
707+ # can cause the partition to be used by a storage
708+ # subsystem and preventing the exclusive open done by the
709+ # wipe_volume call below. See
710+ # https://bugs.launchpad.net/curtin/+bug/1718699 for all
711+ # the gory details.
712+ LOG.debug('Wiping 1M on %s at offset %s', disk, offset)
713+ block.zero_file_at_offsets(disk, [offset], exclusive=False)
714+
715+ table.apply(disk)
716+
717+ for kname, number, offset, size in block.sysfs_partition_data(disk):
718+ offset_sectors = table.bytes2sectors(offset)
719+ wipe = wipes[offset_sectors]
720+ if wipe is not None:
721+ # Wipe the new partitions as needed.
722+ block.wipe_volume(block.kname_to_path(kname), wipe)
723+ resize = resizes.get(offset_sectors)
724+ if resize and resize['direction'] == 'up':
725+ perform_resize(kname, resize)
726+
727+ # Make the names if needed
728+ if 'name' in info:
729+ for action in part_actions:
730+ if action.get('flag') != 'extended':
731+ make_dname(action['id'], storage_config)
732+
733+
734+def partition_handler_v2(info, storage_config, handlers):
735+ pass
736+
737+
738+# vi: ts=4 expandtab syntax=python
739diff --git a/curtin/commands/install_grub.py b/curtin/commands/install_grub.py
740index ba46bd2..74ffdf1 100644
741--- a/curtin/commands/install_grub.py
742+++ b/curtin/commands/install_grub.py
743@@ -62,6 +62,9 @@ def get_grub_package_name(target_arch, uefi, rhel_ver=None):
744 elif target_arch == 'i386':
745 grub_name = 'grub-efi-ia32'
746 grub_target = 'i386-efi'
747+ elif target_arch == 'riscv64':
748+ grub_name = 'grub-efi-riscv64'
749+ grub_target = 'riscv64-efi'
750 else:
751 raise ValueError('Unsupported UEFI arch: %s' % target_arch)
752 else:
753diff --git a/curtin/distro.py b/curtin/distro.py
754index 8b5fbf8..16ce2c5 100644
755--- a/curtin/distro.py
756+++ b/curtin/distro.py
757@@ -23,7 +23,8 @@ from .log import LOG
758
759 DistroInfo = namedtuple('DistroInfo', ('variant', 'family'))
760 DISTRO_NAMES = ['arch', 'centos', 'debian', 'fedora', 'freebsd', 'gentoo',
761- 'opensuse', 'redhat', 'rhel', 'sles', 'suse', 'ubuntu']
762+ 'opensuse', 'redhat', 'rhel', 'sles', 'suse', 'ubuntu',
763+ 'rocky']
764
765
766 # python2.7 lacks PEP 435, so we must make use an alternative for py2.7/3.x
767@@ -37,7 +38,7 @@ DISTROS = distro_enum(*DISTRO_NAMES)
768 OS_FAMILIES = {
769 DISTROS.debian: [DISTROS.debian, DISTROS.ubuntu],
770 DISTROS.redhat: [DISTROS.centos, DISTROS.fedora, DISTROS.redhat,
771- DISTROS.rhel],
772+ DISTROS.rhel, DISTROS.rocky],
773 DISTROS.gentoo: [DISTROS.gentoo],
774 DISTROS.freebsd: [DISTROS.freebsd],
775 DISTROS.suse: [DISTROS.opensuse, DISTROS.sles, DISTROS.suse],
776@@ -382,6 +383,9 @@ def system_upgrade(opts=None, target=None, env=None, allow_daemons=False,
777 osfamily=None):
778 LOG.debug("Upgrading system in %s", target)
779
780+ if not osfamily:
781+ osfamily = get_osfamily(target=target)
782+
783 distro_cfg = {
784 DISTROS.debian: {'function': run_apt_command,
785 'subcommands': ('dist-upgrade', 'autoremove')},
786diff --git a/curtin/storage_config.py b/curtin/storage_config.py
787index 405a1e2..e9e8991 100644
788--- a/curtin/storage_config.py
789+++ b/curtin/storage_config.py
790@@ -79,7 +79,7 @@ STORAGE_CONFIG_SCHEMA = {
791 'required': ['version', 'config'],
792 'definitions': schemas.definitions,
793 'properties': {
794- 'version': {'type': 'integer', 'enum': [1]},
795+ 'version': {'type': 'integer', 'enum': [1, 2]},
796 'config': {
797 'type': 'array',
798 'items': {
799@@ -753,6 +753,8 @@ class BlockdevParser(ProbertParser):
800 return entry
801
802 if entry['type'] == 'partition':
803+ if devname:
804+ entry['path'] = devname
805 attrs = blockdev_data['attrs']
806 if self.is_mpath_partition(blockdev_data):
807 entry['number'] = int(blockdev_data['DM_PART'])
808@@ -798,6 +800,8 @@ class BlockdevParser(ProbertParser):
809 entry['size'] *= 512
810
811 ptype = blockdev_data.get('ID_PART_ENTRY_TYPE')
812+ if ptype is not None:
813+ entry['partition_type'] = ptype
814 flag_name, _flag_code = ptable_uuid_to_flag_entry(ptype)
815
816 if ptable and ptable.get('label') == 'dos':
817@@ -1315,7 +1319,7 @@ def extract_storage_config(probe_data, strict=False):
818 ordered = (dasd + disk + part + format + lvols + lparts + raids +
819 dmcrypts + mounts + bcache + zpool + zfs)
820
821- final_config = {'storage': {'version': 1, 'config': ordered}}
822+ final_config = {'storage': {'version': 2, 'config': ordered}}
823 try:
824 LOG.info('Validating extracted storage config components')
825 validate_config(final_config['storage'])
826@@ -1346,7 +1350,7 @@ def extract_storage_config(probe_data, strict=False):
827
828 LOG.debug("Merging storage config dependencies")
829 merged_config = {
830- 'version': 1,
831+ 'version': 2,
832 'config': merge_config_trees_to_list(ctrees)
833 }
834 LOG.debug("Merged storage config:\n%s",
835@@ -1355,4 +1359,12 @@ def extract_storage_config(probe_data, strict=False):
836 return {'storage': merged_config}
837
838
839+def select_configs(storage_config, **kwargs):
840+ """ Given a set of key=value arguments, return a list of the configs that
841+ match all specified key-value pairs.
842+ """
843+ return [cfg for cfg in storage_config.values()
844+ if all(cfg.get(k) == v for k, v in kwargs.items())]
845+
846+
847 # vi: ts=4 expandtab syntax=python
848diff --git a/curtin/util.py b/curtin/util.py
849index 5b66b55..d3c3b66 100644
850--- a/curtin/util.py
851+++ b/curtin/util.py
852@@ -501,6 +501,13 @@ def chdir(dirname):
853 os.chdir(curdir)
854
855
856+@contextmanager
857+def mount(src, target):
858+ do_mount(src, target)
859+ yield
860+ do_umount(target)
861+
862+
863 def do_mount(src, target, opts=None):
864 # mount src at target with opts and return True
865 # if already mounted, return False
866diff --git a/debian/changelog b/debian/changelog
867index b8e9bb5..da79e7c 100644
868--- a/debian/changelog
869+++ b/debian/changelog
870@@ -1,3 +1,72 @@
871+curtin (22.1-0ubuntu1) kinetic; urgency=medium
872+
873+ * New upstream release. (LP: #1979687)
874+ - deb: fix dependencies [Dan Bungert]
875+ - tox: drop xenial-py3 from default env list [Dan Bungert]
876+ - block/v2: preserve disk label id [Dan Bungert]
877+ - block/v2: docs for partition_type [Dan Bungert]
878+ - block/v2: unit tests for partition_type [Dan Bungert]
879+ - block/v2: raw partition table codes for gpt [Dan Bungert]
880+ - block/v2: allow setting raw partition_type value [Dan Bungert]
881+ - Make sure curthooks do not discard supplied proxy settings
882+ [Olivier Gayot]
883+ - block/v2: resize-friendly ordering of wipe [Dan Bungert]
884+ - block/v2: handle resize when no format action [Dan Bungert]
885+ - block/v2: resize of ntfs [Dan Bungert]
886+ - vmtests: remove out of date skip [Dan Bungert]
887+ - block: provide get_resize_fstypes [Dan Bungert]
888+ - Add support for resize of ext{2,3,4} [Dan Bungert]
889+ - Add riscv64 to supported UEFI architectures [William Wilson]
890+ - block_meta_v2: call make_dname when required [Michael Hudson-Doyle]
891+ - examples: even more tweaks for v2 [Michael Hudson-Doyle]
892+ - Add riscv64 support [Heinrich Schuchardt]
893+ - examples: stop assuming curtin accounts for overhead of logical
894+ partitions [Michael Hudson-Doyle]
895+ - block_meta_v2: zero start of partitions before they are created
896+ [Michael Hudson-Doyle]
897+ - examples: enlarge / for some more vmtests [Michael Hudson-Doyle]
898+ - skip BionicTestPartitionExistingRAID.test_correct_ptype
899+ [Michael Hudson-Doyle]
900+ - examples: boost size of / in multipath-reuse.yaml [Michael Hudson-Doyle]
901+ - examples: add offsets to preserved partitions [Michael Hudson-Doyle]
902+ - block_meta_v2: change how we invoke sfdisk again, restore partprobe call
903+ [Michael Hudson-Doyle]
904+ - block_meta_v2: fix partitioning a device with sector size != 512
905+ [Michael Hudson-Doyle]
906+ - block_meta_v2: fix implicit offset calculation for dos partitions
907+ [Michael Hudson-Doyle]
908+ - block_meta_v2: do not use aliases for partition types
909+ [Michael Hudson-Doyle]
910+ - Remove CentOS 6 tests. [Michael Hudson-Doyle]
911+ - vmtests: boost the size of / in a few tests [Michael Hudson-Doyle]
912+ - examples: sleep after creating bcache device in preserve-bcache.yaml
913+ [Michael Hudson-Doyle]
914+ - vmtests: fix parted invocation in partition-existing-raid.yaml
915+ [Michael Hudson-Doyle]
916+ - vmtests: bump VM memory size to 2048 MiB for all tests
917+ [Michael Hudson-Doyle]
918+ - vmtests: drop assertion that clear-holders ran [Michael Hudson-Doyle]
919+ - block_meta_v2: a few more fixes for v2 partitioning
920+ [Michael Hudson-Doyle]
921+ - block_meta: call realpath on partition node returned by sfdisk
922+ [Michael Hudson-Doyle]
923+ - Add rocky linux as a RHEL-like variant [Dimitri John Ledkov]
924+ - Update pylint version in tox.ini [Michael Hudson-Doyle]
925+ - block_meta: implement v2 partitioning [Michael Hudson-Doyle]
926+ - Stop running CI against Python 2 [Michael Hudson-Doyle]
927+ - Make sure curthooks do not discard APT preferences [Olivier Gayot]
928+ - Remove leftover debug print statement [Olivier Gayot]
929+ - Fix format of examples/apt-source.yaml [Olivier Gayot]
930+ - Implement support for APT preferences in apt-config [Olivier Gayot]
931+ - build-deb: changelog gen with dch [Dan Bungert]
932+ - vmtests uefi: relax the uefi check [Dan Bungert]
933+ - block: output partition device path [Dan Bungert]
934+ - support version 2 curtin storage configs [Michael Hudson-Doyle]
935+ - system-upgrade: lookup os family [Dan Bungert]
936+ - add preserve: true support to the image action [Michael Hudson-Doyle]
937+
938+ -- Dan Bungert <daniel.bungert@canonical.com> Mon, 27 Jun 2022 16:24:54 -0600
939+
940 curtin (21.3-0ubuntu1) jammy; urgency=medium
941
942 * New upstream release.
943diff --git a/debian/changelog.trunk b/debian/changelog.trunk
944deleted file mode 100644
945index 4d943c0..0000000
946--- a/debian/changelog.trunk
947+++ /dev/null
948@@ -1,5 +0,0 @@
949-curtin (UPSTREAM_VER-0ubuntu1) UNRELEASED; urgency=low
950-
951- * Initial release
952-
953- -- Scott Moser <smoser@ubuntu.com> Mon, 29 Jul 2013 16:12:09 -0400
954diff --git a/debian/control b/debian/control
955index 9f0b71d..a35cbf6 100644
956--- a/debian/control
957+++ b/debian/control
958@@ -7,6 +7,7 @@ Build-Depends: debhelper (>= 7),
959 dh-python,
960 python3,
961 python3-apt,
962+ python3-attr,
963 python3-coverage,
964 python3-mock,
965 python3-nose,
966@@ -14,6 +15,8 @@ Build-Depends: debhelper (>= 7),
967 python3-setuptools,
968 python3-yaml
969 Homepage: http://launchpad.net/curtin
970+Vcs-Git: https://git.launchpad.net/curtin
971+Vcs-Browser: https://git.launchpad.net/curtin
972 X-Python3-Version: >= 3.2
973
974 Package: curtin
975@@ -51,6 +54,7 @@ Architecture: all
976 Priority: extra
977 Depends: curtin-common (= ${binary:Version}),
978 python3-apt,
979+ python3-attr,
980 python3-oauthlib,
981 python3-yaml,
982 wget,
983diff --git a/doc/topics/apt_source.rst b/doc/topics/apt_source.rst
984index cf0f8bd..924ee80 100644
985--- a/doc/topics/apt_source.rst
986+++ b/doc/topics/apt_source.rst
987@@ -31,6 +31,8 @@ Features
988
989 - add arbitrary apt.conf settings
990
991+ - add arbitrary apt preferences
992+
993 - provide debconf configurations
994
995 - disabling suites (=pockets)
996diff --git a/doc/topics/storage.rst b/doc/topics/storage.rst
997index 0f33ec0..bbff909 100644
998--- a/doc/topics/storage.rst
999+++ b/doc/topics/storage.rst
1000@@ -13,8 +13,7 @@ Custom storage configuration is handled by the ``block-meta custom`` command
1001 in curtin. Partitioning layout is read as a list of in-order modifications to
1002 make to achieve the desired configuration. The top level configuration key
1003 containing this configuration is ``storage``. This key should contain a
1004-dictionary with at least a version number and the configuration list. The
1005-current config specification is ``version: 1``.
1006+dictionary with at least a version number and the configuration list.
1007
1008 **Config Example**::
1009
1010@@ -27,6 +26,20 @@ current config specification is ``version: 1``.
1011 serial: QM00002
1012 model: QEMU_HARDDISK
1013
1014+Config versions
1015+---------------
1016+
1017+The current version of curtin supports versions ``1`` and ``2``. These
1018+only differ in the interpretation of ``partition`` actions at this
1019+time. ``lvm_partition`` actions will be interpreted differently at
1020+some point in the future.
1021+
1022+.. note::
1023+
1024+ Config version ``2`` is under active development and subject to change.
1025+ Users are advised to use version ``1`` unless features enabled by version
1026+ ``2`` are required.
1027+
1028 Configuration Types
1029 -------------------
1030 Each entry in the config list is a dictionary with several keys which vary
1031@@ -322,13 +335,41 @@ The partition command creates a single partition on a disk. Curtin only needs
1032 to be told which disk to use and the size of the partition. Additional options
1033 are available.
1034
1035+Partition actions are interpreted differently according to the version of the
1036+storage config.
1037+
1038+ * For version 1 configs, the actions are handled one by one and each
1039+ partition is created (or assumed to exist, in the ``preserve: true`` case)
1040+ just after that described by the previous action.
1041+
1042+ * For version 2 configs, the actions are bundled together to create a
1043+ complete description of the partition table, and the ``offset`` of each
1044+ action is respected if present. Any partitions that already exist but are
1045+ not referenced in the new config are (superblock-) wiped and deleted.
1046+
1047+ * Because the numbering of logical partitions is not stable (i.e. if there
1048+ are two logical partitions numbered 5 and 6, and partition 5 is deleted,
1049+ what was partition 6 will become partition 5), curtin checks if a
1050+ partition is deleted or not by checking for the presence of a partition
1051+ action with a matching offset.
1052+
1053+If the disk is being completely repartitioned, the two schemes are effectively
1054+the same.
1055+
1056 **number**: *<number>*
1057
1058-The partition number can be specified using ``number``. However, numbers must
1059-be in order and some situations, such as extended/logical partitions on msdos
1060-partition tables will require special numbering, so it maybe better to omit
1061-the partition number. If the ``number`` key is not present, curtin will attempt
1062-determine the right number to use.
1063+The partition number can be specified using ``number``.
1064+
1065+For GPT partition tables, this will just be the slot in the partition table
1066+that is used to describe this partition.
1067+
1068+For DOS partition tables, a primary or extended partition must have a number
1069+less than or equal to 4. Logical partitions have numbers 5 or greater but are
1070+numbered by the order they are found when parsing the partitions, so the
1071+``number`` field is ignored for them.
1072+
1073+If the ``number`` key is not present, curtin will attempt determine the right
1074+number to use.
1075
1076 **size**: *<size>*
1077
1078@@ -338,8 +379,15 @@ the appropriate SI prefix, i.e. *B, k, M, G, T...*
1079
1080 .. note::
1081
1082- Curtin does not adjust size values. If you specific a size that exceeds the
1083- capacity of a device then installation will fail.
1084+ Curtin does not adjust or inspect size values. If you specify a size that
1085+ exceeds the capacity of a device then installation will fail.
1086+
1087+**offset**: *<offset>*
1088+
1089+The offset at which to create the partition. Only respected in a version 2
1090+config. If the offset field is not present, the partition will be placed after
1091+that described by the preceding (logical or primary, if appropriate) partition
1092+action, or at the start of the disk (or extended partition, as appropriate).
1093
1094 **device**: *<device id>*
1095
1096@@ -368,9 +416,7 @@ only apply to gpt partition tables.
1097 The *logical/extended* partition flags can be used to create logical partitions
1098 on a msdos table. An extended partition should be created containing all of the
1099 empty space on the drive, and logical partitions can be created within it. A
1100-extended partition must already be present to create logical partitions. If the
1101-``number`` flag is set for an extended partition it must be set to 4, and
1102-each logical partition should be numbered starting from 5.
1103+extended partition must already be present to create logical partitions.
1104
1105 On msdos partition tables, the *boot* flag sets the boot parameter to that
1106 partition. On gpt partition tables, the boot flag sets the esp flag on the
1107@@ -385,10 +431,33 @@ partition with the *bios_grub* flag is needed. This partition should be placed
1108 at the beginning of the disk and should be 1MB in size. It should not contain a
1109 filesystem or be mounted anywhere on the system.
1110
1111+**partition_type**: *msdos: byte value in 0xnn style; gpt: GUID*
1112+
1113+Only applicable to v2 storage configuration. If both ``partition_type`` and
1114+``flag`` are set, ``partition_type`` dictates the acutal type.
1115+
1116+The ``partition_type`` field allows for setting arbitrary partition type values
1117+that do not have a matching ``flag``, or cases that are not handled by the
1118+``flag`` system. For example, since the *boot* flag results in both setting
1119+the bootable state for a MSDOS partition table and setting it to type *0xEF*,
1120+one can override this behavior and achieve a bootable partition of a different
1121+type by using ``flag``: *boot* and using ``partition_type``.
1122+
1123 **preserve**: *true, false*
1124
1125 If the preserve flag is set to true, curtin will verify that the partition
1126 exists and that the ``size`` and ``flag`` match the configuration provided.
1127+See also the ``resize`` flag, which adjusts this behavior.
1128+
1129+**resize**: *true, false*
1130+
1131+Only applicable to v2 storage configuration.
1132+If the ``preserve`` flag is set to false, this value is not applicable.
1133+If the ``preserve`` flag is set to true, curtin will adjust the size of the
1134+partition to the new size. When adjusting smaller, the size of the contents
1135+must permit that. When adjusting larger, there must already be a gap beyond
1136+the partition in question.
1137+Resize is supported on filesystems of types ext2, ext3, ext4, ntfs.
1138
1139 **name**: *<name>*
1140
1141diff --git a/examples/apt-source.yaml b/examples/apt-source.yaml
1142index f0f7108..30e30a2 100644
1143--- a/examples/apt-source.yaml
1144+++ b/examples/apt-source.yaml
1145@@ -77,7 +77,7 @@ apt:
1146 # arches is list of architectures the following config applies to
1147 # the special keyword "default" applies to any architecture not explicitly
1148 # listed.
1149- - arches: [amd64, i386, default]
1150+ - arches: [amd64, i386, default]
1151 # uri is just defining the target as-is
1152 uri: http://us.archive.ubuntu.com/ubuntu
1153 #
1154@@ -100,7 +100,7 @@ apt:
1155 # security is optional, if not defined it is set to the same value as primary
1156 security:
1157 uri: http://security.ubuntu.com/ubuntu
1158- [...]
1159+ # [...]
1160
1161 # if no mirrors are specified at all, or all lookups fail it will use:
1162 # primary: http://archive.ubuntu.com/ubuntu
1163@@ -152,6 +152,18 @@ apt:
1164 # The following example is also the builtin default if nothing is specified
1165 add_apt_repo_match: '^[\w-]+:\w'
1166
1167+ # 1.9 preferences
1168+ #
1169+ # Any apt preferences that will be made available to apt
1170+ # see the APT_PREFERENCES(5) man page for details about what can be specified
1171+ preferences:
1172+ - package: python3-*
1173+ pin: origin *ubuntu.com*
1174+ pin-priority: 200
1175+ - package: python-*
1176+ pin: origin *ubuntu.com*
1177+ pin-priority: -1
1178+
1179
1180 ##############################################################################
1181 # Section 2: source list entries
1182diff --git a/examples/tests/basic.yaml b/examples/tests/basic.yaml
1183index 82f5ad1..9b5f7ea 100644
1184--- a/examples/tests/basic.yaml
1185+++ b/examples/tests/basic.yaml
1186@@ -17,7 +17,7 @@ storage:
1187 - id: sda1
1188 type: partition
1189 number: 1
1190- size: 3GB
1191+ size: 4GB
1192 device: sda
1193 flag: boot
1194 - id: sda2
1195diff --git a/examples/tests/basic_iscsi.yaml b/examples/tests/basic_iscsi.yaml
1196index 88516ca..4e9f89a 100644
1197--- a/examples/tests/basic_iscsi.yaml
1198+++ b/examples/tests/basic_iscsi.yaml
1199@@ -12,7 +12,7 @@ storage:
1200 - id: vdb1
1201 type: partition
1202 number: 1
1203- size: 3GB
1204+ size: 4GB
1205 device: vdb
1206 flag: boot
1207 - id: vdb2
1208diff --git a/examples/tests/bcache-partitions.yaml b/examples/tests/bcache-partitions.yaml
1209index 20ccddc..90861bc 100644
1210--- a/examples/tests/bcache-partitions.yaml
1211+++ b/examples/tests/bcache-partitions.yaml
1212@@ -18,7 +18,6 @@ storage:
1213 type: disk
1214 name: rotary1
1215 serial: disk-c
1216- ptable: gpt
1217 wipe: superblock
1218 - id: id_rotary0_part1
1219 type: partition
1220diff --git a/examples/tests/centos6_basic.yaml b/examples/tests/centos6_basic.yaml
1221deleted file mode 100644
1222index 90fc584..0000000
1223--- a/examples/tests/centos6_basic.yaml
1224+++ /dev/null
1225@@ -1,101 +0,0 @@
1226-showtrace: true
1227-storage:
1228- version: 1
1229- config:
1230- - id: sda
1231- type: disk
1232- ptable: msdos
1233- model: QEMU HARDDISK
1234- serial: disk-a
1235- name: main_disk_with_in/\&valid@#dname
1236- wipe: superblock
1237- grub_device: true
1238- - id: sda1
1239- type: partition
1240- number: 1
1241- size: 3GB
1242- device: sda
1243- flag: boot
1244- - id: sda2
1245- type: partition
1246- number: 2
1247- size: 1GB
1248- device: sda
1249- - id: sda3
1250- type: partition
1251- number: 3
1252- size: 1GB
1253- device: sda
1254- name: swap
1255- - id: sda1_root
1256- type: format
1257- fstype: ext3
1258- volume: sda1
1259- label: 'cloudimg-rootfs'
1260- - id: sda2_home
1261- type: format
1262- fstype: ext4
1263- volume: sda2
1264- - id: sda3_swap
1265- type: format
1266- fstype: swap
1267- volume: sda3
1268- - id: sda1_mount
1269- type: mount
1270- path: /
1271- device: sda1_root
1272- - id: sda2_mount
1273- type: mount
1274- path: /home
1275- device: sda2_home
1276- - id: sparedisk_id
1277- type: disk
1278- serial: disk-b
1279- name: sparedisk
1280- wipe: superblock
1281- - id: sparedisk_fat_fmt_id
1282- type: format
1283- fstype: fat32
1284- volume: sparedisk_id
1285- - id: btrfs_disk_id
1286- type: disk
1287- serial: disk-c
1288- name: btrfs_volume
1289- wipe: superblock
1290- - id: btrfs_disk_fmt_id
1291- type: format
1292- fstype: btrfs
1293- volume: btrfs_disk_id
1294- - id: btrfs_disk_mnt_id
1295- type: mount
1296- path: /btrfs
1297- options: 'defaults,noatime'
1298- device: btrfs_disk_fmt_id
1299- - id: pnum_disk
1300- type: disk
1301- serial: disk-d
1302- name: pnum_disk
1303- wipe: superblock
1304- ptable: gpt
1305- - id: pnum_disk_p1
1306- type: partition
1307- number: 1
1308- size: 1GB
1309- device: pnum_disk
1310- - id: pnum_disk_p2
1311- type: partition
1312- number: 2
1313- size: 8MB
1314- device: pnum_disk
1315- flag: prep
1316- wipe: zero
1317- name: prep
1318- - id: pnum_disk_p3
1319- type: partition
1320- number: 10
1321- size: 1GB
1322- device: pnum_disk
1323- - id: swap_mnt
1324- type: mount
1325- path: "none"
1326- device: sda3_swap
1327diff --git a/examples/tests/lvm.yaml b/examples/tests/lvm.yaml
1328index 8eab6b0..1018d1b 100644
1329--- a/examples/tests/lvm.yaml
1330+++ b/examples/tests/lvm.yaml
1331@@ -23,7 +23,7 @@ storage:
1332 flag: boot
1333 - id: sda_extended
1334 type: partition
1335- size: 5G
1336+ size: 5.5G
1337 flag: extended
1338 device: sda
1339 - id: sda2
1340diff --git a/examples/tests/lvm_iscsi.yaml b/examples/tests/lvm_iscsi.yaml
1341index dd7c2b6..1f2ad01 100644
1342--- a/examples/tests/lvm_iscsi.yaml
1343+++ b/examples/tests/lvm_iscsi.yaml
1344@@ -12,7 +12,7 @@ storage:
1345 - id: vdb1
1346 type: partition
1347 number: 1
1348- size: 3GB
1349+ size: 4GB
1350 device: vdb
1351 flag: boot
1352 - id: vdb2
1353@@ -44,7 +44,7 @@ storage:
1354 wipe: superblock
1355 - id: sda_extended
1356 type: partition
1357- size: 5G
1358+ size: 5.5G
1359 flag: extended
1360 device: sda
1361 - id: sda1
1362@@ -100,7 +100,7 @@ storage:
1363 wipe: superblock
1364 - id: sdb_extended
1365 type: partition
1366- size: 4G
1367+ size: 4.5G
1368 flag: extended
1369 device: sdb
1370 - id: sdb1
1371diff --git a/examples/tests/mirrorboot.yaml b/examples/tests/mirrorboot.yaml
1372index 42fdc93..83217d8 100644
1373--- a/examples/tests/mirrorboot.yaml
1374+++ b/examples/tests/mirrorboot.yaml
1375@@ -17,7 +17,7 @@ storage:
1376 flag: bios_grub
1377 - id: sda1
1378 type: partition
1379- size: 3GB
1380+ size: 3.5GB
1381 device: sda
1382 - id: sdb
1383 type: disk
1384@@ -28,7 +28,7 @@ storage:
1385 name: second_disk
1386 - id: sdb1
1387 type: partition
1388- size: 3GB
1389+ size: 3.5GB
1390 device: sdb
1391 - id: mddevice
1392 name: md0
1393diff --git a/examples/tests/multipath-lvm-part-wipe.yaml b/examples/tests/multipath-lvm-part-wipe.yaml
1394index cb18a08..8400d78 100644
1395--- a/examples/tests/multipath-lvm-part-wipe.yaml
1396+++ b/examples/tests/multipath-lvm-part-wipe.yaml
1397@@ -113,7 +113,7 @@ storage:
1398 - id: root_vg_lv1
1399 type: lvm_partition
1400 name: lv1_root
1401- size: 2.5G
1402+ size: 3.5G
1403 volgroup: root_vg
1404 - id: lv1_root_fs
1405 type: format
1406diff --git a/examples/tests/multipath-reuse.yaml b/examples/tests/multipath-reuse.yaml
1407index 24e193e..f008848 100644
1408--- a/examples/tests/multipath-reuse.yaml
1409+++ b/examples/tests/multipath-reuse.yaml
1410@@ -6,8 +6,8 @@ bucket:
1411 - &setup |
1412 parted /dev/disk/by-id/dm-name-mpatha --script -- \
1413 mklabel msdos \
1414- mkpart primary ext4 1GiB 4GiB \
1415- mkpart primary ext4 4GiB 5GiB \
1416+ mkpart primary ext4 1GiB 5GiB \
1417+ mkpart primary ext4 5GiB 6GiB \
1418 set 1 boot on
1419 udevadm settle
1420
1421@@ -32,16 +32,18 @@ storage:
1422 - id: sda1
1423 type: partition
1424 number: 1
1425- size: 3GB
1426+ size: 4GB
1427 device: sda
1428 flag: boot
1429 preserve: true
1430+ offset: 1G
1431 - id: sda2
1432 type: partition
1433 number: 2
1434 size: 1GB
1435 device: sda
1436 preserve: true
1437+ offset: 5G
1438 - id: sda1_root
1439 type: format
1440 fstype: ext4
1441diff --git a/examples/tests/multipath.yaml b/examples/tests/multipath.yaml
1442index 11838d1..a3b536f 100644
1443--- a/examples/tests/multipath.yaml
1444+++ b/examples/tests/multipath.yaml
1445@@ -16,7 +16,7 @@ storage:
1446 - id: sda1
1447 type: partition
1448 number: 1
1449- size: 3GB
1450+ size: 4GB
1451 device: sda
1452 flag: boot
1453 wipe: superblock
1454diff --git a/examples/tests/partition-existing-raid.yaml b/examples/tests/partition-existing-raid.yaml
1455index 07cf8d2..423ab85 100644
1456--- a/examples/tests/partition-existing-raid.yaml
1457+++ b/examples/tests/partition-existing-raid.yaml
1458@@ -15,7 +15,7 @@ bucket:
1459 /dev/disk/by-id/virtio-disk-b-part1 /dev/disk/by-id/virtio-disk-c-part1
1460 udevadm settle
1461 parted /dev/md1 --script -- \
1462- mklabel dos
1463+ mklabel msdos
1464 udevadm settle
1465 mdadm --stop /dev/md1
1466 udevadm settle
1467@@ -51,7 +51,7 @@ storage:
1468 id: id_disk0_part2
1469 device: id_disk0
1470 number: 2
1471- size: 3G
1472+ size: 4G
1473 - type: partition
1474 id: id_disk0_part3
1475 device: id_disk0
1476@@ -63,6 +63,7 @@ storage:
1477 flag: boot
1478 number: 1
1479 size: 8G
1480+ offset: 1G
1481 preserve: true
1482 - type: partition
1483 id: id_disk2_part1
1484@@ -70,6 +71,7 @@ storage:
1485 flag: boot
1486 number: 1
1487 size: 8G
1488+ offset: 1G
1489 preserve: true
1490 - type: raid
1491 id: raid-md1
1492diff --git a/examples/tests/preserve-bcache.yaml b/examples/tests/preserve-bcache.yaml
1493index f614f37..13f8d54 100644
1494--- a/examples/tests/preserve-bcache.yaml
1495+++ b/examples/tests/preserve-bcache.yaml
1496@@ -10,6 +10,7 @@ bucket:
1497 udevadm settle
1498 make-bcache -C /dev/disk/by-id/virtio-disk-b \
1499 -B /dev/disk/by-id/virtio-disk-a-part2 --writeback
1500+ sleep 1
1501 udevadm settle
1502 mkfs.ext4 /dev/bcache0
1503 mount /dev/bcache0 /mnt
1504@@ -46,6 +47,7 @@ storage:
1505 size: 1024M
1506 preserve: true
1507 wipe: superblock
1508+ offset: 1M
1509 - id: id_rotary0_part2
1510 type: partition
1511 name: rotary0-part2
1512@@ -53,6 +55,7 @@ storage:
1513 number: 2
1514 size: 8G
1515 preserve: true
1516+ offset: 1026M
1517 - id: id_bcache0
1518 type: bcache
1519 name: bcache0
1520diff --git a/examples/tests/preserve-lvm.yaml b/examples/tests/preserve-lvm.yaml
1521index a939759..58bfa1f 100644
1522--- a/examples/tests/preserve-lvm.yaml
1523+++ b/examples/tests/preserve-lvm.yaml
1524@@ -47,6 +47,7 @@ storage:
1525 device: main_disk
1526 flag: bios_grub
1527 preserve: true
1528+ offset: 1MB
1529 - id: main_disk_p2
1530 type: partition
1531 number: 2
1532@@ -54,6 +55,7 @@ storage:
1533 device: main_disk
1534 flag: boot
1535 preserve: true
1536+ offset: 3MB
1537 - id: root_vg
1538 type: lvm_volgroup
1539 name: root_vg
1540diff --git a/examples/tests/preserve-partition-wipe-vg-simple.yaml b/examples/tests/preserve-partition-wipe-vg-simple.yaml
1541index e1f0b9e..9876b42 100644
1542--- a/examples/tests/preserve-partition-wipe-vg-simple.yaml
1543+++ b/examples/tests/preserve-partition-wipe-vg-simple.yaml
1544@@ -39,6 +39,7 @@ storage:
1545 number: 1
1546 type: partition
1547 id: disk-sda-part-1
1548+ offset: 2M
1549 - device: disk-sda
1550 size: 3G
1551 flag: linux
1552@@ -47,6 +48,7 @@ storage:
1553 wipe: zero
1554 type: partition
1555 id: disk-sda-part-2
1556+ offset: 4G
1557 - fstype: ext4
1558 volume: disk-sda-part-2
1559 preserve: false
1560diff --git a/examples/tests/preserve-partition-wipe-vg.yaml b/examples/tests/preserve-partition-wipe-vg.yaml
1561index 27a4235..5e35a54 100644
1562--- a/examples/tests/preserve-partition-wipe-vg.yaml
1563+++ b/examples/tests/preserve-partition-wipe-vg.yaml
1564@@ -51,6 +51,7 @@ storage:
1565 wipe: zero
1566 type: partition
1567 id: disk-sda-part-1
1568+ offset: 2M
1569 - device: disk-sda
1570 size: 3G
1571 flag: linux
1572@@ -58,6 +59,7 @@ storage:
1573 wipe: zero
1574 type: partition
1575 id: disk-sda-part-2
1576+ offset: 1G
1577 - device: disk-sdb
1578 flag: linux
1579 size: 3G
1580@@ -65,12 +67,14 @@ storage:
1581 wipe: zero
1582 type: partition
1583 id: disk-sdb-part-1
1584+ offset: 1G
1585 - device: disk-sdb
1586 flag: linux
1587 size: 3G
1588 preserve: true
1589 type: partition
1590 id: disk-sdb-part-2
1591+ offset: 4G
1592 - fstype: ext4
1593 volume: disk-sda-part-2
1594 preserve: false
1595diff --git a/examples/tests/preserve-raid.yaml b/examples/tests/preserve-raid.yaml
1596index 9e0489f..3d39c80 100644
1597--- a/examples/tests/preserve-raid.yaml
1598+++ b/examples/tests/preserve-raid.yaml
1599@@ -52,7 +52,7 @@ storage:
1600 id: id_disk0_part2
1601 device: id_disk0
1602 number: 2
1603- size: 3G
1604+ size: 4G
1605 - type: partition
1606 id: id_disk0_part3
1607 device: id_disk0
1608@@ -65,6 +65,7 @@ storage:
1609 number: 1
1610 size: 8G
1611 preserve: true
1612+ offset: 1G
1613 - type: partition
1614 id: id_disk2_part1
1615 device: id_disk2
1616@@ -72,6 +73,7 @@ storage:
1617 number: 1
1618 size: 8G
1619 preserve: true
1620+ offset: 1G
1621 - type: raid
1622 id: raid-md1
1623 name: md1
1624diff --git a/examples/tests/preserve.yaml b/examples/tests/preserve.yaml
1625index de8a975..2cf692e 100644
1626--- a/examples/tests/preserve.yaml
1627+++ b/examples/tests/preserve.yaml
1628@@ -6,8 +6,8 @@ bucket:
1629 mklabel gpt \
1630 mkpart primary ext4 2MiB 514MiB \
1631 set 1 esp on \
1632- mkpart primary ext4 1GiB 4GiB \
1633- mkpart primary ext4 4GiB 7GiB
1634+ mkpart primary ext4 1GiB 5GiB \
1635+ mkpart primary ext4 6GiB 9GiB
1636 udevadm settle
1637 mkfs.ext4 /dev/disk/by-id/virtio-disk-a-part3
1638 mount /dev/disk/by-id/virtio-disk-a-part3 /mnt
1639@@ -32,18 +32,21 @@ storage:
1640 number: 1
1641 size: 512M
1642 preserve: true
1643+ offset: 2M
1644 - type: partition
1645 id: id_disk0_part2
1646 device: id_disk0
1647 number: 2
1648- size: 3G
1649+ size: 4G
1650 preserve: true
1651+ offset: 1G
1652 - type: partition
1653 id: id_disk0_part3
1654 device: id_disk0
1655 number: 3
1656 size: 3G
1657 preserve: true
1658+ offset: 6G
1659 - type: format
1660 id: id_efi_format
1661 volume: id_disk0_part1
1662diff --git a/examples/tests/reuse-lvm-member-partition.yaml b/examples/tests/reuse-lvm-member-partition.yaml
1663index fd8f602..cad1474 100644
1664--- a/examples/tests/reuse-lvm-member-partition.yaml
1665+++ b/examples/tests/reuse-lvm-member-partition.yaml
1666@@ -69,12 +69,14 @@ storage:
1667 flag: boot
1668 number: 1
1669 size: 1G
1670+ offset: 1G
1671 - type: partition
1672 id: id_disk0_part2
1673 preserve: true
1674 device: id_disk0
1675 number: 2
1676 size: 7G
1677+ offset: 2G
1678 - type: format
1679 id: id_efi_format
1680 volume: id_disk0_part1
1681diff --git a/examples/tests/reuse-msdos-partitions.yaml b/examples/tests/reuse-msdos-partitions.yaml
1682index d444517..f3c6974 100644
1683--- a/examples/tests/reuse-msdos-partitions.yaml
1684+++ b/examples/tests/reuse-msdos-partitions.yaml
1685@@ -43,6 +43,7 @@ storage:
1686 flag: boot
1687 preserve: true
1688 wipe: superblock
1689+ offset: 1M
1690 - id: sda2
1691 type: partition
1692 number: 2
1693@@ -50,6 +51,7 @@ storage:
1694 flag: extended
1695 device: sda
1696 preserve: true
1697+ offset: 3074M
1698 - id: sda5
1699 type: partition
1700 number: 5
1701@@ -58,6 +60,7 @@ storage:
1702 device: sda
1703 preserve: true
1704 wipe: superblock
1705+ offset: 3075M
1706 - id: sda6
1707 type: partition
1708 number: 6
1709@@ -66,6 +69,7 @@ storage:
1710 device: sda
1711 preserve: true
1712 wipe: superblock
1713+ offset: 5123M
1714 - id: sda1_root
1715 type: format
1716 fstype: ext4
1717diff --git a/examples/tests/reuse-raid-member-wipe-partition.yaml b/examples/tests/reuse-raid-member-wipe-partition.yaml
1718index d20b79c..136f96e 100644
1719--- a/examples/tests/reuse-raid-member-wipe-partition.yaml
1720+++ b/examples/tests/reuse-raid-member-wipe-partition.yaml
1721@@ -49,6 +49,7 @@ storage:
1722 flag: boot
1723 number: 1
1724 size: 1G
1725+ offset: 1G
1726 - type: partition
1727 id: id_disk0_part2
1728 preserve: true
1729@@ -56,6 +57,7 @@ storage:
1730 number: 2
1731 size: 7G
1732 wipe: superblock
1733+ offset: 2G
1734 - type: format
1735 id: id_efi_format
1736 volume: id_disk0_part1
1737diff --git a/examples/tests/uefi_basic.yaml b/examples/tests/uefi_basic.yaml
1738index 91a72ae..e6ad351 100644
1739--- a/examples/tests/uefi_basic.yaml
1740+++ b/examples/tests/uefi_basic.yaml
1741@@ -31,7 +31,7 @@ storage:
1742 - device: id_disk0
1743 id: id_disk0_part2
1744 number: 2
1745- size: 3G
1746+ size: 4G
1747 type: partition
1748 wipe: superblock
1749 - fstype: fat32
1750diff --git a/examples/tests/uefi_reuse_esp.yaml b/examples/tests/uefi_reuse_esp.yaml
1751index 7ad7fdf..0232019 100644
1752--- a/examples/tests/uefi_reuse_esp.yaml
1753+++ b/examples/tests/uefi_reuse_esp.yaml
1754@@ -8,7 +8,7 @@ bucket:
1755 mklabel gpt \
1756 mkpart primary fat32 1MiB 513MiB \
1757 set 1 esp on \
1758- mkpart primary ext4 513MiB 3585MiB
1759+ mkpart primary ext4 513MiB 4609MiB
1760
1761 udevadm settle
1762 mkfs.vfat -I -n EFI -F 32 /dev/disk/by-id/virtio-disk-a-part1
1763@@ -67,9 +67,10 @@ storage:
1764 - device: id_disk0
1765 id: id_disk0_part2
1766 number: 2
1767- size: 3G
1768+ size: 4G
1769 type: partition
1770 preserve: true
1771+ offset: 513M
1772 - fstype: fat32
1773 id: id_efi_format
1774 label: efi
1775diff --git a/pylintrc b/pylintrc
1776index 1b5fa1a..7a50917 100644
1777--- a/pylintrc
1778+++ b/pylintrc
1779@@ -7,7 +7,7 @@ jobs=0
1780 # List of members which are set dynamically and missed by pylint inference
1781 # system, and so shouldn't trigger E1101 when accessed. Python regular
1782 # expressions are accepted.
1783-generated-members=redhat,centos,fedora,debian,suse,opensuse,sles,arch,ubuntu,rhel,freebsd,gentoo
1784+generated-members=redhat,centos,fedora,debian,suse,opensuse,sles,arch,ubuntu,rhel,freebsd,gentoo,rocky
1785
1786 # List of module names for which member attributes should not be checked
1787 # (useful for modules/projects where namespaces are manipulated during runtime
1788diff --git a/test-requirements.txt b/test-requirements.txt
1789index f6404c1..1970d03 100644
1790--- a/test-requirements.txt
1791+++ b/test-requirements.txt
1792@@ -3,3 +3,4 @@ mock
1793 nose
1794 pyflakes
1795 coverage
1796+parameterized
1797diff --git a/tests/integration/test_block_meta.py b/tests/integration/test_block_meta.py
1798index bd602b2..e542017 100644
1799--- a/tests/integration/test_block_meta.py
1800+++ b/tests/integration/test_block_meta.py
1801@@ -1,13 +1,21 @@
1802 # This file is part of curtin. See LICENSE file for copyright and license info.
1803
1804-from collections import namedtuple
1805+import dataclasses
1806+from dataclasses import dataclass
1807 import contextlib
1808+import json
1809+import os
1810+from parameterized import parameterized
1811+import re
1812 import sys
1813+from typing import Optional
1814 import yaml
1815-import os
1816
1817 from curtin import block, udev, util
1818
1819+from curtin.commands.block_meta import _get_volume_fstype
1820+from curtin.commands.block_meta_v2 import ONE_MIB_BYTES
1821+
1822 from tests.unittests.helpers import CiTestCase
1823 from tests.integration.webserv import ImageServer
1824
1825@@ -17,10 +25,11 @@ class IntegrationTestCase(CiTestCase):
1826
1827
1828 @contextlib.contextmanager
1829-def loop_dev(image):
1830- dev = util.subp(
1831- ['losetup', '--show', '--find', '--partscan', image],
1832- capture=True, decode='ignore')[0].strip()
1833+def loop_dev(image, sector_size=512):
1834+ dev = util.subp([
1835+ 'losetup', '--show', '--find', '--partscan',
1836+ '--sector-size', str(sector_size), image,
1837+ ], capture=True, decode='ignore')[0].strip()
1838 try:
1839 udev.udevadm_trigger([dev])
1840 yield dev
1841@@ -28,18 +37,114 @@ def loop_dev(image):
1842 util.subp(['losetup', '--detach', dev])
1843
1844
1845-PartData = namedtuple("PartData", ('number', 'offset', 'size'))
1846+@dataclass(order=True)
1847+class PartData:
1848+ number: Optional[int] = None
1849+ offset: Optional[int] = None
1850+ size: Optional[int] = None
1851+ boot: Optional[bool] = None
1852+ partition_type: Optional[str] = None
1853+
1854+ # test cases may initialize the values they care about
1855+ # test utilities shall initialize all fields
1856+ def assertFieldsAreNotNone(self):
1857+ for field in dataclasses.fields(self):
1858+ assert getattr(self, field.name) is not None
1859+
1860+ def __eq__(self, other):
1861+ for field in dataclasses.fields(self):
1862+ myval = getattr(self, field.name)
1863+ otherval = getattr(other, field.name)
1864+ if myval is not None and otherval is not None \
1865+ and myval != otherval:
1866+ return False
1867+ return True
1868+
1869+
1870+def _get_ext_size(dev, part_action):
1871+ num = part_action['number']
1872+ cmd = ['dumpe2fs', '-h', f'{dev}p{num}']
1873+ out = util.subp(cmd, capture=True)[0]
1874+ for line in out.splitlines():
1875+ if line.startswith('Block count'):
1876+ block_count = line.split(':')[1].strip()
1877+ if line.startswith('Block size'):
1878+ block_size = line.split(':')[1].strip()
1879+ return int(block_count) * int(block_size)
1880+
1881+
1882+def _get_ntfs_size(dev, part_action):
1883+ num = part_action['number']
1884+ cmd = ['ntfsresize',
1885+ '--no-action',
1886+ '--force', # needed post-resize, which otherwise demands a CHKDSK
1887+ '--info', f'{dev}p{num}']
1888+ out = util.subp(cmd, capture=True)[0]
1889+ # Sample input:
1890+ # Current volume size: 41939456 bytes (42 MB)
1891+ volsize_matcher = re.compile(r'^Current volume size: ([0-9]+) bytes')
1892+ for line in out.splitlines():
1893+ m = volsize_matcher.match(line)
1894+ if m:
1895+ return int(m.group(1))
1896+ raise Exception('ntfs volume size not found')
1897+
1898+
1899+_get_fs_sizers = {
1900+ 'ext2': _get_ext_size,
1901+ 'ext3': _get_ext_size,
1902+ 'ext4': _get_ext_size,
1903+ 'ntfs': _get_ntfs_size,
1904+}
1905+
1906+
1907+def _get_filesystem_size(dev, part_action, fstype='ext4'):
1908+ if fstype not in _get_fs_sizers.keys():
1909+ raise Exception(f'_get_filesystem_size: no support for {fstype}')
1910+ return _get_fs_sizers[fstype](dev, part_action)
1911+
1912+
1913+def _get_extended_partition_size(dev, num):
1914+ # sysfs reports extended partitions as having 1K size
1915+ # sfdisk seems to have a better idea
1916+ ptable_json = util.subp(['sfdisk', '-J', dev], capture=True)[0]
1917+ ptable = json.loads(ptable_json)
1918+
1919+ nodename = f'{dev}p{num}'
1920+ partitions = ptable['partitiontable']['partitions']
1921+ partition = [part for part in partitions if part['node'] == nodename][0]
1922+ return partition['size'] * 512
1923+
1924+
1925+def _get_disk_label_id(dev):
1926+ ptable_json = util.subp(['sfdisk', '-J', dev], capture=True)[0]
1927+ ptable = json.loads(ptable_json)
1928+ # string in lowercase hex
1929+ return ptable['partitiontable']['id']
1930
1931
1932 def summarize_partitions(dev):
1933- # We don't care about the kname
1934- return sorted(
1935- [PartData(*d[1:]) for d in block.sysfs_partition_data(dev)])
1936+ parts = []
1937+ ptable_json = util.subp(['sfdisk', '-J', dev], capture=True)[0]
1938+ ptable = json.loads(ptable_json)
1939+ partitions = ptable['partitiontable']['partitions']
1940+ for d in block.sysfs_partition_data(dev):
1941+ nodename = f'/dev/{d[0]}'
1942+ partition = [part for part in partitions
1943+ if part['node'] == nodename][0]
1944+ ptype = partition['type']
1945+ boot = partition.get('bootable', False)
1946+ # We don't care about the kname
1947+ pd = PartData(*d[1:], partition_type=ptype, boot=boot)
1948+ pd.assertFieldsAreNotNone()
1949+ parts.append(pd)
1950+ return sorted(parts)
1951
1952
1953 class StorageConfigBuilder:
1954
1955- def __init__(self):
1956+ def __init__(self, *, version):
1957+ self.version = version
1958 self.config = []
1959 self.cur_image = None
1960
1961@@ -47,37 +152,81 @@ class StorageConfigBuilder:
1962 return {
1963 'storage': {
1964 'config': self.config,
1965+ 'version': self.version,
1966 },
1967 }
1968
1969- def add_image(self, *, path, size, create=False, **kw):
1970- action = {
1971- 'type': 'image',
1972- 'id': 'id' + str(len(self.config)),
1973- 'path': path,
1974- 'size': size,
1975- }
1976- action.update(**kw)
1977- self.cur_image = action['id']
1978+ def _add(self, *, type, **kw):
1979+ if type != 'image' and self.cur_image is None:
1980+ raise Exception("no current image")
1981+ action = {'id': 'id' + str(len(self.config))}
1982+ action.update(type=type, **kw)
1983 self.config.append(action)
1984+ return action
1985+
1986+ def add_image(self, *, path, size, create=False, **kw):
1987 if create:
1988 with open(path, "wb") as f:
1989 f.write(b"\0" * int(util.human2bytes(size)))
1990+ action = self._add(type='image', path=path, size=size, **kw)
1991+ self.cur_image = action['id']
1992+ return action
1993
1994 def add_part(self, *, size, **kw):
1995- if self.cur_image is None:
1996- raise Exception("no current image")
1997- action = {
1998- 'type': 'partition',
1999- 'id': 'id' + str(len(self.config)),
2000- 'device': self.cur_image,
2001- 'size': size,
2002- }
2003- action.update(**kw)
2004- self.config.append(action)
2005+ fstype = kw.pop('fstype', None)
2006+ part = self._add(type='partition', device=self.cur_image, size=size,
2007+ **kw)
2008+ if fstype:
2009+ self.add_format(part=part, fstype=fstype)
2010+ return part
2011+
2012+ def add_format(self, *, part, fstype='ext4', **kw):
2013+ return self._add(type='format', volume=part['id'], fstype=fstype, **kw)
2014+
2015+ def set_preserve(self):
2016+ for action in self.config:
2017+ action['preserve'] = True
2018
2019
2020 class TestBlockMeta(IntegrationTestCase):
2021+ def setUp(self):
2022+ self.data = self.random_string()
2023+
2024+ def assertPartitions(self, *args):
2025+ with loop_dev(self.img) as dev:
2026+ self.assertEqual([*args], summarize_partitions(dev))
2027+
2028+ @contextlib.contextmanager
2029+ def mount(self, dev, partition_cfg):
2030+ mnt_point = self.tmp_dir()
2031+ num = partition_cfg['number']
2032+ with util.mount(f'{dev}p{num}', mnt_point):
2033+ yield mnt_point
2034+
2035+ @contextlib.contextmanager
2036+ def open_file_on_part(self, dev, part_action, mode):
2037+ with self.mount(dev, part_action) as mnt_point:
2038+ with open(f'{mnt_point}/data.txt', mode) as fp:
2039+ yield fp
2040+
2041+ def create_data(self, dev, part_action):
2042+ with self.open_file_on_part(dev, part_action, 'w') as fp:
2043+ fp.write(self.data)
2044+
2045+ def check_data(self, dev, part_action):
2046+ with self.open_file_on_part(dev, part_action, 'r') as fp:
2047+ self.assertEqual(self.data, fp.read())
2048+
2049+ def check_fssize(self, dev, part_action, fstype, expected):
2050+ tolerance = 0
2051+ if fstype == 'ntfs':
2052+ # Per ntfsresize manpage, the actual fs size is at least one sector
2053+ # less than requested.
2054+ # In these tests it has been consistently 7 sectors fewer.
2055+ tolerance = 512 * 10
2056+ actual_fssize = _get_filesystem_size(dev, part_action, fstype)
2057+ diff = expected - actual_fssize
2058+ self.assertTrue(0 <= diff <= tolerance, f'difference of {diff}')
2059
2060 def run_bm(self, config, *args, **kwargs):
2061 config_path = self.tmp_path('config.yaml')
2062@@ -102,34 +251,88 @@ class TestBlockMeta(IntegrationTestCase):
2063 ]
2064 util.subp(cmd, env=cmd_env, **kwargs)
2065
2066- def _test_default_offsets(self, ptable):
2067+ def _test_default_offsets(self, ptable, version, sector_size=512):
2068 psize = 40 << 20
2069 img = self.tmp_path('image.img')
2070- config = StorageConfigBuilder()
2071+ config = StorageConfigBuilder(version=version)
2072+ config.add_image(
2073+ path=img, size='200M', ptable=ptable, sector_size=sector_size)
2074+ p1 = config.add_part(size=psize, number=1)
2075+ p2 = config.add_part(size=psize, number=2)
2076+ p3 = config.add_part(size=psize, number=3)
2077+ self.run_bm(config.render())
2078+
2079+ with loop_dev(img, sector_size) as dev:
2080+ self.assertEqual(
2081+ summarize_partitions(dev), [
2082+ PartData(number=1, offset=1 << 20, size=psize),
2083+ PartData(number=2, offset=(1 << 20) + psize, size=psize),
2084+ PartData(number=3, offset=(1 << 20) + 2*psize, size=psize),
2085+ ])
2086+ p1['offset'] = 1 << 20
2087+ p2['offset'] = (1 << 20) + psize
2088+ p3['offset'] = (1 << 20) + 2*psize
2089+ config.set_preserve()
2090+ self.run_bm(config.render())
2091+
2092+ def test_default_offsets_gpt_v1(self):
2093+ self._test_default_offsets('gpt', 1)
2094+
2095+ def test_default_offsets_msdos_v1(self):
2096+ self._test_default_offsets('msdos', 1)
2097+
2098+ def test_default_offsets_gpt_v2(self):
2099+ self._test_default_offsets('gpt', 2)
2100+
2101+ def test_default_offsets_msdos_v2(self):
2102+ self._test_default_offsets('msdos', 2)
2103+
2104+ def test_default_offsets_gpt_v1_4k(self):
2105+ self._test_default_offsets('gpt', 1, 4096)
2106+
2107+ def test_default_offsets_msdos_v1_4k(self):
2108+ self._test_default_offsets('msdos', 1, 4096)
2109+
2110+ def test_default_offsets_gpt_v2_4k(self):
2111+ self._test_default_offsets('gpt', 2, 4096)
2112+
2113+ def test_default_offsets_msdos_v2_4k(self):
2114+ self._test_default_offsets('msdos', 2, 4096)
2115+
2116+ def _test_specified_offsets(self, ptable, version):
2117+ psize = 20 << 20
2118+ img = self.tmp_path('image.img')
2119+ config = StorageConfigBuilder(version=version)
2120 config.add_image(path=img, size='100M', ptable=ptable)
2121- config.add_part(size=psize, number=1)
2122- config.add_part(size=psize, number=2)
2123+ config.add_part(size=psize, number=1, offset=psize)
2124+ config.add_part(size=psize, number=2, offset=psize * 3)
2125 self.run_bm(config.render())
2126
2127 with loop_dev(img) as dev:
2128 self.assertEqual(
2129 summarize_partitions(dev), [
2130- PartData(
2131- number=1, offset=1 << 20, size=psize),
2132- PartData(
2133- number=2, offset=(1 << 20) + psize, size=psize),
2134+ PartData(number=1, offset=psize, size=psize),
2135+ PartData(number=2, offset=psize*3, size=psize),
2136 ])
2137+ config.set_preserve()
2138+ self.run_bm(config.render())
2139+
2140+ def DONT_test_specified_offsets_gpt_v1(self):
2141+ self._test_specified_offsets('gpt', 1)
2142+
2143+ def DONT_test_specified_offsets_msdos_v1(self):
2144+ self._test_specified_offsets('msdos', 1)
2145
2146- def test_default_offsets_gpt(self):
2147- self._test_default_offsets('gpt')
2148+ def test_specified_offsets_gpt_v2(self):
2149+ self._test_specified_offsets('gpt', 2)
2150
2151- def test_default_offsets_msdos(self):
2152- self._test_default_offsets('msdos')
2153+ def test_specified_offsets_msdos_v2(self):
2154+ self._test_specified_offsets('msdos', 2)
2155
2156- def _test_non_default_numbering(self, ptable):
2157+ def _test_non_default_numbering(self, ptable, version):
2158 psize = 40 << 20
2159 img = self.tmp_path('image.img')
2160- config = StorageConfigBuilder()
2161+ config = StorageConfigBuilder(version=version)
2162 config.add_image(path=img, size='100M', ptable=ptable)
2163 config.add_part(size=psize, number=1)
2164 config.add_part(size=psize, number=4)
2165@@ -138,23 +341,30 @@ class TestBlockMeta(IntegrationTestCase):
2166 with loop_dev(img) as dev:
2167 self.assertEqual(
2168 summarize_partitions(dev), [
2169- PartData(
2170- number=1, offset=1 << 20, size=psize),
2171- PartData(
2172- number=4, offset=(1 << 20) + psize, size=psize),
2173+ PartData(number=1, offset=1 << 20, size=psize),
2174+ PartData(number=4, offset=(1 << 20) + psize, size=psize),
2175 ])
2176
2177- def test_non_default_numbering_gpt(self):
2178- self._test_non_default_numbering('gpt')
2179+ def test_non_default_numbering_gpt_v1(self):
2180+ self._test_non_default_numbering('gpt', 1)
2181
2182- def BROKEN_test_non_default_numbering_msdos(self):
2183- self._test_non_default_numbering('msdos')
2184+ def BROKEN_test_non_default_numbering_msdos_v1(self):
2185+ self._test_non_default_numbering('msdos', 2)
2186
2187- def test_logical(self):
2188+ def test_non_default_numbering_gpt_v2(self):
2189+ self._test_non_default_numbering('gpt', 2)
2190+
2191+ def test_non_default_numbering_msdos_v2(self):
2192+ self._test_non_default_numbering('msdos', 2)
2193+
2194+ def _test_logical(self, version):
2195 img = self.tmp_path('image.img')
2196- config = StorageConfigBuilder()
2197+ config = StorageConfigBuilder(version=version)
2198 config.add_image(path=img, size='100M', ptable='msdos')
2199- config.add_part(size='50M', number=1, flag='extended')
2200+ # curtin adds 1MiB to the size of the extend partition per contained
2201+ # logical partition, but only in v1 mode
2202+ size = '97M' if version == 1 else '99M'
2203+ config.add_part(size=size, number=1, flag='extended')
2204 config.add_part(size='10M', number=5, flag='logical')
2205 config.add_part(size='10M', number=6, flag='logical')
2206 self.run_bm(config.render())
2207@@ -163,19 +373,162 @@ class TestBlockMeta(IntegrationTestCase):
2208 self.assertEqual(
2209 summarize_partitions(dev), [
2210 # extended partitions get a strange size in sysfs
2211- PartData(number=1, offset=1 << 20, size=1 << 10),
2212- PartData(number=5, offset=2 << 20, size=10 << 20),
2213+ PartData(number=1, offset=1 << 20, size=1 << 10),
2214+ PartData(number=5, offset=2 << 20, size=10 << 20),
2215 # part 5 takes us to 12 MiB offset, curtin leaves a 1 MiB
2216 # gap.
2217 PartData(number=6, offset=13 << 20, size=10 << 20),
2218 ])
2219+ self.assertEqual(99 << 20, _get_extended_partition_size(dev, 1))
2220
2221 p1kname = block.partition_kname(block.path_to_kname(dev), 1)
2222 self.assertTrue(block.is_extended_partition('/dev/' + p1kname))
2223
2224+ def test_logical_v1(self):
2225+ self._test_logical(1)
2226+
2227+ def test_logical_v2(self):
2228+ self._test_logical(2)
2229+
2230+ def _test_replace_partition(self, ptable):
2231+ psize = 20 << 20
2232+ img = self.tmp_path('image.img')
2233+ config = StorageConfigBuilder(version=2)
2234+ config.add_image(path=img, size='100M', ptable=ptable)
2235+ config.add_part(size=psize, number=1)
2236+ config.add_part(size=psize, number=2)
2237+ self.run_bm(config.render())
2238+
2239+ with loop_dev(img) as dev:
2240+ self.assertEqual(
2241+ summarize_partitions(dev), [
2242+ PartData(number=1, offset=1 << 20, size=psize),
2243+ PartData(number=2, offset=(1 << 20) + psize, size=psize),
2244+ ])
2245+
2246+ config = StorageConfigBuilder(version=2)
2247+ config.add_image(path=img, size='100M', ptable=ptable, preserve=True)
2248+ config.add_part(size=psize, number=1, offset=1 << 20, preserve=True)
2249+ config.add_part(size=psize*2, number=2)
2250+ self.run_bm(config.render())
2251+
2252+ with loop_dev(img) as dev:
2253+ self.assertEqual(
2254+ summarize_partitions(dev), [
2255+ PartData(number=1, offset=1 << 20, size=psize),
2256+ PartData(number=2, offset=(1 << 20) + psize, size=2*psize),
2257+ ])
2258+
2259+ def test_replace_partition_gpt_v2(self):
2260+ self._test_replace_partition('gpt')
2261+
2262+ def test_replace_partition_msdos_v2(self):
2263+ self._test_replace_partition('msdos')
2264+
2265+ def test_delete_logical_partition(self):
2266+ # The test case that resulted in a lot of hair-pulling:
2267+ # deleting a logical partition renumbers any later partitions
2268+ # (so you cannot stably refer to partitions by number!)
2269+ psize = 20 << 20
2270+ img = self.tmp_path('image.img')
2271+ config = StorageConfigBuilder(version=2)
2272+ config.add_image(path=img, size='100M', ptable='msdos')
2273+ config.add_part(size='90M', number=1, flag='extended')
2274+ config.add_part(size=psize, number=5, flag='logical')
2275+ config.add_part(size=psize, number=6, flag='logical')
2276+ self.run_bm(config.render())
2277+
2278+ with loop_dev(img) as dev:
2279+ self.assertEqual(
2280+ summarize_partitions(dev), [
2281+ PartData(number=1, offset=1 << 20, size=1 << 10),
2282+ PartData(number=5, offset=(2 << 20), size=psize),
2283+ PartData(number=6, offset=(3 << 20) + psize, size=psize),
2284+ ])
2285+ self.assertEqual(90 << 20, _get_extended_partition_size(dev, 1))
2286+
2287+ config = StorageConfigBuilder(version=2)
2288+ config.add_image(path=img, size='100M', ptable='msdos', preserve=True)
2289+ config.add_part(size='90M', number=1, flag='extended', preserve=True)
2290+ config.add_part(
2291+ size=psize, number=5, flag='logical', offset=(3 << 20) + psize,
2292+ preserve=True)
2293+ self.run_bm(config.render())
2294+
2295+ with loop_dev(img) as dev:
2296+ self.assertEqual(
2297+ summarize_partitions(dev), [
2298+ PartData(number=1, offset=1 << 20, size=1 << 10),
2299+ PartData(number=5, offset=(3 << 20) + psize, size=psize),
2300+ ])
2301+ self.assertEqual(90 << 20, _get_extended_partition_size(dev, 1))
2302+
2303+ def _test_wiping(self, ptable):
2304+ # Test wiping behaviour.
2305+ #
2306+ # Paritions that should be (superblock, i.e. first and last
2307+ # megabyte) wiped:
2308+ #
2309+ # 1) New partitions
2310+ # 2) Partitions that are being removed, i.e. no longer present
2311+ # 3) Preserved partitions with an explicit wipe
2312+ #
2313+ # Partitions that should not be wiped:
2314+ #
2315+ # 4) Preserved partitions with no wipe field.
2316+ #
2317+ # We test this by creating some partitions with block-meta,
2318+ # writing content to them, then running block-meta again, with
2319+ # each partition matching one of the conditions above.
2320+ img = self.tmp_path('image.img')
2321+ config = StorageConfigBuilder(version=2)
2322+ config.add_image(path=img, size='30M', ptable=ptable)
2323+ config.add_part(size='5M', number=1, offset='5M')
2324+ config.add_part(size='5M', number=2, offset='10M')
2325+ config.add_part(size='5M', number=3, offset='15M')
2326+ config.add_part(size='5M', number=4, offset='20M')
2327+ self.run_bm(config.render())
2328+
2329+ part_offset_sizes = {}
2330+ with loop_dev(img) as dev:
2331+ for kname, number, offset, size in block.sysfs_partition_data(dev):
2332+ content = bytes([number])
2333+ with open(block.kname_to_path(kname), 'wb') as fp:
2334+ fp.write(content*size)
2335+ part_offset_sizes[number] = (offset, size)
2336+
2337+ config = StorageConfigBuilder(version=2)
2338+ config.add_image(path=img, size='30M', ptable=ptable, preserve=True)
2339+ config.add_part(size='5M', number=1, offset='5M')
2340+ # Partition 2 is being deleted.
2341+ config.add_part(
2342+ size='5M', number=3, offset='15M', preserve=True,
2343+ wipe='superblock')
2344+ config.add_part(size='5M', number=4, offset='20M', preserve=True)
2345+ self.run_bm(config.render())
2346+
2347+ expected_content = {1: {0}, 2: {0}, 3: {0}, 4: {4}}
2348+
2349+ with loop_dev(img) as dev:
2350+ with open(dev, 'rb') as fp:
2351+ for nr, (offset, size) in part_offset_sizes.items():
2352+ expected = expected_content[nr]
2353+ fp.seek(offset)
2354+ first = set(fp.read(ONE_MIB_BYTES))
2355+ fp.seek(offset + size - ONE_MIB_BYTES)
2356+ last = set(fp.read(ONE_MIB_BYTES))
2357+ self.assertEqual(first, expected)
2358+ self.assertEqual(last, expected)
2359+
2360+ def test_wiping_gpt(self):
2361+ self._test_wiping('gpt')
2362+
2363+ def test_wiping_msdos(self):
2364+ self._test_wiping('msdos')
2365+
2366 def test_raw_image(self):
2367 img = self.tmp_path('image.img')
2368- config = StorageConfigBuilder()
2369+ config = StorageConfigBuilder(version=1)
2370 config.add_image(path=img, size='2G', ptable='gpt', create=True)
2371
2372 curtin_cfg = config.render()
2373@@ -206,3 +559,432 @@ class TestBlockMeta(IntegrationTestCase):
2374 )
2375 finally:
2376 server.stop()
2377+
2378+ def _do_test_resize(self, start, end, fstype):
2379+ start <<= 20
2380+ end <<= 20
2381+ img = self.tmp_path('image.img')
2382+ config = StorageConfigBuilder(version=2)
2383+ config.add_image(path=img, size='200M', ptable='gpt')
2384+ p1 = config.add_part(size=start, offset=1 << 20, number=1,
2385+ fstype=fstype)
2386+ self.run_bm(config.render())
2387+ with loop_dev(img) as dev:
2388+ self.assertEqual(fstype, _get_volume_fstype(f'{dev}p1'))
2389+ self.create_data(dev, p1)
2390+ self.assertEqual(
2391+ summarize_partitions(dev), [
2392+ PartData(number=1, offset=1 << 20, size=start),
2393+ ])
2394+ self.check_fssize(dev, p1, fstype, start)
2395+
2396+ config.set_preserve()
2397+ p1['resize'] = True
2398+ p1['size'] = end
2399+ self.run_bm(config.render())
2400+ with loop_dev(img) as dev:
2401+ self.check_data(dev, p1)
2402+ self.assertEqual(
2403+ summarize_partitions(dev), [
2404+ PartData(number=1, offset=1 << 20, size=end),
2405+ ])
2406+ self.check_fssize(dev, p1, fstype, end)
2407+
2408+ def test_resize_up_ext2(self):
2409+ self._do_test_resize(40, 80, 'ext2')
2410+
2411+ def test_resize_down_ext2(self):
2412+ self._do_test_resize(80, 40, 'ext2')
2413+
2414+ def test_resize_up_ext3(self):
2415+ self._do_test_resize(40, 80, 'ext3')
2416+
2417+ def test_resize_down_ext3(self):
2418+ self._do_test_resize(80, 40, 'ext3')
2419+
2420+ def test_resize_up_ext4(self):
2421+ self._do_test_resize(40, 80, 'ext4')
2422+
2423+ def test_resize_down_ext4(self):
2424+ self._do_test_resize(80, 40, 'ext4')
2425+
2426+ def test_resize_up_ntfs(self):
2427+ self._do_test_resize(40, 80, 'ntfs')
2428+
2429+ def test_resize_down_ntfs(self):
2430+ self._do_test_resize(80, 40, 'ntfs')
2431+
2432+ def test_resize_logical(self):
2433+ img = self.tmp_path('image.img')
2434+ config = StorageConfigBuilder(version=2)
2435+ config.add_image(path=img, size='100M', ptable='msdos')
2436+ config.add_part(size='50M', number=1, flag='extended', offset=1 << 20)
2437+ config.add_part(size='10M', number=5, flag='logical', offset=2 << 20)
2438+ p6 = config.add_part(size='10M', number=6, flag='logical',
2439+ offset=13 << 20, fstype='ext4')
2440+ self.run_bm(config.render())
2441+
2442+ with loop_dev(img) as dev:
2443+ self.create_data(dev, p6)
2444+ self.assertEqual(
2445+ summarize_partitions(dev), [
2446+ # extended partitions get a strange size in sysfs
2447+ PartData(number=1, offset=1 << 20, size=1 << 10),
2448+ PartData(number=5, offset=2 << 20, size=10 << 20),
2449+ # part 5 takes us to 12 MiB offset, curtin leaves a 1 MiB
2450+ # gap.
2451+ PartData(number=6, offset=13 << 20, size=10 << 20),
2452+ ])
2453+ self.assertEqual(50 << 20, _get_extended_partition_size(dev, 1))
2454+
2455+ config.set_preserve()
2456+ p6['resize'] = True
2457+ p6['size'] = '20M'
2458+ self.run_bm(config.render())
2459+
2460+ with loop_dev(img) as dev:
2461+ self.check_data(dev, p6)
2462+ self.assertEqual(
2463+ summarize_partitions(dev), [
2464+ PartData(number=1, offset=1 << 20, size=1 << 10),
2465+ PartData(number=5, offset=2 << 20, size=10 << 20),
2466+ PartData(number=6, offset=13 << 20, size=20 << 20),
2467+ ])
2468+ self.assertEqual(50 << 20, _get_extended_partition_size(dev, 1))
2469+
2470+ def test_resize_extended(self):
2471+ img = self.tmp_path('image.img')
2472+ config = StorageConfigBuilder(version=2)
2473+ config.add_image(path=img, size='100M', ptable='msdos')
2474+ p1 = config.add_part(size='50M', number=1, flag='extended',
2475+ offset=1 << 20)
2476+ p5 = config.add_part(size='49M', number=5, flag='logical',
2477+ offset=2 << 20)
2478+ self.run_bm(config.render())
2479+
2480+ with loop_dev(img) as dev:
2481+ self.assertEqual(
2482+ summarize_partitions(dev), [
2483+ # extended partitions get a strange size in sysfs
2484+ PartData(number=1, offset=1 << 20, size=1 << 10),
2485+ PartData(number=5, offset=2 << 20, size=49 << 20),
2486+ ])
2487+ self.assertEqual(50 << 20, _get_extended_partition_size(dev, 1))
2488+
2489+ config.set_preserve()
2490+ p1['resize'] = True
2491+ p1['size'] = '99M'
2492+ p5['resize'] = True
2493+ p5['size'] = '98M'
2494+ self.run_bm(config.render())
2495+
2496+ with loop_dev(img) as dev:
2497+ self.assertEqual(
2498+ summarize_partitions(dev), [
2499+ PartData(number=1, offset=1 << 20, size=1 << 10),
2500+ PartData(number=5, offset=2 << 20, size=98 << 20),
2501+ ])
2502+ self.assertEqual(99 << 20, _get_extended_partition_size(dev, 1))
2503+
2504+ def test_split(self):
2505+ img = self.tmp_path('image.img')
2506+ config = StorageConfigBuilder(version=2)
2507+ config.add_image(path=img, size='200M', ptable='gpt')
2508+ config.add_part(size=9 << 20, offset=1 << 20, number=1)
2509+ p2 = config.add_part(size='180M', offset=10 << 20, number=2,
2510+ fstype='ext4')
2511+ self.run_bm(config.render())
2512+ with loop_dev(img) as dev:
2513+ self.create_data(dev, p2)
2514+ self.assertEqual(
2515+ summarize_partitions(dev), [
2516+ PartData(number=1, offset=1 << 20, size=9 << 20),
2517+ PartData(number=2, offset=10 << 20, size=180 << 20),
2518+ ])
2519+ self.assertEqual(180 << 20, _get_filesystem_size(dev, p2))
2520+
2521+ config.set_preserve()
2522+ p2['resize'] = True
2523+ p2['size'] = '80M'
2524+ p3 = config.add_part(size='100M', offset=90 << 20, number=3,
2525+ fstype='ext4')
2526+ self.run_bm(config.render())
2527+ with loop_dev(img) as dev:
2528+ self.check_data(dev, p2)
2529+ self.assertEqual(
2530+ summarize_partitions(dev), [
2531+ PartData(number=1, offset=1 << 20, size=9 << 20),
2532+ PartData(number=2, offset=10 << 20, size=80 << 20),
2533+ PartData(number=3, offset=90 << 20, size=100 << 20),
2534+ ])
2535+ self.assertEqual(80 << 20, _get_filesystem_size(dev, p2))
2536+ self.assertEqual(100 << 20, _get_filesystem_size(dev, p3))
2537+
2538+ def test_partition_unify(self):
2539+ img = self.tmp_path('image.img')
2540+ config = StorageConfigBuilder(version=2)
2541+ config.add_image(path=img, size='200M', ptable='gpt')
2542+ config.add_part(size=9 << 20, offset=1 << 20, number=1)
2543+ p2 = config.add_part(size='40M', offset=10 << 20, number=2,
2544+ fstype='ext4')
2545+ p3 = config.add_part(size='60M', offset=50 << 20, number=3,
2546+ fstype='ext4')
2547+ self.run_bm(config.render())
2548+ with loop_dev(img) as dev:
2549+ self.create_data(dev, p2)
2550+ self.assertEqual(
2551+ summarize_partitions(dev), [
2552+ PartData(number=1, offset=1 << 20, size=9 << 20),
2553+ PartData(number=2, offset=10 << 20, size=40 << 20),
2554+ PartData(number=3, offset=50 << 20, size=60 << 20),
2555+ ])
2556+ self.assertEqual(40 << 20, _get_filesystem_size(dev, p2))
2557+ self.assertEqual(60 << 20, _get_filesystem_size(dev, p3))
2558+
2559+ config = StorageConfigBuilder(version=2)
2560+ config.add_image(path=img, size='200M', ptable='gpt')
2561+ config.add_part(size=9 << 20, offset=1 << 20, number=1)
2562+ p2 = config.add_part(size='100M', offset=10 << 20, number=2,
2563+ fstype='ext4', resize=True)
2564+ config.set_preserve()
2565+ self.run_bm(config.render())
2566+ with loop_dev(img) as dev:
2567+ self.check_data(dev, p2)
2568+ self.assertEqual(
2569+ summarize_partitions(dev), [
2570+ PartData(number=1, offset=1 << 20, size=9 << 20),
2571+ PartData(number=2, offset=10 << 20, size=100 << 20),
2572+ ])
2573+ self.assertEqual(100 << 20, _get_filesystem_size(dev, p2))
2574+
2575+ def test_mix_of_operations_gpt(self):
2576+ # a test that keeps, creates, resizes, and deletes a partition
2577+ # 200 MiB disk, using full disk
2578+ # init size preserve final size
2579+ # p1 - 9 MiB yes 9MiB
2580+ # p2 - 90 MiB yes, resize 139MiB
2581+ # p3 - 99 MiB no 50MiB
2582+ img = self.tmp_path('image.img')
2583+ config = StorageConfigBuilder(version=2)
2584+ config.add_image(path=img, size='200M', ptable='gpt')
2585+ config.add_part(size=9 << 20, offset=1 << 20, number=1)
2586+ p2 = config.add_part(size='90M', offset=10 << 20, number=2,
2587+ fstype='ext4')
2588+ p3 = config.add_part(size='99M', offset=100 << 20, number=3,
2589+ fstype='ext4')
2590+ self.run_bm(config.render())
2591+ with loop_dev(img) as dev:
2592+ self.create_data(dev, p2)
2593+ self.assertEqual(
2594+ summarize_partitions(dev), [
2595+ PartData(number=1, offset=1 << 20, size=9 << 20),
2596+ PartData(number=2, offset=10 << 20, size=90 << 20),
2597+ PartData(number=3, offset=100 << 20, size=99 << 20),
2598+ ])
2599+ self.assertEqual(90 << 20, _get_filesystem_size(dev, p2))
2600+ self.assertEqual(99 << 20, _get_filesystem_size(dev, p3))
2601+
2602+ config = StorageConfigBuilder(version=2)
2603+ config.add_image(path=img, size='200M', ptable='gpt')
2604+ config.add_part(size=9 << 20, offset=1 << 20, number=1)
2605+ p2 = config.add_part(size='139M', offset=10 << 20, number=2,
2606+ fstype='ext4', resize=True)
2607+ config.set_preserve()
2608+ p3 = config.add_part(size='50M', offset=149 << 20, number=3,
2609+ fstype='ext4')
2610+ self.run_bm(config.render())
2611+ with loop_dev(img) as dev:
2612+ self.check_data(dev, p2)
2613+ self.assertEqual(
2614+ summarize_partitions(dev), [
2615+ PartData(number=1, offset=1 << 20, size=9 << 20),
2616+ PartData(number=2, offset=10 << 20, size=139 << 20),
2617+ PartData(number=3, offset=149 << 20, size=50 << 20),
2618+ ])
2619+ self.assertEqual(139 << 20, _get_filesystem_size(dev, p2))
2620+ self.assertEqual(50 << 20, _get_filesystem_size(dev, p3))
2621+
2622+ def test_mix_of_operations_msdos(self):
2623+ # a test that keeps, creates, resizes, and deletes a partition
2624+ # including handling of extended/logical
2625+ # 200 MiB disk, initially only using front 100MiB
2626+ # flag init size preserve final size
2627+ # p1 - primary 9MiB yes 9MiB
2628+ # p2 - extended 89MiB yes, resize 189MiB
2629+ # p3 - logical 37MiB yes, resize 137MiB
2630+ # p4 - logical 50MiB no 50MiB
2631+ img = self.tmp_path('image.img')
2632+ config = StorageConfigBuilder(version=2)
2633+ config.add_image(path=img, size='200M', ptable='msdos')
2634+ p1 = config.add_part(size='9M', offset=1 << 20, number=1,
2635+ fstype='ext4')
2636+ config.add_part(size='89M', offset=10 << 20, number=2, flag='extended')
2637+ p5 = config.add_part(size='36M', offset=11 << 20, number=5,
2638+ flag='logical', fstype='ext4')
2639+ p6 = config.add_part(size='50M', offset=49 << 20, number=6,
2640+ flag='logical', fstype='ext4')
2641+ self.run_bm(config.render())
2642+
2643+ with loop_dev(img) as dev:
2644+ self.create_data(dev, p1)
2645+ self.create_data(dev, p5)
2646+ self.assertEqual(
2647+ summarize_partitions(dev), [
2648+ PartData(number=1, offset=1 << 20, size=9 << 20),
2649+ PartData(number=2, offset=10 << 20, size=1 << 10),
2650+ PartData(number=5, offset=11 << 20, size=36 << 20),
2651+ PartData(number=6, offset=49 << 20, size=50 << 20),
2652+ ])
2653+ self.assertEqual(89 << 20, _get_extended_partition_size(dev, 2))
2654+ self.assertEqual(9 << 20, _get_filesystem_size(dev, p1))
2655+ self.assertEqual(36 << 20, _get_filesystem_size(dev, p5))
2656+ self.assertEqual(50 << 20, _get_filesystem_size(dev, p6))
2657+
2658+ config = StorageConfigBuilder(version=2)
2659+ config.add_image(path=img, size='200M', ptable='msdos')
2660+ p1 = config.add_part(size='9M', offset=1 << 20, number=1,
2661+ fstype='ext4')
2662+ config.add_part(size='189M', offset=10 << 20, number=2,
2663+ flag='extended', resize=True)
2664+ p5 = config.add_part(size='136M', offset=11 << 20, number=5,
2665+ flag='logical', fstype='ext4', resize=True)
2666+ config.set_preserve()
2667+ p6 = config.add_part(size='50M', offset=149 << 20, number=6,
2668+ flag='logical', fstype='ext4')
2669+ self.run_bm(config.render())
2670+
2671+ with loop_dev(img) as dev:
2672+ self.check_data(dev, p1)
2673+ self.check_data(dev, p5)
2674+ self.assertEqual(
2675+ summarize_partitions(dev), [
2676+ PartData(number=1, offset=1 << 20, size=9 << 20),
2677+ PartData(number=2, offset=10 << 20, size=1 << 10),
2678+ PartData(number=5, offset=11 << 20, size=136 << 20),
2679+ PartData(number=6, offset=149 << 20, size=50 << 20),
2680+ ])
2681+ self.assertEqual(189 << 20, _get_extended_partition_size(dev, 2))
2682+ self.assertEqual(9 << 20, _get_filesystem_size(dev, p1))
2683+ self.assertEqual(136 << 20, _get_filesystem_size(dev, p5))
2684+ self.assertEqual(50 << 20, _get_filesystem_size(dev, p6))
2685+
2686+ def test_split_and_wiping(self):
2687+ # regression test for a bug where a partition wipe would happen before
2688+ # a resize was performed, resulting in data loss.
2689+ img = self.tmp_path('image.img')
2690+ config = StorageConfigBuilder(version=2)
2691+ config.add_image(path=img, size='100M', ptable='gpt')
2692+ p1 = config.add_part(size=98 << 20, offset=1 << 20, number=1,
2693+ fstype='ext4')
2694+ self.run_bm(config.render())
2695+ with loop_dev(img) as dev:
2696+ self.assertEqual(
2697+ summarize_partitions(dev), [
2698+ PartData(number=1, offset=1 << 20, size=98 << 20),
2699+ ])
2700+ with self.mount(dev, p1) as mnt_point:
2701+ # Attempt to create files across the partition with gaps
2702+ for i in range(1, 41):
2703+ with open(f'{mnt_point}/{str(i)}', 'wb') as fp:
2704+ fp.write(bytes([i]) * (2 << 20))
2705+ for i in range(1, 41):
2706+ if i % 5 != 0:
2707+ os.remove(f'{mnt_point}/{str(i)}')
2708+
2709+ config = StorageConfigBuilder(version=2)
2710+ config.add_image(path=img, size='100M', ptable='gpt')
2711+ p1 = config.add_part(size=49 << 20, offset=1 << 20, number=1,
2712+ fstype='ext4', resize=True)
2713+ config.set_preserve()
2714+ config.add_part(size=49 << 20, offset=50 << 20, number=2,
2715+ fstype='ext4')
2716+ self.run_bm(config.render())
2717+ with loop_dev(img) as dev:
2718+ self.assertEqual(
2719+ summarize_partitions(dev), [
2720+ PartData(number=1, offset=1 << 20, size=49 << 20),
2721+ PartData(number=2, offset=50 << 20, size=49 << 20),
2722+ ])
2723+ with self.mount(dev, p1) as mnt_point:
2724+ for i in range(5, 41, 5):
2725+ with open(f'{mnt_point}/{i}', 'rb') as fp:
2726+ self.assertEqual(bytes([i]) * (2 << 20), fp.read())
2727+
2728+ def test_parttype_dos(self):
2729+ # msdos partition table partitions shall retain their type
2730+ # create initial situation similar to this
2731+ # Device Boot Start End Sectors Size Id Type
2732+ # /dev/sda1 * 2048 104447 102400 50M 7 HPFS/NTFS/exFA
2733+ # /dev/sda2 104448 208668781 208564334 99.5G 7 HPFS/NTFS/exFA
2734+ # /dev/sda3 208670720 209711103 1040384 508M 27 Hidden NTFS Wi
2735+ self.img = self.tmp_path('image.img')
2736+ config = StorageConfigBuilder(version=2)
2737+ config.add_image(path=self.img, size='200M', ptable='msdos')
2738+ config.add_part(size=50 << 20, offset=1 << 20, number=1,
2739+ fstype='ntfs', flag='boot', partition_type='0x7')
2740+ config.add_part(size=100 << 20, offset=51 << 20, number=2,
2741+ fstype='ntfs', partition_type='0x7')
2742+ config.add_part(size=48 << 20, offset=151 << 20, number=3,
2743+ fstype='ntfs', partition_type='0x27')
2744+ self.run_bm(config.render())
2745+ self.assertPartitions(
2746+ PartData(number=1, offset=1 << 20, size=50 << 20,
2747+ partition_type='7', boot=True),
2748+ PartData(number=2, offset=51 << 20, size=100 << 20,
2749+ partition_type='7', boot=False),
2750+ PartData(number=3, offset=151 << 20, size=48 << 20,
2751+ partition_type='27', boot=False))
2752+
2753+ def test_parttype_gpt(self):
2754+ # gpt partition table partitions shall retain their type
2755+ # create initial situation similar to this
2756+ # # Start (sector) End (sector) Size Code Name
2757+ # 1 2048 206847 100.0 MiB EF00 EFI system part
2758+ # 2 206848 239615 16.0 MiB 0C01 Microsoft reser
2759+ # 3 239616 103811181 49.4 GiB 0700 Basic data part
2760+ # 4 103813120 104853503 508.0 MiB 2700
2761+ esp = 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B'
2762+ msreserved = 'E3C9E316-0B5C-4DB8-817D-F92DF00215AE'
2763+ msdata = 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7'
2764+ winre = 'DE94BBA4-06D1-4D40-A16A-BFD50179D6AC'
2765+ self.img = self.tmp_path('image.img')
2766+ config = StorageConfigBuilder(version=2)
2767+ config.add_image(path=self.img, size='100M', ptable='gpt')
2768+ config.add_part(number=1, offset=1 << 20, size=9 << 20,
2769+ flag='boot', fstype='ntfs')
2770+ config.add_part(number=2, offset=10 << 20, size=20 << 20,
2771+ partition_type=msreserved)
2772+ config.add_part(number=3, offset=30 << 20, size=50 << 20,
2773+ partition_type=msdata, fstype='ntfs')
2774+ config.add_part(number=4, offset=80 << 20, size=19 << 20,
2775+ partition_type=winre, fstype='ntfs')
2776+ self.run_bm(config.render())
2777+ self.assertPartitions(
2778+ PartData(number=1, offset=1 << 20, size=9 << 20,
2779+ partition_type=esp),
2780+ PartData(number=2, offset=10 << 20, size=20 << 20,
2781+ partition_type=msreserved),
2782+ PartData(number=3, offset=30 << 20, size=50 << 20,
2783+ partition_type=msdata),
2784+ PartData(number=4, offset=80 << 20, size=19 << 20,
2785+ partition_type=winre))
2786+
2787+ @parameterized.expand([('msdos',), ('gpt',)])
2788+ def test_disk_label_id_persistent(self, ptable):
2789+ # when the disk is preserved, the disk label id shall also be preserved
2790+ self.img = self.tmp_path('image.img')
2791+ config = StorageConfigBuilder(version=2)
2792+ config.add_image(path=self.img, size='20M', ptable=ptable)
2793+ config.add_part(number=1, offset=1 << 20, size=18 << 20)
2794+ self.run_bm(config.render())
2795+ self.assertPartitions(
2796+ PartData(number=1, offset=1 << 20, size=18 << 20))
2797+ with loop_dev(self.img) as dev:
2798+ orig_label_id = _get_disk_label_id(dev)
2799+
2800+ config.set_preserve()
2801+ self.run_bm(config.render())
2802+ self.assertPartitions(
2803+ PartData(number=1, offset=1 << 20, size=18 << 20))
2804+ with loop_dev(self.img) as dev:
2805+ self.assertEqual(orig_label_id, _get_disk_label_id(dev))
2806diff --git a/tests/unittests/test_apt_source.py b/tests/unittests/test_apt_source.py
2807index 48fb820..267711f 100644
2808--- a/tests/unittests/test_apt_source.py
2809+++ b/tests/unittests/test_apt_source.py
2810@@ -572,6 +572,55 @@ class TestAptSourceConfig(CiTestCase):
2811 'Acquire::ftp::Proxy "foobar3";\n'
2812 'Acquire::https::Proxy "foobar4";\n'))
2813
2814+ def test_preference_to_str(self):
2815+ """ test_preference_to_str - Test converting a preference dict to
2816+ textual representation.
2817+ """
2818+ preference = {
2819+ "package": "*",
2820+ "pin": "release a=unstable",
2821+ "pin-priority": 50,
2822+ }
2823+
2824+ expected = """\
2825+Package: *
2826+Pin: release a=unstable
2827+Pin-Priority: 50
2828+"""
2829+ self.assertEqual(expected, apt_config.preference_to_str(preference))
2830+
2831+ @staticmethod
2832+ def test_apply_apt_preferences():
2833+ """ test_apply_apt_preferences - Test apt preferences configuration
2834+ """
2835+ cfg = {
2836+ "preferences": [
2837+ {
2838+ "package": "*",
2839+ "pin": "release a=unstable",
2840+ "pin-priority": 50,
2841+ }, {
2842+ "package": "dummy-unwanted-package",
2843+ "pin": "origin *ubuntu.com*",
2844+ "pin-priority": -1,
2845+ }
2846+ ]
2847+ }
2848+
2849+ expected_content = """\
2850+Package: *
2851+Pin: release a=unstable
2852+Pin-Priority: 50
2853+
2854+Package: dummy-unwanted-package
2855+Pin: origin *ubuntu.com*
2856+Pin-Priority: -1
2857+"""
2858+ with mock.patch.object(util, "write_file") as mockobj:
2859+ apt_config.apply_apt_preferences(cfg, "preferencesfn")
2860+
2861+ mockobj.assert_called_with("preferencesfn", expected_content)
2862+
2863 def test_mirror(self):
2864 """test_mirror - Test defining a mirror"""
2865 pmir = "http://us.archive.ubuntu.com/ubuntu/"
2866diff --git a/tests/unittests/test_block.py b/tests/unittests/test_block.py
2867index 6d9b776..7a73b69 100644
2868--- a/tests/unittests/test_block.py
2869+++ b/tests/unittests/test_block.py
2870@@ -927,4 +927,12 @@ class TestSfdiskInfo(CiTestCase):
2871 self.assertEqual([], self.m_load_json.call_args_list)
2872
2873
2874+class TestResize(CiTestCase):
2875+ def test_basic(self):
2876+ resizers = 'curtin.commands.block_meta_v2.resizers'
2877+ values = {'a': 1, 'b': 2}
2878+ with mock.patch.dict(resizers, values, clear=True):
2879+ self.assertEqual({'a', 'b'}, block.get_resize_fstypes())
2880+
2881+
2882 # vi: ts=4 expandtab syntax=python
2883diff --git a/tests/unittests/test_commands_block_meta.py b/tests/unittests/test_commands_block_meta.py
2884index 3e22792..9185d4e 100644
2885--- a/tests/unittests/test_commands_block_meta.py
2886+++ b/tests/unittests/test_commands_block_meta.py
2887@@ -3,16 +3,25 @@
2888 from argparse import Namespace
2889 from collections import OrderedDict
2890 import copy
2891-from mock import patch, call
2892+from mock import (
2893+ call,
2894+ Mock,
2895+ patch,
2896+)
2897 import os
2898 import random
2899+import uuid
2900
2901 from curtin.block import dasd
2902-from curtin.commands import block_meta
2903+from curtin.commands import block_meta, block_meta_v2
2904 from curtin import paths, util
2905 from .helpers import CiTestCase
2906
2907
2908+def random_uuid():
2909+ return uuid.uuid4()
2910+
2911+
2912 class TestGetPathToStorageVolume(CiTestCase):
2913
2914 def setUp(self):
2915@@ -2572,6 +2581,8 @@ class TestPartitionVerifySfdisk(CiTestCase):
2916 base = 'curtin.commands.block_meta.'
2917 self.add_patch(base + 'verify_size', 'm_verify_size')
2918 self.add_patch(base + 'verify_ptable_flag', 'm_verify_ptable_flag')
2919+ self.add_patch(base + 'os.path.realpath', 'm_realpath')
2920+ self.m_realpath.side_effect = lambda x: x
2921 self.info = {
2922 'id': 'disk-sda-part-2',
2923 'type': 'partition',
2924@@ -2611,6 +2622,257 @@ class TestPartitionVerifySfdisk(CiTestCase):
2925 self.assertEqual([], self.m_verify_ptable_flag.call_args_list)
2926
2927
2928+class TestPartitionVerifySfdiskV2(CiTestCase):
2929+
2930+ def setUp(self):
2931+ super(TestPartitionVerifySfdiskV2, self).setUp()
2932+ base = 'curtin.commands.block_meta_v2.'
2933+ self.add_patch(base + 'verify_size', 'm_verify_size')
2934+ self.add_patch(base + 'verify_ptable_flag', 'm_verify_ptable_flag')
2935+ self.add_patch(base + 'os.path.realpath', 'm_realpath')
2936+ self.m_realpath.side_effect = lambda x: x
2937+ self.info = {
2938+ 'id': 'disk-sda-part-2',
2939+ 'type': 'partition',
2940+ 'offset': '1GB',
2941+ 'device': 'sda',
2942+ 'number': 2,
2943+ 'size': '5GB',
2944+ 'flag': 'boot',
2945+ }
2946+ self.part_size = int(util.human2bytes(self.info['size']))
2947+ self.devpath = self.random_string()
2948+ self.sfdisk_part_info = {
2949+ 'node': self.devpath,
2950+ 'start': (1 << 30) // 512,
2951+ }
2952+ self.storage_config = {self.info['id']: self.info}
2953+ self.label = self.random_string()
2954+ self.table = Mock()
2955+ self.table.sectors2bytes = lambda x: x * 512
2956+
2957+ def test_partition_verify_sfdisk(self):
2958+ block_meta_v2.partition_verify_sfdisk_v2(self.info, self.label,
2959+ self.sfdisk_part_info,
2960+ self.storage_config,
2961+ self.table)
2962+ self.assertEqual(
2963+ [call(self.devpath, self.part_size, self.sfdisk_part_info)],
2964+ self.m_verify_size.call_args_list)
2965+ self.assertEqual(
2966+ [call(self.devpath, self.info['flag'], self.label,
2967+ self.sfdisk_part_info)],
2968+ self.m_verify_ptable_flag.call_args_list)
2969+
2970+ def test_partition_verify_no_moves(self):
2971+ self.info['preserve'] = True
2972+ self.info['resize'] = True
2973+ self.info['offset'] = '2GB'
2974+ with self.assertRaises(RuntimeError):
2975+ block_meta_v2.partition_verify_sfdisk_v2(
2976+ self.info, self.label, self.sfdisk_part_info,
2977+ self.storage_config, self.table)
2978+
2979+
2980+class TestSfdiskV2(CiTestCase):
2981+ def test_gpt_basic(self):
2982+ table = block_meta_v2.GPTPartTable(512)
2983+ expected = '''\
2984+label: gpt
2985+'''
2986+ self.assertEqual(expected, table.render())
2987+
2988+ def test_gpt_boot(self):
2989+ table = block_meta_v2.GPTPartTable(512)
2990+ table.add(dict(number=1, offset=1 << 20, size=9 << 20, flag='boot'))
2991+ expected = '''\
2992+label: gpt
2993+
2994+1: start=2048 size=18432 type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B'''
2995+ self.assertEqual(expected, table.render())
2996+
2997+ def test_gpt_boot_raw_type(self):
2998+ esp = 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B'
2999+ table = block_meta_v2.GPTPartTable(512)
3000+ table.add(dict(number=1, offset=1 << 20, size=9 << 20,
3001+ partition_type=esp))
3002+ expected = '''\
3003+label: gpt
3004+
3005+1: start=2048 size=18432 type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B'''
3006+ self.assertEqual(expected, table.render())
3007+
3008+ def test_gpt_random_uuid(self):
3009+ ptype = str(random_uuid()).lower()
3010+ table = block_meta_v2.GPTPartTable(512)
3011+ table.add(dict(number=1, offset=1 << 20, size=9 << 20,
3012+ flag='boot', partition_type=ptype))
3013+ expected = f'''\
3014+label: gpt
3015+
3016+1: start=2048 size=18432 type={ptype}'''
3017+ self.assertEqual(expected, table.render())
3018+
3019+ def test_dos_basic(self):
3020+ table = block_meta_v2.DOSPartTable(512)
3021+ expected = '''\
3022+label: dos
3023+'''
3024+ self.assertEqual(expected, table.render())
3025+
3026+ def test_dos_boot(self):
3027+ table = block_meta_v2.DOSPartTable(512)
3028+ table.add(dict(number=1, offset=1 << 20, size=9 << 20, flag='boot'))
3029+ expected = '''\
3030+label: dos
3031+
3032+1: start=2048 size=18432 type=EF bootable'''
3033+ self.assertEqual(expected, table.render())
3034+
3035+ def test_dos_random_code(self):
3036+ ptype = hex(random.randint(0, 0xff))[2:]
3037+ table = block_meta_v2.DOSPartTable(512)
3038+ table.add(dict(number=1, offset=1 << 20, size=9 << 20,
3039+ flag='boot', partition_type=ptype))
3040+ expected = f'''\
3041+label: dos
3042+
3043+1: start=2048 size=18432 type={ptype} bootable'''
3044+ self.assertEqual(expected, table.render())
3045+
3046+
3047+class TestPartitionNeedsResize(CiTestCase):
3048+
3049+ def setUp(self):
3050+ super(TestPartitionNeedsResize, self).setUp()
3051+ base = 'curtin.commands.block_meta_v2.'
3052+ self.add_patch(base + 'os.path.realpath', 'm_realpath')
3053+ self.add_patch(base + '_get_volume_fstype', 'm_get_volume_fstype')
3054+ self.m_realpath.side_effect = lambda x: x
3055+ self.partition = {
3056+ 'id': 'disk-sda-part-2',
3057+ 'type': 'partition',
3058+ 'offset': '1GB',
3059+ 'device': 'sda',
3060+ 'number': 2,
3061+ 'size': '5GB',
3062+ 'flag': 'boot',
3063+ }
3064+ self.devpath = self.random_string()
3065+ self.sfdisk_part_info = {
3066+ 'node': self.devpath,
3067+ 'start': (1 << 30) // 512,
3068+ 'size': (1 << 30) // 512,
3069+ }
3070+ self.format = {
3071+ 'id': 'id-format',
3072+ 'type': 'format',
3073+ 'fstype': 'ext4',
3074+ 'volume': self.partition['id'],
3075+ }
3076+ self.storage_config = {
3077+ self.partition['id']: self.partition,
3078+ self.format['id']: self.format,
3079+ }
3080+ self.table = Mock()
3081+ self.table.sectors2bytes = lambda x: x * 512
3082+
3083+ def test_partition_resize_happy_path(self):
3084+ self.partition['preserve'] = True
3085+ self.partition['resize'] = True
3086+ self.format['preserve'] = True
3087+ self.format['fstype'] = 'ext4'
3088+ self.m_get_volume_fstype.return_value = 'ext4'
3089+ expected = {
3090+ 'fstype': 'ext4',
3091+ 'size': 5 << 30,
3092+ 'direction': 'up',
3093+ }
3094+ actual = block_meta_v2._prepare_resize(
3095+ self.storage_config, self.partition, self.table,
3096+ self.sfdisk_part_info)
3097+ self.assertEqual(expected, actual)
3098+
3099+ def test_partition_resize_no_format_action(self):
3100+ self.partition['preserve'] = True
3101+ self.partition['resize'] = True
3102+ self.storage_config = {self.partition['id']: self.partition}
3103+ self.m_get_volume_fstype.return_value = 'ext4'
3104+ expected = {
3105+ 'fstype': 'ext4',
3106+ 'size': 5 << 30,
3107+ 'direction': 'up',
3108+ }
3109+ actual = block_meta_v2._prepare_resize(
3110+ self.storage_config, self.partition, self.table,
3111+ self.sfdisk_part_info)
3112+ self.assertEqual(expected, actual)
3113+
3114+ def test_partition_resize_change_fs(self):
3115+ self.partition['preserve'] = True
3116+ self.partition['resize'] = True
3117+ self.format['preserve'] = True
3118+ self.format['fstype'] = 'ext3'
3119+ self.m_get_volume_fstype.return_value = 'ext4'
3120+ with self.assertRaises(RuntimeError):
3121+ block_meta_v2._prepare_resize(self.storage_config, self.partition,
3122+ self.table, self.sfdisk_part_info)
3123+
3124+ def test_partition_resize_unsupported_fs(self):
3125+ self.partition['preserve'] = True
3126+ self.partition['resize'] = True
3127+ self.format['preserve'] = True
3128+ self.format['fstype'] = 'reiserfs'
3129+ self.m_get_volume_fstype.return_value = 'resierfs'
3130+ with self.assertRaises(RuntimeError):
3131+ block_meta_v2._prepare_resize(self.storage_config, self.partition,
3132+ self.table, self.sfdisk_part_info)
3133+
3134+ def test_partition_resize_format_preserve_false(self):
3135+ # though the filesystem type is not supported for resize, it's ok
3136+ # because with format preserve=False, we're recreating anyhow
3137+ self.partition['preserve'] = True
3138+ self.partition['resize'] = True
3139+ self.format['preserve'] = False
3140+ self.format['fstype'] = 'reiserfs'
3141+ self.m_get_volume_fstype.return_value = 'reiserfs'
3142+ self.assertIsNone(
3143+ block_meta_v2._prepare_resize(self.storage_config, self.partition,
3144+ self.table, self.sfdisk_part_info))
3145+
3146+ def test_partition_resize_partition_preserve_false(self):
3147+ # not a resize - partition is recreated
3148+ self.partition['preserve'] = False
3149+ self.partition['resize'] = True
3150+ self.format['preserve'] = False
3151+ self.format['fstype'] = 'reiserfs'
3152+ self.m_get_volume_fstype.return_value = 'reiserfs'
3153+ self.assertIsNone(
3154+ block_meta_v2._prepare_resize(self.storage_config, self.partition,
3155+ self.table, self.sfdisk_part_info))
3156+
3157+ def test_partition_resize_equal_size(self):
3158+ # not a resize - the size is the same so leave it alone
3159+ self.partition['preserve'] = True
3160+ self.partition['resize'] = True
3161+ self.partition['size'] = '1GB'
3162+ self.format['preserve'] = True
3163+ self.m_get_volume_fstype.return_value = 'ext4'
3164+ self.assertIsNone(
3165+ block_meta_v2._prepare_resize(self.storage_config, self.partition,
3166+ self.table, self.sfdisk_part_info))
3167+
3168+ def test_partition_resize_unformatted(self):
3169+ # not a resize - an unformatted partition has nothing to preserve
3170+ self.partition['preserve'] = True
3171+ self.partition['resize'] = True
3172+ self.storage_config = {self.partition['id']: self.partition}
3173+ self.m_get_volume_fstype.return_value = ''
3174+ self.assertIsNone(
3175+ block_meta_v2._prepare_resize(self.storage_config, self.partition,
3176+ self.table, self.sfdisk_part_info))
3177+
3178+
3179 class TestPartitionVerifyFdasd(CiTestCase):
3180
3181 def setUp(self):
3182diff --git a/tests/unittests/test_storage_config.py b/tests/unittests/test_storage_config.py
3183index a2308c4..df48a4d 100644
3184--- a/tests/unittests/test_storage_config.py
3185+++ b/tests/unittests/test_storage_config.py
3186@@ -7,7 +7,7 @@ from curtin.storage_config import ProbertParser as baseparser
3187 from curtin.storage_config import (BcacheParser, BlockdevParser, DasdParser,
3188 DmcryptParser, FilesystemParser, LvmParser,
3189 RaidParser, MountParser, ZfsParser)
3190-from curtin.storage_config import ptable_uuid_to_flag_entry
3191+from curtin.storage_config import ptable_uuid_to_flag_entry, select_configs
3192 from curtin import util
3193
3194
3195@@ -285,7 +285,7 @@ class TestBlockdevParser(CiTestCase):
3196 """ BlockdevParser skips invalid ID_WWN_* values. """
3197 self.bdevp.blockdev_data['/dev/sda'] = {
3198 'DEVTYPE': 'disk',
3199- 'DEVNAME': 'sda',
3200+ 'DEVNAME': '/dev/sda',
3201 'ID_SERIAL': 'Corsair_Force_GS_1785234921906',
3202 'ID_SERIAL_SHORT': '1785234921906',
3203 'ID_WWN': '0x0000000000000000',
3204@@ -300,7 +300,7 @@ class TestBlockdevParser(CiTestCase):
3205 """ BlockdevParser skips invalid ID_SERIAL_* values. """
3206 self.bdevp.blockdev_data['/dev/sda'] = {
3207 'DEVTYPE': 'disk',
3208- 'DEVNAME': 'sda',
3209+ 'DEVNAME': '/dev/sda',
3210 'ID_SERIAL': ' ',
3211 'ID_SERIAL_SHORT': 'My Serial is My PassPort',
3212 }
3213@@ -345,10 +345,12 @@ class TestBlockdevParser(CiTestCase):
3214 'id': 'partition-sda1',
3215 'type': 'partition',
3216 'device': 'disk-sda',
3217+ 'path': '/dev/sda1',
3218 'number': 1,
3219 'offset': 1048576,
3220 'size': 499122176,
3221 'flag': 'linux',
3222+ 'partition_type': '0fc63daf-8483-4772-8e79-3d69d8477de4',
3223 }
3224 self.assertDictEqual(expected_dict,
3225 self.bdevp.asdict(blockdev))
3226@@ -375,12 +377,12 @@ class TestBlockdevParser(CiTestCase):
3227 """ BlockdevParser ignores partition with zero start value."""
3228 self.bdevp.blockdev_data['/dev/vda'] = {
3229 'DEVTYPE': 'disk',
3230- 'DEVNAME': 'vda',
3231+ 'DEVNAME': '/dev/vda',
3232 }
3233 test_value = {
3234 'DEVTYPE': 'partition',
3235 'MAJOR': "252",
3236- 'DEVNAME': 'vda1',
3237+ 'DEVNAME': '/dev/vda1',
3238 "DEVPATH":
3239 "/devices/pci0000:00/0000:00:04.0/virtio0/block/vda/vda1",
3240 "ID_PART_ENTRY_TYPE": "0x0",
3241@@ -390,8 +392,10 @@ class TestBlockdevParser(CiTestCase):
3242 'id': 'partition-vda1',
3243 'type': 'partition',
3244 'device': 'disk-vda',
3245+ 'path': '/dev/vda1',
3246 'number': 1,
3247 'size': 784334848,
3248+ 'partition_type': '0x0',
3249 }
3250 self.assertDictEqual(expected_dict, self.bdevp.asdict(test_value))
3251
3252@@ -403,7 +407,8 @@ class TestBlockdevParser(CiTestCase):
3253
3254 # XXX: Parameterize me
3255 def test_blockdev_to_id_raises_valueerror_on_empty_devtype(self):
3256- test_value = {'DEVTYPE': '', 'DEVNAME': 'bar', 'DEVPATH': 'foobar'}
3257+ test_value = {'DEVTYPE': '', 'DEVNAME': '/dev/bar',
3258+ 'DEVPATH': 'foobar'}
3259 with self.assertRaises(ValueError):
3260 self.bdevp.blockdev_to_id(test_value)
3261
3262@@ -415,7 +420,7 @@ class TestBlockdevParser(CiTestCase):
3263
3264 # XXX: Parameterize me
3265 def test_blockdev_to_id_raises_valueerror_on_missing_devtype(self):
3266- test_value = {'DEVNAME': 'bar', 'DEVPATH': 'foobar'}
3267+ test_value = {'DEVNAME': '/dev/bar', 'DEVPATH': 'foobar'}
3268 with self.assertRaises(ValueError):
3269 self.bdevp.blockdev_to_id(test_value)
3270
3271@@ -424,9 +429,10 @@ class TestBlockdevParser(CiTestCase):
3272 self.probe_data = _get_data('probert_storage_lvm.json')
3273 self.bdevp = BlockdevParser(self.probe_data)
3274 blockdev = self.bdevp.blockdev_data['/dev/vda2']
3275- expected_dict = {
3276+ base_expected_dict = {
3277 'id': 'partition-vda2',
3278 'type': 'partition',
3279+ 'path': '/dev/vda2',
3280 'device': 'disk-vda',
3281 'number': 2,
3282 'offset': 3222274048,
3283@@ -435,6 +441,8 @@ class TestBlockdevParser(CiTestCase):
3284 }
3285 for ext_part_entry in ['0xf', '0x5', '0x85', '0xc5']:
3286 blockdev['ID_PART_ENTRY_TYPE'] = ext_part_entry
3287+ expected_dict = base_expected_dict.copy()
3288+ expected_dict['partition_type'] = ext_part_entry
3289 self.assertDictEqual(expected_dict,
3290 self.bdevp.asdict(blockdev))
3291
3292@@ -446,10 +454,12 @@ class TestBlockdevParser(CiTestCase):
3293 'id': 'partition-vda5',
3294 'type': 'partition',
3295 'device': 'disk-vda',
3296+ 'path': '/dev/vda5',
3297 'number': 5,
3298 'offset': 3223322624,
3299 'size': 2147483648,
3300 'flag': 'logical',
3301+ 'partition_type': '0x83',
3302 }
3303 self.assertDictEqual(expected_dict,
3304 self.bdevp.asdict(blockdev))
3305@@ -463,10 +473,12 @@ class TestBlockdevParser(CiTestCase):
3306 'id': 'partition-vdb1',
3307 'type': 'partition',
3308 'device': 'disk-vdb',
3309+ 'path': '/dev/vdb1',
3310 'number': 1,
3311 'offset': 1048576,
3312 'size': 536870912,
3313 'flag': 'boot',
3314+ 'partition_type': '0xb',
3315 }
3316 self.assertDictEqual(expected_dict,
3317 self.bdevp.asdict(blockdev))
3318@@ -480,10 +492,12 @@ class TestBlockdevParser(CiTestCase):
3319 'id': 'partition-vda5',
3320 'type': 'partition',
3321 'device': 'disk-vda',
3322+ 'path': '/dev/vda5',
3323 'number': 5,
3324 'offset': 3223322624,
3325 'size': 2147483648,
3326 'flag': 'boot',
3327+ 'partition_type': '0x83',
3328 }
3329 self.assertDictEqual(expected_dict,
3330 self.bdevp.asdict(blockdev))
3331@@ -536,6 +550,7 @@ class TestBlockdevParser(CiTestCase):
3332 blockdev = self.bdevp.blockdev_data['/dev/dm-2']
3333 expected_dict = {
3334 'device': 'mpath-disk-mpatha',
3335+ 'path': '/dev/dm-2',
3336 'flag': 'linux',
3337 'id': 'mpath-partition-mpatha-part2',
3338 'multipath': 'mpatha',
3339@@ -543,6 +558,7 @@ class TestBlockdevParser(CiTestCase):
3340 'offset': 2097152,
3341 'size': 10734272512,
3342 'type': 'partition',
3343+ 'partition_type': '0fc63daf-8483-4772-8e79-3d69d8477de4',
3344 }
3345 self.assertDictEqual(expected_dict, self.bdevp.asdict(blockdev))
3346
3347@@ -970,7 +986,7 @@ class TestExtractStorageConfig(CiTestCase):
3348 """ verify live-iso extracted storage-config finds target disk. """
3349 extracted = storage_config.extract_storage_config(self.probe_data)
3350 self.assertEqual(
3351- {'storage': {'version': 1,
3352+ {'storage': {'version': 2,
3353 'config': [{'id': 'disk-sda', 'path': '/dev/sda',
3354 'serial': 'QEMU_HARDDISK_QM00001',
3355 'type': 'disk'}]}}, extracted)
3356@@ -985,13 +1001,13 @@ class TestExtractStorageConfig(CiTestCase):
3357 if missing_key != 'blockdev':
3358 self.assertEqual(
3359 {'storage':
3360- {'version': 1,
3361+ {'version': 2,
3362 'config': [{'id': 'disk-sda', 'path': '/dev/sda',
3363 'serial': 'QEMU_HARDDISK_QM00001',
3364 'type': 'disk'}]}}, extracted)
3365 else:
3366 # empty config without blockdev data
3367- self.assertEqual({'storage': {'config': [], 'version': 1}},
3368+ self.assertEqual({'storage': {'config': [], 'version': 2}},
3369 extracted)
3370
3371 @skipUnlessJsonSchema()
3372@@ -1010,10 +1026,17 @@ class TestExtractStorageConfig(CiTestCase):
3373 'raidlevel': 'raid1', 'name': 'md1',
3374 'devices': ['partition-vdb1', 'partition-vdc1'],
3375 'spare_devices': []}, raids[0])
3376- self.assertEqual({'id': 'raid-md1p1', 'type': 'partition',
3377- 'size': 4285530112, 'flag': 'linux', 'number': 1,
3378- 'device': 'raid-md1', 'offset': 1048576},
3379- raid_partitions[0])
3380+ self.assertEqual({
3381+ 'id': 'raid-md1p1',
3382+ 'type': 'partition',
3383+ 'path': '/dev/md1p1',
3384+ 'size': 4285530112,
3385+ 'flag': 'linux',
3386+ 'number': 1,
3387+ 'partition_type': '0fc63daf-8483-4772-8e79-3d69d8477de4',
3388+ 'device': 'raid-md1',
3389+ 'offset': 1048576},
3390+ raid_partitions[0])
3391
3392 @skipUnlessJsonSchema()
3393 def test_find_extended_partition(self):
3394@@ -1108,4 +1131,26 @@ class TestExtractStorageConfig(CiTestCase):
3395 self.assertEqual(expected_dict, bitlocker[0])
3396
3397
3398+class TestSelectConfigs(CiTestCase):
3399+ def test_basic(self):
3400+ id0 = {'a': 1, 'b': 2}
3401+ id1 = {'a': 1, 'c': 3}
3402+ sc = {'id0': id0, 'id1': id1}
3403+
3404+ self.assertEqual([id0, id1], select_configs(sc, a=1))
3405+
3406+ def test_not_found(self):
3407+ id0 = {'a': 1, 'b': 2}
3408+ id1 = {'a': 1, 'c': 3}
3409+ sc = {'id0': id0, 'id1': id1}
3410+
3411+ self.assertEqual([], select_configs(sc, a=4))
3412+
3413+ def test_multi_criteria(self):
3414+ id0 = {'a': 1, 'b': 2}
3415+ id1 = {'a': 1, 'c': 3}
3416+ sc = {'id0': id0, 'id1': id1}
3417+
3418+ self.assertEqual([id0], select_configs(sc, a=1, b=2))
3419+
3420 # vi: ts=4 expandtab syntax=python
3421diff --git a/tests/vmtests/__init__.py b/tests/vmtests/__init__.py
3422index fd6c246..c52c442 100644
3423--- a/tests/vmtests/__init__.py
3424+++ b/tests/vmtests/__init__.py
3425@@ -1930,29 +1930,6 @@ class VMBaseClass(TestCase):
3426 self.assertIn(kpackage, self.debian_packages)
3427
3428 @skip_if_flag('expected_failure')
3429- def test_clear_holders_ran(self):
3430- """ Test curtin install runs block-meta/clear-holders. """
3431- if not self.has_storage_config():
3432- raise SkipTest("This test does not use storage config.")
3433-
3434- install_logfile = 'root/curtin-install.log'
3435- self.output_files_exist([install_logfile])
3436- install_log = self.load_collect_file(install_logfile)
3437-
3438- # validate block-meta called clear-holders at least once
3439- # We match both 'start' and 'finish' strings, so for each
3440- # call we'll have 2 matches.
3441- clear_holders_re = 'cmd-install/.*cmd-block-meta/clear-holders'
3442- events = re.findall(clear_holders_re, install_log)
3443- print('Matched clear-holder events:\n%s' % events)
3444- self.assertGreaterEqual(len(events), 2)
3445-
3446- # dirty_disks mode runs an early block-meta command which
3447- # also runs clear-holders
3448- if self.dirty_disks is True:
3449- self.assertGreaterEqual(len(events), 4)
3450-
3451- @skip_if_flag('expected_failure')
3452 def test_kernel_img_conf(self):
3453 """ Test curtin install kernel-img.conf correctly. """
3454 if self.target_distro != 'ubuntu':
3455@@ -2058,17 +2035,6 @@ class VMBaseClass(TestCase):
3456
3457 return swaps
3458
3459- # we don't yet have a skip_by_date on specific releases
3460- if is_devel_release(self.target_release):
3461- name = "test_swaps_used"
3462- bug = "1894910"
3463- fixby = "2020-10-15"
3464- removeby = "2020-11-01"
3465- raise SkipTest(
3466- "skip_by_date({name}) LP: #{bug} "
3467- "fixby={fixby} removeby={removeby}: ".format(
3468- name=name, bug=bug, fixby=fixby, removeby=removeby))
3469-
3470 expected_swaps = find_fstab_swaps()
3471 proc_swaps = self.load_collect_file("proc-swaps")
3472 for swap in expected_swaps:
3473@@ -2573,7 +2539,6 @@ def prep_partition_for_device(device):
3474 'size': '8M',
3475 'flag': 'prep',
3476 'guid': '9e1a2d38-c612-4316-aa26-8b49521e5a8b',
3477- 'offset': '1M',
3478 'wipe': 'zero',
3479 'grub_device': True,
3480 'device': device}
3481diff --git a/tests/vmtests/releases.py b/tests/vmtests/releases.py
3482index fa755b1..67248bf 100644
3483--- a/tests/vmtests/releases.py
3484+++ b/tests/vmtests/releases.py
3485@@ -7,7 +7,7 @@ class _ReleaseBase(object):
3486 repo = "maas-daily"
3487 arch = get_platform_arch()
3488 target_arch = arch
3489- mem = "1024"
3490+ mem = "2048"
3491
3492
3493 class _UbuntuBase(_ReleaseBase):
3494@@ -72,22 +72,6 @@ class _UbuntuCore20FromFocalBase(_UbuntuCoreUbuntuBase):
3495 release = "focal"
3496 # release for target
3497 target_release = "ubuntu-core-20"
3498- mem = "2048"
3499-
3500-
3501-class _Centos66FromXenialBase(_CentosFromUbuntuBase):
3502- release = "xenial"
3503- target_release = "centos66"
3504-
3505-
3506-class _Centos66FromBionicBase(_CentosFromUbuntuBase):
3507- release = "bionic"
3508- target_release = "centos66"
3509-
3510-
3511-class _Centos66FromFocalBase(_CentosFromUbuntuBase):
3512- release = "focal"
3513- target_release = "centos66"
3514
3515
3516 class _PreciseBase(_UbuntuBase):
3517@@ -148,7 +132,6 @@ class _XenialEdge(_XenialBase):
3518 class _BionicBase(_UbuntuBase):
3519 release = "bionic"
3520 target_release = "bionic"
3521- mem = "2048"
3522 if _UbuntuBase.arch == "arm64":
3523 subarch = "ga-18.04"
3524
3525@@ -164,7 +147,6 @@ class _DiscoBase(_UbuntuBase):
3526 release = "disco"
3527 target_release = "disco"
3528 # squashfs is over 300MB, need more ram
3529- mem = "2048"
3530 if _UbuntuBase.arch == "arm64":
3531 subarch = "ga-19.04"
3532
3533@@ -172,7 +154,6 @@ class _DiscoBase(_UbuntuBase):
3534 class _EoanBase(_UbuntuBase):
3535 release = "eoan"
3536 target_release = "eoan"
3537- mem = "2048"
3538 if _UbuntuBase.arch == "arm64":
3539 subarch = "ga-19.10"
3540
3541@@ -180,7 +161,6 @@ class _EoanBase(_UbuntuBase):
3542 class _FocalBase(_UbuntuBase):
3543 release = "focal"
3544 target_release = "focal"
3545- mem = "2048"
3546 if _UbuntuBase.arch == "arm64":
3547 subarch = "ga-20.04"
3548
3549@@ -188,7 +168,6 @@ class _FocalBase(_UbuntuBase):
3550 class _HirsuteBase(_UbuntuBase):
3551 release = "hirsute"
3552 target_release = "hirsute"
3553- mem = "2048"
3554 if _UbuntuBase.arch == "arm64":
3555 subarch = "ga-21.04"
3556
3557@@ -196,7 +175,6 @@ class _HirsuteBase(_UbuntuBase):
3558 class _ImpishBase(_UbuntuBase):
3559 release = "impish"
3560 target_release = "impish"
3561- mem = "2048"
3562 if _UbuntuBase.arch == "arm64":
3563 subarch = "ga-21.10"
3564
3565@@ -225,11 +203,8 @@ class _Releases(object):
3566
3567 class _CentosReleases(object):
3568 centos70_xenial = _Centos70FromXenialBase
3569- centos66_xenial = _Centos66FromXenialBase
3570 centos70_bionic = _Centos70FromBionicBase
3571- centos66_bionic = _Centos66FromBionicBase
3572 centos70_focal = _Centos70FromFocalBase
3573- centos66_focal = _Centos66FromFocalBase
3574
3575
3576 class _UbuntuCoreReleases(object):
3577diff --git a/tests/vmtests/test_basic.py b/tests/vmtests/test_basic.py
3578index 6059bd9..616d635 100644
3579--- a/tests/vmtests/test_basic.py
3580+++ b/tests/vmtests/test_basic.py
3581@@ -41,10 +41,6 @@ class TestBasicAbs(VMBaseClass):
3582 f="btrfs_uuid_diskc"
3583 if command -v btrfs-debug-tree >/dev/null; then
3584 btrfs-debug-tree -r $dev | awk '/^uuid/ {print $2}' | grep "-"
3585- # btrfs-debug-tree fails in centos66, use btrfs-show instead
3586- if [ "$?" != "0" ]; then
3587- btrfs-show $dev | awk '/uuid/ {print $4}'
3588- fi
3589 else
3590 btrfs inspect-internal dump-super $dev |
3591 awk '/^dev_item.fsid/ {print $2}'
3592@@ -61,9 +57,6 @@ class TestBasicAbs(VMBaseClass):
3593 """)]
3594
3595 def _test_ptable(self, blkid_output, expected):
3596- if self.target_release == "centos66":
3597- raise SkipTest("No PTTYPE blkid output on Centos66")
3598-
3599 if not blkid_output:
3600 raise RuntimeError('_test_ptable requires blkid output file')
3601
3602@@ -100,8 +93,6 @@ class TestBasicAbs(VMBaseClass):
3603 self.assertEqual(kname_uuid, btrfs_uuid)
3604
3605 def _test_partition_is_prep(self, info_file):
3606- if self.target_release == "centos66":
3607- raise SkipTest("Cannot detect PReP partitions in Centos66")
3608 udev_info = self.load_collect_file(info_file).rstrip()
3609 if not udev_info:
3610 raise ValueError('Empty udev_info collect file')
3611@@ -132,10 +123,7 @@ class TestBasicAbs(VMBaseClass):
3612
3613 def test_partition_numbers(self):
3614 # pnum_disk should have partitions 1 2, and 10
3615- if self.target_release != 'centos66':
3616- disk = self._dname_to_kname('pnum_disk')
3617- else:
3618- disk = self._serial_to_kname('disk-d')
3619+ disk = self._dname_to_kname('pnum_disk')
3620
3621 expected = [disk + s for s in ["", "1", "2", "10"]]
3622 self._test_partition_numbers(disk, expected)
3623@@ -220,19 +208,6 @@ class Centos70FocalTestBasic(centos_relbase.centos70_focal,
3624 __test__ = True
3625
3626
3627-class Centos66XenialTestBasic(centos_relbase.centos66_xenial,
3628- CentosTestBasicAbs):
3629- __test__ = True
3630-
3631-
3632-class Centos66BionicTestBasic(centos_relbase.centos66_bionic,
3633- CentosTestBasicAbs):
3634- # Centos66 cannot handle ext4 defaults in Bionic (64bit,meta_csum)
3635- # this conf defaults to ext3
3636- conf_file = "examples/tests/centos6_basic.yaml"
3637- __test__ = True
3638-
3639-
3640 class XenialGAi386TestBasic(relbase.xenial_ga, TestBasicAbs):
3641 __test__ = True
3642 arch_skip = ["arm64", "ppc64el", "s390x"]
3643diff --git a/tests/vmtests/test_network.py b/tests/vmtests/test_network.py
3644index 1b42493..6ff3a16 100644
3645--- a/tests/vmtests/test_network.py
3646+++ b/tests/vmtests/test_network.py
3647@@ -486,11 +486,6 @@ class ImpishTestNetworkBasic(relbase.impish, TestNetworkBasicAbs):
3648 __test__ = True
3649
3650
3651-class Centos66TestNetworkBasic(centos_relbase.centos66_xenial,
3652- CentosTestNetworkBasicAbs):
3653- __test__ = True
3654-
3655-
3656 class Centos70TestNetworkBasic(centos_relbase.centos70_xenial,
3657 CentosTestNetworkBasicAbs):
3658 __test__ = True
3659diff --git a/tests/vmtests/test_network_alias.py b/tests/vmtests/test_network_alias.py
3660index 8b58edd..dd6ba02 100644
3661--- a/tests/vmtests/test_network_alias.py
3662+++ b/tests/vmtests/test_network_alias.py
3663@@ -34,11 +34,6 @@ class CentosTestNetworkAliasAbs(TestNetworkAliasAbs):
3664 pass
3665
3666
3667-class Centos66TestNetworkAlias(centos_relbase.centos66_xenial,
3668- CentosTestNetworkAliasAbs):
3669- __test__ = True
3670-
3671-
3672 class Centos70TestNetworkAlias(centos_relbase.centos70_xenial,
3673 CentosTestNetworkAliasAbs):
3674 __test__ = True
3675diff --git a/tests/vmtests/test_network_bonding.py b/tests/vmtests/test_network_bonding.py
3676index 73bcf60..ad0c1d4 100644
3677--- a/tests/vmtests/test_network_bonding.py
3678+++ b/tests/vmtests/test_network_bonding.py
3679@@ -69,11 +69,6 @@ class ImpishTestBonding(relbase.impish, TestNetworkBondingAbs):
3680 __test__ = True
3681
3682
3683-class Centos66TestNetworkBonding(centos_relbase.centos66_xenial,
3684- CentosTestNetworkBondingAbs):
3685- __test__ = True
3686-
3687-
3688 class Centos70TestNetworkBonding(centos_relbase.centos70_xenial,
3689 CentosTestNetworkBondingAbs):
3690 __test__ = True
3691diff --git a/tests/vmtests/test_network_bridging.py b/tests/vmtests/test_network_bridging.py
3692index 93ecc4b..9c90702 100644
3693--- a/tests/vmtests/test_network_bridging.py
3694+++ b/tests/vmtests/test_network_bridging.py
3695@@ -41,8 +41,6 @@ default_bridge_params_uncheckable = [
3696
3697 # attrs we cannot validate
3698 release_to_bridge_params_uncheckable = {
3699- 'centos66': ['bridge_fd', 'bridge_hello', 'bridge_hw', 'bridge_maxage',
3700- 'bridge_pathcost', 'bridge_portprio'],
3701 'centos70': ['bridge_fd', 'bridge_hello', 'bridge_hw', 'bridge_maxage',
3702 'bridge_pathcost', 'bridge_portprio'],
3703 'xenial': ['bridge_ageing'],
3704@@ -220,11 +218,6 @@ class CentosTestBridgeNetworkAbs(TestBridgeNetworkAbs):
3705 self.assertTrue('bridge' in status)
3706
3707
3708-class Centos66TestBridgeNetwork(centos_relbase.centos66_xenial,
3709- CentosTestBridgeNetworkAbs):
3710- __test__ = True
3711-
3712-
3713 class Centos70TestBridgeNetwork(centos_relbase.centos70_xenial,
3714 CentosTestBridgeNetworkAbs):
3715 __test__ = True
3716diff --git a/tests/vmtests/test_network_ipv6.py b/tests/vmtests/test_network_ipv6.py
3717index 80b8ccf..f524e82 100644
3718--- a/tests/vmtests/test_network_ipv6.py
3719+++ b/tests/vmtests/test_network_ipv6.py
3720@@ -65,11 +65,6 @@ class ImpishTestNetworkIPV6(relbase.impish, TestNetworkIPV6Abs):
3721 __test__ = True
3722
3723
3724-class Centos66TestNetworkIPV6(centos_relbase.centos66_xenial,
3725- CentosTestNetworkIPV6Abs):
3726- __test__ = True
3727-
3728-
3729 class Centos70TestNetworkIPV6(centos_relbase.centos70_xenial,
3730 CentosTestNetworkIPV6Abs):
3731 __test__ = True
3732diff --git a/tests/vmtests/test_network_ipv6_static.py b/tests/vmtests/test_network_ipv6_static.py
3733index f24aab5..cb9caad 100644
3734--- a/tests/vmtests/test_network_ipv6_static.py
3735+++ b/tests/vmtests/test_network_ipv6_static.py
3736@@ -35,11 +35,6 @@ class ImpishTestNetworkIPV6Static(relbase.impish, TestNetworkIPV6StaticAbs):
3737 __test__ = True
3738
3739
3740-class Centos66TestNetworkIPV6Static(centos_relbase.centos66_xenial,
3741- CentosTestNetworkIPV6StaticAbs):
3742- __test__ = True
3743-
3744-
3745 class Centos70TestNetworkIPV6Static(centos_relbase.centos70_xenial,
3746 CentosTestNetworkIPV6StaticAbs):
3747 __test__ = True
3748diff --git a/tests/vmtests/test_network_ipv6_vlan.py b/tests/vmtests/test_network_ipv6_vlan.py
3749index a6eae41..7955101 100644
3750--- a/tests/vmtests/test_network_ipv6_vlan.py
3751+++ b/tests/vmtests/test_network_ipv6_vlan.py
3752@@ -34,11 +34,6 @@ class ImpishTestNetworkIPV6Vlan(relbase.impish, TestNetworkIPV6VlanAbs):
3753 __test__ = True
3754
3755
3756-class Centos66TestNetworkIPV6Vlan(centos_relbase.centos66_xenial,
3757- CentosTestNetworkIPV6VlanAbs):
3758- __test__ = True
3759-
3760-
3761 class Centos70TestNetworkIPV6Vlan(centos_relbase.centos70_xenial,
3762 CentosTestNetworkIPV6VlanAbs):
3763 __test__ = True
3764diff --git a/tests/vmtests/test_network_mtu.py b/tests/vmtests/test_network_mtu.py
3765index a36a752..f112b1c 100644
3766--- a/tests/vmtests/test_network_mtu.py
3767+++ b/tests/vmtests/test_network_mtu.py
3768@@ -201,11 +201,6 @@ class ImpishTestNetworkMtu(relbase.impish, TestNetworkMtuNetworkdAbs):
3769 __test__ = True
3770
3771
3772-class Centos66TestNetworkMtu(centos_relbase.centos66_xenial,
3773- CentosTestNetworkMtuAbs):
3774- __test__ = True
3775-
3776-
3777 class Centos70TestNetworkMtu(centos_relbase.centos70_xenial,
3778 CentosTestNetworkMtuAbs):
3779 __test__ = True
3780diff --git a/tests/vmtests/test_network_static.py b/tests/vmtests/test_network_static.py
3781index 95960af..867cf11 100644
3782--- a/tests/vmtests/test_network_static.py
3783+++ b/tests/vmtests/test_network_static.py
3784@@ -40,11 +40,6 @@ class ImpishTestNetworkStatic(relbase.impish, TestNetworkStaticAbs):
3785 __test__ = True
3786
3787
3788-class Centos66TestNetworkStatic(centos_relbase.centos66_xenial,
3789- CentosTestNetworkStaticAbs):
3790- __test__ = True
3791-
3792-
3793 class Centos70TestNetworkStatic(centos_relbase.centos70_xenial,
3794 CentosTestNetworkStaticAbs):
3795 __test__ = True
3796diff --git a/tests/vmtests/test_network_static_routes.py b/tests/vmtests/test_network_static_routes.py
3797index eb096ee..664c035 100644
3798--- a/tests/vmtests/test_network_static_routes.py
3799+++ b/tests/vmtests/test_network_static_routes.py
3800@@ -43,11 +43,6 @@ class ImpishTestNetworkStaticRoutes(relbase.impish,
3801 __test__ = True
3802
3803
3804-class Centos66TestNetworkStaticRoutes(centos_relbase.centos66_xenial,
3805- CentosTestNetworkStaticRoutesAbs):
3806- __test__ = False
3807-
3808-
3809 class Centos70TestNetworkStaticRoutes(centos_relbase.centos70_xenial,
3810 CentosTestNetworkStaticRoutesAbs):
3811 __test__ = False
3812diff --git a/tests/vmtests/test_network_vlan.py b/tests/vmtests/test_network_vlan.py
3813index 38bc87c..99bad66 100644
3814--- a/tests/vmtests/test_network_vlan.py
3815+++ b/tests/vmtests/test_network_vlan.py
3816@@ -88,11 +88,6 @@ class ImpishTestNetworkVlan(relbase.impish, TestNetworkVlanAbs):
3817 __test__ = True
3818
3819
3820-class Centos66TestNetworkVlan(centos_relbase.centos66_xenial,
3821- CentosTestNetworkVlanAbs):
3822- __test__ = True
3823-
3824-
3825 class Centos70TestNetworkVlan(centos_relbase.centos70_xenial,
3826 CentosTestNetworkVlanAbs):
3827 __test__ = True
3828diff --git a/tests/vmtests/test_preserve_raid.py b/tests/vmtests/test_preserve_raid.py
3829index 4bb977e..04c16b7 100644
3830--- a/tests/vmtests/test_preserve_raid.py
3831+++ b/tests/vmtests/test_preserve_raid.py
3832@@ -56,6 +56,9 @@ class BionicTestPartitionExistingRAID(
3833 relbase.bionic, TestPartitionExistingRAID):
3834 __test__ = True
3835
3836+ def test_correct_ptype(self):
3837+ self.skipTest("lsblk on bionic does not support PTTYPE")
3838+
3839
3840 class FocalTestPartitionExistingRAID(
3841 relbase.focal, TestPartitionExistingRAID):
3842diff --git a/tests/vmtests/test_simple.py b/tests/vmtests/test_simple.py
3843index 0ee87fc..2b91f0b 100644
3844--- a/tests/vmtests/test_simple.py
3845+++ b/tests/vmtests/test_simple.py
3846@@ -29,15 +29,6 @@ class Centos70BionicTestSimple(centos_relbase.centos70_bionic, TestSimple):
3847 __test__ = True
3848
3849
3850-class Centos66XenialTestSimple(centos_relbase.centos66_xenial, TestSimple):
3851- __test__ = True
3852-
3853-
3854-class Centos66BionicTestSimple(centos_relbase.centos66_bionic, TestSimple):
3855- __test__ = False
3856- # LP: #1775424 Centos66 fails with Bionic Ephemeral ext4 features
3857-
3858-
3859 class XenialTestSimple(relbase.xenial, TestSimple):
3860 __test__ = True
3861
3862diff --git a/tests/vmtests/test_uefi_basic.py b/tests/vmtests/test_uefi_basic.py
3863index aa4c650..1a90a7d 100644
3864--- a/tests/vmtests/test_uefi_basic.py
3865+++ b/tests/vmtests/test_uefi_basic.py
3866@@ -17,7 +17,7 @@ class TestBasicAbs(VMBaseClass):
3867 disk_to_check = [('main_disk', 1), ('main_disk', 2)]
3868 extra_collect_scripts = [textwrap.dedent("""
3869 cd OUTPUT_COLLECT_D
3870- ls /sys/firmware/efi/ | cat >ls_sys_firmware_efi
3871+ test -d /sys/firmware/efi ; echo $? >is_efi
3872 cp /sys/class/block/vda/queue/logical_block_size vda_lbs
3873 cp /sys/class/block/vda/queue/physical_block_size vda_pbs
3874 blockdev --getsz /dev/vda | cat >vda_blockdev_getsz
3875@@ -28,24 +28,10 @@ class TestBasicAbs(VMBaseClass):
3876 exit 0
3877 """)]
3878
3879- def test_sys_firmware_efi(self):
3880- self.output_files_exist(["ls_sys_firmware_efi"])
3881- sys_efi_possible = [
3882- 'config_table',
3883- 'efivars',
3884- 'fw_platform_size',
3885- 'fw_vendor',
3886- 'runtime',
3887- 'runtime-map',
3888- 'systab',
3889- 'vars',
3890- ]
3891- efi_lines = self.load_collect_file(
3892- "ls_sys_firmware_efi").strip().split('\n')
3893-
3894- # sys/firmware/efi contents differ based on kernel and configuration
3895- for efi_line in efi_lines:
3896- self.assertIn(efi_line, sys_efi_possible)
3897+ def test_is_efi(self):
3898+ self.output_files_exist(["is_efi"])
3899+ efi_lines = self.load_collect_file("is_efi").strip().split('\n')
3900+ self.assertEqual(['0'], efi_lines)
3901
3902 def test_disk_block_sizes(self):
3903 """ Test disk logical and physical block size are match
3904diff --git a/tools/build-deb b/tools/build-deb
3905index dbe364f..85868d7 100755
3906--- a/tools/build-deb
3907+++ b/tools/build-deb
3908@@ -1,7 +1,7 @@
3909 #!/bin/sh
3910 # This file is part of curtin. See LICENSE file for copyright and license info.
3911
3912-set -e
3913+set -eu
3914
3915 sourcename="curtin"
3916 TEMP_D=""
3917@@ -13,7 +13,7 @@ cleanup() {
3918 [ -z "$TEMP_D" ] || rm -Rf "$TEMP_D"
3919 }
3920
3921-if [ "$1" = "-h" -o "$1" = "--help" ]; then
3922+if [ "${1:-}" = "-h" -o "${1:-}" = "--help" ]; then
3923 cat <<EOF
3924 Usage: ${0##*/}
3925 build a deb of from trunk directory
3926@@ -36,19 +36,13 @@ top_d=$(cd "$(dirname "${0}")"/.. && pwd)
3927 ref=HEAD
3928
3929 if [ $# -eq 0 ]; then
3930- # if no opts given, build source, without depends, and not signed.
3931- set -- -S -d -us -uc
3932+ # if no opts given, build source, without depends.
3933+ set -- -S -d
3934 fi
3935
3936-# grab the first line in the changelog
3937-# hopefully this pulls the version info there
3938-# resulting in something like: UPSTREAM_VER-0ubuntu1
3939-clogver_o=$(sed -n '1s,.*(\([^)]*\)).*,\1,p' debian/changelog.trunk)
3940-
3941 # uver gets 17.1-3-gc85e2562 '17.1' if this is a tag.
3942 uver=$(git describe --long --abbrev=8 "--match=[0-9][0-9]*" "$ref")
3943-clogver_debian=${clogver_o##*-}
3944-clogver_new="${uver}-${clogver_debian}"
3945+clogver_new="${uver}-0ubuntu1"
3946
3947 # uver_base_rel rel gets '17.1'
3948 uver_base_rel=${uver%%-*}
3949@@ -60,7 +54,7 @@ TEMP_D=$(mktemp -d "${TMPDIR:-/tmp}/${bname}.XXXXXX")
3950
3951 trap cleanup EXIT
3952
3953-echo "building version ${uver}, debian_ver=${clogver_debian}"
3954+echo "building version ${uver}"
3955
3956 dir="${sourcename}-$uver"
3957 tarball="${sourcename}_$uver.orig.tar.gz"
3958@@ -73,38 +67,13 @@ cd "${TEMP_D}"
3959 tar xzf "$tarball" || fail "failed extract tarball"
3960
3961 if [ ! -d "$dir" ]; then
3962- # make-tarball will create the directory name based on the
3963- # contents of debian/changelog.trunk in the version provided.
3964- # if that differs from what is here, then user has changes.
3965- for d in ${sourcename}*; do
3966- [ -d "$d" ] && break
3967- done
3968- if [ -d "$d" ]; then
3969- {
3970- echo "WARNING: git at '${uver}' had different version"
3971- echo " in debian/changelog.trunk than your tree. version there"
3972- echo " is '$d' working directory had $uver"
3973- } 1>&2
3974- dir=$d
3975- else
3976- echo "did not find a directory created by make-tarball. sorry." 1>&2
3977- exit
3978- fi
3979+ echo "did not find a directory created by make-tarball. sorry." 1>&2
3980+ exit
3981 fi
3982 cd "$dir" || fail "failed cd $dir"
3983
3984-# move files ending in .trunk to name without .trunk
3985-# ie, this copies debian/changelog.trunk to debian/changelog
3986-for f in debian/*.trunk; do
3987- mv "$f" "${f%.trunk}"
3988-done
3989-
3990-# first line of debian/changelog looks like
3991-# curtin (<version>) UNRELEASED; urgency=low
3992-# fix the version and UNRELEASED
3993-sed -i -e "1s,([^)]*),(${clogver_new})," \
3994- -e "1s,UNRELEASED,${RELEASE}," debian/changelog ||
3995- fail "failed to write debian/changelog"
3996+dch --create --package curtin --newversion "$clogver_new" \
3997+ --distribution "$RELEASE" "Development release"
3998 debuild "$@" || fail "debuild failed"
3999
4000 cd "$TEMP_D"
4001diff --git a/tox.ini b/tox.ini
4002index d9437c5..2fc4027 100644
4003--- a/tox.ini
4004+++ b/tox.ini
4005@@ -3,14 +3,10 @@ minversion = 1.6
4006 skipsdist = True
4007 envlist =
4008 py3-flake8,
4009- py27,
4010 py3,
4011 py3-pyflakes,
4012 py3-pylint,
4013- py27-pylint,
4014- trusty-py27,
4015- block-schema,
4016- xenial-py3
4017+ block-schema
4018
4019 [tox:jenkins]
4020 downloadcache = ~/cache/pip
4021@@ -31,23 +27,6 @@ commands = {envpython} {toxinidir}/tools/noproxy {envpython} -m nose \
4022 basepython = python3
4023 sitepackages = true
4024
4025-[testenv:py27]
4026-basepython = python2.7
4027-sitepackages = true
4028-# https://github.com/pypa/setuptools/issues/1963
4029-deps = {[testenv]deps}
4030- setuptools<45
4031-
4032-# tox uses '--pre' by default to pip install. We don't want that, and
4033-# 'pip_pre=False' isn't available until tox version 1.9.
4034-install_command = pip install {opts} {packages}
4035-
4036-[testenv:py2-flake8]
4037-basepython = python2
4038-deps = {[testenv]deps}
4039- flake8
4040-commands = {envpython} -m flake8 {posargs:curtin}
4041-
4042 [testenv:py3-flake8]
4043 basepython = python3
4044 deps = {[testenv]deps}
4045@@ -64,19 +43,10 @@ commands = {envpython} -m pyflakes {posargs:curtin/ tests/ tools/}
4046 basepython = python3
4047 sitepackages = true
4048 deps = {[testenv]deps}
4049- pylint==2.6.0
4050+ pylint==2.12.2
4051 git+https://git.launchpad.net/simplestreams
4052 commands = {envpython} -m pylint --errors-only {posargs:curtin tests/vmtests}
4053
4054-[testenv:py27-pylint]
4055-# set basepython because tox 1.6 (trusty) does not support generated environments
4056-basepython = python2.7
4057-sitepackages = true
4058-deps = {[testenv]deps}
4059- {[testenv:py27]deps}
4060- pylint==1.8.1
4061-commands = {envpython} -m pylint --errors-only {posargs:curtin}
4062-
4063 [testenv:docs]
4064 deps = {[testenv]deps}
4065 sphinx
4066@@ -107,15 +77,6 @@ basepython = python3
4067 commands =
4068 {toxinidir}/tools/run-pyflakes3 {posargs}
4069
4070-[testenv:trusty-py27]
4071-deps = {[testenv:trusty]deps}
4072- setuptools<45
4073-
4074-basepython = python2.7
4075-sitepackages = true
4076-commands = {envpython} {toxinidir}/tools/noproxy {envpython} -m nose \
4077- {posargs:tests/unittests}
4078-
4079 [testenv:trusty-py3]
4080 deps = {[testenv:trusty]deps}
4081 basepython = python3
4082@@ -129,13 +90,6 @@ deps =
4083 pyyaml==3.11
4084 oauthlib==1.0.3
4085
4086-[testenv:xenial-py27]
4087-basepython = python27
4088-deps = {[testenv:xenial]deps}
4089- {[testenv:py27]deps}
4090-commands = {envpython} {toxinidir}/tools/noproxy {envpython} -m nose \
4091- {posargs:tests/unittests}
4092-
4093 [testenv:xenial-py3]
4094 basepython = python3
4095 sitepackages = true

Subscribers

People subscribed via source and target branches