Merge ~dbungert/curtin:partition-arbitrary-types into curtin:master

Proposed by Dan Bungert
Status: Merged
Merge reported by: Dan Bungert
Merged at revision: 3d3f901afea2ffed5d1a105f2cc6d941bf09f052
Proposed branch: ~dbungert/curtin:partition-arbitrary-types
Merge into: curtin:master
Diff against target: 489 lines (+259/-17)
8 files modified
curtin/block/schemas.py (+5/-0)
curtin/commands/block_meta_v2.py (+13/-3)
curtin/storage_config.py (+2/-0)
doc/topics/storage.rst (+12/-0)
test-requirements.txt (+1/-0)
tests/integration/test_block_meta.py (+134/-8)
tests/unittests/test_commands_block_meta.py (+72/-0)
tests/unittests/test_storage_config.py (+20/-6)
Reviewer Review Type Date Requested Status
Michael Hudson-Doyle Approve
Server Team CI bot continuous-integration Approve
Review via email: mp+420764@code.launchpad.net

Commit message

block/v2: allow setting raw partition_type value

To preserve partition types that do not map to known flag values, we
must be able to capture that original type information and carry it
around in the storage config until used.

If we do not do this, preserved partitions get their type normalized to
another value (usually the equivalent of flags 'linux' or 'boot').

To post a comment you must log in.
Revision history for this message
Michael Hudson-Doyle (mwhudson) wrote :

Seems basically fine but

1) needs documentation
2) it should work with gpt partition tables too?
3) is it ok that this only works with v2 configs? (probably...)

Revision history for this message
Michael Hudson-Doyle (mwhudson) :
review: Needs Fixing
Revision history for this message
Server Team CI bot (server-team-bot) wrote :
review: Approve (continuous-integration)
Revision history for this message
Server Team CI bot (server-team-bot) wrote :
review: Approve (continuous-integration)
e0eb292... by Dan Bungert

block/v2: raw partition table codes for gpt

ac081b6... by Dan Bungert

block/v2: unit tests for partition_type

34d15bc... by Dan Bungert

block/v2: docs for partition_type

Revision history for this message
Server Team CI bot (server-team-bot) wrote :
review: Approve (continuous-integration)
3d3f901... by Dan Bungert

block/v2: preserve disk label id

Windows will report a boot failure if the disk label ID changes and
nothing else.

Revision history for this message
Dan Bungert (dbungert) wrote :

> 1) needs documentation
Done. I debated leaving it undocumented on purpose as too obscure of a detail, but this is fine.

2) it should work with gpt partition tables too?
Done.

3) is it ok that this only works with v2 configs? (probably...)
That's my intention.

Also added disk label ID preservation, that's needed.

Revision history for this message
Server Team CI bot (server-team-bot) wrote :
review: Approve (continuous-integration)
Revision history for this message
Michael Hudson-Doyle (mwhudson) wrote :

Nice, this looks good. Just one question/suggestion but feel free to ignore it.

review: Approve
Revision history for this message
Michael Hudson-Doyle (mwhudson) :

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/curtin/block/schemas.py b/curtin/block/schemas.py
2index 1834343..92f88d0 100644
3--- a/curtin/block/schemas.py
4+++ b/curtin/block/schemas.py
5@@ -304,6 +304,11 @@ PARTITION = {
6 'enum': ['bios_grub', 'boot', 'extended', 'home', 'linux',
7 'logical', 'lvm', 'mbr', 'prep', 'raid', 'swap',
8 '']},
9+ 'partition_type': {'type': 'string',
10+ 'oneOf': [
11+ {'pattern': r'^0x[0-9a-fA-F]{1,2}$'},
12+ {'$ref': '#/definitions/uuid'},
13+ ]},
14 'grub_device': {
15 'type': ['boolean', 'integer'],
16 'minimum': 0,
17diff --git a/curtin/commands/block_meta_v2.py b/curtin/commands/block_meta_v2.py
18index a095c8f..b4838f9 100644
19--- a/curtin/commands/block_meta_v2.py
20+++ b/curtin/commands/block_meta_v2.py
21@@ -98,6 +98,7 @@ class SFDiskPartTable:
22
23 def __init__(self, sector_bytes):
24 self.entries = []
25+ self.label_id = None
26 self._sector_bytes = sector_bytes
27 if ONE_MIB_BYTES % sector_bytes != 0:
28 raise Exception(
29@@ -112,7 +113,11 @@ class SFDiskPartTable:
30 return amount * self._sector_bytes
31
32 def render(self):
33- r = ['label: ' + self.label, ''] + [e.render() for e in self.entries]
34+ r = ['label: ' + self.label]
35+ if self.label_id:
36+ r.extend(['label-id: ' + self.label_id])
37+ r.extend([''])
38+ r.extend([e.render() for e in self.entries])
39 return '\n'.join(r)
40
41 def apply(self, device):
42@@ -150,7 +155,8 @@ class GPTPartTable(SFDiskPartTable):
43 start = self.one_mib_sectors
44 size = self.bytes2sectors(action['size'])
45 uuid = action.get('uuid')
46- type = FLAG_TO_GUID.get(action.get('flag'))
47+ type = action.get('partition_type',
48+ FLAG_TO_GUID.get(action.get('flag')))
49 entry = PartTableEntry(number, start, size, type, uuid)
50 self.entries.append(entry)
51 return entry
52@@ -210,7 +216,7 @@ class DOSPartTable(SFDiskPartTable):
53 prev.start + prev.size,
54 self.one_mib_sectors)
55 size = self.bytes2sectors(action['size'])
56- type = FLAG_TO_MBR_TYPE.get(flag)
57+ type = action.get('partition_type', FLAG_TO_MBR_TYPE.get(flag))
58 if flag == 'boot':
59 bootable = True
60 else:
61@@ -364,6 +370,10 @@ def disk_handler_v2(info, storage_config, handlers):
62 preserved_offsets.add(entry.start)
63 wipes[entry.start] = _wipe_for_action(action)
64
65+ # preserve disk label ids
66+ if info.get('preserve') and sfdisk_info is not None:
67+ table.label_id = sfdisk_info['id']
68+
69 for kname, nr, offset, size in block.sysfs_partition_data(disk):
70 offset_sectors = table.bytes2sectors(offset)
71 resize = resizes.get(offset_sectors)
72diff --git a/curtin/storage_config.py b/curtin/storage_config.py
73index 2ede996..e9e8991 100644
74--- a/curtin/storage_config.py
75+++ b/curtin/storage_config.py
76@@ -800,6 +800,8 @@ class BlockdevParser(ProbertParser):
77 entry['size'] *= 512
78
79 ptype = blockdev_data.get('ID_PART_ENTRY_TYPE')
80+ if ptype is not None:
81+ entry['partition_type'] = ptype
82 flag_name, _flag_code = ptable_uuid_to_flag_entry(ptype)
83
84 if ptable and ptable.get('label') == 'dos':
85diff --git a/doc/topics/storage.rst b/doc/topics/storage.rst
86index aa5ad13..bbff909 100644
87--- a/doc/topics/storage.rst
88+++ b/doc/topics/storage.rst
89@@ -431,6 +431,18 @@ partition with the *bios_grub* flag is needed. This partition should be placed
90 at the beginning of the disk and should be 1MB in size. It should not contain a
91 filesystem or be mounted anywhere on the system.
92
93+**partition_type**: *msdos: byte value in 0xnn style; gpt: GUID*
94+
95+Only applicable to v2 storage configuration. If both ``partition_type`` and
96+``flag`` are set, ``partition_type`` dictates the acutal type.
97+
98+The ``partition_type`` field allows for setting arbitrary partition type values
99+that do not have a matching ``flag``, or cases that are not handled by the
100+``flag`` system. For example, since the *boot* flag results in both setting
101+the bootable state for a MSDOS partition table and setting it to type *0xEF*,
102+one can override this behavior and achieve a bootable partition of a different
103+type by using ``flag``: *boot* and using ``partition_type``.
104+
105 **preserve**: *true, false*
106
107 If the preserve flag is set to true, curtin will verify that the partition
108diff --git a/test-requirements.txt b/test-requirements.txt
109index f6404c1..1970d03 100644
110--- a/test-requirements.txt
111+++ b/test-requirements.txt
112@@ -3,3 +3,4 @@ mock
113 nose
114 pyflakes
115 coverage
116+parameterized
117diff --git a/tests/integration/test_block_meta.py b/tests/integration/test_block_meta.py
118index 8322062..e542017 100644
119--- a/tests/integration/test_block_meta.py
120+++ b/tests/integration/test_block_meta.py
121@@ -1,12 +1,15 @@
122 # This file is part of curtin. See LICENSE file for copyright and license info.
123
124-from collections import namedtuple
125+import dataclasses
126+from dataclasses import dataclass
127 import contextlib
128 import json
129-import sys
130-import yaml
131 import os
132+from parameterized import parameterized
133 import re
134+import sys
135+from typing import Optional
136+import yaml
137
138 from curtin import block, udev, util
139
140@@ -34,7 +37,28 @@ def loop_dev(image, sector_size=512):
141 util.subp(['losetup', '--detach', dev])
142
143
144-PartData = namedtuple("PartData", ('number', 'offset', 'size'))
145+@dataclass(order=True)
146+class PartData:
147+ number: Optional[int] = None
148+ offset: Optional[int] = None
149+ size: Optional[int] = None
150+ boot: Optional[bool] = None
151+ partition_type: Optional[str] = None
152+
153+ # test cases may initialize the values they care about
154+ # test utilities shall initialize all fields
155+ def assertFieldsAreNotNone(self):
156+ for field in dataclasses.fields(self):
157+ assert getattr(self, field.name) is not None
158+
159+ def __eq__(self, other):
160+ for field in dataclasses.fields(self):
161+ myval = getattr(self, field.name)
162+ otherval = getattr(other, field.name)
163+ if myval is not None and otherval is not None \
164+ and myval != otherval:
165+ return False
166+ return True
167
168
169 def _get_ext_size(dev, part_action):
170@@ -92,10 +116,29 @@ def _get_extended_partition_size(dev, num):
171 return partition['size'] * 512
172
173
174+def _get_disk_label_id(dev):
175+ ptable_json = util.subp(['sfdisk', '-J', dev], capture=True)[0]
176+ ptable = json.loads(ptable_json)
177+ # string in lowercase hex
178+ return ptable['partitiontable']['id']
179+
180+
181 def summarize_partitions(dev):
182- # We don't care about the kname
183- return sorted(
184- [PartData(*d[1:]) for d in block.sysfs_partition_data(dev)])
185+ parts = []
186+ ptable_json = util.subp(['sfdisk', '-J', dev], capture=True)[0]
187+ ptable = json.loads(ptable_json)
188+ partitions = ptable['partitiontable']['partitions']
189+ for d in block.sysfs_partition_data(dev):
190+ nodename = f'/dev/{d[0]}'
191+ partition = [part for part in partitions
192+ if part['node'] == nodename][0]
193+ ptype = partition['type']
194+ boot = partition.get('bootable', False)
195+ # We don't care about the kname
196+ pd = PartData(*d[1:], partition_type=ptype, boot=boot)
197+ pd.assertFieldsAreNotNone()
198+ parts.append(pd)
199+ return sorted(parts)
200
201
202 class StorageConfigBuilder:
203@@ -149,6 +192,10 @@ class TestBlockMeta(IntegrationTestCase):
204 def setUp(self):
205 self.data = self.random_string()
206
207+ def assertPartitions(self, *args):
208+ with loop_dev(self.img) as dev:
209+ self.assertEqual([*args], summarize_partitions(dev))
210+
211 @contextlib.contextmanager
212 def mount(self, dev, partition_cfg):
213 mnt_point = self.tmp_dir()
214@@ -860,5 +907,84 @@ class TestBlockMeta(IntegrationTestCase):
215 ])
216 with self.mount(dev, p1) as mnt_point:
217 for i in range(5, 41, 5):
218- with open(f'{mnt_point}/{str(i)}', 'rb') as fp:
219+ with open(f'{mnt_point}/{i}', 'rb') as fp:
220 self.assertEqual(bytes([i]) * (2 << 20), fp.read())
221+
222+ def test_parttype_dos(self):
223+ # msdos partition table partitions shall retain their type
224+ # create initial situation similar to this
225+ # Device Boot Start End Sectors Size Id Type
226+ # /dev/sda1 * 2048 104447 102400 50M 7 HPFS/NTFS/exFA
227+ # /dev/sda2 104448 208668781 208564334 99.5G 7 HPFS/NTFS/exFA
228+ # /dev/sda3 208670720 209711103 1040384 508M 27 Hidden NTFS Wi
229+ self.img = self.tmp_path('image.img')
230+ config = StorageConfigBuilder(version=2)
231+ config.add_image(path=self.img, size='200M', ptable='msdos')
232+ config.add_part(size=50 << 20, offset=1 << 20, number=1,
233+ fstype='ntfs', flag='boot', partition_type='0x7')
234+ config.add_part(size=100 << 20, offset=51 << 20, number=2,
235+ fstype='ntfs', partition_type='0x7')
236+ config.add_part(size=48 << 20, offset=151 << 20, number=3,
237+ fstype='ntfs', partition_type='0x27')
238+ self.run_bm(config.render())
239+ self.assertPartitions(
240+ PartData(number=1, offset=1 << 20, size=50 << 20,
241+ partition_type='7', boot=True),
242+ PartData(number=2, offset=51 << 20, size=100 << 20,
243+ partition_type='7', boot=False),
244+ PartData(number=3, offset=151 << 20, size=48 << 20,
245+ partition_type='27', boot=False))
246+
247+ def test_parttype_gpt(self):
248+ # gpt partition table partitions shall retain their type
249+ # create initial situation similar to this
250+ # # Start (sector) End (sector) Size Code Name
251+ # 1 2048 206847 100.0 MiB EF00 EFI system part
252+ # 2 206848 239615 16.0 MiB 0C01 Microsoft reser
253+ # 3 239616 103811181 49.4 GiB 0700 Basic data part
254+ # 4 103813120 104853503 508.0 MiB 2700
255+ esp = 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B'
256+ msreserved = 'E3C9E316-0B5C-4DB8-817D-F92DF00215AE'
257+ msdata = 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7'
258+ winre = 'DE94BBA4-06D1-4D40-A16A-BFD50179D6AC'
259+ self.img = self.tmp_path('image.img')
260+ config = StorageConfigBuilder(version=2)
261+ config.add_image(path=self.img, size='100M', ptable='gpt')
262+ config.add_part(number=1, offset=1 << 20, size=9 << 20,
263+ flag='boot', fstype='ntfs')
264+ config.add_part(number=2, offset=10 << 20, size=20 << 20,
265+ partition_type=msreserved)
266+ config.add_part(number=3, offset=30 << 20, size=50 << 20,
267+ partition_type=msdata, fstype='ntfs')
268+ config.add_part(number=4, offset=80 << 20, size=19 << 20,
269+ partition_type=winre, fstype='ntfs')
270+ self.run_bm(config.render())
271+ self.assertPartitions(
272+ PartData(number=1, offset=1 << 20, size=9 << 20,
273+ partition_type=esp),
274+ PartData(number=2, offset=10 << 20, size=20 << 20,
275+ partition_type=msreserved),
276+ PartData(number=3, offset=30 << 20, size=50 << 20,
277+ partition_type=msdata),
278+ PartData(number=4, offset=80 << 20, size=19 << 20,
279+ partition_type=winre))
280+
281+ @parameterized.expand([('msdos',), ('gpt',)])
282+ def test_disk_label_id_persistent(self, ptable):
283+ # when the disk is preserved, the disk label id shall also be preserved
284+ self.img = self.tmp_path('image.img')
285+ config = StorageConfigBuilder(version=2)
286+ config.add_image(path=self.img, size='20M', ptable=ptable)
287+ config.add_part(number=1, offset=1 << 20, size=18 << 20)
288+ self.run_bm(config.render())
289+ self.assertPartitions(
290+ PartData(number=1, offset=1 << 20, size=18 << 20))
291+ with loop_dev(self.img) as dev:
292+ orig_label_id = _get_disk_label_id(dev)
293+
294+ config.set_preserve()
295+ self.run_bm(config.render())
296+ self.assertPartitions(
297+ PartData(number=1, offset=1 << 20, size=18 << 20))
298+ with loop_dev(self.img) as dev:
299+ self.assertEqual(orig_label_id, _get_disk_label_id(dev))
300diff --git a/tests/unittests/test_commands_block_meta.py b/tests/unittests/test_commands_block_meta.py
301index 0d73ab6..9185d4e 100644
302--- a/tests/unittests/test_commands_block_meta.py
303+++ b/tests/unittests/test_commands_block_meta.py
304@@ -10,6 +10,7 @@ from mock import (
305 )
306 import os
307 import random
308+import uuid
309
310 from curtin.block import dasd
311 from curtin.commands import block_meta, block_meta_v2
312@@ -17,6 +18,10 @@ from curtin import paths, util
313 from .helpers import CiTestCase
314
315
316+def random_uuid():
317+ return uuid.uuid4()
318+
319+
320 class TestGetPathToStorageVolume(CiTestCase):
321
322 def setUp(self):
323@@ -2669,6 +2674,73 @@ class TestPartitionVerifySfdiskV2(CiTestCase):
324 self.storage_config, self.table)
325
326
327+class TestSfdiskV2(CiTestCase):
328+ def test_gpt_basic(self):
329+ table = block_meta_v2.GPTPartTable(512)
330+ expected = '''\
331+label: gpt
332+'''
333+ self.assertEqual(expected, table.render())
334+
335+ def test_gpt_boot(self):
336+ table = block_meta_v2.GPTPartTable(512)
337+ table.add(dict(number=1, offset=1 << 20, size=9 << 20, flag='boot'))
338+ expected = '''\
339+label: gpt
340+
341+1: start=2048 size=18432 type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B'''
342+ self.assertEqual(expected, table.render())
343+
344+ def test_gpt_boot_raw_type(self):
345+ esp = 'C12A7328-F81F-11D2-BA4B-00A0C93EC93B'
346+ table = block_meta_v2.GPTPartTable(512)
347+ table.add(dict(number=1, offset=1 << 20, size=9 << 20,
348+ partition_type=esp))
349+ expected = '''\
350+label: gpt
351+
352+1: start=2048 size=18432 type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B'''
353+ self.assertEqual(expected, table.render())
354+
355+ def test_gpt_random_uuid(self):
356+ ptype = str(random_uuid()).lower()
357+ table = block_meta_v2.GPTPartTable(512)
358+ table.add(dict(number=1, offset=1 << 20, size=9 << 20,
359+ flag='boot', partition_type=ptype))
360+ expected = f'''\
361+label: gpt
362+
363+1: start=2048 size=18432 type={ptype}'''
364+ self.assertEqual(expected, table.render())
365+
366+ def test_dos_basic(self):
367+ table = block_meta_v2.DOSPartTable(512)
368+ expected = '''\
369+label: dos
370+'''
371+ self.assertEqual(expected, table.render())
372+
373+ def test_dos_boot(self):
374+ table = block_meta_v2.DOSPartTable(512)
375+ table.add(dict(number=1, offset=1 << 20, size=9 << 20, flag='boot'))
376+ expected = '''\
377+label: dos
378+
379+1: start=2048 size=18432 type=EF bootable'''
380+ self.assertEqual(expected, table.render())
381+
382+ def test_dos_random_code(self):
383+ ptype = hex(random.randint(0, 0xff))[2:]
384+ table = block_meta_v2.DOSPartTable(512)
385+ table.add(dict(number=1, offset=1 << 20, size=9 << 20,
386+ flag='boot', partition_type=ptype))
387+ expected = f'''\
388+label: dos
389+
390+1: start=2048 size=18432 type={ptype} bootable'''
391+ self.assertEqual(expected, table.render())
392+
393+
394 class TestPartitionNeedsResize(CiTestCase):
395
396 def setUp(self):
397diff --git a/tests/unittests/test_storage_config.py b/tests/unittests/test_storage_config.py
398index d6b0a36..df48a4d 100644
399--- a/tests/unittests/test_storage_config.py
400+++ b/tests/unittests/test_storage_config.py
401@@ -350,6 +350,7 @@ class TestBlockdevParser(CiTestCase):
402 'offset': 1048576,
403 'size': 499122176,
404 'flag': 'linux',
405+ 'partition_type': '0fc63daf-8483-4772-8e79-3d69d8477de4',
406 }
407 self.assertDictEqual(expected_dict,
408 self.bdevp.asdict(blockdev))
409@@ -394,6 +395,7 @@ class TestBlockdevParser(CiTestCase):
410 'path': '/dev/vda1',
411 'number': 1,
412 'size': 784334848,
413+ 'partition_type': '0x0',
414 }
415 self.assertDictEqual(expected_dict, self.bdevp.asdict(test_value))
416
417@@ -427,7 +429,7 @@ class TestBlockdevParser(CiTestCase):
418 self.probe_data = _get_data('probert_storage_lvm.json')
419 self.bdevp = BlockdevParser(self.probe_data)
420 blockdev = self.bdevp.blockdev_data['/dev/vda2']
421- expected_dict = {
422+ base_expected_dict = {
423 'id': 'partition-vda2',
424 'type': 'partition',
425 'path': '/dev/vda2',
426@@ -439,6 +441,8 @@ class TestBlockdevParser(CiTestCase):
427 }
428 for ext_part_entry in ['0xf', '0x5', '0x85', '0xc5']:
429 blockdev['ID_PART_ENTRY_TYPE'] = ext_part_entry
430+ expected_dict = base_expected_dict.copy()
431+ expected_dict['partition_type'] = ext_part_entry
432 self.assertDictEqual(expected_dict,
433 self.bdevp.asdict(blockdev))
434
435@@ -455,6 +459,7 @@ class TestBlockdevParser(CiTestCase):
436 'offset': 3223322624,
437 'size': 2147483648,
438 'flag': 'logical',
439+ 'partition_type': '0x83',
440 }
441 self.assertDictEqual(expected_dict,
442 self.bdevp.asdict(blockdev))
443@@ -473,6 +478,7 @@ class TestBlockdevParser(CiTestCase):
444 'offset': 1048576,
445 'size': 536870912,
446 'flag': 'boot',
447+ 'partition_type': '0xb',
448 }
449 self.assertDictEqual(expected_dict,
450 self.bdevp.asdict(blockdev))
451@@ -491,6 +497,7 @@ class TestBlockdevParser(CiTestCase):
452 'offset': 3223322624,
453 'size': 2147483648,
454 'flag': 'boot',
455+ 'partition_type': '0x83',
456 }
457 self.assertDictEqual(expected_dict,
458 self.bdevp.asdict(blockdev))
459@@ -551,6 +558,7 @@ class TestBlockdevParser(CiTestCase):
460 'offset': 2097152,
461 'size': 10734272512,
462 'type': 'partition',
463+ 'partition_type': '0fc63daf-8483-4772-8e79-3d69d8477de4',
464 }
465 self.assertDictEqual(expected_dict, self.bdevp.asdict(blockdev))
466
467@@ -1018,11 +1026,17 @@ class TestExtractStorageConfig(CiTestCase):
468 'raidlevel': 'raid1', 'name': 'md1',
469 'devices': ['partition-vdb1', 'partition-vdc1'],
470 'spare_devices': []}, raids[0])
471- self.assertEqual({'id': 'raid-md1p1', 'type': 'partition',
472- 'path': '/dev/md1p1',
473- 'size': 4285530112, 'flag': 'linux', 'number': 1,
474- 'device': 'raid-md1', 'offset': 1048576},
475- raid_partitions[0])
476+ self.assertEqual({
477+ 'id': 'raid-md1p1',
478+ 'type': 'partition',
479+ 'path': '/dev/md1p1',
480+ 'size': 4285530112,
481+ 'flag': 'linux',
482+ 'number': 1,
483+ 'partition_type': '0fc63daf-8483-4772-8e79-3d69d8477de4',
484+ 'device': 'raid-md1',
485+ 'offset': 1048576},
486+ raid_partitions[0])
487
488 @skipUnlessJsonSchema()
489 def test_find_extended_partition(self):

Subscribers

People subscribed via source and target branches