1
diff --git a/curtin/block/clear_holders.py b/curtin/block/clear_holders.py
2
index 4a099cd..bcc16e6 100644
3
--- a/curtin/block/clear_holders.py
4
+++ b/curtin/block/clear_holders.py
5
@@ -166,14 +166,27 @@ def shutdown_mdadm(device):
6
166
166
7
167
    blockdev = block.sysfs_to_devpath(device)
167
    blockdev = block.sysfs_to_devpath(device)
8
168
168
12
169
    LOG.info('Wiping superblock on raid device: %s', device)
169
    LOG.info('Discovering raid devices and spares for %s', device)
10
170
    _wipe_superblock(blockdev, exclusive=False)
11
171
13
172
    md_devs = (
170
    md_devs = (
14
173
        mdadm.md_get_devices_list(blockdev) +
171
        mdadm.md_get_devices_list(blockdev) +
15
174
        mdadm.md_get_spares_list(blockdev))
172
        mdadm.md_get_spares_list(blockdev))
16
175
    mdadm.set_sync_action(blockdev, action="idle")
173
    mdadm.set_sync_action(blockdev, action="idle")
17
176
    mdadm.set_sync_action(blockdev, action="frozen")
174
    mdadm.set_sync_action(blockdev, action="frozen")
18
175
19
176
    LOG.info('Wiping superblock on raid device: %s', device)
20
177
    try:
21
178
        _wipe_superblock(blockdev, exclusive=False)
22
179
    except ValueError as e:
23
180
        # if the array is not functional, writes to the device may fail
24
181
        # and _wipe_superblock will raise ValueError for short writes
25
182
        # which happens on inactive raid volumes.  In that case we
26
183
        # shouldn't give up yet as we still want to disassemble
27
184
        # array and wipe members.  Other errors such as IOError or OSError
28
185
        # are unwelcome and will stop deployment.
29
186
        LOG.debug('Non-fatal error writing to array device %s, '
30
187
                  'proceeding with shutdown: %s', blockdev, e)
31
188
32
189
    LOG.info('Removing raid array members: %s', md_devs)
33
177
    for mddev in md_devs:
190
    for mddev in md_devs:
34
178
        try:
191
        try:
35
179
            mdadm.fail_device(blockdev, mddev)
192
            mdadm.fail_device(blockdev, mddev)
36
@@ -534,8 +547,10 @@ def assert_clear(base_paths):
37
534
    valid = ('disk', 'partition')
547
    valid = ('disk', 'partition')
38
535
    if not isinstance(base_paths, (list, tuple)):
548
    if not isinstance(base_paths, (list, tuple)):
39
536
        base_paths = [base_paths]
549
        base_paths = [base_paths]
42
537
    base_paths = [block.sys_block_path(path) for path in base_paths]
550
    base_paths = [block.sys_block_path(path, strict=False)
43
538
    for holders_tree in [gen_holders_tree(p) for p in base_paths]:
551
                  for path in base_paths]
44
552
    for holders_tree in [gen_holders_tree(p)
45
553
                         for p in base_paths if os.path.exists(p)]:
46
539
        if any(holder_type not in valid and path not in base_paths
554
        if any(holder_type not in valid and path not in base_paths
47
540
               for (holder_type, path) in get_holder_types(holders_tree)):
555
               for (holder_type, path) in get_holder_types(holders_tree)):
48
541
            raise OSError('Storage not clear, remaining:\n{}'
556
            raise OSError('Storage not clear, remaining:\n{}'
49
diff --git a/curtin/commands/block_meta.py b/curtin/commands/block_meta.py
50
index a9110a9..37b93dd 100644
51
--- a/curtin/commands/block_meta.py
52
+++ b/curtin/commands/block_meta.py
53
@@ -67,7 +67,7 @@ def block_meta(args):
54
67
    if devices is None:
67
    if devices is None:
55
68
        devices = []
68
        devices = []
56
69
        if 'storage' in cfg:
69
        if 'storage' in cfg:
58
70
            devices = get_disk_paths_from_storage_config(
70
            devices = get_device_paths_from_storage_config(
59
71
                extract_storage_ordered_dict(cfg))
71
                extract_storage_ordered_dict(cfg))
60
72
        if len(devices) == 0:
72
        if len(devices) == 0:
61
73
            devices = cfg.get('block-meta', {}).get('devices', [])
73
            devices = cfg.get('block-meta', {}).get('devices', [])
62
@@ -1535,20 +1535,24 @@ def zfs_handler(info, storage_config):
63
1535
            util.write_file(state['fstab'], fstab_entry, omode='a')
1535
            util.write_file(state['fstab'], fstab_entry, omode='a')
64
1536
1536
65
1537
1537
69
1538
def get_disk_paths_from_storage_config(storage_config):
1538
def get_device_paths_from_storage_config(storage_config):
70
1539
    """Returns a list of disk paths in a storage config filtering out
1539
    """Returns a list of device paths in a storage config filtering out
71
1540
       preserved or disks which do not have wipe configuration.
1540
       preserved or devices which do not have wipe configuration.
72
1541
1541
73
1542
    :param: storage_config: Ordered dict of storage configation
1542
    :param: storage_config: Ordered dict of storage configation
74
1543
    """
1543
    """
83
1544
    def get_path_if_present(disk, config):
1544
    dpaths = []
84
1545
        return get_path_to_storage_volume(disk, config)
1545
    for (k, v) in storage_config.items():
85
1546
1546
        if v.get('type') in ['disk', 'partition']:
86
1547
    return [get_path_if_present(k, storage_config)
1547
            if config.value_as_boolean(v.get('wipe')):
87
1548
            for (k, v) in storage_config.items()
1548
                if config.value_as_boolean(v.get('preserve')):
88
1549
            if v.get('type') == 'disk' and
1549
                    continue
89
1550
            config.value_as_boolean(v.get('wipe')) and
1550
                try:
90
1551
            not config.value_as_boolean(v.get('preserve'))]
1551
                    dpaths.append(
91
1552
                        get_path_to_storage_volume(k, storage_config))
92
1553
                except Exception:
93
1554
                    pass
94
1555
    return dpaths
95
1552
1556
96
1553
1557
97
1554
def zfsroot_update_storage_config(storage_config):
1558
def zfsroot_update_storage_config(storage_config):
98
diff --git a/examples/tests/reuse-raid-member-partition.yaml b/examples/tests/reuse-raid-member-partition.yaml
99
1555
new file mode 100644
1559
new file mode 100644
100
index 0000000..3fe2d83
101
--- /dev/null
102
+++ b/examples/tests/reuse-raid-member-partition.yaml
103
@@ -0,0 +1,73 @@
104
1
showtrace: true
105
2
106
3
# The point of this test is to test installing to a partition that used to
107
4
# be a RAID member where the other disks that used to be part of the
108
5
# RAID are not present (the scenario that the disk was just grabbed
109
6
# out of a pile of previously used disks and shoved into a server).
110
7
111
8
# So what it does is to create a RAID0 out of two partition from two
112
9
# disks, stop the RAID, wipe one of the disks and then install to the
113
10
# other, reusing the partition that was part of the RAID.
114
11
115
12
bucket:
116
13
  - &setup |
117
14
    parted /dev/disk/by-id/virtio-disk-a --script -- \
118
15
        mklabel gpt              \
119
16
        mkpart primary 1GiB 2GiB \
120
17
        mkpart primary 2GiB 9GiB
121
18
    parted /dev/disk/by-id/virtio-disk-b --script -- \
122
19
        mklabel gpt              \
123
20
        mkpart primary 2GiB 9GiB
124
21
    udevadm settle
125
22
    mdadm --create --metadata 1.2 --level 0 -n 2 /dev/md1 --assume-clean \
126
23
        /dev/disk/by-id/virtio-disk-a-part2 /dev/disk/by-id/virtio-disk-b-part1
127
24
    udevadm settle
128
25
    mdadm --stop /dev/md1
129
26
    udevadm settle
130
27
    mdadm --zero-superblock /dev/disk/by-id/virtio-disk-b-part1
131
28
    wipefs -a /dev/disk/by-id/virtio-disk-b
132
29
    udevadm settle
133
30
134
31
early_commands:
135
32
  00-setup-raid: [sh, -exuc, *setup]
136
33
137
34
storage:
138
35
  config:
139
36
  - type: disk
140
37
    id: id_disk0
141
38
    serial: disk-a
142
39
    ptable: gpt
143
40
    preserve: true
144
41
  - type: disk
145
42
    id: id_disk1
146
43
    serial: disk-b
147
44
  - type: partition
148
45
    id: id_disk0_part1
149
46
    preserve: true
150
47
    device: id_disk0
151
48
    flag: boot
152
49
    number: 1
153
50
    size: 1G
154
51
  - type: partition
155
52
    id: id_disk0_part2
156
53
    preserve: true
157
54
    device: id_disk0
158
55
    number: 2
159
56
    size: 7G
160
57
  - type: format
161
58
    id: id_efi_format
162
59
    volume: id_disk0_part1
163
60
    fstype: fat32
164
61
  - type: format
165
62
    id: id_root_format
166
63
    volume: id_disk0_part2
167
64
    fstype: ext4
168
65
  - type: mount
169
66
    device: id_root_format
170
67
    id: id_root_mount
171
68
    path: /
172
69
  - type: mount
173
70
    id: id_efi_mount
174
71
    device: id_efi_format
175
72
    path: /boot/efi
176
73
  version: 1
177
diff --git a/examples/tests/reuse-raid-member-wipe.yaml b/examples/tests/reuse-raid-member-wipe.yaml
178
0
new file mode 100644
74
new file mode 100644
179
index 0000000..84a2686
180
--- /dev/null
181
+++ b/examples/tests/reuse-raid-member-wipe.yaml
182
@@ -0,0 +1,71 @@
183
1
showtrace: true
184
2
185
3
# The point of this test is to test installing to a disk that contains
186
4
# a partition that used to be a RAID member where the other parts of
187
5
# the RAID are not present (the scenario is that the disk was just
188
6
# grabbed out of a pile of previously used disks and shoved into a
189
7
# server).
190
8
191
9
# So what it does is to create a RAID0 out of partitions on two disks,
192
10
# stop the RAID, wipe the superblock on one of them and then install
193
11
# to the other using a standard partitioning scheme.
194
12
195
13
bucket:
196
14
  - &setup |
197
15
    parted /dev/disk/by-id/virtio-disk-a --script -- \
198
16
        mklabel gpt              \
199
17
        mkpart primary 1GiB 9GiB
200
18
    parted /dev/disk/by-id/virtio-disk-b --script -- \
201
19
        mklabel gpt              \
202
20
        mkpart primary 1GiB 9GiB
203
21
    udevadm settle
204
22
    mdadm --create --metadata 1.2 --level 0 -n 2 /dev/md1 --assume-clean \
205
23
        /dev/disk/by-id/virtio-disk-a-part1 /dev/disk/by-id/virtio-disk-b-part1
206
24
    udevadm settle
207
25
    mdadm --stop /dev/md1
208
26
    udevadm settle
209
27
    mdadm --zero-superblock /dev/disk/by-id/virtio-disk-b-part1
210
28
    wipefs -a /dev/disk/by-id/virtio-disk-b
211
29
    udevadm settle
212
30
213
31
early_commands:
214
32
  00-setup-raid: [sh, -exuc, *setup]
215
33
216
34
storage:
217
35
  config:
218
36
  - type: disk
219
37
    id: id_disk0
220
38
    serial: disk-a
221
39
    ptable: gpt
222
40
    wipe: superblock-recursive
223
41
  - type: disk
224
42
    id: id_disk1
225
43
    serial: disk-b
226
44
  - type: partition
227
45
    id: id_disk0_part1
228
46
    device: id_disk0
229
47
    flag: boot
230
48
    number: 1
231
49
    size: 512M
232
50
  - type: partition
233
51
    id: id_disk0_part2
234
52
    device: id_disk0
235
53
    number: 2
236
54
    size: 8G
237
55
  - type: format
238
56
    id: id_efi_format
239
57
    volume: id_disk0_part1
240
58
    fstype: fat32
241
59
  - type: format
242
60
    id: id_root_format
243
61
    volume: id_disk0_part2
244
62
    fstype: ext4
245
63
  - type: mount
246
64
    device: id_root_format
247
65
    id: id_root_mount
248
66
    path: /
249
67
  - type: mount
250
68
    id: id_efi_mount
251
69
    device: id_efi_format
252
70
    path: /boot/efi
253
71
  version: 1
254
diff --git a/tests/unittests/test_clear_holders.py b/tests/unittests/test_clear_holders.py
255
index e4d8f8d..906ae06 100644
256
--- a/tests/unittests/test_clear_holders.py
257
+++ b/tests/unittests/test_clear_holders.py
258
@@ -566,11 +566,17 @@ class TestClearHolders(CiTestCase):
259
566
        for tree, result in test_trees_and_results:
566
        for tree, result in test_trees_and_results:
260
567
            self.assertEqual(clear_holders.get_holder_types(tree), result)
567
            self.assertEqual(clear_holders.get_holder_types(tree), result)
261
568
568
262
569
    @mock.patch('curtin.block.clear_holders.os.path.exists')
263
569
    @mock.patch('curtin.block.clear_holders.block.sys_block_path')
570
    @mock.patch('curtin.block.clear_holders.block.sys_block_path')
264
570
    @mock.patch('curtin.block.clear_holders.gen_holders_tree')
571
    @mock.patch('curtin.block.clear_holders.gen_holders_tree')
266
571
    def test_assert_clear(self, mock_gen_holders_tree, mock_syspath):
572
    def test_assert_clear(self, mock_gen_holders_tree, mock_syspath, m_ospe):
267
573
        def my_sysblock(p, strict=False):
268
574
            return '/sys/class/block/%s' % os.path.basename(p)
269
575
270
572
        mock_gen_holders_tree.return_value = self.example_holders_trees[0][0]
576
        mock_gen_holders_tree.return_value = self.example_holders_trees[0][0]
272
573
        mock_syspath.side_effect = lambda x: x
577
        mock_syspath.side_effect = my_sysblock
273
578
        # os.path.exists set to True to include all devices in the call list
274
579
        m_ospe.return_value = True
275
574
        device = self.test_blockdev
580
        device = self.test_blockdev
276
575
        with self.assertRaises(OSError):
581
        with self.assertRaises(OSError):
277
576
            clear_holders.assert_clear(device)
582
            clear_holders.assert_clear(device)
278
diff --git a/tests/vmtests/test_reuse_raid_member.py b/tests/vmtests/test_reuse_raid_member.py
279
577
new file mode 100644
583
new file mode 100644
280
index 0000000..3052c9c
281
--- /dev/null
282
+++ b/tests/vmtests/test_reuse_raid_member.py
283
@@ -0,0 +1,62 @@
284
1
# This file is part of curtin. See LICENSE file for copyright and license info.
285
2
286
3
from . import VMBaseClass
287
4
from .releases import base_vm_classes as relbase
288
5
289
6
290
7
class TestReuseRAIDMember(VMBaseClass):
291
8
    """ Curtin can install to a RAID member if other members are missing. """
292
9
    conf_file = "examples/tests/reuse-raid-member-wipe.yaml"
293
10
    extra_disks = ['10G', '10G']
294
11
    uefi = True
295
12
296
13
    def test_simple(self):
297
14
        pass
298
15
299
16
300
17
class TestReuseRAIDMemberPartition(VMBaseClass):
301
18
    """ Curtin can install to a RAID member if other members are missing. """
302
19
    conf_file = "examples/tests/reuse-raid-member-wipe.yaml"
303
20
    extra_disks = ['10G', '10G']
304
21
    uefi = True
305
22
306
23
    def test_simple(self):
307
24
        pass
308
25
309
26
310
27
class BionicTestReuseRAIDMember(relbase.bionic, TestReuseRAIDMember):
311
28
    __test__ = True
312
29
313
30
314
31
class CosmicTestReuseRAIDMember(relbase.cosmic, TestReuseRAIDMember):
315
32
    __test__ = True
316
33
317
34
318
35
class DiscoTestReuseRAIDMember(relbase.disco, TestReuseRAIDMember):
319
36
    __test__ = True
320
37
321
38
322
39
class EoanTestReuseRAIDMember(relbase.eoan, TestReuseRAIDMember):
323
40
    __test__ = True
324
41
325
42
326
43
class BionicTestReuseRAIDMemberPartition(
327
44
        relbase.bionic, TestReuseRAIDMemberPartition):
328
45
    __test__ = True
329
46
330
47
331
48
class CosmicTestReuseRAIDMemberPartition(
332
49
        relbase.cosmic, TestReuseRAIDMemberPartition):
333
50
    __test__ = True
334
51
335
52
336
53
class DiscoTestReuseRAIDMemberPartition(
337
54
        relbase.disco, TestReuseRAIDMemberPartition):
338
55
    __test__ = True
339
56
340
57
341
58
class EoanTestReuseRAIDMemberPartition(
342
59
        relbase.eoan, TestReuseRAIDMemberPartition):
343
60
    __test__ = True
344
61
345
62
# vi: ts=4 expandtab syntax=python
Reviewer	Review Type	Date Requested	Status
Server Team CI bot	continuous-integration		Approve on 2019-09-06
Dan Watkins (community)		2019-07-31	Approve on 2019-08-02
Review via email: mp+370843@code.launchpad.net