Merge ~smoser/curtin:ubuntu/devel into curtin:ubuntu/devel

Proposed by Scott Moser on 2018-08-17
Status: Rejected
Rejected by: Scott Moser on 2018-08-18
Proposed branch: ~smoser/curtin:ubuntu/devel
Merge into: curtin:ubuntu/devel
Diff against target: 772 lines (+412/-43)
19 files modified
curtin/block/clear_holders.py (+3/-0)
curtin/block/lvm.py (+23/-5)
curtin/block/mdadm.py (+2/-3)
curtin/commands/block_meta.py (+2/-1)
curtin/commands/install.py (+2/-1)
curtin/log.py (+43/-0)
curtin/udev.py (+2/-0)
curtin/util.py (+33/-8)
debian/changelog (+10/-0)
examples/tests/dirty_disks_config.yaml (+30/-3)
examples/tests/lvmoverraid.yaml (+98/-0)
examples/tests/vmtest_defaults.yaml (+14/-0)
tests/unittests/test_block_lvm.py (+13/-13)
tests/unittests/test_block_mdadm.py (+4/-5)
tests/unittests/test_clear_holders.py (+5/-2)
tests/unittests/test_util.py (+62/-0)
tests/vmtests/__init__.py (+15/-1)
tests/vmtests/test_lvm_raid.py (+50/-0)
tests/vmtests/test_lvm_root.py (+1/-1)
Reviewer Review Type Date Requested Status
Chad Smith 2018-08-17 Disapprove on 2018-08-18
Server Team CI bot continuous-integration Approve on 2018-08-17
Review via email: mp+353354@code.launchpad.net

Commit message

update changelog (New upstream snapshot 18.1-48-g6a776e15).

Description of the change

see commit message

To post a comment you must log in.
Chad Smith (chad.smith) wrote :

rejected as ryan had a zfs branch that landed after this.
Just pulled in https://code.launchpad.net/~chad.smith/curtin/+git/curtin/+merge/353361 instead

review: Disapprove
Scott Moser (smoser) wrote :

going with chad's with one more upstream commit at
 https://code.launchpad.net/~chad.smith/curtin/+git/curtin/+merge/353361

Unmerged commits

cecdb41... by Scott Moser on 2018-08-17

update changelog (New upstream snapshot 18.1-48-g6a776e15).

cb7fc8d... by Scott Moser on 2018-08-17

merge from master at 18.1-48-g6a776e15

6a776e1... by Ryan Harper on 2018-08-17

clear-holders: rescan for lvm devices after assembling raid arrays

Lvm devices to be found after assembling raid arrays. Add a call after
lvm_scan to activate any discovered vgs and lvs.

LP: #1783413

40d682c... by Ryan Harper on 2018-08-17

vmtest: enable persistent journal and collect at boot time

Target OSes which use systemd-journald may or maynot enable a
persistent journal on the root filesystem. If the target release
supports journald, issue the commands to create a persistent
journal. During the boot phase, issue commands to flush the
journal and collect (and compress) the journal as the last
script before tar'ing the collect data.

Rename vmtest_pollinate.yaml to vmtest_defaults.yaml. This
file is included in all vmtest runs.

1b387fb... by Scott Moser on 2018-08-17

Add timing and logging functions.

This adds some decorators and functions to easily time function call.
It was originally done to get an idea on how much time we spend waiting
for udevadm settle, but the code is generally useful.

Note that the 'cmd_install' timer unfortunately doesn't make it
into the saved-off log as the message is appended after it is copied.

1a201e5... by Scott Moser on 2018-08-13

parse_dpkg_version: support non-numeric in version string.

This fixes parse_dpkg_version for packages with non-numeric (0-9.)
in their versions. It also improves it to work for native packages.
Native packages do not have '-'. Also adds tests of parse_dpkg_version.

LP: #1786795

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/curtin/block/clear_holders.py b/curtin/block/clear_holders.py
2index 9d73b28..a2042d5 100644
3--- a/curtin/block/clear_holders.py
4+++ b/curtin/block/clear_holders.py
5@@ -624,6 +624,9 @@ def start_clear_holders_deps():
6 # all disks and partitions should be sufficient to remove the mdadm
7 # metadata
8 mdadm.mdadm_assemble(scan=True, ignore_errors=True)
9+ # scan and activate for logical volumes
10+ lvm.lvm_scan()
11+ lvm.activate_volgroups()
12 # the bcache module needs to be present to properly detect bcache devs
13 # on some systems (precise without hwe kernel) it may not be possible to
14 # lad the bcache module bcause it is not present in the kernel. if this
15diff --git a/curtin/block/lvm.py b/curtin/block/lvm.py
16index 8643245..eca64f6 100644
17--- a/curtin/block/lvm.py
18+++ b/curtin/block/lvm.py
19@@ -57,14 +57,32 @@ def lvmetad_running():
20 '/run/lvmetad.pid'))
21
22
23-def lvm_scan():
24+def activate_volgroups():
25+ """
26+ Activate available volgroups and logical volumes within.
27+
28+ # found
29+ % vgchange -ay
30+ 1 logical volume(s) in volume group "vg1sdd" now active
31+
32+ # none found (no output)
33+ % vgchange -ay
34+ """
35+
36+ # vgchange handles syncing with udev by default
37+ # see man 8 vgchange and flag --noudevsync
38+ out, _ = util.subp(['vgchange', '--activate=y'], capture=True)
39+ if out:
40+ LOG.info(out)
41+
42+
43+def lvm_scan(activate=True):
44 """
45 run full scan for volgroups, logical volumes and physical volumes
46 """
47- # the lvm tools lvscan, vgscan and pvscan on ubuntu precise do not
48- # support the flag --cache. the flag is present for the tools in ubuntu
49- # trusty and later. since lvmetad is used in current releases of
50- # ubuntu, the --cache flag is needed to ensure that the data cached by
51+ # prior to xenial, lvmetad is not packaged, so even if a tool supports
52+ # flag --cache it has no effect. In Xenial and newer the --cache flag is
53+ # used (if lvmetad is running) to ensure that the data cached by
54 # lvmetad is updated.
55
56 # before appending the cache flag though, check if lvmetad is running. this
57diff --git a/curtin/block/mdadm.py b/curtin/block/mdadm.py
58index e0fe0d3..8eff7fb 100644
59--- a/curtin/block/mdadm.py
60+++ b/curtin/block/mdadm.py
61@@ -184,7 +184,7 @@ def mdadm_create(md_devname, raidlevel, devices, spares=None, md_name=""):
62 cmd.append(device)
63
64 # Create the raid device
65- util.subp(["udevadm", "settle"])
66+ udev.udevadm_settle()
67 util.subp(["udevadm", "control", "--stop-exec-queue"])
68 try:
69 util.subp(cmd, capture=True)
70@@ -208,8 +208,7 @@ def mdadm_create(md_devname, raidlevel, devices, spares=None, md_name=""):
71 raise
72
73 util.subp(["udevadm", "control", "--start-exec-queue"])
74- util.subp(["udevadm", "settle",
75- "--exit-if-exists=%s" % md_devname])
76+ udev.udevadm_settle(exists=md_devname)
77
78
79 def mdadm_examine(devpath, export=MDADM_USE_EXPORT):
80diff --git a/curtin/commands/block_meta.py b/curtin/commands/block_meta.py
81index f5b82cf..63193f5 100644
82--- a/curtin/commands/block_meta.py
83+++ b/curtin/commands/block_meta.py
84@@ -3,7 +3,7 @@
85 from collections import OrderedDict, namedtuple
86 from curtin import (block, config, util)
87 from curtin.block import (bcache, mdadm, mkfs, clear_holders, lvm, iscsi, zfs)
88-from curtin.log import LOG
89+from curtin.log import LOG, logged_time
90 from curtin.reporter import events
91
92 from . import populate_one_subcmd
93@@ -48,6 +48,7 @@ CMD_ARGUMENTS = (
94 )
95
96
97+@logged_time("BLOCK_META")
98 def block_meta(args):
99 # main entry point for the block-meta command.
100 state = util.load_command_environment()
101diff --git a/curtin/commands/install.py b/curtin/commands/install.py
102index 9e5406c..4d2a13f 100644
103--- a/curtin/commands/install.py
104+++ b/curtin/commands/install.py
105@@ -15,7 +15,7 @@ from curtin.block import iscsi
106 from curtin import config
107 from curtin import util
108 from curtin import version
109-from curtin.log import LOG
110+from curtin.log import LOG, logged_time
111 from curtin.reporter.legacy import load_reporter
112 from curtin.reporter import events
113 from . import populate_one_subcmd
114@@ -390,6 +390,7 @@ def migrate_proxy_settings(cfg):
115 cfg['proxy'] = proxy
116
117
118+@logged_time("INSTALL_COMMAND")
119 def cmd_install(args):
120 from .collect_logs import create_log_tarfile
121 cfg = deepcopy(CONFIG_BUILTIN)
122diff --git a/curtin/log.py b/curtin/log.py
123index 4844460..446ba2c 100644
124--- a/curtin/log.py
125+++ b/curtin/log.py
126@@ -1,6 +1,9 @@
127 # This file is part of curtin. See LICENSE file for copyright and license info.
128
129 import logging
130+import time
131+
132+from functools import wraps
133
134 # Logging items for easy access
135 getLogger = logging.getLogger
136@@ -56,6 +59,46 @@ def _getLogger(name='curtin'):
137 if not logging.getLogger().handlers:
138 logging.getLogger().addHandler(NullHandler())
139
140+
141+def _repr_call(name, *args, **kwargs):
142+ return "%s(%s)" % (
143+ name,
144+ ', '.join([str(repr(a)) for a in args] +
145+ ["%s=%s" % (k, repr(v)) for k, v in kwargs.items()]))
146+
147+
148+def log_call(func, *args, **kwargs):
149+ return log_time(
150+ "TIMED %s: " % _repr_call(func.__name__, *args, **kwargs),
151+ func, *args, **kwargs)
152+
153+
154+def log_time(msg, func, *args, **kwargs):
155+ start = time.time()
156+ try:
157+ return func(*args, **kwargs)
158+ finally:
159+ LOG.debug(msg + "%.3f", (time.time() - start))
160+
161+
162+def logged_call():
163+ def decorator(func):
164+ @wraps(func)
165+ def wrapper(*args, **kwargs):
166+ return log_call(func, *args, **kwargs)
167+ return wrapper
168+ return decorator
169+
170+
171+def logged_time(msg):
172+ def decorator(func):
173+ @wraps(func)
174+ def wrapper(*args, **kwargs):
175+ return log_time("TIMED %s: " % msg, func, *args, **kwargs)
176+ return wrapper
177+ return decorator
178+
179+
180 LOG = _getLogger()
181
182 # vi: ts=4 expandtab syntax=python
183diff --git a/curtin/udev.py b/curtin/udev.py
184index 92e38ff..13d9cc5 100644
185--- a/curtin/udev.py
186+++ b/curtin/udev.py
187@@ -2,6 +2,7 @@
188
189 import os
190 from curtin import util
191+from curtin.log import logged_call
192
193
194 def compose_udev_equality(key, value):
195@@ -40,6 +41,7 @@ def generate_udev_rule(interface, mac):
196 return '%s\n' % rule
197
198
199+@logged_call()
200 def udevadm_settle(exists=None, timeout=None):
201 settle_cmd = ["udevadm", "settle"]
202 if exists:
203diff --git a/curtin/util.py b/curtin/util.py
204index 7d06c09..29bf06e 100644
205--- a/curtin/util.py
206+++ b/curtin/util.py
207@@ -38,7 +38,7 @@ except NameError:
208 # python3 does not have a long type.
209 numeric_types = (int, float)
210
211-from .log import LOG
212+from .log import LOG, log_call
213
214 _INSTALLED_HELPERS_PATH = 'usr/lib/curtin/helpers'
215 _INSTALLED_MAIN = 'usr/bin/curtin'
216@@ -661,7 +661,7 @@ class ChrootableTarget(object):
217
218 # if /dev is to be unmounted, udevadm settle (LP: #1462139)
219 if target_path(self.target, "/dev") in self.umounts:
220- subp(['udevadm', 'settle'])
221+ log_call(subp, ['udevadm', 'settle'])
222
223 for p in reversed(self.umounts):
224 do_umount(p)
225@@ -810,13 +810,37 @@ def parse_dpkg_version(raw, name=None, semx=None):
226 """Parse a dpkg version string into various parts and calcualate a
227 numerical value of the version for use in comparing package versions
228
229- returns a dictionary with the results
230+ Native packages (without a '-'), will have the package version treated
231+ as the upstream version.
232+
233+ returns a dictionary with fields:
234+ 'major' (int), 'minor' (int), 'micro' (int),
235+ 'semantic_version' (int),
236+ 'extra' (string), 'raw' (string), 'upstream' (string),
237+ 'name' (present only if name is not None)
238 """
239+ if not isinstance(raw, string_types):
240+ raise TypeError(
241+ "Invalid type %s for parse_dpkg_version" % raw.__class__)
242+
243 if semx is None:
244 semx = (10000, 100, 1)
245
246- upstream = raw.split('-')[0]
247- toks = upstream.split(".", 2)
248+ if "-" in raw:
249+ upstream = raw.rsplit('-', 1)[0]
250+ else:
251+ # this is a native package, package version treated as upstream.
252+ upstream = raw
253+
254+ match = re.search(r'[^0-9.]', upstream)
255+ if match:
256+ extra = upstream[match.start():]
257+ upstream_base = upstream[:match.start()]
258+ else:
259+ upstream_base = upstream
260+ extra = None
261+
262+ toks = upstream_base.split(".", 2)
263 if len(toks) == 3:
264 major, minor, micro = toks
265 elif len(toks) == 2:
266@@ -825,9 +849,10 @@ def parse_dpkg_version(raw, name=None, semx=None):
267 major, minor, micro = (toks[0], 0, 0)
268
269 version = {
270- 'major': major,
271- 'minor': minor,
272- 'micro': micro,
273+ 'major': int(major),
274+ 'minor': int(minor),
275+ 'micro': int(micro),
276+ 'extra': extra,
277 'raw': raw,
278 'upstream': upstream,
279 }
280diff --git a/debian/changelog b/debian/changelog
281index 8f013e6..31c9a7c 100644
282--- a/debian/changelog
283+++ b/debian/changelog
284@@ -1,3 +1,13 @@
285+curtin (18.1-48-g6a776e15-0ubuntu1) UNRELEASED; urgency=medium
286+
287+ * New upstream snapshot.
288+ - clear-holders: rescan for lvm devices after assembling raid arrays
289+ - vmtest: enable persistent journal and collect at boot time
290+ - Add timing and logging functions.
291+ - parse_dpkg_version: support non-numeric in version string.
292+
293+ -- Scott Moser <smoser@ubuntu.com> Fri, 17 Aug 2018 17:38:35 -0400
294+
295 curtin (18.1-44-g2b12b8fc-0ubuntu1) cosmic; urgency=medium
296
297 * New upstream snapshot.
298diff --git a/examples/tests/dirty_disks_config.yaml b/examples/tests/dirty_disks_config.yaml
299index 75d44c3..fb9a0d6 100644
300--- a/examples/tests/dirty_disks_config.yaml
301+++ b/examples/tests/dirty_disks_config.yaml
302@@ -27,6 +27,31 @@ bucket:
303 # disable any rpools to trigger disks with zfs_member label but inactive
304 # pools
305 zpool export rpool ||:
306+ - &lvm_stop |
307+ #!/bin/sh
308+ # This function disables any existing lvm logical volumes that
309+ # have been created during the early storage config stage
310+ # and simulates the effect of booting into a system with existing
311+ # (but inactive) lvm configuration.
312+ for vg in `pvdisplay -C --separator = -o vg_name --noheadings`; do
313+ vgchange -an $vg ||:
314+ done
315+ # disable the automatic pvscan, we want to test that curtin
316+ # can find/enable logical volumes without this service
317+ command -v systemctl && systemctl mask lvm2-pvscan\@.service
318+ # remove any existing metadata written from early disk config
319+ rm -rf /etc/lvm/archive /etc/lvm/backup
320+ - &mdadm_stop |
321+ #!/bin/sh
322+ # This function disables any existing raid devices which may
323+ # have been created during the early storage config stage
324+ # and simulates the effect of booting into a system with existing
325+ # but inactive mdadm configuration.
326+ for md in /dev/md*; do
327+ mdadm --stop $md ||:
328+ done
329+ # remove any existing metadata written from early disk config
330+ rm -f /etc/mdadm/mdadm.conf
331
332 early_commands:
333 # running block-meta custom from the install environment
334@@ -34,9 +59,11 @@ early_commands:
335 # the disks exactly as in this config before the rest of the install
336 # will just blow it all away. We have clean out other environment
337 # that could unintentionally mess things up.
338- blockmeta: [env, -u, OUTPUT_FSTAB,
339+ 01-blockmeta: [env, -u, OUTPUT_FSTAB,
340 TARGET_MOUNT_POINT=/tmp/my.bdir/target,
341 WORKING_DIR=/tmp/my.bdir/work.d,
342 curtin, --showtrace, -v, block-meta, --umount, custom]
343- enable_swaps: [sh, -c, *swapon]
344- disable_rpool: [sh, -c, *zpool_export]
345+ 02-enable_swaps: [sh, -c, *swapon]
346+ 03-disable_rpool: [sh, -c, *zpool_export]
347+ 04-lvm_stop: [sh, -c, *lvm_stop]
348+ 05-mdadm_stop: [sh, -c, *mdadm_stop]
349diff --git a/examples/tests/lvmoverraid.yaml b/examples/tests/lvmoverraid.yaml
350new file mode 100644
351index 0000000..a1d41e9
352--- /dev/null
353+++ b/examples/tests/lvmoverraid.yaml
354@@ -0,0 +1,98 @@
355+storage:
356+ config:
357+ - grub_device: true
358+ id: disk-0
359+ model: QEMU_HARDDISK
360+ name: 'main_disk'
361+ serial: disk-a
362+ preserve: false
363+ ptable: gpt
364+ type: disk
365+ wipe: superblock
366+ - grub_device: false
367+ id: disk-2
368+ name: 'disk-2'
369+ serial: disk-b
370+ preserve: false
371+ type: disk
372+ wipe: superblock
373+ - grub_device: false
374+ id: disk-1
375+ name: 'disk-1'
376+ serial: disk-c
377+ preserve: false
378+ type: disk
379+ wipe: superblock
380+ - grub_device: false
381+ id: disk-3
382+ name: 'disk-3'
383+ serial: disk-d
384+ preserve: false
385+ type: disk
386+ wipe: superblock
387+ - grub_device: false
388+ id: disk-4
389+ name: 'disk-4'
390+ serial: disk-e
391+ preserve: false
392+ type: disk
393+ wipe: superblock
394+ - device: disk-0
395+ flag: bios_grub
396+ id: part-0
397+ preserve: false
398+ size: 1048576
399+ type: partition
400+ - device: disk-0
401+ flag: ''
402+ id: part-1
403+ preserve: false
404+ size: 4G
405+ type: partition
406+ - devices:
407+ - disk-2
408+ - disk-1
409+ id: raid-0
410+ name: md0
411+ raidlevel: 1
412+ spare_devices: []
413+ type: raid
414+ - devices:
415+ - disk-3
416+ - disk-4
417+ id: raid-1
418+ name: md1
419+ raidlevel: 1
420+ spare_devices: []
421+ type: raid
422+ - devices:
423+ - raid-0
424+ - raid-1
425+ id: vg-0
426+ name: vg0
427+ type: lvm_volgroup
428+ - id: lv-0
429+ name: lv-0
430+ size: 3G
431+ type: lvm_partition
432+ volgroup: vg-0
433+ - fstype: ext4
434+ id: fs-0
435+ preserve: false
436+ type: format
437+ volume: part-1
438+ - fstype: ext4
439+ id: fs-1
440+ preserve: false
441+ type: format
442+ volume: lv-0
443+ - device: fs-0
444+ id: mount-0
445+ path: /
446+ type: mount
447+ - device: fs-1
448+ id: mount-1
449+ path: /home
450+ type: mount
451+ version: 1
452+
453diff --git a/examples/tests/vmtest_pollinate.yaml b/examples/tests/vmtest_defaults.yaml
454index e4fac06..b1512a8 100644
455--- a/examples/tests/vmtest_pollinate.yaml
456+++ b/examples/tests/vmtest_defaults.yaml
457@@ -6,5 +6,19 @@ _vmtest_pollinate:
458 [ -d "${cfg%/*}" ] || exit 0
459 echo curtin/vmtest >> "$cfg"
460
461+# this enables a persitent journald if target system has journald
462+# and does not have /var/log/journal directory already
463+_persist_journal:
464+ - &persist_journal |
465+ command -v journalctl && {
466+ jdir=/var/log/journal
467+ [ -e ${jdir} ] || {
468+ mkdir -p ${jdir}
469+ systemd-tmpfiles --create --prefix ${jdir}
470+ }
471+ }
472+ exit 0
473+
474 late_commands:
475 01_vmtest_pollinate: ['curtin', 'in-target', '--', 'sh', '-c', *pvmtest]
476+ 02_persist_journal: ['curtin', 'in-target', '--', 'sh', '-c', *persist_journal]
477diff --git a/tests/unittests/test_block_lvm.py b/tests/unittests/test_block_lvm.py
478index 341f2fa..22fb064 100644
479--- a/tests/unittests/test_block_lvm.py
480+++ b/tests/unittests/test_block_lvm.py
481@@ -75,24 +75,24 @@ class TestBlockLvm(CiTestCase):
482 @mock.patch('curtin.block.lvm.util')
483 def test_lvm_scan(self, mock_util, mock_lvmetad):
484 """check that lvm_scan formats commands correctly for each release"""
485+ cmds = [['pvscan'], ['vgscan', '--mknodes']]
486 for (count, (codename, lvmetad_status, use_cache)) in enumerate(
487- [('precise', False, False), ('precise', True, False),
488- ('trusty', False, False), ('trusty', True, True),
489- ('vivid', False, False), ('vivid', True, True),
490- ('wily', False, False), ('wily', True, True),
491+ [('precise', False, False),
492+ ('trusty', False, False),
493 ('xenial', False, False), ('xenial', True, True),
494- ('yakkety', True, True), ('UNAVAILABLE', True, True),
495 (None, True, True), (None, False, False)]):
496 mock_util.lsb_release.return_value = {'codename': codename}
497 mock_lvmetad.return_value = lvmetad_status
498 lvm.lvm_scan()
499- self.assertEqual(
500- len(mock_util.subp.call_args_list), 2 * (count + 1))
501- for (expected, actual) in zip(
502- [['pvscan'], ['vgscan', '--mknodes']],
503- mock_util.subp.call_args_list[2 * count:2 * count + 2]):
504- if use_cache:
505- expected.append('--cache')
506- self.assertEqual(mock.call(expected, capture=True), actual)
507+ expected = [cmd for cmd in cmds]
508+ for cmd in expected:
509+ if lvmetad_status:
510+ cmd.append('--cache')
511+
512+ calls = [mock.call(cmd, capture=True) for cmd in expected]
513+ self.assertEqual(len(expected), len(mock_util.subp.call_args_list))
514+ mock_util.subp.has_calls(calls)
515+ mock_util.subp.reset_mock()
516+
517
518 # vi: ts=4 expandtab syntax=python
519diff --git a/tests/unittests/test_block_mdadm.py b/tests/unittests/test_block_mdadm.py
520index e2e109c..341e49d 100644
521--- a/tests/unittests/test_block_mdadm.py
522+++ b/tests/unittests/test_block_mdadm.py
523@@ -90,6 +90,8 @@ class TestBlockMdadmCreate(CiTestCase):
524 self.add_patch('curtin.block.mdadm.util', 'mock_util')
525 self.add_patch('curtin.block.mdadm.is_valid_device', 'mock_valid')
526 self.add_patch('curtin.block.mdadm.get_holders', 'mock_holders')
527+ self.add_patch('curtin.block.mdadm.udev.udevadm_settle',
528+ 'm_udevadm_settle')
529
530 # Common mock settings
531 self.mock_valid.return_value = True
532@@ -115,8 +117,6 @@ class TestBlockMdadmCreate(CiTestCase):
533 expected_calls.append(
534 call(["mdadm", "--zero-superblock", d], capture=True))
535
536- side_effects.append(("", "")) # udevadm settle
537- expected_calls.append(call(["udevadm", "settle"]))
538 side_effects.append(("", "")) # udevadm control --stop-exec-queue
539 expected_calls.append(call(["udevadm", "control",
540 "--stop-exec-queue"]))
541@@ -134,9 +134,6 @@ class TestBlockMdadmCreate(CiTestCase):
542 side_effects.append(("", "")) # udevadm control --start-exec-queue
543 expected_calls.append(call(["udevadm", "control",
544 "--start-exec-queue"]))
545- side_effects.append(("", "")) # udevadm settle
546- expected_calls.append(call(["udevadm", "settle",
547- "--exit-if-exists=%s" % md_devname]))
548
549 return (side_effects, expected_calls)
550
551@@ -154,6 +151,8 @@ class TestBlockMdadmCreate(CiTestCase):
552 mdadm.mdadm_create(md_devname=md_devname, raidlevel=raidlevel,
553 devices=devices, spares=spares)
554 self.mock_util.subp.assert_has_calls(expected_calls)
555+ self.m_udevadm_settle.assert_has_calls(
556+ [call(), call(exists=md_devname)])
557
558 def test_mdadm_create_raid0_devshort(self):
559 md_devname = "md0"
560diff --git a/tests/unittests/test_clear_holders.py b/tests/unittests/test_clear_holders.py
561index 6c29171..21f76be 100644
562--- a/tests/unittests/test_clear_holders.py
563+++ b/tests/unittests/test_clear_holders.py
564@@ -779,10 +779,12 @@ class TestClearHolders(CiTestCase):
565 mock_gen_holders_tree.return_value = self.example_holders_trees[1][1]
566 clear_holders.assert_clear(device)
567
568+ @mock.patch('curtin.block.clear_holders.lvm')
569 @mock.patch('curtin.block.clear_holders.zfs')
570 @mock.patch('curtin.block.clear_holders.mdadm')
571 @mock.patch('curtin.block.clear_holders.util')
572- def test_start_clear_holders_deps(self, mock_util, mock_mdadm, mock_zfs):
573+ def test_start_clear_holders_deps(self, mock_util, mock_mdadm, mock_zfs,
574+ mock_lvm):
575 mock_zfs.zfs_supported.return_value = True
576 clear_holders.start_clear_holders_deps()
577 mock_mdadm.mdadm_assemble.assert_called_with(
578@@ -790,11 +792,12 @@ class TestClearHolders(CiTestCase):
579 mock_util.load_kernel_module.assert_has_calls([
580 mock.call('bcache'), mock.call('zfs')])
581
582+ @mock.patch('curtin.block.clear_holders.lvm')
583 @mock.patch('curtin.block.clear_holders.zfs')
584 @mock.patch('curtin.block.clear_holders.mdadm')
585 @mock.patch('curtin.block.clear_holders.util')
586 def test_start_clear_holders_deps_nozfs(self, mock_util, mock_mdadm,
587- mock_zfs):
588+ mock_zfs, mock_lvm):
589 """test that we skip zfs modprobe on unsupported platforms"""
590 mock_zfs.zfs_supported.return_value = False
591 clear_holders.start_clear_holders_deps()
592diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
593index 483cd5d..7fb332d 100644
594--- a/tests/unittests/test_util.py
595+++ b/tests/unittests/test_util.py
596@@ -4,6 +4,7 @@ from unittest import skipIf
597 import mock
598 import os
599 import stat
600+import sys
601 from textwrap import dedent
602
603 from curtin import util
604@@ -1035,4 +1036,65 @@ class TestLoadKernelModule(CiTestCase):
605 self.assertEqual(0, self.m_subp.call_count)
606
607
608+class TestParseDpkgVersion(CiTestCase):
609+ """test parse_dpkg_version."""
610+
611+ def test_none_raises_type_error(self):
612+ self.assertRaises(TypeError, util.parse_dpkg_version, None)
613+
614+ @skipIf(sys.version_info.major < 3, "python 2 bytes are strings.")
615+ def test_bytes_raises_type_error(self):
616+ self.assertRaises(TypeError, util.parse_dpkg_version, b'1.2.3-0')
617+
618+ def test_simple_native_package_version(self):
619+ """dpkg versions must have a -. If not present expect value error."""
620+ self.assertEqual(
621+ {'major': 2, 'minor': 28, 'micro': 0, 'extra': None,
622+ 'raw': '2.28', 'upstream': '2.28', 'name': 'germinate',
623+ 'semantic_version': 22800},
624+ util.parse_dpkg_version('2.28', name='germinate'))
625+
626+ def test_complex_native_package_version(self):
627+ dver = '1.0.106ubuntu2+really1.0.97ubuntu1'
628+ self.assertEqual(
629+ {'major': 1, 'minor': 0, 'micro': 106,
630+ 'extra': 'ubuntu2+really1.0.97ubuntu1',
631+ 'raw': dver, 'upstream': dver, 'name': 'debootstrap',
632+ 'semantic_version': 100106},
633+ util.parse_dpkg_version(dver, name='debootstrap',
634+ semx=(100000, 1000, 1)))
635+
636+ def test_simple_valid(self):
637+ self.assertEqual(
638+ {'major': 1, 'minor': 2, 'micro': 3, 'extra': None,
639+ 'raw': '1.2.3-0', 'upstream': '1.2.3', 'name': 'foo',
640+ 'semantic_version': 10203},
641+ util.parse_dpkg_version('1.2.3-0', name='foo'))
642+
643+ def test_simple_valid_with_semx(self):
644+ self.assertEqual(
645+ {'major': 1, 'minor': 2, 'micro': 3, 'extra': None,
646+ 'raw': '1.2.3-0', 'upstream': '1.2.3',
647+ 'semantic_version': 123},
648+ util.parse_dpkg_version('1.2.3-0', semx=(100, 10, 1)))
649+
650+ def test_upstream_with_hyphen(self):
651+ """upstream versions may have a hyphen."""
652+ cver = '18.2-14-g6d48d265-0ubuntu1'
653+ self.assertEqual(
654+ {'major': 18, 'minor': 2, 'micro': 0, 'extra': '-14-g6d48d265',
655+ 'raw': cver, 'upstream': '18.2-14-g6d48d265',
656+ 'name': 'cloud-init', 'semantic_version': 180200},
657+ util.parse_dpkg_version(cver, name='cloud-init'))
658+
659+ def test_upstream_with_plus(self):
660+ """multipath tools has a + in it."""
661+ mver = '0.5.0+git1.656f8865-5ubuntu2.5'
662+ self.assertEqual(
663+ {'major': 0, 'minor': 5, 'micro': 0, 'extra': '+git1.656f8865',
664+ 'raw': mver, 'upstream': '0.5.0+git1.656f8865',
665+ 'semantic_version': 500},
666+ util.parse_dpkg_version(mver))
667+
668+
669 # vi: ts=4 expandtab syntax=python
670diff --git a/tests/vmtests/__init__.py b/tests/vmtests/__init__.py
671index 68b7442..0249655 100644
672--- a/tests/vmtests/__init__.py
673+++ b/tests/vmtests/__init__.py
674@@ -916,8 +916,9 @@ class VMBaseClass(TestCase):
675 # build iscsi disk args if needed
676 disks.extend(cls.build_iscsi_disks())
677
678+ # class config file and vmtest defaults
679+ configs = [cls.conf_file, 'examples/tests/vmtest_defaults.yaml']
680 # proxy config
681- configs = [cls.conf_file, 'examples/tests/vmtest_pollinate.yaml']
682 cls.proxy = get_apt_proxy()
683 if cls.proxy is not None and not cls.td.restored:
684 proxy_config = os.path.join(cls.td.install, 'proxy.cfg')
685@@ -1800,6 +1801,19 @@ def generate_user_data(collect_scripts=None, apt_proxy=None,
686 exit 0;
687 """)
688
689+ # add journal collection "last" before collect_post
690+ collect_journal = textwrap.dedent("""#!/bin/sh -x
691+ cd OUTPUT_COLLECT_D
692+ # sync and flush journal before copying (if journald enabled)
693+ [ -e /var/log/journal ] && {
694+ journalctl --sync --flush --rotate
695+ cp -a /var/log/journal ./var-log-journal
696+ gzip -9 ./var-log-journal/*/system*.journal
697+ }
698+ exit 0;
699+ """)
700+ collect_scripts.append(collect_journal)
701+
702 scripts = ([collect_prep] + [copy_rootdir] + collect_scripts +
703 [collect_post] + [failsafe_poweroff])
704
705diff --git a/tests/vmtests/test_lvm_raid.py b/tests/vmtests/test_lvm_raid.py
706new file mode 100644
707index 0000000..0c50941
708--- /dev/null
709+++ b/tests/vmtests/test_lvm_raid.py
710@@ -0,0 +1,50 @@
711+# This file is part of curtin. See LICENSE file for copyright and license info.
712+
713+from .releases import base_vm_classes as relbase
714+from .test_mdadm_bcache import TestMdadmAbs
715+from .test_lvm import TestLvmAbs
716+
717+import textwrap
718+
719+
720+class TestLvmOverRaidAbs(TestMdadmAbs, TestLvmAbs):
721+ conf_file = "examples/tests/lvmoverraid.yaml"
722+ active_mdadm = "2"
723+ nr_cpus = 2
724+ dirty_disks = True
725+ extra_disks = ['10G'] * 4
726+
727+ collect_scripts = TestLvmAbs.collect_scripts
728+ collect_scripts += TestMdadmAbs.collect_scripts + [textwrap.dedent("""
729+ cd OUTPUT_COLLECT_D
730+ ls -al /dev/md* > dev_md
731+ cp -a /etc/mdadm etc_mdadm
732+ cp -a /etc/lvm etc_lvm
733+ """)]
734+
735+ fstab_expected = {
736+ '/dev/vg1/lv1': '/srv/data',
737+ '/dev/vg1/lv2': '/srv/backup',
738+ }
739+ disk_to_check = [('main_disk', 1),
740+ ('md0', 0),
741+ ('md1', 0)]
742+
743+ def test_lvs(self):
744+ self.check_file_strippedline("lvs", "lv-0=vg0")
745+
746+ def test_pvs(self):
747+ self.check_file_strippedline("pvs", "vg0=/dev/md0")
748+ self.check_file_strippedline("pvs", "vg0=/dev/md1")
749+
750+
751+class CosmicTestLvmOverRaid(relbase.cosmic, TestLvmOverRaidAbs):
752+ __test__ = True
753+
754+
755+class BionicTestLvmOverRaid(relbase.bionic, TestLvmOverRaidAbs):
756+ __test__ = True
757+
758+
759+class XenialGATestLvmOverRaid(relbase.xenial_ga, TestLvmOverRaidAbs):
760+ __test__ = True
761diff --git a/tests/vmtests/test_lvm_root.py b/tests/vmtests/test_lvm_root.py
762index 8ca69d4..bc8b047 100644
763--- a/tests/vmtests/test_lvm_root.py
764+++ b/tests/vmtests/test_lvm_root.py
765@@ -113,7 +113,7 @@ class XenialTestUefiLvmRootXfs(relbase.xenial, TestUefiLvmRootAbs):
766 }
767
768
769-@VMBaseClass.skip_by_date("1652822", fixby="2019-06-01")
770+@VMBaseClass.skip_by_date("1652822", fixby="2019-06-01", install=False)
771 class XenialTestUefiLvmRootXfsBootXfs(relbase.xenial, TestUefiLvmRootAbs):
772 """This tests xfs root and xfs boot with uefi.
773

Subscribers

People subscribed via source and target branches