Merge lp:~chad.smith/curtin/xenial-sru-1721808 into lp:~curtin-dev/curtin/xenial

Proposed by Chad Smith on 2017-10-06
Status: Merged
Merged at revision: 71
Proposed branch: lp:~chad.smith/curtin/xenial-sru-1721808
Merge into: lp:~curtin-dev/curtin/xenial
Diff against target: 7045 lines (+3340/-962)
91 files modified
curtin/__init__.py (+2/-0)
curtin/block/__init__.py (+69/-20)
curtin/block/iscsi.py (+44/-3)
curtin/block/mdadm.py (+10/-6)
curtin/commands/apply_net.py (+34/-8)
curtin/commands/apt_config.py (+0/-9)
curtin/commands/curthooks.py (+197/-94)
curtin/commands/extract.py (+6/-0)
curtin/commands/install.py (+44/-4)
curtin/futil.py (+24/-1)
curtin/net/__init__.py (+106/-0)
curtin/reporter/handlers.py (+42/-0)
curtin/util.py (+137/-13)
debian/changelog (+33/-0)
doc/index.rst (+1/-0)
doc/topics/apt_source.rst (+9/-6)
doc/topics/config.rst (+18/-0)
doc/topics/curthooks.rst (+109/-0)
doc/topics/integration-testing.rst (+6/-0)
doc/topics/networking.rst (+2/-0)
doc/topics/overview.rst (+45/-47)
doc/topics/reporting.rst (+29/-0)
doc/topics/storage.rst (+2/-0)
examples/network-ipv6-bond-vlan.yaml (+2/-2)
examples/tests/bonding_network.yaml (+1/-4)
examples/tests/centos_basic.yaml (+2/-1)
examples/tests/centos_defaults.yaml (+91/-0)
examples/tests/journald_reporter.yaml (+20/-0)
examples/tests/network_alias.yaml (+29/-31)
examples/tests/network_static_routes.yaml (+10/-15)
examples/tests/network_v2_passthrough.yaml (+8/-0)
setup.py (+16/-2)
tests/unittests/helpers.py (+36/-0)
tests/unittests/test_apt_custom_sources_list.py (+3/-6)
tests/unittests/test_apt_source.py (+4/-7)
tests/unittests/test_basic.py (+4/-4)
tests/unittests/test_block.py (+20/-36)
tests/unittests/test_block_iscsi.py (+187/-18)
tests/unittests/test_block_lvm.py (+2/-2)
tests/unittests/test_block_mdadm.py (+10/-22)
tests/unittests/test_block_mkfs.py (+2/-2)
tests/unittests/test_clear_holders.py (+5/-5)
tests/unittests/test_commands_apply_net.py (+334/-0)
tests/unittests/test_commands_block_meta.py (+6/-19)
tests/unittests/test_commands_install.py (+22/-0)
tests/unittests/test_config.py (+6/-6)
tests/unittests/test_curthooks.py (+241/-57)
tests/unittests/test_feature.py (+5/-2)
tests/unittests/test_gpg.py (+4/-4)
tests/unittests/test_make_dname.py (+4/-4)
tests/unittests/test_net.py (+99/-24)
tests/unittests/test_partitioning.py (+4/-3)
tests/unittests/test_public.py (+54/-0)
tests/unittests/test_reporter.py (+29/-38)
tests/unittests/test_util.py (+201/-52)
tests/unittests/test_version.py (+7/-19)
tests/vmtests/__init__.py (+59/-7)
tests/vmtests/releases.py (+0/-15)
tests/vmtests/test_apt_config_cmd.py (+0/-4)
tests/vmtests/test_basic.py (+0/-13)
tests/vmtests/test_bcache_basic.py (+0/-4)
tests/vmtests/test_centos_basic.py (+35/-0)
tests/vmtests/test_iscsi.py (+0/-4)
tests/vmtests/test_journald_reporter.py (+52/-0)
tests/vmtests/test_lvm.py (+0/-9)
tests/vmtests/test_lvm_iscsi.py (+4/-4)
tests/vmtests/test_mdadm_bcache.py (+3/-59)
tests/vmtests/test_mdadm_iscsi.py (+4/-4)
tests/vmtests/test_multipath.py (+0/-4)
tests/vmtests/test_network.py (+202/-39)
tests/vmtests/test_network_alias.py (+33/-4)
tests/vmtests/test_network_bonding.py (+47/-22)
tests/vmtests/test_network_bridging.py (+77/-17)
tests/vmtests/test_network_enisource.py (+2/-8)
tests/vmtests/test_network_ipv6.py (+29/-4)
tests/vmtests/test_network_ipv6_enisource.py (+8/-6)
tests/vmtests/test_network_ipv6_static.py (+17/-5)
tests/vmtests/test_network_ipv6_vlan.py (+17/-5)
tests/vmtests/test_network_mtu.py (+61/-8)
tests/vmtests/test_network_static.py (+30/-4)
tests/vmtests/test_network_static_routes.py (+19/-6)
tests/vmtests/test_network_vlan.py (+40/-15)
tests/vmtests/test_nvme.py (+0/-9)
tests/vmtests/test_raid5_bcache.py (+0/-9)
tests/vmtests/test_simple.py (+0/-4)
tests/vmtests/test_uefi_basic.py (+0/-19)
tools/build-deb (+3/-1)
tools/curtainer (+14/-8)
tools/find-tgt (+54/-29)
tools/jenkins-runner (+47/-10)
tools/launch (+46/-7)
To merge this branch: bzr merge lp:~chad.smith/curtin/xenial-sru-1721808
Reviewer Review Type Date Requested Status
curtin developers 2017-10-06 Pending
Review via email: mp+331961@code.launchpad.net

Description of the change

Upstream snapshot from Xenial for SRU.

To post a comment you must log in.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'curtin/__init__.py'
2--- curtin/__init__.py 2017-06-12 20:39:06 +0000
3+++ curtin/__init__.py 2017-10-06 16:35:22 +0000
4@@ -23,6 +23,8 @@
5 # can determine which features are supported. Each entry should have
6 # a consistent meaning.
7 FEATURES = [
8+ # curtin can apply centos networking via centos_apply_network_config
9+ 'CENTOS_APPLY_NETWORK_CONFIG',
10 # install supports the 'network' config version 1
11 'NETWORK_CONFIG_V1',
12 # reporter supports 'webhook' type
13
14=== modified file 'curtin/block/__init__.py'
15--- curtin/block/__init__.py 2017-06-12 20:39:06 +0000
16+++ curtin/block/__init__.py 2017-10-06 16:35:22 +0000
17@@ -19,7 +19,6 @@
18 import errno
19 import itertools
20 import os
21-import shlex
22 import stat
23 import sys
24 import tempfile
25@@ -204,30 +203,13 @@
26 return [path_to_kname(device)]
27
28
29-def _shlex_split(str_in):
30- # shlex.split takes a string
31- # but in python2 if input here is a unicode, encode it to a string.
32- # http://stackoverflow.com/questions/2365411/
33- # python-convert-unicode-to-ascii-without-errors
34- if sys.version_info.major == 2:
35- try:
36- if isinstance(str_in, unicode):
37- str_in = str_in.encode('utf-8')
38- except NameError:
39- pass
40-
41- return shlex.split(str_in)
42- else:
43- return shlex.split(str_in)
44-
45-
46 def _lsblock_pairs_to_dict(lines):
47 """
48 parse lsblock output and convert to dict
49 """
50 ret = {}
51 for line in lines.splitlines():
52- toks = _shlex_split(line)
53+ toks = util.shlex_split(line)
54 cur = {}
55 for tok in toks:
56 k, v = tok.split("=", 1)
57@@ -468,7 +450,7 @@
58 for line in out.splitlines():
59 curdev, curdata = line.split(":", 1)
60 data[curdev] = dict(tok.split('=', 1)
61- for tok in _shlex_split(curdata))
62+ for tok in util.shlex_split(curdata))
63 return data
64
65
66@@ -978,4 +960,71 @@
67 else:
68 raise ValueError("wipe mode %s not supported" % mode)
69
70+
71+def storage_config_required_packages(storage_config, mapping):
72+ """Read storage configuration dictionary and determine
73+ which packages are required for the supplied configuration
74+ to function. Return a list of packaged to install.
75+ """
76+
77+ if not storage_config or not isinstance(storage_config, dict):
78+ raise ValueError('Invalid storage configuration. '
79+ 'Must be a dict:\n %s' % storage_config)
80+
81+ if not mapping or not isinstance(mapping, dict):
82+ raise ValueError('Invalid storage mapping. Must be a dict')
83+
84+ if 'storage' in storage_config:
85+ storage_config = storage_config.get('storage')
86+
87+ needed_packages = []
88+
89+ # get reqs by device operation type
90+ dev_configs = set(operation['type']
91+ for operation in storage_config['config'])
92+
93+ for dev_type in dev_configs:
94+ if dev_type in mapping:
95+ needed_packages.extend(mapping[dev_type])
96+
97+ # for any format operations, check the fstype and
98+ # determine if we need any mkfs tools as well.
99+ format_configs = set([operation['fstype']
100+ for operation in storage_config['config']
101+ if operation['type'] == 'format'])
102+ for format_type in format_configs:
103+ if format_type in mapping:
104+ needed_packages.extend(mapping[format_type])
105+
106+ return needed_packages
107+
108+
109+def detect_required_packages_mapping():
110+ """Return a dictionary providing a versioned configuration which maps
111+ storage configuration elements to the packages which are required
112+ for functionality.
113+
114+ The mapping key is either a config type value, or an fstype value.
115+
116+ """
117+ version = 1
118+ mapping = {
119+ version: {
120+ 'handler': storage_config_required_packages,
121+ 'mapping': {
122+ 'bcache': ['bcache-tools'],
123+ 'btrfs': ['btrfs-tools'],
124+ 'ext2': ['e2fsprogs'],
125+ 'ext3': ['e2fsprogs'],
126+ 'ext4': ['e2fsprogs'],
127+ 'lvm_partition': ['lvm2'],
128+ 'lvm_volgroup': ['lvm2'],
129+ 'raid': ['mdadm'],
130+ 'xfs': ['xfsprogs']
131+ },
132+ },
133+ }
134+ return mapping
135+
136+
137 # vi: ts=4 expandtab syntax=python
138
139=== modified file 'curtin/block/iscsi.py'
140--- curtin/block/iscsi.py 2017-06-12 20:39:06 +0000
141+++ curtin/block/iscsi.py 2017-10-06 16:35:22 +0000
142@@ -195,6 +195,15 @@
143 return target_nodes_location
144
145
146+def restart_iscsi_service():
147+ LOG.info('restarting iscsi service')
148+ if util.uses_systemd():
149+ cmd = ['systemctl', 'reload-or-restart', 'open-iscsi']
150+ else:
151+ cmd = ['service', 'open-iscsi', 'restart']
152+ util.subp(cmd, capture=True)
153+
154+
155 def save_iscsi_config(iscsi_disk):
156 state = util.load_command_environment()
157 # A nodes directory will be created in the same directory as the
158@@ -238,11 +247,35 @@
159 return _ISCSI_DISKS
160
161
162+def get_iscsi_disks_from_config(cfg):
163+ """Parse a curtin storage config and return a list
164+ of iscsi disk objects for each configuration present
165+ """
166+ if not cfg:
167+ cfg = {}
168+
169+ sconfig = cfg.get('storage', {}).get('config', {})
170+ if not sconfig:
171+ LOG.warning('Configuration dictionary did not contain'
172+ ' a storage configuration')
173+ return []
174+
175+ # Construct IscsiDisk objects for each iscsi volume present
176+ iscsi_disks = [IscsiDisk(disk['path']) for disk in sconfig
177+ if disk['type'] == 'disk' and
178+ disk.get('path', "").startswith('iscsi:')]
179+ LOG.debug('Found %s iscsi disks in storage config', len(iscsi_disks))
180+ return iscsi_disks
181+
182+
183 def disconnect_target_disks(target_root_path=None):
184 target_nodes_path = util.target_path(target_root_path, '/etc/iscsi/nodes')
185 fails = []
186 if os.path.isdir(target_nodes_path):
187 for target in os.listdir(target_nodes_path):
188+ if target not in iscsiadm_sessions():
189+ LOG.debug('iscsi target %s not active, skipping', target)
190+ continue
191 # conn is "host,port,lun"
192 for conn in os.listdir(
193 os.path.sep.join([target_nodes_path, target])):
194@@ -254,7 +287,9 @@
195 fails.append(target)
196 LOG.warn("Unable to logout of iSCSI target %s: %s",
197 target, e)
198-
199+ else:
200+ LOG.warning('Skipping disconnect: failed to find iscsi nodes path: %s',
201+ target_nodes_path)
202 if fails:
203 raise RuntimeError(
204 "Unable to logout of iSCSI targets: %s" % ', '.join(fails))
205@@ -414,9 +449,15 @@
206
207 def disconnect(self):
208 if self.target not in iscsiadm_sessions():
209+ LOG.warning('Iscsi target %s not in active iscsi sessions',
210+ self.target)
211 return
212
213- util.subp(['sync'])
214- iscsiadm_logout(self.target, self.portal)
215+ try:
216+ util.subp(['sync'])
217+ iscsiadm_logout(self.target, self.portal)
218+ except util.ProcessExecutionError as e:
219+ LOG.warn("Unable to logout of iSCSI target %s from portal %s: %s",
220+ self.target, self.portal, e)
221
222 # vi: ts=4 expandtab syntax=python
223
224=== modified file 'curtin/block/mdadm.py'
225--- curtin/block/mdadm.py 2017-06-12 20:39:06 +0000
226+++ curtin/block/mdadm.py 2017-10-06 16:35:22 +0000
227@@ -273,7 +273,11 @@
228 LOG.debug('%s/sync_max = %s', sync_action, val)
229 if val != "idle":
230 LOG.debug("mdadm: setting array sync_action=idle")
231- util.write_file(sync_action, content="idle")
232+ try:
233+ util.write_file(sync_action, content="idle")
234+ except (IOError, OSError) as e:
235+ LOG.debug("mdadm: (non-fatal) write to %s failed %s",
236+ sync_action, e)
237
238 # Setting the sync_{max,min} may can help prevent the array from
239 # changing back to 'resync' which may prevent the array from being
240@@ -283,11 +287,11 @@
241 if val != "0":
242 LOG.debug("mdadm: setting array sync_{min,max}=0")
243 try:
244- util.write_file(sync_max, content="0")
245- util.write_file(sync_min, content="0")
246- except IOError:
247- LOG.warning('mdadm: failed to set sync_{max,min} values')
248- pass
249+ for sync_file in [sync_max, sync_min]:
250+ util.write_file(sync_file, content="0")
251+ except (IOError, OSError) as e:
252+ LOG.debug('mdadm: (non-fatal) write to %s failed %s',
253+ sync_file, e)
254
255 # one wonders why this command doesn't do any of the above itself?
256 out, err = util.subp(["mdadm", "--manage", "--stop", devpath],
257
258=== modified file 'curtin/commands/apply_net.py'
259--- curtin/commands/apply_net.py 2017-02-08 22:22:44 +0000
260+++ curtin/commands/apply_net.py 2017-10-06 16:35:22 +0000
261@@ -21,6 +21,7 @@
262 from .. import log
263 import curtin.net as net
264 import curtin.util as util
265+from curtin import config
266 from . import populate_one_subcmd
267
268
269@@ -89,15 +90,38 @@
270 sys.stderr.write(msg + "\n")
271 raise Exception(msg)
272
273+ passthrough = False
274 if network_state:
275+ # NB: we cannot support passthrough until curtin can convert from
276+ # network_state to network-config yaml
277 ns = net.network_state.from_state_file(network_state)
278+ raise ValueError('Not Supported; curtin lacks a network_state to '
279+ 'network_config converter.')
280 elif network_config:
281- ns = net.parse_net_config(network_config)
282-
283- net.render_network_state(target=target, network_state=ns)
284+ netcfg = config.load_config(network_config)
285+
286+ # curtin will pass-through the netconfig into the target
287+ # for rendering at runtime unless the target OS does not
288+ # support NETWORK_CONFIG_V2 feature.
289+ LOG.info('Checking cloud-init in target [%s] for network '
290+ 'configuration passthrough support.', target)
291+ try:
292+ passthrough = net.netconfig_passthrough_available(target)
293+ except util.ProcessExecutionError:
294+ LOG.warning('Failed to determine if passthrough is available')
295+
296+ if passthrough:
297+ LOG.info('Passing network configuration through to target: %s',
298+ target)
299+ net.render_netconfig_passthrough(target, netconfig=netcfg)
300+ else:
301+ ns = net.parse_net_config_data(netcfg.get('network', {}))
302+
303+ if not passthrough:
304+ LOG.info('Rendering network configuration in target')
305+ net.render_network_state(target=target, network_state=ns)
306
307 _maybe_remove_legacy_eth0(target)
308- LOG.info('Attempting to remove ipv6 privacy extensions')
309 _disable_ipv6_privacy_extensions(target)
310 _patch_ifupdown_ipv6_mtu_hook(target)
311
312@@ -130,6 +154,7 @@
313 by default; this races with the cloud-image desire to disable them.
314 Resolve this by allowing the cloud-image setting to win. """
315
316+ LOG.debug('Attempting to remove ipv6 privacy extensions')
317 cfg = util.target_path(target, path=path)
318 if not os.path.exists(cfg):
319 LOG.warn('Failed to find ipv6 privacy conf file %s', cfg)
320@@ -143,7 +168,7 @@
321 lines = [f.strip() for f in contents.splitlines()
322 if not f.startswith("#")]
323 if lines == known_contents:
324- LOG.info('deleting file: %s', cfg)
325+ LOG.info('Removing ipv6 privacy extension config file: %s', cfg)
326 util.del_file(cfg)
327 msg = "removed %s with known contents" % cfg
328 curtin_contents = '\n'.join(
329@@ -153,9 +178,10 @@
330 "# net.ipv6.conf.default.use_tempaddr = 2"])
331 util.write_file(cfg, curtin_contents)
332 else:
333- LOG.info('skipping, content didnt match')
334- LOG.debug("found content:\n%s", lines)
335- LOG.debug("expected contents:\n%s", known_contents)
336+ LOG.debug('skipping removal of %s, expected content not found',
337+ cfg)
338+ LOG.debug("Found content in file %s:\n%s", cfg, lines)
339+ LOG.debug("Expected contents in file %s:\n%s", cfg, known_contents)
340 msg = (bmsg + " '%s' exists with user configured content." % cfg)
341 except Exception as e:
342 msg = bmsg + " %s exists, but could not be read. %s" % (cfg, e)
343
344=== modified file 'curtin/commands/apt_config.py'
345--- curtin/commands/apt_config.py 2017-03-01 16:13:56 +0000
346+++ curtin/commands/apt_config.py 2017-10-06 16:35:22 +0000
347@@ -24,7 +24,6 @@
348 import os
349 import re
350 import sys
351-import time
352 import yaml
353
354 from curtin.log import LOG
355@@ -406,20 +405,12 @@
356 if aa_repo_match(source):
357 with util.ChrootableTarget(
358 target, sys_resolvconf=True) as in_chroot:
359- time_entered = time.time()
360 try:
361 in_chroot.subp(["add-apt-repository", source],
362 retries=(1, 2, 5, 10))
363 except util.ProcessExecutionError:
364 LOG.exception("add-apt-repository failed.")
365 raise
366- finally:
367- # workaround to gnupg >=2.x spawning daemons (LP: #1645680)
368- seconds_since = time.time() - time_entered + 1
369- in_chroot.subp(['killall', '--wait', '--quiet',
370- '--younger-than', '%ds' % seconds_since,
371- '--regexp', '(dirmngr|gpg-agent)'],
372- rcs=[0, 1])
373 continue
374
375 sourcefn = util.target_path(target, ent['filename'])
376
377=== modified file 'curtin/commands/curthooks.py'
378--- curtin/commands/curthooks.py 2017-06-12 20:39:06 +0000
379+++ curtin/commands/curthooks.py 2017-10-06 16:35:22 +0000
380@@ -16,6 +16,7 @@
381 # along with Curtin. If not, see <http://www.gnu.org/licenses/>.
382
383 import copy
384+import glob
385 import os
386 import platform
387 import re
388@@ -25,6 +26,7 @@
389
390 from curtin import config
391 from curtin import block
392+from curtin import net
393 from curtin import futil
394 from curtin.log import LOG
395 from curtin import swap
396@@ -65,28 +67,18 @@
397 }
398 }
399
400-
401-def write_files(cfg, target):
402- # this takes 'write_files' entry in config and writes files in the target
403- # config entry example:
404- # f1:
405- # path: /file1
406- # content: !!binary |
407- # f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAwARAAAAAAABAAAAAAAAAAJAVAAAAAAA
408- # f2: {path: /file2, content: "foobar", permissions: '0666'}
409- if 'write_files' not in cfg:
410- return
411-
412- for (key, info) in cfg.get('write_files').items():
413- if not info.get('path'):
414- LOG.warn("Warning, write_files[%s] had no 'path' entry", key)
415- continue
416-
417- futil.write_finfo(path=target + os.path.sep + info['path'],
418- content=info.get('content', ''),
419- owner=info.get('owner', "-1:-1"),
420- perms=info.get('permissions',
421- info.get('perms', "0644")))
422+CLOUD_INIT_YUM_REPO_TEMPLATE = """
423+[group_cloud-init-el-stable]
424+name=Copr repo for el-stable owned by @cloud-init
425+baseurl=https://copr-be.cloud.fedoraproject.org/results/@cloud-init/el-stable/epel-%s-$basearch/
426+type=rpm-md
427+skip_if_unavailable=True
428+gpgcheck=1
429+gpgkey=https://copr-be.cloud.fedoraproject.org/results/@cloud-init/el-stable/pubkey.gpg
430+repo_gpgcheck=0
431+enabled=1
432+enabled_metadata=1
433+"""
434
435
436 def do_apt_config(cfg, target):
437@@ -142,15 +134,9 @@
438 parameters = root=%s
439
440 """ % root_arg
441- zipl_cfg = {
442- "write_files": {
443- "zipl_cfg": {
444- "path": "/etc/zipl.conf",
445- "content": zipl_conf,
446- }
447- }
448- }
449- write_files(zipl_cfg, target)
450+ futil.write_files(
451+ files={"zipl_conf": {"path": "/etc/zipl.conf", "content": zipl_conf}},
452+ base_dir=target)
453
454
455 def run_zipl(cfg, target):
456@@ -648,6 +634,40 @@
457 update_initramfs(target, all_kernels=True)
458
459
460+def detect_required_packages(cfg):
461+ """
462+ detect packages that will be required in-target by custom config items
463+ """
464+
465+ mapping = {
466+ 'storage': block.detect_required_packages_mapping(),
467+ 'network': net.detect_required_packages_mapping(),
468+ }
469+
470+ needed_packages = []
471+ for cfg_type, cfg_map in mapping.items():
472+
473+ # skip missing or invalid config items, configs may
474+ # only have network or storage, not always both
475+ if not isinstance(cfg.get(cfg_type), dict):
476+ continue
477+
478+ cfg_version = cfg[cfg_type].get('version')
479+ if not isinstance(cfg_version, int) or cfg_version not in cfg_map:
480+ msg = ('Supplied configuration version "%s", for config type'
481+ '"%s" is not present in the known mapping.' % (cfg_version,
482+ cfg_type))
483+ raise ValueError(msg)
484+
485+ mapped_config = cfg_map[cfg_version]
486+ found_reqs = mapped_config['handler'](cfg, mapped_config['mapping'])
487+ needed_packages.extend(found_reqs)
488+
489+ LOG.debug('Curtin config dependencies requires additional packages: %s',
490+ needed_packages)
491+ return needed_packages
492+
493+
494 def install_missing_packages(cfg, target):
495 ''' describe which operation types will require specific packages
496
497@@ -655,46 +675,10 @@
498 'pkg1': ['op_name_1', 'op_name_2', ...]
499 }
500 '''
501- custom_configs = {
502- 'storage': {
503- 'lvm2': ['lvm_volgroup', 'lvm_partition'],
504- 'mdadm': ['raid'],
505- 'bcache-tools': ['bcache']},
506- 'network': {
507- 'vlan': ['vlan'],
508- 'ifenslave': ['bond'],
509- 'bridge-utils': ['bridge']},
510- }
511-
512- format_configs = {
513- 'xfsprogs': ['xfs'],
514- 'e2fsprogs': ['ext2', 'ext3', 'ext4'],
515- 'btrfs-tools': ['btrfs'],
516- }
517-
518- needed_packages = []
519+
520 installed_packages = util.get_installed_packages(target)
521- for cust_cfg, pkg_reqs in custom_configs.items():
522- if cust_cfg not in cfg:
523- continue
524-
525- all_types = set(
526- operation['type']
527- for operation in cfg[cust_cfg]['config']
528- )
529- for pkg, types in pkg_reqs.items():
530- if set(types).intersection(all_types) and \
531- pkg not in installed_packages:
532- needed_packages.append(pkg)
533-
534- format_types = set(
535- [operation['fstype']
536- for operation in cfg[cust_cfg]['config']
537- if operation['type'] == 'format'])
538- for pkg, fstypes in format_configs.items():
539- if set(fstypes).intersection(format_types) and \
540- pkg not in installed_packages:
541- needed_packages.append(pkg)
542+ needed_packages = set([pkg for pkg in detect_required_packages(cfg)
543+ if pkg not in installed_packages])
544
545 arch_packages = {
546 's390x': [('s390-tools', 'zipl')],
547@@ -703,16 +687,28 @@
548 for pkg, cmd in arch_packages.get(platform.machine(), []):
549 if not util.which(cmd, target=target):
550 if pkg not in needed_packages:
551- needed_packages.append(pkg)
552+ needed_packages.add(pkg)
553+
554+ # FIXME: This needs cleaning up.
555+ # do not install certain packages on artful as they are no longer needed.
556+ # ifenslave specifically causes issuse due to dependency on ifupdown.
557+ codename = util.lsb_release(target=target).get('codename')
558+ if codename == 'artful':
559+ drops = set(['bridge-utils', 'ifenslave', 'vlan'])
560+ if needed_packages.union(drops):
561+ LOG.debug("Skipping install of %s. Not needed on artful.",
562+ needed_packages.union(drops))
563+ needed_packages = needed_packages.difference(drops)
564
565 if needed_packages:
566+ to_add = list(sorted(needed_packages))
567 state = util.load_command_environment()
568 with events.ReportEventStack(
569 name=state.get('report_stack_prefix'),
570 reporting_enabled=True, level="INFO",
571 description="Installing packages on target system: " +
572- str(needed_packages)):
573- util.install_packages(needed_packages, target=target)
574+ str(to_add)):
575+ util.install_packages(to_add, target=target)
576
577
578 def system_upgrade(cfg, target):
579@@ -737,8 +733,8 @@
580 util.system_upgrade(target=target)
581
582
583-def handle_cloudconfig(cfg, target=None):
584- """write cloud-init configuration files into target
585+def handle_cloudconfig(cfg, base_dir=None):
586+ """write cloud-init configuration files into base_dir.
587
588 cloudconfig format is a dictionary of keys and values of content
589
590@@ -773,9 +769,9 @@
591 cfgvalue['path'] = cfgpath
592
593 # re-use write_files format and adjust target to prepend
594- LOG.debug('Calling write_files with cloudconfig @ %s', target)
595+ LOG.debug('Calling write_files with cloudconfig @ %s', base_dir)
596 LOG.debug('Injecting cloud-config:\n%s', cfg)
597- write_files({'write_files': cfg}, target)
598+ futil.write_files(cfg, base_dir)
599
600
601 def ubuntu_core_curthooks(cfg, target=None):
602@@ -795,17 +791,98 @@
603 if os.path.exists(cloudinit_disable):
604 util.del_file(cloudinit_disable)
605
606- handle_cloudconfig(cloudconfig, target=cc_target)
607+ handle_cloudconfig(cloudconfig, base_dir=cc_target)
608
609 netconfig = cfg.get('network', None)
610 if netconfig:
611 LOG.info('Writing network configuration')
612 ubuntu_core_netconfig = os.path.join(cc_target,
613- "50-network-config.cfg")
614+ "50-curtin-networking.cfg")
615 util.write_file(ubuntu_core_netconfig,
616 content=config.dump_config({'network': netconfig}))
617
618
619+def rpm_get_dist_id(target):
620+ """Use rpm command to extract the '%rhel' distro macro which returns
621+ the major os version id (6, 7, 8). This works for centos or rhel
622+ """
623+ with util.ChrootableTarget(target) as in_chroot:
624+ dist, _ = in_chroot.subp(['rpm', '-E', '%rhel'], capture=True)
625+ return dist.rstrip()
626+
627+
628+def centos_apply_network_config(netcfg, target=None):
629+ """ CentOS images execute built-in curthooks which only supports
630+ simple networking configuration. This hook enables advanced
631+ network configuration via config passthrough to the target.
632+ """
633+
634+ def cloud_init_repo(version):
635+ if not version:
636+ raise ValueError('Missing required version parameter')
637+
638+ return CLOUD_INIT_YUM_REPO_TEMPLATE % version
639+
640+ if netcfg:
641+ LOG.info('Removing embedded network configuration (if present)')
642+ ifcfgs = glob.glob(util.target_path(target,
643+ 'etc/sysconfig/network-scripts') +
644+ '/ifcfg-*')
645+ # remove ifcfg-* (except ifcfg-lo)
646+ for ifcfg in ifcfgs:
647+ if os.path.basename(ifcfg) != "ifcfg-lo":
648+ util.del_file(ifcfg)
649+
650+ LOG.info('Checking cloud-init in target [%s] for network '
651+ 'configuration passthrough support.', target)
652+ passthrough = net.netconfig_passthrough_available(target)
653+ LOG.debug('passthrough available via in-target: %s', passthrough)
654+
655+ # if in-target cloud-init is not updated, upgrade via cloud-init repo
656+ if not passthrough:
657+ cloud_init_yum_repo = (
658+ util.target_path(target,
659+ 'etc/yum.repos.d/curtin-cloud-init.repo'))
660+ # Inject cloud-init daily yum repo
661+ util.write_file(cloud_init_yum_repo,
662+ content=cloud_init_repo(rpm_get_dist_id(target)))
663+
664+ # we separate the installation of repository packages (epel,
665+ # cloud-init-el-release) as we need a new invocation of yum
666+ # to read the newly installed repo files.
667+ YUM_CMD = ['yum', '-y', '--noplugins', 'install']
668+ retries = [1] * 30
669+ with util.ChrootableTarget(target) as in_chroot:
670+ # ensure up-to-date ca-certificates to handle https mirror
671+ # connections
672+ in_chroot.subp(YUM_CMD + ['ca-certificates'], capture=True,
673+ log_captured=True, retries=retries)
674+ in_chroot.subp(YUM_CMD + ['epel-release'], capture=True,
675+ log_captured=True, retries=retries)
676+ in_chroot.subp(YUM_CMD + ['cloud-init-el-release'],
677+ log_captured=True, capture=True,
678+ retries=retries)
679+ in_chroot.subp(YUM_CMD + ['cloud-init'], capture=True,
680+ log_captured=True, retries=retries)
681+
682+ # remove cloud-init el-stable bootstrap repo config as the
683+ # cloud-init-el-release package points to the correct repo
684+ util.del_file(cloud_init_yum_repo)
685+
686+ # install bridge-utils if needed
687+ with util.ChrootableTarget(target) as in_chroot:
688+ try:
689+ in_chroot.subp(['rpm', '-q', 'bridge-utils'],
690+ capture=False, rcs=[0])
691+ except util.ProcessExecutionError:
692+ LOG.debug('Image missing bridge-utils package, installing')
693+ in_chroot.subp(YUM_CMD + ['bridge-utils'], capture=True,
694+ log_captured=True, retries=retries)
695+
696+ LOG.info('Passing network configuration through to target')
697+ net.render_netconfig_passthrough(target, netconfig={'network': netcfg})
698+
699+
700 def target_is_ubuntu_core(target):
701 """Check if Ubuntu-Core specific directory is present at target"""
702 if target:
703@@ -814,6 +891,22 @@
704 return False
705
706
707+def target_is_centos(target):
708+ """Check if CentOS specific file is present at target"""
709+ if target:
710+ return os.path.exists(util.target_path(target, 'etc/centos-release'))
711+
712+ return False
713+
714+
715+def target_is_rhel(target):
716+ """Check if RHEL specific file is present at target"""
717+ if target:
718+ return os.path.exists(util.target_path(target, 'etc/redhat-release'))
719+
720+ return False
721+
722+
723 def curthooks(args):
724 state = util.load_command_environment()
725
726@@ -827,14 +920,28 @@
727 "Use --target or set TARGET_MOUNT_POINT\n")
728 sys.exit(2)
729
730- # if network-config hook exists in target,
731- # we do not run the builtin
732- if util.run_hook_if_exists(target, 'curtin-hooks'):
733- sys.exit(0)
734-
735 cfg = config.load_command_config(args, state)
736 stack_prefix = state.get('report_stack_prefix', '')
737
738+ # if curtin-hooks hook exists in target we can defer to the in-target hooks
739+ if util.run_hook_if_exists(target, 'curtin-hooks'):
740+ # For vmtests to force execute centos_apply_network_config, uncomment
741+ # the value in examples/tests/centos_defaults.yaml
742+ if cfg.get('_ammend_centos_curthooks'):
743+ if cfg.get('cloudconfig'):
744+ handle_cloudconfig(
745+ cfg['cloudconfig'],
746+ base_dir=util.target_path(target, 'etc/cloud/cloud.cfg.d'))
747+
748+ if target_is_centos(target) or target_is_rhel(target):
749+ LOG.info('Detected RHEL/CentOS image, running extra hooks')
750+ with events.ReportEventStack(
751+ name=stack_prefix, reporting_enabled=True,
752+ level="INFO",
753+ description="Configuring CentOS for first boot"):
754+ centos_apply_network_config(cfg.get('network', {}), target)
755+ sys.exit(0)
756+
757 if target_is_ubuntu_core(target):
758 LOG.info('Detected Ubuntu-Core image, running hooks')
759 with events.ReportEventStack(
760@@ -846,13 +953,16 @@
761 with events.ReportEventStack(
762 name=stack_prefix + '/writing-config',
763 reporting_enabled=True, level="INFO",
764- description="writing config files and configuring apt"):
765- write_files(cfg, target)
766+ description="configuring apt configuring apt"):
767 do_apt_config(cfg, target)
768 disable_overlayroot(cfg, target)
769
770 # packages may be needed prior to installing kernel
771- install_missing_packages(cfg, target)
772+ with events.ReportEventStack(
773+ name=stack_prefix + '/installing-missing-packages',
774+ reporting_enabled=True, level="INFO",
775+ description="installing missing packages"):
776+ install_missing_packages(cfg, target)
777
778 # If a /etc/iscsi/nodes/... file was created by block_meta then it
779 # needs to be copied onto the target system
780@@ -880,7 +990,6 @@
781 setup_zipl(cfg, target)
782 install_kernel(cfg, target)
783 run_zipl(cfg, target)
784-
785 restore_dist_interfaces(cfg, target)
786
787 with events.ReportEventStack(
788@@ -908,12 +1017,6 @@
789 detect_and_handle_multipath(cfg, target)
790
791 with events.ReportEventStack(
792- name=stack_prefix + '/installing-missing-packages',
793- reporting_enabled=True, level="INFO",
794- description="installing missing packages"):
795- install_missing_packages(cfg, target)
796-
797- with events.ReportEventStack(
798 name=stack_prefix + '/system-upgrade',
799 reporting_enabled=True, level="INFO",
800 description="updating packages on target system"):
801
802=== modified file 'curtin/commands/extract.py'
803--- curtin/commands/extract.py 2016-05-10 16:13:29 +0000
804+++ curtin/commands/extract.py 2017-10-06 16:35:22 +0000
805@@ -21,6 +21,7 @@
806 import curtin.config
807 from curtin.log import LOG
808 import curtin.util
809+from curtin.futil import write_files
810 from curtin.reporter import events
811
812 from . import populate_one_subcmd
813@@ -122,6 +123,11 @@
814 "do not know how to extract '%s'" %
815 source['uri'])
816
817+ if cfg.get('write_files'):
818+ LOG.info("Applying write_files from config.")
819+ write_files(cfg['write_files'], target)
820+ else:
821+ LOG.info("No write_files in config.")
822 sys.exit(0)
823
824
825
826=== modified file 'curtin/commands/install.py'
827--- curtin/commands/install.py 2017-06-12 20:39:06 +0000
828+++ curtin/commands/install.py 2017-10-06 16:35:22 +0000
829@@ -366,6 +366,27 @@
830 return True
831
832
833+def migrate_proxy_settings(cfg):
834+ """Move the legacy proxy setting 'http_proxy' into cfg['proxy']."""
835+ proxy = cfg.get('proxy', {})
836+ if not isinstance(proxy, dict):
837+ raise ValueError("'proxy' in config is not a dictionary: %s" % proxy)
838+
839+ if 'http_proxy' in cfg:
840+ hp = cfg['http_proxy']
841+ if hp:
842+ if proxy.get('http_proxy', hp) != hp:
843+ LOG.warn("legacy http_proxy setting (%s) differs from "
844+ "proxy/http_proxy (%s), using %s",
845+ hp, proxy['http_proxy'], proxy['http_proxy'])
846+ else:
847+ LOG.debug("legacy 'http_proxy' migrated to proxy/http_proxy")
848+ proxy['http_proxy'] = hp
849+ del cfg['http_proxy']
850+
851+ cfg['proxy'] = proxy
852+
853+
854 def cmd_install(args):
855 cfg = CONFIG_BUILTIN.copy()
856 config.merge_config(cfg, args.config)
857@@ -384,8 +405,10 @@
858 # we default to tgz for old style sources config
859 cfg['sources'][i] = util.sanitize_source(cfg['sources'][i])
860
861- if cfg.get('http_proxy'):
862- os.environ['http_proxy'] = cfg['http_proxy']
863+ migrate_proxy_settings(cfg)
864+ for k in ('http_proxy', 'https_proxy', 'no_proxy'):
865+ if k in cfg['proxy']:
866+ os.environ[k] = cfg['proxy'][k]
867
868 instcfg = cfg.get('install', {})
869 logfile = instcfg.get('log_file')
870@@ -454,9 +477,26 @@
871 '/root/curtin-install.log')
872 if log_target_path:
873 copy_install_log(logfile, workingd.target, log_target_path)
874+ # unmount everything (including iscsi disks)
875 util.do_umount(workingd.target, recursive=True)
876- # need to do some processing on iscsi disks to disconnect?
877- iscsi.disconnect_target_disks(workingd.target)
878+
879+ # The open-iscsi service in the ephemeral environment handles
880+ # disconnecting active sessions. On Artful release the systemd
881+ # unit file has conditionals that are not met at boot time and
882+ # results in open-iscsi service not being started; This breaks
883+ # shutdown on Artful releases.
884+ # Additionally, in release < Artful, if the storage configuration
885+ # is layered, like RAID over iscsi volumes, then disconnecting iscsi
886+ # sessions before stopping the raid device hangs.
887+ # As it turns out, letting the open-iscsi service take down the
888+ # session last is the cleanest way to handle all releases regardless
889+ # of what may be layered on top of the iscsi disks.
890+ #
891+ # Check if storage configuration has iscsi volumes and if so ensure
892+ # iscsi service is active before exiting install
893+ if iscsi.get_iscsi_disks_from_config(cfg):
894+ iscsi.restart_iscsi_service()
895+
896 shutil.rmtree(workingd.top)
897
898 apply_power_state(cfg.get('power_state'))
899
900=== modified file 'curtin/futil.py'
901--- curtin/futil.py 2014-03-26 17:34:57 +0000
902+++ curtin/futil.py 2017-10-06 16:35:22 +0000
903@@ -19,7 +19,8 @@
904 import pwd
905 import os
906
907-from .util import write_file
908+from .util import write_file, target_path
909+from .log import LOG
910
911
912 def chownbyid(fname, uid=None, gid=None):
913@@ -78,3 +79,25 @@
914 omode = "wb"
915 write_file(path, content, mode=decode_perms(perms), omode=omode)
916 chownbyname(path, u, g)
917+
918+
919+def write_files(files, base_dir=None):
920+ """Write files described in the dictionary 'files'
921+
922+ paths are assumed under 'base_dir', which will default to '/'.
923+ A trailing '/' will be applied if not present.
924+
925+ files is a dictionary where each entry has:
926+ path: /file1
927+ content: (bytes or string)
928+ permissions: (optional, default=0644)
929+ owner: (optional, default -1:-1): string of 'uid:gid'."""
930+ for (key, info) in files.items():
931+ if not info.get('path'):
932+ LOG.warn("Warning, write_files[%s] had no 'path' entry", key)
933+ continue
934+
935+ write_finfo(path=target_path(base_dir, info['path']),
936+ content=info.get('content', ''),
937+ owner=info.get('owner', "-1:-1"),
938+ perms=info.get('permissions', info.get('perms', "0644")))
939
940=== modified file 'curtin/net/__init__.py'
941--- curtin/net/__init__.py 2017-03-01 16:13:56 +0000
942+++ curtin/net/__init__.py 2017-10-06 16:35:22 +0000
943@@ -520,7 +520,52 @@
944 return content
945
946
947+def netconfig_passthrough_available(target, feature='NETWORK_CONFIG_V2'):
948+ """
949+ Determine if curtin can pass v2 network config to in target cloud-init
950+ """
951+ LOG.debug('Checking in-target cloud-init for feature: %s', feature)
952+ with util.ChrootableTarget(target) as in_chroot:
953+
954+ cloudinit = util.which('cloud-init', target=target)
955+ if not cloudinit:
956+ LOG.warning('Target does not have cloud-init installed')
957+ return False
958+
959+ available = False
960+ try:
961+ out, _ = in_chroot.subp([cloudinit, 'features'], capture=True)
962+ available = feature in out.splitlines()
963+ except util.ProcessExecutionError:
964+ # we explicitly don't dump the exception as this triggers
965+ # vmtest failures when parsing the installation log file
966+ LOG.warning("Failed to probe cloudinit features")
967+ return False
968+
969+ LOG.debug('cloud-init feature %s available? %s', feature, available)
970+ return available
971+
972+
973+def render_netconfig_passthrough(target, netconfig=None):
974+ """
975+ Extract original network config and pass it
976+ through to cloud-init in target
977+ """
978+ cc = 'etc/cloud/cloud.cfg.d/50-curtin-networking.cfg'
979+ if not isinstance(netconfig, dict):
980+ raise ValueError('Network config must be a dictionary')
981+
982+ if 'network' not in netconfig:
983+ raise ValueError("Network config must contain the key 'network'")
984+
985+ content = config.dump_config(netconfig)
986+ cc_passthrough = os.path.sep.join((target, cc,))
987+ LOG.info('Writing network config to %s: %s', cc, cc_passthrough)
988+ util.write_file(cc_passthrough, content=content)
989+
990+
991 def render_network_state(target, network_state):
992+ LOG.debug("rendering eni from netconfig")
993 eni = 'etc/network/interfaces'
994 netrules = 'etc/udev/rules.d/70-persistent-net.rules'
995 cc = 'etc/cloud/cloud.cfg.d/curtin-disable-cloudinit-networking.cfg'
996@@ -542,4 +587,65 @@
997 """Returns the string value of an interface's MAC Address"""
998 return read_sys_net(ifname, "address", enoent=False)
999
1000+
1001+def network_config_required_packages(network_config, mapping=None):
1002+
1003+ if network_config is None:
1004+ network_config = {}
1005+
1006+ if not isinstance(network_config, dict):
1007+ raise ValueError('Invalid network configuration. Must be a dict')
1008+
1009+ if mapping is None:
1010+ mapping = {}
1011+
1012+ if not isinstance(mapping, dict):
1013+ raise ValueError('Invalid network mapping. Must be a dict')
1014+
1015+ # allow top-level 'network' key
1016+ if 'network' in network_config:
1017+ network_config = network_config.get('network')
1018+
1019+ # v1 has 'config' key and uses type: devtype elements
1020+ if 'config' in network_config:
1021+ dev_configs = set(device['type']
1022+ for device in network_config['config'])
1023+ else:
1024+ # v2 has no config key
1025+ dev_configs = set(cfgtype for (cfgtype, cfg) in
1026+ network_config.items() if cfgtype not in ['version'])
1027+
1028+ needed_packages = []
1029+ for dev_type in dev_configs:
1030+ if dev_type in mapping:
1031+ needed_packages.extend(mapping[dev_type])
1032+
1033+ return needed_packages
1034+
1035+
1036+def detect_required_packages_mapping():
1037+ """Return a dictionary providing a versioned configuration which maps
1038+ network configuration elements to the packages which are required
1039+ for functionality.
1040+ """
1041+ mapping = {
1042+ 1: {
1043+ 'handler': network_config_required_packages,
1044+ 'mapping': {
1045+ 'bond': ['ifenslave'],
1046+ 'bridge': ['bridge-utils'],
1047+ 'vlan': ['vlan']},
1048+ },
1049+ 2: {
1050+ 'handler': network_config_required_packages,
1051+ 'mapping': {
1052+ 'bonds': ['ifenslave'],
1053+ 'bridges': ['bridge-utils'],
1054+ 'vlans': ['vlan']}
1055+ },
1056+ }
1057+
1058+ return mapping
1059+
1060+
1061 # vi: ts=4 expandtab syntax=python
1062
1063=== modified file 'curtin/reporter/handlers.py'
1064--- curtin/reporter/handlers.py 2017-02-08 22:22:44 +0000
1065+++ curtin/reporter/handlers.py 2017-10-06 16:35:22 +0000
1066@@ -80,7 +80,49 @@
1067 LOG.warn("failed posting event: %s [%s]" % (event.as_string(), e))
1068
1069
1070+class JournaldHandler(ReportingHandler):
1071+
1072+ def __init__(self, level="DEBUG", identifier="curtin_event"):
1073+ super(JournaldHandler, self).__init__()
1074+ if isinstance(level, int):
1075+ pass
1076+ else:
1077+ input_level = level
1078+ try:
1079+ level = getattr(logging, level.upper())
1080+ except Exception:
1081+ LOG.warn("invalid level '%s', using WARN", input_level)
1082+ level = logging.WARN
1083+ self.level = level
1084+ self.identifier = identifier
1085+
1086+ def publish_event(self, event):
1087+ # Ubuntu older than precise will not have python-systemd installed.
1088+ try:
1089+ from systemd import journal
1090+ except ImportError:
1091+ raise
1092+ level = str(getattr(journal, "LOG_" + event.level, journal.LOG_DEBUG))
1093+ extra = {}
1094+ if hasattr(event, 'result'):
1095+ extra['CURTIN_RESULT'] = event.result
1096+ journal.send(
1097+ event.as_string(),
1098+ PRIORITY=level,
1099+ SYSLOG_IDENTIFIER=self.identifier,
1100+ CURTIN_EVENT_TYPE=event.event_type,
1101+ CURTIN_MESSAGE=event.description,
1102+ CURTIN_NAME=event.name,
1103+ **extra
1104+ )
1105+
1106+
1107 available_handlers = DictRegistry()
1108 available_handlers.register_item('log', LogHandler)
1109 available_handlers.register_item('print', PrintHandler)
1110 available_handlers.register_item('webhook', WebHookHandler)
1111+# only add journald handler on systemd systems
1112+try:
1113+ available_handlers.register_item('journald', JournaldHandler)
1114+except ImportError:
1115+ print('journald report handler not supported; no systemd module')
1116
1117=== modified file 'curtin/util.py'
1118--- curtin/util.py 2017-06-12 20:39:06 +0000
1119+++ curtin/util.py 2017-10-06 16:35:22 +0000
1120@@ -23,6 +23,7 @@
1121 import os
1122 import platform
1123 import re
1124+import shlex
1125 import shutil
1126 import socket
1127 import subprocess
1128@@ -57,6 +58,8 @@
1129 _INSTALLED_MAIN = '/usr/bin/curtin'
1130
1131 _LSB_RELEASE = {}
1132+_USES_SYSTEMD = None
1133+_HAS_UNSHARE_PID = None
1134
1135 _DNS_REDIRECT_IP = None
1136
1137@@ -66,21 +69,31 @@
1138
1139 def _subp(args, data=None, rcs=None, env=None, capture=False,
1140 shell=False, logstring=False, decode="replace",
1141- target=None, cwd=None, log_captured=False):
1142+ target=None, cwd=None, log_captured=False, unshare_pid=None):
1143 if rcs is None:
1144 rcs = [0]
1145-
1146 devnull_fp = None
1147- try:
1148- if target_path(target) != "/":
1149- args = ['chroot', target] + list(args)
1150-
1151- if not logstring:
1152- LOG.debug(("Running command %s with allowed return codes %s"
1153- " (shell=%s, capture=%s)"), args, rcs, shell, capture)
1154- else:
1155- LOG.debug(("Running hidden command to protect sensitive "
1156- "input/output logstring: %s"), logstring)
1157+
1158+ tpath = target_path(target)
1159+ chroot_args = [] if tpath == "/" else ['chroot', target]
1160+ sh_args = ['sh', '-c'] if shell else []
1161+ if isinstance(args, string_types):
1162+ args = [args]
1163+
1164+ try:
1165+ unshare_args = _get_unshare_pid_args(unshare_pid, tpath)
1166+ except RuntimeError as e:
1167+ raise RuntimeError("Unable to unshare pid (cmd=%s): %s" % (args, e))
1168+
1169+ args = unshare_args + chroot_args + sh_args + list(args)
1170+
1171+ if not logstring:
1172+ LOG.debug(("Running command %s with allowed return codes %s"
1173+ " (capture=%s)"), args, rcs, capture)
1174+ else:
1175+ LOG.debug(("Running hidden command to protect sensitive "
1176+ "input/output logstring: %s"), logstring)
1177+ try:
1178 stdin = None
1179 stdout = None
1180 stderr = None
1181@@ -94,7 +107,7 @@
1182 stdin = subprocess.PIPE
1183 sp = subprocess.Popen(args, stdout=stdout,
1184 stderr=stderr, stdin=stdin,
1185- env=env, shell=shell, cwd=cwd)
1186+ env=env, shell=False, cwd=cwd)
1187 # communicate in python2 returns str, python3 returns bytes
1188 (out, err) = sp.communicate(data)
1189
1190@@ -128,6 +141,63 @@
1191 return (out, err)
1192
1193
1194+def _has_unshare_pid():
1195+ global _HAS_UNSHARE_PID
1196+ if _HAS_UNSHARE_PID is not None:
1197+ return _HAS_UNSHARE_PID
1198+
1199+ if not which('unshare'):
1200+ _HAS_UNSHARE_PID = False
1201+ return False
1202+ out, err = subp(["unshare", "--help"], capture=True, decode=False,
1203+ unshare_pid=False)
1204+ joined = b'\n'.join([out, err])
1205+ _HAS_UNSHARE_PID = b'--fork' in joined and b'--pid' in joined
1206+ return _HAS_UNSHARE_PID
1207+
1208+
1209+def _get_unshare_pid_args(unshare_pid=None, target=None, euid=None):
1210+ """Get args for calling unshare for a pid.
1211+
1212+ If unshare_pid is False, return empty list.
1213+ If unshare_pid is True, check if it is usable. If not, raise exception.
1214+ if unshare_pid is None, then unshare if
1215+ * euid is 0
1216+ * 'unshare' with '--fork' and '--pid' is available.
1217+ * target != /
1218+ """
1219+ if unshare_pid is not None and not unshare_pid:
1220+ # given a false-ish other than None means no.
1221+ return []
1222+
1223+ if euid is None:
1224+ euid = os.geteuid()
1225+
1226+ tpath = target_path(target)
1227+
1228+ unshare_pid_in = unshare_pid
1229+ if unshare_pid is None:
1230+ unshare_pid = False
1231+ if tpath != "/" and euid == 0:
1232+ if _has_unshare_pid():
1233+ unshare_pid = True
1234+
1235+ if not unshare_pid:
1236+ return []
1237+
1238+ # either unshare was passed in as True, or None and turned to True.
1239+ if euid != 0:
1240+ raise RuntimeError(
1241+ "given unshare_pid=%s but euid (%s) != 0." %
1242+ (unshare_pid_in, euid))
1243+
1244+ if not _has_unshare_pid():
1245+ raise RuntimeError(
1246+ "given unshare_pid=%s but no unshare command." % unshare_pid_in)
1247+
1248+ return ['unshare', '--fork', '--pid', '--']
1249+
1250+
1251 def subp(*args, **kwargs):
1252 """Run a subprocess.
1253
1254@@ -160,6 +230,10 @@
1255 means to run, sleep 1, run, sleep 3, run and then return exit code.
1256 :param target:
1257 run the command as 'chroot target <args>'
1258+ :param unshare_pid:
1259+ unshare the pid namespace.
1260+ default value (None) is to unshare pid namespace if possible
1261+ and target != /
1262
1263 :return
1264 if not capturing, return is (None, None)
1265@@ -1275,6 +1349,9 @@
1266 if not path:
1267 return target
1268
1269+ if not isinstance(path, string_types):
1270+ raise ValueError("Unexpected input for path: %s" % path)
1271+
1272 # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /.
1273 while len(path) and path[0] == "/":
1274 path = path[1:]
1275@@ -1290,4 +1367,51 @@
1276 __call__ = ChrootableTarget.subp
1277
1278
1279+def shlex_split(str_in):
1280+ # shlex.split takes a string
1281+ # but in python2 if input here is a unicode, encode it to a string.
1282+ # http://stackoverflow.com/questions/2365411/
1283+ # python-convert-unicode-to-ascii-without-errors
1284+ if sys.version_info.major == 2:
1285+ try:
1286+ if isinstance(str_in, unicode):
1287+ str_in = str_in.encode('utf-8')
1288+ except NameError:
1289+ pass
1290+
1291+ return shlex.split(str_in)
1292+ else:
1293+ return shlex.split(str_in)
1294+
1295+
1296+def load_shell_content(content, add_empty=False, empty_val=None):
1297+ """Given shell like syntax (key=value\nkey2=value2\n) in content
1298+ return the data in dictionary form. If 'add_empty' is True
1299+ then add entries in to the returned dictionary for 'VAR='
1300+ variables. Set their value to empty_val."""
1301+
1302+ data = {}
1303+ for line in shlex_split(content):
1304+ key, value = line.split("=", 1)
1305+ if not value:
1306+ value = empty_val
1307+ if add_empty or value:
1308+ data[key] = value
1309+
1310+ return data
1311+
1312+
1313+def uses_systemd():
1314+ """ Check if current enviroment uses systemd by testing if
1315+ /run/systemd/system is a directory; only present if
1316+ systemd is available on running system.
1317+ """
1318+
1319+ global _USES_SYSTEMD
1320+ if _USES_SYSTEMD is None:
1321+ _USES_SYSTEMD = os.path.isdir('/run/systemd/system')
1322+
1323+ return _USES_SYSTEMD
1324+
1325+
1326 # vi: ts=4 expandtab syntax=python
1327
1328=== modified file 'debian/changelog'
1329--- debian/changelog 2017-06-12 20:52:38 +0000
1330+++ debian/changelog 2017-10-06 16:35:22 +0000
1331@@ -1,3 +1,36 @@
1332+curtin (0.1.0~bzr532-0ubuntu1~16.04.1) xenial; urgency=medium
1333+
1334+ * New upstream snapshot. (LP: #1721808)
1335+ - vmtest: fix artful networking
1336+ - docs: Trivial doc fix for enabling proposed.
1337+ - setup.py: fix to allow installation into a virtualenv
1338+ - doc: update documentation on curtin-hooks and non-ubuntu installation.
1339+ - reporter: Add journald reporter to send events to journald
1340+ - vmtests: add option to tar disk images after test run
1341+ - install: ensure iscsi service is running to handle shutdown properly
1342+ - mdadm: handle write failures to sysfs entries when stopping mdadm
1343+ - vmtest: catch exceptions in curtin-log-print
1344+ - iscsi: use curtin storage config to disconnect iscsi targets
1345+ - vmtests: bump skip_by_date values out to give cloud-init SRU more time
1346+ - vmtest: get info about collected symlinks and then delete them.
1347+ - Update network cloud-init related skiptest dates, SRU still pending
1348+ - tests: Add CiTestCase common parent for all curtin tests.
1349+ - vmtests: Remove force flag for centos curthooks
1350+ - tools/jenkins-runner: improve tgtd cleanup logic
1351+ - tests: Drop EOL Wily Vivid and Yakkety tests.
1352+ - Disable yum plugins when installing packages, update ca-certs for https
1353+ - Rename centos_network_curthooks -> centos_apply_network_config.
1354+ - tests: in centos_defaults use write_files for grub serial.
1355+ - write_files: write files after extract, change write_files signature.
1356+ - pass network configuration through to target for ubuntu and centos
1357+ - tests: disable yakkety tests.
1358+ - tools/launch: automatically pass on proxy settings to curtin
1359+ - Add top level 'proxy' to config, deprecate top level http_proxy.
1360+ - tools/curtainer: fix to enable deb-src for -proposed.
1361+ - Use unshare to put chroot commands in own pid namespace.
1362+
1363+ -- Chad Smith <chad.smith@canonical.com> Fri, 06 Oct 2017 10:07:36 -0600
1364+
1365 curtin (0.1.0~bzr505-0ubuntu1~16.04.1) xenial-proposed; urgency=medium
1366
1367 * debian/new-upstream-snapshot: create tarball in .. otherwise it
1368
1369=== modified file 'doc/index.rst'
1370--- doc/index.rst 2016-10-03 18:42:29 +0000
1371+++ doc/index.rst 2017-10-06 16:35:22 +0000
1372@@ -17,6 +17,7 @@
1373 topics/apt_source
1374 topics/networking
1375 topics/storage
1376+ topics/curthooks
1377 topics/reporting
1378 topics/development
1379 topics/integration-testing
1380
1381=== modified file 'doc/topics/apt_source.rst'
1382--- doc/topics/apt_source.rst 2016-10-03 18:42:29 +0000
1383+++ doc/topics/apt_source.rst 2017-10-06 16:35:22 +0000
1384@@ -135,7 +135,9 @@
1385
1386 apt:
1387 sources:
1388- proposed.list: deb $MIRROR $RELEASE-proposed main restricted universe multiverse
1389+ proposed.list:
1390+ source: |
1391+ deb $MIRROR $RELEASE-proposed main restricted universe multiverse
1392
1393 * Make debug symbols available
1394
1395@@ -143,11 +145,12 @@
1396
1397 apt:
1398 sources:
1399- ddebs.list: |
1400- deb http://ddebs.ubuntu.com $RELEASE main restricted universe multiverse
1401-  deb http://ddebs.ubuntu.com $RELEASE-updates main restricted universe multiverse
1402-  deb http://ddebs.ubuntu.com $RELEASE-security main restricted universe multiverse
1403- deb http://ddebs.ubuntu.com $RELEASE-proposed main restricted universe multiverse
1404+ ddebs.list:
1405+ source: |
1406+ deb http://ddebs.ubuntu.com $RELEASE main restricted universe multiverse
1407+  deb http://ddebs.ubuntu.com $RELEASE-updates main restricted universe multiverse
1408+  deb http://ddebs.ubuntu.com $RELEASE-security main restricted universe multiverse
1409+ deb http://ddebs.ubuntu.com $RELEASE-proposed main restricted universe multiverse
1410
1411 Timing
1412 ~~~~~~
1413
1414=== modified file 'doc/topics/config.rst'
1415--- doc/topics/config.rst 2016-10-03 18:42:29 +0000
1416+++ doc/topics/config.rst 2017-10-06 16:35:22 +0000
1417@@ -24,6 +24,7 @@
1418 - multipath (``multipath``)
1419 - network (``network``)
1420 - power_state (``power_state``)
1421+- proxy (``proxy``)
1422 - reporting (``reporting``)
1423 - restore_dist_interfaces: (``restore_dist_interfaces``)
1424 - sources (``sources``)
1425@@ -177,6 +178,7 @@
1426 http_proxy
1427 ~~~~~~~~~~
1428 Curtin will export ``http_proxy`` value into the installer environment.
1429+**Deprecated**: This setting is deprecated in favor of ``proxy`` below.
1430
1431 **http_proxy**: *<HTTP Proxy URL>*
1432
1433@@ -348,6 +350,22 @@
1434 message: Bye Bye
1435
1436
1437+proxy
1438+~~~~~
1439+Curtin will put ``http_proxy``, ``https_proxy`` and ``no_proxy``
1440+into its install environment. This is in affect for curtin's process
1441+and subprocesses.
1442+
1443+**proxy**: A dictionary containing http_proxy, https_proxy, and no_proxy.
1444+
1445+**Example**::
1446+
1447+ proxy:
1448+ http_proxy: http://squid.proxy:3728/
1449+ https_proxy: http://squid.proxy:3728/
1450+ no_proxy: localhost,127.0.0.1,10.0.2.1
1451+
1452+
1453 reporting
1454 ~~~~~~~~~
1455 Configure installation reporting (see Reporting section for details).
1456
1457=== added file 'doc/topics/curthooks.rst'
1458--- doc/topics/curthooks.rst 1970-01-01 00:00:00 +0000
1459+++ doc/topics/curthooks.rst 2017-10-06 16:35:22 +0000
1460@@ -0,0 +1,109 @@
1461+========================================
1462+Curthooks / New OS Support
1463+========================================
1464+Curtin has built-in support for installation of Ubuntu.
1465+Other operating systems are supported through a mechanism called
1466+'curthooks' or 'curtin-hooks'.
1467+
1468+A curtin install runs through different stages. See the
1469+:ref:`Stages <stages>`
1470+documentation for function of each stage.
1471+The stages communicate with each other via data in a working directory and
1472+environment variables as described in
1473+:ref:`Command Environment`.
1474+
1475+Curtin handles partitioning, filesystem creation and target filesystem
1476+population for all operating systems. Curthooks are the mechanism provided
1477+so that the operating system can customize itself before reboot. This
1478+customization typically would need to include:
1479+
1480+ - ensuring that appropriate device drivers are loaded on first boot
1481+ - consuming the network interfaces file and applying its declarations.
1482+ - ensuring that necessary packages are installed to utilize storage
1483+ configuration or networking configuration.
1484+ - making the system boot (running grub-install or equivalent).
1485+
1486+Image provided curtin-hooks
1487+---------------------------
1488+An image provides curtin hooks support by containing a file
1489+``/curtin/curtin-hooks``.
1490+
1491+If an Ubuntu image image contains this path it will override the builtin
1492+curtin support.
1493+
1494+The ``curtin-hooks`` program should be executable in the filesystem and
1495+will be executed without any arguments. It will be executed in the install
1496+environment, *not* the target environment. A change of root to the
1497+target environment can be done with ``curtin in-target``.
1498+
1499+The hook is provided with some environment variables that can be used
1500+to find more information. See the :ref:`Command Environment` doc for
1501+details. Specifically interesting to this stage are:
1502+
1503+ - ``OUTPUT_NETWORK_CONFIG``: This is a path to the file created during
1504+ network discovery stage.
1505+ - ``OUTPUT_FSTAB``: This is a path to the file created during partitioning
1506+ stage.
1507+ - ``CONFIG``: This is a path to the curtin config file. It is provided so
1508+ that additional configuration could be provided through to the OS
1509+ customization.
1510+
1511+.. **TODO**: We should add 'PYTHON' or 'CURTIN_PYTHON' to this environment
1512+ so that the hook can easily run a python program with the same python
1513+ that curtin ran with (ie, python2 or python3).
1514+
1515+
1516+Networking configuration
1517+------------------------
1518+Access to the network configuration that is desired is inside the config
1519+and is in the format described in :ref:`networking`.
1520+
1521+.. TODO: We should guarantee that the presence
1522+ of network config v1 in the file OUTPUT_NETWORK_CONFIG.
1523+
1524+The curtin-hooks program must read the configuration from the
1525+path contained in ``OUTPUT_NETWORK_CONFIG`` and then set up
1526+the installed system to use it.
1527+
1528+If the installed system has cloud-init at version 17.1 or higher, it may
1529+be possible to simply copy this section into the target in
1530+``/etc/cloud/cloud.cfg.d/`` and let cloud-init render the correct
1531+networking on first boot.
1532+
1533+Storage configuration
1534+---------------------
1535+Access to the storage configuration that was set up is inside the config
1536+and is in the format described in :ref:`storage`.
1537+
1538+.. TODO: We should guarantee that the presence
1539+ of storage config v1 in the file OUTPUT_STORAGE_CONFIG.
1540+ This would mean the user would not have to pull it out
1541+ of CONFIG. We should guarantee its presence and format
1542+ even in the 'simple' path.
1543+
1544+To apply this storage configuration, the curthooks may need to:
1545+
1546+ * update /etc/fstab to add the expected mounts entries. The environment
1547+ variable ``OUTPUT_FSTAB`` contains a path to a file that may be suitable
1548+ for use.
1549+
1550+ * install any packages that are not already installed that are required
1551+ to boot with the provided storage config. For example, if the storage
1552+ layout includes raid you may need to install the mdadm package.
1553+
1554+ * update or create an initramfs.
1555+
1556+
1557+System boot
1558+-----------
1559+In Ubuntu, curtin will run 'grub-setup' and to install grub. This covers
1560+putting the bootloader onto the disk(s) that are marked as
1561+``grub_device``. The provided hook will need to do the equivalent
1562+operation.
1563+
1564+finalize hook
1565+-------------
1566+There is one other hook that curtin will invoke in an install, called
1567+``finalize``. This program is invoked in the same environment as
1568+``curtin-hooks`` above. It is intended to give the OS a final opportunity
1569+make updates before reboot. It is called before ``late_commands``.
1570
1571=== modified file 'doc/topics/integration-testing.rst'
1572--- doc/topics/integration-testing.rst 2017-06-12 20:39:06 +0000
1573+++ doc/topics/integration-testing.rst 2017-10-06 16:35:22 +0000
1574@@ -161,6 +161,12 @@
1575 - ``logs``: install and boot logs
1576 - ``collect``: data collected by the boot phase
1577
1578+- ``CURTIN_VMTEST_TAR_DISKS``: default 0
1579+
1580+ Vmtest writes out disk image files sparsely into a disks directory
1581+ If this flag is set to a non-zero number, vmtest will tar all disks in
1582+ the directory into a single disks.tar and remove the sparse disk files.
1583+
1584 - ``CURTIN_VMTEST_TOPDIR``: default $TMPDIR/vmtest-<timestamp>
1585
1586 Vmtest puts all test data under this value. By default, it creates
1587
1588=== modified file 'doc/topics/networking.rst'
1589--- doc/topics/networking.rst 2016-10-03 18:42:29 +0000
1590+++ doc/topics/networking.rst 2017-10-06 16:35:22 +0000
1591@@ -1,3 +1,5 @@
1592+.. _networking:
1593+
1594 ==========
1595 Networking
1596 ==========
1597
1598=== modified file 'doc/topics/overview.rst'
1599--- doc/topics/overview.rst 2016-10-03 18:42:29 +0000
1600+++ doc/topics/overview.rst 2017-10-06 16:35:22 +0000
1601@@ -4,6 +4,8 @@
1602
1603 Curtin is intended to be a bare bones "installer". Its goal is to take data from a source, and get it onto disk as quick as possible and then boot it. The key difference from traditional package based installers is that curtin assumes the thing its installing is intelligent and will do the right thing.
1604
1605+.. _Stages:
1606+
1607 Stages
1608 ------
1609 A usage of curtin will go through the following stages:
1610@@ -22,6 +24,32 @@
1611
1612 Curtin's assumption is that a fairly rich Linux (Ubuntu) environment is booted.
1613
1614+.. _Command Environment:
1615+
1616+Command Environment
1617+~~~~~~~~~~~~~~~~~~~
1618+Stages and commands invoked by curtin always have the following environment
1619+variables defined.
1620+
1621+- ``WORKING_DIR``: This is for inter-command state. It will be the same
1622+ directory for each command run and will only be deleted at the end of the
1623+ install. Files referenced in other environment variables will be in
1624+ this directory.
1625+
1626+- ``TARGET_MOUNT_POINT``: The path in the filesystem where the target
1627+ filesystem will be mounted.
1628+
1629+- ``OUTPUT_NETWORK_CONFIG``: After the network discovery stage, this file
1630+ should contain networking config information that should then be written
1631+ to the target.
1632+
1633+- ``OUTPUT_FSTAB``: After partitioning and filesystem creation, this file
1634+ will contain fstab(5) style content representing mounts.
1635+
1636+- ``CONFIG``: This variable contains a path to a yaml formatted file with
1637+ the fully rendered config.
1638+
1639+
1640 Early Commands
1641 ~~~~~~~~~~~~~~
1642 Early commands are executed on the system, and non-zero exit status will terminate the installation process. These commands are intended to be used for things like
1643@@ -48,32 +76,23 @@
1644 10_wipe_filesystems: curtin wipe --quick --all-unused-disks
1645 50_setup_raid: curtin disk-setup --all-disks raid0 /
1646
1647-**Command environment**
1648-
1649-Partitioning commands have the following environment variables available to them:
1650-
1651-- ``WORKING_DIR``: This is simply for some sort of inter-command state. It will be the same directory for each command run and will only be deleted at the end of all partitioning_commands.
1652-- ``OUTPUT_FSTAB``: This is the target path for a fstab file. After all partitioning commands have been run, a file should exist, formatted per fstab(5) that describes how the filesystems should be mounted.
1653-- ``TARGET_MOUNT_POINT``:
1654-
1655-
1656-Network Discovery and Setup
1657-~~~~~~~~~~~~~~~~~~~~~~~~~~~
1658-Networking is done in a similar fashion to partitioning. A series of commands, specified in the config are run. At the end of these commands, a interfaces(5) style file is expected to be written to ``OUTPUT_INTERFACES``.
1659-
1660-Note, that as with fstab, this file is not copied verbatim to the target filesystem, but rather made available to the OS customization stage. That stage may just copy the file verbatim, but may also parse it, and use that as input.
1661-
1662-**Config Example**::
1663-
1664- network_commands:
1665- 10_netconf: curtin network copy-existing
1666-
1667-**Command environment**
1668-
1669-Networking commands have the following environment variables available to them:
1670-
1671-- ``WORKING_DIR``: This is simply for some sort of inter-command state. It will be the same directory for each command run and will only be deleted at the end of all network_commands.
1672-- ``OUTPUT_INTERFACES``: This is the target path for an interfaces style file. After all commands have been run, a file should exist, formatted per interfaces(5) that describes the systems network setup.
1673+
1674+Network Discovery
1675+~~~~~~~~~~~~~~~~~
1676+Networking configuration is *discovered* in the 'network' stage.
1677+The default command run at this stage is ``curtin net-meta auto``. After
1678+execution, it will write the discovered networking to the file specified
1679+in the environment variable ``OUTPUT_NETWORK_CONFIG``. The format of this
1680+file is as described in :ref:`networking`.
1681+
1682+If curtin's config has a network section, the net-meta will simply parrot the
1683+data to the output file. If there is no network section, then its default
1684+behavior is to copy existing config from the running environment.
1685+
1686+Note, that as with fstab, this file is not copied verbatim to the target
1687+filesystem, but rather made available to the OS customization stage. That
1688+stage may just copy the file verbatim, but may also parse it, and apply the
1689+settings.
1690
1691 Extraction of sources
1692 ~~~~~~~~~~~~~~~~~~~~~
1693@@ -88,27 +107,6 @@
1694
1695 wget $URL | tar -Sxvzf
1696
1697-Hook for installed OS to customize itself
1698-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1699-After extraction of sources, the source that was extracted is then given a chance to customize itself for the system. This customization may include:
1700- - ensuring that appropriate device drivers are loaded on first boot
1701- - consuming the network interfaces file and applying its declarations.
1702- - ensuring that necessary packages
1703-
1704-**Config Example**::
1705-
1706- config_hook: {{TARGET_MP}}/opt/curtin/config-hook
1707-
1708-**Command environment**
1709- - ``INTERFACES``: This is a path to the file created during networking stage
1710- - ``FSTAB``: This is a path to the file created during partitioning stage
1711- - ``CONFIG``: This is a path to the curtin config file. It is provided so that additional configuration could be provided through to the OS customization.
1712-
1713-**Helpers**
1714-
1715-Curtin provides some helpers to make the OS customization easier.
1716- - `curtin in-target`: run the command while chrooted into the target.
1717-
1718 Final Commands
1719 ~~~~~~~~~~~~~~
1720
1721
1722=== modified file 'doc/topics/reporting.rst'
1723--- doc/topics/reporting.rst 2016-10-03 18:42:29 +0000
1724+++ doc/topics/reporting.rst 2017-10-06 16:35:22 +0000
1725@@ -10,6 +10,7 @@
1726 Reporting consists of notification of a series of 'events. Each event has:
1727 - **event_type**: 'start' or 'finish'
1728 - **description**: human readable text
1729+ - **level**: the log level of the event, DEBUG/INFO/WARN etc.
1730 - **name**: and id for this event
1731 - **result**: only present when event_type is 'finish', its value is one of "SUCCESS", "WARN", or "FAIL". A result of WARN indicates something is likely wrong, but a non-fatal error. A result of "FAIL" is fatal.
1732 - **origin**: literal value 'curtin'
1733@@ -75,6 +76,34 @@
1734 is specified then all messages with a lower priority than specified will be
1735 ignored. Default is INFO.
1736
1737+Journald Reporter
1738+-----------------
1739+
1740+The journald reporter sends the events to systemd's `journald`_. To enable,
1741+provide curtin with config like::
1742+
1743+ reporting:
1744+ mylistener:
1745+ type: journald
1746+ identifier: "my_identifier"
1747+ level: DEBUG
1748+
1749+The event's fields are mapped to fields of the resulting journal entry
1750+as follows:
1751+
1752+- **description** maps to **CURTIN_MESSAGE**
1753+- **level** maps to **PRIORITY**
1754+- **name** maps to **CURTIN_NAME**
1755+- **event_type** maps to **CURTIN_EVENT_TYPE**
1756+- **result**, if present, maps to **CURTIN_RESULT**
1757+
1758+The configured `identifier`, which defaults to "curtin_event", becomes
1759+the entry's **SYSLOG_IDENTIFIER**.
1760+
1761+The python-systemd package must be installed to use this handler.
1762+
1763+.. _`journald`: https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html
1764+
1765 Example Events
1766 ~~~~~~~~~~~~~~
1767 The following is an example event that would be posted::
1768
1769=== modified file 'doc/topics/storage.rst'
1770--- doc/topics/storage.rst 2017-06-12 20:39:06 +0000
1771+++ doc/topics/storage.rst 2017-10-06 16:35:22 +0000
1772@@ -1,3 +1,5 @@
1773+.. _storage:
1774+
1775 =======
1776 Storage
1777 =======
1778
1779=== modified file 'examples/network-ipv6-bond-vlan.yaml'
1780--- examples/network-ipv6-bond-vlan.yaml 2016-10-03 18:42:29 +0000
1781+++ examples/network-ipv6-bond-vlan.yaml 2017-10-06 16:35:22 +0000
1782@@ -3,10 +3,10 @@
1783 config:
1784 - name: interface0
1785 type: physical
1786- mac_address: BC:76:4E:06:96:B3
1787+ mac_address: bc:76:4e:06:96:b3
1788 - name: interface1
1789 type: physical
1790- mac_address: BC:76:4E:04:88:41
1791+ mac_address: bc:76:4e:04:88:41
1792 - type: bond
1793 bond_interfaces:
1794 - interface0
1795
1796=== modified file 'examples/tests/bonding_network.yaml'
1797--- examples/tests/bonding_network.yaml 2016-10-03 18:00:41 +0000
1798+++ examples/tests/bonding_network.yaml 2017-10-06 16:35:22 +0000
1799@@ -16,8 +16,7 @@
1800 mac_address: "52:54:00:12:34:04"
1801 # Bond.
1802 - type: bond
1803- name: bond0
1804- mac_address: "52:54:00:12:34:06"
1805+ name: bond1
1806 bond_interfaces:
1807 - interface1
1808 - interface2
1809@@ -26,8 +25,6 @@
1810 subnets:
1811 - type: static
1812 address: 10.23.23.2/24
1813- - type: static
1814- address: 10.23.24.2/24
1815
1816 curthooks_commands:
1817 # use curtin to disable open-iscsi ifupdown hooks for precise; they're
1818
1819=== modified file 'examples/tests/centos_basic.yaml'
1820--- examples/tests/centos_basic.yaml 2017-01-18 16:01:35 +0000
1821+++ examples/tests/centos_basic.yaml 2017-10-06 16:35:22 +0000
1822@@ -9,5 +9,6 @@
1823 mac_address: "52:54:00:12:34:00"
1824 subnets:
1825 - type: static
1826- address: 10.0.2.15/24
1827+ address: 10.0.2.15
1828+ netmask: 255.255.255.0
1829 gateway: 10.0.2.2
1830
1831=== added file 'examples/tests/centos_defaults.yaml'
1832--- examples/tests/centos_defaults.yaml 1970-01-01 00:00:00 +0000
1833+++ examples/tests/centos_defaults.yaml 2017-10-06 16:35:22 +0000
1834@@ -0,0 +1,91 @@
1835+hook_commands:
1836+ builtin: null
1837+
1838+# To force curtin to run centos_apply_network_config vmtest, uncomment
1839+# _ammend_centos_curthooks: True
1840+
1841+write_files:
1842+ grub_serial_console:
1843+ path: '/root/curtin-send-console-to-serial'
1844+ permissions: '0755'
1845+ owner: 'root:root'
1846+ content: |
1847+ # update grub1 and grub2 configs to write to serial console.
1848+ CONPARM="console=ttyS0,115200"
1849+ grub1conf="/boot/grub/grub.conf"
1850+ grub2conf="/boot/grub2/grub.cfg"
1851+ grub2def="/etc/default/grub"
1852+
1853+ rerror() { perror "$?" "$@"; return $r; }
1854+ perror() { local r="$1"; shift; error "$@"; return $r; }
1855+ error() { echo "GRUB_SERIAL:" "ERROR:" "$@" 1>&2; }
1856+ info() { echo "GRUB_SERIAL:" "$@" 1>&2; }
1857+ fail() { error "$@"; exit 1; }
1858+ bk() {
1859+ local ofile="$1" bk="$1.dist.curtin"
1860+ shift
1861+ [ -e "$ofile" ] || return 0
1862+ cp "$ofile" "$bk" || rerror "failed backup ($ofile -> $bk):" "$@";
1863+ }
1864+
1865+ update_grub1() {
1866+ local cfg="$1" r=""
1867+ [ -e "$cfg" ] ||
1868+ { info "no grub1 cfg '$cfg'"; return 0; }
1869+ bk "$cfg" "grub1 config" || return
1870+ if ! grep "^serial" "$cfg"; then
1871+ cat >> "$cfg" <<EOF
1872+ #curtin added
1873+ serial --unit=0 --speed=115200
1874+ terminal --timeout=2 serial console
1875+ EOF
1876+ r=$?
1877+ [ $r -eq 0 ] ||
1878+ { perror $r "failed to append to grub1 cfg '$cfg'"; return; }
1879+ fi
1880+ sed -i -e '/linux16/n' -e '/console=/n' \
1881+ -e "s/root=\([^ ]*\)/root=\1 ${CONPARM}/" "$cfg" ||
1882+ { rerror "failed to update grub1 cfg '$cfg'."; return; }
1883+ info "updated grub1 cfg '$cfg'."
1884+ }
1885+
1886+ update_grub2() {
1887+ local cfg="$1" defgrub="$2"
1888+ [ -e "$cfg" ] || { info "no grub2 config '$cfg'"; return 0; }
1889+ bk "$cfg" "grub2 config" || return
1890+ sed -i -e '/kernel/n' -e '/console=/n' \
1891+ -e "s/root=\([^ ]*\)/root=\1 ${CONPARM}/" "$cfg" ||
1892+ { rerror "failed to update grub2 '$cfg'"; return; }
1893+
1894+ # update /etc/default/grub. any GRUB_CMDLINE_LINUX remove
1895+ # any console= and add conparm at the beginning.
1896+ local var="GRUB_CMDLINE_LINUX" msg="updated grub2 '$cfg'."
1897+ if [ ! -e "$defgrub" ]; then
1898+ msg="$msg. no defaults file '$defgrub'."
1899+ else
1900+ bk "$defgrub" "grub2 defaults file" || return
1901+ msg="$msg. updated defaults file '$defgrub'."
1902+ sed -i \
1903+ -e "/$var=/!n" \
1904+ -e 's/console=[^ "]*//g' \
1905+ -e "s/$var=\"/$var=\"${CONPARM}/" "$defgrub" ||
1906+ { rerror "grub2 default update failed on $defgrub"; return; }
1907+ fi
1908+ info "$msg"
1909+ }
1910+
1911+ update_grub1 "$grub1conf" || fail "failed update grub1"
1912+ update_grub2 "$grub2conf" "$grub2def" || fail "failed update grub2"
1913+
1914+late_commands:
1915+ # centos66 images include grub 0.97 which will detect vmtests' ephemeral disk
1916+ # and the install disk which leaves grub configured with two disks. When
1917+ # vmtest reboots into installed disk, there is only one disk and the grub
1918+ # map is no longer valid. Here in 00_grub, we switch hd1 to hd0. MAAS
1919+ # is not affected as their ephemeral image (iscsi or http) is not discovered
1920+ # by grub and therefor the device.map doesn't contain a second device. Cent7
1921+ # has grub2 which uses root by UUID
1922+ 00_grub1_boot: [curtin, in-target, --, sed, -i.curtin, -e,
1923+ 's|(hd1,0)|(hd0,0)|g', /boot/grub/grub.conf]
1924+ # vmtest wants output to go to serial console so we update grub inside.
1925+ 00_grub_serial: [curtin, in-target, --, '/root/curtin-send-console-to-serial']
1926
1927=== added file 'examples/tests/journald_reporter.yaml'
1928--- examples/tests/journald_reporter.yaml 1970-01-01 00:00:00 +0000
1929+++ examples/tests/journald_reporter.yaml 2017-10-06 16:35:22 +0000
1930@@ -0,0 +1,20 @@
1931+reporting:
1932+ journald:
1933+ type: journald
1934+ level: DEBUG
1935+
1936+journal_cmds:
1937+ - &copy_journal_log |
1938+ journalctl -b -o short-precise --no-pager -t curtin_event \
1939+ > ${TARGET_MOUNT_POINT}/root/journalctl.curtin_events.log
1940+
1941+ # use sed to make the json file loadable (listify the json)
1942+ - &copy_journal_json |
1943+ journalctl -b -o json-pretty --no-pager -t curtin_event \
1944+ | sed -e '1i [' -e 's|^}|},|g' -e '$s|^},|}|' -e '$a]' \
1945+ > ${TARGET_MOUNT_POINT}/root/journalctl.curtin_events.json
1946+
1947+# extract the journald entries for curtin
1948+late_commands:
1949+ 00_copy_journal__log: [sh, -c, *copy_journal_log]
1950+ 01_copy_journal_json: [sh, -c, *copy_journal_json]
1951
1952=== modified file 'examples/tests/network_alias.yaml'
1953--- examples/tests/network_alias.yaml 2016-10-03 18:42:29 +0000
1954+++ examples/tests/network_alias.yaml 2017-10-06 16:35:22 +0000
1955@@ -8,29 +8,27 @@
1956 mac_address: "52:54:00:12:34:00"
1957 subnets:
1958 - type: static
1959- address: 192.168.1.2/24
1960- mtu: 1501
1961+ address: 10.47.98.1/24
1962 - type: static
1963 address: 2001:4800:78ff:1b:be76:4eff:fe06:ffac
1964 netmask: 'ffff:ffff:ffff:ffff::'
1965- mtu: 1480
1966 # multi_v4_alias: multiple v4 addrs on same interface
1967 - type: physical
1968 name: interface1
1969 mac_address: "52:54:00:12:34:02"
1970 subnets:
1971 - type: static
1972- address: 192.168.2.2/22
1973+ address: 192.168.20.2/24
1974 routes:
1975- - network: 192.168.0.0
1976- netmask: 255.255.252.0
1977- gateway: 192.168.2.1
1978+ - gateway: 192.168.20.1
1979+ netmask: 255.255.255.0
1980+ network: 10.242.47.0
1981 - type: static
1982- address: 10.23.23.7/23
1983+ address: 10.23.22.7/23
1984 routes:
1985- - gateway: 10.23.23.1
1986- netmask: 255.255.254.0
1987- network: 10.23.22.0
1988+ - gateway: 10.23.22.2
1989+ netmask: 255.255.255.0
1990+ network: 10.49.253.0
1991 # multi_v6_alias: multiple v6 addrs on same interface
1992 - type: physical
1993 name: interface2
1994@@ -51,17 +49,17 @@
1995 mac_address: "52:54:00:12:34:06"
1996 subnets:
1997 - type: static
1998- address: 192.168.7.7/22
1999+ address: 192.168.80.8/24
2000 routes:
2001- - network: 192.168.0.0
2002- netmask: 255.255.252.0
2003- gateway: 192.168.7.1
2004+ - gateway: 192.168.80.1
2005+ netmask: 255.255.255.0
2006+ network: 10.189.34.0
2007 - type: static
2008- address: 10.99.99.23/23
2009+ address: 10.99.10.23/23
2010 routes:
2011- - gateway: 10.99.99.1
2012- netmask: 255.255.254.0
2013- network: 10.99.98.0
2014+ - gateway: 10.99.10.1
2015+ netmask: 255.255.255.0
2016+ network: 10.77.154.0
2017 - type: static
2018 address: 2001:4800:78ff:1b:be76:4eff:beef:4000
2019 netmask: 'ffff:ffff:ffff:ffff::'
2020@@ -86,17 +84,17 @@
2021 address: 2001:4800:78ff:1b:be76:4eff:debe:9000
2022 netmask: 'ffff:ffff:ffff:ffff::'
2023 - type: static
2024- address: 192.168.100.100/22
2025+ address: 192.168.100.100/24
2026 routes:
2027- - network: 192.168.0.0
2028- netmask: 255.255.252.0
2029- gateway: 192.168.100.1
2030+ - gateway: 192.168.100.1
2031+ netmask: 255.255.255.0
2032+ network: 10.28.219.0
2033 - type: static
2034 address: 10.17.142.2/23
2035 routes:
2036 - gateway: 10.17.142.1
2037- netmask: 255.255.254.0
2038- network: 10.17.142.0
2039+ netmask: 255.255.255.0
2040+ network: 10.82.49.0
2041 # multi_v6_and_v4_mix_order: multiple v4 and v6 addr, mixed order
2042 - type: physical
2043 name: interface5
2044@@ -109,17 +107,17 @@
2045 address: 2001:4800:78ff:1b:be76:4eff:baaf:c000
2046 netmask: 'ffff:ffff:ffff:ffff::'
2047 - type: static
2048- address: 192.168.200.200/22
2049+ address: 192.168.200.200/24
2050 routes:
2051- - network: 192.168.0.0
2052- netmask: 255.255.252.0
2053- gateway: 192.168.200.1
2054+ - gateway: 192.168.200.1
2055+ netmask: 255.255.255.0
2056+ network: 10.71.23.0
2057 - type: static
2058 address: 10.252.2.2/23
2059 routes:
2060 - gateway: 10.252.2.1
2061- netmask: 255.255.254.0
2062- network: 10.252.2.0
2063+ netmask: 255.255.255.0
2064+ network: 10.3.7.0
2065 - type: static
2066 address: 2001:4800:78ff:1b:be76:4eff:baaf:b000
2067 netmask: 'ffff:ffff:ffff:ffff::'
2068
2069=== modified file 'examples/tests/network_static_routes.yaml'
2070--- examples/tests/network_static_routes.yaml 2017-02-08 22:22:44 +0000
2071+++ examples/tests/network_static_routes.yaml 2017-10-06 16:35:22 +0000
2072@@ -10,18 +10,13 @@
2073 - address: 172.23.31.42/26
2074 gateway: 172.23.31.2
2075 type: static
2076- - type: route
2077- id: 4
2078- metric: 0
2079- destination: 10.0.0.0/12
2080- gateway: 172.23.31.1
2081- - type: route
2082- id: 5
2083- metric: 0
2084- destination: 192.168.0.0/16
2085- gateway: 172.23.31.1
2086- - type: route
2087- id: 6
2088- metric: 1
2089- destination: 10.200.0.0/16
2090- gateway: 172.23.31.1
2091+ routes:
2092+ - gateway: 172.23.31.1
2093+ network: 10.0.0.0/12
2094+ metric: 0
2095+ - gateway: 172.23.31.1
2096+ network: 192.168.0.0/16
2097+ metric: 0
2098+ - gateway: 172.23.31.1
2099+ network: 10.200.0.0/16
2100+ metric: 1
2101
2102=== added file 'examples/tests/network_v2_passthrough.yaml'
2103--- examples/tests/network_v2_passthrough.yaml 1970-01-01 00:00:00 +0000
2104+++ examples/tests/network_v2_passthrough.yaml 2017-10-06 16:35:22 +0000
2105@@ -0,0 +1,8 @@
2106+showtrace: true
2107+network:
2108+ version: 2
2109+ ethernets:
2110+ interface0:
2111+ match:
2112+ mac_address: "52:54:00:12:34:00"
2113+ dhcp4: true
2114
2115=== modified file 'setup.py'
2116--- setup.py 2016-10-03 18:42:29 +0000
2117+++ setup.py 2017-10-06 16:35:22 +0000
2118@@ -1,6 +1,7 @@
2119 from distutils.core import setup
2120 from glob import glob
2121 import os
2122+import sys
2123
2124 import curtin
2125
2126@@ -8,6 +9,19 @@
2127 def is_f(p):
2128 return os.path.isfile(p)
2129
2130+
2131+def in_virtualenv():
2132+ try:
2133+ if sys.real_prefix == sys.prefix:
2134+ return False
2135+ else:
2136+ return True
2137+ except AttributeError:
2138+ return False
2139+
2140+
2141+USR = "usr" if in_virtualenv() else "/usr"
2142+
2143 setup(
2144 name="curtin",
2145 description='The curtin installer',
2146@@ -27,9 +41,9 @@
2147 ],
2148 scripts=glob('bin/*'),
2149 data_files=[
2150- ('/usr/share/doc/curtin',
2151+ (USR + '/share/doc/curtin',
2152 [f for f in glob('doc/*') if is_f(f)]),
2153- ('/usr/lib/curtin/helpers',
2154+ (USR + '/lib/curtin/helpers',
2155 [f for f in glob('helpers/*') if is_f(f)])
2156 ]
2157 )
2158
2159=== modified file 'tests/unittests/helpers.py'
2160--- tests/unittests/helpers.py 2017-02-08 22:22:44 +0000
2161+++ tests/unittests/helpers.py 2017-10-06 16:35:22 +0000
2162@@ -19,6 +19,10 @@
2163 import imp
2164 import importlib
2165 import mock
2166+import os
2167+import shutil
2168+import tempfile
2169+from unittest import TestCase
2170
2171
2172 def builtin_module_name():
2173@@ -43,3 +47,35 @@
2174 m_patch = '{}.open'.format(mod_name)
2175 with mock.patch(m_patch, m_open, create=True):
2176 yield m_open
2177+
2178+
2179+class CiTestCase(TestCase):
2180+ """Common testing class which all curtin unit tests subclass."""
2181+
2182+ def add_patch(self, target, attr, **kwargs):
2183+ """Patches specified target object and sets it as attr on test
2184+ instance also schedules cleanup"""
2185+ if 'autospec' not in kwargs:
2186+ kwargs['autospec'] = True
2187+ m = mock.patch(target, **kwargs)
2188+ p = m.start()
2189+ self.addCleanup(m.stop)
2190+ setattr(self, attr, p)
2191+
2192+ def tmp_dir(self, dir=None, cleanup=True):
2193+ """Return a full path to a temporary directory for the test run."""
2194+ if dir is None:
2195+ tmpd = tempfile.mkdtemp(
2196+ prefix="curtin-ci-%s." % self.__class__.__name__)
2197+ else:
2198+ tmpd = tempfile.mkdtemp(dir=dir)
2199+ self.addCleanup(shutil.rmtree, tmpd)
2200+ return tmpd
2201+
2202+ def tmp_path(self, path, _dir=None):
2203+ # return an absolute path to 'path' under dir.
2204+ # if dir is None, one will be created with tmp_dir()
2205+ # the file is not created or modified.
2206+ if _dir is None:
2207+ _dir = self.tmp_dir()
2208+ return os.path.normpath(os.path.abspath(os.path.join(_dir, path)))
2209
2210=== modified file 'tests/unittests/test_apt_custom_sources_list.py'
2211--- tests/unittests/test_apt_custom_sources_list.py 2016-10-03 18:42:29 +0000
2212+++ tests/unittests/test_apt_custom_sources_list.py 2017-10-06 16:35:22 +0000
2213@@ -3,10 +3,7 @@
2214 """
2215 import logging
2216 import os
2217-import shutil
2218-import tempfile
2219
2220-from unittest import TestCase
2221
2222 import yaml
2223 import mock
2224@@ -14,6 +11,7 @@
2225
2226 from curtin import util
2227 from curtin.commands import apt_config
2228+from .helpers import CiTestCase
2229
2230 LOG = logging.getLogger(__name__)
2231
2232@@ -85,12 +83,11 @@
2233 """)
2234
2235
2236-class TestAptSourceConfigSourceList(TestCase):
2237+class TestAptSourceConfigSourceList(CiTestCase):
2238 """TestAptSourceConfigSourceList - Class to test sources list rendering"""
2239 def setUp(self):
2240 super(TestAptSourceConfigSourceList, self).setUp()
2241- self.new_root = tempfile.mkdtemp()
2242- self.addCleanup(shutil.rmtree, self.new_root)
2243+ self.new_root = self.tmp_dir()
2244 # self.patchUtils(self.new_root)
2245
2246 @staticmethod
2247
2248=== modified file 'tests/unittests/test_apt_source.py'
2249--- tests/unittests/test_apt_source.py 2017-03-01 16:13:56 +0000
2250+++ tests/unittests/test_apt_source.py 2017-10-06 16:35:22 +0000
2251@@ -4,11 +4,8 @@
2252 import glob
2253 import os
2254 import re
2255-import shutil
2256 import socket
2257-import tempfile
2258
2259-from unittest import TestCase
2260
2261 import mock
2262 from mock import call
2263@@ -16,6 +13,7 @@
2264 from curtin import util
2265 from curtin import gpg
2266 from curtin.commands import apt_config
2267+from .helpers import CiTestCase
2268
2269
2270 EXPECTEDKEY = u"""-----BEGIN PGP PUBLIC KEY BLOCK-----
2271@@ -62,14 +60,13 @@
2272 ChrootableTargetStr = "curtin.commands.apt_config.util.ChrootableTarget"
2273
2274
2275-class TestAptSourceConfig(TestCase):
2276+class TestAptSourceConfig(CiTestCase):
2277 """ TestAptSourceConfig
2278 Main Class to test apt configs
2279 """
2280 def setUp(self):
2281 super(TestAptSourceConfig, self).setUp()
2282- self.tmp = tempfile.mkdtemp()
2283- self.addCleanup(shutil.rmtree, self.tmp)
2284+ self.tmp = self.tmp_dir()
2285 self.aptlistfile = os.path.join(self.tmp, "single-deb.list")
2286 self.aptlistfile2 = os.path.join(self.tmp, "single-deb2.list")
2287 self.aptlistfile3 = os.path.join(self.tmp, "single-deb3.list")
2288@@ -930,7 +927,7 @@
2289 orig, apt_config.disable_suites(["proposed"], orig, rel))
2290
2291
2292-class TestDebconfSelections(TestCase):
2293+class TestDebconfSelections(CiTestCase):
2294
2295 @mock.patch("curtin.commands.apt_config.debconf_set_selections")
2296 def test_no_set_sel_if_none_to_set(self, m_set_sel):
2297
2298=== modified file 'tests/unittests/test_basic.py'
2299--- tests/unittests/test_basic.py 2013-07-29 16:12:09 +0000
2300+++ tests/unittests/test_basic.py 2017-10-06 16:35:22 +0000
2301@@ -1,7 +1,7 @@
2302-from unittest import TestCase
2303-
2304-
2305-class TestImport(TestCase):
2306+from .helpers import CiTestCase
2307+
2308+
2309+class TestImport(CiTestCase):
2310 def test_import(self):
2311 import curtin
2312 self.assertFalse(getattr(curtin, 'BOGUS_ENTRY', None))
2313
2314=== modified file 'tests/unittests/test_block.py'
2315--- tests/unittests/test_block.py 2017-06-12 20:39:06 +0000
2316+++ tests/unittests/test_block.py 2017-10-06 16:35:22 +0000
2317@@ -1,19 +1,16 @@
2318-from unittest import TestCase
2319 import functools
2320 import os
2321 import mock
2322-import tempfile
2323-import shutil
2324 import sys
2325
2326 from collections import OrderedDict
2327
2328-from .helpers import simple_mocked_open
2329+from .helpers import CiTestCase, simple_mocked_open
2330 from curtin import util
2331 from curtin import block
2332
2333
2334-class TestBlock(TestCase):
2335+class TestBlock(CiTestCase):
2336
2337 @mock.patch("curtin.block.util")
2338 def test_get_volume_uuid(self, mock_util):
2339@@ -103,7 +100,7 @@
2340 block.lookup_disk(serial)
2341
2342
2343-class TestSysBlockPath(TestCase):
2344+class TestSysBlockPath(CiTestCase):
2345 @mock.patch("curtin.block.get_blockdev_for_partition")
2346 @mock.patch("os.path.exists")
2347 def test_existing_valid_devname(self, m_os_path_exists, m_get_blk):
2348@@ -177,19 +174,13 @@
2349 block.sys_block_path('/dev/cciss/c0d0p1'))
2350
2351
2352-class TestWipeFile(TestCase):
2353+class TestWipeFile(CiTestCase):
2354 def __init__(self, *args, **kwargs):
2355 super(TestWipeFile, self).__init__(*args, **kwargs)
2356
2357- def tfile(self, *args):
2358- # return a temp file in a dir that will be cleaned up
2359- tmpdir = tempfile.mkdtemp()
2360- self.addCleanup(shutil.rmtree, tmpdir)
2361- return os.path.sep.join([tmpdir] + list(args))
2362-
2363 def test_non_exist_raises_file_not_found(self):
2364 try:
2365- p = self.tfile("enofile")
2366+ p = self.tmp_path("enofile")
2367 block.wipe_file(p)
2368 raise Exception("%s did not raise exception" % p)
2369 except Exception as e:
2370@@ -198,7 +189,7 @@
2371
2372 def test_non_exist_dir_raises_file_not_found(self):
2373 try:
2374- p = self.tfile("enodir", "file")
2375+ p = self.tmp_path(os.path.sep.join(["enodir", "file"]))
2376 block.wipe_file(p)
2377 raise Exception("%s did not raise exception" % p)
2378 except Exception as e:
2379@@ -207,7 +198,7 @@
2380
2381 def test_default_is_zero(self):
2382 flen = 1024
2383- myfile = self.tfile("def_zero")
2384+ myfile = self.tmp_path("def_zero")
2385 util.write_file(myfile, flen * b'\1', omode="wb")
2386 block.wipe_file(myfile)
2387 found = util.load_file(myfile, decode=False)
2388@@ -219,7 +210,7 @@
2389 def reader(size):
2390 return size * b'\1'
2391
2392- myfile = self.tfile("reader_used")
2393+ myfile = self.tmp_path("reader_used")
2394 # populate with nulls
2395 util.write_file(myfile, flen * b'\0', omode="wb")
2396 block.wipe_file(myfile, reader=reader, buflen=flen)
2397@@ -236,15 +227,15 @@
2398 data['x'] = data['x'][size:]
2399 return buf
2400
2401- myfile = self.tfile("reader_twice")
2402+ myfile = self.tmp_path("reader_twice")
2403 util.write_file(myfile, flen * b'\xff', omode="wb")
2404 block.wipe_file(myfile, reader=reader, buflen=20)
2405 found = util.load_file(myfile, decode=False)
2406 self.assertEqual(found, expected)
2407
2408 def test_reader_fhandle(self):
2409- srcfile = self.tfile("fhandle_src")
2410- trgfile = self.tfile("fhandle_trg")
2411+ srcfile = self.tmp_path("fhandle_src")
2412+ trgfile = self.tmp_path("fhandle_trg")
2413 data = '\n'.join(["this is source file." for f in range(0, 10)] + [])
2414 util.write_file(srcfile, data)
2415 util.write_file(trgfile, 'a' * len(data))
2416@@ -254,7 +245,7 @@
2417 self.assertEqual(data, found)
2418
2419 def test_exclusive_open_raise_missing(self):
2420- myfile = self.tfile("no-such-file")
2421+ myfile = self.tmp_path("no-such-file")
2422
2423 with self.assertRaises(ValueError):
2424 with block.exclusive_open(myfile) as fp:
2425@@ -265,7 +256,7 @@
2426 @mock.patch('os.open')
2427 def test_exclusive_open(self, mock_os_open, mock_os_fdopen, mock_os_close):
2428 flen = 1024
2429- myfile = self.tfile("my_exclusive_file")
2430+ myfile = self.tmp_path("my_exclusive_file")
2431 util.write_file(myfile, flen * b'\1', omode="wb")
2432 mock_fd = 3
2433 mock_os_open.return_value = mock_fd
2434@@ -288,7 +279,7 @@
2435 mock_os_close,
2436 mock_util_fuser):
2437 flen = 1024
2438- myfile = self.tfile("my_exclusive_file")
2439+ myfile = self.tmp_path("my_exclusive_file")
2440 util.write_file(myfile, flen * b'\1', omode="wb")
2441 mock_os_open.side_effect = OSError("NO_O_EXCL")
2442 mock_holders.return_value = ['md1']
2443@@ -310,7 +301,7 @@
2444 def test_exclusive_open_fdopen_failure(self, mock_os_open,
2445 mock_os_fdopen, mock_os_close):
2446 flen = 1024
2447- myfile = self.tfile("my_exclusive_file")
2448+ myfile = self.tmp_path("my_exclusive_file")
2449 util.write_file(myfile, flen * b'\1', omode="wb")
2450 mock_fd = 3
2451 mock_os_open.return_value = mock_fd
2452@@ -328,7 +319,7 @@
2453 self.assertEqual([], mock_os_close.call_args_list)
2454
2455
2456-class TestWipeVolume(TestCase):
2457+class TestWipeVolume(CiTestCase):
2458 dev = '/dev/null'
2459
2460 @mock.patch('curtin.block.lvm')
2461@@ -366,7 +357,7 @@
2462 block.wipe_volume(self.dev, mode='invalidmode')
2463
2464
2465-class TestBlockKnames(TestCase):
2466+class TestBlockKnames(CiTestCase):
2467 """Tests for some of the kname functions in block"""
2468 def test_determine_partition_kname(self):
2469 part_knames = [(('sda', 1), 'sda1'),
2470@@ -430,7 +421,7 @@
2471 block.kname_to_path(kname)
2472
2473
2474-class TestPartTableSignature(TestCase):
2475+class TestPartTableSignature(CiTestCase):
2476 blockdev = '/dev/null'
2477 dos_content = b'\x00' * 0x1fe + b'\x55\xAA' + b'\x00' * 0xf00
2478 gpt_content = b'\x00' * 0x200 + b'EFI PART' + b'\x00' * (0x200 - 8)
2479@@ -493,7 +484,7 @@
2480 block.check_efi_signature(self.blockdev))
2481
2482
2483-class TestNonAscii(TestCase):
2484+class TestNonAscii(CiTestCase):
2485 @mock.patch('curtin.block.util.subp')
2486 def test_lsblk(self, mock_subp):
2487 # lsblk can write non-ascii data, causing shlex to blow up
2488@@ -519,14 +510,7 @@
2489 block.blkid()
2490
2491
2492-class TestSlaveKnames(TestCase):
2493- def add_patch(self, target, attr, autospec=True):
2494- """Patches specified target object and sets it as attr on test
2495- instance also schedules cleanup"""
2496- m = mock.patch(target, autospec=autospec)
2497- p = m.start()
2498- self.addCleanup(m.stop)
2499- setattr(self, attr, p)
2500+class TestSlaveKnames(CiTestCase):
2501
2502 def setUp(self):
2503 super(TestSlaveKnames, self).setUp()
2504
2505=== modified file 'tests/unittests/test_block_iscsi.py'
2506--- tests/unittests/test_block_iscsi.py 2017-06-12 20:39:06 +0000
2507+++ tests/unittests/test_block_iscsi.py 2017-10-06 16:35:22 +0000
2508@@ -1,23 +1,13 @@
2509 import mock
2510+import os
2511
2512-from unittest import TestCase
2513 from curtin.block import iscsi
2514-
2515-
2516-class IscsiTestBase(TestCase):
2517- def setUp(self):
2518- super(IscsiTestBase, self).setUp()
2519-
2520- def add_patch(self, target, attr):
2521- """Patches specified target object and sets it as attr on test
2522- instance also schedules cleanup"""
2523- m = mock.patch(target, autospec=True)
2524- p = m.start()
2525- self.addCleanup(m.stop)
2526- setattr(self, attr, p)
2527-
2528-
2529-class TestBlockIscsiPortalParsing(IscsiTestBase):
2530+from curtin import util
2531+from .helpers import CiTestCase
2532+
2533+
2534+class TestBlockIscsiPortalParsing(CiTestCase):
2535+
2536 def test_iscsi_portal_parsing_string(self):
2537 with self.assertRaisesRegexp(ValueError, 'not a string'):
2538 iscsi.assert_valid_iscsi_portal(1234)
2539@@ -490,7 +480,7 @@
2540 self.assertEquals(i.target, 'iqn.2017-04.com.example.test:target-name')
2541
2542
2543-class TestBlockIscsiVolPath(IscsiTestBase):
2544+class TestBlockIscsiVolPath(CiTestCase):
2545 # non-iscsi backed disk returns false
2546 # regular iscsi-backed disk returns true
2547 # layered setup without an iscsi member returns false
2548@@ -569,4 +559,183 @@
2549 with self.assertRaises(ValueError):
2550 iscsi.volpath_is_iscsi(None)
2551
2552+
2553+class TestBlockIscsiDiskFromConfig(CiTestCase):
2554+ # Test iscsi parsing of storage config for iscsi configure disks
2555+
2556+ def setUp(self):
2557+ super(TestBlockIscsiDiskFromConfig, self).setUp()
2558+ self.add_patch('curtin.block.iscsi.util.subp', 'mock_subp')
2559+
2560+ def test_parse_iscsi_disk_from_config(self):
2561+ """Test parsing iscsi volume path creates the same iscsi disk"""
2562+ target = 'curtin-659d5f45-4f23-46cb-b826-f2937b896e09'
2563+ iscsi_path = 'iscsi:10.245.168.20::20112:1:' + target
2564+ cfg = {
2565+ 'storage': {
2566+ 'config': [{'type': 'disk',
2567+ 'id': 'iscsidev1',
2568+ 'path': iscsi_path,
2569+ 'name': 'iscsi_disk1',
2570+ 'ptable': 'msdos',
2571+ 'wipe': 'superblock'}]
2572+ }
2573+ }
2574+ expected_iscsi_disk = iscsi.IscsiDisk(iscsi_path)
2575+ iscsi_disk = iscsi.get_iscsi_disks_from_config(cfg).pop()
2576+ # utilize IscsiDisk str method for equality check
2577+ self.assertEqual(str(expected_iscsi_disk), str(iscsi_disk))
2578+
2579+ def test_parse_iscsi_disk_from_config_no_iscsi(self):
2580+ """Test parsing storage config with no iscsi disks included"""
2581+ cfg = {
2582+ 'storage': {
2583+ 'config': [{'type': 'disk',
2584+ 'id': 'ssd1',
2585+ 'path': 'dev/slash/foo1',
2586+ 'name': 'the-fast-one',
2587+ 'ptable': 'gpt',
2588+ 'wipe': 'superblock'}]
2589+ }
2590+ }
2591+ expected_iscsi_disks = []
2592+ iscsi_disks = iscsi.get_iscsi_disks_from_config(cfg)
2593+ self.assertEqual(expected_iscsi_disks, iscsi_disks)
2594+
2595+ def test_parse_iscsi_disk_from_config_invalid_iscsi(self):
2596+ """Test parsing storage config with no iscsi disks included"""
2597+ cfg = {
2598+ 'storage': {
2599+ 'config': [{'type': 'disk',
2600+ 'id': 'iscsidev2',
2601+ 'path': 'iscsi:garbage',
2602+ 'name': 'noob-city',
2603+ 'ptable': 'msdos',
2604+ 'wipe': 'superblock'}]
2605+ }
2606+ }
2607+ with self.assertRaises(ValueError):
2608+ iscsi.get_iscsi_disks_from_config(cfg)
2609+
2610+ def test_parse_iscsi_disk_from_config_empty(self):
2611+ """Test parse_iscsi_disks handles empty/invalid config"""
2612+ expected_iscsi_disks = []
2613+ iscsi_disks = iscsi.get_iscsi_disks_from_config({})
2614+ self.assertEqual(expected_iscsi_disks, iscsi_disks)
2615+
2616+ cfg = {'storage': {'config': []}}
2617+ iscsi_disks = iscsi.get_iscsi_disks_from_config(cfg)
2618+ self.assertEqual(expected_iscsi_disks, iscsi_disks)
2619+
2620+ def test_parse_iscsi_disk_from_config_none(self):
2621+ """Test parse_iscsi_disks handles no config"""
2622+ expected_iscsi_disks = []
2623+ iscsi_disks = iscsi.get_iscsi_disks_from_config({})
2624+ self.assertEqual(expected_iscsi_disks, iscsi_disks)
2625+
2626+ cfg = None
2627+ iscsi_disks = iscsi.get_iscsi_disks_from_config(cfg)
2628+ self.assertEqual(expected_iscsi_disks, iscsi_disks)
2629+
2630+
2631+class TestBlockIscsiDisconnect(CiTestCase):
2632+ # test that when disconnecting iscsi targets we
2633+ # check that the target has an active session before
2634+ # issuing a disconnect command
2635+
2636+ def setUp(self):
2637+ super(TestBlockIscsiDisconnect, self).setUp()
2638+ self.add_patch('curtin.block.iscsi.util.subp', 'mock_subp')
2639+ self.add_patch('curtin.block.iscsi.iscsiadm_sessions',
2640+ 'mock_iscsi_sessions')
2641+ # fake target_root + iscsi nodes dir
2642+ self.target_path = self.tmp_dir()
2643+ self.iscsi_nodes = os.path.join(self.target_path, 'etc/iscsi/nodes')
2644+ util.ensure_dir(self.iscsi_nodes)
2645+
2646+ def _fmt_disconnect(self, target, portal):
2647+ return ['iscsiadm', '--mode=node', '--targetname=%s' % target,
2648+ '--portal=%s' % portal, '--logout']
2649+
2650+ def _setup_nodes(self, sessions, connection):
2651+ # setup iscsi_nodes dir (<fakeroot>/etc/iscsi/nodes) with content
2652+ for s in sessions:
2653+ sdir = os.path.join(self.iscsi_nodes, s)
2654+ connpath = os.path.join(sdir, connection)
2655+ util.ensure_dir(sdir)
2656+ util.write_file(connpath, content="")
2657+
2658+ def test_disconnect_target_disk(self):
2659+ """Test iscsi disconnecting multiple sessions, all present"""
2660+
2661+ sessions = [
2662+ 'curtin-53ab23ff-a887-449a-80a8-288151208091',
2663+ 'curtin-94b62de1-c579-42c0-879e-8a28178e64c5',
2664+ 'curtin-556aeecd-a227-41b7-83d7-2bb471c574b4',
2665+ 'curtin-fd0f644b-7858-420f-9997-3ea2aefe87b9'
2666+ ]
2667+ connection = '10.245.168.20,16395,1'
2668+ self._setup_nodes(sessions, connection)
2669+
2670+ self.mock_iscsi_sessions.return_value = "\n".join(sessions)
2671+
2672+ iscsi.disconnect_target_disks(self.target_path)
2673+
2674+ expected_calls = []
2675+ for session in sessions:
2676+ (host, port, _) = connection.split(',')
2677+ disconnect = self._fmt_disconnect(session, "%s:%s" % (host, port))
2678+ calls = [
2679+ mock.call(['sync']),
2680+ mock.call(disconnect, capture=True, log_captured=True),
2681+ mock.call(['udevadm', 'settle']),
2682+ ]
2683+ expected_calls.extend(calls)
2684+
2685+ self.mock_subp.assert_has_calls(expected_calls, any_order=True)
2686+
2687+ def test_disconnect_target_disk_skip_disconnected(self):
2688+ """Test iscsi does not attempt to disconnect already closed sessions"""
2689+ sessions = [
2690+ 'curtin-53ab23ff-a887-449a-80a8-288151208091',
2691+ 'curtin-94b62de1-c579-42c0-879e-8a28178e64c5',
2692+ 'curtin-556aeecd-a227-41b7-83d7-2bb471c574b4',
2693+ 'curtin-fd0f644b-7858-420f-9997-3ea2aefe87b9'
2694+ ]
2695+ connection = '10.245.168.20,16395,1'
2696+ self._setup_nodes(sessions, connection)
2697+ # Test with all sessions are already disconnected
2698+ self.mock_iscsi_sessions.return_value = ""
2699+
2700+ iscsi.disconnect_target_disks(self.target_path)
2701+
2702+ self.mock_subp.assert_has_calls([], any_order=True)
2703+
2704+ @mock.patch('curtin.block.iscsi.iscsiadm_logout')
2705+ def test_disconnect_target_disk_raises_runtime_error(self, mock_logout):
2706+ """Test iscsi raises RuntimeError if we fail to logout"""
2707+ sessions = [
2708+ 'curtin-53ab23ff-a887-449a-80a8-288151208091',
2709+ ]
2710+ connection = '10.245.168.20,16395,1'
2711+ self._setup_nodes(sessions, connection)
2712+ self.mock_iscsi_sessions.return_value = "\n".join(sessions)
2713+ mock_logout.side_effect = util.ProcessExecutionError()
2714+
2715+ with self.assertRaises(RuntimeError):
2716+ iscsi.disconnect_target_disks(self.target_path)
2717+
2718+ expected_calls = []
2719+ for session in sessions:
2720+ (host, port, _) = connection.split(',')
2721+ disconnect = self._fmt_disconnect(session, "%s:%s" % (host, port))
2722+ calls = [
2723+ mock.call(['sync']),
2724+ mock.call(disconnect, capture=True, log_captured=True),
2725+ mock.call(['udevadm', 'settle']),
2726+ ]
2727+ expected_calls.extend(calls)
2728+
2729+ self.mock_subp.assert_has_calls([], any_order=True)
2730+
2731 # vi: ts=4 expandtab syntax=python
2732
2733=== modified file 'tests/unittests/test_block_lvm.py'
2734--- tests/unittests/test_block_lvm.py 2016-10-03 18:42:29 +0000
2735+++ tests/unittests/test_block_lvm.py 2017-10-06 16:35:22 +0000
2736@@ -1,10 +1,10 @@
2737 from curtin.block import lvm
2738
2739-from unittest import TestCase
2740+from .helpers import CiTestCase
2741 import mock
2742
2743
2744-class TestBlockLvm(TestCase):
2745+class TestBlockLvm(CiTestCase):
2746 vg_name = 'ubuntu-volgroup'
2747
2748 @mock.patch('curtin.block.lvm.util')
2749
2750=== modified file 'tests/unittests/test_block_mdadm.py'
2751--- tests/unittests/test_block_mdadm.py 2017-06-12 20:39:06 +0000
2752+++ tests/unittests/test_block_mdadm.py 2017-10-06 16:35:22 +0000
2753@@ -1,27 +1,15 @@
2754-from unittest import TestCase
2755 from mock import call, patch
2756 from curtin.block import dev_short
2757 from curtin.block import mdadm
2758 from curtin import util
2759+from .helpers import CiTestCase
2760 import os
2761 import subprocess
2762 import textwrap
2763
2764
2765-class MdadmTestBase(TestCase):
2766- def setUp(self):
2767- super(MdadmTestBase, self).setUp()
2768-
2769- def add_patch(self, target, attr):
2770- """Patches specified target object and sets it as attr on test
2771- instance also schedules cleanup"""
2772- m = patch(target, autospec=True)
2773- p = m.start()
2774- self.addCleanup(m.stop)
2775- setattr(self, attr, p)
2776-
2777-
2778-class TestBlockMdadmAssemble(MdadmTestBase):
2779+class TestBlockMdadmAssemble(CiTestCase):
2780+
2781 def setUp(self):
2782 super(TestBlockMdadmAssemble, self).setUp()
2783 self.add_patch('curtin.block.mdadm.util', 'mock_util')
2784@@ -94,7 +82,7 @@
2785 rcs=[0, 1, 2])
2786
2787
2788-class TestBlockMdadmCreate(MdadmTestBase):
2789+class TestBlockMdadmCreate(CiTestCase):
2790 def setUp(self):
2791 super(TestBlockMdadmCreate, self).setUp()
2792 self.add_patch('curtin.block.mdadm.util', 'mock_util')
2793@@ -243,7 +231,7 @@
2794 self.mock_util.subp.assert_has_calls(expected_calls)
2795
2796
2797-class TestBlockMdadmExamine(MdadmTestBase):
2798+class TestBlockMdadmExamine(CiTestCase):
2799 def setUp(self):
2800 super(TestBlockMdadmExamine, self).setUp()
2801 self.add_patch('curtin.block.mdadm.util', 'mock_util')
2802@@ -328,7 +316,7 @@
2803 self.assertEqual(data, {})
2804
2805
2806-class TestBlockMdadmStop(MdadmTestBase):
2807+class TestBlockMdadmStop(CiTestCase):
2808 def setUp(self):
2809 super(TestBlockMdadmStop, self).setUp()
2810 self.add_patch('curtin.block.mdadm.util.lsb_release', 'mock_util_lsb')
2811@@ -495,7 +483,7 @@
2812 self.mock_util_write_file.assert_has_calls(expected_writes)
2813
2814
2815-class TestBlockMdadmRemove(MdadmTestBase):
2816+class TestBlockMdadmRemove(CiTestCase):
2817 def setUp(self):
2818 super(TestBlockMdadmRemove, self).setUp()
2819 self.add_patch('curtin.block.mdadm.util', 'mock_util')
2820@@ -521,7 +509,7 @@
2821 self.mock_util.subp.assert_has_calls(expected_calls)
2822
2823
2824-class TestBlockMdadmQueryDetail(MdadmTestBase):
2825+class TestBlockMdadmQueryDetail(CiTestCase):
2826 def setUp(self):
2827 super(TestBlockMdadmQueryDetail, self).setUp()
2828 self.add_patch('curtin.block.mdadm.util', 'mock_util')
2829@@ -599,7 +587,7 @@
2830 '93a73e10:427f280b:b7076c02:204b8f7a')
2831
2832
2833-class TestBlockMdadmDetailScan(MdadmTestBase):
2834+class TestBlockMdadmDetailScan(CiTestCase):
2835 def setUp(self):
2836 super(TestBlockMdadmDetailScan, self).setUp()
2837 self.add_patch('curtin.block.mdadm.util', 'mock_util')
2838@@ -634,7 +622,7 @@
2839 self.assertEqual(None, data)
2840
2841
2842-class TestBlockMdadmMdHelpers(MdadmTestBase):
2843+class TestBlockMdadmMdHelpers(CiTestCase):
2844 def setUp(self):
2845 super(TestBlockMdadmMdHelpers, self).setUp()
2846 self.add_patch('curtin.block.mdadm.util', 'mock_util')
2847
2848=== modified file 'tests/unittests/test_block_mkfs.py'
2849--- tests/unittests/test_block_mkfs.py 2016-10-03 18:42:29 +0000
2850+++ tests/unittests/test_block_mkfs.py 2017-10-06 16:35:22 +0000
2851@@ -1,10 +1,10 @@
2852 from curtin.block import mkfs
2853
2854-from unittest import TestCase
2855+from .helpers import CiTestCase
2856 import mock
2857
2858
2859-class TestBlockMkfs(TestCase):
2860+class TestBlockMkfs(CiTestCase):
2861 test_uuid = "fb26cc6c-ae73-11e5-9e38-2fb63f0c3155"
2862
2863 def _get_config(self, fstype):
2864
2865=== modified file 'tests/unittests/test_clear_holders.py'
2866--- tests/unittests/test_clear_holders.py 2017-06-12 20:39:06 +0000
2867+++ tests/unittests/test_clear_holders.py 2017-10-06 16:35:22 +0000
2868@@ -1,12 +1,12 @@
2869-from unittest import TestCase
2870 import mock
2871-
2872-from curtin.block import clear_holders
2873 import os
2874 import textwrap
2875
2876-
2877-class TestClearHolders(TestCase):
2878+from curtin.block import clear_holders
2879+from .helpers import CiTestCase
2880+
2881+
2882+class TestClearHolders(CiTestCase):
2883 test_blockdev = '/dev/null'
2884 test_syspath = '/sys/class/block/null'
2885 remove_retries = [0.2] * 150 # clear_holders defaults to 30 seconds
2886
2887=== added file 'tests/unittests/test_commands_apply_net.py'
2888--- tests/unittests/test_commands_apply_net.py 1970-01-01 00:00:00 +0000
2889+++ tests/unittests/test_commands_apply_net.py 2017-10-06 16:35:22 +0000
2890@@ -0,0 +1,334 @@
2891+from mock import patch, call
2892+import copy
2893+import os
2894+
2895+from curtin.commands import apply_net
2896+from curtin import util
2897+from .helpers import CiTestCase
2898+
2899+
2900+class TestApplyNet(CiTestCase):
2901+
2902+ def setUp(self):
2903+ super(TestApplyNet, self).setUp()
2904+
2905+ base = 'curtin.commands.apply_net.'
2906+ patches = [
2907+ (base + '_maybe_remove_legacy_eth0', 'm_legacy'),
2908+ (base + '_disable_ipv6_privacy_extensions', 'm_ipv6_priv'),
2909+ (base + '_patch_ifupdown_ipv6_mtu_hook', 'm_ipv6_mtu'),
2910+ ('curtin.net.netconfig_passthrough_available', 'm_netpass_avail'),
2911+ ('curtin.net.render_netconfig_passthrough', 'm_netpass_render'),
2912+ ('curtin.net.parse_net_config_data', 'm_net_parsedata'),
2913+ ('curtin.net.render_network_state', 'm_net_renderstate'),
2914+ ('curtin.net.network_state.from_state_file', 'm_ns_from_file'),
2915+ ('curtin.config.load_config', 'm_load_config'),
2916+ ]
2917+ for (tgt, attr) in patches:
2918+ self.add_patch(tgt, attr)
2919+
2920+ self.target = "my_target"
2921+ self.network_config = {
2922+ 'network': {
2923+ 'version': 1,
2924+ 'config': {},
2925+ }
2926+ }
2927+ self.ns = {
2928+ 'interfaces': {},
2929+ 'routes': [],
2930+ 'dns': {
2931+ 'nameservers': [],
2932+ 'search': [],
2933+ }
2934+ }
2935+
2936+ def test_apply_net_notarget(self):
2937+ self.assertRaises(Exception,
2938+ apply_net.apply_net, None, "", "")
2939+
2940+ def test_apply_net_nostate_or_config(self):
2941+ self.assertRaises(Exception,
2942+ apply_net.apply_net, "")
2943+
2944+ def test_apply_net_target_and_state(self):
2945+ self.m_ns_from_file.return_value = self.ns
2946+
2947+ self.assertRaises(ValueError,
2948+ apply_net.apply_net, self.target,
2949+ network_state=self.ns, network_config=None)
2950+
2951+ def test_apply_net_target_and_config(self):
2952+ self.m_load_config.return_value = self.network_config
2953+ self.m_netpass_avail.return_value = False
2954+ self.m_net_parsedata.return_value = self.ns
2955+
2956+ apply_net.apply_net(self.target, network_state=None,
2957+ network_config=self.network_config)
2958+
2959+ self.m_netpass_avail.assert_called_with(self.target)
2960+
2961+ self.m_net_renderstate.assert_called_with(target=self.target,
2962+ network_state=self.ns)
2963+ self.m_legacy.assert_called_with(self.target)
2964+ self.m_ipv6_priv.assert_called_with(self.target)
2965+ self.m_ipv6_mtu.assert_called_with(self.target)
2966+
2967+ def test_apply_net_target_and_config_passthrough(self):
2968+ self.m_load_config.return_value = self.network_config
2969+ self.m_netpass_avail.return_value = True
2970+
2971+ netcfg = "network_config.yaml"
2972+ apply_net.apply_net(self.target, network_state=None,
2973+ network_config=netcfg)
2974+
2975+ self.assertFalse(self.m_ns_from_file.called)
2976+ self.m_load_config.assert_called_with(netcfg)
2977+ self.m_netpass_avail.assert_called_with(self.target)
2978+ nc = self.network_config
2979+ self.m_netpass_render.assert_called_with(self.target, netconfig=nc)
2980+
2981+ self.assertFalse(self.m_net_renderstate.called)
2982+ self.m_legacy.assert_called_with(self.target)
2983+ self.m_ipv6_priv.assert_called_with(self.target)
2984+ self.m_ipv6_mtu.assert_called_with(self.target)
2985+
2986+ def test_apply_net_target_and_config_passthrough_nonet(self):
2987+ nc = {'storage': {}}
2988+ self.m_load_config.return_value = nc
2989+ self.m_netpass_avail.return_value = True
2990+
2991+ netcfg = "network_config.yaml"
2992+
2993+ apply_net.apply_net(self.target, network_state=None,
2994+ network_config=netcfg)
2995+
2996+ self.assertFalse(self.m_ns_from_file.called)
2997+ self.m_load_config.assert_called_with(netcfg)
2998+ self.m_netpass_avail.assert_called_with(self.target)
2999+ self.m_netpass_render.assert_called_with(self.target, netconfig=nc)
3000+
3001+ self.assertFalse(self.m_net_renderstate.called)
3002+ self.m_legacy.assert_called_with(self.target)
3003+ self.m_ipv6_priv.assert_called_with(self.target)
3004+ self.m_ipv6_mtu.assert_called_with(self.target)
3005+
3006+ def test_apply_net_target_and_config_passthrough_v2_not_available(self):
3007+ nc = copy.deepcopy(self.network_config)
3008+ nc['network']['version'] = 2
3009+ self.m_load_config.return_value = nc
3010+ self.m_netpass_avail.return_value = False
3011+ self.m_net_parsedata.return_value = self.ns
3012+
3013+ netcfg = "network_config.yaml"
3014+
3015+ apply_net.apply_net(self.target, network_state=None,
3016+ network_config=netcfg)
3017+
3018+ self.assertFalse(self.m_ns_from_file.called)
3019+ self.m_load_config.assert_called_with(netcfg)
3020+ self.m_netpass_avail.assert_called_with(self.target)
3021+ self.assertFalse(self.m_netpass_render.called)
3022+ self.m_net_parsedata.assert_called_with(nc['network'])
3023+
3024+ self.m_net_renderstate.assert_called_with(
3025+ target=self.target, network_state=self.ns)
3026+ self.m_legacy.assert_called_with(self.target)
3027+ self.m_ipv6_priv.assert_called_with(self.target)
3028+ self.m_ipv6_mtu.assert_called_with(self.target)
3029+
3030+
3031+class TestApplyNetPatchIfupdown(CiTestCase):
3032+
3033+ @patch('curtin.util.write_file')
3034+ def test_apply_ipv6_mtu_hook(self, mock_write):
3035+ target = 'mytarget'
3036+ prehookfn = 'if-pre-up.d/mtuipv6'
3037+ posthookfn = 'if-up.d/mtuipv6'
3038+ mode = 0o755
3039+
3040+ apply_net._patch_ifupdown_ipv6_mtu_hook(target,
3041+ prehookfn=prehookfn,
3042+ posthookfn=posthookfn)
3043+
3044+ precfg = util.target_path(target, path=prehookfn)
3045+ postcfg = util.target_path(target, path=posthookfn)
3046+ precontents = apply_net.IFUPDOWN_IPV6_MTU_PRE_HOOK
3047+ postcontents = apply_net.IFUPDOWN_IPV6_MTU_POST_HOOK
3048+
3049+ hook_calls = [
3050+ call(precfg, precontents, mode=mode),
3051+ call(postcfg, postcontents, mode=mode),
3052+ ]
3053+ mock_write.assert_has_calls(hook_calls)
3054+
3055+ @patch('curtin.util.write_file')
3056+ def test_apply_ipv6_mtu_hook_write_fail(self, mock_write):
3057+ """Write failure raises IOError"""
3058+ target = 'mytarget'
3059+ prehookfn = 'if-pre-up.d/mtuipv6'
3060+ posthookfn = 'if-up.d/mtuipv6'
3061+ mock_write.side_effect = (IOError)
3062+
3063+ self.assertRaises(IOError,
3064+ apply_net._patch_ifupdown_ipv6_mtu_hook,
3065+ target,
3066+ prehookfn=prehookfn,
3067+ posthookfn=posthookfn)
3068+ self.assertEqual(1, mock_write.call_count)
3069+
3070+ @patch('curtin.util.write_file')
3071+ def test_apply_ipv6_mtu_hook_invalid_target(self, mock_write):
3072+ """Invalid target path fail before calling util.write_file"""
3073+ invalid_target = {}
3074+ prehookfn = 'if-pre-up.d/mtuipv6'
3075+ posthookfn = 'if-up.d/mtuipv6'
3076+
3077+ self.assertRaises(ValueError,
3078+ apply_net._patch_ifupdown_ipv6_mtu_hook,
3079+ invalid_target,
3080+ prehookfn=prehookfn,
3081+ posthookfn=posthookfn)
3082+ self.assertEqual(0, mock_write.call_count)
3083+
3084+ @patch('curtin.util.write_file')
3085+ def test_apply_ipv6_mtu_hook_invalid_prepost_fn(self, mock_write):
3086+ """Invalid prepost filenames fail before calling util.write_file"""
3087+ target = "mytarget"
3088+ invalid_prehookfn = {'a': 1}
3089+ invalid_posthookfn = {'b': 2}
3090+
3091+ self.assertRaises(ValueError,
3092+ apply_net._patch_ifupdown_ipv6_mtu_hook,
3093+ target,
3094+ prehookfn=invalid_prehookfn,
3095+ posthookfn=invalid_posthookfn)
3096+ self.assertEqual(0, mock_write.call_count)
3097+
3098+
3099+class TestApplyNetPatchIpv6Priv(CiTestCase):
3100+
3101+ @patch('curtin.util.del_file')
3102+ @patch('curtin.util.load_file')
3103+ @patch('os.path')
3104+ @patch('curtin.util.write_file')
3105+ def test_disable_ipv6_priv_extentions(self, mock_write, mock_ospath,
3106+ mock_load, mock_del):
3107+ target = 'mytarget'
3108+ path = 'etc/sysctl.d/10-ipv6-privacy.conf'
3109+ ipv6_priv_contents = (
3110+ 'net.ipv6.conf.all.use_tempaddr = 2\n'
3111+ 'net.ipv6.conf.default.use_tempaddr = 2')
3112+ expected_ipv6_priv_contents = '\n'.join(
3113+ ["# IPv6 Privacy Extensions (RFC 4941)",
3114+ "# Disabled by curtin",
3115+ "# net.ipv6.conf.all.use_tempaddr = 2",
3116+ "# net.ipv6.conf.default.use_tempaddr = 2"])
3117+ mock_ospath.exists.return_value = True
3118+ mock_load.side_effect = [ipv6_priv_contents]
3119+
3120+ apply_net._disable_ipv6_privacy_extensions(target)
3121+
3122+ cfg = util.target_path(target, path=path)
3123+ mock_write.assert_called_with(cfg, expected_ipv6_priv_contents)
3124+
3125+ @patch('curtin.util.load_file')
3126+ @patch('os.path')
3127+ def test_disable_ipv6_priv_extentions_decoderror(self, mock_ospath,
3128+ mock_load):
3129+ target = 'mytarget'
3130+ mock_ospath.exists.return_value = True
3131+
3132+ # simulate loading of binary data
3133+ mock_load.side_effect = (Exception)
3134+
3135+ self.assertRaises(Exception,
3136+ apply_net._disable_ipv6_privacy_extensions,
3137+ target)
3138+
3139+ @patch('curtin.util.load_file')
3140+ @patch('os.path')
3141+ def test_disable_ipv6_priv_extentions_notfound(self, mock_ospath,
3142+ mock_load):
3143+ target = 'mytarget'
3144+ path = 'foo.conf'
3145+ mock_ospath.exists.return_value = False
3146+
3147+ apply_net._disable_ipv6_privacy_extensions(target, path=path)
3148+
3149+ # source file not found
3150+ cfg = util.target_path(target, path)
3151+ mock_ospath.exists.assert_called_with(cfg)
3152+ self.assertEqual(0, mock_load.call_count)
3153+
3154+
3155+class TestApplyNetRemoveLegacyEth0(CiTestCase):
3156+
3157+ @patch('curtin.util.del_file')
3158+ @patch('curtin.util.load_file')
3159+ @patch('os.path')
3160+ def test_remove_legacy_eth0(self, mock_ospath, mock_load, mock_del):
3161+ target = 'mytarget'
3162+ path = 'eth0.cfg'
3163+ cfg = util.target_path(target, path)
3164+ legacy_eth0_contents = (
3165+ 'auto eth0\n'
3166+ 'iface eth0 inet dhcp')
3167+
3168+ mock_ospath.exists.return_value = True
3169+ mock_load.side_effect = [legacy_eth0_contents]
3170+
3171+ apply_net._maybe_remove_legacy_eth0(target, path)
3172+
3173+ mock_del.assert_called_with(cfg)
3174+
3175+ @patch('curtin.util.del_file')
3176+ @patch('curtin.util.load_file')
3177+ @patch('os.path')
3178+ def test_remove_legacy_eth0_nomatch(self, mock_ospath, mock_load,
3179+ mock_del):
3180+ target = 'mytarget'
3181+ path = 'eth0.cfg'
3182+ legacy_eth0_contents = "nomatch"
3183+ mock_ospath.join.side_effect = os.path.join
3184+ mock_ospath.exists.return_value = True
3185+ mock_load.side_effect = [legacy_eth0_contents]
3186+
3187+ self.assertRaises(Exception,
3188+ apply_net._maybe_remove_legacy_eth0,
3189+ target, path)
3190+
3191+ self.assertEqual(0, mock_del.call_count)
3192+
3193+ @patch('curtin.util.del_file')
3194+ @patch('curtin.util.load_file')
3195+ @patch('os.path')
3196+ def test_remove_legacy_eth0_badload(self, mock_ospath, mock_load,
3197+ mock_del):
3198+ target = 'mytarget'
3199+ path = 'eth0.cfg'
3200+ mock_ospath.exists.return_value = True
3201+ mock_load.side_effect = (Exception)
3202+
3203+ self.assertRaises(Exception,
3204+ apply_net._maybe_remove_legacy_eth0,
3205+ target, path)
3206+
3207+ self.assertEqual(0, mock_del.call_count)
3208+
3209+ @patch('curtin.util.del_file')
3210+ @patch('curtin.util.load_file')
3211+ @patch('os.path')
3212+ def test_remove_legacy_eth0_notfound(self, mock_ospath, mock_load,
3213+ mock_del):
3214+ target = 'mytarget'
3215+ path = 'eth0.conf'
3216+ mock_ospath.exists.return_value = False
3217+
3218+ apply_net._maybe_remove_legacy_eth0(target, path)
3219+
3220+ # source file not found
3221+ cfg = util.target_path(target, path)
3222+ mock_ospath.exists.assert_called_with(cfg)
3223+ self.assertEqual(0, mock_load.call_count)
3224+ self.assertEqual(0, mock_del.call_count)
3225
3226=== modified file 'tests/unittests/test_commands_block_meta.py'
3227--- tests/unittests/test_commands_block_meta.py 2017-06-12 20:39:06 +0000
3228+++ tests/unittests/test_commands_block_meta.py 2017-10-06 16:35:22 +0000
3229@@ -1,24 +1,11 @@
3230-from unittest import TestCase
3231 from mock import patch, call
3232 from argparse import Namespace
3233
3234 from curtin.commands import block_meta
3235-
3236-
3237-class BlockMetaTestBase(TestCase):
3238- def setUp(self):
3239- super(BlockMetaTestBase, self).setUp()
3240-
3241- def add_patch(self, target, attr):
3242- """Patches specified target object and sets it as attr on test
3243- instance also schedules cleanup"""
3244- m = patch(target, autospec=True)
3245- p = m.start()
3246- self.addCleanup(m.stop)
3247- setattr(self, attr, p)
3248-
3249-
3250-class TestBlockMetaSimple(BlockMetaTestBase):
3251+from .helpers import CiTestCase
3252+
3253+
3254+class TestBlockMetaSimple(CiTestCase):
3255 def setUp(self):
3256 super(TestBlockMetaSimple, self).setUp()
3257 self.target = "my_target"
3258@@ -120,10 +107,10 @@
3259 [call(['mount', devname, self.target])])
3260
3261
3262-class TestBlockMeta(BlockMetaTestBase):
3263+class TestBlockMeta(CiTestCase):
3264+
3265 def setUp(self):
3266 super(TestBlockMeta, self).setUp()
3267- # self.target = tempfile.mkdtemp()
3268
3269 basepath = 'curtin.commands.block_meta.'
3270 self.add_patch(basepath + 'get_path_to_storage_volume', 'mock_getpath')
3271
3272=== added file 'tests/unittests/test_commands_install.py'
3273--- tests/unittests/test_commands_install.py 1970-01-01 00:00:00 +0000
3274+++ tests/unittests/test_commands_install.py 2017-10-06 16:35:22 +0000
3275@@ -0,0 +1,22 @@
3276+import copy
3277+
3278+from curtin.commands import install
3279+from .helpers import CiTestCase
3280+
3281+
3282+class TestMigrateProxy(CiTestCase):
3283+ def test_legacy_moved_over(self):
3284+ """Legacy setting should get moved over."""
3285+ proxy = "http://my.proxy:3128"
3286+ cfg = {'http_proxy': proxy}
3287+ install.migrate_proxy_settings(cfg)
3288+ self.assertEqual(cfg, {'proxy': {'http_proxy': proxy}})
3289+
3290+ def test_no_legacy_new_only(self):
3291+ """If only new 'proxy', then no change is expected."""
3292+ proxy = "http://my.proxy:3128"
3293+ cfg = {'proxy': {'http_proxy': proxy, 'https_proxy': proxy,
3294+ 'no_proxy': "10.2.2.2"}}
3295+ expected = copy.deepcopy(cfg)
3296+ install.migrate_proxy_settings(cfg)
3297+ self.assertEqual(expected, cfg)
3298
3299=== modified file 'tests/unittests/test_config.py'
3300--- tests/unittests/test_config.py 2015-10-02 16:19:07 +0000
3301+++ tests/unittests/test_config.py 2017-10-06 16:35:22 +0000
3302@@ -1,12 +1,12 @@
3303-from unittest import TestCase
3304 import copy
3305 import json
3306 import textwrap
3307
3308 from curtin import config
3309-
3310-
3311-class TestMerge(TestCase):
3312+from .helpers import CiTestCase
3313+
3314+
3315+class TestMerge(CiTestCase):
3316 def test_merge_cfg_string(self):
3317 d1 = {'str1': 'str_one'}
3318 d2 = {'dict1': {'d1.e1': 'd1-e1'}}
3319@@ -16,7 +16,7 @@
3320 self.assertEqual(d1, expected)
3321
3322
3323-class TestCmdArg2Cfg(TestCase):
3324+class TestCmdArg2Cfg(CiTestCase):
3325 def test_cmdarg_flat(self):
3326 self.assertEqual(config.cmdarg2cfg("foo=bar"), {'foo': 'bar'})
3327
3328@@ -50,7 +50,7 @@
3329 self.assertEqual(via_merge, via_merge_cmdarg)
3330
3331
3332-class TestConfigArchive(TestCase):
3333+class TestConfigArchive(CiTestCase):
3334 def test_archive_dict(self):
3335 myarchive = _replace_consts(textwrap.dedent("""
3336 _ARCH_HEAD_
3337
3338=== modified file 'tests/unittests/test_curthooks.py'
3339--- tests/unittests/test_curthooks.py 2017-06-12 20:39:06 +0000
3340+++ tests/unittests/test_curthooks.py 2017-10-06 16:35:22 +0000
3341@@ -1,29 +1,14 @@
3342 import os
3343-from unittest import TestCase
3344 from mock import call, patch, MagicMock
3345-import shutil
3346-import tempfile
3347
3348 from curtin.commands import curthooks
3349 from curtin import util
3350 from curtin import config
3351 from curtin.reporter import events
3352-
3353-
3354-class CurthooksBase(TestCase):
3355- def setUp(self):
3356- super(CurthooksBase, self).setUp()
3357-
3358- def add_patch(self, target, attr, autospec=True):
3359- """Patches specified target object and sets it as attr on test
3360- instance also schedules cleanup"""
3361- m = patch(target, autospec=autospec)
3362- p = m.start()
3363- self.addCleanup(m.stop)
3364- setattr(self, attr, p)
3365-
3366-
3367-class TestGetFlashKernelPkgs(CurthooksBase):
3368+from .helpers import CiTestCase
3369+
3370+
3371+class TestGetFlashKernelPkgs(CiTestCase):
3372 def setUp(self):
3373 super(TestGetFlashKernelPkgs, self).setUp()
3374 self.add_patch('curtin.util.subp', 'mock_subp')
3375@@ -57,7 +42,7 @@
3376 self.mock_is_uefi_bootable.assert_called_once_with()
3377
3378
3379-class TestCurthooksInstallKernel(CurthooksBase):
3380+class TestCurthooksInstallKernel(CiTestCase):
3381 def setUp(self):
3382 super(TestCurthooksInstallKernel, self).setUp()
3383 self.add_patch('curtin.util.has_pkg_available', 'mock_haspkg')
3384@@ -70,7 +55,7 @@
3385 'fallback-package': 'mock-fallback',
3386 'mapping': {}}}
3387 # Tests don't actually install anything so we just need a name
3388- self.target = tempfile.mktemp()
3389+ self.target = self.tmp_dir()
3390
3391 def test__installs_flash_kernel_packages_when_needed(self):
3392 kernel_package = self.kernel_cfg.get('kernel', {}).get('package', {})
3393@@ -94,14 +79,11 @@
3394 [kernel_package], target=self.target)
3395
3396
3397-class TestUpdateInitramfs(CurthooksBase):
3398+class TestUpdateInitramfs(CiTestCase):
3399 def setUp(self):
3400 super(TestUpdateInitramfs, self).setUp()
3401 self.add_patch('curtin.util.subp', 'mock_subp')
3402- self.target = tempfile.mkdtemp()
3403-
3404- def tearDown(self):
3405- shutil.rmtree(self.target)
3406+ self.target = self.tmp_dir()
3407
3408 def _mnt_call(self, point):
3409 target = os.path.join(self.target, point)
3410@@ -134,7 +116,7 @@
3411 self.mock_subp.assert_has_calls(subp_calls)
3412
3413
3414-class TestInstallMissingPkgs(CurthooksBase):
3415+class TestInstallMissingPkgs(CiTestCase):
3416 def setUp(self):
3417 super(TestInstallMissingPkgs, self).setUp()
3418 self.add_patch('platform.machine', 'mock_machine')
3419@@ -176,11 +158,38 @@
3420 self.assertEqual([], self.mock_install_packages.call_args_list)
3421
3422
3423-class TestSetupGrub(CurthooksBase):
3424+class TestSetupZipl(CiTestCase):
3425+
3426+ def setUp(self):
3427+ super(TestSetupZipl, self).setUp()
3428+ self.target = self.tmp_dir()
3429+
3430+ @patch('curtin.block.get_devices_for_mp')
3431+ @patch('platform.machine')
3432+ def test_noop_non_s390x(self, m_machine, m_get_devices):
3433+ m_machine.return_value = 'non-s390x'
3434+ curthooks.setup_zipl(None, self.target)
3435+ self.assertEqual(0, m_get_devices.call_count)
3436+
3437+ @patch('curtin.block.get_devices_for_mp')
3438+ @patch('platform.machine')
3439+ def test_setup_zipl_writes_etc_zipl_conf(self, m_machine, m_get_devices):
3440+ m_machine.return_value = 's390x'
3441+ m_get_devices.return_value = ['/dev/mapper/ubuntu--vg-root']
3442+ curthooks.setup_zipl(None, self.target)
3443+ m_get_devices.assert_called_with(self.target)
3444+ with open(os.path.join(self.target, 'etc', 'zipl.conf')) as stream:
3445+ content = stream.read()
3446+ self.assertIn(
3447+ '# This has been modified by the MAAS curtin installer',
3448+ content)
3449+
3450+
3451+class TestSetupGrub(CiTestCase):
3452
3453 def setUp(self):
3454 super(TestSetupGrub, self).setUp()
3455- self.target = tempfile.mkdtemp()
3456+ self.target = self.tmp_dir()
3457 self.add_patch('curtin.util.lsb_release', 'mock_lsb_release')
3458 self.mock_lsb_release.return_value = {
3459 'codename': 'xenial',
3460@@ -203,9 +212,6 @@
3461 self.mock_in_chroot_subp.side_effect = iter(self.in_chroot_subp_output)
3462 self.mock_chroot.return_value = self.mock_in_chroot
3463
3464- def tearDown(self):
3465- shutil.rmtree(self.target)
3466-
3467 def test_uses_old_grub_install_devices_in_cfg(self):
3468 cfg = {
3469 'grub_install_devices': ['/dev/vdb']
3470@@ -434,17 +440,13 @@
3471 self.mock_in_chroot_subp.call_args_list[0][0])
3472
3473
3474-class TestUbuntuCoreHooks(CurthooksBase):
3475+class TestUbuntuCoreHooks(CiTestCase):
3476 def setUp(self):
3477 super(TestUbuntuCoreHooks, self).setUp()
3478 self.target = None
3479
3480- def tearDown(self):
3481- if self.target:
3482- shutil.rmtree(self.target)
3483-
3484 def test_target_is_ubuntu_core(self):
3485- self.target = tempfile.mkdtemp()
3486+ self.target = self.tmp_dir()
3487 ubuntu_core_path = os.path.join(self.target, 'system-data',
3488 'var/lib/snapd')
3489 util.ensure_dir(ubuntu_core_path)
3490@@ -457,7 +459,7 @@
3491 self.assertFalse(is_core)
3492
3493 def test_target_is_ubuntu_core_noncore_target(self):
3494- self.target = tempfile.mkdtemp()
3495+ self.target = self.tmp_dir()
3496 non_core_path = os.path.join(self.target, 'curtin')
3497 util.ensure_dir(non_core_path)
3498 self.assertTrue(os.path.isdir(non_core_path))
3499@@ -469,7 +471,7 @@
3500 @patch('curtin.commands.curthooks.handle_cloudconfig')
3501 def test_curthooks_no_config(self, mock_handle_cc, mock_del_file,
3502 mock_write_file):
3503- self.target = tempfile.mkdtemp()
3504+ self.target = self.tmp_dir()
3505 cfg = {}
3506 curthooks.ubuntu_core_curthooks(cfg, target=self.target)
3507 self.assertEqual(len(mock_handle_cc.call_args_list), 0)
3508@@ -478,7 +480,7 @@
3509
3510 @patch('curtin.commands.curthooks.handle_cloudconfig')
3511 def test_curthooks_cloud_config_remove_disabled(self, mock_handle_cc):
3512- self.target = tempfile.mkdtemp()
3513+ self.target = self.tmp_dir()
3514 uc_cloud = os.path.join(self.target, 'system-data', 'etc/cloud')
3515 cc_disabled = os.path.join(uc_cloud, 'cloud-init.disabled')
3516 cc_path = os.path.join(uc_cloud, 'cloud.cfg.d')
3517@@ -496,7 +498,7 @@
3518 curthooks.ubuntu_core_curthooks(cfg, target=self.target)
3519
3520 mock_handle_cc.assert_called_with(cfg.get('cloudconfig'),
3521- target=cc_path)
3522+ base_dir=cc_path)
3523 self.assertFalse(os.path.exists(cc_disabled))
3524
3525 @patch('curtin.util.write_file')
3526@@ -504,7 +506,7 @@
3527 @patch('curtin.commands.curthooks.handle_cloudconfig')
3528 def test_curthooks_cloud_config(self, mock_handle_cc, mock_del_file,
3529 mock_write_file):
3530- self.target = tempfile.mkdtemp()
3531+ self.target = self.tmp_dir()
3532 cfg = {
3533 'cloudconfig': {
3534 'file1': {
3535@@ -518,7 +520,7 @@
3536 cc_path = os.path.join(self.target,
3537 'system-data/etc/cloud/cloud.cfg.d')
3538 mock_handle_cc.assert_called_with(cfg.get('cloudconfig'),
3539- target=cc_path)
3540+ base_dir=cc_path)
3541 self.assertEqual(len(mock_write_file.call_args_list), 0)
3542
3543 @patch('curtin.util.write_file')
3544@@ -526,7 +528,7 @@
3545 @patch('curtin.commands.curthooks.handle_cloudconfig')
3546 def test_curthooks_net_config(self, mock_handle_cc, mock_del_file,
3547 mock_write_file):
3548- self.target = tempfile.mkdtemp()
3549+ self.target = self.tmp_dir()
3550 cfg = {
3551 'network': {
3552 'version': '1',
3553@@ -541,13 +543,13 @@
3554 netcfg_path = os.path.join(self.target,
3555 'system-data',
3556 'etc/cloud/cloud.cfg.d',
3557- '50-network-config.cfg')
3558+ '50-curtin-networking.cfg')
3559 netcfg = config.dump_config({'network': cfg.get('network')})
3560 mock_write_file.assert_called_with(netcfg_path,
3561 content=netcfg)
3562 self.assertEqual(len(mock_del_file.call_args_list), 0)
3563
3564- @patch('curtin.commands.curthooks.write_files')
3565+ @patch('curtin.commands.curthooks.futil.write_files')
3566 def test_handle_cloudconfig(self, mock_write_files):
3567 cc_target = "tmpXXXX/systemd-data/etc/cloud/cloud.cfg.d"
3568 cloudconfig = {
3569@@ -561,20 +563,202 @@
3570 }
3571
3572 expected_cfg = {
3573- 'write_files': {
3574- 'file1': {
3575- 'path': '50-cloudconfig-file1.cfg',
3576- 'content': cloudconfig['file1']['content']},
3577- 'foobar': {
3578- 'path': '50-cloudconfig-foobar.cfg',
3579- 'content': cloudconfig['foobar']['content']}
3580- }
3581+ 'file1': {
3582+ 'path': '50-cloudconfig-file1.cfg',
3583+ 'content': cloudconfig['file1']['content']},
3584+ 'foobar': {
3585+ 'path': '50-cloudconfig-foobar.cfg',
3586+ 'content': cloudconfig['foobar']['content']}
3587 }
3588- curthooks.handle_cloudconfig(cloudconfig, target=cc_target)
3589+ curthooks.handle_cloudconfig(cloudconfig, base_dir=cc_target)
3590 mock_write_files.assert_called_with(expected_cfg, cc_target)
3591
3592 def test_handle_cloudconfig_bad_config(self):
3593 with self.assertRaises(ValueError):
3594- curthooks.handle_cloudconfig([], target="foobar")
3595+ curthooks.handle_cloudconfig([], base_dir="foobar")
3596+
3597+
3598+class TestDetectRequiredPackages(CiTestCase):
3599+ test_config = {
3600+ 'storage': {
3601+ 1: {
3602+ 'bcache': {
3603+ 'type': 'bcache', 'name': 'bcache0', 'id': 'cache0',
3604+ 'backing_device': 'sda3', 'cache_device': 'sdb'},
3605+ 'lvm_partition': {
3606+ 'id': 'lvol1', 'name': 'lv1', 'volgroup': 'vg1',
3607+ 'type': 'lvm_partition'},
3608+ 'lvm_volgroup': {
3609+ 'id': 'vol1', 'name': 'vg1', 'devices': ['sda', 'sdb'],
3610+ 'type': 'lvm_volgroup'},
3611+ 'raid': {
3612+ 'id': 'mddevice', 'name': 'md0', 'type': 'raid',
3613+ 'raidlevel': 5, 'devices': ['sda1', 'sdb1', 'sdc1']},
3614+ 'ext2': {
3615+ 'id': 'format0', 'fstype': 'ext2', 'type': 'format'},
3616+ 'ext3': {
3617+ 'id': 'format1', 'fstype': 'ext3', 'type': 'format'},
3618+ 'ext4': {
3619+ 'id': 'format2', 'fstype': 'ext4', 'type': 'format'},
3620+ 'btrfs': {
3621+ 'id': 'format3', 'fstype': 'btrfs', 'type': 'format'},
3622+ 'xfs': {
3623+ 'id': 'format4', 'fstype': 'xfs', 'type': 'format'}}
3624+ },
3625+ 'network': {
3626+ 1: {
3627+ 'bond': {
3628+ 'name': 'bond0', 'type': 'bond',
3629+ 'bond_interfaces': ['interface0', 'interface1'],
3630+ 'params': {'bond-mode': 'active-backup'},
3631+ 'subnets': [
3632+ {'type': 'static', 'address': '10.23.23.2/24'},
3633+ {'type': 'static', 'address': '10.23.24.2/24'}]},
3634+ 'vlan': {
3635+ 'id': 'interface1.2667', 'mtu': 1500, 'name':
3636+ 'interface1.2667', 'type': 'vlan', 'vlan_id': 2667,
3637+ 'vlan_link': 'interface1',
3638+ 'subnets': [{'address': '10.245.184.2/24',
3639+ 'dns_nameservers': [], 'type': 'static'}]},
3640+ 'bridge': {
3641+ 'name': 'br0', 'bridge_interfaces': ['eth0', 'eth1'],
3642+ 'type': 'bridge', 'params': {
3643+ 'bridge_stp': 'off', 'bridge_fd': 0,
3644+ 'bridge_maxwait': 0},
3645+ 'subnets': [
3646+ {'type': 'static', 'address': '192.168.14.2/24'},
3647+ {'type': 'static', 'address': '2001:1::1/64'}]}},
3648+ 2: {
3649+ 'vlan': {
3650+ 'vlans': {
3651+ 'en-intra': {'id': 1, 'link': 'eno1', 'dhcp4': 'yes'},
3652+ 'en-vpn': {'id': 2, 'link': 'eno1'}}},
3653+ 'bridge': {
3654+ 'bridges': {
3655+ 'br0': {
3656+ 'interfaces': ['wlp1s0', 'switchports'],
3657+ 'dhcp4': True}}}}
3658+ },
3659+ }
3660+
3661+ def _fmt_config(self, config_items):
3662+ res = {}
3663+ for item, item_confs in config_items.items():
3664+ version = item_confs['version']
3665+ res[item] = {'version': version}
3666+ if version == 1:
3667+ res[item]['config'] = [self.test_config[item][version][i]
3668+ for i in item_confs['items']]
3669+ elif version == 2 and item == 'network':
3670+ for cfg_item in item_confs['items']:
3671+ res[item].update(self.test_config[item][version][cfg_item])
3672+ else:
3673+ raise NotImplementedError
3674+ return res
3675+
3676+ def _test_req_mappings(self, req_mappings):
3677+ for (config_items, expected_reqs) in req_mappings:
3678+ cfg = self._fmt_config(config_items)
3679+ actual_reqs = curthooks.detect_required_packages(cfg)
3680+ self.assertEqual(set(actual_reqs), set(expected_reqs),
3681+ 'failed for config: {}'.format(config_items))
3682+
3683+ def test_storage_v1_detect(self):
3684+ self._test_req_mappings((
3685+ ({'storage': {
3686+ 'version': 1,
3687+ 'items': ('lvm_partition', 'lvm_volgroup', 'btrfs', 'xfs')}},
3688+ ('lvm2', 'xfsprogs', 'btrfs-tools')),
3689+ ({'storage': {
3690+ 'version': 1,
3691+ 'items': ('raid', 'bcache', 'ext3', 'xfs')}},
3692+ ('mdadm', 'bcache-tools', 'e2fsprogs', 'xfsprogs')),
3693+ ({'storage': {
3694+ 'version': 1,
3695+ 'items': ('raid', 'lvm_volgroup', 'lvm_partition', 'ext3',
3696+ 'ext4', 'btrfs')}},
3697+ ('lvm2', 'mdadm', 'e2fsprogs', 'btrfs-tools')),
3698+ ({'storage': {
3699+ 'version': 1,
3700+ 'items': ('bcache', 'lvm_volgroup', 'lvm_partition', 'ext2')}},
3701+ ('bcache-tools', 'lvm2', 'e2fsprogs')),
3702+ ))
3703+
3704+ def test_network_v1_detect(self):
3705+ self._test_req_mappings((
3706+ ({'network': {
3707+ 'version': 1,
3708+ 'items': ('bridge',)}},
3709+ ('bridge-utils',)),
3710+ ({'network': {
3711+ 'version': 1,
3712+ 'items': ('vlan', 'bond')}},
3713+ ('vlan', 'ifenslave')),
3714+ ({'network': {
3715+ 'version': 1,
3716+ 'items': ('bond', 'bridge')}},
3717+ ('ifenslave', 'bridge-utils')),
3718+ ({'network': {
3719+ 'version': 1,
3720+ 'items': ('vlan', 'bridge', 'bond')}},
3721+ ('ifenslave', 'bridge-utils', 'vlan')),
3722+ ))
3723+
3724+ def test_mixed_v1_detect(self):
3725+ self._test_req_mappings((
3726+ ({'storage': {
3727+ 'version': 1,
3728+ 'items': ('raid', 'bcache', 'ext4')},
3729+ 'network': {
3730+ 'version': 1,
3731+ 'items': ('vlan',)}},
3732+ ('mdadm', 'bcache-tools', 'e2fsprogs', 'vlan')),
3733+ ({'storage': {
3734+ 'version': 1,
3735+ 'items': ('lvm_partition', 'lvm_volgroup', 'xfs')},
3736+ 'network': {
3737+ 'version': 1,
3738+ 'items': ('bridge', 'bond')}},
3739+ ('lvm2', 'xfsprogs', 'bridge-utils', 'ifenslave')),
3740+ ({'storage': {
3741+ 'version': 1,
3742+ 'items': ('ext3', 'ext4', 'btrfs')},
3743+ 'network': {
3744+ 'version': 1,
3745+ 'items': ('bond', 'vlan')}},
3746+ ('e2fsprogs', 'btrfs-tools', 'vlan', 'ifenslave')),
3747+ ))
3748+
3749+ def test_network_v2_detect(self):
3750+ self._test_req_mappings((
3751+ ({'network': {
3752+ 'version': 2,
3753+ 'items': ('bridge',)}},
3754+ ('bridge-utils',)),
3755+ ({'network': {
3756+ 'version': 2,
3757+ 'items': ('vlan',)}},
3758+ ('vlan',)),
3759+ ({'network': {
3760+ 'version': 2,
3761+ 'items': ('vlan', 'bridge')}},
3762+ ('vlan', 'bridge-utils')),
3763+ ))
3764+
3765+ def test_mixed_storage_v1_network_v2_detect(self):
3766+ self._test_req_mappings((
3767+ ({'network': {
3768+ 'version': 2,
3769+ 'items': ('bridge', 'vlan')},
3770+ 'storage': {
3771+ 'version': 1,
3772+ 'items': ('raid', 'bcache', 'ext4')}},
3773+ ('vlan', 'bridge-utils', 'mdadm', 'bcache-tools', 'e2fsprogs')),
3774+ ))
3775+
3776+ def test_invalid_version_in_config(self):
3777+ with self.assertRaises(ValueError):
3778+ curthooks.detect_required_packages({'network': {'version': 3}})
3779+
3780
3781 # vi: ts=4 expandtab syntax=python
3782
3783=== modified file 'tests/unittests/test_feature.py'
3784--- tests/unittests/test_feature.py 2017-06-12 20:39:06 +0000
3785+++ tests/unittests/test_feature.py 2017-10-06 16:35:22 +0000
3786@@ -1,9 +1,9 @@
3787-from unittest import TestCase
3788+from .helpers import CiTestCase
3789
3790 import curtin
3791
3792
3793-class TestExportsFeatures(TestCase):
3794+class TestExportsFeatures(CiTestCase):
3795 def test_has_storage_v1(self):
3796 self.assertIn('STORAGE_CONFIG_V1', curtin.FEATURES)
3797
3798@@ -15,3 +15,6 @@
3799
3800 def test_has_reporting_events_webhook(self):
3801 self.assertIn('REPORTING_EVENTS_WEBHOOK', curtin.FEATURES)
3802+
3803+ def test_has_centos_apply_network_config(self):
3804+ self.assertIn('CENTOS_APPLY_NETWORK_CONFIG', curtin.FEATURES)
3805
3806=== modified file 'tests/unittests/test_gpg.py'
3807--- tests/unittests/test_gpg.py 2017-02-08 22:22:44 +0000
3808+++ tests/unittests/test_gpg.py 2017-10-06 16:35:22 +0000
3809@@ -1,12 +1,12 @@
3810-from unittest import TestCase
3811 from mock import call, patch
3812 import textwrap
3813
3814 from curtin import gpg
3815 from curtin import util
3816-
3817-
3818-class TestCurtinGpg(TestCase):
3819+from .helpers import CiTestCase
3820+
3821+
3822+class TestCurtinGpg(CiTestCase):
3823
3824 @patch('curtin.util.subp')
3825 def test_export_armour(self, mock_subp):
3826
3827=== modified file 'tests/unittests/test_make_dname.py'
3828--- tests/unittests/test_make_dname.py 2016-10-03 18:42:29 +0000
3829+++ tests/unittests/test_make_dname.py 2017-10-06 16:35:22 +0000
3830@@ -1,13 +1,13 @@
3831-from unittest import TestCase
3832 import mock
3833
3834 import textwrap
3835 import uuid
3836
3837 from curtin.commands import block_meta
3838-
3839-
3840-class TestMakeDname(TestCase):
3841+from .helpers import CiTestCase
3842+
3843+
3844+class TestMakeDname(CiTestCase):
3845 state = {'scratch': '/tmp/null'}
3846 rules_d = '/tmp/null/rules.d'
3847 rule_file = '/tmp/null/rules.d/{}.rules'
3848
3849=== modified file 'tests/unittests/test_net.py'
3850--- tests/unittests/test_net.py 2017-03-01 16:13:56 +0000
3851+++ tests/unittests/test_net.py 2017-10-06 16:35:22 +0000
3852@@ -1,15 +1,14 @@
3853-from unittest import TestCase
3854+import mock
3855 import os
3856-import shutil
3857-import tempfile
3858 import yaml
3859
3860-from curtin import net
3861+from curtin import config, net, util
3862 import curtin.net.network_state as network_state
3863+from .helpers import CiTestCase
3864 from textwrap import dedent
3865
3866
3867-class TestNetParserData(TestCase):
3868+class TestNetParserData(CiTestCase):
3869
3870 def test_parse_deb_config_data_ignores_comments(self):
3871 contents = dedent("""\
3872@@ -234,13 +233,11 @@
3873 }, ifaces)
3874
3875
3876-class TestNetParser(TestCase):
3877+class TestNetParser(CiTestCase):
3878
3879 def setUp(self):
3880- self.target = tempfile.mkdtemp()
3881-
3882- def tearDown(self):
3883- shutil.rmtree(self.target)
3884+ super(TestNetParser, self).setUp()
3885+ self.target = self.tmp_dir()
3886
3887 def make_config(self, path=None, name=None, contents=None,
3888 parse=True):
3889@@ -386,9 +383,10 @@
3890 self.assertEqual({}, observed)
3891
3892
3893-class TestNetConfig(TestCase):
3894+class TestNetConfig(CiTestCase):
3895 def setUp(self):
3896- self.target = tempfile.mkdtemp()
3897+ super(TestNetConfig, self).setUp()
3898+ self.target = self.tmp_dir()
3899 self.config_f = os.path.join(self.target, 'config')
3900 self.config = '''
3901 # YAML example of a simple network config
3902@@ -435,9 +433,6 @@
3903 ns.parse_config()
3904 return ns
3905
3906- def tearDown(self):
3907- shutil.rmtree(self.target)
3908-
3909 def test_parse_net_config_data(self):
3910 ns = self.get_net_state()
3911 net_state_from_cls = ns.network_state
3912@@ -503,24 +498,19 @@
3913 auto interface1
3914 iface interface1 inet manual
3915 bond-mode active-backup
3916- bond-master bond0
3917+ bond-master bond1
3918
3919 auto interface2
3920 iface interface2 inet manual
3921 bond-mode active-backup
3922- bond-master bond0
3923+ bond-master bond1
3924
3925- auto bond0
3926- iface bond0 inet static
3927+ auto bond1
3928+ iface bond1 inet static
3929 address 10.23.23.2/24
3930 bond-mode active-backup
3931- hwaddress ether 52:54:00:12:34:06
3932 bond-slaves none
3933
3934- # control-alias bond0
3935- iface bond0 inet static
3936- address 10.23.24.2/24
3937-
3938 source /etc/network/interfaces.d/*.cfg
3939 """)
3940 net_ifaces = net.render_interfaces(ns.network_state)
3941@@ -654,6 +644,91 @@
3942 self.assertEqual(sorted(ifaces.split('\n')),
3943 sorted(net_ifaces.split('\n')))
3944
3945+ @mock.patch('curtin.util.subp')
3946+ @mock.patch('curtin.util.which')
3947+ @mock.patch.object(util.ChrootableTarget, "__enter__", new=lambda a: a)
3948+ def test_netconfig_passthrough_available(self, mock_which, mock_subp):
3949+ cloud_init = '/usr/bin/cloud-init'
3950+ mock_which.return_value = cloud_init
3951+ mock_subp.return_value = ("NETWORK_CONFIG_V1\nNETWORK_CONFIG_V2\n", '')
3952+
3953+ available = net.netconfig_passthrough_available(self.target)
3954+
3955+ self.assertEqual(True, available,
3956+ "netconfig passthrough was NOT available")
3957+ mock_which.assert_called_with('cloud-init', target=self.target)
3958+ mock_subp.assert_called_with([cloud_init, 'features'],
3959+ capture=True, target=self.target)
3960+
3961+ @mock.patch('curtin.net.LOG')
3962+ @mock.patch('curtin.util.subp')
3963+ @mock.patch('curtin.util.which')
3964+ @mock.patch.object(util.ChrootableTarget, "__enter__", new=lambda a: a)
3965+ def test_netconfig_passthrough_available_no_cloudinit(self, mock_which,
3966+ mock_subp, mock_log):
3967+ mock_which.return_value = None
3968+
3969+ available = net.netconfig_passthrough_available(self.target)
3970+
3971+ self.assertEqual(False, available,
3972+ "netconfig passthrough was available")
3973+ self.assertTrue(mock_log.warning.called)
3974+ self.assertFalse(mock_subp.called)
3975+
3976+ @mock.patch('curtin.util.subp')
3977+ @mock.patch('curtin.util.which')
3978+ @mock.patch.object(util.ChrootableTarget, "__enter__", new=lambda a: a)
3979+ def test_netconfig_passthrough_available_feature_not_found(self,
3980+ mock_which,
3981+ mock_subp):
3982+ cloud_init = '/usr/bin/cloud-init'
3983+ mock_which.return_value = cloud_init
3984+ mock_subp.return_value = ('NETWORK_CONFIG_V1\n', '')
3985+
3986+ available = net.netconfig_passthrough_available(self.target)
3987+
3988+ self.assertEqual(False, available,
3989+ "netconfig passthrough was available")
3990+ mock_which.assert_called_with('cloud-init', target=self.target)
3991+ mock_subp.assert_called_with([cloud_init, 'features'],
3992+ capture=True, target=self.target)
3993+
3994+ @mock.patch('curtin.net.LOG')
3995+ @mock.patch('curtin.util.subp')
3996+ @mock.patch('curtin.util.which')
3997+ @mock.patch.object(util.ChrootableTarget, "__enter__", new=lambda a: a)
3998+ def test_netconfig_passthrough_available_exc(self, mock_which, mock_subp,
3999+ mock_log):
4000+ cloud_init = '/usr/bin/cloud-init'
4001+ mock_which.return_value = cloud_init
4002+ mock_subp.side_effect = util.ProcessExecutionError
4003+
4004+ available = net.netconfig_passthrough_available(self.target)
4005+
4006+ self.assertEqual(False, available,
4007+ "netconfig passthrough was available")
4008+ mock_which.assert_called_with('cloud-init', target=self.target)
4009+ mock_subp.assert_called_with([cloud_init, 'features'],
4010+ capture=True, target=self.target)
4011+ self.assertTrue(mock_log.warning.called)
4012+
4013+ @mock.patch('curtin.util.write_file')
4014+ def test_render_netconfig_passthrough(self, mock_writefile):
4015+ netcfg = yaml.safe_load(self.config)
4016+ pt_config = 'etc/cloud/cloud.cfg.d/50-curtin-networking.cfg'
4017+ target_config = os.path.sep.join((self.target, pt_config),)
4018+
4019+ net.render_netconfig_passthrough(self.target, netconfig=netcfg)
4020+
4021+ content = config.dump_config(netcfg)
4022+ mock_writefile.assert_called_with(target_config, content=content)
4023+
4024+ def test_render_netconfig_passthrough_nonetcfg(self):
4025+ netcfg = None
4026+ self.assertRaises(ValueError,
4027+ net.render_netconfig_passthrough,
4028+ self.target, netconfig=netcfg)
4029+
4030 def test_routes_rendered(self):
4031 # as reported in bug 1649652
4032 conf = [
4033
4034=== modified file 'tests/unittests/test_partitioning.py'
4035--- tests/unittests/test_partitioning.py 2015-10-02 16:19:07 +0000
4036+++ tests/unittests/test_partitioning.py 2017-10-06 16:35:22 +0000
4037@@ -1,6 +1,7 @@
4038-import unittest
4039+from unittest import skip
4040 import mock
4041 import curtin.commands.block_meta
4042+from .helpers import CiTestCase
4043
4044 from sys import version_info
4045 if version_info.major == 2:
4046@@ -11,8 +12,8 @@
4047 parted = None # FIXME: remove these tests entirely. This is here for flake8
4048
4049
4050-@unittest.skip
4051-class TestBlock(unittest.TestCase):
4052+@skip
4053+class TestBlock(CiTestCase):
4054 storage_config = {
4055 "sda": {"id": "sda", "type": "disk", "ptable": "msdos",
4056 "serial": "DISK_1", "grub_device": "True"},
4057
4058=== added file 'tests/unittests/test_public.py'
4059--- tests/unittests/test_public.py 1970-01-01 00:00:00 +0000
4060+++ tests/unittests/test_public.py 2017-10-06 16:35:22 +0000
4061@@ -0,0 +1,54 @@
4062+
4063+from curtin import block
4064+from curtin import config
4065+from curtin import futil
4066+from curtin import util
4067+
4068+from curtin.commands import curthooks
4069+from .helpers import CiTestCase
4070+
4071+
4072+class TestPublicAPI(CiTestCase):
4073+ """Test entry points known to be used externally.
4074+
4075+ Curtin's only known external library user is the curthooks
4076+ that are present in the MAAS images. This will test for presense
4077+ of the modules and entry points that are used there.
4078+
4079+ This unit test is present to just test entry points. Function
4080+ behavior should be present elsewhere."""
4081+
4082+ def assert_has_callables(self, module, expected):
4083+ self.assertEqual(expected, _module_has(module, expected, callable))
4084+
4085+ def test_block(self):
4086+ """Verify expected attributes in curtin.block."""
4087+ self.assert_has_callables(
4088+ block,
4089+ ['get_devices_for_mp', 'get_blockdev_for_partition', '_lsblock'])
4090+
4091+ def test_config(self):
4092+ """Verify exported attributes in curtin.config."""
4093+ self.assert_has_callables(config, ['load_config'])
4094+
4095+ def test_util(self):
4096+ """Verify exported attributes in curtin.util."""
4097+ self.assert_has_callables(
4098+ util, ['RunInChroot', 'load_command_environment'])
4099+
4100+ def test_centos_apply_network_config(self):
4101+ """MAAS images use centos_apply_network_config from cmd.curthooks."""
4102+ self.assert_has_callables(curthooks, ['centos_apply_network_config'])
4103+
4104+ def test_futil(self):
4105+ """Verify exported attributes in curtin.futil."""
4106+ self.assert_has_callables(futil, ['write_files'])
4107+
4108+
4109+def _module_has(module, names, nfilter=None):
4110+ found = [(name, getattr(module, name))
4111+ for name in names if hasattr(module, name)]
4112+ if nfilter is not None:
4113+ found = [(name, attr) for name, attr in found if nfilter(attr)]
4114+
4115+ return [name for name, _ in found]
4116
4117=== modified file 'tests/unittests/test_reporter.py'
4118--- tests/unittests/test_reporter.py 2017-03-01 16:13:56 +0000
4119+++ tests/unittests/test_reporter.py 2017-10-06 16:35:22 +0000
4120@@ -21,7 +21,6 @@
4121 unicode_literals,
4122 )
4123
4124-from unittest import TestCase
4125 from mock import patch
4126
4127 from curtin.reporter.legacy import (
4128@@ -39,13 +38,12 @@
4129 from curtin.reporter import handlers
4130 from curtin import url_helper
4131 from curtin.reporter import events
4132+from .helpers import CiTestCase
4133
4134-import os
4135-import tempfile
4136 import base64
4137
4138
4139-class TestLegacyReporter(TestCase):
4140+class TestLegacyReporter(CiTestCase):
4141
4142 @patch('curtin.reporter.legacy.LOG')
4143 def test_load_reporter_logs_empty_cfg(self, mock_LOG):
4144@@ -72,7 +70,7 @@
4145 self.assertTrue(mock_LOG.error.called)
4146
4147
4148-class TestMAASReporter(TestCase):
4149+class TestMAASReporter(CiTestCase):
4150 def test_load_factory_raises_exception_wrong_options(self):
4151 options = {'wrong': 'wrong'}
4152 self.assertRaises(
4153@@ -86,7 +84,7 @@
4154 self.assertIsInstance(reporter, MAASReporter)
4155
4156
4157-class TestReporter(TestCase):
4158+class TestReporter(CiTestCase):
4159 config = {'element1': {'type': 'webhook', 'level': 'INFO',
4160 'consumer_key': "ck_foo",
4161 'consumer_secret': 'cs_foo',
4162@@ -175,39 +173,32 @@
4163 @patch('curtin.reporter.events.report_event')
4164 def test_report_finished_post_files(self, mock_report_event):
4165 test_data = b'abcdefg'
4166- tmp = tempfile.mkstemp()
4167- try:
4168- with open(tmp[1], 'wb') as fp:
4169- fp.write(test_data)
4170- events.report_finish_event(self.ev_name, self.ev_desc,
4171- post_files=[tmp[1]])
4172- event = self._get_reported_event(mock_report_event)
4173- files = event.as_dict().get('files')
4174- self.assertTrue(len(files) == 1)
4175- self.assertEqual(files[0].get('path'), tmp[1])
4176- self.assertEqual(files[0].get('encoding'), 'base64')
4177- self.assertEqual(files[0].get('content'),
4178- base64.b64encode(test_data).decode())
4179- finally:
4180- os.remove(tmp[1])
4181+ tmpfname = self.tmp_path('testfile')
4182+ with open(tmpfname, 'wb') as fp:
4183+ fp.write(test_data)
4184+ events.report_finish_event(self.ev_name, self.ev_desc,
4185+ post_files=[tmpfname])
4186+ event = self._get_reported_event(mock_report_event)
4187+ files = event.as_dict().get('files')
4188+ self.assertTrue(len(files) == 1)
4189+ self.assertEqual(files[0].get('path'), tmpfname)
4190+ self.assertEqual(files[0].get('encoding'), 'base64')
4191+ self.assertEqual(files[0].get('content'),
4192+ base64.b64encode(test_data).decode())
4193
4194 @patch('curtin.url_helper.OauthUrlHelper')
4195 def test_webhook_handler_post_files(self, mock_url_helper):
4196 test_data = b'abcdefg'
4197- tmp = tempfile.mkstemp()
4198- tmpfname = tmp[1]
4199- try:
4200- with open(tmpfname, 'wb') as fp:
4201- fp.write(test_data)
4202- event = events.FinishReportingEvent('test_event_name',
4203- 'test event description',
4204- post_files=[tmpfname],
4205- level='INFO')
4206- webhook_handler = handlers.WebHookHandler('127.0.0.1:8000',
4207- level='INFO')
4208- webhook_handler.publish_event(event)
4209- webhook_handler.oauth_helper.geturl.assert_called_with(
4210- url='127.0.0.1:8000', data=event.as_dict(),
4211- headers=webhook_handler.headers, retries=None)
4212- finally:
4213- os.remove(tmpfname)
4214+ tmpfname = self.tmp_path('testfile')
4215+ with open(tmpfname, 'wb') as fp:
4216+ fp.write(test_data)
4217+ event = events.FinishReportingEvent('test_event_name',
4218+ 'test event description',
4219+ post_files=[tmpfname],
4220+ level='INFO')
4221+ webhook_handler = handlers.WebHookHandler('127.0.0.1:8000',
4222+ level='INFO')
4223+ webhook_handler.publish_event(event)
4224+ webhook_handler.oauth_helper.geturl.assert_called_with(
4225+ url='127.0.0.1:8000', data=event.as_dict(),
4226+ headers=webhook_handler.headers, retries=None)
4227
4228=== modified file 'tests/unittests/test_util.py'
4229--- tests/unittests/test_util.py 2017-06-12 20:39:06 +0000
4230+++ tests/unittests/test_util.py 2017-10-06 16:35:22 +0000
4231@@ -1,16 +1,14 @@
4232-from unittest import TestCase, skipIf
4233+from unittest import skipIf
4234 import mock
4235 import os
4236 import stat
4237-import shutil
4238-import tempfile
4239 from textwrap import dedent
4240
4241 from curtin import util
4242-from .helpers import simple_mocked_open
4243-
4244-
4245-class TestLogTimer(TestCase):
4246+from .helpers import CiTestCase, simple_mocked_open
4247+
4248+
4249+class TestLogTimer(CiTestCase):
4250 def test_logger_called(self):
4251 data = {}
4252
4253@@ -24,16 +22,14 @@
4254 self.assertIn("mymessage", data['msg'])
4255
4256
4257-class TestDisableDaemons(TestCase):
4258+class TestDisableDaemons(CiTestCase):
4259 prcpath = "usr/sbin/policy-rc.d"
4260
4261 def setUp(self):
4262- self.target = tempfile.mkdtemp()
4263+ super(TestDisableDaemons, self).setUp()
4264+ self.target = self.tmp_dir()
4265 self.temp_prc = os.path.join(self.target, self.prcpath)
4266
4267- def tearDown(self):
4268- shutil.rmtree(self.target)
4269-
4270 def test_disable_daemons_in_root_works(self):
4271 ret = util.disable_daemons_in_root(self.target)
4272 self.assertTrue(ret)
4273@@ -55,8 +51,10 @@
4274 self.assertTrue(os.path.exists(self.temp_prc))
4275
4276
4277-class TestWhich(TestCase):
4278+class TestWhich(CiTestCase):
4279+
4280 def setUp(self):
4281+ super(TestWhich, self).setUp()
4282 self.orig_is_exe = util.is_exe
4283 util.is_exe = self.my_is_exe
4284 self.orig_path = os.environ.get("PATH")
4285@@ -103,8 +101,10 @@
4286 self.assertEqual(found, "/usr/bin2/fuzz")
4287
4288
4289-class TestLsbRelease(TestCase):
4290+class TestLsbRelease(CiTestCase):
4291+
4292 def setUp(self):
4293+ super(TestLsbRelease, self).setUp()
4294 self._reset_cache()
4295
4296 def _reset_cache(self):
4297@@ -143,7 +143,7 @@
4298 self.assertEqual(util.lsb_release(), expected)
4299
4300
4301-class TestSubp(TestCase):
4302+class TestSubp(CiTestCase):
4303
4304 stdin2err = ['bash', '-c', 'cat >&2']
4305 stdin2out = ['cat']
4306@@ -160,6 +160,12 @@
4307 decode_type = str
4308 nodecode_type = bytes
4309
4310+ def setUp(self):
4311+ super(TestSubp, self).setUp()
4312+ self.add_patch(
4313+ 'curtin.util._get_unshare_pid_args', 'mock_get_unshare_pid_args',
4314+ return_value=[])
4315+
4316 def printf_cmd(self, *args):
4317 # bash's printf supports \xaa. So does /usr/bin/printf
4318 # but by using bash, we remove dependency on another program.
4319@@ -296,12 +302,29 @@
4320 calls = m_popen.call_args_list
4321 popen_args, popen_kwargs = calls[-1]
4322 target = util.target_path(kwargs.get('target', None))
4323+ unshcmd = self.mock_get_unshare_pid_args.return_value
4324 if target == "/":
4325- self.assertEqual(cmd, popen_args[0])
4326+ self.assertEqual(unshcmd + list(cmd), popen_args[0])
4327 else:
4328- self.assertEqual(['chroot', target] + list(cmd), popen_args[0])
4329+ self.assertEqual(unshcmd + ['chroot', target] + list(cmd),
4330+ popen_args[0])
4331 return calls
4332
4333+ def test_args_can_be_a_tuple(self):
4334+ """subp can take a tuple for cmd rather than a list."""
4335+ my_cmd = tuple(['echo', 'hi', 'mom'])
4336+ calls = self._subp_wrap_popen(my_cmd, {})
4337+ args, kwargs = calls[0]
4338+ # subp was called with cmd as a tuple. That may get converted to
4339+ # a list before subprocess.popen. So only compare as lists.
4340+ self.assertEqual(1, len(calls))
4341+ self.assertEqual(list(my_cmd), list(args[0]))
4342+
4343+ def test_args_can_be_a_string(self):
4344+ """subp("cat") is acceptable, as suprocess.call("cat") works fine."""
4345+ out, err = util.subp("cat", data=b'hi mom', capture=True, decode=False)
4346+ self.assertEqual(b'hi mom', out)
4347+
4348 def test_with_target_gets_chroot(self):
4349 args, kwargs = self._subp_wrap_popen(["my-command"],
4350 {'target': "/mytarget"})[0]
4351@@ -342,8 +365,94 @@
4352 # since we fail a few times, it needs to have been called again.
4353 self.assertEqual(len(r), len(rcs))
4354
4355-
4356-class TestHuman2Bytes(TestCase):
4357+ def test_unshare_pid_return_is_used(self):
4358+ """The return of _get_unshare_pid_return needs to be in command."""
4359+ my_unshare_cmd = ['do-unshare-command', 'arg0', 'arg1', '--']
4360+ self.mock_get_unshare_pid_args.return_value = my_unshare_cmd
4361+ my_kwargs = {'target': '/target', 'unshare_pid': True}
4362+ r = self._subp_wrap_popen(['apt-get', 'install'], my_kwargs)
4363+ self.assertEqual(1, len(r))
4364+ args, kwargs = r[0]
4365+ self.assertEqual(
4366+ [mock.call(my_kwargs['unshare_pid'], my_kwargs['target'])],
4367+ self.mock_get_unshare_pid_args.call_args_list)
4368+ expected = (my_unshare_cmd + ['chroot', '/target'] +
4369+ ['apt-get', 'install'])
4370+ self.assertEqual(expected, args[0])
4371+
4372+
4373+class TestGetUnsharePidArgs(CiTestCase):
4374+ """Test the internal implementation for when to unshare."""
4375+
4376+ def setUp(self):
4377+ super(TestGetUnsharePidArgs, self).setUp()
4378+ self.add_patch('curtin.util._has_unshare_pid', 'mock_has_unshare_pid',
4379+ return_value=True)
4380+ # our trusty tox environment with mock 1.0.1 will stack trace
4381+ # if autospec is not disabled here.
4382+ self.add_patch('curtin.util.os.geteuid', 'mock_geteuid',
4383+ autospec=False, return_value=0)
4384+
4385+ def assertOff(self, result):
4386+ self.assertEqual([], result)
4387+
4388+ def assertOn(self, result):
4389+ self.assertEqual(['unshare', '--fork', '--pid', '--'], result)
4390+
4391+ def test_unshare_pid_none_and_not_root_means_off(self):
4392+ """If not root, then expect off."""
4393+ self.assertOff(util._get_unshare_pid_args(None, "/foo", 500))
4394+ self.assertOff(util._get_unshare_pid_args(None, "/", 500))
4395+
4396+ self.mock_geteuid.return_value = 500
4397+ self.assertOff(util._get_unshare_pid_args(None, "/"))
4398+ self.assertOff(
4399+ util._get_unshare_pid_args(unshare_pid=None, target="/foo"))
4400+
4401+ def test_unshare_pid_none_and_no_unshare_pid_means_off(self):
4402+ """No unshare support and unshare_pid is None means off."""
4403+ self.mock_has_unshare_pid.return_value = False
4404+ self.assertOff(util._get_unshare_pid_args(None, "/target", 0))
4405+
4406+ def test_unshare_pid_true_and_no_unshare_pid_raises(self):
4407+ """Passing unshare_pid in as True and no command should raise."""
4408+ self.mock_has_unshare_pid.return_value = False
4409+ expected_msg = 'no unshare command'
4410+ with self.assertRaisesRegexp(RuntimeError, expected_msg):
4411+ util._get_unshare_pid_args(True)
4412+
4413+ with self.assertRaisesRegexp(RuntimeError, expected_msg):
4414+ util._get_unshare_pid_args(True, "/foo", 0)
4415+
4416+ def test_unshare_pid_true_and_not_root_raises(self):
4417+ """When unshare_pid is True for non-root an error is raised."""
4418+ expected_msg = 'euid.* != 0'
4419+ with self.assertRaisesRegexp(RuntimeError, expected_msg):
4420+ util._get_unshare_pid_args(True, "/foo", 500)
4421+
4422+ self.mock_geteuid.return_value = 500
4423+ with self.assertRaisesRegexp(RuntimeError, expected_msg):
4424+ util._get_unshare_pid_args(True)
4425+
4426+ def test_euid0_target_not_slash(self):
4427+ """If root and target is not /, then expect on."""
4428+ self.assertOn(util._get_unshare_pid_args(None, target="/foo", euid=0))
4429+
4430+ def test_euid0_target_slash(self):
4431+ """If root and target is /, then expect off."""
4432+ self.assertOff(util._get_unshare_pid_args(None, "/", 0))
4433+ self.assertOff(util._get_unshare_pid_args(None, target=None, euid=0))
4434+
4435+ def test_unshare_pid_of_false_means_off(self):
4436+ """Any unshare_pid value false-ish other than None means no unshare."""
4437+ self.assertOff(
4438+ util._get_unshare_pid_args(unshare_pid=False, target=None))
4439+ self.assertOff(util._get_unshare_pid_args(False, "/target", 1))
4440+ self.assertOff(util._get_unshare_pid_args(False, "/", 0))
4441+ self.assertOff(util._get_unshare_pid_args("", "/target", 0))
4442+
4443+
4444+class TestHuman2Bytes(CiTestCase):
4445 GB = 1024 * 1024 * 1024
4446 MB = 1024 * 1024
4447
4448@@ -397,52 +506,42 @@
4449 util.bytes2human(util.human2bytes(size_str)), size_str)
4450
4451
4452-class TestSetUnExecutable(TestCase):
4453+class TestSetUnExecutable(CiTestCase):
4454 tmpf = None
4455 tmpd = None
4456
4457- def tearDown(self):
4458- if self.tmpf:
4459- if os.path.exists(self.tmpf):
4460- os.unlink(self.tmpf)
4461- self.tmpf = None
4462- if self.tmpd:
4463- shutil.rmtree(self.tmpd)
4464- self.tmpd = None
4465-
4466- def tempfile(self, data=None):
4467- fp, self.tmpf = tempfile.mkstemp()
4468- if data:
4469- fp.write(data)
4470- os.close(fp)
4471- return self.tmpf
4472+ def setUp(self):
4473+ super(CiTestCase, self).setUp()
4474+ self.tmpd = self.tmp_dir()
4475
4476 def test_change_needed_returns_original_mode(self):
4477- tmpf = self.tempfile()
4478+ tmpf = self.tmp_path('testfile')
4479+ util.write_file(tmpf, '')
4480 os.chmod(tmpf, 0o755)
4481 ret = util.set_unexecutable(tmpf)
4482 self.assertEqual(ret, 0o0755)
4483
4484 def test_no_change_needed_returns_none(self):
4485- tmpf = self.tempfile()
4486+ tmpf = self.tmp_path('testfile')
4487+ util.write_file(tmpf, '')
4488 os.chmod(tmpf, 0o600)
4489 ret = util.set_unexecutable(tmpf)
4490 self.assertEqual(ret, None)
4491
4492 def test_change_does_as_expected(self):
4493- tmpf = self.tempfile()
4494+ tmpf = self.tmp_path('testfile')
4495+ util.write_file(tmpf, '')
4496 os.chmod(tmpf, 0o755)
4497 ret = util.set_unexecutable(tmpf)
4498 self.assertEqual(ret, 0o0755)
4499 self.assertEqual(stat.S_IMODE(os.stat(tmpf).st_mode), 0o0644)
4500
4501 def test_strict_no_exists_raises_exception(self):
4502- self.tmpd = tempfile.mkdtemp()
4503 bogus = os.path.join(self.tmpd, 'bogus')
4504 self.assertRaises(ValueError, util.set_unexecutable, bogus, True)
4505
4506
4507-class TestTargetPath(TestCase):
4508+class TestTargetPath(CiTestCase):
4509 def test_target_empty_string(self):
4510 self.assertEqual("/etc/passwd", util.target_path("", "/etc/passwd"))
4511
4512@@ -484,7 +583,7 @@
4513 util.target_path("/target/", "///my/path/"))
4514
4515
4516-class TestRunInChroot(TestCase):
4517+class TestRunInChroot(CiTestCase):
4518 """Test the legacy 'RunInChroot'.
4519
4520 The test works by mocking ChrootableTarget's __enter__ to do nothing.
4521@@ -514,7 +613,7 @@
4522 m_subp.assert_called_with(cmd, target=target)
4523
4524
4525-class TestLoadFile(TestCase):
4526+class TestLoadFile(CiTestCase):
4527 """Test utility 'load_file'"""
4528
4529 def test_load_file_simple(self):
4530@@ -545,7 +644,7 @@
4531 self.assertEqual(loaded_contents, contents)
4532
4533
4534-class TestIpAddress(TestCase):
4535+class TestIpAddress(CiTestCase):
4536 """Test utility 'is_valid_ip{,v4,v6}_address'"""
4537
4538 def test_is_valid_ipv6_address(self):
4539@@ -570,10 +669,11 @@
4540 '2002:4559:1FE2:0000:0000:0000:4559:1FE2'))
4541
4542
4543-class TestLoadCommandEnvironment(TestCase):
4544+class TestLoadCommandEnvironment(CiTestCase):
4545+
4546 def setUp(self):
4547- self.tmpd = tempfile.mkdtemp()
4548- self.addCleanup(shutil.rmtree, self.tmpd)
4549+ super(TestLoadCommandEnvironment, self).setUp()
4550+ self.tmpd = self.tmp_dir()
4551 all_names = {
4552 'CONFIG',
4553 'OUTPUT_FSTAB',
4554@@ -616,7 +716,7 @@
4555 self.fail("unexpected key error raised: %s" % e)
4556
4557
4558-class TestWaitForRemoval(TestCase):
4559+class TestWaitForRemoval(CiTestCase):
4560 def test_wait_for_removal_missing_path(self):
4561 with self.assertRaises(ValueError):
4562 util.wait_for_removal(None)
4563@@ -684,14 +784,12 @@
4564 ])
4565
4566
4567-class TestGetEFIBootMGR(TestCase):
4568+class TestGetEFIBootMGR(CiTestCase):
4569
4570 def setUp(self):
4571 super(TestGetEFIBootMGR, self).setUp()
4572- mock_chroot = mock.patch(
4573- 'curtin.util.ChrootableTarget', autospec=False)
4574- self.mock_chroot = mock_chroot.start()
4575- self.addCleanup(mock_chroot.stop)
4576+ self.add_patch(
4577+ 'curtin.util.ChrootableTarget', 'mock_chroot', autospec=False)
4578 self.mock_in_chroot = mock.MagicMock()
4579 self.mock_in_chroot.__enter__.return_value = self.mock_in_chroot
4580 self.in_chroot_subp_output = []
4581@@ -753,4 +851,55 @@
4582 }, observed)
4583
4584
4585+class TestUsesSystemd(CiTestCase):
4586+
4587+ def setUp(self):
4588+ super(TestUsesSystemd, self).setUp()
4589+ self._reset_cache()
4590+ self.add_patch('curtin.util.os.path.isdir', 'mock_isdir')
4591+
4592+ def _reset_cache(self):
4593+ util._USES_SYSTEMD = None
4594+
4595+ def test_uses_systemd_on_systemd(self):
4596+ """ Test that uses_systemd returns True if sdpath is a dir """
4597+ # systemd_enabled
4598+ self.mock_isdir.return_value = True
4599+ result = util.uses_systemd()
4600+ self.assertEqual(True, result)
4601+ self.assertEqual(1, len(self.mock_isdir.call_args_list))
4602+
4603+ def test_uses_systemd_cached(self):
4604+ """Test that we cache the uses_systemd result"""
4605+
4606+ # reset_cache should ensure it's unset
4607+ self.assertEqual(None, util._USES_SYSTEMD)
4608+
4609+ # systemd enabled
4610+ self.mock_isdir.return_value = True
4611+
4612+ # first time
4613+ first_result = util.uses_systemd()
4614+
4615+ # check the cache value
4616+ self.assertEqual(first_result, util._USES_SYSTEMD)
4617+
4618+ # second time
4619+ second_result = util.uses_systemd()
4620+
4621+ # results should match between tries
4622+ self.assertEqual(True, first_result)
4623+ self.assertEqual(True, second_result)
4624+
4625+ # isdir should only be called once
4626+ self.assertEqual(1, len(self.mock_isdir.call_args_list))
4627+
4628+ def test_uses_systemd_on_non_systemd(self):
4629+ """ Test that uses_systemd returns False if sdpath is not a dir """
4630+ # systemd not available
4631+ self.mock_isdir.return_value = False
4632+ result = util.uses_systemd()
4633+ self.assertEqual(False, result)
4634+
4635+
4636 # vi: ts=4 expandtab syntax=python
4637
4638=== modified file 'tests/unittests/test_version.py'
4639--- tests/unittests/test_version.py 2017-02-08 22:22:44 +0000
4640+++ tests/unittests/test_version.py 2017-10-06 16:35:22 +0000
4641@@ -1,28 +1,16 @@
4642-from unittest import TestCase
4643 import mock
4644 import subprocess
4645 import os
4646
4647 from curtin import version
4648 from curtin import __version__ as old_version
4649-
4650-
4651-class CurtinVersionBase(TestCase):
4652- def setUp(self):
4653- super(CurtinVersionBase, self).setUp()
4654-
4655- def add_patch(self, target, attr):
4656- """Patches specified target object and sets it as attr on test
4657- instance also schedules cleanup"""
4658- m = mock.patch(target, autospec=True)
4659- p = m.start()
4660- self.addCleanup(m.stop)
4661- setattr(self, attr, p)
4662-
4663-
4664-class TestCurtinVersion(CurtinVersionBase):
4665-
4666- def setUp(self):
4667+from .helpers import CiTestCase
4668+
4669+
4670+class TestCurtinVersion(CiTestCase):
4671+
4672+ def setUp(self):
4673+ super(TestCurtinVersion, self).setUp()
4674 self.add_patch('subprocess.check_output', 'mock_subp')
4675 self.add_patch('os.path', 'mock_path')
4676
4677
4678=== modified file 'tests/vmtests/__init__.py'
4679--- tests/vmtests/__init__.py 2017-06-12 20:39:06 +0000
4680+++ tests/vmtests/__init__.py 2017-10-06 16:35:22 +0000
4681@@ -38,6 +38,7 @@
4682 CURTIN_VMTEST_IMAGE_SYNC = os.environ.get("CURTIN_VMTEST_IMAGE_SYNC", "1")
4683 IMAGE_SYNCS = []
4684 TARGET_IMAGE_FORMAT = "raw"
4685+TAR_DISKS = bool(int(os.environ.get("CURTIN_VMTEST_TAR_DISKS", "0")))
4686
4687
4688 DEFAULT_BRIDGE = os.environ.get("CURTIN_VMTEST_BRIDGE", "user")
4689@@ -335,7 +336,12 @@
4690 __test__ = False
4691 arch_skip = []
4692 boot_timeout = BOOT_TIMEOUT
4693- collect_scripts = []
4694+ collect_scripts = [textwrap.dedent("""
4695+ cd OUTPUT_COLLECT_D
4696+ dpkg-query --show \
4697+ --showformat='${db:Status-Abbrev}\t${Package}\t${Version}\n' \
4698+ > debian-packages.txt 2> debian-packages.txt.err
4699+ """)]
4700 conf_file = "examples/tests/basic.yaml"
4701 nr_cpus = None
4702 dirty_disks = False
4703@@ -368,6 +374,8 @@
4704 target_krel = None
4705 target_ftype = "vmtest.root-tgz"
4706
4707+ _debian_packages = None
4708+
4709 def shortDescription(self):
4710 return None
4711
4712@@ -593,7 +601,7 @@
4713 logger.debug("Interface name: {}".format(ifname))
4714 iface = interfaces.get(ifname)
4715 hwaddr = iface.get('mac_address')
4716- if hwaddr:
4717+ if iface['type'] == 'physical' and hwaddr:
4718 macs.append(hwaddr)
4719 netdevs = []
4720 if len(macs) > 0:
4721@@ -685,6 +693,12 @@
4722 configs.append(excfg)
4723 logger.debug('Added extra config {}'.format(excfg))
4724
4725+ if cls.target_distro == "centos":
4726+ centos_default = 'examples/tests/centos_defaults.yaml'
4727+ configs.append(centos_default)
4728+ logger.info('Detected centos, adding default config %s',
4729+ centos_default)
4730+
4731 if cls.multipath:
4732 disks = disks * cls.multipath_num_paths
4733
4734@@ -871,8 +885,11 @@
4735 raise
4736
4737 # capture curtin install log and webhook timings
4738- util.subp(["tools/curtin-log-print", "--dumpfiles", cls.td.logs,
4739- cls.reporting_log], capture=True)
4740+ try:
4741+ util.subp(["tools/curtin-log-print", "--dumpfiles", cls.td.logs,
4742+ cls.reporting_log], capture=True)
4743+ except util.ProcessExecutionError as error:
4744+ logger.debug('tools/curtin-log-print failed: %s', error)
4745
4746 logger.info(
4747 "%s: setUpClass finished. took %.02f seconds. Running testcases.",
4748@@ -929,7 +946,8 @@
4749 clean_working_dir(cls.td.tmpdir, success,
4750 keep_pass=KEEP_DATA['pass'],
4751 keep_fail=KEEP_DATA['fail'])
4752-
4753+ if TAR_DISKS:
4754+ tar_disks(cls.td.tmpdir)
4755 cls.cleanIscsiState(success,
4756 keep_pass=KEEP_DATA['pass'],
4757 keep_fail=KEEP_DATA['fail'])
4758@@ -1143,6 +1161,18 @@
4759 fp.write(json.dumps(data, indent=2, sort_keys=True,
4760 separators=(',', ': ')) + "\n")
4761
4762+ @property
4763+ def debian_packages(self):
4764+ if self._debian_packages is None:
4765+ data = self.load_collect_file("debian-packages.txt")
4766+ pkgs = {}
4767+ for line in data.splitlines():
4768+ # lines are <status>\t<
4769+ status, pkg, ver = line.split('\t')
4770+ pkgs[pkg] = {'status': status, 'version': ver}
4771+ self._debian_packages = pkgs
4772+ return self._debian_packages
4773+
4774
4775 class PsuedoVMBaseClass(VMBaseClass):
4776 # This mimics much of the VMBaseClass just with faster setUpClass
4777@@ -1332,8 +1362,13 @@
4778 output_device = '/dev/disk/by-id/virtio-%s' % OUTPUT_DISK_NAME
4779
4780 collect_prep = textwrap.dedent("mkdir -p " + output_dir)
4781- collect_post = textwrap.dedent(
4782- 'tar -C "%s" -cf "%s" .' % (output_dir, output_device))
4783+ collect_post = textwrap.dedent("""\
4784+ cd {output_dir}\n
4785+ # remove any symlinks, but archive information about them.
4786+ # %Y target's file type, %P = path, %l = target of symlink
4787+ find -type l -printf "%Y\t%P\t%l\n" -delete > symlinks.txt
4788+ tar -cf "{output_device}" .
4789+ """).format(output_dir=output_dir, output_device=output_device)
4790
4791 # copy /root for curtin config and install.log
4792 copy_rootdir = textwrap.dedent("cp -a /root " + output_dir)
4793@@ -1410,6 +1445,23 @@
4794 KEEP_DATA.update(data)
4795
4796
4797+def tar_disks(tmpdir, outfile="disks.tar", diskmatch=".img"):
4798+ """ Tar up files in ``tmpdir``/disks that ends with the pattern supplied"""
4799+
4800+ disks_dir = os.path.join(tmpdir, "disks")
4801+ if os.path.exists(disks_dir):
4802+ outfile = os.path.join(disks_dir, outfile)
4803+ disks = [os.path.join(disks_dir, disk) for disk in
4804+ os.listdir(disks_dir) if disk.endswith(diskmatch)]
4805+ cmd = ["tar", "--create", "--file=%s" % outfile,
4806+ "--verbose", "--remove-files", "--sparse"]
4807+ cmd.extend(disks)
4808+ logger.info('Taring %s disks sparsely to %s', len(disks), outfile)
4809+ util.subp(cmd, capture=True)
4810+ else:
4811+ logger.error('Failed to find "disks" dir under tmpdir: %s', tmpdir)
4812+
4813+
4814 def boot_log_wrap(name, func, cmd, console_log, timeout, purpose):
4815 logger.debug("%s[%s]: booting with timeout=%s log=%s cmd: %s",
4816 name, purpose, timeout, console_log, ' '.join(cmd))
4817
4818=== modified file 'tests/vmtests/releases.py'
4819--- tests/vmtests/releases.py 2017-06-12 20:39:06 +0000
4820+++ tests/vmtests/releases.py 2017-10-06 16:35:22 +0000
4821@@ -77,22 +77,10 @@
4822 target_release = "trusty"
4823
4824
4825-class _VividBase(_UbuntuBase):
4826- release = "vivid"
4827-
4828-
4829-class _WilyBase(_UbuntuBase):
4830- release = "wily"
4831-
4832-
4833 class _XenialBase(_UbuntuBase):
4834 release = "xenial"
4835
4836
4837-class _YakketyBase(_UbuntuBase):
4838- release = "yakkety"
4839-
4840-
4841 class _ZestyBase(_UbuntuBase):
4842 release = "zesty"
4843
4844@@ -110,10 +98,7 @@
4845 trusty_hwe_w = _TrustyHWEW
4846 trusty_hwe_x = _TrustyHWEX
4847 trustyfromxenial = _TrustyFromXenial
4848- vivid = _VividBase
4849- wily = _WilyBase
4850 xenial = _XenialBase
4851- yakkety = _YakketyBase
4852 zesty = _ZestyBase
4853 artful = _ArtfulBase
4854
4855
4856=== modified file 'tests/vmtests/test_apt_config_cmd.py'
4857--- tests/vmtests/test_apt_config_cmd.py 2017-06-12 20:39:06 +0000
4858+++ tests/vmtests/test_apt_config_cmd.py 2017-10-06 16:35:22 +0000
4859@@ -55,10 +55,6 @@
4860 __test__ = True
4861
4862
4863-class YakketyTestAptConfigCMDCMD(relbase.yakkety, TestAptConfigCMD):
4864- __test__ = True
4865-
4866-
4867 class ZestyTestAptConfigCMDCMD(relbase.zesty, TestAptConfigCMD):
4868 __test__ = True
4869
4870
4871=== modified file 'tests/vmtests/test_basic.py'
4872--- tests/vmtests/test_basic.py 2017-06-12 20:39:06 +0000
4873+++ tests/vmtests/test_basic.py 2017-10-06 16:35:22 +0000
4874@@ -202,19 +202,10 @@
4875 __test__ = True
4876
4877
4878-class WilyTestBasic(relbase.wily, TestBasicAbs):
4879- # EOL - 2016-07-28
4880- __test__ = False
4881-
4882-
4883 class XenialTestBasic(relbase.xenial, TestBasicAbs):
4884 __test__ = True
4885
4886
4887-class YakketyTestBasic(relbase.yakkety, TestBasicAbs):
4888- __test__ = True
4889-
4890-
4891 class ZestyTestBasic(relbase.zesty, TestBasicAbs):
4892 __test__ = True
4893
4894@@ -323,10 +314,6 @@
4895 __test__ = True
4896
4897
4898-class YakketyTestScsiBasic(relbase.yakkety, TestBasicScsiAbs):
4899- __test__ = True
4900-
4901-
4902 class ZestyTestScsiBasic(relbase.zesty, TestBasicScsiAbs):
4903 __test__ = True
4904
4905
4906=== modified file 'tests/vmtests/test_bcache_basic.py'
4907--- tests/vmtests/test_bcache_basic.py 2017-06-12 20:39:06 +0000
4908+++ tests/vmtests/test_bcache_basic.py 2017-10-06 16:35:22 +0000
4909@@ -59,10 +59,6 @@
4910 __test__ = True
4911
4912
4913-class YakketyBcacheBasic(relbase.yakkety, TestBcacheBasic):
4914- __test__ = True
4915-
4916-
4917 class ZestyBcacheBasic(relbase.zesty, TestBcacheBasic):
4918 __test__ = True
4919
4920
4921=== modified file 'tests/vmtests/test_centos_basic.py'
4922--- tests/vmtests/test_centos_basic.py 2017-01-18 16:01:35 +0000
4923+++ tests/vmtests/test_centos_basic.py 2017-10-06 16:35:22 +0000
4924@@ -1,5 +1,6 @@
4925 from . import VMBaseClass
4926 from .releases import centos_base_vm_classes as relbase
4927+from .test_network import TestNetworkBaseTestsAbs
4928
4929 import textwrap
4930
4931@@ -9,10 +10,20 @@
4932 __test__ = False
4933 conf_file = "examples/tests/centos_basic.yaml"
4934 extra_kern_args = "BOOTIF=eth0-52:54:00:12:34:00"
4935+ # XXX: command | tee output is required for Centos under SELinux
4936+ # http://danwalsh.livejournal.com/22860.html
4937 collect_scripts = [textwrap.dedent(
4938 """
4939 cd OUTPUT_COLLECT_D
4940 cat /etc/fstab > fstab
4941+ rpm -qa | cat >rpm_qa
4942+ ifconfig -a | cat >ifconfig_a
4943+ ip a | cat >ip_a
4944+ cp -a /etc/sysconfig/network-scripts .
4945+ cp -a /var/log/messages .
4946+ cp -a /var/log/cloud-init* .
4947+ cp -a /var/lib/cloud ./var_lib_cloud
4948+ cp -a /run/cloud-init ./run_cloud-init
4949 """)]
4950 fstab_expected = {
4951 'LABEL=cloudimg-rootfs': '/',
4952@@ -40,3 +51,27 @@
4953 # FIXME: test is disabled because the grub config script in target
4954 # specifies drive using hd(1,0) syntax, which breaks when the
4955 # installation medium is removed. other than this, the install works
4956+
4957+
4958+class CentosTestBasicNetworkAbs(TestNetworkBaseTestsAbs):
4959+ conf_file = "examples/tests/centos_basic.yaml"
4960+ extra_kern_args = "BOOTIF=eth0-52:54:00:12:34:00"
4961+ collect_scripts = TestNetworkBaseTestsAbs.collect_scripts + [
4962+ textwrap.dedent("""
4963+ cd OUTPUT_COLLECT_D
4964+ cp -a /etc/sysconfig/network-scripts .
4965+ cp -a /var/log/cloud-init* .
4966+ cp -a /var/lib/cloud ./var_lib_cloud
4967+ cp -a /run/cloud-init ./run_cloud-init
4968+ """)]
4969+
4970+ def test_etc_network_interfaces(self):
4971+ pass
4972+
4973+ def test_etc_resolvconf(self):
4974+ pass
4975+
4976+
4977+class Centos70BasicNetworkFromXenialTestBasic(relbase.centos70fromxenial,
4978+ CentosTestBasicNetworkAbs):
4979+ __test__ = True
4980
4981=== modified file 'tests/vmtests/test_iscsi.py'
4982--- tests/vmtests/test_iscsi.py 2017-06-12 20:39:06 +0000
4983+++ tests/vmtests/test_iscsi.py 2017-10-06 16:35:22 +0000
4984@@ -59,10 +59,6 @@
4985 __test__ = True
4986
4987
4988-class YakketyTestIscsiBasic(relbase.yakkety, TestBasicIscsiAbs):
4989- __test__ = True
4990-
4991-
4992 class ZestyTestIscsiBasic(relbase.zesty, TestBasicIscsiAbs):
4993 __test__ = True
4994
4995
4996=== added file 'tests/vmtests/test_journald_reporter.py'
4997--- tests/vmtests/test_journald_reporter.py 1970-01-01 00:00:00 +0000
4998+++ tests/vmtests/test_journald_reporter.py 2017-10-06 16:35:22 +0000
4999@@ -0,0 +1,52 @@
5000+from . import VMBaseClass
The diff has been truncated for viewing.

Subscribers

People subscribed via source and target branches

to all changes: