Merge ~chad.smith/cloud-init:ubuntu/xenial into cloud-init:ubuntu/xenial

Proposed by Chad Smith on 2018-03-27
Status: Merged
Merged at revision: e9ebc06ea8d64e5a50fa17995a639527115bc544
Proposed branch: ~chad.smith/cloud-init:ubuntu/xenial
Merge into: cloud-init:ubuntu/xenial
Diff against target: 7954 lines (+4621/-638)
105 files modified
.pylintrc (+11/-1)
ChangeLog (+110/-0)
cloudinit/apport.py (+3/-3)
cloudinit/cloud.py (+3/-2)
cloudinit/cmd/main.py (+29/-6)
cloudinit/cmd/tests/test_clean.py (+2/-1)
cloudinit/cmd/tests/test_main.py (+161/-0)
cloudinit/cmd/tests/test_status.py (+2/-1)
cloudinit/config/cc_keys_to_console.py (+1/-3)
cloudinit/config/cc_puppet.py (+44/-15)
cloudinit/config/cc_resizefs.py (+22/-0)
cloudinit/config/cc_runcmd.py (+4/-2)
cloudinit/config/cc_salt_minion.py (+65/-20)
cloudinit/config/cc_set_hostname.py (+35/-6)
cloudinit/config/cc_snap.py (+230/-0)
cloudinit/config/cc_snap_config.py (+7/-0)
cloudinit/config/cc_snappy.py (+8/-0)
cloudinit/config/cc_ssh_authkey_fingerprints.py (+4/-5)
cloudinit/config/cc_ubuntu_advantage.py (+173/-0)
cloudinit/config/tests/test_snap.py (+490/-0)
cloudinit/config/tests/test_ubuntu_advantage.py (+269/-0)
cloudinit/distros/arch.py (+1/-4)
cloudinit/distros/freebsd.py (+6/-0)
cloudinit/distros/opensuse.py (+2/-3)
cloudinit/ec2_utils.py (+2/-4)
cloudinit/net/cmdline.py (+22/-2)
cloudinit/net/netplan.py (+14/-21)
cloudinit/net/network_state.py (+11/-1)
cloudinit/settings.py (+2/-0)
cloudinit/sources/DataSourceAliYun.py (+1/-1)
cloudinit/sources/DataSourceAzure.py (+11/-22)
cloudinit/sources/DataSourceCloudSigma.py (+1/-1)
cloudinit/sources/DataSourceConfigDrive.py (+10/-0)
cloudinit/sources/DataSourceGCE.py (+8/-9)
cloudinit/sources/DataSourceHetzner.py (+106/-0)
cloudinit/sources/DataSourceIBMCloud.py (+325/-0)
cloudinit/sources/DataSourceOVF.py (+16/-5)
cloudinit/sources/DataSourceOpenNebula.py (+75/-34)
cloudinit/sources/DataSourceScaleway.py (+4/-4)
cloudinit/sources/__init__.py (+17/-4)
cloudinit/sources/helpers/hetzner.py (+26/-0)
cloudinit/sources/tests/test_init.py (+97/-1)
cloudinit/stages.py (+1/-2)
cloudinit/subp.py (+57/-0)
cloudinit/tests/helpers.py (+35/-19)
cloudinit/tests/test_subp.py (+61/-0)
cloudinit/tests/test_util.py (+169/-0)
cloudinit/url_helper.py (+14/-10)
cloudinit/util.py (+115/-24)
cloudinit/version.py (+1/-1)
config/cloud.cfg.tmpl (+9/-3)
debian/changelog (+57/-3)
debian/patches/azure-use-walinux-agent.patch (+1/-1)
debian/patches/ds-identify-behavior-xenial.patch (+3/-3)
doc/examples/cloud-config-chef.txt (+2/-2)
doc/rtd/conf.py (+1/-0)
doc/rtd/topics/capabilities.rst (+8/-6)
doc/rtd/topics/debugging.rst (+31/-26)
doc/rtd/topics/modules.rst (+2/-0)
doc/rtd/topics/network-config.rst (+2/-2)
doc/rtd/topics/tests.rst (+10/-10)
packages/debian/control.in (+2/-1)
tests/cloud_tests/bddeb.py (+1/-1)
tests/cloud_tests/platforms/ec2/__init__.py (+0/-0)
tests/cloud_tests/platforms/lxd/__init__.py (+0/-0)
tests/cloud_tests/platforms/lxd/platform.py (+0/-4)
tests/cloud_tests/platforms/nocloudkvm/__init__.py (+0/-0)
tests/cloud_tests/platforms/nocloudkvm/instance.py (+1/-1)
tests/cloud_tests/platforms/nocloudkvm/platform.py (+0/-4)
tests/cloud_tests/platforms/platforms.py (+12/-2)
tests/cloud_tests/releases.yaml (+3/-0)
tests/cloud_tests/testcases.yaml (+3/-0)
tests/cloud_tests/testcases/__init__.py (+3/-0)
tests/cloud_tests/testcases/base.py (+168/-5)
tests/cloud_tests/testcases/main/command_output_simple.py (+2/-15)
tests/cloud_tests/testcases/modules/salt_minion.py (+10/-0)
tests/cloud_tests/testcases/modules/salt_minion.yaml (+9/-1)
tests/cloud_tests/testcases/modules/snap.py (+16/-0)
tests/cloud_tests/testcases/modules/snap.yaml (+18/-0)
tests/cloud_tests/testcases/modules/snappy.py (+2/-0)
tests/cloud_tests/util.py (+5/-1)
tests/cloud_tests/verify.py (+7/-4)
tests/data/mount_parse_ext.txt (+19/-0)
tests/data/mount_parse_zfs.txt (+21/-0)
tests/data/zpool_status_simple.txt (+10/-0)
tests/unittests/test_datasource/test_azure.py (+22/-15)
tests/unittests/test_datasource/test_common.py (+4/-0)
tests/unittests/test_datasource/test_gce.py (+19/-1)
tests/unittests/test_datasource/test_hetzner.py (+117/-0)
tests/unittests/test_datasource/test_ibmcloud.py (+262/-0)
tests/unittests/test_datasource/test_opennebula.py (+177/-89)
tests/unittests/test_ds_identify.py (+161/-6)
tests/unittests/test_handler/test_handler_apt_source_v1.py (+2/-1)
tests/unittests/test_handler/test_handler_bootcmd.py (+7/-12)
tests/unittests/test_handler/test_handler_ntp.py (+6/-12)
tests/unittests/test_handler/test_handler_resizefs.py (+60/-12)
tests/unittests/test_handler/test_handler_runcmd.py (+4/-10)
tests/unittests/test_handler/test_handler_set_hostname.py (+53/-4)
tests/unittests/test_handler/test_schema.py (+16/-19)
tests/unittests/test_net.py (+58/-81)
tests/unittests/test_util.py (+135/-0)
tools/ds-identify (+99/-28)
tools/pipremove (+14/-0)
tools/run-centos (+78/-13)
tox.ini (+6/-3)
Reviewer Review Type Date Requested Status
Server Team CI bot continuous-integration Approve on 2018-03-28
Scott Moser 2018-03-27 Pending
Review via email: mp+342248@code.launchpad.net

Commit message

Sync of tip for SRU into Xenial.

Also git cherry-pick isc-dhcp-client package dependency per
5b630c3419c5e28946cd1cd58091d78e89df354a.

To post a comment you must log in.

FAILED: Continuous integration, rev:2436136535a30f5bc10eea483ba71c26d00a004b
https://jenkins.ubuntu.com/server/job/cloud-init-ci/942/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    FAILED: Ubuntu LTS: Build

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/942/rebuild

review: Needs Fixing (continuous-integration)
Scott Moser (smoser) wrote :

c-i failed, so we need to update/refresh some quilt patches it looks like
though i'm confused by why that would fail but our daily artful be fine:
 https://code.launchpad.net/~cloud-init-dev/+recipe/cloud-init-daily-artful

anyway... the other thing from me typing in IRC:

21:38 <@smoser> blackboxsw: i would probably not add the new depends into x, a
21:39 <@smoser> it probably *is* strictly required.
21:39 <@smoser> but because isc-dhcp-client is part of ubuntu-minimal
21:40 <@smoser> and you're really expected to create a ubuntu without 'ubuntu-minimal'
21:40 <@smoser> it is not likely that cloud-init is to be installed in such a place.
21:43 <@smoser> so.. i'd just leave it be. additionally, adding depends or recommends like that to a stable can be problematic
21:43 <@smoser> see someone complaining about a similar add
21:43 <@smoser> https://bugs.launchpad.net/ubuntu/+source/initramfs-tools/+bug/1633643
21:43 <ubot5> Ubuntu bug 1633643 in initramfs-tools (Ubuntu) "unnecessary dependency upon isc-dhcp-client" [Undecided,Invalid]
21:44 <@smoser> and then also the squashfuse bug (bug 1628289)
21:44 <ubot5> bug 1628289 in Snappy "snapd should depend on squashfuse (for use in containers)" [Undecided,In progress] https://launchpad.net/bugs/1628289

FAILED: Continuous integration, rev:4790ce9e7e774ba3e46420623a74949472e999e9
https://jenkins.ubuntu.com/server/job/cloud-init-ci/946/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    FAILED: Ubuntu LTS: Build

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/946/rebuild

review: Needs Fixing (continuous-integration)

PASSED: Continuous integration, rev:eb1fc028d452d6616f98ade26736bb5dbe1eaaf5
https://jenkins.ubuntu.com/server/job/cloud-init-ci/951/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/951/rebuild

review: Approve (continuous-integration)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
diff --git a/.pylintrc b/.pylintrc
index 05a086d..0bdfa59 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -46,7 +46,17 @@ reports=no
46# (useful for modules/projects where namespaces are manipulated during runtime46# (useful for modules/projects where namespaces are manipulated during runtime
47# and thus existing member attributes cannot be deduced by static analysis. It47# and thus existing member attributes cannot be deduced by static analysis. It
48# supports qualified module names, as well as Unix pattern matching.48# supports qualified module names, as well as Unix pattern matching.
49ignored-modules=six.moves,pkg_resources,httplib,http.client,paramiko,simplestreams49ignored-modules=
50 http.client,
51 httplib,
52 pkg_resources,
53 six.moves,
54 # cloud_tests requirements.
55 boto3,
56 botocore,
57 paramiko,
58 pylxd,
59 simplestreams
5060
51# List of class names for which member attributes should not be checked (useful61# List of class names for which member attributes should not be checked (useful
52# for classes with dynamically set attributes). This supports the use of62# for classes with dynamically set attributes). This supports the use of
diff --git a/ChangeLog b/ChangeLog
index 31c2dcb..daa7ccf 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,113 @@
118.2:
2 - Hetzner: Exit early if dmi system-manufacturer is not Hetzner.
3 - Add missing dependency on isc-dhcp-client to trunk ubuntu packaging.
4 (LP: #1759307)
5 - FreeBSD: resizefs module now able to handle zfs/zpool.
6 [Dominic Schlegel] (LP: #1721243)
7 - cc_puppet: Revert regression of puppet creating ssl and ssl_cert dirs
8 - Enable IBMCloud datasource in settings.py.
9 - IBMCloud: Initial IBM Cloud datasource.
10 - tests: remove jsonschema from xenial tox environment.
11 - tests: Fix newly added schema unit tests to skip if no jsonschema.
12 - ec2: Adjust ec2 datasource after exception_cb change.
13 - Reduce AzurePreprovisioning HTTP timeouts.
14 [Douglas Jordan] (LP: #1752977)
15 - Revert the logic of exception_cb in read_url.
16 [Kurt Garloff] (LP: #1702160, #1298921)
17 - ubuntu-advantage: Add new config module to support
18 ubuntu-advantage-tools
19 - Handle global dns entries in netplan (LP: #1750884)
20 - Identify OpenTelekomCloud Xen as OpenStack DS.
21 [Kurt Garloff] (LP: #1756471)
22 - datasources: fix DataSource subclass get_hostname method signature
23 (LP: #1757176)
24 - OpenNebula: Update network to return v2 config rather than ENI.
25 [Akihiko Ota]
26 - Add Hetzner Cloud DataSource
27 - net: recognize iscsi root cases without ip= on kernel command line.
28 (LP: #1752391)
29 - tests: fix flakes warning for unused variable
30 - tests: patch leaked stderr messages from snap unit tests
31 - cc_snap: Add new module to install and configure snapd and snap
32 packages.
33 - tests: Make pylint happy and fix python2.6 uses of assertRaisesRegex.
34 - netplan: render bridge port-priority values (LP: #1735821)
35 - util: Fix subp regression. Allow specifying subp command as a string.
36 (LP: #1755965)
37 - doc: fix all warnings issued by 'tox -e doc'
38 - FreeBSD: Set hostname to FQDN. [Dominic Schlegel] (LP: #1753499)
39 - tests: fix run_tree and bddeb
40 - tests: Fix some warnings in tests that popped up with newer python.
41 - set_hostname: When present in metadata, set it before network bringup.
42 (LP: #1746455)
43 - tests: Centralize and re-use skipTest based on json schema presense.
44 - This commit fixes get_hostname on the AzureDataSource.
45 [Douglas Jordan] (LP: #1754495)
46 - shellify: raise TypeError on bad input.
47 - Make salt minion module work on FreeBSD.
48 [Dominic Schlegel] (LP: #1721503)
49 - Simplify some comparisions. [Rémy Léone]
50 - Change some list creation and population to literal. [Rémy Léone]
51 - GCE: fix reading of user-data that is not base64 encoded. (LP: #1752711)
52 - doc: fix chef install from apt packages example in RTD.
53 - Implement puppet 4 support [Romanos Skiadas] (LP: #1446804)
54 - subp: Fix subp usage with non-ascii characters when no system locale.
55 (LP: #1751051)
56 - salt: configure grains in grains file rather than in minion config.
57 [Daniel Wallace]
58
5918.1:
60 - OVF: Fix VMware support for 64-bit platforms. [Sankar Tanguturi]
61 - ds-identify: Fix searching for iso9660 OVF cdroms. (LP: #1749980)
62 - SUSE: Fix groups used for ownership of cloud-init.log [Robert Schweikert]
63 - ds-identify: check /writable/system-data/ for nocloud seed.
64 (LP: #1747070)
65 - tests: run nosetests in cloudinit/ directory, fix py26 fallout.
66 - tools: run-centos: git clone rather than tar.
67 - tests: add support for logs with lxd from snap and future lxd 3.
68 (LP: #1745663)
69 - EC2: Fix get_instance_id called against cached datasource pickle.
70 (LP: #1748354)
71 - cli: fix cloud-init status to report running when before result.json
72 (LP: #1747965)
73 - net: accept network-config in netplan format for renaming interfaces
74 (LP: #1709715)
75 - Fix ssh keys validation in ssh_util [Tatiana Kholkina]
76 - docs: Update RTD content for cloud-init subcommands.
77 - OVF: Extend well-known labels to include OVFENV. (LP: #1698669)
78 - Fix potential cases of uninitialized variables. (LP: #1744796)
79 - tests: Collect script output as binary, collect systemd journal, fix lxd.
80 - HACKING.rst: mention setting user name and email via git config.
81 - Azure VM Preprovisioning support. [Douglas Jordan] (LP: #1734991)
82 - tools/read-version: Fix read-version when in a git worktree.
83 - docs: Fix typos in docs and one debug message. [Florian Grignon]
84 - btrfs: support resizing if root is mounted ro.
85 [Robert Schweikert] (LP: #1734787)
86 - OpenNebula: Improve network configuration support.
87 [Akihiko Ota] (LP: #1719157, #1716397, #1736750)
88 - tests: Fix EC2 Platform to return console output as bytes.
89 - tests: Fix attempted use of /run in a test case.
90 - GCE: Improvements and changes to ssh key behavior for default user.
91 [Max Illfelder] (LP: #1670456, #1707033, #1707037, #1707039)
92 - subp: make ProcessExecutionError have expected types in stderr, stdout.
93 - tests: when querying ntp server, do not do dns resolution.
94 - Recognize uppercase vfat disk labels [James Penick] (LP: #1598783)
95 - tests: remove zesty as supported OS to test [Joshua Powers]
96 - Do not log warning on config files that represent None. (LP: #1742479)
97 - tests: Use git hash pip dependency format for pylxd.
98 - tests: add integration requirements text file [Joshua Powers]
99 - MAAS: add check_instance_id based off oauth tokens. (LP: #1712680)
100 - tests: update apt sources list test [Joshua Powers]
101 - tests: clean up image properties [Joshua Powers]
102 - tests: rename test ssh keys to avoid appearance of leaking private keys.
103 [Joshua Powers]
104 - tests: Enable AWS EC2 Integration Testing [Joshua Powers]
105 - cli: cloud-init clean handles symlinks (LP: #1741093)
106 - SUSE: Add a basic test of network config rendering. [Robert Schweikert]
107 - Azure: Only bounce network when necessary. (LP: #1722668)
108 - lint: Fix lints seen by pylint version 1.8.1.
109 - cli: Fix error in cloud-init modules --mode=init. (LP: #1736600)
110
117.2:11117.2:
2 - ds-identify: failure in NoCloud due to unset variable usage.112 - ds-identify: failure in NoCloud due to unset variable usage.
3 (LP: #1737704)113 (LP: #1737704)
diff --git a/cloudinit/apport.py b/cloudinit/apport.py
index 221f341..618b016 100644
--- a/cloudinit/apport.py
+++ b/cloudinit/apport.py
@@ -14,9 +14,9 @@ except ImportError:
1414
15KNOWN_CLOUD_NAMES = [15KNOWN_CLOUD_NAMES = [
16 'Amazon - Ec2', 'AliYun', 'AltCloud', 'Azure', 'Bigstep', 'CloudSigma',16 'Amazon - Ec2', 'AliYun', 'AltCloud', 'Azure', 'Bigstep', 'CloudSigma',
17 'CloudStack', 'DigitalOcean', 'GCE - Google Compute Engine', 'MAAS',17 'CloudStack', 'DigitalOcean', 'GCE - Google Compute Engine',
18 'NoCloud', 'OpenNebula', 'OpenStack', 'OVF', 'Scaleway', 'SmartOS',18 'Hetzner Cloud', 'MAAS', 'NoCloud', 'OpenNebula', 'OpenStack', 'OVF',
19 'VMware', 'Other']19 'Scaleway', 'SmartOS', 'VMware', 'Other']
2020
21# Potentially clear text collected logs21# Potentially clear text collected logs
22CLOUDINIT_LOG = '/var/log/cloud-init.log'22CLOUDINIT_LOG = '/var/log/cloud-init.log'
diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py
index ba61678..6d12c43 100644
--- a/cloudinit/cloud.py
+++ b/cloudinit/cloud.py
@@ -78,8 +78,9 @@ class Cloud(object):
78 def get_locale(self):78 def get_locale(self):
79 return self.datasource.get_locale()79 return self.datasource.get_locale()
8080
81 def get_hostname(self, fqdn=False):81 def get_hostname(self, fqdn=False, metadata_only=False):
82 return self.datasource.get_hostname(fqdn=fqdn)82 return self.datasource.get_hostname(
83 fqdn=fqdn, metadata_only=metadata_only)
8384
84 def device_name_to_device(self, name):85 def device_name_to_device(self, name):
85 return self.datasource.device_name_to_device(name)86 return self.datasource.device_name_to_device(name)
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
index d2f1b77..3f2dbb9 100644
--- a/cloudinit/cmd/main.py
+++ b/cloudinit/cmd/main.py
@@ -40,6 +40,7 @@ from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE,
4040
41from cloudinit import atomic_helper41from cloudinit import atomic_helper
4242
43from cloudinit.config import cc_set_hostname
43from cloudinit.dhclient_hook import LogDhclient44from cloudinit.dhclient_hook import LogDhclient
4445
4546
@@ -215,12 +216,10 @@ def main_init(name, args):
215 if args.local:216 if args.local:
216 deps = [sources.DEP_FILESYSTEM]217 deps = [sources.DEP_FILESYSTEM]
217218
218 early_logs = []219 early_logs = [attempt_cmdline_url(
219 early_logs.append(220 path=os.path.join("%s.d" % CLOUD_CONFIG,
220 attempt_cmdline_url(221 "91_kernel_cmdline_url.cfg"),
221 path=os.path.join("%s.d" % CLOUD_CONFIG,222 network=not args.local)]
222 "91_kernel_cmdline_url.cfg"),
223 network=not args.local))
224223
225 # Cloud-init 'init' stage is broken up into the following sub-stages224 # Cloud-init 'init' stage is broken up into the following sub-stages
226 # 1. Ensure that the init object fetches its config without errors225 # 1. Ensure that the init object fetches its config without errors
@@ -354,6 +353,11 @@ def main_init(name, args):
354 LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s",353 LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s",
355 mode, name, iid, init.is_new_instance())354 mode, name, iid, init.is_new_instance())
356355
356 if mode == sources.DSMODE_LOCAL:
357 # Before network comes up, set any configured hostname to allow
358 # dhcp clients to advertize this hostname to any DDNS services
359 # LP: #1746455.
360 _maybe_set_hostname(init, stage='local', retry_stage='network')
357 init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL))361 init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL))
358362
359 if mode == sources.DSMODE_LOCAL:363 if mode == sources.DSMODE_LOCAL:
@@ -370,6 +374,7 @@ def main_init(name, args):
370 init.setup_datasource()374 init.setup_datasource()
371 # update fully realizes user-data (pulling in #include if necessary)375 # update fully realizes user-data (pulling in #include if necessary)
372 init.update()376 init.update()
377 _maybe_set_hostname(init, stage='init-net', retry_stage='modules:config')
373 # Stage 7378 # Stage 7
374 try:379 try:
375 # Attempt to consume the data per instance.380 # Attempt to consume the data per instance.
@@ -683,6 +688,24 @@ def status_wrapper(name, args, data_d=None, link_d=None):
683 return len(v1[mode]['errors'])688 return len(v1[mode]['errors'])
684689
685690
691def _maybe_set_hostname(init, stage, retry_stage):
692 """Call set-hostname if metadata, vendordata or userdata provides it.
693
694 @param stage: String representing current stage in which we are running.
695 @param retry_stage: String represented logs upon error setting hostname.
696 """
697 cloud = init.cloudify()
698 (hostname, _fqdn) = util.get_hostname_fqdn(
699 init.cfg, cloud, metadata_only=True)
700 if hostname: # meta-data or user-data hostname content
701 try:
702 cc_set_hostname.handle('set-hostname', init.cfg, cloud, LOG, None)
703 except cc_set_hostname.SetHostnameError as e:
704 LOG.debug(
705 'Failed setting hostname in %s stage. Will'
706 ' retry in %s stage. Error: %s.', stage, retry_stage, str(e))
707
708
686def main_features(name, args):709def main_features(name, args):
687 sys.stdout.write('\n'.join(sorted(version.FEATURES)) + '\n')710 sys.stdout.write('\n'.join(sorted(version.FEATURES)) + '\n')
688711
diff --git a/cloudinit/cmd/tests/test_clean.py b/cloudinit/cmd/tests/test_clean.py
index 6713af4..5a3ec3b 100644
--- a/cloudinit/cmd/tests/test_clean.py
+++ b/cloudinit/cmd/tests/test_clean.py
@@ -165,10 +165,11 @@ class TestClean(CiTestCase):
165 wrap_and_call(165 wrap_and_call(
166 'cloudinit.cmd.clean',166 'cloudinit.cmd.clean',
167 {'Init': {'side_effect': self.init_class},167 {'Init': {'side_effect': self.init_class},
168 'sys.exit': {'side_effect': self.sys_exit},
168 'sys.argv': {'new': ['clean', '--logs']}},169 'sys.argv': {'new': ['clean', '--logs']}},
169 clean.main)170 clean.main)
170171
171 self.assertRaisesCodeEqual(0, context_manager.exception.code)172 self.assertEqual(0, context_manager.exception.code)
172 self.assertFalse(173 self.assertFalse(
173 os.path.exists(self.log1), 'Unexpected log {0}'.format(self.log1))174 os.path.exists(self.log1), 'Unexpected log {0}'.format(self.log1))
174175
diff --git a/cloudinit/cmd/tests/test_main.py b/cloudinit/cmd/tests/test_main.py
175new file mode 100644176new file mode 100644
index 0000000..dbe421c
--- /dev/null
+++ b/cloudinit/cmd/tests/test_main.py
@@ -0,0 +1,161 @@
1# This file is part of cloud-init. See LICENSE file for license information.
2
3from collections import namedtuple
4import copy
5import os
6from six import StringIO
7
8from cloudinit.cmd import main
9from cloudinit.util import (
10 ensure_dir, load_file, write_file, yaml_dumps)
11from cloudinit.tests.helpers import (
12 FilesystemMockingTestCase, wrap_and_call)
13
14mypaths = namedtuple('MyPaths', 'run_dir')
15myargs = namedtuple('MyArgs', 'debug files force local reporter subcommand')
16
17
18class TestMain(FilesystemMockingTestCase):
19
20 with_logs = True
21
22 def setUp(self):
23 super(TestMain, self).setUp()
24 self.new_root = self.tmp_dir()
25 self.cloud_dir = self.tmp_path('var/lib/cloud/', dir=self.new_root)
26 os.makedirs(self.cloud_dir)
27 self.replicateTestRoot('simple_ubuntu', self.new_root)
28 self.cfg = {
29 'datasource_list': ['None'],
30 'runcmd': ['ls /etc'], # test ALL_DISTROS
31 'system_info': {'paths': {'cloud_dir': self.cloud_dir,
32 'run_dir': self.new_root}},
33 'write_files': [
34 {
35 'path': '/etc/blah.ini',
36 'content': 'blah',
37 'permissions': 0o755,
38 },
39 ],
40 'cloud_init_modules': ['write-files', 'runcmd'],
41 }
42 cloud_cfg = yaml_dumps(self.cfg)
43 ensure_dir(os.path.join(self.new_root, 'etc', 'cloud'))
44 self.cloud_cfg_file = os.path.join(
45 self.new_root, 'etc', 'cloud', 'cloud.cfg')
46 write_file(self.cloud_cfg_file, cloud_cfg)
47 self.patchOS(self.new_root)
48 self.patchUtils(self.new_root)
49 self.stderr = StringIO()
50 self.patchStdoutAndStderr(stderr=self.stderr)
51
52 def test_main_init_run_net_stops_on_file_no_net(self):
53 """When no-net file is present, main_init does not process modules."""
54 stop_file = os.path.join(self.cloud_dir, 'data', 'no-net') # stop file
55 write_file(stop_file, '')
56 cmdargs = myargs(
57 debug=False, files=None, force=False, local=False, reporter=None,
58 subcommand='init')
59 (item1, item2) = wrap_and_call(
60 'cloudinit.cmd.main',
61 {'util.close_stdin': True,
62 'netinfo.debug_info': 'my net debug info',
63 'util.fixup_output': ('outfmt', 'errfmt')},
64 main.main_init, 'init', cmdargs)
65 # We should not run write_files module
66 self.assertFalse(
67 os.path.exists(os.path.join(self.new_root, 'etc/blah.ini')),
68 'Unexpected run of write_files module produced blah.ini')
69 self.assertEqual([], item2)
70 # Instancify is called
71 instance_id_path = 'var/lib/cloud/data/instance-id'
72 self.assertFalse(
73 os.path.exists(os.path.join(self.new_root, instance_id_path)),
74 'Unexpected call to datasource.instancify produced instance-id')
75 expected_logs = [
76 "Exiting. stop file ['{stop_file}'] existed\n".format(
77 stop_file=stop_file),
78 'my net debug info' # netinfo.debug_info
79 ]
80 for log in expected_logs:
81 self.assertIn(log, self.stderr.getvalue())
82
83 def test_main_init_run_net_runs_modules(self):
84 """Modules like write_files are run in 'net' mode."""
85 cmdargs = myargs(
86 debug=False, files=None, force=False, local=False, reporter=None,
87 subcommand='init')
88 (item1, item2) = wrap_and_call(
89 'cloudinit.cmd.main',
90 {'util.close_stdin': True,
91 'netinfo.debug_info': 'my net debug info',
92 'util.fixup_output': ('outfmt', 'errfmt')},
93 main.main_init, 'init', cmdargs)
94 self.assertEqual([], item2)
95 # Instancify is called
96 instance_id_path = 'var/lib/cloud/data/instance-id'
97 self.assertEqual(
98 'iid-datasource-none\n',
99 os.path.join(load_file(
100 os.path.join(self.new_root, instance_id_path))))
101 # modules are run (including write_files)
102 self.assertEqual(
103 'blah', load_file(os.path.join(self.new_root, 'etc/blah.ini')))
104 expected_logs = [
105 'network config is disabled by fallback', # apply_network_config
106 'my net debug info', # netinfo.debug_info
107 'no previous run detected'
108 ]
109 for log in expected_logs:
110 self.assertIn(log, self.stderr.getvalue())
111
112 def test_main_init_run_net_calls_set_hostname_when_metadata_present(self):
113 """When local-hostname metadata is present, call cc_set_hostname."""
114 self.cfg['datasource'] = {
115 'None': {'metadata': {'local-hostname': 'md-hostname'}}}
116 cloud_cfg = yaml_dumps(self.cfg)
117 write_file(self.cloud_cfg_file, cloud_cfg)
118 cmdargs = myargs(
119 debug=False, files=None, force=False, local=False, reporter=None,
120 subcommand='init')
121
122 def set_hostname(name, cfg, cloud, log, args):
123 self.assertEqual('set-hostname', name)
124 updated_cfg = copy.deepcopy(self.cfg)
125 updated_cfg.update(
126 {'def_log_file': '/var/log/cloud-init.log',
127 'log_cfgs': [],
128 'syslog_fix_perms': ['syslog:adm', 'root:adm', 'root:wheel'],
129 'vendor_data': {'enabled': True, 'prefix': []}})
130 updated_cfg.pop('system_info')
131
132 self.assertEqual(updated_cfg, cfg)
133 self.assertEqual(main.LOG, log)
134 self.assertIsNone(args)
135
136 (item1, item2) = wrap_and_call(
137 'cloudinit.cmd.main',
138 {'util.close_stdin': True,
139 'netinfo.debug_info': 'my net debug info',
140 'cc_set_hostname.handle': {'side_effect': set_hostname},
141 'util.fixup_output': ('outfmt', 'errfmt')},
142 main.main_init, 'init', cmdargs)
143 self.assertEqual([], item2)
144 # Instancify is called
145 instance_id_path = 'var/lib/cloud/data/instance-id'
146 self.assertEqual(
147 'iid-datasource-none\n',
148 os.path.join(load_file(
149 os.path.join(self.new_root, instance_id_path))))
150 # modules are run (including write_files)
151 self.assertEqual(
152 'blah', load_file(os.path.join(self.new_root, 'etc/blah.ini')))
153 expected_logs = [
154 'network config is disabled by fallback', # apply_network_config
155 'my net debug info', # netinfo.debug_info
156 'no previous run detected'
157 ]
158 for log in expected_logs:
159 self.assertIn(log, self.stderr.getvalue())
160
161# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/tests/test_status.py b/cloudinit/cmd/tests/test_status.py
index 4a5a8c0..37a8993 100644
--- a/cloudinit/cmd/tests/test_status.py
+++ b/cloudinit/cmd/tests/test_status.py
@@ -380,10 +380,11 @@ class TestStatus(CiTestCase):
380 wrap_and_call(380 wrap_and_call(
381 'cloudinit.cmd.status',381 'cloudinit.cmd.status',
382 {'sys.argv': {'new': ['status']},382 {'sys.argv': {'new': ['status']},
383 'sys.exit': {'side_effect': self.sys_exit},
383 '_is_cloudinit_disabled': (False, ''),384 '_is_cloudinit_disabled': (False, ''),
384 'Init': {'side_effect': self.init_class}},385 'Init': {'side_effect': self.init_class}},
385 status.main)386 status.main)
386 self.assertRaisesCodeEqual(0, context_manager.exception.code)387 self.assertEqual(0, context_manager.exception.code)
387 self.assertEqual('status: running\n', m_stdout.getvalue())388 self.assertEqual('status: running\n', m_stdout.getvalue())
388389
389# vi: ts=4 expandtab syntax=python390# vi: ts=4 expandtab syntax=python
diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py
index efedd4a..aff4010 100644
--- a/cloudinit/config/cc_keys_to_console.py
+++ b/cloudinit/config/cc_keys_to_console.py
@@ -63,9 +63,7 @@ def handle(name, cfg, cloud, log, _args):
63 ["ssh-dss"])63 ["ssh-dss"])
6464
65 try:65 try:
66 cmd = [helper_path]66 cmd = [helper_path, ','.join(fp_blacklist), ','.join(key_blacklist)]
67 cmd.append(','.join(fp_blacklist))
68 cmd.append(','.join(key_blacklist))
69 (stdout, _stderr) = util.subp(cmd)67 (stdout, _stderr) = util.subp(cmd)
70 util.multi_log("%s\n" % (stdout.strip()),68 util.multi_log("%s\n" % (stdout.strip()),
71 stderr=False, console=True)69 stderr=False, console=True)
diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py
index 28b1d56..4190a20 100644
--- a/cloudinit/config/cc_puppet.py
+++ b/cloudinit/config/cc_puppet.py
@@ -21,6 +21,13 @@ under ``version``, and defaults to ``none``, which selects the latest version
21in the repos. If the ``puppet`` config key exists in the config archive, this21in the repos. If the ``puppet`` config key exists in the config archive, this
22module will attempt to start puppet even if no installation was performed.22module will attempt to start puppet even if no installation was performed.
2323
24The module also provides keys for configuring the new puppet 4 paths and
25installing the puppet package from the puppetlabs repositories:
26https://docs.puppet.com/puppet/4.2/reference/whered_it_go.html
27The keys are ``package_name``, ``conf_file`` and ``ssl_dir``. If unset, their
28values will default to ones that work with puppet 3.x and with distributions
29that ship modified puppet 4.x that uses the old paths.
30
24Puppet configuration can be specified under the ``conf`` key. The31Puppet configuration can be specified under the ``conf`` key. The
25configuration is specified as a dictionary containing high-level ``<section>``32configuration is specified as a dictionary containing high-level ``<section>``
26keys and lists of ``<key>=<value>`` pairs within each section. Each section33keys and lists of ``<key>=<value>`` pairs within each section. Each section
@@ -44,6 +51,9 @@ in pem format as a multi-line string (using the ``|`` yaml notation).
44 puppet:51 puppet:
45 install: <true/false>52 install: <true/false>
46 version: <version>53 version: <version>
54 conf_file: '/etc/puppet/puppet.conf'
55 ssl_dir: '/var/lib/puppet/ssl'
56 package_name: 'puppet'
47 conf:57 conf:
48 agent:58 agent:
49 server: "puppetmaster.example.org"59 server: "puppetmaster.example.org"
@@ -63,9 +73,17 @@ from cloudinit import helpers
63from cloudinit import util73from cloudinit import util
6474
65PUPPET_CONF_PATH = '/etc/puppet/puppet.conf'75PUPPET_CONF_PATH = '/etc/puppet/puppet.conf'
66PUPPET_SSL_CERT_DIR = '/var/lib/puppet/ssl/certs/'
67PUPPET_SSL_DIR = '/var/lib/puppet/ssl'76PUPPET_SSL_DIR = '/var/lib/puppet/ssl'
68PUPPET_SSL_CERT_PATH = '/var/lib/puppet/ssl/certs/ca.pem'77PUPPET_PACKAGE_NAME = 'puppet'
78
79
80class PuppetConstants(object):
81
82 def __init__(self, puppet_conf_file, puppet_ssl_dir, log):
83 self.conf_path = puppet_conf_file
84 self.ssl_dir = puppet_ssl_dir
85 self.ssl_cert_dir = os.path.join(puppet_ssl_dir, "certs")
86 self.ssl_cert_path = os.path.join(self.ssl_cert_dir, "ca.pem")
6987
7088
71def _autostart_puppet(log):89def _autostart_puppet(log):
@@ -92,22 +110,29 @@ def handle(name, cfg, cloud, log, _args):
92 return110 return
93111
94 puppet_cfg = cfg['puppet']112 puppet_cfg = cfg['puppet']
95
96 # Start by installing the puppet package if necessary...113 # Start by installing the puppet package if necessary...
97 install = util.get_cfg_option_bool(puppet_cfg, 'install', True)114 install = util.get_cfg_option_bool(puppet_cfg, 'install', True)
98 version = util.get_cfg_option_str(puppet_cfg, 'version', None)115 version = util.get_cfg_option_str(puppet_cfg, 'version', None)
116 package_name = util.get_cfg_option_str(
117 puppet_cfg, 'package_name', PUPPET_PACKAGE_NAME)
118 conf_file = util.get_cfg_option_str(
119 puppet_cfg, 'conf_file', PUPPET_CONF_PATH)
120 ssl_dir = util.get_cfg_option_str(puppet_cfg, 'ssl_dir', PUPPET_SSL_DIR)
121
122 p_constants = PuppetConstants(conf_file, ssl_dir, log)
99 if not install and version:123 if not install and version:
100 log.warn(("Puppet install set false but version supplied,"124 log.warn(("Puppet install set false but version supplied,"
101 " doing nothing."))125 " doing nothing."))
102 elif install:126 elif install:
103 log.debug(("Attempting to install puppet %s,"),127 log.debug(("Attempting to install puppet %s,"),
104 version if version else 'latest')128 version if version else 'latest')
105 cloud.distro.install_packages(('puppet', version))129
130 cloud.distro.install_packages((package_name, version))
106131
107 # ... and then update the puppet configuration132 # ... and then update the puppet configuration
108 if 'conf' in puppet_cfg:133 if 'conf' in puppet_cfg:
109 # Add all sections from the conf object to puppet.conf134 # Add all sections from the conf object to puppet.conf
110 contents = util.load_file(PUPPET_CONF_PATH)135 contents = util.load_file(p_constants.conf_path)
111 # Create object for reading puppet.conf values136 # Create object for reading puppet.conf values
112 puppet_config = helpers.DefaultingConfigParser()137 puppet_config = helpers.DefaultingConfigParser()
113 # Read puppet.conf values from original file in order to be able to138 # Read puppet.conf values from original file in order to be able to
@@ -115,20 +140,23 @@ def handle(name, cfg, cloud, log, _args):
115 # (TODO(harlowja) is this really needed??)140 # (TODO(harlowja) is this really needed??)
116 cleaned_lines = [i.lstrip() for i in contents.splitlines()]141 cleaned_lines = [i.lstrip() for i in contents.splitlines()]
117 cleaned_contents = '\n'.join(cleaned_lines)142 cleaned_contents = '\n'.join(cleaned_lines)
118 puppet_config.readfp(StringIO(cleaned_contents),143 # Move to puppet_config.read_file when dropping py2.7
119 filename=PUPPET_CONF_PATH)144 puppet_config.readfp( # pylint: disable=W1505
145 StringIO(cleaned_contents),
146 filename=p_constants.conf_path)
120 for (cfg_name, cfg) in puppet_cfg['conf'].items():147 for (cfg_name, cfg) in puppet_cfg['conf'].items():
121 # Cert configuration is a special case148 # Cert configuration is a special case
122 # Dump the puppet master ca certificate in the correct place149 # Dump the puppet master ca certificate in the correct place
123 if cfg_name == 'ca_cert':150 if cfg_name == 'ca_cert':
124 # Puppet ssl sub-directory isn't created yet151 # Puppet ssl sub-directory isn't created yet
125 # Create it with the proper permissions and ownership152 # Create it with the proper permissions and ownership
126 util.ensure_dir(PUPPET_SSL_DIR, 0o771)153 util.ensure_dir(p_constants.ssl_dir, 0o771)
127 util.chownbyname(PUPPET_SSL_DIR, 'puppet', 'root')154 util.chownbyname(p_constants.ssl_dir, 'puppet', 'root')
128 util.ensure_dir(PUPPET_SSL_CERT_DIR)155 util.ensure_dir(p_constants.ssl_cert_dir)
129 util.chownbyname(PUPPET_SSL_CERT_DIR, 'puppet', 'root')156
130 util.write_file(PUPPET_SSL_CERT_PATH, cfg)157 util.chownbyname(p_constants.ssl_cert_dir, 'puppet', 'root')
131 util.chownbyname(PUPPET_SSL_CERT_PATH, 'puppet', 'root')158 util.write_file(p_constants.ssl_cert_path, cfg)
159 util.chownbyname(p_constants.ssl_cert_path, 'puppet', 'root')
132 else:160 else:
133 # Iterate through the config items, we'll use ConfigParser.set161 # Iterate through the config items, we'll use ConfigParser.set
134 # to overwrite or create new items as needed162 # to overwrite or create new items as needed
@@ -144,8 +172,9 @@ def handle(name, cfg, cloud, log, _args):
144 puppet_config.set(cfg_name, o, v)172 puppet_config.set(cfg_name, o, v)
145 # We got all our config as wanted we'll rename173 # We got all our config as wanted we'll rename
146 # the previous puppet.conf and create our new one174 # the previous puppet.conf and create our new one
147 util.rename(PUPPET_CONF_PATH, "%s.old" % (PUPPET_CONF_PATH))175 util.rename(p_constants.conf_path, "%s.old"
148 util.write_file(PUPPET_CONF_PATH, puppet_config.stringify())176 % (p_constants.conf_path))
177 util.write_file(p_constants.conf_path, puppet_config.stringify())
149178
150 # Set it up so it autostarts179 # Set it up so it autostarts
151 _autostart_puppet(log)180 _autostart_puppet(log)
diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py
index cec22bb..c8e1752 100644
--- a/cloudinit/config/cc_resizefs.py
+++ b/cloudinit/config/cc_resizefs.py
@@ -84,6 +84,10 @@ def _resize_ufs(mount_point, devpth):
84 return ('growfs', devpth)84 return ('growfs', devpth)
8585
8686
87def _resize_zfs(mount_point, devpth):
88 return ('zpool', 'online', '-e', mount_point, devpth)
89
90
87def _get_dumpfs_output(mount_point):91def _get_dumpfs_output(mount_point):
88 dumpfs_res, err = util.subp(['dumpfs', '-m', mount_point])92 dumpfs_res, err = util.subp(['dumpfs', '-m', mount_point])
89 return dumpfs_res93 return dumpfs_res
@@ -148,6 +152,7 @@ RESIZE_FS_PREFIXES_CMDS = [
148 ('ext', _resize_ext),152 ('ext', _resize_ext),
149 ('xfs', _resize_xfs),153 ('xfs', _resize_xfs),
150 ('ufs', _resize_ufs),154 ('ufs', _resize_ufs),
155 ('zfs', _resize_zfs),
151]156]
152157
153RESIZE_FS_PRECHECK_CMDS = {158RESIZE_FS_PRECHECK_CMDS = {
@@ -188,6 +193,13 @@ def maybe_get_writable_device_path(devpath, info, log):
188 log.debug("Not attempting to resize devpath '%s': %s", devpath, info)193 log.debug("Not attempting to resize devpath '%s': %s", devpath, info)
189 return None194 return None
190195
196 # FreeBSD zpool can also just use gpt/<label>
197 # with that in mind we can not do an os.stat on "gpt/whatever"
198 # therefore return the devpath already here.
199 if devpath.startswith('gpt/'):
200 log.debug('We have a gpt label - just go ahead')
201 return devpath
202
191 try:203 try:
192 statret = os.stat(devpath)204 statret = os.stat(devpath)
193 except OSError as exc:205 except OSError as exc:
@@ -231,6 +243,16 @@ def handle(name, cfg, _cloud, log, args):
231243
232 (devpth, fs_type, mount_point) = result244 (devpth, fs_type, mount_point) = result
233245
246 # if we have a zfs then our device path at this point
247 # is the zfs label. For example: vmzroot/ROOT/freebsd
248 # we will have to get the zpool name out of this
249 # and set the resize_what variable to the zpool
250 # so the _resize_zfs function gets the right attribute.
251 if fs_type == 'zfs':
252 zpool = devpth.split('/')[0]
253 devpth = util.get_device_info_from_zpool(zpool)
254 resize_what = zpool
255
234 info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what)256 info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what)
235 log.debug("resize_info: %s" % info)257 log.debug("resize_info: %s" % info)
236258
diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py
index 449872f..539cbd5 100644
--- a/cloudinit/config/cc_runcmd.py
+++ b/cloudinit/config/cc_runcmd.py
@@ -39,8 +39,10 @@ schema = {
39 using ``sh``.39 using ``sh``.
4040
41 .. note::41 .. note::
42 all commands must be proper yaml, so you have to quote any characters42
43 yaml would eat (':' can be problematic)"""),43 all commands must be proper yaml, so you have to quote any characters
44 yaml would eat (':' can be problematic)
45 """),
44 'distros': distros,46 'distros': distros,
45 'examples': [dedent("""\47 'examples': [dedent("""\
46 runcmd:48 runcmd:
diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py
index 2b38837..d6a21d7 100644
--- a/cloudinit/config/cc_salt_minion.py
+++ b/cloudinit/config/cc_salt_minion.py
@@ -12,7 +12,9 @@ key is present in the config parts, then salt minion will be installed and
12started. Configuration for salt minion can be specified in the ``conf`` key12started. Configuration for salt minion can be specified in the ``conf`` key
13under ``salt_minion``. Any conf values present there will be assigned in13under ``salt_minion``. Any conf values present there will be assigned in
14``/etc/salt/minion``. The public and private keys to use for salt minion can be14``/etc/salt/minion``. The public and private keys to use for salt minion can be
15specified with ``public_key`` and ``private_key`` respectively.15specified with ``public_key`` and ``private_key`` respectively. Optionally if
16you have a custom package name, service name or config directory you can
17specify them with ``pkg_name``, ``service_name`` and ``config_dir``.
1618
17**Internal name:** ``cc_salt_minion``19**Internal name:** ``cc_salt_minion``
1820
@@ -23,8 +25,14 @@ specified with ``public_key`` and ``private_key`` respectively.
23**Config keys**::25**Config keys**::
2426
25 salt_minion:27 salt_minion:
28 pkg_name: 'salt-minion'
29 service_name: 'salt-minion'
30 config_dir: '/etc/salt'
26 conf:31 conf:
27 master: salt.example.com32 master: salt.example.com
33 grains:
34 role:
35 - web
28 public_key: |36 public_key: |
29 ------BEGIN PUBLIC KEY-------37 ------BEGIN PUBLIC KEY-------
30 <key data>38 <key data>
@@ -39,7 +47,34 @@ import os
3947
40from cloudinit import util48from cloudinit import util
4149
42# Note: see http://saltstack.org/topics/installation/50# Note: see https://docs.saltstack.com/en/latest/topics/installation/
51# Note: see https://docs.saltstack.com/en/latest/ref/configuration/
52
53
54class SaltConstants(object):
55 """
56 defines default distribution specific salt variables
57 """
58 def __init__(self, cfg):
59
60 # constants tailored for FreeBSD
61 if util.is_FreeBSD():
62 self.pkg_name = 'py27-salt'
63 self.srv_name = 'salt_minion'
64 self.conf_dir = '/usr/local/etc/salt'
65 # constants for any other OS
66 else:
67 self.pkg_name = 'salt-minion'
68 self.srv_name = 'salt-minion'
69 self.conf_dir = '/etc/salt'
70
71 # if there are constants given in cloud config use those
72 self.pkg_name = util.get_cfg_option_str(cfg, 'pkg_name',
73 self.pkg_name)
74 self.conf_dir = util.get_cfg_option_str(cfg, 'config_dir',
75 self.conf_dir)
76 self.srv_name = util.get_cfg_option_str(cfg, 'service_name',
77 self.srv_name)
4378
4479
45def handle(name, cfg, cloud, log, _args):80def handle(name, cfg, cloud, log, _args):
@@ -49,39 +84,49 @@ def handle(name, cfg, cloud, log, _args):
49 " no 'salt_minion' key in configuration"), name)84 " no 'salt_minion' key in configuration"), name)
50 return85 return
5186
52 salt_cfg = cfg['salt_minion']87 s_cfg = cfg['salt_minion']
88 const = SaltConstants(cfg=s_cfg)
5389
54 # Start by installing the salt package ...90 # Start by installing the salt package ...
55 cloud.distro.install_packages(('salt-minion',))91 cloud.distro.install_packages(const.pkg_name)
5692
57 # Ensure we can configure files at the right dir93 # Ensure we can configure files at the right dir
58 config_dir = salt_cfg.get("config_dir", '/etc/salt')94 util.ensure_dir(const.conf_dir)
59 util.ensure_dir(config_dir)
6095
61 # ... and then update the salt configuration96 # ... and then update the salt configuration
62 if 'conf' in salt_cfg:97 if 'conf' in s_cfg:
63 # Add all sections from the conf object to /etc/salt/minion98 # Add all sections from the conf object to minion config file
64 minion_config = os.path.join(config_dir, 'minion')99 minion_config = os.path.join(const.conf_dir, 'minion')
65 minion_data = util.yaml_dumps(salt_cfg.get('conf'))100 minion_data = util.yaml_dumps(s_cfg.get('conf'))
66 util.write_file(minion_config, minion_data)101 util.write_file(minion_config, minion_data)
67102
103 if 'grains' in s_cfg:
104 # add grains to /etc/salt/grains
105 grains_config = os.path.join(const.conf_dir, 'grains')
106 grains_data = util.yaml_dumps(s_cfg.get('grains'))
107 util.write_file(grains_config, grains_data)
108
68 # ... copy the key pair if specified109 # ... copy the key pair if specified
69 if 'public_key' in salt_cfg and 'private_key' in salt_cfg:110 if 'public_key' in s_cfg and 'private_key' in s_cfg:
70 if os.path.isdir("/etc/salt/pki/minion"):111 pki_dir_default = os.path.join(const.conf_dir, "pki/minion")
71 pki_dir_default = "/etc/salt/pki/minion"112 if not os.path.isdir(pki_dir_default):
72 else:113 pki_dir_default = os.path.join(const.conf_dir, "pki")
73 pki_dir_default = "/etc/salt/pki"
74114
75 pki_dir = salt_cfg.get('pki_dir', pki_dir_default)115 pki_dir = s_cfg.get('pki_dir', pki_dir_default)
76 with util.umask(0o77):116 with util.umask(0o77):
77 util.ensure_dir(pki_dir)117 util.ensure_dir(pki_dir)
78 pub_name = os.path.join(pki_dir, 'minion.pub')118 pub_name = os.path.join(pki_dir, 'minion.pub')
79 pem_name = os.path.join(pki_dir, 'minion.pem')119 pem_name = os.path.join(pki_dir, 'minion.pem')
80 util.write_file(pub_name, salt_cfg['public_key'])120 util.write_file(pub_name, s_cfg['public_key'])
81 util.write_file(pem_name, salt_cfg['private_key'])121 util.write_file(pem_name, s_cfg['private_key'])
122
123 # we need to have the salt minion service enabled in rc in order to be
124 # able to start the service. this does only apply on FreeBSD servers.
125 if cloud.distro.osfamily == 'freebsd':
126 cloud.distro.updatercconf('salt_minion_enable', 'YES')
82127
83 # restart salt-minion. 'service' will start even if not started. if it128 # restart salt-minion. 'service' will start even if not started. if it
84 # was started, it needs to be restarted for config change.129 # was started, it needs to be restarted for config change.
85 util.subp(['service', 'salt-minion', 'restart'], capture=False)130 util.subp(['service', const.srv_name, 'restart'], capture=False)
86131
87# vi: ts=4 expandtab132# vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py
index aa3dfe5..3d2b2da 100644
--- a/cloudinit/config/cc_set_hostname.py
+++ b/cloudinit/config/cc_set_hostname.py
@@ -32,22 +32,51 @@ will be used.
32 hostname: <fqdn/hostname>32 hostname: <fqdn/hostname>
33"""33"""
3434
35import os
36
37
38from cloudinit.atomic_helper import write_json
35from cloudinit import util39from cloudinit import util
3640
3741
42class SetHostnameError(Exception):
43 """Raised when the distro runs into an exception when setting hostname.
44
45 This may happen if we attempt to set the hostname early in cloud-init's
46 init-local timeframe as certain services may not be running yet.
47 """
48 pass
49
50
38def handle(name, cfg, cloud, log, _args):51def handle(name, cfg, cloud, log, _args):
39 if util.get_cfg_option_bool(cfg, "preserve_hostname", False):52 if util.get_cfg_option_bool(cfg, "preserve_hostname", False):
40 log.debug(("Configuration option 'preserve_hostname' is set,"53 log.debug(("Configuration option 'preserve_hostname' is set,"
41 " not setting the hostname in module %s"), name)54 " not setting the hostname in module %s"), name)
42 return55 return
43
44 (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud)56 (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud)
57 # Check for previous successful invocation of set-hostname
58
59 # set-hostname artifact file accounts for both hostname and fqdn
60 # deltas. As such, it's format is different than cc_update_hostname's
61 # previous-hostname file which only contains the base hostname.
62 # TODO consolidate previous-hostname and set-hostname artifact files and
63 # distro._read_hostname implementation so we only validate one artifact.
64 prev_fn = os.path.join(cloud.get_cpath('data'), "set-hostname")
65 prev_hostname = {}
66 if os.path.exists(prev_fn):
67 prev_hostname = util.load_json(util.load_file(prev_fn))
68 hostname_changed = (hostname != prev_hostname.get('hostname') or
69 fqdn != prev_hostname.get('fqdn'))
70 if not hostname_changed:
71 log.debug('No hostname changes. Skipping set-hostname')
72 return
73 log.debug("Setting the hostname to %s (%s)", fqdn, hostname)
45 try:74 try:
46 log.debug("Setting the hostname to %s (%s)", fqdn, hostname)
47 cloud.distro.set_hostname(hostname, fqdn)75 cloud.distro.set_hostname(hostname, fqdn)
48 except Exception:76 except Exception as e:
49 util.logexc(log, "Failed to set the hostname to %s (%s)", fqdn,77 msg = "Failed to set the hostname to %s (%s)" % (fqdn, hostname)
50 hostname)78 util.logexc(log, msg)
51 raise79 raise SetHostnameError("%s: %s" % (msg, e))
80 write_json(prev_fn, {'hostname': hostname, 'fqdn': fqdn})
5281
53# vi: ts=4 expandtab82# vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_snap.py b/cloudinit/config/cc_snap.py
54new file mode 10064483new file mode 100644
index 0000000..34a53fd
--- /dev/null
+++ b/cloudinit/config/cc_snap.py
@@ -0,0 +1,230 @@
1# Copyright (C) 2018 Canonical Ltd.
2#
3# This file is part of cloud-init. See LICENSE file for license information.
4
5"""Snap: Install, configure and manage snapd and snap packages."""
6
7import sys
8from textwrap import dedent
9
10from cloudinit import log as logging
11from cloudinit.config.schema import (
12 get_schema_doc, validate_cloudconfig_schema)
13from cloudinit.settings import PER_INSTANCE
14from cloudinit.subp import prepend_base_command
15from cloudinit import util
16
17
18distros = ['ubuntu']
19frequency = PER_INSTANCE
20
21LOG = logging.getLogger(__name__)
22
23schema = {
24 'id': 'cc_snap',
25 'name': 'Snap',
26 'title': 'Install, configure and manage snapd and snap packages',
27 'description': dedent("""\
28 This module provides a simple configuration namespace in cloud-init to
29 both setup snapd and install snaps.
30
31 .. note::
32 Both ``assertions`` and ``commands`` values can be either a
33 dictionary or a list. If these configs are provided as a
34 dictionary, the keys are only used to order the execution of the
35 assertions or commands and the dictionary is merged with any
36 vendor-data snap configuration provided. If a list is provided by
37 the user instead of a dict, any vendor-data snap configuration is
38 ignored.
39
40 The ``assertions`` configuration option is a dictionary or list of
41 properly-signed snap assertions which will run before any snap
42 ``commands``. They will be added to snapd's assertion database by
43 invoking ``snap ack <aggregate_assertion_file>``.
44
45 Snap ``commands`` is a dictionary or list of individual snap
46 commands to run on the target system. These commands can be used to
47 create snap users, install snaps and provide snap configuration.
48
49 .. note::
50 If 'side-loading' private/unpublished snaps on an instance, it is
51 best to create a snap seed directory and seed.yaml manifest in
52 **/var/lib/snapd/seed/** which snapd automatically installs on
53 startup.
54
55 **Development only**: The ``squashfuse_in_container`` boolean can be
56 set true to install squashfuse package when in a container to enable
57 snap installs. Default is false.
58 """),
59 'distros': distros,
60 'examples': [dedent("""\
61 snap:
62 assertions:
63 00: |
64 signed_assertion_blob_here
65 02: |
66 signed_assertion_blob_here
67 commands:
68 00: snap create-user --sudoer --known <snap-user>@mydomain.com
69 01: snap install canonical-livepatch
70 02: canonical-livepatch enable <AUTH_TOKEN>
71 """), dedent("""\
72 # LXC-based containers require squashfuse before snaps can be installed
73 snap:
74 commands:
75 00: apt-get install squashfuse -y
76 11: snap install emoj
77
78 """), dedent("""\
79 # Convenience: the snap command can be omitted when specifying commands
80 # as a list and 'snap' will automatically be prepended.
81 # The following commands are equivalent:
82 snap:
83 commands:
84 00: ['install', 'vlc']
85 01: ['snap', 'install', 'vlc']
86 02: snap install vlc
87 03: 'snap install vlc'
88 """)],
89 'frequency': PER_INSTANCE,
90 'type': 'object',
91 'properties': {
92 'snap': {
93 'type': 'object',
94 'properties': {
95 'assertions': {
96 'type': ['object', 'array'], # Array of strings or dict
97 'items': {'type': 'string'},
98 'additionalItems': False, # Reject items non-string
99 'minItems': 1,
100 'minProperties': 1,
101 'uniqueItems': True
102 },
103 'commands': {
104 'type': ['object', 'array'], # Array of strings or dict
105 'items': {
106 'oneOf': [
107 {'type': 'array', 'items': {'type': 'string'}},
108 {'type': 'string'}]
109 },
110 'additionalItems': False, # Reject non-string & non-list
111 'minItems': 1,
112 'minProperties': 1,
113 'uniqueItems': True
114 },
115 'squashfuse_in_container': {
116 'type': 'boolean'
117 }
118 },
119 'additionalProperties': False, # Reject keys not in schema
120 'required': [],
121 'minProperties': 1
122 }
123 }
124}
125
126# TODO schema for 'assertions' and 'commands' are too permissive at the moment.
127# Once python-jsonschema supports schema draft 6 add support for arbitrary
128# object keys with 'patternProperties' constraint to validate string values.
129
130__doc__ = get_schema_doc(schema) # Supplement python help()
131
132SNAP_CMD = "snap"
133ASSERTIONS_FILE = "/var/lib/cloud/instance/snapd.assertions"
134
135
136def add_assertions(assertions):
137 """Import list of assertions.
138
139 Import assertions by concatenating each assertion into a
140 string separated by a '\n'. Write this string to a instance file and
141 then invoke `snap ack /path/to/file` and check for errors.
142 If snap exits 0, then all assertions are imported.
143 """
144 if not assertions:
145 return
146 LOG.debug('Importing user-provided snap assertions')
147 if isinstance(assertions, dict):
148 assertions = assertions.values()
149 elif not isinstance(assertions, list):
150 raise TypeError(
151 'assertion parameter was not a list or dict: {assertions}'.format(
152 assertions=assertions))
153
154 snap_cmd = [SNAP_CMD, 'ack']
155 combined = "\n".join(assertions)
156
157 for asrt in assertions:
158 LOG.debug('Snap acking: %s', asrt.split('\n')[0:2])
159
160 util.write_file(ASSERTIONS_FILE, combined.encode('utf-8'))
161 util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True)
162
163
164def run_commands(commands):
165 """Run the provided commands provided in snap:commands configuration.
166
167 Commands are run individually. Any errors are collected and reported
168 after attempting all commands.
169
170 @param commands: A list or dict containing commands to run. Keys of a
171 dict will be used to order the commands provided as dict values.
172 """
173 if not commands:
174 return
175 LOG.debug('Running user-provided snap commands')
176 if isinstance(commands, dict):
177 # Sort commands based on dictionary key
178 commands = [v for _, v in sorted(commands.items())]
179 elif not isinstance(commands, list):
180 raise TypeError(
181 'commands parameter was not a list or dict: {commands}'.format(
182 commands=commands))
183
184 fixed_snap_commands = prepend_base_command('snap', commands)
185
186 cmd_failures = []
187 for command in fixed_snap_commands:
188 shell = isinstance(command, str)
189 try:
190 util.subp(command, shell=shell, status_cb=sys.stderr.write)
191 except util.ProcessExecutionError as e:
192 cmd_failures.append(str(e))
193 if cmd_failures:
194 msg = 'Failures running snap commands:\n{cmd_failures}'.format(
195 cmd_failures=cmd_failures)
196 util.logexc(LOG, msg)
197 raise RuntimeError(msg)
198
199
200# RELEASE_BLOCKER: Once LP: #1628289 is released on xenial, drop this function.
201def maybe_install_squashfuse(cloud):
202 """Install squashfuse if we are in a container."""
203 if not util.is_container():
204 return
205 try:
206 cloud.distro.update_package_sources()
207 except Exception as e:
208 util.logexc(LOG, "Package update failed")
209 raise
210 try:
211 cloud.distro.install_packages(['squashfuse'])
212 except Exception as e:
213 util.logexc(LOG, "Failed to install squashfuse")
214 raise
215
216
217def handle(name, cfg, cloud, log, args):
218 cfgin = cfg.get('snap', {})
219 if not cfgin:
220 LOG.debug(("Skipping module named %s,"
221 " no 'snap' key in configuration"), name)
222 return
223
224 validate_cloudconfig_schema(cfg, schema)
225 if util.is_true(cfgin.get('squashfuse_in_container', False)):
226 maybe_install_squashfuse(cloud)
227 add_assertions(cfgin.get('assertions', []))
228 run_commands(cfgin.get('commands', []))
229
230# vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_snap_config.py b/cloudinit/config/cc_snap_config.py
index e82c081..afe297e 100644
--- a/cloudinit/config/cc_snap_config.py
+++ b/cloudinit/config/cc_snap_config.py
@@ -4,11 +4,15 @@
4#4#
5# This file is part of cloud-init. See LICENSE file for license information.5# This file is part of cloud-init. See LICENSE file for license information.
66
7# RELEASE_BLOCKER: Remove this deprecated module in 18.3
7"""8"""
8Snap Config9Snap Config
9-----------10-----------
10**Summary:** snap_config modules allows configuration of snapd.11**Summary:** snap_config modules allows configuration of snapd.
1112
13**Deprecated**: Use :ref:`snap` module instead. This module will not exist
14in cloud-init 18.3.
15
12This module uses the same ``snappy`` namespace for configuration but16This module uses the same ``snappy`` namespace for configuration but
13acts only only a subset of the configuration.17acts only only a subset of the configuration.
1418
@@ -154,6 +158,9 @@ def handle(name, cfg, cloud, log, args):
154 LOG.debug('No snappy config provided, skipping')158 LOG.debug('No snappy config provided, skipping')
155 return159 return
156160
161 log.warning(
162 'DEPRECATION: snap_config module will be dropped in 18.3 release.'
163 ' Use snap module instead')
157 if not(util.system_is_snappy()):164 if not(util.system_is_snappy()):
158 LOG.debug("%s: system not snappy", name)165 LOG.debug("%s: system not snappy", name)
159 return166 return
diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py
index eecb817..bab80bb 100644
--- a/cloudinit/config/cc_snappy.py
+++ b/cloudinit/config/cc_snappy.py
@@ -1,10 +1,14 @@
1# This file is part of cloud-init. See LICENSE file for license information.1# This file is part of cloud-init. See LICENSE file for license information.
22
3# RELEASE_BLOCKER: Remove this deprecated module in 18.3
3"""4"""
4Snappy5Snappy
5------6------
6**Summary:** snappy modules allows configuration of snappy.7**Summary:** snappy modules allows configuration of snappy.
78
9**Deprecated**: Use :ref:`snap` module instead. This module will not exist
10in cloud-init 18.3.
11
8The below example config config would install ``etcd``, and then install12The below example config config would install ``etcd``, and then install
9``pkg2.smoser`` with a ``<config-file>`` argument where ``config-file`` has13``pkg2.smoser`` with a ``<config-file>`` argument where ``config-file`` has
10``config-blob`` inside it. If ``pkgname`` is installed already, then14``config-blob`` inside it. If ``pkgname`` is installed already, then
@@ -271,6 +275,10 @@ def handle(name, cfg, cloud, log, args):
271 LOG.debug("%s: 'auto' mode, and system not snappy", name)275 LOG.debug("%s: 'auto' mode, and system not snappy", name)
272 return276 return
273277
278 log.warning(
279 'DEPRECATION: snappy module will be dropped in 18.3 release.'
280 ' Use snap module instead')
281
274 set_snappy_command()282 set_snappy_command()
275283
276 pkg_ops = get_package_ops(packages=mycfg['packages'],284 pkg_ops = get_package_ops(packages=mycfg['packages'],
diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py
index 35d8c57..98b0e66 100755
--- a/cloudinit/config/cc_ssh_authkey_fingerprints.py
+++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py
@@ -77,11 +77,10 @@ def _pprint_key_entries(user, key_fn, key_entries, hash_meth='md5',
77 tbl = SimpleTable(tbl_fields)77 tbl = SimpleTable(tbl_fields)
78 for entry in key_entries:78 for entry in key_entries:
79 if _is_printable_key(entry):79 if _is_printable_key(entry):
80 row = []80 row = [entry.keytype or '-',
81 row.append(entry.keytype or '-')81 _gen_fingerprint(entry.base64, hash_meth) or '-',
82 row.append(_gen_fingerprint(entry.base64, hash_meth) or '-')82 entry.options or '-',
83 row.append(entry.options or '-')83 entry.comment or '-']
84 row.append(entry.comment or '-')
85 tbl.add_row(row)84 tbl.add_row(row)
86 authtbl_s = tbl.get_string()85 authtbl_s = tbl.get_string()
87 authtbl_lines = authtbl_s.splitlines()86 authtbl_lines = authtbl_s.splitlines()
diff --git a/cloudinit/config/cc_ubuntu_advantage.py b/cloudinit/config/cc_ubuntu_advantage.py
88new file mode 10064487new file mode 100644
index 0000000..16b1868
--- /dev/null
+++ b/cloudinit/config/cc_ubuntu_advantage.py
@@ -0,0 +1,173 @@
1# Copyright (C) 2018 Canonical Ltd.
2#
3# This file is part of cloud-init. See LICENSE file for license information.
4
5"""Ubuntu advantage: manage ubuntu-advantage offerings from Canonical."""
6
7import sys
8from textwrap import dedent
9
10from cloudinit import log as logging
11from cloudinit.config.schema import (
12 get_schema_doc, validate_cloudconfig_schema)
13from cloudinit.settings import PER_INSTANCE
14from cloudinit.subp import prepend_base_command
15from cloudinit import util
16
17
18distros = ['ubuntu']
19frequency = PER_INSTANCE
20
21LOG = logging.getLogger(__name__)
22
23schema = {
24 'id': 'cc_ubuntu_advantage',
25 'name': 'Ubuntu Advantage',
26 'title': 'Install, configure and manage ubuntu-advantage offerings',
27 'description': dedent("""\
28 This module provides configuration options to setup ubuntu-advantage
29 subscriptions.
30
31 .. note::
32 Both ``commands`` value can be either a dictionary or a list. If
33 the configuration provided is a dictionary, the keys are only used
34 to order the execution of the commands and the dictionary is
35 merged with any vendor-data ubuntu-advantage configuration
36 provided. If a ``commands`` is provided as a list, any vendor-data
37 ubuntu-advantage ``commands`` are ignored.
38
39 Ubuntu-advantage ``commands`` is a dictionary or list of
40 ubuntu-advantage commands to run on the deployed machine.
41 These commands can be used to enable or disable subscriptions to
42 various ubuntu-advantage products. See 'man ubuntu-advantage' for more
43 information on supported subcommands.
44
45 .. note::
46 Each command item can be a string or list. If the item is a list,
47 'ubuntu-advantage' can be omitted and it will automatically be
48 inserted as part of the command.
49 """),
50 'distros': distros,
51 'examples': [dedent("""\
52 # Enable Extended Security Maintenance using your service auth token
53 ubuntu-advantage:
54 commands:
55 00: ubuntu-advantage enable-esm <token>
56 """), dedent("""\
57 # Enable livepatch by providing your livepatch token
58 ubuntu-advantage:
59 commands:
60 00: ubuntu-advantage enable-livepatch <livepatch-token>
61
62 """), dedent("""\
63 # Convenience: the ubuntu-advantage command can be omitted when
64 # specifying commands as a list and 'ubuntu-advantage' will
65 # automatically be prepended.
66 # The following commands are equivalent
67 ubuntu-advantage:
68 commands:
69 00: ['enable-livepatch', 'my-token']
70 01: ['ubuntu-advantage', 'enable-livepatch', 'my-token']
71 02: ubuntu-advantage enable-livepatch my-token
72 03: 'ubuntu-advantage enable-livepatch my-token'
73 """)],
74 'frequency': PER_INSTANCE,
75 'type': 'object',
76 'properties': {
77 'ubuntu-advantage': {
78 'type': 'object',
79 'properties': {
80 'commands': {
81 'type': ['object', 'array'], # Array of strings or dict
82 'items': {
83 'oneOf': [
84 {'type': 'array', 'items': {'type': 'string'}},
85 {'type': 'string'}]
86 },
87 'additionalItems': False, # Reject non-string & non-list
88 'minItems': 1,
89 'minProperties': 1,
90 'uniqueItems': True
91 }
92 },
93 'additionalProperties': False, # Reject keys not in schema
94 'required': ['commands']
95 }
96 }
97}
98
99# TODO schema for 'assertions' and 'commands' are too permissive at the moment.
100# Once python-jsonschema supports schema draft 6 add support for arbitrary
101# object keys with 'patternProperties' constraint to validate string values.
102
103__doc__ = get_schema_doc(schema) # Supplement python help()
104
105UA_CMD = "ubuntu-advantage"
106
107
108def run_commands(commands):
109 """Run the commands provided in ubuntu-advantage:commands config.
110
111 Commands are run individually. Any errors are collected and reported
112 after attempting all commands.
113
114 @param commands: A list or dict containing commands to run. Keys of a
115 dict will be used to order the commands provided as dict values.
116 """
117 if not commands:
118 return
119 LOG.debug('Running user-provided ubuntu-advantage commands')
120 if isinstance(commands, dict):
121 # Sort commands based on dictionary key
122 commands = [v for _, v in sorted(commands.items())]
123 elif not isinstance(commands, list):
124 raise TypeError(
125 'commands parameter was not a list or dict: {commands}'.format(
126 commands=commands))
127
128 fixed_ua_commands = prepend_base_command('ubuntu-advantage', commands)
129
130 cmd_failures = []
131 for command in fixed_ua_commands:
132 shell = isinstance(command, str)
133 try:
134 util.subp(command, shell=shell, status_cb=sys.stderr.write)
135 except util.ProcessExecutionError as e:
136 cmd_failures.append(str(e))
137 if cmd_failures:
138 msg = (
139 'Failures running ubuntu-advantage commands:\n'
140 '{cmd_failures}'.format(
141 cmd_failures=cmd_failures))
142 util.logexc(LOG, msg)
143 raise RuntimeError(msg)
144
145
146def maybe_install_ua_tools(cloud):
147 """Install ubuntu-advantage-tools if not present."""
148 if util.which('ubuntu-advantage'):
149 return
150 try:
151 cloud.distro.update_package_sources()
152 except Exception as e:
153 util.logexc(LOG, "Package update failed")
154 raise
155 try:
156 cloud.distro.install_packages(['ubuntu-advantage-tools'])
157 except Exception as e:
158 util.logexc(LOG, "Failed to install ubuntu-advantage-tools")
159 raise
160
161
162def handle(name, cfg, cloud, log, args):
163 cfgin = cfg.get('ubuntu-advantage')
164 if cfgin is None:
165 LOG.debug(("Skipping module named %s,"
166 " no 'ubuntu-advantage' key in configuration"), name)
167 return
168
169 validate_cloudconfig_schema(cfg, schema)
170 maybe_install_ua_tools(cloud)
171 run_commands(cfgin.get('commands', []))
172
173# vi: ts=4 expandtab
diff --git a/cloudinit/config/tests/test_snap.py b/cloudinit/config/tests/test_snap.py
0new file mode 100644174new file mode 100644
index 0000000..c5b4a9d
--- /dev/null
+++ b/cloudinit/config/tests/test_snap.py
@@ -0,0 +1,490 @@
1# This file is part of cloud-init. See LICENSE file for license information.
2
3import re
4from six import StringIO
5
6from cloudinit.config.cc_snap import (
7 ASSERTIONS_FILE, add_assertions, handle, maybe_install_squashfuse,
8 run_commands, schema)
9from cloudinit.config.schema import validate_cloudconfig_schema
10from cloudinit import util
11from cloudinit.tests.helpers import (
12 CiTestCase, mock, wrap_and_call, skipUnlessJsonSchema)
13
14
15SYSTEM_USER_ASSERTION = """\
16type: system-user
17authority-id: LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp
18brand-id: LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp
19email: foo@bar.com
20password: $6$E5YiAuMIPAwX58jG$miomhVNui/vf7f/3ctB/f0RWSKFxG0YXzrJ9rtJ1ikvzt
21series:
22- 16
23since: 2016-09-10T16:34:00+03:00
24until: 2017-11-10T16:34:00+03:00
25username: baz
26sign-key-sha3-384: RuVvnp4n52GilycjfbbTCI3_L8Y6QlIE75wxMc0KzGV3AUQqVd9GuXoj
27
28AcLBXAQAAQoABgUCV/UU1wAKCRBKnlMoJQLkZVeLD/9/+hIeVywtzsDA3oxl+P+u9D13y9s6svP
29Jd6Wnf4FTw6sq1GjBE4ZA7lrwSaRCUJ9Vcsvf2q9OGPY7mOb2TBxaDe0PbUMjrSrqllSSQwhpNI
30zG+NxkkKuxsUmLzFa+k9m6cyojNbw5LFhQZBQCGlr3JYqC0tIREq/UsZxj+90TUC87lDJwkU8GF
31s4CR+rejZj4itIcDcVxCSnJH6hv6j2JrJskJmvObqTnoOlcab+JXdamXqbldSP3UIhWoyVjqzkj
32+to7mXgx+cCUA9+ngNCcfUG+1huGGTWXPCYkZ78HvErcRlIdeo4d3xwtz1cl/w3vYnq9og1XwsP
33Yfetr3boig2qs1Y+j/LpsfYBYncgWjeDfAB9ZZaqQz/oc8n87tIPZDJHrusTlBfop8CqcM4xsKS
34d+wnEY8e/F24mdSOYmS1vQCIDiRU3MKb6x138Ud6oHXFlRBbBJqMMctPqWDunWzb5QJ7YR0I39q
35BrnEqv5NE0G7w6HOJ1LSPG5Hae3P4T2ea+ATgkb03RPr3KnXnzXg4TtBbW1nytdlgoNc/BafE1H
36f3NThcq9gwX4xWZ2PAWnqVPYdDMyCtzW3Ck+o6sIzx+dh4gDLPHIi/6TPe/pUuMop9CBpWwez7V
37v1z+1+URx6Xlq3Jq18y5pZ6fY3IDJ6km2nQPMzcm4Q=="""
38
39ACCOUNT_ASSERTION = """\
40type: account-key
41authority-id: canonical
42revision: 2
43public-key-sha3-384: BWDEoaqyr25nF5SNCvEv2v7QnM9QsfCc0PBMYD_i2NGSQ32EF2d4D0
44account-id: canonical
45name: store
46since: 2016-04-01T00:00:00.0Z
47body-length: 717
48sign-key-sha3-384: -CvQKAwRQ5h3Ffn10FILJoEZUXOv6km9FwA80-Rcj-f-6jadQ89VRswH
49
50AcbBTQRWhcGAARAA0KKYYQWuHOrsFVi4p4l7ZzSvX7kLgJFFeFgOkzdWKBTHEnsMKjl5mefFe9j
51qe8NlmJdfY7BenP7XeBtwKp700H/t9lLrZbpTNAPHXYxEWFJp5bPqIcJYBZ+29oLVLN1Tc5X482
52vCiDqL8+pPYqBrK2fNlyPlNNSum9wI70rDDL4r6FVvr+osTnGejibdV8JphWX+lrSQDnRSdM8KJ
53UM43vTgLGTi9W54oRhsA2OFexRfRksTrnqGoonCjqX5wO3OFSaMDzMsO2MJ/hPfLgDqw53qjzuK
54Iec9OL3k5basvu2cj5u9tKwVFDsCKK2GbKUsWWpx2KTpOifmhmiAbzkTHbH9KaoMS7p0kJwhTQG
55o9aJ9VMTWHJc/NCBx7eu451u6d46sBPCXS/OMUh2766fQmoRtO1OwCTxsRKG2kkjbMn54UdFULl
56VfzvyghMNRKIezsEkmM8wueTqGUGZWa6CEZqZKwhe/PROxOPYzqtDH18XZknbU1n5lNb7vNfem9
572ai+3+JyFnW9UhfvpVF7gzAgdyCqNli4C6BIN43uwoS8HkykocZS/+Gv52aUQ/NZ8BKOHLw+7an
58Q0o8W9ltSLZbEMxFIPSN0stiZlkXAp6DLyvh1Y4wXSynDjUondTpej2fSvSlCz/W5v5V7qA4nIc
59vUvV7RjVzv17ut0AEQEAAQ==
60
61AcLDXAQAAQoABgUCV83k9QAKCRDUpVvql9g3IBT8IACKZ7XpiBZ3W4lqbPssY6On81WmxQLtvsM
62WTp6zZpl/wWOSt2vMNUk9pvcmrNq1jG9CuhDfWFLGXEjcrrmVkN3YuCOajMSPFCGrxsIBLSRt/b
63nrKykdLAAzMfG8rP1d82bjFFiIieE+urQ0Kcv09Jtdvavq3JT1Tek5mFyyfhHNlQEKOzWqmRWiL
643c3VOZUs1ZD8TSlnuq/x+5T0X0YtOyGjSlVxk7UybbyMNd6MZfNaMpIG4x+mxD3KHFtBAC7O6kL
65eX3i6j5nCY5UABfA3DZEAkWP4zlmdBEOvZ9t293NaDdOpzsUHRkoi0Zez/9BHQ/kwx/uNc2WqrY
66inCmu16JGNeXqsyinnLl7Ghn2RwhvDMlLxF6RTx8xdx1yk6p3PBTwhZMUvuZGjUtN/AG8BmVJQ1
67rsGSRkkSywvnhVJRB2sudnrMBmNS2goJbzSbmJnOlBrd2WsV0T9SgNMWZBiov3LvU4o2SmAb6b+
68rYwh8H5QHcuuYJuxDjFhPswIp6Wes5T6hUicf3SWtObcDS4HSkVS4ImBjjX9YgCuFy7QdnooOWE
69aPvkRw3XCVeYq0K6w9GRsk1YFErD4XmXXZjDYY650MX9v42Sz5MmphHV8jdIY5ssbadwFSe2rCQ
706UX08zy7RsIb19hTndE6ncvSNDChUR9eEnCm73eYaWTWTnq1cxdVP/s52r8uss++OYOkPWqh5nO
71haRn7INjH/yZX4qXjNXlTjo0PnHH0q08vNKDwLhxS+D9du+70FeacXFyLIbcWllSbJ7DmbumGpF
72yYbtj3FDDPzachFQdIG3lSt+cSUGeyfSs6wVtc3cIPka/2Urx7RprfmoWSI6+a5NcLdj0u2z8O9
73HxeIgxDpg/3gT8ZIuFKePMcLDM19Fh/p0ysCsX+84B9chNWtsMSmIaE57V+959MVtsLu7SLb9gi
74skrju0pQCwsu2wHMLTNd1f3PTHmrr49hxetTus07HSQUApMtAGKzQilF5zqFjbyaTd4xgQbd+PK
75CjFyzQTDOcUhXpuUGt/IzlqiFfsCsmbj2K4KdSNYMlqIgZ3Azu8KvZLIhsyN7v5vNIZSPfEbjde
76ClU9r0VRiJmtYBUjcSghD9LWn+yRLwOxhfQVjm0cBwIt5R/yPF/qC76yIVuWUtM5Y2/zJR1J8OF
77qWchvlImHtvDzS9FQeLyzJAOjvZ2CnWp2gILgUz0WQdOk1Dq8ax7KS9BQ42zxw9EZAEPw3PEFqR
78IQsRTONp+iVS8YxSmoYZjDlCgRMWUmawez/Fv5b9Fb/XkO5Eq4e+KfrpUujXItaipb+tV8h5v3t
79oG3Ie3WOHrVjCLXIdYslpL1O4nadqR6Xv58pHj6k"""
80
81
82class FakeCloud(object):
83 def __init__(self, distro):
84 self.distro = distro
85
86
87class TestAddAssertions(CiTestCase):
88
89 with_logs = True
90
91 def setUp(self):
92 super(TestAddAssertions, self).setUp()
93 self.tmp = self.tmp_dir()
94
95 @mock.patch('cloudinit.config.cc_snap.util.subp')
96 def test_add_assertions_on_empty_list(self, m_subp):
97 """When provided with an empty list, add_assertions does nothing."""
98 add_assertions([])
99 self.assertEqual('', self.logs.getvalue())
100 m_subp.assert_not_called()
101
102 def test_add_assertions_on_non_list_or_dict(self):
103 """When provided an invalid type, add_assertions raises an error."""
104 with self.assertRaises(TypeError) as context_manager:
105 add_assertions(assertions="I'm Not Valid")
106 self.assertEqual(
107 "assertion parameter was not a list or dict: I'm Not Valid",
108 str(context_manager.exception))
109
110 @mock.patch('cloudinit.config.cc_snap.util.subp')
111 def test_add_assertions_adds_assertions_as_list(self, m_subp):
112 """When provided with a list, add_assertions adds all assertions."""
113 self.assertEqual(
114 ASSERTIONS_FILE, '/var/lib/cloud/instance/snapd.assertions')
115 assert_file = self.tmp_path('snapd.assertions', dir=self.tmp)
116 assertions = [SYSTEM_USER_ASSERTION, ACCOUNT_ASSERTION]
117 wrap_and_call(
118 'cloudinit.config.cc_snap',
119 {'ASSERTIONS_FILE': {'new': assert_file}},
120 add_assertions, assertions)
121 self.assertIn(
122 'Importing user-provided snap assertions', self.logs.getvalue())
123 self.assertIn(
124 'sertions', self.logs.getvalue())
125 self.assertEqual(
126 [mock.call(['snap', 'ack', assert_file], capture=True)],
127 m_subp.call_args_list)
128 compare_file = self.tmp_path('comparison', dir=self.tmp)
129 util.write_file(compare_file, '\n'.join(assertions).encode('utf-8'))
130 self.assertEqual(
131 util.load_file(compare_file), util.load_file(assert_file))
132
133 @mock.patch('cloudinit.config.cc_snap.util.subp')
134 def test_add_assertions_adds_assertions_as_dict(self, m_subp):
135 """When provided with a dict, add_assertions adds all assertions."""
136 self.assertEqual(
137 ASSERTIONS_FILE, '/var/lib/cloud/instance/snapd.assertions')
138 assert_file = self.tmp_path('snapd.assertions', dir=self.tmp)
139 assertions = {'00': SYSTEM_USER_ASSERTION, '01': ACCOUNT_ASSERTION}
140 wrap_and_call(
141 'cloudinit.config.cc_snap',
142 {'ASSERTIONS_FILE': {'new': assert_file}},
143 add_assertions, assertions)
144 self.assertIn(
145 'Importing user-provided snap assertions', self.logs.getvalue())
146 self.assertIn(
147 "DEBUG: Snap acking: ['type: system-user', 'authority-id: Lqv",
148 self.logs.getvalue())
149 self.assertIn(
150 "DEBUG: Snap acking: ['type: account-key', 'authority-id: canonic",
151 self.logs.getvalue())
152 self.assertEqual(
153 [mock.call(['snap', 'ack', assert_file], capture=True)],
154 m_subp.call_args_list)
155 compare_file = self.tmp_path('comparison', dir=self.tmp)
156 combined = '\n'.join(assertions.values())
157 util.write_file(compare_file, combined.encode('utf-8'))
158 self.assertEqual(
159 util.load_file(compare_file), util.load_file(assert_file))
160
161
162class TestRunCommands(CiTestCase):
163
164 with_logs = True
165
166 def setUp(self):
167 super(TestRunCommands, self).setUp()
168 self.tmp = self.tmp_dir()
169
170 @mock.patch('cloudinit.config.cc_snap.util.subp')
171 def test_run_commands_on_empty_list(self, m_subp):
172 """When provided with an empty list, run_commands does nothing."""
173 run_commands([])
174 self.assertEqual('', self.logs.getvalue())
175 m_subp.assert_not_called()
176
177 def test_run_commands_on_non_list_or_dict(self):
178 """When provided an invalid type, run_commands raises an error."""
179 with self.assertRaises(TypeError) as context_manager:
180 run_commands(commands="I'm Not Valid")
181 self.assertEqual(
182 "commands parameter was not a list or dict: I'm Not Valid",
183 str(context_manager.exception))
184
185 def test_run_command_logs_commands_and_exit_codes_to_stderr(self):
186 """All exit codes are logged to stderr."""
187 outfile = self.tmp_path('output.log', dir=self.tmp)
188
189 cmd1 = 'echo "HI" >> %s' % outfile
190 cmd2 = 'bogus command'
191 cmd3 = 'echo "MOM" >> %s' % outfile
192 commands = [cmd1, cmd2, cmd3]
193
194 mock_path = 'cloudinit.config.cc_snap.sys.stderr'
195 with mock.patch(mock_path, new_callable=StringIO) as m_stderr:
196 with self.assertRaises(RuntimeError) as context_manager:
197 run_commands(commands=commands)
198
199 self.assertIsNotNone(
200 re.search(r'bogus: (command )?not found',
201 str(context_manager.exception)),
202 msg='Expected bogus command not found')
203 expected_stderr_log = '\n'.join([
204 'Begin run command: {cmd}'.format(cmd=cmd1),
205 'End run command: exit(0)',
206 'Begin run command: {cmd}'.format(cmd=cmd2),
207 'ERROR: End run command: exit(127)',
208 'Begin run command: {cmd}'.format(cmd=cmd3),
209 'End run command: exit(0)\n'])
210 self.assertEqual(expected_stderr_log, m_stderr.getvalue())
211
212 def test_run_command_as_lists(self):
213 """When commands are specified as a list, run them in order."""
214 outfile = self.tmp_path('output.log', dir=self.tmp)
215
216 cmd1 = 'echo "HI" >> %s' % outfile
217 cmd2 = 'echo "MOM" >> %s' % outfile
218 commands = [cmd1, cmd2]
219 mock_path = 'cloudinit.config.cc_snap.sys.stderr'
220 with mock.patch(mock_path, new_callable=StringIO):
221 run_commands(commands=commands)
222
223 self.assertIn(
224 'DEBUG: Running user-provided snap commands',
225 self.logs.getvalue())
226 self.assertEqual('HI\nMOM\n', util.load_file(outfile))
227 self.assertIn(
228 'WARNING: Non-snap commands in snap config:', self.logs.getvalue())
229
230 def test_run_command_dict_sorted_as_command_script(self):
231 """When commands are a dict, sort them and run."""
232 outfile = self.tmp_path('output.log', dir=self.tmp)
233 cmd1 = 'echo "HI" >> %s' % outfile
234 cmd2 = 'echo "MOM" >> %s' % outfile
235 commands = {'02': cmd1, '01': cmd2}
236 mock_path = 'cloudinit.config.cc_snap.sys.stderr'
237 with mock.patch(mock_path, new_callable=StringIO):
238 run_commands(commands=commands)
239
240 expected_messages = [
241 'DEBUG: Running user-provided snap commands']
242 for message in expected_messages:
243 self.assertIn(message, self.logs.getvalue())
244 self.assertEqual('MOM\nHI\n', util.load_file(outfile))
245
246
247@skipUnlessJsonSchema()
248class TestSchema(CiTestCase):
249
250 with_logs = True
251
252 def test_schema_warns_on_snap_not_as_dict(self):
253 """If the snap configuration is not a dict, emit a warning."""
254 validate_cloudconfig_schema({'snap': 'wrong type'}, schema)
255 self.assertEqual(
256 "WARNING: Invalid config:\nsnap: 'wrong type' is not of type"
257 " 'object'\n",
258 self.logs.getvalue())
259
260 @mock.patch('cloudinit.config.cc_snap.run_commands')
261 def test_schema_disallows_unknown_keys(self, _):
262 """Unknown keys in the snap configuration emit warnings."""
263 validate_cloudconfig_schema(
264 {'snap': {'commands': ['ls'], 'invalid-key': ''}}, schema)
265 self.assertIn(
266 'WARNING: Invalid config:\nsnap: Additional properties are not'
267 " allowed ('invalid-key' was unexpected)",
268 self.logs.getvalue())
269
270 def test_warn_schema_requires_either_commands_or_assertions(self):
271 """Warn when snap configuration lacks both commands and assertions."""
272 validate_cloudconfig_schema(
273 {'snap': {}}, schema)
274 self.assertIn(
275 'WARNING: Invalid config:\nsnap: {} does not have enough'
276 ' properties',
277 self.logs.getvalue())
278
279 @mock.patch('cloudinit.config.cc_snap.run_commands')
280 def test_warn_schema_commands_is_not_list_or_dict(self, _):
281 """Warn when snap:commands config is not a list or dict."""
282 validate_cloudconfig_schema(
283 {'snap': {'commands': 'broken'}}, schema)
284 self.assertEqual(
285 "WARNING: Invalid config:\nsnap.commands: 'broken' is not of type"
286 " 'object', 'array'\n",
287 self.logs.getvalue())
288
289 @mock.patch('cloudinit.config.cc_snap.run_commands')
290 def test_warn_schema_when_commands_is_empty(self, _):
291 """Emit warnings when snap:commands is an empty list or dict."""
292 validate_cloudconfig_schema(
293 {'snap': {'commands': []}}, schema)
294 validate_cloudconfig_schema(
295 {'snap': {'commands': {}}}, schema)
296 self.assertEqual(
297 "WARNING: Invalid config:\nsnap.commands: [] is too short\n"
298 "WARNING: Invalid config:\nsnap.commands: {} does not have enough"
299 " properties\n",
300 self.logs.getvalue())
301
302 @mock.patch('cloudinit.config.cc_snap.run_commands')
303 def test_schema_when_commands_are_list_or_dict(self, _):
304 """No warnings when snap:commands are either a list or dict."""
305 validate_cloudconfig_schema(
306 {'snap': {'commands': ['valid']}}, schema)
307 validate_cloudconfig_schema(
308 {'snap': {'commands': {'01': 'also valid'}}}, schema)
309 self.assertEqual('', self.logs.getvalue())
310
311 @mock.patch('cloudinit.config.cc_snap.add_assertions')
312 def test_warn_schema_assertions_is_not_list_or_dict(self, _):
313 """Warn when snap:assertions config is not a list or dict."""
314 validate_cloudconfig_schema(
315 {'snap': {'assertions': 'broken'}}, schema)
316 self.assertEqual(
317 "WARNING: Invalid config:\nsnap.assertions: 'broken' is not of"
318 " type 'object', 'array'\n",
319 self.logs.getvalue())
320
321 @mock.patch('cloudinit.config.cc_snap.add_assertions')
322 def test_warn_schema_when_assertions_is_empty(self, _):
323 """Emit warnings when snap:assertions is an empty list or dict."""
324 validate_cloudconfig_schema(
325 {'snap': {'assertions': []}}, schema)
326 validate_cloudconfig_schema(
327 {'snap': {'assertions': {}}}, schema)
328 self.assertEqual(
329 "WARNING: Invalid config:\nsnap.assertions: [] is too short\n"
330 "WARNING: Invalid config:\nsnap.assertions: {} does not have"
331 " enough properties\n",
332 self.logs.getvalue())
333
334 @mock.patch('cloudinit.config.cc_snap.add_assertions')
335 def test_schema_when_assertions_are_list_or_dict(self, _):
336 """No warnings when snap:assertions are a list or dict."""
337 validate_cloudconfig_schema(
338 {'snap': {'assertions': ['valid']}}, schema)
339 validate_cloudconfig_schema(
340 {'snap': {'assertions': {'01': 'also valid'}}}, schema)
341 self.assertEqual('', self.logs.getvalue())
342
343
344class TestHandle(CiTestCase):
345
346 with_logs = True
347
348 def setUp(self):
349 super(TestHandle, self).setUp()
350 self.tmp = self.tmp_dir()
351
352 @mock.patch('cloudinit.config.cc_snap.run_commands')
353 @mock.patch('cloudinit.config.cc_snap.add_assertions')
354 @mock.patch('cloudinit.config.cc_snap.validate_cloudconfig_schema')
355 def test_handle_no_config(self, m_schema, m_add, m_run):
356 """When no snap-related configuration is provided, nothing happens."""
357 cfg = {}
358 handle('snap', cfg=cfg, cloud=None, log=self.logger, args=None)
359 self.assertIn(
360 "DEBUG: Skipping module named snap, no 'snap' key in config",
361 self.logs.getvalue())
362 m_schema.assert_not_called()
363 m_add.assert_not_called()
364 m_run.assert_not_called()
365
366 @mock.patch('cloudinit.config.cc_snap.run_commands')
367 @mock.patch('cloudinit.config.cc_snap.add_assertions')
368 @mock.patch('cloudinit.config.cc_snap.maybe_install_squashfuse')
369 def test_handle_skips_squashfuse_when_unconfigured(self, m_squash, m_add,
370 m_run):
371 """When squashfuse_in_container is unset, don't attempt to install."""
372 handle(
373 'snap', cfg={'snap': {}}, cloud=None, log=self.logger, args=None)
374 handle(
375 'snap', cfg={'snap': {'squashfuse_in_container': None}},
376 cloud=None, log=self.logger, args=None)
377 handle(
378 'snap', cfg={'snap': {'squashfuse_in_container': False}},
379 cloud=None, log=self.logger, args=None)
380 self.assertEqual([], m_squash.call_args_list) # No calls
381 # snap configuration missing assertions and commands will default to []
382 self.assertIn(mock.call([]), m_add.call_args_list)
383 self.assertIn(mock.call([]), m_run.call_args_list)
384
385 @mock.patch('cloudinit.config.cc_snap.maybe_install_squashfuse')
386 def test_handle_tries_to_install_squashfuse(self, m_squash):
387 """If squashfuse_in_container is True, try installing squashfuse."""
388 cfg = {'snap': {'squashfuse_in_container': True}}
389 mycloud = FakeCloud(None)
390 handle('snap', cfg=cfg, cloud=mycloud, log=self.logger, args=None)
391 self.assertEqual(
392 [mock.call(mycloud)], m_squash.call_args_list)
393
394 def test_handle_runs_commands_provided(self):
395 """If commands are specified as a list, run them."""
396 outfile = self.tmp_path('output.log', dir=self.tmp)
397
398 cfg = {
399 'snap': {'commands': ['echo "HI" >> %s' % outfile,
400 'echo "MOM" >> %s' % outfile]}}
401 mock_path = 'cloudinit.config.cc_snap.sys.stderr'
402 with mock.patch(mock_path, new_callable=StringIO):
403 handle('snap', cfg=cfg, cloud=None, log=self.logger, args=None)
404 self.assertEqual('HI\nMOM\n', util.load_file(outfile))
405
406 @mock.patch('cloudinit.config.cc_snap.util.subp')
407 def test_handle_adds_assertions(self, m_subp):
408 """Any configured snap assertions are provided to add_assertions."""
409 assert_file = self.tmp_path('snapd.assertions', dir=self.tmp)
410 compare_file = self.tmp_path('comparison', dir=self.tmp)
411 cfg = {
412 'snap': {'assertions': [SYSTEM_USER_ASSERTION, ACCOUNT_ASSERTION]}}
413 wrap_and_call(
414 'cloudinit.config.cc_snap',
415 {'ASSERTIONS_FILE': {'new': assert_file}},
416 handle, 'snap', cfg=cfg, cloud=None, log=self.logger, args=None)
417 content = '\n'.join(cfg['snap']['assertions'])
418 util.write_file(compare_file, content.encode('utf-8'))
419 self.assertEqual(
420 util.load_file(compare_file), util.load_file(assert_file))
421
422 @mock.patch('cloudinit.config.cc_snap.util.subp')
423 @skipUnlessJsonSchema()
424 def test_handle_validates_schema(self, m_subp):
425 """Any provided configuration is runs validate_cloudconfig_schema."""
426 assert_file = self.tmp_path('snapd.assertions', dir=self.tmp)
427 cfg = {'snap': {'invalid': ''}} # Generates schema warning
428 wrap_and_call(
429 'cloudinit.config.cc_snap',
430 {'ASSERTIONS_FILE': {'new': assert_file}},
431 handle, 'snap', cfg=cfg, cloud=None, log=self.logger, args=None)
432 self.assertEqual(
433 "WARNING: Invalid config:\nsnap: Additional properties are not"
434 " allowed ('invalid' was unexpected)\n",
435 self.logs.getvalue())
436
437
438class TestMaybeInstallSquashFuse(CiTestCase):
439
440 with_logs = True
441
442 def setUp(self):
443 super(TestMaybeInstallSquashFuse, self).setUp()
444 self.tmp = self.tmp_dir()
445
446 @mock.patch('cloudinit.config.cc_snap.util.is_container')
447 def test_maybe_install_squashfuse_skips_non_containers(self, m_container):
448 """maybe_install_squashfuse does nothing when not on a container."""
449 m_container.return_value = False
450 maybe_install_squashfuse(cloud=FakeCloud(None))
451 self.assertEqual([mock.call()], m_container.call_args_list)
452 self.assertEqual('', self.logs.getvalue())
453
454 @mock.patch('cloudinit.config.cc_snap.util.is_container')
455 def test_maybe_install_squashfuse_raises_install_errors(self, m_container):
456 """maybe_install_squashfuse logs and raises package install errors."""
457 m_container.return_value = True
458 distro = mock.MagicMock()
459 distro.update_package_sources.side_effect = RuntimeError(
460 'Some apt error')
461 with self.assertRaises(RuntimeError) as context_manager:
462 maybe_install_squashfuse(cloud=FakeCloud(distro))
463 self.assertEqual('Some apt error', str(context_manager.exception))
464 self.assertIn('Package update failed\nTraceback', self.logs.getvalue())
465
466 @mock.patch('cloudinit.config.cc_snap.util.is_container')
467 def test_maybe_install_squashfuse_raises_update_errors(self, m_container):
468 """maybe_install_squashfuse logs and raises package update errors."""
469 m_container.return_value = True
470 distro = mock.MagicMock()
471 distro.update_package_sources.side_effect = RuntimeError(
472 'Some apt error')
473 with self.assertRaises(RuntimeError) as context_manager:
474 maybe_install_squashfuse(cloud=FakeCloud(distro))
475 self.assertEqual('Some apt error', str(context_manager.exception))
476 self.assertIn('Package update failed\nTraceback', self.logs.getvalue())
477
478 @mock.patch('cloudinit.config.cc_snap.util.is_container')
479 def test_maybe_install_squashfuse_happy_path(self, m_container):
480 """maybe_install_squashfuse logs and raises package install errors."""
481 m_container.return_value = True
482 distro = mock.MagicMock() # No errors raised
483 maybe_install_squashfuse(cloud=FakeCloud(distro))
484 self.assertEqual(
485 [mock.call()], distro.update_package_sources.call_args_list)
486 self.assertEqual(
487 [mock.call(['squashfuse'])],
488 distro.install_packages.call_args_list)
489
490# vi: ts=4 expandtab
diff --git a/cloudinit/config/tests/test_ubuntu_advantage.py b/cloudinit/config/tests/test_ubuntu_advantage.py
0new file mode 100644491new file mode 100644
index 0000000..f2a59fa
--- /dev/null
+++ b/cloudinit/config/tests/test_ubuntu_advantage.py
@@ -0,0 +1,269 @@
1# This file is part of cloud-init. See LICENSE file for license information.
2
3import re
4from six import StringIO
5
6from cloudinit.config.cc_ubuntu_advantage import (
7 handle, maybe_install_ua_tools, run_commands, schema)
8from cloudinit.config.schema import validate_cloudconfig_schema
9from cloudinit import util
10from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema
11
12
13# Module path used in mocks
14MPATH = 'cloudinit.config.cc_ubuntu_advantage'
15
16
17class FakeCloud(object):
18 def __init__(self, distro):
19 self.distro = distro
20
21
22class TestRunCommands(CiTestCase):
23
24 with_logs = True
25
26 def setUp(self):
27 super(TestRunCommands, self).setUp()
28 self.tmp = self.tmp_dir()
29
30 @mock.patch('%s.util.subp' % MPATH)
31 def test_run_commands_on_empty_list(self, m_subp):
32 """When provided with an empty list, run_commands does nothing."""
33 run_commands([])
34 self.assertEqual('', self.logs.getvalue())
35 m_subp.assert_not_called()
36
37 def test_run_commands_on_non_list_or_dict(self):
38 """When provided an invalid type, run_commands raises an error."""
39 with self.assertRaises(TypeError) as context_manager:
40 run_commands(commands="I'm Not Valid")
41 self.assertEqual(
42 "commands parameter was not a list or dict: I'm Not Valid",
43 str(context_manager.exception))
44
45 def test_run_command_logs_commands_and_exit_codes_to_stderr(self):
46 """All exit codes are logged to stderr."""
47 outfile = self.tmp_path('output.log', dir=self.tmp)
48
49 cmd1 = 'echo "HI" >> %s' % outfile
50 cmd2 = 'bogus command'
51 cmd3 = 'echo "MOM" >> %s' % outfile
52 commands = [cmd1, cmd2, cmd3]
53
54 mock_path = '%s.sys.stderr' % MPATH
55 with mock.patch(mock_path, new_callable=StringIO) as m_stderr:
56 with self.assertRaises(RuntimeError) as context_manager:
57 run_commands(commands=commands)
58
59 self.assertIsNotNone(
60 re.search(r'bogus: (command )?not found',
61 str(context_manager.exception)),
62 msg='Expected bogus command not found')
63 expected_stderr_log = '\n'.join([
64 'Begin run command: {cmd}'.format(cmd=cmd1),
65 'End run command: exit(0)',
66 'Begin run command: {cmd}'.format(cmd=cmd2),
67 'ERROR: End run command: exit(127)',
68 'Begin run command: {cmd}'.format(cmd=cmd3),
69 'End run command: exit(0)\n'])
70 self.assertEqual(expected_stderr_log, m_stderr.getvalue())
71
72 def test_run_command_as_lists(self):
73 """When commands are specified as a list, run them in order."""
74 outfile = self.tmp_path('output.log', dir=self.tmp)
75
76 cmd1 = 'echo "HI" >> %s' % outfile
77 cmd2 = 'echo "MOM" >> %s' % outfile
78 commands = [cmd1, cmd2]
79 with mock.patch('%s.sys.stderr' % MPATH, new_callable=StringIO):
80 run_commands(commands=commands)
81
82 self.assertIn(
83 'DEBUG: Running user-provided ubuntu-advantage commands',
84 self.logs.getvalue())
85 self.assertEqual('HI\nMOM\n', util.load_file(outfile))
86 self.assertIn(
87 'WARNING: Non-ubuntu-advantage commands in ubuntu-advantage'
88 ' config:',
89 self.logs.getvalue())
90
91 def test_run_command_dict_sorted_as_command_script(self):
92 """When commands are a dict, sort them and run."""
93 outfile = self.tmp_path('output.log', dir=self.tmp)
94 cmd1 = 'echo "HI" >> %s' % outfile
95 cmd2 = 'echo "MOM" >> %s' % outfile
96 commands = {'02': cmd1, '01': cmd2}
97 with mock.patch('%s.sys.stderr' % MPATH, new_callable=StringIO):
98 run_commands(commands=commands)
99
100 expected_messages = [
101 'DEBUG: Running user-provided ubuntu-advantage commands']
102 for message in expected_messages:
103 self.assertIn(message, self.logs.getvalue())
104 self.assertEqual('MOM\nHI\n', util.load_file(outfile))
105
106
107@skipUnlessJsonSchema()
108class TestSchema(CiTestCase):
109
110 with_logs = True
111
112 def test_schema_warns_on_ubuntu_advantage_not_as_dict(self):
113 """If ubuntu-advantage configuration is not a dict, emit a warning."""
114 validate_cloudconfig_schema({'ubuntu-advantage': 'wrong type'}, schema)
115 self.assertEqual(
116 "WARNING: Invalid config:\nubuntu-advantage: 'wrong type' is not"
117 " of type 'object'\n",
118 self.logs.getvalue())
119
120 @mock.patch('%s.run_commands' % MPATH)
121 def test_schema_disallows_unknown_keys(self, _):
122 """Unknown keys in ubuntu-advantage configuration emit warnings."""
123 validate_cloudconfig_schema(
124 {'ubuntu-advantage': {'commands': ['ls'], 'invalid-key': ''}},
125 schema)
126 self.assertIn(
127 'WARNING: Invalid config:\nubuntu-advantage: Additional properties'
128 " are not allowed ('invalid-key' was unexpected)",
129 self.logs.getvalue())
130
131 def test_warn_schema_requires_commands(self):
132 """Warn when ubuntu-advantage configuration lacks commands."""
133 validate_cloudconfig_schema(
134 {'ubuntu-advantage': {}}, schema)
135 self.assertEqual(
136 "WARNING: Invalid config:\nubuntu-advantage: 'commands' is a"
137 " required property\n",
138 self.logs.getvalue())
139
140 @mock.patch('%s.run_commands' % MPATH)
141 def test_warn_schema_commands_is_not_list_or_dict(self, _):
142 """Warn when ubuntu-advantage:commands config is not a list or dict."""
143 validate_cloudconfig_schema(
144 {'ubuntu-advantage': {'commands': 'broken'}}, schema)
145 self.assertEqual(
146 "WARNING: Invalid config:\nubuntu-advantage.commands: 'broken' is"
147 " not of type 'object', 'array'\n",
148 self.logs.getvalue())
149
150 @mock.patch('%s.run_commands' % MPATH)
151 def test_warn_schema_when_commands_is_empty(self, _):
152 """Emit warnings when ubuntu-advantage:commands is empty."""
153 validate_cloudconfig_schema(
154 {'ubuntu-advantage': {'commands': []}}, schema)
155 validate_cloudconfig_schema(
156 {'ubuntu-advantage': {'commands': {}}}, schema)
157 self.assertEqual(
158 "WARNING: Invalid config:\nubuntu-advantage.commands: [] is too"
159 " short\nWARNING: Invalid config:\nubuntu-advantage.commands: {}"
160 " does not have enough properties\n",
161 self.logs.getvalue())
162
163 @mock.patch('%s.run_commands' % MPATH)
164 def test_schema_when_commands_are_list_or_dict(self, _):
165 """No warnings when ubuntu-advantage:commands are a list or dict."""
166 validate_cloudconfig_schema(
167 {'ubuntu-advantage': {'commands': ['valid']}}, schema)
168 validate_cloudconfig_schema(
169 {'ubuntu-advantage': {'commands': {'01': 'also valid'}}}, schema)
170 self.assertEqual('', self.logs.getvalue())
171
172
173class TestHandle(CiTestCase):
174
175 with_logs = True
176
177 def setUp(self):
178 super(TestHandle, self).setUp()
179 self.tmp = self.tmp_dir()
180
181 @mock.patch('%s.run_commands' % MPATH)
182 @mock.patch('%s.validate_cloudconfig_schema' % MPATH)
183 def test_handle_no_config(self, m_schema, m_run):
184 """When no ua-related configuration is provided, nothing happens."""
185 cfg = {}
186 handle('ua-test', cfg=cfg, cloud=None, log=self.logger, args=None)
187 self.assertIn(
188 "DEBUG: Skipping module named ua-test, no 'ubuntu-advantage' key"
189 " in config",
190 self.logs.getvalue())
191 m_schema.assert_not_called()
192 m_run.assert_not_called()
193
194 @mock.patch('%s.maybe_install_ua_tools' % MPATH)
195 def test_handle_tries_to_install_ubuntu_advantage_tools(self, m_install):
196 """If ubuntu_advantage is provided, try installing ua-tools package."""
197 cfg = {'ubuntu-advantage': {}}
198 mycloud = FakeCloud(None)
199 handle('nomatter', cfg=cfg, cloud=mycloud, log=self.logger, args=None)
200 m_install.assert_called_once_with(mycloud)
201
202 @mock.patch('%s.maybe_install_ua_tools' % MPATH)
203 def test_handle_runs_commands_provided(self, m_install):
204 """When commands are specified as a list, run them."""
205 outfile = self.tmp_path('output.log', dir=self.tmp)
206
207 cfg = {
208 'ubuntu-advantage': {'commands': ['echo "HI" >> %s' % outfile,
209 'echo "MOM" >> %s' % outfile]}}
210 mock_path = '%s.sys.stderr' % MPATH
211 with mock.patch(mock_path, new_callable=StringIO):
212 handle('nomatter', cfg=cfg, cloud=None, log=self.logger, args=None)
213 self.assertEqual('HI\nMOM\n', util.load_file(outfile))
214
215
216class TestMaybeInstallUATools(CiTestCase):
217
218 with_logs = True
219
220 def setUp(self):
221 super(TestMaybeInstallUATools, self).setUp()
222 self.tmp = self.tmp_dir()
223
224 @mock.patch('%s.util.which' % MPATH)
225 def test_maybe_install_ua_tools_noop_when_ua_tools_present(self, m_which):
226 """Do nothing if ubuntu-advantage-tools already exists."""
227 m_which.return_value = '/usr/bin/ubuntu-advantage' # already installed
228 distro = mock.MagicMock()
229 distro.update_package_sources.side_effect = RuntimeError(
230 'Some apt error')
231 maybe_install_ua_tools(cloud=FakeCloud(distro)) # No RuntimeError
232
233 @mock.patch('%s.util.which' % MPATH)
234 def test_maybe_install_ua_tools_raises_update_errors(self, m_which):
235 """maybe_install_ua_tools logs and raises apt update errors."""
236 m_which.return_value = None
237 distro = mock.MagicMock()
238 distro.update_package_sources.side_effect = RuntimeError(
239 'Some apt error')
240 with self.assertRaises(RuntimeError) as context_manager:
241 maybe_install_ua_tools(cloud=FakeCloud(distro))
242 self.assertEqual('Some apt error', str(context_manager.exception))
243 self.assertIn('Package update failed\nTraceback', self.logs.getvalue())
244
245 @mock.patch('%s.util.which' % MPATH)
246 def test_maybe_install_ua_raises_install_errors(self, m_which):
247 """maybe_install_ua_tools logs and raises package install errors."""
248 m_which.return_value = None
249 distro = mock.MagicMock()
250 distro.update_package_sources.return_value = None
251 distro.install_packages.side_effect = RuntimeError(
252 'Some install error')
253 with self.assertRaises(RuntimeError) as context_manager:
254 maybe_install_ua_tools(cloud=FakeCloud(distro))
255 self.assertEqual('Some install error', str(context_manager.exception))
256 self.assertIn(
257 'Failed to install ubuntu-advantage-tools\n', self.logs.getvalue())
258
259 @mock.patch('%s.util.which' % MPATH)
260 def test_maybe_install_ua_tools_happy_path(self, m_which):
261 """maybe_install_ua_tools installs ubuntu-advantage-tools."""
262 m_which.return_value = None
263 distro = mock.MagicMock() # No errors raised
264 maybe_install_ua_tools(cloud=FakeCloud(distro))
265 distro.update_package_sources.assert_called_once_with()
266 distro.install_packages.assert_called_once_with(
267 ['ubuntu-advantage-tools'])
268
269# vi: ts=4 expandtab
diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py
index f87a343..b814c8b 100644
--- a/cloudinit/distros/arch.py
+++ b/cloudinit/distros/arch.py
@@ -129,11 +129,8 @@ class Distro(distros.Distro):
129 if pkgs is None:129 if pkgs is None:
130 pkgs = []130 pkgs = []
131131
132 cmd = ['pacman']132 cmd = ['pacman', "-Sy", "--quiet", "--noconfirm"]
133 # Redirect output133 # Redirect output
134 cmd.append("-Sy")
135 cmd.append("--quiet")
136 cmd.append("--noconfirm")
137134
138 if args and isinstance(args, str):135 if args and isinstance(args, str):
139 cmd.append(args)136 cmd.append(args)
diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py
index aa468bc..754d3df 100644
--- a/cloudinit/distros/freebsd.py
+++ b/cloudinit/distros/freebsd.py
@@ -132,6 +132,12 @@ class Distro(distros.Distro):
132 LOG.debug("Using network interface %s", bsddev)132 LOG.debug("Using network interface %s", bsddev)
133 return bsddev133 return bsddev
134134
135 def _select_hostname(self, hostname, fqdn):
136 # Should be FQDN if available. See rc.conf(5) in FreeBSD
137 if fqdn:
138 return fqdn
139 return hostname
140
135 def _read_system_hostname(self):141 def _read_system_hostname(self):
136 sys_hostname = self._read_hostname(filename=None)142 sys_hostname = self._read_hostname(filename=None)
137 return ('rc.conf', sys_hostname)143 return ('rc.conf', sys_hostname)
diff --git a/cloudinit/distros/opensuse.py b/cloudinit/distros/opensuse.py
index a219e9f..162dfa0 100644
--- a/cloudinit/distros/opensuse.py
+++ b/cloudinit/distros/opensuse.py
@@ -67,11 +67,10 @@ class Distro(distros.Distro):
67 if pkgs is None:67 if pkgs is None:
68 pkgs = []68 pkgs = []
6969
70 cmd = ['zypper']
71 # No user interaction possible, enable non-interactive mode70 # No user interaction possible, enable non-interactive mode
72 cmd.append('--non-interactive')71 cmd = ['zypper', '--non-interactive']
7372
74 # Comand is the operation, such as install73 # Command is the operation, such as install
75 if command == 'upgrade':74 if command == 'upgrade':
76 command = 'update'75 command = 'update'
77 cmd.append(command)76 cmd.append(command)
diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
index d6c61e4..dc3f0fc 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/ec2_utils.py
@@ -135,10 +135,8 @@ class MetadataMaterializer(object):
135135
136136
137def _skip_retry_on_codes(status_codes, _request_args, cause):137def _skip_retry_on_codes(status_codes, _request_args, cause):
138 """Returns if a request should retry based on a given set of codes that138 """Returns False if cause.code is in status_codes."""
139 case retrying to be stopped/skipped.139 return cause.code not in status_codes
140 """
141 return cause.code in status_codes
142140
143141
144def get_instance_userdata(api_version='latest',142def get_instance_userdata(api_version='latest',
diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py
index 7b2cc9d..9e9fe0f 100755
--- a/cloudinit/net/cmdline.py
+++ b/cloudinit/net/cmdline.py
@@ -9,12 +9,15 @@ import base64
9import glob9import glob
10import gzip10import gzip
11import io11import io
12import os
1213
13from . import get_devicelist14from . import get_devicelist
14from . import read_sys_net_safe15from . import read_sys_net_safe
1516
16from cloudinit import util17from cloudinit import util
1718
19_OPEN_ISCSI_INTERFACE_FILE = "/run/initramfs/open-iscsi.interface"
20
1821
19def _klibc_to_config_entry(content, mac_addrs=None):22def _klibc_to_config_entry(content, mac_addrs=None):
20 """Convert a klibc written shell content file to a 'config' entry23 """Convert a klibc written shell content file to a 'config' entry
@@ -103,9 +106,13 @@ def _klibc_to_config_entry(content, mac_addrs=None):
103 return name, iface106 return name, iface
104107
105108
109def _get_klibc_net_cfg_files():
110 return glob.glob('/run/net-*.conf') + glob.glob('/run/net6-*.conf')
111
112
106def config_from_klibc_net_cfg(files=None, mac_addrs=None):113def config_from_klibc_net_cfg(files=None, mac_addrs=None):
107 if files is None:114 if files is None:
108 files = glob.glob('/run/net-*.conf') + glob.glob('/run/net6-*.conf')115 files = _get_klibc_net_cfg_files()
109116
110 entries = []117 entries = []
111 names = {}118 names = {}
@@ -160,10 +167,23 @@ def _b64dgz(b64str, gzipped="try"):
160 return _decomp_gzip(blob, strict=gzipped != "try")167 return _decomp_gzip(blob, strict=gzipped != "try")
161168
162169
170def _is_initramfs_netconfig(files, cmdline):
171 if files:
172 if 'ip=' in cmdline or 'ip6=' in cmdline:
173 return True
174 if os.path.exists(_OPEN_ISCSI_INTERFACE_FILE):
175 # iBft can configure networking without ip=
176 return True
177 return False
178
179
163def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None):180def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None):
164 if cmdline is None:181 if cmdline is None:
165 cmdline = util.get_cmdline()182 cmdline = util.get_cmdline()
166183
184 if files is None:
185 files = _get_klibc_net_cfg_files()
186
167 if 'network-config=' in cmdline:187 if 'network-config=' in cmdline:
168 data64 = None188 data64 = None
169 for tok in cmdline.split():189 for tok in cmdline.split():
@@ -172,7 +192,7 @@ def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None):
172 if data64:192 if data64:
173 return util.load_yaml(_b64dgz(data64))193 return util.load_yaml(_b64dgz(data64))
174194
175 if 'ip=' not in cmdline and 'ip6=' not in cmdline:195 if not _is_initramfs_netconfig(files, cmdline):
176 return None196 return None
177197
178 if mac_addrs is None:198 if mac_addrs is None:
diff --git a/cloudinit/net/netplan.py b/cloudinit/net/netplan.py
index d3788af..6344348 100644
--- a/cloudinit/net/netplan.py
+++ b/cloudinit/net/netplan.py
@@ -311,12 +311,12 @@ class Renderer(renderer.Renderer):
311 if newname is None:311 if newname is None:
312 continue312 continue
313 br_config.update({newname: value})313 br_config.update({newname: value})
314 if newname == 'path-cost':314 if newname in ['path-cost', 'port-priority']:
315 # <interface> <cost> -> <interface>: int(<cost>)315 # <interface> <value> -> <interface>: int(<value>)
316 newvalue = {}316 newvalue = {}
317 for costval in value:317 for val in value:
318 (port, cost) = costval.split()318 (port, portval) = val.split()
319 newvalue[port] = int(cost)319 newvalue[port] = int(portval)
320 br_config.update({newname: newvalue})320 br_config.update({newname: newvalue})
321321
322 if len(br_config) > 0:322 if len(br_config) > 0:
@@ -336,22 +336,15 @@ class Renderer(renderer.Renderer):
336 _extract_addresses(ifcfg, vlan)336 _extract_addresses(ifcfg, vlan)
337 vlans.update({ifname: vlan})337 vlans.update({ifname: vlan})
338338
339 # inject global nameserver values under each physical interface339 # inject global nameserver values under each all interface which
340 if nameservers:340 # has addresses and do not already have a DNS configuration
341 for _eth, cfg in ethernets.items():341 if nameservers or searchdomains:
342 nscfg = cfg.get('nameservers', {})342 nscfg = {'addresses': nameservers, 'search': searchdomains}
343 addresses = nscfg.get('addresses', [])343 for section in [ethernets, wifis, bonds, bridges, vlans]:
344 addresses += nameservers344 for _name, cfg in section.items():
345 nscfg.update({'addresses': addresses})345 if 'nameservers' in cfg or 'addresses' not in cfg:
346 cfg.update({'nameservers': nscfg})346 continue
347347 cfg.update({'nameservers': nscfg})
348 if searchdomains:
349 for _eth, cfg in ethernets.items():
350 nscfg = cfg.get('nameservers', {})
351 search = nscfg.get('search', [])
352 search += searchdomains
353 nscfg.update({'search': search})
354 cfg.update({'nameservers': nscfg})
355348
356 # workaround yaml dictionary key sorting when dumping349 # workaround yaml dictionary key sorting when dumping
357 def _render_section(name, section):350 def _render_section(name, section):
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index fe667d8..6d63e5c 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -47,7 +47,7 @@ NET_CONFIG_TO_V2 = {
47 'bridge_maxage': 'max-age',47 'bridge_maxage': 'max-age',
48 'bridge_maxwait': None,48 'bridge_maxwait': None,
49 'bridge_pathcost': 'path-cost',49 'bridge_pathcost': 'path-cost',
50 'bridge_portprio': None,50 'bridge_portprio': 'port-priority',
51 'bridge_stp': 'stp',51 'bridge_stp': 'stp',
52 'bridge_waitport': None}}52 'bridge_waitport': None}}
5353
@@ -708,6 +708,7 @@ class NetworkStateInterpreter(object):
708708
709 gateway4 = None709 gateway4 = None
710 gateway6 = None710 gateway6 = None
711 nameservers = {}
711 for address in cfg.get('addresses', []):712 for address in cfg.get('addresses', []):
712 subnet = {713 subnet = {
713 'type': 'static',714 'type': 'static',
@@ -723,6 +724,15 @@ class NetworkStateInterpreter(object):
723 gateway4 = cfg.get('gateway4')724 gateway4 = cfg.get('gateway4')
724 subnet.update({'gateway': gateway4})725 subnet.update({'gateway': gateway4})
725726
727 if 'nameservers' in cfg and not nameservers:
728 addresses = cfg.get('nameservers').get('addresses')
729 if addresses:
730 nameservers['dns_nameservers'] = addresses
731 search = cfg.get('nameservers').get('search')
732 if search:
733 nameservers['dns_search'] = search
734 subnet.update(nameservers)
735
726 subnets.append(subnet)736 subnets.append(subnet)
727737
728 routes = []738 routes = []
diff --git a/cloudinit/settings.py b/cloudinit/settings.py
index c120498..dde5749 100644
--- a/cloudinit/settings.py
+++ b/cloudinit/settings.py
@@ -36,6 +36,8 @@ CFG_BUILTIN = {
36 'SmartOS',36 'SmartOS',
37 'Bigstep',37 'Bigstep',
38 'Scaleway',38 'Scaleway',
39 'Hetzner',
40 'IBMCloud',
39 # At the end to act as a 'catch' when none of the above work...41 # At the end to act as a 'catch' when none of the above work...
40 'None',42 'None',
41 ],43 ],
diff --git a/cloudinit/sources/DataSourceAliYun.py b/cloudinit/sources/DataSourceAliYun.py
index 7ac8288..22279d0 100644
--- a/cloudinit/sources/DataSourceAliYun.py
+++ b/cloudinit/sources/DataSourceAliYun.py
@@ -22,7 +22,7 @@ class DataSourceAliYun(EC2.DataSourceEc2):
22 super(DataSourceAliYun, self).__init__(sys_cfg, distro, paths)22 super(DataSourceAliYun, self).__init__(sys_cfg, distro, paths)
23 self.seed_dir = os.path.join(paths.seed_dir, "AliYun")23 self.seed_dir = os.path.join(paths.seed_dir, "AliYun")
2424
25 def get_hostname(self, fqdn=False, _resolve_ip=False):25 def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False):
26 return self.metadata.get('hostname', 'localhost.localdomain')26 return self.metadata.get('hostname', 'localhost.localdomain')
2727
28 def get_public_ssh_keys(self):28 def get_public_ssh_keys(self):
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 4bcbf3a..0ee622e 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -20,7 +20,7 @@ from cloudinit import net
20from cloudinit.net.dhcp import EphemeralDHCPv420from cloudinit.net.dhcp import EphemeralDHCPv4
21from cloudinit import sources21from cloudinit import sources
22from cloudinit.sources.helpers.azure import get_metadata_from_fabric22from cloudinit.sources.helpers.azure import get_metadata_from_fabric
23from cloudinit.url_helper import readurl, wait_for_url, UrlError23from cloudinit.url_helper import readurl, UrlError
24from cloudinit import util24from cloudinit import util
2525
26LOG = logging.getLogger(__name__)26LOG = logging.getLogger(__name__)
@@ -49,7 +49,6 @@ DEFAULT_FS = 'ext4'
49AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77'49AZURE_CHASSIS_ASSET_TAG = '7783-7084-3265-9085-8269-3286-77'
50REPROVISION_MARKER_FILE = "/var/lib/cloud/data/poll_imds"50REPROVISION_MARKER_FILE = "/var/lib/cloud/data/poll_imds"
51IMDS_URL = "http://169.254.169.254/metadata/reprovisiondata"51IMDS_URL = "http://169.254.169.254/metadata/reprovisiondata"
52IMDS_RETRIES = 5
5352
5453
55def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid):54def find_storvscid_from_sysctl_pnpinfo(sysctl_out, deviceid):
@@ -223,6 +222,8 @@ DEF_PASSWD_REDACTION = 'REDACTED'
223222
224223
225def get_hostname(hostname_command='hostname'):224def get_hostname(hostname_command='hostname'):
225 if not isinstance(hostname_command, (list, tuple)):
226 hostname_command = (hostname_command,)
226 return util.subp(hostname_command, capture=True)[0].strip()227 return util.subp(hostname_command, capture=True)[0].strip()
227228
228229
@@ -449,36 +450,24 @@ class DataSourceAzure(sources.DataSource):
449 headers = {"Metadata": "true"}450 headers = {"Metadata": "true"}
450 LOG.debug("Start polling IMDS")451 LOG.debug("Start polling IMDS")
451452
452 def sleep_cb(response, loop_n):453 def exc_cb(msg, exception):
453 return 1
454
455 def exception_cb(msg, exception):
456 if isinstance(exception, UrlError) and exception.code == 404:454 if isinstance(exception, UrlError) and exception.code == 404:
457 return455 return True
458 LOG.warning("Exception during polling. Will try DHCP.",
459 exc_info=True)
460
461 # If we get an exception while trying to call IMDS, we456 # If we get an exception while trying to call IMDS, we
462 # call DHCP and setup the ephemeral network to acquire the new IP.457 # call DHCP and setup the ephemeral network to acquire the new IP.
463 raise exception458 return False
464459
465 need_report = report_ready460 need_report = report_ready
466 for i in range(IMDS_RETRIES):461 while True:
467 try:462 try:
468 with EphemeralDHCPv4() as lease:463 with EphemeralDHCPv4() as lease:
469 if need_report:464 if need_report:
470 self._report_ready(lease=lease)465 self._report_ready(lease=lease)
471 need_report = False466 need_report = False
472 wait_for_url([url], max_wait=None, timeout=60,467 return readurl(url, timeout=1, headers=headers,
473 status_cb=LOG.info,468 exception_cb=exc_cb, infinite=True).contents
474 headers_cb=lambda url: headers, sleep_time=1,469 except UrlError:
475 exception_cb=exception_cb,470 pass
476 sleep_time_cb=sleep_cb)
477 return str(readurl(url, headers=headers))
478 except Exception:
479 LOG.debug("Exception during polling-retrying dhcp" +
480 " %d more time(s).", (IMDS_RETRIES - i),
481 exc_info=True)
482471
483 def _report_ready(self, lease):472 def _report_ready(self, lease):
484 """Tells the fabric provisioning has completed473 """Tells the fabric provisioning has completed
diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py
index 4eaad47..c816f34 100644
--- a/cloudinit/sources/DataSourceCloudSigma.py
+++ b/cloudinit/sources/DataSourceCloudSigma.py
@@ -84,7 +84,7 @@ class DataSourceCloudSigma(sources.DataSource):
8484
85 return True85 return True
8686
87 def get_hostname(self, fqdn=False, resolve_ip=False):87 def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False):
88 """88 """
89 Cleans up and uses the server's name if the latter is set. Otherwise89 Cleans up and uses the server's name if the latter is set. Otherwise
90 the first part from uuid is being used.90 the first part from uuid is being used.
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index b8db626..c7b5fe5 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -14,6 +14,7 @@ from cloudinit import util
1414
15from cloudinit.net import eni15from cloudinit.net import eni
1616
17from cloudinit.sources.DataSourceIBMCloud import get_ibm_platform
17from cloudinit.sources.helpers import openstack18from cloudinit.sources.helpers import openstack
1819
19LOG = logging.getLogger(__name__)20LOG = logging.getLogger(__name__)
@@ -255,6 +256,15 @@ def find_candidate_devs(probe_optical=True):
255 # an unpartitioned block device (ex sda, not sda1)256 # an unpartitioned block device (ex sda, not sda1)
256 devices = [d for d in candidates257 devices = [d for d in candidates
257 if d in by_label or not util.is_partition(d)]258 if d in by_label or not util.is_partition(d)]
259
260 if devices:
261 # IBMCloud uses config-2 label, but limited to a single UUID.
262 ibm_platform, ibm_path = get_ibm_platform()
263 if ibm_path in devices:
264 devices.remove(ibm_path)
265 LOG.debug("IBMCloud device '%s' (%s) removed from candidate list",
266 ibm_path, ibm_platform)
267
258 return devices268 return devices
259269
260270
diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py
index 2da34a9..d816262 100644
--- a/cloudinit/sources/DataSourceGCE.py
+++ b/cloudinit/sources/DataSourceGCE.py
@@ -90,7 +90,7 @@ class DataSourceGCE(sources.DataSource):
90 public_keys_data = self.metadata['public-keys-data']90 public_keys_data = self.metadata['public-keys-data']
91 return _parse_public_keys(public_keys_data, self.default_user)91 return _parse_public_keys(public_keys_data, self.default_user)
9292
93 def get_hostname(self, fqdn=False, resolve_ip=False):93 def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False):
94 # GCE has long FDQN's and has asked for short hostnames.94 # GCE has long FDQN's and has asked for short hostnames.
95 return self.metadata['local-hostname'].split('.')[0]95 return self.metadata['local-hostname'].split('.')[0]
9696
@@ -213,16 +213,15 @@ def read_md(address=None, platform_check=True):
213 if md['availability-zone']:213 if md['availability-zone']:
214 md['availability-zone'] = md['availability-zone'].split('/')[-1]214 md['availability-zone'] = md['availability-zone'].split('/')[-1]
215215
216 encoding = instance_data.get('user-data-encoding')216 if 'user-data' in instance_data:
217 if encoding:217 # instance_data was json, so values are all utf-8 strings.
218 ud = instance_data['user-data'].encode("utf-8")
219 encoding = instance_data.get('user-data-encoding')
218 if encoding == 'base64':220 if encoding == 'base64':
219 md['user-data'] = b64decode(instance_data.get('user-data'))221 ud = b64decode(ud)
220 else:222 elif encoding:
221 LOG.warning('unknown user-data-encoding: %s, ignoring', encoding)223 LOG.warning('unknown user-data-encoding: %s, ignoring', encoding)
222224 ret['user-data'] = ud
223 if 'user-data' in md:
224 ret['user-data'] = md['user-data']
225 del md['user-data']
226225
227 ret['meta-data'] = md226 ret['meta-data'] = md
228 ret['success'] = True227 ret['success'] = True
diff --git a/cloudinit/sources/DataSourceHetzner.py b/cloudinit/sources/DataSourceHetzner.py
229new file mode 100644228new file mode 100644
index 0000000..5c75b65
--- /dev/null
+++ b/cloudinit/sources/DataSourceHetzner.py
@@ -0,0 +1,106 @@
1# Author: Jonas Keidel <jonas.keidel@hetzner.com>
2# Author: Markus Schade <markus.schade@hetzner.com>
3#
4# This file is part of cloud-init. See LICENSE file for license information.
5#
6"""Hetzner Cloud API Documentation.
7 https://docs.hetzner.cloud/"""
8
9from cloudinit import log as logging
10from cloudinit import net as cloudnet
11from cloudinit import sources
12from cloudinit import util
13
14import cloudinit.sources.helpers.hetzner as hc_helper
15
16LOG = logging.getLogger(__name__)
17
18BASE_URL_V1 = 'http://169.254.169.254/hetzner/v1'
19
20BUILTIN_DS_CONFIG = {
21 'metadata_url': BASE_URL_V1 + '/metadata',
22 'userdata_url': BASE_URL_V1 + '/userdata',
23}
24
25MD_RETRIES = 60
26MD_TIMEOUT = 2
27MD_WAIT_RETRY = 2
28
29
30class DataSourceHetzner(sources.DataSource):
31 def __init__(self, sys_cfg, distro, paths):
32 sources.DataSource.__init__(self, sys_cfg, distro, paths)
33 self.distro = distro
34 self.metadata = dict()
35 self.ds_cfg = util.mergemanydict([
36 util.get_cfg_by_path(sys_cfg, ["datasource", "Hetzner"], {}),
37 BUILTIN_DS_CONFIG])
38 self.metadata_address = self.ds_cfg['metadata_url']
39 self.userdata_address = self.ds_cfg['userdata_url']
40 self.retries = self.ds_cfg.get('retries', MD_RETRIES)
41 self.timeout = self.ds_cfg.get('timeout', MD_TIMEOUT)
42 self.wait_retry = self.ds_cfg.get('wait_retry', MD_WAIT_RETRY)
43 self._network_config = None
44 self.dsmode = sources.DSMODE_NETWORK
45
46 def get_data(self):
47 if not on_hetzner():
48 return False
49 nic = cloudnet.find_fallback_nic()
50 with cloudnet.EphemeralIPv4Network(nic, "169.254.0.1", 16,
51 "169.254.255.255"):
52 md = hc_helper.read_metadata(
53 self.metadata_address, timeout=self.timeout,
54 sec_between=self.wait_retry, retries=self.retries)
55 ud = hc_helper.read_userdata(
56 self.userdata_address, timeout=self.timeout,
57 sec_between=self.wait_retry, retries=self.retries)
58
59 self.userdata_raw = ud
60 self.metadata_full = md
61
62 """hostname is name provided by user at launch. The API enforces
63 it is a valid hostname, but it is not guaranteed to be resolvable
64 in dns or fully qualified."""
65 self.metadata['instance-id'] = md['instance-id']
66 self.metadata['local-hostname'] = md['hostname']
67 self.metadata['network-config'] = md.get('network-config', None)
68 self.metadata['public-keys'] = md.get('public-keys', None)
69 self.vendordata_raw = md.get("vendor_data", None)
70
71 return True
72
73 @property
74 def network_config(self):
75 """Configure the networking. This needs to be done each boot, since
76 the IP information may have changed due to snapshot and/or
77 migration.
78 """
79
80 if self._network_config:
81 return self._network_config
82
83 _net_config = self.metadata['network-config']
84 if not _net_config:
85 raise Exception("Unable to get meta-data from server....")
86
87 self._network_config = _net_config
88
89 return self._network_config
90
91
92def on_hetzner():
93 return util.read_dmi_data('system-manufacturer') == "Hetzner"
94
95
96# Used to match classes to dependencies
97datasources = [
98 (DataSourceHetzner, (sources.DEP_FILESYSTEM, )),
99]
100
101
102# Return a list of data sources that match this set of dependencies
103def get_datasource_list(depends):
104 return sources.list_from_depends(depends, datasources)
105
106# vi: ts=4 expandtab
diff --git a/cloudinit/sources/DataSourceIBMCloud.py b/cloudinit/sources/DataSourceIBMCloud.py
0new file mode 100644107new file mode 100644
index 0000000..02b3d56
--- /dev/null
+++ b/cloudinit/sources/DataSourceIBMCloud.py
@@ -0,0 +1,325 @@
1# This file is part of cloud-init. See LICENSE file for license information.
2"""Datasource for IBMCloud.
3
4IBMCloud is also know as SoftLayer or BlueMix.
5IBMCloud hypervisor is xen (2018-03-10).
6
7There are 2 different api exposed launch methods.
8 * template: This is the legacy method of launching instances.
9 When booting from an image template, the system boots first into
10 a "provisioning" mode. There, host <-> guest mechanisms are utilized
11 to execute code in the guest and provision it.
12
13 Cloud-init will disable itself when it detects that it is in the
14 provisioning mode. It detects this by the presence of
15 a file '/root/provisioningConfiguration.cfg'.
16
17 When provided with user-data, the "first boot" will contain a
18 ConfigDrive-like disk labeled with 'METADATA'. If there is no user-data
19 provided, then there is no data-source.
20
21 Cloud-init never does any network configuration in this mode.
22
23 * os_code: Essentially "launch by OS Code" (Operating System Code).
24 This is a more modern approach. There is no specific "provisioning" boot.
25 Instead, cloud-init does all the customization. With or without
26 user-data provided, an OpenStack ConfigDrive like disk is attached.
27
28 Only disks with label 'config-2' and UUID '9796-932E' are considered.
29 This is to avoid this datasource claiming ConfigDrive. This does
30 mean that 1 in 8^16 (~4 billion) Xen ConfigDrive systems will be
31 incorrectly identified as IBMCloud.
32
33TODO:
34 * is uuid (/sys/hypervisor/uuid) stable for life of an instance?
35 it seems it is not the same as data's uuid in the os_code case
36 but is in the template case.
37
38"""
39import base64
40import json
41import os
42
43from cloudinit import log as logging
44from cloudinit import sources
45from cloudinit.sources.helpers import openstack
46from cloudinit import util
47
48LOG = logging.getLogger(__name__)
49
50IBM_CONFIG_UUID = "9796-932E"
51
52
53class Platforms(object):
54 TEMPLATE_LIVE_METADATA = "Template/Live/Metadata"
55 TEMPLATE_LIVE_NODATA = "UNABLE TO BE IDENTIFIED."
56 TEMPLATE_PROVISIONING_METADATA = "Template/Provisioning/Metadata"
57 TEMPLATE_PROVISIONING_NODATA = "Template/Provisioning/No-Metadata"
58 OS_CODE = "OS-Code/Live"
59
60
61PROVISIONING = (
62 Platforms.TEMPLATE_PROVISIONING_METADATA,
63 Platforms.TEMPLATE_PROVISIONING_NODATA)
64
65
66class DataSourceIBMCloud(sources.DataSource):
67
68 dsname = 'IBMCloud'
69 system_uuid = None
70
71 def __init__(self, sys_cfg, distro, paths):
72 super(DataSourceIBMCloud, self).__init__(sys_cfg, distro, paths)
73 self.source = None
74 self._network_config = None
75 self.network_json = None
76 self.platform = None
77
78 def __str__(self):
79 root = super(DataSourceIBMCloud, self).__str__()
80 mstr = "%s [%s %s]" % (root, self.platform, self.source)
81 return mstr
82
83 def _get_data(self):
84 results = read_md()
85 if results is None:
86 return False
87
88 self.source = results['source']
89 self.platform = results['platform']
90 self.metadata = results['metadata']
91 self.userdata_raw = results.get('userdata')
92 self.network_json = results.get('networkdata')
93 vd = results.get('vendordata')
94 self.vendordata_pure = vd
95 self.system_uuid = results['system-uuid']
96 try:
97 self.vendordata_raw = sources.convert_vendordata(vd)
98 except ValueError as e:
99 LOG.warning("Invalid content in vendor-data: %s", e)
100 self.vendordata_raw = None
101
102 return True
103
104 def check_instance_id(self, sys_cfg):
105 """quickly (local check only) if self.instance_id is still valid
106
107 in Template mode, the system uuid (/sys/hypervisor/uuid) is the
108 same as found in the METADATA disk. But that is not true in OS_CODE
109 mode. So we read the system_uuid and keep that for later compare."""
110 if self.system_uuid is None:
111 return False
112 return self.system_uuid == _read_system_uuid()
113
114 @property
115 def network_config(self):
116 if self.platform != Platforms.OS_CODE:
117 # If deployed from template, an agent in the provisioning
118 # environment handles networking configuration. Not cloud-init.
119 return {'config': 'disabled', 'version': 1}
120 if self._network_config is None:
121 if self.network_json is not None:
122 LOG.debug("network config provided via network_json")
123 self._network_config = openstack.convert_net_json(
124 self.network_json, known_macs=None)
125 else:
126 LOG.debug("no network configuration available.")
127 return self._network_config
128
129
130def _read_system_uuid():
131 uuid_path = "/sys/hypervisor/uuid"
132 if not os.path.isfile(uuid_path):
133 return None
134 return util.load_file(uuid_path).strip().lower()
135
136
137def _is_xen():
138 return os.path.exists("/proc/xen")
139
140
141def _is_ibm_provisioning():
142 return os.path.exists("/root/provisioningConfiguration.cfg")
143
144
145def get_ibm_platform():
146 """Return a tuple (Platform, path)
147
148 If this is Not IBM cloud, then the return value is (None, None).
149 An instance in provisioning mode is considered running on IBM cloud."""
150 label_mdata = "METADATA"
151 label_cfg2 = "CONFIG-2"
152 not_found = (None, None)
153
154 if not _is_xen():
155 return not_found
156
157 # fslabels contains only the first entry with a given label.
158 fslabels = {}
159 try:
160 devs = util.blkid()
161 except util.ProcessExecutionError as e:
162 LOG.warning("Failed to run blkid: %s", e)
163 return (None, None)
164
165 for dev in sorted(devs.keys()):
166 data = devs[dev]
167 label = data.get("LABEL", "").upper()
168 uuid = data.get("UUID", "").upper()
169 if label not in (label_mdata, label_cfg2):
170 continue
171 if label in fslabels:
172 LOG.warning("Duplicate fslabel '%s'. existing=%s current=%s",
173 label, fslabels[label], data)
174 continue
175 if label == label_cfg2 and uuid != IBM_CONFIG_UUID:
176 LOG.debug("Skipping %s with LABEL=%s due to uuid != %s: %s",
177 dev, label, uuid, data)
178 continue
179 fslabels[label] = data
180
181 metadata_path = fslabels.get(label_mdata, {}).get('DEVNAME')
182 cfg2_path = fslabels.get(label_cfg2, {}).get('DEVNAME')
183
184 if cfg2_path:
185 return (Platforms.OS_CODE, cfg2_path)
186 elif metadata_path:
187 if _is_ibm_provisioning():
188 return (Platforms.TEMPLATE_PROVISIONING_METADATA, metadata_path)
189 else:
190 return (Platforms.TEMPLATE_LIVE_METADATA, metadata_path)
191 elif _is_ibm_provisioning():
192 return (Platforms.TEMPLATE_PROVISIONING_NODATA, None)
193 return not_found
194
195
196def read_md():
197 """Read data from IBM Cloud.
198
199 @return: None if not running on IBM Cloud.
200 dictionary with guaranteed fields: metadata, version
201 and optional fields: userdata, vendordata, networkdata.
202 Also includes the system uuid from /sys/hypervisor/uuid."""
203 platform, path = get_ibm_platform()
204 if platform is None:
205 LOG.debug("This is not an IBMCloud platform.")
206 return None
207 elif platform in PROVISIONING:
208 LOG.debug("Cloud-init is disabled during provisioning: %s.",
209 platform)
210 return None
211
212 ret = {'platform': platform, 'source': path,
213 'system-uuid': _read_system_uuid()}
214
215 try:
216 if os.path.isdir(path):
217 results = metadata_from_dir(path)
218 else:
219 results = util.mount_cb(path, metadata_from_dir)
220 except BrokenMetadata as e:
221 raise RuntimeError(
222 "Failed reading IBM config disk (platform=%s path=%s): %s" %
223 (platform, path, e))
224
225 ret.update(results)
226 return ret
227
228
229class BrokenMetadata(IOError):
230 pass
231
232
233def metadata_from_dir(source_dir):
234 """Walk source_dir extracting standardized metadata.
235
236 Certain metadata keys are renamed to present a standardized set of metadata
237 keys.
238
239 This function has a lot in common with ConfigDriveReader.read_v2 but
240 there are a number of inconsistencies, such key renames and as only
241 presenting a 'latest' version which make it an unlikely candidate to share
242 code.
243
244 @return: Dict containing translated metadata, userdata, vendordata,
245 networkdata as present.
246 """
247
248 def opath(fname):
249 return os.path.join("openstack", "latest", fname)
250
251 def load_json_bytes(blob):
252 return json.loads(blob.decode('utf-8'))
253
254 files = [
255 # tuples of (results_name, path, translator)
256 ('metadata_raw', opath('meta_data.json'), load_json_bytes),
257 ('userdata', opath('user_data'), None),
258 ('vendordata', opath('vendor_data.json'), load_json_bytes),
259 ('networkdata', opath('network_data.json'), load_json_bytes),
260 ]
261
262 results = {}
263 for (name, path, transl) in files:
264 fpath = os.path.join(source_dir, path)
265 raw = None
266 try:
267 raw = util.load_file(fpath, decode=False)
268 except IOError as e:
269 LOG.debug("Failed reading path '%s': %s", fpath, e)
270
271 if raw is None or transl is None:
272 data = raw
273 else:
274 try:
275 data = transl(raw)
276 except Exception as e:
277 raise BrokenMetadata("Failed decoding %s: %s" % (path, e))
278
279 results[name] = data
280
281 if results.get('metadata_raw') is None:
282 raise BrokenMetadata(
283 "%s missing required file 'meta_data.json'" % source_dir)
284
285 results['metadata'] = {}
286
287 md_raw = results['metadata_raw']
288 md = results['metadata']
289 if 'random_seed' in md_raw:
290 try:
291 md['random_seed'] = base64.b64decode(md_raw['random_seed'])
292 except (ValueError, TypeError) as e:
293 raise BrokenMetadata(
294 "Badly formatted metadata random_seed entry: %s" % e)
295
296 renames = (
297 ('public_keys', 'public-keys'), ('hostname', 'local-hostname'),
298 ('uuid', 'instance-id'))
299 for mdname, newname in renames:
300 if mdname in md_raw:
301 md[newname] = md_raw[mdname]
302
303 return results
304
305
306# Used to match classes to dependencies
307datasources = [
308 (DataSourceIBMCloud, (sources.DEP_FILESYSTEM,)),
309]
310
311
312# Return a list of data sources that match this set of dependencies
313def get_datasource_list(depends):
314 return sources.list_from_depends(depends, datasources)
315
316
317if __name__ == "__main__":
318 import argparse
319
320 parser = argparse.ArgumentParser(description='Query IBM Cloud Metadata')
321 args = parser.parse_args()
322 data = read_md()
323 print(util.json_dumps(data))
324
325# vi: ts=4 expandtab
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 6e62f98..dc914a7 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -95,11 +95,20 @@ class DataSourceOVF(sources.DataSource):
95 "VMware Customization support")95 "VMware Customization support")
96 elif not util.get_cfg_option_bool(96 elif not util.get_cfg_option_bool(
97 self.sys_cfg, "disable_vmware_customization", True):97 self.sys_cfg, "disable_vmware_customization", True):
98 deployPkgPluginPath = search_file("/usr/lib/vmware-tools",98
99 "libdeployPkgPlugin.so")99 search_paths = (
100 if not deployPkgPluginPath:100 "/usr/lib/vmware-tools", "/usr/lib64/vmware-tools",
101 deployPkgPluginPath = search_file("/usr/lib/open-vm-tools",101 "/usr/lib/open-vm-tools", "/usr/lib64/open-vm-tools")
102 "libdeployPkgPlugin.so")102
103 plugin = "libdeployPkgPlugin.so"
104 deployPkgPluginPath = None
105 for path in search_paths:
106 deployPkgPluginPath = search_file(path, plugin)
107 if deployPkgPluginPath:
108 LOG.debug("Found the customization plugin at %s",
109 deployPkgPluginPath)
110 break
111
103 if deployPkgPluginPath:112 if deployPkgPluginPath:
104 # When the VM is powered on, the "VMware Tools" daemon113 # When the VM is powered on, the "VMware Tools" daemon
105 # copies the customization specification file to114 # copies the customization specification file to
@@ -111,6 +120,8 @@ class DataSourceOVF(sources.DataSource):
111 msg="waiting for configuration file",120 msg="waiting for configuration file",
112 func=wait_for_imc_cfg_file,121 func=wait_for_imc_cfg_file,
113 args=("cust.cfg", max_wait))122 args=("cust.cfg", max_wait))
123 else:
124 LOG.debug("Did not find the customization plugin.")
114125
115 if vmwareImcConfigFilePath:126 if vmwareImcConfigFilePath:
116 LOG.debug("Found VMware Customization Config File at %s",127 LOG.debug("Found VMware Customization Config File at %s",
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index ce47b6b..d4a4111 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -20,7 +20,6 @@ import string
2020
21from cloudinit import log as logging21from cloudinit import log as logging
22from cloudinit import net22from cloudinit import net
23from cloudinit.net import eni
24from cloudinit import sources23from cloudinit import sources
25from cloudinit import util24from cloudinit import util
2625
@@ -91,19 +90,19 @@ class DataSourceOpenNebula(sources.DataSource):
91 return False90 return False
9291
93 self.seed = seed92 self.seed = seed
94 self.network_eni = results.get('network-interfaces')93 self.network = results.get('network-interfaces')
95 self.metadata = md94 self.metadata = md
96 self.userdata_raw = results.get('userdata')95 self.userdata_raw = results.get('userdata')
97 return True96 return True
9897
99 @property98 @property
100 def network_config(self):99 def network_config(self):
101 if self.network_eni is not None:100 if self.network is not None:
102 return eni.convert_eni_data(self.network_eni)101 return self.network
103 else:102 else:
104 return None103 return None
105104
106 def get_hostname(self, fqdn=False, resolve_ip=None):105 def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False):
107 if resolve_ip is None:106 if resolve_ip is None:
108 if self.dsmode == sources.DSMODE_NETWORK:107 if self.dsmode == sources.DSMODE_NETWORK:
109 resolve_ip = True108 resolve_ip = True
@@ -143,18 +142,42 @@ class OpenNebulaNetwork(object):
143 def mac2network(self, mac):142 def mac2network(self, mac):
144 return self.mac2ip(mac).rpartition(".")[0] + ".0"143 return self.mac2ip(mac).rpartition(".")[0] + ".0"
145144
146 def get_dns(self, dev):145 def get_nameservers(self, dev):
147 return self.get_field(dev, "dns", "").split()146 nameservers = {}
147 dns = self.get_field(dev, "dns", "").split()
148 dns.extend(self.context.get('DNS', "").split())
149 if dns:
150 nameservers['addresses'] = dns
151 search_domain = self.get_field(dev, "search_domain", "").split()
152 if search_domain:
153 nameservers['search'] = search_domain
154 return nameservers
148155
149 def get_domain(self, dev):156 def get_mtu(self, dev):
150 return self.get_field(dev, "domain")157 return self.get_field(dev, "mtu")
151158
152 def get_ip(self, dev, mac):159 def get_ip(self, dev, mac):
153 return self.get_field(dev, "ip", self.mac2ip(mac))160 return self.get_field(dev, "ip", self.mac2ip(mac))
154161
162 def get_ip6(self, dev):
163 addresses6 = []
164 ip6 = self.get_field(dev, "ip6")
165 if ip6:
166 addresses6.append(ip6)
167 ip6_ula = self.get_field(dev, "ip6_ula")
168 if ip6_ula:
169 addresses6.append(ip6_ula)
170 return addresses6
171
172 def get_ip6_prefix(self, dev):
173 return self.get_field(dev, "ip6_prefix_length", "64")
174
155 def get_gateway(self, dev):175 def get_gateway(self, dev):
156 return self.get_field(dev, "gateway")176 return self.get_field(dev, "gateway")
157177
178 def get_gateway6(self, dev):
179 return self.get_field(dev, "gateway6")
180
158 def get_mask(self, dev):181 def get_mask(self, dev):
159 return self.get_field(dev, "mask", "255.255.255.0")182 return self.get_field(dev, "mask", "255.255.255.0")
160183
@@ -171,13 +194,11 @@ class OpenNebulaNetwork(object):
171 return default if val in (None, "") else val194 return default if val in (None, "") else val
172195
173 def gen_conf(self):196 def gen_conf(self):
174 global_dns = self.context.get('DNS', "").split()197 netconf = {}
175198 netconf['version'] = 2
176 conf = []199 netconf['ethernets'] = {}
177 conf.append('auto lo')
178 conf.append('iface lo inet loopback')
179 conf.append('')
180200
201 ethernets = {}
181 for mac, dev in self.ifaces.items():202 for mac, dev in self.ifaces.items():
182 mac = mac.lower()203 mac = mac.lower()
183204
@@ -185,29 +206,49 @@ class OpenNebulaNetwork(object):
185 # dev stores the current system name.206 # dev stores the current system name.
186 c_dev = self.context_devname.get(mac, dev)207 c_dev = self.context_devname.get(mac, dev)
187208
188 conf.append('auto ' + dev)209 devconf = {}
189 conf.append('iface ' + dev + ' inet static')210
190 conf.append(' #hwaddress %s' % mac)211 # Set MAC address
191 conf.append(' address ' + self.get_ip(c_dev, mac))212 devconf['match'] = {'macaddress': mac}
192 conf.append(' network ' + self.get_network(c_dev, mac))
193 conf.append(' netmask ' + self.get_mask(c_dev))
194213
214 # Set IPv4 address
215 devconf['addresses'] = []
216 mask = self.get_mask(c_dev)
217 prefix = str(net.mask_to_net_prefix(mask))
218 devconf['addresses'].append(
219 self.get_ip(c_dev, mac) + '/' + prefix)
220
221 # Set IPv6 Global and ULA address
222 addresses6 = self.get_ip6(c_dev)
223 if addresses6:
224 prefix6 = self.get_ip6_prefix(c_dev)
225 devconf['addresses'].extend(
226 [i + '/' + prefix6 for i in addresses6])
227
228 # Set IPv4 default gateway
195 gateway = self.get_gateway(c_dev)229 gateway = self.get_gateway(c_dev)
196 if gateway:230 if gateway:
197 conf.append(' gateway ' + gateway)231 devconf['gateway4'] = gateway
232
233 # Set IPv6 default gateway
234 gateway6 = self.get_gateway6(c_dev)
235 if gateway:
236 devconf['gateway6'] = gateway6
198237
199 domain = self.get_domain(c_dev)238 # Set DNS servers and search domains
200 if domain:239 nameservers = self.get_nameservers(c_dev)
201 conf.append(' dns-search ' + domain)240 if nameservers:
241 devconf['nameservers'] = nameservers
202242
203 # add global DNS servers to all interfaces243 # Set MTU size
204 dns = self.get_dns(c_dev)244 mtu = self.get_mtu(c_dev)
205 if global_dns or dns:245 if mtu:
206 conf.append(' dns-nameservers ' + ' '.join(global_dns + dns))246 devconf['mtu'] = mtu
207247
208 conf.append('')248 ethernets[dev] = devconf
209249
210 return "\n".join(conf)250 netconf['ethernets'] = ethernets
251 return(netconf)
211252
212253
213def find_candidate_devs():254def find_candidate_devs():
@@ -393,10 +434,10 @@ def read_context_disk_dir(source_dir, asuser=None):
393 except TypeError:434 except TypeError:
394 LOG.warning("Failed base64 decoding of userdata")435 LOG.warning("Failed base64 decoding of userdata")
395436
396 # generate static /etc/network/interfaces437 # generate Network Configuration v2
397 # only if there are any required context variables438 # only if there are any required context variables
398 # http://opennebula.org/documentation:rel3.8:cong#network_configuration439 # http://docs.opennebula.org/5.4/operation/references/template.html#context-section
399 ipaddr_keys = [k for k in context if re.match(r'^ETH\d+_IP$', k)]440 ipaddr_keys = [k for k in context if re.match(r'^ETH\d+_IP.*$', k)]
400 if ipaddr_keys:441 if ipaddr_keys:
401 onet = OpenNebulaNetwork(context)442 onet = OpenNebulaNetwork(context)
402 results['network-interfaces'] = onet.gen_conf()443 results['network-interfaces'] = onet.gen_conf()
diff --git a/cloudinit/sources/DataSourceScaleway.py b/cloudinit/sources/DataSourceScaleway.py
index b0b19c9..e2502b0 100644
--- a/cloudinit/sources/DataSourceScaleway.py
+++ b/cloudinit/sources/DataSourceScaleway.py
@@ -113,9 +113,9 @@ def query_data_api_once(api_address, timeout, requests_session):
113 retries=0,113 retries=0,
114 session=requests_session,114 session=requests_session,
115 # If the error is a HTTP/404 or a ConnectionError, go into raise115 # If the error is a HTTP/404 or a ConnectionError, go into raise
116 # block below.116 # block below and don't bother retrying.
117 exception_cb=lambda _, exc: exc.code == 404 or (117 exception_cb=lambda _, exc: exc.code != 404 and (
118 isinstance(exc.cause, requests.exceptions.ConnectionError)118 not isinstance(exc.cause, requests.exceptions.ConnectionError)
119 )119 )
120 )120 )
121 return util.decode_binary(resp.contents)121 return util.decode_binary(resp.contents)
@@ -215,7 +215,7 @@ class DataSourceScaleway(sources.DataSource):
215 def get_public_ssh_keys(self):215 def get_public_ssh_keys(self):
216 return [key['key'] for key in self.metadata['ssh_public_keys']]216 return [key['key'] for key in self.metadata['ssh_public_keys']]
217217
218 def get_hostname(self, fqdn=False, resolve_ip=False):218 def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False):
219 return self.metadata['hostname']219 return self.metadata['hostname']
220220
221 @property221 @property
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index a05ca2f..df0b374 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -276,21 +276,34 @@ class DataSource(object):
276 return "iid-datasource"276 return "iid-datasource"
277 return str(self.metadata['instance-id'])277 return str(self.metadata['instance-id'])
278278
279 def get_hostname(self, fqdn=False, resolve_ip=False):279 def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False):
280 """Get hostname or fqdn from the datasource. Look it up if desired.
281
282 @param fqdn: Boolean, set True to return hostname with domain.
283 @param resolve_ip: Boolean, set True to attempt to resolve an ipv4
284 address provided in local-hostname meta-data.
285 @param metadata_only: Boolean, set True to avoid looking up hostname
286 if meta-data doesn't have local-hostname present.
287
288 @return: hostname or qualified hostname. Optionally return None when
289 metadata_only is True and local-hostname data is not available.
290 """
280 defdomain = "localdomain"291 defdomain = "localdomain"
281 defhost = "localhost"292 defhost = "localhost"
282 domain = defdomain293 domain = defdomain
283294
284 if not self.metadata or 'local-hostname' not in self.metadata:295 if not self.metadata or 'local-hostname' not in self.metadata:
296 if metadata_only:
297 return None
285 # this is somewhat questionable really.298 # this is somewhat questionable really.
286 # the cloud datasource was asked for a hostname299 # the cloud datasource was asked for a hostname
287 # and didn't have one. raising error might be more appropriate300 # and didn't have one. raising error might be more appropriate
288 # but instead, basically look up the existing hostname301 # but instead, basically look up the existing hostname
289 toks = []302 toks = []
290 hostname = util.get_hostname()303 hostname = util.get_hostname()
291 fqdn = util.get_fqdn_from_hosts(hostname)304 hosts_fqdn = util.get_fqdn_from_hosts(hostname)
292 if fqdn and fqdn.find(".") > 0:305 if hosts_fqdn and hosts_fqdn.find(".") > 0:
293 toks = str(fqdn).split(".")306 toks = str(hosts_fqdn).split(".")
294 elif hostname and hostname.find(".") > 0:307 elif hostname and hostname.find(".") > 0:
295 toks = str(hostname).split(".")308 toks = str(hostname).split(".")
296 elif hostname:309 elif hostname:
diff --git a/cloudinit/sources/helpers/hetzner.py b/cloudinit/sources/helpers/hetzner.py
297new file mode 100644310new file mode 100644
index 0000000..2554530
--- /dev/null
+++ b/cloudinit/sources/helpers/hetzner.py
@@ -0,0 +1,26 @@
1# Author: Jonas Keidel <jonas.keidel@hetzner.com>
2# Author: Markus Schade <markus.schade@hetzner.com>
3#
4# This file is part of cloud-init. See LICENSE file for license information.
5
6from cloudinit import log as logging
7from cloudinit import url_helper
8from cloudinit import util
9
10LOG = logging.getLogger(__name__)
11
12
13def read_metadata(url, timeout=2, sec_between=2, retries=30):
14 response = url_helper.readurl(url, timeout=timeout,
15 sec_between=sec_between, retries=retries)
16 if not response.ok():
17 raise RuntimeError("unable to read metadata at %s" % url)
18 return util.load_yaml(response.contents.decode())
19
20
21def read_userdata(url, timeout=2, sec_between=2, retries=30):
22 response = url_helper.readurl(url, timeout=timeout,
23 sec_between=sec_between, retries=retries)
24 if not response.ok():
25 raise RuntimeError("unable to read userdata at %s" % url)
26 return response.contents
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index af15115..e7fda22 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -1,13 +1,15 @@
1# This file is part of cloud-init. See LICENSE file for license information.1# This file is part of cloud-init. See LICENSE file for license information.
22
3import inspect
3import os4import os
4import six5import six
5import stat6import stat
67
7from cloudinit.helpers import Paths8from cloudinit.helpers import Paths
9from cloudinit import importer
8from cloudinit.sources import (10from cloudinit.sources import (
9 INSTANCE_JSON_FILE, DataSource)11 INSTANCE_JSON_FILE, DataSource)
10from cloudinit.tests.helpers import CiTestCase, skipIf12from cloudinit.tests.helpers import CiTestCase, skipIf, mock
11from cloudinit.user_data import UserDataProcessor13from cloudinit.user_data import UserDataProcessor
12from cloudinit import util14from cloudinit import util
1315
@@ -108,6 +110,74 @@ class TestDataSource(CiTestCase):
108 self.assertEqual('userdata_raw', datasource.userdata_raw)110 self.assertEqual('userdata_raw', datasource.userdata_raw)
109 self.assertEqual('vendordata_raw', datasource.vendordata_raw)111 self.assertEqual('vendordata_raw', datasource.vendordata_raw)
110112
113 def test_get_hostname_strips_local_hostname_without_domain(self):
114 """Datasource.get_hostname strips metadata local-hostname of domain."""
115 tmp = self.tmp_dir()
116 datasource = DataSourceTestSubclassNet(
117 self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
118 self.assertTrue(datasource.get_data())
119 self.assertEqual(
120 'test-subclass-hostname', datasource.metadata['local-hostname'])
121 self.assertEqual('test-subclass-hostname', datasource.get_hostname())
122 datasource.metadata['local-hostname'] = 'hostname.my.domain.com'
123 self.assertEqual('hostname', datasource.get_hostname())
124
125 def test_get_hostname_with_fqdn_returns_local_hostname_with_domain(self):
126 """Datasource.get_hostname with fqdn set gets qualified hostname."""
127 tmp = self.tmp_dir()
128 datasource = DataSourceTestSubclassNet(
129 self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
130 self.assertTrue(datasource.get_data())
131 datasource.metadata['local-hostname'] = 'hostname.my.domain.com'
132 self.assertEqual(
133 'hostname.my.domain.com', datasource.get_hostname(fqdn=True))
134
135 def test_get_hostname_without_metadata_uses_system_hostname(self):
136 """Datasource.gethostname runs util.get_hostname when no metadata."""
137 tmp = self.tmp_dir()
138 datasource = DataSourceTestSubclassNet(
139 self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
140 self.assertEqual({}, datasource.metadata)
141 mock_fqdn = 'cloudinit.sources.util.get_fqdn_from_hosts'
142 with mock.patch('cloudinit.sources.util.get_hostname') as m_gethost:
143 with mock.patch(mock_fqdn) as m_fqdn:
144 m_gethost.return_value = 'systemhostname.domain.com'
145 m_fqdn.return_value = None # No maching fqdn in /etc/hosts
146 self.assertEqual('systemhostname', datasource.get_hostname())
147 self.assertEqual(
148 'systemhostname.domain.com',
149 datasource.get_hostname(fqdn=True))
150
151 def test_get_hostname_without_metadata_returns_none(self):
152 """Datasource.gethostname returns None when metadata_only and no MD."""
153 tmp = self.tmp_dir()
154 datasource = DataSourceTestSubclassNet(
155 self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
156 self.assertEqual({}, datasource.metadata)
157 mock_fqdn = 'cloudinit.sources.util.get_fqdn_from_hosts'
158 with mock.patch('cloudinit.sources.util.get_hostname') as m_gethost:
159 with mock.patch(mock_fqdn) as m_fqdn:
160 self.assertIsNone(datasource.get_hostname(metadata_only=True))
161 self.assertIsNone(
162 datasource.get_hostname(fqdn=True, metadata_only=True))
163 self.assertEqual([], m_gethost.call_args_list)
164 self.assertEqual([], m_fqdn.call_args_list)
165
166 def test_get_hostname_without_metadata_prefers_etc_hosts(self):
167 """Datasource.gethostname prefers /etc/hosts to util.get_hostname."""
168 tmp = self.tmp_dir()
169 datasource = DataSourceTestSubclassNet(
170 self.sys_cfg, self.distro, Paths({'run_dir': tmp}))
171 self.assertEqual({}, datasource.metadata)
172 mock_fqdn = 'cloudinit.sources.util.get_fqdn_from_hosts'
173 with mock.patch('cloudinit.sources.util.get_hostname') as m_gethost:
174 with mock.patch(mock_fqdn) as m_fqdn:
175 m_gethost.return_value = 'systemhostname.domain.com'
176 m_fqdn.return_value = 'fqdnhostname.domain.com'
177 self.assertEqual('fqdnhostname', datasource.get_hostname())
178 self.assertEqual('fqdnhostname.domain.com',
179 datasource.get_hostname(fqdn=True))
180
111 def test_get_data_write_json_instance_data(self):181 def test_get_data_write_json_instance_data(self):
112 """get_data writes INSTANCE_JSON_FILE to run_dir as readonly root."""182 """get_data writes INSTANCE_JSON_FILE to run_dir as readonly root."""
113 tmp = self.tmp_dir()183 tmp = self.tmp_dir()
@@ -200,3 +270,29 @@ class TestDataSource(CiTestCase):
200 "WARNING: Error persisting instance-data.json: 'utf8' codec can't"270 "WARNING: Error persisting instance-data.json: 'utf8' codec can't"
201 " decode byte 0xaa in position 2: invalid start byte",271 " decode byte 0xaa in position 2: invalid start byte",
202 self.logs.getvalue())272 self.logs.getvalue())
273
274 def test_get_hostname_subclass_support(self):
275 """Validate get_hostname signature on all subclasses of DataSource."""
276 # Use inspect.getfullargspec when we drop py2.6 and py2.7
277 get_args = inspect.getargspec # pylint: disable=W1505
278 base_args = get_args(DataSource.get_hostname) # pylint: disable=W1505
279 # Import all DataSource subclasses so we can inspect them.
280 modules = util.find_modules(os.path.dirname(os.path.dirname(__file__)))
281 for loc, name in modules.items():
282 mod_locs, _ = importer.find_module(name, ['cloudinit.sources'], [])
283 if mod_locs:
284 importer.import_module(mod_locs[0])
285 for child in DataSource.__subclasses__():
286 if 'Test' in child.dsname:
287 continue
288 self.assertEqual(
289 base_args,
290 get_args(child.get_hostname), # pylint: disable=W1505
291 '%s does not implement DataSource.get_hostname params'
292 % child)
293 for grandchild in child.__subclasses__():
294 self.assertEqual(
295 base_args,
296 get_args(grandchild.get_hostname), # pylint: disable=W1505
297 '%s does not implement DataSource.get_hostname params'
298 % grandchild)
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index d045268..bc4ebc8 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -132,8 +132,7 @@ class Init(object):
132 return initial_dirs132 return initial_dirs
133133
134 def purge_cache(self, rm_instance_lnk=False):134 def purge_cache(self, rm_instance_lnk=False):
135 rm_list = []135 rm_list = [self.paths.boot_finished]
136 rm_list.append(self.paths.boot_finished)
137 if rm_instance_lnk:136 if rm_instance_lnk:
138 rm_list.append(self.paths.instance_link)137 rm_list.append(self.paths.instance_link)
139 for f in rm_list:138 for f in rm_list:
diff --git a/cloudinit/subp.py b/cloudinit/subp.py
140new file mode 100644139new file mode 100644
index 0000000..0ad0930
--- /dev/null
+++ b/cloudinit/subp.py
@@ -0,0 +1,57 @@
1# This file is part of cloud-init. See LICENSE file for license information.
2"""Common utility functions for interacting with subprocess."""
3
4# TODO move subp shellify and runparts related functions out of util.py
5
6import logging
7
8LOG = logging.getLogger(__name__)
9
10
11def prepend_base_command(base_command, commands):
12 """Ensure user-provided commands start with base_command; warn otherwise.
13
14 Each command is either a list or string. Perform the following:
15 - If the command is a list, pop the first element if it is None
16 - If the command is a list, insert base_command as the first element if
17 not present.
18 - When the command is a string not starting with 'base-command', warn.
19
20 Allow flexibility to provide non-base-command environment/config setup if
21 needed.
22
23 @commands: List of commands. Each command element is a list or string.
24
25 @return: List of 'fixed up' commands.
26 @raise: TypeError on invalid config item type.
27 """
28 warnings = []
29 errors = []
30 fixed_commands = []
31 for command in commands:
32 if isinstance(command, list):
33 if command[0] is None: # Avoid warnings by specifying None
34 command = command[1:]
35 elif command[0] != base_command: # Automatically prepend
36 command.insert(0, base_command)
37 elif isinstance(command, str):
38 if not command.startswith('%s ' % base_command):
39 warnings.append(command)
40 else:
41 errors.append(str(command))
42 continue
43 fixed_commands.append(command)
44
45 if warnings:
46 LOG.warning(
47 'Non-%s commands in %s config:\n%s',
48 base_command, base_command, '\n'.join(warnings))
49 if errors:
50 raise TypeError(
51 'Invalid {name} config.'
52 ' These commands are not a string or list:\n{errors}'.format(
53 name=base_command, errors='\n'.join(errors)))
54 return fixed_commands
55
56
57# vi: ts=4 expandtab
diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py
index 0080c72..999b1d7 100644
--- a/cloudinit/tests/helpers.py
+++ b/cloudinit/tests/helpers.py
@@ -173,17 +173,15 @@ class CiTestCase(TestCase):
173 dir = self.tmp_dir()173 dir = self.tmp_dir()
174 return os.path.normpath(os.path.abspath(os.path.join(dir, path)))174 return os.path.normpath(os.path.abspath(os.path.join(dir, path)))
175175
176 def assertRaisesCodeEqual(self, expected, found):176 def sys_exit(self, code):
177 """Handle centos6 having different context manager for assertRaises.177 """Provide a wrapper around sys.exit for python 2.6
178 with assertRaises(Exception) as e:178
179 raise Exception("BOO")179 In 2.6, this code would produce 'cm.exception' with value int(2)
180180 rather than the SystemExit that was raised by sys.exit(2).
181 centos6 will have e.exception as an integer.181 with assertRaises(SystemExit) as cm:
182 anything nwere will have it as something with a '.code'"""182 sys.exit(2)
183 if isinstance(found, int):183 """
184 self.assertEqual(expected, found)184 raise SystemExit(code)
185 else:
186 self.assertEqual(expected, found.code)
187185
188186
189class ResourceUsingTestCase(CiTestCase):187class ResourceUsingTestCase(CiTestCase):
@@ -285,10 +283,15 @@ class FilesystemMockingTestCase(ResourceUsingTestCase):
285 def patchOS(self, new_root):283 def patchOS(self, new_root):
286 patch_funcs = {284 patch_funcs = {
287 os.path: [('isfile', 1), ('exists', 1),285 os.path: [('isfile', 1), ('exists', 1),
288 ('islink', 1), ('isdir', 1)],286 ('islink', 1), ('isdir', 1), ('lexists', 1)],
289 os: [('listdir', 1), ('mkdir', 1),287 os: [('listdir', 1), ('mkdir', 1),
290 ('lstat', 1), ('symlink', 2)],288 ('lstat', 1), ('symlink', 2)]
291 }289 }
290
291 if hasattr(os, 'scandir'):
292 # py27 does not have scandir
293 patch_funcs[os].append(('scandir', 1))
294
292 for (mod, funcs) in patch_funcs.items():295 for (mod, funcs) in patch_funcs.items():
293 for f, nargs in funcs:296 for f, nargs in funcs:
294 func = getattr(mod, f)297 func = getattr(mod, f)
@@ -411,6 +414,19 @@ except AttributeError:
411 return decorator414 return decorator
412415
413416
417try:
418 import jsonschema
419 assert jsonschema # avoid pyflakes error F401: import unused
420 _missing_jsonschema_dep = False
421except ImportError:
422 _missing_jsonschema_dep = True
423
424
425def skipUnlessJsonSchema():
426 return skipIf(
427 _missing_jsonschema_dep, "No python-jsonschema dependency present.")
428
429
414# older versions of mock do not have the useful 'assert_not_called'430# older versions of mock do not have the useful 'assert_not_called'
415if not hasattr(mock.Mock, 'assert_not_called'):431if not hasattr(mock.Mock, 'assert_not_called'):
416 def __mock_assert_not_called(mmock):432 def __mock_assert_not_called(mmock):
@@ -422,12 +438,12 @@ if not hasattr(mock.Mock, 'assert_not_called'):
422 mock.Mock.assert_not_called = __mock_assert_not_called438 mock.Mock.assert_not_called = __mock_assert_not_called
423439
424440
425# older unittest2.TestCase (centos6) do not have assertRaisesRegex441# older unittest2.TestCase (centos6) have only the now-deprecated
426# And setting assertRaisesRegex to assertRaisesRegexp causes442# assertRaisesRegexp. Simple assignment makes pylint complain, about
427# https://github.com/PyCQA/pylint/issues/1653 . So the workaround.443# users of assertRaisesRegex so we use getattr to trick it.
444# https://github.com/PyCQA/pylint/issues/1946
428if not hasattr(unittest2.TestCase, 'assertRaisesRegex'):445if not hasattr(unittest2.TestCase, 'assertRaisesRegex'):
429 def _tricky(*args, **kwargs):446 unittest2.TestCase.assertRaisesRegex = (
430 return unittest2.TestCase.assertRaisesRegexp447 getattr(unittest2.TestCase, 'assertRaisesRegexp'))
431 unittest2.TestCase.assertRaisesRegex = _tricky
432448
433# vi: ts=4 expandtab449# vi: ts=4 expandtab
diff --git a/cloudinit/tests/test_subp.py b/cloudinit/tests/test_subp.py
434new file mode 100644450new file mode 100644
index 0000000..448097d
--- /dev/null
+++ b/cloudinit/tests/test_subp.py
@@ -0,0 +1,61 @@
1# This file is part of cloud-init. See LICENSE file for license information.
2
3"""Tests for cloudinit.subp utility functions"""
4
5from cloudinit import subp
6from cloudinit.tests.helpers import CiTestCase
7
8
9class TestPrependBaseCommands(CiTestCase):
10
11 with_logs = True
12
13 def test_prepend_base_command_errors_on_neither_string_nor_list(self):
14 """Raise an error for each command which is not a string or list."""
15 orig_commands = ['ls', 1, {'not': 'gonna work'}, ['basecmd', 'list']]
16 with self.assertRaises(TypeError) as context_manager:
17 subp.prepend_base_command(
18 base_command='basecmd', commands=orig_commands)
19 self.assertEqual(
20 "Invalid basecmd config. These commands are not a string or"
21 " list:\n1\n{'not': 'gonna work'}",
22 str(context_manager.exception))
23
24 def test_prepend_base_command_warns_on_non_base_string_commands(self):
25 """Warn on each non-base for commands of type string."""
26 orig_commands = [
27 'ls', 'basecmd list', 'touch /blah', 'basecmd install x']
28 fixed_commands = subp.prepend_base_command(
29 base_command='basecmd', commands=orig_commands)
30 self.assertEqual(
31 'WARNING: Non-basecmd commands in basecmd config:\n'
32 'ls\ntouch /blah\n',
33 self.logs.getvalue())
34 self.assertEqual(orig_commands, fixed_commands)
35
36 def test_prepend_base_command_prepends_on_non_base_list_commands(self):
37 """Prepend 'basecmd' for each non-basecmd command of type list."""
38 orig_commands = [['ls'], ['basecmd', 'list'], ['basecmda', '/blah'],
39 ['basecmd', 'install', 'x']]
40 expected = [['basecmd', 'ls'], ['basecmd', 'list'],
41 ['basecmd', 'basecmda', '/blah'],
42 ['basecmd', 'install', 'x']]
43 fixed_commands = subp.prepend_base_command(
44 base_command='basecmd', commands=orig_commands)
45 self.assertEqual('', self.logs.getvalue())
46 self.assertEqual(expected, fixed_commands)
47
48 def test_prepend_base_command_removes_first_item_when_none(self):
49 """Remove the first element of a non-basecmd when it is None."""
50 orig_commands = [[None, 'ls'], ['basecmd', 'list'],
51 [None, 'touch', '/blah'],
52 ['basecmd', 'install', 'x']]
53 expected = [['ls'], ['basecmd', 'list'],
54 ['touch', '/blah'],
55 ['basecmd', 'install', 'x']]
56 fixed_commands = subp.prepend_base_command(
57 base_command='basecmd', commands=orig_commands)
58 self.assertEqual('', self.logs.getvalue())
59 self.assertEqual(expected, fixed_commands)
60
61# vi: ts=4 expandtab
diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py
index ba6bf69..3f37dbb 100644
--- a/cloudinit/tests/test_util.py
+++ b/cloudinit/tests/test_util.py
@@ -3,6 +3,7 @@
3"""Tests for cloudinit.util"""3"""Tests for cloudinit.util"""
44
5import logging5import logging
6from textwrap import dedent
67
7import cloudinit.util as util8import cloudinit.util as util
89
@@ -16,6 +17,25 @@ MOUNT_INFO = [
16]17]
1718
1819
20class FakeCloud(object):
21
22 def __init__(self, hostname, fqdn):
23 self.hostname = hostname
24 self.fqdn = fqdn
25 self.calls = []
26
27 def get_hostname(self, fqdn=None, metadata_only=None):
28 myargs = {}
29 if fqdn is not None:
30 myargs['fqdn'] = fqdn
31 if metadata_only is not None:
32 myargs['metadata_only'] = metadata_only
33 self.calls.append(myargs)
34 if fqdn:
35 return self.fqdn
36 return self.hostname
37
38
19class TestUtil(CiTestCase):39class TestUtil(CiTestCase):
2040
21 def test_parse_mount_info_no_opts_no_arg(self):41 def test_parse_mount_info_no_opts_no_arg(self):
@@ -44,3 +64,152 @@ class TestUtil(CiTestCase):
44 m_mount_info.return_value = ('/dev/sda1', 'btrfs', '/', 'ro,relatime')64 m_mount_info.return_value = ('/dev/sda1', 'btrfs', '/', 'ro,relatime')
45 is_rw = util.mount_is_read_write('/')65 is_rw = util.mount_is_read_write('/')
46 self.assertEqual(is_rw, False)66 self.assertEqual(is_rw, False)
67
68
69class TestShellify(CiTestCase):
70
71 def test_input_dict_raises_type_error(self):
72 self.assertRaisesRegex(
73 TypeError, 'Input.*was.*dict.*xpected',
74 util.shellify, {'mykey': 'myval'})
75
76 def test_input_str_raises_type_error(self):
77 self.assertRaisesRegex(
78 TypeError, 'Input.*was.*str.*xpected', util.shellify, "foobar")
79
80 def test_value_with_int_raises_type_error(self):
81 self.assertRaisesRegex(
82 TypeError, 'shellify.*int', util.shellify, ["foo", 1])
83
84 def test_supports_strings_and_lists(self):
85 self.assertEqual(
86 '\n'.join(["#!/bin/sh", "echo hi mom", "'echo' 'hi dad'",
87 "'echo' 'hi' 'sis'", ""]),
88 util.shellify(["echo hi mom", ["echo", "hi dad"],
89 ('echo', 'hi', 'sis')]))
90
91
92class TestGetHostnameFqdn(CiTestCase):
93
94 def test_get_hostname_fqdn_from_only_cfg_fqdn(self):
95 """When cfg only has the fqdn key, derive hostname and fqdn from it."""
96 hostname, fqdn = util.get_hostname_fqdn(
97 cfg={'fqdn': 'myhost.domain.com'}, cloud=None)
98 self.assertEqual('myhost', hostname)
99 self.assertEqual('myhost.domain.com', fqdn)
100
101 def test_get_hostname_fqdn_from_cfg_fqdn_and_hostname(self):
102 """When cfg has both fqdn and hostname keys, return them."""
103 hostname, fqdn = util.get_hostname_fqdn(
104 cfg={'fqdn': 'myhost.domain.com', 'hostname': 'other'}, cloud=None)
105 self.assertEqual('other', hostname)
106 self.assertEqual('myhost.domain.com', fqdn)
107
108 def test_get_hostname_fqdn_from_cfg_hostname_with_domain(self):
109 """When cfg has only hostname key which represents a fqdn, use that."""
110 hostname, fqdn = util.get_hostname_fqdn(
111 cfg={'hostname': 'myhost.domain.com'}, cloud=None)
112 self.assertEqual('myhost', hostname)
113 self.assertEqual('myhost.domain.com', fqdn)
114