Merge ~chad.smith/cloud-init:ubuntu/devel into cloud-init:ubuntu/devel

Proposed by Chad Smith on 2018-03-01
Status: Merged
Merged at revision: 964c869024b422c3966975cc6d23981537084b0a
Proposed branch: ~chad.smith/cloud-init:ubuntu/devel
Merge into: cloud-init:ubuntu/devel
Diff against target: 371 lines (+141/-27)
10 files modified
cloudinit/config/cc_puppet.py (+40/-14)
cloudinit/config/cc_salt_minion.py (+9/-0)
cloudinit/sources/DataSourceGCE.py (+7/-8)
cloudinit/util.py (+7/-2)
debian/changelog (+13/-0)
doc/examples/cloud-config-chef.txt (+2/-2)
tests/cloud_tests/testcases/modules/salt_minion.py (+5/-0)
tests/cloud_tests/testcases/modules/salt_minion.yaml (+5/-0)
tests/unittests/test_datasource/test_gce.py (+19/-1)
tests/unittests/test_util.py (+34/-0)
Reviewer Review Type Date Requested Status
Server Team CI bot continuous-integration Approve on 2018-03-01
Scott Moser 2018-03-01 Pending
Review via email: mp+340252@code.launchpad.net

Description of the change

Sync tip of master to pull in GCE fix for LP: #1752711 and other changes for Bionic

To post a comment you must log in.

PASSED: Continuous integration, rev:964c869024b422c3966975cc6d23981537084b0a
https://jenkins.ubuntu.com/server/job/cloud-init-ci/804/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/804/rebuild

review: Approve (continuous-integration)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py
index 28b1d56..57a170f 100644
--- a/cloudinit/config/cc_puppet.py
+++ b/cloudinit/config/cc_puppet.py
@@ -21,6 +21,13 @@ under ``version``, and defaults to ``none``, which selects the latest version
21in the repos. If the ``puppet`` config key exists in the config archive, this21in the repos. If the ``puppet`` config key exists in the config archive, this
22module will attempt to start puppet even if no installation was performed.22module will attempt to start puppet even if no installation was performed.
2323
24The module also provides keys for configuring the new puppet 4 paths and
25installing the puppet package from the puppetlabs repositories:
26https://docs.puppet.com/puppet/4.2/reference/whered_it_go.html
27The keys are ``package_name``, ``conf_file`` and ``ssl_dir``. If unset, their
28values will default to ones that work with puppet 3.x and with distributions
29that ship modified puppet 4.x that uses the old paths.
30
24Puppet configuration can be specified under the ``conf`` key. The31Puppet configuration can be specified under the ``conf`` key. The
25configuration is specified as a dictionary containing high-level ``<section>``32configuration is specified as a dictionary containing high-level ``<section>``
26keys and lists of ``<key>=<value>`` pairs within each section. Each section33keys and lists of ``<key>=<value>`` pairs within each section. Each section
@@ -44,6 +51,9 @@ in pem format as a multi-line string (using the ``|`` yaml notation).
44 puppet:51 puppet:
45 install: <true/false>52 install: <true/false>
46 version: <version>53 version: <version>
54 conf_file: '/etc/puppet/puppet.conf'
55 ssl_dir: '/var/lib/puppet/ssl'
56 package_name: 'puppet'
47 conf:57 conf:
48 agent:58 agent:
49 server: "puppetmaster.example.org"59 server: "puppetmaster.example.org"
@@ -63,9 +73,17 @@ from cloudinit import helpers
63from cloudinit import util73from cloudinit import util
6474
65PUPPET_CONF_PATH = '/etc/puppet/puppet.conf'75PUPPET_CONF_PATH = '/etc/puppet/puppet.conf'
66PUPPET_SSL_CERT_DIR = '/var/lib/puppet/ssl/certs/'
67PUPPET_SSL_DIR = '/var/lib/puppet/ssl'76PUPPET_SSL_DIR = '/var/lib/puppet/ssl'
68PUPPET_SSL_CERT_PATH = '/var/lib/puppet/ssl/certs/ca.pem'77PUPPET_PACKAGE_NAME = 'puppet'
78
79
80class PuppetConstants(object):
81
82 def __init__(self, puppet_conf_file, puppet_ssl_dir, log):
83 self.conf_path = puppet_conf_file
84 self.ssl_dir = puppet_ssl_dir
85 self.ssl_cert_dir = os.path.join(puppet_ssl_dir, "certs")
86 self.ssl_cert_path = os.path.join(self.ssl_cert_dir, "ca.pem")
6987
7088
71def _autostart_puppet(log):89def _autostart_puppet(log):
@@ -92,22 +110,29 @@ def handle(name, cfg, cloud, log, _args):
92 return110 return
93111
94 puppet_cfg = cfg['puppet']112 puppet_cfg = cfg['puppet']
95
96 # Start by installing the puppet package if necessary...113 # Start by installing the puppet package if necessary...
97 install = util.get_cfg_option_bool(puppet_cfg, 'install', True)114 install = util.get_cfg_option_bool(puppet_cfg, 'install', True)
98 version = util.get_cfg_option_str(puppet_cfg, 'version', None)115 version = util.get_cfg_option_str(puppet_cfg, 'version', None)
116 package_name = util.get_cfg_option_str(
117 puppet_cfg, 'package_name', PUPPET_PACKAGE_NAME)
118 conf_file = util.get_cfg_option_str(
119 puppet_cfg, 'conf_file', PUPPET_CONF_PATH)
120 ssl_dir = util.get_cfg_option_str(puppet_cfg, 'ssl_dir', PUPPET_SSL_DIR)
121
122 p_constants = PuppetConstants(conf_file, ssl_dir, log)
99 if not install and version:123 if not install and version:
100 log.warn(("Puppet install set false but version supplied,"124 log.warn(("Puppet install set false but version supplied,"
101 " doing nothing."))125 " doing nothing."))
102 elif install:126 elif install:
103 log.debug(("Attempting to install puppet %s,"),127 log.debug(("Attempting to install puppet %s,"),
104 version if version else 'latest')128 version if version else 'latest')
105 cloud.distro.install_packages(('puppet', version))129
130 cloud.distro.install_packages((package_name, version))
106131
107 # ... and then update the puppet configuration132 # ... and then update the puppet configuration
108 if 'conf' in puppet_cfg:133 if 'conf' in puppet_cfg:
109 # Add all sections from the conf object to puppet.conf134 # Add all sections from the conf object to puppet.conf
110 contents = util.load_file(PUPPET_CONF_PATH)135 contents = util.load_file(p_constants.conf_path)
111 # Create object for reading puppet.conf values136 # Create object for reading puppet.conf values
112 puppet_config = helpers.DefaultingConfigParser()137 puppet_config = helpers.DefaultingConfigParser()
113 # Read puppet.conf values from original file in order to be able to138 # Read puppet.conf values from original file in order to be able to
@@ -116,19 +141,19 @@ def handle(name, cfg, cloud, log, _args):
116 cleaned_lines = [i.lstrip() for i in contents.splitlines()]141 cleaned_lines = [i.lstrip() for i in contents.splitlines()]
117 cleaned_contents = '\n'.join(cleaned_lines)142 cleaned_contents = '\n'.join(cleaned_lines)
118 puppet_config.readfp(StringIO(cleaned_contents),143 puppet_config.readfp(StringIO(cleaned_contents),
119 filename=PUPPET_CONF_PATH)144 filename=p_constants.conf_path)
120 for (cfg_name, cfg) in puppet_cfg['conf'].items():145 for (cfg_name, cfg) in puppet_cfg['conf'].items():
121 # Cert configuration is a special case146 # Cert configuration is a special case
122 # Dump the puppet master ca certificate in the correct place147 # Dump the puppet master ca certificate in the correct place
123 if cfg_name == 'ca_cert':148 if cfg_name == 'ca_cert':
124 # Puppet ssl sub-directory isn't created yet149 # Puppet ssl sub-directory isn't created yet
125 # Create it with the proper permissions and ownership150 # Create it with the proper permissions and ownership
126 util.ensure_dir(PUPPET_SSL_DIR, 0o771)151 util.ensure_dir(p_constants.ssl_dir, 0o771)
127 util.chownbyname(PUPPET_SSL_DIR, 'puppet', 'root')152 util.chownbyname(p_constants.ssl_dir, 'puppet', 'root')
128 util.ensure_dir(PUPPET_SSL_CERT_DIR)153 util.ensure_dir(p_constants.ssl_cert_dir)
129 util.chownbyname(PUPPET_SSL_CERT_DIR, 'puppet', 'root')154 util.chownbyname(p_constants.ssl_cert_dir, 'puppet', 'root')
130 util.write_file(PUPPET_SSL_CERT_PATH, cfg)155 util.write_file(p_constants.ssl_cert_path, cfg)
131 util.chownbyname(PUPPET_SSL_CERT_PATH, 'puppet', 'root')156 util.chownbyname(p_constants.ssl_cert_path, 'puppet', 'root')
132 else:157 else:
133 # Iterate through the config items, we'll use ConfigParser.set158 # Iterate through the config items, we'll use ConfigParser.set
134 # to overwrite or create new items as needed159 # to overwrite or create new items as needed
@@ -144,8 +169,9 @@ def handle(name, cfg, cloud, log, _args):
144 puppet_config.set(cfg_name, o, v)169 puppet_config.set(cfg_name, o, v)
145 # We got all our config as wanted we'll rename170 # We got all our config as wanted we'll rename
146 # the previous puppet.conf and create our new one171 # the previous puppet.conf and create our new one
147 util.rename(PUPPET_CONF_PATH, "%s.old" % (PUPPET_CONF_PATH))172 util.rename(p_constants.conf_path, "%s.old"
148 util.write_file(PUPPET_CONF_PATH, puppet_config.stringify())173 % (p_constants.conf_path))
174 util.write_file(p_constants.conf_path, puppet_config.stringify())
149175
150 # Set it up so it autostarts176 # Set it up so it autostarts
151 _autostart_puppet(log)177 _autostart_puppet(log)
diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py
index 2b38837..5112a34 100644
--- a/cloudinit/config/cc_salt_minion.py
+++ b/cloudinit/config/cc_salt_minion.py
@@ -25,6 +25,9 @@ specified with ``public_key`` and ``private_key`` respectively.
25 salt_minion:25 salt_minion:
26 conf:26 conf:
27 master: salt.example.com27 master: salt.example.com
28 grains:
29 role:
30 - web
28 public_key: |31 public_key: |
29 ------BEGIN PUBLIC KEY-------32 ------BEGIN PUBLIC KEY-------
30 <key data>33 <key data>
@@ -65,6 +68,12 @@ def handle(name, cfg, cloud, log, _args):
65 minion_data = util.yaml_dumps(salt_cfg.get('conf'))68 minion_data = util.yaml_dumps(salt_cfg.get('conf'))
66 util.write_file(minion_config, minion_data)69 util.write_file(minion_config, minion_data)
6770
71 if 'grains' in salt_cfg:
72 # add grains to /etc/salt/grains
73 grains_config = os.path.join(config_dir, 'grains')
74 grains_data = util.yaml_dumps(salt_cfg.get('grains'))
75 util.write_file(grains_config, grains_data)
76
68 # ... copy the key pair if specified77 # ... copy the key pair if specified
69 if 'public_key' in salt_cfg and 'private_key' in salt_cfg:78 if 'public_key' in salt_cfg and 'private_key' in salt_cfg:
70 if os.path.isdir("/etc/salt/pki/minion"):79 if os.path.isdir("/etc/salt/pki/minion"):
diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py
index 2da34a9..bebc991 100644
--- a/cloudinit/sources/DataSourceGCE.py
+++ b/cloudinit/sources/DataSourceGCE.py
@@ -213,16 +213,15 @@ def read_md(address=None, platform_check=True):
213 if md['availability-zone']:213 if md['availability-zone']:
214 md['availability-zone'] = md['availability-zone'].split('/')[-1]214 md['availability-zone'] = md['availability-zone'].split('/')[-1]
215215
216 encoding = instance_data.get('user-data-encoding')216 if 'user-data' in instance_data:
217 if encoding:217 # instance_data was json, so values are all utf-8 strings.
218 ud = instance_data['user-data'].encode("utf-8")
219 encoding = instance_data.get('user-data-encoding')
218 if encoding == 'base64':220 if encoding == 'base64':
219 md['user-data'] = b64decode(instance_data.get('user-data'))221 ud = b64decode(ud)
220 else:222 elif encoding:
221 LOG.warning('unknown user-data-encoding: %s, ignoring', encoding)223 LOG.warning('unknown user-data-encoding: %s, ignoring', encoding)
222224 ret['user-data'] = ud
223 if 'user-data' in md:
224 ret['user-data'] = md['user-data']
225 del md['user-data']
226225
227 ret['meta-data'] = md226 ret['meta-data'] = md
228 ret['success'] = True227 ret['success'] = True
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 338fb97..02dc2ce 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -1746,7 +1746,7 @@ def chmod(path, mode):
1746def write_file(filename, content, mode=0o644, omode="wb", copy_mode=False):1746def write_file(filename, content, mode=0o644, omode="wb", copy_mode=False):
1747 """1747 """
1748 Writes a file with the given content and sets the file mode as specified.1748 Writes a file with the given content and sets the file mode as specified.
1749 Resotres the SELinux context if possible.1749 Restores the SELinux context if possible.
17501750
1751 @param filename: The full path of the file to write.1751 @param filename: The full path of the file to write.
1752 @param content: The content to write to the file.1752 @param content: The content to write to the file.
@@ -1865,8 +1865,13 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False,
1865 if not isinstance(data, bytes):1865 if not isinstance(data, bytes):
1866 data = data.encode()1866 data = data.encode()
18671867
1868 # Popen converts entries in the arguments array from non-bytes to bytes.
1869 # When locale is unset it may use ascii for that encoding which can
1870 # cause UnicodeDecodeErrors. (LP: #1751051)
1871 bytes_args = [x if isinstance(x, six.binary_type) else x.encode("utf-8")
1872 for x in args]
1868 try:1873 try:
1869 sp = subprocess.Popen(args, stdout=stdout,1874 sp = subprocess.Popen(bytes_args, stdout=stdout,
1870 stderr=stderr, stdin=stdin,1875 stderr=stderr, stdin=stdin,
1871 env=env, shell=shell)1876 env=env, shell=shell)
1872 (out, err) = sp.communicate(data)1877 (out, err) = sp.communicate(data)
diff --git a/debian/changelog b/debian/changelog
index 2552bb6..27dba2c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,16 @@
1cloud-init (18.1-5-g40e77380-0ubuntu1) bionic; urgency=medium
2
3 * New upstream snapshot.
4 - GCE: fix reading of user-data that is not base64 encoded. (LP: #1752711)
5 - doc: fix chef install from apt packages example in RTD.
6 - Implement puppet 4 support [Romanos Skiadas] (LP: #1446804)
7 - subp: Fix subp usage with non-ascii characters when no system locale.
8 (LP: #1751051)
9 - salt: configure grains in grains file rather than in minion config.
10 [Daniel Wallace]
11
12 -- Chad Smith <chad.smith@canonical.com> Thu, 01 Mar 2018 15:47:04 -0700
13
1cloud-init (18.1-0ubuntu1) bionic; urgency=medium14cloud-init (18.1-0ubuntu1) bionic; urgency=medium
215
3 * New upstream snapshot.16 * New upstream snapshot.
diff --git a/doc/examples/cloud-config-chef.txt b/doc/examples/cloud-config-chef.txt
index 58d5fdc..defc5a5 100644
--- a/doc/examples/cloud-config-chef.txt
+++ b/doc/examples/cloud-config-chef.txt
@@ -12,8 +12,8 @@
1212
13# Key from https://packages.chef.io/chef.asc13# Key from https://packages.chef.io/chef.asc
14apt:14apt:
15 source1:15 sources:
16 source: "deb http://packages.chef.io/repos/apt/stable $RELEASE main"16 source1: "deb http://packages.chef.io/repos/apt/stable $RELEASE main"
17 key: |17 key: |
18 -----BEGIN PGP PUBLIC KEY BLOCK-----18 -----BEGIN PGP PUBLIC KEY BLOCK-----
19 Version: GnuPG v1.4.12 (Darwin)19 Version: GnuPG v1.4.12 (Darwin)
diff --git a/tests/cloud_tests/testcases/modules/salt_minion.py b/tests/cloud_tests/testcases/modules/salt_minion.py
index c697db2..f13b48a 100644
--- a/tests/cloud_tests/testcases/modules/salt_minion.py
+++ b/tests/cloud_tests/testcases/modules/salt_minion.py
@@ -26,4 +26,9 @@ class Test(base.CloudTestCase):
26 self.assertIn('<key data>', out)26 self.assertIn('<key data>', out)
27 self.assertIn('------END PUBLIC KEY-------', out)27 self.assertIn('------END PUBLIC KEY-------', out)
2828
29 def test_grains(self):
30 """Test master value in config."""
31 out = self.get_data_file('grains')
32 self.assertIn('role: web', out)
33
29# vi: ts=4 expandtab34# vi: ts=4 expandtab
diff --git a/tests/cloud_tests/testcases/modules/salt_minion.yaml b/tests/cloud_tests/testcases/modules/salt_minion.yaml
index f20d24f..ab0e05b 100644
--- a/tests/cloud_tests/testcases/modules/salt_minion.yaml
+++ b/tests/cloud_tests/testcases/modules/salt_minion.yaml
@@ -17,6 +17,8 @@ cloud_config: |
17 ------BEGIN PRIVATE KEY------17 ------BEGIN PRIVATE KEY------
18 <key data>18 <key data>
19 ------END PRIVATE KEY-------19 ------END PRIVATE KEY-------
20 grains:
21 role: web
20collect_scripts:22collect_scripts:
21 minion: |23 minion: |
22 #!/bin/bash24 #!/bin/bash
@@ -30,5 +32,8 @@ collect_scripts:
30 minion.pub: |32 minion.pub: |
31 #!/bin/bash33 #!/bin/bash
32 cat /etc/salt/pki/minion/minion.pub34 cat /etc/salt/pki/minion/minion.pub
35 grains: |
36 #!/bin/bash
37 cat /etc/salt/grains
3338
34# vi: ts=4 expandtab39# vi: ts=4 expandtab
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index f77c2c4..eb3cec4 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -38,11 +38,20 @@ GCE_META_ENCODING = {
38 'instance/hostname': 'server.project-baz.local',38 'instance/hostname': 'server.project-baz.local',
39 'instance/zone': 'baz/bang',39 'instance/zone': 'baz/bang',
40 'instance/attributes': {40 'instance/attributes': {
41 'user-data': b64encode(b'/bin/echo baz\n').decode('utf-8'),41 'user-data': b64encode(b'#!/bin/echo baz\n').decode('utf-8'),
42 'user-data-encoding': 'base64',42 'user-data-encoding': 'base64',
43 }43 }
44}44}
4545
46GCE_USER_DATA_TEXT = {
47 'instance/id': '12345',
48 'instance/hostname': 'server.project-baz.local',
49 'instance/zone': 'baz/bang',
50 'instance/attributes': {
51 'user-data': '#!/bin/sh\necho hi mom\ntouch /run/up-now\n',
52 }
53}
54
46HEADERS = {'Metadata-Flavor': 'Google'}55HEADERS = {'Metadata-Flavor': 'Google'}
47MD_URL_RE = re.compile(56MD_URL_RE = re.compile(
48 r'http://metadata.google.internal/computeMetadata/v1/.*')57 r'http://metadata.google.internal/computeMetadata/v1/.*')
@@ -135,7 +144,16 @@ class TestDataSourceGCE(test_helpers.HttprettyTestCase):
135 shostname = GCE_META_PARTIAL.get('instance/hostname').split('.')[0]144 shostname = GCE_META_PARTIAL.get('instance/hostname').split('.')[0]
136 self.assertEqual(shostname, self.ds.get_hostname())145 self.assertEqual(shostname, self.ds.get_hostname())
137146
147 def test_userdata_no_encoding(self):
148 """check that user-data is read."""
149 _set_mock_metadata(GCE_USER_DATA_TEXT)
150 self.ds.get_data()
151 self.assertEqual(
152 GCE_USER_DATA_TEXT['instance/attributes']['user-data'].encode(),
153 self.ds.get_userdata_raw())
154
138 def test_metadata_encoding(self):155 def test_metadata_encoding(self):
156 """user-data is base64 encoded if user-data-encoding is 'base64'."""
139 _set_mock_metadata(GCE_META_ENCODING)157 _set_mock_metadata(GCE_META_ENCODING)
140 self.ds.get_data()158 self.ds.get_data()
141159
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index 4a92e74..89ae40f 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -8,7 +8,9 @@ import shutil
8import stat8import stat
9import tempfile9import tempfile
1010
11import json
11import six12import six
13import sys
12import yaml14import yaml
1315
14from cloudinit import importer, util16from cloudinit import importer, util
@@ -733,6 +735,38 @@ class TestSubp(helpers.CiTestCase):
733 self.assertEqual("/target/my/path/",735 self.assertEqual("/target/my/path/",
734 util.target_path("/target/", "///my/path/"))736 util.target_path("/target/", "///my/path/"))
735737
738 def test_c_lang_can_take_utf8_args(self):
739 """Independent of system LC_CTYPE, args can contain utf-8 strings.
740
741 When python starts up, its default encoding gets set based on
742 the value of LC_CTYPE. If no system locale is set, the default
743 encoding for both python2 and python3 in some paths will end up
744 being ascii.
745
746 Attempts to use setlocale or patching (or changing) os.environ
747 in the current environment seem to not be effective.
748
749 This test starts up a python with LC_CTYPE set to C so that
750 the default encoding will be set to ascii. In such an environment
751 Popen(['command', 'non-ascii-arg']) would cause a UnicodeDecodeError.
752 """
753 python_prog = '\n'.join([
754 'import json, sys',
755 'from cloudinit.util import subp',
756 'data = sys.stdin.read()',
757 'cmd = json.loads(data)',
758 'subp(cmd, capture=False)',
759 ''])
760 cmd = [BASH, '-c', 'echo -n "$@"', '--',
761 self.utf8_valid.decode("utf-8")]
762 python_subp = [sys.executable, '-c', python_prog]
763
764 out, _err = util.subp(
765 python_subp, update_env={'LC_CTYPE': 'C'},
766 data=json.dumps(cmd).encode("utf-8"),
767 decode=False)
768 self.assertEqual(self.utf8_valid, out)
769
736770
737class TestEncode(helpers.TestCase):771class TestEncode(helpers.TestCase):
738 """Test the encoding functions"""772 """Test the encoding functions"""

Subscribers

People subscribed via source and target branches